import gettext
import os
import re
import langtable
import locale as locale_mod
import glob
from collections import namedtuple
import functools

from pyanaconda.core import constants
from pyanaconda.core.util import setenv, execWithRedirect
from pyanaconda.core.string import upcase_first_letter
from pyanaconda.modules.common.constants.services import BOSS

from pyanaconda.anaconda_loggers import get_module_logger
log = get_module_logger(__name__)

SCRIPTS_SUPPORTED_BY_CONSOLE = {'Latn', 'Cyrl', 'Grek'}

class LocalizationConfigError(Exception):
    """Exception class for localization configuration related problems"""

    pass

class InvalidLocaleSpec(LocalizationConfigError):
    """Exception class for the errors related to invalid locale specs"""

    pass

def is_valid_langcode(langcode):
    """Check if the given locale has a language specified.

    :return: whether the language or locale is valid
    :rtype: bool
    """
    parsed = langtable.parse_locale(langcode)
    return bool(parsed.language)

def raise_on_invalid_locale(arg):
    """Helper to abort when a locale is not valid.

    :raise: InvalidLocaleSpec
    """
    if not is_valid_langcode(arg):
        raise InvalidLocaleSpec("'{}' is not a valid locale".format(arg))

def get_language_id(locale):
    """Return language id without territory or anything else."""
    return langtable.parse_locale(locale).language

@functools.cache
def get_common_languages():
    """Return common languages to prioritize them"""
    return langtable.list_common_languages()

def is_supported_locale(locale):
    """Function that tells if the given locale is supported by the Anaconda or
    not. We consider locales supported by the langtable as supported by the
    Anaconda.

    :param locale: locale to test
    :type locale: str
    :return: whether the given locale is supported or not
    :rtype: bool
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    en_name = get_english_name(locale)
    return bool(en_name)

def locale_supported_in_console(locale):
    """Function that tells if the given locale can be displayed by the Linux console.

    The Linux console can display Latin, Cyrillic and Greek characters reliably,
    but others such as Japanese, can't be correctly installed.

    :param str locale: locale to test
    :return: whether the given locale is supported by the console or not
    :rtype: bool
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    locale_scripts = get_locale_scripts(locale)
    return set(locale_scripts).issubset(SCRIPTS_SUPPORTED_BY_CONSOLE)

def find_best_locale_match(locale, langcodes):
    """Find the best match for the locale in a list of langcodes. This is useful
    when e.g. pt_BR is a locale and there are possibilities to choose an item
    (e.g. rnote) for a list containing both pt and pt_BR or even also pt_PT.

    :param locale: a valid locale (e.g. en_US.UTF-8 or sr_RS.UTF-8@latin, etc.)
    :type locale: str
    :param langcodes: a list or generator of langcodes (e.g. en, en_US, en_US@latin, etc.)
    :type langcodes: list(str) or generator(str)
    :return: the best matching langcode from the list of None if none matches
    :rtype: str or None
    """
    # Parse the locale.
    if not is_valid_langcode(locale):
        return None

    locale_parsed = langtable.parse_locale(locale)

    # Get a score for each langcode.
    scores = {}

    for langcode in langcodes:
        if not is_valid_langcode(langcode):
            continue

        langcode_parsed = langtable.parse_locale(langcode)

        # Don't match a non-POSIX locale with a POSIX langcode.
        if langcode_parsed.variant == "POSIX" and locale_parsed.variant != "POSIX":
            continue

        score = _evaluate_locales(locale_parsed, langcode_parsed)

        # Matches matching only script or encoding or both are not useful.
        # The score of 100 requires at least territory to have matched.
        if score <= 100:
            continue

        scores[langcode] = score

    # Find the best one.
    return max(scores.keys(), key=scores.get, default=None)

def _evaluate_locales(locale, langcode):
    """Compare a locale with a langcode.

    :param locale: a parsed locale
    :param langcode: a parsed langcode
    :return: a score
    """
    return \
        _evaluate_values(locale.language, langcode.language, 1000) + \
        _evaluate_values(locale.territory, langcode.territory, 100) + \
        _evaluate_values(locale.script, langcode.script, 10) + \
        _evaluate_values(locale.variant, langcode.variant, 10) + \
        _evaluate_values(locale.encoding, langcode.encoding, 1)

def _evaluate_values(locale_value, langcode_value, weight):
    """Compare a locale value with a langcode value.

    :param locale_value: an item of the parsed locale
    :param langcode_value: an item of the parsed langcode
    :return: a score
    """
    if locale_value and langcode_value:
        if locale_value == langcode_value:
            # Match.
            return weight
        else:
            # Not match.
            return -weight

    if langcode_value and not locale_value:
        # The langcode has something the locale doesn't have.
        return -weight

    return 0

def setup_locale(locale, localization_proxy=None, text_mode=False):
    """Procedure setting the system to use the given locale and store it in to the
    localization module (if given). DOES NOT PERFORM ANY CHECKS OF THE GIVEN
    LOCALE.

    $LANG must be set by the caller in order to set the language used by gettext.
    Doing this in a thread-safe way is up to the caller.

    We also try to set a proper console font for the locale in text mode.
    If the font for the locale can't be displayed in the Linux console,
    we fall back to the English locale.

    This function returns the locale that was used in the setlocale call, which,
    depending on what the environment and interface is able to support, may be
    different from the locale requested.

    :param str locale: locale to setup
    :param localization_proxy: DBus proxy of the localization module or None
    :param bool text_mode: if the locale is being setup for text mode
    :return: the locale that was actually set
    :rtype: str
    """
    if localization_proxy:
        localization_proxy.Language = locale

    # not all locales might be displayable in text mode
    if text_mode:
        # check if the script corresponding to the locale/language
        # can be displayed by the Linux console
        # * all scripts for the given locale/language need to be
        #   supported by the linux console
        # * otherwise users might get a screen full of white rectangles
        #   (also known as "tofu") in text mode
        # then we also need to check if we have information about what
        # font to use for correctly displaying the given language/locale

        script_supported = locale_supported_in_console(locale)
        log.debug("scripts found for locale %s: %s", locale, get_locale_scripts(locale))

        console_fonts = get_locale_console_fonts(locale)
        log.debug("console fonts found for locale %s: %s", locale, console_fonts)

        font_set = False
        if script_supported and console_fonts:
            # try to set console font
            for font in console_fonts:
                if set_console_font(font):
                    # console font set successfully, skip the rest
                    font_set = True
                    break

        if not font_set:
            log.warning("can't set console font for locale %s", locale)
            # report what exactly went wrong
            if not script_supported:
                log.warning("script not supported by console for locale %s", locale)
            if not console_fonts:  # no fonts known for locale
                log.warning("no console font found for locale %s", locale)
            if script_supported and console_fonts:
                log.warning("none of the suggested fonts can be set for locale %s", locale)
            log.warning("falling back to the English locale")
            locale = constants.DEFAULT_LANG
            os.environ["LANG"] = locale  # pylint: disable=environment-modify

    # set the locale to the value we have selected
    # Since glibc does not install all locales, an installable locale may not
    # actually be available right now. Give it a shot and fallback.
    log.debug("setting locale to: %s", locale)
    setenv("LANG", locale)

    try:
        locale_mod.setlocale(locale_mod.LC_ALL, locale)
    except locale_mod.Error as e:
        log.debug("setlocale failed: %s", e)
        locale = constants.DEFAULT_LANG
        setenv("LANG", locale)
        locale_mod.setlocale(locale_mod.LC_ALL, locale)

    set_modules_locale(locale)

    return locale

def set_modules_locale(locale):
    """Set locale of all modules."""
    boss_proxy = BOSS.get_proxy()
    boss_proxy.SetLocale(locale)

def get_english_name(locale):
    """Function returning english name for the given locale.

    :param locale: locale to return english name for
    :type locale: str
    :return: english name for the locale or empty string if unknown
    :rtype: st
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    name = langtable.language_name(languageId=locale, languageIdQuery="en")
    return upcase_first_letter(name)

def get_native_name(locale):
    """Function returning native name for the given locale.

    :param locale: locale to return native name for
    :type locale: str
    :return: english name for the locale or empty string if unknown
    :rtype: st
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.language_name(languageId=locale)

def get_available_translations(localedir=None):
    """Method that generates (i.e. returns a generator) available translations
    for the installer in the given localedir.

    :type localedir: str
    :return: generator yielding available translations (languages)
    :rtype: generator yielding strings
    """
    localedir = localedir or gettext._default_localedir

    # usually there are no message files for en
    messagefiles = sorted(glob.glob(localedir + "/*/LC_MESSAGES/anaconda.mo") +
                          ["blob/en/blob/blob"])
    trans_gen = (path.split(os.path.sep)[-3] for path in messagefiles)

    langs = set()

    for trans in trans_gen:
        lang = get_language_id(trans)
        if lang and lang not in langs:
            langs.add(lang)
            # check if there are any locales for the language
            locales = get_language_locales(lang)
            if not locales:
                continue

            yield lang

@functools.lru_cache(2048)
def locale_has_translation(locale):
    """Does the locale have a translation available?

    Checks if a given locale will receive a gettext translation. That could be either because the
    locale has a translation, or because there is another translation to fall back onto. In
    reality, the fallback is mostly of the type "ja_JP" -> "ja".

    For English, always return true, because that is the "untranslated" state which does not need
    translation files present to work.

    :param str locale: locale to check
    :return bool: is there a translation
    """
    if get_language_id(locale) == "en":
        return True

    files = gettext.find("anaconda", None, [locale], True)
    return bool(files)

def get_language_locales(lang):
    """Function returning all locales available for the given language.

    :param lang: language to get available locales for
    :type lang: str
    :return: a list of available locales
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(lang)

    return langtable.list_locales(languageId=lang)

def get_territory_locales(territory):
    """Function returning list of locales for the given territory. The list is
    sorted from the most probable locale to the least probable one (based on
    langtable's ranking.

    :param territory: territory to return locales for
    :type territory: str
    :return: list of locales
    :rtype: list of strings
    """
    return langtable.list_locales(territoryId=territory)

def get_locale_keyboards(locale):
    """Function returning preferred keyboard layouts for the given locale.

    :param locale: locale string
    :type locale: str
    :return: list of preferred keyboard layouts
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.list_keyboards(languageId=locale)

def get_common_keyboard_layouts():
    """Function returning common keyboard layouts carrying high ranks.

    :return: list of common keyboard layouts
    :rtype: list of strings
    """
    return langtable.list_common_keyboards()

def get_locale_timezones(locale):
    """Function returning preferred timezones for the given locale.

    :param locale: locale string
    :type locale: str
    :return: list of preferred timezones
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.list_timezones(languageId=locale)

def get_locale_console_fonts(locale):
    """Function returning preferred console fonts for the given locale.

    :param str locale: locale string
    :return: list of preferred console fonts
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.list_consolefonts(languageId=locale)

def get_locale_scripts(locale):
    """Function returning preferred scripts (writing systems) for the given locale.

    :param locale: locale string
    :type locale: str
    :return: list of preferred scripts
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.list_scripts(languageId=locale)

def get_xlated_timezone(tz_spec_part):
    """Function returning translated name of a region, city or complete timezone
    name according to the current value of the $LANG variable.

    :param tz_spec_part: a region, city or complete timezone name
    :type tz_spec_part: str
    :return: translated name of the given region, city or timezone
    :rtype: str
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    locale = os.environ.get("LANG", constants.DEFAULT_LANG)

    raise_on_invalid_locale(locale)

    xlated = langtable.timezone_name(tz_spec_part, languageIdQuery=locale)
    return xlated

def get_firmware_language(text_mode=False):
    """Procedure that returns the firmware language information (if any).

    :param boot text_mode: if the locale is being setup for text mode
    :return: the firmware language translated into a locale string, or None
    :rtype: str
    """
    try:
        n = "/sys/firmware/efi/efivars/PlatformLang-8be4df61-93ca-11d2-aa0d-00e098032b8c"
        with open(n, 'r') as f:
            d = f.read()
    except OSError:
        return None

    # the contents of the file are:
    # 4-bytes of attribute data that we don't care about
    # NUL terminated ASCII string like 'en-US'.
    if len(d) < 10:
        log.debug("PlatformLang was too short")
        return None
    d = d[4:]
    if d[2] != '-':
        log.debug("PlatformLang was malformed")
        return None

    # they use - and we use _, so fix it...
    d = d[:2] + '_' + d[3:-1]

    # UEFI 2.3.1 Errata C specifies 2 aliases in common use that
    # aren't part of RFC 4646, but are allowed in PlatformLang.
    # Because why make anything simple?
    if d.startswith('zh_chs'):
        d = 'zh_Hans'
    elif d.startswith('zh_cht'):
        d = 'zh_Hant'
    d += '.UTF-8'

    if not is_supported_locale(d):
        log.debug("PlatformLang was '%s', which is unsupported.", d)
        return None

    locales = get_language_locales(d)
    if not locales:
        log.debug("No locales found for the PlatformLang '%s'.", d)
        return None

    log.debug("Using UEFI PlatformLang '%s' ('%s') as our language.", d, locales[0])
    return locales[0]

_DateFieldSpec = namedtuple("DateFieldSpec", ["format", "suffix"])

def resolve_date_format(year, month, day, fail_safe=True):
    """Puts the year, month and day objects in the right order according to the
    currently set locale and provides format specification for each of the
    fields.

    :param year: any object or value representing year
    :type year: any
    :param month: any object or value representing month
    :type month: any
    :param day: any object or value representing day
    :type day: any
    :param bool fail_safe: whether to fall back to default in case of invalid
                           format or raise exception instead
    :returns: a pair where the first field contains a tuple with the year, month
              and day objects/values put in the right order and where the second
              field contains a tuple with three :class:`_DateFieldSpec` objects
              specifying formats respectively to the first (year, month, day)
              field, e.g. ((year, month, day), (y_fmt, m_fmt, d_fmt))
    :rtype: tuple
    :raise ValueError: in case currently set locale has unsupported date
                       format and fail_safe is set to False
    """
    FAIL_SAFE_DEFAULT = "%Y-%m-%d"

    def order_terms_formats(fmt_str):
        # see date (1), 'O' (not '0') is a mystery, 'E' is Buddhist calendar, '(.*)'
        # is an arbitrary suffix
        field_spec_re = re.compile(r'([-_0OE^#]*)([yYmbBde])(.*)')

        # see date (1)
        fmt_str = fmt_str.replace("%F", "%Y-%m-%d")

        # e.g. "%d.%m.%Y" -> ['d.', 'm.', 'Y']
        fields = fmt_str.split("%")[1:]

        ordered_terms = []
        ordered_formats = []
        for field in fields:
            match = field_spec_re.match(field)
            if not match:
                # ignore fields we are not interested in (like %A for weekday name, etc.)
                continue

            prefix, item, suffix = match.groups()
            if item in ("d", "e"):
                # "e" is the same as "_d"
                ordered_terms.append(day)
            elif item in ("Y", "y"):
                # 4-digit year, 2-digit year
                ordered_terms.append(year)
            elif item in ("m", "b", "B"):
                # month number, short month name, long month name
                ordered_terms.append(month)

            # "%" + prefix + item gives a format for date/time formatting functions
            ordered_formats.append(_DateFieldSpec("%" + prefix + item, suffix.strip()))

        if len(ordered_terms) != 3 or len(ordered_formats) != 3:
            raise ValueError("Not all fields successfully identified in the format '%s'" % fmt_str)

        return (tuple(ordered_terms), tuple(ordered_formats))

    fmt_str = locale_mod.nl_langinfo(locale_mod.D_FMT)

    if not fmt_str or "%" not in fmt_str:
        if fail_safe:
            # use some sane default
            fmt_str = FAIL_SAFE_DEFAULT
        else:
            raise ValueError("Invalid date format string for current locale: '%s'" % fmt_str)

    try:
        return order_terms_formats(fmt_str)
    except ValueError:
        if not fail_safe:
            raise
        else:
            # if this call fails too, something is going terribly wrong and we
            # should be informed about it
            return order_terms_formats(FAIL_SAFE_DEFAULT)

def set_console_font(font):
    """Try to set console font to the given value.

    :param str font: console font name
    :returns: True on success, False on failure
    :rtype: bool
    """
    log.debug("setting console font to %s", font)
    rc = execWithRedirect("setfont", [font])
    if rc == 0:
        log.debug("console font set successfully to %s", font)
        return True
    else:
        log.error("setting console font to %s failed", font)
        return False

def setup_locale_environment(locale=None, text_mode=False, prefer_environment=False):
    """Clean and configure the local environment variables.

    This function will attempt to determine the desired locale and configure
    the process environment (os.environ) in the least surprising way. If a
    locale argument is provided, it will be attempted first. After that, this
    function will attempt to use the language environment variables in a manner
    similar to gettext(3) (in order, $LANGUAGE, $LC_ALL, $LC_MESSAGES, $LANG),
    followed by the UEFI PlatformLang, followed by a default.

    When this function returns, $LANG will be set, and $LANGUAGE, $LC_ALL,
    and $LC_MESSAGES will not be set, because they get in the way when changing
    the language after startup.

    This function must be run before any threads are started. This function
    modifies the process environment, which is not thread-safe.

    :param str locale: locale to setup if provided
    :param bool text_mode: if the locale is being setup for text mode
    :param bool prefer_environment: whether the process environment, if available, overrides the locale parameter
    :return: None
    :rtype: None
    """
    # pylint: disable=environment-modify

    # Look for a locale in the environment. If the variable is setup but
    # empty it doesn't count, and some programs (KDE) actually do this.
    # If prefer_environment is set, the environment locale can override
    # the parameter passed in. This can be used, for example, by initial-setup,
    # to prefer the possibly-more-recent environment settings before falling back
    # to a locale set at install time and saved in the kickstart.
    if not locale or prefer_environment:
        for varname in ("LANGUAGE", "LC_ALL", "LC_MESSAGES", "LANG"):
            if varname in os.environ and os.environ[varname]:
                locale = os.environ[varname]
                break

    # Look for a locale in the firmware if there was nothing in the environment
    if not locale:
        locale = get_firmware_language(text_mode)

    # parse the locale using langtable
    if locale:
        try:
            env_langs = get_language_locales(locale)
            # the first language is the best match
            locale = env_langs[0]
        except (InvalidLocaleSpec, IndexError):
            log.error("Invalid locale '%s' given on command line, kickstart or environment", locale)
            locale = None

    # If langtable returned no locales, or if nothing was configured, fall back to the default
    if not locale:
        locale = constants.DEFAULT_LANG

    # Save the locale in the environment
    os.environ["LANG"] = locale

    # Cleanup the rest of the environment variables
    for varname in ("LANGUAGE", "LC_ALL", "LC_MESSAGES"):
        if varname in os.environ:
            del os.environ[varname]


import torch
from transformers import BlenderbotTokenizer, BlenderbotForConditionalGeneration

# Init is ran on server startup
# Load your model to GPU as a global variable here using the variable name "model"
def init():
    global model
    global tokenizer

    tokenizer = BlenderbotTokenizer.from_pretrained("facebook/blenderbot-3B")
    model = BlenderbotForConditionalGeneration.from_pretrained("facebook/blenderbot-3B").to("cuda")

# Inference is ran for every server call
# Reference your preloaded global model variable here.
def inference(model_inputs:dict) -> dict:
    global model
    global tokenizer

    # Parse out your arguments
    prompt = model_inputs.get('prompt', None)
    if prompt == None:
        return {'message': "No prompt provided"}

    # Run the model
    inputs = tokenizer(prompt, return_tensors="pt").to("cuda")

    reply_ids = model.generate(**inputs)

    result = tokenizer.batch_decode(reply_ids)

    # Return the results as a dictionary
    return result


#!/usr/bin/python3
"""
Copy notes from previous beta releases to make notes for an ESR
"""

import argparse
import os
import re
import json
from pathlib import Path
from urllib.request import urlopen
from packaging.version import Version

from tools_lib import guess_release_date, yaml, notes_template
from ruamel.yaml.comments import CommentedSeq as CS

topdir = Path(os.path.dirname(__file__)) / ".."
beta_dir = Path(os.path.join(topdir, "beta"))
release_dir = Path(os.path.join(topdir, "release"))
ver_notes_yaml = "{this_esr}.yml"

MERCURIAL_TAGS_URL = "https://hg.mozilla.org/releases/comm-esr{major}/json-tags"
RELEASE_TAG_RE = re.compile(r"THUNDERBIRD_([\d_b]+)_RELEASE")
CHANGESET_URL_TEMPLATE = "https://hg.mozilla.org/releases/comm-esr{major}/json-pushes?fromchange={from_version}&tochange={to_version}&full=1"
BUG_NUMBER_REGEX = re.compile(r"bug \d+", re.IGNORECASE)
BACKOUT_REGEX = re.compile(r"back(\s?)out|backed out|backing out", re.IGNORECASE)

PD_VERSIONS = "https://product-details.mozilla.org/1.0/thunderbird_versions.json"
CUR_VERSION_URL = (
    "https://hg.mozilla.org/releases/{repo}/raw-file/{rev}/mail/config/version.txt"
)

ESR_VERSIONS = ("102", "115")

class RelNote:
    def __init__(self, bugs, tag, note_text):
        bugs.sort()
        self.bugs = bugs
        self.tag = tag
        self.note = note_text

    @property
    def output(self):
        rv = {"note": self.note, "tag": self.tag}
        if self.bugs:
            rv["bugs"] = self.bugs
        return rv

def sec_version(this_esr):
    if this_esr.endswith("esr"):
        this_esr = this_esr[:-3]
    version = Version(this_esr)

    if version.micro == 0:
        return f"thunderbird{version.major}.{version.minor}"
    return False

def load_notes(this_esr, previous_esr, previous_esr_rev):
    beta_notes = {}
    new_notes = []
    rel_notes = []

    esr_major = previous_esr.split(".")[0]
    min_version = Version("{}.0.0".format(esr_major))

    print(
        """Generating notes for Thunderbird {this_esr}
Previous esr: {previous_esr}
    """.format(
            this_esr=this_esr, previous_esr=previous_esr
        )
    )

    bug_list, backout_list = get_buglist(esr_major, previous_esr_rev)
    bugs_fixed = set.union(bug_list, backout_list)

    note_files = os.listdir(str(beta_dir))
    for _file in note_files:
        if not _file.endswith("beta.yml"):
            continue
        version = Version(_file[:-8])
        # Exclude versions lower than the beta for the previous esr
        # anything newer than the current beta
        if version < min_version:
            continue

        with open(os.path.join(beta_dir, _file), "r") as fp:
            doc = yaml.load(fp)
            beta_notes[_file[:-8]] = doc

    versions = sorted([Version(v) for v in beta_notes.keys()])

    for ver_str in [str(v) for v in versions]:
        ver_notes = beta_notes[ver_str]["notes"]
        for note in ver_notes:
            if "bugs" not in note:
                continue
            if note["tag"] == "unresolved":
                continue

            rel_note = RelNote(note["bugs"], note["tag"], note["note"])

            new_notes.append(rel_note)

    noted_bugs = set()
    for note in new_notes:
        # Only add this note if it is for bugs that are new/changed/fixed in this release
        if any([b for b in note.bugs if b in bugs_fixed]):
            # Remove references to other bugs that might be on the note
            for b2 in note.bugs:
                if b2 not in bugs_fixed:
                    note.bugs.remove(b2)
                else:
                    noted_bugs.add(b2)
            rel_notes.append(note.output)

    sec_v = sec_version(this_esr)
    if sec_v:  # sec_version returns False for non-.0 esrs (like 115.5.1)
        sec_url = notes_template.TMPL_SEC_NOTE.format(thunderbird_version=sec_v)
        sec_note = RelNote([], "fixed", sec_url)
        rel_notes.append(sec_note.output)

    RELEASE_TEMPLATE = f"TMPL_{esr_major}_TEXT"
    new_yaml = yaml.load(notes_template.TMPL_HEADER)
    new_yaml["release"]["text"] = getattr(notes_template, RELEASE_TEMPLATE)
    new_yaml["release"][
        "import_system_requirements"
    ] = notes_template.REQUIREMENTS_IMPORT[esr_major]
    release_date, human_date = guess_release_date()
    new_yaml["release"]["release_date"] = release_date

    notes_sequence = CS()
    note_counter = 0
    for tag_name in ["new", "changed", "fixed"]:
        tag_notes = [n for n in rel_notes if n["tag"] == tag_name]
        if tag_notes:
            notes_sequence.extend(tag_notes)
            notes_sequence.yaml_set_comment_before_after_key(
                note_counter, "{}\n".format(tag_name.capitalize()), 2
            )
            note_counter += len(tag_notes)

    new_yaml["notes"] = notes_sequence

    out_yaml = ver_notes_yaml.format(this_esr=this_esr)
    with open(release_dir / out_yaml, "w") as fp:
        yaml.dump(new_yaml, fp)

    print("Wrote notes to {}.".format(out_yaml))

    no_note_bugs = sorted(bugs_fixed - noted_bugs)
    if no_note_bugs:
        # Print out bugs where no note is available
        print("")
        print("Bugs that do not have an associated note:")

        bugzilla_url = "https://bugzilla.mozilla.org/rest/bug?include_fields=id,summary&id={}".format(
            ",".join([str(b) for b in no_note_bugs])
        )
        with urlopen(bugzilla_url) as response:
            data = json.load(response)

        for note in data["bugs"]:
            print("{} - {}".format(note["id"], note["summary"]))

def get_bugs_in_changeset(changeset_data):
    unique_bugs, unique_backout_bugs = set(), set()
    for data in changeset_data.values():
        for changeset in data["changesets"]:
            if is_excluded_change(changeset):
                continue

            changeset_desc = changeset["desc"]
            bug_re = BUG_NUMBER_REGEX.search(changeset_desc)

            if bug_re:
                bug_number = bug_re.group().split(" ")[1]

                if is_backout_bug(changeset_desc):
                    unique_backout_bugs.add(int(bug_number))
                else:
                    unique_bugs.add(int(bug_number))

    return unique_bugs, unique_backout_bugs

def is_excluded_change(changeset):
    excluded_change_keywords = [
        "a=test-only",
        "a=release",
    ]
    return any(keyword in changeset["desc"] for keyword in excluded_change_keywords)

def is_backout_bug(changeset_description):
    return bool(BACKOUT_REGEX.search(changeset_description))

def get_buglist(esr_major, from_rev):
    """Retrieve the list of bugs since the last release"""
    to_rev = "tip"
    changeset_url = CHANGESET_URL_TEMPLATE.format(
        major=esr_major, from_version=from_rev, to_version=to_rev
    )
    print(changeset_url)
    with urlopen(changeset_url) as response:
        data = json.load(response)
    unique_bugs, unique_backout_bugs = get_bugs_in_changeset(data)

    if unique_bugs:
        print("Fixed bugs:")
        print(" ".join([str(bug) for bug in sorted(unique_bugs)]))
        print("")
    if unique_backout_bugs:
        print("Backout bugs:")
        print(" ".join([str(bug) for bug in sorted(unique_backout_bugs)]))

    return unique_bugs, unique_backout_bugs

def fetch(url, data_type="text"):
    with urlopen(url) as response:
        if data_type == "json":
            data = json.load(response)
        else:
            data = response.read().strip().decode("utf-8")
    return data

def get_prev_release_rev(tags):
    for tag in tags[:10]:
        match = RELEASE_TAG_RE.match(tag["tag"])
        if match:
            print("Using tag {} at rev {}.".format(tag["tag"], tag["node"]))
            return tag["node"]
    raise Exception("Did not find a RELEASE tag in the first 10 tags")

def get_version_from_rev(repo, rev):
    return fetch(CUR_VERSION_URL.format(repo=repo, rev=rev))

def get_versions(major_version):
    versions = {}

    esr_tags = fetch(MERCURIAL_TAGS_URL.format(major=major_version), data_type="json")
    prev_esr_rev = get_prev_release_rev(esr_tags["tags"])
    versions["prev_esr_rev"] = prev_esr_rev
    prev_esr_version = get_version_from_rev(f"comm-esr{major_version}", prev_esr_rev)
    versions["previous_esr"] = prev_esr_version

    this_esr_version = get_version_from_rev(f"comm-esr{major_version}", "tip")
    versions["this_esr"] = this_esr_version

    return versions

def main():
    parser = argparse.ArgumentParser(description="Make ESR notes")
    parser.add_argument(
        "major_ver",
        metavar="major_version",
        choices=ESR_VERSIONS,
        help="Major ESR version",
    )
    args = parser.parse_args()

    versions = get_versions(args.major_ver)

    load_notes(versions["this_esr"], versions["previous_esr"], versions["prev_esr_rev"])

if __name__ == "__main__":
    main()


import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from sklearn.linear_model import LinearRegression
from scipy.stats import linregress
from sklearn.preprocessing import PolynomialFeatures
import statsmodels.api as sm

df = pd.read_excel(r'E:\大一下学期\SI100\Project\cv信息的提取_汇总_新_带领域(最终版本).xlsx')
groups = df.groupby(df.columns[2])
fields = {
    'Economics of Education': 'education',
    'Applied Microeconomics': 'micro',
    'Labor Economics': 'labor',
    'Environment Economics': 'environ',
    'Public Economics': 'public',
    'Macroeconomics': 'macro',
    'Behavioral Economics': 'behav',
    'Political Economics': 'poli',
    'International Trade': 'trade'or'inter',
    'Regional Economics': 'region',
    'Finance': 'finan',
    'Industrial Organization': 'organization',
    'Game Theory': 'game',
    'Monetary Economics': 'monetary',
    'Urban Economics': 'urban',
    'Econometrics': 'econometrics',
    'Mathematical Economics': 'math',
    'Health Economics': 'health',
    'Economics of Population': 'popu'
}

result = []
for group_name, data in groups:
    data_str = ','.join(data[data.columns[4]].astype(str))
    data_str = data_str.lower()
    GDP = data[data.columns[3]].unique().astype(int)
    GDP = GDP.item()
    GDP = GDP / 10000
    pop = data[data.columns[5]].unique().astype(int)
    pop = pop.item()
    pop = pop / 1000000
    field_counts = {field: 0 for field in fields}

    for field, keyword in fields.items():
        count = data_str.count(keyword)
        field_counts[field] += count

    score = 0
    for field, count in field_counts.items():
        if field == 'Economics of Education':
            score += 5 * count
        elif field == 'Applied Microeconomics':
            score += 5 * count
        elif field == 'Labor Economics':
            score += 5 * count
        elif field == 'Environment Economics':
            score += 4 * count
        elif field == 'Public Economics':
            score += 4 * count
        elif field == 'Macroeconomics':
            score += 8 * count
        elif field == 'Behavioral Economics':
            score += 4 * count
        elif field == 'Political Economics':
            score += 3 * count
        elif field == 'International Trade':
            score += 6 * count
        elif field == 'Regional Economics':
            score += 2 * count
        elif field == 'Finance':
            score += 7 * count
        elif field == 'Industrial Organization':
            score += 7 * count
        elif field == 'Game Theory':
            score += 7 * count
        elif field == 'Monetary Economics':
            score += 8 * count
        elif field == 'Urban Economics':
            score += 4 * count
        elif field == 'Econometrics':
            score += 10 * count
        elif field == 'Mathematical Economics':
            score += 10 * count
        elif field == 'Health Economics':
            score += 4 * count
        elif field == 'Economics of Population':
            score += 4 * count

    line_count = len(data)
    data_count = sum(len(item.split(',')) for item in data[data.columns[4]].astype(str))

    result.append({'国家': group_name, '人数': len(data), '国家得分均值': score / data_count, '人均GDP(万美元)': GDP, '人口(百万人)': pop})

result_df = pd.DataFrame(result)
print(result_df)
column_names = result_df.columns
plt.scatter(result_df[column_names[3]], result_df[column_names[2]], s=result_df[column_names[1]], alpha=0.5)
plt.show()

variables = result_df.iloc[:, [3, 4]].values
y = result_df.iloc[:, 2].values
variables = sm.add_constant(variables)
model = sm.OLS(y, variables)
results = model.fit()
print(results.summary())


import stanza
import nltk
import keyword_list as kl
#Uncomment if you want to use this file separatly
# nlp = stanza.Pipeline("en")

def test_stanza(sentence,indice):
    '''
    input:
        sentence: str
        indice: int

    output:
        dict

    Cette fonction prend en argument une phrase et un indice i et
    renvoie les informations stanza du i-eme mot de la phrase sous la
    forme d'un dictionnaire
    '''
    doc = nlp(sentence)
    phrase = doc.sentences[0].to_dict()
    return phrase[indice]

def is_commer(sword):
    '''
    renvoie True si le mot est ','  ';'  ou  ':'
    et False sinon
    '''
    return sword['lemma'] in [',',';',':']

def Conjugaison(sentence,indice,time,method='Any'):
    '''
    input:
        sentence: str
        indice: int
        time: str
        method: str

    output:
        bool

    Cette fonction prend en argument une phrase contenant un mot clé se trouvant à
    la position indice ainsi qu'un temps (exemple 'Past') et un méthode qui peut
    être un entier, 'Any', 'Next' ou 'UC' (untill commer) la fonction renvoie True si il existe un verbe
    conjugué au temps time dans la phrase.

    method:     int -> on regarde à la position indice + method
                'Any' -> on regarde partout
                'UC' -> on regarde jusqu'à la virgule
                'Next' -> on regarde le premier verbe (ou jusqu'à la virgule)

    Exemple:    sentence: 'If I were here'
                indice: 0
                time: 'Past'
                method: 2
    Cet appel renvoie True car à la position indice + method = 2 il y a bien un verbe
    'were' conjugué au passé.

    Remarque: la méthode 'Any' permet de rechercher n'importe où dans la phrase
    '''
    if time == 'Past':
        time_condition = ['Tense=Past']
    elif time == '-ing':
        time_condition = ['Tense=Pres|VerbForm=Part','VerbForm=Ger']
    else:
        raise Exception('''Le temps rentré n'est pas pris en charge''')
    doc = nlp(sentence)
    phrase = doc.sentences[0].to_dict()
    if method=='Any':
        for mot in phrase:
            if 'feats' in mot:
                for tc in time_condition:
                    if tc in mot['feats']:
                        return True
    elif method == 'UC': #untill commer
        for k in range(indice,len(phrase)):
            mot = phrase[k]
            if is_commer(mot):
                break
            if 'feats' in mot:
                for tc in time_condition:
                    if tc in mot['feats']:
                        return True
    elif method == 'Next':
        for k in range(indice,len(phrase)):
            mot = phrase[k]
            if is_commer(mot):
                break
            if mot['upos'] == 'VERB' or mot['upos'] == 'AUX':
                if 'feats' in mot:
                    for tc in time_condition:
                        if tc in mot['feats']:
                            return True
                    return False
                break
    elif type(method) == int:
        if indice+method < 0 or indice+method >= len(phrase):
            return False
        mot = phrase[indice+method]
        if 'feats' in mot:
            for tc in time_condition:
                if tc in mot['feats']:
                    return True
    return False

def sentencify(text):
    '''
    Input : Un texte complet (string).
    Output : Une liste contenant la liste des phrases du texte ainsi
    que le numéro de la phrase,
    l'indice de début et de fin (list of (string,int,int,int)).
    '''
    liste_phrases = []
    debut_phrase = 0
    num_phrase = 0
    indice_texte = 0
    while indice_texte < len(text):
        if text[indice_texte] in ['.','!','?']:
            if indice_texte+2<len(text) and text[indice_texte+2] == '.':
                indice_texte += 2
            liste_phrases.append((text[debut_phrase:indice_texte+1],num_phrase,debut_phrase,indice_texte))
            debut_phrase = indice_texte + 2
            num_phrase += 1
        indice_texte += 1
    return liste_phrases

def wordify(sentence):
    '''
    Entree : Une phrase (string).
    Sortie : liste des mots constitutifs de la phrase (list of string).
    '''
    return nltk.word_tokenize(sentence)

def is_exception(sentence,list_excep,indice):
    '''
    input:
        sentence: str
        list_excep: list((str,method,str)) avec method qui peut être un int ou 'Any'
        indice: int

    output:
        bool

    la fonction renvoie True si le mot clé à la position indice dans la phrase est une exception
    selon la règle donnée dans list_excep

    Remarque : list_excep ne contient qu'une seule exception, c'est une liste car une exception peut être issue
    de plusieurs conditions exemple When + passé + ! => exception

    Remarque 2 : La terminologie est trompeuse, cette fonction permet à la fois de savoir si le mot
    clé est une exception et si c'est une forme composite.
    '''
    for excep in list_excep:
        if excep[0] == 'Word':
            liste_mots = wordify(sentence.upper())
            if type(excep[1]) == int:
                if ((indice + excep[1] < 0) or (indice + excep[1] >= len(liste_mots)) or (liste_mots[indice+excep[1]] != excep[2])):
                    return False
            else:
                #Rentrer ici les différents mots clés d'emplacement
                if  excep[1] == 'Any':
                    for k in range(len(liste_mots)):
                        if not liste_mots[k] == excep[2]:
                            return False
                else:
                    raise Exception("Cette méthode n'est pas traitée")
        elif excep[0] == 'Verb':
            if not Conjugaison(sentence,indice,excep[2],method=excep[1]):
                return False
        elif excep[0] == 'Upos':
            liste_mots = wordify(sentence.upper())
            if type(excep[1]) == int:
                if (indice + excep[1] < 0) or (indice + excep[1] >= len(liste_mots)):
                    return False
                phrase = nlp(sentence)
                sword = phrase.sentences[0].to_dict()[indice + excep[1]]
                return ('upos' in sword) and (excep[2] in sword['upos'])
            else:
                raise Exception('''Pour l'instant, le upos n'est traité qu'avec la méthode int''')
    return True

def is_rule(sentence):
    '''
    Entree : une phrase (string).
    Sortie : un booléen qui donne si la phrase est une condition (bool).
    '''
    if len(sentence) == 0 or sentence[-1] == '?':
        return False
    liste_mots = wordify(sentence.upper())
    for i in range(len(liste_mots)):
        if liste_mots[i] in kl.keyword_list:
            is_rule = True
            if 'Composite' in kl.keyword_list[liste_mots[i]]:
                is_rule = False
                for list_comp in kl.keyword_list[liste_mots[i]]['Composite']:
                    if is_exception(sentence,list_comp,i):
                        is_rule = True
                        break
            if is_rule and 'Exception' in kl.keyword_list[liste_mots[i]]:
                for list_excep in kl.keyword_list[liste_mots[i]]['Exception']:
                    if is_exception(sentence,list_excep,i):
                        is_rule = False
                        break
            if is_rule:
                return True
    return False

def keyword_extract(text):
    '''
    Entree : un texte (string).
    Sortie : une liste de couple (phrase,indice) où phrase est une phrase
    contenant une condition et indice est le numéro (commençant à 0) de la phrase
    en question (list of (string,int)).
    '''
    indices = []
    i = 0
    liste_phrases = sentencify(text)
    for sentence in [sent for sent,_,_,_ in liste_phrases]:
        su = sentence.upper()
        if is_rule(su):
            indices.append(i)
        i += 1
    return [(liste_phrases[i]) for i in indices]

print(sentencify('If yes then no. Do not worry.'))


# -*- coding: utf-8 -*-
"""
Created on Sun Jul 26 14:12:50 2020
Models for AE-Classifier
@author: Lahiru D. Chamain
"""

import tensorflow as tf
from ops import resblock,bottle_resblock,get_residual_layer,conv,batch_norm,max_pooling,global_avg_pooling,fully_conneted,relu
from ops import entropyLoss
from utils import quantize,transpose_NHWC_to_NCHW,_Network3D, bitcost_to_bpp
from collections import namedtuple

# z is the bottleneck before quantization
EncoderOutput = namedtuple('EncoderOutput', ['qbar', 'qhard', 'symbols', 'z'])

# returned by _Network._quantize
_QuantizerOutput = namedtuple('_QuantizerOutput', ['qbar', 'qsoft', 'qhard', 'symbols'])

class AE(object):
        def __init__(self,codec = 'v3',ncenters=2,H_target=0.4,res_n=18,label_dim=1000,is_training=True,reuse = False):
            self.is_training = is_training
            self.reuse = reuse
            self.res_n = res_n
            self.label_dim=label_dim
            self.ncenters = ncenters
            self._centers = self.createCenters(self.ncenters,-2,2)
            self.pc = _Network3D(kernel_size=3, num_centers=self.ncenters) ##kernelsize,numcenters
            self.H_target = H_target
            self.codec = codec

        def createCenters(self,num_cernters,rmin,rmax):
            centerInitializer = tf.random_uniform_initializer(minval=rmin, maxval=rmax, seed=666)
            centerVariable = tf.get_variable('centers', shape=(num_cernters,), dtype=tf.float32,
                                            initializer=centerInitializer)
            return centerVariable

        def quantizer(self, x, data_format = 'NHWC'):
                with tf.variable_scope("quantizer", reuse=self.reuse):
                    qsoft, qhard, symbols = quantize(x, self._centers, sigma=1,data_format = data_format)
                    with tf.name_scope('qbar'):
                        qbar = qsoft + tf.stop_gradient(qhard - qsoft)
                    return _QuantizerOutput(qbar, qsoft, qhard, symbols)
                    return x
        ##################################################################################
        # Encoders
        ##################################################################################

        def encoderV3(self, x):
            with tf.variable_scope("encoder", reuse=self.reuse):

                if self.res_n < 50 :
                    residual_block = resblock
                else :
                    residual_block = bottle_resblock

                residual_list = get_residual_layer(self.res_n)

                ch =32
                x = conv(x, channels=ch, kernel=7, stride=2, scope='conv1') #80x64
                x = max_pooling(x,kernel=3,stride=2,name='maxpool1')

                for i in range(residual_list[0]) :
                    x = residual_block(x, channels=ch, is_training=self.is_training, downsample=False, scope='resblock0_' + str(i)) #40x32
                x = residual_block(x, channels=ch, is_training=self.is_training, downsample=True, scope='resblock1_0')

                for i in range(1, residual_list[1]) :
                    x = residual_block(x, channels=ch, is_training=self.is_training, downsample=False, scope='resblock1_' + str(i)) #20x32

                x = residual_block(x, channels=ch, is_training=self.is_training, downsample=False, scope='resblock2_0_0') #20x32
                x = batch_norm(x, self.is_training, scope='batch_norm')
                qout = self.quantizer(x,data_format = 'NCHW')
                return EncoderOutput(qout.qbar, qout.qhard, qout.symbols, x)

        def encoderV4(self, x):
            with tf.variable_scope("encoder", reuse=self.reuse):

                if self.res_n < 50 :
                    residual_block = resblock
                else :
                    residual_block = bottle_resblock

                residual_list = get_residual_layer(self.res_n)

                ch =128
                x = conv(x, channels=ch, kernel=7, stride=2, scope='conv1')
                x = max_pooling(x,kernel=3,stride=2,name='maxpool1')

                for i in range(residual_list[0]) :
                    x = residual_block(x, channels=ch, is_training=self.is_training, downsample=False, scope='resblock0_' + str(i)) #40x32
                x = residual_block(x, channels=ch, is_training=self.is_training, downsample=True, scope='resblock1_0')

                for i in range(1, residual_list[1]) :
                    x = residual_block(x, channels=ch, is_training=self.is_training, downsample=False, scope='resblock1_' + str(i)) #20x32

                x = residual_block(x, channels=ch, is_training=self.is_training, downsample=False, scope='resblock2_0_0')
                x = conv(x, channels=ch//4, kernel=3, stride=1, scope='conv2')
                x = batch_norm(x, self.is_training, scope='batch_norm')
                qout = self.quantizer(x,data_format = 'NCHW')

                return EncoderOutput(qout.qbar, qout.qhard, qout.symbols, x)

        ##################################################################################
        # Classifiers
        ##################################################################################
        def classifierV3(self, x):
            with tf.variable_scope("classifier", reuse=self.reuse):
                    if self.res_n < 50 :
                        residual_block = resblock
                    else :
                        residual_block = bottle_resblock
                    residual_list = get_residual_layer(self.res_n)

                    ch = 64
                    x = conv(x, channels=ch*4, kernel=3, stride=1, scope='conv3')

                    x = residual_block(x, channels=ch*4, is_training=self.is_training, downsample=False, scope='resblock2_0_1')

                    for i in range(1, residual_list[2]) :
                        x = residual_block(x, channels=ch*4, is_training=self.is_training, downsample=False, scope='resblock2_' + str(i)) #10x256

                    x = residual_block(x, channels=ch*8, is_training=self.is_training, downsample=True, scope='resblock_3_0')

                    for i in range(1, residual_list[3]) :
                        x = residual_block(x, channels=ch*8, is_training=self.is_training, downsample=False, scope='resblock_3_' + str(i)) #5x512

                    x = batch_norm(x, self.is_training, scope='batch_norm')
                    x = relu(x)

                    x = global_avg_pooling(x)
                    x = fully_conneted(x, units=self.label_dim, scope='logit')
                    return x

        ##################################################################################
        # Models
        ##################################################################################
        def networkV3(self, x):
            with tf.variable_scope("network", reuse=self.reuse):

                z = self.encoderV3(x)
                y = self.classifierV3(z.qbar)
                pc_in = z.qbar
                pc_in = transpose_NHWC_to_NCHW(pc_in)
                bc_train = self.pc.bitcost(pc_in, transpose_NHWC_to_NCHW(z.symbols), is_training=self.is_training, pad_value=self.pc.auto_pad_value(self._centers))
                bpp_train = bitcost_to_bpp(bc_train, transpose_NHWC_to_NCHW(x))
                H_loss = entropyLoss(bpp_train,self.H_target)

                return  y,bpp_train,H_loss

        def networkV4(self, x):
            with tf.variable_scope("network", reuse=self.reuse):

                z = self.encoderV4(x)
                y = self.classifierV3(z.qbar)
                pc_in = z.qbar
                pc_in = transpose_NHWC_to_NCHW(pc_in)
                bc_train = self.pc.bitcost(pc_in, transpose_NHWC_to_NCHW(z.symbols), is_training=self.is_training, pad_value=self.pc.auto_pad_value(self._centers))
                bpp_train = bitcost_to_bpp(bc_train, transpose_NHWC_to_NCHW(x))
                H_loss = entropyLoss(bpp_train,self.H_target)

                return  y,bpp_train,H_loss

        def network(self,x):
            if(self.codec == 'v3'):
                return self.networkV3(x)
            elif(self.codec == 'v4'):
                return self.networkV4(x)
            else:
                print('Unknown configuration')
                return None


import time
import scrapy
from autohomebot.items import PcautoItem

class PcautoSpider(scrapy.Spider):
    name = 'pcauto'
    allowed_domains = ['price.pcauto.com.cn']
    start_urls = [
        'https://price.pcauto.com.cn/comment/sg90/'
    ]

    def info(self, message, isPrint=True):
        # 控制台显示消息
        if isPrint == True:
            print('[' + time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(time.time())) + '][INFO]' + message)

        # Log文件输出
        self.logger.info(message)

    def warning(self, message, logOutput=True):
        # 控制台显示消息
        print('[' + time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(time.time())) + '][WARNING]' + message)

        # Log文件输出
        if logOutput == True:
            self.logger.warning(message)

    def error(self, message, logOutput=True):
        # 控制台显示消息
        print('[' + time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(time.time())) + '][ERROR]' + message)

        # Log文件输出
        if logOutput == True:
            self.logger.error(message)

    # 　获取评论详情
    def parse(self, response):

        self.info('太平洋汽车网:[{}]、.状态:[{}]'.format(response.url, response.status))
        title = response.xpath('//h1/text()').extract_first()
        try:
            if response.xpath('//*[@class="next"]'):
                next_page = response.xpath('//*[@class="next"]/@href').extract_first()
                next_url = "https:" + next_page
                yield scrapy.Request(next_url,callback=self.parse)
                # print(next_url)
            for each in response.xpath('//*[@class="litDy clearfix"]'):
                item = PcautoItem()
                username = each.xpath('.//*[@class="txline"]/p/a/text()').extract_first()
                pushtime = each.xpath('.//*[@class="txline"]/span/a/text()').extract_first()
                car_type = each.xpath('.//*[@class="line"][1]/a/text()').extract_first()
                buy_time = each.xpath('.//*[@class="line" and em[contains(text(),"购买时间")]]/text()').extract_first()
                buy_loca = each.xpath('.//*[@class="line" and em[contains(text(),"购买地点")]]/text()').extract_first()
                buy_shop = each.xpath('.//*[@class="line" and em[contains(text(),"购买商家")]]/text()').extract_first()
                buy_cost = each.xpath('.//*[@class="line" and em[contains(text(),"裸车价格")]]/i/text()').extract_first()
                oil_cost = each.xpath('.//*[@class="line" and em[contains(text(),"平均油耗")]]/i/text()').extract_first()
                mileage_ = each.xpath('.//*[@class="line" and em[contains(text(),"行驶里程")]]/text()').extract_first()
                buyimage = each.xpath('.//*[@class="toptit clearfix"]/a/text()').extract()
                cmt_detl = each.xpath('.//*[@class="dianPing clearfix"]')
                comment_detail = {}
                if cmt_detl:
                    comment_detail["优点"] = cmt_detl.xpath('./div[b[contains(text(),"优点")]]/span/text()').extract_first()
                    comment_detail["缺点"] = cmt_detl.xpath('./div[b[contains(text(),"缺点")]]/span/text()').extract_first()
                    comment_detail["外观"] = cmt_detl.xpath('./div[b[contains(text(),"外观")]]/span/text()').extract_first()
                    comment_detail["内饰"] = cmt_detl.xpath('./div[b[contains(text(),"内饰")]]/span/text()').extract_first()
                    comment_detail["空间"] = cmt_detl.xpath('./div[b[contains(text(),"空间")]]/span/text()').extract_first()
                    comment_detail["配置"] = cmt_detl.xpath('./div[b[contains(text(),"配置")]]/span/text()').extract_first()
                    comment_detail["动力"] = cmt_detl.xpath('./div[b[contains(text(),"动力")]]/span/text()').extract_first()
                    comment_detail["操控"] = cmt_detl.xpath('./div[b[contains(text(),"操控")]]/span/text()').extract_first()
                    comment_detail["油耗"] = cmt_detl.xpath('./div[b[contains(text(),"油耗")]]/span/text()').extract_first()
                    comment_detail["舒适"] = cmt_detl.xpath('./div[b[contains(text(),"舒适")]]/span/text()').extract_first()

                # print(username, pushtime, car_type, buy_time, buy_loca, buy_shop, buy_cost, oil_cost, mileage_, zh_Score,
                #       buyimage, cmt_detl)
                item['username'] = username
                item['pushtiem'] = pushtime
                item['car_type'] = car_type
                item['buy_time'] = buy_time
                item['buy_loca'] = buy_loca
                if buy_shop:
                    item['buy_shop'] = buy_shop.strip()
                else:
                    item['buy_shop'] = None
                if buy_cost:
                    item['buy_cost'] = buy_cost + '万元'
                else:
                    item['buy_cost'] = None
                if oil_cost:
                    item['oil_cost'] = oil_cost + 'L/100km'
                else:
                    item['oil_cost'] = None
                item['mileage_'] = mileage_
                item['carimage'] = buyimage
                item['cmt_detl'] = comment_detail
                item['update_datetime'] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                item['collection'] = title
                yield item
        except Exception as e:
            self.error('【parse_detail出错】{},{}'.format(response.url, e))


import os
import pandas as pd
import csv
import requests as r
import datetime as dt
import re

# All Events Data

url="https://nuforc.org/webreports/ndxevent.html"

resp=r.get(url).text
html=resp.split("
')[0].strip()
    row['record_key']=each_record.split('>')[0].split('.html')[0].strip()
    row['record_date']=each_record.split('>')[1].strip('')[4].strip('</TD')
    records.append(row)

# df=pd.DataFrame(records)
# df

eventkeys=row.keys()

# All Events Data - truth checker
with open('all_events.csv','w',newline='') as recordsfile:
    dict_writer=csv.DictWriter(recordsfile, eventkeys)
    dict_writer.writeheader()
    dict_writer.writerows(records)

# All States

states = r.get(url).text
states_records=states.split("
<A HREF=")[1:]

# Alternative approach
# data = pd.read_html('https://nuforc.org/webreports/ndxe202303.html')
# data[0]
# :D


import argparse
import collections
import difflib
import enum
import hashlib
import operator
import os
import stat
import struct
import sys
import time
import zlib

# git索引(.git/index)中的数据
# 相当于只有属性没有方法的类
IndexEntry = collections.namedtuple('IndexEntry',
                                    [
                                        'ctime_s', 'ctime_n', 'mtime_s', 'mtime_n', 'dev', 'ino', 'mode', 'uid',
                                        'gid', 'size', 'sha1', 'flags', 'path',
                                    ])

class ObjectType(enum.Enum):
    """
    enum,Git对象的类型 -> blobs, commits, trees
    """
    commit = 1
    tree = 2
    blob = 3

def read_file(path):
    """
    读取path指向的文件
    :param path:文件路径
    :return data:byte data 文件数据
    """
    with open(path, 'rb') as f:
        return f.read()

def write_file(path, data):
    """
    将byte data写入path指向的文件
    :param path:文件路径
    :param data:写入数据
    """
    with open(path, 'wb') as f:
        f.write(data)

def init(repo):
    """
    初始化 .git 目录
    :param repo:根文件路径
    :return:
    """
    os.mkdir(repo)
    os.mkdir(os.path.join(repo, '.git'))

    for name in ['objects', 'refs', 'refs/heads']:
        os.mkdir(os.path.join(repo, '.git', name))

    write_file(os.path.join(repo, '.git', 'HEAD'), b'ref: refs/heads/master')
    print('initialized empty repository: {}'.format(repo))

def hash_object(data, object_type, write=True):
    """
    计算给定类型对象数据的哈希值并写入IndexEntry
    如果“写”为真,将SHA-1 hash作为十六进制字符串返回
    :param data:byte 数据
    :param object_type:Git对象的类型 -> blobs, commits, trees
    :param write:标志位,是否写入
    :return sha1:hash ID
    """
    header = '{} {}'.format(object_type, len(data)).encode()
    full_data = header + b'\x00' + data
    sha1 = hashlib.sha1(full_data).hexdigest()
    if write:
        path = os.path.join('.git', 'objects', sha1[:2], sha1[2:])
        if not os.path.exists(path):
            os.makedirs(os.path.dirname(path), exist_ok=True)
            write_file(path, zlib.compress(full_data))
    return sha1

def find_object(sha1_prefix):
    """
    查找具有给定SHA-1前缀的对象并返回对象中对象的路径
    如果没有对象或多个对象，则引发ValueError带有这个SHA-1前缀
    :param sha1_prefix:SHA-1前缀
    :return:具有SHA-1前缀对象的路径
    """
    if len(sha1_prefix) < 2:
        raise ValueError('哈希前缀必须是2个或更多字符')
    obj_dir = os.path.join('.git', 'objects', sha1_prefix[:2])
    rest = sha1_prefix[2:]
    objects = [name for name in os.listdir(obj_dir) if name.startswith(rest)]
    if not objects:
        raise ValueError('object {!r} not found'.format(sha1_prefix))
    if len(objects) >= 2:
        raise ValueError('多个 objects ({}) 带有前缀 {!r}'.format(
            len(objects), sha1_prefix))
    return os.path.join(obj_dir, objects[0])

def read_object(sha1_prefix):
    """
    读取具有给定SHA-1前缀的对象并返回元组(object_type, data_bytes)
    如果未找到，则引发 ValueError。
    :param sha1_prefix:SHA-1前缀
    :return (object_type, data_bytes):返回以sha1_prefix为SHA-1前缀对象的data&type
    """
    path = find_object(sha1_prefix)
    full_data = zlib.decompress(read_file(path))
    nul_index = full_data.index(b'\x00')
    header = full_data[:nul_index]
    obj_type, size_str = header.decode().split()
    size = int(size_str)
    data = full_data[nul_index + 1:]
    assert size == len(data), '预期大小 {}，得到 {} 个字节'.format(
        size, len(data))
    return (obj_type, data)

def cat_file(mode, sha1_prefix):
    """
    将具有给定 SHA-1 前缀的对象的内容（或信息）进行标准输出
    如果 mode 是 'commit'、'tree' 或 'blob'，则打印对象的数据字节
    如果模式是'size'，则打印对象的大小
    如果模式是'type',则打印对象的类型
    如果模式是'pretty'，则打印一个对象的美化版本
    :param mode:'commit', 'tree', 'blob', 'size', 'type', 'pretty'
    :param sha1_prefix:SHA-1前缀
    """
    obj_type, data = read_object(sha1_prefix)
    if mode in ['commit', 'tree', 'blob']:
        if obj_type != mode:
            raise ValueError('expected object type {}, get {}'.format(
                mode, obj_type))
        # 标准输出
        sys.stdout.buffer.write(data)
    elif mode == 'size':
        print(len(data))
    elif mode == 'type':
        print(obj_type)
    elif mode == 'pretty':
        if obj_type in ['commit', 'blob']:
            sys.stdout.buffer.write(data)
        elif obj_type == 'tree':
            for mode, path, sha1 in read_tree(data=data):
                type_str = 'tree' if stat.S_ISDIR(mode) else 'blob'
                print('{:06o} {} {}\t{}'.format(mode, type_str, sha1, path))
        else:
            assert False, 'unhandled object type {!r}'.format(obj_type)
    else:
        raise ValueError('unexpected mode {!r}'.format(mode))

def read_index():
    """
    读取 git 索引文件并返回 IndexEntry 对象列表。
    git index 前12字节为标头,末尾20字节为SHA-1,其余字节每隔62字节添加路径数据
    :return entries:IndexEntry 对象列表
    """
    try:
        data = read_file(os.path.join('.git', 'index'))
    except FileNotFoundError:
        return []
    # 设置git index
    digset = hashlib.sha1(data[:-20]).digest()  # 后20字节为SHA-1
    assert digset == data[-20:], 'invalid index checksum'
    # 签名、版本、条目数
    signature, version, num_entries = struct.unpack('!4sLL', data[:12])  # 前12字节为标头,4sLL -> 12字节 -> 4*char + long + long
    assert signature == b'DIRC', 'invalid index signature {}'.format(signature)
    assert version == 2, 'unknown index version {}'.format(version)
    # 数据
    entry_data = data[12:-20]
    entries = []
    i = 0
    while i + 62 < len(entry_data):
        fields_end = i + 62
        # LLLLLLLLLL20sH -> 62字节 -> 10*long + 20*char + short
        fields = struct.unpack('!LLLLLLLLLL20sH', entry_data[i:fields_end])
        path_end = entry_data.index(b'\x00', fields_end)
        path = entry_data[fields_end:path_end]
        entry = IndexEntry(*(fields + (path.decode(),)))
        entries.append(entry)
        entry_len = ((62 + len(path) + 8) // 8) * 8
        i += entry_len
    assert len(entries) == num_entries
    return entries

def ls_files(details=False):
    """
    打印IndexEntry中的所有文件（以及它们的模式和哈希，如果 -s已指定）
    :param details: -s 是否被指定
    :return:
    """
    for entry in read_index():
        if details:
            stage = (entry.flags >> 12) & 3
            print('{:6o} {} {:}\t{}'.format(entry.mode, entry.sha1.hex(), stage, entry.path))
        else:
            print(entry.path)

def get_status():
    """
    将IndexEntry中的文件与当前目录树中的文件进行比较,获取工作的状态
    :return (sorted(changed), sorted(new), sorted(deleted)):status 工作的状态
    """
    paths = set()
    for root, dirs, files in os.walk('.'):
        dirs[:] = [d for d in dirs if d != '.git']
        for file in files:
            path = os.path.join(root, file)
            path = path.replace('\\', '/')
            if path.startswith('./'):
                path = path[2:]
            paths.add(path)
    entries_by_path = {e.path: e for e in read_index()}
    entry_paths = set(entries_by_path)
    changed = {p for p in (paths & entry_paths)
                               if hash_object(read_file(p), 'blob', write=False) !=
                               entries_by_path[p].sha1.hex()}
    new = paths - entry_paths
    deleted = entry_paths - paths
    return (sorted(changed), sorted(new), sorted(deleted))

def status():
    """
    显示git状态，打印出哪些文件被修改、新建和删除
    """
    changed, new, deleted = get_status()
    if changed:
        print('changed files:')
        for path in changed:
            print('   ', path)
    if new:
        print('new files:')
        for path in new:
            print('   ', path)
    if deleted:
        print('deleted files:')
        for path in deleted:
            print('   ', path)

def diff():
    """
    打印每个修改文件的差异
    """
    changed, _, _ = get_status()
    entries_by_path = {e.path: e for e in read_index()}
    for i, path in enumerate(changed):
        sha1 = entries_by_path[path].sha1.hex()
        obj_type, data = read_object(sha1)
        assert obj_type == 'blob'
        index_lines = data.decode().splitlines()
        working_lines = read_file(path).decode().splitlines()
        diff_lines = difflib.unified_diff(
            index_lines, working_lines,
            '{} (index)'.format(path),
            '{} (working copy)'.format(path),
            lineterm='')
        for line in diff_lines:
            print(line)
        if i < len(changed) - 1:
            print('-' * 70)

def write_index(entries):
    """
    将 IndexEntry 对象列表写入 git 索引文件
    :param entries:IndexEntry 对象列表
    """
    packed_entries = []
    for entry in entries:
        entry_head = struct.pack('!LLLLLLLLLL20sH',
                                 entry.ctime_s, entry.ctime_n, entry.mtime_s, entry.mtime_n,
                                 entry.dev, entry.ino, entry.mode, entry.uid, entry.gid,
                                 entry.size, entry.sha1, entry.flags)
        path = entry.path.encode()
        length = ((62 + len(path) + 8) // 8) * 8
        packed_entry = entry_head + path + b'\x00' * (length - 62 - len(path))
        packed_entries.append(packed_entry)
    header = struct.pack('!4sLL', b'DIRC', 2, len(entries))
    all_data = header + b''.join(packed_entries)
    digest = hashlib.sha1(all_data).digest()
    write_file(os.path.join('.git', 'index'), all_data + digest)

def add(paths):
    """
    将所有文件路径添加到 git index
    :param paths:文件路径
    """
    paths = [p.replace('\\', '/') for p in paths]
    all_entries = read_index()
    entries = [e for e in all_entries if e.path not in paths]
    for path in paths:
        sha1 = hash_object(read_file(path), 'blob')
        st = os.stat(path)
        flags = len(path.encode())
        assert flags < (1 << 12)
        entry = IndexEntry(
            int(st.st_ctime), 0, int(st.st_mtime), 0, st.st_dev,
            st.st_ino, st.st_mode, st.st_uid, st.st_gid, st.st_size,
            bytes.fromhex(sha1), flags, path)
        entries.append(entry)
        print('Add Succeed')
    entries.sort(key=operator.attrgetter('path'))
    write_index(entries)

def write_tree():
    """
    从当前IndexEntry写入树对象
    :return:hash ID tree
    """
    tree_entries = []
    for entry in read_index():
        assert '/' not in entry.path, '当前仅支持单个顶级目录'
        mode_path = '{:o} {}'.format(entry.mode, entry.path).encode()
        tree_entry = mode_path + b'\x00' + entry.sha1
        tree_entries.append(tree_entry)
    return hash_object(b''.join(tree_entries), 'tree')

def read_tree(sha1=None, data=None):
    """Read tree object with given SHA-1 (hex string) or data, and return list
    of (mode, path, sha1) tuples.
    """
    if sha1 is not None:
        obj_type, data = read_object(sha1)
        assert obj_type == 'tree'
    elif data is None:
        raise TypeError('must specify "sha1" or "data"')
    i = 0
    entries = []
    for _ in range(1000):
        end = data.find(b'\x00', i)
        if end == -1:
            break
        mode_str, path = data[i:end].decode().split()
        mode = int(mode_str, 8)
        digest = data[end + 1:end + 21]
        entries.append((mode, path, digest.hex()))
        i = end + 1 + 20
    return entries

def get_local_master_hash():
    """
    获取本地主分支当前的提交Hash（SHA-1 字符串）
    :return SHA1 String:主分支当前的提交Hash || None
    """
    master_path = os.path.join('.git', 'refs', 'heads', 'master')
    try:
        return read_file(master_path).decode().strip()
    except FileNotFoundError:
        return None

def commit(message, author=None):
    """
    使用给定的消息将索引的当前状态提交给 master。
    返回提交对象的哈希
    :param message:commit信息
    :param author:提交者信息
    :return sha1:commit SHA-1 hash ID
    """
    tree = write_tree()  # 当前目录（或IndexEntry）的快照,tree object
    parent = get_local_master_hash()  # 上一节点
    if author is None:
        author = '{} <{}>'.format('MorbidArk', '1482506841@qq.com')
    timestamp = int(time.mktime(time.localtime()))
    utc_offset = -time.timezone
    author_time = '{} {}{:02}{:02}'.format(timestamp,
                                           '+' if utc_offset > 0 else '-',
                                           abs(utc_offset) // 3600,
                                           (abs(utc_offset) // 60) % 60)
    lines = ['tree' + tree]
    if parent:
        lines.append('parent ' + parent)
    lines.append('author {} {}'.format(author, author_time))
    lines.append('committer {} {}'.format(author, author_time))
    lines.append('')
    lines.append(message)
    lines.append('')
    data = '\n'.join(lines).encode()
    sha1 = hash_object(data, 'commit')
    master_path = os.path.join('.git', 'refs', 'heads', 'master')
    write_file(master_path, (sha1 + '\n').encode())
    print('committed to master: {:7}'.format(sha1))
    return sha1

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    sub_parsers = parser.add_subparsers(dest='command', metavar='command')
    sub_parsers.required = True

    sub_parser = sub_parsers.add_parser('add',
                                        help='add file(s) to index')
    sub_parser.add_argument('paths', nargs='+', metavar='path',
                            help='path(s) of files to add')

    sub_parser = sub_parsers.add_parser('cat-file',
                                        help='display contents of object')
    valid_modes = ['commit', 'tree', 'blob', 'size', 'type', 'pretty']
    sub_parser.add_argument('mode', choices=valid_modes,
                            help='object type (commit, tree, blob) or display mode (size, '
                                 'type, pretty)')
    sub_parser.add_argument('hash_prefix',
                            help='SHA-1 hash (or hash prefix) of object to display')

    sub_parser = sub_parsers.add_parser('commit',
                                        help='commit current state of index to master branch')
    sub_parser.add_argument('-a', '--author',
                            help='commit author in format "A U Thor <author@example.com>" '
                                 '(uses GIT_AUTHOR_NAME and GIT_AUTHOR_EMAIL environment '
                                 'variables by default)')
    sub_parser.add_argument('-m', '--message', required=True,
                            help='text of commit message')

    sub_parser = sub_parsers.add_parser('diff',
                                        help='show diff of files changed (between index and working '
                                             'copy)')

    sub_parser = sub_parsers.add_parser('hash-object',
                                        help='hash contents of given path (and optionally write to '
                                             'object store)')
    sub_parser.add_argument('path',
                            help='path of file to hash')
    sub_parser.add_argument('-t', choices=['commit', 'tree', 'blob'],
                            default='blob', dest='type',
                            help='type of object (default %(default)r)')
    sub_parser.add_argument('-w', action='store_true', dest='write',
                            help='write object to object store (as well as printing hash)')

    sub_parser = sub_parsers.add_parser('init',
                                        help='initialize a new repo')
    sub_parser.add_argument('repo',
                            help='directory name for new repo')

    sub_parser = sub_parsers.add_parser('ls-files',
                                        help='list files in index')
    sub_parser.add_argument('-s', '--stage', action='store_true',
                            help='show object details (mode, hash, and stage number) in '
                                 'addition to path')

    sub_parser = sub_parsers.add_parser('status',
                                        help='show status of working copy')

    args = parser.parse_args()
    if args.command == 'add':
        add(args.paths)
    elif args.command == 'cat-file':
        try:
            cat_file(args.mode, args.hash_prefix)
        except ValueError as error:
            print(error, file=sys.stderr)
            sys.exit(1)
    elif args.command == 'commit':
        commit(args.message, author=args.author)
    elif args.command == 'diff':
        diff()
    elif args.command == 'hash-object':
        sha1 = hash_object(read_file(args.path), args.type, write=args.write)
        print(sha1)
    elif args.command == 'init':
        init(args.repo)
    elif args.command == 'ls-files':
        ls_files(details=args.stage)
    elif args.command == 'status':
        status()
    else:
        assert False, 'unexpected command {!r}'.format(args.command)


import tensorflow as tf
import numpy as np
import config.config_param as cfg

anchors = cfg.anchors
anchor_masks = cfg.anchor_masks

def yolo_head(y_pred, anchors, calc_loss=False):
    """
    yolo_head split raw prediction from yolo_body into x,y,w,h and bounding box then process with normalization
    and create grid of three sizes
    :param y_pred: raw prediction from yolo_body
    :param anchors: three different anchor box from cfg
    :param calc_loss: True when train, False when predict
    :return: bounding box from prediction and grid
    """
    grid_size = tf.shape(y_pred)[1]

    # reshape_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...classes))

    box_xy, box_wh, confidence, class_probs = tf.split(y_pred, (2, 2, 1, cfg.num_classes), axis=-1)
    # i.e. box_xy.shape = batch_size, grid, grid, num_bbox, xy
    # (8, 13, 13, 3, 2)
    # (8, 26, 26, 3, 2)
    # (8, 52, 52, 3, 2)

    # sigmoid let result lie between [0,1]
    box_xy = tf.sigmoid(box_xy)
    confidence = tf.sigmoid(confidence)
    class_probs = tf.sigmoid(class_probs)

    # create grid

    grid_y = tf.tile(tf.reshape(tf.range(grid_size), [-1, 1, 1, 1]), [1, grid_size, 1, 1])
    grid_x = tf.tile(tf.reshape(tf.range(grid_size), [1, -1, 1, 1]), [grid_size, 1, 1, 1])
    grid = tf.concat([grid_x, grid_y], axis=-1)
    grid = tf.cast(grid, tf.float32)

    # -----------------------------normalization x,y,w,h-------------------------------------------------------------
    # cell is offset from the top left corner of the image by (c x , c y ) and the bounding box prior has width and
    # height p w , p h
    # bx = σ(tx ) + cx
    # by = σ(ty ) + cy
    # bw = pwe^tw
    # bh = phe^th
    # ----------------------------------------------------------------------------------------------------------------
    box_xy = (box_xy + grid) / tf.cast(grid_size, tf.float32)
    box_wh = tf.exp(box_wh) * anchors / cfg.input_shape

    pred_xywh = tf.concat([box_xy, box_wh], axis=-1)  # original xywh for loss

    if calc_loss:
        return pred_xywh, grid

    return box_xy, box_wh, confidence, class_probs


from sortedcontainers import SortedDict
class MyCalendarThree:

    def __init__(self):
        self.d=SortedDict()

    def book(self, start: int, end: int) -> int:

        self.d[start]=self.d.get(start,0)+1

        self.d[end]=self.d.get(end,0)-1

        cur=0
        curMax=0
        for value in self.d.values():
            cur+=value
            curMax=max(cur,curMax)

        return curMax

# Your MyCalendarThree object will be instantiated and called as such:
# obj = MyCalendarThree()
# param_1 = obj.book(start,end)


import pytest

from problem_56_merge import Solution

solution = Solution()

@pytest.mark.parametrize(
    "intervals, want",
    (
        (
            [[1, 3], [2, 6], [8, 10], [15, 18]],
            [[1, 6], [8, 10], [15, 18]],
        ),
        (
            [[1, 4], [4, 5]],
            [[1, 5]],
        ),
    ),
)
def test_merge(intervals, want):
    got = solution.merge(intervals)
    assert got == want


from django.urls import path
from .views import *

login_cms = LoginCmsViewSet.as_view()

logout_cms = LogoutCmsViewSet.as_view()

urlpatterns = [
    path('login', login_cms, name='cms_login'),
    path('logout', logout_cms, name='cms_logout'),
]


"""
Develop Model Evaluation Functions: model_eval.py
    Here are the helper functions for model evaluation
"""
##### Libraries #######################################################
import pandas as pd
import numpy as np
import sys
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score, precision_score, recall_score
from statistics import mean

# extend pirectory to get helper functions
sys.path.insert(0, "./helper_functions")

from reporting import dtype_selector

# pandas config
pd.options.mode.chained_assignment = None # stops slice warning

##### Functions and Classes to be called ###########################################
def cv_mse_stats(name, model, X, y):
    """
    Calculates mean accuracy, percision, and recall

    Parameters:
    ===========
    name: String name of the trial
    model: Sklearn algorithm
    X: Dataframe or Array of features
    y: Dataframe or Array of target values

    Returns:
    ========
    scoring: array[name, mean(accuracy), mean(percision), mean(recall)]
    """
    # container
    scoring = [name]

    for scoring_metric in ["accuracy", "precision", "recall"]:
        measurement = cross_val_score(model,
                                      X,
                                      y,
                                      scoring = scoring_metric,
                                      cv = 5,
                                      verbose = 0,
                                      n_jobs = 4)

        scoring.append(mean(measurement))

    return scoring

def stratified_cv(features, labels, cv = 5, threshold = 0.5, model = None):
    """
    Performs stratified cv testing

    Parameters:
    ===========
    features: DataFrame of features (Required)
    labels: Series of target labels (Required)
    cv: int
        the number of cross validations desired
    threshold: float
        the probability threshold for class prediction
    model: estimator
        Able to take a pretrained model, or use a standard logistic regression

    Returns:
    ========
    results: Dataframe that contains AUC metrics
    """
    # initialize the splitting
    sss = StratifiedKFold(n_splits = cv)

    # create containers
    accuracy = []
    precision = []
    recall = []

    # begin cv
    for train_index, test_index in sss.split(features, labels):
        # initialize logistic regressor
        if model == None:
            model_cl = LogisticRegression(max_iter = 400, random_state = 42)

        if model != None:
            model_cl = model

        # split data
        train_features = features.iloc[train_index, :]
        test_features = features.iloc[test_index, :]
        train_labels = labels.iloc[train_index]
        test_labels = labels.iloc[test_index]

        # train model
        model_cl.fit(train_features, train_labels)

        # predict
        pred_prob = model_cl.predict_proba(test_features)
        pred = [int(i > threshold) for i in pred_prob[:, 1]]

        # scoring
        accuracy.append(accuracy_score(test_labels, pred))
        precision.append(precision_score(test_labels, pred))
        recall.append(recall_score(test_labels, pred))

    results =  pd.DataFrame(data = {
        "accuracy": accuracy,
        "precision": precision,
        "recall": recall
    })

    return results

def threshold_testing(features, labels, model, model_name):
    """
    Tests the model across mulitple thresholds

    Parameters:
    ===========
    features: Dataframe or Array
    labels: Series, array, or list
    model: estimator
    model_name: String

    Returns:
    ========
    threshold_performance_df: Dataframe
        contains
    """
    threshold_performance = []
    for threshold_value in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]:
        # get AUC measures for theshold
        results = stratified_cv(features,
                                labels,
                                threshold = threshold_value,
                                model = model)

        # organize results
        mean_results = results.mean(axis = 0)
        full_metric = [model_name,
                       threshold_value,
                       round(mean_results[0], 4),
                       round(mean_results[1], 4),
                       round(mean_results[2], 4)]

        # add to container
        threshold_performance = np.concatenate(
            (threshold_performance, full_metric),
            axis = 0
        )

    threshold_performance_df = pd.DataFrame(
        data = threshold_performance.reshape((8, 5)),
        columns = ["Model", "Threshold", "accuracy", "precision", "recall"]
    )

    return threshold_performance_df
def temp():
    """
    TEMP

    Parameters:
    ===========

    Returns:
    ========
    """
    return 0


import csv
import itertools

with open('30_25_0.001_1_hwtimes.txt', 'r') as in_file:
    lines = in_file.read().splitlines()
    stripped = [line.replace(","," ").split() for line in lines]
    grouped = itertools.izip(*[stripped]*1)
    with open('30_25_0.001_1_hwtimes.csv', 'w') as out_file:
        writer = csv.writer(out_file)
        writer.writerow(('title', 'intro', 'tagline'))
        for group in grouped:
            writer.writerows(group)


"""
* @program: tensorflow-yolo1
* @description:
* @author: thesong
* @create: 2020-10-20 14:07
"""

from utils import VOC

if __name__ == '__main__':
    voc = VOC.VOC()
    print(voc.get_voc_size())
    print(voc.toTfrecord())


from django.shortcuts import render
from web.models import *
from django.shortcuts import render, get_object_or_404
from django.core.paginator import Paginator, EmptyPage

# Create your views here.
def home(request):
    stores = Store.objects.all()
    category= Category.objects.all()

    # Create a dictionary to store the first associated coupon for each store
    store_coupons = {}

    for store in stores:
        # Get the first coupon associated with the store, if any
        coupon = Coupon.objects.filter(store=store).first()

        # Add the store and its associated coupon (if any) to the dictionary
        store_coupons[store] = coupon

    context = {
        "stores": stores,
        "store_coupons": store_coupons,
        "categories": category
    }
    return render(request, 'index.html', context)

def stores(request):
    # Get all stores from the database and order them by name
    stores = Store.objects.order_by('name')

    # Create a list to store the grouped stores
    store_groups = []

    # Initialize variables to keep track of the current letter and stores for that letter
    current_letter = None
    current_stores = []

    for store in stores:
        first_letter = store.name[0].upper()

        if first_letter != current_letter:
            # If the letter changes, add the current stores to the list
            if current_letter is not None:
                store_groups.append((current_letter, current_stores))
            current_letter = first_letter
            current_stores = []

        current_stores.append(store)

    # Add the last group of stores
    if current_stores:
        store_groups.append((current_letter, current_stores))

    context = {
        'store_groups': store_groups
    }

    return render(request, 'stores.html', context)

def category(request):
    category= Category.objects.all()
    context = {
        'categories': category
    }
    return render(request, 'category.html', context=context)

def coupans(request, page=1):
    page = int(page)
    items_per_page = 15

    # Query all coupons
    all_coupons = Coupon.objects.filter(is_deal=False)

    # Get the count of code (where is_deal is False) and deal (where is_deal is True) coupons
    code_count = Coupon.objects.filter(is_deal=False).count()
    deal_count = Coupon.objects.filter(is_deal=True).count()

    # Get the total count of all coupons

    total_count = Coupon.objects.all().count()

    # Paginate all coupons
    paginator = Paginator(all_coupons, items_per_page)

    # Get the current page
    coupons = paginator.page(page)

    # If the current page is not the first page, get the coupons from the previous page
    if page > 1:
        previous_page = paginator.page(page - 1)
        previous_coupons = previous_page.object_list
    else:
        previous_coupons = []

    return render(request, 'coupans.html', {
        'coupons': coupons,
        'previous_coupons': previous_coupons,
        'code_count': code_count,
        'deal_count': deal_count,
        'total_count': total_count  # Pass the total count of all coupons to the template
    })

def dealCoupans(request, page=1):
    page = int(page)
    items_per_page = 15

    # Query all coupons
    all_coupons = Coupon.objects.filter(is_deal=True)

    # Get the count of code (where is_deal is False) and deal (where is_deal is True) coupons
    code_count = Coupon.objects.filter(is_deal=False).count()
    deal_count = Coupon.objects.filter(is_deal=True).count()

    # Get the total count of all coupons

    total_count = Coupon.objects.all().count()

    # Paginate all coupons
    paginator = Paginator(all_coupons, items_per_page)

    # Get the current page
    coupons = paginator.page(page)

    # If the current page is not the first page, get the coupons from the previous page
    if page > 1:
        previous_page = paginator.page(page - 1)
        previous_coupons = previous_page.object_list
    else:
        previous_coupons = []

    return render(request, 'dealCoupans.html', {
        'coupons': coupons,
        'previous_coupons': previous_coupons,
        'code_count': code_count,
        'deal_count': deal_count,
        'total_count': total_count  # Pass the total count of all coupons to the template
    })

def blog(request):
    return render(request, 'blog.html')

def about_us(request):
    return render(request, 'about.html')

def privacy_policy(request):
    return render(request, 'privacy-policy.html')

def contact_us(request):
    return render(request, 'contact_us.html')

def base(request):
    stores= Store.objects.all()

    context = {
        'stores': stores
    }
    return render(request, 'base.html', context=context)

def store_details(request,pk, page=1):
    page = int(page)
    items_per_page = 15

    # Query all coupons
    all_coupons = Coupon.objects.filter(store__id=pk)
    store = Store.objects.filter(id=pk)

    # Get the count of code (where is_deal is False) and deal (where is_deal is True) coupons
    code_count = all_coupons.filter(is_deal=False).count()
    deal_count = all_coupons.filter(is_deal=True).count()

    # Get the total count of all coupons

    total_count = all_coupons.count()

    # Paginate all coupons
    paginator = Paginator(all_coupons, items_per_page)

    # Get the current page
    coupons = paginator.page(page)
    categories = Category.objects.all()

    # If the current page is not the first page, get the coupons from the previous page
    if page > 1:
        previous_page = paginator.page(page - 1)
        previous_coupons = previous_page.object_list
    else:
        previous_coupons = []

    return render(request, 'store_details.html', {
        'coupons': coupons,
        'previous_coupons': previous_coupons,
        'code_count': code_count,
        'deal_count': deal_count,
        'total_count': total_count,
        'stores': store,
        'categories': categories
    })

def coupan_detail(request, pk):
    # Retrieve the coupon using the provided primary key
    coupon = get_object_or_404(Coupon, pk=pk)

    # Access the associated store's data
    store = coupon.store

    return render(request, 'coupan_detail.html', {'coupons': coupon, 'stores': store})

def deal_detail(request, pk):
    # Retrieve the coupon using the provided primary key
    coupon = get_object_or_404(Coupon, pk=pk)

    # Access the associated store's data
    store = coupon.store

    return render(request, 'deal_details.html', {'coupons': coupon, 'stores': store})


from flask import render_template, request
import requests
from .import bp as main
from flask_login import login_required

# used to display pokemon data
@main.route('/pokemon', methods=['GET', 'POST'])
@login_required
def pokemon():
    if request.method == 'POST':
        name = request.form.get('name')
        url = f'https://pokeapi.co/api/v2/pokemon/{name}'
        response = requests.get(url)
        if not response.ok:
            # return -1
            error_string="Uh oh! Something went wrong"
            render_template("pokemon.html.j2", error=error_string)
        else:
            data = response.json()
            if not data:
                error_string=f"There is no Pokemon info for {name}, check the spelling"
                return render_template("pokemon.html.j2", error=error_string)
                # name, atleast one ability's name, base_experience, and
                # the URL for its sprite
            complete_pokemon = []
            char_dict = {
                    "Name":data['forms'][0]['name'],
                    "Ability":data['abilities'][0]['ability']['name'],
                    "Base Experience":data['base_experience'],
                    "Sprite URL":data['sprites']['front_shiny']
                    }
            complete_pokemon.append(char_dict)
            return render_template('pokemon.html.j2', pokemon=complete_pokemon)
    return render_template("pokemon.html.j2")


import socket
import textwrap
from tempfile import NamedTemporaryFile

import requests
from matplotlib.image import imread
from PIL import Image, ImageDraw, ImageFont

def is_connected(url="http://www.google.com/", timeout=3):
    try:
        requests.head(url, timeout=timeout)
        return True
    except requests.ConnectionError as ex:
        print(ex)
        return False

def get_ip():
    """
    Retrieve primary IP address on the local box
    see https://stackoverflow.com/a/28950776/91468
    :return:
    """
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    try:
        # doesn't even have to be reachable
        s.connect(("8.8.8.8", 1))
        ip = s.getsockname()[0]
    except Exception:
        ip = "127.0.0.1"
    finally:
        s.close()
    return ip

def get_hostname():
    return socket.gethostname()

def number_to_string(number):
    """
    Transform a number to a string. If number is smaller than 1000, displays the last 5 significant figures.
    :param number: number to convert to string
    :return: string representation of the number
    """
    number_string = ""
    if number > 100:
        number_string = format(int(number), ",")
    else:
        number_string = str(float("%.5g" % number))
    return number_string

def get_percentage_diff(current, previous):
    if current == previous:
        return 0
    try:
        return (float(current - previous) / previous) * 100.0
    except ZeroDivisionError:
        return float("inf")

# see https://kavigupta.org/2019/05/18/Setting-the-size-of-figures-in-matplotlib/
def get_size(fig, dpi=100):
    with NamedTemporaryFile(suffix=".png") as f:
        fig.savefig(f.name, bbox_inches="tight", dpi=dpi)
        height, width, _channels = imread(f.name).shape
        return width / dpi, height / dpi

def set_size(fig, size, dpi=100, eps=1e-2, give_up=2, min_size_px=10):
    target_width, target_height = size
    set_width, set_height = target_width, target_height  # reasonable starting point
    deltas = []  # how far we have
    while True:
        fig.set_size_inches([set_width, set_height])
        actual_width, actual_height = get_size(fig, dpi=dpi)
        set_width *= target_width / actual_width
        set_height *= target_height / actual_height
        deltas.append(
            abs(actual_width - target_width) + abs(actual_height - target_height)
        )
        if deltas[-1] < eps:
            return True
        if len(deltas) > give_up and sorted(deltas[-give_up:]) == deltas[-give_up:]:
            return False
        if set_width * dpi < min_size_px or set_height * dpi < min_size_px:
            return False

def empty_image(width, height):
    """
    Returns an empty canvas/image to draw  on
    :return:
    """
    return Image.new("1", (width, height), 255)  # 255: clear the frame

def place_text(
    img,
    text,
    x_offset=0,
    y_offset=0,
    font_size=40,
    font_name="Forum-Regular",
    fill=0,
):
    """
    Put some text at a location on the image.
    """
    draw = ImageDraw.Draw(img)
    font = ImageFont.truetype("/usr/share/fonts/TTF/DejaVuSans.ttf", font_size)
    draw.text((x_offset, y_offset), text, font=font, fill=fill)

def place_centered_text(
    img,
    text,
    x_offset=0,
    y_offset=0,
    font_size=40,
    font_name="Forum-Regular",
    fill=0,
):
    """
    Put some centered text at a location on the image.
    """
    font = ImageFont.truetype("/usr/share/fonts/TTF/DejaVuSans.ttf", font_size)
    img_width, img_height = img.size
    text_width, _ = font.getsize(text)
    text_height = font_size
    draw_x = (img_width - text_width) // 2 + x_offset
    draw_y = (img_height - text_height) // 2 + y_offset
    place_text(img, text, draw_x, draw_y, font_size, font_name, fill)

def place_text_right(
    img,
    text,
    x_offset=0,
    y_offset=0,
    font_size=40,
    font_name="Forum-Regular",
    fill=0,
):
    font = ImageFont.truetype("/usr/share/fonts/TTF/DejaVuSans.ttf", font_size)
    img_width, img_height = img.size
    text_width, _ = font.getsize(text)
    draw_x = (img_width - text_width) + x_offset
    draw_y = y_offset
    place_text(img, text, draw_x, draw_y, font_size, font_name, fill)

def write_wrapped_lines(
    img,
    text,
    font_size=16,
    y_text=20,
    height=15,
    width=25,
    font_name="Roboto-Light",
):
    lines = textwrap.wrap(text, width)
    num_lines = 0
    for line in lines:
        place_centered_text(img, line, 0, y_text, font_size, font_name)
        y_text += height
        num_lines += 1
    return img


from .batch_filter import BatchFilter
from gunpowder.array import ArrayKey, Array
from gunpowder.batch import Batch
import logging

logger = logging.getLogger(__name__)

class AsType(BatchFilter):
    """Cast arrays to a different datatype (ex: np.float32 --> np.uint8).

    Args:

        source (:class:`ArrayKey`):

            The key of the array to cast.

        target_dtype (str or dtype):

            The voxel size of the target.

        target (:class:`ArrayKey`, optional):

            The key of the array to store the cast ``source``.

    """

    def __init__(self, source, target_dtype, target=None):
        assert isinstance(source, ArrayKey)
        if target is not None:
            assert isinstance(target, ArrayKey)
            self.target = target
        else:
            self.target = source

        self.source = source
        self.target_dtype = target_dtype

    def setup(self):
        spec = self.spec[self.source].copy()
        spec.dtype = self.target_dtype
        if self.target is not self.source:
            self.provides(self.target, spec)
        else:
            self.updates(self.source, spec)
        self.enable_autoskip()

    def process(self, batch, request):
        source = batch.arrays[self.source]
        source_data = source.data

        cast_data = source_data.astype(self.target_dtype)

        target_spec = source.spec.copy()
        target_spec.dtype = cast_data.dtype
        target_array = Array(cast_data, target_spec)

        # create output array
        outputs = Batch()
        outputs.arrays[self.target] = target_array

        return outputs