# CommandLineXMLValidator
# Based upon https://emredjan.github.io/blog/2017/04/08/validating-xml/
# Version 1.4
# Fabrice Kinnar - 2019

# Import required modules
import time
import sys
import itertools
import pyodbc
import copy

from glob import glob
from argparse import ArgumentParser
from pypika import MSSQLQuery, Table, Tables

from StringTools import StringTools
from ANSIColor import ANSIColor
from MessageQueue import MessageQueue
from XMLValidator import XMLValidator
from MobileValidator import MobileValidator

# CommandLineXMLValidator Class
#
# Encapsulate the command line tool that validate a XML file
#
# PUBLIC INTERFACE:
# - CommandLineXMLValidator()   : Constructor
# - run()                       : Perform the job
#
# CLASS METHODS:
# - divisor                     : Build a section divisor string
# - division_title              : Build a division title
# - build_line                  : Build a message line
# - build_data_line             : Build a data line
class CommandLineXMLValidator:
    # CONSTANTS
    MESSAGE_LENGTH = 40  # max lenght of a console message
    TABULATION = 3  # number of spaces per tabulation
    DIVISOR_CHAR = "─"  # character used for section divisor
    DIVISOR_LENGTH = 80  # length for section divisor
    COLUMN_SEPARATOR = " : "  # columns separator for data line
    COLUMN_WIDTH = 20  # width for columns

    LANGUAGES_LIST = ["NL", "FR"]  # supported LANGUAGES_LIST for labels

    # Database connection
    DATABASE_SERVER = "2008-SQL"
    DATABASE_NAME = "agrprod"
    CONNECTION_STRING = "DRIVER={ODBC Driver 17 for SQL Server};SERVER=" + DATABASE_SERVER + ";DATABASE=" + DATABASE_NAME + ";Trusted_Connection=yes;"

    # divisor (class method)
    #
    # Build a section divisor string
    @classmethod
    def divisor(cls):
        return StringTools.uniform(cls.DIVISOR_CHAR, cls.DIVISOR_LENGTH)

    # division_title (class method)
    #
    # Build a division title
    #   text : text to use as a title
    @classmethod
    def division_title(cls, text):
        return StringTools.right_aligned(text, cls.DIVISOR_LENGTH)

    # build_line (class method)
    #
    # Build a message line
    #   indent : indentation to use (n * TABULATION)
    #   text   : text to display on the line
    #   tail   : text to add to the end of the text (optional)
    # Returned string is truncated to MESSAGE_LENGTH
    @classmethod
    def build_line(cls, indent, text, tail=None):
        indent *= cls.TABULATION  # indentation in spaces
        len_text = len(text)  # length of given text

        # Length difference between the text and the unescaped text (useful for tabulation when there are escape sequences)
        diff_len = len_text - len(ANSIColor.unescape(text))

        # The first two parts of the line (indentation + text)
        parts = [StringTools.blank(indent), text]

        # If there is a tail, add it to the parts
        if tail:
            len_tail = len(tail)
            diff_len += len_tail - len(ANSIColor.unescape(tail))
            parts += [StringTools.blank(cls.MESSAGE_LENGTH + diff_len - indent - len_text - len_tail), tail]

        # Join the string parts and limit to the MESSAGE_LENGTH size
        return "".join(parts)[0 : cls.MESSAGE_LENGTH + diff_len]

    # build_data_line (class method)
    #
    # Build a data line
    #   indent : indentation to use (n * TABULATION)
    #   text   : text to display on the line
    #   value  : value for the specified text (optional)
    @classmethod
    def build_data_line(cls, indent, text, value=None):
        if value:
            return f"{cls.build_line(indent, text, cls.COLUMN_SEPARATOR)}{value}"
        return cls.build_line(indent, text)

    # format_warning (class method)
    #
    # ANSI format a text as a warning
    #   text   : text to format
    @classmethod
    def format_warning(cls, text):
        return ANSIColor.escape(ANSIColor.Mode.bold, ANSIColor.Color.yellow, None, text)

    # format_error (class method)
    #
    # ANSI format a text as an error
    #   text   : text to format
    @classmethod
    def format_error(cls, text):
        return ANSIColor.escape(ANSIColor.Mode.bold, ANSIColor.Color.white, ANSIColor.Color.red, text)

    # format_error (class method)
    #
    # ANSI format a text as an error
    #   text   : text to format
    @classmethod
    def format_errors_array(cls, texts):
        return [ANSIColor.escape(ANSIColor.Mode.bold, ANSIColor.Color.white, ANSIColor.Color.red, text) for text in texts]

    # CONSTRUCTOR
    #
    # Initialise private data
    def __init__(self):
        # Initialise the timer
        self.__execution_time = time.perf_counter()
        self.__cpu_time = time.process_time()

        # Parse command line arguments
        self.__parameters = self.__parse_arguments()

        # Retreive files to check (must be AFTER __parse_arguments())
        self.__files_to_check = glob(self.__parameters.xml_file, recursive=self.__parameters.deep)

        # Initialise private members
        self.__messages = MessageQueue()  # Message queue
        self.__number_of_errors = 0  # Number of detected errors
        self.__number_of_warnings = 0  # Number of detected warnings
        self.__validator = None  # Validator that we used (XMLValidator or MobileValidator)

        # To avoid multiple accesses to the DB, cache some data
        # List of browsers defined in Agresso
        self.__agresso_browsers_list = []
        # Dictionary of columns defined in Agresso (key : <browser>@<client>, value : list of columns)
        self.__agresso_columns_dict = {}

        # Intersection of found datasets (List but initialised to None to detect the first call)
        self.__datasets_intersection = None

        # If the users specified a list of comma separated clients, build a set whith them
        self.__specified_clients_set = set(self.__parameters.client.strip().split(",") if self.__parameters.client else [])

    # PUBLIC: run
    #
    # Execute the validation and display the result
    def run(self):
        # Output initialisation messages (if needed)
        self.__push_initialisation_message()

        # Perform the validation
        self.__perform_validation()

        # Perform deep analysis (if needed)
        self.__perform_analysis()

        # Show statistics (if needed)
        self.__push_statistics()

        # Show a little report (if needed)
        self.__push_report()

        # Output messages
        self.__messages.execute(print)

        return self.__number_of_errors

    # PRIVATE: __parse_arguments
    #
    # Parse the command line arguments
    def __parse_arguments(self):
        argument_parser = ArgumentParser()
        argument_parser.add_argument("xml_file", help="xml file (or wildcard) to validate")
        argument_parser.add_argument("-s", "--schema", help="xsd schema file to use")
        argument_parser.add_argument("-m", "--mobile", action="store_true", help="perform Mobile specific validation")
        argument_parser.add_argument("-v", "--verbose", action="store_true", help="verbose output")
        argument_parser.add_argument("-q", "--quiet", action="store_true", help="quiet mode, show errors only")
        argument_parser.add_argument("-r", "--report", action="store_true", help="show a report after validation")
        argument_parser.add_argument("-d", "--deep", action="store_true", help="look deeply for wildcard to handle subfolders (ex. src/**/*.xml)")
        argument_parser.add_argument("-a", "--analyse", action="store_true", help="analyse a Mobile template and display a summary of its content")
        argument_parser.add_argument("-c", "--check", action="store_true", help="perform some tests and display several statistics about Mobile template")
        argument_parser.add_argument("-l", "--client", help="specify client for Mobile template")
        return argument_parser.parse_args()

    # PRIVATE: __push_initialisation_message
    #
    # Put initialisation messages in the message queue if needed
    def __push_initialisation_message(self):
        # For a non verbose mode, we do not do anything
        if not self.__parameters.verbose:
            return None

        # Build a initialisation sentence
        message_parts = ["Checking XML structure"]
        if self.__parameters.schema:
            if self.__parameters.mobile:
                message_parts.append(", schema and mobile info")
            else:
                message_parts.append(" and schema")
        elif self.__parameters.mobile:
            message_parts.append(" and mobile info")

        # Push the message in the queue
        self.__messages.push("".join(message_parts))
        self.__messages.push()

    # PRIVATE: __push_report
    #
    # Put report messages in the message queue if needed
    def __push_report(self):
        # When no report is needed, terminate here
        if not self.__parameters.report:
            return None

        # Build some information messages about files we worked on
        number_of_files = len(self.__files_to_check)
        str_files_fragment = f"{number_of_files} {StringTools.plural(('file', 's'), number_of_files)} checked"
        str_errors_fragment = f"{self.__number_of_errors} {StringTools.plural(('error', 's'), self.__number_of_errors)}"
        str_warnings_fragment = f"{self.__number_of_warnings} {StringTools.plural(('warning', 's'), self.__number_of_warnings)}"

        # Compute timer
        self.__execution_time = time.perf_counter() - self.__execution_time
        self.__cpu_time = time.process_time() - self.__cpu_time
        str_execution_time_fragment = f"Execution time: {self.__execution_time:.2f}s"
        str_cpu_time_fragment = f"CPU time: {self.__cpu_time:.2f}s"

        # Push messages in the queue
        self.__messages.push()
        if self.__number_of_errors > 0:
            self.__messages.push(CommandLineXMLValidator.format_error(f"{str_files_fragment}, {str_warnings_fragment}, {str_errors_fragment}"))
        elif self.__number_of_warnings > 0:
            self.__messages.push(CommandLineXMLValidator.format_warning(f"{str_files_fragment}, {str_warnings_fragment}, {str_errors_fragment}"))
        else:
            self.__messages.push(f"{str_files_fragment}, {str_warnings_fragment}, {str_errors_fragment}")

        self.__messages.push(f"{str_execution_time_fragment}, {str_cpu_time_fragment}")

    # PRIVATE: __perform_validation
    #
    # Perform actual validation and put relevant info in the message queue
    def __perform_validation(self):
        # Loop through all files we have to check
        for xml_filename in self.__files_to_check:
            # Build a validator
            self.__validator = None
            try:
                # Choose between MobileValidator and XMLValidator using "mobile" parameter
                self.__validator = (
                    MobileValidator(xml_filename, self.__parameters.schema)
                    if self.__parameters.mobile
                    else XMLValidator(xml_filename, self.__parameters.schema)
                )
            except Exception as e:
                # The validator has not been built, log the problem
                self.__number_of_errors += 1
                self.__messages.push(CommandLineXMLValidator.format_error(f"{xml_filename} : Cannot build the validator - {type(e).__name__} !!"))

            # The validator has not been built, loop to the next file
            if self.__validator is None:
                continue

            # Check the XML structure
            rc = None
            try:
                # The check method could raise an exception, that's why it is enclosed in a "try" statement
                rc = self.__validator.check()
            finally:
                # If an exception has been raised, rc is None, otherwise we have an actual result
                self.__handle_validation_result(rc)
                if rc is None and not self.__parameters.quiet:
                    self.__messages.push(f"{xml_filename} : validation complete")

    # PRIVATE: __handle_validation_result
    #
    # Handle a validation result to enqueue relevant messages
    #   result : the validation result to work with
    def __handle_validation_result(self, result):
        # If an exception has been raised, result is None, there in nothing to do
        if result is None:
            return None

        # Take the error into account and enqueue the message
        self.__number_of_errors += 1
        if self.__parameters.verbose:
            # In verbose mode, tell the user a little bit more
            self.__messages.push(f"XML file : {self.__validator.get_xml_filename()}")

            # Append messages (unless they are empty)
            self.__messages.push_array(result)

            # Append an error message
            self.__messages.push("Validation failed !!")
        else:
            # In non verbose mode, be a little bit more laconic
            self.__messages.push(f"{self.__validator.get_xml_filename()}  : validation failed !!")

    # PRIVATE: __is_mobile
    #
    # Do we have a MobileValidator in hands
    def __is_mobile(self):
        # Did we build a MobileValidator (it collects info during validation)
        if not isinstance(self.__validator, MobileValidator):
            if not self.__parameters.quiet:
                self.__number_of_warnings += 1
                self.__messages.push(
                    CommandLineXMLValidator.format_warning("Mobile validation must be used to perform a deep analysis of a mobile template !!")
                )
            return False

        return True

    # PRIVATE: __perform_analysis
    #
    # Perform deep analysis of the given mobile template and put relevant info in the message queue
    def __perform_analysis(self):
        # Do we need to perform a deep analysis
        if not self.__parameters.analyse:
            return None

        # Did we build a MobileValidator (it collects info during validation)
        if not self.__is_mobile():
            return None

        # Push a section header
        self.__push_section_header("MOBILE DEEP ANALYSIS")

        # Get the root element
        xml_root = self.__validator.get_xml_document().getroot()

        # Get item element
        # Normally, there should be only one single element but work with an array, who knows...
        # I do not validate here, as the schema is supposed to be correct !
        template_items = xml_root.findall("item")
        number_of_items = len(template_items)

        # If we have more than one item, log the info
        if number_of_items > 1:
            self.__messages.push(CommandLineXMLValidator.build_data_line(0, "Number of item elements", number_of_items))

        # Analyse every items
        for i in range(0, number_of_items):
            self.__messages.push(CommandLineXMLValidator.build_data_line(0, f">  ITEM {i+1} OF {number_of_items}"))
            self.__perform_item_analysis(template_items[i])

    # PRIVATE: __perform_item_analysis
    #
    # Perform deep analysis of the given item and put relevant info in the message queue
    #   item_element : xml element for the item
    def __perform_item_analysis(self, item_element):
        # Retreive key
        self.__push_tags(None, ["key"], item_element, 1)

        # Analyse labels
        self.__perform_labels_analysis(1, item_element, "name/label")

        # Retreive pages
        item_pages = item_element.findall("pages/page")
        number_of_pages = len(item_pages)
        self.__messages.push(CommandLineXMLValidator.build_data_line(1, "Pages", number_of_pages))

        # Analyse pages
        for i in range(0, number_of_pages):
            self.__messages.push(CommandLineXMLValidator.build_data_line(1, f">  PAGE {i+1} OF {number_of_pages}"))
            self.__perform_page_analysis(item_pages[i])

    # PRIVATE: __perform_page_analysis
    #
    # Perform deep analysis of the given page and put relevant info in the message queue
    #   page_element : xml element for the page
    def __perform_page_analysis(self, page_element):
        # Retreive position
        self.__push_tags(None, ["position"], page_element, 2)

        # Analyse labels
        self.__perform_labels_analysis(2, page_element, "title/label")

        # Retreive templates
        templates = page_element.findall("templates/template")
        number_of_templates = len(templates)
        self.__messages.push(CommandLineXMLValidator.build_data_line(2, "Templates", number_of_templates))

        # Analyse templates
        for i in range(0, number_of_templates):
            title = CommandLineXMLValidator.build_line(1, f">  {i+1} OF {number_of_templates}")
            self.__perform_template_analysis(title, templates[i])

    # PRIVATE: __perform_labels_analysis
    def __perform_labels_analysis(self, indent, page_element, xpath):
        # Display labels
        labels = self.__get_labels(page_element, xpath)
        for (language, label) in labels:
            self.__messages.push(CommandLineXMLValidator.build_data_line(indent, f"Label {language}", label))

        # Duplication detection
        distinct_labels = {label for (_, label) in labels}
        number_of_duplications = len(labels) - len(distinct_labels)
        if number_of_duplications:
            self.__number_of_warnings += number_of_duplications
            warning_title = CommandLineXMLValidator.format_warning("WARNING")
            warning_message = CommandLineXMLValidator.format_warning(
                f"{StringTools.plural([f'{number_of_duplications} label duplication', 's'], number_of_duplications)} found !!"
            )
            self.__messages.push(CommandLineXMLValidator.build_data_line(indent, warning_title, warning_message))

    # PRIVATE: __perform_template_analysis
    #
    # Perform deep analysis of the given template and put relevant info in the message queue
    #   template_element : xml element for the template
    def __perform_template_analysis(self, title, template_element, indent=3):
        # Retreive some tags
        self.__push_tags_compact(title, ["key", "type"], template_element, indent)

        template_type = template_element.find("type").text
        if template_type.lower() == "table":
            # Retreive templates
            templates = template_element.findall("values/templates/template")
            number_of_templates = len(templates)
            self.__messages.push(CommandLineXMLValidator.build_data_line(indent + 2, "Templates", number_of_templates))

            # Analyse templates
            for i in range(0, number_of_templates):
                title = CommandLineXMLValidator.build_line(indent + 1, f">  {i+1} OF {number_of_templates}")
                self.__perform_template_analysis(title, templates[i])

    # PRIVATE: __get_labels
    #
    # Get labels in both French and Dutch
    #   father    : the father element for the labels we need
    #   base_path : the path for the label without NL or FR
    # Return them in an array
    def __get_labels(self, father, base_path):
        output_labels = []
        for l in CommandLineXMLValidator.LANGUAGES_LIST:
            output_label = [l]
            label = father.find(f"{base_path}{l}")
            if label is not None:
                output_label.append(label.text)
                output_labels.append(output_label)

        return output_labels

    # PRIVATE: __push_tags
    #
    # Push some tags and their texts in the message queue
    #   tags   : an ARRAY of tag to retreive
    #   father : father element for the given tags
    #   indent : indentation to use for enqueuing the messages
    def __push_tags(self, title, tags, father, indent):
        if title:
            self.__messages.push(title)
        for tag in tags:
            tag_element = father.find(tag)
            self.__messages.push(CommandLineXMLValidator.build_data_line(indent, tag, tag_element.text))

    # PRIVATE: __push_tags_compact
    #
    # Push some tags and their texts in the message queue (compact mode)
    #   tags   : an ARRAY of tag to retreive
    #   father : father element for the given tags
    #   indent : indentation to use for enqueuing the messages
    def __push_tags_compact(self, title, tags, father, indent):
        tag_values = []
        for tag in tags:
            tag_element = father.find(tag)
            tag_values.append(StringTools.fixed_width(tag_element.text, CommandLineXMLValidator.COLUMN_WIDTH))

        self.__messages.push(CommandLineXMLValidator.build_data_line(indent, title, " ".join(tag_values)))

    # PRIVATE: __push_section_header
    #
    # Push a section header in the queue
    #   title : title for the section
    def __push_section_header(self, title):
        # Build a title section
        self.__messages.push(CommandLineXMLValidator.divisor())
        self.__messages.push(CommandLineXMLValidator.division_title(title))
        self.__messages.push(CommandLineXMLValidator.divisor())

    # PRIVATE: __push_statistics_header
    #
    # Push statistics header in the message queue (if needed)
    def __push_statistics_header(self):
        self.__push_section_header("MOBILE STATISTICS AND TESTS")
        if self.__specified_clients_set:
            self.__messages.push(
                CommandLineXMLValidator.build_data_line(
                    0, StringTools.plural(("Specified client", "s"), len(self.__specified_clients_set)), f"[{', '.join(self.__specified_clients_set)}]"
                )
            )

    # PRIVATE: __push_statistics_simple_tags
    #
    # Push statistics simple tags in the message queue (if needed)
    #   xml_root : root xml element
    #   tags : an array of dictionaries that describe tags to explore
    #          [
    #             {
    #                 title : the label to display
    #                 tag   : xpath to the targeted element
    #             }
    #          ]
    def __push_statistics_simple_tags(self, xml_root, tags):
        for tag in tags:
            self.__messages.push(CommandLineXMLValidator.build_data_line(0, tag["title"], len(xml_root.findall(tag["tag"]))))

    # PRIVATE: __push_statistics_validated_tags
    #
    # Push statistics validated tags in the message queue (if needed)
    #   xml_root : root xml element
    #   tags : an array of dictionaries that describe tags to explore
    #          [
    #              {
    #                 title         : the label to display
    #                 base          : xpath to immediate parent of the targeted element
    #                 tag           : targeted element tag name
    #                 validator     : callback to verify groups of tags (optional)            [lambda key, count]
    #                 key_validator : callback to verify tags one by one (optional)           [lambda tag]
    #                 summary       : callback to display a summary at the end (optional)     [lambda]
    #              }
    #          ]
    def __push_statistics_validated_tags(self, xml_root, tags):
        for tag in tags:
            self.__push_statistics_entry(xml_root, tag)

    # PRIVATE: __push_statistics_entry
    #
    # Push a statistics entry in the queue
    def __push_statistics_entry(self, xml_root, tag):
        # Push the title in the queue
        self.__messages.push(tag["title"])

        # Get every given tag from XML
        tags = self.__get_sorted_tags_from_xpath(xml_root, tag["base"] + tag["tag"])

        # Run every validation hooks
        keys = [
            {"id": "validator", "method": self.__push_statistics_validated_tags_grouped},
            {"id": "key_validator", "method": self.__push_statistics_validated_tags_isolated},
            {"id": "summary", "method": self.__push_statistics_validated_tags_summary},
        ]
        for key in keys:
            callback = tag.get(key["id"])  # get the call back from the tag definition
            if callable(callback):  # if it is callable (ie. is not None)
                key["method"](callback, tags)  # call it

    # PRIVATE: __push_statistics_validated_tags_grouped
    #
    # Push grouped validation of tags in the message queue
    def __push_statistics_validated_tags_grouped(self, callback, tags):
        for (key, group) in itertools.groupby(tags, key=lambda x: x.text):  # group by node.text
            size = str(len(list(group)))
            count = f"{StringTools.fixed_width(size, CommandLineXMLValidator.COLUMN_WIDTH)} {callback(key, size)}"
            self.__messages.push(CommandLineXMLValidator.build_data_line(1, key, count))  # push the result message

    # PRIVATE: __push_statistics_validated_tags_isolated
    #
    # Push isolated validation of tags in the message queue
    def __push_statistics_validated_tags_isolated(self, callback, tags):
        for tag in tags:
            rc_isolate = callback(tag)  # call it
            if rc_isolate:  # return None when tag is not (yet) tested
                for (rc_key, rc_value) in rc_isolate:
                    self.__messages.push(CommandLineXMLValidator.build_data_line(1, rc_key, rc_value))  # and push its result

    # PRIVATE: __push_statistics_validated_tags_summary
    #
    # Push statistics validated tags summary in the message queue
    # The third dummy parameter is there to keep the same signature
    # as __push_statistics_validated_tags_grouped, and __push_statistics_validated_tags_isolated
    def __push_statistics_validated_tags_summary(self, callback, _):
        (rc_key, rc_value) = callback()  # call it
        self.__messages.push(CommandLineXMLValidator.build_data_line(1, rc_key, rc_value))  # and push its result

    # PRIVATE: __push_statistics
    #
    # Push statistics in the message queue (if needed)
    def __push_statistics(self):
        # Do we need to display statistics
        if not self.__parameters.check:
            return None

        # Did we build a MobileValidator (it collects info during validation)
        if not self.__is_mobile():
            return None

        # Push a section header
        self.__push_statistics_header()

        # Get the root element
        xml_root = self.__validator.get_xml_document().getroot()

        # Retreive several count of tags
        simple_tags = [
            dict(title="Number of items", tag="item"),
            dict(title="Number of pages", tag="item/pages/page"),
            dict(title="Number of templates", tag="item/pages/page/templates/template"),
        ]
        self.__push_statistics_simple_tags(xml_root, simple_tags)

        # Display a list of used types
        validated_tags = [
            dict(
                title="Template types",
                base="item/pages/page/templates/template/",
                tag="type",
                validator=lambda key, count: self.__test_template_type(key, count),
            ),
            dict(
                title="Datasets",
                base="item/pages/page/templates/template/values/",
                tag="dataset",
                validator=lambda key, count: self.__test_dataset(key, count),
                summary=lambda: self.__summary_datasets(),
            ),
            dict(title="Columns", base="item/pages/page/templates/template/", tag="values", key_validator=lambda tag: self.__test_column(tag)),
        ]
        self.__push_statistics_validated_tags(xml_root, validated_tags)

    # PRIVATE: __get_sorted_tags_from_xpath
    #
    # Retreive all tags for a given xpath and sort them alphabetically
    def __get_sorted_tags_from_xpath(self, xml_root, xpaths):
        tags = []

        for xpath in xpaths.strip().split(","):  # tags targeted by xpath can be comma separated
            for element in xml_root.findall(xpath):  # for each tag, we get an array of element
                tags += self.__split_tag_element(element)

        tags.sort(key=lambda x: x.text)  # itertools need the list to be sorted
        return tags

    # PRIVATE: __split_tag_element
    #
    # Split a tag element into several tag when values are comma separated
    def __split_tag_element(self, tag_element):
        elements = []

        if not tag_element.text:
            return elements

        for key in tag_element.text.strip().split(","):  # in an emement, values can be comma separated
            new_element = copy.copy(tag_element)  # copy the given element
            new_element.text = key  # and attribute the current key to it
            elements.append(new_element)  # for each key, add a node with the real key

        return elements

    # PRIVATE: __test_template_type
    #
    # Test template type
    def __test_template_type(self, key, count):
        if key.lower() == "lookup":
            if key != "Lookup":
                # if we validated against the schema, this error has already been taken into account
                if not self.__parameters.schema:
                    self.__number_of_errors += 1
                return CommandLineXMLValidator.format_error(f'Should be "Lookup", instead of "{key}"!!')
        elif not key.islower():
            # if we validated against the schema, this error has already been taken into account
            if not self.__parameters.schema:
                self.__number_of_errors += 1
            return CommandLineXMLValidator.format_error(f"Should be {key.lower()} !!")

        return ""

    # PRIVATE: __get_agresso_browsers
    #
    # Retreive Agresso browsers from database
    def __get_agresso_browsers(self):
        # If we have not accessed database yet, retreive browsers
        if not self.__agresso_browsers_list:
            # Execute the request and get the result
            # select distinct col_0, col_01, col_04 from acrsystemplate where col_04 like '%ef_ds%' order by col_04
            acrsystemplate = Table("acrsystemplate")
            q = (
                MSSQLQuery.select(acrsystemplate.col_0, acrsystemplate.col_01, acrsystemplate.col_04)
                .from_(acrsystemplate)
                .where(acrsystemplate.col_04.like("%ef_ds%"))
                .orderby(acrsystemplate.col_04)
                .distinct()
            )
            request = str(q)

            # Connect to the database and get a cursor
            cursor = pyodbc.connect(CommandLineXMLValidator.CONNECTION_STRING).cursor()
            with cursor.execute(request):
                for row in cursor.fetchall():
                    self.__agresso_browsers_list.append(row)  # Store the currect row

        # Return browsers to client
        return self.__agresso_browsers_list

    # PRIVATE: __test_dataset
    #
    # Test dataset
    def __test_dataset(self, browser_name, count):
        # Search for the given brower
        browsers = self.__get_agresso_browsers()
        clients = {client for (_, client, browser) in browsers if browser == browser_name}

        if len(clients) == 0:
            # We did not find the given browser
            self.__number_of_errors += 1
            return CommandLineXMLValidator.format_error("not found in Agresso !!")

        # Calcultate intersection of client (with clients found during previous step)
        if self.__datasets_intersection is not None:
            self.__datasets_intersection = self.__datasets_intersection & clients
        else:
            # For the first call, when we did not specify any client on the command line, we don't have any intersection yet :
            # initialise it with the clients we've just found
            self.__datasets_intersection = clients

        return f"for [{', '.join(clients)}]"

    # PRIVATE: __summary_dataset
    #
    # Summary for a dataset section
    def __summary_datasets(self):
        # if there is no intersection here, that's because there are no common client
        if not self.__datasets_intersection:
            self.__number_of_errors += 1
            return CommandLineXMLValidator.format_errors_array([">  ERROR !!", "NO COMMON CLIENT FOUND !!"])

        # check if all given client are available
        missing_datasets = self.__specified_clients_set - self.__datasets_intersection
        if missing_datasets:
            self.__number_of_errors += 1
            return CommandLineXMLValidator.format_errors_array([">  ERROR !!", f"[{', '.join(missing_datasets)}] MISSING !!"])

        # there is no error, just return a summary
        return [StringTools.plural((">  COMMON CLIENT", "S"), len(self.__datasets_intersection)), f"[{', '.join(self.__datasets_intersection)}]"]

    # PRIVATE: __test_column
    #
    # Test values
    def __test_column(self, tag):
        dataset = tag.find("dataset")

        # Filter out value tags that do not contain dataset
        if dataset is None:
            return None

        # Lookup for columns
        columns = tag.findall("search") + tag.findall("result") + tag.findall("return")

        # Build a set to remove duplicates
        columns_set = set()
        for column_list in columns:
            for column in column_list.text.split(","):
                columns_set.add((column, dataset.text))

        # There must be at least one column
        if len(columns_set) == 0:
            self.__number_of_errors += 1
            return CommandLineXMLValidator.format_errors_array([">  ERROR !!", f"{dataset.text}: No column found !!"])

        # Test columns agains Agresso DB
        rc = []
        for (column, dataset) in columns_set:
            decoded_column_name = self.__decode_column_name(column)
            if self.__test_column_against_agresso_for_all_clients(decoded_column_name, dataset, self.__datasets_intersection):
                dataset_info = [f"in {dataset}"]  # dataset name
                if decoded_column_name != column:  # if the real column name is different than the encoded one
                    dataset_info.append(f"({decoded_column_name})")  # add the real column name between parenthesis
                rc.append([f"{column}", " ".join(dataset_info)])  # append the constructed message to the result
            else:
                self.__number_of_errors += 1
                rc.append(CommandLineXMLValidator.format_errors_array([f"{column}", f"NOT FOUND IN {dataset} !!"]))

        return rc

    # PRIVATE: __test_column_agains_agresso
    #
    # Test a column against Agress DB
    def __test_column_against_agresso(self, column, browser, client):
        # Build a key for the dictionary upon browser and client
        key = f"{browser}@{client}"
        if key in self.__agresso_columns_dict and column in self.__agresso_columns_dict[key]:
            return True

        # Execute the request and get the result
        # select colname from acrsystemplate inner join arabrtcolinfo on acrsystemplate.col_0 = arabrtcolinfo.brt_id where col_04 = '{browser}' and col_01 = '{client}'
        acrsystemplate, arabrtcolinfo = Tables("acrsystemplate", "arabrtcolinfo")
        q = (
            MSSQLQuery.select(arabrtcolinfo.colname)
            .from_(acrsystemplate)
            .join(arabrtcolinfo)
            .on(acrsystemplate.col_0 == arabrtcolinfo.brt_id)
            .where(acrsystemplate.col_04 == f"{browser}" and acrsystemplate.col_01 == f"{client}")
        )
        request = str(q)

        # Connect to the database and get a cursor
        found = False
        self.__agresso_columns_dict[key] = []
        cursor = pyodbc.connect(CommandLineXMLValidator.CONNECTION_STRING).cursor()
        with cursor.execute(request):
            for row in cursor.fetchall():
                self.__agresso_columns_dict[key].append(row.colname)
                if row.colname == column:
                    found = True  # Search through the result

        # Return browsers to client
        return found

    # PRIVATE: __test_column_against_agresso_for_all_clients
    #
    # Test a column against Agress DB
    def __test_column_against_agresso_for_all_clients(self, column, browser, clients):
        for client in clients:
            if not self.__test_column_against_agresso(column, browser, client):
                return False

        return True

    # PRIVATE: __decode_column_name
    #
    # If column contains hashtags, they are weirdly encoded as _x0023_
    def __decode_column_name(self, column):
        return column.replace("_x0023_", "#")


from contextlib import contextmanager
from functools import lru_cache
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker

ORMBase = declarative_base()

@lru_cache()
def get_engine(*args, **kwargs):
    return create_engine(*args, **kwargs)

@lru_cache()
def get_sessionmaker(*args, **kwargs):
    return sessionmaker(*args, **kwargs)

@contextmanager
def get_session(*args, **kwargs):
    session = get_sessionmaker(*args, **kwargs)()
    ORMBase.metadata.create_all(session.bind)
    try:
        yield session
        session.commit()
        session.close()
    except:
        session.rollback()
        session.close()
        raise


from django.conf.urls import url,include
from .views import *

urlpatterns = [
    url(r'^add/$', add_to_cart, name='add_to_cart'),
    url(r'^view_cart', view_cart, name='view_cart'),
    url(r'^remove/(\d+)$', remove_from_cart, name='remove_from_cart'),
    url(r'^update/(\d+)$', update_quantity, name='update_quantity'),
    url(r'^coupon/apply$',apply_coupon,name="apply_coupon"),
     url(r'^addc/$', add_to_cart_product_list, name='add_to_cart_product_list'),

    ]


"""
Base HyPC format

Description:
    Base HyPC format script used to:
        - Convert ASCII point cloud to HyPC point cloud
    Also contains methods to update existing HyPC point clouds

Required inputs:
    -ASCII based point cloud file

Created by: Christopher Iseli
Last Modified: 20/10/2018 (Christopher Iseli)
"""

import numpy as np
import pandas as pd
import time, math
from scipy.spatial import cKDTree
import pickle

## ===== Settings ====##

inputFname = "pointCloud_sample.txt"
outputFname = "pointCloud_sample1.hypc"

##--------------------##

class hypc:
    def __init__(self, asciiFname, max_segment = 200000):
        start_time = time.clock()
        print("Importing:", inputFname)
        self.df = pd.read_csv(asciiFname, sep=" ", names=column_names)

        print("Point cloud loaded in: ",time.clock()-start_time)
        self.df.loc[:, 'x':'z'] = self.df.loc[:, 'x':'z'].astype(dtype=np.float64)
        self.df.loc[:, '1':'21'] = self.df.loc[:, '1':'21'].astype(dtype=np.uint16)
        self.df['class'] = np.zeros(self.df.shape[0])
        self.df['segment_id'] = np.empty(self.df.shape[0])
        self.df = self.df.sort_values('x')
        self.df = self.df.reset_index(drop=True)

        # self.segment

        print("Creating K-DTree structure")
        self.kdTree = cKDTree(self.df.loc[:, ['x', 'y', 'z']], compact_nodes=True, balanced_tree=False)
        mins,maxes = self.kdTree.mins,self.kdTree.maxes
        self.extents = [[mins[0],maxes[0]],[mins[1],maxes[1]],[mins[2],maxes[2]]]
        self.medians = [np.median(self.kdTree.data[:,0]),np.median(self.kdTree.data[:,1]),np.median(self.kdTree.data[:,2])]
        self.df = self.df.drop(['x','y','z'],axis=1)
        self.segment_indices,self.segment_bounds = segmentPoints(self)
        print("\nNew HyPC point cloud created..", time.clock()-start_time)
        saveHyPC(self,outputFname)
        print("Time for completion: ", time.clock() - start_time)
        # self.segmentPoints()
        self.stats()

    def stats(self):
        shape = self.df.shape
        print("\nImported ASCII file with:")
        print("\tNum. Points: " +str(shape[0]))
        print("\tAttributes: " + str(shape[1]))
        print("\nExtents: ")
        print("\t X: "+str(self.extents[0]))
        print("\t Y: " + str(self.extents[1]))
        print("\t Z: " + str(self.extents[2]))

def updateHyPC(pointCloud,outFname):

    mins, maxes = pointCloud.kdTree.mins, pointCloud.kdTree.maxes
    pointCloud.extents = [[mins[0], maxes[0]], [mins[1], maxes[1]], [mins[2], maxes[2]]]
    pointCloud.medians = [np.median(pointCloud.kdTree.data[:, 0]), np.median(pointCloud.kdTree.data[:, 1]),np.median(pointCloud.kdTree.data[:, 2])]

    pointCloud.segment_indices, pointCloud.segment_bounds = segmentPoints(pointCloud)
    indices = []
    for idxs in pointCloud.segment_indices:
        indices.append(np.max(idxs))

    saveHyPC(pointCloud, outFname)

def segmentPoints(pointCloud,max_points=200000, num_segments = None):
    if pointCloud.df.shape[0]<200000:
        # segment_indices = [pointCloud.df.index.values.astype(np.uint32)]
        # segment_bounds =  [pointCloud.extents[0][0],pointCloud.extents[1][0],pointCloud.extents[0][1],pointCloud.extents[1][1]]
        num_segments=2
    else:
        num_segments = int(math.sqrt(pointCloud.df.shape[0]/max_points))
    start = time.clock()
    min_x,max_x = pointCloud.extents[0]
    min_y,max_y = pointCloud.extents[1]
    # x_size = int(np.floor((max_y - min_y) / num_segments))
    # y_size = int(np.floor((max_x - min_x) / num_segments))
    x_size = (max_x - min_x) / num_segments
    y_size = (max_y - min_y) / num_segments
    x_segment_bounds,xstep = np.linspace(min_x+x_size, max_x, num= num_segments,retstep = True)
    y_segment_bounds,ystep = np.linspace(min_y+y_size, max_y, num=num_segments,retstep=True)

    # Create bins
    bins_x = np.searchsorted(x_segment_bounds, pointCloud.kdTree.data[:, 0])
    bins_y = np.searchsorted(y_segment_bounds, pointCloud.kdTree.data[:, 1])

    x_segment_bounds =np.insert(x_segment_bounds,0,min_x)
    y_segment_bounds = np.insert(y_segment_bounds, 0, min_y)

    pointCloud.df["bins_x"] = bins_x.astype(np.uint16)
    pointCloud.df["bins_y"] = bins_y.astype(np.uint16)

    segments = pointCloud.df.groupby(['bins_x', 'bins_y'])
    # print("Time to create segments: ", time.clock()-start)

    segment_indices = []
    segment_groups = []
    for i,segment in enumerate(segments):
        segment_indices.append(segment[1].index.values.astype(np.uint32))
        segment_groups.append(list(segment[0]))
    segment_indices = np.array(segment_indices)
    segment_bounds = []

    for i, group in enumerate(segment_groups):
        x_start, x_end = x_segment_bounds[group[0]],x_segment_bounds[group[0]]+xstep
        y_start, y_end = y_segment_bounds[group[1]], y_segment_bounds[group[1]]+ystep
        rect = [x_start,y_start,x_end,y_end]
        segment_bounds.append(rect)

    return segment_indices,segment_bounds

def saveHyPC(hypc_obj,fname):
    print("Saving point cloud to :", fname)
    with open(fname, "wb") as f:
        pickle.dump(hypc_obj, f, pickle.HIGHEST_PROTOCOL)

if __name__ == "__main__":
    column_names = ['x','y','z','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19','20','21']
    cloud = hypc(inputFname)


import unittest

import bs4
import requests

DEV_URL = 'http://localhost:8000'

class FunctionalTests(unittest.TestCase):
    def test_server_is_running(self):
        text = 'django'
        req = requests.get(DEV_URL)
        page = bs4.BeautifulSoup(req.content, 'html.parser')
        self.assertIn(text, page.title.string.lower())

if __name__ == '__main__':
    unittest.main()


import logging
import sys
import threading
import os
import click

from anime_downloader.sites import get_anime_class, ALL_ANIME_SITES
from anime_downloader import util
from anime_downloader.__version__ import __version__

logger = logging.getLogger(__name__)

echo = click.echo
sitenames = [v[1] for v in ALL_ANIME_SITES]

class SiteThread(threading.Thread):
    def __init__(self, site, *args, **kwargs):
        self.site = site
        self.exception = None
        super().__init__(*args, **kwargs)

    def run(self):
        try:
            ani = get_anime_class(self.site)

            # this should be more dynamic
            sr = ani.search('naruto')[0]

            anime = ani(sr.url)

            stream_url = anime[0].source().stream_url
        except Exception as e:
            self.exception = e

@click.command()
@click.argument('test_query', default='naruto')
def command(test_query):
    """Test all sites to see which ones are working and which ones aren't. Test naruto as a default."""
    util.print_info(__version__)
    logger = logging.getLogger("anime_downloader")
    logger.setLevel(logging.ERROR)

    threads = []

    for site in sitenames:
        t = SiteThread(site, daemon=True)
        t.start()
        threads.append(t)

    for thread in threads:
        if os.name == 'nt':
            p, f = 'Works: ', "Doesn't work: "  # Emojis doesn't work in cmd
        else:
            p, f = '✅ ', '❌ '
        thread.join(timeout=10)
        if not thread.is_alive():
            if not thread.exception:
                # echo(click.style('Works ', fg='green') + site)
                echo(click.style(p, fg='green') + thread.site)
            else:
                logging.debug('Error occurred during testing')
                logging.debug(thread.exception)
                echo(click.style(f, fg='red') + thread.site)
        else:
            logging.debug('timeout during testing')
            echo(click.style(f, fg='red') + thread.site)


# -*- coding: utf-8 -*-
"""
Created on Tue Mar  5 09:26:33 2019

@author: Tristan O'Hanlon

This will create a datasset of global total cloud cover with latitude.
Data is stored in a 2D array cccm_tcc_lat
[:,0] = latitude
[:,1] = total cloud cover fraction
"""
import time
import numpy as np
import os
from pyhdf import SD
import matplotlib.pyplot as plt

start = time.time()

cf=[] # create a blank array to add cloud amount data
lat=[] # create a blank array to add latitude data
counter=0

# The directory where your HDF files are stored
os.chdir('E:/University/University/MSc/Models/Data/CCCM/2010')

# Load every file in the directory
for filename in os.listdir():

    # Load the file
    f = SD.SD(filename)

    # Get the latitude data as a list
    lat = lat+f.select('Colatitude of subsatellite point at surface at observation').get().tolist()

    # Get the cloud cover data as a list. Since this is the 'cloud free' data, need to invert later
    cf = cf+f.select('Cloud area enhanced').get().tolist()

end = time.time()
print(end - start)

start = time.time()

if len(lat) != len(cf):
    exit('Invalid sizes of lat and cf data')

# Join the two lists as if they were two columns side by side, into a list of two elements each
#print("round lats")
lat[:] = [(round(v*2,0)/2-90)*-1 for v in lat]

lat = np.array(lat)
cf = np.array(cf)

#cf[cf > 100] = None

end = time.time()
print(end - start)

start = time.time()

combined = np.vstack((lat, cf)).T

#print("get unique lats")
unique = np.unique(lat)
#print(unique)

#print ("combined")
#print (combined)

#print ("sorted")
# Add a column for every additional column, -1 will sort by the first column
combined = combined[np.lexsort(np.transpose(combined)[:-1])]
#print (combined)

# Averages of (lat,cloud cover) array
averages_total = unique.size
cccm_tcc2_lat = np.empty((averages_total,2),dtype=float)

# Current subtotal of current lat
subtotal = 0.0
# Current number of cloud cover entries in subtotal
number = 0
# Set the current lat to false
current_lat = None

# Iterate through all of the (lat,cloud cover) elements and subtotal the same lat values
i = 0
end = time.time()
print(end - start)

start = time.time()
for item in combined:

    if current_lat is None:
        """
        print("setting current_lat to item[0]")
        print("(current_lat == item[0]) = ", end='')
        print(current_lat == item[0])
        """
        current_lat = item[0];

#    if item[1] is None:
#        pass

    # If the lat is not the same as last time, then perform the average calc and reset everything
    if item[0] != current_lat:

        # Find the average value. 1 - is the inverse of the 'cloud free' precentage.
        average = subtotal / number / 100
        #average = subtotal / number / 100
        """
        print("--------")
        print("lat: ", end='')
        print(current_lat, end='')
        print(", avg: ", end='')
        print(average, end='')
        print(", subtotal: ", end='')
        print(subtotal, end='')
        print(", number: ", end='')
        print(number)
        """
        # Append the average
        cccm_tcc2_lat[i] = [current_lat, average]
        # Reset the subtotal
        subtotal = 0.0
        number = 0
        # Set the current latitude
        current_lat = item[0]
        # Move to the next index in the averages array
        i+=1

    # Add the next value to the subtotal
    number+=1
    subtotal+=item[1]

# Catch the last entry in the for loop
average = subtotal / number / 100
#average = subtotal / number / 100
cccm_tcc2_lat[i] = [current_lat, average]

end = time.time()
print(end - start)
"""
print ("averages")
# Iterate through all of the (lat,cloud cover) elements
for item in averages:
    print("[", end='')
    print(item[0], end='')
    print(", ", end='')
    print(item[1], end='')
    print("]\n", end='')
"""
plt.figure()
fig, ax = plt.subplots()

#ax.plot(cccm_tcc_lat[:,0],cccm_tcc_lat[:,1], '-r', label='Derived from MODIS radiances by the enhanced cloud mask')
#ax.plot(cccm_tcc1_lat[:,0],cccm_tcc1_lat[:,1], '--g', label='Derived from MODIS radiances by the standard CERES-MODIS cloud mask')
ax.plot(cccm_tcc2_lat[:,0],cccm_tcc2_lat[:,1], '--b', label='MODIS')
ax.legend(loc='lower center', bbox_to_anchor=(0.5, -0.3),
          ncol=4, fancybox=True, shadow=True);

ax.set_ylabel('Cloud Fraction', color='r')
ax.set_xlabel('Latitude')

plt.title('Global Cloud Fraction vs Latitude - 2010')
plt.show()


"""
给定一个非空且只包含非负数的整数数组 nums，数组的度的定义是指数组里任一元素出现频数的最大值。

你的任务是在 nums 中找到与 nums 拥有相同大小的度的最短连续子数组，返回其长度。

示例 1：

输入：[1, 2, 2, 3, 1]
输出：2
解释：
输入数组的度是2，因为元素1和2的出现频数最大，均为2.
连续子数组里面拥有相同度的有如下所示:
[1, 2, 2, 3, 1], [1, 2, 2, 3], [2, 2, 3, 1], [1, 2, 2], [2, 2, 3], [2, 2]
最短连续子数组[2, 2]的长度为2，所以返回2.
示例 2：

输入：[1,2,2,3,1,4,2]
输出：6

提示：

nums.length 在1到 50,000 区间范围内。
nums[i] 是一个在 0 到 49,999 范围内的整数。
通过次数38,237提交次数66,082

来源：力扣（LeetCode）
链接：https://leetcode-cn.com/problems/degree-of-an-array
著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
"""
from typing import *
from collections import Counter

class Solution:

    def findShortestSubArray(self, nums: List[int]) -> int:
        def check(counter, freq):
            _, cur_freq = counter.most_common(1)[0]
            return cur_freq >= freq

        counter = Counter(nums)
        _, freq = counter.most_common(1)[0]

        left, right = 0, 0
        num_counter, length = Counter(), len(nums)
        while right < len(nums):
            n = nums[right]
            right += 1
            num_counter[n] += 1
            flag = False
            while check(num_counter, freq):
                flag = True
                num_counter[nums[left]] -= 1
                left += 1
            if flag:
                tmp = right - left + 1
                length = tmp if tmp < length else length
        return length

if __name__ == '__main__':
    s = Solution()
    print(s.findShortestSubArray([1,2,2,3,1,4,2]))


def separator():
    print("-------------------")

# The lambda operator or lambda function is a way to create small anonymous
# functions, i.e. functions without a name.
# These functions are throw-away functions, i.e. they are just needed where they
# have been created. Lambda functions are mainly used in combination with the
# functions filter(), map() and reduce().
sum_of_two = lambda x, y: x + y
print(sum_of_two(3, 7))

separator()

# map(func, seq)
# The first argument func is the name of a function and the second a sequence
# (e.g. a list) seq. map() applies the function func to all the elements of the
# sequence seq.
C = [39.2, 36.5, 37.3, 38, 37.8]
F = list(map(lambda x: (float(9)/5)*x + 32, C))
print(F)

C = list(map(lambda x: (float(5)/9)*(x-32), F))
print(C)

# map() can be applied to more than one list. The lists have to have the same
# length. map() will apply its lambda function to the elements of the argument
# lists, i.e. it first applies to the elements with the 0th index, then to the
# elements with the 1st index until the n-th index is reached
a = [1, 2, 3, 4]
b = [17, 12, 11, 12]
c = [-1, -4, 5, 9]

print(list(map(lambda x,y:x+y, a,b)))
print(list(map(lambda x,y,z: x+y+z, a,b,c)))
print(list(map(lambda x,y,z : 2.5*x + 2*y - z, a,b,c)))

separator()

# filter(function, sequence)
# offers an elegant way to filter out all the elements of a sequence "sequence",
# for which the function function returns True. i.e. an item will be produced by
# the iterator result of filter(function, sequence) if item is included in the
# sequence "sequence" and if function(item) returns True.
fibonacci = [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55]
odd_numbers = list(filter(lambda x: x % 2, fibonacci))
print(odd_numbers)

separator()

# reduce(func, seq)
# continually applies the function func() to the sequence seq. It returns a
# single value.
import functools
red = functools.reduce(lambda x, y: x+y, [47, 11, 42, 13])
print(red)

# Calculating the sum of the numbers from 1 to 100:
print(functools.reduce(lambda x, y: x+y, range(1, 101)))


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

iris = pd.read_csv('C:\\0Krzysztof\WD\iris.data', names=["SepalLengthCm", "SepalWidthCm", "PetalLengthCm", "PetalWidthCm", "Species"])

iris_setosa = iris[iris["Species"]=="Iris-setosa"]
iris_versicolor = iris[iris["Species"]=="Iris-versicolor"]
iris_virginica = iris[iris["Species"]=="Iris-virginica"]

x3 = iris_virginica["SepalLengthCm"]
y3 = iris_virginica["PetalLengthCm"]
plt.scatter(x3, y3, color="r", label="Iris virginica")

x = iris_setosa["SepalLengthCm"]
y = iris_setosa["PetalLengthCm"]
plt.scatter(x, y, color="g", label="Iris setosa")

x2 = iris_versicolor["SepalLengthCm"]
y2 = iris_versicolor["PetalLengthCm"]
plt.scatter(x2, y2, color="b", label="Iris versicolor")

plt.xlabel("Sepal length (cm)")
plt.ylabel("Petal length (cm)")
plt.title("Iris dataset: petal length and sepal length")
plt.legend(loc="lower right")
plt.plot()
plt.show()


import discord
from discord.ext import commands

# TODO

class CustomChannels():
    def __init__(self, bot, utils):
        self.bot = bot
        self.utils = utils

        if 'custom_channels' not in self.utils.settings:
            self.utils.settings['custom_channels'] = {}
        if 'category' not in self.utils.settings['custom_channels']:
            self.utils.settings['custom_channels']['category'] = None
        if 'public_channels' not in self.utils.settings['custom_channels']:
            self.utils.settings['custom_channels']['public_channels'] = []
        if 'private_channels' not in self.utils.settings['custom_channels']:
            self.utils.settings['custom_channels']['private_channels'] = []
        if 'invite_message' not in self.utils.settings['custom_channels']:
            self.utils.settings['custom_channels']['invite_message'] = "{member.mention}, is toegevoegd aan dit kanaal!"

        self.utils.save_settings()

        self.offtopic_category = self.utils.settings['custom_channels']['category']
        self.public_channels = self.utils.settings['custom_channels']['public_channels']
        self.private_channels = self.utils.settings['custom_channels']['private_channels']
        self.invite_message = self.utils.settings['custom_channels']['invite_message']
        self.description_message = "De beschrijving van dit kanaal is aangepast."

    async def on_guild_channel_delete(self, channel):
        if channel.id in self.public_channels:
            public_channels = self.utils.settings['custom_channels']['public_channels']
            public_channels.remove(channel.id)

            self.utils.settings['custom_channels']['public_channels'] = public_channels
            self.public_channels = self.utils.settings['custom_channels']['public_channels']
            self.utils.save_settings()
        elif channel.id in self.private_channels:
            private_channels = self.utils.settings['custom_channels']['private_channels']
            private_channels.remove(channel.id)

            self.utils.settings['custom_channels']['private_channels'] = private_channels
            self.private_channels = self.utils.settings['custom_channels']['private_channels']
            self.utils.save_settings()

    @commands.group(name='customchannel', aliases=['cc'])
    async def customchannel(self, context):
        '''Laat gebruikers hun eigen kanalen aanmaken'''
        if not context.invoked_subcommand:
            await self.utils.send_cmd_help(context)

    @customchannel.command(name='list', aliases=['lijst'])
    async def customchannel_list_directory(self, context):
        '''Laat alle publieke kanalen zien'''

        channels = '\n\n'
        for channel in self.public_channels:
            ch = self.bot.get_channel(channel)
            channels += '{} ({})\n'.format(ch.mention, ch.id)
        embed = discord.Embed(color=discord.Color.red(), description=channels, title='Alle publieke kanalen')
        await context.send(embed=embed)
        # await context.message.add_reaction('\U0001F44D')

    @customchannel.command(name='join', aliases=['betreed'])
    async def customchannel_join_channel(self, context, channel: discord.TextChannel):
        '''Betreed een publiek kanaal'''
        if channel.id in self.public_channels:
            channel = self.bot.get_channel(channel.id)
            member = context.author
            await channel.set_permissions(member,
                                          read_messages=True)
            await context.message.delete()

    @customchannel.command(name='description', aliases=['beschrijving'])
    async def customchannel_change_description_channel(self, context, *, description: str):
        '''Verander beschrijving van een kanaal'''
        channel = context.channel
        description = ''.join(description)
        if channel.id in self.public_channels or channel.id in self.private_channels:
            channel = self.bot.get_channel(channel.id)
            await channel.edit(topic=description)
            await context.channel.send(self.description_message)
            await context.message.delete()

    @customchannel.command(name='category')
    @commands.is_owner()
    async def customchannel_set_category(self, context, categorie: discord.CategoryChannel):
        '''Kies welke categorie je wilt gebruiken om alle kanalen in kwijt te kunnen'''
        self.utils.settings['custom_channels']['category'] = categorie.id
        self.offtopic_category = self.utils.settings['custom_channels']['category']
        self.utils.save_settings()
        await context.message.add_reaction('\U0001F44D')

    @customchannel.command(name='private', aliases=['prive'])
    async def customchannel_new_channel(self, context, *, channel_name: str):
        '''Open een nieuw private kanaal'''
        if self.offtopic_category:
            guild = context.guild
            channel_name = channel_name.replace(' ', '-')

            overwrites = {
                guild.default_role: discord.PermissionOverwrite(read_messages=False),
                guild.me: discord.PermissionOverwrite(read_messages=True),
                context.author: discord.PermissionOverwrite(read_messages=True)
                }

            category = discord.utils.get(guild.categories,
                                         id=self.offtopic_category)

            channel = await guild.create_text_channel(channel_name,
                                                      category=category,
                                                      overwrites=overwrites)

            private_channels = self.utils.settings['custom_channels']['private_channels']
            private_channels.append(channel.id)

            self.utils.settings['custom_channels']['private_channels'] = private_channels
            self.private_channels = self.utils.settings['custom_channels']['private_channels']
            self.utils.save_settings()

            await context.message.add_reaction('\U0001F44D')

    @customchannel.command(name='public', aliases=['publiek'])
    async def customchannel_new_public_channel(self, context, *, channel_name: str):
        '''Open een nieuw publiek kanaal'''
        if self.offtopic_category:
            guild = context.guild
            channel_name = channel_name.replace(' ', '-')

            overwrites = {
                guild.default_role: discord.PermissionOverwrite(read_messages=False),
                guild.me: discord.PermissionOverwrite(read_messages=True),
                context.author: discord.PermissionOverwrite(read_messages=True)
                }

            category = discord.utils.get(guild.categories,
                                         id=self.offtopic_category)

            channel = await guild.create_text_channel(channel_name,
                                                      category=category,
                                                      overwrites=overwrites)

            public_channels = self.utils.settings['custom_channels']['public_channels']
            public_channels.append(channel.id)

            self.utils.settings['custom_channels']['public_channels'] = public_channels
            self.public_channels = self.utils.settings['custom_channels']['public_channels']
            self.utils.save_settings()

            await context.message.add_reaction('\U0001F44D')

    @customchannel.command(name='invite', aliases=['verzoek'])
    async def customchannel_invite_to_channel(self, context, member: discord.Member):
        '''Nodig iemand uit om een private kanaal te betreden'''
        await context.channel.set_permissions(member,
                                              read_messages=True)
        await context.channel.send(self.invite_message.format(member=member))
        await context.message.delete()

    @customchannel.command(name='set_invite')
    @commands.is_owner()
    async def customchannel_set_invite_message(self, context, *, message: str):
        '''Stel een bericht in waarmee de uitgenodigde mensen begroet moeten worden.'''
        self.utils.settings['custom_channels']['invite_message'] = ''.join(message)
        self.invite_message = self.utils.settings['custom_channels']['invite_message']
        self.utils.save_settings()
        await context.message.add_reaction('\U0001F44D')

    # Kleuters...
    # @privatechannel.command(name='schop', aliases=['kick'])
    # async def privatechannel_kick_from_channel(self, context, member: discord.Member):
    #    '''Schop iemand uit het private kanaal'''
    #    await context.channel.set_permissions(member,
    #                                          read_messages=False)
    #    await context.message.add_reaction('\U0001F44D')

    @customchannel.command(name='leave', aliases=['verlaat'])
    async def customchannel_leave_from_channel(self, context):
        '''Verlaat een private kanaal'''
        member = context.author
        await context.channel.set_permissions(member,
                                              read_messages=False)
        await context.message.delete()


#!/usr/bin/python
# _*_ coding:utf-8 _*_
# 音乐爬虫

if __name__ == "__main__":
    # num = 245610
    # print(int(num/2000))
    # print(int(num/30000))
    import requests
    from bs4 import BeautifulSoup
    file = open("music.html","r")
    html_doc = file.read()
    # print(html_doc)

    soup = BeautifulSoup(html_doc, 'html.parser')
    # print(soup.prettify())
    content1 = soup.find(id="content1")

    li = content1.find_all("li")

    #目标音乐站
    url = "http://96.ierge.cn/"
    musicid = []
    musicname = []
    musicurl = []
    for x in li:
        mtitle = x.a.get("title")
        musicname.append(mtitle)
        ids = x.input.get("title")
        musicid.append(ids)
        n1 = int(int(ids)/30000)
        n2 = int(int(ids)/2000)
        xurl = url + "/" + str(n1) + "/" + str(n2) + "/" + ids+".mp3"
        musicurl.append(xurl)
        # print x.input.get("title")
        print(xurl)
        print(mtitle)

        # url = 'http://96.ierge.cn/7/109/218490.mp3'
        r = requests.get(xurl)
        with open("music/"+mtitle+".mp3", "wb") as code:
            code.write(r.content)
        print(" 下载完成。。。。")


# -*- coding: utf-8 -*-
"""
Django settings for quips project.

For more information on this file, see
https://docs.djangoproject.com/en/dev/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/dev/ref/settings/
"""
from __future__ import absolute_import, unicode_literals

import environ

try:
    from psycopg2cffi import compat

    compat.register()
except ModuleNotFoundError:
    # only use psycopg2cffi if available
    pass

ROOT_DIR = environ.Path(__file__) - 3  # (/a/b/myfile.py - 3 = /)
APPS_DIR = ROOT_DIR.path("quips")

env = environ.Env()

# APP CONFIGURATION
# ------------------------------------------------------------------------------
DJANGO_APPS = (
    # Default Django apps:
    "django.contrib.auth",
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.sites",
    "django.contrib.messages",
    "django.contrib.staticfiles",
    # Useful template tags:
    # 'django.contrib.humanize',
    # Admin
    "django.contrib.admin",
)
THIRD_PARTY_APPS = (
    "memoize",  # function memoization
    "rest_framework",  # DRF
)

# Apps specific for this project go here.
LOCAL_APPS = (
    "quips.quips",
    "quips.api",
    "quips.slackapp",
    "quips.website",
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS

# MIDDLEWARE CONFIGURATION
# ------------------------------------------------------------------------------
MIDDLEWARE = (
    # WhiteNoise really thinks it needs to be first
    # http://whitenoise.evans.io/en/stable/django.html#django-middleware
    "whitenoise.middleware.WhiteNoiseMiddleware",
    # Make sure djangosecure.middleware.SecurityMiddleware is listed first
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.middleware.common.CommonMiddleware",
    "django.middleware.csrf.CsrfViewMiddleware",
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    "django.contrib.messages.middleware.MessageMiddleware",
    "django.middleware.clickjacking.XFrameOptionsMiddleware",
)

# MIGRATIONS CONFIGURATION
# ------------------------------------------------------------------------------
MIGRATION_MODULES = {"sites": "quips.contrib.sites.migrations"}

# DJANGO REST FRAMEWORK
# ------------------------------------------------------------------------------
REST_FRAMEWORK = {
    "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination",
    "PAGE_SIZE": 10,
}

# DEBUG
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#debug
DEBUG = env.bool("DJANGO_DEBUG", False)

# FIXTURE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-FIXTURE_DIRS
FIXTURE_DIRS = (str(APPS_DIR.path("fixtures")),)

# EMAIL CONFIGURATION
# ------------------------------------------------------------------------------
EMAIL_BACKEND = env(
    "DJANGO_EMAIL_BACKEND", default="django.core.mail.backends.smtp.EmailBackend"
)

# MANAGER CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#admins
ADMINS = (("""Brad Smith""", "bradster@infinitewarp.com"),)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#managers
MANAGERS = ADMINS

# DATABASE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
DATABASES = {
    # Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
    "default": env.db("DATABASE_URL", default="postgres:///quips"),
}
DATABASES["default"]["ATOMIC_REQUESTS"] = True

# GENERAL CONFIGURATION
# ------------------------------------------------------------------------------
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = "UTC"

# See: https://docs.djangoproject.com/en/dev/ref/settings/#language-code
LANGUAGE_CODE = "en-us"

# See: https://docs.djangoproject.com/en/dev/ref/settings/#site-id
SITE_ID = 1

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-i18n
USE_I18N = True

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-l10n
USE_L10N = True

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-tz
USE_TZ = True

# TEMPLATE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#templates
TEMPLATES = [
    {
        # See: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-TEMPLATES-BACKEND
        "BACKEND": "django.template.backends.django.DjangoTemplates",
        # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-dirs
        "DIRS": [
            str(APPS_DIR.path("templates")),
        ],
        "OPTIONS": {
            # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-debug
            "debug": DEBUG,
            # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-loaders
            # https://docs.djangoproject.com/en/dev/ref/templates/api/#loader-types
            "loaders": [
                "django.template.loaders.filesystem.Loader",
                "django.template.loaders.app_directories.Loader",
            ],
            # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-context-processors
            "context_processors": [
                "django.template.context_processors.debug",
                "django.template.context_processors.request",
                "django.contrib.auth.context_processors.auth",
                "django.template.context_processors.i18n",
                "django.template.context_processors.media",
                "django.template.context_processors.static",
                "django.template.context_processors.tz",
                "django.contrib.messages.context_processors.messages",
                # Your stuff: custom template context processors go here
            ],
        },
    },
]

# See: http://django-crispy-forms.readthedocs.org/en/latest/install.html#template-packs
CRISPY_TEMPLATE_PACK = "bootstrap3"

# STATIC FILE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
STATIC_ROOT = str(ROOT_DIR("staticfiles"))

# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
STATIC_URL = "/static/"

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
STATICFILES_DIRS = (str(APPS_DIR.path("static")),)

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-finders
STATICFILES_FINDERS = (
    "django.contrib.staticfiles.finders.FileSystemFinder",
    "django.contrib.staticfiles.finders.AppDirectoriesFinder",
)

# MEDIA CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
MEDIA_ROOT = str(APPS_DIR("media"))

# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
MEDIA_URL = "/media/"

# URL Configuration
# ------------------------------------------------------------------------------
ROOT_URLCONF = "config.urls"

# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
WSGI_APPLICATION = "config.wsgi.application"

# AUTHENTICATION CONFIGURATION
# ------------------------------------------------------------------------------
AUTHENTICATION_BACKENDS = ("django.contrib.auth.backends.ModelBackend",)

# Custom user app defaults

# SLUGLIFIER
AUTOSLUG_SLUGIFY_FUNCTION = "slugify.slugify"

# Location of root django.contrib.admin URL, use {% url 'admin:index' %}
ADMIN_URL = "admin/"

# Your common stuff: Below this line define 3rd party library settings
DEFAULT_QUIP_UUID = env("DEFAULT_QUIP_UUID", default=None)
RANDOM_QUIP_CACHE_DURATION = env.int(
    "RANDOM_QUIP_CACHE_DURATION", default=60 * 60 * 24
)  # default to one day
GIPHY_HTML_VIDEO = env.bool("GIPHY_HTML_VIDEO", default=True)

# New in Django 3.2:
# 3.1 and older default is 32-bit AutoField. 3.2 now recommends 64-bit BigAutoField.
# See "Customizing type of auto-created primary keys" in the 3.2 release notes:
# https://docs.djangoproject.com/en/3.2/releases/3.2/
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"


import nltk
import requests
import json

def do_pos_tagging(text):
    url = 'http://localhost:9000/?properties={"annotators":"pos,openie","outputFormat":"json"}'
    response = requests.post(url, data=text)
    return response.json()

def extract_named_entities(text):
    text = do_pos_tagging(text)
    return text['sentences'][0]['tokens']

def remove_dbspotlight_annotations(spotlight_result, pos_tagged_result):
    dup = pos_tagged_result.copy()
    for idx, p in enumerate(pos_tagged_result):
        if p['pos'] in ('NN', 'NNS'):
            for s in spotlight_result['annotated_text']['location_data']:
                if int(p['characterOffsetBegin']) == int(s['start']) or int(p['characterOffsetEnd']) == int(s['end']):
                    dup.remove(p)
        else:
            dup.remove(p)
    return dup


import json
import slugify
import csv

out = {
    "spo": {
        "id":"spo",
        "short_name":'SPO',
        'name': "Strana Práv Občanů",
        "friendly_name": 'spo',
        "vote":{}
    },
    "svobodni": {
        "id":"svobodni",
        "short_name":'Svobodní',
        'name': "Strana svobodných občanů",
        "friendly_name": 'svobodni',
        "vote":{}
    },
    "pirati": {
        "id":"pirati",
        "short_name":'Piráti',
        'name': "Česká pirátská strana",
        "friendly_name": 'pirati',
        "vote":{}
    },
    "zeleni": {
        "id":"zeleni",
        "short_name":'Zelení',
        'name': "Strana zelených",
        "friendly_name": 'zeleni',
        "vote":{}
    },
}
with open("answers2.csv") as infile:
    csvr = csv.reader(infile)
    for row in csvr:
        out['spo']['vote'][row[5]] = int(row[1])
        out['svobodni']['vote'][row[5]] = int(row[2])
        out['pirati']['vote'][row[5]] = int(row[3])
        out['zeleni']['vote'][row[5]] = int(row[4])

with open("answers2.json", "w") as outfile:
    json.dump(out,outfile)


# use the TimeDistributed layer to process the output from the LSTM hidden layer.
from numpy import array
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import TimeDistributed
from keras.layers import LSTM
# prepare sequence
length = 5
seq = array([i/float(length) for i in range(length)])
print(seq)

# There are two key points to remember when using the TimeDistributed wrapper layer:
#  The input must be (at least) 3D. This often means that you will need to configure your last LSTM layer
#   prior to your TimeDistributed wrapped Dense layer to return sequences (e.g. set the "return_sequences" argument to "True").
#  The output will be 3D. This means that if your TimeDistributed wrapped Dense layer is your output layer and you are
#   predicting a sequence, you will need to resize your y array into a 3D vector.

X = seq.reshape(1, length, 1)
# Define the shape of the output as having 1 sample, 5 time steps, and 1 feature, just like the input sequence
y = seq.reshape(1, length, 1)

# define LSTM configuration
n_neurons = length
n_batch = 1
# The TimeDistributed achieves trick by applying the same Dense layer (same weights) to the LSTMs outputs for one time step at a time.
# In this way, the output layer only needs one connection to each LSTM unit (plus one bias).
# For this reason, the number of training epochs needs to be increased to account for the smaller network capacity.
n_epoch = 1000
# create LSTM
model = Sequential()
# Define the LSTM hidden layer to return sequences rather than single values
# This has the effect of each LSTM unit returning a sequence of 5 outputs, one for each
# time step in the input data, instead of single output value
model.add(LSTM(n_neurons, input_shape=(length, 1), return_sequences=True))
# Use the TimeDistributed on the output layer to wrap a fully connected Dense layer with a single output.
# The single output value in the output layer is key. It highlights that we intend to output one time step from the sequence
# for each time step in the input. It just so happens that we will process 5 time steps of the input sequence at a time.
model.add(TimeDistributed(Dense(1)))
model.compile(loss='mean_squared_error', optimizer='adam')
print(model.summary())
# train LSTM
model.fit(X, y, epochs=n_epoch, batch_size=n_batch, verbose=0)
# evaluate
result = model.predict(X, batch_size=n_batch, verbose=0)
for value in result[0,:,0]:
	print('%.1f' % value)


from __future__ import annotations

import math
from typing import TYPE_CHECKING

import cairocffi
import xcffib.xproto

from libqtile.backend import base

if TYPE_CHECKING:
    from typing import Optional

    from libqtile.backend.base import Internal
    from libqtile.core.manager import Qtile

class Drawer(base.Drawer):
    """A helper class for drawing to Internal windows.

    The underlying surface here is an XCBSurface backed by a pixmap. We draw to the
    pixmap starting at offset 0, 0, and when the time comes to display to the window (on
    draw()), we copy the appropriate portion of the pixmap onto the window. In the event
    that our drawing area is resized, we invalidate the underlying surface and pixmap
    and recreate them when we need them again with the new geometry.
    """
    def __init__(self, qtile: Qtile, win: Internal, width: int, height: int):
        base.Drawer.__init__(self, qtile, win, width, height)
        self._xcb_surface = None
        self._pixmap = None
        self._gc = None

    def finalize(self):
        self._free_xcb_surface()
        self._free_pixmap()
        self._free_gc()
        base.Drawer.finalize(self)

    @property
    def width(self):
        return self._width

    @width.setter
    def width(self, width):
        if width > self._width:
            self._free_xcb_surface()
            self._free_pixmap()
        self._width = width

    @property
    def height(self):
        return self._height

    @height.setter
    def height(self, height):
        if height > self._height:
            self._free_xcb_surface()
            self._free_pixmap()
        self._height = height

    @property
    def pixmap(self):
        if self._pixmap is None:
            # draw here since the only use case of this function is in the
            # systray widget which expects a filled pixmap.
            self.draw()
        return self._pixmap

    def _create_gc(self):
        gc = self.qtile.core.conn.conn.generate_id()
        self.qtile.core.conn.conn.core.CreateGC(
            gc,
            self._win.wid,
            xcffib.xproto.GC.Foreground | xcffib.xproto.GC.Background,
            [
                self.qtile.core.conn.default_screen.black_pixel,
                self.qtile.core.conn.default_screen.white_pixel,
            ],
        )
        return gc

    def _free_gc(self):
        if self._gc is not None:
            self.qtile.core.conn.conn.core.FreeGC(self._gc)
            self._gc = None

    def _create_xcb_surface(self):
        surface = cairocffi.XCBSurface(
            self.qtile.core.conn.conn,
            self._pixmap,
            self._find_root_visual(),
            self.width,
            self.height,
        )
        return surface

    def _free_xcb_surface(self):
        if self._xcb_surface is not None:
            self._xcb_surface.finish()
            self._xcb_surface = None

    def _create_pixmap(self):
        pixmap = self.qtile.core.conn.conn.generate_id()
        self.qtile.core.conn.conn.core.CreatePixmap(
            self.qtile.core.conn.default_screen.root_depth,
            pixmap,
            self._win.wid,
            self.width,
            self.height,
        )
        return pixmap

    def _free_pixmap(self):
        if self._pixmap is not None:
            self.qtile.core.conn.conn.core.FreePixmap(self._pixmap)
            self._pixmap = None

    def paint_to(self, drawer):
        # If XCBSurface has been invalidated, we need to draw now to create it
        if self._xcb_surface is None:
            self.draw()
        drawer.ctx.set_source_surface(self._xcb_surface)
        drawer.ctx.paint()

    def _paint(self):
        # Only attempt to run RecordingSurface's operations if it actually has
        # some. self.surface.ink_extents() computes the bounds of the current
        # list of operations. If any of the bounds are not 0.0 then the surface
        # has operations and we should paint them.
        if any(not math.isclose(0.0, i) for i in self.surface.ink_extents()):
            # Paint RecordingSurface operations to the XCBSurface
            ctx = cairocffi.Context(self._xcb_surface)
            ctx.set_source_surface(self.surface, 0, 0)
            ctx.paint()

            # Clear RecordingSurface of operations
            self._reset_surface()

    def draw(
        self,
        offsetx: int = 0,
        offsety: int = 0,
        width: Optional[int] = None,
        height: Optional[int] = None,
    ):
        # If this is our first draw, create the gc
        if self._gc is None:
            self._gc = self._create_gc()

        # If the Drawer has been resized/invalidated we need to recreate these
        if self._xcb_surface is None:
            self._pixmap = self._create_pixmap()
            self._xcb_surface = self._create_xcb_surface()

        # paint stored operations(if any) to XCBSurface
        self._paint()

        # Finally, copy XCBSurface's underlying pixmap to the window.
        self.qtile.core.conn.conn.core.CopyArea(  # type: ignore
            self._pixmap,
            self._win.wid,
            self._gc,
            0, 0,  # srcx, srcy
            offsetx, offsety,  # dstx, dsty
            self.width if width is None else width,
            self.height if height is None else height
        )

    def _find_root_visual(self):
        for i in self.qtile.core.conn.default_screen.allowed_depths:
            for v in i.visuals:
                if v.visual_id == self.qtile.core.conn.default_screen.root_visual:
                    return v


from flask import make_response, request, jsonify
from app.models import userModel
from app.auth import auth

def response_message(response):
    if response[0] == 201 or response[0] == 200:
        return make_response(jsonify({
            'status' : response[0],
            'token' : response[1],
            'User' : response[2]
        }), response[0])
    else:
        return make_response(jsonify({
            'status' : response[0],
            'error' : response[1]
        }), response[0])

@auth.route('/auth/signup', methods=['POST'])
def register_user():
    data = request.get_json(force=True)
    resp = userModel.create_account(data)

    message = response_message(resp)
    return message

@auth.route('/auth/login', methods=['POST'])
def user_login():
    data = request.get_json()
    resp = userModel.user_sign_in(data)

    message = response_message(resp)
    return message


# -*- coding: UTF-8 -*-
from crawl_tss.storage import MongoConn

__author__ = 'shen'

from scrapy.spider import Spider
from scrapy.selector import Selector

from crawl_tss.items import MessageItem

from crawl_tss import pipelines

class NewestMessage(Spider):
    name = "newestMessages"

    pipeline = set([
        pipelines.NewestMessages
    ])

    start_urls = ["http://218.94.159.102/tss/en/home/index.html"]

    def parse(self, response):
        sel = Selector(response)

        all_announcements = sel.xpath("//table[@class='forum_text']/tr")[2:11]

        all_topics = sel.xpath("//table[@class='forum_text']/tr")[15:24]

        # writeContext(response.body)

        items = []

        MongoConn.new_topics.drop()
        MongoConn.new_announcements.drop()

        for announcement in all_announcements:
            item = MessageItem()
            item['type'] = "announcement"
            item['title'] = announcement.xpath("td[1]/a[1]/@title").extract()
            item['link'] = announcement.xpath("td[1]/a[1]/@href").extract()
            item['courseName'] = announcement.xpath("td[3]/a[1]/@title").extract()
            item['time'] = announcement.xpath("td[2]").extract()
            items.append(item)

        for topic in all_topics:
            item = MessageItem()
            item['type'] = "topic"
            item['title'] = topic.xpath("td[1]/a[1]/@title").extract()
            item['link'] = topic.xpath("td[1]/a[1]/@href").extract()
            item['courseName'] = topic.xpath("td[3]/a[1]/@title").extract()
            item['time'] = topic.xpath("td[2]").extract()
            items.append(item)

        return items


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^$', views.index),
    url(r'^courses$', views.courses),
    url(r'^confirmation/(?P<id>\d+)$', views.confirmation),
    url(r'^keep$', views.keep),
    url(r'^delete/(?P<id>\d+)$', views.delete),
    # url(r'^comments$', views.comments)
]


from Crypto.PublicKey import RSA
from Crypto.Cipher import PKCS1_OAEP
import time
import matplotlib.pyplot as plt

#Three empty lists to store the values for time taken for execution
key = list()
enc = list()
dec = list()

#Generates and returns the key object containing the private and public keys
def newkeys(keysize):
   key = RSA.generate(keysize)
   private, public = key, key.publickey()
   return public, private

#Encrypts the message with the public key using OAEP padding
def encrypt(message, pub_key):
   cipher = PKCS1_OAEP.new(pub_key)
   return cipher.encrypt(message)

#Decrypts the encrypted message with the generated private key
def decrypt(ciphertext, priv_key):
   cipher = PKCS1_OAEP.new(priv_key)
   return cipher.decrypt(ciphertext)

def main():
   #The chosedn key lengths(in bits)
   key_lengths = [1024, 2048, 3072, 4096, 5120]

   for i in key_lengths:
      start = time.time()
      public, private = newkeys(i)
      end = time.time()
      key.append(1000*(end-start))

      start = time.time()
      ciphertext = encrypt(b"A message for encryption", public)
      end = time.time()
      enc.append(1000*(end-start))

      start = time.time()
      message = decrypt(ciphertext, private)
      end = time.time()
      dec.append(1000*(end-start))

   #print(key, enc, dec)

   '''
   Plotting the graph with the key lengths on the x-axis and logscale
   of time taken(in milliseconds) on the y-axis
   '''
   plt.figure()

   plt.plot(key_lengths, key, ".-b", label="Key_generation")
   plt.plot(key_lengths, enc, ".-g", label="Encryption")
   plt.plot(key_lengths, dec, ".-y", label="Decrption")
   plt.yscale("log")
   plt.xlabel("Key sizes")
   plt.ylabel("Time(ms)")
   plt.title("Log scale of time")
   plt.legend()

   plt.show()

if __name__ == "__main__":
	main()


import json
from datetime import datetime

from channels.db import database_sync_to_async
from channels.generic.websocket import AsyncJsonWebsocketConsumer
from django.contrib.contenttypes.models import ContentType
from django.core.paginator import Paginator

from chat.exceptions import ClientError
from friend.models import FriendRequest, FriendList
from notification.constants import DEFAULT_NOTIFICATION_PAGE_SIZE, GENERAL_MSG_TYPE_NOTIFICATIONS_PAYLOAD, \
    GENERAL_MSG_TYPE_UPDATED_NOTIFICATION, GENERAL_MSG_TYPE_PAGINATION_EXHAUSTED, \
    GENERAL_MSG_TYPE_NOTIFICATIONS_REFRESH_PAYLOAD
from notification.models import Notification
from notification.utils import LazyNotificationEncoder

class NotificationConsumer(AsyncJsonWebsocketConsumer):
    """
    Passing data to and from header.html. Notifications are displayed as "drop-downs" in the nav bar.
    There is two major categories of notifications:
        1. General Notifications
            1. FriendRequest
            2. FriendList
        1. Chat Notifications
            1. UnreadChatRoomMessages
    """

    async def connect(self):
        """
        Called when the websocket is handshaking as part of initial connection.
        """
        print("NotificationConsumer: connect: " + str(self.scope["user"]))
        await self.accept()

    async def disconnect(self, code):
        """
        Called when the WebSocket closes for any reason.
        """
        print("NotificationConsumer: disconnect")

    async def receive_json(self, content):
        """
        Called when we get a text frame. Channels will JSON-decode the payload
        for us and pass it as the first argument.
        """
        command = content.get("command", None)
        print("NotificationConsumer: receive_json. Command: " + command)
        try:
            if command == "get_general_notifications":
                payload = await get_general_notifications(self.scope["user"], content.get("page_number", None))
                if payload is None:
                    await self.general_pagination_exhausted()
                else:
                    payload = json.loads(payload)
                    await self.send_general_notifications_payload(payload['notifications'], payload['new_page_number'])
            elif command == "accept_friend_request":
                notification_id = content['notification_id']
                payload = await accept_friend_request(self.scope['user'], notification_id)
                if payload is None:
                    raise ClientError(402, "Something went wrong. Try refreshing the browser.")
                else:
                    payload = json.loads(payload)
                    await self.send_updated_friend_request_notification(payload['notification'])
            elif command == "decline_friend_request":
                notification_id = content['notification_id']
                payload = await decline_friend_request(self.scope['user'], notification_id)
                if payload is None:
                    raise ClientError("Something went wrong. Try refreshing the browser.")
                else:
                    payload = json.loads(payload)
                    await self.send_updated_friend_request_notification(payload['notification'])
            elif command == "refresh_general_notifications":
                payload = await refresh_general_notifications(self.scope["user"], content['oldest_timestamp'],
                                                              content['newest_timestamp'])
                if payload is None:
                    raise ClientError("Something went wrong. Try refreshing the browser.")
                else:
                    payload = json.loads(payload)
                    await self.send_general_refreshed_notifications_payload(payload['notifications'])
        except Exception as e:
            print("EXCEPTION: receive_json: " + str(e))
            pass

    async def display_progress_bar(self, shouldDisplay):
        print("NotificationConsumer: display_progress_bar: " + str(shouldDisplay))
        await self.send_json(
            {
                "progress_bar": shouldDisplay,
            },
        )

    async def send_general_notifications_payload(self, notifications, new_page_number):
        """
        Called by receive_json when ready to send a json array of the notifications
        """
        # print("NotificationConsumer: send_general_notifications_payload")
        await self.send_json(
            {
                "general_msg_type": GENERAL_MSG_TYPE_NOTIFICATIONS_PAYLOAD,
                "notifications": notifications,
                "new_page_number": new_page_number,
            },
        )

    async def send_updated_friend_request_notification(self, notification):
        """
        After a friend request is accepted or declined, send the updated notification to template
        payload contains 'notification' and 'response':
        1. payload['notification']
        2. payload['response']
        """
        await self.send_json({
            "general_msg_type": GENERAL_MSG_TYPE_UPDATED_NOTIFICATION,
            "notification": notification,
        })

    async def general_pagination_exhausted(self):
        """
        Called by receive_json when pagination is exhausted for general notifications
        """
        # print("General Pagination DONE... No more notifications.")
        await self.send_json({
            "general_msg_type": GENERAL_MSG_TYPE_PAGINATION_EXHAUSTED,
        })

    async def send_general_refreshed_notifications_payload(self, notifications):
        """
        Called by receive_json when ready to send a json array of the notifications
        """
        # print("NotificationConsumer: send_general_refreshed_notifications_payload: " + str(notifications))
        await self.send_json(
            {
                "general_msg_type": GENERAL_MSG_TYPE_NOTIFICATIONS_REFRESH_PAYLOAD,
                "notifications": notifications,
            },
        )

@database_sync_to_async
def get_general_notifications(user, page_number):
    """
    Get General Notifications with Pagination (next page of results).
    This is for appending to the bottom of the notifications list.
    General Notifications are:
    1. FriendRequest
    2. FriendList
    """
    if user.is_authenticated:
        friend_request_ct = ContentType.objects.get_for_model(FriendRequest)
        friend_list_ct = ContentType.objects.get_for_model(FriendList)
        notifications = Notification.objects.filter(target=user,
                                                    content_type__in=[friend_request_ct, friend_list_ct]).order_by(
            '-timestamp')
        p = Paginator(notifications, DEFAULT_NOTIFICATION_PAGE_SIZE)

        payload = {}
        if len(notifications) > 0:
            if int(page_number) <= p.num_pages:
                s = LazyNotificationEncoder()
                serialized_notifications = s.serialize(p.page(page_number).object_list)
                payload['notifications'] = serialized_notifications
                new_page_number = int(page_number) + 1
                payload['new_page_number'] = new_page_number
        else:
            return None
    else:
        raise ClientError("User must be authenticated to get notifications.")

    return json.dumps(payload)

@database_sync_to_async
def accept_friend_request(user, notification_id):
    """
    接收好友请求
    """
    payload = {}
    if user.is_authenticated:
        try:
            notification = Notification.objects.get(pk=notification_id)
            friend_request = notification.content_object
            # confirm this is the correct user
            if friend_request.receiver == user:
                # accept the request and get the updated notification
                updated_notification = friend_request.accept()

                # return the notification associated with this FriendRequest
                s = LazyNotificationEncoder()
                payload['notification'] = s.serialize([updated_notification])[0]
                return json.dumps(payload)
        except Notification.DoesNotExist:
            raise ClientError(202, "Database Error.")
    return None

@database_sync_to_async
def decline_friend_request(user, notification_id):
    """
    Decline a friend request
    """
    payload = {}
    if user.is_authenticated:
        try:
            notification = Notification.objects.get(pk=notification_id)
            friend_request = notification.content_object
            # confirm this is the correct user
            if friend_request.receiver == user:
                # accept the request and get the updated notification
                updated_notification = friend_request.decline()

                # return the notification associated with this FriendRequest
                s = LazyNotificationEncoder()
                payload['notification'] = s.serialize([updated_notification])[0]
                return json.dumps(payload)
        except Notification.DoesNotExist:
            raise ClientError("An error occurred with that notification. Try refreshing the browser.")
    return None

@database_sync_to_async
def refresh_general_notifications(user, oldest_timestamp, newest_timestamp):
    """
    Retrieve the general notifications newer than the oldest one on the screen and younger than the newest one the screen.
    The result will be: Notifications currently visible will be updated
    """
    payload = {}
    if user.is_authenticated:
        oldest_ts = oldest_timestamp[0:oldest_timestamp.find("+")] # remove timezone because who cares
        oldest_ts = datetime.strptime(oldest_ts, '%Y-%m-%d %H:%M:%S.%f')
        newest_ts = newest_timestamp[0:newest_timestamp.find("+")] # remove timezone because who cares
        newest_ts = datetime.strptime(newest_ts, '%Y-%m-%d %H:%M:%S.%f')
        friend_request_ct = ContentType.objects.get_for_model(FriendRequest)
        friend_list_ct = ContentType.objects.get_for_model(FriendList)
        notifications = Notification.objects.filter(target=user,
                                                    content_type__in=[friend_request_ct, friend_list_ct],
                                                    timestamp__gte=oldest_ts,
                                                    timestamp__lte=newest_ts).order_by('-timestamp')

        s = LazyNotificationEncoder()
        payload['notifications'] = s.serialize(notifications)
    else:
        raise ClientError("你必须登录以获得通知.")

    return json.dumps(payload)


from ROOT import TH2F, gStyle, TCanvas
import ROOT
import json
import numpy as np
import sys

def main():
    args = sys.argv
    f = open(args[1], 'r')
    json_data = json.load(f)
    pix_list = ["Col", "Enable", "Hitbus", "InjEn", "TDAC"]
    col_list = np.linspace(264., 399., 136.)
    row_list = np.linspace(1., 191., 191.)
    #print(col_list)

    output = ROOT.TFile(args[1]+'enable_map.root','RECREATE')
    h2D = TH2F("h2D", "", 136, 264, 400, 192, 0, 192)
    gStyle.SetPalette(1)
#    ["PixelConfig"][{"Col", "Enable", "Hitbus", "InjEn", "TDAC"},{}]
    for col in col_list:
        #print(col)
        #print("Col:{} \t".format(col))#json_data["RD53A"]["PixelConfig"][col]))
        colnum = json_data["RD53A"]["PixelConfig"][int(col)]
        for row in row_list:
            if colnum["Enable"][int(row)] == 1:
                h2D.Fill(col, row)
    output.Write()
    can = ROOT.TCanvas()
    h2D.Draw("colz")

    h2D.SetXTitle("col")
    h2D.SetYTitle("row")
    h2D.SetStats(0)
    can.SaveAs(args[1]+'.png')

if __name__=='__main__':
    main()