from django import forms

class PanelistForm(forms.Form):
    email = forms.EmailField(label='Email address')
    name = forms.CharField(label='Name', max_length=280)
    returning = forms.BooleanField(
        label='Returning Panelist',
        help_text="Have you been a panelist at ConFusion before?",
        required=False)
    bio = forms.CharField(
        label='Bio',
        widget=forms.Textarea(attrs={'rows':3}),
        help_text="New panelist? Tell us a little about yourself. (This is for our edification only and won't be published).",
        required=False)

class PanelSubmissionForm(forms.Form):
    email = forms.EmailField(
        label='Contact Email',
        max_length=280,
        help_text="If you'd like to be on or moderate this panel, please use the same email you used on the Panelist Signup Form.")
    title = forms.CharField(
        label='Panel Title',
        max_length=280,
        help_text="Panel titles and descriptions may be edited for style, clarity, or fun.")
    description = forms.CharField(
        label='Panel Description',
        widget=forms.Textarea(attrs={'rows':4}),
        help_text="This should be a draft of the panel description as it would appear in the program book.")
    notes = forms.CharField(
        label='Notes for ConFusion Planners',
        widget=forms.Textarea(attrs={'rows':3}),
        required=False,
        help_text="Does this panel require a/v? Did you pre-arrange it with other panelists? Any other notes for us?"
        )


import re
import json
from .scraper import Scraper

class MetasploitParser(Scraper):
    def __init__(self, filename=None, name=None, exploit_type=None, title=None, platform=None, exploit=None, mongoclient=None, date=None):
        ext = ['.rb']
        super().__init__(filename, name, exploit_type, title, platform, exploit, mongoclient, date, ext)
        self.brackets = {
            '{': '}',
            '[': ']',
            '(': ')',
            '}': '{',
            ']': '[',
            ')': '('
        }

    def parse_infos(self):
        cves = self.db['cves']

        self.exploit = self.exploit.replace('\\', '\\\\')

        query = self.parsed_col.find_one({"filename": self.filename})
        if query is not None:
            parsed = query['parsed']
            if parsed:
                return

        error = False
        parsed_file = True
        try:
            description = re.findall(r"[\'\"]Description[\'\"]\s*=>\s*(.*)", self.exploit, re.DOTALL)
            if description:
                description = description[0]
                description = re.sub(r'\s+', ' ', description)
                start, end, is_q = self.find_desc(description)
                if start == -1:
                    desc = re.findall(r"[\'\"]Description[\'\"]\s*=>\s*[\'\"](.*?)[\'\"],", self.exploit, re.DOTALL)[0]
                    desc, = desc[0]
                else:
                    desc = description[start:end]
                    description = description.replace("'", '"')
                    if is_q:
                        desc = desc[start + 3:end - 1]
                    else:
                        desc = desc[start + 2:end - 1]
            else:
                desc = ""

            array = re.findall(r"\\?'(.*?)\\?\\?'[^\w]", self.exploit)  # Get all quoted strings
            for string in array:
                string_1 = string
                if '#' in string:
                    if len(string) == 1:
                        string_1 = '# '
                    else:
                        string_1 = string.replace('#',
                                                  'dash')  # Remove '#' inside them to not lose data when eliminate comments
                    self.exploit = re.sub(re.escape(string), "{}".format(string_1), self.exploit)
                string_1 = string_1.strip('"')  # Replace '"' with '`' to avoid conflict in JSON
                if '"' in string_1:
                    self.exploit = re.sub(re.escape(string_1), string_1.replace('"', '`'), self.exploit)
            self.exploit = re.sub(r'(?:[,\s](#[^{\n]?.*\n{1}\w.*|#[^{\n]?.*)|(#[\'"].*?[\'"]?\n))', '',
                                  self.exploit)  # Delete comments
            self.exploit = re.sub(r'=begin.*?=end', '', self.exploit, flags=re.DOTALL)  # Delete =begin and =end
            self.exploit = re.sub(r'@.*\s*=\s*.*', '', self.exploit)  # Delete annotations

            lista = re.findall(
                r'\bdef initialize(?:\(\s*i|\s*super|\(\s*.*?\s*\)?\s*super)(.*?)(?:\bregister_options\b|\bend\b)\s*(?:def|[A-Za-z_0-9]+\s*=)',
                self.exploit, re.DOTALL)[0]  # Get the main initialize function
            nospace = lista + 'end of the body'  # Add a flag
            nospace1 = re.findall(
                r",?\s*({?\s*[\'\"](?:Name|Description)[\'\"].*?),?\s*\)\s*\)?\s*(?:(?:opt.*)?(?:de)?register_(?:advanced_)?(?:evasion_)?options|end of the body)",
                nospace, re.M | re.S)  # Get rid of the begin and the end of the function, keep only the 'variables'
            if nospace1:
                nospace = nospace1[0]
            title = re.findall(r"[\'\"]Name[\'\"]\s*=>\s*['\"](.*?[\s]*.*?)['\"],", nospace)[0]  # Get title
            nospace = re.sub(r'=>', ':', nospace)  # Replace '=>' with ':'
            nospace = re.sub(r'\s+', ' ', nospace)  # Replace all two+ spaces with only one
            nospace = nospace.replace('\\', '\\\\')  # Encode '\'

            if '{' not in nospace[:4]:
                nospace = '{' + nospace + '}'  # Add brackets

            platform = re.findall(r'(%w{|%w\(|%w\[)(.*?)(}|\)|\]),?', nospace)  # Find platform

            nospace = re.sub(r"('BadChars' : )(?:'.*?'(?=\s?,\s|\s?})|\".*?\"?(?=\s?,\s'|\s?})|.*?(, '))",
                             r"\g<1>''\g<2>", nospace)  # Replace 'BadChars'

            nospace = re.sub(r'0x0*[0-9a-fA-F][0-9a-fA-F]*', '0', nospace)  # Replace hex numbers with '0'
            nospace = re.sub(r"'EncoderType' : (.*?)(,|}|\])", r"'EncoderType' : ''\g<2>",
                             nospace)  # Replace 'EncoderType'
            nospace = re.sub(r"\]\s*\[", "], [", nospace)  # Add ',' between '] ['
            authors = re.findall(r"'Authors?' : \[\s*(.*?)\],? '", nospace)  # Find the 'Author' (array)
            if not authors:
                authors = re.findall(r"'Authors?' : \s*(.*?[\s']),? '", nospace)  # Find the 'Author' (only one)
            if authors:
                authors = re.split(r", (?=(?:[^\"']*[\"'][^\"']*[\"'])*[^\"']*$)",
                                   authors[0])  # Split authors by the most outer ','
                for author in authors:
                    author = author.strip()
                    if not ((author.startswith('"') and author.endswith('"')) or (
                            author.startswith("'") and author.endswith("'"))):
                        nospace = re.sub(re.escape(author), "", nospace)  # Delete wrong name format
                    else:
                        author_rep = author[1:-1].replace('"', '')  # Delete '"' from 'O"Shea'
                        author_rep = author_rep.replace("'", '')  # Delete '"' from 'O"Shea'
                        nospace = re.sub(re.escape(author[1:-1]), author_rep, nospace)
            desc = '"' + desc.replace('"', "'") + '"'  # Replace '"' with "'"
            title = title.replace('"', "'")
            nospace = re.sub(r"(?!\B'[^']*)(\w*_LICENSE|ARCH_\w*)(\s*\]*?)(?![^']*'\B)", r"'\g<1>'",
                             nospace)  # Add quotation marks to 'License' and 'Arch' values
            nospace = re.sub(r'\'', '"', nospace)  # Transform single quotes to duble quotes
            nospace = re.sub(r'(.*?)" "(.*)', r'\g<1>", "\g<2>', nospace)  # Add ',' between 'a_string" "another_one'
            nospace = re.sub(r'(,)\s*,', r' \g<1>', nospace)  # Remove two consecutive ','
            nospace = re.sub(r'(,)\s*([}\]\)])', r' \g<2>', nospace)  # Remove ',' from ', }' (not allowed on JSON)
            nospace = re.sub(r'"\\\\\\\\x[a-zA-Z0-9\\x"\.\(\)+\s]+,', r'"",',
                             nospace)  # Remove some encoded hex values
            nospace = re.sub(r'(\[|{)\s*,', r' \g<1>', nospace)  # Remove ',' from '{, ' (not allowed on JSON)
            nospace = re.sub(r'("BrowserRequirements" : )({.*})\s*([},\]])(?=\s*["{\[]+|$)', r'\g<1>""\g<3>',
                             nospace)  # Delete 'BrowserRequirements'
            nospace = re.sub(r'"Stance" : (.*?)(,|})', r'"Stance" : "\g<1>"\g<2>',
                             nospace)  # Add quotation marks to 'Stance' value

            nospace = re.sub(r'"\s*:\s*(\(.*?\)\s*)([,}])', r'" : "\g<1>"\g<2>',
                             nospace)  # Add quotation marks to ': (some_unquoted_value)'
            nospace = re.sub(r'([^:,\[{\(\)+\.]) ([{"\[\'])', r'\g<1>, \g<2>', nospace)  # Add ',' to ') ('

            nospace = re.sub(r'(\s*?{\s*?|\s*?,\s*?)(["])?([a-zA-Z0-9_]+)(["])?\s?:\s+([^\/\[]+?)',
                             r'\g<1>"\g<3>" : \g<5>', nospace)  # Add quotation marks to 'lowercase_string: value'
            description = re.findall(r'"Description"\s*:\s*(.*)', nospace)
            if description:
                description = description[0]
                start, end, _ = self.find_desc(description)
                if start != -1:
                    nospace = nospace.replace(description[start:end], desc.replace('::', ':~:'))
            nospace = re.sub(
                r'("PrependEncoder" :\s+)(.*\.[\s\n\r]*[\w]+\(.*?\).*?(}?,?\s*")|[^"\(]\S*[^"},]|\(.*\)|".*?"\s*([,}]))',
                r'\g<1>""\g<3>\g<4>', nospace)  # Delete 'PrependEncoder'
            nospace = re.sub(r'("Targets"\s*:\s*)(\d+.*?\d+)', r'\g<1>"\g<2>"',
                             nospace)  # Add quotation marks to 'Targets: 1..2'
            nospace = re.sub(r'(\s*[{:,\[])\s+(:\w+)', r'\g<1> "\g<2>"', nospace)  # Add quotation marks to ':some_value'
            nospace = re.sub(r'"Name"\s*:\s*"(.*?)", "',
                             '"Name" : "' + title.replace('\n', ' ').replace('::', ':~:') + '", "',
                             nospace)  # Replace title
            if platform:
                _, platform, _ = platform[0]
                nospace = re.sub(r'(%w{|%w\(|%w\[)(.*?)(}|\)|\]),?', '"{}",'.format(platform),
                                 nospace)  # Replace platform
            nospace = re.sub(r'("Space" : )[^"][0-9_,+A-Za-z\*\.\)\(\s\-\$]*', r'\g<1> 1024, ',
                             nospace)  # Add constant value to 'Space' adress

            nospace = re.sub(
                r'"Prepend" : (?:.*\.[\s\n\r]*[\w]+\(.*?\).*?(}?,?\s*")|"?.*?"?\s*([,}]\s*[",])|"?.*?"\s*?)',
                r'\g<1>\g<2>', nospace)  # Remove 'Prepend' value (may contain method calls - first regex)
            nospace = re.sub(r'(,)\s*,', r' \g<1>', nospace)
            nospace = re.sub(r'(,)\s*([}\]\)])', r' \g<2>', nospace)
            nospace = re.sub(r'(\[|{)\s*,', r' \g<1>', nospace)
            nospace = re.sub(r',\s*\d+\s*[\-\+\*]\s*\d+', '', nospace)

            nospace = re.sub(r'(:\s+\[\s*)([A-Z_]+[^a-z0-9-"\]}{\[]+)(\s*\])', r'\g<1>"\g<2>"\g<3>',
                             nospace)  # Add quotation marks to ': CAPITAL_LETTERS'
            nospace = re.sub(r'"Signature" : \/.*?\/.*?,', '', nospace)  # Remove Signature
            arithmetics = re.findall(r'(:\s*)(\d+)(\s*)(\*|\+|-)(\s*)(\d+)(,?)',
                                     nospace)  # Find arithmetic operations to replace with actual value
            while (arithmetics):
                for operation in arithmetics:
                    beg, a, sp1, sign, sp2, b, ending = operation
                    result = 0
                    if sign == '+':
                        result = int(a) + int(b)
                    elif sign == '*':
                        result = int(a) * int(b)

                    nospace = re.sub(re.escape('{}{}{}{}{}{}{}'.format(beg, a, sp1, sign, sp2, b, ending)),
                                     beg + str(result) + ending, nospace)
                arithmetics = re.findall(r'(:\s*)(\d+)(\s*)(\*|\+|-)(\s*)(\d+)(,?)',
                                         nospace)  # Find arithmetic operations to replace with actual value

            hex_values = re.findall(r'(?:\\x[0-9a-fA-F]+)+', nospace)  # Find hex escaped values
            for group in hex_values:
                nospace = re.sub(re.escape(group), "Some string", nospace)  # Replace them with a constant
            nospace = re.sub(r': nil', ': null', nospace)  # Replace 'nil'
            targets = re.findall(r'("Targets" : \[\s*\[?.*?[,\s]+.*\s*\]\s*\])', nospace)  # Edit the 'Targets' array
            if targets:
                to_delete = ['References', 'Author', 'Disclosure']
                values = [x in targets[0] for x in to_delete]
                if len(set(values)) == 1 and values[0] == False:
                    nospace = re.sub(r'("Targets" : \[\s*\[?.*?[,\s]+.*\s*\]\s*\])', r'', nospace)
                    nospace = re.sub(r'(,)\s*,', r' \g<1>', nospace)
                    nospace = re.sub(r'(,)\s*([}\]\)])', r' \g<2>', nospace)
                    nospace = re.sub(r'(\[|{)\s*,', r' \g<1>', nospace)
                    nospace = re.sub(r',\s*\d+\s*[\-\+\*]\s*\d+', r'', nospace)
                else:
                    to_edit = re.findall(r'\[\s*\[?\s*(.*?\s*})\s*\]\s*\]?', targets[0])  # Get the arrays
                    if not to_edit:
                        targets[0] = re.sub(r'"References" : \[\s*(.*?)\s*\]\s*\]', '', targets[0])
                        to_edit = re.findall(r'\[\s*\[?\s*(.*?\s*\])\s*\]?\s*\]?', targets[0])  # Can be only one
                    for i, match in enumerate(to_edit):
                        if match.endswith(']'):  # Some problematic endings
                            if not re.findall(r'"Arch" : \[', match):
                                match = match[:-1]
                        check_brackets = re.findall(r'^".*?"\s*,\s*(.*)', match)[
                            0]  # Check if after "some string" exist a '{' eg. ["some_string", {}]
                        if not check_brackets.startswith(
                                '{'):  # If not, add them, otherwise JSON will throw an exception (can't have "key" : "value" inside arrays only if it is an object)
                            to_insert = '{ ' + check_brackets + ' }'
                            match1 = match.replace(check_brackets, to_insert)
                            nospace = re.sub(re.escape(match), match1, nospace)
            nospace = re.sub(r'\\"([^,])', r"'\g<1>",
                             nospace)  # Some problematic endings from Windows paths eg. "C:\\"(treated like escape for '"')
            references = re.findall(r'"References" : \[\s*(.*?)\s*\]\s*,\s*"',
                                    nospace)  # Find references eg. ([ ["CVE", "YYYY-NR"] ])
            if references:
                references = references[0]
                if references:
                    if references.startswith('[') and not references.endswith(']'):
                        nospace = re.sub(re.escape(references), references + ' ]', nospace)
                    if not references.startswith(
                            '['):  # Some anomalies like '"URL" : "SRC"' or '"URL", "SRC", "URL2", "SRC2"' so trying to create the correct format
                        new_array_1 = references.split(', ')
                        new_array = []

                        for string in new_array_1:
                            if re.findall(r'(".*?")\s*:\s*(".*?")', string):
                                new_array.append(string)
                            else:
                                values = string.strip('"').split(' ')
                                new_array.extend('"' + string + '"' for string in values)
                        alt_array = []
                        alt_array.extend(
                            '[ {} ]'.format(arr) for arr in new_array if re.findall(r'(".*?")\s*:\s*(".*?")', arr))  #
                        non_array = [arr for arr in new_array if not re.findall(r'(".*?")\s*:\s*(".*?")', arr)]
                        if len(non_array) % 2 == 0:
                            new_array = ['[ {}, {} ]'.format(non_array[i], non_array[i + 1]) for i in
                                         range(0, len(non_array), 2)]
                            alt_array.extend(new_array)
                        references = ', '.join(alt_array)
                    references_replaced = re.sub(r'("\s*(?:\\\\\\\\|\+),?\s*")', '',
                                                 references)  # Eliminate multiline strings
                    references_replaced = re.sub(r'(\[\s*".*?")\s*:\s*(".*?"\s*\])', r'\g<1>, \g<2>',
                                                 references_replaced)  # Replace ':' with ','
                    nospace = re.sub(re.escape(references), references_replaced,
                                     nospace)  # Replace the old format with the newly created on

            nospace = re.sub(r':?\s([^"]\w*::.*?[^"])([,}\]])', r': ""\g<2>',
                             nospace)  # Add quotation marks to ': MSF::CONSOLE' like values
            nospace = re.sub(r'("DECODERSTUB"\s*:\s*).*?\)', r'\g<1>""', nospace)
            myfile = json.dumps(nospace.replace('\\', '\\\\'))  # Escape the 'backslash' and dumps to JSON

            jsonf = json.loads(json.loads(myfile))
            title = ''

            if self.collection.find_one({"filename": self.name}) is not None:
                title = self.collection.find_one({"filename": self.name})['cve']
            else:
                if 'References' in jsonf:
                    for arr in jsonf.get('References'):
                        if arr and arr[0] == 'CVE':
                            title = title + ' ' + arr[0] + '-' + arr[1]
                    title = title.lstrip()

            if not title:
                title = re.sub(r'\s', '_', self.title)
                title = re.sub(r'\.', '@', title)
                title = self.title + '_' + title

            myDict = {
                "EDB-ID": self.name,
                "Vulnerability": title,
                "Date": self.date,
                "Versions": json.loads(self.get_version_from_name(jsonf.get('Description'))),
            }

            arr = ['Name', 'Description', 'Platform', 'References', 'Targets']

            for key in arr:
                if key in jsonf.keys():
                    if key == 'Name':
                        myDict[key] = self.title
                    else:
                        myDict[key] = jsonf.get(key)
                else:
                    if key == 'Name':
                        myDict[key] = self.title
                    elif key == 'Platform':
                        myDict[key] = self.platform

            myDict['Type'] = self.exploit_type

            URIs = self.parse_url()

            myDict['URI'] = list(set(URIs))
            cves.update({"EDB-ID": self.name}, myDict, upsert=True)
        except Exception as e:
            error, parsed_file = self.found_error(e)
        finally:
            self.update_parsed_obj_db(parsed_file, error)

    def parse_url(self):

        register_options = re.findall(r'register_options\((.*?)],?(?:\s*self\.class)?\)?\s*(?:end)', self.exploit,
                                      re.DOTALL)
        if register_options:
            register_options = register_options[0]

        self.register_options = register_options

        URIs = re.findall(r'(GET|POST|PATCH|PUT) (.*) HTTP', self.exploit)
        if URIs:
            URIs = ['/' + URIs[i][1].lstrip('/') for i in range(len(URIs)) if URIs[i][1] != '/']
        uris = re.findall(r'\s*[\'\"](\/[a-zA-Z0-9_.\-~#{}]*?)[\'\"]\s*', self.exploit)
        URIs.extend(self.construct_url(uris))
        normalize = re.findall(r'normalize_uri\((.*)\)', self.exploit)
        if normalize:
            urls = {}
            for string in normalize:

                to_search_1 = re.findall(r'[?!^\'"](.*?)[?!^\'"]', string)
                splitted = string.replace(' ', '').replace("'", '').split(',')
                to_search = [word for word in splitted if word not in to_search_1]
                if to_search:
                    for i in range(len(to_search)):
                        if 'uri' in to_search[i] or 'path' in to_search[i] or 'datastore' in to_search[i].lower():
                            word = re.findall(r"'TARGETURI',\s*\[(.*?)\]", self.exploit)
                            if word:
                                word = word[0].replace(' ', '').split(',')[2]
                            else:
                                word = 'to_del'
                        elif 'rand' in to_search[i]:
                            continue
                        else:
                            word = re.findall(re.escape(to_search[i]) + r'\s*=\s*[\'"]?(.*?)[\'"]\n', self.exploit)
                            if not word:
                                word = 'to_del'
                            else:
                                word = word[0]
                        urls[to_search[i]] = word
            normaliz = []
            for string in normalize:
                for key in urls.keys():
                    if key in string:
                        string = string.replace(key, urls[key])
                normaliz.append(string.replace('to_del,', ''))
            URIs.extend(self.construct_url(['/' + '/'.join(
                s.strip('/').replace("'", '').replace(',', '').replace('"', '').replace(' ', '') for s in
                normaliz[i].split(',') if 'uri' not in s and 'nil' not in s) for i in range(len(normaliz))]))
        uris = re.findall(r"'uri'\s*=>\s*[\'\"]\/?#{.*?}(\S*)\?\S*[\'\"]", self.exploit)
        URIs.extend(self.construct_url(uris))

        for i, uri in enumerate(URIs):
            regex = re.findall(r'(.*?)%[qQwW][\(\[{](.*?)[\]\)}]', uri)
            if regex:
                regex = regex[0]
                URIs[i] = regex[0] + regex[1]
        return URIs

    # There are some problems with regex so parsing the description was the best way to get all the necessary info
    def find_desc(self, description):
        start = -1
        end = -1
        count = 0
        found = False
        is_q = False
        delimiter = ''
        for i, word in enumerate(description):

            if word == '%' and start == -1:  # Find the %[qQ] and the delimiter '(', '[' or '{'
                start = i
                if description[i + 1] in 'qQ':
                    is_q = True
                    delimiter = description[i + 2]
                else:
                    delimiter = description[i + 1]
            if word in '{[(':
                found = True
                count += 1
            elif word in '}])':
                count -= 1

            if (count == 0 and found and word == self.revert(
                    delimiter)):  # If there's a matching paranthesis and a odd number of them
                return (start, i + 1, is_q)
            elif (description[i - 1] == self.revert(delimiter) and word == ',' and "".join(description[i + 1:].split())[
                0] in "\'\""):  # There can be some anomalies and try to find the coresponding paranthesis in construction like ")'Author"
                return (start, i, is_q)
        return (start, end, is_q)

    def revert(self, char):
        return self.brackets.get(char)

    def construct_url(self, uris):
        URIs = []
        register_options = self.register_options
        if uris:
            for i in range(len(uris)):

                if '/bin/' in uris[i] or 'cmd' in uris[i] or '/c' == uris[i] or '/>' in uris[i] or '\", ' in uris[i] or "/', " in uris[i] or '/' == uris[i]:  # If we found '/bin/sh' or '/cmd' continue
                    continue
                elif 'path' in uris[i] or 'target' in uris[i] or 'base' in uris[
                    i]:  # If there's a construction like targeturi.path continue
                    continue
                elif '\\\\r' in uris[i] or '\\\\n' in uris[i] or '/@' in uris[i] or 'rand' in uris[i]:
                    continue
                elif 'datastore' in uris[i].lower():  # If url = datastore[TARGETURI]
                    word = re.findall(r'datastore\[(.*)\]', uris[i])  # Get the datastore key
                    if word:
                        word = word[0]
                    else:
                        continue
                    if word not in register_options:  # If the key is not in register_options, then the value is manually introduced in metasploit framework
                        continue
                    to_replace_arr = re.findall(r"'.*?{}',\s*\[(.*)\]".format(re.escape(word)), register_options)[
                        0]  # Get the array from construction like 'TARGETURI',[boolean, COMMENT', '/PATH']
                    to_replace_word = to_replace_arr.replace(' ', '').split(',')
                    if len(to_replace_word) > 2:
                        to_replace_word = to_replace_word[2][1:-1]  # Get the value from the array and strip the "'"
                    else:
                        continue
                    if to_replace_word == '/':
                        continue
                    URIs.extend(
                        [re.sub(r'datastore\[(.*)\]', to_replace_word, uris[i])])  # Replace the datastore with the value
                elif 'dash{' not in str(uris[i]) and '#{' not in str(uris[i]):
                    if '/' != uris[i]:  # If the path is '/', don't care
                        URIs.extend(['/' + uris[i].lstrip('/')])  # add the '/' at the begining of the path
                elif 'dash{' in str(uris[i]) or '#{' in str(uris[i]):
                    to_search = re.findall('(?:dash|#){(.*)}', uris[i])  # The url is like '/dash{url}/'
                    if to_search:
                        urls = []
                        for i in range(len(to_search)):
                            urls.extend(re.findall(to_search[i] + r'\s*=\s*[\'"]?(.*?)[\'"]\n',
                                                   self.exploit))  # Try to find that variable
                        if urls:
                            URIs.extend(['/' + re.sub(r'(?:dash|#){(.*)}', url, uris[i]).lstrip('/') for url in urls if
                                         url != '/'])  # Add the new uri to the main array
        return URIs


import torch
from torch import nn  # 神经网路蝉蛹

# from torch import functional as F  # 旧版
from torch.nn import functional as F  # 常用函数
from torch import optim  # 优化工具包
import torchvision  # 视觉
from matplotlib import pyplot as plt
from utils import plot_image, plot_curve, one_hot

# 1. 加载数据集, 通过touchvision工具包直接下载, 并行读取多张图片. 这里的图片是28x28
# 70k图片是用来训练,
batch_size = 512
train_loader = torch.utils.data.DataLoader(
    torchvision.datasets.MNIST(
        "data/mnist_data",
        train=True,
        download=True,
        transform=torchvision.transforms.Compose(
            [
                torchvision.transforms.ToTensor(),
                # 数据下载以后不是一个正态分布, 可以使用这个来进行正态分布, 主要影响的是性能, 其他的都不变
                torchvision.transforms.Normalize((0.1307,), (0.3081,)),
            ]
        ),
    ),
    batch_size=batch_size,  # 每次训练加载多少张图片
    shuffle=True,
)

test_loader = torch.utils.data.DataLoader(
    torchvision.datasets.MNIST(
        "data/mnist_data/",
        train=False,
        download=True,
        transform=torchvision.transforms.Compose(
            [
                torchvision.transforms.ToTensor(),
                torchvision.transforms.Normalize((0.1307,), (0.3081,)),
            ]
        ),
    ),
    batch_size=batch_size,
    shuffle=False,
)

# 显示图片
x, y = next(iter(test_loader))
# 如果不添加Normalize, 会均匀地分布在0-1之间, 而现在就变换成了这样, 是的数据在0附近
# torch.Size([512, 1, 28, 28]) torch.Size([512]) tensor(-0.4242) tensor(2.8215)
print(x.shape, y.shape, x.min(), x.max())
# plot_image(x, y, "image sample")  # 打印数据

###
# 创建网络
###
class Net(nn.Module):
    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

        ## 新建三层, 每一层wx+b
        self.fc1 = nn.Linear(28 * 28, 256)  # 输入28*28, 输出256
        self.fc2 = nn.Linear(256, 64)
        self.fc3 = nn.Linear(64, 10)  # 输出层

    ## 定义一个计算过程, 输入包含x
    def forward(self, x):
        # x: [b, 1, 28, 28]
        # h1 = relu(wx+b)
        x = F.relu(self.fc1(x))
        # h2 = relu(h1w2+b2)
        x = F.relu(self.fc2(x))
        # h3 = h2w3+b3  暂时不加激活函数
        x = self.fc3(x)
        return x

net = Net()

# net.parameters() 当中包含的就是 [w1,b1,w2,b2,w3,b3]
# Momentum: SGD 在 ravines 的情况下容易被困住， ravines 就是曲面的一个方向比另一个方向更陡，
# 这时 SGD 会发生震荡而迟迟不能接近极小值
optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9)

train_loss = []
## 测试/训练
# 训练三遍
for epoch in range(3):
    # 对内部层数迭代一次, 每一次迭代拿到一个batch512个文件
    for batch_inx, (x, y) in enumerate(train_loader):
        # print(x.shape, y.shape)  # torch.Size([512, 1, 28, 28]) torch.Size([512])
        # 将向量flattern平铺拉伸
        x = x.view(x.size(0), 28 * 28)  # => [b, 10]
        out = net(x)
        # 将y转换为Onehot
        y_onehot = one_hot(y)
        # print(y_onehot.shape) # torch.Size([512, 10])
        loss = F.mse_loss(out, y_onehot)  # 计算mse

        # 清零梯度, grad = 0, 否则的话grad就会加载上一次grad的总和上
        optimizer.zero_grad()
        loss.backward()
        # 更新梯度 w' = w - lr*grad
        optimizer.step()

        train_loss.append(loss.item())
        if batch_inx % 10 == 0:
            print(epoch, batch_inx, loss.item())
    """ end batch """
""" end epoch """

# 获得了[w1, b1, w2, b2, w3, b3]
# plot_curve(train_loss) # 打印训练的loss值

# 我们观察一下测试
total_correct = 0
for x, y in test_loader:
    x = x.view(x.size(0), 28 * 28)
    out = net(x)
    # out: [512, 10] => pred: [b]
    pred = out.argmax(dim=1)
    correct = pred.eq(y).sum().float()
    total_correct += correct

# 获取所有数据集的数量
total_num = len(test_loader.dataset)
acc = total_correct / total_num
print("================================")
print("acc:{}".format(acc))  # acc:0.6884999871253967
print("================================")

# 我们对之前的sample进行预测
x, y = next(iter(test_loader))
out = net(x.view(x.size(0), 28 * 28))
pred = out.argmax(dim=1)
plot_image(x, pred, "test")


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

from django.core.management.base import BaseCommand

from crashstats.cron import JOBS, DEFAULT_FREQUENCY
from crashstats.cron.models import Job

class Command(BaseCommand):
    help = "List available cron jobs."

    def handle(self, **options):
        for job_spec in JOBS:
            self.stdout.write(job_spec["cmd"])

            cmdline = [job_spec["cmd"]]
            if job_spec.get("cmd_args"):
                cmdline.extend(job_spec["cmd_args"])
            if job_spec.get("backfill"):
                cmdline.append("--run-time=RUNTIME")
            if job_spec.get("last_success"):
                cmdline.append("--last-success=LASTSUCCESS")
            cmdline = " ".join(cmdline)
            self.stdout.write("    cmdline:      %s" % cmdline)

            schedule = []
            schedule.append("every " + job_spec.get("frequency", DEFAULT_FREQUENCY))
            if job_spec.get("time"):
                schedule.append(job_spec["time"] + " UTC")
            schedule = " @ ".join(schedule)
            self.stdout.write("    schedule:     %s" % schedule)

            try:
                job = Job.objects.get(app_name=job_spec["cmd"])
                self.stdout.write("    last run:     %s" % job.last_run)
                self.stdout.write("    last_success: %s" % job.last_success)
                self.stdout.write("    next run:     %s" % job.next_run)
                if job.last_error:
                    self.stdout.write("    " + job.last_error)
            except Job.DoesNotExist:
                self.stdout.write("    Never run.")


from django.shortcuts import get_object_or_404
from rest_framework import status
from rest_framework.permissions import IsAdminUser, IsAuthenticated
from rest_framework.response import Response
from rest_framework.views import APIView

from litner.models import LitnerModel, LitnerKarNameModel, LitnerKarNameDBModel
from litner.serializer import LitnerSerializer, LitnerDetailSerializer, LitnerTakeExamSerializer

class LitnerListView(APIView):
    permission_classes = [IsAuthenticated]

    def get(self, request, pk=None):
        if pk is None:
            try:
                leitners = LitnerModel.objects.all()
                serializer = LitnerSerializer(leitners, many=True)
                return Response({'data': serializer.data}, status.HTTP_200_OK)
            except Exception as ins:
                return Response({'message': str(ins)},status.HTTP_404_NOT_FOUND)
        else:
            try:
                leitners = LitnerModel.objects.get(pk=pk)
                serializer = LitnerDetailSerializer(leitners)
                serializer = LitnerDetailSerializer(leitners)
                return Response({'data': serializer.data}, status.HTTP_200_OK)
            except Exception as ins:
                return Response({'message': 'Leitner notFound'}, status.HTTP_404_NOT_FOUND)

    def post(self, request, pk):

        exam = get_object_or_404(LitnerModel, pk=pk)
        try:
            exam.is_open = True
            karname = LitnerKarNameModel.objects.get(user=request.user, exam_id=exam)
            answers = LitnerKarNameDBModel.objects.filter(karname=karname)
            is_corrects = []
            is_false = []
            is_null = []
            print(answers)
            for answer in answers:
                if not answer.is_correct is None:
                    if answer.is_correct == True:
                        is_corrects.append(
                            {
                                'question_id': answer.question.id,
                                # 'question_text': answer.question.question_text,
                                # 'answer_text': answer.answerss
                            })
                    else:
                        if answer.is_correct == False:
                            is_false.append({
                                'question_id': answer.question.id,
                                # 'question_text': answer.question.question_text,
                                # 'answer_text': answer.answerss
                            })
                else:
                    is_null.append({
                        'question_id': answer.question.id,
                        # 'question_text': answer.question.question_text,
                        # 'answer_text': answer.answerss
                    })
            result = {
                'True answers': is_corrects,
                'False answers': is_false,
                'None answers': is_null
            }
            exam.save()
            return Response(result)

        except:
            # just questions that have answer
            # [
            #     {
            #         "question": 1,
            #         "is_correct": true or false
            #     }
            # ]
            data = {'user': request.user.id, "exam_id": pk}
            if request.data:
                data['karname'] = request.data
            else:
                data['karname'] = []
            serializer = LitnerTakeExamSerializer(data=data, context={'request': request, 'exam': pk})
            if serializer.is_valid():
                serializer.save()
                return Response(serializer.data, status.HTTP_200_OK)
            else:
                return Response(serializer.errors, status.HTTP_404_NOT_FOUND)

    def put(self, request, pk):
        # request.data example: you should send new questions and their choices
        # [
        #     {
        #         "question":1,    #our questuin
        #         "choice":2      #client's answer
        #     },
        #     ...
        # ]
        data = {'user': request.user.id, "exam_id": pk}
        if request.data:
            data['karname'] = request.data
        else:
            data['karname'] = []
        exam = get_object_or_404(LitnerModel, pk=pk)
        karname = get_object_or_404(LitnerKarNameModel, user=request.user, exam_id=exam)
        serializer = LitnerTakeExamSerializer(instance=karname, data=data, context={'request': request, 'exam': pk},
                                              partial=True)
        if serializer.is_valid():
            try:
                serializer.save()
            except Exception:
                return Response(serializer.errors, status.HTTP_404_NOT_FOUND)
        else:
            return Response(serializer.errors, status.HTTP_404_NOT_FOUND)
        return Response(serializer.data)

class LitnerView(APIView):
    permission_classes = [IsAdminUser]

    def post(self, request):
        data = LitnerDetailSerializer(data=request.data)
        data.is_valid(raise_exception=True)
        data.save()
        return Response(data.data)

    def delete(self, request, pk):
        try:
            instance = LitnerModel.objects.get(pk=pk)
            instance.delete()
            return Response({"massage": "leitner remove successfully"}, status.HTTP_204_NO_CONTENT)
        except Exception:
            return Response(
                {"massage": "leitner not found"}, status.HTTP_404_NOT_FOUND)

    def put(self, request):
        pass

# a = {
#     'exam': 1,
#     'user': 20,
#     'answer': [
#         {
#             "id_question": 5,
#             "choice": None  # null
#         }
#     ],
#     # 'true_answer': [
#     #     {
#     #         "question_text": "سوال اول آزمون ریاضی که بود؟",
#     #         "answers_text": "رضا"
#     #     },
#     #     {
#     #         "question_text": "سوال 2 آزمون مامایی که بود؟",
#     #         "answers_text": "احمد"
#     #     },
#     #     {
#     #         "question_text": "سوال اول آزمون عربی که بود؟",
#     #         "answers_text": "علی"
#     #     },
#     # ],
#     # 'no_answer': [
#     #     {
#     #         "question_text": "امیر نمره ریاضیش چند بود؟",
#     #         "answers_text": "10"
#     #     },
#     #     {
#     #         "question_text": "سوال 2 آزمون مامایی که بود؟",
#     #         "answers_text": "احمد"
#     #     },
#     #     {
#     #         "question_text": "سوال اول آزمون عربی که بود؟",
#     #         "answers_text": "علی"
#     #     },
#     # ],
#     # 'false_answer': [
#     #     {
#     #         "question_text": "سوال اول آزمون ریاضی که بود؟",
#     #         "answers_text": "رضا"
#     #     },
#     #     {
#     #         "question_text": "سوال 2 آزمون مامایی که بود؟",
#     #         "answers_text": "احمد"
#     #     },
#     #     {
#     #         "question_text": "سوال اول آزمون عربی که بود؟",
#     #         "answers_text": "علی"
#     #     },
#     # ],
# }


from card import Card
from suit import Suit
from random import shuffle
from player import Player
from init_ai import init_AI
from second_ai import second_AI
from team import Team
from comp import Comp
from termcolor import colored

import random

game_run = True
match_run = True
# human = Player(1)
human = second_AI(1,0, 1)
teammate = second_AI(1, 1, 0)
opponent_1 = second_AI(2, 2, 3)
opponent_2 = second_AI(2, 3, 2)
team1 = Team(human, teammate)
team2 = Team(opponent_1, opponent_2)

list_of_players_1 = [human, opponent_1, teammate, opponent_2]
list_of_players_2 = [opponent_1, teammate, opponent_2, human]
list_of_players_3 = [teammate, opponent_2, human, opponent_1]
list_of_players_4 = [opponent_2, human, opponent_1, teammate, ]
# the order if the player at each position wins, this is with the player and teammate being at
# posistion 0 and 1
iteration_of_players = {
    0: list_of_players_1,
    1: list_of_players_3,
    2: list_of_players_2,
    3: list_of_players_4

}

# cards on table is a dic that keeps the cards in a specific order,
# 0 is the player, 1 is the teammate, 2 and 3 or the opponents cards.
cards_on_table = {
    0: Card(Suit.NONE, -1),
    1: Card(Suit.NONE, -1),
    2: Card(Suit.NONE, -1),
    3: Card(Suit.NONE, -1)
}
inital_suit = Suit.HEART

def __init_deck():
    deck = list()
    for i in range(2, 15):
        deck.append(Card(Suit.HEART, i))
        deck.append(Card(Suit.DIAMOND, i))
        deck.append(Card(Suit.SPADE, i))
        deck.append(Card(Suit.CLUB, i))

    return deck

def shuffle_cards(current_deck):
    for i in range(random.randint(1, 10)):
        shuffle(current_deck)
        return current_deck

def deal_5(deck):
    five_cards = list()
    for i in range(5):
        five_cards.append(deck.pop())
    return five_cards

def deal_4(deck):
    four_cards = list()
    for i in range(4):
        four_cards.append(deck.pop())
    return four_cards

def shift_list(winning_pos):
    if winning_pos == 0:
        return 0
    elif winning_pos == 1:
        return -1
    elif winning_pos == 2:
        return -2
    return 1

deck = __init_deck()
current_cards = shuffle_cards(deck)

# pick the first hawkhem(the first king)
index_of_king = random.randint(0, 3)
index_of_starter = index_of_king
first = True
comp = Comp()
while match_run:

    for player in iteration_of_players[index_of_king]:
        player.add_cards(deal_5(current_cards))
        if first:
            comp.set_hokm(player.select_hokm())
            print('Hokm is {0}'.format(comp.hokm))
            first = False

    # break
    for player in iteration_of_players[index_of_king]:
        player.hokm = comp.hokm

    for i in range(2):
        for player in iteration_of_players[index_of_king]:
            player.add_cards(deal_4(current_cards))

    # print(teammate.play_card(None, None, -1))
    # for player in iteration_of_players[index_of_king]:
    #      print(player.player_number, '   ', player.hand)
    # break
    while game_run:
        for player in iteration_of_players[index_of_starter]:
            comp.set_and_comp(player.player_number,
                              player.play_card(comp.initial_suit, comp.switcher_2, comp.high_card_index))

        index_of_starter = comp.high_card_index

        comp.print_cards()
        # allow AI to know which cards have been played.
        for player in iteration_of_players[index_of_starter]:
            player.update_cards_played(comp.switcher_2, comp.initial_suit)

        if comp.high_card_index < 2:

            team1.hands_won = team1.hands_won + 1
            print(colored('Team1 won the hand', 'cyan'))
            string1 = '\nteam 1 has {0} points team2 has {1} points\n'.format(team1.hands_won, team2.hands_won)
            print(colored(string1, 'red'))
            if team1.hands_won == 7:
                team1.games_won = team1.games_won + 1

                string1 = 'End of Game team 1 won {0} games and team 2 won {1}'.format(team1.games_won,
                                                                                team2.games_won)
                print(colored(string1, 'magenta', attrs=['bold', 'dark']))
                team1.hands_won = 0
                team2.hands_won = 0
                if index_of_king > 1:
                    index_of_king = (index_of_king - 1) % 2
                comp.new_game_reset()
                break
        else:

            team2.hands_won = team2.hands_won + 1
            print(colored('Team2 won the hand', 'yellow'))
            string1 = '\nTeam1 has {0} points Team2 has {1} points\n'.format(team1.hands_won, team2.hands_won)
            print(colored(string1, 'red'))
            if team2.hands_won == 7:
                team2.games_won = team2.games_won + 1

                string1 = 'End of Game team 1 won {0} games and team 2 won {1}'.format(team1.games_won,
                                                                                team2.games_won)
                print(colored(string1, 'magenta', attrs=['bold', 'dark']))
                team1.hands_won = 0
                team2.hands_won = 0
                if index_of_king < 1:
                    index_of_king = index_of_king + 2
                comp.new_game_reset()
                break
        comp.after_hand_reset()

    index_of_starter = index_of_king
    first = True
    current_cards = shuffle_cards(__init_deck())
    for player in iteration_of_players[index_of_starter]:
        player.reset_hand()

    if team1.games_won == 7 or team2.games_won == 7:
        string1 = 'End of Match team 1 won {0} games and team 2 won {1}'.format(team1.games_won, team2.games_won)
        print(colored(string1, 'magenta', attrs=['bold', 'dark']))
        break


"""
All the functions to properly build the object hierarchies.
"""
import collections.abc
import enum
import warnings
from typing import Any, Iterable, Iterator, Mapping, MutableMapping, Optional, Union, cast

from kopf._cogs.helpers import thirdparty
from kopf._cogs.structs import bodies, dicts
from kopf._core.actions import execution
from kopf._core.intents import causes

K8sObject = Union[MutableMapping[Any, Any], thirdparty.PykubeObject, thirdparty.KubernetesModel]
K8sObjects = Union[K8sObject, Iterable[K8sObject]]

class _UNSET(enum.Enum):
    token = enum.auto()

def append_owner_reference(
        objs: K8sObjects,
        owner: Optional[bodies.Body] = None,
        *,
        controller: Optional[bool] = True,
        block_owner_deletion: Optional[bool] = True,
) -> None:
    """
    Append an owner reference to the resource(s), if it is not yet there.

    Note: the owned objects are usually not the one being processed,
    so the whole body can be modified, no patches are needed.
    """
    real_owner = _guess_owner(owner)
    owner_ref = bodies.build_owner_reference(
        real_owner, controller=controller, block_owner_deletion=block_owner_deletion
    )
    for obj in cast(Iterator[K8sObject], dicts.walk(objs)):
        # Pykube is yielded as a usual dict, no need to specially treat it.
        if isinstance(obj, collections.abc.MutableMapping):
            refs = obj.setdefault('metadata', {}).setdefault('ownerReferences', [])
            if not any(ref.get('uid') == owner_ref['uid'] for ref in refs):
                refs.append(owner_ref)
        elif isinstance(obj, thirdparty.KubernetesModel):
            if obj.metadata is None:
                obj.metadata = thirdparty.V1ObjectMeta()
            if obj.metadata.owner_references is None:
                obj.metadata.owner_references = []
            refs = obj.metadata.owner_references
            if not any(ref.uid == owner_ref['uid'] for ref in refs):
                refs.append(thirdparty.V1OwnerReference(
                    api_version=owner_ref['apiVersion'],
                    kind=owner_ref['kind'],
                    name=owner_ref['name'],
                    uid=owner_ref['uid'],
                    controller=owner_ref['controller'],
                    block_owner_deletion=owner_ref['blockOwnerDeletion'],
                ))
        else:
            raise TypeError(f"K8s object class is not supported: {type(obj)}")

def remove_owner_reference(
        objs: K8sObjects,
        owner: Optional[bodies.Body] = None,
) -> None:
    """
    Remove an owner reference to the resource(s), if it is there.

    Note: the owned objects are usually not the one being processed,
    so the whole body can be modified, no patches are needed.
    """
    real_owner = _guess_owner(owner)
    owner_ref = bodies.build_owner_reference(real_owner)
    for obj in cast(Iterator[K8sObject], dicts.walk(objs)):
        # Pykube is yielded as a usual dict, no need to specially treat it.
        if isinstance(obj, collections.abc.MutableMapping):
            refs = obj.setdefault('metadata', {}).setdefault('ownerReferences', [])
            if any(ref.get('uid') == owner_ref['uid'] for ref in refs):
                refs[:] = [ref for ref in refs if ref.get('uid') != owner_ref['uid']]
        elif isinstance(obj, thirdparty.KubernetesModel):
            if obj.metadata is None:
                obj.metadata = thirdparty.V1ObjectMeta()
            if obj.metadata.owner_references is None:
                obj.metadata.owner_references = []
            refs = obj.metadata.owner_references
            if any(ref.uid == owner_ref['uid'] for ref in refs):
                refs[:] = [ref for ref in refs if ref.uid != owner_ref['uid']]
        else:
            raise TypeError(f"K8s object class is not supported: {type(obj)}")

def label(
        objs: K8sObjects,
        labels: Union[Mapping[str, Union[None, str]], _UNSET] = _UNSET.token,
        *,
        forced: bool = False,
        nested: Optional[Union[str, Iterable[dicts.FieldSpec]]] = None,
        force: Optional[bool] = None,  # deprecated
) -> None:
    """
    Apply the labels to the object(s).
    """
    nested = [nested] if isinstance(nested, str) else nested
    if force is not None:
        warnings.warn("force= is deprecated in kopf.label(); use forced=...", DeprecationWarning)
        forced = force

    # Try to use the current object being handled if possible.
    if isinstance(labels, _UNSET):
        real_owner = _guess_owner(None)
        labels = real_owner.get('metadata', {}).get('labels', {})
    if isinstance(labels, _UNSET):
        raise RuntimeError("Impossible error: labels are not resolved.")  # for type-checking

    # Set labels based on the explicitly specified or guessed ones.
    for obj in cast(Iterator[K8sObject], dicts.walk(objs, nested=nested)):
        # Pykube is yielded as a usual dict, no need to specially treat it.
        if isinstance(obj, collections.abc.MutableMapping):
            obj_labels = obj.setdefault('metadata', {}).setdefault('labels', {})
        elif isinstance(obj, thirdparty.KubernetesModel):
            if obj.metadata is None:
                obj.metadata = thirdparty.V1ObjectMeta()
            if obj.metadata.labels is None:
                obj.metadata.labels = {}
            obj_labels = obj.metadata.labels
        else:
            raise TypeError(f"K8s object class is not supported: {type(obj)}")

        for key, val in labels.items():
            if forced:
                obj_labels[key] = val
            else:
                obj_labels.setdefault(key, val)

def harmonize_naming(
        objs: K8sObjects,
        name: Union[None, str, _UNSET] = _UNSET.token,
        *,
        forced: bool = False,
        strict: bool = False,
) -> None:
    """
    Adjust the names or prefixes of the objects.

    In strict mode, the provided name is used as is. It can be helpful
    if the object is referred by that name in other objects.

    In non-strict mode (the default), the object uses the provided name
    as a prefix, while the suffix is added by Kubernetes remotely.
    The actual name should be taken from Kubernetes response
    (this is the recommended scenario).

    If the objects already have their own names, auto-naming is not applied,
    and the existing names are used as is.
    """

    # Try to use the current object being handled if possible.
    if isinstance(name, _UNSET):
        real_owner = _guess_owner(None)
        name = real_owner.get('metadata', {}).get('name', None)
    if isinstance(name, _UNSET):
        raise RuntimeError("Impossible error: the name is not resolved.")  # for type-checking
    if name is None:
        raise LookupError("Name must be set explicitly: couldn't find it automatically.")

    # Set name/prefix based on the explicitly specified or guessed name.
    for obj in cast(Iterator[K8sObject], dicts.walk(objs)):
        # Pykube is yielded as a usual dict, no need to specially treat it.
        if isinstance(obj, collections.abc.MutableMapping):
            noname = 'metadata' not in obj or not set(obj['metadata']) & {'name', 'generateName'}
            if forced or noname:
                if strict:
                    obj.setdefault('metadata', {})['name'] = name
                    if 'generateName' in obj['metadata']:
                        del obj['metadata']['generateName']
                else:
                    obj.setdefault('metadata', {})['generateName'] = f'{name}-'
                    if 'name' in obj['metadata']:
                        del obj['metadata']['name']
        elif isinstance(obj, thirdparty.KubernetesModel):
            if obj.metadata is None:
                obj.metadata = thirdparty.V1ObjectMeta()
            noname = obj.metadata.name is None and obj.metadata.generate_name is None
            if forced or noname:
                if strict:
                    obj.metadata.name = name
                    if obj.metadata.generate_name is not None:
                        obj.metadata.generate_name = None
                else:
                    obj.metadata.generate_name = f'{name}-'
                    if obj.metadata.name is not None:
                        obj.metadata.name = None
        else:
            raise TypeError(f"K8s object class is not supported: {type(obj)}")

def adjust_namespace(
        objs: K8sObjects,
        namespace: Union[None, str, _UNSET] = _UNSET.token,
        *,
        forced: bool = False,
) -> None:
    """
    Adjust the namespace of the objects.

    If the objects already have the namespace set, it will be preserved.

    It is a common practice to keep the children objects in the same
    namespace as their owner, unless explicitly overridden at time of creation.
    """

    # Try to use the current object being handled if possible.
    if isinstance(namespace, _UNSET):
        real_owner = _guess_owner(None)
        namespace = real_owner.get('metadata', {}).get('namespace', None)
    if isinstance(namespace, _UNSET):
        raise RuntimeError("Impossible error: the namespace is not resolved.")  # for type-checking

    # Set namespace based on the explicitly specified or guessed namespace.
    for obj in cast(Iterator[K8sObject], dicts.walk(objs)):
        # Pykube is yielded as a usual dict, no need to specially treat it.
        if isinstance(obj, collections.abc.MutableMapping):
            if forced or obj.get('metadata', {}).get('namespace') is None:
                obj.setdefault('metadata', {})['namespace'] = namespace
        elif isinstance(obj, thirdparty.KubernetesModel):
            if obj.metadata is None:
                obj.metadata = thirdparty.V1ObjectMeta()
            if forced or obj.metadata.namespace is None:
                obj.metadata.namespace = namespace
        else:
            raise TypeError(f"K8s object class is not supported: {type(obj)}")

def adopt(
        objs: K8sObjects,
        owner: Optional[bodies.Body] = None,
        *,
        forced: bool = False,
        strict: bool = False,
        nested: Optional[Union[str, Iterable[dicts.FieldSpec]]] = None,
) -> None:
    """
    The children should be in the same namespace, named after their parent, and owned by it.
    """
    real_owner = _guess_owner(owner)
    real_owner_name = real_owner.get('metadata', {}).get('name', None)
    real_owner_namespace = real_owner.get('metadata', {}).get('namespace', None)
    real_owner_labels = real_owner.get('metadata', {}).get('labels', {})
    append_owner_reference(objs, owner=real_owner)
    harmonize_naming(objs, forced=forced, strict=strict, name=real_owner_name)
    adjust_namespace(objs, forced=forced, namespace=real_owner_namespace)
    label(objs, forced=forced, nested=nested, labels=real_owner_labels)

def _guess_owner(
        owner: Optional[bodies.Body],
) -> bodies.Body:
    if owner is not None:
        return owner

    try:
        cause = execution.cause_var.get()
    except LookupError:
        pass
    else:
        if cause is not None and isinstance(cause, causes.ResourceCause):
            return cause.body

    raise LookupError("Owner must be set explicitly, since running outside of a handler.")


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.1
#   kernelspec:
#     display_name: Python 3.8.10 ('ml4t')
#     language: python
#     name: python3
# ---

# + [markdown] id="sNw45BR0m62X" pycharm={"name": "#%% md\n"}
# # Importing dependencies
# We used the following libraries in the first task:
# - `pandas` to manipulate the data
# - `scikit-learn` for imputing and scaling the data
# - `seaborn` and `matplotlib` for visualization.

# + id="4CRS48qom62Z" pycharm={"name": "#%%\n"}
import pandas as pd
from sklearn import preprocessing
import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns

# + [markdown] id="XyY6wwS-m62a" pycharm={"name": "#%% md\n"}
# # Data exploration
# Test
#
# Let's load the training dataset from the corresponding .csv file.
# Since we know that the columns represent the mean/standard deviation of the positions and angles of the 60 points, respectively, followed by the label name and code, let us rename the columns accordingly to allow for easier reading.

# + id="KNoDyk_fm62a" pycharm={"name": "#%%\n"}
def load_data():
  training_data = pd.read_csv('train-final.csv', header=None)

  name_mappings = {
      # Feature columns
      **{i:f'positions_mean_{i}' for i in range(60)},
      **{i:f'positions_std_{i}' for i in range(60, 120)},
      **{i:f'angles_mean_{i}' for i in range(120, 180)},
      **{i:f'angles_std_{i}' for i in range(180, 240)},
      # Label columns
      **{240: 'label_name', 241: 'label_code'},
  }

  training_data.rename(name_mappings, axis=1, inplace=True)
  training_feature_columns = training_data.columns[:-2]

  training_features = training_data[training_feature_columns]
  training_labels = training_data.label_name

  return training_features, training_labels

training_features, training_labels = load_data()

# + [markdown] id="_aYlx2ckm62b" pycharm={"name": "#%% md\n"}
# Let's show some of the data

# + colab={"base_uri": "https://localhost:8080/", "height": 300} id="paURPIp3m62b" outputId="4b22de09-f76e-4f25-8e2d-9d356e7ca70d" pycharm={"name": "#%%\n"}
training_features.head()

# + [markdown] id="DYPTNtFmm62c" pycharm={"name": "#%% md\n"}
# How many different labels do we have in the training dataset?

# + colab={"base_uri": "https://localhost:8080/"} id="l8hDw1bgm62c" outputId="7f333633-1d1c-4483-8fbc-0c1f9acf96a4" pycharm={"name": "#%%\n"}
number_of_classes = training_labels.nunique()
number_of_classes

# + [markdown] id="2NwxuXvqm62c" pycharm={"name": "#%% md\n"}
# Now let's take a look at how many occurrences we have of each label.

# + colab={"base_uri": "https://localhost:8080/", "height": 322} id="_O-CkrFfm62d" outputId="ad9afcd5-4836-4ad1-fa9b-7d7a5aa5954e" pycharm={"name": "#%%\n"}
training_labels.value_counts().plot(kind='bar', figsize=(10, 4))

# + [markdown] id="YlISrYJpm62d" pycharm={"name": "#%% md\n"}
# We can see that `child` is the most common label in the training dataset and that `go` is the least common label.

# + [markdown] id="AhOTmkwRm62d" pycharm={"name": "#%% md\n"}
# Now let's look for columns that have missing values. The missing values are in the following columns (along with the missing value count):
#

# + colab={"base_uri": "https://localhost:8080/"} id="rTfhsEE8m62f" outputId="62eb3157-0b72-4f9c-a1bf-91cf110ced34" pycharm={"name": "#%%\n"}
# Look for columns that have missing values
columns_null_sum = training_features.isnull().sum()
columns_with_nulls = columns_null_sum[columns_null_sum > 0]

print(
    "Total amount of missing values in the dataframe:",
    training_features.isnull().sum().sum()
)
print(
    "Missing values in the following column indexes (and missing value count):"
)
print(columns_with_nulls)

# + [markdown] id="yprFNiVkm62j" pycharm={"name": "#%% md\n"}
# Some classifiers are more sensitive to the range, mean & outliers of the features, such as linear regression models, for example.
# In order to be able to train a wide range of classifiers and compare them, we will need to preprocess the data for scaling and outlier treatment.

# + [markdown] id="t9yB47dLm62k" pycharm={"name": "#%% md\n"}
# Let's see if the dataset also contains outliers. There are quite a few way to detect outliers (Source:
# [Outlier detection methods in Scikit-Learn](https://scikit-learn.org/stable/modules/outlier_detection.html)):
# - Isolation forest
# - Local outlier factor
# - One-class support vector machine (SVM)
# - Elliptic envelope
#
# We start by doing a boxplot for all features to get a visual indication of the outlier situation.

# + colab={"base_uri": "https://localhost:8080/", "height": 337} id="r77J-ZiOm62k" outputId="0bddf0cf-5a7c-4708-823f-a73359d67ac3" pycharm={"name": "#%%\n"}
training_features.boxplot(figsize=(18,7))
plt.xticks([1], [''])

# + [markdown] id="m4tCX_XRnYRS" pycharm={"name": "#%% md\n"}
# Based on the boxplot, there appears to be many columns with outliers. Many classifiers, e.g. linear classifiers like Logistic Regression will not handle outliers well, so we need to find a way to handle also outliers.

# + [markdown] id="J1sVekRXoT4Q" pycharm={"name": "#%% md\n"}
# # Methods

# + [markdown] id="Kq-ne6mOolOy" pycharm={"name": "#%% md\n"}
# ## Outliers
#
# As we saw in the boxplot above, there are many columns with outliers. And while there are many methods to detect outliers,let's begin with just identifying the values that are farthest from the mean.
#
# A simple approach is to identify the values that lie outside of 3$\sigma$ (that is, three times the standard deviation) as outliers, and drop the rows that have at least one outlier. Let's give it a try.

# + id="sOaxwVV_m62k" pycharm={"name": "#%%\n"}
#training_features_outliers_marked = training_features[abs(training_features) <= 3]
from scipy import stats

training_features_outliers_marked = training_features[
    np.abs(stats.zscore(training_features.fillna(training_features.mean()))) < 3
]

# + colab={"base_uri": "https://localhost:8080/", "height": 300} id="gGFZoxbwpqaE" outputId="e4bbb462-72f6-46b3-e83a-1d5e98fe2561" pycharm={"name": "#%%\n"}
training_features_outliers_marked.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 281} id="j2OIbvJTm62l" outputId="9170fed3-adf5-4aab-e724-f6e342249732" pycharm={"name": "#%%\n"}
training_features_outliers_removed = training_features_outliers_marked.dropna()
training_features_outliers_removed.boxplot(figsize=(18,7))
plt.xticks([1], [''])
print("Number of rows left", training_features_outliers_removed.shape[0])

# + [markdown] id="r8fsvlJZm62l" pycharm={"name": "#%% md\n"}
# The boxplot now looks better, except for the second part (columns 60 to 120), which is `positions_std_i`.
# We can also see that if we remove all the rows with at least one detected outlier, we are left with less than half of the original data! This is due to the large number of features.
#
# We need another method for this dataset, let's instead cap the outliers to 3 sigma.

# + id="-artO4WUm62l" pycharm={"name": "#%%\n"}
def pipeline_outliers(df, std_cap=3):
  df = df.copy()

  for column in df.columns:

    mean = df[column].mean(skipna = True)
    std = df[column].std(skipna = True)

    df[column] = np.clip(df[column], -(mean + std_cap*std), mean + std_cap*std)

  return df

# + colab={"base_uri": "https://localhost:8080/", "height": 337} id="oDGrLk2Jm62l" outputId="3145fc38-ed0e-4bd3-f389-77c2a98ec641" pycharm={"name": "#%%\n"}
df = pipeline_outliers(training_features)
df.boxplot(figsize=(18,7))
plt.xticks([1], [''])

# + [markdown] id="QHeOA5iqm62g" pycharm={"name": "#%% md\n"}
# ## Missing data

# + [markdown] id="T-HJFZs8m62g" pycharm={"name": "#%% md\n"}
# As we saw above, there are 6 columns that have missing values (3 or 4 missing values each). Many classifiers do not handle missing values directly, such as Logistic Regression and SVM, for example. As such we need to find a way to manage the missing values.

# + [markdown] id="ENEICm2Om62g" pycharm={"name": "#%% md\n"}
# There are many different ways of handling missing values and we will explore a few of them here. To get started, let's examine the features/columns that contain missing data. The two visualizations chosen for each of the features/columns are:
#
# - *Histogram* - This will give a good indication of the distribution, for example if it appears to be normal.
# - *Boxplot* - We get some additional information from the boxplot showing the median, quartiles as well as outliers.
#
# Let's plot the distributions for the columns with missing data:

# + colab={"base_uri": "https://localhost:8080/", "height": 332} id="3-dLLDaPm62g" outputId="66d2de3f-72c3-4e08-80cc-832bca38404b" pycharm={"name": "#%%\n"}
def plot_distributions_for_columns(dataframe, columns):
    # Plot distributions for each of the columns that have missing values
    figure, axes = plt.subplots(2, len(columns), figsize=(12, 6))

    for index, column in enumerate(columns):
        # plot a histogram of the column for the first row
        dataframe[column].plot(
            kind='hist', ax=axes[0, index], title=column, bins=15
        )
        # Do a box plot as well
        sns.boxplot(y=dataframe[column], ax=axes[1, index]).set_title(column)

    plt.tight_layout()

plot_distributions_for_columns(training_features, columns_with_nulls.index)

# + [markdown] id="TBDoZnucm62h" pycharm={"name": "#%% md\n"}
# The distributions are quite different, which mean we may need different imputation techniques for each column.
# For example, the columns `positions_mean_9` and `positions_mean_15` are clearly not normal distributions and replacing missing values with the mean would likely not be ideal.
# For instance in the case of `positions_mean_9` the mean is close to 0 where few other samples are, moreover this feature may even need to be split into two separate features as it appears to be the combination of two gaussian distributions.
#
# Also, we do not know if the missing values themselves have a significance, i.e. we might want to create a separate column to indicate that a missing value is present or not. There are relatively few rows that have missing values, though, that may limit the usefulness of this technique and usefulness will also depend on the classifier used in the end.
#
# For now, we will implement support for the following imputation strategies:
# - Replacing with **mean**
# - **Drop the rows** containing at least one missing value
# - **K-Nearest Neighbour (KNN)** imputation, i.e. use the mean value of the K nearest neighbours

# + id="AjTqPq0Qm62h" pycharm={"name": "#%%\n"}
from sklearn.impute import KNNImputer, SimpleImputer

def impute(dataframe, imputer_class, **kwargs):
    imputer = imputer_class(**kwargs)
    dataframe = dataframe.copy()
    dataframe[dataframe.columns] = imputer.fit_transform(dataframe.values)
    return dataframe

# KNN imputation
def impute_knn(dataframe):
    return impute(dataframe, KNNImputer, n_neighbors=2, weights='uniform')

# Drop rows
def impute_drop_rows(dataframe):
    return dataframe.dropna()

# Mean imputation
def impute_mean(dataframe):
    return impute(
        dataframe, SimpleImputer, missing_values=np.nan, strategy='mean'
    )

# + [markdown] id="o09ETbgLm62h" pycharm={"name": "#%% md\n"}
# Let's compare the mean and KNN imputation methods.

# + id="C8MiBZ8Jm62h" pycharm={"name": "#%%\n"}
columns_null_sum = training_features.isnull().sum()
columns_with_nulls = columns_null_sum[columns_null_sum > 0]

# + id="PUSYg7HNm62h" pycharm={"name": "#%%\n"}
#training_features = training_data[training_feature_columns]
all_na_values = training_features.isna()

# + [markdown] id="FTGyuLvum62h" pycharm={"name": "#%% md\n"}
# First let's execute both imputations individually.
# Starting with the mean imputation, the imputed values are the following:

# + colab={"base_uri": "https://localhost:8080/"} id="v14LUk-Km62i" outputId="3a6b15cc-155c-4759-bd5f-40c0ce506048" pycharm={"name": "#%%\n"}
training_data_mean_imputed = impute_mean(training_features)
training_data_mean_imputed.values[all_na_values]

# + [markdown] id="lGKhqpOsm62i" pycharm={"name": "#%% md\n"}
# While the KNN-imputed values look like this:

# + colab={"base_uri": "https://localhost:8080/"} id="FscTkmDLm62i" outputId="c888a8ae-11c4-450b-d89f-b553060a65ea" pycharm={"name": "#%%\n"}
training_data_knn_imputed = impute_knn(training_features)
training_data_knn_imputed.values[all_na_values]

# + [markdown] id="EtCVnW3um62i" pycharm={"name": "#%% md\n"}
# ### Comparison of mean and KNN
# Now it is time to compare the results and visualize the differences on a histogram

# + colab={"base_uri": "https://localhost:8080/", "height": 265} id="Ne7NYuHXm62i" outputId="53fa39df-ee95-4aac-a6d6-c46eef07f36a" pycharm={"name": "#%%\n"}
differences = (
    training_data_mean_imputed.values[all_na_values]
    - training_data_knn_imputed.values[all_na_values]
)
plt.hist(differences);

# + [markdown] id="rQTrrd-qm62i" pycharm={"name": "#%% md\n"}
# We can conclude that the differences between the methods are not significant, the mode is close to zero, for most of the missing values the two methods give very similar imputed values - more than half of the values are in the range [-0.1,0.1]
#
# Now let's look at how the difference is distributed for each column using a boxplot and a swarm plot.
#

# + colab={"base_uri": "https://localhost:8080/", "height": 297} id="WGeg5UJDm62j" outputId="7b4edc51-7010-4610-f78f-8f5fcdde7b0b" pycharm={"name": "#%%\n"}
na_rows, na_columns = np.where(all_na_values)
dataframe_differences_columns = pd.DataFrame(
    {'diff': differences, 'column': training_features.columns[na_columns]}
)

fig, axs = plt.subplots(ncols=2)

axis = sns.boxplot(
    x='column', data=dataframe_differences_columns, y='diff', ax=axs[0]
)
axis.set_xticklabels(axis.get_xticklabels(), rotation=90);

axis = sns.swarmplot(
    x='column', data=dataframe_differences_columns, y='diff', ax=axs[1]
)
axis.set_xticklabels(axis.get_xticklabels(), rotation=90);

fig.tight_layout()

# + [markdown] id="BeH5QKZUm62j" pycharm={"name": "#%% md\n"}
# When comparing the imputed values between the 'mean' and 'KNN' approaches, we see that they produce very similar values for columns `positions_mean_7` and `positions_mean_16` but larger differences by varying degrees for the other columns. The largest (absolute) deviance is for `positions_mean_9`.
#
#
# Based on this it is likely that we may need to employ different imputation techniques depending on the column/feature.

# + [markdown] id="7EvO3QTJm62j" pycharm={"name": "#%% md\n"}
# For now, however, let's use KNN imputation and make sure that after the imputation there are no more missing values in our dataset. We will bring these three different methods into Task 2 of the project.
#

# + id="cJCnEdNqok3O" pycharm={"name": "#%%\n"}
# Let's create a pipeline function that uses the KNN imputation
def pipeline_missing_values(df):
  df = df.copy()

  columns_null_sum = df.isnull().sum()
  columns_with_nulls = columns_null_sum[columns_null_sum > 0]

  df[columns_with_nulls.index] = impute_knn(df[columns_with_nulls.index])
  assert df.isnull().sum().sum() == 0

  return df

# + [markdown] id="UIZibV1-m62j" pycharm={"name": "#%% md\n"}
# ## Scaling
# First we will take a look at how the data looks by feature/column. While the dataset contains a lot of features, we can use a boxplot to get an overview understanding of how the different columns compare.

# + [markdown] id="1UU_s9ajm62j" pycharm={"name": "#%% md\n"}
# Let's make a boxplot for every feature to get an overview of how they all relate in terms of range & centre. No need to have labels for the feature names, we just want to show all of them in one plot.

# + colab={"base_uri": "https://localhost:8080/", "height": 337} id="zZ8mc_GWm62k" outputId="2af47191-5b38-4fc8-f431-1002b2a0cc98" pycharm={"name": "#%%\n"}
training_features.boxplot(figsize=(18, 7))
plt.xticks([1], [''])

# + [markdown] id="IabX4HxSm62k" pycharm={"name": "#%% md\n"}
# The columns are clearly not all scaled to the same range and they have different means. As such a scaler that centers the mean and normalizes (scale to the variance) may be suitable. Let's use scikit's `StandardScaler` for this.
#
# The `StandardScaler`normalizes the data so that the mean becomes zero, and the variance one, i.e. the scaled dataset follows a *standard* normal distribution.

# + colab={"base_uri": "https://localhost:8080/", "height": 300} id="aeXO-9R9m62k" outputId="3e3b1dda-14ed-4bb7-f572-190ac361ba85" pycharm={"name": "#%%\n"}
def pipeline_scale(dataframe):
    scaler = preprocessing.StandardScaler()
    scaled_values = scaler.fit_transform(dataframe.values)
    return pd.DataFrame(scaled_values)

training_features_scaled = pipeline_scale(training_features)
training_features_scaled.head()

# + [markdown] id="z3ewuEEFm62k" pycharm={"name": "#%% md\n"}
# Looks good, i.e. the mean is about 0 and the standard deviation is around 1. All the columns have now been scaled. Let's rerun the boxplot.

# + colab={"base_uri": "https://localhost:8080/", "height": 335} id="W1uf0D3QsqPH" outputId="8c9271bc-c2aa-4a2c-e1fd-10d410f6655d" pycharm={"name": "#%%\n"}
training_features_scaled.boxplot(figsize=(18, 7))
plt.xticks([1], [''])

# + [markdown] id="Unie-y19uJpv" pycharm={"name": "#%% md\n"}
# # Pipeline

# + [markdown] id="wTBLe3qVvs2f" pycharm={"name": "#%% md\n"}
# In the Methods section above we have defined the pipeline methods needed for handling outliers, missing data and scaling. Let's put them all together.

# + colab={"base_uri": "https://localhost:8080/", "height": 337} id="1kLsdpktv7pS" outputId="ca6ad9cb-0932-4d61-d04c-84b137dc5174" pycharm={"name": "#%%\n"}
# Load the data
training_features, training_labels = load_data()

# Plot before pipeline
training_features.boxplot(figsize=(18, 7))
plt.xticks([1], [''])

# + [markdown] id="NqTkoxZQv7EW" pycharm={"name": "#%% md\n"}
# Run the pipeline and rerun the boxplot on the resulting dataset

# + colab={"base_uri": "https://localhost:8080/", "height": 281} id="8TDqvcNTzgQ4" outputId="9b00742b-403e-41bd-a19f-badf2f8cffb3" pycharm={"name": "#%%\n"}
df = pipeline_outliers(training_features, std_cap=3)
df = pipeline_missing_values(df)
df = pipeline_scale(df)

df.boxplot(figsize=(18, 7))
plt.xticks([1], [''])


import json
import os

import geopandas as gpd
import numpy
from geomet import wkt

from noise_api.noise_analysis.road_info import RoadInfo

cwd = os.path.dirname(os.path.abspath(__file__))

# road_type_ids from IffStar NoiseModdeling
road_types_iffstar_noise_modelling = {
    "boulevard": 56,  # in boulevard 70km/h
    "street": 53,  # extra boulevard Street 50km/h
    "alley": 54,  # extra boulevard Street <50km/,
    "railroad": 99,  # railroad
}

all_roads = []

def reset_all_roads():
    global all_roads
    all_roads = []

# opens a json from path
def open_geojson(path):
    with open(path) as f:
        return json.load(f)

# extract traffic data from road properties
def get_car_traffic_data(road_properties):
    #  https://d-nb.info/97917323X/34
    # p. 68, assuming a mix of type a) and type b) for car traffic around grasbrook (strong morning peak)
    # most roads in hamburg belong to type a) or b)
    # assuming percentage of daily traffic in peak hour = 11%
    # truck traffic: the percentage of daily traffic in peak hour seems to be around 8%
    # source. https://bast.opus.hbz-nrw.de/opus45-bast/frontdoor/deliver/index/docId/1921/file/SVZHeft29.pdf

    # Sources daily traffic: HafenCity GmbH , Standortanalyse, S. 120

    if road_properties["road_type"] == "railroad":
        return None, None, None

    car_traffic = int(int(road_properties["car_traffic_daily"]) * 0.11)
    truck_traffic = int(int(road_properties["truck_traffic_daily"]) * 0.08)
    max_speed = int(road_properties["max_speed"])

    return max_speed, car_traffic, truck_traffic

# source for train track data = http://laermkartierung1.eisenbahn-bundesamt.de/mb3/app.php/application/eba
def get_train_track_data(road_properties):
    if not road_properties["road_type"] == "railroad":
        return None, None, None, None

    train_speed = road_properties["train_speed"]
    train_per_hour = road_properties["trains_per_hour"]
    ground_type = road_properties["ground_type"]
    has_anti_vibration = road_properties["has_anti_vibration"]

    return train_speed, train_per_hour, ground_type, has_anti_vibration

def apply_traffic_settings_to_design_roads(design_roads, traffic_settings):
    max_speed = traffic_settings["max_speed"]
    traffic_quota = traffic_settings["traffic_quota"]

    for road in design_roads["features"]:
        # only adjust traffic settings of manipulatable roads
        if (
            "traffic_settings_adjustable" in list(road["properties"].keys())
            and road["properties"]["traffic_settings_adjustable"]
        ):
            road["properties"]["max_speed"] = max_speed
            road["properties"]["truck_traffic_daily"] = (
                road["properties"]["truck_traffic_daily"] * traffic_quota
            )
            road["properties"]["car_traffic_daily"] = (
                road["properties"]["car_traffic_daily"] * traffic_quota
            )

    return design_roads

def get_road_queries(traffic_settings, roads_gdf):
    roads_geojson = json.loads(roads_gdf.to_json())
    roads_geojson = apply_traffic_settings_to_design_roads(
        roads_geojson, traffic_settings
    )
    road_features = roads_geojson["features"]

    for feature in road_features:
        id = feature["properties"]["id"]
        road_type = get_road_type(feature["properties"])
        coordinates = feature["geometry"]["coordinates"]

        # input road type might not be defined. road is not imported # TODO consider using a fallback
        if road_type == 0:
            continue
        if feature["geometry"]["type"] == "MultiLineString":
            # beginning point of the road
            start_point = coordinates[0][0]
            # end point of the road
            end_point = coordinates[-1][-1]
        else:
            # beginning point of the road
            start_point = coordinates[0]
            # end point of the road
            end_point = coordinates[1]
            # build string containing all coordinates

        geom = wkt.dumps(feature["geometry"], decimals=0)

        max_speed, car_traffic, truck_traffic = get_car_traffic_data(
            feature["properties"]
        )
        (
            train_speed,
            train_per_hour,
            ground_type,
            has_anti_vibration,
        ) = get_train_track_data(feature["properties"])

        # init new RoadInfo object
        road_info = RoadInfo(
            id,
            geom,
            road_type,
            start_point,
            end_point,
            max_speed,
            car_traffic,
            truck_traffic,
            train_speed,
            train_per_hour,
            ground_type,
            has_anti_vibration,
        )
        all_roads.append(road_info)

    nodes = create_nodes(all_roads)
    sql_insert_strings = []
    for road in all_roads:
        sql_insert_string = get_insert_query_for_road(road, nodes)
        sql_insert_strings.append(sql_insert_string)

    return sql_insert_strings

# returns sql queries for the traffic table,
def get_traffic_queries():
    sql_insert_strings_noisy_roads = []
    nodes = create_nodes(all_roads)
    for road in all_roads:
        node_from = get_node_for_point(road.get_start_point(), nodes)
        node_to = get_node_for_point(road.get_end_point(), nodes)

        if (
            road.get_road_type_for_query()
            == road_types_iffstar_noise_modelling["railroad"]
        ):
            # train traffic
            train_speed = road.get_train_speed()
            trains_per_hour = road.get_train_per_hour()
            ground_type = road.get_ground_type_train_track()
            has_anti_vibration = road.is_anti_vibration()

            sql_insert_string = (
                "INSERT INTO roads_traffic (node_from,node_to, train_speed, "
                "trains_per_hour, ground_type, has_anti_vibration) "
                "VALUES ({0},{1},{2},{3},{4},{5});".format(
                    node_from,
                    node_to,
                    train_speed,
                    trains_per_hour,
                    ground_type,
                    has_anti_vibration,
                )
            )
        else:
            # car traffic
            traffic_cars = road.get_car_traffic()
            traffic_trucks = road.get_truck_traffic()
            max_speed = road.get_max_speed()
            load_speed = max_speed * 0.9
            junction_speed = max_speed * 0.85

            sql_insert_string = (
                "INSERT INTO roads_traffic (node_from,node_to,load_speed,junction_speed,max_speed,"
                "lightVehicleCount,heavyVehicleCount) "
                "VALUES ({0},{1},{2},{3},{4},{5},{6});".format(
                    node_from,
                    node_to,
                    load_speed,
                    junction_speed,
                    max_speed,
                    traffic_cars,
                    traffic_trucks,
                )
            )
        sql_insert_strings_noisy_roads.append(sql_insert_string)

    return sql_insert_strings_noisy_roads

# returns a wkt string for a multipolygon containing all buildings
def get_buildings_geom_as_wkt(buildings_gdf: gpd.GeoDataFrame) -> str:
    # simplify complex geometries to speed up calculation and avoid hickups with spatial db.
    buildings_gdf.geometry = buildings_gdf.geometry.simplify(0.1)

    # calculation of noise results is 5sec. faster if building geometries are provided as single multipolygon
    return f"'{buildings_gdf.geometry.unary_union}'"

# create nodes for all roads - nodes are connection points of roads
def create_nodes(all_roads):
    nodes = []
    for road in all_roads:
        coordinates_start_point = road.get_start_point()
        nodes.append(coordinates_start_point)
        coordinates_end_point = road.get_end_point()
        nodes.append(coordinates_end_point)
    unique_nodes = []

    # filter for duplicates
    for node in nodes:
        if not any(
            numpy.array_equal(node, unique_node) for unique_node in unique_nodes
        ):
            unique_nodes.append(node)
    return unique_nodes

def get_node_for_point(point, nodes):
    dict_of_nodes = {i: nodes[i] for i in range(0, len(nodes))}
    for node_id, node in dict_of_nodes.items():
        if point == node:
            return node_id
    print("could not find node for point", point)
    exit()

def get_road_type(road_properties):
    # if not in road types continue
    for output_road_type in road_types_iffstar_noise_modelling.keys():
        if road_properties["road_type"] == output_road_type:
            return road_types_iffstar_noise_modelling[output_road_type]

    print("no matching noise road_type_found for", road_properties["road_type"])
    return 0

def get_insert_query_for_road(road, nodes):
    node_from = get_node_for_point(road.get_start_point(), nodes)
    node_to = get_node_for_point(road.get_end_point(), nodes)

    sql_insert_string = (
        "INSERT INTO roads_geom (the_geom,NUM,node_from,node_to,road_type) "
        "VALUES (ST_GeomFromText('{0}'),{1},{2},{3},{4});".format(
            road.get_geom(),
            road.get_road_id(),
            node_from,
            node_to,
            road.get_road_type_for_query(),
        )
    )

    return sql_insert_string


import json

from app.common.constants import ModuleName
from app.userpanel.models import Calculation
from tests.integration.constants import URL

def test_get_hardy_weinberg_ok(test_client):
    response = test_client.get(URL.HARDY_WEINBERG_GET)
    assert response.status_code == 200
    assert b'Hardy-Weinberg equilibrium' in response.data

def test_post_hardy_weinberg_ok(test_client):
    data = dict()
    data["ho"] = 4
    data["he"] = 3
    data["rho"] = 2
    data["alfa"] = 0.05

    response = test_client.post(URL.HARDY_WEINBERG_POST, data=json.dumps(data), content_type='application/json')
    calculation = Calculation.query.first()

    assert response.status_code == 200
    assert ModuleName.HARDY_WEINBERG in calculation.module_name


import logging

from utils import messages
import helpers

from descriptors import EndpointAddress
from task import Task
import zmq

logger = logging.getLogger('zmqpipeline.distributor')

class Distributor(object):
    """
    Responsible for distributing (pushing) tasks to workers. What gets distributed is determined
    by Tasks, which are user-implementations that configure how the distributor works.

    The pipeline pattern assumes there is only one distributor with one or more registered tasks.
    """
    task_classes = {}

    def __init__(self, collector_endpoint, collector_ack_endpoint,
                 receive_metadata = False, metadata_endpoint = None):
        """
        Instantiate a distributor.

        :param EndpointAddress collector_endpoint: The endpoint of the collector
        :param EndpointAddress collector_ack_endpoint: The endpoint of the collector for receiving ACKs
        :param bool receive_metadata: When true, the distributor will wait for meta data from a meta data worker before distributing tasks to clients
        :param EndpointAddress metadata_endpoint: The endpoint from which to receive meta data from a meta data worker.
        :return: A Distributor object
        """
        logger.info('Initializing distributor')
        self.context = zmq.Context()
        self.metadata = {}

        if not isinstance(collector_endpoint, EndpointAddress):
            raise TypeError('collector_endpoint must be an EndpointAddress instance')
        if not isinstance(collector_ack_endpoint, EndpointAddress):
            raise TypeError('collector_ack_endpoint must be an EndpointAddress instance')

        logger.info('Connecting to collector endpoint %s', collector_endpoint)
        self.sink = self.context.socket(zmq.PUSH)
        self.sink.connect(collector_endpoint)

        logger.info('Connecting to collector ACK endpoint %s', collector_ack_endpoint)
        self.sink_ack = self.context.socket(zmq.PULL)
        self.sink_ack.connect(collector_ack_endpoint)

        if receive_metadata:
            logger.debug('Setting up receiving metadata')
            if not metadata_endpoint:
                raise AttributeError('metadata_endpoint is required to receive metadata')
            if not isinstance(metadata_endpoint, EndpointAddress):
                raise TypeError('metadata_endpoint must be an EndpointAddress instance')

            logger.info('Connecting to meta data worker endpoint %s', metadata_endpoint)
            self.metadata_client = self.context.socket(zmq.REP)
            self.metadata_client.bind(metadata_endpoint)
        else:
            self.metadata_client = None

        self.poller = zmq.Poller()
        self.poller.register(self.sink_ack, zmq.POLLIN)

        self.client_addresses = {}

        # keyed on worker ids
        self.worker_initialized = {}
        self.clients = {}
        self.tasks = {}

        for taskcls in self.registered_task_classes:
            self.register_task_instance(taskcls)

        self.receive_metadata = receive_metadata
        if receive_metadata:
            self.wait_for_metadata()

        # data received from collector
        self.ack_data = {}

    @property
    def registered_task_classes(self):
        return self.task_classes.values()

    @classmethod
    def register_task(cls, task):
        if task.task_type not in cls.task_classes:
            cls.task_classes[task.task_type] = task

    def wait_for_metadata(self):
        logger.info('Waiting to receive metadata')
        msg = self.metadata_client.recv()
        data, task, msgtype = messages.get(msg)
        assert msgtype == messages.MESSAGE_TYPE_META_DATA
        assert task == ''

        logger.debug('Storing metadata: %s', data)

        if data:
            self.metadata = data

        logger.info('Meta data received - sending success response')
        self.metadata_client.send(messages.create_success())

        logger.info('Sending meta data to collector')
        self.sink.send(messages.create_metadata(self.metadata))

    def process_sink_ack(self):
        msg = self.sink_ack.recv()
        data, tt, msgtype = messages.get(msg)

        self.tasks[tt].n_acks += 1
        ack_id = data.get('_id', 0)

        logger.debug('Received ACK ID %d - %d acks from task type %s', ack_id, self.tasks[tt].n_acks, tt)
        # logger.info('Received ACK ID %d - %d acks from task type %s - msgtype: %s', ack_id, self.tasks[tt].n_acks, tt, msgtype)
        if data:
            self.ack_data = data
        else:
            self.ack_data = {}

    def register_task_instance(self, taskcls):
        task = taskcls()
        logger.debug('Registering Task with type %s', task.task_type)

        logger.info('Connecting task type %s to endpoint %s', task.task_type, task.endpoint)
        s = self.context.socket(zmq.ROUTER)
        s.bind(helpers.endpoint_binding(task.endpoint))

        task.client = s
        self.clients[task.task_type] = s
        self.tasks[task.task_type] = task

        logger.debug('Registering task %s', task.task_type)
        Task.register_task_instance(task)
        self.poller.register(s, zmq.POLLIN)

    def add_client_address(self, task_type, addr):
        if task_type not in self.client_addresses:
            self.client_addresses[task_type] = set()
        if addr not in self.client_addresses[task_type]:
            logger.info('Adding new client address: %s for task type: %s', addr, task_type)

        self.client_addresses[task_type].add(addr)

    def run(self):
        """
        Runs the distributor and blocks. Call this immediately after instantiating the distributor.

        Since this method will block the calling program, it should be the last thing the calling code
        does before exiting. Run() will automatically shutdown the distributor gracefully when finished.

        :return: None
        """
        logger.info('Distributor is running (main loop processing)')

        while True:
            try:
                socks = dict(self.poller.poll())
            except KeyboardInterrupt:
                break

            if self.sink_ack in socks:
                self.process_sink_ack()

            alltasks = self.tasks.values()
            if all([t.is_complete for t in alltasks]):
                "exit main loop if all registered tasks are complete"
                logger.info('All tasks complete - exiting distributor main loop')
                break

            for task_type, task in self.tasks.items():
                if task.is_complete:
                    # task finished - skip
                    continue

                if task.is_task_ready_for_initialization() or task.received_init_signal:
                    # task either needs to be initialized, or has been initialized and is ready for processing

                    if task.received_init_signal and not task.is_available_for_handling(self.ack_data):
                        # only consult the task about handling availability if the init signal has been received
                        continue

                    address, empty, msg = task.client.recv_multipart()
                    self.add_client_address(task_type, address)

                    if not self.worker_initialized.get(address):
                        logger.debug('Initial signal for task %s from worker %s recieved', task_type, address)
                        self.worker_initialized[address] = True
                        task.received_init_signal = True

                        if hasattr(task, 'initialize'):
                            logger.info('Initializing task type %s', task.task_type)
                            task.initialize(metadata = self.metadata)

                        logger.debug('Sending success reply to worker address: %s', address)
                        task.client.send_multipart([
                            address, b'', messages.create_metadata(self.metadata)
                        ])

                    else:
                        data, tt, msgtype = messages.get(msg)
                        data = data or {}

                        if msgtype == messages.MESSAGE_TYPE_READY:
                            # invoke registered task's handler
                            logger.debug('Invoking task.handle for task type: %s, client address %s - passing data: %s', task.task_type, address, data)
                            sdata = task.handle(data, address, msgtype, self.ack_data) or {}
                            if not isinstance(sdata, dict) and sdata is not None:
                                raise TypeError('Task handler must return a dictionary or nothing')

                            sdata.update(data)

                            logger.debug('Sending to collector - task type: %s - data: %s', tt, sdata)
                            task.client.send_multipart([
                                address, b'', messages.create_data(tt, sdata)
                            ])

        # shutdown collector + workers
        self.shutdown()

    def shutdown(self):
        """
        Shuts down the distributor. This is automatically called when run() is complete and the distributor
        exits gracefully. Client code should only invoke this method directly on exiting prematurely,
        for example on a KeyboardInterruptException

        :return: None
        """
        logger.info('Shutting down collector and workers')

        logger.debug('Sending END signal to collector')
        self.sink.send(messages.create_end())

        for task_type, task in self.tasks.items():
            client_addrs = self.client_addresses[task_type]
            n_clients = len(client_addrs)

            logger.debug('Shutting down %d clients for task type %s', n_clients, task_type)
            for _ in client_addrs:
                address, empty, msg = task.client.recv_multipart()

                logger.debug('Sending END signal to worker address %s - task type %s', address, task_type)
                task.client.send_multipart([
                    address, b'', messages.create_end(task = task_type)
                ])


import wandb

import pytorch_lightning as pl
from pytorch_lightning.loggers import WandbLogger
from pytorch_lightning.callbacks import ModelCheckpoint
from pytorch_lightning.callbacks.early_stopping import EarlyStopping

from experiment import SegmentationModel3d

if __name__ == '__main__':

    defaults = {
        'learning_rate': 0.0001,
        'loss': 'dice',
        'alpha': 0.9,
        'blocks': 4,
        'batch_size': 10,
        'initial_features': 64,

        'p_dropout': 0.0,

        'p_affine_or_elastic': 0.0,
        'p_elastic': 0.2,
        'p_affine': 0.8,

        'patch_size': 48,
        'samples_per_volume': 10,
        'queue_length': 80,
        'patch_overlap': 4,
        'random_sample_ratio': 4,

        'log_image_every_n': 3,

        'data_path': '/data/training',
    }

    wandb.init(
        project="aneurism_detection",
        config=defaults
    )

    hparams = wandb.config._as_dict()

    model = SegmentationModel3d(hparams)

    wandb_logger = WandbLogger(
        project="aneurism_detection")

    checkpoint_callback = ModelCheckpoint(
        dirpath=f"checkpoints/{wandb_logger.experiment.name}/",
        # save_top_k=1,
        # monitor="step",
        filename="{epoch:02d}-{avg_train_jaccard:.2f}",
    )

    early_stop_callback = EarlyStopping(
        monitor="avg_validation_sensitivity",
        patience=4,
        mode="max")

    trainer = pl.Trainer(
        #fast_dev_run=True,
        gpus=[0],
        #limit_train_batches=0.02, limit_val_batches=0.1,

        profiler="simple",

        callbacks=[checkpoint_callback],

        logger=wandb_logger,
        log_every_n_steps=1,
        check_val_every_n_epoch=3,)

    trainer.fit(model, ckpt_path='checkpoints/pretty-frog-165/epoch=95-avg_train_jaccard=0.00.ckpt' )


# 29Plot.py

import matplotlib.pyplot as graph      # terminal에서 pip install matplotlib (설치 먼저!)
import numpy as np

x = np.arange(5) # [0,1,2,3,4]
y = np.arange(5) # [0,1,2,3,4]

graph.plot(x+1, y+5, "c-", x+10, y+20, "md")
# 색 → b : blue, r : red, m : magenta, g : green, y : yellow, c : cyan
# 모양(marker) → o : circle, 8: octagon, p : pentagon, D : diamond,
#               s : square, * : star, x : x모양, d : thin diamond, ^ v < > (상하좌우)

graph.xlabel("x")
graph.ylabel("y")
graph.title("Plot Line Marker")
graph.grid(True)                       # 그리드 보이게 해줘
graph.savefig("d:/plot_marker.png")    # 저장도 해줘(save figure)
graph.show()


import collections

# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

# Solution 1, hashmap to store all connections
class Solution:
    def dfs(self, node, d):
        if node is None:
            return
        if node.left:
            d[node.val].add(node.left.val)
            d[node.left.val].add(node.val)
            self.dfs(node.left, d)

        if node.right:
            d[node.val].add(node.right.val)
            d[node.right.val].add(node.val)
            self.dfs(node.right, d)

    def bfs(self, start, steps, d):
        seen = set()
        q = [start]
        seen.add(start)
        curr = 0
        while curr < steps:
            tmp = []
            for p in q:
                for x in d[p]:
                    if x not in seen:
                        seen.add(x)
                        tmp.append(x)
            curr += 1
            q = tmp

        return q

    def distanceK(self, root, target, K):
        """
        :type root: TreeNode
        :type target: TreeNode
        :type K: int
        :rtype: List[int]
        """
        d = collections.defaultdict(set)

        self.dfs(root, d)

        res = self.bfs(target.val, K, d)

        return res

# Solution 2, Add parent to each node
class Solution:
    def add_parent(self, node, parent):
        if node is None:
            return
        node.parent = parent

        self.add_parent(node.left, node)
        self.add_parent(node.right, node)

    def bfs(self, node, steps):
        q = [node]
        seen = set()
        seen.add(node.val)
        curr = 0

        while curr < steps:
            tmp = []

            for p in q:
                for x in [p.parent, p.left, p.right]:
                    if x and x.val not in seen:
                        tmp.append(x)
                        seen.add(x.val)
            curr += 1
            q = tmp
        res = [p.val for p in q]

        return res

    def distanceK(self, root, target, K):
        """
        :type root: TreeNode
        :type target: TreeNode
        :type K: int
        :rtype: List[int]
        """

        self.add_parent(root, None)

        res = self.bfs(target, K)

        return res

Algorithms/0863 All Nodes Distance K in Binary Tree.py