import contextlib
import pyodbc
import subprocess

# Populate the names of the API DB, the database server, and the current ICTVonline database.
apiDB = "ICTV_API"
dbServer = "ICTVDEV"
taxonomyDB = "ICTVonline39"

# Export taxonomy JSON files for non-species and species taxa in every MSL release.
def exportTaxonomyJSON(apiDB_, dbServer_, taxonomyDB_, treeID_):

    # Validate parameters
    if apiDB_ in (None, '') or not apiDB_.strip():
        raise Exception("The apiDB parameter is invalid")

    if dbServer_ in (None, '') or not dbServer_.strip():
        raise Exception("The dbServer parameter is invalid")

    if taxonomyDB_ in (None, '') or not taxonomyDB_.strip():
        raise Exception("The taxonomyDB parameter is invalid")

    # The database connection string
    dbConnectionString = ("Driver={SQL Server Native Client 11.0};"
        f"Server={dbServer_};"
        f"Database={apiDB_};"
        "Trusted_Connection=yes;")

    # SQL to drop existing and create new taxonomy views for this taxonomy database.
    viewsSQL = f"EXEC dbo.createIctvOnlineViews @ictvOnlineDbName = '{taxonomyDB_}' "

    treeConstraint = ""
    treeLimit = ""

    # If a tree ID was provided, we will limit the tree ID query to one result.
    if treeID_ not in (None, ''):
        treeConstraint = f"AND tree_id = {treeID_} "
        treeLimit = "TOP 1 "

    # SQL to retrieve tree ID(s) in the taxonomy database's taxonomy "table of contents" table.
    treeSQL = f"""
        SELECT {treeLimit}tree_id
        FROM v_taxonomy_toc
        WHERE msl_release_num IS NOT NULL
        {treeConstraint}
        ORDER BY tree_id ASC
    """

    # Open the database connection
    with contextlib.closing(pyodbc.connect(dbConnectionString)) as dbConnection:

        # Create views for this taxonomy database.
        viewCursor = dbConnection.cursor()
        viewCursor.execute(viewsSQL)

        # Get all tree IDs in the taxonomy database.
        treeCursor = dbConnection.cursor()

        # Create views for this taxonomy database.
        # Iterate over all tree IDs that are returned.
        for row in treeCursor.execute(treeSQL):

            # Get the tree ID as a string.
            treeID = str(row.tree_id)

            # Get the year from the tree ID.
            year = treeID[0:4]
            formattedYear = year if treeID[4] == "0" else f"{year}.{treeID[4]}"

            # Create the command line text to run sqlcmd for non-species taxa.
            nonSpeciesCMD = (f"sqlcmd -S {dbServer_} "
                f"-Q \"EXEC [{apiDB_}].dbo.exportNonSpeciesTaxonomyJSON @treeID = {treeID}\" "
                f"-o \"JSON\\nonSpecies_{formattedYear}.json\" "
                "-y 0 ")

            # Run the command
            subprocess.run(nonSpeciesCMD, shell=True)

             # Create the command line text to run sqlcmd for species taxa.
            speciesCMD = (f"sqlcmd -S {dbServer_} "
                f"-Q \"EXEC [{apiDB_}].dbo.exportSpeciesTaxonomyJSON @treeID = {treeID}\" "
                f"-o \"JSON\\species_{formattedYear}.json\" "
                "-y 0 ")

            # Run the command
            subprocess.run(speciesCMD, shell=True)

if __name__ == '__main__':

    exportTaxonomyJSON(apiDB, dbServer, taxonomyDB, None)


import numpy as np
from geopy import Nominatim
from geopy.extra.rate_limiter import RateLimiter
import pandas as pd
import os,glob
import time
import configparser

print('process Started')

path_fig = configparser.ConfigParser()
path_fig.read('test_us.ini')
excel_path =  path_fig.get('test','excel_path')
input_path =  path_fig.get('test','input_path')
output_path =  path_fig.get('test','output_path')

#split dataset into chuncks
df = pd.read_excel(excel_path)
ss = np.array_split(df, 175)

for test in range(len(ss)):
	ss[test].to_csv(f'{input_path}uscitysplit{test}.csv')
# print('wait 1.30 min')
time.sleep(10)

# function for finding location coordinates
def process_operation(df,i):
    i = i
    df['CSM'] = df[df.columns[0:2]].apply(lambda x: ','.join(x.dropna().astype(str)),axis=1)
    geolocator = Nominatim(user_agent="my_request")

    #applying the rate limiter wrapper
    geocode = RateLimiter(geolocator.geocode, min_delay_seconds=1)

    #Applying the method to pandas DataFrame
    df['location'] = df['CSM'].apply(geocode)
    df['Lat'] = df['location'].apply(lambda x: x.latitude if x else None)
    df['Lon'] = df['location'].apply(lambda x: x.longitude if x else None)
    df.to_csv(f'{output_path}uscity_corr_{i}.csv')
    print(f'saved as csv uscity_corr_{i}.csv successfully')
    print('process completed',df.shape)

# read all splited csv and apply with function
all_files = glob.glob(os.path.join(input_path, "*.csv"))

all_files.sort()

for i in range(len(all_files)):
    print(f'processing with {all_files[i]}')
    df = pd.read_csv(f'{all_files[i]}')
    df.drop(['Unnamed: 0'], axis=1,inplace=True)
    # print(df)
    try:
        process_operation(df,i)
    except:
        time.sleep(500)
        process_operation(df,i)

#   Merge all files generated coordinates csv
merge_all_files = glob.glob(os.path.join(output_path, "*.csv"))
df_final = pd.concat((pd.read_csv(f) for f in merge_all_files), ignore_index=True)
df_final.drop(['Unnamed: 0', 'location','CSM',], axis=1,inplace=True)
df_final.to_csv('US-City_Coordinates_testrun.csv')

print('process Completed')


import datetime, time
import os
from difflib import SequenceMatcher

#size of the item length
#decided in the stringMargin method
sizeX = 0
# predefined width for the box that will contain items
DEFAULT_SIZE_X=50
sizeY = 8
# takes an input of list of contents that need to be printed in a box
# creates an artificial spaces around the item to fit it in the box
# if the list is less than 8 elemnts it will add empty string items
#if any item in the list is longer than 50 the box is resized to fit it
# to keep the box at a standard height
def stringMargin(listOfItemsForBox):

    #resize the height of the box
    listOfItemsForBox.append("")
    while len(listOfItemsForBox) < sizeY:
       listOfItemsForBox.append("")

    # refere to the outer sizeX
    global sizeX

    sizeX = len(max(listOfItemsForBox, key=len)) + 10
    if sizeX < DEFAULT_SIZE_X :
        sizeX = DEFAULT_SIZE_X
    # the final output of the function
    modifiedList = []
    for item in listOfItemsForBox :
        # calculates needed length
        needed_length = sizeX - len(item)
         # a standard right intended
        rightIndent = 6
        if item == listOfItemsForBox[0]:
            rightIndent = 3
        leftIndent = needed_length - rightIndent
        leftMargin = len(item) + leftIndent

         # creates left margin
        item = item.ljust(leftMargin)
        rightMargin = len(item) + rightIndent
         # adds right indent spaces into the item
        item = item.rjust(rightMargin, " ")
        modifiedList.append(item)
    return modifiedList

# function to generate a box
#takes input for the actual content of the box
#outputs a box
def boxGenerator(listOfContent):
	# clears the terminal before the box so the box stays on top
    os.system('clear')

    #takes the input list to stringMargin() to modify it
    listOfContent = stringMargin(listOfContent)

    # prints a corner then stays in the same line
    print(u'\u250F', end="", flush=True)
    for i in range(0, sizeX):
    	# prints the top-bar
        print(u'\u2501', end="", flush=True)

	# prints a corner
    print(u'\u2513')

    #  the body of the box
    for i in range(0, len(listOfContent)):
        # prints a vertical pipe
        print(u'\u2503', end="", flush=True)

        #prints the actual content with spaces around it
        print(listOfContent[i], end="", flush=True)

        # prints a vertical pipe
        print(u'\u2503')

    # prints a T style corner
    print(u'\u2523', end="", flush=True)

    # prints the bottom bar
    for i in range(0, sizeX):
    	# prints a horizontal bar
        print(u'\u2501', end="", flush=True)

    # prints a corner
    print(u'\u251B')

    #prints a corner
    print(u'\u2517', end="", flush=True)

    # prints a vertical line
    for i in range(0, 15):
        print(u'\u2501', end="", flush=True)

#cleans the sentence from special characters
def sentenceModification(sentence):
    sentence = sentence.lstrip().rstrip().lower()
    for chars in sentence:
        if not chars.isalpha() and chars !=" ":
            sentence = sentence.replace(chars, "")
    return sentence

#dictionary that contains word counts
fileWordsInformation = {
    'total Number of words': 0,
    'correctly spelt words': 0,
    'incorrectly spelt words': 0,
    'ignored words': 0,
    'marked words': 0,
    'words added to dictionary': 0
}

# checks words in sentences
def checkWordsForSentences(sentence):
    #cleans the sentence from any special characters
    sentence = sentenceModification(sentence)
    #splits the sentence into list of words
    listofWords = sentence.split()
    fileWordsInformation["total Number of words"] = len(listofWords)
    fileWordsInformation["correctly spelt words"] = 0
    fileWordsInformation["incorrectly spelt words"] = 0

    #irretates through the list of words to find any matching in the dictionary
    #prints the word is spelt correctly if it was found in the dictionary
    for word in listofWords:
        if word in EnglishWords:
            boxGenerator(["W O R D     F O U N D", "", "", word,
            "Spelt correctly"])
            #introduce a lag so the screen above does not disapper
            # if next screens come
            dummyInput = input("enter any key to go next")
            fileWordsInformation["correctly spelt words"] += 1
        else:
            boxGenerator(["W O R D   N O T  F O U N D", "", "", word,
             "Not found in dictionary"])
            dummyInput = input("enter any key to go next")
            fileWordsInformation["incorrectly spelt words"] += 1
    finalBoxList = ["F I N A L   R E S U L T", ""]

    #prints the fileWordsInformation which contains number of words,
    # spelt correctly not spelt correctly
    for x, y in fileWordsInformation.items():
        tempStr = "Number of " + x + " : " + str(y)
        finalBoxList.append(tempStr)
        #to not print the file related counts
        if len(finalBoxList) == 5:
            break
    #generate final information
    boxGenerator(finalBoxList)

#a funtion to spell check for words
#takes the input for the file and file name
#creates a file with the name 2+file name
#saves the spellchecked words to the new file
#offeres the user the chance to use other words
#instead of the incorrect ones
def checkWordsForFiles(sentence, file):
    sentence = sentenceModification(sentence)
    listofWords = sentence.split()
    fileWordsInformation["total Number of words"] = len(listofWords)
    correctedWords = ""
    resultFile = open("2" + file, "w")
    for word in listofWords:
        if word in EnglishWords:
            correctedWords += word + " "
            fileWordsInformation["correctly spelt words"] += 1
        # case the word is spelt incorrectly
        else:
            fileWordsInformation["incorrectly spelt words"] += 1
            listOfRaitos = []
            wordsFromRatios = []
            #saves the list of all ratios against the word from dictonary
            for matchedWord in EnglishWords:
                ratio = SequenceMatcher(None, matchedWord, word).ratio()
                listOfRaitos.append(ratio)
            desiredInput = False
            # insure correct input from the user
            while not desiredInput:
            	#takes the maximum ratio available to the word
                requiredWordIndex = listOfRaitos.index(max(listOfRaitos))
                desiredWord = EnglishWords[requiredWordIndex]
                boxGenerator(["W O R D   N O T   F O U N D", "", word, "", "did you mean", "", desiredWord])
                changeWordInput = input(" Enter [y] or [n]: ")
                print("\n")
                if changeWordInput == "y":
                    correctedWords += desiredWord + " "
                    desiredInput = True

                #in case the user wants to save the word to a dictionary
                #or mark as incorrect or ignore it
                elif changeWordInput == "n":
                    desiredInput = True
                    correctInput = False
                    # asks the user to add the word into dictionart
                    # ignore the word or mark it
                    # while loop is to ensure correct input from the user
                    while not correctInput:
                        boxGenerator(["W O R D   N O T   F O U N D", "", word, "", "1. Ignore the word.",
                                      "2. Mark the word as incorrect.", "3. Add word to dictionary."])
                        incorrectWordsHandling = input(" Enter choice: ")
                        print("\n")
                        if incorrectWordsHandling == "1":
                            correctedWords += "!" + word + "! "
                            fileWordsInformation["ignored words"] += 1
                            correctInput = True
                        elif incorrectWordsHandling == "2":
                            correctedWords += "?" + word + "? "
                            fileWordsInformation["marked words"] += 1
                            correctInput = True
                        elif incorrectWordsHandling == "3":
                            correctedWords += "*" + word + "* "
                            fileWordsInformation["words added to dictionary"] += 1
                            englishDict.write("\n" + word)
                            correctInput = True
    #prints the data into the corrected file
    resultFile.write(datetime.datetime.now().strftime("%d-%m-%Y %H:%M:%S\n"))
    resultFile.write("\n")
    finalBoxList = ["F I N A L   R E S U L T", ""]
    for x, y in fileWordsInformation.items():
        tempStr = "Number of " + x + " : " + str(y)
        temp2Str = tempStr + "\n"
        finalBoxList.append(tempStr)
        resultFile.write(temp2Str)
    resultFile.write("\n")
    #writes all the words after correction to the file
    resultFile.write(correctedWords)
    resultFile.close()

    #generates feedback with word counts, correct, incorrect,ignored...etc
    boxGenerator(finalBoxList)
    dummyInput = input(" enter any key to go next ")

# the main screen
terminated = False
englishDictReadOnly = open("EnglishWords.txt")
# creates a list to contain the words in the file
EnglishWords = englishDictReadOnly.read().split("\n")
englishDictReadOnly.close()
# reopens the file in append mode so it can be edited
englishDict = open("EnglishWords.txt", "a")
while not terminated:

	#generates the main screen
    boxGenerator(["S P E L L   C H E C K E R", "", "1. Check a file", "2. Check a sentence", "", "", "0.Quit"])
    usrInput = input(" Enter choice: ")

    # file mode
    if usrInput == "1":
        fileWrittenCorrect = False
        userFileNameInput = ""
        #validates input for file name
        while not fileWrittenCorrect:
        	# file mode main screen
            boxGenerator(["L O A D    F I L E", "", "Enter a file name", "then press [enter]"])
            if userFileNameInput == "":
                userFileNameInput = input(" Filename: ")
            startTime = time.clock()

            #checks if the file name exists
            if os.path.exists(userFileNameInput):
                boxGenerator(["L O A D    F I L E", "", "", "Correct File name", "", "Done"])
                checkWordsForFiles(open(userFileNameInput).read(), userFileNameInput)
                fileWrittenCorrect = True
                open(userFileNameInput).close()
                endTime = time.clock() - startTime
                #shows the time in microseconds
                boxGenerator(["C H E C K E D   F I L E", "", "TimeElapsed", str(endTime * 10 ** 6), "microseconds"])
            else:
                boxGenerator(["L O A D    F I L E", "", "Enter a file name or press q to quit", "then press [enter]"])
                userFileNameInput = input(" Filename: ")
                if (userFileNameInput == "q"):
                    break
	# sentence mode
    elif usrInput == "2":

    	# prints sentence screen
        boxGenerator(["S E N T E N C E   M O D E", "", "Enter a sentence", "then press [enter]"])
        checkWordsForSentences(input(" Enter sentence to spellcheck: "))
        usrInput = input(" Press q [enter] to quit or any other key [enter] to go again: ")
        # termenates the program
        if usrInput == "q":
            terminated = True

    # quits the program
    elif usrInput == "0" or usrInput == "q":
        terminated = True
        englishDict.close()


'''
Created on Oct 20, 2016

@author: christian
'''
import regex as re
import string

from andromeda.config.constants import Constants

class Tokenizer:

    def __init__(self):
        self._raw_string = ''
        self._char_map = []
        self._init_char_map()

    def _init_char_map(self):
        self._char_map = []
        for c in self._raw_string:
            self._char_map.append((Constants.TOKENIZER__TOKEN_CLASS__UNKNOWN, 0))

    def _set_token_class(self, start_pos, end_pos, token_class, token_class_nr=0, overwrite=False):
        for pos in range(start_pos, end_pos):
            if self._char_map[pos][0] == Constants.TOKENIZER__TOKEN_CLASS__UNKNOWN or overwrite == True:
                self._char_map[pos] = (token_class, token_class_nr)

    def tokenize(self, s):
        self._raw_string = s.strip()
        self._init_char_map()

        self._match_whitespaces(Constants.TOKENIZER__TOKEN_CLASS__WHITESPACE)
        #self._match_unicode_strings(Contstants.TOKENIZER__TOKEN_CLASS__UNICODE)
        self._match_urls(Constants.TOKENIZER__TOKEN_CLASS__URL)
        self._match_emails(Constants.TOKENIZER__TOKEN_CLASS__EMAIL)
        self._match_special_twitter_concepts('@', Constants.TOKENIZER__TOKEN_CLASS__SPECIAL_TERM_USER)
        self._match_special_twitter_concepts('#', Constants.TOKENIZER__TOKEN_CLASS__SPECIAL_TERM_TOPIC)
        self._match_html_entities(Constants.TOKENIZER__TOKEN_CLASS__HTML_ENTITY)
        #self._match_time_formats(Contstants.TOKENIZER__TOKEN_CLASS__TIME_FORMAT)
        self._match_numbers(Constants.TOKENIZER__TOKEN_CLASS__NUMBER)
        self._match_brackets()
        self._match_quotes()
        self._match_emoticons(Constants.TOKENIZER__TOKEN_CLASS__EMOTICON)
        #self._match_laughter(Contstants.TOKENIZER__TOKEN_CLASS__LAUGHTER)
        self._match_character_substrings('-=\+<>', 2, Constants.TOKENIZER__TOKEN_CLASS__CHARACTER_SUBSTRING)
        self._match_repeated_chars(Constants.TOKENIZER__PUNCTUATION_MARKS__TERMINAL_POINTS, Constants.TOKENIZER__TOKEN_CLASS__PUNCTUATION_TERMINAL_POINTS_REPEATED)
        self._match_repeated_chars(Constants.TOKENIZER__PUNCTUATION_MARKS__PAUSING_POINTS, Constants.TOKENIZER__TOKEN_CLASS__PUNCTUATION_PAUSING_POINTS_REPEATED)
        self._match_punctuation_marks(Constants.TOKENIZER__PUNCTUATION_MARKS__TERMINAL_POINTS, Constants.TOKENIZER__TOKEN_CLASS__PUNCTUATION_TERMINAL_POINTS)
        self._match_punctuation_marks(Constants.TOKENIZER__PUNCTUATION_MARKS__PAUSING_POINTS, Constants.TOKENIZER__TOKEN_CLASS__PUNCTUATION_PAUSING_POINTS)
        self._match_alphanumeric_words(Constants.TOKENIZER__TOKEN_CLASS__ALPHANUM)
        #self._match_negation_abbreviation(Contstants.TOKENIZER__TOKEN_CLASS__ALPHANUM_NEGATION_ABBREVIATION)
        self._match_selected_characters('\'&@*+', Constants.TOKENIZER__TOKEN_CLASS__ALPHANUM)
        self._match_unicode_characters(Constants.TOKENIZER__TOKEN_CLASS__UNICODE)
        return self._generate_token_list()

    def _match_html_entities(self, token_class):
        p = re.compile('&[^\s]*;')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_unicode_strings(self, token_class):
        token_class_nr = 0
        for size in [8,4]:
            p = re.compile(r'(\\[U])(\w{%s})' % size, re.IGNORECASE)
            for m in p.finditer(self._raw_string):
                token_class_nr += 1
                self._set_token_class(m.start(1), m.end(2), token_class, token_class_nr)

    def _match_urls(self, token_class):
        p = re.compile('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+#]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+[^ \<\.;:"\'\)\]\}]')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_emails(self, token_class):
        p = re.compile('([a-z0-9!#$%&\'*+\/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&\'*+\/=?^_`{|}~-]+)*(@|\sat\s)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(\.|\sdot\s))+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)')
        for m in p.finditer(self._raw_string, re.UNICODE):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_whitespaces(self, token_class):
        p = re.compile(' ')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_repeated_chars(self, character_list, token_class):
        pattern_core = '|'.join(character_list)
        p = re.compile('['+pattern_core+']{2,}')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_character_substrings(self, character_list, min_length, token_class):
        core_pattern = ''.join(character_list)
        p = re.compile('['+core_pattern+']{2,}')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_quotes(self):
        p = re.compile('(?<=\W)"([^"'+Constants.TOKENIZER__PUNCTUATION_MARKS__PAUSING_POINTS+Constants.TOKENIZER__PUNCTUATION_MARKS__TERMINAL_POINTS+']*)"(?=\W)')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[0] + 1, Constants.TOKENIZER__TOKEN_CLASS__QUOTE_OPEN)
            self._set_token_class(m.span()[1] - 1, m.span()[1], Constants.TOKENIZER__TOKEN_CLASS__QUOTE_CLOSE)
        p = re.compile('(?<=\W)\'([^\':;\.,]*)\'(?=\W)')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[0] + 1, Constants.TOKENIZER__TOKEN_CLASS__QUOTE_OPEN)
            self._set_token_class(m.span()[1] - 1, m.span()[1], Constants.TOKENIZER__TOKEN_CLASS__QUOTE_CLOSE)

    def _match_brackets(self):
        bracket_pairs = [('(', ')'), ('[', ']'), ('{', '}'), ('<', '>')]
        for bracket_pair in bracket_pairs:
            p = re.compile('\\'+bracket_pair[0]+'([^\\'+bracket_pair[0]+'\\'+bracket_pair[1]+']*)\\'+bracket_pair[1])
            for m in p.finditer(self._raw_string):
                self._set_token_class(m.span()[0], m.span()[0] + 1, Constants.TOKENIZER__TOKEN_CLASS__BRACKET_OPEN)
                self._set_token_class(m.span()[1] - 1, m.span()[1], Constants.TOKENIZER__TOKEN_CLASS__BRACKET_CLOSE)

    def _match_punctuation_marks(self, character_list, token_class):
        pattern_core = '|'.join(character_list)
        p = re.compile(r'(['+pattern_core+'])')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.start(1), m.end(1), token_class)

    def _match_alphanumeric_words(self, token_class):
        p = re.compile('([a-zA-Z0-9]+)(?:[-[a-zA-Z0-9]|\']*([a-zA-Z0-9])?)?')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_special_twitter_concepts(self, start_char, token_class):
        p = re.compile(start_char+'([A-Za-z_]+[A-Za-z0-9_]+)')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_numbers(self, token_class):
        p = re.compile('(?<=\s)([+-]?\d+(?:.\d)?\d*)(?=\W)')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_time_formats(self, token_class):
        p = re.compile(r'(?:(?:([01]?\d|2[0-3]):)?([0-5]?\d):)?([0-5]?\d)\s?(am|pm|a\.m\.|p\.m\.)?')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_laughter(self, token_class):
        pattern_core = '|'.join(Contstants.TOKENIZER__EMOTEXTS_CORE_PATTERNS_LIST)
        p = re.compile(r'\b[a-z]*(' + pattern_core + r'){2,}[a-z]*\b', re.IGNORECASE)
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_emoticons(self, token_class):
        # Generic patterns
        p = re.compile("(^|\s+)([%s]?)([%s])([%s]?)([%s]+)([,\.]|$|\s+)" % tuple(map(re.escape, [Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_TOP, Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_EYES, Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_NOSES, Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_MOUTHS])))
        for m in p.finditer(self._raw_string, overlapped=True):
            self._set_token_class(m.start(2), m.end(5), token_class)
        # Generic patterns (mirrored orientation)
        p = re.compile("(^|\s+)([%s]+)([%s]?)([%s])([%s]?)([,\.]|$|\s+)" % tuple(map(re.escape, [Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_MOUTHS, Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_NOSES, Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_EYES, Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_TOP])))
        for m in p.finditer(self._raw_string, overlapped=True):
            self._set_token_class(m.start(2), m.end(5), token_class)
        # Fixed patterns
        p = re.compile('|'.join(map(re.escape, Constants.TOKENIZER__EMOTICONS_FIXED_PATTERNS)))
        for m in p.finditer(self._raw_string, overlapped=True):
            self._set_token_class(m.span()[0], m.span()[1], token_class, overwrite=True)

    def _match_unicode_characters(self, token_class):
        token_class_nr = 0
        p = re.compile("\W", re.UNICODE)
        for m in p.finditer(self._raw_string):
            c = self._raw_string[m.span()[0]:m.span()[1]]
            if c not in string.printable:
                token_class_nr += 1
                self._set_token_class(m.span()[0], m.span()[1], token_class, token_class_nr)

    def _match_selected_characters(self, character_list, token_class):
        pattern_core = '|'.join(character_list)
        p = re.compile('['+pattern_core+']{1}')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_negation_abbreviation(self, token_class):
        p = re.compile('([A-Za-z0-9_]+)([^n])([n]+\'[t]+)' , re.IGNORECASE)
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.start(3), m.end(3), token_class, overwrite=True)

    def __match_time_formats(self, token_class):
        p = re.compile(r'(?:(?:([01]?\d|2[0-3]):)?([0-5]?\d):)?([0-5]?\d)\s?(am|pm|a\.m\.|p\.m\.)?')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _generate_token_list(self):
        token_list = []
        token = ''
        current_token_class = -1
        current_token_class_nr = -1
        for pos, val in enumerate(self._char_map):
            token_class = val[0]
            token_class_nr = val[1]

            if token_class != current_token_class or token_class_nr != current_token_class_nr or token_class == Constants.TOKENIZER__TOKEN_CLASS__NON_ASCII:
                if not token.isspace() and current_token_class >= 0:
                    token_list.append({ Constants.TOKENIZER__TOKEN_FEATURE__BASE :
                                            { Constants.TOKENIZER__TOKEN_FEATURE__TOKEN: token,
                                              Constants.TOKENIZER__TOKEN_FEATURE__CLASS: current_token_class,
                                              Constants.TOKENIZER__TOKEN_FEATURE__FIRST_CHARACTER_INDEX: (pos - len(token)),
                                              Constants.TOKENIZER__TOKEN_FEATURE__CHARACTER_COUNT: len(token)
                                            }
                                      })
                token = ''
                current_token_class = token_class
                current_token_class_nr = token_class_nr
            token += self._raw_string[pos]

        if token != '':
            token_list.append({ Constants.TOKENIZER__TOKEN_FEATURE__BASE :
                                    { Constants.TOKENIZER__TOKEN_FEATURE__TOKEN: token,
                                      Constants.TOKENIZER__TOKEN_FEATURE__CLASS: current_token_class,
                                      Constants.TOKENIZER__TOKEN_FEATURE__FIRST_CHARACTER_INDEX: (pos+1 - len(token)),
                                      Constants.TOKENIZER__TOKEN_FEATURE__CHARACTER_COUNT: len(token)
                                    }
                              })
        return token_list

if __name__ == "__main__":

    s = '@DemoUser Watched Spectre yesterday HAHAHAHA It was good, but not the "masterpiece" everyone had expected :('
    #s = '@DemoUser Can I do it? \U0001F61E \U0001F52B'
    #s = 'peoples\' consumers \'decide on what they really want\': SINGAPORE An elderly customer bought... http://t.co/dwSxPnUbJJ #singapore'
    #s = 'new \'Your Signature Emoji\' feature (http://bla.com)'
    s = ':\'( :\'( \'test\' yep'
    #s = ':\'( :\'( "test" yes'
    s = '\U0001F602\U0001F602Woooow\u2026Deadpool\U0001F602'.decode('unicode-escape')

    tokenizer = Tokenizer()
    token_list = tokenizer.tokenize(s)


import cv2
import sys
sys.path.append("game/")
import wrapped_flappy_bird as game
import random
import numpy as np
from TFBrain import TFBrain
from TFBrain import Experience
from tqdm import tqdm

# Preprocess raw image to 80*80 image

def preprocess(img):
    img = cv2.resize(img, (80,80))
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    ret, img = cv2.threshold(img,1,255,cv2.THRESH_BINARY)
    img = np.reshape(img, (80,80,1))
    return img

def init_state(observation):
    state = np.stack((observation, observation, observation, observation), axis = 2)
    return state

def proceed_state(state, next_observation):
    next_state = np.append(state[:,:,1:], next_observation, axis = 2)
    return next_state

def learn_flappybird():
    brain_config = {
        'network_type': 'cnn',
        'learning': True,
        'num_actions': 2,
        'experience_size': 50000,
        'gamma': 0.99,
        'observe_age': 1000,
        'explore_age': 300000,
        'explore_burin': 1000,
        'epsilon_min': 0.0,
        'epsilon_test_time': 0.0,
        'batch_size': 32,
        'lookback_window': 3
    }

    brain = TFBrain(brain_config)
    brain.show_configs()

    bird_env = game.GameState()
    frame_per_action = 1

    action = np.array([1,0])
    observation, reward, chain_end = bird_env.frame_step(action)
    observation = preprocess(observation)
    observation = np.reshape(observation, (observation.shape[0], observation.shape[1]))
    state = init_state(observation)

    i = 0
    while 1!= 0:
        if i % frame_per_action == 0:
            action = brain.decide(state, determistic=False)
        else:
            # Do nothing
            action = np.array([1,0])
        i += 1
        next_observation, reward, chain_end = bird_env.frame_step(action)
        next_observation = preprocess(next_observation)
        next_state = proceed_state(state, next_observation)
        experience = Experience(state, action, reward, next_state, chain_end)
        brain.learn(experience)
        state = next_state

def play_flappybird():
    brain_config = {
        'network_type': 'cnn',
        'learning': False,
        'num_actions': 2,
        'lookback_window': 3
    }

    brain = TFBrain(brain_config)
    brain.show_configs()

    bird_env = game.GameState()
    frame_per_action = 1

    action = np.array([1,0])
    observation, reward, chain_end = bird_env.frame_step(action)
    observation = preprocess(observation)
    observation = np.reshape(observation, (observation.shape[0], observation.shape[1]))
    state = init_state(observation)

    i = 0
    while 1!= 0:
        if i % frame_per_action == 0:
            action = brain.decide(state, determistic=True)
        else:
            # Do nothing
            action = np.array([1,0])
        i += 1
        next_observation, reward, chain_end = bird_env.frame_step(action)
        next_observation = preprocess(next_observation)
        next_state = proceed_state(state, next_observation)
        experience = Experience(state, action, reward, next_state, chain_end)
        state = next_state

def main():
    learn_flappybird()
    # play_flappybird()

if __name__ == '__main__':
    main()


from PIL import Image, ImageDraw
from matplotlib import pyplot as plt
import numpy as np
from scipy.spatial import cKDTree
from scipy.spatial import Voronoi
from skimage.draw import polygon
import random

def voronoi(points, size):
    # Add points at edges to eliminate infinite ridges
    edge_points = size*np.array([[-1, -1], [-1, 2], [2, -1], [2, 2]])
    new_points = np.vstack([points, edge_points])

    # Calculate Voronoi tessellation
    vor = Voronoi(new_points)
    return vor

width = height = size = 3600
n = 384

x = np.random.randint(0, width, (n,)) #随机采样n个点
y = np.random.randint(0, height, (n,))

points = np.vstack([x, y]).T #x和y依次组成n个点[[x, y],...]

cyan = (12, 236, 221)
yellow = (255, 243, 56)
pink = (196, 0, 255)
magenta = (255, 103, 231)

new_points = points.copy()

for k in range(100):
    vor = voronoi(new_points, size)

    im = Image.new('RGB', (width, height), (255, 255, 255))
    draw = ImageDraw.Draw(im)

    for i, point in enumerate(new_points):
        circle_size = np.array([20, 20])
        p1 = point - circle_size/2
        p2 = point + circle_size/2
        draw.ellipse((*p1, *p2), fill=(0, 0, 255))

    new_points = []

    for i, region in enumerate(vor.regions):
        if len(region) == 0 or -1 in region: continue
        poly = np.array([vor.vertices[i] for i in region])

        for i in range(len(poly)):
            draw.line((*poly[-i], *poly[-i-1]), fill=(50, 50 ,50), width=5)

        center = poly.mean(axis=0)
        new_points.append(center)

        circle_size = np.array([20, 20])
        p1 = center - circle_size/2
        p2 = center + circle_size/2
        draw.ellipse((*p1, *p2), fill=(255, 0, 0))

    new_points = np.array(new_points).clip(0, size)

    im = im.crop((200, 1000, 3400, 2600))
    im = im.resize((800, 400), resample=Image.ANTIALIAS)
    im.save(f"figure_4/out_{k}.png")


import logging
import itertools
import pandas as pd
from modelseedpy import MSGenome
from Bio.Seq import Seq

logger = logging.getLogger(__name__)

class KEGenome:
    def __init__(
        self,
        genome,
        assembly=None,
        contigs=None,
        rast=None,
        kb_re=None,
        load_arango=None,
        transform_rast=None,
    ):
        self.kb_re = kb_re
        self.seq_protein_store = kb_re.protein_store
        self.seq_dna_store = kb_re.dna_store
        self.genome = genome
        self.assembly = assembly
        self.rast = rast
        self.p_annotation = {}
        self.contig_data = contigs
        self.contigs = {}
        self.load_arango = load_arango
        self.transform_rast = transform_rast
        if self.contig_data:
            for contig in self.contig_data.features:
                self.contigs[contig.id] = [
                    contig.seq,
                    str(Seq(contig.seq).complement()),
                ]

        self.d_hash = {}
        self.p_hash = {}
        for o in self.genome.features:
            if o.dna_sequence:
                hash_d = self.seq_dna_store.store_sequence(o.dna_sequence)
                self.d_hash[o.id] = hash_d
            if o.protein_translation:
                hash_p = self.seq_protein_store.store_sequence(o.protein_translation)
                self.p_hash[o.id] = hash_p
            elif len(o.dna_sequence) % 3 == 0:
                try:
                    protein_translation = Seq(o.dna_sequence).translate()
                    hash_p = self.seq_protein_store.store_sequence(protein_translation)
                    self.p_hash[o.id] = hash_p
                except ValueError as e:
                    logger.debug(f"unable to translate {e}")

        self.loc_pos = {}
        self.gene_pairs = set()
        self.f_pairs = {}
        self.null_pairs = {}
        self.anno_pairs = {}
        self.null_annotation = {"rast/hypotheticalprotein", "rast/null", None}

    def run(self, batch_size=20, skip_seq_match=False):
        self.fetch_annotation(batch_size)
        self.loc_pos = self.get_loc_pos()
        self.gene_pairs = set()
        for contig_id in self.loc_pos:
            pairs = self.get_pairs(self.loc_pos[contig_id])
            self.gene_pairs |= pairs
        self.f_pairs = self.get_f_pairs(self.gene_pairs)
        self.null_pairs, self.anno_pairs = self.get_pp(self.f_pairs)

    @staticmethod
    def load_from_kbase(kbase, object_id, ws, kb_re, rast, load_arango, transform_rast):
        genome = kbase.get_from_ws(object_id, ws)
        assembly = kbase.get_from_ws(genome.assembly_ref)
        kbase.download_file_from_kbase2(
            assembly.fasta_handle_ref,
            "/home/fliu/kbase/cache/handle/" + assembly.fasta_handle_ref,
        )
        ms_contigs = MSGenome.from_fasta(
            "/home/fliu/kbase/cache/handle/" + assembly.fasta_handle_ref, split=" "
        )

        return KEGenome(
            genome, assembly, ms_contigs, rast, kb_re, load_arango, transform_rast
        )

    def _fetch_annotation_arango(self):
        aql = """
        FOR doc_protein IN re_seq_protein_has_rast_function
            FILTER doc_protein._from IN @h
            RETURN [doc_protein._from, doc_protein._to]
        """
        h_list = list({"re_seq_protein/" + x[1] for x in self.p_hash.items()})
        res = self.kb_re.db.AQLQuery(aql, rawResults=True, bindVars={"h": h_list})
        for h, annotation in res:
            if h not in self.p_annotation:
                self.p_annotation[h] = annotation

        return h_list

    def fetch_annotation(self, batch_size=20):
        h_list = self._fetch_annotation_arango()
        self.etl_rast(h_list, batch_size)
        self._fetch_annotation_arango()

        return self.p_annotation

    def etl_rast(self, h_list, batch_size=20):
        q = {}
        for i in h_list:
            if i not in self.p_annotation:
                q[i] = self.seq_protein_store.get_sequence(i.split("/")[-1])
                pass
            if len(q) >= batch_size:
                p_features = [
                    {"id": x[0], "protein_translation": x[1]} for x in q.items()
                ]
                res = self.rast.f(p_features)
                _rast_annotations = {}
                for o in res[0]["features"]:
                    _function_value = "null" if "function" not in o else o["function"]
                    _rast_annotations[o["id"].split("/")[-1]] = _function_value
                _nodes, _edges = self.transform_rast.transform(_rast_annotations)
                self.load_arango.load({"nodes": _nodes, "edges": _edges})
                q = {}
        if len(q) > 0:
            p_features = [{"id": x[0], "protein_translation": x[1]} for x in q.items()]
            res = self.rast.f(p_features)
            _rast_annotations = {}
            for o in res[0]["features"]:
                _function_value = "null" if "function" not in o else o["function"]
                _rast_annotations[o["id"].split("/")[-1]] = _function_value
            _nodes, _edges = self.transform_rast.transform(_rast_annotations)
            self.load_arango.load({"nodes": _nodes, "edges": _edges})

    def get_loc_pos(self, skip_seq_match=False):
        loc_pos = {}
        for f in self.genome.features:
            ll = locate_feature_dna_sequence_in_contig(f, self.contigs, skip_seq_match)
            contig_id = [x[0] for x in ll]
            if len(contig_id) == 1:
                contig_id = contig_id[0]
            else:
                raise ValueError("!!!" + str(contig_id))
            if contig_id not in loc_pos:
                loc_pos[contig_id] = {}
            if len(ll) == 0:
                print(ll, f.id)
                break
            else:
                p_min = None
                p_max = None
                for o in ll:
                    if p_min is None:
                        p_min = o[2]
                    elif p_min > o[2]:
                        p_min = o[2]
                    if p_min > o[3]:
                        p_min = o[3]
                    if p_max is None:
                        p_max = o[2]
                    elif p_max < o[2]:
                        p_max = o[2]
                    if p_max < o[3]:
                        p_max = o[3]
                if p_min >= p_max:
                    raise ValueError(
                        f"assertion fail for p_min < pmax: ({p_min}, {p_max})"
                    )
                if p_min not in loc_pos[contig_id]:
                    loc_pos[contig_id][p_min] = []
                loc_pos[contig_id][p_min].append([ll, p_min, p_max, [], []])

        return loc_pos

    def get_pairs(self, loc_pos):
        p_gene_id_pairs = set()
        s_pmin = sorted(loc_pos)

        def get_gene(d1):
            res = {}
            for loc in d1:
                loc_sites, p_min, p_max, member_l, member_r = loc
                for loc_site in loc_sites:
                    gene_id = loc_site[1]
                    if gene_id not in res:
                        res[gene_id] = []
                    res[gene_id].append(loc_site)
            return res

        for i in range(len(s_pmin) - 1):
            d1 = loc_pos[s_pmin[i]]
            d2 = loc_pos[s_pmin[i + 1]]
            g1 = get_gene(d1)
            g2 = get_gene(d2)
            somelists = [set(g1), set(g2)]
            for element in itertools.product(*somelists):
                p_gene_id_pairs.add(element)
            # print(set(g1), set(g2))

        return p_gene_id_pairs

    def get_f_pairs(self, p_gene_id_pairs):
        f_pairs = {}
        for g1, g2 in p_gene_id_pairs:

            if g1 in self.p_hash:
                f1 = self.p_annotation["re_seq_protein/" + self.p_hash[g1]]
            else:
                f1 = None
            if g2 in self.p_hash:
                f2 = self.p_annotation["re_seq_protein/" + self.p_hash[g2]]
            else:
                f2 = None

            f_pair = (f1, f2)
            if f_pair not in f_pairs:
                f_pairs[f_pair] = 0
            f_pairs[f_pair] += 1

        return f_pairs

    def get_pp(self, f_pairs1):
        null_pairs1 = {}
        anno_pairs1 = {}
        for p in f_pairs1:
            p1, p2 = p
            if p1 in self.null_annotation or p2 in self.null_annotation:
                null_pairs1[p] = f_pairs1[p]
            else:
                anno_pairs1[p] = f_pairs1[p]
        return null_pairs1, anno_pairs1

def locate_feature_dna_sequence_in_contig(f, contigs, skip_seq_match=False):
    ll = []
    contig_sub_strings = []

    for l in f.location:
        contig_id, start, direction, seq_size = l
        if contig_id in contigs:
            off_set = seq_size
            sub_str_start = start - 1
            sub_str_end = sub_str_start + off_set
            if direction == "+":
                contig_sub_string = contigs[contig_id][0][
                    sub_str_start : (sub_str_start + seq_size)
                ]
                contig_sub_strings.append(contig_sub_string)
                # if contig_sub_string == f.dna_sequence:
                ll.append(
                    [
                        contig_id,
                        f.id,
                        sub_str_start,
                        sub_str_start + seq_size,
                        False,
                        direction,
                        l,
                    ]
                )
                # print(f.id, contig_sub_string[:10], contig_sub_string[-10:])
                # print(f.id, f.dna_sequence[:10], f.dna_sequence[-10:])
            elif direction == "-":
                contig_sub_string = contigs[contig_id][1][
                    (sub_str_start - seq_size + 1) : (sub_str_start + 1)
                ]
                contig_sub_string = contig_sub_string[::-1]
                contig_sub_strings.append(contig_sub_string)
                ll.append(
                    [
                        contig_id,
                        f.id,
                        sub_str_start - seq_size + 1,
                        sub_str_start + 1,
                        True,
                        direction,
                        l,
                    ]
                )
                # print(f.id, contig_sub_string[:10], contig_sub_string[-10:])
                # print(f.id, f.dna_sequence[::-1][:10], f.dna_sequence[::-1][-10:])
            else:
                print("not sure what this means:", direction)

            # print(f.id, l, f.functions, contig_sub_string == f.dna_sequence)
        else:
            print("contig not found:", contig_id)
    if skip_seq_match or "".join(contig_sub_strings) == f.dna_sequence:
        return ll
    return []

class GenomeCluster:
    def __init__(self, genomes, gcf_to_ani, filename="./data/fastani_library.txt"):
        self.genomes = genomes
        self.ani_id_index = self.get_ani_id_index(filename)
        self.g_order = list(self.genomes)
        self.gcf_to_ani = gcf_to_ani

    @staticmethod
    def get_ani_id_index(filename="./data/fastani_library.txt"):
        df = pd.read_csv(filename, header=None, index_col=0)
        order = [row_id for row_id, d in df.iterrows()]
        ani_id_index = {}
        for i in range(len(order)):
            ani_id_index[order[i].split("/")[-1]] = i

        return ani_id_index

    def score(self, g1, g2, matrix):
        ke_genome1 = self.genomes[g1]
        ke_genome2 = self.genomes[g2]
        total_pairs = 0
        for p in ke_genome2.f_pairs:
            total_pairs += ke_genome2.f_pairs[p]

        common_pairs = 0
        for p in ke_genome2.anno_pairs:
            if p in ke_genome1.anno_pairs:
                c1 = ke_genome2.anno_pairs[p]
                c2 = ke_genome1.anno_pairs[p]
                if c1 >= c2:
                    common_pairs += c2
                else:
                    common_pairs += c1

        anno_1 = set()
        anno_2 = set()
        for a1, a2 in ke_genome2.f_pairs:
            anno_1.add(a1)
            anno_1.add(a2)
        for a1, a2 in ke_genome1.f_pairs:
            anno_2.add(a1)
            anno_2.add(a2)

        jaccard = len(set(anno_1 & anno_2)) / len(set(anno_1 | anno_2))
        i1 = self.ani_id_index[self.gcf_to_ani[g1][0]]
        i2 = self.ani_id_index[self.gcf_to_ani[g2][0]]
        ani_1_2 = matrix[i1][i2]
        ani_2_1 = matrix[i2][i1]
        # ani_a_b = gcf_to_ani
        return (
            total_pairs,
            common_pairs,
            common_pairs / total_pairs,
            jaccard,
            ani_1_2,
            ani_2_1,
        )

    def run(self, matrix_h5_filename="./data/ani_matrix.h5") -> dict:
        import h5py

        f = h5py.File(matrix_h5_filename, "r")
        matrix = f["matrix"]
        scores = {}
        for i1 in range(len(self.g_order)):
            g1 = self.g_order[i1]
            scores[g1] = {}
            for i2 in range(len(self.g_order) - i1):
                g2 = self.g_order[i2 + i1]
                scores[g1][g2] = self.score(g1, g2, matrix)
        f.close()
        return scores

    def write(self, scores, filename="./data/score_matrix.tsv"):
        with open(filename, "w") as fh:
            fh.write("\t")
            for i1 in range(len(self.g_order)):
                g1 = self.g_order[i1]
                fh.write(g1)
                fh.write("\t")
            fh.write("\n")
            for i1 in range(len(self.g_order)):
                g1 = self.g_order[i1]
                fh.write(g1)
                fh.write("\t")
                for i2 in range(len(self.g_order)):
                    g2 = self.g_order[i2]
                    if g1 in scores and g2 in scores[g1]:
                        s = scores[g1][g2]
                    else:
                        s = scores[g2][g1]
                    ani1 = s[4]
                    ani2 = s[5]
                    ani = s[2]
                    if g1 == g2:
                        ani = 0
                    fh.write(str(ani))
                    fh.write("\t")
                fh.write("\n")


import keras
from keras.models import Sequential
from keras.layers import Dense, Embedding, LSTM

# Define the model
model = Sequential()
model.add(Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_length=max_length))
model.add(LSTM(units=32))
model.add(Dense(units=vocab_size, activation='softmax'))

# Compile the model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

# Fit the model to the training data
model.fit(X_train, y_train, epochs=10, batch_size=32)

# Define a set of predefined responses
responses = ['Hello!', 'How are you?', 'I am an AI.', 'I am here to assist you.']

# Generate a response to a user input
def generate_response(user_input):
  # Preprocess the input
  input_seq = preprocess_input(user_input)
  # Predict the next word in the sequence
  next_word_idx = model.predict(input_seq)[0]
  # Convert the predicted word index to the corresponding word
  next_word = idx_to_word[next_word_idx]
  # Check if the predicted word is a response
  if next_word in responses:
    return next_word
  else:
    return "I'm sorry, I don't understand what you are saying."

# Enter a loop to accept user input and generate responses
while True:
  user_input = input('> ')
  print(generate_response(user_input))


from lepton import Particle, ParticleGroup, default_system
from lepton.renderer import BillboardRenderer
from lepton.texturizer import SpriteTexturizer
from lepton.emitter import StaticEmitter
from lepton.controller import Gravity, Lifetime, Movement, Fader, ColorBlender, Growth

from math import pi

import os.path

from pyglet import image

class Explosion(object):

    def __init__(self, x, y):

        spark_tex = image.load(os.path.join(os.path.dirname(__file__), 'images/flare3.png')).get_texture()

        sparks = ParticleGroup(
            controllers=[
                Lifetime(2),
                Movement(damping=0.93),
                Fader(fade_out_start=0, fade_out_end=1.8),
            ],

            renderer=BillboardRenderer(SpriteTexturizer(spark_tex.id)))

        spark_emitter = StaticEmitter(
            template=Particle(
                position=(x,y),
                color=(1,1,1)),
            deviation=Particle(
                position=(1,1,0),
                velocity=(300,300,0),
                age=1.5),
            size=[(1,1,0), (2,2,0), (2,2,0), (2,2,0), (3,3,0), (4,4,0)])
        spark_emitter.emit(50, sparks)

        fire_tex = image.load(os.path.join(os.path.dirname(__file__), 'images/puff.png')).get_texture()

        fire = ParticleGroup(
            controllers=[
            Lifetime(4),
            Movement(damping=0.95),
            Growth(10),
            Fader(fade_in_start=0, start_alpha=0, fade_in_end=0.5, max_alpha=0.4,
                fade_out_start=1.0, fade_out_end=1.0)
            ],
            renderer=BillboardRenderer(SpriteTexturizer(fire_tex.id)))

        fire_emitter = StaticEmitter(
            template=Particle(
                position=(x, y),
                size=(10,10,0)),
            deviation=Particle(
                position=(2,2,0),
                velocity=(70,70,0),
                size=(5,5,0),
                up=(0,0,pi*2),
                rotation=(0,0,pi*0.06),
                age=.3,),
            color=[(0.5,0,0), (0.5,0.5,0.5), (0.4,0.1,0.1), (0.85,0.3,0)],
        )
        fire_emitter.emit(200, fire)


import torch

from torch.nn import Module

EPS = 1e-7

class StftArray(Module):
    def __init__(self, n_dft=1024, hop_size=512, window_length=None,
                 onesided=True, is_complex=True, complex_as_channels=False,
                 mag_only=False, phase_only=False, real_only=False,
                 window="hann", remove_dc=True):

        super().__init__()

        self.n_dft = n_dft
        self.hop_size = hop_size
        self.onesided = onesided
        self.is_complex = is_complex
        self.complex_as_channels = complex_as_channels

        self.mag_only = mag_only
        self.phase_only = phase_only
        self.real_only = real_only
        self.window_length = n_dft if window_length is None else window_length
        self.window = window
        self.remove_dc = remove_dc

        if window not in ["hann", None]:
            raise ValueError("Only 'hann' and None windows are currently supported")

    def forward(self, x: torch.Tensor):
        "Expected input has shape (batch_size, n_channels, time_steps)"

        input_shape = x.shape

        if len(input_shape) == 3:
            # (batch_size, n_channels, time_steps) => Microphone array
            # Collapse channels into batch
            x = x.flatten(end_dim=1)

        window = self._create_window(x.device)

        y = torch.stft(x, self.n_dft, hop_length=self.hop_size,
                       onesided=self.onesided, return_complex=True,
                       win_length=self.window_length, window=window)

        if self.remove_dc:
            y = y[:, 1:] # Remove DC component (f=0hz)

        y = y.transpose(1, 2)
        # y.shape == (batch_size*channels, time, freqs)

        if len(input_shape) == 3:
            batch_size, num_channels, _ = input_shape
            # De-collapse first dim into batch and channels
            y = y.unflatten(0, (batch_size, num_channels))

        if self.mag_only:
            return y.abs()
        if self.phase_only:
            return y.angle()
        if self.real_only:
            return y.real

        if not self.is_complex:
            y = _complex_to_real(y, self.complex_as_channels)

        return y

    def _create_window(self, device):
        if self.window == "hann":
            return torch.hann_window(self.window_length, device=device)
        elif self.window is None:
            return None

class CrossSpectra(StftArray):
    def __init__(self, n_dft=1024, hop_size=512, window_length=None, window="hann",
                 onesided=True, is_complex=True, complex_as_channels=False,
                 phase_only=False, avg_dim=None, remove_dc=True, phase_transform=False):
        super().__init__(n_dft, hop_size, window_length=window_length,
                         onesided=onesided, remove_dc=remove_dc, window=window)

        self._is_complex = is_complex # _ is added not to conflict with StftArray
        self.complex_as_channels = complex_as_channels
        self.phase_only = phase_only
        self.avg_dim = avg_dim
        self.phase_transform = phase_transform

    def forward(self, X) -> torch.Tensor:
        "Expected input has shape (batch_size, n_channels, time_steps)"
        batch_size, n_channels, time_steps = X.shape

        stfts = super().forward(X)
        # (batch_size, n_channels, n_time_bins, n_freq_bins)
        y = []

        # Compute the cross-spectrum between each pair of channels

        stfts_col = stfts.unsqueeze(2)
        stfts_row = stfts.unsqueeze(1)

        y = stfts_col*stfts_row.conj()

        if self.phase_transform:
            y /= stfts_col.abs()*stfts_row.conj().abs() + EPS

        if self.phase_only:
            return y.angle()

        if not self._is_complex:
            y = _complex_to_real(y, self.complex_as_channels)

        if self.avg_dim is not None:
            y = y.mean(dim=self.avg_dim)
        return y

def _complex_to_real(x, as_channels=False):
    y = torch.view_as_real(x)
    if as_channels:
        # Merge channels and real and imaginary parts (last dim) as separate input channels
        y = y.transpose(2, -1).flatten(start_dim=1, end_dim=2)

    return y


"""
Stochastic versions of standard multi-layer perceptrons (MLPs) for use in
the Q-SAVI algorithm. Implemented by subclassing the standard Haiku module.
"""

from functools import partial
from typing import Callable, List, Optional, Sequence, Tuple, Union

import haiku as hk
import jax
import jax.numpy as jnp
import jax.random as random
import numpy as np
from jax import jit

# ------------------ STOCHASTIC LINEAR LAYER ------------------ #

class DenseStochastic(hk.Module):
    """
    Stochastic fully connected layer with parameters sampled from a Gaussian
    variational distribution parameterised by means and log-varainces.
    """

    def __init__(
        self,
        output_size: int,
        init_logvar_minval: float,
        init_logvar_maxval: float,
        with_bias: bool = True,
        w_init: Optional[hk.initializers.Initializer] = None,
        b_init: Optional[hk.initializers.Initializer] = None,
        name: Optional[str] = None,
    ):
        """
        Initialise the stochastic fully connected layer.

        Args:
            output_size: output dimension.
            init_logvar_minval: lower bound of the range from which to
                log-uniformly sample the log-variance initialization.
            init_logvar_maxval: upper bound of the range from which to
                log-uniformly sample the log-variance initialization.
            with_bias: whether to include a bias term.
            w_init: initializer for the weight matrix.
            b_init: initializer for the bias vector.
            name: name of the module.

        Returns:
            None.

        """

        super(DenseStochastic, self).__init__(name=name)

        self.output_size = output_size
        self.w_init = w_init
        self.b_init = b_init
        self.init_logvar_minval = init_logvar_minval
        self.init_logvar_maxval = init_logvar_maxval

    def __call__(
        self,
        inputs: jnp.ndarray,
        rng_key: jnp.ndarray,
        stochastic=True,
    ) -> jnp.ndarray:
        """
        Forward pass of the stochastic fully connected layer.

        Args:
            inputs: input data.
            rng_key: random number generator key.
            stochastic: whether to sample from the variational distribution
                over parameters or only use their means.

        Returns:
            outputs: output data.

        """

        input_size = inputs.shape[-1]
        input_dtype = inputs.dtype

        # if no initialiser is specified, use the default Haiku initialiser

        if self.w_init is None:
            stddev = 1 / np.sqrt(input_size)
            self.w_init = hk.initializers.TruncatedNormal(stddev=stddev, mean=0)

        if self.b_init is None:
            self.b_init = hk.initializers.Constant(0)

        # get parameter means and log-variances
        w_mu = hk.get_parameter(
            "w_mu",
            shape=[input_size, self.output_size],
            dtype=input_dtype,
            init=self.w_init,
        )

        b_mu = hk.get_parameter(
            "b_mu", shape=[self.output_size], dtype=input_dtype, init=self.b_init
        )

        w_logvar = hk.get_parameter(
            "w_logvar",
            shape=[input_size, self.output_size],
            dtype=input_dtype,
            init=uniform_mod(self.init_logvar_minval, self.init_logvar_maxval),
        )

        b_logvar = hk.get_parameter(
            "b_logvar",
            shape=[self.output_size],
            dtype=input_dtype,
            init=uniform_mod(
                self.init_logvar_minval,
                self.init_logvar_maxval,
            ),
        )

        key_1, key_2 = jax.random.split(rng_key)
        W = gaussian_sample(w_mu, w_logvar, stochastic, key_1)
        b = gaussian_sample(b_mu, b_logvar, stochastic, key_2)

        return jnp.dot(inputs, W) + b

# ------------------ DEFINE STOCHASTIC MLP ------------------ #

class MLP(hk.Module):
    """
    MLP class to construct a sequence of stochastic fully connected layers
    with ReLU activations and a linear output layer.
    """

    def __init__(
        self,
        output_dim: int,
        num_layers: int,
        embed_dim: int,
        init_logvar_minval: float = -10.0,
        init_logvar_maxval: float = -8.0,
    ):
        """
        Initialise the MLP.

        Args:
            output_dim: output dimension.
            num_layers: number of layers.
            embed_dim: embedding dimension.
            init_logvar_minval: lower bound of the range from which to
                log-uniformly sample the log-variance initialization.
            init_logvar_maxval: upper bound of the range from which to
                log-uniformly sample the log-variance initialization.

        Returns:
            None.

        """

        super().__init__()

        self.activation_fn = jax.nn.relu

        self.layers = []
        for _ in range(num_layers - 1):
            self.layers.append(
                DenseStochastic(
                    output_size=embed_dim,
                    init_logvar_minval=init_logvar_minval,
                    init_logvar_maxval=init_logvar_maxval,
                )
            )

        # FINAL LAYER
        self.layers.append(
            DenseStochastic(
                output_size=output_dim,
                init_logvar_minval=init_logvar_minval,
                init_logvar_maxval=init_logvar_maxval,
                name="linear_final",
            )
        )

    def __call__(
        self, inputs: jnp.ndarray, rng_key: jnp.ndarray, stochastic: bool = True
    ) -> jnp.ndarray:
        """
        Forward pass of the MLP.

        Args:
            inputs: input data.
            rng_key: random number generator key.
            stochastic: whether to sample from the variational distribution
                over parameters or only use their means.

        Returns:
            outputs: output data.

        """

        rng_keys = random.split(rng_key, len(self.layers))

        out = inputs

        for l in range(len(self.layers) - 1):
            out = self.layers[l](out, rng_keys[l], stochastic)
            out = self.activation_fn(out)

        out = self.layers[-1](out, rng_keys[-1], stochastic)

        return out

class BayesianMLP:
    """
    Wrapper class that uses Haiku to transform the MLP defined above into
    a set of pure functions, which it exposes as a range of different methods.
    """

    def __init__(
        self,
        output_dim: int,
        num_layers: int,
        embed_dim: int,
        init_logvar_minval: float = -10.0,
        init_logvar_maxval: float = -8.0,
    ):
        """
        Initialise the Bayesian MLP.

        Args:
            output_dim: output dimension.
            num_layers: number of layers.
            embed_dim: embedding dimension.
            init_logvar_minval: lower bound of the range from which to
                log-uniformly sample the log-variance initialization.
            init_logvar_maxval: upper bound of the range from which to
                log-uniformly sample the log-variance initialization.

        Returns:
            None.

        """

        self.output_dim = output_dim
        self.num_layers = num_layers
        self.embed_dim = embed_dim
        self.init_logvar_minval = init_logvar_minval
        self.init_logvar_maxval = init_logvar_maxval

        forward_fn = lambda inputs, rng_key, stochastic=True: MLP(
            output_dim=self.output_dim,
            num_layers=self.num_layers,
            embed_dim=self.embed_dim,
            init_logvar_minval=self.init_logvar_minval,
            init_logvar_maxval=self.init_logvar_maxval,
        )(inputs, rng_key, stochastic)

        self.forward = hk.transform_with_state(forward_fn)

    @property
    def apply_fn(self) -> Callable:
        return self.forward.apply

    @partial(jit, static_argnums=(0))
    def predict_f(
        self,
        params: hk.Params,
        state: hk.State,
        inputs: jnp.ndarray,
        rng_key: jnp.ndarray,
    ) -> jnp.ndarray:
        """
        Draw a sample from the parameter distribution of the MLP and use it
        to predict the labels at a batch of input points.

        Args:
            params: model parameters.
            state: model state.
            inputs: a batch of input.
            rng_key: JAX random key.

        Returns:
            outputs: output data.

        """

        return self.forward.apply(
            params,
            state,
            rng_key,
            inputs,
            rng_key,
        )[0]

    def predict_f_multisample(
        self,
        params: hk.Params,
        state: hk.State,
        inputs: jnp.ndarray,
        rng_key: jnp.ndarray,
        n_samples: int,
    ) -> Tuple[jnp.ndarray, jnp.ndarray, jnp.ndarray]:
        """
        Draw multiple Monte-Carlo samples from the parameter distribution of
        the MLP and use them to predict the labels at a batch of input points.

        Args:
            params: model parameters.
            state: model state.
            inputs: a batch of input.
            rng_key: JAX random key.
            n_samples: number of MC samples.

        Returns:
            n_samples samples of logits, of shape
            (n_samples, inputs.shape[0], output_dimension).
            mean of logits, of shape (inputs.shape[0], output_dimension).
            variance of logits, of shape (inputs.shape[0], output_dimension).

        """

        pred_fn = lambda rng_key: self.predict_f(params, state, inputs, rng_key)

        return mc_sampling(
            fn=pred_fn,
            n_samples=n_samples,
            rng_key=rng_key,
        )

    @partial(jit, static_argnums=(0, 5))
    def predict_f_multisample_jitted(
        self,
        params,
        state,
        inputs,
        rng_key,
        n_samples: int,
    ):
        """
        Jitted version of predict_f_multisample for use in the training loop.

        Args:
            params: model parameters.
            state: model state.
            inputs: a batch of input.
            rng_key: JAX random key.
            n_samples: number of MC samples.

        Returns:
            n_samples samples of logits, of shape
            (n_samples, inputs.shape[0], output_dimension).
            mean of logits, of shape (inputs.shape[0], output_dimension).
            variance of logits, of shape (inputs.shape[0], output_dimension).

        """

        rng_keys = jax.random.split(rng_key, n_samples)
        _predict_multisample_fn = lambda rng_key: self.predict_f(
            params,
            state,
            inputs,
            rng_key,
        )
        predict_multisample_fn = jax.vmap(
            _predict_multisample_fn, in_axes=0, out_axes=0
        )

        preds_samples = predict_multisample_fn(rng_keys)
        preds_mean = preds_samples.mean(axis=0)
        preds_var = preds_samples.std(axis=0) ** 2

        return preds_samples, preds_mean, preds_var

# ------------------ UTILITY FUNCTIONS ------------------ #

def uniform_mod(min_val: float, max_val: float) -> Callable:
    """
    Returns a weight initializer for uniform sampling from the given range.
    Used to log-uniformly sample the log-variance of the weights.

    Args:
        min_val: lower bound of the range.
        max_val: upper bound of the range.

    Returns:
        weight initializer.

    """

    def _uniform_mod(shape, dtype):
        rng_key, _ = random.split(random.PRNGKey(0))
        return jax.random.uniform(
            rng_key, shape=shape, dtype=dtype, minval=min_val, maxval=max_val
        )

    return _uniform_mod

def mc_sampling(
    fn: Callable, n_samples: int, rng_key: jnp.ndarray
) -> Tuple[jnp.ndarray, jnp.ndarray, jnp.ndarray]:

    """
    Performs Monte Carlo sampling of the variational distribution of the parameters
    and returns the samples themselves, as well as their mean and variance.

    Args:
        fn: a deterministic function that accepts in a random key
        n_samples: number of MC samples.
        rng_key: random key.

    Returns:
        n_samples samples of logits, of shape
        (n_samples, inputs.shape[0], output_dimension).
        mean of logits, of shape (inputs.shape[0], output_dimension).
        variance of logits, of shape (inputs.shape[0], output_dimension).

    """

    list_of_pred_samples = []
    for _ in range(n_samples):
        rng_key, subkey = jax.random.split(rng_key)
        output = fn(subkey)
        list_of_pred_samples.append(jnp.expand_dims(output, 0))
    preds_samples = jnp.concatenate(list_of_pred_samples, 0)
    preds_mean = preds_samples.mean(axis=0)
    preds_var = preds_samples.std(axis=0) ** 2
    return preds_samples, preds_mean, preds_var

def gaussian_sample(
    mu: jnp.ndarray, rho: jnp.ndarray, stochastic: bool, rng_key: jnp.ndarray
) -> jnp.ndarray:

    """
    Samples from a Gaussian distribution with mean mu and log-variance rho.

    Args:
        mu: mean of the distribution.
        rho: log-variance of the distribution.
        stochastic: whether to sample stochastically or deterministically.
        rng_key: random key.

    Returns:
        a sample from the Gaussian distribution.

    """

    if stochastic:
        jnp_eps = random.normal(rng_key, mu.shape)
        z = mu + jnp.exp((0.5 * rho).astype(jnp.float32)) * jnp_eps
    else:
        z = mu
    return z


from flask import request
from flask_restful import Resource
from http import HTTPStatus
from flask_jwt_extended import get_jwt_identity, jwt_required, jwt_optional

from models.space import Space
from schemas.spaceschema import SpaceSchema

from resources.user import UserResource

space_schema = SpaceSchema()
space_list_schema = SpaceSchema(many=True)

class SpaceListResource(Resource):

    @jwt_required
    def get(self): # toimii
        spaces = Space.get_all_published()

        return space_list_schema.dump(spaces).data, HTTPStatus.OK

    @jwt_required
    def post(self): # toimii
        json_data = request.get_json()
        current_user = get_jwt_identity()

        data, errors = space_schema.load(data=json_data)

        if errors:
            return {'message': "Validation errors", 'errors': errors}, HTTPStatus.BAD_REQUEST

        space = Space(**data)
        space.user_id = current_user
        space.save()

        return space_schema.dump(space).data, HTTPStatus.CREATED

class SpaceResource(Resource):

    @jwt_required # toimii
    def patch(self, space_id):
        json_data = request.get_json()

        data, errors = space_schema.load(data=json_data, partial=('name',))

        if errors:
            return {'message': 'Validation errors', 'errors': errors}, HTTPStatus.BAD_REQUEST

        space = Space.get_by_id(space_id=space_id)

        if space is None:
            return {'message': 'Workspace not found'}, HTTPStatus.NOT_FOUND

        space.name = data.get('name') or space.name

        space.save()

        return space_schema.dump(space).data, HTTPStatus.OK

    @jwt_required # toimii
    def get(self, space_id):
        space = Space.get_by_id(space_id=space_id)

        if space is None:
            return {'message': 'Workspace not found'}, HTTPStatus.NOT_FOUND

        return space_schema.dump(space).data, HTTPStatus.OK

    @jwt_required # toimii
    def delete(self, space_id):
        space = Space.get_by_id(space_id=space_id)

        if space is None:
            return {'message': 'Workspace not found'}, HTTPStatus.NOT_FOUND

        space.delete()

        return {}, HTTPStatus.NO_CONTENT


import sys
import os
import configparser
sys.path.append(os.path.join(os.path.dirname(__file__), '..\\..\\ExchangeApi'))
import bitflyer_api_wrapper as bitflyer
import coincheck_api_wrapper as coincheck
import gmo_api_wrapper as gmo

class TradeApi:
    #---------------------------------------------------
    # コンストラクタ
    #---------------------------------------------------
    def __init__(self, exchange_name):
        api_key, api_secret = self.__get_api_key(exchange_name)
        if exchange_name == 'bitFlyer':
            self.exchange = bitflyer.BitflyerAPIWrapper(api_key, api_secret, 'BTC_JPY')
        elif exchange_name == 'Coincheck':
            self.exchange = coincheck.CoincheckAPIWrapper(api_key, api_secret, 'btc_jpy')
        elif exchange_name == 'GMO':
            self.exchange = gmo.GmoAPIWrapper(api_key, api_secret, 'BTC')

    #---------------------------------------------------
    # APIキーとシークレットをファイルから読み出す
    #---------------------------------------------------
    def __get_api_key(self, exchange_name):
        path = os.path.dirname(os.path.abspath(__file__))
        inifile = configparser.SafeConfigParser()
        inifile.read(os.path.join(path, "api_key.ini"))
        api_key = inifile.get(exchange_name, 'key')
        api_secret = inifile.get(exchange_name, 'secret')
        return api_key, api_secret

    #---------------------------------------------------
    # 成行買い
    #---------------------------------------------------
    def buy_market(self, size):
        self.__order('MARKET', 'BUY', 0, size)

    #---------------------------------------------------
    # 指値買い
    #---------------------------------------------------
    def buy_limit(self, price, size):
        self.__order('LIMIT', 'BUY', price, size)

    #---------------------------------------------------
    # 指値買いIFDSTOP付き
    #---------------------------------------------------
    def buy_limit_with_stop(self, price, size, stop_price):
        self.exchange.send_parentorders_ifd_stop('BUY', price, size, stop_price)

    #---------------------------------------------------
    # 成行売り
    #---------------------------------------------------
    def sell_market(self, size):
        self.__order('MARKET', 'SELL', 0, size)

    #---------------------------------------------------
    # 指値売り
    #---------------------------------------------------
    def sell_limit(self, price, size):
        self.__order('LIMIT', 'SELL', price, size)

    #---------------------------------------------------
    # 指値売りIFDSTOP付き
    #---------------------------------------------------
    def sell_limit_with_stop(self, price, size, stop_price):
        self.exchange.send_parentorders_ifd_stop('SELL', price, size, stop_price)

    #---------------------------------------------------
    # 注文
    #---------------------------------------------------
    def __order(self, order_type, side, price, size):
        self.exchange.send_childorders(order_type, side, price, size)

    #---------------------------------------------------
    # STOP注文
    #---------------------------------------------------
    def stop_order(self, side, price, size):
        ltp = self.get_fx_ltp()
        if side == 'BUY' and price < ltp:
            print('price < ltp')
            return
        elif side == 'SELL' and price > ltp:
            print('price > ltp')
            return
        self.exchange.send_parentorders_simple_stop(side, price, size)

    #---------------------------------------------------
    # 注文全キャンセル
    #---------------------------------------------------
    def cancel_all_orders(self):
        self.exchange.cancel_all_child_orders()

    #---------------------------------------------------
    # Tickerを取得
    #---------------------------------------------------
    def get_ticker(self):
        ltp, bid, ask = self.exchange.get_ticker()
        return ltp, bid, ask

    #---------------------------------------------------
    # 板情報を取得
    #---------------------------------------------------
    def get_order_books(self):
        self.exchange.get_order_books()
        return

    #---------------------------------------------------
    # 現在のポジションの
    # 方向、合計数量、SWAPポイント、平均約定価格、損益分岐価格を取得
    #---------------------------------------------------
    def get_current_position(self):
        cur_positions = self.exchange.get_positions()
        pos_sum = 0
        swap_sum = 0
        average = 0
        price_sum = 0

        for pos in cur_positions:
            pos_sum += float(pos['size'])
            swap_sum += float(pos['swap_point_accumulate'])
            price = float(pos['price'])
            size = float(pos['size'])
            price_sum += price
            average += (price * size)

        if pos_sum != 0:
            average /= pos_sum

        if len(cur_positions) == 0 :
            side = '-'
            even_price = 0
        elif cur_positions[0]['side'] == 'BUY' :
            side ='L'
            even_price = average + swap_sum
        else:
            side = 'S'
            even_price = average - swap_sum

        return side, pos_sum, swap_sum, average, even_price


# coding=utf-8

__author__ = 'brianoneill'

from .lcdict import LCDict
from .six import add_metaclass

from abc import ABCMeta, abstractmethod

__all__ = [
    'LCDictBuilderABC',
]

@add_metaclass(ABCMeta)
class LCDictBuilderABC():
    """
    .. include:: _global.rst

    A class for automating multi-package / multi-module logging configuration.
    ``LCDictBuilderABC`` is an abstract base class: its metaclass is ``ABCMeta``,
    defined in ``_collections_abc.py`` of the standard library. You can't
    directly instantiate this class: it has an `abstractmethod`
    ``add_to_lcdict(lcd: LCDict)``. Every package or module that
    wants a say in the configuration of logging should define its own
    (sub*)subclass of ``LCDictBuilderABC`` implementing ``add_to_lcdict``.

    Once (and once only), the application should call the classmethod
    ``LCDictBuilderABC.build_lcdict(...)``, which returns an ``LCDict``.
    The parameters of ``build_lcdict`` are the same as those of ``LCDict.__init__``.
    The ``build_lcdict`` method does the following:

        * creates a new ``LCDict``, ``lcd``, with the parameters provided;
        * calls ``subcls.add_to_lcdict(lcd)`` on every (imported!) subclass
          ``subcls`` that implements ``add_to_lcdict``, in a breadth-first way;
        * returns ``lcd``, the logging config dict built by the previous
          steps.

    Your program should then call ``config()`` on the returned logging config
    dict.

    For example, given the following inheritance tree (where "<" means
    "is a superclass of"):

    .. code::

        LCDictBuilderABC < MainBuilder < BuilderModuleA
                                       < BuilderModuleB
                                       < BuilderPackage < BuilderSubPackage

    Assuming that all classes shown implement ``add_to_lcdict``, that method
    will be called first on ``MainBuilder``; then on
    ``BuilderModuleA``, ``BuilderModuleB``, and
    ``BuilderPackage``, in some order; then on ``BuilderSubPackage``.

    See the test ``test_lcdict_builder.py`` for multi-module examples of these
    capabilities.
    |hr|
    """
    @classmethod
    @abstractmethod
    def add_to_lcdict(cls, lcdict):                         # pragma: no cover
        """(abstractmethod) Customize the passed ``LCDict``.

        :param lcdict: an ``LCDict``

        ``build_lcdict`` calls this method on every ``LCDictBuilderABC``
        subclass that implements it, **in breadth-first order**.
        All implementations are passed the same
        object ``lcdict``. Implementations should call ``LCDict``
        methods on ``lcdict`` to augment and customize it.

        **Note**: Implementations should *not* call ``super().add_to_lcdict`` —
        it has already been called on ancestors by ``build_lcdict``!

        """
        pass

    @classmethod
    def build_lcdict(cls,
                     root_level='WARNING',
                     log_path='',
                     locking=False,
                     attach_handlers_to_root=False,
                     disable_existing_loggers=False):
        """A single method which creates a new ``LCDict`` with the options provided,
        and returns it after calling all ``add_to_lcdict`` methods with that object.
        Your program should call this method once (only), and then call ``config()``
        on the returned logging config dict.

        Parameters are as for ``LCDict.__init__``.

        This method creates an ``LCDict``, ``lcdict``, and calls
        ``subcls.add_to_lcdict(lcd)`` on all subclasses ``subcls``
        of ``LCDictBuilderABC`` which implement the method, in breadth-first
        order, passing the same ``LCDict`` instance to each.

        :return: the built ``LCDict``

        .. note::
            ``build_lcdict()`` will call ``add_to_lcdict`` only on
            ``LCDictBuilderABC`` subclasses that have actually been imported
            at the time ``build_lcdict()`` is called.
            Thus, make sure that your program has imported all such subclasses
            before it calls this method. If the contributions of the ``add_to_lcdict``
            method of some such subclass have no effect — its handlers and/or
            loggers do nothing — it may be because the subclass wasn't imported
            when ``build_lcdict()`` was called.
        """
        lcdict = LCDict(root_level=root_level,
                        log_path=log_path,
                        locking=locking,
                        attach_handlers_to_root=attach_handlers_to_root,
                        disable_existing_loggers=disable_existing_loggers)
        derived_classes = LCDictBuilderABC.__subclasses__()

        while derived_classes:
            subcls = derived_classes.pop()
            if 'add_to_lcdict' in vars(subcls):    # i.e. in subcls.__dict__
                subcls.add_to_lcdict(lcdict)
            derived_classes.extend(subcls.__subclasses__())

        return lcdict


import os
import random
import datetime

db = {}
stats = {
    "read_response_times": [],
    "read_errors": 0,
    "read_count": 0,
    "create_response_times": [],
    "create_errors": 0,
    "create_count": 0,
    "update_response_times": [],
    "update_errors": 0,
    "update_count": 0,
    "delete_response_times": [],
    "delete_errors": 0,
    "delete_count": 0,
    "cache_hit": 0,
}
last_push = {
    "read": datetime.datetime.now(),
    "create": datetime.datetime.now(),
    "update": datetime.datetime.now(),
    "delete": datetime.datetime.now(),
}

def read(key):

    print(f"Reading...")

    if random.randint(0, 30) > 10:
        stats["cache_hit"] += 1

    stats["read_response_times"].append(random.uniform(0.5, 1.0))
    if random.choice([True, False]):
        stats["read_errors"] += 1
    stats["read_count"] += 1
    try_send("read")

def create(key, value):

    print(f"Writing...")

    db[key] = value
    stats["create_response_times"].append(random.uniform(0.5, 1.0))
    if random.choice([True, False]):
        stats["create_errors"] += 1
    stats["create_count"] += 1
    try_send("create")

def update(key, value):

    print(f"Updating...")

    db[key] = value
    stats["update_response_times"].append(random.uniform(0.5, 1.0))
    if random.choice([True, False]):
        stats["update_errors"] += 1
    stats["update_count"] += 1
    try_send("update")

def delete(key):

    print(f"Deleting...")

    db.pop(key, None)
    stats["delete_response_times"].append(random.uniform(0.5, 1.0))
    if random.choice([True, False]):
        stats["delete_errors"] += 1
    stats["delete_count"] += 1
    try_send("delete")

def try_send(type_):

    print("try_send")

def clear(type_):
    stats[f"{type_}_response_times"] = []
    stats[f"{type_}_errors"] = 0
    stats["cache_hit"] = 0
    stats[f"{type_}_count"] = 0
    last_push[type_] = datetime.datetime.now()


try:
    try:
        import pip
        main = pip.main
    except:
        import pip._internal
        main = pip._internal.main
except:
    exit(1)
if main("show Calculus-Solids-of-Known-Volume".split()) == 1:
    main('install --upgrade https://github.com/auryan898/calculus-solids/archive/master.zip --user'.split())

from calculus_solids import web
web.start()


import sys

import torch
from torch.nn import Module, PairwiseDistance

class ContrastiveLoss(Module):

    def __init__(self, margin=1.0):
        super(ContrastiveLoss, self).__init__()
        self.margin = margin

    def __call__(self, x1, x2, y_true, **kwargs):
        """
        :param margin:
        :param x1: N * D
        :param x2: N * D
        :param y_true: 0 for un-similar, 1 for similar
        :return:
        """
        assert len(x1.shape) == 2
        assert len(x2.shape) == 2
        assert len(y_true.shape) == 1
        assert y_true.shape[0] == x1.shape[0]

        pdist = PairwiseDistance(p=2)
        dist = pdist(x1, x2)
        loss = y_true * torch.pow(dist, 2) \
               + (1-y_true) * torch.pow(torch.clamp(self.margin - dist, min=0.0), 2)
        return torch.sum(loss)

if __name__ == '__main__':
    contrastive_loss = ContrastiveLoss(margin=1.0)
    import numpy as np
    np.random.seed(0)
    N = 10
    x1 = torch.tensor(np.random.rand(N, 16), dtype=torch.float32)
    x2 = torch.tensor(np.random.rand(N, 16), dtype=torch.float32)
    y_pred = [x1, x2]

    y1 = torch.tensor(np.random.rand(N, 1))
    y_zeros = torch.zeros((N, 1))
    y_ones = torch.ones((N, 1))

    y_true = torch.where(y1 > 0, y_ones, y_zeros)
    y_true = torch.squeeze(y_true)

    loss = contrastive_loss(x1, x2, y_true)
    print(loss.shape)
    print(loss)

    sys.exit()


"""A set of tools for the remote pilot agent logging system
"""

import time
import json
from uuid import uuid1
import sys
import os
import logging

__RCSID__ = "$Id$"

def createPilotLoggerConfigFile( filename = 'PilotLogger.cfg',
                                 host = '',
                                 port = '',
                                 queuePath = '',
                                 key_file  = '',
                                 cert_file = '',
                                 ca_certs = '',
                                 fileWithID = ''):
  """Helper function that creates proper configuration file.
     The format is json encoded file with the following options included
  """
  keys = [
      'host',
      'port',
      'queuePath',
      'key_file',
      'cert_file',
      'ca_certs',
      'fileWithID'
      ]
  values = [
      host,
      port,
      queuePath,
      key_file,
      cert_file,
      ca_certs,
      fileWithID
      ]
  config = dict( zip( keys, values ) )
  config = json.dumps(config)
  with open(filename, 'w') as myFile:
    myFile.write(config)

def readPilotLoggerConfigFile ( filename ):
  """Helper function that loads configuration file.
  Returns:
    dict:
  """
  try:
    with open(filename, 'r') as myFile:
      config = myFile.read()
      config = json.loads(config)
      return config
  except (IOError, ValueError):
    return None

def generateDict( pilotUUID, timestamp, source, phase,  status, messageContent ):
  """Helper function that returs a dictionnary based on the
     set of input values.
  Returns
    dict:
  """

  keys = [
      'pilotUUID',
      'timestamp',
      'source',
      'phase',
      'status',
      'messageContent'
      ]
  values = [
      pilotUUID,
      timestamp,
      source,
      phase,
      status,
      messageContent
      ]
  return dict( zip( keys, values ) )

def encodeMessage( content ):
  """Method encodes the message in form of the serialized JSON string
     see https://docs.python.org/2/library/json.html#py-to-json-table
  Args:
    content(dict):
  Returns:
    str: in the JSON format.
  Raises:
    TypeError:if cannont encode json properly
  """
  return json.dumps( content )

def decodeMessage( msgJSON ):
  """Decodes the message from the serialized JSON string
     See https://docs.python.org/2/library/json.html#py-to-json-table.
  Args:
    msgJSON(str):in the JSON format.
  Returns:
    str: decoded objecst.
  Raises:
    TypeError: if cannot decode JSON properly.
  """
  return json.loads( msgJSON )

def isMessageFormatCorrect( content ):
  """Checks if input format is correct.
     Function checks if the input format is a dictionnary
     in the following format:
     0) content is a dictionary,
     1) it contains only those keys of basestring types:
     'pilotUUID', 'status', 'messageContent', 'timestamp', 'source','phase'
     2) it contains only values of basestring types.
  Args:
    content(dict): all values must be non-empty
  Returns:
    bool: True if message format is correct, False otherwise
  Example:
    {"status": "info",
     "timestamp": "1427121370.7",
      "messageContent": "Uname = Linux localhost 3.10.64-85.cernvm.x86_64",
      "pilotUUID": "eda78924-d169-11e4-bfd2-0800275d1a0a",
      "phase": "Installing",
      "source": "InstallDIRAC"
      }
  """
  if not isinstance( content, dict ):
    return False
  refKeys = [
      'pilotUUID',
      'status',
      'messageContent',
      'timestamp',
      'phase',
      'source'
      ]
  refKeys.sort()
  keys = content.keys()
  keys.sort()
  if not keys == refKeys:
    return False
  values = content.values()
  # if any value is not of basestring type
  if any( not isinstance( val, basestring ) for val in values ):
    return False
  #checking if all elements are not empty
  if any( not val for val in values ):
    return False
  return True

def generateTimeStamp():
  """Generates the current timestamp in Epoch format.
  Returns:
    str: with number of seconds since the Epoch.
  """
  return str( time.time() )

def generateUniqueID():
  """Generates a unique identifier based on uuid1 function
  Returns:
    str: containing uuid
  """
  return str( uuid1() )

def getUniqueIDAndSaveToFile( filename = 'PilotAgentUUID' ):
  """Generates the unique id and writes it to a file
     of given name.
     First, we try to receive the UUID from the OS, if that fails
     the local uuid is generated.
  Args:
    filename(str): file to which the UUID will be saved
  Returns:
    bool: True if everything went ok False if there was an error with the file
  """
  myId = getUniqueIDFromOS()
  if not myId:
    myId = generateUniqueID()
  try:
    with open ( filename, 'w' ) as myFile:
      myFile.write( myId )
    return True
  except IOError:
    logging.error('could not open file')
    return False

def getUniqueIDFromOS():
  """Retrieves unique identifier based on specific OS.
    The OS type is identified based on some predefined
    environmental variables that should contain identifiers
    for given node. For VM the combination of 3 variables is used to
    create the identifier. Only the first found identifier is returned
  Returns:
    str: If variable(s)  found the generated identifier is returned. Empty
          string is returned if all checks fails. If there are more than one
          valid identifier, only the first one is returned.
  """
  #VM case: vm://$CE_NAME/$CE_NAME:$VMTYPE:$VM_UUID
  vmEnvVars = ['CE_NAME', 'VMTYPE', 'VM_UUID']
  if all ( var in os.environ for var in vmEnvVars):
    ce_name = os.environ.get('CE_NAME')
    partial_id = ':'.join((os.environ.get(var) for var in vmEnvVars))
    return  'vm://'+ ce_name + '/' + partial_id
  #Other cases: $envVar
  envVars = ['CREAM_JOBID', 'GRID_GLOBAL_JOBID']
  ids = ( os.environ.get(var) for var in envVars if var in os.environ)
  return next(ids, '')

def main():
  """Is used to generate the pilot uuid
     and save it to a file even
     before any DIRAC related part is installed.
  """
  filename = ' '.join( sys.argv[1:] )
  if not filename:
    getUniqueIDAndSaveToFile()
  else:
    getUniqueIDAndSaveToFile( filename )

if __name__ == '__main__':
  main()


import json

import pandas as pd
import matplotlib.pyplot as plt
import requests
from bs4 import BeautifulSoup

file = open("ingredient_list.json", "r")
item_list = []

class Data:
    def __init__(self):
        self.adjusted = True
        self.item_counts = json.load(file)
        self.graph = "not set"
        self.dumped = []

    def remove_keyword(self):
        pass

    def add_count(self, item):
        items = item.split("and") if item.find("and") > 0 else [item]
        for item in items:
            if item in self.item_counts:
                self.item_counts[item] += 1
            else:
                self.dumped.append(item)

    def sort_counts(self, reverse=False):
        self.item_counts = dict(sorted(self.item_counts.items(), key=lambda x: x[1], reverse=reverse))

    def setUp_graph(self, threshold):
        values = []
        names = []
        threshold -= 1
        other_count = 0
        x = 0
        for i in self.item_counts:
            if x < threshold:
                names.append(i)
                values.append(self.item_counts[i])
            else:
                other_count += self.item_counts[i]
            x = x + 1
        values.append(other_count)
        names.append("other")
        self.graph = pd.DataFrame({'count': values}, index=names)

    def print_pie_chart(self):
        self.graph.plot.pie(y='count', figsize=(50, 50), fontsize=30)
        plt.show()
        plt.close()


from MeiTuan_Spider.core.utils import Spider, Tool
from MeiTuan_Spider.settings import city_href_path
from lxml import etree
from collections import defaultdict

class CityLinks(object):
    # 城市链接
    def __init__(self):
        self.spider = Spider()
        self.file = Tool()
        self.data = defaultdict(dict)
        self.url = 'https://www.meituan.com/changecity/'

    def get_page(self):
        html = self.spider.crawl(url=self.url, plain='text')
        content = etree.HTML(html)
        divs = content.xpath('//*[@class="alphabet-city-area"]/div')
        if divs:
            for div in divs:
                city_label = div.xpath('./span[@class="city-label"]/text()')[0]
                cities = div.xpath('./span[@class="cities"]/a')
                for city in cities:
                    href = city.xpath('./@href')[0]
                    name = city.xpath('./text()')[0]
                    self.data[city_label].update({name: 'https:' + href})

        self.file.save_to_json(city_href_path, self.data)

if __name__ == '__main__':
    city = CityLinks()
    city.get_page()


# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# http://www.sphinx-doc.org/en/master/config

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
from subprocess import check_call

import six

sys.path.insert(0, os.path.abspath("../.."))

# -- General configuration ---------------------------------------------------

# Add any Sphinx extension model names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.autosectionlabel',
    'sphinx.ext.autosummary',
    'sphinx.ext.coverage',
    'sphinx.ext.doctest',
    'sphinx.ext.githubpages',
    'sphinx.ext.ifconfig',
    'sphinx.ext.intersphinx',
    'sphinx.ext.mathjax',
    'sphinx.ext.napoleon',
    'sphinx.ext.todo',
    'sphinx.ext.viewcode'
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

pygments_style = 'sphinx'

# -- Project information -----------------------------------------------------

project = 'torecsys'
copyright = '2019, Jasper, Li Wai Yin'
author = 'Jasper, Li Wai Yin'

# The full version, including alpha/beta/rc tags
version = "dev-0.0.1"
release = "dev"

# The master toctree document.
master_doc = 'index'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# Disable docstring inheritance
autodoc_inherit_docstrings = True

# autosectionlabel throws warnings if section names are duplicated.
# The following tells autosectionlabel to not throw a warning for
# duplicated section names that are in different documents.
autosectionlabel_prefix_document = True

# Configuration of sphinx
add_function_parentheses = False
add_module_names = False
autoclass_content = "both"
autodoc_mock_imports = ["torch", "torchaudio", "torchvision", "torchtext"]

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = 'nature'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_title = "Torecsys Documentation - Torescys Master Documentation"
html_short_title = "torecsys documentation"
html_baseurl = "https://torecsys.readthedocs.io/"
html_static_path = ['_static']
html_logo = "_static/img/pytorch-logo-dark.svg"
html_favicon = "_static/img/pytorch-logo-flame.ico"

# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
intersphinx_mapping = {
    'python': ('https://docs.python.org/3.9', None),
    'numpy': ('https://docs.scipy.org/doc/numpy/', None)
}

def on_init(app):  # pylint: disable=unused-argument
    """
    Run sphinx-apidoc after Sphinx initialization.
    Read the Docs won't run tox or custom shell commands, so we need this to
    avoid checking in the generated reStructuredText files.
    """
    docs_path = os.path.abspath(os.path.dirname(__file__))
    root_path = os.path.abspath(os.path.join(docs_path, '../..'))
    apidoc_path = 'sphinx-apidoc'
    if hasattr(sys, 'real_prefix'):  # Check to see if we are in a virtualenv
        # If we are, assemble the path manually
        bin_path = os.path.abspath(os.path.join(sys.prefix, 'bin'))
        apidoc_path = os.path.join(bin_path, apidoc_path)
    check_call([apidoc_path, '-o', docs_path, os.path.join(root_path, 'torecsys')])

def setup(app):
    """Sphinx extension: run sphinx-apidoc."""
    event = 'builder-inited' if six.PY3 else b'builder-inited'
    app.connect(event, on_init)