import argparse
import json
import os
import pprint
import string
from datetime import timedelta
from email.message import Message
from email.parser import Parser

from networkx import DiGraph

import sampledata
import settings
from cascade.models import Project, ActSequence, CascadeTree, CascadeNode, ParamTypes
from db.managers import DBManager
from settings import logger
from utils.time_utils import time_measure, str_to_datetime

def split_to_words(in_str):
    in_str = in_str.strip()
    for ch in string.punctuation:
        in_str = in_str.replace(ch, ' ')
    return in_str.lower().split()

def jaccard(set1, set2):
    if not set1 or not set2:
        return 0
    else:
        return len(set1 & set2) / len(set1 | set2)

def find_thread(threads, email):
    words = set(split_to_words(email.subject))
    for i in range(len(threads)):
        thread = threads[i]
        if all([jaccard(set(split_to_words(subject)), words) > 0.5 for subject in thread.subjects]):
            if email.from_addr in thread.users or any([u in thread.users for u in email.to_addr]):
                return i
    return None

def extract_dir_file_names(path):
    file_names = []
    for root, dirs, files in os.walk(path):
        file_names.extend(os.path.join(root, f) for f in files)
    return file_names

def extract_file_names(root_dir, dir_name):
    if os.path.exists(os.path.join(root_dir, dir_name, 'all_documents')):
        logger.debug('files of all_documents returned')
        return extract_dir_file_names(os.path.join(root_dir, dir_name, 'all_documents'))
    else:
        file_names = []
        if os.path.exists(os.path.join(root_dir, dir_name, 'inbox')):
            file_names = extract_dir_file_names(os.path.join(root_dir, dir_name, 'inbox'))
        if os.path.exists(os.path.join(root_dir, dir_name, 'sent_items')):
            file_names.extend(extract_dir_file_names(os.path.join(root_dir, dir_name, 'sent_items')))
        logger.debug('files of inbox & sent_items returned')
        return file_names

class Email:
    def __init__(self, path: str):
        try:
            with open(path) as f:
                data = f.read()
        except UnicodeDecodeError:
            with open(path, encoding="ISO-8859-1") as f:
                data = f.read()
        message = Parser().parsestr(data)

        self.from_addr = message['from']
        if message['to'] is None:
            self.to_addr = []
        else:
            self.to_addr = message['to'].replace("\n", "").replace("\t", "").replace(" ", "").split(",")
        self.date = str_to_datetime(message['date'][:-12], '%a, %d %b %Y %H:%M:%S', 'US/Pacific')
        self.subject = message['subject']
        self.path = path

class Thread:
    def __init__(self):
        self.emails = []
        self.users = set()
        self.subjects = set()

    def add_email(self, email):
        self.emails.append(email)
        self.users.add(email.from_addr)
        if email.to_addr:
            self.users.update(email.to_addr)
        if email.subject:
            self.subjects.add(email.subject)

def save_file_names(threads, save_path):
    logger.info('saving threads ...')
    file_names = [[email.path for email in thread.emails] for thread in threads]
    with open(save_path, 'w') as f:
        json.dump(file_names, f, indent=2)

def extract_threads(path):
    threads = []
    for dir_name in sorted(os.listdir(path)):
        logger.info('reading emails of %s ...', dir_name)

        file_names = extract_file_names(path, dir_name)
        logger.info('number of emails : %d', len(file_names))

        i = 0
        for fname in file_names:
            email = Email(fname)
            if email.subject and email.to_addr:
                thread_index = find_thread(threads, email)
                if thread_index is not None:
                    threads[thread_index].add_email(email)
                else:
                    new_thread = Thread()
                    new_thread.add_email(email)
                    threads.append(new_thread)
            i += 1
            if i % 100 == 0:
                logger.debug('%d emails read', i)

    threads = list(filter(lambda thread: len(thread.emails) > 1, threads))

    logger.info('removing the threads with subject Re, FW, or FWD ...')
    new_threads = []
    for thread in threads:
        # Remove the threads with single email. Remove the threads with subject Re, FW, or FWD.
        if len(thread.emails) < 2 or any(
                split_to_words(subj) in [['re'], ['fw'], ['fwd']] for subj in thread.subjects):
            continue
        new_threads.append(thread)

    return new_threads

def load_threads(save_path):
    logger.info('loading threads from the saved file ...')
    with open(save_path) as f:
        file_names = json.load(f)

    threads = []

    for thr_fnames in file_names:
        thr = Thread()

        for fname in thr_fnames:
            email = Email(fname)
            thr.add_email(email)
        threads.append(thr)

    return threads

def save_cascades(threads, db_name):
    manager = DBManager(db_name)

    # Drop the database.
    manager.client.drop_database(db_name)

    db = manager.db
    users_map = {}  # dictionary from email addresses (usernames) to their _id
    all_post_cascades = []
    all_reshares = []
    act_sequences = {}
    trees = {}

    i = 0
    logger.info('processing %d threads ...', len(threads))

    # Iterate on threads.
    for thread in threads:
        users = [{'username': user} for user in thread.users]
        db.users.insert_many(users)
        users_map.update({user['username']: user['_id'] for user in users})
        posts_map = {}  # dictionary from email addresses to their first posts in this thread.
        reshare_pairs = []  # list of the pairs (post1, post2) where post2 is a reshare of post1

        # Iterate on the emails of this thread in the order of their dates.
        for email in sorted(thread.emails, key=lambda m: m.date):
            if email.from_addr in posts_map:
                from_post = posts_map[email.from_addr]
                if from_post['datetime'] is None:
                    from_post['datetime'] = email.date
            else:
                from_post = {'author_id': users_map[email.from_addr], 'datetime': email.date}
                posts_map[email.from_addr] = from_post

            if email.to_addr:
                for to_addr in email.to_addr:
                    if to_addr not in posts_map:
                        to_post = {'author_id': users_map[to_addr], 'datetime': None}
                        posts_map[to_addr] = to_post
                        reshare_pairs.append((from_post, to_post))

        # Set the datetime of the posts without datetime.
        for source_post, dest_post in reshare_pairs:
            if dest_post['datetime'] is None:
                # The destination time is set 1 day after the source time if there is no datetime.
                dest_post['datetime'] = source_post['datetime'] + timedelta(days=1)

        # Insert the posts into db.
        posts = list(posts_map.values())
        db.posts.insert_many(posts)
        logger.debug('%d posts inserted', len(posts))

        # Extract the activation sequence of the thread.
        times = [post['datetime'] for post in posts]
        first_time, last_time = min(times), max(times)
        act_seq = extract_act_sequence(posts, first_time, last_time)

        # Add the thread reshares to the list of all reshares.
        reshares = [{
            'reshared_post_id': pair[0]['_id'],
            'post_id': pair[1]['_id'],
            'ref_user_id': pair[0]['author_id'],
            'user_id': pair[1]['author_id'],
            'ref_datetime': pair[0]['datetime'],
            'datetime': pair[1]['datetime']
        } for pair in reshare_pairs]
        all_reshares.extend(reshares)

        # Extract the cascade tree of the thread.
        tree = extract_tree(reshares)

        # Insert the cascade into db and add the activation sequence and the tree to the related dictionaries.
        res = db.cascades.insert_one({
            'size': len(thread.users),
            'count': len(posts_map),
            'depth': tree.depth,
            'first_time': first_time,
            'last_time': last_time
        })
        cascade_id = res.inserted_id
        act_sequences[cascade_id] = act_seq
        trees[cascade_id] = tree

        # Add the thread post_cascades to the list of all post_cascades.
        post_cascades = [{'post_id': post['_id'],
                          'cascade_id': cascade_id,
                          'author_id': post['author_id'],
                          'datetime': post['datetime']} for post in posts]
        all_post_cascades.extend(post_cascades)

        i += 1
        if i % 1000 == 0:
            logger.info('%d threads done', i)

    # Extract the graph
    graph = extract_graph(all_reshares)

    # Create a project containing all cascades.
    create_project(act_sequences, graph, trees, db_name)

    # Insert post_cascades and reshares into db.
    logger.info('inserting %d post_cascades', len(all_post_cascades))
    db.postcascades.insert_many(all_post_cascades)
    logger.info('done. inserting %d reshares', len(all_reshares))
    db.reshares.insert_many(all_reshares)
    logger.info('done')

def create_project(act_sequences, graph, trees, db_name):
    """
    Create a project containing all cascades (with random training, validation, and test sets). Then save the graph,
    trees, and activation sequences data in the project data directory.
    :param act_sequences:
    :param graph:
    :param trees:
    :param db_name:
    :return:
    """
    logger.info('creating project "enron-all" ...')
    project_name = 'enron-all'
    sampledata.Command().sample_data(db_name, project_name)
    project = Project(project_name)
    project.save_param(graph, 'graph', ParamTypes.GRAPH)
    project.save_trees(trees)
    project.save_act_sequences(act_sequences)

def extract_graph(all_reshares):
    edges = {(reshare['ref_user_id'], reshare['user_id']) for reshare in all_reshares}
    graph = DiGraph()
    graph.add_edges_from(edges)
    return graph

def extract_act_sequence(posts, first_time, last_time):
    sorted_posts = sorted(posts, key=lambda p: p['datetime'])
    users = [post['author_id'] for post in sorted_posts]
    times = [(post['datetime'] - first_time).total_seconds() / (3600.0 * 24 * 30) for post in
             sorted_posts]  # number of months
    rel_last_time = (last_time - first_time).total_seconds() / (3600.0 * 24 * 30)  # number of months
    return ActSequence(users, times, rel_last_time)

def extract_tree(reshares):
    roots = []
    nodes = {}

    for reshare in sorted(reshares, key=lambda resh: resh['datetime']):
        if reshare['ref_user_id'] not in nodes:
            node = CascadeNode(reshare['ref_user_id'], reshare['ref_datetime'])
            nodes[node.user_id] = node
            roots.append(node)
        else:
            node = nodes[reshare['ref_user_id']]

        if reshare['user_id'] not in nodes:
            child_node = CascadeNode(reshare['user_id'], reshare['datetime'], reshare['ref_user_id'])
            nodes[child_node.user_id] = child_node
            node.children.append(child_node)

    return CascadeTree(roots)

@time_measure()
def main(args):
    threads = extract_threads(args.path)

    save_path = os.path.join(settings.BASE_PATH, 'data', 'enron_threads.json')
    save_file_names(threads, save_path)
    pprint.pprint({i: threads[i].subjects for i in range(len(threads))})

    # threads = load_threads(save_path)

    save_cascades(threads, args.db)

if __name__ == '__main__':
    parser = argparse.ArgumentParser('Read Enron dataset')
    parser.add_argument("-p", "--path", required=True, help="dataset directory path")
    parser.add_argument('-d', '--db', required=True, help="db name in which the documents must be inserted")
    args = parser.parse_args()
    main(args)


import json
import math

import nltk
from gensim import corpora
from gensim import models
from mrjob.job import MRJob
from mrjob.step import MRStep
from nltk.tokenize import sent_tokenize, word_tokenize

# Change the value of this constant to the output of script random_paper_selector.py
RANDOM_PAPER = {
    "author": "[{'name': 'Ahmed Osman'}, {'name': 'Wojciech Samek'}]",
    "day": 1,
    "id": "1802.00209v1",
    "link": "[{'rel': 'alternate', 'href': 'http://arxiv.org/abs/1802.00209v1', 'type': 'text/html'}, {'rel': 'related', 'href': 'http://arxiv.org/pdf/1802.00209v1', 'type': 'application/pdf', 'title': 'pdf'}]",
    "month": 2,
    "summary": "We propose an architecture for VQA which utilizes recurrent layers to\ngenerate visual and textual attention. The memory characteristic of the\nproposed recurrent attention units offers a rich joint embedding of visual and\ntextual features and enables the model to reason relations between several\nparts of the image and question. Our single model outperforms the first place\nwinner on the VQA 1.0 dataset, performs within margin to the current\nstate-of-the-art ensemble model. We also experiment with replacing attention\nmechanisms in other state-of-the-art models with our implementation and show\nincreased accuracy. In both cases, our recurrent attention mechanism improves\nperformance in tasks requiring sequential or relational reasoning on the VQA\ndataset.",
    "tag": "[{'term': 'cs.AI', 'scheme': 'http://arxiv.org/schemas/atom', 'label': None}, {'term': 'cs.CL', 'scheme': 'http://arxiv.org/schemas/atom', 'label': None}, {'term': 'cs.CV', 'scheme': 'http://arxiv.org/schemas/atom', 'label': None}, {'term': 'cs.NE', 'scheme': 'http://arxiv.org/schemas/atom', 'label': None}, {'term': 'stat.ML', 'scheme': 'http://arxiv.org/schemas/atom', 'label': None}]",
    "title": "Dual Recurrent Attention Units for Visual Question Answering",
    "year": 2018
}

# Global variable used to store the text to vec representation of the random paper's summary
random_result = {}

# Download relevant NLTK model if not already present
nltk.download('punkt')

def compute_random_paper_aspects():
    """
    This method computes the computationally relevant data on the given random paper
    :return: random_result: the text to vec representation of the random paper's summary
    """

    random_paper_summary = RANDOM_PAPER["summary"]
    random_paper_summary = random_paper_summary.replace("\n", " ")
    random_data = []

    # Iterate through each sentence in the random paper summary
    for i in sent_tokenize(random_paper_summary):
        temp = []

        # Tokenize the sentence into words
        for j in word_tokenize(i):
            temp.append(j.lower())

        random_data.append(temp)
    # Compute the dictionary of words for the random paper
    random_paper_dictionary = corpora.Dictionary(random_data)

    # From the dictionary of words, form a new dictionary of words in
    # their alphabetical order and their index in this alphabetical order
    random_paper_vector = random_paper_dictionary.token2id

    # For each sentence, convert each word into a tuple with the
    # following structure: word_id, nb of times that a word appears in the sentence
    random_corpus = [random_paper_dictionary.doc2bow(sentence) for sentence in random_data]

    # Compute the TF-IDF model for the random paper to obtain the term frequency
    # in the summary with respect to the length of the summary
    random_tfidf = models.TfidfModel(random_corpus)

    # In random_result, return a dictionary containing the words in their alphabetical
    # order and the total TF-IDF score of each word that appears in the random paper
    random_vec = []
    for document in random_tfidf[random_corpus]:
        for word, score in document:
            random_vec.append((word, score))
    num_dict = {}
    for t in random_vec:
        if t[0] in num_dict:
            num_dict[t[0]] = num_dict[t[0]] + t[1]
        else:
            num_dict[t[0]] = t[1]

    global random_result
    for k, v in random_paper_vector.items():
        if v in num_dict:
            random_result[k] = num_dict.get(v)
    return random_result

class SimilarPaperRecommendations(MRJob):

    def mapper_paper_summary(self, _, line):
        """
        This mapper yields the paper id and the paper summary for each paper in the JSON
        :param _: None
        :param line: one line from the input file
        :return: (paper_id, paper_summary)
        """

        # For input file type JSON, get the id and the summary of all the papers
        paper_data = json.loads(line)
        paper_id = paper_data["id"]
        paper_summary = paper_data["summary"]

        yield paper_id, paper_summary

    def mapper_compute_cosine_similarity(self, paper_id, paper_summary):
        """
        This mapper computes the cosine similarity between a random paper (given)
        and each paper in the JSON and yields the paper id, paper summary
        :param paper_id: unique id as defined in the input data
        :param paper_summary: corresponding paper summary
        :return: None, (cosine_similarity, (paper_id, paper_summary))
        """

        global random_result
        paper_summary = paper_summary.replace("\n", " ")

        paper_data = []
        # Iterate through each sentence in the paper summary
        for i in sent_tokenize(paper_summary):
            temp = []

            # Tokenize the sentence into words
            for j in word_tokenize(i):
                temp.append(j.lower())

            paper_data.append(temp)

        # Compute the dictionary of words for each paper
        paper_dictionary = corpora.Dictionary(paper_data)

        # From the dictionary of words, form a new dictionary of words in
        # their alphabetical order and their index in this alphabetical order
        paper_vector = paper_dictionary.token2id

        # For each sentence, convert each word into a tuple with the
        # following structure: word_id, nb of times that a word appears in the sentence
        paper_corpus = [paper_dictionary.doc2bow(sentence) for sentence in paper_data]

        # Compute the TF-IDF model for the paper to obtain the term frequency
        # in the summary with respect to the length of the summary
        paper_tfidf = models.TfidfModel(paper_corpus)

        # In paper_result, return a dictionary containing the words in their alphabetical
        # order and the total score of each word that appears in each paper
        paper_vec = []
        for document in paper_tfidf[paper_corpus]:
            for word, score in document:
                paper_vec.append((word, score))
        num_dict = {}
        for t in paper_vec:
            if t[0] in num_dict:
                num_dict[t[0]] = num_dict[t[0]] + t[1]
            else:
                num_dict[t[0]] = t[1]

        paper_result = {}
        for k, v in paper_vector.items():
            if v in num_dict:
                paper_result[k] = num_dict.get(v)

        # Compute the cosine similarity between each paper and the randomly selected paper,
        # but avoid this computation in case the paper and the random paper are the same;
        # The cosine similarity implementation was done without using a library that computes it
        # automatically, and can be found here:
        # https://stackoverflow.com/questions/18424228/cosine-similarity-between-2-number-lists
        # The formula used in the cosine similarity computation will be presented in
        # the documentation
        if paper_result != random_result:

            # Select the terms that are present in the paper and the random paper
            intersection = set(random_result.keys()) & set(paper_result.keys())
            numerator = sum([random_result[x] * paper_result[x] for x in intersection])
            sum1 = sum([random_result[x] ** 2 for x in random_result.keys()])
            sum2 = sum([paper_result[x] ** 2 for x in paper_result.keys()])
            denominator = math.sqrt(sum1) * math.sqrt(sum2)
            if not denominator:
                cosine_similarity = 0.0
            else:
                cosine_similarity = round(float(numerator) / denominator, 3)

            yield None, (cosine_similarity, (paper_id, paper_summary))

    def reducer_find_highest_similarity(self, _, similarity_paper_pair):
        """
        This reducer orders the list of papers by their similarity compared with
        the randomly selected paper and returns the paper which is most similar
        to the randomly selected paper
        :param _: None
        :param similarity_paper_pair:(cosine_similarity, (paper_id, paper_summary))
        :return: sorted_similarity_paper_pairs[0]:
        the (cosine_similarity, (paper_id, paper_summary)) construct for the paper
        with the highest similarity compared to the randomly selected paper
        """

        # Sort by cosine similarity of the paper
        sorted_similarity_paper_pairs = sorted(similarity_paper_pair, key=lambda x: x[0], reverse=True)
        yield sorted_similarity_paper_pairs[0]

    def steps(self):
        return [
            MRStep(mapper=self.mapper_paper_summary),
            MRStep(mapper=self.mapper_compute_cosine_similarity,
                   reducer=self.reducer_find_highest_similarity)
        ]

if __name__ == '__main__':
    compute_random_paper_aspects()
    SimilarPaperRecommendations.run()


import json
from pprint import pprint
import xmltodict
from utils import *
import uiautomator2 as u2
from loguru import logger

last_step = ""

few_shot = """
Now that you are an automated testing program for Android software, \
what you have to do is test the functionality of the software as completely as possible \
and check for any problems. I will tell you the information of the current program interface \
by asking questions, and you will tell me the next step of the test by answering.

When you encounter components with similar names, you can look at them as the same category and test one or more of them. When you encounter many operation options, you tend to click on them from smallest to largest, and tend to click on the component with "menu button" in its name.

Now that you are an automated testing program for Android software, what you have to do is test the functionality of the software as completely as possible and check for any problems. I will tell you the information of the current program interface by asking questions, and you will tell me the next step of the test by answering.

"""

start = True

def next(is_start: bool, step: int):
    """
    :param is_start:
    :param step:
    :return:
    """
    global last_step
    global start

    logger.info("A new turn ...")

    start = False
    logger.info('reload the page...')
    d = u2.connect()
    print(d.info)
    page_source = d.dump_hierarchy(compressed=True, pretty=True)
    xml_file = open(save_path + 'hierarchy.xml', 'w', encoding='utf-8')
    xml_file.write(page_source)
    xml_file.close()
    xml_file = open(save_path + 'hierarchy.xml', 'r', encoding='utf-8')
    logger.info('reading hierarchy tree xml file...')
    data_dict = xmltodict.parse(xml_file.read())
    data_str = json.dumps(data_dict)
    json_file = open(save_path + 'hierarchy.json', 'w', encoding='utf-8')
    json_file.write(data_str)
    json_file.close()

    all_components = getAllComponents(data_dict)
    logger.info("There is {} components on current page.".format(str(len(all_components))))

    running_info = get_running_info()
    # ActivityName
    activity_name = running_info['activity'].replace('.', ' ').split(' ')[-1].replace('Activit', '')
    print("activity_name: {}".format(activity_name))

    jsonfile = open(r'./json/{}.json'.format(appname), 'r')
    jsondata = json.load(jsonfile)
    pprint(jsondata)

    logger.info('searching for describable components...')
    components_list = []
    for e_component in all_components:
        info = get_common_desc(e_component)
        desc = info['desc']
        if desc != "":
            e_component['@desc'] = desc
            components_list.append(e_component)

    origin_list = [e["@desc"] for e in components_list]
    renamed_list = rename_duplicate(origin_list)

    for i, e in enumerate(renamed_list):
        components_list[i]["@desc"] = e

    logger.info('searching for clickable components...')
    clickable_list = []

    for e_component in components_list:
        if e_component['@clickable'] == 'true':
            info = {"desc": e_component["@desc"], "bounds": e_component["@bounds"]}
            clickable_list.append(info)

    logger.info('searching for editable components...')
    edit_list = []
    for e_component in components_list:
        if '@class' in e_component and (e_component['@class'] == 'android.widget.EditText' or
                                        e_component['@class'] == 'android.widget.AutoCompleteTextView'):
            edit_list.append(e_component)

    print("--- There are describable components: ---")
    for e in components_list:
        print(e)
    print("----------------------------------------")

    print("--- There are clickable components: ---")
    for e in clickable_list:
        print(e)
    print("----------------------------------------")

    print("--- There are editable components: ---")
    for e in edit_list:
        print(e)
    print("----------------------------------------")

    up_half, down_half = split_page(components_list)

    up_half = [e["@desc"] for e in up_half]
    down_half = [e["@desc"] for e in down_half]

    print("up half: {}".format(up_half))
    print("down half: {}".format(down_half))

    # WidgetC WidgetID/WidgetT WidgetAct WidgetV
    widget_list = []
    for e in components_list:
        temp = {}
        #
        temp["WidgetC"] = e["@class"].split(".")[-1]
        #
        if e["@text"] == "" and e["@content-desc"] == "":
            temp["widgetID"] = e["@desc"]
        else:
            temp["widgetT"] = e["@desc"]
        #
        temp["WidgetAct"] = "none"
        if "@clickable" in e and e["@clickable"] == "true":
            temp["WidgetAct"] = "Click"
        #
        temp["WidgetV"] = "0"
        widget_list.append(temp)
    pprint(widget_list)

    jsondata["Page information attribute"] = [{
        "ActivityName": activity_name,
        "Widgets": [e["@desc"] for e in components_list],
        "Layouts": [
            {
                "Upper half": up_half,
                "Lower half": down_half
            }
        ]
        # "VisitTime": "1",
        # "Duplicate": "0"
    }]
    jsondata["Widget information attribute"] = [{}]
    for i, e in enumerate(widget_list):
        jsondata["Widget information attribute"][0]["Widget_{}".format(str(i + 1))] = [e]

    print("json data:")
    pprint(jsondata)

    jsonstr = json.dumps(jsondata)
    f = open('./json/{}-step{}.json'.format(appname, str(step + 1)), 'w')
    f.write(jsonstr)
    f.close()

    prompt = "We want to test the {} App, which has {} main function pages, namely: ".format(appname, str(len(jsondata["Global information attribute"][0]["Activities"])))

    for e in jsondata["Global information attribute"][0]["Activities"]:
        prompt += '"{}", '.format(e)
    prompt = prompt[:-2]

    prompt += '.\nThe recommended test sequence is: '

    for e in jsondata["Global information attribute"][0]["Priority"]:
        prompt += '"{}", '.format(e)
    prompt = prompt[:-2]

    prompt += '.\nThe function UI page we are currently testing is "{}".\n'.format(activity_name)

    prompt += 'The number of expoloration recorded on the current page is {}.\n'.format(jsondata["Page information attribute"][0]["VisitTime"])

    prompt += "Now we can do these:\n"

    opt_id = 1
    for e in clickable_list:
        prompt += '{}. click "{}"\n'.format(str(opt_id), e["desc"])
        opt_id += 1
    prompt += "{}. return to previous page\n".format(str(opt_id))
    prompt += "What do you choose to do? Please answer directly with one of the options. \n"
    print(prompt)

    res = getOutput(few_shot + prompt)

    print(res)

    if res.lower().find("click") != -1:
        #
        target = get_quote(res)[0]
        click(target, components_list)
        last_step = 'click "{}"'.format(target)

    #
    if res.lower().find("return to previous page") != -1:
        get_back()
        last_step = "return to previous page"

    print(last_step)

    #
    new_running_info = get_running_info()
    #
    new_activity_name = new_running_info['activity'].replace('.', ' ').split(' ')[-1].replace('Activit', '')
    print("new activity_name: {}".format(new_activity_name))

    #
    if is_start:
        g_file = open('./gragh/{}.txt'.format(appname), 'w', encoding='utf-8')
        g_file.close()

    #
    if not os.path.exists('./gragh/{}.txt'.format(appname)):
        g_file = open('./gragh/{}.txt'.format(appname), 'w', encoding='utf-8')
        g_file.close()
    g_file = open('./gragh/{}.txt'.format(appname), 'r', encoding='utf-8')
    lines = len(g_file.readlines())
    g_file.close()
    #
    g_file = open('./gragh/{}.txt'.format(appname), 'a', encoding='utf-8')
    g_file.write("Step {}: {} <- {} -> {}\n".format(str(lines + 1), activity_name, res, new_activity_name))

    is_start = False

if __name__ == '__main__':
    for step in range(5):
        next(start, step)


import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--hyps', nargs='+', default=[])
parser.add_argument('--labels', nargs='+', default=[])
parser.add_argument('--output', type=str, required=True)
args = parser.parse_args()

assert len(args.hyps) == len(args.labels)
assert len(args.hyps) > 0

import os
import numpy as np
from imageio import imread
import matplotlib.pyplot as plt

#fig = plt.figure(figsize=(19.2, 4.8))
fig = plt.figure()

for idx, (hyp, label) in enumerate(zip(args.hyps, args.labels)):
    _im = imread(os.path.join(hyp, '15.png'))

    ax = fig.add_subplot(1, len(args.hyps), idx+1)
    ax.set_axis_off()
    ax.set_title(label)

    ax.imshow(_im)

plt.savefig(args.output)


import numpy as np
import matplotlib.pyplot as plt

def spot_counts(spots, bin_size=30, make_plot=True):
    """
    Compute and plot the monthly spot number.

    Following Hathaway (2015), LRSP 12: 4, we compute the number of spots
    emerging in a particular month, then smooth the time series with a
    sliding 13-month window, centered on each month. This corresponds to
    the "International Sunspot Number" (see section 3.1).

    This function generalizes the spot count window size, but keeps
    the month window by default.

    Parameters
    ----------
    spots (astropy Table):
        The output of `regions` containing the table of star spots.

    bin_size (float, 30):
        The window length in days for the spot count.

    make_plot (bool, optional, default=True):
        Whether to make and display the plot.

    Returns
    -------
    T (array):
        The times corresponding to each spot count.

    N (array):
        The monthly spot counts.
    """
    days = spots["nday"]

    # first bin spot counts by month
    d = bin_size/2
    time = []
    nspots = []
    while d < days.max():
        time.append(d)
        nspots.append(sum((d-bin_size/2 < days) & (days <= d+bin_size/2)))
        d += bin_size

    # smooth monthly spot counts
    time = np.array(time)
    nspots = np.array(nspots)
    bin_size *= 13
    d = bin_size/2
    T = []
    N = []
    while d < days.max():
        T.append(d/365)
        N.append(np.mean(nspots[((d-bin_size/2) < time) & (time <= (d+bin_size/2))]))
        d += bin_size/13

    if make_plot:
        plt.figure()
        plt.plot(time/365, nspots, "ko", ms=3, label="Monthly Spot Count")
        plt.plot(T, N, "r", label="13-Month Smoothing")
        plt.xlabel("Time (years)")
        plt.ylabel("Monthly Sunspot Number")
        plt.legend()
        plt.show()

    return np.array(T), np.array(N)


import time, itertools
from helpers import *

t = time.time()

# Read XMAS data numbers to array
input = mypath + "d9-input.txt"
with open(input) as f:
    XMASdata = [int(line) for line in f]

# Part 1
preamble = XMASdata[:25]
available_numbers = preamble.copy()

for i in range(25, len(XMASdata)):
    combinations = list(itertools.combinations(available_numbers, 2))
    combination_sums = map(lambda x: sum(x), combinations)
    if XMASdata[i] not in combination_sums:
        no_sum = XMASdata[i]
        break
    available_numbers = available_numbers[1:]
    available_numbers.append(XMASdata[i])

dropstar(17, no_sum, t)

# Part 2
available_numbers = preamble.copy()

for i in range(25, len(XMASdata)):
    combinations = list(itertools.combinations(available_numbers, 2))
    combination_sums = map(lambda x: sum(x), combinations)
    current_number = XMASdata[i]
    if current_number not in combination_sums:
        # print("No sum possible in 2 of prior 25 numbers for {0}".format(current_number))
        n = 3
        keep_looking = True
        while keep_looking:
            for j in range(0, len(XMASdata)):
                next_group = XMASdata[j : j + n]
                if current_number == sum(next_group):
                    weakness = min(next_group) + max(next_group)
                    keep_looking = False
            n += 1
    available_numbers = available_numbers[1:]
    available_numbers.append(XMASdata[i])

dropstar(18, weakness, t)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import numpy as np
import math
import tensorflow as tf
import gensim.models.keyedvectors as word2vec
from pathlib import Path
from nni_child_model.data.ptb import *

global only_sentence, use_embed_layer, phrase_min_length, slot_num
only_sentence = False
phrase_min_length=0
embed_dim = 300
slot_num = 1

def set_random_seed(seed):
  print("-" * 80)
  print("set random seed for data reading: {}".format(seed))
  np.random.seed(seed)

def sst_load_trees(filename):
  trees = read_trees(filename)
  return trees

def sst_get_id_input(content, word_id_dict, max_input_length):
  words = content.split(' ')
  sentence = []
  field = []
  index, f_index = 0, 1
  for i in range(0, max(max_input_length, len(words))):
    if len(words) <= index:
      id = word_id_dict["<pad>"]
    else:
      word = words[index].strip()
      if word in word_id_dict:
        id = word_id_dict[word]
      else:
        id = word_id_dict["<unknown>"]
    if index < max_input_length:
      sentence.append(id)
      if len(words) <= index:
        field.append(0)
      else:
        field.append(f_index)
    index += 1
    if "<sep>" in word_id_dict and id == word_id_dict["<sep>"]:
      f_index += 1
  return sentence, field

def sst_get_phrases(trees, sample_ratio=1.0,
                    is_binary=False, only_sentence=False):
  all_phrases = []
  for tree in trees:
    if only_sentence == True:
      sentence = get_sentence_by_tree(tree)
      label = int(tree.label)
      pair = (sentence, label)
      all_phrases.append(pair)
    else:
      phrases = get_phrases_by_tree(tree)
      sentence = get_sentence_by_tree(tree)
      pair = (sentence, int(tree.label))
      all_phrases.append(pair)
      all_phrases.extend(phrases)
  np.random.shuffle(all_phrases)
  result_phrases = []
  for pair in all_phrases:
    if is_binary:
      phrase = pair[0]
      label = pair[1]
      if label <= 1:
        pair = (phrase, 0)
      elif label >= 3:
        pair = (phrase, 1)
      else:
        continue
    if sample_ratio == 1.0:
      result_phrases.append(pair)
    else:
      rand_portion = np.random.random()
      if rand_portion < sample_ratio:
        result_phrases.append(pair)
  return result_phrases

def get_phrases_by_tree(tree):
  phrases = []
  if tree == None:
    return phrases
  if tree.is_leaf():
    pair = (tree.word, int(tree.label))
    phrases.append(pair)
    return phrases
  left_child_phrases = get_phrases_by_tree(tree.subtrees[0])
  right_child_phrases = get_phrases_by_tree(tree.subtrees[1])
  phrases.extend(left_child_phrases)
  phrases.extend(right_child_phrases)
  sentence = get_sentence_by_tree(tree)
  pair = (sentence, int(tree.label))
  phrases.append(pair)
  return phrases

def get_sentence_by_tree(tree):
  sentence = ""
  if tree == None:
    return sentence
  if tree.is_leaf():
    return tree.word
  left_sentence = get_sentence_by_tree(tree.subtrees[0])
  right_sentence = get_sentence_by_tree(tree.subtrees[1])
  sentence = left_sentence + " " + right_sentence
  return sentence.strip()

def get_word_id_dict(word_num_dict, word_id_dict, min_count):
  for word in word_num_dict:
    count = word_num_dict[word]
    if count >= min_count:
      index = len(word_id_dict)
      if word not in word_id_dict:
        word_id_dict[word] = index
  return word_id_dict

def load_word_num_dict(phrases, word_num_dict):
  for (sentence, label) in phrases:
    words = sentence.split(' ')
    for cur_word in words:
      word = cur_word.strip()
      if word not in word_num_dict:
        word_num_dict[word] = 1
      else:
        count = word_num_dict[word]
        word_num_dict[word] += 1
  return word_num_dict

def init_trainable_embedding(embedding, word_id_dict,
                word_embed_model, unknown_word_embed):
  embed_dim = unknown_word_embed.shape[0]
  embedding[0] = np.zeros(embed_dim)
  embedding[1] = unknown_word_embed
  for word in word_id_dict:
    id = word_id_dict[word]
    if id == 0 or id == 1:
      continue
    if word in word_embed_model:
      embedding[id] = word_embed_model[word]
    else:
      embedding[id] = np.random.rand(embed_dim) / 2.0 - 0.25
  print("init embedding: {0}".format(embedding))
  return embedding

def sst_get_trainable_data(phrases, word_id_dict,
                           split_label, max_input_length, is_binary):
  doc, bow_doc, labels, datasets = [], [], [], []

  for (phrase, label) in phrases:
    if len(phrase.split(' ')) < phrase_min_length:
      continue
    phrase_input, field_input = sst_get_id_input(phrase,
                                                 word_id_dict,
                                                 max_input_length)
    doc.append(phrase_input)
    bow_doc.append(field_input)
    labels.append(int(label))
    datasets.append("sst")
  labels = np.array(labels, dtype=np.int32)
  datasets = np.array(datasets, dtype=np.str)
  if split_label == 1:
    split_label_str = "train"
  elif split_label == 2:
    split_label_str = "test"
  else:
    split_label_str = "valid"
  doc = np.reshape(doc, [-1, max_input_length])  #(N, len)
  doc = doc.astype(np.int32)
  bow_doc = np.reshape(bow_doc, [-1, max_input_length])
  bow_doc = bow_doc.astype(np.int32)
  print(split_label_str, doc.shape, labels.shape, datasets.shape)
  print(doc)
  return doc, bow_doc, labels, datasets

def load_glove_model(filename):
  embedding_dict = {}
  with open(filename) as f:
    for line in f:
      vocab_word, vec = line.strip().split(' ', 1)
      embed_vector = list(map(float, vec.split()))
      embedding_dict[vocab_word] = embed_vector
  return embedding_dict

def load_embedding(embedding_model):
  word_embed_model = {}

  if embedding_model == "word2vec" or embedding_model == "all":
    embedding_data_path = '/data/yujwang/word2vec-GoogleNews-vectors'
    embedding_data_file = os.path.join(embedding_data_path,'GoogleNews-vectors-negative300.bin')
    word_embed_model["word2vec"] = word2vec.KeyedVectors.load_word2vec_format(embedding_data_file, binary=True)

  if embedding_model == "glove" or embedding_model == "all":
    embedding_data_path = '/data/yujwang'
    embedding_data_file = os.path.join(embedding_data_path, 'glove.840B.300d.txt')
    word_embed_model["glove"] = load_glove_model(embedding_data_file)

  unknown_word_embed = np.random.rand(embed_dim)    #create random vector in [0, 1)
  unknown_word_embed = (unknown_word_embed - 0.5) / 2.0
  return word_embed_model, unknown_word_embed

def read_data_sst(word_id_dict, word_num_dict, data_path, max_input_length,
                  embedding_model, min_count, train_ratio, valid_ratio,
                  is_binary=False, is_valid=False, cache={}):
  """Reads SST format data. Always returns NHWC format

  Returns:
    sentences: np tensor of size [N, H, W, C=1]
    labels: np tensor of size [N]
  """

  doc, labels, datasets = {}, {}, {}

  if len(cache) == 0:
    print("-" * 80)
    print("Reading SST data")

    train_file_name = os.path.join(data_path, 'train.txt')
    valid_file_name = os.path.join(data_path, 'dev.txt')
    test_file_name = os.path.join(data_path, 'test.txt')

    train_trees = sst_load_trees(train_file_name)
    train_phrases = sst_get_phrases(train_trees,
                                    train_ratio,
                                    is_binary,
                                    only_sentence)
    print("finish load train_phrases")
    valid_trees = sst_load_trees(valid_file_name)
    valid_phrases = sst_get_phrases(valid_trees,
                                    valid_ratio,
                                    is_binary,
                                    only_sentence or is_valid)
    if is_valid == False:
      train_phrases = train_phrases + valid_phrases
      valid_phrases = None
    test_trees = sst_load_trees(test_file_name)
    test_phrases = sst_get_phrases(test_trees,
                                   1.0,
                                   is_binary,
                                   only_sentence=True)
    print("finish load test_phrases")

    cache["train"] = train_phrases
    cache["valid"] = valid_phrases
    cache["test"] = test_phrases
  else:
    train_phrases = cache["train"]
    valid_phrases = cache["valid"]
    test_phrases = cache["test"]

  #get word_id_dict
  word_id_dict["<pad>"] = 0
  word_id_dict["<unknown>"] = 1
  load_word_num_dict(train_phrases, word_num_dict)
  print("finish load train words: {0}".format(len(word_num_dict)))
  if valid_phrases != None:
    load_word_num_dict(valid_phrases, word_num_dict)
  load_word_num_dict(test_phrases, word_num_dict)
  print("finish load test words: {0}".format(len(word_num_dict)))
  word_id_dict = get_word_id_dict(word_num_dict, word_id_dict, min_count)
  print("after trim words: {0}".format(len(word_id_dict)))

  if embedding_model != "none":
    word_embed_model, unknown_word_embed = load_embedding(embedding_model)
    embedding = {}
    for model_name in word_embed_model:
      embedding[model_name] = np.random.random(
              [len(word_id_dict), embed_dim]).astype(np.float32) / 2.0 - 0.25
      embedding[model_name] = init_trainable_embedding(embedding[model_name],
              word_id_dict, word_embed_model[model_name],
              unknown_word_embed)
    embedding["none"] = np.random.random(
            [len(word_id_dict), embed_dim]).astype(np.float32) / 2.0 - 0.25
    embedding["none"][0] = np.zeros([embed_dim])
    embedding["field"] = np.random.random(
            [slot_num + 1, embed_dim]).astype(np.float32) / 2.0 - 0.25
    embedding["field"][0] = np.zeros([embed_dim])

  else:
    embedding = {}
    embedding["none"] = np.random.random(
      [len(word_id_dict), embed_dim]).astype(np.float32) / 2.0 - 0.25
    embedding["none"][0] = np.zeros([embed_dim])
    embedding["field"] = np.random.random(
      [slot_num + 1, embed_dim]).astype(np.float32) / 2.0 - 0.25
    embedding["field"][0] = np.zeros([embed_dim])

  print("finish initialize word embedding")

  doc["train"], doc["train_bow_ids"], labels["train"], datasets["train"] \
          = sst_get_trainable_data(train_phrases, word_id_dict,
                   1, max_input_length, is_binary)
  doc["test"], doc["test_bow_ids"], labels["test"], datasets["test"] \
          = sst_get_trainable_data(test_phrases, word_id_dict,
                  2, max_input_length, is_binary)
  if is_valid == True:
    doc["valid"], doc["valid_bow_ids"], labels["valid"], datasets["valid"]\
            = sst_get_trainable_data(valid_phrases, word_id_dict,
                     3, max_input_length, is_binary)
  else:
    doc["valid"], doc["valid_bow_ids"], labels["valid"], datasets["valid"]\
            = None, None, None, None

  return doc, labels, datasets, embedding


import requests
import pandas as pd

params = {
    'query': '코로나19'
}
headers = {
    'X-Naver-Client-Id': 'D27XOJWmhRiNEgPtei5H',
    'X-Naver-Client-Secret': 'Dis2xjQawO'
}
res = requests.get('https://openapi.naver.com/v1/search/blog.json',
                   params=params, headers=headers)
result = res.json()

print(type(result))

result_lists = []

for item in result['items']:
    result_lists.append(
        [item['title'], item['link'], item['description'], item['postdate']])

df = pd.DataFrame(result_lists, columns=['타이틀','링크','설명','날짜'])
df.to_csv('corona_result.csv')
print('mission completed')


import logging
import os
import pickle

import torch
from torch.utils.data import Dataset

from .config import OVERWRITE_CACHE, DATASET_FILENAME, BOT_TOKEN
from app_config import DATA_DIR, CACHE_DIR

logger = logging.getLogger(__name__)

def remove_prefix(input_string, prefix):
    return input_string[len(prefix):] if prefix and input_string.startswith(prefix) else input_string

class ConversationDataset(Dataset):
    def __init__(self, tokenizer):
        cached_features_file = os.path.join(CACHE_DIR,  os.path.splitext(DATASET_FILENAME)[0] + "_CACHED_FEATURES")
        if os.path.exists(cached_features_file) and not OVERWRITE_CACHE:
            logger.info("Loading extracted features from cached file {}".format(cached_features_file))
            with open(cached_features_file, "rb") as cached_file:
                self.utterances, self.responses = pickle.load(cached_file)
        else:
            logger.info("Extracting features from dataset file {}".format(DATASET_FILENAME))
            self.utterances = []
            self.responses = []
            with open(os.path.join(DATA_DIR, DATASET_FILENAME)) as f:
                file_content = f.readlines()
                i = 0
                while i < len(file_content):
                    if file_content[i].startswith(tokenizer.bos_token):
                        j = i
                        while j < len(file_content) and file_content[j].startswith(tokenizer.bos_token):
                            j += 1
                        if j < len(file_content):
                            bot_response = remove_prefix(file_content[j], BOT_TOKEN).strip()
                            k = j + 1
                            while k < len(file_content) and file_content[k].startswith(BOT_TOKEN):
                                bot_response += ' ' + remove_prefix(file_content[k], BOT_TOKEN).strip()
                                k += 1
                            self.utterances.append(tokenizer.encode(remove_prefix(file_content[j - 1],
                                                                                tokenizer.bos_token).strip(),
                                                                    truncation=True))
                            self.responses.append(tokenizer.encode(bot_response, truncation=True))
                            i = k
                            continue
                    i += 1
            logger.info("Saving extracted features into cached file {}".format(cached_features_file))
            with open(cached_features_file, "wb") as handle:
                pickle.dump((self.utterances, self.responses), handle, protocol=pickle.HIGHEST_PROTOCOL)

    def __len__(self):
        return len(self.utterances)

    def __getitem__(self, index):
        return torch.tensor(self.utterances[index], dtype=torch.long), torch.tensor(self.responses[index],
                                                                                dtype=torch.long)


import os
import pickle
import time
from typing import Dict, List, Optional, Tuple

import cv2
import numpy as np
import pyglet

from sip._src import Keypoints, CAMERA_LANDMARK_NAMES
from sip._src.choregraphy import Choregraphy
from sip._src.keypoint import (
    keypoints_to_time_series,
    split_keypoint,
    get_keypoints_from_stream,
)
from sip._src.score import alt_cosine_similarity
from sip._src.sequence import split_sequence
from sip._src.video import get_duration, test_camera

def get_chore_sequence_splits(
    chore: Choregraphy, split_duration: float
) -> Tuple[Dict[str, List[np.ndarray]]]:
    """Get segments of keypoints time series from a choregraphy
    Args:
        chore (Choregraphy)
        split_duration (float): duration in seconds of a segment

    Returns:
        split_time_keypoints (List[Dict[str, np.ndarray]])
            where keys are landmarks names
            and values are sequences of list for coordinates
        split_time_visible (List[Dict[str, np.ndarray]])
            where keys are landmarks names
            and values are booleans to tell if joint is visible
    """
    keypoints = chore.keypoints
    t_keypoints, t_visible = keypoints_to_time_series(keypoints)

    length = get_duration(chore.video_path)
    n_splits = int(length / split_duration * 1000) + 1

    split_t_keypoints = {
        key: split_sequence(t_keypoints[key], n_splits) for key in t_keypoints
    }
    split_t_visible = {
        key: split_sequence(t_visible[key], n_splits) for key in t_visible
    }

    return split_t_keypoints, split_t_visible

def _fill_buffer(dt, vid: cv2.VideoCapture) -> None:
    """Capture keypoints and add to global buffer

    Args:
        dt: for pyglet use
        vid: the user's camera cv2 stream
    """
    global BUFFER

    frame_keypoints = get_keypoints_from_stream(vid, CAMERA_LANDMARK_NAMES)[0]
    BUFFER.append(frame_keypoints)

def _score(
    dt,
    chore_buffers: List[List[Keypoints]],
    score_label: pyglet.text.Label,
    difficulty: int,
    save_segments: bool = False,
    save_path: str = "./",
) -> None:
    """Score a segment

    Args:
        dt: for pyglet use
        chore_buffers (List[List[Keypoints]]): from the original choregraphy
        score_label (pyglet.text.Label): the score pyglet text Label
        difficulty (int): the game's difficulty
        save_segments (bool): if True, every segment will be saved on disk
        save_path (str): the path where segments are saved ("./" by default)
    """
    global BUFFER, CUR_SEGMENT, PREV_SCORE, TOTAL_SCORE, TOTAL_VISIBILITY

    if save_segments:
        pickle.dump(
            BUFFER, open(os.path.join(save_path, f"buffer_{CUR_SEGMENT}"), "wb")
        )
        pickle.dump(
            chore_buffers[CUR_SEGMENT],
            open(os.path.join(save_path, f"choreat_{CUR_SEGMENT}"), "wb"),
        )

    score, visibility = alt_cosine_similarity(
        BUFFER, chore_buffers[CUR_SEGMENT], difficulty
    )

    BUFFER = []
    CUR_SEGMENT += 1
    PREV_SCORE = 1 if score > 0.80 else 0
    TOTAL_SCORE += score
    TOTAL_VISIBILITY += visibility

    score_label.text = f"{score:.2f} - {visibility:.2f}"

def launch_game(chore: Choregraphy, difficulty: int):
    """Dance along and score in real time"""
    fps = test_camera(True)
    time.sleep(3)

    # defining global variables
    global BUFFER, CUR_SEGMENT, PREV_SCORE, TOTAL_SCORE, TOTAL_VISIBILITY

    BUFFER = []  # buffer to store current segment
    CUR_SEGMENT = 0  # current segment
    PREV_SCORE = 0  # how well user performed last segment
    TOTAL_SCORE = 0.0
    TOTAL_VISIBILITY = 0.0

    # making window
    title = "School Idol Project"
    window = pyglet.window.Window(fullscreen=True, caption=title)

    # loading user's camera
    vid = cv2.VideoCapture(0)

    # loading choregraphy's video
    video_path = chore.video_path
    video_player = pyglet.media.Player()
    video_media = pyglet.media.load(video_path)
    video_player.queue(video_media)
    video_player.play()

    # loading good segment gif
    good_animation = pyglet.image.load_animation("./resources/test.gif")
    sprite = pyglet.sprite.Sprite(img=good_animation)

    # loading score text canvas
    label = pyglet.text.Label(
        "",
        font_name="Times New Roman",
        font_size=36,
        x=0,
        y=50,
        anchor_x="left",
        anchor_y="center",
    )

    # splitting the chore in segments
    n_splits = int(get_duration(video_path) // 1) + 1
    chore_buffers = split_keypoint(chore.keypoints, n_splits)

    # defining functions to fill buffer and score segment
    pyglet.clock.schedule_interval(_fill_buffer, 1 / fps, vid)
    pyglet.clock.schedule_interval(
        _score, 1, chore_buffers, label, difficulty, save_segments=False
    )

    # defining pyglet on_draw function
    @window.event
    def on_draw():
        window.clear()

        if video_player.source and video_player.source.video_format:
            texture = video_player.get_texture()
            w = texture.width
            texture.blit(960 - w // 2, 0)
        else:
            pyglet.app.exit()

        if PREV_SCORE:
            sprite.draw()

        label.draw()

    pyglet.app.run()
    window.close()
    vid.release()

    return TOTAL_SCORE / n_splits, TOTAL_VISIBILITY / n_splits


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 16-11-6 下午3:35
# @Author  : Jyy
# @Site    :
# @File    : application.py
# @Software: PyCharm
from __future__ import division, print_function
from flask import Flask
from chat_room.view import app as blue
import os

def get_project_dir():
    return os.path.split(os.path.abspath(__file__))[0]

def get_application():
    proj_dir = get_project_dir()
    flask_params = {
        'static_folder': os.path.join(proj_dir, 'static'),
        'template_folder': os.path.join(proj_dir, 'template'),
    }

    app = Flask('chat_room', **flask_params)
    app.debug = True
    app.config['TEMPLATES_AUTO_RELOAD'] = True

    # blueprints

    app.register_blueprint(blue)
    return app


from typing import Iterator, List, Union, Tuple, Any
from datetime import datetime
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split

from tensorflow import keras
import tensorflow_addons as tfa
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import layers, models, Model
from tensorflow.python.keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint
from tensorflow.keras.losses import MeanAbsoluteError, MeanAbsolutePercentageError
from tensorflow.keras.models import Sequential
from tensorflow.keras.applications import EfficientNetB0
from tensorflow.keras.utils import plot_model
from tensorflow.keras.callbacks import History

# the next 3 lines of code are for my machine and setup due to https://github.com/tensorflow/tensorflow/issues/43174
import tensorflow as tf

physical_devices = tf.config.list_physical_devices("GPU")
tf.config.experimental.set_memory_growth(physical_devices[0], True)

def visualize_augmentations(data_generator: ImageDataGenerator, df: pd.DataFrame):
    """Visualizes the keras augmentations with matplotlib in 3x3 grid. This function is part of create_generators() and
    can be accessed from there.

    Parameters
    ----------
    data_generator : Iterator
        The keras data generator of your training data.
    df : pd.DataFrame
        The Pandas DataFrame containing your training data.
    """
    # super hacky way of creating a small dataframe with one image
    series = df.iloc[2]

    df_augmentation_visualization = pd.concat([series, series], axis=1).transpose()

    iterator_visualizations = data_generator.flow_from_dataframe(  # type: ignore
        dataframe=df_augmentation_visualization,
        x_col="image_location",
        y_col="price",
        class_mode="raw",
        target_size=(224, 224),  # size of the image
        batch_size=1,  # use only one image for visualization
    )

    for i in range(9):
        ax = plt.subplot(3, 3, i + 1)  # create a 3x3 grid
        batch = next(iterator_visualizations)  # get the next image of the generator (always the same image)
        img = batch[0]  # type: ignore
        img = img[0, :, :, :]  # remove one dimension for plotting without issues
        plt.imshow(img)
    plt.show()
    plt.close()

def get_mean_baseline(train: pd.DataFrame, val: pd.DataFrame) -> float:
    """Calculates the mean MAE and MAPE baselines by taking the mean values of the training data as prediction for the
    validation target feature.

    Parameters
    ----------
    train : pd.DataFrame
        Pandas DataFrame containing your training data.
    val : pd.DataFrame
        Pandas DataFrame containing your validation data.

    Returns
    -------
    float
        MAPE value.
    """
    y_hat = train["price"].mean()
    val["y_hat"] = y_hat
    mae = MeanAbsoluteError()
    mae = mae(val["price"], val["y_hat"]).numpy()  # type: ignore
    mape = MeanAbsolutePercentageError()
    mape = mape(val["price"], val["y_hat"]).numpy()  # type: ignore

    print(mae)
    print("mean baseline MAPE: ", mape)

    return mape

def split_data(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
    """Accepts a Pandas DataFrame and splits it into training, testing and validation data. Returns DataFrames.

    Parameters
    ----------
    df : pd.DataFrame
        Your Pandas DataFrame containing all your data.

    Returns
    -------
    Union[pd.DataFrame, pd.DataFrame, pd.DataFrame]
        [description]
    """
    train, val = train_test_split(df, test_size=0.2, random_state=1)  # split the data with a validation size o 20%
    train, test = train_test_split(
        train, test_size=0.125, random_state=1
    )  # split the data with an overall  test size of 10%

    print("shape train: ", train.shape)  # type: ignore
    print("shape val: ", val.shape)  # type: ignore
    print("shape test: ", test.shape)  # type: ignore

    print("Descriptive statistics of train:")
    print(train.describe())  # type: ignore
    return train, val, test  # type: ignore

def create_generators(
    df: pd.DataFrame, train: pd.DataFrame, val: pd.DataFrame, test: pd.DataFrame, plot_augmentations: Any
) -> Tuple[Iterator, Iterator, Iterator]:
    """Accepts four Pandas DataFrames: all your data, the training, validation and test DataFrames. Creates and returns
    keras ImageDataGenerators. Within this function you can also visualize the augmentations of the ImageDataGenerators.

    Parameters
    ----------
    df : pd.DataFrame
        Your Pandas DataFrame containing all your data.
    train : pd.DataFrame
        Your Pandas DataFrame containing your training data.
    val : pd.DataFrame
        Your Pandas DataFrame containing your validation data.
    test : pd.DataFrame
        Your Pandas DataFrame containing your testing data.

    Returns
    -------
    Tuple[Iterator, Iterator, Iterator]
        keras ImageDataGenerators used for training, validating and testing of your models.
    """
    train_generator = ImageDataGenerator(
        rescale=1.0 / 255,
        rotation_range=5,
        width_shift_range=0.1,
        height_shift_range=0.1,
        brightness_range=(0.75, 1),
        shear_range=0.1,
        zoom_range=[0.75, 1],
        horizontal_flip=True,
        validation_split=0.2,
    )  # create an ImageDataGenerator with multiple image augmentations
    validation_generator = ImageDataGenerator(
        rescale=1.0 / 255
    )  # except for rescaling, no augmentations are needed for validation and testing generators
    test_generator = ImageDataGenerator(rescale=1.0 / 255)
    # visualize image augmentations
    if plot_augmentations == True:
        visualize_augmentations(train_generator, df)

    train_generator = train_generator.flow_from_dataframe(
        dataframe=train,
        x_col="image_location",  # this is where your image data is stored
        y_col="price",  # this is your target feature
        class_mode="raw",  # use "raw" for regressions
        target_size=(224, 224),
        batch_size=32,  # increase or decrease to fit your GPU
    )

    validation_generator = validation_generator.flow_from_dataframe(
        dataframe=val, x_col="image_location", y_col="price", class_mode="raw", target_size=(224, 224), batch_size=128,
    )
    test_generator = test_generator.flow_from_dataframe(
        dataframe=test, x_col="image_location", y_col="price", class_mode="raw", target_size=(224, 224), batch_size=128,
    )
    return train_generator, validation_generator, test_generator

def get_callbacks(model_name: str) -> List[Union[TensorBoard, EarlyStopping, ModelCheckpoint]]:
    """Accepts the model name as a string and returns multiple callbacks for training the keras model.

    Parameters
    ----------
    model_name : str
        The name of the model as a string.

    Returns
    -------
    List[Union[TensorBoard, EarlyStopping, ModelCheckpoint]]
        A list of multiple keras callbacks.
    """
    logdir = (
        "logs/scalars/" + model_name + "_" + datetime.now().strftime("%Y%m%d-%H%M%S")
    )  # create a folder for each model.
    tensorboard_callback = TensorBoard(log_dir=logdir)
    # use tensorboard --logdir logs/scalars in your command line to startup tensorboard with the correct logs

    early_stopping_callback = EarlyStopping(
        monitor="val_mean_absolute_percentage_error",
        min_delta=1,  # model should improve by at least 1%
        patience=10,  # amount of epochs  with improvements worse than 1% until the model stops
        verbose=2,
        mode="min",
        restore_best_weights=True,  # restore the best model with the lowest validation error
    )

    model_checkpoint_callback = ModelCheckpoint(
        "./data/models/" + model_name + ".h5",
        monitor="val_mean_absolute_percentage_error",
        verbose=0,
        save_best_only=True,  # save the best model
        mode="min",
        save_freq="epoch",  # save every epoch
    )  # saving eff_net takes quite a bit of time
    return [tensorboard_callback, early_stopping_callback, model_checkpoint_callback]

def small_cnn() -> Sequential:
    """A very small custom convolutional neural network with image input dimensions of 224x224x3.

    Returns
    -------
    Sequential
        The keras Sequential model.
    """
    model = models.Sequential()
    model.add(layers.Conv2D(32, (3, 3), activation="relu", input_shape=(224, 224, 3)))
    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Conv2D(64, (3, 3), activation="relu"))
    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Conv2D(64, (3, 3), activation="relu"))

    model.add(layers.Flatten())
    model.add(layers.Dense(64, activation="relu"))
    model.add(layers.Dense(1))

    return model

def run_model(
    model_name: str,
    model_function: Model,
    lr: float,
    train_generator: Iterator,
    validation_generator: Iterator,
    test_generator: Iterator,
) -> History:
    """This function runs a keras model with the Ranger optimizer and multiple callbacks. The model is evaluated within
    training through the validation generator and afterwards one final time on the test generator.

    Parameters
    ----------
    model_name : str
        The name of the model as a string.
    model_function : Model
        Keras model function like small_cnn()  or adapt_efficient_net().
    lr : float
        Learning rate.
    train_generator : Iterator
        keras ImageDataGenerators for the training data.
    validation_generator : Iterator
        keras ImageDataGenerators for the validation data.
    test_generator : Iterator
        keras ImageDataGenerators for the test data.

    Returns
    -------
    History
        The history of the keras model as a History object. To access it as a Dict, use history.history. For an example
        see plot_results().
    """

    callbacks = get_callbacks(model_name)
    model = model_function
    model.summary()
    plot_model(model, to_file=model_name + ".png", show_shapes=True)

    radam = tfa.optimizers.RectifiedAdam(learning_rate=lr)
    ranger = tfa.optimizers.Lookahead(radam, sync_period=6, slow_step_size=0.5)
    optimizer = ranger

    model.compile(
        optimizer=optimizer, loss="mean_absolute_error", metrics=[MeanAbsoluteError(), MeanAbsolutePercentageError()]
    )
    history = model.fit(
        train_generator,
        epochs=100,
        validation_data=validation_generator,
        callbacks=callbacks,
        workers=6,  # adjust this according to the number of CPU cores of your machine
    )

    model.evaluate(
        test_generator, callbacks=callbacks,
    )
    return history  # type: ignore

def adapt_efficient_net() -> Model:
    """This code uses adapts the most up-to-date version of EfficientNet with NoisyStudent weights to a regression
    problem. Most of this code is adapted from the official keras documentation.

    Returns
    -------
    Model
        The keras model.
    """
    inputs = layers.Input(
        shape=(224, 224, 3)
    )  # input shapes of the images should always be 224x224x3 with EfficientNetB0
    # use the downloaded and converted newest EfficientNet wheights
    model = EfficientNetB0(include_top=False, input_tensor=inputs, weights="efficientnetb0_notop.h5")
    # Freeze the pretrained weights
    model.trainable = False

    # Rebuild top
    x = layers.GlobalAveragePooling2D(name="avg_pool")(model.output)
    x = layers.BatchNormalization()(x)
    top_dropout_rate = 0.4
    x = layers.Dropout(top_dropout_rate, name="top_dropout")(x)
    outputs = layers.Dense(1, name="pred")(x)

    # Compile
    model = keras.Model(inputs, outputs, name="EfficientNet")

    return model

def plot_results(model_history_small_cnn: History, model_history_eff_net: History, mean_baseline: float):
    """This function uses seaborn with matplotlib to plot the trainig and validation losses of both input models in an
    sns.relplot(). The mean baseline is plotted as a horizontal red dotted line.

    Parameters
    ----------
    model_history_small_cnn : History
        keras History object of the model.fit() method.
    model_history_eff_net : History
        keras History object of the model.fit() method.
    mean_baseline : float
        Result of the get_mean_baseline() function.
    """

    # create a dictionary for each model history and loss type
    dict1 = {
        "MAPE": model_history_small_cnn.history["mean_absolute_percentage_error"],
        "type": "training",
        "model": "small_cnn",
    }
    dict2 = {
        "MAPE": model_history_small_cnn.history["val_mean_absolute_percentage_error"],
        "type": "validation",
        "model": "small_cnn",
    }
    dict3 = {
        "MAPE": model_history_eff_net.history["mean_absolute_percentage_error"],
        "type": "training",
        "model": "eff_net",
    }
    dict4 = {
        "MAPE": model_history_eff_net.history["val_mean_absolute_percentage_error"],
        "type": "validation",
        "model": "eff_net",
    }

    # convert the dicts to pd.Series and concat them to a pd.DataFrame in the long format
    s1 = pd.DataFrame(dict1)
    s2 = pd.DataFrame(dict2)
    s3 = pd.DataFrame(dict3)
    s4 = pd.DataFrame(dict4)
    df = pd.concat([s1, s2, s3, s4], axis=0).reset_index()
    grid = sns.relplot(data=df, x=df["index"], y="MAPE", hue="model", col="type", kind="line", legend=False)
    grid.set(ylim=(20, 100))  # set the y-axis limit
    for ax in grid.axes.flat:
        ax.axhline(
            y=mean_baseline, color="lightcoral", linestyle="dashed"
        )  # add a mean baseline horizontal bar to each plot
        ax.set(xlabel="Epoch")
    labels = ["small_cnn", "eff_net", "mean_baseline"]  # custom labels for the plot

    plt.legend(labels=labels)
    plt.savefig("training_validation.png")
    plt.show()

def run(small_sample=False):
    """Run all the code of this file.

    Parameters
    ----------
    small_sample : bool, optional
        If you just want to check if the code is working, set small_sample to True, by default False
    """

    df = pd.read_pickle("./data/df.pkl")
    df["image_location"] = (
        "./data/processed_images/" + df["zpid"] + ".png"
    )  # add the correct path for the image locations.
    if small_sample == True:
        df = df.iloc[0:1000]  # set small_sampe to True if you want to check if your code works without long waiting
    train, val, test = split_data(df)  # split your data
    mean_baseline = get_mean_baseline(train, val)
    train_generator, validation_generator, test_generator = create_generators(
        df=df, train=train, val=val, test=test, plot_augmentations=True
    )

    small_cnn_history = run_model(
        model_name="small_cnn",
        model_function=small_cnn(),
        lr=0.001,
        train_generator=train_generator,
        validation_generator=validation_generator,
        test_generator=test_generator,
    )

    eff_net_history = run_model(
        model_name="eff_net",
        model_function=adapt_efficient_net(),
        lr=0.5,
        train_generator=train_generator,
        validation_generator=validation_generator,
        test_generator=test_generator,
    )

    plot_results(small_cnn_history, eff_net_history, mean_baseline)

if __name__ == "__main__":
    run(small_sample=False)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# create data for the plot
nbins = 5
totalN = 100

rawdata = np.ceil(np.logspace(np.log10(1/2), np.log10(nbins-.01), totalN))

# prepare data for pie chart
uniquenums = np.unique(rawdata)
data4pie = np.zeros(len(uniquenums))

for i in range(len(uniquenums)):
    data4pie[i] = sum(rawdata == uniquenums[i])

plt.pie(data4pie, labels=100*data4pie/sum(data4pie))
plt.show()


import asyncio
import qrcode
from qrcode.exceptions import DataOverflowError

import math
import urllib

from PIL import Image, ImageQt

from PyQt5.QtCore import pyqtProperty, pyqtSignal, pyqtSlot, QObject, QRect, QPoint
from PyQt5.QtGui import QImage, QColor
from PyQt5.QtQuick import QQuickImageProvider

from electrum.logging import get_logger
from electrum.qrreader import get_qr_reader
from electrum.i18n import _
from electrum.util import profiler, get_asyncio_loop

class QEQRParser(QObject):
    _logger = get_logger(__name__)

    busyChanged = pyqtSignal()
    dataChanged = pyqtSignal()
    imageChanged = pyqtSignal()

    def __init__(self, text=None, parent=None):
        super().__init__(parent)

        self._busy = False
        self._image = None
        self._data = None

        self._text = text
        self.qrreader = get_qr_reader()
        if not self.qrreader:
            raise Exception(_("The platform QR detection library is not available."))

    @pyqtSlot('QImage')
    def scanImage(self, image=None):
        if self._busy:
            self._logger.warning("Already processing an image. Check 'busy' property before calling scanImage")
            return

        if image is None:
            self._logger.warning("No image to decode")
            return

        self._busy = True
        self.busyChanged.emit()

        # self.logImageStats(image)
        self._parseQR(image)

    def logImageStats(self, image):
        self._logger.info(f'width: {image.width()} height: {image.height()} depth: {image.depth()} format: {image.format()}')

    def _parseQR(self, image):
        self.w = image.width()
        self.h = image.height()
        img_crop_rect = self._get_crop(image, 360)
        frame_cropped = image.copy(img_crop_rect)

        async def co_parse_qr(image):
            # Convert to Y800 / GREY FourCC (single 8-bit channel)
            # This creates a copy, so we don't need to keep the frame around anymore
            frame_y800 = image.convertToFormat(QImage.Format_Grayscale8)

            self.frame_id = 0
            # Read the QR codes from the frame
            self.qrreader_res = self.qrreader.read_qr_code(
                frame_y800.constBits().__int__(),
                frame_y800.byteCount(),
                frame_y800.bytesPerLine(),
                frame_y800.width(),
                frame_y800.height(),
                self.frame_id
                )

            if len(self.qrreader_res) > 0:
                result = self.qrreader_res[0]
                self._data = result
                self.dataChanged.emit()

            self._busy = False
            self.busyChanged.emit()

        asyncio.run_coroutine_threadsafe(co_parse_qr(frame_cropped), get_asyncio_loop())

    def _get_crop(self, image: QImage, scan_size: int) -> QRect:
        """
        Returns a QRect that is scan_size x scan_size in the middle of the resolution
        """
        self.scan_pos_x = (image.width() - scan_size) // 2
        self.scan_pos_y = (image.height() - scan_size) // 2
        return QRect(self.scan_pos_x, self.scan_pos_y, scan_size, scan_size)

    @pyqtProperty(bool, notify=busyChanged)
    def busy(self):
        return self._busy

    @pyqtProperty('QImage', notify=imageChanged)
    def image(self):
        return self._image

    @pyqtProperty(str, notify=dataChanged)
    def data(self):
        return self._data.data

    @pyqtProperty('QPoint', notify=dataChanged)
    def center(self):
        (x,y) = self._data.center
        return QPoint(x+self.scan_pos_x, y+self.scan_pos_y)

    @pyqtProperty('QVariant', notify=dataChanged)
    def points(self):
        result = []
        for item in self._data.points:
            (x,y) = item
            result.append(QPoint(x+self.scan_pos_x, y+self.scan_pos_y))
        return result

class QEQRImageProvider(QQuickImageProvider):
    def __init__(self, max_size, parent=None):
        super().__init__(QQuickImageProvider.Image)
        self._max_size = max_size
        self.qimg = None

    _logger = get_logger(__name__)

    @profiler
    def requestImage(self, qstr, size):
        # Qt does a urldecode before passing the string here
        # but BIP21 (and likely other uri based specs) requires urlencoding,
        # so we re-encode percent-quoted if a known 'scheme' is found in the string
        # (unknown schemes might be found when a colon is in a serialized TX, which
        # leads to mangling of the tx, so we check for supported schemes.)
        uri = urllib.parse.urlparse(qstr)
        if uri.scheme and uri.scheme in ['bitcoin', 'lightning']:
            # urlencode request parameters
            query = urllib.parse.parse_qs(uri.query)
            query = urllib.parse.urlencode(query, doseq=True, quote_via=urllib.parse.quote)
            uri = uri._replace(query=query)
            qstr = urllib.parse.urlunparse(uri)

        qr = qrcode.QRCode(version=1, border=2)
        qr.add_data(qstr)

        # calculate best box_size
        pixelsize = min(self._max_size, 400)
        try:
            modules = 17 + 4 * qr.best_fit() + qr.border * 2
            qr.box_size = math.floor(pixelsize/modules)

            qr.make(fit=True)

            pimg = qr.make_image(fill_color='black', back_color='white')
            self.qimg = ImageQt.ImageQt(pimg)
        except DataOverflowError:
            # fake it
            modules = 17 + qr.border * 2
            box_size = math.floor(pixelsize/modules)
            self.qimg = QImage(box_size * modules, box_size * modules, QImage.Format_RGB32)
            self.qimg.fill(QColor('gray'))
        return self.qimg, self.qimg.size()

# helper for placing icon exactly where it should go on the QR code
# pyqt5 is unwilling to accept slots on QEQRImageProvider, so we need to define
# a separate class (sigh)
class QEQRImageProviderHelper(QObject):
    def __init__(self, max_size, parent=None):
        super().__init__(parent)
        self._max_size = max_size

    @pyqtSlot(str, result='QVariantMap')
    def getDimensions(self, qstr):
        qr = qrcode.QRCode(version=1, border=2)
        qr.add_data(qstr)

        # calculate best box_size
        pixelsize = min(self._max_size, 400)
        try:
            modules = 17 + 4 * qr.best_fit() + qr.border * 2
            valid = True
        except DataOverflowError:
            # fake it
            modules = 17 + qr.border * 2
            valid = False

        qr.box_size = math.floor(pixelsize/modules)
        # calculate icon width in modules
        icon_modules = int(modules / 5)
        icon_modules += (icon_modules+1) % 2  # force odd

        return {
            'modules': modules,
            'box_size': qr.box_size,
            'icon_modules': icon_modules,
            'valid': valid
        }


from web3 import Web3
import settings

class transaction_result:
    def __init__(self):
        self.is_harvest = 1
        self.transaction_status = 0
        self.harvest_amount = 0
        self.transaction_hx = ''
        self.gas_consume = 0

def harvest_and_restake(is_harvest, restake_amount):
    web3 = Web3(Web3.HTTPProvider(settings.rpc))
    if web3.isConnected() != True:
        print('Rpc Connection Failed')

    contract = web3.eth.contract(settings.contract_address, abi=settings.get_contractAbi(settings.contract_address))

    gas = settings.gas
    gasPrice = int(web3.eth.gas_price*settings.gas_price_multiples)
    transaction_result.harvest_amount = contract.functions.pendingReward(settings.wallet_address).call()

    if is_harvest == 1:
       deposit_amount = 0
       transaction_result.is_harvest = 1
    else:
       deposit_amount = restake_amount
       transaction_result.is_harvest = 0

    txn = contract.functions.deposit(deposit_amount).buildTransaction({
        'gas': gas,
        'gasPrice': gasPrice,
        'nonce': web3.eth.getTransactionCount(settings.wallet_address)
        })

    sign_txn = web3.eth.account.signTransaction(txn, private_key=settings.private_key)

    while True:
        try:
            hx = web3.eth.sendRawTransaction(sign_txn.rawTransaction)
            break
        except ValueError:
            print("Nonce error, Trying again")
            txn = contract.functions.deposit(deposit_amount).buildTransaction({
                    'gas': gas,
                    'gasPrice': gasPrice,
                    'nonce': web3.eth.getTransactionCount(settings.wallet_address)
                    })
            sign_txn = web3.eth.account.signTransaction(txn, private_key=settings.private_key)

    transaction_result.transaction_hx = hx
    receipt = web3.eth.wait_for_transaction_receipt(hx, timeout=settings.transaction_timeout*60)
    transaction_result.gas_consume = gasPrice * receipt.gasUsed

    if receipt.status == 1:
        if is_harvest == 1:
            print('Harvest Succeed')
            transaction_result.transaction_status = 1
        else:
            print('Restake Succeed')
    else:
        if is_harvest == 1:
            print('Harvest Failed')
        else:
            print('Restake Failed')
    return transaction_result


import os

from in_place import InPlace
from re import sub, finditer
import requests

regex = "\[!\[.*]\(.*\/rsrc\/\d*(?P<url>.+?)(?:%3F|\)).*?\]\(.*?\)"

def fixfile(contents):
    for x in finditer(regex, contents):
        path = "./static/img" + x["url"]
        os.makedirs(os.path.dirname(path), exist_ok = True)
        img_data = requests.get("https://sites.google.com/a/puredarwin.org/puredarwin" + x["url"]).content
        with open(path, "wb") as file:
            file.write(img_data)
    contents = sub(regex, r"![](/img\g<url>)", contents)
    return contents

def loop(path):
    for file in os.scandir(path):
        if file.is_dir():
            loop(file.path)
        else:
            with InPlace(file) as f:
                text = fixfile(f.read())
                f.write(text)

loop("docs/")


# Chapter05-2
# 파이썬 심화
# 파이썬 클래스 특별 메소드 심화 활용 및 상속
# Special Method, Class ABC

# class 선언
class VectorP(object):
    # private 선언
    def __init__(self, x, y):
        self.__x = float(x)
        self.__y = float(y)

    def __iter__(self):
        return (i for i in (self.__x, self.__y)) # Generator

    @property
    def x(self):
        return self.__x

    @x.setter
    def x(self, v):
        self.__x = v

    @property
    def y(self):
        return self.__y

    @y.setter
    def y(self, v):
        if v < -273:
            raise ValueError("Temperature below -273 is not possible")
        self.__y = v

# 객체 선언
v5 = VectorP(20, 40)

# print('EX1-2 -', v5.__x, v5.__y)

print('EX1-3 -',dir(v5), v5.__dict__)
print('EX1-4 -', v5.x, v5.y) # 타 언어와 달리 근본적인 해결책 X, 파이썬 개발자 간의 암묵적 약속

# Iter 확인
for v in v5:
    print('EX1-5 -', v)

print()
print()

# __slot__
# 파이선 인터프리터에게 통보
# 해당 클래스가 가지는 속성을 제한
# __dict__ 속성 최적화 -> 다수 객체 생성시 -> 메모리 사용(공간) 대폭 감소

# 성능 비교 (클래스 2개 선언)
class TestA(object):
    __slots__ = ('a',)

class TestB(object):
    pass

use_slot = TestA()
no_slot = TestB()

print('EX2-1 -', use_slot)
# print('EX2-2 -', use_slot.__dict__)
print('EX2-3 -', no_slot)
print('EX2-4 -', no_slot.__dict__)

use_slot.a = 10
# use_slot.b = 10

# 메모리 사용량 비교
import timeit

# 측정을 위한 함수 선언
def repeat_outer(obj):
    def repeat_inner():
        obj.a = 'test'
        del obj.a
    return repeat_inner

print('EX3-1 -', min(timeit.repeat(repeat_outer(use_slot), number=1000)))
print('EX3-2 -', min(timeit.repeat(repeat_outer(no_slot), number=1000)))

print()
print()

# 객체 슬라이싱
class ObjectS:
    def __init__(self):
        self._numbers = [n for n in range(1, 10000, 3)]

    def __len__(self):
        return len(self._numbers)

    def __getitem__(self, idx):
        return self._numbers[idx]

s = ObjectS()

print('EX3-1 -', s.__dict__)
print('EX3-2 -', len(s._numbers))
print('EX3-3 -', len(s))
print('EX3-4 -', s[1:10])
print('EX3-5 -', s[-1])
print('EX3-6 -', s[::10])

print()
print()

# 파이썬 추상클래스
# 참고 : https://docs.python.org/3/library/collections.abc.html

from collections.abc import Sequence

# Sequence 상속 받지 않았지만, 자동으로 __iter__, __contains__ 기능 작동
# 객체 전체를 자동으로 조사 -> 시퀀스 프로토콜

class IterTestA():
    def __getitem__(self, idx):
        return range(1, 50, 2)[idx] # range(1, 50, 2)

i1 = IterTestA()

print('EX4-1 -', i1[4])
print('EX4-2 -', i1[4]) # [idx] 제거 후
print('EX4-3 -', 3 in i1[1:10])
# print('EX4-4 -', [i for i in i1[:]])

print()
print()

# Sequence 상속
# 요구사항인 추상메소드 모두 구현해야 동작

class IterTestB(Sequence):
    def __getitem__(self, idx):
        return range(1, 50, 2)[idx] # range(1, 50, 2)

    def __len__(self, idx):
        return len(range(1, 50, 2)[idx])

i2 = IterTestB()

print('EX4-5 -', i2[4])
# print('EX4-6 -', len(i2[:]))
print('EX4-7 -', len(i2[1:6]))

print()
print()

# abc 활용 예제
import abc

class RandomMachine(abc.ABC): # metaclass=abc.ABCMeta (3.4 이하 버전)
    # __metaclass__ = abc.ABCMeta

    # 추상 메소드
    @abc.abstractmethod
    def load(self, iterobj):
        '''iterable 항목 추가'''

    # 추상 메소드
    @abc.abstractmethod
    def pick(self, iterobj):
        '''무작위 항목 뽑기'''

    def inspect(self):
        items = []
        while True:
            try:
                items.append(self.pick())
            except LookupError:
                break
        return tuple(sorted(items))

import random

class CraneMachine(RandomMachine):
    def __init__(self, items):
        self._randomizer = random.SystemRandom()
        self._items = []
        self.load(items)

    def load(self, items):
        self._items.extend(items)
        self._randomizer.shuffle(self._items)

    def pick(self):
        try:
            return self._items.pop()
        except IndexError:
            raise LookupError('Empty Crane Box.')

    def __call__(self):
        return self.pick()

    # 메소드 오버라이드
    # def inspect(self):
    #     print('Override Test')

# 서브 클래스 확인
print('EX5-1 -', issubclass(RandomMachine, CraneMachine))
print('EX5-2 -', issubclass(CraneMachine, RandomMachine))
# 상속 구조 확인
print('EX5-3 -', CraneMachine.__mro__)

cm = CraneMachine(range(1,100)) # 추상메소드 구현 안하면 에러

print('EX5-4 -', cm._items)
print('EX5-5- ', cm.pick())
print('EX5-6- ', cm())
print('EX5-7- ', cm.inspect())


import json
from collections import OrderedDict

from pel.hexdump import hexdump
from pel.datastream import DataStream
from pel.hwdiags.parserdata import ParserData

def _parse_signature_list(version: int, data: memoryview) -> str:
    """
    Parser for the signature list.
    """

    stream = DataStream(data, byte_order='big', is_signed=False)
    parser = ParserData()
    out    = OrderedDict()

    # The first 4 bytes contains the number of signatures in this data.
    sig_count = stream.get_int(4)

    # Iterate each signature.
    out["Signature List"] = []
    for i in range(0, sig_count):
        # Extact the three words of the signature.
        a = stream.get_mem(4).hex()
        b = stream.get_mem(4).hex()
        c = stream.get_mem(4).hex()

        # Get the signature data.
        out["Signature List"].append(parser.get_signature(a, b, c))

    # Convert to JSON format and dump to a string.
    return json.dumps(out)

def _parse_register_dump(version: int, data: memoryview) -> str:
    """
    Parser for the register dump.
    """

    stream = DataStream(data, byte_order='big', is_signed=False)
    parser = ParserData()
    out    = OrderedDict()

    # The register dump will just be a list of strings where each line is either
    # a chip description or the register data associated with the chip.
    dump = []

    reg_name_len   = 25
    data_chunk_len = 4
    chip_desc_len  = 31 + reg_name_len + data_chunk_len

    # The first 4 bytes contains the number of chips containing data.
    chip_count = stream.get_int(4)

    for c in range(0, chip_count):
        # Each chip will have the following information:
        #   4 byte chip model/EC
        #   2 byte chip position
        #   1 byte node position
        #   4 byte number of registers
        model_ec = stream.get_mem(4).hex()
        chip_pos = stream.get_int(2)
        node_pos = stream.get_int(1)
        num_regs = stream.get_int(4)

        # Get the chip description. To follow legacy output, pad the right side
        # with '*' characters.
        chip_desc = parser.get_chip_desc(model_ec, node_pos, chip_pos) + ' '
        dump.append(chip_desc.ljust(chip_desc_len, '*'))

        for r in range(0, num_regs):
            # Each register will have the following information:
            #   3 byte register ID
            #   1 byte register instance
            #   1 byte data size
            #   * byte data buffer (* depends on value of data size)
            reg_id    = stream.get_mem(3).hex()
            reg_inst  = stream.get_int(1)
            data_size = stream.get_int(1)
            data_buf  = stream.get_mem(data_size).hex()

            # Get the register data from the parser.
            reg_name, reg_addr = parser.get_reg_data(model_ec, reg_id, reg_inst)

            # Crop longer names and pad with spaces on the right.
            reg_name = reg_name[0 : reg_name_len]
            reg_name = reg_name.ljust(reg_name_len)

            # Split the data buffer up into chunks for readability.
            chunks = []
            for i in range(0, len(data_buf), data_chunk_len):
                chunks.append(data_buf[i : i + data_chunk_len])

            data_buf = ' '.join(chunks)

            # Add the register info.
            dump.append("  %s (%s) %s" % (reg_name, reg_addr, data_buf.upper()))

    out["Register Dump"] = dump

    # Convert to JSON format and dump to a string.
    return json.dumps(out)

def _parse_callout_ffdc(version: int, data: memoryview) -> str:
    """
    Parser for callout list FFDC.
    """

    # The data in this section is simply a string representation of JSON data.
    # When the JSON data is written to the section, it will include a null
    # character at the end because it is a string. Before converting the
    # memoryview of the data to a string the null character must be removed.
    # Otherwise, the json.loads() function will throw an exception due to the
    # unexpected character at the end.

    # Convert the data into a string.
    s = data.tobytes().rstrip(b'\0').decode('utf8')

    # It seems odd to convert the JSON string to python structures just to turn
    # around convert it back to a string. However, we still need to do this so
    # that the output looks nice. Otherwise, the data will all be in one line.

    # Convert to JSON format and dump to a string.
    return json.dumps( { "Callout List FFDC": json.loads(s) } )

def _parse_hb_scratch_regs(version: int, data: memoryview) -> str:
    """
    Parser for the Hostboot scratch registers.
    """

    stream = DataStream(data, byte_order='big', is_signed=False)

    # Extract the hex string values of the data.
    cfamAddr  = '0x' + stream.get_mem(4).hex()
    cfamValue = '0x' + stream.get_mem(4).hex()
    scomAddr  = '0x' + stream.get_mem(8).hex()
    scomValue = '0x' + stream.get_mem(8).hex()

    # Just make a simple list for now.
    out = {
        cfamAddr: cfamValue,
        scomAddr: scomValue
    }

    # Convert to JSON format and dump to a string.
    return json.dumps({"Hostboot Scratch Registers": out})

def _parse_scratch_reg_sig(version: int, data: memoryview) -> str:
    """
    Parser for the error signature stored in the Hostboot scratch registers.
    """

    stream = DataStream(data, byte_order='big', is_signed=False)

    # Extract the hex string values of the data.
    chipId = '0x' + stream.get_mem(4).hex()
    sigId = '0x' + stream.get_mem(4).hex()

    out = {
        'Chip ID': chipId,
        'Signature ID': sigId
    }

    # Convert to JSON format and dump to a string.
    return json.dumps({"Scratch Register Error Signature": out})

def _parse_default(version: int, data: memoryview) -> str:
    """
    Default parser for user data sections that are not currently supported.
    """

    return json.dumps(None)

def parseUDToJson(subtype: int, version: int, data: memoryview) -> str:
    """
    Default function required by all component PEL user data parsers.
    """

    # Determine which parser to use.
    parsers = {
        1: _parse_signature_list,
        2: _parse_register_dump,
        3: _parse_callout_ffdc,
        4: _parse_hb_scratch_regs,
        5: _parse_scratch_reg_sig,
    }
    subtype_func = parsers.get(subtype, _parse_default)

    # Return the parsed output of this data.
    return subtype_func(version, data)


from django.urls import path

from . import views

urlpatterns = [
    path('register/', views.register, name='register'),
    path('login/', views.login, name='login'),
    path('logout/', views.logout, name='logout'),
    path('home/', views.home, name='home'),
    path('delete_user/', views.delete_user, name='delete_user'),
]


# Author: Chang Liu
# Some codes are refered from https://github.com/facebookresearch/pycls

"""Stage module."""

import copy
import torch
import torch.nn as nn
from typing import Optional, Dict, Any

from pytorch_dl.core.utils import build_module_from_cfg
from pytorch_dl.models.blocks import ResResidualBlock

_BLOCKS: Dict[str, nn.Module] = {
    "ResResidualBlock": ResResidualBlock
}

__all__ = ["Stages", "ResStage"]

class Stages(nn.Module):
    def __init__(
            self,
            d: int,
            block_cfg: Dict[str, Any]
        ) -> None:
        super(Stages, self).__init__()
        block = build_module_from_cfg("block", block_cfg, _BLOCKS)
        for i in range(d):
            self.add_module(f"block_{i + 1}", block)

        self._cfg = {
            "type": type(self).__name__,
            "d": d,
            "block": block_cfg
        }

    def forward(self, X: torch.Tensor) -> torch.Tensor:
        for block in self.children():
            X = block(X)
        return X

    @property
    def cfg(self) -> Dict[str, Any]:
        return copy.deepcopy(self._cfg)

class ResStage(Stages):
    def __init__(
            self,
            d: int,
            stride: int,
            c_in: int,
            c_out: int,
            c_b: Optional[int] = None
        ) -> None:
        block_cfg = {
            "type": "ResResidualBlock",
            "stride": stride,
            "c_in": c_in,
            "c_out": c_out
        }
        if c_b:
            block_cfg.update({
                "c_b": c_b,
                "trans_block_type": "ResBottleneckBlock"
            })
        else:
            block_cfg.update({
                "c_b": None,
                "trans_block_type": "ResBasicBlock"
            })

        super(ResStage, self).__init__(d, block_cfg)


from django.db import models
from django import forms

# Create your models here.
class Contact(models.Model):
    Name = models.CharField(max_length=20)
    Email = models.EmailField(max_length=20)
    Phone_no = models.IntegerField(max_length=20)
    Messege = models.TextField(max_length=500)


import sys, random, thread,time
from PyQt5.QtWidgets import QApplication, QDialog, QWidget
from PyQt5 import QtGui

# locks ---------------
PAINT_LOCK = thread.allocate_lock()

class Example(QWidget):
    def __init__(self):
        super(Example, self).__init__()
        self.initUI()
    def initUI(self):
        self.setGeometry(300, 300, 280, 170)
        self.setWindowTitle('Points')
        self.show()

    def paintEvent(self, e):
        #PAINT_LOCK.acquire()

        qp = QtGui.QPainter()
        qp.begin(self)
        self.drawPoints(qp)
        qp.end()

        #PAINT_LOCK.release()

    def drawPoints(self, qp):
        qp.setPen(QtGui.QPen(QtGui.QColor(255, 0, 0)))
        size = self.size()
        for i in range(1000):
            x = random.randint(1, size.width()-1)
            y = random.randint(1, size.height()-1)
            qp.drawRect(x, y, 3, 3)

def repaint_repeat(ex, interval):
    while(True):
        ex.update()

        time.sleep(interval)

    thread.exit_thread()

app = QApplication(sys.argv)

ex = Example()

thread.start_new_thread(repaint_repeat, (ex, 0.04))
#window = QDialog()
#window.show()

sys.exit(app.exec_())


import googletrans
from tkinter import *
from tkinter import ttk

from googletrans import Translator
from PIL import Image, ImageTk

source  = ""
destination = ""
t = ""
# print(googletrans.LANGUAGES)
def comboAction(event):
    global source
    global destination
    source = combo1.get()
    destination = combo2.get()

def Traduct(event):

    trans  = Translator()
    global t
    t = ""
    t = T1.get("1.0" , END)

    translated = trans.translate(t, src= source, dest= destination)
    T2.delete('1.0', END)
    T2.insert(END , translated.text)

root = Tk()
root.geometry("800x300")
root.config(bg='#99FCD6')

l = Label(root, text = "LANGUAGE TRANSLATOR\n",font=('calibre',35, 'bold'),bg='#7CFDEA',justify='center')
l.pack()
l.place(x=460,y=30)
labelChooseLang = Label(root, text = "Source language",font=('calibre',10, 'bold'),bg='#99FCD6')
labelChooseLang.config(justify='center')
labelChooseLang.pack()
labelChooseLang.place(x = 180+80 , y = 200)
labelLangTraduct = Label(root, text = "Destination language",font=('calibre',10, 'bold'),bg='#99FCD6')
labelLangTraduct.config(justify='center')
labelLangTraduct.pack()
labelLangTraduct.place(x = 530+400 , y = 200)

img=Image.open('C:/Users/Tamboli/OneDrive/Desktop/Processing/bidirect.png')
resize_image = img.resize((60, 60))
img = ImageTk.PhotoImage(resize_image)
m=Label(root,image=img)
m.config(image=img)
m.pack()

m.place(x=700,y=330)
# m.place(x=900,y=380+130)

languages =['fr' , 'en' , 'es' , 'ar','hi','mr']

combo1 = ttk.Combobox(root, values = languages )
combo1.place(x = 330+80 , y = 200)
combo1.current(0)
combo1.bind("<<ComboboxSelected>>", comboAction)

combo2 = ttk.Combobox(root, values = languages)
combo2.place(x = 690+400 , y = 200)
combo2.current(0)
combo2.bind("<<ComboboxSelected>>", comboAction)

T1 = Text(root )
T1.place(x = 200 , y = 300 , width = 400 , height = 150)
T1.bind("<Return>" , Traduct)

T2 = Text(root)
T2.place(x = 690+200 , y = 300 , width = 400 , height = 150)

root.mainloop()