#!/usr/bin/python3
#encoding:utf8

from wxpy import *
import re
import sqlite3
import time

class select():
    dict = {'0417': 'ict', '0450': 'ibiz', '0455': 'iecon', '0500': 'eng1', '0511': 'eng2', '0580': 'imath',
            '0610': 'ibio', '0620': 'ichem', '0625': 'iphy', '9231': 'fm', '9608': 'cs', '9609': 'abiz',
            '9695': 'lit', '9698': 'psy', '9700': 'abio', '9701': 'achem', '9702': 'aphy', '9708': 'aecon',
            '9709': 'amath'}
    #判断输入的指令是否合法
    def legal(self,data):
        inputs = data.split(' ')
        if re.match(u'^建议', data, re.U) :
            return None
        else:
            for input in inputs:
                if len(input) == 4 and input.isdigit():
                    continue
                elif (len(input) == 2 or len(input) == 1) and input.isdigit():
                    continue
                elif input.lower() in dict.values():
                    continue
                elif input in ['ms', 'qp', 's', 'w','S','W']:
                    continue
                elif re.match('^[0-9]{4}[_]{1}.[0-9]{2}[_]{1}[a-z]{2}[_]{1}[0-9]{2}', input):
                    continue
                else:
                    return False

    def search(self, input):
        lzn = select()

        if lzn.legal(input) == False:
            return None
        elif re.match(u'^建议', input, re.U) or re.match(u'谢', input, re.U):
            return None

        conditions = input.split(' ')
        conn = sqlite3.connect('file_location.sqlite')
        cur = conn.cursor()
        command = ["SELECT location FROM File where"]

        for condition in conditions:
            if len(condition) == 4 and condition.isdigit():
                #year
                if condition.startswith('2'):
                    year = int(condition)
                    command.append('year = %d' % year)
                #syallbus
                elif re.match('^[0][0-9]{3}', condition) or re.match('^[9][0-9]{3}', condition):
                    syllabus = int(condition)
                    command.append('syllabus = %d' % syllabus)
            #month
            elif condition.lower() in ['s','w']:
                month = condition.lower()
                command.append('month = %s' % '"'+month+'"')
            #p_
            elif condition == '1' or condition == '2' or condition == '3' or condition == '4' \
                    or condition == '5' or condition == '6' or condition == '7':
                paper_number = int(condition)
                command.append('paper_number = %d' % paper_number)
            #p__
            elif condition.isdigit() and len(condition) == 2 and condition[0] in ['1','2','3','4','5','6','7']\
                    and condition[1] in ['1','2','3']:
                component = int(condition)
                command.append('component = %d'%component)
            #type:ms or qp
            elif condition == 'ms':
                type = 'ms'
                command.append('type = %s' % '"' + type + '"')
            elif condition == 'qp':
                type = 'qp'
                command.append('type = %s' % '"' + type + '"')
            #subject
            elif condition.lower() in dict.values():
                convert = {'ict': 'ICT', 'ibiz': 'IBiz', 'iecon': 'IEcon', 'eng1': 'Eng1', 'eng2': 'Eng2', 'imath': 'IMath',
                 'ibio': 'IBio', 'ichem': 'IChem', 'iphy': 'IPhy', 'fm': 'FM', 'cs': 'CS', 'abiz': 'ABiz',
                'lit': 'Lit', 'psy': 'Psy', 'abio': 'ABio', 'achem': 'AChem', 'aphy': 'APhy', 'aecon': 'AEcon',
                 'amath': 'AMath'}
                subject = convert[condition.lower()]
                command.append('subject = %s' % '"' + subject + '"')
            elif re.match('^[0-9]{4}[_]{1}.[0-9]{2}[_]{1}[a-z]{2}[_]{1}[0-9]{2}', condition):
                file_name = condition
                command.append('file_name = %s' % '"'+file_name+'"')

        order = ' and '.join(command).replace(' and', '', 1)
        print (order)
        cur.execute(order)

        result = []

        rows = cur.fetchall()
        if len(rows) == 0:
            return '小助手在你给的条件下没有找到一张卷子...😅'
        elif len(rows) > 10:
            return '符合你要求的卷子太多了[捂脸]...请你再多加些条件...'
        else:
            for row in rows:
                for paper in row:
                    result.append(paper)
            return result

bot = Bot(cache_path=True, console_qr=-2)
group = bot.groups()
friends = bot.friends(update=True)
None_group = bot.groups().search(None)

@bot.register(friends,except_self=True)
def return_pdf(msg):
    content = msg.text
    lzn = select()
    if lzn.legal(content) is None and lzn.search(content) is not None:
        results = lzn.search(content)
        if lzn.legal(content) is None and len(results) < 10:
            msg.reply('在你给的条件下我一共找到了%d张卷子，给你( つ•̀ω•́)つ'%len(results))
            for result in results:
                msg.reply_file(result)
        elif lzn.legal(content) is None and len(results) >= 10:
            msg.reply(lzn.search(content))
    elif re.match('谢', content):
        msg.reply('和机器人客气什么[嘿哈]')
    elif re.match('建议',content):
        conn = sqlite3.connect('feedbacks.db')
        conn.text_factory=str
        cur = conn.cursor()
        cur.execute('''INSERT INTO feedback (user_name, time, content)
                                VALUES ( ?, ?, ?)''', (str(msg.chat), int(time.time()), content))
        conn.commit()
        msg.reply('谢谢你的反馈😁小(cheng)助(xu)手(yuan)会尽快做出调整的')
    elif lzn.legal(content) == False:
        msg.reply( '看不懂你的要求啊😢请你再检查一下你输入的条件...')

@bot.register(group)
def ignore(msg):
    return

@bot.register(msg_types=FRIENDS)
def auto_accept_friends(msg):
    global friends
    new_friend = bot.accept_friend(msg.card)
    new_friend.send('同学你好～下面是考试小助手的使用指南')
    new_friend.send_file('ksxzs.pdf')
    new_friend.send('刚加小助手的同学需要等它好友列表更新后才能得到回复，抱歉[捂脸]')
    friends = bot.friends(update = True)

embed()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# ======================================================================================================================
# Imports
# ======================================================================================================================
import sys
import json
from jinja2 import Environment, FileSystemLoader, TemplateError
import pkg_resources
import click

# ======================================================================================================================
# Globals
# ======================================================================================================================
TEMPLATES_DIR = 'data'
TEMPLATE = 'molecule.yml.j2'
OUTPUT_FILE = 'molecule.yml'
__version__ = '1.1.0'

# ======================================================================================================================
# Functions
# ======================================================================================================================
def _load_input_file(file_path):
    """Read and validate the input file contents.

    Args:
        file_path (str): A string representing a valid file path.
    Returns:
        dict: The exit code to return to the shell.
    Raises:
        RuntimeError: invalid path.
    """

    try:
        with open(file_path, 'r') as f:
            json_inventory = json.loads(f.read())
    except IOError:
        raise RuntimeError('Invalid path "{}" for inventory file!'.format(file_path))

    return json_inventory

def generate_hosts_inventory(json_inventory):
    """Build a dictionary of hosts and associated groups from a Ansible JSON inventory file as keys.

    Args:
        json_inventory (dict): A dictionary object representing a Ansible JSON inventory file.
    Returns:
        dict(list): A dictionary of hosts with each host having a list of associated groups.
            { 'host_name': ['group1', 'group2'] }
    """

    inventory_child_groups = {}

    try:
        inventory_hosts = {k: set() for k in json_inventory['_meta']['hostvars'].keys()}
        inventory_groups = {k: v for (k, v) in json_inventory.items() if k != '_meta'}
    except KeyError:
        raise RuntimeError('Expected key(s) missing from inventory file! ("_meta", hostvars")')

    for group_name, group_info in inventory_groups.items():
        if 'children' in group_info and len(group_info['children']) > 0:
            for child in group_info['children']:
                if child in inventory_child_groups.keys():
                    inventory_child_groups[child].add(group_name)
                else:
                    inventory_child_groups[child] = {group_name}

    for group_name, group_info in inventory_groups.items():
        if 'hosts' in group_info.keys():
            for host in group_info['hosts']:
                inventory_hosts[host].add(group_name)

                if group_name in inventory_child_groups.keys():
                    inventory_hosts[host].update(inventory_child_groups[group_name])

    return inventory_hosts

def render_molecule_template(scenario, inventory_hosts, template_file):
    """Create a molecule config file from a template.

    Args:
        scenario (str): The scenario name to be defined as the scenario in the rendering
        inventory_hosts (dict(list(str)): A dictionary of inventory hosts with each host having a list of associated
            groups.
        template_file (str): The template file to use for rendering.
    Returns:
        str: A molecule config file populated with hosts and groups.
    """

    template_path = pkg_resources.resource_filename('moleculerize', TEMPLATES_DIR)
    j2_env = Environment(loader=FileSystemLoader(template_path),
                         trim_blocks=True,
                         lstrip_blocks=True,
                         keep_trailing_newline=True
                         )

    try:
        return j2_env.get_template(template_file).render(
            hosts=inventory_hosts, scenario=scenario)
    except TemplateError as e:
        print(e)
        raise RuntimeError('Template "{}" not found!'.format(template_file))

# ======================================================================================================================
# Main
# ======================================================================================================================
@click.command()
@click.argument('inv_file', type=click.Path(exists=True))
@click.option('--scenario', '-s',
              type=click.STRING,
              default='default',
              help='Molecule config scenario to create.')
@click.option('--template', '-t',
              type=click.STRING,
              default=TEMPLATE,
              help='Molecule config template file')
@click.option('--output', '-o',
              type=click.STRING,
              default=OUTPUT_FILE,
              help='Output file path for molecule config.')
def main(inv_file, scenario, template, output):
    """Build molecule config files from an Ansible dynamic inventory file

    \b
    Required Arguments:
        INV_FILE        A valid Ansible dynamic inventory file
    """

    exit_code = 0

    try:
        inventory_hosts = generate_hosts_inventory(_load_input_file(inv_file))

        try:
            with open(output, 'wb') as f:
                f.write(render_molecule_template(scenario, inventory_hosts, template))
        except IOError:
            raise RuntimeError('Cannot write "{}" Molecule configuration file!'.format(output))

        print("Scenario: {}".format(scenario))
        print("Inventory file: {}".format(inv_file))
        print("Template file: {0}/{1}".format(TEMPLATES_DIR, template))
        print("Output file: {}".format(output))

        print("\nSuccess!")
    except RuntimeError as e:
        exit_code = 1
        print(e)

        print("\nFailed!")

    return exit_code

if __name__ == '__main__':
    sys.exit()


# -*- coding:utf-8 -*-

"""
月と太陽の暦(TEX)
2017/12/10
"""
import ephem
from moon_func import *
from moon_calc import *
import calend_func

year = 2018

moon = ephem.Moon()
sun = ephem.Sun()

location = ephem.Observer()

location.name = '福岡'
location.lon, location.lat = '130.4000', '33.5833'

location.elevation = 0.0

# tex -----------
print("\\documentclass[12pt.a4j]{jsarticle}")
print("\\begin{document}")
print("\\pagestyle{empty}")
print("\\begin{center}")

for month in range(1, 13):

    rise_dic    = {}
    set_dic     = {}
    ecl_dic     = {}

    days = calend_func.month_days(year, month)

    for d in range(0, days):

       day = d+1

       date0 = str(year)+str('/%02d' % month)+str('/%02d' % day)
       location.date = str(date0+' 9:00') #UT+9hr =JST
       date_location = location.date

       today     = Moon(location)
       today.rise(location)
       rise_dic.update({today.index:today})

       location.date= date_location
       today     = Moon(location)
       today.set(location)
       set_dic.update({today.index:today})

       location.date= date_location
       today     = Moon(location)
       today.noon(location)
       ecl_dic.update({today.index:today})

    # tex -----

    print("  {\\large %4d 年 %2d 月}" % (year, month))
    print("  {\\Large 　　　太陽と月の暦   （%s） }" % location.name)
    print("  \\begin{table}[ht]")
    print("  \\begin{center}")
    print("     \\begin{tabular}{|rc|ccc|rc|ccc|}")
    print("     \\hline")
    print("     \\multicolumn{2}{|c|}{日（曜）} & \\multicolumn{3}{c|}{日出　―　日没} & \\multicolumn{2}{c|}{月齢（潮）} & \\multicolumn{3}{c|}{月出　―　月没}\\\\")
    print("     \\hline")

    for d in range(0, days):
        day = d + 1
        date0 = str(year)+str('/%02d' % month)+str('/%02d' % day)
        location.date = str(date0+' 9:00')

        weekday = get_weekday(date0)

        date1  = repr(ephem.localtime(location.date))
        date2  = eval(date1)
        date_str   = date2.strftime('%Y%m%d')
        mm         = int(date2.strftime('%m'))
        dd         = int(date2.strftime('%d'))

        #--- Sun rise -----
        today = Sun(location)
        today.sun_rise(location)
        sun_rise = today.sun_rise

        #--- Sun set -----
        today = Sun(location)
        today.sun_set(location)
        sun_set = today.sun_set

        rise       = rise_dic.get(date_str)

        if (rise is None):
            rise_str     = " --:-- "
        else:
            rise_str = rise.hhmm

        set             = set_dic.get(date_str)

        if (set is None):
            set_str     = " --:-- "
        else:
            set_str     = set.hhmm

        ecl             = ecl_dic.get(date_str)

        if (ecl is None):
            ecl_str     = " ECL none! "
        else:
            ecl_tname   = ecl.tname
            moon_age    = ecl.moonage

        print(' %2d & %s & %s &-& %s & %4.1f & %s & %s &-& %s \\\\'    %(dd,weekday,sun_rise,sun_set,moon_age,ecl_tname,rise_str,set_str))

    print("    \\hline")
    print("    \\end{tabular}")
    print("    \\end{center}")
    print("\\end{table}")
    print("\\newpage")

print("\\end{center}")
print("\\end{document}")


import os
import sys
sys.path.append(os.path.abspath(os.path.dirname(__file__) + '/' + '../..'))

import matplotlib as mpl

mpl.use('pgf')
pgf_with_latex = {
    "pgf.texsystem": "xelatex",
    "pgf.rcfonts": False,
    "text.usetex": True,
    "font.family": "Times New Roman",
    "pgf.preamble": [
        r"\usepackage{fontspec}",
        r"\setmainfont{Times New Roman}",
        r"\usepackage{unicode-math}",
        r"\setmathfont{xits-math.otf}"
        ]
    }
mpl.rcParams.update(pgf_with_latex)

import numpy as np
import datetime as dt
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import matplotlib.patches as mpatches
import seaborn as sns

from python.tools import (
    clean_folder
)

################
## Parameters ##
################

input_folder  = './get_graphs/input'
output_folder = './get_graphs/output/plot_raw_growth_rate'
days_infectious = 7

#######################
## Construct dataset ##
#######################

clean_folder(output_folder)

# Read data
df = pd.read_csv('{}/dataset.csv'.format(input_folder))

# Merge in estimates
df_temp = pd.read_csv('{}/estimated_R.csv'.format(input_folder))
mask = df_temp['days_infectious'] == days_infectious
df_temp = df_temp.loc[mask, ]
df = pd.merge(df, df_temp[['Date', 'Country/Region', 'R']], how = 'left')
df['gr_infected_smoothed'] = (1 / float(days_infectious)) * (df['R'] - 1)
df['Date'] = pd.to_datetime(df['Date'])

# Plot raw data on growth of infected
# for China, Italy, and US
country_list = ['China', 'Italy', 'US', ]
fig, ax = plt.subplots(figsize = (5.0, 4.0))
colors = ['r', 'b', 'g']
df_data_graph_temp = pd.DataFrame()
for country, color in zip(country_list, colors):
    mask = (df['Country/Region'] == country)
    df_temp = df.loc[mask, ].copy()
    plt.plot(df_temp['Date'], df_temp['gr_infected_{}'.format(days_infectious)],
             color = color,
             linestyle = '-',
             alpha = 0.9,
             label = country)
    plt.plot(df_temp['Date'], df_temp['gr_infected_smoothed'],
             color = color,
             linestyle = '--',
             alpha = 0.9)
    df_data_graph_temp = pd.concat([df_data_graph_temp, df_temp[['Country/Region', 'Date',
                                                                 'gr_infected_{}'.format(days_infectious), 'gr_infected_smoothed']]])
plt.legend(frameon = False,
           fontsize = 12)

plt.ylabel('Growth Rate Number of Infected', fontsize = 12)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b-%d'))
plt.gca().xaxis.set_major_locator(mdates.DayLocator(interval = 14))
df_data_graph_temp.to_csv('{}/growth_rate_countries.csv'.format(output_folder), index = False)
fig.savefig("{}/raw_data_infected.png".format(output_folder), bbox_inches = 'tight', dpi = 600)
fig.savefig("{}/raw_data_infected.pgf".format(output_folder), bbox_inches = 'tight', dpi = 600)


from os import path
from pandas import DataFrame
import json

from pytimeparse import parse as pt_parse

from contentai_metadata_flatten.parsers import Flatten

class Parser(Flatten):
    def __init__(self, path_content, logger=None):
        super().__init__(path_content, logger=logger)
        self.EXTRACTOR = "dsai_metadata"
        self.SCORE_DEFAULT_FIXED = 0.75

    @staticmethod
    def known_types():
        """Return the output types for this generator
        :return: list.  List of output types (file types) for this generator
        """
        return ['keyword', 'identity', 'tag', 'scene', 'topic', 'brand', 'shot', 'transcript']

    def parse(self, run_options):
        """Flatten CAE Indexing results

        :param: run_options (dict): specific runtime information
        :returns: (DataFrame): DataFrame on successful decoding and export, None (or exception) otherwise
        """
        dict_data = self.get_extractor_results(self.EXTRACTOR, "metadata.json")

        list_items = []
        list_keywords = []
        if "keywords" in dict_data:  # loop over keywords
            for local_obj in dict_data['keywords']:
                list_keywords.append({'tag':local_obj, 'tag_type':"keyword"})

        # TODO: populate with other keywords (e.g. trending?)

        key_sentence = {}
        if "smartTags" in dict_data:  # loop over smart tags
            for local_type in ["programNE", "epgNE", "commercialNE"]:
                if local_type in dict_data["smartTags"]:
                    for local_obj in dict_data["smartTags"][local_type]:
                        if "neType" in local_obj and "namedEntity" in local_obj and "sentNumber" in local_obj:  # validate object
                            insight_obj = {"tag": local_obj["namedEntity"], "tag_type": "entity",  # could be 'brand' too or 'identity'?
                                "details": {"type": local_obj["neType"], 'description': local_obj['neLabel'], 'weight': local_obj['weight'] } }
                            for sent_id in local_obj["sentNumber"]:
                                sent_id = int(sent_id)
                                if sent_id not in key_sentence:  # save this instance data a this sentence
                                    key_sentence[sent_id] = []
                                key_sentence[sent_id].append(insight_obj)

        if "sent" in dict_data:  # loop over transcripts
            insight_obj = dict_data["sent"]
            for local_obj in insight_obj:
                if "text" in local_obj and "start" in local_obj and len(local_obj["text"]) > 0:  # validate object
                    time_begin = float(local_obj['start'])/1000
                    time_duration = float(local_obj['duration'])/1000
                    detail_obj = { "transcript": local_obj["text"] }
                    if "ccstart" in local_obj:
                        detail_obj['caption'] = {"time_begin": float(local_obj['ccstart'])/1000}
                        detail_obj['caption']["time_end"] = float(local_obj['ccduration'])/1000 + detail_obj['caption']["time_begin"]
                    list_items.append( {"time_begin": time_begin, "source_event": "speech", "tag_type": "transcript",
                        "time_end": time_begin + time_duration, "time_event": time_begin, "tag": Flatten.TAG_TRANSCRIPT,
                        "score": self.SCORE_DEFAULT_FIXED, "details": json.dumps(detail_obj), "extractor": self.EXTRACTOR})

                    # process other named entities that indicted this sentence
                    sent_id = int(local_obj["number"])
                    if sent_id in key_sentence:
                        for insight_obj in key_sentence[sent_id]:
                            list_items.append( {"time_begin": time_begin, "source_event": "speech", "tag_type": insight_obj['tag_type'],
                                "time_end": time_begin + time_duration, "time_event": time_begin, "tag": insight_obj['tag'],
                                "score": self.SCORE_DEFAULT, "details": json.dumps(insight_obj['details']), "extractor": self.EXTRACTOR})

                    # now process quickly for keywords
                    lower_scan = local_obj["text"].lower()
                    for insight_obj in list_keywords:
                        if insight_obj['tag'].lower() in lower_scan:   # just check for presence
                            list_items.append( {"time_begin": time_begin, "source_event": "speech", "tag_type": insight_obj['tag_type'],
                                "time_end": time_begin + time_duration, "time_event": time_begin, "tag": insight_obj['tag'],
                                "score": self.SCORE_DEFAULT, "details": "", "extractor": self.EXTRACTOR})

        if "silence" in dict_data:  # loop over audio
            for local_obj in dict_data['silence']:
                if "start" in local_obj and "duration" in local_obj:  # validate object
                    time_begin = float(local_obj['start'])/1000
                    time_duration = float(local_obj['duration'])/1000
                    list_items.append({"time_begin": time_begin, "source_event": "audio", "tag_type": "tag",
                        "time_end": time_begin + time_duration, "time_event": time_begin, "tag": "silence",
                        "score": self.SCORE_DEFAULT_FIXED, "details": "", "extractor": self.EXTRACTOR})

        if "audio" in dict_data:  # loop over audio concepts
            if 'regions' in dict_data['audio']:
                for local_obj in dict_data['audio']['regions']:
                    if "start" in local_obj and "duration" in local_obj and 'concepts' in local_obj:  # validate object
                        time_begin = float(local_obj['start'])/1000
                        time_duration = float(local_obj['duration'])/1000
                        for score_obj in local_obj['concepts']:
                            list_items.append({"time_begin": time_begin, "source_event": "audio", "tag_type": "tag",
                                "time_end": time_begin + time_duration, "time_event": time_begin, "tag": score_obj['name'],
                                "score": round(float(score_obj['score']), self.ROUND_DIGITS), "details": "", "extractor": self.EXTRACTOR})

        if "commercial" in dict_data:  # loop over scenes
            for local_obj in dict_data['commercial']:
                if "start" in local_obj and "duration" in local_obj:  # validate object
                    time_begin = float(local_obj['start'])/1000
                    time_duration = float(local_obj['duration'])/1000
                    list_items.append({"time_begin": time_begin, "source_event": "video", "tag_type": "scene",
                        "time_end": time_begin + time_duration, "time_event": time_begin, "tag": "commercial",
                        "score": self.SCORE_DEFAULT, "details": "", "extractor": self.EXTRACTOR})

        for local_type in ['tms', 'iab']:  # loop over TMS and IAB concepts
            if local_type in dict_data and 'regions' in dict_data[local_type]:
                for local_obj in dict_data[local_type]['regions']:
                    if "start" in local_obj and "duration" in local_obj and 'concepts' in local_obj:  # validate object
                        time_begin = float(local_obj['start'])/1000
                        time_duration = float(local_obj['duration'])/1000
                        for score_obj in local_obj['concepts']:
                            list_items.append({"time_begin": time_begin, "source_event": "video", "tag_type": "topic",
                                "time_end": time_begin + time_duration, "time_event": time_begin, "tag": score_obj['name'],
                                "score": round(float(score_obj['score']), self.ROUND_DIGITS), "details": "", "extractor": self.EXTRACTOR})

        if "mmimg" in dict_data:  # overall validation
            if 'images' in dict_data['mmimg'] and 'width' in dict_data['mmimg'] and 'height' in dict_data['mmimg']:  # overall validation
                img_width = float(dict_data['mmimg']['width'])
                img_height = float(dict_data['mmimg']['width'])
                img_timing = {}

                img_id_last = -1
                kfcluster_max = self.SCORE_DEFAULT   # this is given in raw distance, so we're going to min/max normalize
                for local_obj in dict_data['mmimg']['images']:  # loop over images for timing construction
                    time_begin = float(local_obj['start'])/1000
                    img_id = int(local_obj['id'])
                    img_timing[img_id] = {'time_begin': time_begin, 'time_end': time_begin}
                    if (img_id - 1) in img_timing:  # seek back for end timing
                        img_timing[img_id - 1]['time_end'] = img_timing[img_id]['time_begin'] - 1/1000
                    img_id_last = img_id
                    if 'kfcluster' in local_obj and len(local_obj['kfcluster']):
                        kfcluster_max = max(kfcluster_max, float(local_obj['kfcluster']['score']))
                if img_id_last > -1 and 'duration' in dict_data:  # time the final image/shot
                    img_timing[img_id_last]['time_end'] = float(dict_data['duration'])/1000 - 1/1000

                for local_obj in dict_data['mmimg']['images']:  # loop over images for other extraction
                    details_obj = {}
                    img_id = int(local_obj['id'])
                    if 'type' in local_obj:  # udpate 0.7.0, make into an array
                        details_obj['shot_type'] = [local_obj['type']]
                    # first, publish the shot for this image
                    list_items.append( {"time_begin": img_timing[img_id]['time_begin'], "source_event": "video", "tag_type": "shot",
                        "time_end": img_timing[img_id]['time_end'], "time_event": img_timing[img_id]['time_begin'], "tag": "shot",
                        "score": self.SCORE_DEFAULT_FIXED, "details": json.dumps(details_obj),
                        "extractor": self.EXTRACTOR})

                    if "face" in local_obj:  # process faces
                        for insight_obj in local_obj['face']:
                            details_obj = {}
                            if 'x' in insight_obj and 'w' in insight_obj:
                                details_obj['box'] = {'w': round(float(insight_obj['w']) / img_width, self.ROUND_DIGITS),
                                    'h': round(float(insight_obj['h']) / img_width, self.ROUND_DIGITS),
                                    'l': round(float(insight_obj['x']) / img_height, self.ROUND_DIGITS),
                                    't': round(float(insight_obj['y']) / img_height, self.ROUND_DIGITS) }
                            if 'rec' in insight_obj:   # specific identity
                                list_items.append( {"time_begin": img_timing[img_id]['time_begin'], "source_event": "face", "tag_type": "identity",
                                    "time_end": img_timing[img_id]['time_end'], "time_event": img_timing[img_id]['time_begin'],
                                    "tag": insight_obj['rec']['name'].replace("_", " "),
                                    "score": float(insight_obj['rec']['confidence']), "details": json.dumps(details_obj),
                                    "extractor": self.EXTRACTOR})

                            if 'cluster' in insight_obj:   # general face cluster
                                list_items.append( {"time_begin": img_timing[img_id]['time_begin'], "source_event": "face", "tag_type": "identity",
                                    "time_end": img_timing[img_id]['time_end'], "time_event": img_timing[img_id]['time_begin'],
                                    "tag": f"face_cluster_{insight_obj['cluster']['id']}",
                                    "score": min(self.SCORE_DEFAULT_FIXED, float(insight_obj['cluster']['score'])), "details": json.dumps(details_obj),
                                    "extractor": self.EXTRACTOR})

                    object_map = {'logo': 'brand', 'object': 'tag'}
                    for local_type in object_map:  # loop over logo and object
                        if local_type in local_obj:
                            for insight_obj in local_obj[local_type]:
                                details_obj = {}
                                if 'x' in insight_obj and 'w' in insight_obj:
                                    details_obj['box'] = {'w': round(float(insight_obj['w']) / img_width, self.ROUND_DIGITS),
                                        'h': round(float(insight_obj['h']) / img_width, self.ROUND_DIGITS),
                                        'l': round(float(insight_obj['x']) / img_height, self.ROUND_DIGITS),
                                        't': round(float(insight_obj['y']) / img_height, self.ROUND_DIGITS) }
                                list_items.append( {"time_begin": img_timing[img_id]['time_begin'], "source_event": "image", "tag_type": object_map[local_type],
                                    "time_end": img_timing[img_id]['time_end'], "time_event": img_timing[img_id]['time_begin'],
                                    "tag": insight_obj['name'].replace("_", " "),
                                    "score": round(min(self.SCORE_DEFAULT_FIXED, float(insight_obj['score'])), self.ROUND_DIGITS), "details": json.dumps(details_obj),
                                    "extractor": self.EXTRACTOR})

                    if 'concept' in local_obj:   # process concepts
                        for insight_obj in local_obj['concept']:
                            list_items.append({"time_begin": img_timing[img_id]['time_begin'], "source_event": "image", "tag_type": "tag",
                                "time_end": img_timing[img_id]['time_end'], "time_event": img_timing[img_id]['time_begin'],
                                "tag": insight_obj['name'], "score": round(float(insight_obj['score']), self.ROUND_DIGITS), "details": "", "extractor": self.EXTRACTOR})

                    if 'kfcluster' in local_obj and len(local_obj['kfcluster']):   # process kfcluster (duplicate frames)
                        details_obj = local_obj['kfcluster']
                        # TODO: investigate whether kfcluster score is a distance or a similarity; this code assumes distance!
                        list_items.append({"time_begin": img_timing[img_id]['time_begin'], "source_event": "image", "tag_type": "scene",
                            "time_end": img_timing[img_id]['time_end'], "time_event": img_timing[img_id]['time_begin'],
                            "tag": "duplicate", "score": 1 - round(float(local_obj['kfcluster']['score']) / kfcluster_max, self.ROUND_DIGITS),
                            "details": json.dumps(details_obj), "extractor": self.EXTRACTOR})

        if "mmpara" in dict_data:  # loop over paragraph segments to make scenes (from speech)
            for local_obj in dict_data['mmpara']:
                if "start" in local_obj and "duration" in local_obj:  # validate object
                    time_begin = float(local_obj['start'])/1000
                    time_duration = float(local_obj['duration'])/1000
                    details_obj = {}
                    if "sentstart" in local_obj and "sentend" in local_obj:  # retain number of sentences
                        details_obj = {'sentences': int(local_obj["sentend"]) - int(local_obj["sentstart"]) + 1}
                    list_items.append({"time_begin": time_begin, "source_event": "speech", "tag_type": "scene",
                        "time_end": time_begin + time_duration, "time_event": time_begin, "tag": "story",
                        "score": self.SCORE_DEFAULT, "details": json.dumps(details_obj), "extractor": self.EXTRACTOR})

        # TODO: additional parsing for these data
        # viewers  --> ??
        # segments  --> scenes?

        if len(list_items) > 0:   # return the whole thing as dataframe
            return DataFrame(list_items)

        if run_options["verbose"]:
            self.logger.critical(f"Missing nested sections from source '{self.EXTRACTOR}'")
        return None


#!/usr/bin/python
# -*- coding: utf-8 -*-
"""Document: network script, keep network always working, using python3"""

import os
import sys
import time
import logging, random
import subprocess
from adb import ADB

PING_RESULT = 0
NETWORK_RESULT = 0
LOGPATH = "output"       # log保存的目录

def DisableNetwork():
    ''' disable network card '''
    result = os.system(u"netsh interface set interface 以太网 disable")
    if result == 1:
        print("disable network card failed")
    else:
        print("disable network card successfully")

def ping():
    trytime = 30
    success = 0
    adb = ADB()
    cmd = "ping -c 4 {0} | grep -q 'ttl=' && echo '{0} ok' || echo '{0} failed'".format('61.135.169.125')
    print(cmd)
    for i in range(trytime):
        result = adb.shell(cmd)
        if 'ok' in result[1]: success += 1
        print(result)
    return success

def init():
    time.sleep(5)
    logPath = os.path.join(LOGPATH, time.strftime('%m-%d-%H_%M_%S',time.localtime(time.time())))
    if not os.path.exists(logPath):
        os.makedirs(logPath)
    global SERIAL_LOG_PATH
    global LOG_PATH
    SERIAL_LOG_PATH = os.path.join(logPath, "serial.log")
    LOG_PATH = os.path.join(logPath, "this.log")
    logging.basicConfig(level=logging.INFO, format='%(asctime)s [line:%(lineno)d] %(levelname)s %(message)s',
                        datefmt='%a, %d %b %Y %H:%M:%S', filename=LOG_PATH, filemode='w')

if __name__ == '__main__':

    logging.info("Start Test!")
    print(ping())
    # while True:
    #     PING_RESULT = ping()
    #     if PING_RESULT == 0:
    #         time.sleep(20)
    #     else:
    #         logging.error(time.clock(),"Ping fail")
    #         DisableNetwork()
    #         time.sleep(10)


# coding:utf-8
import json
import threading

from ttvcloud.ApiInfo import ApiInfo
from ttvcloud.Credentials import Credentials
from ttvcloud.ServiceInfo import ServiceInfo
from ttvcloud.base.Service import Service

class EditService(Service):
    _instance_lock = threading.Lock()

    def __new__(cls, *args, **kwargs):
        if not hasattr(EditService, "_instance"):
            with EditService._instance_lock:
                if not hasattr(EditService, "_instance"):
                    EditService._instance = object.__new__(cls)
        return EditService._instance

    def __init__(self, region='cn-north-1'):
        self.service_info = EditService.get_service_info(region)
        self.api_info = EditService.get_api_info()
        super(EditService, self).__init__(self.service_info, self.api_info)

    @staticmethod
    def get_service_info(region):
        service_info_map = {
            'cn-north-1': ServiceInfo("open.bytedanceapi.com", {'Accept': 'application/json'},
                                      Credentials('', '', 'edit', 'cn-north-1'), 5, 5),
            'ap-singapore-1': ServiceInfo("open.ap-singapore-1.bytedanceapi.com", {'Accept': 'application/json'},
                                          Credentials('', '', 'edit', 'ap-singapore-1'), 5, 5),
            'us-east-1': ServiceInfo("open.us-east-1.bytedanceapi.com", {'Accept': 'application/json'},
                                     Credentials('', '', 'edit', 'us-east-1'), 5, 5),
        }
        service_info = service_info_map.get(region, None)
        if not service_info:
            raise Exception('Cant find the region, please check it carefully')

        return service_info

    @staticmethod
    def get_api_info():
        api_info = {"SubmitDirectEditTaskAsync": ApiInfo("POST", "/", {"Action": "SubmitDirectEditTaskAsync",
                                                                       "Version": "2018-01-01"}, {}, {}),
                    "SubmitDirectEditTaskSync": ApiInfo("POST", "/", {"Action": "SubmitDirectEditTaskSync",
                                                                      "Version": "2018-01-01"}, {}, {}),
                    "GetDirectEditResult": ApiInfo("POST", "/", {"Action": "GetDirectEditResult",
                                                                 "Version": "2018-01-01"}, {}, {}),
                    "SubmitTemplateTaskAsync": ApiInfo("POST", "/", {"Action": "SubmitTemplateTaskAsync",
                                                                     "Version": "2018-01-01"}, {}, {})
                    }
        return api_info

    def submit_direct_edit_task_async(self, body):
        res = self.json('SubmitDirectEditTaskAsync', {}, body)
        if res == '':
            raise Exception("empty response")
        res_json = json.loads(res)
        return res_json

    def submit_direct_edit_task_sync(self, body):
        res = self.json('SubmitDirectEditTaskSync', {}, body)
        if res == '':
            raise Exception("empty response")
        res_json = json.loads(res)
        return res_json

    def get_direct_edit_result(self, body):
        res = self.json('GetDirectEditResult', {}, body)
        if res == '':
            raise Exception("empty response")
        res_json = json.loads(res)
        return res_json

    def submit_template_task_async(self, body):
        res = self.json('SubmitTemplateTaskAsync', {}, body)
        if res == '':
            raise Exception("empty response")
        res_json = json.loads(res)
        return res_json


import tkinter as tk
from tkinter import filedialog
from tkinter import simpledialog
import matplotlib

matplotlib.use("TkAgg")
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import Datasci

# Functions for the buttons of the GUI
def givedata():
    """
    Take input from user about the details of the vehicle being tested
    """
    Make = simpledialog.askstring(
        "Input", "What is the Make of the Test Vehicle", parent=root
    )
    if Make is not None:
        print(Make)

    Model = simpledialog.askstring(
        "Input", "What is the Model of the Test Vehicle", parent=root
    )
    if Model is not None:
        print(Model)

    Year = simpledialog.askinteger(
        "Input", "What is year of the model?", parent=root, minvalue=0, maxvalue=2025
    )
    if Year is not None:
        print(Year)

    Dateformat = simpledialog.askstring(
        "Input", "What is the date? (format: xx/xx/xx)", parent=root
    )
    if Dateformat is not None:
        print(Dateformat)

    Submodel = simpledialog.askstring(
        "Input",
        "What is the submodel of the vehicle?",
        parent=root,
    )
    if Submodel is not None:
        print(Submodel)

    tire_size = simpledialog.askstring(
        "Input",
        "What is the tire size? (Front/Rear) [cm]",
        parent=root,
    )
    if tire_size is not None:
        print(tire_size)

    rolling_radius = simpledialog.askstring(
        "Input",
        "What is the rolling radius size? (Front/Rear) [cm]",
        parent=root,
    )
    if rolling_radius is not None:
        print(rolling_radius)

    vehicle_weight = simpledialog.askinteger(
        "Input",
        "What is gross vehicle weight of the vehicle? [kgf],[N]",
        parent=root,
        minvalue=0,
        maxvalue=10000,
    )
    if vehicle_weight is not None:
        print(vehicle_weight)

    cg_height = simpledialog.askinteger(
        "Input",
        "What is the height of the center of gravity? [m]",
        parent=root,
        minvalue=0,
        maxvalue=2025,
    )
    if cg_height is not None:
        print(cg_height)

    wheelbase = simpledialog.askinteger(
        "Input",
        "What is length  of the wheelbase? [m]",
        parent=root,
        minvalue=0,
        maxvalue=2025,
    )
    if wheelbase is not None:
        print(wheelbase)

    brake_brand = simpledialog.askstring(
        "Input",
        "What is the name/brand of the brake?",
        parent=root,
    )
    if brake_brand is not None:
        print(brake_brand)

    brake_size = simpledialog.askstring(
        "Input",
        "What is the brake size? (eff.radius/outside diam./thickness)[mm]",
        parent=root,
    )
    if brake_size is not None:
        print(brake_size)

    caliper_size = simpledialog.askstring(
        "Input",
        "What is the caliper size? (piston diameter/number of pistons)[mm]",
        parent=root,
    )
    if caliper_size is not None:
        print(caliper_size)

    length_pad = simpledialog.askinteger(
        "Input",
        "What is the length of the Lining (pad) in the sliding direction? [mm]",
        parent=root,
        minvalue=0,
        maxvalue=500,
    )
    if length_pad is not None:
        print(length_pad)

    width_pad = simpledialog.askinteger(
        "Input",
        "What is the width of the Lining (pad) in the perpendicular direction? [mm]",
        parent=root,
        minvalue=0,
        maxvalue=500,
    )
    if width_pad is not None:
        print(width_pad)

    thickness_pad = simpledialog.askinteger(
        "Input",
        "What is the thickness of the Lining (pad)? [mm]",
        parent=root,
        minvalue=0,
        maxvalue=500,
    )
    if thickness_pad is not None:
        print(thickness_pad)

    area_pad = simpledialog.askinteger(
        "Input",
        "What is the area of the Lining (pad)? [mm^2]",
        parent=root,
        minvalue=0,
        maxvalue=5000,
    )
    if area_pad is not None:
        print(area_pad)

    wheel_drive = simpledialog.askstring(
        "Input",
        "What kind of drive system does the vehicle have? (FX2/RX2,AWD,4WD)",
        parent=root,
    )
    if wheel_drive is not None:
        print(wheel_drive)

    inertia = simpledialog.askinteger(
        "Input",
        "What is the calculated Inertia? [kgm^2]",
        parent=root,
        minvalue=0,
        maxvalue=5000,
    )
    if inertia is not None:
        print(inertia)

    shaft_speed = simpledialog.askinteger(
        "Input",
        "What is the shaft speed at 50 km/h? [km/h]",
        parent=root,
        minvalue=0,
        maxvalue=5000,
    )
    if shaft_speed is not None:
        print(shaft_speed)

    braking_torque = simpledialog.askinteger(
        "Input",
        "What is the braking torque at 0.3g? [N*m]",
        parent=root,
        minvalue=0,
        maxvalue=5000,
    )
    if braking_torque is not None:
        print(braking_torque)

    friction_material = simpledialog.askstring(
        "Input",
        "What is the friction material on the pad?",
        parent=root,
    )
    if friction_material is not None:
        print(friction_material)

    return [
        Make,
        Model,
        Dateformat,
        Year,
        Submodel,
        vehicle_weight,
        cg_height,
        wheelbase,
        rolling_radius,
        tire_size,
        brake_brand,
        brake_size,
        thickness_pad,
        caliper_size,
        length_pad,
        width_pad,
        wheel_drive,
        inertia,
        shaft_speed,
        braking_torque,
        friction_material,
        area_pad,
    ]

def PDFconverter():
    """
    convert HTML into PDF dyanamically
    """
    # tools for creating variable template environment converting HTML to PDF
    from jinja2 import Environment, FileSystemLoader
    from weasyprint import HTML, CSS

    # loading the HTML file into an environment to pass variables to it
    file_loader = FileSystemLoader(".")
    env = Environment(loader=file_loader)
    template = env.get_template("FDPBrakes/base.html")

    # list of variables to pass
    [
        Make,
        Model,
        Dateformat,
        Year,
        Submodel,
        vehicle_weight,
        cg_height,
        wheelbase,
        rolling_radius,
        tire_size,
        brake_brand,
        brake_size,
        thickness_pad,
        caliper_size,
        length_pad,
        width_pad,
        wheel_drive,
        inertia,
        shaft_speed,
        braking_torque,
        friction_material,
        area_pad,
    ] = givedata()
    [
        figure1,
        figure2,
        figure3,
        figure4,
        figure5,
        figure6,
        figure7,
        figure8,
    ] = DataSci2()
    template_vars = {
        "Report_Standard": Make,
        "Make_And_Model": Model,
        "Dateformat": Dateformat,
        "Year": Year,
        "Submodel": Submodel,
        "vehicle_weight": vehicle_weight,
        "cg_height": cg_height,
        "wheelbase": wheelbase,
        "rolling_radius": rolling_radius,
        "tire_size": tire_size,
        "brake_brand": brake_brand,
        "brake_size": brake_size,
        "thickness_pad": thickness_pad,
        "caliper_size": caliper_size,
        "length_pad": length_pad,
        "width_pad": width_pad,
        "wheel_drive": wheel_drive,
        "shaft_speed": shaft_speed,
        "braking_torque": braking_torque,
        "friction_material": friction_material,
        "area_pad": area_pad,
        "inertia": inertia,
        "fig2_1": figure1,
        "fig2_2": figure2,
        "fig2_3": figure3,
        "fig2_4": figure4,
        "fig2_5": figure5,
        "fig2_6": figure6,
        "fig2_7": figure7,
        "fig2_8": figure8,
        "fig_1": Datasci.fig1,
        "fig_2": Datasci.fig2,
        "fig_3": Datasci.fig3,
        "fig_4": Datasci.fig4,
        "fig_5": Datasci.fig5,
        "fig_7": Datasci.fig7,
    }

    # Rendering the variables in and then converting the HTML to PDF
    html_out = template.render(template_vars)
    HTML(string=html_out, base_url=".").write_pdf(
        "mockreport.pdf",
        stylesheets=[
            "/Users/Ty/Desktop/FDP_brakes_proj_local/FDPBrakes/Static/basic.css"
        ],
    )

def getdata():
    """
    prompt user for path of the excel file that contains all the test data
    """
    file_path = filedialog.askopenfilename(
        title="Select File", filetypes=[("Excel files", ".xlsx .xls")]
    )
    return file_path

def DataSci2():
    """ This module takes in data and creates graphs from the dataframes"""
    data_path = getdata()
    df1 = pd.read_excel(data_path, sheet_name="Three Sections")
    df1 = df1[df1["Final Temp"] > 0]
    df2 = pd.read_excel(data_path, sheet_name="Three Sections #2")

    sns.set_palette("bright")

    def figure1():
        figure1 = sns.lineplot(
            x="Stop Number",
            y="Friction Level",
            data=df1,
            hue="Test Section",
        )
        return figure1.figure.savefig(
            "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure1.svg"
        )

    def figure2():
        figure2 = sns.lineplot(
            x="Stop Number",
            y="Friction Level",
            hue="Test Section",
            palette="bright",
            data=df2,
        )
        return figure2.figure.savefig(
            "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure2.svg"
        )

    def figure3():
        figure3 = sns.relplot(
            x="Friction Level",
            y="Final Temp",
            col="Test Section",
            palette="bright",
            kind="line",
            data=df1,
        )
        return figure3.savefig(
            "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure3.svg"
        )

    def figure4():
        figure4 = sns.relplot(
            x="Friction Level",
            y="Final Temp",
            col="Test Section",
            palette="bright",
            kind="scatter",
            data=df2,
        )
        return figure4.savefig(
            "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure4.svg"
        )

    def figure5():
        figure5 = sns.catplot(x="Test Section", y="Friction Level", data=df1)
        return figure5.savefig(
            "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure5.svg"
        )

    def figure6():
        fig6 = sns.catplot(
            x="Test Section", y="Friction Level", kind="violin", data=df2
        )
        return fig6.savefig(
            "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure6.svg"
        )

    def figure7():
        fig7 = sns.boxplot(x="Test Section", y="Friction Level", data=df1, whis=np.inf)
        fig7 = sns.stripplot(x="Test Section", y="Friction Level", data=df1, color=".3")
        return fig7.figure.savefig(
            "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure7.svg"
        )

    def figure8():
        y = df1.groupby(["Test Section"]).mean()
        fig8 = sns.barplot(x="Test Section", y="Friction Level", data=y.reset_index())
        return fig8.figure.savefig(
            "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure8.svg"
        )

    plt.clf()
    figure1()
    plt.clf()
    figure2()
    plt.clf()
    figure3()
    plt.clf()
    figure4()
    plt.clf()
    figure6()
    plt.clf()
    figure5()
    plt.clf()
    figure7()
    plt.clf()
    figure8()

    figure1 = "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure1.svg"
    figure2 = "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure2.svg"
    figure3 = "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure3.svg"
    figure4 = "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure4.svg"
    figure5 = "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure5.svg"
    figure6 = "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure6.svg"
    figure7 = "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure7.svg"
    figure8 = "/Users/Ty/Desktop/FDP_brakes_proj_local/Figures/figure8.svg"
    return [figure1, figure2, figure3, figure4, figure5, figure6, figure7, figure8]

if __name__ == "__main__":
    root = tk.Tk()
    b1 = tk.Button(
        root, text="Input Variables, Select Excel Sheet, and Run", command=PDFconverter
    )
    b1.pack()
    root.mainloop()


import sys
from typing import Union

sys.path.append("..")

from utils import constants

import matplotlib.pyplot as plt
import matplotlib
import numpy as np

matplotlib.style.use("seaborn-dark")

def save_episode_duration_graph(
    filename: str,
    durations: list,
    learner: str,
    beaten: Union[int, bool],
    mean_length: int = 10,
) -> None:
    """Creates and saves a plot of episode durations average over the mean length
    puts a vertical line in where the game was beaten for the first time

    Parameters
    ----------
    filename : str
        name of the file to be saved with its location
    durations : list
        list of the duration of each episode over training
    learner : str
        name of the learner
    beaten : int or bool
        an integer of which episode beat the game for the first time, or a bool = False
        showing that the game was never beaten
    mean_length : int, optional
        the period over which to calculate running averages, by default 10
    """
    assert beaten > 0 or beaten == False, "beaten must be a positive integer or False"
    assert mean_length > 0, "mean length must be a positive integer"
    fig, ax = plt.subplots(dpi=600)
    ax.set_xlabel("Episode")
    ax.set_ylabel("Episode duration")
    ax.set_title(f"Episode duration mean over {mean_length} episodes for {learner}")
    means = [
        np.array(durations[x : x + mean_length]).mean()
        for x in range(len(durations) - mean_length)
    ]
    ax.plot(
        [i for i in range(mean_length, len(durations))],
        means,
        color="green",
        linewidth=2,
    )
    ax.annotate(
        f"{len(durations)} episodes",
        (len(durations) - 1, means[-1]),
        xytext=(len(durations) - 1 + 200, means[-1]),
    )
    if beaten:
        plt.axvline(x=beaten)
        plt.text(x=beaten + 5, y=min(means) + 10, s="Game beaten", rotation=90)
    ax.grid()
    fig.savefig(filename)

def save_episode_reward_graph(
    filename: str,
    rewards: list,
    learner: str,
    proportion_decay_over: float,
    episodes: int,
    beaten: int,
    mean_length: int = 10,
) -> None:
    """Creates and saves a plot of episode durations average over the mean length
    puts a vertical line in where the game was beaten for the first time. Also puts
    a vertical line where epsilon was decayed to its minimal value

    Parameters
    ----------
    filename : str
        name of the file to be saved with its location
    rewards : list
        list of episode rewards
    learner : str
        name of the learner
    proportion_decay_over : float
        proportion as a decimal for which epsilon was decayed over
    episodes : int
        number of episodes that were used for training
    beaten : int
        integer for which the game was beaten first or False showing game was not beaten
    mean_length : int, optional
        length to calculate the running average over, by default 10
    """
    assert (
        0 < proportion_decay_over <= 1
    ), "proportion to decay over should be a decimal between 0 and 1"
    assert i > 0 or i == False, "beaten must be a positive integer or False"
    assert mean_length > 0, "mean length must be a positive integer"
    fig, ax = plt.subplots(dpi=600)
    ax.set_xlabel("Episode")
    ax.set_ylabel("Episode total reward")
    ax.set_title(f"Episode reward mean over {mean_length} episodes for {learner}")
    means = [
        np.array(rewards[x : x + mean_length]).mean()
        for x in range(len(rewards) - mean_length)
    ]
    ax.plot(
        [i for i in range(mean_length, len(rewards))],
        means,
        color="green",
        linewidth=2,
    )
    ax.grid()
    plt.axvline(x=proportion_decay_over * episodes)
    plt.text(
        x=proportion_decay_over * episodes + 5,
        y=min(means) + 10,
        s="Epsilon fully decayed",
        rotation=90,
    )
    if beaten:
        plt.axvline(x=beaten, color="r")
        plt.text(x=beaten + 5, y=min(means) + 10, s="Game beaten", rotation=90)
    fig.savefig(filename)

def save_unique_states_graph(filename: str, unique_states: list, learner: str) -> None:
    """Plots the cumulative number of unique states that have been seen at each
    stage of learning

    Parameters
    ----------
    filename : str
        Name of file to be saved and location
    unique_states : list
        List of unique states (cumulative) seen at each stage of learning
    learner : str
        Name of learner
    """
    fig, ax = plt.subplots(dpi=600)
    ax.set_xlabel("Episode")
    ax.set_ylabel("Unique states")
    ax.set_title(f"Number of unique states seen for {learner}")
    ax.plot(
        [i for i in range(len(unique_states))],
        unique_states,
        color="green",
        linewidth=2,
    )
    ax.grid()
    fig.savefig(filename)


import importlib
import traceback
import xcalar.container.context as ctx

logger = ctx.get_logger()

def dynaload(name, path):
    try:
        spec = importlib.util.spec_from_file_location(name, path)
        if spec is None:
            raise Exception(f'No {name} plugin found in {path}')
        return spec.loader.load_module()
    except Exception as e:
        logger.warn(
            f'Failed to load {name} from {path}\n{traceback.format_exc()}')
        raise e


from data import BinPackingExample, Lister, FFD
import itertools

# Select the one you intend to use
import gurobipy as gp
import pyomo as po
import pyscipopt as pso
import mip

def bpp(s, B):
    n = len(s)
    U = len(FFD(s, B))

    # Write here Your model for the compact formulation
    return model

def solveBinPacking(s, B):
    n = len(s)
    U = len(FFD(s, B))
    model = bpp(s, B)
    ## adapt here depending on the solver
    solver = po.SolverFactory('glpk')
    results = solver.solve(model)
    bins = [[] for i in range(U)]
    for (i, j) in model.var_indices:
        if model.x[i, j]() > .5:
            bins[j].append(s[i])
    for i in range(bins.count([])):
        bins.remove([])
    for b in bins:
        b.sort()
    bins.sort()
    return bins

if __name__ == '__main__':
    # s, B = Lister()
    s, B = BinPackingExample()
    bins = solveBinPacking(s, B)
    print(len(bins))
    for b in bins:
        print((b, sum(b)))


import os
import sys
import argparse
import csv
import math
import json
import random
import logging
from collections import defaultdict

import networkx as nx
import numpy as np
import torch

from misc import util
import MatterSim

csv.field_size_limit(sys.maxsize)

angle_inc = math.pi / 6.
NUM_VIEWS = 36

IMAGE_W = 640
IMAGE_H = 480
VFOV = 60

class ImageFeatures(object):

    def __init__(self, image_feature_file, image_feat_size, angle_feat_size):

        logging.info('Loading image features from %s' % image_feature_file)
        tsv_fieldnames = ['scanId', 'viewpointId', 'image_w','image_h', 'vfov', 'features']

        default_features = np.zeros((NUM_VIEWS, image_feat_size), dtype=np.float32)
        self.features = defaultdict(lambda: default_features)

        with open(image_feature_file, "rt") as tsv_in_file:
            reader = csv.DictReader(
                tsv_in_file, delimiter='\t', fieldnames=tsv_fieldnames)
            for item in reader:

                assert int(item['image_h']) == IMAGE_H
                assert int(item['image_w']) == IMAGE_W
                assert int(item['vfov']) == VFOV

                long_id = item['scanId'] + '_' + item['viewpointId']
                features = np.frombuffer(util.decode_base64(item['features']),
                    dtype=np.float32).reshape((NUM_VIEWS, image_feat_size))
                self.features[long_id] = features

    def get_features(self, scan, viewpoint):
        long_id = scan + '_' + viewpoint
        return self.features[long_id]

class MatterportWorldMeta(object):

    def __init__(self, config):

        self.config = config
        self.angle_feat_size = config.nav_agent.model.angle_feat_size
        self.image_feat_size = config.nav_agent.model.image_feat_size

        self.load_graphs()
        self.load_cached_adjacent_lists()
        self.load_image_features()
        self.make_angle_features()

    def load_graphs(self):
        scan_list_file = os.path.join(
            self.config.data_dir, 'connectivity/scans.txt')
        scans = set(open(scan_list_file).read().strip().split('\n'))
        self.graphs = {}
        self.paths = {}
        self.distances = {}
        for scan in scans:
            graph = util.load_nav_graphs(scan, self.config.data_dir)
            self.graphs[scan] = graph
            self.paths[scan] = dict(nx.all_pairs_dijkstra_path(graph))
            self.distances[scan] = dict(nx.all_pairs_dijkstra_path_length(graph))

    def load_cached_adjacent_lists(self):
        cached_action_space_path = os.path.join(
            self.config.data_dir, 'panoramic_action_space.json')
        with open(os.path.join(cached_action_space_path)) as f:
            self.cached_adj_loc_lists = json.load(f)

    def load_image_features(self):
        image_feature_path = os.path.join(self.config.data_dir, 'img_features',
            self.config.world.image_feature_file)
        self.featurizer = ImageFeatures(image_feature_path, self.image_feat_size, self.angle_feat_size)

    def make_angle_features(self):
        self.angle_features = []
        for i in range(NUM_VIEWS):
            embeddings = np.zeros((NUM_VIEWS, self.angle_feat_size), dtype=np.float32)
            base_heading = (i % 12) * angle_inc
            for j in range(NUM_VIEWS):
                heading = (j % 12) * angle_inc
                heading -= base_heading
                elevation = ((j // 12) - 1) * angle_inc
                embeddings[j, :] = util.build_angle_features(heading, elevation, self.angle_feat_size)
            self.angle_features.append(embeddings)

class MatterportWorld(object):

    STOP_ACTION = 0

    def __init__(self, meta):

        self.config = meta.config

        self.featurizer = meta.featurizer
        self.graphs = meta.graphs
        self.paths = meta.paths
        self.distances = meta.distances
        self.cached_adj_loc_lists = meta.cached_adj_loc_lists
        self.angle_features = meta.angle_features
        self.angle_feat_size = meta.angle_feat_size
        self.image_feat_size = meta.image_feat_size

        self.init_simulator()

        self.random = random.Random(self.config.seed)

    def init(self, poses):
        self.sim.newEpisode(*list(zip(*poses)))
        return MatterportState(self)

    def init_simulator(self):
        self.sim_batch_size = self.config.trainer.batch_size
        self.sim = MatterSim.Simulator()
        self.sim.setRenderingEnabled(False)
        self.sim.setDiscretizedViewingAngles(True)
        self.sim.setCameraResolution(IMAGE_W, IMAGE_H)
        self.sim.setCameraVFOV(math.radians(VFOV))
        self.sim.setNavGraphPath(
            os.path.join(self.config.data_dir, 'connectivity'))
        self.sim.setBatchSize(self.sim_batch_size)
        self.sim.initialize()

    def get_shortest_path(self, scan, start, end):
        return self.paths[scan][start][end]

    def get_weighted_distance(self, scan, start, end):
        return self.distances[scan][start][end]

    def get_unweighted_distance(self, scan, start, end):
        return len(self.get_shortest_path(scan, start, end)) - 1

    def get_path_length(self, scan, path):
        length = 0
        for i, v in enumerate(path[1:]):
            u = path[i]
            length += self.get_weighted_distance(scan, u, v)
        return length

class MatterportState(object):

    def __init__(self, world):

        self.world = world

        self.states = []

        for sim_state in world.sim.getState():

            state = util.Struct()
            state.scan      = sim_state.scanId
            state.viewpoint = sim_state.location.viewpointId
            state.view_id   = sim_state.viewIndex
            state.heading   = sim_state.heading
            state.elevation = sim_state.elevation
            state.location  = sim_state.location

            long_id = '_'.join(
                [state.scan, state.viewpoint, str(state.view_id % 12)])
            state.adj_loc_list = world.cached_adj_loc_lists[long_id]

            state.curr_view_features = world.featurizer.get_features(
                state.scan, state.viewpoint)

            state.action_embeddings = util.build_action_embeddings(
                state.adj_loc_list, state.curr_view_features, world.angle_feat_size)

            state.curr_view_features = np.concatenate(
                (state.curr_view_features, world.angle_features[state.view_id]), -1)

            self.states.append(state)

    def __getitem__(self, i):
        return self.states[i]

    def __iter__(self):
        return iter(self.states)

    def __len__(self):
        return len(self.states)

    def step(self, actions):

        navigable_view_indices = []
        next_viewpoint_ids = []
        next_view_indices = []

        for i, (state, action) in enumerate(zip(self.states, actions)):
            loc = state.adj_loc_list[action]
            next_viewpoint_ids.append(loc['nextViewpointId'])
            next_view_indices.append(loc['absViewIndex'])
            navigable_view_indices.append(loc['absViewIndex'])

        new_states = self._navigate_to_locations(next_viewpoint_ids,
            next_view_indices, navigable_view_indices)

        return new_states

    def _navigate_to_locations(self,
            next_viewpoint_ids, next_view_indices, navigable_view_indices):

        sim = self.world.sim

        # Rotate to the view index assigned to the next viewpoint
        heading_deltas, elevation_deltas = util.calculate_headings_and_elevations_for_views(
            sim, navigable_view_indices)

        sim.makeAction(
            [0] * len(heading_deltas), heading_deltas, elevation_deltas)

        states = sim.getState()
        locationIds = []

        assert len(states) == len(next_viewpoint_ids) == len(navigable_view_indices)

        zipped_info = zip(states,
                          next_viewpoint_ids,
                          navigable_view_indices)

        for i, (state, next_viewpoint_id, navigable_view_index) in enumerate(zipped_info):

            # Check if rotation was done right
            assert state.viewIndex == navigable_view_index

            # Find index of the next viewpoint
            index = None
            for i, loc in enumerate(state.navigableLocations):
                if loc.viewpointId == next_viewpoint_id:
                    index = i
                    break
            assert index is not None, state.scanId + ' ' + state.location.viewpointId + ' ' + next_viewpoint_id
            locationIds.append(index)

        # Rotate to the target view index
        heading_deltas, elevation_deltas = util.calculate_headings_and_elevations_for_views(
            sim, next_view_indices)

        sim.makeAction(locationIds, heading_deltas, elevation_deltas)

        # Final check

        states = sim.getState()
        zipped_info = zip(states, next_viewpoint_ids, next_view_indices)
        for state, next_viewpoint_id, next_view_index in zipped_info:
            assert state.viewIndex == next_view_index
            assert state.location.viewpointId == next_viewpoint_id

        return MatterportState(self.world)


from typing import Dict, List, Optional
from tests.mocks.services import MockMLModel, MockSLAM
from viam.services.slam import Pose, SLAM
from viam.services.mlmodel import Metadata, MLModel
from viam.utils import ValueTypes

class ExampleMLModel(MLModel):
    def __init__(self, name: str):
        self.input_data = MockMLModel.INPUT_DATA
        self.output_data = MockMLModel.OUTPUT_DATA
        self.meta = MockMLModel.META
        super().__init__(name)

    async def infer(self, input_data: Dict[str, ValueTypes], *, timeout: Optional[float] = None) -> Dict[str, ValueTypes]:
        return self.output_data

    async def metadata(self, *, timeout: Optional[float] = None) -> Metadata:
        return self.meta

class ExampleSLAM(SLAM):
    def __init__(self, name: str):
        self.position = MockSLAM.POSITION
        self.internal_chunks = MockSLAM.INTERNAL_STATE_CHUNKS
        self.point_cloud_chunks = MockSLAM.POINT_CLOUD_PCD_CHUNKS
        super().__init__(name)

    async def get_internal_state(self, **kwargs) -> List[bytes]:
        return self.internal_chunks

    async def get_point_cloud_map(self, **kwargs) -> List[bytes]:
        return self.point_cloud_chunks

    async def get_position(self, **kwargs) -> Pose:
        return self.position


#!/usr/bin/python3.4

#####################################################
##              This is the buggyBot               ##
##   the first robotics Project from dandrews7396  ##
#####################################################

from datetime import datetime
import time
import os
import numpy as np
import explorerhat as eh
import cv2
import scipy
from picamera import PiCamera
from io import BytesIO, StringIO
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
from mpu6050 import mpu6050

## Set-up a class that forms the basis for our buggyBot this will allow us to update
## the bot as we go.

class buggyBot():
    """A buggy that's going to roam over the house. A lot of these will be None."""

    def __init__(self):
        self.start_time = None # Start system time
        self.time = None # Start the navigation time
        self.total_time = None # Total time passed
        self.img = None # No image to begin with
        self.img_array = np.empty((240, 320, 3), dtype=np.uint8) # Image will also be saved as np array, so no need to convert.
        self.yaw = None # Not yet initialised
        self.pitch = None # Not yet initialised
        self.roll = None # Not yet initialised
        self.vel = None # Not yet initialised
        self.steer = None # Not yet initialised
        self.motor1 = eh.motor.one
        self.motor2 = eh.motor.two
        self.max_vel = 100
        ## Add in address using sudo i2cdetect -y 1
        self.mpu_sensor = mpu6050(0x68)
        self.x_acc_ctr = None   ## These will need
        self.y_acc_ctr = None   ## Populating on flat, level
        self.z_acc_ctr = None   ## When stationary.
        self.camera = PiCamera()
        self.buggy_vision = np.zeros((160,230,3), dtype = np.float)
        self.world_map = np.zeros((200, 200, 3), dtype = np.float)
        self._img_file = './images/'

    def update(self):
        """ Update buggyBot as time passes."""
        if self.start_time == None:
            self.start_time = time.time()
            self.total_time = 0

            if self.mpu_sensor.read_accel_range != 2:
                self.mpu_sensor.set_accel_range(0x00)
            if self.mpu_sensor.read_gyro_range != 250:
                self.mpu_sensor.set_gyro_range(0x00)
            self.camera.resolution = (320, 240)
            self.camera.framerate = 24
            self.camera.rotation = 180
            self.img, self.img_array = self._take_picture()
            self.display_image(init=True)

        else:
            tot_time = time.time() - self.start_time
            if np.isfinite(tot_time):
                self.total_time = tot_time
            ## If using an arduino, see `accelArdData.py` for helper code
            mpu_data = self.mpu_sensor.get_all_data()
            ## Output mpu_data to console, for debug
            print(mpu_data)
            mpu_temp = mpu_data[0]
            mpu_accel = mpu_data[1]
            mpu_gyro = mpu_data[2]
            self.pitch = self._calc_pitch(mpu_accel)
            self.roll = self._calc_roll(mpu_accel)
            self.img, self.img_array = self._take_picture()
            self.display_image()

    def _calc_pitch(self, accel_data):
        """ Calculate pitch angle of buggyBot."""
        x = accel_data["x"]
        y = accel_data["y"]
        z = accel_data["z"]
        denom = np.sqrt(y**2 + z**2)
        return np.arctan2(x, denom)

    def _calc_roll(self, accel_data):
        """ Calculate roll angle of buggyBot."""
        x = accel_data["x"]
        y = accel_data["y"]
        z = accel_data["z"]
        denom = np.sqrt(x**2 + z**2)
        return np.arctan2(y, denom)

    def _take_picture(self):
        """ Take picture as both standard image an np array."""
        timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3]
        array_out = np.empty((240, 320, 3), dtype=np.uint8)
        self.camera.capture(array_out, 'rgb')
        std_img = self.camera.capture(self._img_file + '{}.jpg'.format(timestamp))
        return std_img, array_out

    def display_image(self, init=False):
        """Display latest image, overlay with IMU data"""
        plt.clf()
        images = sorted(os.listdir(self._img_file))
        imgpath = self._img_file + images[-1]
        img = mpimg.imread(imgpath)
        plt.imshow(img)
        if self.pitch and self.roll:
            txt = 'Pitch: {:.3f}\nRoll: {:.3f}'.format(self.pitch, self.roll)
            overlay = plt.text(310, 50, txt,
                               horizontalalignment='right', color='white')
        plt.axis('off')
        plt.draw()
        if init:
            plt.show(block=False)

    def forwards(self, speed=100):
        """
        This method is taken straight from the gpiozero library.
        Simply calls the forward method for both motors via explorerhat

        Speed is defaulted to 100, to match explorerhat default.
        """

        self.motor1.forwards(speed)
        self.motor2.forwards(speed)

    def backwards(self, speed=100):
        """
        Again taken from gpiozero, calls explorerhat motor `backwards()`
        method for both motors.

        speed is again set to 100, as per the explorerhat default.
        """

        self.motor1.backwards(speed)
        self.motor2.backwards(speed)

    def stop(self):
        """
        Calls speed = 0 on both motors, via the explorerhat
        `motor.stop()` method.
        """
        self.motor1.stop()
        self.motor2.stop()

    ## These following methods will need to be augmented
    ## with either an internal or external function to
    ## match a required heading, once an integrated
    ## heading sensor has been installed.

    ## Can be used 'as is' for an RC solution.

    def turn_left(self, speed=100):
        """
        Call motor methods in opposite directions,
        in order to turn left.

        left backwards, right forwards. same speed.
        """
        self.motor1.backwards(speed)
        self.motor2.forwards(speed)

    def turn_right(self, speed=100):
        """
        As before, opposite direction.
        """
        self.motor1.forwards(speed)
        self.motor2.backwards(speed)


#!/usr/bin/env python
import matplotlib.pyplot as plt
import numpy as np
from skimage.io import imread
from skimage import img_as_ubyte

from skimage.util import random_noise

def task_5_example1(do_savefig = False):
    coins = imread("data/hand.png")
    bc    = np.bincount(coins.ravel(), minlength = 256)
    cdf   = np.cumsum(bc) / bc.sum()
    mask1 = (coins > 50) & (coins < 90)
    mask2 = (coins > 105) & (coins < 120)
    mask3 = (coins > 185) & (coins < 205)
    th1 = mask1
    th2 = mask1 | mask2
    th3 = mask1 | mask2 | mask3

    xs = range(256)

    fig, axes = plt.subplots(2, 3, figsize = (9, 6))
    ax1, ax2, ax3, ax4, ax5, ax6 = axes.ravel()

    ax1.imshow(coins, cmap = "gray")

    ax2b = ax2.twinx()
    ax2.bar(xs, bc)
    ax2b.plot(xs, cdf, color = "red")

    ax3.imshow(th1, cmap = "gray")
    ax4.imshow(th2, cmap = "gray")
    ax5.imshow(th3, cmap = "gray")
    ax6.axis("off")

    ax1.set_title("hand.tiff")
    ax2.set_title("Histogram/CDF")
    ax3.set_title("Thresholding band at 50..90.")
    ax4.set_title("Thresholding bands at\n50..90 and 105..120.")
    ax5.set_title("Thresholding bands at 50..90,\n105..120, and 185..205.")

    fig.tight_layout()
    fig.show()
    if do_savefig:
        fig.savefig("figures/task_5_1_example1.png")

def task_5_example2(do_savefig = False):
    coins  = imread("data/coins.png")
    coins2 = img_as_ubyte(random_noise(coins, mode = "gaussian"))

    bc  = np.bincount(coins.ravel(), minlength = 256)
    bc2 = np.bincount(coins2.ravel(), minlength = 256)
    bc2[0] = bc2[-1] = 0

    cdf  = np.cumsum(bc)  / bc.sum()
    cdf2 = np.cumsum(bc2) / bc2.sum()

    th  = coins  > 95
    th2 = coins2 > 125

    xs = range(256)

    fig, [[ax1, ax2, ax_foo],
          [ax3, ax4, ax_bar]] = plt.subplots(2, 3, figsize = (12, 8))
    ax2b = ax2.twinx()
    ax4b = ax4.twinx()

    ax1.imshow(coins, cmap = "gray")

    ax2.bar(xs, bc)
    ax2b.plot(cdf, color = "red")

    ax3.imshow(coins2, cmap = "gray")
    ax4.bar(xs, bc2)
    ax4b.plot(cdf2, color = "red")

    ax1.set_title("Original img")
    ax2.set_title("Histogram/CDF (original img)")

    ax3.set_title("Noisy img (Gaussian noise)")
    ax4.set_title("Histogram/CDF (noisy img)")

    ax_foo.imshow(th, cmap = "gray")
    ax_foo.set_title("Original img thresholded at 100")
    ax_bar.imshow(th2, cmap = "gray")
    ax_bar.set_title("Noisy img thresholded at 125")

    fig.tight_layout()
    fig.show()
    if do_savefig:
        fig.savefig("figures/task_5_1_example2.png")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Sep  8 12:22:39 2021

@author: Attila
"""

import argparse
import os
import shutil
import time
import math

import torch
import torch.nn as nn
import torch.nn.parallel
import torch.multiprocessing as mp
import torch.backends.cudnn as cudnn
import torch.distributed as dist
import torch.optim
import torch.utils.data
import torch.utils.data.distributed

from torchvision.models.resnet import resnet18, resnet50
from imagenet_tfrecord import ImageNet_TFRecord

from torch.nn.parallel import DistributedDataParallel as DDP
from network import marco_softmax

# item() is a recent addition, so this helps with backward compatibility.
def to_python_float(t):
    if hasattr(t, 'item'):
        return t.item()
    else:
        return t[0]

def main_process(args):
    # set address for master process to localhost since we use a single node
    os.environ['MASTER_ADDR'] = 'localhost'
    os.environ['MASTER_PORT'] = '12355'

    # use all gpus pytorch can find
    args.world_size = torch.cuda.device_count()
    print('Found {} GPUs:'.format(args.world_size))
    for i in range(args.world_size):
        print('{} : {}'.format(i, torch.cuda.get_device_name(i)))

    # total batch size = batch size per gpu * ngpus
    args.total_batch_size = args.world_size * args.batch_size

    # TODO: find out what this stuff does
    print("\nCUDNN VERSION: {}\n".format(torch.backends.cudnn.version()))
    cudnn.benchmark = True
    assert torch.backends.cudnn.enabled, "Amp requires cudnn backend to be enabled."

    if not len(args.data):
        raise Exception("error: No data set provided")

    # start processes for all gpus
    mp.spawn(gpu_process, nprocs=args.world_size, args=(args,))

def gpu_process(gpu, args):
    # each gpu runs in a separate proces
    torch.cuda.set_device(gpu)
    torch.distributed.init_process_group(backend='nccl', init_method='env://',
                                         rank=gpu, world_size=args.world_size)

    # create model
    model = resnet50(pretrained=args.pretrained)
    model = nn.Sequential(model, marco_softmax(1000))

    # Set cudnn to deterministic setting
    if args.deterministic:
        cudnn.benchmark = False
        cudnn.deterministic = True
        torch.manual_seed(gpu)
        torch.set_printoptions(precision=10)

    # push model to gpu
    model = model.cuda(gpu)

    # Scale learning rate based on global batch size
    args.lr = args.lr*float(args.batch_size*args.world_size)/256.
    optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum,
                                weight_decay=args.weight_decay)

    # Use DistributedDataParallel for distributed training
    model = DDP(model, device_ids=[gpu], output_device=gpu)

    # define loss function (criterion) and optimizer
    criterion = nn.NLLLoss().cuda(gpu)
    best_prec1 = 0

    # Optionally resume from a checkpoint
    if args.resume:
        # Use a local scope to avoid dangling references
        def resume():
            if os.path.isfile(args.resume):
                print("=> loading checkpoint '{}'".format(args.resume))
                checkpoint = torch.load(args.resume, map_location=lambda storage, loc: storage.cuda(gpu))
                args.start_epoch = checkpoint['epoch']
                best_prec1 = checkpoint['best_prec1']
                model.load_state_dict(checkpoint['state_dict'])
                optimizer.load_state_dict(checkpoint['optimizer'])
                print("=> loaded checkpoint '{}' (epoch {})"
                      .format(args.resume, checkpoint['epoch']))
                return best_prec1
            else:
                print("=> no checkpoint found at '{}'".format(args.resume))
                return 0
        best_prec1 = resume()

    # Data loading code
    train_loader = ImageNet_TFRecord(args.data, 'train', args.batch_size, args.workers,
                                     gpu, args.world_size, augment=True)
    val_loader = ImageNet_TFRecord(args.data, 'val', args.batch_size, args.workers,
                                   gpu, args.world_size, augment=False)

    # only evaluate model, no training
    if args.evaluate:
        validate(val_loader, model, criterion, gpu, args)
        return

    total_time = AverageMeter()
    for epoch in range(args.start_epoch, args.epochs):
        # train for one epoch
        avg_train_time = train(train_loader, model, criterion, optimizer, epoch, gpu, args)
        total_time.update(avg_train_time)

        # if in test mode quit after 1st epoch
        if args.test:
            break

        # evaluate on validation set
        [prec1, prec5] = validate(val_loader, model, criterion, gpu, args)

        # remember best prec@1 and save checkpoint
        if gpu == 0:
            is_best = prec1 > best_prec1
            best_prec1 = max(prec1, best_prec1)
            save_checkpoint({
                'epoch': epoch + 1,
                'state_dict': model.state_dict(),
                'best_prec1': best_prec1,
                'optimizer': optimizer.state_dict(),
            }, is_best)
            if epoch == args.epochs - 1:
                print('##Top-1 {0}\n'
                      '##Top-5 {1}\n'
                      '##Perf  {2}'.format(
                          prec1,
                          prec5,
                          args.total_batch_size / total_time.avg))

        train_loader.reset()
        val_loader.reset()

def train(train_loader, model, criterion, optimizer, epoch, gpu, args):
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to train mode
    model.train()
    end = time.time()
    for i, data in enumerate(train_loader):
        input = data[0]["data"]
        target = data[0]["label"].squeeze().cuda(gpu).long()
        train_loader_len = int(math.ceil(train_loader._size / args.batch_size))

        # lr schedule
        adjust_learning_rate(args.lr, optimizer, epoch, i, train_loader_len)

        # if in test mode, quit after 100 iterations
        if args.test and i > 100:
            break

        # compute output
        output = model(input)
        loss = criterion(torch.log(output), target)

        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if i % args.print_freq == 0:
            # Every print_freq iterations, check the loss, accuracy, and speed.
            # For best performance, it doesn't make sense to print these metrics every
            # iteration, since they incur an allreduce and some host<->device syncs.

            # Measure accuracy
            prec1, prec5 = accuracy(output.data, target, topk=(1, 5))

            # Average loss and accuracy across processes for logging
            reduced_loss = reduce_tensor(loss.data, args.world_size)
            prec1 = reduce_tensor(prec1, args.world_size)
            prec5 = reduce_tensor(prec5, args.world_size)

            # # to_python_float incurs a host<->device sync
            losses.update(to_python_float(reduced_loss), input.size(0))
            top1.update(to_python_float(prec1), input.size(0))
            top5.update(to_python_float(prec5), input.size(0))

            torch.cuda.synchronize()
            batch_time.update((time.time() - end)/args.print_freq)
            end = time.time()

            if gpu == 0:  # only print for main process
                print('Epoch: [{0}][{1}/{2}]\t'
                      'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                      # 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                      'Speed {3:.3f} ({4:.3f})\t'
                      'Loss {loss.val:.10f} ({loss.avg:.4f})\t'
                      'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                      'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                           epoch, i, train_loader_len,
                           args.world_size*args.batch_size/batch_time.val,
                           args.world_size*args.batch_size/batch_time.avg,
                           batch_time=batch_time,
                           loss=losses, top1=top1, top5=top5))

    return batch_time.avg

def validate(val_loader, model, criterion, gpu, args):
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to evaluate mode
    model.eval()

    end = time.time()

    for i, data in enumerate(val_loader):
        input = data[0]["data"]
        target = data[0]["label"].squeeze().cuda(gpu).long()
        val_loader_len = int(val_loader._size / args.batch_size)

        # compute output
        with torch.no_grad():
            output = model(input)
            loss = criterion(torch.log(output), target)

        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        reduced_loss = reduce_tensor(loss.data, args.world_size)
        prec1 = reduce_tensor(prec1, args.world_size)
        prec5 = reduce_tensor(prec5, args.world_size)
        losses.update(to_python_float(reduced_loss), input.size(0))
        top1.update(to_python_float(prec1), input.size(0))
        top5.update(to_python_float(prec5), input.size(0))

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        # TODO:  Change timings to mirror train().
        if gpu == 0 and i % args.print_freq == 0:
            print('Test: [{0}/{1}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Speed {2:.3f} ({3:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                  'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                       i, val_loader_len,
                       args.world_size * args.batch_size / batch_time.val,
                       args.world_size * args.batch_size / batch_time.avg,
                       batch_time=batch_time, loss=losses,
                       top1=top1, top5=top5))

    print(' * Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'.format(top1=top1, top5=top5))

    return [top1.avg, top5.avg]

def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'):
    torch.save(state, filename)
    if is_best:
        shutil.copyfile(filename, 'model_best.pth.tar')

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def adjust_learning_rate(lr, optimizer, epoch, step, len_epoch):
    """LR schedule that should yield 76% converged accuracy with batch size 256"""
    factor = epoch // 30

    if epoch >= 80:
        factor = factor + 1

    lr = lr*(0.1**factor)

    """Warmup"""
    if epoch < 5:
        lr = lr*float(1 + step + epoch*len_epoch)/(5.*len_epoch)

    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res

def reduce_tensor(tensor, world_size):
    rt = tensor.clone()
    dist.all_reduce(rt, op=dist.reduce_op.SUM)
    rt /= world_size
    return rt

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='PyTorch ImageNet Training')
    parser.add_argument('data', metavar='DIR', nargs='*', help='path(s) to dataset',
                        default='/tudelft.net/staff-bulk/ewi/insy/CV-DataSets/imagenet/tfrecords')
    parser.add_argument('-j', '--workers', default=2, type=int, metavar='N',
                        help='number of data loading workers per GPU (default: 2)')
    parser.add_argument('--epochs', default=90, type=int, metavar='N',
                        help='number of total epochs to run')
    parser.add_argument('--start-epoch', default=0, type=int, metavar='N',
                        help='manual epoch number (useful on restarts)')
    parser.add_argument('-b', '--batch-size', default=64, type=int,
                        metavar='N', help='mini-batch size per process (default: 64)')
    parser.add_argument('--lr', '--learning-rate', default=0.001, type=float,
                        metavar='LR', help='Initial learning rate.  Will be scaled by <global batch size>/64: args.lr = args.lr*float(args.batch_size*args.world_size)/256.  A warmup schedule will also be applied over the first 5 epochs.')
    parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
                        help='momentum')
    parser.add_argument('--weight-decay', '--wd', default=1e-4, type=float,
                        metavar='W', help='weight decay (default: 1e-4)')
    parser.add_argument('--print_freq', '-p', default=10, type=int,
                        metavar='N', help='print frequency (default: 10)')
    parser.add_argument('--resume', default='', type=str, metavar='PATH',
                        help='path to latest checkpoint (default: none)')
    parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true',
                        help='evaluate model on validation set')
    parser.add_argument('--pretrained', dest='pretrained', action='store_true',
                        help='use pre-trained model')
    parser.add_argument('--dali_cpu', action='store_true',
                        help='Runs CPU based version of DALI pipeline.')
    parser.add_argument('--deterministic', action='store_true')
    parser.add_argument('-t', '--test', action='store_true',
                        help='Run short training script.')
    args = parser.parse_args()

    print(args)

    main_process(args)


import datetime
import psutil
from django.shortcuts import render
from django.views.generic import View

# import forms
from djangoaddicts.hostutils.forms import HostProcessFilterForm

class GetHostProcesses(View):
    @staticmethod
    def get_process_count(process_list: list, status: str) -> int:
        """get a count of processes for a given status

        Args:
            process_list (list): list of processes as returned from psutil.process_iter(
            status (str): name of process status to count

        Returns:
            int: number of processes of 'status'
        """
        count = 0
        for process in process_list:
            try:
                if process.status() == status:
                    count += 1
            except psutil.NoSuchProcess:
                continue
        return count

    def get(self, request):
        """Get host prcesses"""
        context = {}
        process_list = list(psutil.process_iter())
        filter_form = HostProcessFilterForm(request.GET or None)
        context["counts"] = {
            "running": self.get_process_count(process_list, "running"),
            "sleeping": self.get_process_count(process_list, "sleeping"),
            "idle": self.get_process_count(process_list, "idle"),
            "stopped": self.get_process_count(process_list, "stopped"),
            "zombie": self.get_process_count(process_list, "zombie"),
            "dead": self.get_process_count(process_list, "dead"),
        }

        if request.GET.dict().get("clear", None):
            context["clear_filter"] = False

        else:
            if filter_form.is_valid():
                context["clear_filter"] = True

                if filter_form.cleaned_data.get("status", None):
                    filtered_process_list = []
                    for i in process_list:
                        if i.status() in filter_form.cleaned_data["status"]:
                            filtered_process_list.append(i)
                    process_list = filtered_process_list

                if filter_form.cleaned_data.get("created_at__gte", None):
                    filtered_process_list = []
                    for i in process_list:
                        if i.create_time() > filter_form.cleaned_data["created_at__gte"].timestamp():
                            filtered_process_list.append(i)
                    process_list = filtered_process_list
                if filter_form.cleaned_data.get("created_at__lte", None):
                    filtered_process_list = []
                    for i in process_list:
                        if i.create_time() < filter_form.cleaned_data["created_at__lte"].timestamp():
                            filtered_process_list.append(i)
                    process_list = filtered_process_list

        context["process_list"] = process_list
        context["title"] = "Host Processes"
        context["now"] = datetime.datetime.now()
        context["subtitle"] = psutil.os.uname()[1]
        filter_form = {}
        filter_form["form"] = HostProcessFilterForm(request.GET or None)
        filter_form["modal_name"] = "filter_processes"
        filter_form["modal_size"] = "modal-lg"
        filter_form["modal_title"] = "Filter Host Processes"
        filter_form["hx_method"] = "hx-get"
        filter_form["hx_url"] = "/hostutils/get_host_processes"
        filter_form["hx_target"] = "id_process_list_container"
        filter_form["method"] = "GET"
        filter_form["action"] = "Filter"
        context["filter_form"] = filter_form
        return render(request, template_name="hostutils/bs5/snippets/host_process_card_swap.htm", context=context)


from pyspark.sql import SparkSession
from pyspark.sql.functions import *
spark = SparkSession.builder.getOrCreate()
rawDF = spark.read.json("cricket.json", multiLine = "true")
innings=rawDF.select(explode("innings").alias("innings"))
#topDF = innings.select(col("innings.team"),explode("innings.overs.deliveries").alias("overs"),col("overs.batter"),col("overs.bowler"))

#top1DF=topDF.select(explode("overs.batter").alias("batter"),explode("overs.bowler").alias("bowler"))
top1DF=innings.select(col("innings.team"),explode("innings.overs.deliveries").alias("batbowl"))
topDF=top1DF.select(col("batbowl.batter").alias("batter"),col("batbowl.bowler").alias("bowler"))
top2DF=topDF.select(col("batter"),col("bowler"))
#length=len(top2DF.select('batter').take(1)[0][0])
#top2DF.select([top2DF.batter]+[top2DF.batter[i] for i in range(length)]).unique().show()
top6DF=top2DF.withColumn("batter",arrays_zip("batter")).select("bowler",explode("batter").alias("merge")).select("bowler",col("merge.batter"))
runs1=innings.withColumn("deliveries",explode("innings.overs.deliveries")).withColumn("runs",explode("deliveries.runs")).select("runs.total")
df1=top6DF.withColumn("id",monotonically_increasing_id())
df2=runs1.withColumn("id",monotonically_increasing_id())
df3=df2.join(df1,"id","outer").drop("id")
#df4=df3.groupBy("batter","bowler").sum().alias("total")
#df4=df3.groupBy("batter","bowler").agg(sum("total").alias("total"))
df8=df3.groupBy("batter","bowler").agg(sum("total"))
#df8.show(df8.count())
df5=df8.orderBy("batter",ascending=False)
#top3DF=top2DF.drop_duplicates(['bowler'])
top3DF=df5.withColumn("result",array_distinct("bowler"))
#top4DF=top3DF.withColumn("result1",array_distinct("batter"))
#top2DF.select("batter",top2DF.element[0],top2DF.element[1]).show()
#top3DF.show()
top3DF.show(top3DF.count())
#print((top3DF.count(), len(top3DF.column)))


#-*-coding:utf8;-*-
#qpy:console

from geojson import Point, Feature, FeatureCollection, dump
import csv

input_csv = '/storage/emulated/0/Download/All stations Taiwan.csv'
geojson_output = '/storage/emulated/0/Download/All stations Taiwan.geojson'

data = []
attributes = []

with open(input_csv) as csv_file:
    print('Reading CSV.....')
    reader = csv.reader(csv_file)
    for row in reader:
        data.append(row)

#print(data)
attributes = data[0]
input_data = data[1:]
#print(input_data)
#print(attributes)
features = []
num_stations = len(input_data)
num_attributes = len(attributes)

print('Writing Geojson file....')
for i in range(num_stations):
    #print(i)
    lon = float(input_data[i][num_attributes-2])
    lat = float(input_data[i][num_attributes-1])
    point = Point((lon, lat))
    feature_dict = {}

    for j in range(num_attributes-2):
        feature_dict[attributes[j]] = str(input_data[i][j])

    features.append(Feature(geometry=point, properties=feature_dict))

# add more features...
# features.append(...)

feature_collection = FeatureCollection(features)

with open(geojson_output, 'w') as f:
   dump(feature_collection, f)


import logging

from config import CRAWLER_LOG_FILE_PATH
from crawler import Crawler
from device import Device
from path import Path
import os
package = "com.tencent.mm"
root_activity = "com.tencent.mm.ui.LauncherUI"
guide_directory = os.path.abspath("./data/wechat_red_packet2")
device_serial = ""
strings_path =  os.path.abspath("./strings/wechat_strings.txt")
cluster_dir =  os.path.abspath("./data/wechat001")

logging.basicConfig(level=logging.DEBUG,
                    filename=CRAWLER_LOG_FILE_PATH,
                    datefmt='%m-%d %H:%M:%S',
                    format='[%(asctime)s] [%(levelname)s] [%(filename)s] [%(funcName)s] - %(message)s')

if __name__ == "__main__":
    logging.info("Program started.")

    device = Device(device_serial)
    crawler = Crawler(package, root_activity, device, strings_path, cluster_dir)

    guide_path = Path()
    guide_path.load(guide_directory)
    crawler.follow_guide_path(guide_path)