import argparse
import sys
import os
import gpxpy
# had to clone the latest github version of gpxfile to get it to work:
# pip install git+https://github.com/tkrajina/gpxpy.git
from glob import glob
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from mpl_toolkits.mplot3d import Axes3D

def plot_files(lats, lons, alts, names):
    # make plot
    lat = [item for sublist in lats for item in sublist]
    lon = [item for sublist in lats for item in sublist]
    mapb = Basemap(projection='gall', resolution='l',
                   urcrnrlat=max(lat)+.01,
                   llcrnrlon=max(lon)+.01,
                   llcrnrlat=min(lat)-.01,
                   urcrnrlon=min(lon)-.01,
                   )
    colors = plt.cm.rainbow(np.linspace(0, 1, len(lats)))
    fig = plt.figure(figsize=(12,8))
    ax = Axes3D(fig)
    #ax = fig.add_subplot(111, projection='3d')
    for xx in range(len(lats)):
        lat = lats[xx]
        lon = lons[xx]
        alt = alts[xx]
        name = names[xx]
        yu, xu = mapb(np.asarray(lon),
                   np.asarray(lat))
        #mapb.scatter(yu, xu, c=alt, s=3, edgecolor="None", alpha=0.5)
        #parallels = np.linspace(min(lat),max(lat),5)
        #mapb.drawparallels(parallels,labels=[True, False, False, False],fontsize=10)

        #meridians = np.linspace(min(lon),max(lon),5)
        #mapb.drawmeridians(meridians,labels=[False, True, False, False],fontsize=10)
        ax.scatter(yu, xu, alt, edgecolor="None", c=colors[xx], label=name)
        #plt.colorbar()
    plt.legend()
    plt.show()

def parse_gpx(gpxfile):
    gpxfo = open(gpxfile, 'r')
    gpx = gpxpy.parse(gpxfo)
    lat = []
    lon = []
    t = []
    alt = []
    yaw = []
    pitch = []
    roll = []
    for track in gpx.tracks:
        for segment in track.segments:
            for point in segment.points:
                lat.append(point.latitude)
                lon.append(point.longitude)
                alt.append(point.elevation)
                print(point)
    return lat, lon, alt

if __name__ == '__main__':
    aparser = argparse.ArgumentParser(description="Read gpx")
    aparser.add_argument('--f', dest='input_files',
                         help="file to read data from", nargs="+")
    aparser.add_argument('--g', dest='glob', default='',
                         help="recursive glob search for files", type=str)
    #parser.add_argument('output_file', help="file to write data to")
    # parse command line
    try:
        args = aparser.parse_args()
    except :
        aparser.print_help()
        sys.exit()
    ifiles = []
    if len(args.glob):
        # do glob search based on paramters
        if str(args.glob) == 'thesis':
            print("searching thesis space")
            g = '/Volumes/johannah_external/thesis-work/201511_sea_state_DRI_Sikululiaq/uas_data/*/n2/log/*.gpx'
        else:
            g = args.glob
        ifiles = sorted(glob(g))
    else:
        ifiles = args.input_files

    print(ifiles)
    if not len(ifiles):
        print("no valid files found")
        aparser.print_help()
        sys.exit()
    lats = []
    lons = []
    alts= []
    cs = []
    names = []
    for xx, ifile in enumerate(ifiles):
        if not os.path.exists(ifile):
            print("Error: input file, %s does not exist" %ifile)
        else:
            if 1:
                lat,lon,alt = parse_gpx(ifile)
                lats.append(lat)
                lons.append(lon)
                alts.append(alt)
                names.append(os.path.split(ifiles[xx])[1])
    if len(lats):
        plot_files(lats, lons, alts, names)


import pandas as pd  # Create Data Frame (as excel tables)
import numpy as np   # Evaluate types from pandas
import re            # Evaluate Regular expressions
import pyodbc        # Save informmation to Microsoft Access Data Base
import pyproj        # Proj coordinates from planes to geographycs (lon, lat)
import os            # Manage directory and files path

def promaxdb2df(doc):
    """
    Return pandas DataFrame from csv Promax geometry file.

    Argumets:
    doc -- csv Promax geometry file
    skiprows -- number of skipped rows of header file
    name -- header column names in DataFrame
    """
    skiprows, header = _skiprows_header_before(doc, '<')
    names = header.split()
    names = list(map(_correcth2Obug, names))
    names[0] = names[0] + '*'
    names = names + ['']
    df = pd.read_table(doc, delimiter='|', engine='python', skiprows=skiprows,
                       header=None, names=names, skipfooter=1)\
        .dropna(axis=1, how='all')
    df[names[0]] = df[names[0]].apply(lambda x: int(re.sub(r'^<\s+', '', x)))
    df.set_index(names[0], inplace=True)
    return df

def _skiprows_header_after(doc, mark):
    return_next = False
    for n, r in enumerate(open(doc)):
        if return_next:
            return n, r
        if r[:len(mark)] == mark:             # Found mark!. So break next line
            return_next = True

def _skiprows_header_before(doc, mark):
    for n, r in enumerate(open(doc)):
        if r[:len(mark)] == mark:             # Found mark!. So break next line
            return n, rold
        rold = r

def _widths_skiprows(doc):
    skiprows, header = _skiprows_header_after(doc, '<<')
    w = [m.start() for m in re.finditer(r'\s+', header)]    # Get width columns
    w = [t-s for s, t in zip(w, w[1:])]             # Difference between values
    w[0] = w[0]+1                               # A space more to evit problems
    return w, skiprows

def _correcth2Obug(name):
    """
    Frecuently H2O_DEP is written as H20_DEP, changing the letter O by number 0
    This function correct this error and return H2O_DEP (letter O)
    """
    p = re.compile(r'H2[0oO]_DEP')
    return p.sub('H2O_DEP', name)

def geo2df(doc, source):
    """
    Return pandas DataFrame from csv Promax geometry file.

    Argumets:
    doc -- csv Promax geometry file
    source -- (SIN, SRF or CDP)
    """
    widths, skiprows = _widths_skiprows(doc)
    df = pd.read_fwf(doc, skiprows=skiprows, widths=widths).dropna(how='all')
    df.set_index('STATION', inplace=True)
    warningmess = 'Warning: source should be \"SRF, SIN or CDP\"'
    print(warningmess) if source not in ('SIN', 'SRF', 'CDP') else None
    df.index.names = ['%sGEO*' % source]
    return df

def promax2meta(doc, target):
    """
    Return meta information (Line or Area) of csv Promax geometry file.

    Arguments:
    doc -- csv Promax geometry file
    target -- meta information to get (Line or Area)
    """
    ptarget = r'' + re.escape(target) + r'\s*[=:]\s*\"?([\w-]+)\"?'
    for line in open(doc):
        result = (re.search(ptarget, line, re.I))
        if result:
            return result.group(1)

def targetInCol(df, target):
    """
    Return meta information (Line or Area) from information in a column of DF.

    Arguments:
    doc -- csv Promax geometry file
    target -- meta information to get (Line or Area)
    """
    c = list(df.columns)
    ptarget = r''+re.escape(target)
    i = [i for i, x in enumerate(c) if re.search(ptarget, x, re.I)]
    return df.iloc[0][i[0]] if i else None

def _chano(ano):
    return '19' + ano if len(ano) == 2 else ano

def normNameLine(name):
    """
    Return normalized name of Line to ANH standars

    Arguments:
    name -- name of Line to normalize
    """
    pline = re.compile(r'^([a-zA-Z]+)[_-]*([0-9]+)[_-]+([0-9]+)')
    search = pline.search(name)
    try:
        name, ano, num = pline.search(name).groups()
    except (AttributeError, TypeError):
        print('Error: Line \"%s\" don\'t match as Line name' % name)
    else:
        return (name + '-' + _chano(ano) + '-' + num).upper()

def normNameArea(name):
    """
    Return normalized name of Area to ANH standars

    Arguments:
    name -- name of Area to normalize
    """
    parea = re.compile(r'^([a-zA-Z]+)[_-]+([0-9]+)')
    try:
        name, ano = parea.search(name).groups()
    except (AttributeError, TypeError):
        print('Error: Area \"%s\" don\'t match as Area name' % name)
    else:
        return (name + '-' + ano).upper()

def addmeta(df, *values):
    """
    update inplace DataFrame with meta information
    Arguments:
    df -- pandas DataFrame to modify
    values -- (colheader, value) to insert in DataFrame
    """
    normName = {
        'Line': normNameLine,
        'Area': normNameArea
    }
    for name, value in values:
        nsname = _chname(name)  # starname = ALGO*, nsname = nostarname = ALGO
        if targetInCol(df, nsname):
            df[nsname.upper()] = \
                normName[nsname.capitalize()](targetInCol(df, nsname))
            df.rename(columns={nsname: name}, inplace=True)
        else:
            df[name] = value

def planesMagna2geo(*planes):
    """
    Return tuple geographycs WGS84 coordinates:
    (lat1, lat2, ...), (lon1, lon2, ...)
    from magna Bogota planes Gauss-Kruger projection coordinates (x, y)

    Arguments:
    planes: Tuple coordinates (x1, x2, ...), (y1, y2, ...)
    """
    # Create instance of projection using EPSG:3116
    # MAGNA-SIRGAS / Colombia Bogota zone
    # (http://spatialreference.org/ref/epsg/3116/)
    # Validated with (Augustin Codazzi, 2004, pag 56-65):
    # http://www.igac.gov.co/wps/wcm/connect/91311780469f77c3aff6bf923ecdf8fe/aspectos+practicos.pdf?MOD=AJPERES
    magna = pyproj.Proj('+proj=tmerc \
        +lat_0=4.596200416666666 +lon_0=-74.07750791666666 \
        +k=1 +x_0=1000000 +y_0=1000000 \
        +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs')
    return magna(*planes, inverse=True)

def _chtype(x):
    if x == 'float64': return 'double'
    if x == 'int64': return 'int'
    if x == 'O': return 'text'

def _chformat(x):
    if type(x) == np.dtype(float): return float(x)
    if type(x) == np.dtype('int64'): return float(x)
    if type(x) == np.dtype(str): return str(x)

def _chname(x):
    """
    Returns changed name. Removing the last *
    """
    return x[:-1] if x[-1] == '*' else x

def cols(df):
    """
    Returns [(index, type, props), (col1, type1, props1), ...]
    for field names DataFrame in correct format to Microsoft Access DataBase

    Arguments:
    df: DataFrame
    """
    c = df.columns.values.tolist()
    t = list(map(_chtype, df.dtypes.tolist()))
    # Is primary? True if field name ends with *
    p = ['PRIMARY KEY' if x[-1] == '*' else '' for x in c]
    c = list(map(_chname, c))

    return list(zip(
                [_chname(df.index.name)] + c,
                [_chtype(df.index.dtype)] + t,
                ['PRIMARY KEY'] + p
                ))

def createTable(doc, fields, tableName):
    """
    Create table in Microsoft Access DataBase

    Arguments:
    doc: C:\\Route\\To\\FILE.accdb
    fields: field1, field2, ... , fieldn (String of fields sep by commas)
    tableName: Name of Table in Access
    """
    odbc = 'DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=%s;' % doc
    conn = pyodbc.connect(odbc)
    cursor = conn.cursor()

    try:
        cursor.execute("CREATE TABLE %s(%s)" % (tableName, ','.join(fields)))
        conn.commit()
    except pyodbc.ProgrammingError:
        print('Warning: Table %s already exists' % tableName)

    cursor.close()
    del cursor
    conn.close()

def addPrimaryKeys(doc, primaryKeys, tableName):
    """
    Add Primay keys at table in Microsoft Access DataBase

    Arguments:
    doc: C:\\Users\\Route\\To\\FILE.accdb
    fields: primary1, primary2, ... , primarym (String of fields sep by commas)
    tableName: Name of Table in Access
    """
    odbc = 'DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=%s;' % doc
    conn = pyodbc.connect(odbc)
    cursor = conn.cursor()
    try:
        cursor.execute("ALTER TABLE %s ADD PRIMARY KEY (%s)"
                       % (tableName, ','.join(primaryKeys)))
        conn.commit()
    except Exception:
        print('Warning: Primary keys (%s) already exits. Not altered table %s'
              % (','.join(primaryKeys), tableName))

    cursor.close()
    del cursor
    conn.close()

def to_access(doc, df, tableName=None):
    """
    Save Pandas DataFrame to Microsoft Access DataBase

    Arguments:
    doc: C:\\Users\\Route\\To\\FILE.accdb
    df: Pandas DataFrame
    tableName: Name of access table (optional: name of first column)
    """
    ctp = cols(df)                    # (Col,type, isPrimaryKey) list of tuples
    c = [c for c, t, p in ctp]        # Only cols list
    fields = [c + ' ' + t for c, t, p in ctp]
    pk = [c for c, t, p in ctp if p == 'PRIMARY KEY']  # prim keys
    tableName = c[0] if not tableName else tableName
    print('Inserting data of Table %s in access ...' % tableName)

    createTable(doc, fields, tableName)
    addPrimaryKeys(doc, pk, tableName)

    # Insert DataFrame
    odbc = 'DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=%s;' % doc
    conn = pyodbc.connect(odbc)
    cursor = conn.cursor()

    sql_ins = "INSERT INTO %s(%s) VALUES (%s)" \
        % (tableName, ','.join(c), ','.join(['?']*len(c)))

    for row in df.itertuples():
        try:
            cursor.execute(sql_ins, tuple(map(_chformat, row)))
            conn.commit()
        except pyodbc.IntegrityError:
            print('%s = %s already exists. Not inserted' % (c[0], row[0]))

    cursor.close()
    del cursor
    conn.close()

def promaxcsvfiles(datafolder):
    info = []
    for dirname, dirnames, filenames in os.walk(datafolder):
        for filename in filenames:
            path = os.path.join(dirname, filename)
            base, dom = os.path.split(path)
            dom = os.path.splitext(dom)[0]
            base, line = os.path.split(base)
            base, area = os.path.split(base)
            info = info + [[area, line, dom, path]]
    return info

def to_database(AREA, LINE, PROMAX_FILE, ACCESS_DATABASE_FILE):
    # line = normNameLine(promax2meta(SIN_PROMAX_FILE, 'Line'))

    dom = promaxdb2df(PROMAX_FILE)  # dom = cdp, sin or srf
    addmeta(dom, ('LINE*', LINE), ('AREA*', AREA))
    dom['LONGITUDE'], dom['LATITUDE'] = planesMagna2geo(
         list(dom['X_COORD']), list(dom['Y_COORD']))

    to_access(ACCESS_DATABASE_FILE, dom)

def main():
    workfolder = 'C:\\Users\\Usuario9\\Desktop\\JOHN\\1.ANH_SISMICA\\'
    ACCESS_DATABASE_FILE = workfolder + 'DOC_SISMICA\\GEODATABASE2.mdb'
    datafolder = workfolder + 'DOC_SISMICA\\PROMAX_CSV'
    info = promaxcsvfiles(datafolder)
    for index, (area, line, dom, path) in enumerate(info):
        print('[%.1f%%] [%s %s %s] [file no. %d of %d] ...'
              % (index/len(info)*100, area, line, dom, index+1, len(info)))
        to_database(area, line, path, ACCESS_DATABASE_FILE)

if __name__ == "__main__":
    main()


import argparse
import os
import shutil
import torch
import platform
import torch.multiprocessing as multiprocessing

from distar.ctools.utils import read_config

from distar.actor import Actor
from distar.agent.import_helper import import_module

from distar.ctools.worker.coordinator.coordinator import Coordinator, create_coordinator_app
from distar.ctools.worker.league.league import League
from distar.ctools.worker.league.league_api import create_league_app
import time

def learner_run(config, args):
    import threading
    def start_app(learner):
        torch.set_num_threads(1)
        learner_app = learner.create_rl_learner_app(learner)
        learner_app.run(host=learner._ip, port=learner._port, debug=False, use_reloader=False)

    config.learner.player_id = args.player_id
    learner_config_path = os.path.join(os.getcwd(), 'experiments', config.common.experiment_name,
                                       config.learner.player_id, 'rl_user_config.yaml')
    address_path = os.path.join(os.getcwd(), 'experiments', config.common.experiment_name, config.learner.player_id,
                                'address')
    cluster_address_path = os.path.join(os.getcwd(), 'experiments', config.common.experiment_name,
                                        config.learner.player_id, 'clusters')
    if os.path.exists(address_path):
        shutil.rmtree(address_path, ignore_errors=True)
    if os.path.exists(cluster_address_path):
        shutil.rmtree(cluster_address_path, ignore_errors=True)
    try:
        os.makedirs(os.path.dirname(learner_config_path))
        os.makedirs(cluster_address_path)
    except:
        pass
    shutil.copyfile(args.config, learner_config_path)
    RLLearner = import_module(config.learner.agent, 'RLLearner')
    if config.learner.use_distributed:
        learner = RLLearner(config, "torch", args.init_method, args.rank, args.world_size)
    else:
        learner = RLLearner(config, "single_node")

    thread = threading.Thread(target=start_app, args=(learner,), daemon=True)
    thread.start()
    learner.run()

def actor_run(config, args):
    if args.gpu_batch_inference == 'true':
        config.actor.gpu_batch_inference = True
    else:
        config.actor.gpu_batch_inference = False
    if config.actor.job_type == 'train':
        exp_name = config.common.experiment_name
        replay_path = os.path.abspath(config.env.replay_dir)
        new_replay_path = os.path.join(replay_path, exp_name)
        config.env.replay_dir = new_replay_path
    actor = Actor(config)
    actor.run()

def league_run(config, args):
    exp_config_path = os.path.join(os.getcwd(), 'experiments', config.common.experiment_name, 'rl_user_config.yaml')
    try:
        os.makedirs(os.path.dirname(exp_config_path))
    except:
        pass
    shutil.copyfile(args.config, exp_config_path)

    time_label = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime(time.time()))
    config_dir = os.path.join(os.getcwd(), 'experiments', config.common.experiment_name, 'config')
    if not os.path.exists(config_dir):
        os.makedirs(config_dir)
    bk_exp_config_path = os.path.join(config_dir, f'rl_user_config_{time_label}.yaml')
    shutil.copyfile(args.config, bk_exp_config_path)

    default_model_path = os.path.join(os.path.dirname(__file__), 'sl_model.pth')
    for idx, p in enumerate(config.league.active_players.checkpoint_path):
        if p == 'default':
            config.league.active_players.checkpoint_path[idx] = default_model_path
    for idx, p in enumerate(config.league.historical_players.checkpoint_path):
        if p == 'default':
            config.league.historical_players.checkpoint_path[idx] = default_model_path
    for idx, p in enumerate(config.league.active_players.teacher_path):
        if p == 'default':
            config.league.active_players.teacher_path[idx] = default_model_path
    league = League(config)
    league_app = create_league_app(league)
    league_app.run(host=config.communication.coordinator_ip, port=config.communication.league_port, debug=False,
                   use_reloader=False)

def coordinator_run(config, args):
    coordinator = Coordinator(config)
    coordinator_app = create_coordinator_app(coordinator)
    coordinator_app.run(host=config.communication.coordinator_ip, port=config.communication.coordinator_port,
                        debug=False, use_reloader=False)

if __name__ == '__main__':
    if os.path.exists(r'C:\Program Files (x86)\StarCraft II'):
        sc2path = r'C:\Program Files (x86)\StarCraft II'
    elif os.path.exists('/Applications/StarCraft II'):
        sc2path = '/Applications/StarCraft II'
    else:
        assert 'SC2PATH' in os.environ.keys(), 'please add StarCraft2 installation path to your environment variables!'
        sc2path = os.environ['SC2PATH']
        assert os.path.exists(sc2path), 'SC2PATH: {} does not exist!'.format(sc2path)
    if not os.path.exists(os.path.join(sc2path, 'Maps/Ladder2019Season2')):
        shutil.copytree(os.path.join(os.path.dirname(__file__), '../envs/maps/Ladder2019Season2'), os.path.join(sc2path, 'Maps/Ladder2019Season2'))

    parser = argparse.ArgumentParser(description="rl_train")
    parser.add_argument("--config", default=os.path.join(os.path.dirname(__file__), 'rl_user_config.yaml'))
    parser.add_argument("--type", default=None)
    parser.add_argument("--task", default='bot')
    parser.add_argument("--player_id", default='MP0')
    parser.add_argument("--gpu_batch_inference", default='false')
    parser.add_argument("--init_method", type=str, default=None)
    parser.add_argument("--rank", type=int, default=0)
    parser.add_argument("--world_size", type=int, default=1)
    args = parser.parse_args()
    config = read_config(args.config)
    config.common.type = 'rl'
    config.actor.traj_len = config.learner.data.trajectory_length
    config.learner.var_record_type = 'alphastar'
    if args.init_method is not None:
        config.learner.use_distributed = True
    if args.task == 'bot':
        config.league.vs_bot = True
        config.learner.use_value_feature = False
    elif args.task == 'selfplay':
        config.league.vs_bot = False
    else:
        raise NotImplementedError
    context_str = 'spawn'
    mp_context = multiprocessing.get_context(context_str)

    if args.type is None:
        p_coordinator = mp_context.Process(target=coordinator_run, args=(config, args))
        p_coordinator.start()
        time.sleep(10)
        p_league = mp_context.Process(target=league_run, args=(config, args))
        p_league.start()
        time.sleep(10)
        p_learner = mp_context.Process(target=learner_run, args=(config, args))
        p_learner.start()
        time.sleep(10)
        actor_run(config, args)
    elif args.type == 'coordinator':
        coordinator_run(config, args)
    elif args.type == 'learner':
        learner_run(config, args)
    elif args.type == 'league':
        league_run(config, args)
    elif args.type == 'actor':
        actor_run(config, args)


import os
import json
import time

from collections import OrderedDict
from work import *

def json_userData_update(userName_code_match):
    ## 슬랙 봇 최초 실행하거나, 새로운 유저가 들어왔을 때 돌아간다.
    exitence = os.path.isfile("data.json")  ## 파일 있는지 체크
    if exitence == False:  ## 파일이 없다면.
        file_data = OrderedDict()
        file_data["userName_code_match"] = userName_code_match  ## 현재 슬랙에 참여하고 있는 사람들 정보 json에 저장한다.

        with open('data.json', 'w', encoding="utf-8") as make_file:  ## json 파일 작성.
            json.dump(file_data, make_file, ensure_ascii=False, indent="\t")
    else:  ## 파일이 있다면 현재 있는 사람들 데이터만 업데이트.
        ## 파일 데이터 불러온다.
        with open('data.json', encoding="utf-8") as data_file:
            file_data = json.load(data_file, object_pairs_hook=OrderedDict)
            ## 파일 데이터의 userName_code_match만 새로 쓴다 -> 업데이트
            file_data["userName_code_match"] = userName_code_match
        ## json 파일 쓰기.
        with open('data.json', 'w', encoding="utf-8") as make_file:
            json.dump(file_data, make_file, ensure_ascii=False, indent="\t")

def command_User_Event(command_Number, eventJson):
    userCode = eventJson["user"]
    nowDate = time.strftime('%Y%m%d')
    nowTime = time.strftime('%H%M')
    channel = eventJson["channel"]

    with open('data.json', encoding="utf-8") as data_file:
        jsonData = json.load(data_file, object_pairs_hook=OrderedDict)

    ## 사용자별 오브젝트가 만들어졌는지? 안만들어져있으면 만들어라.
    json_file_user_insert(jsonData, userCode)
    ## 사용자 안에서 오늘날짜 만들어졌는지? 안만들어져있으면 만들자.
    json_file_date_insert(jsonData, userCode, nowDate)

    DM_str = "준비중입니다."
    if command_Number == 0:

        ## 시작 시간 배열에 넣기.
        insert_juge = json_file_startTime_insert(jsonData, userCode, nowDate, nowTime)
        if insert_juge:
            userChannelUpdate(jsonData, userCode, channel)
            userName = jsonData["userName_code_match"][userCode]
            strTime = time.strftime("%Y년 %m월 %d일 %H시 %M분")
            DM_str = userName + "님이 " + strTime + "부터 일하는 시간을 체크합니다."
            return DM_str
        else:
            DM_str = "이미 일을 하고 있습니다."
            return DM_str
    elif command_Number == 1:
        insert_juge, exceptionDM = json_file_reststart_insert(jsonData, userCode, nowDate, nowTime)
        if insert_juge:
            userChannelUpdate(jsonData, userCode, channel)
            userName = jsonData["userName_code_match"][userCode]
            strTime = time.strftime("%Y년 %m월 %d일 %H시 %M분")
            DM_str = userName + "님이 " + strTime + "부터 쉬기 시작했습니다."
            return DM_str
        else:
            DM_str = exceptionDM
            return DM_str
    elif command_Number == 2:
        ## 휴식 정지
        insert_juge, exceptionDM = json_file_restend_insert(jsonData, userCode, nowDate, nowTime)
        if insert_juge:
            userChannelUpdate(jsonData, userCode, channel)
            userName = jsonData["userName_code_match"][userCode]
            strTime = time.strftime("%Y년 %m월 %d일 %H시 %M분")
            DM_str = userName + "님이 " + strTime + "부터 다시 일하기 시작했습니다."
            return DM_str
        else:
            DM_str = exceptionDM
            return DM_str
    elif command_Number == 3:
        insert_juge, exceptionDM = json_file_workingend_insert(jsonData, userCode, nowDate, nowTime)
        if insert_juge:
            userChannelUpdate(jsonData, userCode, channel)
            userName = jsonData["userName_code_match"][userCode]
            strTime = time.strftime("%Y년 %m월 %d일 %H시 %M분")
            DM_str = userName + "님이 " + strTime + "에 일하기를 종료했습니다."
            return DM_str
        else:
            DM_str = exceptionDM
            return DM_str
    elif command_Number == 4:
        insert_juge, exceptionDM = json_file_yes_insert(jsonData, userCode, nowDate, nowTime)

        if insert_juge:
            userChannelUpdate(jsonData, userCode, channel)
            userName = jsonData["userName_code_match"][userCode]
            strTime = time.strftime("%Y년 %m월 %d일 %H시 %M분")
            DM_str = userName + "님이 " + strTime + "에 일을 하고 있습니다."
            return DM_str
        else:
            DM_str = exceptionDM
            return DM_str
        return DM_str
        pass
    elif command_Number == 5:
        ## 오늘 현재까지 일한 시간 체크.
        insert_juge, exceptionDM = today_working_timeStatus(jsonData, userCode, nowDate, nowTime)
        if insert_juge:
            strTime = time.strftime("%Y년 %m월 %d일")
            userName = jsonData["userName_code_match"][userCode]
            DM_str = "[" + strTime + "] " + userName + "님이 " + exceptionDM
            return DM_str
        else:
            DM_str = exceptionDM
            return DM_str
    elif command_Number == 6:
        # workingCheckList = working_check_List()

        # reset_Working_Event(workingCheckList)
        return DM_str
    elif command_Number == 7:
        exceptionDM = user_state_feedback(jsonData, userCode, nowDate)

        return exceptionDM
    else:
        return DM_str

def today_working_timeStatus(jsonData, userCode, nowDate, nowTime):
    size = len(jsonData["workingTimeData"][userCode][nowDate])

    if size > 0:
        startEndList = working_time_startEndList(jsonData, userCode, nowDate, nowTime)
        resultMinute = startEndTime_calculation(startEndList)
        h = str(int(resultMinute / 60))
        m = str(resultMinute % 60)
        return True, h + "시간" + m + "분 동안 일했습니다."
    else:
        return False, "일을 시작하지 않았습니다."

    return False, None

def startEndTime_calculation(startEndList):
    resultMinute = 0
    for startEnd in startEndList:
        startTime = (int(startEnd[0][0:2]) * 60) + int(startEnd[0][2:4])
        endTime = (int(startEnd[1][0:2]) * 60) + int(startEnd[1][2:4])
        resultMinute += (endTime - startTime)
    return resultMinute

def working_time_startEndList(jsonData, userCode, nowDate, nowTime):
    size = len(jsonData["workingTimeData"][userCode][nowDate])
    workdingLog_list = []
    startEndList = [None, None]

    if size > 0:
        for i in range(size):
            workingLog = jsonData["workingTimeData"][userCode][nowDate][i]
            state = list(workingLog.keys())[0]
            time = list(workingLog.values())[0]

            if state == "workingStartTime" or state == "workingCheckTime" or state == "restEndTime":
                startEndList[0] = time

            elif state == "nonResponseTime" or state == "restStartTime":
                startEndList[1] = time
                workdingLog_list.append(startEndList)
                startEndList = [None, None]

            elif state == "workingEndTime":
                if startEndList[0] == None:
                    startEndList[0] = time
                    startEndList[1] = time
                else:
                    startEndList[1] = time
                workdingLog_list.append(startEndList)
                startEndList = [None, None]

            if i == size - 1:
                if startEndList[0] != None and startEndList[1] == None:
                    startEndList[1] = nowTime
                    workdingLog_list.append(startEndList)
                    startEndList = [None, None]

        return workdingLog_list

def user_state_feedback(jsonData, userCode, nowDate):
    size = len(jsonData["workingTimeData"][userCode][nowDate])
    if size > 0:
        state = list(jsonData["workingTimeData"][userCode][nowDate][size - 1].keys())[0]
        time = list(jsonData["workingTimeData"][userCode][nowDate][size - 1].values())[0]

        if state == "workingStartTime" or state == "workingCheckTime" or state == "restEndTime":
            return time[0:2] + "시 " + time[2:4] + "분 " + "에 working Check하고 일하고 있어요.. 힘내세요!!"
        elif state == "restStartTime":
            return time[0:2] + "시 " + time[2:4] + "분 " + "부터 쉬고 있어요"
        elif state == "nonResponseTime":
            return time[0:2] + "시 " + time[2:4] + "분 " + "부터 응답이 없어요 다시 일을 하실려면, 응, yes, y 입력하세요"
        elif state == "workingEndTime":
            return time[0:2] + "시 " + time[2:4] + "분 " + "에 일하기를 그만했어요. 오늘 하루 수고했어요"

    else:
        return "일을 시작하지 않았어요!"
    pass

def json_file_yes_insert(jsonData, userCode, nowDate, nowTime):
    size = len(jsonData["workingTimeData"][userCode][nowDate])
    if size > 0:
        state = list(jsonData["workingTimeData"][userCode][nowDate][size - 1].keys())[0]
        if state == "nonResponseTime" or state == "restEndTime" or state == "workingCheckTime":
            yesWorkingTime = {"workingCheckTime": nowTime}
            jsonData["workingTimeData"][userCode][nowDate].append(yesWorkingTime)
            write_json_data(jsonData)
            return True, None
        else:
            return False, "휴식 중입니다. ~~"
    else:
        return False, "일을 시작하지 않았습니다"

def json_file_workingend_insert(jsonData, userCode, nowDate, nowTime):
    size = len(jsonData["workingTimeData"][userCode][nowDate])
    if size > 0:
        state = list(jsonData["workingTimeData"][userCode][nowDate][size - 1].keys())[0]
        if state != "workingEndTime":
            workingEndInsert = {"workingEndTime": nowTime}
            jsonData["workingTimeData"][userCode][nowDate].append(workingEndInsert)
            write_json_data(jsonData)
            return True, None
        else:
            return False, "이미 일하는 것을 종료했습니다."
    else:
        return False, "일을 시작 하지 않았습니다."

def json_file_restend_insert(jsonData, userCode, nowDate, nowTime):
    size = len(jsonData["workingTimeData"][userCode][nowDate])
    if size > 0:
        state = list(jsonData["workingTimeData"][userCode][nowDate][size - 1].keys())[0]
        if state == "restStartTime":
            restEndInsert = {"restEndTime": nowTime}
            jsonData["workingTimeData"][userCode][nowDate].append(restEndInsert)
            write_json_data(jsonData)
            return True, None
        else:
            return False, "휴식중이 아닙니다."
    else:
        return False, "일을 시작 하지 않았습니다."

def json_file_reststart_insert(jsonData, userCode, nowDate, nowTime):
    ## 마지막에 일시작, 휴식끝, 워킹체크가 있어야지 쓸수있음.
    ## 휴식시작, 일종료 있거나 일시작이 없으면 쓸수 없음.
    size = len(jsonData["workingTimeData"][userCode][nowDate])
    if size > 0:
        state = list(jsonData["workingTimeData"][userCode][nowDate][size - 1].keys())[0]
        if state == "workingStartTime" or state == "restEndTime" or state == "workingCheckTime":
            restStartInsert = {"restStartTime": nowTime}
            jsonData["workingTimeData"][userCode][nowDate].append(restStartInsert)
            write_json_data(jsonData)
            return True, None
        elif state == "restStartTime":
            exceptionDM = "이미 쉬고 있습니다."
        elif state == "nonResponseTime":
            exceptionDM = "응답이 없어 시간 체크되지 않았습니다.\n 응, yes, y 중에서 입력해주세요."
        elif state == "workingEndTime":
            exceptionDM = "일을 시작하지 않았습니다."
        else:
            exceptionDM = "exceptionDM"

        return False, exceptionDM
    else:
        exceptionDM = "일을 시작하지 않았습니다."
        return False, exceptionDM

def json_file_startTime_insert(jsonData, userCode, nowDate, nowTime):
    ## 일 시작 json 파일 작성.
    size = len(jsonData["workingTimeData"][userCode][nowDate])
    if size == 0:
        startInsert = {"workingStartTime": nowTime}
        jsonData["workingTimeData"][userCode][nowDate].append(startInsert)
        write_json_data(jsonData)
        return True
    else:
        state = list(jsonData["workingTimeData"][userCode][nowDate][size - 1].keys())[0]
        if state == "workingEndTime":
            startInsert = {"workingStartTime": nowTime}
            jsonData["workingTimeData"][userCode][nowDate].append(startInsert)
            write_json_data(jsonData)
            return True
        else:
            return False

def json_file_date_insert(jsonData, userCode, nowDate):
    # 오늘 날짜가 저장되었는지 확인한다.
    if nowDate in jsonData["workingTimeData"][userCode]:
        pass
    else:
        jsonData["workingTimeData"][userCode][nowDate] = []
        ## 추가 하고 파일 데이터 쓰기.
        write_json_data(jsonData)

def json_file_user_insert(jsonData, userCode):
    ## 유저가 저장되었는지 확인한다.
    if userCode in jsonData["workingTimeData"]:
        pass
    else:  ## 유저 데이터가 없을 때.
        jsonData["workingTimeData"][userCode] = {}
        ## 추가 하고 파일 데이터 쓰기.
        write_json_data(jsonData)

def write_json_data(jsonData):
    ## 파일 쓰기.
    with open('data.json', 'w', encoding="utf-8") as make_file:
        json.dump(jsonData, make_file, ensure_ascii=False, indent="\t")

def working_check_List():
    workingCheckList = {}

    ## 마지막 상태에 따라서 DM 날릴지 확인하는 로직.
    nowDate = time.strftime('%Y%m%d')
    with open('data.json', encoding="utf-8") as data_file:
        jsonData = json.load(data_file, object_pairs_hook=OrderedDict)

    for userCode, objectValue in jsonData["workingTimeData"].items():
        if nowDate in objectValue:
            arrayValue = list(objectValue[nowDate])
            size = len(arrayValue)
            userLastState = list(arrayValue[size - 1].keys())[0]
            userLastTime = list(arrayValue[size - 1].values())[0]
            ## 일시작, 휴식끝, 30분 DM 인것만 체크.
            if userLastState == "workingStartTime" or userLastState == "restEndTime" or userLastState == "workingCheckTime":
                workingCheckList[userCode] = userLastTime

    return workingCheckList

def workingCheckDM(userCode, nowTime):
    with open('data.json', encoding="utf-8") as data_file:
        jsonData = json.load(data_file, object_pairs_hook=OrderedDict)

    nowDate = time.strftime('%Y%m%d')

    channel = jsonData["userDMChannel"][userCode]

    nonTimeInsert = {"nonResponseTime": nowTime}
    jsonData["workingTimeData"][userCode][nowDate].append(nonTimeInsert)
    write_json_data(jsonData)
    # return True
    dm = "<@" + userCode + "> 거기있니?"
    select_channel_DM(channel, dm)

def userChannelUpdate(jsonData, userCode, channel):
    jsonData["userDMChannel"][userCode] = channel
    write_json_data(jsonData)

def reset_Working_Event(workingCheckList):
    ## 12시에 현재 일하고 있는 사람들 json 파일에 일종료 메시지 넣기.
    ## 오늘 일했던 사람들에게 하루동알 일한 시간 DM 날리기.
    with open('data.json', encoding="utf-8") as data_file:
        jsonData = json.load(data_file, object_pairs_hook=OrderedDict)

    nowTime = time.strftime('%H%M')
    nowDate = time.strftime('%Y%m%d')
    workingEndInsert = {"workingEndTime": nowTime}

    for key, value in workingCheckList.items():
        jsonData["workingTimeData"][key][nowDate].append(workingEndInsert)

    write_json_data(jsonData)

    ## 오늘 일한 모든 사람에게 일한 시간 DM 날리기.
    for userCode, objectValue in jsonData["workingTimeData"].items():
        if nowDate in objectValue:
            startEndList = working_time_startEndList(jsonData, userCode, nowDate, nowTime)
            resultMinute = startEndTime_calculation(startEndList)
            h = str(int(resultMinute / 60))
            m = str(resultMinute % 60)

            channel = jsonData["userDMChannel"][userCode]
            todayfeedBackDM = "<@" + userCode + "> 님이 [" + nowDate + "] 하루" + h + "시간" + m + "분 동안 일했습니다."
            select_channel_DM(channel, todayfeedBackDM)


from time import perf_counter
from fastapi import APIRouter
from fastapi import Response
from rio_tiler.errors import TileOutsideBounds
from rasterio.errors import RasterioIOError
from rio_tiler.utils import render
from rio_tiler.io import COGReader
from decouple import config
from cogtiler.classify import ftl, hfi, elevation, aspect, slope
from cogtiler import utils

logger = utils.getLogger()

router = APIRouter(
    prefix="/v0.0.1"
)

cache_control = config('CACHE-CONTROL', 'max-age=604800')
cache_expiry_seconds = 604800  # 10,080 minutes, 168 hours, 7 days

@router.get('/tile/{z}/{x}/{y}')
def tile_xyz(z: int, x: int, y: int, path: str, source: str, filter: str = None, tilesize: int = 512) -> Response:
    """
    Prepare your images:
    ```bash
    gdal_translate ftl2018.bin ftl_2018.tif -co TILED=YES -co COMPRESS=DEFLATE
    gdaladdo -r nearest ftl_2018.tif 2 4 8 16 32
    gdal_translate ftl_2018.tif ftl_2018_cloudoptimized.tif -co TILED=YES -co COMPRESS=LZW -co COPY_SRC_OVERVIEWS=YES
    ```
    """
    start = perf_counter()
    s3_url = f's3://{path}'
    key = f'/tile/{z}/{x}/{y}?path={path}&source={source}&filter={filter}&tilesize={tilesize}'
    cache = utils.create_redis()
    logger.info('%s ; s3_url: %s', key, s3_url)
    try:
        cached_data = cache.get(key)
    except Exception as error:
        cached_data = None
        logger.error(error, exc_info=error)
    if cached_data:
        logger.info('redis cache hit %s', key)
        response = Response(content=cached_data, media_type="image/png")
        response.headers["Cache-Control"] = cache_control
        return response
    else:
        logger.info('redis cache miss %s', key)
    try:
        with COGReader(s3_url) as cog:
            try:
                img = cog.tile(x, y, z, tilesize=tilesize)
            except TileOutsideBounds:
                response = Response(status_code=404)
                response.headers["Cache-Control"] = cache_control
                return response
        if source == 'ftl':
            data, mask = ftl.classify(img.data, filter)
        elif source == 'hfi':
            data, mask = hfi.classify(img.data)
        elif source == 'elevation':
            data, mask = elevation.classify(img.data)
        elif source == 'slope':
            data, mask = slope.classify(img.data)
        elif source == 'aspect':
            data, mask = aspect.classify(img.data)
    except RasterioIOError as e:
        # If the file is not found, return 404
        logger.error(e, exc_info=True)
        return Response(status_code=500)
    except Exception as e:
        logger.error(e, exc_info=True)
        return Response(status_code=500)
    # TODO: This part (`render`) gives an error:
    # "ERROR 4: `/vsimem/xxxx.tif' not recognized as a supported file format."
    # We can replace this with a different method. It's just taking an array of rgba values
    # and turning it into a png file.
    rendered = render(data, mask)
    response = Response(content=rendered, media_type="image/png")
    try:
        cache.set(key, rendered, ex=cache_expiry_seconds)
    except Exception as error:  # pylint: disable=broad-except
        logger.error(error, exc_info=error)
    # cache it for a week - we probably only need to cache for a day or two, but this is on
    # the client, so we don't care.
    response.headers["Cache-Control"] = cache_control
    end = perf_counter()
    delta = end - start
    logger.info('/ftl/%s/%s/%s?path=%s&source=%s took %f seconds', z, x, y, path, source, delta)
    return response

@router.get('/value/{band}/{lat}/{lon}')
def value(band: int, lat: float, lon: float, path: str) -> Response:
    s3_url = f's3://{path}'
    try:
        with COGReader(s3_url) as cog:
            point = cog.point(lon, lat, indexes=band)
        # TODO: put in nice json response. This could even be a nice geojson thing
        return point[band - 1]
    except RasterioIOError:
        # If the file is not found, return 404
        response = Response(status_code=404)
        return response


# noqa D100
import mock

def test_touch_setup(cap1xxx):
    """Test that touch initialises Cap1166."""
    from gfxhat import touch

    touch.setup()

    cap1xxx.Cap1166.assert_called_with(i2c_addr=touch.I2C_ADDR)

def test_touch_names():
    """Test that get_name returns a sensible value."""
    from gfxhat import touch

    assert touch.get_name(0) == 'up'

def test_touch_set_led(cap1xxx):
    """Test that set_led calls Cap1166.set_led_state."""
    from gfxhat import touch

    touch.set_led(0, True)

    cap1xxx.Cap1166().set_led_state.assert_called_with(touch.LED_MAPPING[0], True)

def test_touch_high_sensitivity(cap1xxx):
    """Test that high sensitivity calls custom commands.

    Tests should be updated if/when this behaviour is formalised in Cap1xxx.

    """
    from gfxhat import touch

    touch.high_sensitivity()

    cap1xxx.Cap1166()._write_byte.assert_has_calls([
        mock.call(0x00, 0b11000000),
        mock.call(0x1f, 0b00000000)
    ])

def test_touch_repeat(cap1xxx):
    """Test that touch repeat calls Cap1166.enable_repeat()."""
    from gfxhat import touch

    touch.enable_repeat(True)
    cap1xxx.Cap1166().enable_repeat.assert_called_with(0b11111111)

    touch.enable_repeat(False)
    cap1xxx.Cap1166().enable_repeat.assert_called_with(0b00000000)

def test_touch_repeat_rate(cap1xxx):
    """Test that touch repeat rate calls Cap1166.set_repeat_rate()."""
    from gfxhat import touch

    touch.set_repeat_rate(35)
    cap1xxx.Cap1166().set_repeat_rate.assert_called_with(35)

def test_attach_handler(cap1xxx):
    """Test that both handler modes call Cap1166.on()."""
    from gfxhat import touch

    decorator = touch.on([1])

    def handler():
        pass

    decorator(handler)

    cap1xxx.Cap1166().on.assert_has_calls([
        mock.call(channel=1, event='press', handler=handler),
        mock.call(channel=1, event='release', handler=handler),
        mock.call(channel=1, event='held', handler=handler),
    ])

    touch.on([0], handler=handler)

    cap1xxx.Cap1166().on.assert_has_calls([
        mock.call(channel=0, event='press', handler=handler),
        mock.call(channel=0, event='release', handler=handler),
        mock.call(channel=0, event='held', handler=handler),
    ])


import datetime

import pandas as pd

import numpy as np

#atomman imports
from atomman.tools import uber_open_rmode
from DataModelDict import DataModelDict as DM

class Log(object):
    """Object for representing a LAMMPS log output"""

    def __init__(self, log_info=None):
        """Intitializes a Log object."""

        #Initialize simulation properties
        self.__simulations = []
        self.__lammps_version = None
        self.__lammps_date = None

        #Read log data if supplied
        if log_info is not None:
            self.read(log_info)

    def read(self, log_info, append=True):
        """Parses a LAMMPS screen output/log file for thermodynamic data."""

        #Rset properties and values if append is False
        if append is False:
            self.__simulations = []
            self.__lammps_version = None
            self.__lammps_date = None

        #Strings found before run and mimize simulations
        sim_trigger = ['Memory usage per processor =','Per MPI rank memory allocation (min/avg/max) =']

        #Handle file names, strings and open file-like objects equivalently
        with uber_open_rmode(log_info) as log_info:

            headers = []
            footers = []
            i = 0

            #for all lines in file/output
            for line in log_info:

                #skip blank lines
                if len(line.split()) == 0:
                    continue

                #Save the LAMMPS version information
                if line[:8] == 'LAMMPS (' and self.lammps_version is None:
                    month = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4,
                             'May': 5, 'Jun': 6, 'Jul': 7, 'Aug': 8,
                             'Sep': 9, 'Oct': 10,'Nov': 11,'Dec': 12}
                    self.__lammps_version = line.strip()[8:-1]
                    d = self.lammps_version.split('-')[0].split()
                    self.__lammps_date = datetime.date(int(d[2]), month[d[1]], int(d[0]))

                #Check for strings listed before run and minimize simulations
                if any([trigger in line for trigger in sim_trigger]):
                    headers.append(i+1)

                #This follows both run and minimize simulations
                elif 'Loop time of' in line:
                    footers.append(i-1)

                i += 1

            #Add last line to footers for incomplete logs
            footers.append(i)

            #Reset file pointer
            log_info.seek(0)

            #for all lines in file/output
            for header, footer in zip(headers, footers):

                #Initialize simulation data dictionary
                sim = {}

                #Read thermo data and reset file pointer
                sim['thermo'] = pd.read_csv(log_info, header=header, nrows=footer-header, sep='\s+', engine='python', skip_blank_lines=True)
                log_info.seek(0)

                #Append simulation results
                self.__simulations.append(sim)

    @property
    def simulations(self):
        return self.__simulations

    @property
    def lammps_version(self):
        return self.__lammps_version

    @property
    def lammps_date(self):
        return self.__lammps_date

    def flatten(self, style='last'):
        """Combines all simulations into one

        Keyword Arguments:
        style -- specifies which values to use for duplicate time steps
            - first uses the values from the earlier simulation
            - last uses the values from the later simulation
        """

        for sim in self.simulations:
            if 'thermo' in sim and len(sim['thermo']) > 0:
                assert 'Step' in sim['thermo'], 'All simulation thermos must have Step key in order to flatten'
        merged_df = self.simulations[0]['thermo']
        for i in xrange(1, len(self.simulations)):
            if 'thermo' in self.simulations[i]:
                thermo = self.simulations[i]['thermo']
                if   style == 'first':
                    merged_df = pd.concat([merged_df, thermo[thermo.Step > merged_df.Step.max()]], ignore_index=True)
                elif style == 'last':
                    merged_df = pd.concat([merged_df[merged_df.Step < thermo.Step.min()], thermo], ignore_index=True)
                else:
                    raise ValueError('Unsupported style')
        self.__simulations = [{'thermo':merged_df}]

    def model(self, flatten=None):
        """Returns a DataModelDict containing the information."""
        if flatten is not None:
            self.flatten(flatten)

        #Create DataModelDict root
        log_model = DM()
        log_model['LAMMPS-log-thermo-data'] = DM()

        #Loop over all simulations
        for sim in self.simulations:
            sim_model = DM()

            #Convert to DataModelDict
            sim_model['thermo'] = DM()
            thermo = sim['thermo']
            for j in thermo:
                sim_model['thermo'][str(j)] = thermo[j].values.tolist()

            #Append simulation results to DataModelDict root
            log_model['LAMMPS-log-thermo-data'].append('simulation', sim_model)

        return log_model


#!/usr/bin/env python

import os
import os.path
import urllib.parse
import json
import numpy
import face_recognition
import PIL
from flask import Flask, render_template, jsonify, request, redirect

app = Flask(__name__)
faces = numpy.load('faces.npy', allow_pickle=True)
encodings = numpy.array([face[2] for face in faces])
records = {}

FINNA_BASE_URL = 'https://finna.fi'

RECORDDIR = 'records'

MAXDIM = 2048

for recf in os.listdir(RECORDDIR):
    with open(os.path.join(RECORDDIR, recf)) as recs:
        for line in recs:
            rec = json.loads(line)
            records[rec['id']] = rec

def make_record(recid, bbox, distance):
    rec = records[recid]
    ret = {
        'id': recid,
        'location': bbox,
        'imgurl': FINNA_BASE_URL + rec['images'][0],
        'pageurl': FINNA_BASE_URL + '/Record/' + urllib.parse.quote(recid),
        'title': rec.get('title', ''),
        'authors': rec.get('nonPresenterAuthors', None),
        'buildings': rec['buildings'][0]['translated'],
        'distance': distance
    }
    if 'year' in rec:
        ret['year'] = rec['year']
    return ret

def find_similar(file_stream):
    img = PIL.Image.open(file_stream)
    img = img.convert('RGB')
    scale = None
    if img.height > MAXDIM or img.width > MAXDIM:
        origheight = img.height
        img.thumbnail((MAXDIM, MAXDIM))
        scale = origheight / img.height
    img = numpy.array(img)
    face_locations = face_recognition.face_locations(img)
    if not face_locations:
        return None
    input_encodings = face_recognition.face_encodings(img, face_locations)
    face_encoding = input_encodings[0]
    if scale:
        face_locations = [[int(loc * scale) for loc in location] for location in face_locations]
    face_location = face_locations[0]
    distances = face_recognition.face_distance(encodings, face_encoding)
    ranking = distances.argsort()[:10]
    return {'location': face_location,
            'similar': [make_record(faces[idx][0], faces[idx][1], distances[idx])
                        for idx in ranking]}

@app.route('/', methods=['GET'])
def index():
    return render_template('index.html')

@app.route('/similar', methods=['POST'])
def similar():
    if 'imagefile' not in request.files:
        return redirect('/')

    imagefile = request.files['imagefile']
    return jsonify(find_similar(imagefile))

if __name__ == "__main__":
    app.run(host='0.0.0.0', port=5000, debug=True)


import threading
import time
import traceback

from consts.proj_vars import ProjVar
from utils.clients.ssh import SSHClient, ControllerClient, NATBoxClient
from utils.exceptions import ThreadingError
from utils.tis_log import LOG

TIMEOUT_ERR = "Thread did not terminate within timeout. " \
              "Thread details: {} {} {}"
EVENT_TIMEOUT = "Event did not occur within timeout."
INFINITE_WAIT_EVENT_EXPECTED = "There is only one thread (Main Thread), " \
                               "waiting is likely to wait indefinitely"

class MThread(threading.Thread):
    """
    Multi threading class. Allows multiple threads to be run simultaneously.
    e.g. nova_helper.create_flavor('threading', 'auto', vcpus=2, ram=1024) is
    equivalent to...
        thread_1 = MThread(nova_helper.create_flavor, 'threading', 'auto',
        vcpus=2, ram=1024)
        thread_1.start_thread()
        thread_1.wait_for_thread_end()

    Other commands can be run between start_thread and wait_for_thread_end
    The function's output can be retrieved from thread_1.get_output()
    name should NOT be changed
    """
    total_threads = 0
    running_threads = []

    def __init__(self, func, *args, **kwargs):
        """

        Args:
            func (runnable): name of function to run. e.g.
            nova-helper.create_flavor. NOT nova_helper.create_flavor()
            *args:
            **kwargs:
        """
        threading.Thread.__init__(self)
        MThread.total_threads += 1
        self.thread_id = MThread.total_threads
        self.func = func
        self.args = args
        self.kwargs = kwargs
        self._output = None
        self._output_returned = threading.Event()
        self.timeout = None
        self._err = None

    def get_output(self, wait=True, timeout=None):
        """
        Get return value of self.func

        Args:
            wait (bool): Whether or not to wait for the output from self.func
            timeout:

        Returns: return value from self.func

        """
        if wait:
            if timeout is None:
                timeout = self.timeout

            if timeout:
                end_time = time.time() + timeout
                while time.time() < end_time:
                    if self._err:
                        raise ThreadingError(str(self._err))
                    if self._output_returned.is_set():
                        # LOG.info("{}: {} returned: {}".format(self.name,
                        # self.func.__name__, self._output.__str__))
                        break

        return self._output

    def get_error_info(self):
        return self._err

    def start_thread(self, timeout=3600):
        """
        Starts a thread.
        Test must wait for thread to terminate or it can continue running
        during other tests.

        Args:
            timeout (int): how long to wait for the thread to finish

        Returns:

        """
        self.timeout = timeout
        self.__start_thread_base()

    def __start_thread_base(self):
        self.start()

    def run(self):
        """
        Do not run this command. Start threads from start_thread functions
        Returns:

        """
        LOG.info("Starting {}".format(self.name))
        # run the function
        try:
            MThread.running_threads.append(self)
            LOG.info("Connecting to lab fip in new thread...")
            lab = ProjVar.get_var('lab')
            lab_fip = lab['floating ip']
            con_ssh = SSHClient(lab_fip)
            con_ssh.connect(use_current=False)
            ControllerClient.set_active_controller(con_ssh)

            if ProjVar.get_var('IS_DC'):
                LOG.info("Connecting to subclouds fip in new thread...")
                ControllerClient.set_active_controller(con_ssh, 'RegionOne')
                con_ssh_dict = ControllerClient.get_active_controllers_map()
                for name in con_ssh_dict:
                    if name in lab:
                        subcloud_fip = lab[name]['floating ip']
                        subcloud_ssh = SSHClient(subcloud_fip)
                        try:
                            subcloud_ssh.connect(use_current=False)
                            ControllerClient.set_active_controller(subcloud_ssh,
                                                                   name=name)
                        except:
                            if name == ProjVar.get_var('PRIMARY_SUBCLOUD'):
                                raise
                            LOG.warning('Cannot connect to {}'.format(name))

            LOG.info("Connecting to NatBox in new thread...")
            NATBoxClient.set_natbox_client()

            LOG.info("Execute function {}({}, {})".format(self.func.__name__,
                                                          self.args,
                                                          self.kwargs))
            self._output = self.func(*self.args, **self.kwargs)
            LOG.info("{} returned: {}".format(self.func.__name__,
                                              self._output.__str__()))
            self._output_returned.set()
        except:
            err = traceback.format_exc()
            # LOG.error("Error found in thread call {}".format(err))
            self._err = err
            raise
        finally:
            LOG.info("Terminating thread: {}".format(self.thread_id))
            if ProjVar.get_var('IS_DC'):
                ssh_clients = ControllerClient.get_active_controllers(
                    current_thread_only=True)
                for con_ssh in ssh_clients:
                    con_ssh.close()
            else:
                ControllerClient.get_active_controller().close()

            natbox_ssh = NATBoxClient.get_natbox_client()
            if natbox_ssh:
                natbox_ssh.close()

            LOG.debug("{} has finished".format(self.name))
            MThread.running_threads.remove(self)

    def wait_for_thread_end(self, timeout=3600, fail_ok=False):
        """
        Waits for thread (self) to finish executing.
        All tests should wait for threads to end before proceeding to
        teardown, unless it is expected to continue,
        e.g. LOG.tc_step will not work during setup or teardown
        Raise error if thread is still running after timeout
        Args:
            timeout (int): how long to wait for the thread to finish.
            self.timeout is preferred.
            fail_ok (bool): fail_ok=False will raise error if wait times out
            or fails test if thread exited due to error

        Returns (bool): True if thread is not running, False/exception otherwise

        """
        if not self.is_alive():
            LOG.info("{} was already finished".format(self.name))
            if self._err:
                if not fail_ok:
                    raise ThreadingError(
                        "Error in thread: {}".format(self._err))
                LOG.error("Error found in thread call {}".format(self._err))
            return True, self._err

        if not timeout:
            timeout = self.timeout

        LOG.info("Wait for {} to finish".format(self.name))
        self.join(timeout)

        if not fail_ok:
            assert not self._err, "{} ran into an error: {}".format(self.name,
                                                                    self._err)

        if not self.is_alive():
            LOG.info("{} has finished".format(self.name))
        else:
            # Thread didn't finish before timeout
            LOG.error("{} did not finish within timeout".format(self.name))
            if fail_ok:
                return False, self._err
            raise ThreadingError(
                TIMEOUT_ERR.format(self.func, self.args, self.kwargs))

        return True, self._err

def get_multi_threads():
    return MThread.running_threads

def is_multi_thread_active():
    if len(get_multi_threads()) == 0:
        return False
    return True

class Events(threading.Event):
    """
    Blocks thread progress when wait_for_event is called
    (Main Thread)               (Thread-1)
    event = Events()
    event.wait_for_event()
    # waits
                                event.set()
    # continues
    """

    def __init__(self, message="Simple event"):
        threading.Event.__init__(self)
        self.message = message
        self.msg_lock = threading.Lock()

    def set(self):
        """
        Set event flag to true. Allows threads waiting for the event to continue
        """
        threading.Event.set(self)
        LOG.info("Event \"{}\" flag set to True".format(self.message))

    def clear(self):
        threading.Event.clear(self)
        LOG.info("Event \"{}\" flag set to False".format(self.message))

    def wait_for_event(self, timeout=3600, fail_ok=False):
        """
        Waits for this Events object to have its flag set to True
        Args:
            timeout:
            fail_ok:

        Returns (bool): True if event flag set to true, False/exception
        otherwise

        """
        if self.is_set():
            LOG.info("Event \"{}\" has already been set".format(self.message))
            return True
        if not timeout:
            LOG.warning(
                "No timeout was specified. This can lead to waiting "
                "indefinitely")

        if not is_multi_thread_active() and not timeout:
            LOG.error(
                "There are no other running threads that can set this event "
                "to true. "
                "This would wait indefinitely")
            raise ThreadingError(INFINITE_WAIT_EVENT_EXPECTED)

        LOG.info(
            "Waiting for event \"{}\" flag set to true".format(self.message))
        if not threading.Event.wait(self, timeout):
            if fail_ok:
                LOG.error(
                    "Timed out waiting for event \"{}\" flag set to "
                    "True".format(
                        self.message))
                return False
            raise ThreadingError(EVENT_TIMEOUT)

        if self.msg_lock.acquire(False):
            # only one message should appear once event is set
            LOG.info("Threads continuing")
            time.sleep(1)
            self.msg_lock.release()
        return True

class TiSBarrier(threading.Barrier):
    """
    Blocks threads calling wait() until specified number (parties) of threads
    are waiting or timeout expires.
    (Main Thread)               (Thread-1)              (Thread-2)
    barr = TiSBarrier(2)
                                barr.wait()
                                # Thread-1 waits
                                                        barr.wait()
                                # both continue
    """

    def __init__(self, parties, action=None, timeout=180):
        """
        Args:
            parties (int): number of threads to wait for
            action (function): additional function to call when barrier breaks
            timeout (int): max wait time
        """
        if len(get_multi_threads()) + 1 < parties:
            LOG.warning(
                "This barrier will wait for more threads than are currently "
                "running")
        threading.Barrier.__init__(self, parties, action, timeout)

        self.timeout = timeout
        LOG.info("Created a barrier waiting for {} threads".format(parties))

    def wait(self, timeout=None):
        """
        Block thread until barrier breaks
        Exception is raised if timeout expires
        Args:
            timeout (int): max wait time at barrier. Preferred over
            self.timeout, if given

        Returns (int): number from 0 to (parties - 1), randomly assigned and
        unique for each thread that waited.
                       -1 if thread did not wait

        """
        self.barrier_status()
        if self.broken:
            return -1
        if not timeout:
            if not self.timeout:
                LOG.warning("This thread does not have a timeout for wait()")
            else:
                timeout = self.timeout

        try:
            LOG.info("Start waiting at barrier")
            id_ = threading.Barrier.wait(self, timeout)
            # only one thread will report the barrier being passed
            if id_ == 0:
                self._barrier_passed()
            return id
        except:
            LOG.info("Barrier broke before {} threads were waiting".format(
                self.parties))
            raise

    def barrier_status(self):
        if not self.broken:
            LOG.info(
                '{} threads are waiting... Barrier waiting for {} '
                'threads'.format(
                    self.n_waiting, self.parties))
        else:
            LOG.info("This barrier has already been passed.")

    def _barrier_passed(self):
        LOG.info("Barrier has been passed. {} threads continuing".format(
            self.parties))

class TiSLock:
    """
    RLock class with added logging
    lock can be acquired using 'lock.acquire()' or 'with lock as have_lock'
    lock released using 'lock.release()' or exiting with scope
    Do Not call __enter__ or __exit__ explicitly

    (Main Thread)               (Thread-1)
    lock = TiSLock()
    lock.acquire()
    # edit test.txt
                                lock.acquire()
                                # waits
    lock.release()
                                # continues
    """

    def __init__(self, blocking=True, timeout=180):
        """
        Create a locking object
        Args:
            blocking (bool): wait for lock to be released if it is locked
            during first acquire attempt
            timeout (int): how long to wait for lock to be released. Should
            set to -1 if blocking is false
                           -1 with blocking True will have no timeout
        """
        self._lock = threading.RLock()
        self.blocking = blocking
        self.timeout = timeout if blocking else -1

    def acquire(self, **kwargs):
        if 'blocking' in kwargs:
            self.blocking = kwargs['blocking']
        if 'timeout' in kwargs:
            self.timeout = kwargs['timeout']
        return self.__enter__()

    def release(self):
        return self.__exit__()

    def __enter__(self):
        """
        Make sure that True is returned.
        Threads will continue if timeout is reached even though lock was not
        acquired

        Returns (bool): True if lock was acquired, False otherwise

        """
        blocking_msg = "" if self.blocking else "not "
        timeout_msg = self.timeout if self.timeout >= 0 else "None"
        msg = "Attempting to acquire lock, {}blocking, timeout - {}".format(
            blocking_msg, timeout_msg)
        LOG.debug(msg)
        got_lock = False
        try:
            got_lock = self._lock.acquire(self.blocking, self.timeout)
        finally:
            if got_lock:
                LOG.debug("Acquired lock")
            else:
                LOG.debug("Could not acquire lock")
            return got_lock

    def __exit__(self, *args):
        LOG.debug("Releasing lock")
        try:
            return self._lock.release()
        except RuntimeError:
            LOG.error("Lock did not release, lock was unlocked already")
            raise
        except:
            LOG.error("An unexpected error was caught when unlocking lock")
            raise


# %%
import os
import pandas as pd
from sklearn import tree
from sklearn import model_selection
from sklearn import metrics
from sklearn import preprocessing
from sklearn import ensemble
from sklearn import pipeline
from sklearn import decomposition
from sklearn.model_selection import RandomizedSearchCV

import random

import xgboost as xgb

from feature_engine.imputation import ArbitraryNumberImputer
from feature_engine.discretisation import DecisionTreeDiscretiser

# %%
ORACLE_DIR = os.path.dirname(os.path.abspath(__file__))
#ORACLE_DIR = os.path.abspath(".")
SRC_DIR = os.path.dirname(ORACLE_DIR)
BASE_DIR = os.path.dirname(SRC_DIR)
DATA_DIR = os.path.join(BASE_DIR, "data")

DATA_TRAIN_DIR = os.path.join(DATA_DIR, "train")
TABLE_PATH = os.path.join(DATA_TRAIN_DIR, "tb_abt_oracle")

file_path = [i for i in os.listdir(TABLE_PATH) if i.endswith(".csv")][-1]
file_path = os.path.join(TABLE_PATH, file_path)

# %%

# Importando o dataset
print("Importando dataset...")
df_full = pd.read_csv(file_path)

columns = df_full.columns.tolist()
features =list(set(columns) - set(['match_id', 'dt_start_time', 'radiant_win']))
target = 'radiant_win'

df_full[features] = df_full[features].dropna(how="all")
print("ok.")

# %%
# Separa entre treinamento e teste

print("Separando em base de treino e teste...")
X_train, X_test, y_train, y_test = model_selection.train_test_split( df_full[features],
                                                                     df_full[target],
                                                                     random_state=42,
                                                                     test_size=0.1 )

print("ok.")

# %%

print("Ajustando modelo em nosso pipeline...")
arbitrary_imputer = ArbitraryNumberImputer(arbitrary_number=-999,
                                           variables=features)

disc = DecisionTreeDiscretiser(cv=3,
                               scoring='roc_auc',
                               variables=features,
                               regression=False,
                               random_state=42)

pca = decomposition.PCA(n_components=120,
                        random_state=42)

best_pars = {'subsample': 0.7, 'n_estimators': 100, 'max_depth': 5, 'learning_rate': 0.2}

clf_xgb = xgb.XGBClassifier(nthread=8,
                            eval_metric='auc',
                            random_state=42,
                            **best_pars)

"""
params = {
    "n_estimators":[100,500,600,700,800],
    "max_depth":[4,5,6,7,10],
    "subsample":[0.1,0.2,0,3, 0.7, 0.75,0.8,0.9, 0.99],
    "learning_rate":[0.1, 0.2, 0.5, 0.7, 0.8, 0.9, 0.99],
}

random_search = RandomizedSearchCV(clf_xgb,
                                   params,
                                   scoring="roc_auc",
                                   n_jobs=1,
                                   random_state=42,
                                   n_iter=20,
                                   cv=4)
"""

my_pipe = pipeline.Pipeline( [("imputer", arbitrary_imputer),
                              ("discretizator", disc),
                              ("pca", pca),
                              ("model", clf_xgb)])

my_pipe.fit(X_train, y_train)

print("ok.")

# %%
print("Analisando performance na base de teste...")
y_test_pred = my_pipe.predict(X_test)
y_test_prob = my_pipe.predict_proba(X_test)
auc_score = metrics.roc_auc_score(y_test, y_test_prob[:,1])
print("AUC:", auc_score)
print("ok.")

# %%

print("Salvando o modelo...")
model_dict = {"model": my_pipe,
              "features"  : X_train.columns.tolist(),
              "auc_test" : auc_score
              }

model = pd.Series(model_dict)
model.to_pickle(os.path.join(ORACLE_DIR, "models", "model_ale.pkl"))
print("Ok.")


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import csv
import datetime
import yaml
import os

NORDEA_DATA_FOLDER = 'nordea_data'

def read_values(filename):
    with open(filename, 'r') as input_file:
        file_data = input_file.readlines()
        rows = [row.strip().split('\t') for row in file_data if row != '\r\n']
        return rows

def read_settings():
    with open('settings.yaml', 'r') as input_file:
        settings = yaml.load(input_file)
        return settings

class Account():
    def __init__(self, account_number, name, transactions=[], balance=0, balance_date=datetime.datetime.now().date()):
        self.account_number = account_number
        self.name = name
        self.transactions = []
        self.add_transactions(transactions)
        self.init_balance = balance
        self.init_balance_date = balance_date
        self.saaja_maksaja_includes = None
        self.viesti_includes = None
        # self.calculate_balances(balance, balance_date)

    def __str__(self):
        return '{} {} {}'.format(self.account_number, self.name, len(self.transactions))

    def add_transactions(self, transaction_list):
        transaction_list = [trans for trans in transaction_list if trans not in self.transactions]
        print('adding ', len(transaction_list))
        self.transactions = self.transactions + transaction_list
        print(len(self.transactions), 'transactions')
        dates = [trans.kirjauspaiva for trans in self.transactions]

    def calculate_balances(self):
        balance = self.init_balance
        balance_date = self.init_balance_date
        self.transactions = sorted(self.transactions, reverse=True, key=lambda transaction: transaction.kirjauspaiva)
        self.balances = []
        self.day_changes = []
        current_date = None
        current_date_change = 0
        for transaction in self.transactions:
            if current_date != transaction.kirjauspaiva:
                if current_date is not None:
                    self.day_changes.append({"date": current_date, "change": current_date_change})
                current_date = transaction.kirjauspaiva
                current_date_change = 0
            current_date_change = current_date_change + transaction.value
        self.day_changes.append({"date": current_date, "change": current_date_change})

        # self.balances.append({"amount": balance, "date": balance_date})
        date = balance_date
        for change in self.day_changes:
            change_date = change['date']
            while change_date != date:
                self.balances.append({"amount": balance, "date": date})
                try:
                    date = date - datetime.timedelta(days=1)
                except:
                    print(self.balances[-1])
                    raise
            balance = balance - change['change']
        self.balances.append({"amount": balance, "date": date})
        if self.name == 'Nordnet':
            print(self.balances)

    def add_transaction_if_included(self, transaction):
        if self.includes_transaction(transaction):
            opposite = transaction.opposite()
            if opposite not in self.transactions:
                self.transactions.append(opposite)

    def includes_transaction(self, transaction):
        if self.saaja_maksaja_includes is not None:
            if self.saaja_maksaja_includes.lower() in transaction.saaja_maksaja.lower():
                return True
        if self.viesti_includes is not None:
            if self.viesti_includes.lower() in transaction.viesti.lower():
                return True
        return False

class Transaction():
    def __init__(
        self, tili, row
        # kirjauspaiva, arvopaiva, maksupaiva, value, saaja_maksaja, tilinumero, bic,
        # tapahtuma, viite, maksan_viite, viesti, kortinnumero, kuitti
    ):
        while len(row) <= 12:
            row.append('')
        self.row = row
        self.tili = tili
        self.kirjauspaiva = self.parse_date(row[0])
        self.arvopaiva = self.parse_date(row[1])
        self.maksupaiva = self.parse_date(row[2])
        self.value = float(row[3].replace(',', '.'))
        self.saaja_maksaja = row[4].decode('utf-8')
        self.tilinumero = row[5]
        self.bic = row[6]
        self.tapahtuma = row[7]
        self.viite = row[8]
        self.maksan_viite = row[9]
        self.viesti = row[10]
        self.kortinnumero = row[11]
        self.kuitti = row[12]

    def __str__(self):
        return '{} {} {} {}'.format(self.tili, self.kirjauspaiva, self.value, self.saaja_maksaja)

    def parse_date(self, date_string):
        if not date_string:
            return ''
        date = datetime.datetime.strptime(date_string, '%d.%m.%Y').date()
        return date

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

    def opposite(self):
        opposite = Transaction(self.tili, self.row)
        opposite.value = -self.value
        return opposite

class Assets():
    def __init__(self):
        self.accounts = {}
        self.settings = read_settings()
        self.virtual_accounts = []

    def transactions_from_folder(self):
        filenames = os.listdir(NORDEA_DATA_FOLDER)
        for filename in filenames:
            self.transactions_from_file(NORDEA_DATA_FOLDER + '/' + filename)

    def calculate_balances(self):
        for account in self.get_accounts():
            account.calculate_balances()

    def get_accounts(self):
        return [self.accounts[key] for key in self.accounts.keys()] + self.virtual_accounts

    def transactions_from_file(self, filename):
        print(filename)
        data = read_values(filename)
        tilinumero = data[0][1]

        transactions = [Transaction(tilinumero, row) for row in data[2:]]
        self.add_transactions(transactions, tilinumero)

    def add_transactions(self, transactions, tilinumero):
        if tilinumero in self.accounts:
            self.accounts[tilinumero].add_transactions(transactions)
        else:
            account_name = tilinumero
            account = Account(tilinumero, account_name, transactions, balance=self.settings.get('balances', {}).get(tilinumero, 0))
            self.add_account(account)
        for account in self.virtual_accounts:
            for transaction in transactions:
                account.add_transaction_if_included(transaction)

    def add_account(self, account):
        self.accounts[account.account_number] = account

    def external_accounts_from_settings(self):
        external_accounts = self.settings.get('externalaccounts', {})
        for name in external_accounts.keys():
            ext_account = Account(name, name, balance=self.settings.get('balances', {}).get(name))
            account_settings = external_accounts[name]
            ext_account.saaja_maksaja_includes = account_settings.get('saaja_maksaja')
            ext_account.viesti_includes = account_settings.get('viesti')
            self.virtual_accounts.append(ext_account)

    def classifications_from_settings(self):
        self.classifications = self.settings.get('classification')

    def get_all_transactions(self):
        transactions = []
        for account in self.get_accounts():
            transactions = transactions + account.transactions
        return transactions

    def classify_transactions(self, start, end):
        classified = {}
        for transaction in self.get_all_transactions():
            if start <= transaction.kirjauspaiva < end:
                self.add_to_classified(classified, transaction)
        return classified

    def add_to_classified(self, classified, transaction):
        saaja_maksaja = transaction.saaja_maksaja
        type = self.classify_saaja_maksaja(saaja_maksaja)
        self.add_to_dict(classified, type, transaction.value)

    def add_to_dict(self, sum_dict, key, value):
        if key in sum_dict:
            sum_dict[key] += value
        else:
            sum_dict[key] = value

    def classify_saaja_maksaja(self, saaja_maksaja):
        for key in self.classifications:
            try:
                for search_value in self.classifications[key]:
                    if search_value.lower() in saaja_maksaja.lower():
                        saaja_maksaja = key
            except:
                print(saaja_maksaja)
                print(type(saaja_maksaja))
                raise
        return saaja_maksaja

# tilinumero = data[0][1]
# transactions = [Transaction(tilinumero, row) for row in data[2:]]
# transactions2 = [Transaction(tilinumero, row) for row in data[2:]]

# account = Account(tilinumero, 'Käyttötili', transactions, balance=settings.get('balance', 0))

assets = Assets()

assets.external_accounts_from_settings()
assets.transactions_from_folder()
assets.classifications_from_settings()
assets.calculate_balances()
today = datetime.datetime.utcnow().date()

min_date = datetime.datetime(2016, 1, 1).date()
date = datetime.date.today()
months = [date]
spendings = []
total = {}
while date >= min_date:
    if date.day == 1:
        months.append(date)
    date -= datetime.timedelta(days=1)
for i in range(len(months) - 1):
    classified = assets.classify_transactions(months[i + 1], months[i])
    spendings.append(classified)
    for key in classified:
        assets.add_to_dict(total, key, classified[key])
    print(months[i + 1].strftime("%B %Y"))
    print(sorted([[key, classified[key]] for key in classified], key=lambda a: a[1]))
sorted_pairs = sorted([[key, total[key]] for key in total], key=lambda a: a[1])
ordered_targets = [p[0] for p in sorted_pairs]
spending_matrix = []
spending_matrix.append([None] + [t.encode('utf-8') for t in ordered_targets])

for i, spending in enumerate(spendings):
    spending_matrix.append([months[i + 1]] + [spending.get(key, 0) for key in ordered_targets])

print(total)
print(sorted_pairs)

# print(external_names)
# for name in external_names:
#     print(external_accounts[name])
csv_headers = ['date', 'balance']

account_names = [acc.name for acc in assets.get_accounts()]

csv_matrix = []
i = 0
values = {}
while True:
    row = []
    no_data_count = 0
    date = None
    for account in assets.get_accounts():
        try:
            balance = account.balances[i]
            values[account.name] = balance['amount']
            date = balance['date']
        except IndexError:
            no_data_count += 1

        row.append(values.get(account.name, 0))
        value = balance
    i += 1
    value_list = [values[key] for key in values.keys()]
    total = sum(value_list)
    csv_matrix.append([date] + row + [total])
    if no_data_count == len(account_names):
        break
    if i > 10000:
        print('too much')
        break

csv_matrix = list(reversed(csv_matrix))
csv_matrix = [['date'] + account_names + ['total']] + csv_matrix

filtered_csv = []
for index, row in enumerate(csv_matrix):
    if index % 5 == 0:
        filtered_csv.append(row)

with open('out_data.csv', 'w') as out_csv:
    writer = csv.writer(out_csv)
    writer.writerows(csv_matrix)

with open('spending.csv', 'w') as out_csv:
    writer = csv.writer(out_csv)
    writer.writerows(spending_matrix)


# -*- coding: utf-8 -*-
from functools import reduce

def factorial(n):
    print(n)
    return reduce(lambda x,y:x*y, range(1, n+1))

if __name__ == '__main__':
    print("请输入最大阶乘的分母:")
    count = input()
    count = int(count)
    if count % 2 == 0:
        print("必须输入奇数，当前输入为[%s]" % count)
        assert False
    r = 0
    for i in range(1, count+1)[0::2]:
        r += 1 / factorial(i)
    print(r)
    print("%.2f" % r)


import pygame
import random

# Constants
SCREEN_WIDTH = 700
SCREEN_HEIGHT = 400
CELL_SIZE = 100
PLAYER_SPEED = CELL_SIZE
FRUIT_DROP_SPEED = CELL_SIZE

# Colors
WHITE = (255, 255, 255)
GREEN = (0, 255, 0)

# Initialize pygame
pygame.init()

# Load images
player_image = pygame.image.load("C:/Users/megha/A Thesis/Flappy Bird/Game_Of_catch/basket.png")  # Replace with your image path
player_image = pygame.transform.scale(player_image, (100, 90))

fruit_image = pygame.image.load("C:/Users/megha/A Thesis/Flappy Bird/Game_Of_catch/fruit.png")     # Replace with your image path
fruit_image = pygame.transform.scale(fruit_image, (80, 80))

screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
pygame.display.set_caption('Catch the Falling Fruit')
BLACK = (0, 0, 0)

def draw_grid():
    for x in range(0, SCREEN_WIDTH, CELL_SIZE):
        pygame.draw.line(screen, BLACK, (x, 0), (x, SCREEN_HEIGHT))
    for y in range(0, SCREEN_HEIGHT, CELL_SIZE):
        pygame.draw.line(screen, BLACK, (0, y), (SCREEN_WIDTH, y))

class Player:
    def __init__(self):
        self.x = 0
        self.y = SCREEN_HEIGHT - CELL_SIZE

    def move(self, direction):
        new_x = self.x + direction * CELL_SIZE
        if 0 <= new_x < SCREEN_WIDTH:
            self.x = new_x

    def draw(self):
        screen.blit(player_image, (self.x, self.y))

class Fruit:
    def __init__(self, x):
        self.x = x
        self.y = 0

    def move(self):
        self.y += FRUIT_DROP_SPEED/FPS

    def draw(self):
        screen.blit(fruit_image, (self.x, self.y))

    def is_caught(self, player):
        return self.y == player.y and self.x == player.x

    def is_missed(self):
        return self.y > SCREEN_HEIGHT

    def remove(self):
        screen.blit(fruit_image, (self.x, self.y))

player = Player()
fruits = []
score = 0

FPS = 30
fruit_drop_interval = FPS  # 30 frames, which means the fruit will drop once every second
fruit_drop_counter = 0

FRUIT_DROP_SPEED = CELL_SIZE

running = True
while running:
    screen.fill(WHITE)
    draw_grid()
    fruit_drop_counter += 1

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False
        elif event.type == pygame.KEYDOWN:
            if event.key == pygame.K_LEFT:
                player.move(-1)
                score -= 0.1
            elif event.key == pygame.K_RIGHT:
                player.move(1)
                score -= 0.1

    # Randomly spawn a fruit
    if random.random() < 0.02:  # 2% chance per frame to spawn a fruit
        x = random.choice([i * CELL_SIZE for i in range(4)])
        fruits.append(Fruit(x))

    # Check if it's time to move the fruits
    if fruit_drop_counter >= fruit_drop_interval:
        for fruit in fruits:
            fruit.move()
        fruit_drop_counter = 0

    # Move and draw fruits
    for fruit in fruits[:]:
        fruit.move()
        fruit.draw()

        if fruit.is_caught(player):
            score += 10
            fruit.remove(fruit)
        elif fruit.is_missed():
            score -= 10
            fruit.remove(fruit)

    player.draw()

    # Show score
    score_display = pygame.font.Font(None, 36).render(str(score), True, GREEN)
    screen.blit(score_display, (10, 10))

    pygame.display.flip()
    pygame.time.Clock().tick(FPS)

pygame.quit()


from src import utils
from os import getenv
from redis import Redis
import config
import messagebird

redis = Redis().from_url(getenv("REDIS_URL"))
msg_client = messagebird.Client(getenv("MESSAGEBIRD"))

def send_sms(src, dst, text, key=None):
	if key is not None:
		key = key.encode()
		keys = redis.lrange("keys", 0, -1)
		if key not in keys:
			return utils.error("Invalid key")

	fixed_src = utils.fix_number(src)
	if fixed_src.replace(config.phone_prepend, "").isdecimal() and len(fixed_src) == config.phone_total_length:
		src = fixed_src
	dst = utils.fix_number(dst)

	if len(dst) != config.phone_total_length:
		return utils.error("Destination number is not 8 characters long")

	if len(text) == 0 or (len(text.encode()) > 150 and key is not None):
		return utils.error("Invalid message length")

	if src.lower() == "nicesms" and key is not None:
		return utils.error("Reserved sender!")

	try:
		message = msg_client.message_create(src, dst, text)
		if key is not None:
			redis.lrem("keys", 0, key)
	except messagebird.ErrorException as e:
		print(e)
		return utils.error("Unknown Error! Contact the admin. (Key not used)")

	if key:
		print(f"Sent sms | {src} => {dst} | Key: {key.decode()} | Text: {text}")
	else:
		print(f"Sent sms | {src} => {dst} | Text: {text}")

	return message

def make_call(src, dst, text, key=None):
	if key is not None:
		key = key.encode()
		keys = redis.lrange("keys", 0, -1)
		if key not in keys:
			return utils.error("Invalid key")

	src = utils.fix_number(src)
	dst = utils.fix_number(dst)

	if len(dst) != 11 or len(src) != 11:
		return utils.error("Destination and Source numbers are not 8 characters long")

	if len(text) == 0 or len(text.encode()) > 140:
		return utils.error("Invalid tts message length")

	try:
		result = msg_client.voice_message_create(dst, text, params={"originator": src, "language": "da-DK"})
		if key is not None:
			redis.lrem("keys", 0, key)
	except messagebird.ErrorException:
		return utils.error("Unknown Error! Contact the admin. (Key not used)")

	if key:
		print(f"Made call | {src} => {dst} | Key: {key.decode()} | Text: {text}")
	else:
		print(f"Made call | {src} => {dst} | Text: {text}")

	return result

def get_msg(msg_id):
	try:
		return msg_client.message(msg_id)
	except messagebird.ErrorException:
		try:
			return msg_client.voice_message(msg_id)
		except messagebird.ErrorException:
			return utils.error("Message not found!")


import sys
from collections import defaultdict
from typing import Any
from typing import Callable
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from typing import Type

from xsdata.exceptions import XmlContextError
from xsdata.formats.bindings import T
from xsdata.formats.dataclass.compat import class_types
from xsdata.formats.dataclass.models.builders import XmlMetaBuilder
from xsdata.formats.dataclass.models.elements import XmlMeta
from xsdata.models.enums import DataType
from xsdata.utils.constants import return_input

class XmlContext:
    """
    The service provider for binding operations' metadata.

    :param element_name_generator: Default element name generator
    :param attribute_name_generator: Default attribute name generator
    :param class_type: Default class type `dataclasses`
    :param models_package: Restrict auto locate to a specific package
    """

    __slots__ = (
        "element_name_generator",
        "attribute_name_generator",
        "class_type",
        "cache",
        "xsi_cache",
        "sys_modules",
        "models_package",
    )

    def __init__(
        self,
        element_name_generator: Callable = return_input,
        attribute_name_generator: Callable = return_input,
        class_type: str = "dataclasses",
        models_package: Optional[str] = None,
    ):
        self.element_name_generator = element_name_generator
        self.attribute_name_generator = attribute_name_generator
        self.class_type = class_types.get_type(class_type)

        self.cache: Dict[Type, XmlMeta] = {}
        self.xsi_cache: Dict[str, List[Type]] = defaultdict(list)
        self.models_package = models_package
        self.sys_modules = 0

    def reset(self):
        self.cache.clear()
        self.xsi_cache.clear()
        self.sys_modules = 0

    def get_builder(
        self, globalns: Optional[Dict[str, Callable]] = None
    ) -> XmlMetaBuilder:
        return XmlMetaBuilder(
            class_type=self.class_type,
            element_name_generator=self.element_name_generator,
            attribute_name_generator=self.attribute_name_generator,
            globalns=globalns,
        )

    def fetch(
        self,
        clazz: Type,
        parent_ns: Optional[str] = None,
        xsi_type: Optional[str] = None,
    ) -> XmlMeta:
        """
        Fetch the model metadata of the given dataclass type, namespace and xsi
        type.

        :param clazz: The requested dataclass type
        :param parent_ns: The inherited parent namespace
        :param xsi_type: if present it means that the given clazz is
            derived and the lookup procedure needs to check and match a
            dataclass model to the qualified name instead
        """
        meta = self.build(clazz, parent_ns)
        subclass = None
        if xsi_type and meta.target_qname != xsi_type:
            subclass = self.find_subclass(clazz, xsi_type)

        return self.build(subclass, parent_ns) if subclass else meta

    def build_xsi_cache(self):
        """Index all imported dataclasses by their xsi:type qualified name."""
        if len(sys.modules) == self.sys_modules:
            return

        self.xsi_cache.clear()
        builder = self.get_builder()
        for clazz in self.get_subclasses(object):
            if self.is_binding_model(clazz):
                meta = builder.build_class_meta(clazz)

                if meta.target_qname:
                    self.xsi_cache[meta.target_qname].append(clazz)

        self.sys_modules = len(sys.modules)

    def is_binding_model(self, clazz: Type[T]) -> bool:
        if not self.class_type.is_model(clazz):
            return False

        return not self.models_package or (
            hasattr(clazz, "__module__")
            and isinstance(clazz.__module__, str)
            and clazz.__module__.startswith(self.models_package)
        )

    def find_types(self, qname: str) -> List[Type[T]]:
        """
        Find all classes that match the given xsi:type qname.

        - Ignores native schema types, xs:string, xs:float, xs:int, ...
        - Rebuild cache if new modules were imported since last run

        :param qname: Qualified name
        """
        if not DataType.from_qname(qname):
            self.build_xsi_cache()
            if qname in self.xsi_cache:
                return self.xsi_cache[qname]

        return []

    def find_type(self, qname: str) -> Optional[Type[T]]:
        """
        Return the most recently imported class that matches the given xsi:type
        qname.

        :param qname: Qualified name
        """
        types: List[Type] = self.find_types(qname)
        return types[-1] if types else None

    def find_type_by_fields(self, field_names: Set[str]) -> Optional[Type[T]]:
        """
        Find a dataclass from all the imported modules that matches the given
        list of field names.

        :param field_names: A unique list of field names
        """

        def get_field_diff(clazz: Type) -> int:
            meta = self.cache[clazz]
            local_names = {var.local_name for var in meta.get_all_vars()}
            return len(local_names - field_names)

        self.build_xsi_cache()
        choices = [
            (clazz, get_field_diff(clazz))
            for types in self.xsi_cache.values()
            for clazz in types
            if self.local_names_match(field_names, clazz)
        ]

        choices.sort(key=lambda x: (x[1], x[0].__name__))
        return choices[0][0] if len(choices) > 0 else None

    def find_subclass(self, clazz: Type, qname: str) -> Optional[Type]:
        """
        Compare all classes that match the given xsi:type qname and return the
        first one that is either a subclass or shares the same parent class as
        the original class.

        :param clazz: The search dataclass type
        :param qname: Qualified name
        """
        types: List[Type] = self.find_types(qname)
        for tp in types:
            # Why would an xml node with have an xsi:type that points
            # to parent class is beyond me but it happens, let's protect
            # against that scenario <node xsi:type="nodeAbstract" />
            if issubclass(clazz, tp):
                continue

            for tp_mro in tp.__mro__:
                if tp_mro is not object and tp_mro in clazz.__mro__:
                    return tp

        return None

    def build(
        self,
        clazz: Type,
        parent_ns: Optional[str] = None,
        globalns: Optional[Dict[str, Callable]] = None,
    ) -> XmlMeta:
        """
        Fetch from cache or build the binding metadata for the given class and
        parent namespace.

        :param clazz: A dataclass type
        :param parent_ns: The inherited parent namespace
        """
        if clazz not in self.cache:
            builder = self.get_builder(globalns)
            self.cache[clazz] = builder.build(clazz, parent_ns)
        return self.cache[clazz]

    def build_recursive(self, clazz: Type, parent_ns: Optional[str] = None):
        """Build the binding metadata for the given class and all of its
        dependencies."""
        if clazz not in self.cache:
            meta = self.build(clazz, parent_ns)
            for var in meta.get_all_vars():
                types = var.element_types if var.elements else var.types
                for tp in types:
                    if self.class_type.is_model(tp):
                        self.build_recursive(tp, meta.namespace)

    def local_names_match(self, names: Set[str], clazz: Type) -> bool:
        try:
            meta = self.build(clazz)
            local_names = {var.local_name for var in meta.get_all_vars()}
            return not names.difference(local_names)
        except (XmlContextError, NameError):
            # The dataclass includes unsupported typing annotations
            # Let's remove it from xsi_cache
            builder = self.get_builder()
            target_qname = builder.build_class_meta(clazz).target_qname
            if target_qname and target_qname in self.xsi_cache:
                self.xsi_cache[target_qname].remove(clazz)

            return False

    @classmethod
    def is_derived(cls, obj: Any, clazz: Type) -> bool:
        """
        Return whether the given obj is derived from the given dataclass type.

        :param obj: A dataclass instance
        :param clazz: A dataclass type
        """
        if obj is None:
            return False

        if isinstance(obj, clazz):
            return True

        return any(x is not object and isinstance(obj, x) for x in clazz.__bases__)

    @classmethod
    def get_subclasses(cls, clazz: Type):
        try:
            for subclass in clazz.__subclasses__():
                yield from cls.get_subclasses(subclass)
                yield subclass
        except TypeError:
            pass


from parse import load_dataframes
import pandas as pd
import shutil

def sort_stores_by_score(dataframes, n=20, min_reviews=30):
    """
    Req. 1-2-1 각 음식점의 평균 평점을 계산하여 높은 평점의 음식점 순으로 `n`개의 음식점을 정렬하여 리턴합니다
    -> 현재 음식점 별 평균 평점을 구하는 코드는 완성되어 있음
    우리가 해야할 것 : 평균 평점을 기준으로 음식점을 정렬하여 리턴

    Req. 1-2-2 리뷰 개수가 `min_reviews` 미만인 음식점은 제외합니다.
    """
    stores_reviews = pd.merge(
        dataframes["stores"], dataframes["reviews"], left_on="id", right_on="store"
    )

    # 리뷰 개수가 `min_reviews` 미만인 음식점은 제외합니다.
    # store 컬럼을 기준으로 group by를한다. 그리고 store_name을 표시한다.
    # 이 이유는, 이후 mean()을 하였을 때 정수형 컬럼을 제외하고 다 생략되지만, 이름을 확인하기 위해 stor_name을 추가한다.
    scores_group = stores_reviews[stores_reviews['review_cnt'] >= min_reviews].groupby(["store", "store_name", "branch", "review_cnt"])

    # 평균 평점을 기준으로 음식점을 정렬하여 리턴
    scores = scores_group.mean().sort_values(by=['score'], axis=0, ascending=False) # 평균 구하기

    return scores.head(n=n).reset_index()

def get_most_reviewed_stores(dataframes, n=20):
    """
    Req. 1-2-3 가장 많은 리뷰를 받은 `n`개의 음식점을 정렬하여 리턴합니다
    """
    stores_reviews = pd.merge(
        dataframes["stores"], dataframes["reviews"], left_on="id", right_on="store"
    )
    reviews = stores_reviews.groupby(["store", "store_name", "branch", "review_cnt", "tel", "area"])
    reviews_mean = reviews.mean()   # groupby를 했으면, 연산은 꼭 해야한다.

    stores_sorted_by_reviews = reviews_mean.sort_values(by='review_cnt', ascending=False)
    return stores_sorted_by_reviews.head(n=n).reset_index()

def get_most_active_users(dataframes, n=20):
    """
    Req. 1-2-4 가장 많은 리뷰를 작성한 `n`명의 유저를 정렬하여 리턴합니다.
    """
     # user와 review를 병합
    users_reviews = pd.merge(
        dataframes["users"], dataframes["reviews"], left_on="id", right_on="user"
    )
    # 확인하고 싶은 컬럼을 넣고, size() 함수를 사용하면 몇 개의 데이터가 group으로 묶였는지 새로운 컬럼이 생성된다. dataFrameGroup의 경우, count()와 같다.
    # 이후 reset_index, rename을 사용해 원하는 컬럼명을 설정한 뒤 사용한다.
    users = users_reviews.groupby(["user", "gender", "age"]).size().reset_index(name='reviews_cnt')

    users_sorted_by_review_counts = users.sort_values(by='reviews_cnt', ascending=False)
    return users_sorted_by_review_counts.head(n=n).reset_index()

def main():
    data = load_dataframes() # DUMP_FILE = os.path.join(DATA_DIR, "dump.pkl")

    # 단순 출력 형식문
    term_w = shutil.get_terminal_size()[0] - 1
    separater = "-" * term_w

    stores_most_scored = sort_stores_by_score(data)

    print("[최고 평점 음식점]")
    print(f"{separater}\n")
    for i, store in stores_most_scored.iterrows():
        print(
            "{rank}위: {store}({score}점)".format(
                rank=i + 1, store=store.store_name, score=store.score
            )
        )
    print(f"\n{separater}\n\n")

if __name__ == "__main__":
    main()