import json
import numpy as np
import pandas as pd
from pathlib import Path
import argparse
import torch
from tqdm import tqdm

def preprocess(data_name):
    OUT_DF = '../data/{}/ml_{}.csv'.format(data_name, data_name)
    OUT_ALL_TRAIN_DF = '../data/{}/ml_{}_all_train.csv'.format(data_name, data_name)
    OUT_TRAIN_DF = '../data/{}/ml_{}_train.csv'.format(data_name, data_name)
    OUT_VAL_DF = '../data/{}/ml_{}_val.csv'.format(data_name, data_name)
    OUT_TEST_DF = '../data/{}/ml_{}_test.csv'.format(data_name, data_name)

    OUT_EDGE_FEAT = '../data/{}/ml_{}_edge.npy'.format(data_name, data_name)
    OUT_NODE_FEAT = '../data/{}/ml_{}_node.npy'.format(data_name, data_name)

    names = [f'{data_name}.npz']
    items = [np.load(f'../data/{data_name}'+'/'+name) for name in names]

    x = items[0]['x']
    y = items[0]['y']  # .reshape(-1,1)
    edge_index = items[0]['edge_index']
    edge_type = items[0]['edge_type']
    ts = items[0]['edge_timestamp'].astype(np.float32)
    train_mask = items[0]['train_mask']
    valid_mask = items[0]['valid_mask']
    test_mask = items[0]['test_mask']

    # x = torch.tensor(x, dtype=torch.float).contiguous()
    # y = torch.tensor(y, dtype=torch.int64)
    # edge_index = torch.tensor(edge_index.transpose(), dtype=torch.int64).contiguous()
    # edge_type = torch.tensor(edge_type, dtype=torch.float)
    # train_mask = torch.tensor(train_mask, dtype=torch.int64)
    # valid_mask = torch.tensor(valid_mask, dtype=torch.int64)
    # test_mask = torch.tensor(test_mask, dtype=torch.int64)

    src, dst = edge_index[:, 0], edge_index[:, 1]
    idx = np.arange(src.shape[0])
    label_src = np.zeros(src.shape[0])
    label_dst = np.zeros(dst.shape[0])

    train_map_src = {}
    val_map_src = {}
    test_map_src = {}
    train_ts_src = {}
    val_ts_src = {}
    test_ts_src = {}

    train_map_dst = {}
    val_map_dst = {}
    test_map_dst = {}
    train_ts_dst = {}
    val_ts_dst = {}
    test_ts_dst = {}

    all_train_idx_src = []
    all_train_idx_dst = []

    for it, (s, d, t) in tqdm(enumerate(zip(src, dst, ts))):
        label_src[it] = y[s]
        label_dst[it] = y[d]
        # src node
        if s in train_mask:
            all_train_idx_src.append(it)
            if (s in train_map_src) and train_ts_src[s] >= t:
                pass
            else:
                train_map_src[s] = it
                train_ts_src[s] = t
        elif s in valid_mask:
            if (s in val_map_src) and val_ts_src[s] >= t:
                pass
            else:
                val_map_src[s] = it
                val_ts_src[s] = t
        elif s in test_mask:
            if (s in test_map_src) and test_ts_src[s] >= t:
                pass
            else:
                test_map_src[s] = it
                test_ts_src[s] = t
        # dst node
        if d in train_mask:
            all_train_idx_dst.append(it)
            if (d in train_map_dst) and train_ts_dst[d] >= t:
                pass
            else:
                train_map_dst[d] = it
                train_ts_dst[d] = t
        elif d in valid_mask:
            if (d in val_map_dst) and val_ts_dst[d] >= t:
                pass
            else:
                val_map_dst[d] = it
                val_ts_dst[d] = t
        elif d in test_mask:
            if (d in test_map_dst) and test_ts_dst[d] >= t:
                pass
            else:
                test_map_dst[d] = it
                test_ts_dst[d] = t

    for node in train_mask:
        if (node in train_map_src) and (node in train_map_dst):
            t_src, t_dst = train_ts_src[node], train_ts_dst[node]
            if t_src >= t_dst:
                train_map_dst.pop(node)
            else:
                train_map_src.pop(node)

    for node in valid_mask:
        if (node in val_map_src) and (node in val_map_dst):
            t_src, t_dst = val_ts_src[node], val_ts_dst[node]
            if t_src >= t_dst:
                val_map_dst.pop(node)
            else:
                val_map_src.pop(node)

    for node in test_mask:
        if (node in test_map_src) and (node in test_map_dst):
            t_src, t_dst = test_ts_src[node], test_ts_dst[node]
            if t_src >= t_dst:
                test_map_dst.pop(node)
            else:
                test_map_src.pop(node)

    train_idx_mask_src = list(train_map_src.values())
    val_idx_mask_src = list(val_map_src.values())
    test_idx_mask_src = list(test_map_src.values())

    train_idx_mask_dst = list(train_map_dst.values())
    val_idx_mask_dst = list(val_map_dst.values())
    test_idx_mask_dst = list(test_map_dst.values())

    train_src = np.concatenate([src[train_idx_mask_src], dst[train_idx_mask_dst]])
    train_dst =  np.concatenate([dst[train_idx_mask_src], src[train_idx_mask_dst]])
    train_ts =  np.concatenate([ts[train_idx_mask_src], ts[train_idx_mask_dst]])
    train_idx =  np.concatenate([idx[train_idx_mask_src], idx[train_idx_mask_dst]])
    train_label =  np.concatenate([label_src[train_idx_mask_src], label_dst[train_idx_mask_dst]])

    all_train_src = np.concatenate([src[all_train_idx_src], dst[all_train_idx_dst]])
    all_train_dst =  np.concatenate([dst[all_train_idx_src], src[all_train_idx_dst]])
    all_train_ts =  np.concatenate([ts[all_train_idx_src], ts[all_train_idx_dst]])
    all_train_idx =  np.concatenate([idx[all_train_idx_src], idx[all_train_idx_dst]])
    all_train_label =  np.concatenate([label_src[all_train_idx_src], label_dst[all_train_idx_dst]])

    val_src =  np.concatenate([src[val_idx_mask_src], dst[val_idx_mask_dst]])
    val_dst =  np.concatenate([dst[val_idx_mask_src], src[val_idx_mask_dst]])
    val_ts =  np.concatenate([ts[val_idx_mask_src], ts[val_idx_mask_dst]])
    val_idx =  np.concatenate([idx[val_idx_mask_src], idx[val_idx_mask_dst]])
    val_label =  np.concatenate([label_src[val_idx_mask_src], label_dst[val_idx_mask_dst]])

    test_src =  np.concatenate([src[test_idx_mask_src], dst[test_idx_mask_dst]])
    test_dst =  np.concatenate([dst[test_idx_mask_src], src[test_idx_mask_dst]])
    test_ts =  np.concatenate([ts[test_idx_mask_src], ts[test_idx_mask_dst]])
    test_idx =  np.concatenate([idx[test_idx_mask_src], ts[test_idx_mask_dst]])
    test_label =  np.concatenate([label_src[test_idx_mask_src], label_dst[test_idx_mask_dst]])

    print(f'train nodes: {train_src.shape[0]}, val nodes: {val_src.shape[0]}, test nodes: {test_src.shape[0]}')
    print(f'all train shape: {all_train_src.shape[0]}')

    df = pd.DataFrame({
        'src': src,
        'dst': dst,
        'ts': ts,
        'label': label_src,
        'idx': idx
    })
    df.sort_values("ts", inplace=True)

    all_train_df = pd.DataFrame({
        'src': all_train_src,
        'dst': all_train_dst,
        'ts': all_train_ts,
        'label': all_train_label,
        'idx': all_train_idx
    })
    all_train_df.sort_values("ts", inplace=True)

    train_df = pd.DataFrame({
        'src': train_src,
        'dst': train_dst,
        'ts': train_ts,
        'label': train_label,
        'idx': train_idx
    })
    train_df.sort_values("ts", inplace=True)

    val_df = pd.DataFrame({
        'src': val_src,
        'dst': val_dst,
        'ts': val_ts,
        'label': val_label,
        'idx': val_idx
    })
    val_df.sort_values("ts", inplace=True)

    test_df = pd.DataFrame({
        'src': test_src,
        'dst': test_dst,
        'ts': test_ts,
        'label': test_label,
        'idx': test_idx
    })
    test_df.sort_values("ts", inplace=True)

    df.to_csv(OUT_DF)
    all_train_df.to_csv(OUT_ALL_TRAIN_DF)
    train_df.to_csv(OUT_TRAIN_DF)
    val_df.to_csv(OUT_VAL_DF)
    test_df.to_csv(OUT_TEST_DF)

    np.save(OUT_EDGE_FEAT, edge_type)
    np.save(OUT_NODE_FEAT, x)

def run(data_name, bipartite=True):
    Path("../data/").mkdir(parents=True, exist_ok=True)
    preprocess(data_name)

parser = argparse.ArgumentParser('Interface for TGN data preprocessing')
parser.add_argument('--data', type=str, help='Dataset name',
                    default='dgraphfin')
parser.add_argument('--bipartite', action='store_true', help='Whether the graph is bipartite')

args = parser.parse_args()

run(args.data, bipartite=args.bipartite)


import yfinance as yf
import pandas as pd
import plotly
import plotly.graph_objects as go
import numpy as np

def df_indicators_columns(df):
    df["var"] = df[["High","Low","Close"]].std(axis = 1)
    df["mean"] = df[["High","Low","Close"]].mean(axis = 1)

    df["mean_rolling_15"] = df["mean"].rolling(15).mean()
    df["std_rolling_15"] = df["mean"].rolling(15).std()
    df["mean_std_rolling_plus_15"] = df["mean_rolling_15"] + df["std_rolling_15"]
    df["mean_std_rolling_less_15"] = df["mean_rolling_15"] - df["std_rolling_15"]

    df["mean_rolling_45"] = df["mean"].rolling(45).mean()
    df["std_rolling_45"] = df["mean"].rolling(45).std()
    df["mean_std_rolling_plus_45"] = df["mean_rolling_45"] + df["std_rolling_45"]
    df["mean_std_rolling_less_45"] = df["mean_rolling_45"] - df["std_rolling_45"]

    df["mean_rolling_100"] = df["mean"].rolling(100).mean()
    df["std_rolling_100"] = df["mean"].rolling(100).std()
    df["mean_std_rolling_plus_100"] = df["mean_rolling_100"] + df["std_rolling_100"]
    df["mean_std_rolling_less_100"] = df["mean_rolling_100"] - df["std_rolling_100"]

    return df

def graph_mean(df):
    if "Date" in list(df.columns):
        date_column = "Date"
    else:
        date_column = "Datetime"
    fig = go.Figure()
    fig.update_layout(title="Typical daily price", width=600, hovermode='x unified')

    fig.add_trace(go.Scatter(x=df[date_column], y=df["mean"], mode='lines', name="price"))
    return fig

def graph_Bollinger(df_bollinger, bollinger_days):
    df_bollinger["mean_rolling"] = df_bollinger["mean"].rolling(bollinger_days).mean()
    df_bollinger["std_rolling"] = df_bollinger["mean"].rolling(bollinger_days).std()
    df_bollinger["mean_std_rolling_plus"] = df_bollinger["mean_rolling"] + df_bollinger["std_rolling"]
    df_bollinger["mean_std_rolling_less"] = df_bollinger["mean_rolling"] - df_bollinger["std_rolling"]

    if "Date" in list(df_bollinger.columns):
        date_column = "Date"
    else:
        date_column = "Datetime"

    fig_boll = go.Figure()
    fig_boll.update_layout(title="Bollinger bands", width=600, hovermode='x unified')

    fig_boll.add_trace(go.Scatter(x=df_bollinger[date_column], y=df_bollinger["mean_rolling"], mode='lines', name=str(bollinger_days) + " rolling avg"))
    fig_boll.add_trace(go.Scatter(x=df_bollinger[date_column], y=df_bollinger["mean_std_rolling_plus"], mode='lines', name=str(bollinger_days) + " upper band"))
    fig_boll.add_trace(go.Scatter(x=df_bollinger[date_column], y=df_bollinger["mean_std_rolling_less"], mode='lines', name=str(bollinger_days) + " lower band"))

    fig_boll.add_trace(go.Scatter(x=df_bollinger[date_column], y=df_bollinger["mean"], mode='lines', name="Typical Price"))

    return fig_boll

def mean_deviation_calculation(df_CCI, CCI_days):
    df_CCI["mean_rolling"] = df_CCI["mean"].rolling(CCI_days).mean()
    mean_deviations = []

    for index, row in df_CCI.iterrows():
        #print(row["mean_rolling" + days])
        if str(row["mean_rolling"]) == "nan":
            mean_deviations.append(np.nan)
        else:
            mean_rolling = row["mean_rolling"]
            df_temp = df_CCI.iloc[:index+1]
            df_temp = df_temp.iloc[-20:]
            #print(len(df_temp))

            df_temp["diff"] = df_temp["mean"] - mean_rolling
            df_temp["diff"] = df_temp["diff"].abs()

            mean_deviation = df_temp["diff"].sum() / 20
            #print(mean_deviation)
            mean_deviations.append(mean_deviation)
        #print(mean_deviations)
    df_CCI["mean_deviation"] = mean_deviations
    return df_CCI

def graph_CCI(df_CCI, CCI_days):
    if "Date" in list(df_CCI.columns):
        date_column = "Date"
    else:
        date_column = "Datetime"
    df_CCI = mean_deviation_calculation(df_CCI, CCI_days)
    fig_CCI = go.Figure()
    fig_CCI.update_layout(title="CCI", width=600, hovermode='x unified')

    #for days in ["_15", "_45", "_100"]:
    df_CCI["CCI"] = (df_CCI["mean"] - df_CCI["mean_rolling"]) / (0.15 * df_CCI["mean_deviation"]) * 10
    fig_CCI.add_trace(go.Scatter(x=df_CCI[date_column], y=df_CCI["CCI"], mode='lines', name="CCI " + str(CCI_days)))
    return fig_CCI

def pos_DM_calculation(df):
    pos_DMs = [np.nan]
    for index, row in df.iloc[1:].iterrows():
        current_high = row["High"]
        previous_high = df.iloc[index-1:index]["High"].values[0]

        pos_DM = current_high - previous_high
        pos_DMs.append(pos_DM)
    df["pos_DM"] = pos_DMs

    return df

def TR_calculation(df):
    TRs = [np.nan]
    for index, row in df.iloc[1:].iterrows():
        current_high = row["High"]
        current_low = row["Low"]
        current_high_current_low = current_high-current_low

        previous_close = df.iloc[index-1:index]["Close"].values[0]
        current_high_previous_close = abs(current_high-previous_close)

        current_low_previous_close = abs(current_low-previous_close)

        TR = max([current_high_current_low, current_high_previous_close, current_low_previous_close])
        TRs.append(TR)
    df["TR"] = TRs

    return df

def neg_DM_calculation(df):
    neg_DMs = [np.nan]
    for index, row in df.iloc[1:].iterrows():
        current_low = row["Low"]
        previous_low = df.iloc[index-1:index]["Low"].values[0]

        neg_DM = previous_low - current_low
        neg_DMs.append(neg_DM)
    df["neg_DM"] = neg_DMs

    return df

def graph_ADX(df_ADX):
    if "Date" in list(df_ADX.columns):
        date_column = "Date"
    else:
        date_column = "Datetime"

    df_ADX = pos_DM_calculation(df_ADX)
    df_ADX = TR_calculation(df_ADX)
    df_ADX = neg_DM_calculation(df_ADX)
    df_ADX["pos_DM_rolling"] = df_ADX["pos_DM"].rolling(15).sum()
    df_ADX["neg_DM_rolling"] = df_ADX["neg_DM"].rolling(15).sum()
    df_ADX["TR_rolling"] = df_ADX["TR"].rolling(15).sum()

    df_ADX["pos_DI"] = ((df_ADX["pos_DM_rolling"]) - ((df_ADX["pos_DM_rolling"]) / 14) + (df_ADX["pos_DM"])) / ((df_ADX["TR_rolling"]) - ((df_ADX["TR_rolling"]) / 14) + (df_ADX["TR_rolling"]))
    df_ADX["pos_DI"] = df_ADX["pos_DI"]*100
    df_ADX["neg_DI"] = ((df_ADX["neg_DM_rolling"]) - ((df_ADX["neg_DM_rolling"]) / 14) + (df_ADX["neg_DM"])) / ((df_ADX["TR_rolling"]) - ((df_ADX["TR_rolling"]) / 14) + (df_ADX["TR_rolling"]))
    df_ADX["neg_DI"] = df_ADX["neg_DI"]*100
    df_ADX["ADX"] = df_ADX["pos_DI"] - df_ADX["neg_DI"]
    df_ADX["ADX"] = df_ADX["ADX"].abs()

    fig_ADX = go.Figure()
    fig_ADX.update_layout(title="ADX", width=600, hovermode='x unified')

    fig_ADX.add_trace(go.Scatter(x=df_ADX[date_column], y=df_ADX["pos_DI"], mode='lines', name="+ DI"))
    fig_ADX.add_trace(go.Scatter(x=df_ADX[date_column], y=df_ADX["neg_DI"], mode='lines', name="- DI"))
    fig_ADX.add_trace(go.Scatter(x=df_ADX[date_column], y=df_ADX["ADX"], mode='lines', name="ADX"))

    return fig_ADX


import asyncio
import dsnet
from dsnet.core import QueryType
import pytest
from sqlalchemy import create_engine
import msgpack
from starlette.testclient import TestClient

from dsnetserver.main import app
from dsnetserver import __version__
from dsnetserver.main import DATABASE_URL, database
from dsnetserver.models import metadata, pigeonhole_message_table, broadcast_query_table
from dsnet.message import PigeonHoleNotification

db = None

def setup_module(module):
    global db
    db = create_engine(DATABASE_URL)
    metadata.create_all(db)

def teardown_module(module):
    asyncio.run(database.execute(pigeonhole_message_table.delete()))
    asyncio.run(database.execute(broadcast_query_table.delete()))

def test_root():
    client = TestClient(app)
    response = client.get("/")
    assert response.status_code == 200
    assert response.json() == {"message": "Datashare Network Server version %s" % __version__,
                               "server_version": __version__,
                               "core_version": dsnet.__version__,
                               "query_type": QueryType.CLEARTEXT}

def test_post_get_ph_message():
    client = TestClient(app)

    response = client.post("/ph/deadbeef", content=b'binary message')
    assert response.status_code == 200

    response = client.get("/ph/deadbeef")
    assert response.status_code == 200
    assert response.headers['Content-Type'] == 'application/octet-stream'
    assert response.content == b'binary message'

def test_post_get_ph_message_with_wrong_address():
    client = TestClient(app)

    response = client.get("/ph/unknown")
    assert response.status_code == 404

def test_get_ph_messages_by_shortened_address():
    client = TestClient(app)

    response = client.post("/ph/beefc0fe", content=b'binary message 1')
    assert response.status_code == 200

    response = client.post("/ph/beefc0c0", content=b'binary message 2')
    assert response.status_code == 200

    response = client.get("/ph/beefc0")
    assert response.status_code == 200
    assert response.headers['Content-Type'] == 'application/octet-stream'
    messages = msgpack.unpackb(response.content, raw=False)
    assert len(messages) == 2
    assert messages[0] == b'binary message 1'
    assert messages[1] == b'binary message 2'

@pytest.mark.timeout(5)
def test_post_broadcast():
    client = TestClient(app)

    with client.websocket_connect('/notifications') as websocket:
        response = client.post("/bb/broadcast", content=b'query payload')
        assert response.status_code == 200
        data = websocket.receive_bytes()
        assert data == b'query payload'

@pytest.mark.timeout(5)
def test_post_response():
    client = TestClient(app)

    with client.websocket_connect('/notifications') as websocket:
        response = client.post("/ph/deadbeef", content=b'response data')
        assert response.status_code == 200
        data = websocket.receive_bytes()
        assert data == PigeonHoleNotification('deadbe').to_bytes()


# This is part of MiMiCPy

"""

This module contains Shell class to take care of low level
paramiko code for opening a remote shell and running commands

"""

from os.path import expanduser
import time
from ..utils.errors import MiMiCPyError, defaultHook

class Shell:
    def __init__(self, name, config=None):

        self.name = name

        if config == None:
            config = expanduser("~")+'/.ssh/config'

        self.config = config

        conf = self._lookup(name)

        if 'proxyjump' in conf:
            proxy_name, port = conf['proxyjump'].split(':')
            vmtransport = self._getssh(proxy_name).get_transport()
            dest_addr = (self._lookup(name)['hostname'], int(port))
            local_addr = (self._lookup(proxy_name)['hostname'], int(port))
            vmchannel = vmtransport.open_channel("direct-tcpip", dest_addr, local_addr)

            self.ssh = self._getssh(name, sock=vmchannel)

        else:
            self.ssh =  self._getssh(name)

        self.sftp = self.ssh.open_sftp()
        self.decoder = 'utf-8'
        self.is_open = True

        self.ssh_bg = None

    def _lookup(self, name):
        try:
            import paramiko
        except ImportError:
            raise MiMiCPyError("Paramiko python package not installed! Install it to remotely run commands")

        config = paramiko.SSHConfig()
        try:
            f = open(self.config)
        except FileNotFoundError:
            raise FileNotFoundError(f"No SSH config file found in {self.config}")

        config.parse(f)
        conf = config.lookup(name)

        return conf

    def _getssh(self, name, sock=None):
        try:
            import paramiko
        except ImportError:
            raise MiMiCPyError("Paramiko python package not installed! Install it to remote host")

        conf = self._lookup(name)

        if 'hostname' not in conf:
            raise MiMiCPyError(f"Hostname not found for {name} in SSH config file")
        if 'user' not in conf:
            raise MiMiCPyError(f"Username not found for {name} in SSH config file")

        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        if sock is None:
            ssh.connect(hostname=conf['hostname'], username=conf['user'], look_for_keys=False)
        else:
            ssh.connect(hostname=conf['hostname'], username=conf['user'], look_for_keys=False, sock=sock)

        return ssh

    def __del__(self):
        if not hasattr(self, 'is_open'): return
        if self.is_open:
            self.is_open = False
            self.sftp.close()
            self.ssh.close()
            if self.ssh_bg: self.ssh_bg.__del__()

    def runbg(self, cmd, hook=None, dirc='', query_rate=3):
        if self.ssh_bg == None:
            self.ssh_bg = _ShellBGRun(self.ssh, self.decoder)

        self.ssh_bg.run(f'cd {self.pwd()}/{dirc}')
        self.ssh_bg.run(self.loader_str)
        if not hook:
            hook = defaultHook

        out = self.ssh_bg.run(cmd.split(';')[-1], hook, query_rate)

        return out
        # do not destory ssh_bg, as this stops the process
        # kill it in the deconstructor

    def run(self, cmd, stdin=None, hook=None, fresh=False, dirc=''):
        if not fresh and self.loader_str:
            cmd = self.loader_str + ' ; ' + cmd

        cmd = f'cd {self.pwd()}/{dirc} ; ' + cmd

        tran = self.ssh.get_transport()
        chan = tran.open_session()
        chan.get_pty()
        stdout = chan.makefile('rb')
        chan.exec_command(cmd)

        if stdin:
            sin = chan.makefile('wb')
            sin.channel.send(stdin+'\n')
            sin.channel.shutdown_write()

        out = stdout.read().decode(self.decoder)

        if not fresh:
            out = out.replace(self.loader_out, '')

        if not hook:
            hook = defaultHook

        hook(cmd.split(';')[-1], out)

        return out

    def __enter__(self): return self

    def __exit__(self, exc_type, exc_val, exc_tb): self.__del__()

class _ShellBGRun:

    def __init__(self, ssh, decoder):
        self.chan = ssh.invoke_shell()
        self.stdout = ''
        self.decoder = decoder

    def queryStdout(self, buff = 1024):

       while self.chan.recv_ready():
           self.stdout += self.chan.recv(buff).decode(self.decoder)

    def run(self, cmd, hook=None, query_rate=0.2):

        self.queryStdout()

        startout = self.stdout

        self.chan.send(cmd+'\n')

        prev = startout

        while True:
            time.sleep(query_rate)

            self.queryStdout()

            if prev != self.stdout:
                text = self.stdout.replace(prev, '')

                prev = self.stdout

                if hook:
                    if hook(cmd, text): break

            else: break

        lines = self.stdout.replace(startout, '').splitlines()[1:-1]

        return '\n'.join(lines)

    def __del__(self):
        self.chan.close()


#!/usr/bin/env python3

import json
import subprocess
import sys

TARGET_TAG = 'latest'
ECS_CLUSTER = 'rust-ecs-prod'

def main():
    """CLI entrypoint of the program."""
    if len(sys.argv) < 2:
        usage()
    repository_name = sys.argv[1]
    images = get_images(repository_name)
    isRetry = None
    while True:
        selected_image_index = let_user_pick_image(images, isRetry)
        if selected_image_index == -1:
            exit(0)

        if selected_image_index is not None:
            break
        else:
            isRetry = True

    image = images[selected_image_index]
    imageTags = image.get("imageTags",[])
    if TARGET_TAG in imageTags:
        err(f"selected image already tagged as {TARGET_TAG}")

    eprint(f"selected option: {image}\n")
    manifest = get_image_manifest(repository_name, image["imageDigest"])
    retag_image(repository_name,manifest)
    image_pushed_at = format_time(image["imagePushedAt"])
    eprint(f"image pushed at {image_pushed_at} retaged as '{TARGET_TAG}'\n")
    if can_redeploy(repository_name):
        redeployed = force_redeploy()
        print("{}".format("successfully rollback and re-deploy" if redeployed else "Successfully rolled back the image, but redeploying the service with the same name failed"))
    else:
        eprint(f"Successfully rolled back the image, but no service with the same name to redeploy found")

def let_user_pick_image(images, isRetry=None):
    if isRetry is None:
        print("Please choosean image to rollback:\n")
        for idx, image in enumerate(images):
            image_pushed_at = format_time(image["imagePushedAt"])
            tags = ", ".join(image.get("imageTags", []))
            print("{}) {} {}".format(idx+1,image_pushed_at,"("+tags+")" if tags != "" else ""))
        print("")
    else:
        print(f"Invalid value, please check the list of possible values...\n")

    i = input("Enter image number (or 0 to exit): ")
    try:
        idx = int(i)
        if -1 < idx <= len(images):
            return idx-1
    except:
        pass
    return None

def get_images(repository_name):
    """Call ecr to get available images"""
    eprint("obtaining available images")
    try:
        out = json.loads( run_command([
            "aws", "ecr", "describe-images",
            "--repository-name", repository_name,
            "--no-paginate"
        ]).stdout)
    except subprocess.CalledProcessError as e:
        err(f"failed to get availabe images from repository: {e}" )

    return list(sorted(out["imageDetails"], key=lambda image: image["imagePushedAt"], reverse=True))

def get_image_manifest(repository_name, imageDigest):
    """Call ecr batch-get-image to get the image manifest"""
    try:
        out = json.loads( run_command([
            "aws", "ecr", "batch-get-image",
            "--repository-name", repository_name,
            "--image-ids", "imageDigest={}".format(imageDigest)
        ]).stdout)
    except subprocess.CalledProcessError as e:
        err(f"failed to get availabe images from repository: {e}" )

    return out["images"][0]["imageManifest"]

def retag_image(repository_name, manifest):
    """ """
    try:
        out = json.loads( run_command([
            "aws", "ecr", "put-image",
            "--repository-name", repository_name,
            "--image-manifest", manifest,
            "--image-tag", TARGET_TAG
        ]).stdout)
    except subprocess.CalledProcessError as e:
        err(f"failed to re-tag image as {TARGET_TAG}")

    return True

def can_redeploy(repository_name):
    """return True IFF there is a service with the same name of the repository."""
    try:
        out = json.loads( run_command([
            "aws", "ecs", "list-services",
            "--cluster", ECS_CLUSTER,
            "--no-paginate"
        ]).stdout)

        services = out["serviceArns"]
        for service in services:
            # last part of arn is the service name.
            if repository_name == service.split('/')[-1]:
                return True

        return False
    except subprocess.CalledProcessError as e:
        err(f"failed to list services in cluste {ECS_CLUSTER}")

def force_redeploy(service_name):
    """Force redeploy on ecs"""
    try:
        out = json.loads( run_command([
            "aws", "ecs", "update-service",
            "--cluster", ECS_CLUSTER,
            "--service", service_name,
            "--force-new-deployment"
        ]).stdout)

        return True
    except subprocess.CalledProcessError as e:
        err(f"failed to re-deploy service {service_name}")

###############
#  Utilities  #
###############

def format_time(time):
    return time.replace("T", " ").replace("+", " +")

def usage():
    """ print usage help and exit."""
    print("error: missing argument, you need to pass the repository name to use. e.g:")
    print("aws-rollback.py <repository name>")
    exit(1)

def eprint(*args, **kwargs):
    """Just like print(), but outputs on stderr."""
    print(*args, file=sys.stderr, **kwargs)

def err(*args, **kwargs):
    """Show the error message and exit with status 1."""
    eprint("error:", *args, **kwargs)
    exit(1)

def run_command(*args, **kwargs):
    """Run a CLI program capturing stdout. Raise an exception if it fails."""
    return subprocess.run(
        *args,
        stdout=subprocess.PIPE,
        check=True,
        **kwargs,
    )

####################
#  CLI Invocation  #
####################

if __name__ == "__main__":
    main()


from setuptools import setup, find_packages

with open('README.md', 'r') as f:
    long_description = f.read()

setup(
    name='Flask-Discussion',
    version='0.1.1',

    description='Comment system integration for Flask applications',
    long_description=long_description,
    long_description_content_type='text/markdown',

    url='https://github.com/rmed/flask-discussion',

    author='Rafael Medina García',
    author_email='rafamedgar@gmail.com',

    license='MIT',

    classifiers=[
        'Development Status :: 4 - Beta',
        'Framework :: Flask',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python'
    ],

    keywords='flask discussion comments commenting',

    packages=find_packages(),

    include_package_data=True,

    install_requires=[
        'Flask>=1.1.1',
    ]
)


import fingerprint_feature_extractor
import cv2
import numpy as np

# Read the fingerprint images
img1 = cv2.imread('enhanced/1.jpg', 0)
img2 = cv2.imread('enhanced/3.jpg', 0)

# Extract minutiae features from both fingerprints
FeaturesTerminations1, FeaturesBifurcations1 = fingerprint_feature_extractor.extract_minutiae_features(img1, spuriousMinutiaeThresh=10, invertImage=False, showResult=False, saveResult=False)
FeaturesTerminations2, FeaturesBifurcations2 = fingerprint_feature_extractor.extract_minutiae_features(img2, spuriousMinutiaeThresh=10, invertImage=False, showResult=False, saveResult=False)

# Combine both types of features for each fingerprint into a single list
Features1 = FeaturesTerminations1 + FeaturesBifurcations1
Features2 = FeaturesTerminations2 + FeaturesBifurcations2

# Create keypoint objects for each feature in both fingerprints
# kp1 = [cv2.KeyPoint(x=f.locX, y=f.locY, _size=10) for f in Features1]
# kp2 = [cv2.KeyPoint(x=f.locX, y=f.locY, _size=10) for f in Features2]
kp1 = [cv2.KeyPoint(x=f.locX, y=f.locY, size=10) for f in Features1]
kp2 = [cv2.KeyPoint(x=f.locX, y=f.locY, size=10) for f in Features2]

# Create a SIFT descriptor object
sift = cv2.SIFT_create()

# Calculate SIFT descriptors for each keypoint in both fingerprints
_, des1 = sift.compute(img1, kp1)
_, des2 = sift.compute(img2, kp2)

# Create a BFMatcher object to match SIFT descriptors
bf = cv2.BFMatcher()

# Match SIFT descriptors of both fingerprints using the BFMatcher object
matches = bf.match(des1, des2)

# Sort the matches in ascending order of their distances
matches = sorted(matches, key=lambda x: x.distance)

# Keep only the top 10% of matches based on distance
matches = matches[:int(len(matches) * 0.1)]

if len(matches) > 0:
    print("Fingerprints matched!")
else:
    print("Fingerprints do not match.")


import requests
import json
import numpy as np
import pandas as pd
from datetime import datetime
from config import *

################ CACHING & DATA RETRIEVAL ###################
# -----------------------------------------------------------------------------
# Constants
# -----------------------------------------------------------------------------
CACHE_FNAME = 'cache_file.json'
DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
DEBUG = False
apiKey = apiKeyETHERSCAN

# -----------------------------------------------------------------------------
# Load cache file
# -----------------------------------------------------------------------------
try:
    with open(CACHE_FNAME, 'r') as cache_file:
        cache_json = cache_file.read()
        CACHE_DICTION = json.loads(cache_json)
except:
    CACHE_DICTION = {}

def has_cache_expired(timestamp_str, expire_in_days): # BUG 1
    """Check if cache timestamp is over expire_in_days old"""
    # gives current datetime
    now = datetime.now()

    # datetime.strptime converts a formatted string into datetime object
    cache_timestamp = datetime.strptime(timestamp_str, DATETIME_FORMAT)

    # subtracting two datetime objects gives you a timedelta object
    delta = now - cache_timestamp
    delta_in_days = delta.days

    # now that we have days as integers, we can just use comparison
    # and decide if cache has expired or not
    if delta_in_days < expire_in_days: #BUG 2
        return False
    else:
        return True

def params_unique_combination(baseurl, params_d, private_keys=["api_key"]):
    alphabetized_keys = sorted(params_d.keys())
    res = []
    for k in alphabetized_keys:
        if k not in private_keys:
            res.append("{}-{}".format(k, params_d[k]))
    return baseurl + "_".join(res)

def get_from_cache(url, params_d):
    """If URL exists in cache and has not expired, return the html, else return None"""
    cache_key = params_unique_combination(url, params_d)
    if cache_key in CACHE_DICTION:
        url_dict = CACHE_DICTION[cache_key]
 #       html = CACHE_DICTION[url]['html']
        if has_cache_expired(url_dict['timestamp'], url_dict['expire_in_days']):
            # also remove old copy from cache
            del CACHE_DICTION[cache_key]
            html = None
        else:
            html = CACHE_DICTION[cache_key]['html']
    else:
        html = None

    return html

def set_in_cache(url, params_d, html, expire_in_days):
    """Add URL and html to the cache dictionary, and save the whole dictionary to a file as json"""
    cache_key = params_unique_combination(url, params_d)

    CACHE_DICTION[cache_key] = {
        'html': html,
        'timestamp': datetime.now().strftime(DATETIME_FORMAT),
        'expire_in_days': expire_in_days
    }

    with open(CACHE_FNAME, 'w') as cache_file:
        cache_json = json.dumps(CACHE_DICTION)
        cache_file.write(cache_json)

def get_html_from_url(url, params_d, expire_in_days=7): #Added params_d
    """Check in cache, if not found, load html, save in cache and then return that html"""
    # check in cache
    html = get_from_cache(url, params_d)
 #   print(html)
    if html is not None:
        if DEBUG:
            print('Loading from cache: {0}'.format(url))
    else:
 #       if DEBUG:
        print('Fetching a fresh copy: {0}'.format(url))
 #       print()

        # fetch fresh
        response = requests.get(url, params=params_d)

        # Deleted line about encoding because it was messing up my shit

        html = response.text

        # cache it
        set_in_cache(url, params_d, html, expire_in_days)

    return html

######################## END CACHING #############################################

def RequestWalletAddress(address):
    #contractAddress = '0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2'
    params_d = {}
    baseurl = "https://api.etherscan.io/api"
    params_d['module'] = "account"
    params_d['action'] = "tokentx"
    params_d['contractaddress'] = '0xdac17f958d2ee523a2206206994597c13d831ec7' # tether
    params_d['address'] = address
    #params_d['offset'] = 100
    params_d['sort'] = "asc"
    params_d['apikey'] = apiKey

    google_results = get_html_from_url(baseurl, params_d, expire_in_days=1)
    #print(google_results)
    results = json.loads(google_results)['result']

    return results

def ResultScan(results, address):
    outAddress = {} # who are they submitting to
    inAddress = {} # who is sending to them

    for transaction in results:
        if transaction['from'].upper() == address.upper(): # outgoing transactions
            if transaction['to'] not in outAddress:
                outAddress[transaction['to']] = 0
            outAddress[transaction['to']] = outAddress[transaction['to']] + 1
        if transaction['to'].upper () == address.upper(): # incoming transactions
            if transaction['from'] not in inAddress:
                inAddress[transaction['from']] = 0
            inAddress[transaction['from']] = inAddress[transaction['from']] + 1

    return outAddress, inAddress

def GetTransactionInfo(df, origAddress):

    ###Transaction Info: Receivers

    # create new transactions doc
    transactions = pd.DataFrame()

    addresses = pd.unique(df['to'])

    for address in addresses:
        if address.upper() == origAddress.upper():
            continue

        # grab data
        sub = df[df['to'] == address]

        # collect data
        rowDict = {}
        rowDict["address"] = address
        rowDict["numTransactions"] = int(sub.shape[0]) # number of rows
        rowDict["sumMoney"] = np.sum(sub['value'])
        rowDict["type"] = "OUT"

        transactions = transactions.append(rowDict, ignore_index = True)

    ### Transaction Info: Sources
    addresses = pd.unique(df['from'])

    for address in addresses:

        if address.upper() == origAddress.upper():
            continue

        # grab data
        sub = df[df['from'] == address]

        # collect data
        rowDict = {}
        rowDict["address"] = address
        rowDict["numTransactions"] = int(sub.shape[0]) # number of rows
        rowDict["sumMoney"] = np.sum(sub['value'])
        rowDict["type"] = "IN"

        transactions = transactions.append(rowDict, ignore_index = True)

    return transactions


from abc import abstractmethod
from aiogram import Bot
from aiogram import types
from taxi_bot.database_handler import DataBase
from taxi_bot.load_config import Config
from taxi_bot.logger import Logger
from taxi_bot.buttons import keyboard_generator
from aiogram.types import ReplyKeyboardRemove
from aiogram.dispatcher.filters.state import State, StatesGroup

class RegistrationForm(StatesGroup):
    phone_number_request = State()

class BaseHandler:

    def __init__(
            self,
            db: DataBase,
            bot: Bot,
            config: Config,
            kbs: dict,
            logger: Logger,
        ):
        self._db = db
        self._bot = bot
        self._config = config
        self._kbs = kbs
        self._logger = logger

    @abstractmethod
    async def __call__(self, *args, **kwargs):
        raise NotImplementedError

    async def set_state(self, state, field, value):
        async with state.proxy() as data:
            data[field] = value

    async def update_state(self, state, field, value):
        try:
            async with state.proxy() as data:
                data[field] = data[field] + value
        except KeyError:
            return

    async def get_state(self, state, field):
        try:
            async with state.proxy() as data:
                value = data[field]
            return value
        except KeyError:
            return

    async def finish_state(self, state):
        if await state.get_state():
            await state.finish()

    async def create_user(self, message, referral=None, phone_number_requaried=True, state=None):
        phone_number = self._db.create_user(message, referral)
        chat_id = message.from_user.id
        if phone_number_requaried and not phone_number:
            await RegistrationForm.phone_number_request.set()
            await self.send_message(chat_id, None, self._config.messages['phone_number_request'], 'request_contact')
        return phone_number

    def phone_number_parser(self, string):
        phone_number = ''.join([c for c in string if c in '1234567890'])
        if len(phone_number) < 10:
            return
        if phone_number[0] in ['7', '8'] and len(phone_number) == 11:
            phone_number = phone_number[1:]
        if phone_number[0] in ['9'] and len(phone_number) == 10:
            return f'+7{phone_number}'
        return

    async def show_order(self, order, chat_id, kb_name=None, delete_old=False):
        import re
        order_id = order.order_id
        order_dt = self.get_time_from_dt(order.order_dt)
        destination = order.location_to
        price = f"{order.price} рублей"
        geo = re.match(r'\d{1,3}\.\d+|\d{1,3}\.\d+', order.location_from)
        if geo:
            lat, lon = order.location_from.split('|')
            await self.send_venue(
                chat_id=chat_id,
                lat=lat,
                lon=lon,
                destination=destination,
                price=price,
                order_id=order_id,
                kb_name=kb_name,
                delete_old=delete_old,
            )
        else:
            text = [
                f"Откуда: {order.location_from}",
                f"Куда: {destination}",
                f"Стоимость: {price}",
                f"Время заказа: {order_dt}",
            ]
            text = '\n'.join(text)
            await self.send_message(
                chat_id=chat_id,
                order_id=order_id,
                text=text,
                kb_name=kb_name,
                delete_old=delete_old,
            )

    async def show_active_orders(self, driver_id):
        active_orders = self._db.get_orders_by_status(100)
        for order in active_orders:
            await self.show_order(order, driver_id, 'driver_accept_refuse')

    async def delete_old_messages(self, order_id=None, chat_id=None, message_id=None, force=False):
        orders = self._db.get_log_messages(order_id=order_id, chat_id=chat_id, message_id=message_id)
        log_ids = list()
        for order in orders:
            chat_id, message_id, order_id = order.chat_id, order.message_id, order.order_id
            if (order.shown==2) and (not force):
                continue
            try:
                await self._bot.delete_message(chat_id, message_id)
            except Exception as err:
                self.log_error(chat_id, message_id, order_id, self, err)
            log_ids.append(order.log_id)
        self._db.update_log_status(log_ids=log_ids)

    async def send_message(self, chat_id, order_id, text, kb_name=None, kb_suffix=None, delete_old=False, shown=1):
        kb_suffix = kb_suffix if kb_suffix else order_id
        kb = keyboard_generator(self._config.buttons[kb_name], kb_suffix) if kb_name else None
        try:
            message = await self._bot.send_message(
                chat_id=chat_id,
                text=text,
                reply_markup=kb,
                parse_mode='html'
            )
            if delete_old:
                await self.delete_old_messages(chat_id=chat_id)
                if order_id:
                    await self.delete_old_messages(order_id=order_id)
            self.log_message(chat_id, message.message_id, order_id, self, text, shown=shown)
            return message
        except Exception as err:
            self.log_error(chat_id, None, order_id, self, err)

    def time_delta(self, dt1, dt2, format=True):
        """
        dt1 >= dt2
        """
        if dt1 == dt2:
            return
        if not dt1 or not dt2:
            return ''
        delta_seconds = (dt1 - dt2).seconds
        if not format:
            return delta_seconds
        hours = str(delta_seconds//3600).zfill(2)
        minutes = str(delta_seconds%3600//60).zfill(2)
        seconds = str(delta_seconds%60).zfill(2)
        return f"{hours}:{minutes}:{seconds}"

    def get_time_from_dt(self, dt):
        return (dt, dt.strftime('%H:%M:%S')) if dt else (None, '')

    def get_date_from_dt(self, dt):
        return (dt, dt.strftime('%Y-%m-%d')) if dt else (None, '')

    async def show_admin_order(self, order, callback_query=None):
        from datetime import datetime

        order_id = order.order_id
        passenger = self._db.get_user_by_id(order.passenger_id)
        driver = self._db.get_user_by_id(order.driver_id)
        driver_contact = f"{self.tg_user_link(order.driver_id, driver.first_name)} {driver.phone_number}" if driver else 'Водитель не назначен'
        order_date = order.order_dt.date()
        order_dt, order_dt_str = self.get_time_from_dt(order.order_dt)
        accept_dt, accept_dt_str = self.get_time_from_dt(order.accept_dt)
        wait_dt, wait_dt_str = self.get_time_from_dt(order.wait_dt)
        pick_dt, pick_dt_str = self.get_time_from_dt(order.pick_dt)
        end_dt, end_dt_str = self.get_time_from_dt(order.end_dt)
        now, now_str = self.get_time_from_dt(datetime.now())
        location_from = order.location_from.replace('|', ',')
        location_from = "" + location_from + ""
        text = [
            f"#{order_id}",
            f"Дата заказа: {order_date}",
            f"Create:  {order_dt_str}",
            f"Accept: {accept_dt_str} ({self.time_delta(accept_dt, order_dt)})",
            f"Wait:     {wait_dt_str} ({self.time_delta(wait_dt, accept_dt)})",
            f"Pick:     {pick_dt_str} ({self.time_delta(pick_dt, wait_dt)})",
            f"End:      {end_dt_str} ({self.time_delta(end_dt, pick_dt)})",
            f"{self.tg_user_link(order.passenger_id, 'Пассажир')} {passenger.phone_number}",
            driver_contact,
            f"Откуда: {location_from}",
            f"Куда: {order.location_to}",
            f"Цена: {order.price}",
            f"Статус: ({order.order_status}) {self.map_status(order.order_status)}",
            f"Время обновления: {now_str}",
        ]
        text = '\n'.join(text)
        if not callback_query:
            await self.send_message(self._config.ADMIN_ID, order_id, text, 'order_details', shown=2)
            await self.send_message(self._config.WIFE_ID, order_id, text, 'order_details', shown=2)
        else:
            chat_id, message_id, order_id, optionals = await self.message_data(callback_query)
            await self.edit_message(chat_id, message_id, order_id, text, 'order_details')
            await self.answer_callback_query(callback_query)

    async def show_admin_user(self, user_id):
        # from datetime import datetime
        # def str_date_from_dt(dt):
        #     return dt.strftime('%Y-%m-%d')

        user = self._db.get_user_by_id(user_id)
        if user:
            user_info = [
                f'ID: {user_id}',
                self.tg_user_link(user_id, user.first_name),
                f'Phone number: {user.phone_number}',
                f'Registration date: {self.get_date_from_dt(user.user_registration_date)[1]}',
                f'Active: {user.active}',
            ]
            if user.referral:
                user_info.append(f"Referral: {self.tg_user_link(user.referral, 'referral link')}")
            if user.driver_status:
                user_info += [
                    f'Driver registration: {self.get_date_from_dt(user.driver_registration_date)[1]}',
                    f'Driver status: {user.driver_status}',
                    f'Driver car: {user.driver_car}',
                    f'Driver balance: {user.driver_balance}',
                    f'Driver shift_id: {user.driver_shift_id}',
                ]
            user_info = '\n'.join(user_info)
            await self.send_message(self._config.ADMIN_ID, None, user_info, kb_name='admin_user_actions', kb_suffix=user_id)

    def create_shift_report(self, shift):
        from datetime import datetime

        start_shift = shift.shift_start
        end_shift = shift.shift_end
        total_trips = shift.total_trips
        total_income = shift.total_income
        if end_shift:
            shift_duration = self.time_delta(end_shift, start_shift)
            text = '\n'.join([
                f"Статистика:",
                f"Вы работали с {start_shift.strftime('%H:%M')} до {end_shift.strftime('%H:%M')}",
                f"Продолжительность смены: {shift_duration}",
                f"Закрытых заказов: {total_trips}",
                f"Заработано: {total_income} рублей",
            ])
        else:
            shift_duration = self.time_delta(datetime.now(), start_shift)
            text = '\n'.join([
                f"Статистика:",
                f"Вы работаете с {start_shift.strftime('%H:%M')}",
                f"Продолжительность смены: {shift_duration}",
                f"Закрытых заказов: {total_trips}",
                f"Заработано: {total_income} рублей",
            ])
        return text

    async def edit_message(self, chat_id, message_id, order_id, text=None, kb_name=None):
        kb = keyboard_generator(self._config.buttons[kb_name], order_id) if kb_name else None
        if text:
            try:
                message = await self._bot.edit_message_text(
                    chat_id=chat_id,
                    message_id=message_id,
                    text=text,
                    reply_markup=kb,
                    parse_mode='html'
                )
                self.log_info(chat_id, message.message_id, order_id, self, f'{text}|{kb_name}')
            except Exception as err:
                self.log_error(chat_id, None, order_id, self, err)
        else:
            try:
                message = await self._bot.edit_message_reply_markup(
                    chat_id=chat_id,
                    message_id=message_id,
                    reply_markup=kb,
                )
                self.log_info(chat_id, message.message_id, order_id, self, f'{kb_name}')
            except Exception as err:
                self.log_error(chat_id, None, order_id, self, err)

    async def edit_reply_markup(self, callback_query: types.CallbackQuery, kb_name, order_id):
        chat_id = callback_query.from_user.id
        try:
            message = await callback_query.message.edit_reply_markup(keyboard_generator(self._config.buttons[kb_name], order_id))
            self.log_info(chat_id, message.message_id, order_id, self, f'edit_kb: {kb_name}')
        except Exception as err:
            self.log_error(chat_id, None, order_id, self, err)

    async def send_venue(self, chat_id, lat, lon, destination, price, order_id, kb_name=None, delete_old=False):
        kb = keyboard_generator(self._config.buttons[kb_name], order_id) if kb_name else None
        destination = f"Куда: {destination}"
        try:
            message = await self._bot.send_venue(
                chat_id=chat_id,
                latitude=lat,
                longitude=lon,
                title=destination,
                address=price,
                reply_markup=kb,
            )
            if delete_old:
                await self.delete_old_messages(chat_id=chat_id)
                if order_id:
                    await self.delete_old_messages(order_id=order_id)
            text = f'lat: {lat}; lon: {lon} dest: {destination}; price: {price}'
            self.log_message(chat_id, message.message_id, order_id, self, text)
        except Exception as err:
            self.log_error(chat_id, None, order_id, self, err)

    async def remove_inline_markup(self, query, order_id):
        chat_id, message_id, order_id, optionals = await self.message_data(query)
        try:
            if isinstance(query, types.Message):
                message = await query.delete_reply_markup()
            elif isinstance(query, types.CallbackQuery):
                message = await query.message.delete_reply_markup()
            self.log_info(chat_id, message.message_id, order_id, self, 'remove_kb')
        except Exception as err:
            self.log_error(chat_id, message_id, order_id, self, err)

    async def discard_reply_markup(self, chat_id, order_id):
        try:
            message = await self._bot.send_message(chat_id, "msg_text", reply_markup=ReplyKeyboardRemove())
        except Exception as err:
            self.log_error(chat_id, None, order_id, self, err)
        await message.delete()

    async def message_data(self, input_, order_id=None):
        chat_id = input_.from_user.id
        optionals = dict()
        if isinstance(input_, types.CallbackQuery):
            message_id = input_.message.message_id
            try:
                order_id = int(input_.data.split('@')[-1])
            except:
                pass
            optionals['data'] = input_.data
            optionals['text'] = input_.message.text
            self.log_info(chat_id, message_id, order_id, self, optionals['data'])
            await self.answer_callback_query(input_)
        elif isinstance(input_, types.Message):
            message_id = input_.message_id
            if 'text' in input_:
                optionals['text'] = input_.text
                self.log_message(chat_id, message_id, order_id, self, optionals['text'])
            elif 'location' in input_:
                optionals['location'] = f"{input_.location.latitude}|{input_.location.longitude}"
                self.log_message(chat_id, message_id, order_id, self, optionals['location'])
            elif 'contact' in input_:
                phone_number = input_.contact.phone_number
                if phone_number.startswith('+'):
                    optionals['phone_number'] = phone_number
                else:
                    optionals['phone_number'] = '+' + phone_number
                self.log_message(chat_id, message_id, order_id, self, optionals['phone_number'])
            else:
                self.log_message(chat_id, message_id, order_id, self, input_.content_type)
        else:
            self.log_message(chat_id, message_id, order_id, self, input_.content_type)
        return chat_id, message_id, order_id, optionals

    def log_info(self, chat_id, message_id, order_id, _self, message):
        self._db.log_message('INFO', chat_id, message_id, order_id, _self, message, 0)

    def log_message(self, chat_id, message_id, order_id, _self, message, shown=1):
        self._db.log_message('MESSAGE', chat_id, message_id, order_id, _self, message, shown)

    def log_error(self, chat_id, message_id, order_id, _self, message):
        self._db.log_message('ERROR', chat_id, message_id, order_id, _self, message, 0)

    async def answer_callback_query(self, callback_query):
        try:
            await self._bot.answer_callback_query(callback_query.id)
        except Exception as err:
            self.log_error(callback_query.from_user.id, None, None, self, err)

    def map_status(self, order_status):
        status_mapper = {
            70 : 'Ввод адреса отправления',
            80 : 'Ввод адреса назначения ',
            90 : 'Ввод цены              ',
            73 : 'Отмена пассажиром      ',
            83 : 'Отмена пассажиром      ',
            93 : 'Отмена пассажиром      ',
            100: 'Активный заказ         ',
            200: 'Взят в работу          ',
            250: 'Водитель ожидает       ',
            300: 'В пути                 ',
            400: 'Заказ завершен         ',
            103: 'Отмена пассажиром      ',
            203: 'Отмена пассажиром      ',
            253: 'Отмена пассажиром      ',
            303: 'Отмена пассажиром      ',
            107: 'Отмена водителем       ',
            207: 'Отмена водителем       ',
            257: 'Отмена водителем       ',
            307: 'Отмена водителем       ',
            50 : 'Отмена пассажиром      ',
            150: 'Отмена водителем       ',
        }
        if order_status in status_mapper.keys():
            return status_mapper[order_status]
        else:
            return order_status

    def tg_user_link(self, chat_id, first_name):
        return f'{first_name}'

    def get_referral_link(self, chat_id):
        return f"{self._config.BOT_LINK}?start={chat_id}"

    def get_url_link(self, text, url):
        from aiogram.utils.markdown import hlink
        return hlink(text, url)


COVER_META_TAG = '.meta'
COVER_WIDTH = 320
COVER_HEIGHT = 200

THUMB_VIDEO_POSITION = 0.25
FOLDER_COVER_FILE = '.cover.jpg'
MKV_COVER_FILE = 'cover.jpg'
EVENT_COOLDOWN_SECONDS = 1

import sys
import os
import io
import ast
import stat
import json
import time
import zipfile
import enzyme
import hashlib
import logging
import argparse
import subprocess
import collections
import PIL.Image
import PIL.ImageOps

import loghelper
import colorpicker
from watch import Watcher
from worker import Pool
import dbs

loghelper.set_up_logging(console_level=loghelper.VERBOSE, file_level=loghelper.NOTSET, filename='clerk.log')
log = loghelper.get_logger('Clerk', loghelper.Color.Red)
# Enzyme spams the logs with stuff we don't care about
logging.getLogger('enzyme').setLevel(logging.CRITICAL)
log.info('Starting Clerk.')

# Parse command line arguments
parser = argparse.ArgumentParser(description='Fabella Clerk. Watches video library for changes, updates indices and state.')
parser.add_argument('--once', '-o', action='store_true', help="Don't watch the library; just update everything and quit.")
parser.add_argument('path', type=str, help='Path to video library')
args = parser.parse_args()

try:
	import setproctitle
	setproctitle.setproctitle(' '.join(['clerk'] + sys.argv[1:]))
except ModuleNotFoundError:
	log.warning("Couldn't load setproctitle module; not changing process name")

def run_command(command):
	try:
		return subprocess.run(command, capture_output=True, check=True)
	except subprocess.CalledProcessError as e:
		log.error(f'Command returned {e.returncode}: {command}')
		for line in e.stderr.decode('utf-8').splitlines():
			log.error(line)
		raise

class TileError(Exception):
	pass

class BaseTile:
	def to_json(self):
		"""Return the attributes as json."""
		data = {
			'name': self.name,
			'isdir': self.isdir,
			'fingerprint': self.fingerprint,
			'tile_color': self.tile_color,
		}
		if not self.isdir:
			data['duration'] = self.duration

		return data

	def scale_encode(self, fd):
		"""Takes file-like object, reads image from it, scales, encodes to JPEG, returns bytes."""
		try:
			with PIL.Image.open(fd) as cover:
				cover = cover.convert('RGB')
				cover = PIL.ImageOps.fit(cover, (COVER_WIDTH, COVER_HEIGHT))
		except PIL.UnidentifiedImageError as e:
			raise TileError(f'Loading image for {self.path}: {str(e)}')

		# Choose a representative color from the cover image
		# Don't like setting this from here, but we need it later anyway.
		self.tile_color = '#' + ''.join(f'{c:02x}' for c in colorpicker.pick(cover))

		buffer = io.BytesIO()
		cover.save(buffer, format='JPEG', quality=90, subsampling=0, optimize=True)
		return buffer.getvalue()

	def get_folder_cover(self):
		"""Find cover image for folder, scale, return bytes."""
		cover_file = os.path.join(self.full_path, FOLDER_COVER_FILE)
		if not os.path.isfile(cover_file):
			raise TileError(f'Cover image {cover_file} not found')

		with open(cover_file, 'rb') as fd:
			log.info(f'Found cover {cover_file}')
			return self.scale_encode(fd)

	def get_file_cover(self):
		"""Find cover image for file, scale, return bytes."""
		# FIXME: Hmm. Not sure; image files are ignored earlier in the process anyway.
		if self.name.endswith(('.jpg', '.png')):
			log.info(f'Using image file as its own cover: {self.full_path}')
			with open(self.full_path, 'rb') as fd:
				return self.scale_encode(fd)

		if self.name.endswith('.mkv'):
			try:
				with open(self.full_path, 'rb') as fd:
					mkv = enzyme.MKV(fd)
					for a in mkv.attachments:
						if a.mimetype == 'image/jpeg' and a.filename == MKV_COVER_FILE:
							log.info(f'Found embedded cover in {self.full_path}')
							return self.scale_encode(a.data)
			except (OSError, enzyme.exceptions.Error) as e:
				raise TileError(f'Processing {self.full_path}: {e}')

		# If we got here, no embedded cover was found, generate thumbnail
		if self.name.endswith(dbs.VIDEO_EXTENSIONS):
			log.info(f'Generating thumbnail for {self.full_path}')
			try:
				duration = self.get_video_duration()
				# Bit dirty, but we need it later anyway.
				self.duration = round(duration)
				duration = str(duration * THUMB_VIDEO_POSITION)

				sp = run_command(['ffmpeg', '-ss', duration, '-threads', '1', '-i', self.full_path, '-vf', 'scale=1280:720,thumbnail', '-frames:v', '1', '-f', 'apng', '-'])
				return self.scale_encode(io.BytesIO(sp.stdout))
			except subprocess.CalledProcessError:
				raise TileError(f'Processing {self.full_path}: Command returned error')

		raise TileError(f'Processing {self.full_path}: unknown filetype to generate cover image from')

	def get_video_duration(self):
		if self.name.endswith('.mkv'):
			try:
				with open(self.full_path, 'rb') as fd:
					mkv = enzyme.MKV(fd)
					duration = mkv.info.duration
					return duration.seconds + duration.microseconds / 1000000
			except (OSError, enzyme.exceptions.Error) as e:
				log.error(f'Processing {self.full_path}: {e}')
				return None
		else:
			try:
				sp = run_command(['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=nokey=1:noprint_wrappers=1', self.full_path])
				return float(sp.stdout)
			except (subprocess.CalledProcessError, ValueError) as e:
				log.error(f'Getting video duration for {self.name}: {e}')
				return none

	def analyze(self):
		if self.cover_needs_update:
			try:
				if self.isdir:
					self.cover_image = self.get_folder_cover()
				else:
					self.cover_image = self.get_file_cover()
			except TileError as e:
				log.error(str(e))
				self.cover_image = None
			self.cover_needs_update = False

		# Maybe duration was set from getting the cover, maybe not.
		if self.duration is None and not self.isdir and self.name.endswith(dbs.VIDEO_EXTENSIONS):
			try:
				self.duration = self.get_video_duration()
				self.duration = round(self.duration) if self.duration is not None else None
			except subprocess.CalledProcessError as e:
				log.error(f'Couldn\'t determine video duration: {e}')

	def __eq__(self, other):
		if other is None:
			return False
		return (self.name, self.isdir, self.fingerprint) == \
			(other.name, other.isdir, other.fingerprint)

	def __lt__(self, other):
		return self.sortkey < other.sortkey

	@property
	def sortkey(self):
		simplename = os.path.splitext(self.name)[0].strip().casefold()
		for a in ['a ', 'an ', 'the ', 'de ', 'een ']:
			if simplename.startswith(a):
				simplename = simplename[len(a):].strip()
				break
		# Sort on case-insensitive simplified name, then raw name (for deterministic ordering).
		return (simplename, self.name)

	def __str__(self):
		return f'{self.__class__.__name__}({self.name}, isdir={self.isdir}, fingerprint={self.fingerprint})'

	def __repr__(self):
		return self.__str__()

class IndexedTile(BaseTile):
	def __init__(self, path, data):
		self.name = data['name']
		self.isdir = data['isdir']
		self.fingerprint = data['fingerprint']
		self.tile_color = data['tile_color']
		self.duration = None if self.isdir else data['duration']

		self.path = path
		self.full_path = os.path.join(path, self.name)
		self.cover_image = None
		self.cover_needs_update = True

class RealTile(BaseTile):
	def __init__(self, parent_path, name):
		self.name = name
		self.path = path
		self.full_path = os.path.join(path, name)

		# Not yet determined
		self.duration = None
		self.tile_color = None
		self.cover_image = None
		self.cover_needs_update = True

		# Get file attrs
		try:
			st = os.stat(self.full_path)
		except OSError as e:
			# Can't stat the file we were just created for? Fatal.
			raise ValueError(repr(e))

		if stat.S_ISDIR(st.st_mode):
			self.isdir = True
			# Folder, check cover image in it instead
			try:
				st = os.stat(os.path.join(self.full_path, FOLDER_COVER_FILE))
				self.fingerprint = f'inode={st.st_ino}:size={st.st_size}:mtime={st.st_mtime_ns}'
			except FileNotFoundError:
				self.fingerprint = None
		else:
			self.isdir = False
			self.fingerprint = f'inode={st.st_ino}:size={st.st_size}:mtime={st.st_mtime_ns}'

	def valid(self):
		"""True if the tile is "valid". Meaning the name/filetype/etc checks out."""
		if self.name.startswith('.'):
			return False

		if self.isdir:
			return True

		# File
		return self.name.endswith(dbs.VIDEO_EXTENSIONS)

class Meta:
	def __init__(self, *, version):
		self.version = version

	@classmethod
	def from_json(cls, data):
		return cls(version=data['meta']['version'])

	@classmethod
	def from_tiles(cls, tiles):
		return cls(version=dbs.INDEX_META_VERSION)

	def to_json(self):
		return {
			'version': self.version,
		}

	@classmethod
	def full_json(cls, tiles):
		return {
			'meta': Meta.from_tiles(tiles).to_json(),
			'files': [tile.to_json() for tile in sorted(tiles)],
		}

	@classmethod
	def fingerprint(cls, tiles):
		tiles = [[t.name, t.isdir, t.fingerprint] for t in tiles]
		return hashlib.sha256(json.dumps(tiles).encode('utf8')).hexdigest()

	def __eq__(self, other):
		if other is None:
			return False
		return self.version == other.version

	def __str__(self):
		return f'Meta(version={self.version})'

	def __repr__(self):
		return self.__str__()

def scan(path, pool):
	log.debug(f'Scanning {path}')
	if not os.path.isdir(path):
		log.info(f'{path} is gone, nothing to do')
		return

	index_db_name = os.path.join(path, dbs.INDEX_DB_NAME)
	orig_index = dbs.json_read(index_db_name, dbs.INDEX_DB_SCHEMA)

	#### Check meta version, extract file info index
	indexes = []
	indexed_meta = None
	if orig_index:
		try:
			indexed_meta = Meta.from_json(orig_index)
			if indexed_meta.version == dbs.INDEX_META_VERSION:
				indexes = list(orig_index['files'])
			else:
				log.warning(f'Skipping outdated index DB version: {index_db_name}')
		except (KeyError, TypeError) as e:
			log.error(f'Error parsing index DB version {index_db_name}: {str(e)}')

	#### Convert index to tiles
	indexed_tiles = []
	for data in indexes:
		try:
			tile = IndexedTile(path, data)
			indexed_tiles.append(tile)
		except ValueError as e:
			log.error(f'Error parsing json for tile in {index_db_name}: {e}')
	del indexes

	#### Covers DB
	cover_db_name = os.path.join(path, dbs.COVER_DB_NAME)
	cover_db_fingerprint = None
	try:
		with zipfile.ZipFile(cover_db_name, 'r') as fd:
			log.debug(f'Found existing covers DB {cover_db_name}')
			cover_meta = json.loads(fd.read(COVER_META_TAG))
			if cover_meta['version'] != dbs.INDEX_META_VERSION:
				log.info(f'Existing {cover_db_name} outdated version, discarding')
			elif cover_meta['dimensions'] != f'{COVER_WIDTH}x{COVER_HEIGHT}':
				log.info(f'Existing {cover_db_name} has wrong cover dimensions, discarding')
			elif cover_meta['fingerprint'] != Meta.fingerprint(indexed_tiles):
				log.warning(f'Existing {cover_db_name} fingerprint doesn\'t match {index_db_name}, discarding')
			else:
				for tile in indexed_tiles:
					tile.cover_image = fd.read(tile.name)
					if tile.cover_image == b'':
						tile.cover_image = None
					tile.cover_needs_update = False
				cover_db_fingerprint = cover_meta['fingerprint']
	except FileNotFoundError:
		log.info(f'Cover DB {cover_db_name} missing')
	except (OSError, zipfile.BadZipFile, json.JSONDecodeError, KeyError, TypeError) as e:
		log.error(f'Parsing {cover_db_name}: {e}')

	#### List actual files, convert into tiles
	real_tiles = []
	try:
		names = os.listdir(path)
	except FileNotFoundError:
		log.warning(f'Directory disappeared while we were working on it: {path}')
		return

	for name in names:
		try:
			tile = RealTile(path, name)
			real_tiles.append(tile)
		except ValueError as e:
			log.error(f'Error inspecting {path} {name}: {repr(e)}')

	# Filter and sort
	real_tiles = [tile for tile in real_tiles if tile.valid()]
	real_tiles = sorted(real_tiles)

	#### If the index matches reality, we're done.
	index_needs_update = True
	if indexed_tiles == real_tiles and indexed_meta == Meta.from_tiles(real_tiles):
		log.info(f'Existing index DB {index_db_name} is up to date, skipping')
		index_needs_update = False

	#### Determine what we can reuse
	# Abuses real_tiles as the end-result
	indexed_tiles = {t.name: t for t in indexed_tiles}
	for i in range(len(real_tiles)):
		name = real_tiles[i].name
		if real_tiles[i] == indexed_tiles.get(name):
			log.debug(f'Tile for {name} is up to date, reusing')
			real_tiles[i] = indexed_tiles[name]
		else:
			log.debug(f'Tile for {name} is stale, re-inspecting')

	#### Update covers/tile_color/duration etc; this is the expensive part
	for tile in real_tiles:
		if tile.cover_needs_update:
			pool.schedule(tile.analyze)
	pool.join()

	#### Write index
	if index_needs_update:
		dbs.json_write(index_db_name, Meta.full_json(real_tiles))

	#### Write covers
	# FIXME: error checking
	real_fingerprint = Meta.fingerprint(real_tiles)
	if cover_db_fingerprint == real_fingerprint:
		log.info(f'Existing cover DB {cover_db_name} is up to date, skipping')
	else:
		if real_tiles:
			log.info(f'Writing new cover DB {cover_db_name}')
			with zipfile.ZipFile(cover_db_name + dbs.NEW_SUFFIX, 'w') as fd:
				meta = {
					'version': dbs.INDEX_META_VERSION,
					'dimensions': f'{COVER_WIDTH}x{COVER_HEIGHT}',
					'fingerprint': real_fingerprint,
				}
				fd.writestr(COVER_META_TAG, json.dumps(meta, indent=4))

				# Write cover images
				for tile in real_tiles:
					fd.writestr(tile.name, tile.cover_image or b'')
			os.rename(cover_db_name + dbs.NEW_SUFFIX, cover_db_name)
		else:
			if os.path.isfile(cover_db_name):
				log.info(f'No files here, removing {cover_db_name}')
				os.remove(cover_db_name)
			else:
				log.debug(f'No files here, not writing {cover_db_name}')

def process_state_queue(path, roots):
	if not os.path.isdir(path):
		log.debug(f'{path} is gone, nothing to do')
		return

	log.info(f'Processing state events for {path}')

	queue_dir_name = os.path.join(path, dbs.QUEUE_DIR_NAME)
	state_db_name = os.path.join(path, dbs.STATE_DB_NAME)

	new = not os.path.isdir(queue_dir_name)

	# Ensure queue dir exists
	os.makedirs(queue_dir_name, exist_ok=True)
	os.chmod(queue_dir_name, 0o775)

	#### Load original state
	previous_state = dbs.json_read(state_db_name, dbs.STATE_DB_SCHEMA)
	# Deep copy for later
	orig_state = {n: dict(s) for n, s in previous_state.items()}

	if new:
		# This is a new directory. Setting orig_state = None will force a propagation
		# of state to the parent (notably, for position: 1).
		orig_state = None

	# Load filenames from index
	index = dbs.json_read([path, dbs.INDEX_DB_NAME], dbs.INDEX_DB_SCHEMA, default={'files': []})['files']
	new_state = {}

	# Match index to previous state on name AND fingerprint
	remaining = []
	for idx in index:
		name = idx['name']
		state = previous_state.get(name, {})
		fp = state.get('fingerprint', None)
		if fp is not None and fp == idx['fingerprint']:
			new_state[name] = state
			previous_state.pop(name)
			log.debug(f'Found name+fingerprint match in previous state for "{name}"')
		else:
			remaining.append(idx)

	# Match index to previous state on just fingerprint; these were probably renamed
	index = remaining
	remaining = []
	duplicate_fps = {None} \
		| {fp for fp, c in collections.Counter([i.get('fingerprint') for i in index]).items() if c > 1} \
		| {fp for fp, c in collections.Counter([s.get('fingerprint') for s in previous_state.values()]).items() if c > 1}
	if duplicate_fps != {None}:
		log.warning(f'Duplicate fingerprints will be ignored: {duplicate_fps}')
	previous_state_by_fp = {s['fingerprint']: (n, s) for n, s in previous_state.items() if s.get('fingerprint') not in duplicate_fps}
	for idx in index:
		name = idx['name']
		fp = idx.get('fingerprint')
		if fp in duplicate_fps:
			remaining.append(idx)
			continue
		try:
			oldname, state = previous_state_by_fp[fp]
		except KeyError:
			remaining.append(idx)
			continue
		else:
			new_state[name] = state
			previous_state.pop(oldname)
			log.info(f'Found fingerprint match in previous state for "{name}"; renamed from "{oldname}"')

	# Match index to previous state on just name; these were probably updated
	index = remaining
	remaining = []
	for idx in index:
		name = idx['name']
		fp = idx.get('fingerprint')
		try:
			state = previous_state[name]
		except KeyError:
			new_state[name] = {} if fp is None else {'fingerprint': fp}
			log.info(f'New file "{name}" has no match in previous state')
		else:
			log.info(f'Found name match in previous state for "{name}" fingerprint changed from {state.get("fingerprint")} to {fp}')
			if fp is None:
				state.pop('fingerprint', None)
			else:
				state['fingerprint'] = fp
			new_state[name] = state
			previous_state.pop(name)

	# Remaining entries from previous state are unmatched; they were probably removed
	for n, s in previous_state.items():
		log.info(f'"{n}" in previous state was not matched to any current file; ignored')

	#### Collect any state update files
	state_queue = {}
	try:
		for f in os.scandir(queue_dir_name):
			try:
				if not f.is_file() or f.path.endswith(dbs.NEW_SUFFIX):
					continue
				updates = dbs.json_read(f.path, dbs.STATE_UPDATE_SCHEMA, default={})
				state_queue[f.stat().st_mtime, f.path] = updates
			except OSError as e:
				log.error(f'Reading {f.path}: {str(e)}')
	except OSError as e:
		log.error(f'Reading {queue_dir_name}: {str(e)}')
		state_queue = {}

	#### Apply the requested state updates
	for meta, updates in sorted(state_queue.items()):
		for name, update in updates.items():
			log.debug(f'State update for {name}: {update}')

			if name not in new_state:
				new_state[name] = {}
			this_state = new_state[name]

			if 'position' in update:
				if update['position'] > 0:
					this_state['position'] = update['position']
				else:
					this_state.pop('position', None)

			if 'tagged' in update:
				if update['tagged']:
					this_state['tagged'] = True
				else:
					this_state.pop('tagged', None)

	#### Write new state
	if new_state == orig_state:
		log.debug('State unchanged, not updating.')
	else:
		if new_state:
			dbs.json_write(state_db_name, new_state)
		else:
			log.debug(f'Empty state; removing {state_db_name}')
			try:
				os.unlink(state_db_name)
			except FileNotFoundError:
				pass
			except OSError as e:
				log.error(f'Removing {state_db_name}: {str(e)}')

		# Propagate state upwards (but not outside root dir)
		if path not in roots:
			flat = {'tagged': any(s.get('tagged', False) for s in new_state.values())}

			if any(0 < s.get('position', 0) < 1 for s in new_state.values()):
				flat['position'] = 0.5
			elif any(s.get('position', 0) == 0 for s in new_state.values()):
				flat['position'] = 0
			else:
				flat['position'] = 1

			dbs.json_write([os.path.dirname(path), dbs.QUEUE_DIR_NAME, ...], {os.path.basename(path): flat})

	for update_mtime, update_name in state_queue.keys():
		try:
			log.debug(f'Removing {update_name}')
			os.unlink(update_name)
		except OSError as e:
			log.error(f'Removing {update_name}: {str(e)}')

#roots = [os.path.abspath(root) for root in sys.argv[1:]]
roots = [os.path.abspath(args.path)]
if not roots:
	print('Must specify at least one root')
	exit(1)
watcher = Watcher(roots)
for root in roots:
	watcher.push(root, recursive=True)

analyze_pool = Pool('analyze', threads=4)
scan_dirty = {}
state_dirty = {}
for event in watcher.events(timeout=EVENT_COOLDOWN_SECONDS):
	if event:
		log.debug(f'Got event: {event}')

	now = time.time()

	if event:
		if event.isdir and not event.hidden():
			# Case: path/ itself
			if event.evtype in {'modified', 'created'}:
				scan_dirty[event.path] = now

			# Case: path/foo/
			if event.evtype in {'created', 'deleted'}:
				watcher.push(os.path.dirname(event.path))

		if not event.isdir:
			# Case: path/.fabella/queue/foo
			if os.path.dirname(event.path).endswith('/' + dbs.QUEUE_DIR_NAME):
				if not event.path.endswith(dbs.NEW_SUFFIX):
					state_dirty[os.path.dirname(os.path.dirname(os.path.dirname(event.path)))] = now

			# Case: path/.fabella/state.json.gz
			elif event.path.endswith('/' + dbs.STATE_DB_NAME):
				state_dirty[os.path.dirname(os.path.dirname(event.path))] = now

			# Case: path/.fabella/index.json.gz
			elif event.path.endswith('/' + dbs.INDEX_DB_NAME):
				watcher.push(os.path.dirname(os.path.dirname(event.path)))

			# Case: path/.fabella/covers.zip
			elif event.path.endswith('/' + dbs.COVER_DB_NAME):
				watcher.push(os.path.dirname(os.path.dirname(event.path)))

			# Case: path/.cover.jpg
			elif event.path.endswith('/' + FOLDER_COVER_FILE):
				watcher.push(os.path.dirname(os.path.dirname(event.path)))

			# Case: path/foo.bar
			else:
				# Only do something for file extensions we care about
				if event.path.endswith(dbs.VIDEO_EXTENSIONS):
					watcher.push(os.path.dirname(event.path))

	# Items needing a full scan (index/cover update)
	for path, age in list(scan_dirty.items()):
		# Only process index events after a little while. In case a file is being
		# written, this postpones scanning until the file is completely done.
		if now - age > EVENT_COOLDOWN_SECONDS:
			del scan_dirty[path]
			scan(path, pool=analyze_pool)
			state_dirty[path] = now

	# Items needing a state update
	for path, age in list(state_dirty.items()):
		# Don't think we need a cooldown for state updates.
		#if now - age > EVENT_COOLDOWN_SECONDS:
		del state_dirty[path]
		process_state_queue(path, roots)

	if args.once and not scan_dirty and not state_dirty:
		break


from flask import render_template
from department_app.forms.department_form import DepartmentForm
from department_app.views.departments_views_service import *

@app.route('/', methods=['GET'])
def main_page():
    """
    Renders 'main_page.html' template.
    Depending on the choice of the user redirects to 'departments' page or 'employees' page.

    :return: rendered 'main_page.html' template
    """
    app.logger.debug('main_page.html was rendered')
    return render_template('main_page.html')

@app.route('/departments', methods=['GET', 'POST'])
def departments():
    """
    Renders 'departments.html' template.
    Calls 'departments_service' method with additional logic.
    Redirects to 'create' page in case of creating a department.
    Redirects to 'department_get' page in case of view of a particular department.

    :return: rendered 'departments.html' template
    """
    form = DepartmentForm()
    filtered_departments = departments_service(form)
    app.logger.debug('departments.html was rendered')
    return render_template('departments.html', departments=filtered_departments, form=form)

@app.route('/departments/<int:department_id>', methods=['GET', 'POST'])
def department_get(department_id):
    """
    Renders 'department.html' template.
    Calls 'department_get_service' method with additional logic.
    Redirects to 'add_employee' page in case of adding new employee to department.
    Redirects to 'edit_department' page in case of editing department.

    :param department_id: id of department
    :return: rendered 'department.html' template
    """
    department, employees = department_get_service(department_id)
    app.logger.debug('department.html was rendered')
    return render_template('department.html', department=department,
                           employees=employees)

@app.route('/departments/<int:department_id>/add', methods=['GET', 'POST'])
def add_employee(department_id):
    """
    Renders 'employee_add.html' template.
    Calls 'add_employee_service' method with additional logic.

    :param department_id: id of department
    :return: rendered 'employee_add.html' template
    """
    department_info = add_employee_service(department_id)
    app.logger.debug('employee_add.html was rendered')
    return render_template('employee_add.html', department=department_info)

@app.route('/departments/<int:department_id>/edit', methods=['GET', 'POST'])
def edit_department(department_id):
    """
    Renders 'department_edit.html' template
    Calls 'edit_department_service' method with additional logic.
    Redirects to 'department_remove' when trying to remove department.

    :param department_id: id of department
    :return: rendered 'department_edit.html' template
    """
    department_info = edit_department_service(department_id)
    app.logger.debug('department_edit.html was rendered')
    return render_template('department_edit.html', department=department_info)

@app.route('/departments/create_department', methods=['GET', 'POST'])
def create():
    """
    Renders 'department_create.html' template
    Calls 'create_department_service' method with additional logic.

    :return: rendered 'department_create.html' template
    """
    create_department_service()
    app.logger.debug('department_create.html was rendered')
    return render_template('department_create.html')

@app.route('/departments/<int:department_id>/remove', methods=['GET', 'POST'])
def department_remove(department_id):
    """
    Renders 'department_remove.html' template.
    Calls 'department_name_id' method to get id and name of department.
    Calls 'department_remove_service' method with additional logic.
    Upon confirmation of the removal of the department, it is deleted from the database,
    redirects to 'removed_department' page.

    :param department_id: id of department
    :return: rendered 'department_remove.html' template
    """
    department_info = department_name_id(department_id)
    if request.method == 'POST':
        return department_remove_service(department_id)
    app.logger.debug(f'department_remove.html was rendered')
    return render_template('department_remove.html', department=department_info)

@app.route('/departments/removed_department', methods=['GET'])
def removed_department():
    """
    Renders 'removed_department.html' template.
    A flash notification about the success of the operation pops up.
    Arrow back redirects on 'departments' page.

    :return: rendered 'removed_department.html' template
    """
    app.logger.debug('Department was removed')
    flash('The department was successfully removed.', category='success')
    app.logger.debug('removed_department.html was rendered')
    return render_template('removed_department.html')


import sys, os, traceback
import ast
from statistics import mean

import torch
from sklearn.metrics import mean_absolute_error,mean_squared_error
import numpy as np
from sklearn.decomposition import PCA
import json
# from sklearn.manifold import TSNE
from torchmetrics import MeanAbsolutePercentageError
def csm(A,B):
    num=np.dot(A,B.T)/(np.sqrt(np.sum(A**2,axis=1)[:,np.newaxis])*np.sqrt(np.sum(B**2,axis=1))[np.newaxis,:])
    return num
def adjustScoreForMatrix(scores):
    try:
        minValue=np.min(scores)
        maxValue=np.max(scores)
        distance=maxValue-minValue
        scores=(scores-minValue)/distance
    except Exception as e:
        traceback.print_exc()
    return scores

def getReductionEmb(nl_vecs,code_vecs,reductType,reductSize):
    lenNLVecs=len(nl_vecs)
    lenCodeVecs=len(code_vecs)
    nl_vecs_transform=[]
    code_vecs_transform=[]
    all_vecs=nl_vecs+code_vecs
    # print('len all {}'.format(len(all_vecs)))
    if reductType=='adhoc':
        all_vecs_transform=[element[:reductSize] for element in all_vecs]
        nl_vecs_transform = all_vecs_transform[:lenNLVecs]
        code_vecs_transform = all_vecs_transform[lenNLVecs:]
    elif reductType=='pca':
        pca = PCA(n_components=reductSize)
        all_vecs_transform =pca.fit_transform(all_vecs)
        nl_vecs_transform = all_vecs_transform[:lenNLVecs]
        code_vecs_transform = all_vecs_transform[lenNLVecs:]
    # else:
    #     tsne = TSNE(n_components=reductSize,
    #                 perplexity=40,
    #                 random_state=42,
    #                 n_iter=5000,
    #                 n_jobs=-1)
    #     all_vecs_transform = tsne.fit_transform(all_vecs)
    #     nl_vecs_transform = all_vecs_transform[:lenNLVecs]
    #     code_vecs_transform = all_vecs_transform[lenNLVecs:]

    return nl_vecs_transform.tolist(),code_vecs_transform.tolist()

def createDirIfNotExist(fopOutput):
    try:
        # Create target Directory
        os.makedirs(fopOutput, exist_ok=True)
        #print("Directory ", fopOutput, " Created ")
    except FileExistsError:
        print("Directory ", fopOutput, " already exists")


import pytest

from machaon.app import AppRoot
from machaon.process import ProcessMessage, Spirit, TempSpirit, Process
from machaon.ui.tk import tkLauncher

def gettext(log):
    return log.get(1.0, "end")

lastlineindex = ("end linestart -2 lines", "end linestart -2 lines lineend")
def getlastline(log):
    return log.get(*lastlineindex)
def getlasttag(log):
    return log.tag_names(lastlineindex[0])

@pytest.fixture
def approot():
    app = AppRoot()
    app.initialize(ui="tk", title="test", ignore_args=True)
    app.boot_ui()
    return app

def test_message_window(approot):
    spi = Spirit(approot)
    wnd = approot.get_ui()

    assert "\n" == gettext(wnd.log)
    wnd.insert_screen_text("message", "test-message")
    assert "test-message\n\n" == gettext(wnd.log)
    assert "test-message" == getlastline(wnd.log)

    wnd.insert_screen_text("error", "test-error-message")
    assert "test-error-message" == getlastline(wnd.log)
    assert ("error",) == getlasttag(wnd.log)

    wnd.insert_screen_text("hyperlink", "test-hyperlink", link="www.hellowork.go.jp", linktag="message-em")
    assert "test-hyperlink" == getlastline(wnd.log)
    assert set(("message-em","clickable","hlink-1")) == set(getlasttag(wnd.log))

    #wnd.log.mark_set("CURRENT", "end")
    l1 = wnd.log.get("end linestart -2 lines", "end linestart -1 lines")
    l2 = wnd.log.get("end linestart -3 lines", "end linestart -2 lines")

    wnd.message_handler(ProcessMessage(tag="delete-message", line=-1, count=1))
    assert "test-error-message" == getlastline(wnd.log)

    #wnd.run_mainloop()

def test_textindex(approot):
    from machaon.ui.tk import TextIndex as textindex

    assert textindex("1.2").line == 1 and textindex("1.2").char == 2
    assert str(textindex("2.3")) == "2.3"
    assert str(textindex("10.2").move(char=5)) == "10.5"
    assert str(textindex("10.2").move(line=0, char=0)) == "0.0"
    assert str(textindex(line=3, char="end")) == "3.end"
    i = textindex("9.7")
    assert i.shifted(line=-1, char=1).string() == "8.8"
    assert i.string() == "9.7"
    assert textindex("1.0").compare(textindex("1.3")) == 1
    assert textindex("1.2").compare(textindex("2.3")) == 1
    assert textindex("2.3").compare(textindex("2.3")) == 0
    assert textindex("2.3").compare(textindex("1.2")) == -1


#(c) Mircea Andrecut,  mircea.andrecut@gmail.com (2022)
import math
import bz2
import lzma
import numpy as np
from collections import Counter
import matplotlib.pyplot as plt

def runCA(C,L):
    C,M,Q = bytearray(C.encode()),len(C), np.zeros((L,L))
    for J in range(M,L):
        C = [(C[n-2] + C[n-1] + C[n]) % 256 for n in range(J)]
        C.append(sum(C) % 256)
        Q[J,0:J+1] = C
    return 255-Q

def entropy(C):
    p, lns = Counter(C), float(len(C))
    return -sum(count/lns * math.log(count/lns, 256) for count in p.values())

if __name__== "__main__":
    C = "abcdefghijklmnop"
    C = runCA(C,32768)
    F = np.fft.fft(C[-1]-np.mean(C[-1]))[:len(C)//2]

    fig = plt.figure(figsize=(10,10))
    plt.imshow(C[0:1024,0:1024].T, cmap='gray',interpolation=None)
    plt.xlabel('t'); plt.ylabel('C(t)')
    plt.xlim([0, 1024]); plt.ylim([0, 1024])
    fig.savefig("fig1.pdf",bbox_inches="tight")

    fig = plt.figure(figsize=(10,6))
    plt.hist(C[-1],256, facecolor='gray', density=True)
    plt.xlim([0, 256]);plt.ylim([0, 0.02])
    plt.xlabel('s'); plt.ylabel('p(s)')
    fig.savefig("fig2.pdf",bbox_inches="tight")

    fig = plt.figure(figsize=(8,4))
    plt.plot(np.arange(len(F)),np.abs(F),linewidth=0.1,color="gray")
    plt.xlim([0, len(F)]); plt.ylim([0, 1.1*np.max(np.abs(F))])
    plt.xlabel('$f$'); plt.ylabel('$|F(f)|$')
    fig.savefig("fig3a.pdf",bbox_inches="tight")

    fig = plt.figure(figsize=(8,4))
    plt.hist(np.abs(F),256, facecolor="gray", density=True)
    plt.xlim([0, np.max(np.abs(F))])
    plt.xlabel('$|F(f)|$'); plt.ylabel('$p(|F(f)|)$')
    fig.savefig("fig3b.pdf",bbox_inches="tight")

    fig = plt.figure(figsize=(8,4))
    plt.hist(np.angle(F),256, facecolor="gray", density=True)
    plt.xlim([np.min(np.angle(F)),np.max(np.angle(F))])
    plt.xlabel(r'$\varphi$'); plt.ylabel(r'$p(\varphi)$')
    fig.savefig("fig3c.pdf",bbox_inches="tight")

    H = entropy(bytes(C[-1]))
    Z = lzma.compress(bytes(C[-1]))
    print("Entropy = ", H)
    print("LZMA compression ratio = ", len(Z)/len(C[-1]))
    Z = bz2.compress(bytes(C[-1]))
    print("BZIP2 compression ratio = ", len(Z)/len(C[-1]))


"""
Metrics for computing evalutation results
"""

import numpy as np
import skimage.measure as measure
import skimage.morphology as morphology

class Metric(object):
    """
    Compute evaluation result

    Args:
        max_label:
            max label index in the data (0 denoting background)
        n_runs:
            number of test runs
    """
    def __init__(self, max_label=20, n_runs=None):
        self.labels = list(range(max_label + 1))  # all class labels
        self.n_runs = 1 if n_runs is None else n_runs

        # list of list of array, each array save the TP/FP/FN statistic of a testing sample
        self.tp_lst = [[] for _ in range(self.n_runs)]
        self.fp_lst = [[] for _ in range(self.n_runs)]
        self.fn_lst = [[] for _ in range(self.n_runs)]

    def record(self, pred, target, labels=None, n_run=None):
        """
        Record the evaluation result for each sample and each class label, including:
            True Positive, False Positive, False Negative

        Args:
            pred:
                predicted mask array, expected shape is H x W
            target:
                target mask array, expected shape is H x W
            labels:
                only count specific label, used when knowing all possible labels in advance
        """
        assert pred.shape == target.shape

        if self.n_runs == 1:
            n_run = 0

        # array to save the TP/FP/FN statistic for each class (plus BG)
        tp_arr = np.full(len(self.labels), np.nan)
        fp_arr = np.full(len(self.labels), np.nan)
        fn_arr = np.full(len(self.labels), np.nan)

        if labels is None:
            labels = self.labels
        else:
            labels = [0,] + labels

        for j, label in enumerate(labels):
            # Get the location of the pixels that are predicted as class j
            idx = np.where(np.logical_and(pred == j, target != 255))
            pred_idx_j = set(zip(idx[0].tolist(), idx[1].tolist()))
            # Get the location of the pixels that are class j in ground truth
            idx = np.where(target == j)
            target_idx_j = set(zip(idx[0].tolist(), idx[1].tolist()))

            if target_idx_j:  # if ground-truth contains this class
                tp_arr[label] = len(set.intersection(pred_idx_j, target_idx_j))
                fp_arr[label] = len(pred_idx_j - target_idx_j)
                fn_arr[label] = len(target_idx_j - pred_idx_j)

        self.tp_lst[n_run].append(tp_arr)
        self.fp_lst[n_run].append(fp_arr)
        self.fn_lst[n_run].append(fn_arr)

    def get_dice(self, labels=None, n_run=None):
        """
        Compute mean IoU

        Args:
            labels:
                specify a subset of labels to compute mean IoU, default is using all classes
        """
        if labels is None:
            labels = self.labels
        # Sum TP, FP, FN statistic of all samples
        if n_run is None:
            tp_sum = [np.vstack(self.tp_lst[run])
                      for run in range(self.n_runs)]
            fp_sum = [np.vstack(self.fp_lst[run])
                      for run in range(self.n_runs)]
            fn_sum = [np.vstack(self.fn_lst[run])
                      for run in range(self.n_runs)]

            # Compute mean IoU classwisely
            # Average across n_runs, then average over classes
            mIoU_class = np.vstack([np.mean(2*tp_sum[run] / (2*tp_sum[run] + fp_sum[run] + fn_sum[run]),axis=0).take(labels)
                                    for run in range(self.n_runs)])
            mIoU = np.mean(mIoU_class, axis=1)

            return (mIoU_class.mean(axis=0), mIoU_class.std(axis=0),
                    mIoU.mean(axis=0), mIoU.std(axis=0))
        else:
            tp_sum = np.vstack(self.tp_lst[n_run])
            fp_sum = np.vstack(self.fp_lst[n_run])
            fn_sum = np.vstack(self.fn_lst[n_run])

            # Compute mean IoU classwisely and average over classes
            mIoU_class = np.nanmean((2*tp_sum / (2*tp_sum + fp_sum + fn_sum)), axis=0).take(labels)
            #mIoU = mIoU_class.mean()
            mIoU_std = np.nanstd((2*tp_sum / (2*tp_sum + fp_sum + fn_sum)), axis=0).take(labels)
            return mIoU_class, mIoU_std

    def get_dice_batch(self, labels=None, n_run=None):
        """
        Compute mean IoU

        Args:
            labels:
                specify a subset of labels to compute mean IoU, default is using all classes
        """
        if labels is None:
            labels = self.labels
        # Sum TP, FP, FN statistic of all samples
        if n_run is None:
            tp_sum = [np.nansum(np.vstack(self.tp_lst[run]), axis=0).take(labels)
                      for run in range(self.n_runs)]
            fp_sum = [np.nansum(np.vstack(self.fp_lst[run]), axis=0).take(labels)
                      for run in range(self.n_runs)]
            fn_sum = [np.nansum(np.vstack(self.fn_lst[run]), axis=0).take(labels)
                      for run in range(self.n_runs)]

            # Compute mean IoU classwisely
            # Average across n_runs, then average over classes
            mIoU_class = np.vstack([2*tp_sum[run] / (2*tp_sum[run] + fp_sum[run] + fn_sum[run])
                                    for run in range(self.n_runs)])
            mIoU = mIoU_class.mean(axis=1)

            return (np.nanmean(mIoU_class, axis=0), np.nanstd(mIoU_class, axis=0),
                    mIoU.mean(axis=0), mIoU.std(axis=0))
        else:
            tp_sum = np.sum(np.vstack(self.tp_lst[n_run]), axis=0).take(labels)
            fp_sum = np.sum(np.vstack(self.fp_lst[n_run]), axis=0).take(labels)
            fn_sum = np.sum(np.vstack(self.fn_lst[n_run]), axis=0).take(labels)

            # Compute mean IoU classwisely and average over classes
            mIoU_class = 2*tp_sum / (2*tp_sum + fp_sum + fn_sum)
            mIoU = mIoU_class.mean()

            return mIoU_class, mIoU

    def get_mIoU(self, labels=None, n_run=None):
        """
        Compute mean IoU

        Args:
            labels:
                specify a subset of labels to compute mean IoU, default is using all classes
        """
        if labels is None:
            labels = self.labels
        # Sum TP, FP, FN statistic of all samples
        if n_run is None:
            tp_sum = [np.nansum(np.vstack(self.tp_lst[run]), axis=0).take(labels)
                      for run in range(self.n_runs)]
            fp_sum = [np.nansum(np.vstack(self.fp_lst[run]), axis=0).take(labels)
                      for run in range(self.n_runs)]
            fn_sum = [np.nansum(np.vstack(self.fn_lst[run]), axis=0).take(labels)
                      for run in range(self.n_runs)]

            # Compute mean IoU classwisely
            # Average across n_runs, then average over classes
            mIoU_class = np.vstack([tp_sum[run] / (tp_sum[run] + fp_sum[run] + fn_sum[run])
                                    for run in range(self.n_runs)])
            mIoU = mIoU_class.mean(axis=1)

            return (mIoU_class.mean(axis=0), mIoU_class.std(axis=0),
                    mIoU.mean(axis=0), mIoU.std(axis=0))
        else:
            tp_sum = np.nansum(np.vstack(self.tp_lst[n_run]), axis=0).take(labels)
            fp_sum = np.nansum(np.vstack(self.fp_lst[n_run]), axis=0).take(labels)
            fn_sum = np.nansum(np.vstack(self.fn_lst[n_run]), axis=0).take(labels)

            # Compute mean IoU classwisely and average over classes
            mIoU_class = tp_sum / (tp_sum + fp_sum + fn_sum)
            mIoU = mIoU_class.mean()

            return mIoU_class, mIoU

    def get_mIoU_binary(self, n_run=None):
        """
        Compute mean IoU for binary scenario
        (sum all foreground classes as one class)
        """
        # Sum TP, FP, FN statistic of all samples
        if n_run is None:
            tp_sum = [np.nansum(np.vstack(self.tp_lst[run]), axis=0)
                      for run in range(self.n_runs)]
            fp_sum = [np.nansum(np.vstack(self.fp_lst[run]), axis=0)
                      for run in range(self.n_runs)]
            fn_sum = [np.nansum(np.vstack(self.fn_lst[run]), axis=0)
                      for run in range(self.n_runs)]

            # Sum over all foreground classes
            tp_sum = [np.c_[tp_sum[run][0], np.nansum(tp_sum[run][1:])]
                      for run in range(self.n_runs)]
            fp_sum = [np.c_[fp_sum[run][0], np.nansum(fp_sum[run][1:])]
                      for run in range(self.n_runs)]
            fn_sum = [np.c_[fn_sum[run][0], np.nansum(fn_sum[run][1:])]
                      for run in range(self.n_runs)]

            # Compute mean IoU classwisely and average across classes
            mIoU_class = np.vstack([tp_sum[run] / (tp_sum[run] + fp_sum[run] + fn_sum[run])
                                    for run in range(self.n_runs)])
            mIoU = mIoU_class.mean(axis=1)

            return (mIoU_class.mean(axis=0), mIoU_class.std(axis=0),
                    mIoU.mean(axis=0), mIoU.std(axis=0))
        else:
            tp_sum = np.nansum(np.vstack(self.tp_lst[n_run]), axis=0)
            fp_sum = np.nansum(np.vstack(self.fp_lst[n_run]), axis=0)
            fn_sum = np.nansum(np.vstack(self.fn_lst[n_run]), axis=0)

            # Sum over all foreground classes
            tp_sum = np.c_[tp_sum[0], np.nansum(tp_sum[1:])]
            fp_sum = np.c_[fp_sum[0], np.nansum(fp_sum[1:])]
            fn_sum = np.c_[fn_sum[0], np.nansum(fn_sum[1:])]

            mIoU_class = tp_sum / (tp_sum + fp_sum + fn_sum)
            mIoU = mIoU_class.mean()

            return mIoU_class, mIoU

def positive_lesion_rate(tgt, y_last, thre, direct='gt'):
    #tgt is the perspective direction, it is always first: 3d np array
    tgt=tgt.astype(np.bool)
    y_last=y_last.astype(np.bool)
    tgt=morphology.remove_small_holes(morphology.remove_small_objects(tgt,4, connectivity=2),4,connectivity=2)
    y_last=morphology.remove_small_holes(morphology.remove_small_objects(y_last,4, connectivity=2),4,connectivity=2)

    label_image, number_of_labels=measure.label(tgt, background = 0, return_num = True, connectivity=1)
    measures=np.empty(number_of_labels)
    i=0
    th_count=[0]*(len(thre))
    number_of_lesions_=number_of_labels+0
    for j,region in zip(range(number_of_labels),measure.regionprops(label_image, intensity_image=tgt)):
        lesion=np.zeros(tgt.shape)
        if region.area<4:
            measures=measures[:-1]
            number_of_lesions_=number_of_lesions_-1
        else:
            #lesion[minx:maxx, miny:maxy, minz:maxz]=label_image[minx:maxx, miny:maxy, minz:maxz]
            lesion[label_image==j+1]=1
            #print(label_image.min(), label_image.max())
            measure1=calculate_lesion(lesion, y_last)
            #measure1,measure2, measure3=calculate_pn(img[minx:maxx, miny:maxy, minz:maxz], tgt[minx:maxx, miny:maxy, minz:maxz])
            measures[i]=measure1
            i+=1
            for k, th in enumerate(thre):
                if measure1>th:
                    th_count[k]+=1

    return measures, th_count, number_of_lesions_

def calculate_lesion(y_first, y_last):
    # first is always the denominator
    # fist  is considered as lesion pred, then for that lesion, fp+tp=lesion
    y_first=y_first.reshape(1,-1)
    y_last = y_last.reshape(1, -1)

    tp = np.sum(y_last * y_first, axis=-1)
    #lesion_sum=np.sum(y_pred, axis=-1)
    #tn = np.sum((1 - y_last) * (1 - y_first), axis=-1)
    fp = np.sum((1 - y_last) * y_first, axis=-1)
    #fn = np.sum(y_true * (1 - y_pred), axis=-1)
    return tp/(tp+fp)