# -*- coding: utf-8 -*-
"""
Created on Thu Jul  9 16:18:13 2020

@author: Dropex
"""
import sys
import requests
import pandas as pd
import numpy as np

class TradeAsset:
    """
    symbol: BTCUSDT, XAUUSD, AMZN, etc.
    interval: 1h, 4h, 1d, etc.
    exchange: Binance, BitMex, ByBit, etc.
    """
    def __init__(self, symbol, interval, exchange):
        self.symbol = symbol
        self.interval = interval
        self.exchange = exchange

    #Use exchange API to get market information
    def getklines(self):
        if self.exchange == 'Binance':
            url_base='https://api.binance.com/api/v1/klines'
            PARAMS = {'symbol':self.symbol, 'interval':self.interval}
            myheaders = {}
            try:
                myrequest = requests.get(url = url_base, headers = myheaders, params = PARAMS)
                self.klines = myrequest.json()
            except:
                self.klines = [sys.exc_info()[0]]
        else:
            self.klines = ['Exchange not defined']

    #Generate pandas dataframe for further analysis
    def dataframe(self):
        if len(self.klines) > 2:
            df = pd.DataFrame(self.klines)
        else:
            df = pd.DataFrame()
        dfcolumns=len(df.columns) >= 12
        if dfcolumns and self.exchange == 'Binance':
            df.columns=['OpenTime','Open','High','Low','Close','Volume','CloseTime','QuoteAssetVol','NoOfTrades','BuyVolume','TakerbuyquoteassetVol','Nothing']
            df=df[['OpenTime','Open','High','Low','Close','Volume','CloseTime','BuyVolume']]
            df[['Open','High','Low','Close','Volume','BuyVolume']] = df[['Open','High','Low','Close','Volume','BuyVolume']].astype(float)
            df['SellVolume']=df['Volume']-df['BuyVolume']
            #Average from Open and Close values, used for Volume Profile
            df['PriceAverage']=(df['Open']+df['Close'])/2
            self.df = df
        else:
            self.df = pd.DataFrame()

    #EMA of last n periods for Close column
    # df dataframe with column Close
    def ema(self, df, n, sma):
        alpha=2/(n+1)
        ema=np.nan
        lastema=sma
        if 500 >= len(df) >= 2*n+1 > 0 and 'Close' in df.columns:
            for i in range(n):
                ema=alpha*df.loc[len(df['Close'])-n+i,'Close']+(1-alpha)*lastema
                lastema=ema
        return ema

    #Add EMA of n periods column
    # df dataframe with column Close
    def addema(self,df,n):
        ema_list=[]
        for i in df.index.values:
            df_slice=df[0:i+1]
            sma=self.sma(df_slice[0:len(df_slice)-n],n)
            ema_list.append(self.ema(df_slice,n,sma))
        self.df['ema'+str(n)]=ema_list

    #Simple Media Average of last n periods for Close column
    # df dataframe with column Close
    def sma(self, df, n):
        if 500 >= len(df) >= n > 0 and 'Close' in df.columns:
            return df['Close'][len(df['Close'])-n:len(df['Close'])].mean()
        else:
            return np.nan

    #Add SMA of n periods column
    # df dataframe with column Close
    def addsma(self, df, n):
        sma_list=[]
        for i in df.index.values:
            df_slice=df[0:i+1]
            sma_list.append(self.sma(df_slice,n))
        self.df['sma'+str(n)]=sma_list

    #RSI of n periods
    # df dataframe with column Close
    def rsi(self, df, n):
        if 500 >= len(df) >= 2*n+1 > 0 and 'Close' in df.columns:
            #RMA for avgain and avloss
            avgain=0
            avloss=0
            alpha=(1.0/n)
            gainlist=[]
            losslist=[]
            for i in range(n,0,-1):
                closedelta=df.loc[len(df)-n-i,'Close']-df.loc[len(df)-n-i-1,'Close']
                if closedelta > 0:
                    gainlist.append(closedelta)
                else:
                    losslist.append(abs(closedelta))
            avgain=np.array(gainlist).sum()/n
            avloss=np.array(losslist).sum()/n
            for i in range(n,0,-1):
                closedelta=df.loc[len(df)-i,'Close']-df.loc[len(df)-i-1,'Close']
                if closedelta > 0:
                    avgain=closedelta*alpha+(1-alpha)*avgain
                    avloss=(1-alpha)*avloss
                else:
                    avloss=abs(closedelta)*alpha+(1-alpha)*avloss
                    avgain=(1-alpha)*avgain
            rs=avgain/avloss
            rsi=100-(100/(1+rs))
            return rsi
        else:
            return np.nan

    #Add RSI of 14 periods column
    # df dataframe with column Close
    def addrsi(self, df):
        rsi_list=[]
        n=14
        for i in df.index.values:
            df_slice=df[0:i+1]
            rsi_list.append(self.rsi(df_slice,n))
        self.df['rsi'+str(n)]=rsi_list

    #MACD
    # df dataframe with column Close
    def macd(self, df):
        if 500 >= len(df) >= 2*26+17+1 > 0 and 'Close' in df.columns:
            macd_list=[]
            for i in range(0,18):
                firstsma12=self.sma(df[0:len(df)-17+i-12],12)
                firstsma26=self.sma(df[0:len(df)-17+i-26],26)
                macd_list.append(self.ema(df[0:len(df)-17+i],12,firstsma12)-self.ema(df[0:len(df)-17+i],26,firstsma26))
            signal=self.emaoflist(macd_list[9:18],self.smaoflist(macd_list[0:9]))
            hist=macd_list[-1]-signal
            return macd_list[-1],signal,hist
        else:
            return np.nan,np.nan,np.nan

    #Add macd,signal and histogram columns
    # df dataframe with column Close
    def addmacd(self, df):
        macd_list=[]
        signal_list=[]
        hist_list=[]
        for i in df.index.values:
            df_slice=df[0:i+1]
            macd,signal,hist=self.macd(df_slice)
            macd_list.append(macd)
            signal_list.append(signal)
            hist_list.append(hist)
        self.df['macd']=macd_list
        self.df['signal']=signal_list
        self.df['histogram']=hist_list

    #Calculate simple media average of a list
    def smaoflist(self,list):
        n=len(list)
        sma=0
        for i in list:
            sma=sma+i
        sma=sma/n
        return sma

    #Calculate exponential media average of a list
    # sma is simple media average of last n values before the initial value in list
    def emaoflist(self,list, sma):
        n=len(list)
        lastema=sma
        ema=0
        alpha = 2 / (n + 1)
        for i in list:
            ema=(1-alpha)*lastema+alpha*i
            lastema=ema
        return ema

    def di(self,df):
        if 500 >= len(df) >= 2*14+1 > 0 and 'Close' in df.columns and 'High' in df.columns and 'Low' in df.columns:
            upDMlist=[]
            downDMlist=[]
            trlist=[]
            upDIlist=[]
            downDIlist=[]
            for i in range(0,28):
                upDM=df.loc[len(df)-28+i,'High']-df.loc[len(df)-28+i-1,'High']
                downDM=df.loc[len(df)-28+i-1,'Low']-df.loc[len(df)-28+i,'Low']
                if not (upDM > 0 and upDM > downDM):
                    upDM=0
                if not (downDM > 0 and downDM > upDM):
                    downDM=0
                upDMlist.append(upDM)
                downDMlist.append(downDM)
                tr=max(df.loc[len(df)-28+i,'High']-df.loc[len(df)-28+i,'Low'], abs(df.loc[len(df)-28+i,'High']-df.loc[len(df)-28+i-1,'Close']), abs(df.loc[len(df)-28+i,'Low']-df.loc[len(df)-28+i-1,'Close']))
                trlist.append(tr)
            atrsma=self.smaoflist(trlist[0:14])
            atr=self.emaoflist(trlist[14:28],atrsma)
            sma=self.smaoflist(upDMlist[0:14])
            ema=self.emaoflist(upDMlist[14:28],sma)
            upDI=100*ema/atr
            sma=self.smaoflist(downDMlist[0:14])
            ema=self.emaoflist(downDMlist[14:28],sma)
            downDI=100*ema/atr
            return upDI,downDI
        else:
            return 0,0

    def adx(self,df):
        if 500 >= len(df) >= 2*14+1 > 0 and 'Close' in df.columns and 'High' in df.columns and 'Low' in df.columns:
            adxlist=[]
            for i in range(0,28):
                upDI,downDI=self.di(df[0:len(df)-28+i+1])
                if upDI+downDI == 0:
                    upDI=downDI=0.5
                adxlist.append(abs((upDI-downDI)/(upDI+downDI)))
            sma=self.smaoflist(adxlist[0:14])
            ema=self.emaoflist(adxlist[14:28],sma)
            adx=100*ema
            return upDI,downDI,adx
        else:
            return np.nan,np.nan,np.nan

    def addadx(self,df):
        upDI_list=[]
        downDI_list=[]
        adx_list=[]
        for i in df.index.values:
            df_slice=df[0:i+1]
            upDI,downDI,adx=self.adx(df_slice)
            upDI_list.append(upDI)
            downDI_list.append(downDI)
            adx_list.append(adx)
        self.df['upDI']=upDI_list
        self.df['downDI']=downDI_list
        self.df['adx']=adx_list


from unittest import mock

import pytest

from jrnl.os_compat import on_posix
from jrnl.os_compat import on_windows
from jrnl.os_compat import split_args

@pytest.mark.parametrize(
    "systems",
    [
        ["linux", False],
        ["win32", True],
        ["cygwin", False],
        ["msys", False],
        ["darwin", False],
        ["os2", False],
        ["os2emx", False],
        ["riscos", False],
        ["atheos", False],
        ["freebsd7", False],
        ["freebsd8", False],
        ["freebsdN", False],
        ["openbsd6", False],
    ],
)
def test_on_windows(systems):
    osname, expected_on_windows = systems[0], systems[1]
    with mock.patch("jrnl.os_compat.platform", osname):
        assert on_windows() == expected_on_windows

@pytest.mark.parametrize(
    "systems",
    [
        ["linux", True],
        ["win32", False],
        ["cygwin", True],
        ["msys", True],
        ["darwin", True],
        ["os2", True],
        ["os2emx", True],
        ["riscos", True],
        ["atheos", True],
        ["freebsd7", True],
        ["freebsd8", True],
        ["freebsdN", True],
        ["openbsd6", True],
    ],
)
def test_on_posix(systems):
    osname, expected_on_posix = systems[0], systems[1]
    with mock.patch("jrnl.os_compat.platform", osname):
        assert on_posix() == expected_on_posix

@pytest.mark.parametrize(
    "args",
    [
        ["notepad", ["notepad"]],
        ["subl -w", ["subl", "-w"]],
        [
            '"C:\\Program Files\\Sublime Text 3\\subl.exe" -w',
            ['"C:\\Program Files\\Sublime Text 3\\subl.exe"', "-w"],
        ],
    ],
)
def test_split_args_on_windows(args):
    input_arguments, expected_split_args = args[0], args[1]
    with mock.patch("jrnl.os_compat.on_windows", lambda: True):
        assert split_args(input_arguments) == expected_split_args

@pytest.mark.parametrize(
    "args",
    [
        ["vim", ["vim"]],
        [
            'vim -f +Goyo +Limelight "+set spell linebreak"',
            ["vim", "-f", "+Goyo", "+Limelight", '"+set spell linebreak"'],
        ],
    ],
)
def test_split_args_on_not_windows(args):
    input_arguments, expected_split_args = args[0], args[1]
    with mock.patch("jrnl.os_compat.on_windows", lambda: True):
        assert split_args(input_arguments) == expected_split_args


import os
import numpy as np
from delft.sequenceLabelling import Sequence
from delft.utilities.Tokenizer import tokenizeAndFilter
from delft.utilities.Embeddings import Embeddings,test
from delft.utilities.Utilities import stats
from delft.sequenceLabelling.reader import load_data_and_labels_xml_file, load_data_and_labels_conll, load_data_and_labels_lemonde, load_data_and_labels_ontonotes
from sklearn.model_selection import train_test_split
from sklearn.metrics import precision_recall_fscore_support, f1_score
from numpy import nditer, mean
import keras.backend as K
import argparse
import time
from NERDA.datasets import get_conll_data
from NERDA.models import NERDA

tag_scheme = [            'B_NE001',
                            'B_NE003',
                            'B_NE005',
                            'B_NE006',
                            'B_NE007',
                            'B_NE008',
                            'B_NE009',
                            'I_NE001',
                            'I_NE003',
                            'I_NE005',
                            'I_NE006',
                            'I_NE009',
]
tag_outside='O'
# hyperparameters for network
dropout = 0.1
# hyperparameters for training camembert
training_hyperparameters = {
'epochs' : 14,
'warmup_steps' : 700,
'train_batch_size': 32,
'learning_rate': 0.0001
}
def voting(ys=[],coeffs=[],coef_alltime=False):
    unique_ys = list(set(ys))#no001, noo1, noo2, n002
    # no001,noo2
    if len(unique_ys) > 1:
        maximum = ''
        max_occ = 1
        for unique_y in unique_ys:
            temp_occ = ys.count(unique_y)
            if temp_occ > max_occ:
                max_occ = temp_occ
                maximum = unique_y
        if max_occ > len(ys)//2:
            if not coef_alltime:
                return maximum
            else:
                max_coefs = [0 for x in range(len(unique_ys))]
                for i,unique_y in enumerate(unique_ys):
                    indices = [i for i, x in enumerate(ys) if x == unique_y]
                    for j in indices:
                        max_coefs[i] += coeffs[j]
                max_coef_index = max_coefs.index(max(max_coefs))
                return unique_ys[max_coef_index]
        else:
            max_coefs = [0 for x in range(len(unique_ys))]
            for i,unique_y in enumerate(unique_ys):
                indices = [i for i, x in enumerate(ys) if x == unique_y]
                for j in indices:
                    max_coefs[i] += coeffs[j]
            max_coef_index = max_coefs.index(max(max_coefs))
            print(f'conflict max(max_coefs):{max(max_coefs)}\n{*ys,}\t{unique_ys[max_coef_index]}')
            return unique_ys[max_coef_index]
    else : return ys[0]

# usual eval on CoNLL 2003 eng.testb
def eval(model,
         dataset_type='conll2003',
         lang='en',
         architecture='BidLSTM_CRF',
         use_ELMo=False,
         use_BERT=False,
         data_path=None,):

    print('Loading CoNLL-2003 NER data...')
    x_test, y_test = load_data_and_labels_conll('data/sequenceLabelling/CoNLL-2003/eng.testb')
    x_test_list, y_test_list = x_test.tolist(), y_test.tolist()

    new_y_test = []
    for y in y_test_list:
        temp = []
        for p_y in y:
            temp.append(p_y.replace('E_', 'I_'))
        new_y_test.append(temp)

    print(f'x_test shape {x_test.shape}\n \
            y_test shape {y_test.shape}')
    stats(x_eval=x_test, y_eval=y_test)

    coeffs = []
    y_predecteds = []
    print(f'======== {model}')
    model_name1 = 'ner-en-conll2003-BidLSTM_CRF-wiki.fr'
    model_name2 = 'ner-en-conll2003-BidLSTM_CRF-elmo-fr-embeddings-11epocs_3rd-run_2-folds'
    model_name3 = 'ner-en-conll2003-BidLSTM_CRF-bert-base-en-emebddings-11epocs'
    model_name4 = 'ner-en-conll2003-BidLSTM_CRF-glove-840B-embeddings-11epocs'
    model_name5 = 'ner-en-conll2003-BidLSTM_CRF-word2vec-embeddings-25epocs'
    # model_name2 = 'ner-en-conll2003-BidLSTM_CRF-elmo-fr-embeddings-11epocs'
    # model_name4 = 'ner-en-conll2003-BidLSTM_CRF-wiki.fr-embeddings-11epocs'
    if model == 'camembert':
        camembert_model = NERDA(device="cuda:0",
                            tag_scheme = tag_scheme,
                            tag_outside = tag_outside,
                            transformer = '../camembert/camembert_jurica_512/',
                            )
        start_time = time.time()
        print("\nEvaluation on test set:")
        y_predecteds.append(camembert_model.predict(x_test_list))
    elif model == 'w2vec':
        model = Sequence(model_name5)
        model.load()
        start_time = time.time()
        print("\nEvaluation on test set:")
        y_predecteds.append(model.eval(x_test, y_test))
        coeffs.append(1)
    elif model == 'fasttext':
        model = Sequence(model_name1)
        model.load()
        start_time = time.time()
        print("\nEvaluation on test set:")
        y_predecteds.append(model.eval(x_test, y_test))
        coeffs.append(1)
    elif model == 'elmo':
        model = Sequence(model_name2)
        model.load()
        start_time = time.time()
        print("\nEvaluation on test set:")
        y_predecteds.append(model.eval(x_test, y_test))
        coeffs.append(1)
    elif model == 'bert':
        model = Sequence(model_name3)
        model.load()
        start_time = time.time()
        print("\nEvaluation on test set:")
        y_predecteds.append(model.eval(x_test, y_test))
        coeffs.append(1)
    elif model == 'e0':
        model5 = Sequence(model_name5)
        model1 = Sequence(model_name1)
        model1.load()
        model5.load()
        start_time = time.time()
        print("\nEvaluation on test set:")
        y_predecteds.append(model1.eval(x_test, y_test))
        y_predecteds.append(model5.eval(x_test, y_test))
        coeffs = [.4,.6]
    elif model == 'e1':
        model2 = Sequence(model_name2)
        model3 = Sequence(model_name3)
        model2.load()
        model3.load()
        start_time = time.time()
        print("\nEvaluation on test set:")
        y_predecteds.append(model2.eval(x_test, y_test))
        y_predecteds.append(model3.eval(x_test, y_test))
        coeffs = [.6,.4]
    elif model == 'e2':
        model1 = Sequence(model_name1)
        model5 = Sequence(model_name5)
        model2 = Sequence(model_name2)
        model3 = Sequence(model_name3)
        model1.load()
        model5.load()
        model2.load()
        model3.load()
        start_time = time.time()
        print("\nEvaluation on test set:")
        y_predecteds.append(model1.eval(x_test, y_test))
        y_predecteds.append(model5.eval(x_test, y_test))
        y_predecteds.append(model2.eval(x_test, y_test))
        y_predecteds.append(model3.eval(x_test, y_test))
        coeffs = [.3,.3, .2,.2]
    else:
        print('unknown model')
        return
    print(f'coeffs {coeffs}')

    with open('y.txt', 'w') as f:
        y_voted = []
        ys = []
        if len(y_predecteds) == 1:
            f.write(f'X_test\t\tY_test\t\tModel\n')#\t\tGlove
            for x, y, y1 in \
            zip(x_test, y_test, \
            y_predecteds[0]):
                for p_x, p_y, p_y1 in zip(x, y,y1):
                    ys_transformed = []
                    for tr in [ p_y1]:
                        if tr == '<PAD>' or tr == 'O':
                            ys_transformed.append('NE000')
                        else:
                            ys_transformed.append(tr)
                    #[p_y1, p_y2, p_y3, p_y4, p_y5],\
                    y_v = p_y1
                    if y_v == '<PAD>':
                        y_v= 'NE000'
                    y_voted.append(y_v)
                    ys.append(p_y)
                    f.write(f'{p_x}\t\t{p_y}\t\t{p_y1}\t\t{y_v}\n')
        elif len(y_predecteds) == 2:
            f.write(f'X_test\t\tY_test\t\tModel1\t\tModel2\n')#\t\tGlove
            for x, y, y1,y5 in \
            zip(x_test, y_test, \
            y_predecteds[0], y_predecteds[1]):
                for p_x, p_y, p_y1,p_y5 in zip(x, y,y1,y5):
                    ys_transformed = []
                    for tr in [ p_y1,p_y5]:
                        if tr == '<PAD>' or tr == 'O':
                            ys_transformed.append('NE000')
                        else:
                            ys_transformed.append(tr)
                    #[p_y1, p_y2, p_y3, p_y4, p_y5],\
                    y_v = voting(  ys=ys_transformed, \
                                            coeffs=coeffs )
                    if y_v == '<PAD>':
                        y_v= 'NE000'
                    y_voted.append(y_v)
                    ys.append(p_y)
                    f.write(f'{p_x}\t\t{p_y}\t\t{p_y1}\t\t{p_y5}\t\t{y_v}\n')
        elif len(y_predecteds) == 4:
            f.write(f'X_test\t\tY_test\t\tModel1\t\tModel2\t\tModel3\t\tModel4\n')#\t\tGlove
            for x, y, y1,y2,y3,y5 in \
            zip(x_test, y_test, \
            y_predecteds[0], y_predecteds[1], y_predecteds[2], y_predecteds[3]):
                for p_x, p_y, p_y1, p_y2, p_y3,p_y5 in zip(x, y,y1, y2, y3,y5):
                    ys_transformed = []
                    for tr in [ p_y1, p_y2, p_y3,p_y5]:
                        if tr == '<PAD>' or tr == 'O':
                            ys_transformed.append('NE000')
                        else:
                            ys_transformed.append(tr)
                    #[p_y1, p_y2, p_y3, p_y4, p_y5],\
                    y_v = voting(  ys=ys_transformed, \
                                            coeffs=coeffs )
                    if y_v == '<PAD>':
                        y_v= 'NE000'
                    y_voted.append(y_v)
                    ys.append(p_y)
                    f.write(f'{p_x}\t\t{p_y}\t\t{p_y1}\t\t{p_y2}\t\t{p_y3}\t\t{p_y5}\t\t{y_v}\n')

    runtime = round(time.time() - start_time, 3)
    # print(f'Diasappear {set(ys) - set(y_voted)}')
    metrics = precision_recall_fscore_support(ys, y_voted, average=None,)
    metrics2 = f1_score(ys, y_voted, average=None,)
    # print(f'\nmetrics all in one {metrics}\n\n')
    print('============== Voting scores =============\n\tprecision\trecall\tfscore\tsupport\n')
    for label, precision, recall, fscore, support in zip(list(set(ys)),
                                                        nditer(metrics[0]),\
                                                        nditer(metrics[1]), \
                                                        nditer(metrics[2]), \
                                                        nditer(metrics[3]),):
        print(f'{label}\t\t{round(float(precision),4)*100}\t\t{round(float(recall),4)*100}\t\t{round(float(fscore),4)*100}\t\t{support}')
    print('==========================================')
    print(f'AVG:\t{mean(metrics[0])}\t{mean(metrics[1])}\t{mean(metrics[2])}\t{mean(metrics[3])}')

    print('============== Voting scores (2) =============\n\tf1_score\n')
    for label, fscore in zip(list(set(ys)), \
                            nditer(metrics[0]),):
        print(f'{label}\t\t{round(float(fscore),4)*100}')
    print('==========================================')
    print(f'AVG:\t{mean(metrics[0])}')

    print("runtime: %s seconds " % (runtime))

if __name__ == "__main__":

    architectures = ['BidLSTM_CRF', 'BidLSTM_CNN_CRF', 'BidLSTM_CNN_CRF', 'BidGRU_CRF', 'BidLSTM_CNN', 'BidLSTM_CRF_CASING',
                     'bert-base-en', 'bert-large-en', 'scibert', 'biobert']

    parser = argparse.ArgumentParser(
        description = "Neural Named Entity Recognizers")

    parser.add_argument("action", help="one of [train, train_eval, eval, tag]")
    parser.add_argument("--fold-count", type=int, default=1, help="number of folds or re-runs to be used when training")
    parser.add_argument("--lang", default='en', help="language of the model as ISO 639-1 code")
    parser.add_argument("--dataset-type",default='conll2003', help="dataset to be used for training the model")
    parser.add_argument("--train-with-validation-set", action="store_true", help="Use the validation set for training together with the training set")
    parser.add_argument("--architecture",default='BidLSTM_CRF', help="type of model architecture to be used, one of "+str(architectures))
    parser.add_argument("--use-ELMo", action="store_true", help="Use ELMo contextual embeddings")
    parser.add_argument("--use-BERT", action="store_true", help="Use BERT extracted features (embeddings)")
    parser.add_argument("--data-path", default=None, help="path to the corpus of documents for training (only use currently with Ontonotes corpus in orginal XML format)")
    parser.add_argument("--file-in", default=None, help="path to a text file to annotate")
    parser.add_argument("--file-out", default=None, help="path for outputting the resulting JSON NER anotations")
    parser.add_argument("--model", default=None, help="model or ensemble you want to use")
    parser.add_argument(
        "--embedding", default=None,
        help=(
            "The desired pre-trained word embeddings using their descriptions in the file"
            " embedding-registry.json."
            " Be sure to use here the same name as in the registry ('glove-840B', 'fasttext-crawl', 'word2vec'),"
            " and that the path in the registry to the embedding file is correct on your system."
        )
    )

    args = parser.parse_args()

    action = args.action
    model = args.model
    if action not in ('train', 'tag', 'eval', 'train_eval'):
        print('action not specifed, must be one of [train, train_eval, eval, tag]')
    lang = args.lang
    dataset_type = args.dataset_type
    train_with_validation_set = args.train_with_validation_set
    use_ELMo = args.use_ELMo
    use_BERT = args.use_BERT
    architecture = args.architecture
    if architecture not in architectures and architecture.lower().find("bert") == -1:
        print('unknown model architecture, must be one of', architectures)
    data_path = args.data_path
    file_in = args.file_in
    file_out = args.file_out

    # name of embeddings refers to the file embedding-registry.json
    # be sure to use here the same name as in the registry ('glove-840B', 'fasttext-crawl', 'word2vec'),
    # and that the path in the registry to the embedding file is correct on your system
    # below we set the default embeddings value
    if args.embedding is None:
        embeddings_name = 'glove-840B'
        if lang == 'en':
            if dataset_type == 'conll2012':
                embeddings_name = 'fasttext-crawl'
        elif lang == 'fr':
            embeddings_name = 'wiki.fr'
    else:
        embeddings_name = args.embedding

    if action == 'train':
        train(embeddings_name,
            dataset_type,
            lang,
            architecture=architecture,
            use_ELMo=use_ELMo,
            use_BERT=use_BERT,
            data_path=data_path)

    if action == 'train_eval':
        import tensorflow as tf
        strategy = tf.distribute.MirroredStrategy( ["GPU:0", "GPU:1"] )
        with strategy.scope():
            if args.fold_count < 1:
                raise ValueError("fold-count should be equal or more than 1")
            train_eval(embeddings_name,
                dataset_type,
                lang,
                architecture=architecture,
                fold_count=args.fold_count,
                train_with_validation_set=train_with_validation_set,
                use_ELMo=use_ELMo,
                use_BERT=use_BERT,
                data_path=data_path)

    if action == 'eval':
        eval(model, dataset_type, lang, architecture=architecture, use_ELMo=use_ELMo, use_BERT=use_BERT)

    if action == 'tag':
        if lang is not 'en' and lang is not 'fr':
            print("Language not supported:", lang)
        else:
            print(file_in)
            result = annotate("json",
                            dataset_type,
                            lang,
                            architecture=architecture,
                            use_ELMo=use_ELMo,
                            use_BERT=use_BERT,
                            file_in=file_in,
                            file_out=file_out)
            """if result is not None:
                if file_out is None:
                    print(json.dumps(result, sort_keys=False, indent=4, ensure_ascii=False))
            """

    try:
        # see https://github.com/tensorflow/tensorflow/issues/3388
        K.clear_session()
    except:
        # TF could complain in some case
        print("\nLeaving TensorFlow...")


#pip install pyinstaller
#pip install xlrd ## xlrd는 정책변경으로 xls 만 읽을 수 있어서 openpyxl 로 교체.
#pip install openpyxl

#import xlrd
import openpyxl

import sys
import os
import timeit
import enum

#------------------------------------------------------------------------
# 시간 체크.
#------------------------------------------------------------------------
timer_stack_ = list()
def BeginTimer(tag : str):
	start_time = timeit.default_timer()
	timer_stack_.append((tag, start_time))
def EndTimer() -> (str, float):
	stack_count = len(timer_stack_)
	if stack_count > 0:
		last_time = timer_stack_.pop()
		tag = str(last_time[0])
		start_time = float(last_time[1])
		end_time = timeit.default_timer()
		return (("\t" * (stack_count - 2)) + tag, end_time - start_time)
	return ("", 0.0)
def PrintTimer():
	print(end())
def PrintTimer(data : (str, float)):
	tag = data[0]
	pass_time = data[1]
	print(f"{tag} : {pass_time:.4f}s")

#------------------------------------------------------------------------
# C# 에서 사용중인 키워드 (전체).
# 팁 : 최소한의 검사용으로 작성 해놓았지만 C#의 버전업시 어떻게 바뀔지는 모르기 때문에...
# 가급적이면 필드명 등에서 대소문자를 적절히 혼용해서 키워드와의 중복을 피할 것.
#------------------------------------------------------------------------
csharp_strong_keyword_list_ = [ "abstract", "as", "base", "bool", "break", "byte", "case", "catch", "char", "checked", "class", "const", "delegate", "do", "double", "else", "enum", "event", "explicit", "extern", "false", "finally", "fixed", "float", "for", "foreach", "goto", "if", "implicit", "in", "int", "interface", "internal", "lock", "long", "namespace", "new", "null", "object", "function", "declare", "define", "out", "override", "params", "private", "protected", "public", "readonly", "ref", "return", "sbyte", "sealed", "short", "sizeof", "stackalloc", "static", "string", "struct", "switch", "this", "throw", "true", "try", "typeof", "uint", "ulong", "unchecked", "unsafe", "ushort", "using", "virtual", "void", "volatile", "while" ]

#------------------------------------------------------------------------
# C#에서 사용중인 키워드 (데이터타입제외).
#------------------------------------------------------------------------
csharp_weak_keyword_list_ = [ "abstract", "as", "base", "break", "byte", "case", "catch", "checked", "class", "const", "delegate", "do", "else", "enum", "event", "explicit", "extern", "false", "finally", "fixed", "for", "foreach", "goto", "if", "implicit", "in", "interface", "internal", "lock", "namespace", "new", "null", "object", "function", "declare", "define", "out", "override", "params", "private", "protected", "public", "readonly", "ref", "return", "sealed", "sizeof", "stackalloc", "static", "struct", "switch", "this", "throw", "true", "try", "typeof", "unchecked", "unsafe", "using", "virtual", "void", "volatile", "while" ]

#------------------------------------------------------------------------
# JSON 데이터 타입.
#------------------------------------------------------------------------
class DataKind(enum.Enum):
	UNKNOWN = enum.auto()
	BOOLEAN = enum.auto()
	NUMBER = enum.auto()
	REAL = enum.auto()
	TEXT = enum.auto()

#------------------------------------------------------------------------
# 절대 경로 반환.
#------------------------------------------------------------------------
def GetAbsPath(path : str = "") -> str:
	if os.path.isfile(path) or os.path.isdir(path):
		return os.path.abspath(path)

	if getattr(sys, 'frozen', False):
		return os.path.abspath(os.path.dirname(sys.executable) + path) # 응용프로그램 경로.
	elif __file__: # 현재 스크립트파일의 경로.
		return os.path.abspath(os.path.dirname(__file__) + path)

#------------------------------------------------------------------------
# 이름규칙에 알맞는지 확인.
#------------------------------------------------------------------------
def IsCorrectName(value : str):
	if not value:
		return False

	# 알파벳 ==> 숫자의 순으로 호출되어야하며 알파벳과 숫자 외의 문자열은 실패.
	is_correct = False
	for val in value:
		if val.isalpha():
			is_correct = True
		elif val.isnumeric():
			if not is_correct:
				return False
		elif val == "_":
			is_correct = True
		else:
			return False
	return True

#------------------------------------------------------------------------
# 필드 값을 JSON 값으로 변환.
#------------------------------------------------------------------------
def DataValueToJsonValue(value : str, kind : DataKind, is_array : bool):
	if kind == DataKind.TEXT:
		return f"\"{value}\""
	elif is_array:
		result = str()
		result += "["
		val_list = value.split(",")
		val_list_count = len(val_list)
		for val_index in range(val_list_count):
			result += f"{field_value}"
			if val_index + 1 < val_list_count:
				result += ", "
		result += "]"
	else:
		return f"{value}"

#------------------------------------------------------------------------
# 2차원 스트링배열 형태의 기본 테이블 정보.
#------------------------------------------------------------------------
class NativeTable:
	name_ : str
	row_list_ : list # row[col[]]
	def __init__(self):
		self.name_ = str()
		self.row_list_ = list()

#------------------------------------------------------------------------
# 필드 정보 (1개의 열에 해당).
#------------------------------------------------------------------------
class Field:
	name_  : str
	type_ : str
	kind_ : DataKind
	is_array_ : bool
	comment_ : str
	data_ : list
	def __init__(self):
		self.name_ = str()
		self.type_ = str()
		self.kind_ = DataKind.UNKNOWN
		self.is_array_ = bool()
		self.comment_ = str()
		self.data_ = list()

#------------------------------------------------------------------------
# 테이블 정보.
#------------------------------------------------------------------------
class Table:
	name_ : str
	field_dict_ : dict # key:index, value:Field
	data_count_ : int
	def __init__(self):
		self.name_ = str()
		field_dict_ = dict()
		data_count_ = int()

#------------------------------------------------------------------------
# 규격화된 XLSX 파일에서 네이티브테이블을 만들어냄.
#------------------------------------------------------------------------
def CreateNativeTableListFromXlsxFile(file_name : str, startswith = "#") -> list:
	nativetable_list = list()
	#xlrd은 더이상 사용하지 않음.
	#book = xlrd.open_workbook(file_name)
	#for sheet in book.sheets():
	#	if sheet.name.startswith(startswith):
	#		native_table = NativeTable()
	#		native_table.name_ = sheet.name.replace(startswith, "")
	#		for row_index in range(sheet.nrows):
	#			cell_list = sheet.row_values(row_index)
	#			cols = list()
	#			for cell in cell_list:
	#				cols.append(str(cell))
	#			native_table.row_list_.append(cols)
	#		native_table_list.append(native_table)
	#return native_table_list
	book = openpyxl.load_workbook(file_name)
	for sheetname in book.sheetnames:
		if not sheetname.startswith(startswith):
			continue;
		sheet = book[sheetname]
		if not sheet:
			continue;
		nativetable = NativeTable()
		nativetable.name_ = sheetname.replace(startswith, "")
		for row_index in range(1, sheet.max_row + 1):
			cols = list()
			for col_index in range(1, sheet.max_column + 1):
				cell = sheet.cell(row = row_index, column = col_index)
				if cell.value:
					cols.append(str(cell.value))
				else:
					cols.append("")
			nativetable.row_list_.append(cols)
		nativetable_list.append(nativetable)
	return nativetable_list

#------------------------------------------------------------------------
# 네이티브테이블을 가지고 테이블을 만들어냄.
#------------------------------------------------------------------------
def CreateTable(native_table : NativeTable):
	table = Table()
	table.name_ = native_table.name_
	table.data_count_ = 0
	table.field_dict_ = dict()

	# 맨 앞 셀의 이름을 찾으면 다 넣고 종료.
	# 문자열이 존재하면 해당 필드가 존재하는 것.
	for row in native_table.row_list_:
		rowtype = row[0].lower()
		if rowtype == "field":
			rowdata = row[1:]
			for col_index in range(len(rowdata)):

				# 값이 존재하는 컬럼만.
				value = rowdata[col_index]
				if value:
					# 전체 키워드 검사.
					if value == csharp_strong_keyword_list_ or not IsCorrectName(value):
						print(f"Error!! FieldName='{value}'")
						sys.exit(-1)

					field = Field()
					field.name_ = value
					table.field_dict_[col_index] = field

			break

	# 필드와 동일한 인덱스에서 내용을 다 채워넣음.
	for row in native_table.row_list_:
		rowtype = row[0].lower()
		if rowtype == "type" or rowtype == "comment" or rowtype == "data":
			row_columns = row[1:]
			for col_index in range(len(row_columns)):

				# 필드제목이 있어 필드로 인정되는 컬럼만.
				if col_index in table.field_dict_.keys():
					value = row_columns[col_index]
					field = table.field_dict_[col_index]
					if rowtype == "type":

						# 배열여부 판단하고 배열표시 삭제 (끝이 배열[]로 끝나면 배열).
						type_length = len(value)
						if type_length > 2 and value[type_length - 2] == "[" and value[type_length - 1] == "]":
							value = value[:-2]
							field.is_array_ = True

						# 타입명 제외 키워드 검사.
						if value == csharp_weak_keyword_list_ or not IsCorrectName(value):
							print(f"Error!! FieldType='{value}'")
							sys.exit(-1)

						field.type_ = value

						if field.type_ in ["char", "sbyte", "short", "int", "long", "byte", "ushort", "uint", "ulong"]:
							field.kind_ = DataKind.NUMBER
						elif field.type_ in ["float", "double"]:
							field.kind_ = DataKind.REAL
						elif field.type_ in ["bool"]:
							field.kind_ = DataKind.BOOLEAN
						else:
							field.kind_ = DataKind.TEXT
							field.is_array_ = False # 문자열타입은 배열일 수 없음.

					elif rowtype == "comment":
						field.comment_ = value
					elif rowtype == "data":
						field.data_.append(value)

	# 데이터 범위 체크....

	# 빈 데이터 기본값 설정 + 최대데이터 카운트 셋팅.
	for field in table.field_dict_.values():
		data_count = len(field.data_)
		if table.data_count_ < data_count:
			table.data_count_ = data_count

		for data_index in range(data_count):
			value = field.data_[data_index]

			# 값이 존재하지 않을 때.
			if value == "":
				# 정수형.
				if field.kind_ == DataKind.NUMBER:
					field.data_[data_index] = "0"
				# 실수형.
				elif field.kind_ == DataKind.REAL:
					field.data_[data_index] = "0.0"
				# 논리형.
				elif field.kind_ == DataKind.BOOLEAN:
					field.data_[data_index] = "false"
			else:
				if field.kind_ == DataKind.NUMBER:
					field.data_[data_index] = str(int(float(value)))
				elif field.kind_ == DataKind.REAL:
					field.data_[data_index] = str(float(value))
				elif field.kind_ == DataKind.BOOLEAN:
					field.data_[data_index] = "true" if value.lower() == "true" else "false"

	return table

#------------------------------------------------------------------------
# TABLE을 JSON 파일로 변환.
#------------------------------------------------------------------------
def CreateJsonFileFromTable(file_name : str, table : Table):
	result = str()
	result += "[\n"

	field_count = len(table.field_dict_)
	for row_index in range(table.data_count_):
		result += "\t{\n"
		field_index = 0
		for field in table.field_dict_.values():
			field_name = field.name_
			field_value = DataValueToJsonValue(field.data_[row_index], field.kind_, field.is_array_)

			result += f"\t\t\"{field_name}\": {field_value}"
			if field_index + 1 < field_count:
				result += ",\n"
			else:
				result += "\n"
			field_index += 1
		if row_index + 1 < table.data_count_:
			result += "\t},\n"
		else:
			result += "\t}\n"
	result += "]"
	with open(file_name, "w+", encoding = "utf8") as file:
		file.write(result)

#------------------------------------------------------------------------
# 테이블을 CS 파일로 변환.
#------------------------------------------------------------------------
def CreateCsFileFromTable(file_name : str, table : Table, make_enum_fields : list):
	table_name = table.name_
	result = str()

	#result += "using DagraacSystems;\n"
	#result += "using DagraacSystems.Table;\n"
	#result += "using DagraacSystems.TableExtension;\n"
	#result += "using System.Collections.Generic;\n"
	result += "\n"
	result += "\n"

	if make_enum_fields != None:
		for make_enum_field in make_enum_fields:
			result += f"public enum {table_name}_{make_enum_field}\n" # CommonTable_Name
			result += "{\n"
			for field in table.field_dict_.values():
				if field.name_ == make_enum_field and (field.kind_ == DataKind.NUMBER or field.kind_ == DataKind.TEXT):
					for data in field.data_:
						if data:
							result += f"\t{data},\n" # 열거체 작성을 원하는 필드의 모든 값은 서로 유니크여야함.
			result += "}\n"
			result += "\n"
			result += "\n"

	# 클래스.
	result += f"public class {table_name}Data : DagraacSystems.ITableData\n"
	result += "{\n"
	for field in table.field_dict_.values():
		field_name = field.name_
		field_type = field.type_
		field_comment = field.comment_
		if field_comment:
			result += f"\t/// <summary>{field_comment}</summary>\n"
		result += f"\tpublic {field_type} {field_name};\n\n"
	#result += "\n"

	#int ToFields();
	result += "\tpublic System.Collections.Generic.List<DagraacSystems.FieldInfo> ToFields()\n"
	result += "\t{\n"
	result += "\t\treturn new System.Collections.Generic.List<DagraacSystems.FieldInfo>()\n"
	result += "\t\t{\n"
	field_index = 0
	for field in table.field_dict_.values():
		field_name = field.name_
		result += f"\t\t\tnew DagraacSystems.FieldInfo(\"{field_name}\", {field_name}.GetType(), {field_name}),\n"
		field_index += 1
	result += "\t\t};\n"
	result += "\t}\n"
	result += "\n"

	#int GetFieldIndex(string name);
	result += "\tpublic int GetFieldIndex(string name)\n"
	result += "\t{\n"
	result += "\t\tswitch (name)\n"
	result += "\t\t{\n"
	field_index = 0
	for field in table.field_dict_.values():
		field_name = field.name_
		result += f"\t\t\tcase \"{field_name}\": return {field_index};\n"
		field_index += 1
	result += "\t\t}\n"
	result += "\t\treturn -1;\n"
	result += "\t}\n"
	result += "\n"

	#string GetFieldName(int index);
	result += "\tpublic string GetFieldName(int index)\n"
	result += "\t{\n"
	result += "\t\tswitch (index)\n"
	result += "\t\t{\n"
	field_index = 0
	for field in table.field_dict_.values():
		field_name = field.name_
		result += f"\t\t\tcase {field_index}: return \"{field_name}\";\n"
		field_index += 1
	result += "\t\t}\n"
	result += "\t\treturn string.Empty;\n"
	result += "\t}\n"
	result += "\n"

	#System.Type GetFieldType(int index);
	result += "\tpublic System.Type GetFieldType(int index)\n"
	result += "\t{\n"
	result += "\t\tswitch (index)\n"
	result += "\t\t{\n"
	field_index = 0
	for field in table.field_dict_.values():
		field_name = field.name_
		field_type = field.type_
		field_comment = field.comment_
		result += f"\t\t\tcase {field_index}: return {field_name}.GetType();\n"
		field_index += 1
	result += "\t\t}\n"
	result += "\t\treturn null;\n"
	result += "\t}\n"
	result += "\n"

	#object GetFieldValue(int index);
	result += "\tpublic object GetFieldValue(int index)\n"
	result += "\t{\n"
	result += "\t\tswitch (index)\n"
	result += "\t\t{\n"
	field_index = 0
	for field in table.field_dict_.values():
		field_name = field.name_
		field_type = field.type_
		field_comment = field.comment_
		result += f"\t\t\tcase {field_index}: return (object){field_name};\n"
		field_index += 1
	result += "\t\t}\n"
	result += "\t\treturn null;\n"
	result += "\t}\n"
	result += "\n"

	#int GetFieldCount();
	result += "\tpublic int GetFieldCount()\n"
	result += "\t{\n"
	field_count = len(table.field_dict_)
	result += f"\t\treturn {field_count};\n"
	result += "\t}\n"
	#result += "\n"
	result += "}"

	with open(file_name, "w+", encoding = "utf8") as file:
		file.write(result)

#------------------------------------------------------------------------
# 처리 시작.
#------------------------------------------------------------------------
def OnMain(args):
	print("TableExporter 0.0.4")
	print("made by dagraac")
	print()

	# 필수 인자.
	# 시트를 json과 cs로 익스포트.
	xlsx_file_name = GetAbsPath(str(args[1]))
	json_path = GetAbsPath(str(args[2]))
	cs_path = GetAbsPath(str(args[3]))

	# 추가 인자.
	make_enum_fields = None
	if len(args) > 4:
		make_enum_fields = str(args[4]).split(';')

	BeginTimer(f"\nTotal Time")
	native_table_list = CreateNativeTableListFromXlsxFile(xlsx_file_name)
	for native_table in native_table_list:
		if native_table:
			print(f"{native_table.name_}")
			table = CreateTable(native_table)

			if table:
				json_file_name = f"{json_path}\\{table.name_}.json"
				BeginTimer(f"\t{json_file_name}")
				CreateJsonFileFromTable(json_file_name, table)
				PrintTimer(EndTimer())

				cs_file_name = f"{cs_path}\\{table.name_}Data.cs"
				BeginTimer(f"\t{cs_file_name}")
				CreateCsFileFromTable(cs_file_name, table, make_enum_fields)
				PrintTimer(EndTimer())

	PrintTimer(EndTimer())

#------------------------------------------------------------------------
# 진입점.
#------------------------------------------------------------------------
if __name__ == "__main__":
	OnMain(sys.argv)

# 파일 xlsx json cs

# TableExporter^
#  "D:\Projects\Example\Design\Tables\ExampleTable.xlsx"^
#  "D:\Projects\Example\Client\Assets\Resources\Tables"^
#  "D:\Projects\Example\Client\Assets\Scripts\Tables\Data"

# TableExporter^
#  ".\ExampleTable.xlsx"^
#  "..\..\Client\Assets\Resources\Tables"^
#  "..\..\Client\Assets\Scripts\Tables\Data"


#background.py

import pyglet,viewport
#background_batch=pyglet.graphics.Batch()

pyglet.resource.path.append('./images')
pyglet.resource.reindex()

background1_image=pyglet.resource.image('ocean_floor_background.png')
background2_image=pyglet.resource.image('jungle_background.jpg')
background3_image=pyglet.resource.image('space_background.jpg')

bg_map={1:background1_image,
        2:background2_image,
        3:background3_image}

class Background(pyglet.sprite.Sprite):
        def __init__(self,level,*args,**kwargs):
                super(Background,self).__init__(bg_map[level],*args,**kwargs)

                #self.scale=.999


import logging

from PyQt6.QtCore import Qt
from PyQt6.QtGui import QIcon
from PyQt6.QtWidgets import QFileDialog, QFrame, QHBoxLayout, QLabel, QLineEdit, QPushButton, QWidget, QTableWidget, \
    QTableWidgetItem, QAbstractItemView

from wannadb_parsql.parsql import Parser
from wannadb_parsql.rewrite import update_query_attribute_list, rewrite_query
from wannadb_ui.common import BUTTON_FONT, CODE_FONT, CODE_FONT_BOLD, LABEL_FONT, MainWindowContent, \
    MainWindowContentSection, CustomScrollableListItem, CustomScrollableList, RED, CODE_FONT_SMALLER, \
    INPUT_DOCS_COLUMN_NAME

logger = logging.getLogger(__name__)

class DocumentBaseViewerWidget(MainWindowContent):
    def __init__(self, main_window):
        super(DocumentBaseViewerWidget, self).__init__(main_window, "Document Base Viewer")

        self.everything_populated = False

        # Stats
        self.num_documents_nuggets = QLabel("0 documents | 0 nuggets")
        self.num_documents_nuggets.setFont(LABEL_FONT)
        self.controls_widget_layout.addWidget(self.num_documents_nuggets)

        self.export_table_button = QPushButton("Export Structured Data to CSV")
        self.export_table_button.setFont(BUTTON_FONT)
        self.export_table_button.setIcon(QIcon("wannadb_ui/resources/table.svg"))
        self.export_table_button.clicked.connect(self.main_window.save_table_to_csv_task)
        self.controls_widget_layout.addWidget(self.export_table_button)

        # query
        self.query = MainWindowContentSection(self, "Query")
        self.layout.addWidget(self.query)

        self.query_input_box = QWidget()
        self.query_input_box_layout = QHBoxLayout(self.query_input_box)
        self.query_input_box_layout.setContentsMargins(0, 0, 0, 0)

        self.query_box = QLineEdit()
        self.query_box.setPlaceholderText("Enter a SQL query or manually specify the target attributes below")
        self.query_box.setFont(CODE_FONT)
        self.query_box.setStyleSheet("border: none")
        self.query_input_box_layout.addWidget(self.query_box, stretch=1)

        self.enter_query_button = QPushButton("Run Query")
        self.enter_query_button.setFont(BUTTON_FONT)
        self.enter_query_button.setIcon(QIcon("wannadb_ui/resources/text_cursor.svg"))
        self.enter_query_button.setAutoDefault(True)
        self.enter_query_button.clicked.connect(self._parse_query)
        self.query_input_box_layout.addWidget(self.enter_query_button, alignment=Qt.AlignmentFlag.AlignRight)

        self.query.layout.addWidget(self.query_input_box)

        # attributes
        self.attributes = MainWindowContentSection(self, "Attributes")
        self.layout.addWidget(self.attributes)

        self.attributes_controls = QWidget()
        self.attributes_controls_layout = QHBoxLayout(self.attributes_controls)
        self.attributes_controls_layout.setContentsMargins(0, 0, 0, 0)

        self.add_attribute_button = QPushButton("Add Attribute")
        self.add_attribute_button.setFont(BUTTON_FONT)
        self.add_attribute_button.setIcon(QIcon("wannadb_ui/resources/plus.svg"))
        self.add_attribute_button.clicked.connect(self.main_window.add_attribute_task)
        self.attributes_controls_layout.addWidget(self.add_attribute_button, stretch=1)

        self.populate_remaining_attributes_button = QPushButton("Populate Remaining Attributes")
        self.populate_remaining_attributes_button.setFont(BUTTON_FONT)
        self.populate_remaining_attributes_button.setIcon(QIcon("wannadb_ui/resources/run_run.svg"))
        self.populate_remaining_attributes_button.clicked.connect(self.main_window.interactive_table_population_task)
        self.attributes_controls_layout.addWidget(self.populate_remaining_attributes_button, stretch=1)

        self.attributes_list = CustomScrollableList(self, AttributeWidget, self.attributes_controls)
        self.attributes.layout.addWidget(self.attributes_list)

        # result visualization
        self.results = MainWindowContentSection(self, "Results")
        self.layout.addWidget(self.results)
        self.results_table = QTableWidget()
        self.results_table.setFont(CODE_FONT_SMALLER)
        self.results_table.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers)
        self.results.layout.addWidget(self.results_table)

    def update_document_base(self, document_base):
        def _attr_populated(a: str):
            return all(a in d.attribute_mappings.keys() for d in document_base.documents)

        # update documents & nuggets counts
        self.num_documents_nuggets.setText(f"{len(document_base.documents)} documents | {len(document_base.nuggets)} nuggets")

        # update attributes
        self.attributes_list.update_item_list(document_base.attributes, document_base)

        # Everything already populated?
        # Will be used to enable or disable populate and export buttons depending on the completeness of matching
        self.everything_populated = all(_attr_populated(a.name) for a in document_base.attributes)

        # update query
        attributes = [str(a).replace("'", "") for a in document_base.attributes]
        # Take existing query and add manually added attributes if needed
        original_query = self.query_box.text()
        if original_query == "":
            original_query = "SELECT *"
        columns, original_query_parsed = Parser().parse(original_query)
        query = update_query_attribute_list(original_query_parsed, attributes)

        if len(attributes) > 0:
            self.query_box.setText(query)
        else:
            self.query_box.setText("")

        # update cache db
        if self.main_window.cache_db is not None:
            attribute_names = [INPUT_DOCS_COLUMN_NAME]
            attribute_names.extend(attributes)
            query = update_query_attribute_list(original_query_parsed, attribute_names)
            columns, parsed = Parser().parse(query)

            def _get_values_for_column(column):
                for i, document in enumerate(document_base.documents):
                    value = ""
                    if column.name in document.attribute_mappings.keys() and len(
                            document.attribute_mappings[column.name]) > 0:
                        value = document.attribute_mappings[column.name][0].text
                    if value == "":
                        continue
                    yield i, value

            for column in columns:
                if column.name != INPUT_DOCS_COLUMN_NAME and self.main_window.cache_db.table_empty(column.name):
                    logger.info(f"Populating cache table for attribute '{column.name}'")
                    self.main_window.cache_db.store_many(column, _get_values_for_column(column))

            logger.info(f"Columns: {columns}")

            _, rewritten_query = rewrite_query(columns, original_query_parsed)
            logger.info(f"Executing query: '{rewritten_query}'")
            results = self.main_window.cache_db.execute_queries(rewritten_query)[0]

            # update results
            self.results_table.setRowCount(len(results))
            attribute_count = len(attribute_names)
            self.results_table.setColumnCount(attribute_count)
            self.results_table.setHorizontalHeaderLabels(attribute_names)

            for index, row in results.iterrows():
                for c, attribute_name in enumerate(attribute_names):
                    a = row[attribute_name]
                    if a is not None:
                        self.results_table.setItem(index, c, QTableWidgetItem(str(a)))
                    else:
                        self.results_table.setItem(index, c, QTableWidgetItem(""))

            self.results_table.resizeColumnsToContents()

    def enable_input(self):
        self.enter_query_button.setEnabled(True)
        # Enable buttons only conditionally (if needed/possible)
        self.populate_remaining_attributes_button.setEnabled(not self.everything_populated)
        self.export_table_button.setEnabled(self.everything_populated)
        # TODO Apply this to menu entries, too
        self.add_attribute_button.setEnabled(True)
        self.attributes_list.enable_input()

    def disable_input(self):
        self.enter_query_button.setDisabled(True)
        self.populate_remaining_attributes_button.setDisabled(True)
        self.export_table_button.setDisabled(True)
        self.add_attribute_button.setDisabled(True)
        self.attributes_list.disable_input()

    def _parse_query(self):
        if self.main_window.document_base is not None:
            query = self.query_box.text()
            try:
                attributes, parsed = Parser().parse(query)

                attribute_names = [str(attribute) for attribute in attributes]
                logger.info(f"Derived attribute names: {attribute_names}")

                self.main_window.to_busy_state()
                # noinspection PyUnresolvedReferences
                self.main_window.add_attributes.emit(attribute_names, self.main_window.document_base)
            except Exception as e:
                logger.error(str(e))
                self.main_window.status_widget_message.setText(str(e))
                self.main_window.status_widget_progress.setRange(0, 100)
                self.main_window.status_widget_progress.setValue(0)

class AttributeWidget(CustomScrollableListItem):
    def __init__(self, document_base_viewer):
        super(AttributeWidget, self).__init__(document_base_viewer)
        self.document_base_viewer = document_base_viewer
        self.attribute = None

        self.setFixedHeight(40)
        self.setObjectName("attributeWidget")
        self.setStyleSheet("QWidget#attributeWidget { background-color: white}")

        self.layout = QHBoxLayout(self)
        self.layout.setContentsMargins(20, 0, 20, 0)
        self.layout.setSpacing(40)

        self.attribute_name = QLabel()
        self.attribute_name.setFont(CODE_FONT_BOLD)
        self.layout.addWidget(self.attribute_name, alignment=Qt.AlignmentFlag.AlignLeft)

        self.num_matched = QLabel("matches: -")
        self.num_matched.setFont(CODE_FONT)
        self.layout.addWidget(self.num_matched, alignment=Qt.AlignmentFlag.AlignLeft)

        self.buttons_widget = QWidget()
        self.buttons_layout = QHBoxLayout(self.buttons_widget)
        self.buttons_layout.setContentsMargins(0, 0, 0, 0)
        self.buttons_layout.setSpacing(10)
        self.layout.addWidget(self.buttons_widget, alignment=Qt.AlignmentFlag.AlignRight)

        self.start_matching_button = QPushButton()
        self.start_matching_button.setIcon(QIcon("wannadb_ui/resources/run.svg"))
        self.start_matching_button.setToolTip("Populate the cells of this attribute..")
        self.start_matching_button.clicked.connect(self._start_matching_button_clicked)
        self.start_matching_button.hide()
        self.buttons_layout.addWidget(self.start_matching_button)

        self.forget_matches_button = QPushButton()
        self.forget_matches_button.setIcon(QIcon("wannadb_ui/resources/redo.svg"))
        self.forget_matches_button.setToolTip("Clear the cells of this attribute.")
        self.forget_matches_button.clicked.connect(self._forget_matches_button_clicked)
        self.buttons_layout.addWidget(self.forget_matches_button)

        self.remove_button = QPushButton()
        self.remove_button.setIcon(QIcon("wannadb_ui/resources/trash.svg"))
        self.remove_button.setToolTip("Remove this attribute.")
        self.remove_button.clicked.connect(self._remove_button_clicked)
        self.buttons_layout.addWidget(self.remove_button)

    def update_item(self, item, params=None):
        self.attribute = item

        if len(params.attributes) == 0:
            max_attribute_name_len = 10
        else:
            max_attribute_name_len = max(len(attribute.name) for attribute in params.attributes)
        self.attribute_name.setText(self.attribute.name + (" " * (max_attribute_name_len - len(self.attribute.name))))

        mappings_in_some_documents = False
        no_mappings_in_some_documents = False
        num_matches = 0
        for document in params.documents:
            if self.attribute.name in document.attribute_mappings.keys():
                mappings_in_some_documents = True
                if document.attribute_mappings[self.attribute.name] != []:
                    num_matches += 1
            else:
                no_mappings_in_some_documents = True

        if not mappings_in_some_documents and no_mappings_in_some_documents:
            self.num_matched.setStyleSheet(f"color: {RED}")
            self.num_matched.setText("not yet populated")
            self.forget_matches_button.hide()
            self.start_matching_button.show()
        elif mappings_in_some_documents and no_mappings_in_some_documents:
            self.num_matched.setText("only partly populated!")
            self.num_matched.setStyleSheet(f"color: {RED}")
            self.start_matching_button.hide()
            self.forget_matches_button.show()
        else:
            self.num_matched.setText(f"{num_matches} populated cells")
            self.num_matched.setStyleSheet(f"color: black")
            self.start_matching_button.hide()
            self.forget_matches_button.show()

    def enable_input(self):
        self.forget_matches_button.setEnabled(True)
        self.remove_button.setEnabled(True)
        self.start_matching_button.setEnabled(True)

    def disable_input(self):
        self.forget_matches_button.setDisabled(True)
        self.remove_button.setDisabled(True)
        self.start_matching_button.setDisabled(True)

    def _forget_matches_button_clicked(self):
        self.document_base_viewer.main_window.forget_matches_for_attribute_with_given_name_task(self.attribute.name)

    def _start_matching_button_clicked(self):
        self.document_base_viewer.main_window.match_attribute_task(self.attribute.name)

    def _remove_button_clicked(self):
        self.document_base_viewer.main_window.remove_attribute_with_given_name_task(self.attribute.name)

class DocumentBaseCreatorWidget(MainWindowContent):
    def __init__(self, main_window) -> None:
        super(DocumentBaseCreatorWidget, self).__init__(main_window, "Create Document Base")

        self.documents = MainWindowContentSection(self, "Documents:")
        self.layout.addWidget(self.documents)

        self.documents_explanation = QLabel(
            "Enter the path of the directory that contains the documents as .txt files."
        )
        self.documents_explanation.setFont(LABEL_FONT)
        self.documents.layout.addWidget(self.documents_explanation)

        self.path_widget = QFrame()
        self.path_layout = QHBoxLayout(self.path_widget)
        self.path_layout.setContentsMargins(20, 0, 20, 0)
        self.path_layout.setSpacing(10)
        self.path_widget.setObjectName("pathWidget")
        self.path_widget.setStyleSheet("QFrame#pathWidget { background-color: white }")
        self.path_widget.setFixedHeight(40)
        self.documents.layout.addWidget(self.path_widget)

        self.path = QLineEdit()
        self.path.setFont(CODE_FONT_BOLD)
        self.path.setStyleSheet("border: none")
        self.path_layout.addWidget(self.path)

        self.edit_path_button = QPushButton()
        self.edit_path_button.setIcon(QIcon("wannadb_ui/resources/folder.svg"))
        self.edit_path_button.clicked.connect(self._edit_path_button_clicked)
        self.path_layout.addWidget(self.edit_path_button)

        self.attributes = MainWindowContentSection(self, "Attributes:")
        self.layout.addWidget(self.attributes)

        self.labels_explanation = QLabel("Enter the attribute names.")
        self.labels_explanation.setFont(LABEL_FONT)
        self.attributes.layout.addWidget(self.labels_explanation)

        self.create_attribute_button = QPushButton("New Attribute")
        self.create_attribute_button.setFont(BUTTON_FONT)
        self.create_attribute_button.clicked.connect(self._create_attribute_button_clicked)

        self.attribute_names = []
        self.attributes_list = CustomScrollableList(self, AttributeCreatorWidget, self.create_attribute_button)
        self.attributes.layout.addWidget(self.attributes_list)

        self.buttons_widget = QWidget()
        self.buttons_layout = QHBoxLayout(self.buttons_widget)
        self.buttons_layout.setContentsMargins(0, 0, 0, 0)
        self.buttons_layout.setSpacing(10)
        self.buttons_layout.setAlignment(Qt.AlignmentFlag.AlignRight)
        self.attributes.layout.addWidget(self.buttons_widget)

        self.cancel_button = QPushButton("Cancel")
        self.cancel_button.setFont(BUTTON_FONT)
        self.cancel_button.clicked.connect(self._cancel_button_clicked)
        self.buttons_layout.addWidget(self.cancel_button)

        self.create_document_base_button = QPushButton("Create Document Base")
        self.create_document_base_button.setFont(BUTTON_FONT)
        self.create_document_base_button.clicked.connect(self._create_document_base_button_clicked)
        self.buttons_layout.addWidget(self.create_document_base_button)

    def enable_input(self):
        self.edit_path_button.setEnabled(True)
        self.create_attribute_button.setEnabled(True)
        self.cancel_button.setEnabled(True)
        self.create_document_base_button.setEnabled(True)
        self.attributes_list.enable_input()

    def disable_input(self):
        self.edit_path_button.setDisabled(True)
        self.create_attribute_button.setDisabled(True)
        self.cancel_button.setDisabled(True)
        self.create_document_base_button.setDisabled(True)
        self.attributes_list.disable_input()

    def initialize_for_new_document_base(self):
        self.path.setText("")
        self.path.setFocus()
        self.attribute_names = []
        self.attributes_list.update_item_list([])

    def delete_attribute(self, attribute_name):
        self.attribute_names = []
        for attribute_widget in self.attributes_list.item_widgets[:self.attributes_list.num_visible_item_widgets]:
            self.attribute_names.append(attribute_widget.name.text())
        self.attribute_names.remove(attribute_name)
        self.attributes_list.update_item_list(self.attribute_names)
        self.attributes_list.last_item_widget().name.setFocus()

    def _edit_path_button_clicked(self):
        path = str(QFileDialog.getExistingDirectory(self, "Choose a directory of text files."))
        if path != "":
            path = f"{path}/*.txt"
            self.path.setText(path)

    def _create_attribute_button_clicked(self):
        self.attribute_names = []
        for attribute_widget in self.attributes_list.item_widgets[:self.attributes_list.num_visible_item_widgets]:
            self.attribute_names.append(attribute_widget.name.text())
        self.attribute_names.append("")
        self.attributes_list.update_item_list(self.attribute_names)
        self.attributes_list.last_item_widget().name.setFocus()

    def _cancel_button_clicked(self):
        self.main_window.to_start_state()

    def _create_document_base_button_clicked(self):
        self.attribute_names = []
        for attribute_widget in self.attributes_list.item_widgets[:self.attributes_list.num_visible_item_widgets]:
            self.attribute_names.append(attribute_widget.name.text())
        self.main_window.create_document_base_task(self.path.text(), self.attribute_names)

class AttributeCreatorWidget(CustomScrollableListItem):
    def __init__(self, document_base_creator_widget) -> None:
        super(AttributeCreatorWidget, self).__init__(document_base_creator_widget)
        self.document_base_creator_widget = document_base_creator_widget

        self.setFixedHeight(40)
        self.setObjectName("attributeCreatorWidget")
        self.setStyleSheet("QWidget#attributeCreatorWidget { background-color: white}")

        self.layout = QHBoxLayout(self)
        self.layout.setContentsMargins(20, 0, 20, 0)
        self.layout.setSpacing(10)

        self.name = QLineEdit()
        self.name.setFont(CODE_FONT_BOLD)
        self.name.setStyleSheet("border: none")
        self.layout.addWidget(self.name)

        self.delete_button = QPushButton()
        self.delete_button.setIcon(QIcon("wannadb_ui/resources/trash.svg"))
        self.delete_button.clicked.connect(self._delete_button_clicked)
        self.layout.addWidget(self.delete_button)

    def update_item(self, item, params=None):
        self.name.setText(item)

    def _delete_button_clicked(self):
        self.document_base_creator_widget.delete_attribute(self.name.text())

    def enable_input(self):
        self.delete_button.setEnabled(True)

    def disable_input(self):
        self.delete_button.setEnabled(False)

class DocumentBaseCreatingWidget(MainWindowContent):
    def __init__(self, main_window) -> None:
        super(DocumentBaseCreatingWidget, self).__init__(main_window, "Creating the Document Base")

        self.steps = MainWindowContentSection(self, "Please wait while WannaDB prepares the document base.")
        self.layout.addWidget(self.steps)

    def enable_input(self):
        pass

    def disable_input(self):
        pass


import glob
from importlib import import_module
from os.path import dirname, basename, isfile, join

class ModuleLoader:
    def __init__(self, file, namespaceGlobals):
        self.file = file
        self.globals = namespaceGlobals
        self.namespace = None
        self.all = []
        self.__loadDirectory()

    ##
    # returns array of module names
    # found within the current module directory.
    def Modules(self):
        if len(self.all) <= 0:
            self.__loadModules()
        return self.all

    ##
    # loads classes from all of the current directory modules
    # set global variabes for the passed in file for the loaded classes
    # and returns an array of class names.
    # this only loads and sets classes where the class name is the same as the module name.
    def Classes(self):
        if len(self.all) <= 0:
            self.__loadClasses()
        return self.all

    ##
    # returns the app namespace for the current module directory.
    def getAppModuleNamespace(self):
        if self.namespace is None:
            index = self.directory.find('/app/')
            if index >= 0:
                self.namespace = self.directory[index+1::].replace('/','.')
        return self.namespace

    ##
    # set the member variables:
    #  directory, files, modules
    def __loadDirectory(self):
        self.directory = dirname(self.file)
        self.files = glob.glob(join(self.directory, "*.py"))
        self.modules = [ basename(f)[:-3] for f in self.files if isfile(f) and not f.endswith('__init__.py')]

    ##
    # sets the values intended for the __all__ container
    # with all module names found  within the current directory
    def __loadModules(self):
        self.all = [ m for m in self.modules ]

    ##
    # sets the values intended for the __all__ container
    # with all module-class names found  within the current directory
    def __loadClasses(self):
        namespace = self.getAppModuleNamespace()
        for className in self.modules:
            namespaceModule = f"{namespace}.{className}"
            module = import_module(namespaceModule)

            middleware = getattr(module,className)

            self.globals[className] = middleware
            self.all.extend([className])


"""
File Tasks
"""

from __future__ import division, print_function, absolute_import, unicode_literals
import re
import os
import time
import logging
import magic
from zg.util import config, url_valid, md5file, sha512file, to_bytes
from .helpers import TaskMapper
from zg.app import celery
from zg.task import image

logger = logging.getLogger('zg.task.file')

# map tasks by mimetype:
file_mapper = TaskMapper()
file_mapper.by_mimetype('image/*')(image.discover)

def gen_checksum(filename):
    return md5file(filename)

def gen_cryptohash(filename):
    return sha512file(filename)

def validate_file(tempfile):
    logger.debug('validate file: %s', tempfile)
    if not os.path.isfile(tempfile):
        logger.error('tempfile validation failure: no tempfile')
        return (None, None)

    max_size = to_bytes(config('upload.max_size'))
    filesize = os.path.getsize(tempfile)
    if filesize > max_size:
        logger.error('tempfile validation failure: tempfile is too large!')
        return (None, None)

    ma = magic.Magic(mime=True)
    mimetype = ma.from_file(tempfile)
    if mimetype not in config('upload.accept_mimetypes'):
        logger.error('tempfile validation failure: mimetype not allowed!')
        return (None, None)

    return (filesize, mimetype)

@celery.task
def discover(item_id, tempfile, **data):
    """Discover a local file."""
    logger.info('discover(item: {} tempfile: {} data: {})'.format(item_id, tempfile, data))
    url = data.pop('url', None)
    upload = data.pop('upload', None)
    try:
        filesize, mimetype = validate_file(tempfile)
        if not filesize: raise Exception('tempfile validation failure')
        task = file_mapper.get_task_for_mimetype(mimetype)
        if not task: raise Exception('no applicable task found!')
        task.delay(item_id, tempfile, mimetype=mimetype, url=url, upload=upload)
    except Exception as e:
        logger.error('file discover failure: ' + str(e))
        logger.error(e)
        os.remove(tempfile)
        logger.debug('temporary file deleted: ' + tempfile)


from collections import defaultdict
vertex = int(input())
Hash = defaultdict(list)
matrix = []
for _ in range(vertex):
  row = list(map(int ,input().split()))
  matrix.append(row)

for i in range(len(matrix)):
  for j in range(len(matrix[0])):
    if Hash[i+1]:
        Hash[i+1][0] +=matrix[i][j]
    else:
       Hash[i+1]=[matrix[i][j],0]
    if Hash[j+1]:
        Hash[j+1][1] +=matrix[i][j]
    else:
       Hash[j+1]=[0 ,matrix[i][j]]
source = [0]
sink = [0]
for key in Hash:
    if Hash[key][0] == 0:
      sink[0] += 1
      sink.append(key)
    if Hash[key][1] == 0:
       source[0] += 1
       source.append(key)
source = source[:1] + sorted(source[1:])
sink = sink[:1] + sorted(sink[1:])
print(*source)
print(*sink)


import numpy as np
import tensorflow as tf
from tqdm import tqdm
from argparse import ArgumentParser
from rand_agent import RandAgent
from dnn_agent import DNNAgent
from dlru_agent import DLRUAgent
from env import Env

def main(args):
    env = Env(normalisation=args.normalisation)

    if args.agent_type == 'rand':
        agent = RandAgent(nb_actions=4)
    elif args.agent_type == 'dlru':
        agent = DLRUAgent(nb_actions=4, env=Env(normalisation=args.normalisation))
    elif args.agent_type == 'dnn':
        agent = DNNAgent(nb_actions=4, dnn_name=args.dnn_name)
        agent.policy = tf.keras.models.load_model('example/model.hdf5')

    scores = []
    for idx in tqdm(range(args.nb_episodes), desc='Episodes'):
        step = 0
        state_np = env.reset()
        done = False
        while not done:
            action_np = agent.choose_actions(np.expand_dims(state_np, axis=0))[0]
            new_state_np, reward_np, done, info = env.step(action_np)
            score_np = env.score * args.normalisation
            agent.store_transition(state_np, action_np, reward_np, score_np)
            state_np = new_state_np
            if env.invalid_moves_cnt >= args.max_invalid_moves:
                print("Max invalid mover reached")
                done = True
            step += 1
        scores.append(score_np)

        agent.terminal_state = state_np
        agent.dump_trajectory(args.normalisation, idx)

        agent.reset_memory()

    print("Avg score: {}, std: {}".format(np.mean(scores), np.std(scores)))

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('--agent_type', type=str, default='dnn')
    parser.add_argument('--nb_episodes', type=int, default=1)
    parser.add_argument('--max_invalid_moves', type=int, default=2000)
    parser.add_argument('--dnn_name', type=str, default='dnn5')
    parser.add_argument('--normalisation', type=float, default=1024.0)
    args = parser.parse_args()
    main(args)


"""Allows rendering the content of the scene in the coco file format."""

import datetime
from itertools import groupby
import json
import os
from typing import Optional, Dict, Union, Tuple, List

import numpy as np
from skimage import measure
import cv2
import bpy

from blenderproc.python.utility.LabelIdMapping import LabelIdMapping

def write_coco_annotations(output_dir: str, instance_segmaps: List[np.ndarray],
                           instance_attribute_maps: List[dict],
                           colors: List[np.ndarray], color_file_format: str = "PNG",
                           mask_encoding_format: str = "rle", supercategory: str = "coco_annotations",
                           append_to_existing_output: bool = True,
                           jpg_quality: int = 95, label_mapping: Optional[LabelIdMapping] = None,
                           file_prefix: str = "", indent: Optional[Union[int, str]] = None):
    """ Writes coco annotations in the following steps:
    1. Locate the seg images
    2. Locate the rgb maps
    3. Locate the seg mappings
    4. Read color mappings
    5. For each frame write the coco annotation

    :param output_dir: Output directory to write the coco annotations
    :param instance_segmaps: List of instance segmentation maps
    :param instance_attribute_maps: per-frame mappings with idx, class and optionally supercategory/bop_dataset_name
    :param colors: List of color images. Does not support stereo images, enter left and right inputs subsequently.
    :param color_file_format: Format to save color images in
    :param mask_encoding_format: Encoding format of the binary masks. Default: 'rle'. Available: 'rle', 'polygon'.
    :param supercategory: name of the dataset/supercategory to filter for, e.g. a specific BOP dataset set
                          by 'bop_dataset_name' or any loaded object with specified 'cp_supercategory'
    :param append_to_existing_output: If true and if there is already a coco_annotations.json file in the output
                                      directory, the new coco annotations will be appended to the existing file.
                                      Also, the rgb images will be named such that there are no collisions.
    :param jpg_quality: The desired quality level of the jpg encoding
    :param label_mapping: The label mapping which should be used to label the categories based on their ids.
                          If None, is given then the `name` field in the csv files is used or - if not existing -
                          the category id itself is used.
    :param file_prefix: Optional prefix for image file names
    :param indent: If indent is a non-negative integer or string, then the annotation output
                   will be pretty-printed with that indent level. An indent level of 0, negative, or "" will
                   only insert newlines. None (the default) selects the most compact representation.
                   Using a positive integer indent indents that many spaces per level.
                   If indent is a string (such as "\t"), that string is used to indent each level.
    """

    if len(colors) > 0 and len(colors[0].shape) == 4:
        raise ValueError("BlenderProc currently does not support writing coco annotations for stereo images. "
                         "However, you can enter left and right images / segmaps separately.")

    # Create output directory
    os.makedirs(os.path.join(output_dir, 'images'), exist_ok=True)

    coco_annotations_path = os.path.join(output_dir, "coco_annotations.json")
    # Calculate image numbering offset, if append_to_existing_output is activated and coco data exists
    if append_to_existing_output and os.path.exists(coco_annotations_path):
        with open(coco_annotations_path, 'r', encoding="utf-8") as fp:
            existing_coco_annotations = json.load(fp)
        image_offset = max(image["id"] for image in existing_coco_annotations["images"]) + 1
    else:
        image_offset = 0
        existing_coco_annotations = None

    # collect all RGB paths
    new_coco_image_paths = []

    # for each rendered frame
    for frame in range(bpy.context.scene.frame_start, bpy.context.scene.frame_end):
        color_rgb = colors[frame - bpy.context.scene.frame_start]

        # Reverse channel order for opencv
        color_bgr = color_rgb.copy()
        color_bgr[..., :3] = color_bgr[..., :3][..., ::-1]

        if color_file_format == 'PNG':
            target_base_path = f'images/{file_prefix}{frame + image_offset:06d}.png'
            target_path = os.path.join(output_dir, target_base_path)
            cv2.imwrite(target_path, color_bgr)
        elif color_file_format == 'JPEG':
            target_base_path = f'images/{file_prefix}{frame + image_offset:06d}.jpg'
            target_path = os.path.join(output_dir, target_base_path)
            cv2.imwrite(target_path, color_bgr, [int(cv2.IMWRITE_JPEG_QUALITY), jpg_quality])
        else:
            raise RuntimeError(f'Unknown color_file_format={color_file_format}. Try "PNG" or "JPEG"')

        new_coco_image_paths.append(target_base_path)

    coco_output = _CocoWriterUtility.generate_coco_annotations(instance_segmaps,
                                                               instance_attribute_maps,
                                                               new_coco_image_paths,
                                                               supercategory,
                                                               mask_encoding_format,
                                                               existing_coco_annotations,
                                                               label_mapping)

    print("Writing coco annotations to " + coco_annotations_path)
    with open(coco_annotations_path, 'w', encoding="utf-8") as fp:
        json.dump(coco_output, fp, indent=indent)

def binary_mask_to_rle(binary_mask: np.ndarray) -> Dict[str, List[int]]:
    """Converts a binary mask to COCOs run-length encoding (RLE) format. Instead of outputting
    a mask image, you give a list of start pixels and how many pixels after each of those
    starts are included in the mask.
    :param binary_mask: a 2D binary numpy array where '1's represent the object
    :return: Mask in RLE format
    """
    rle: Dict[str, List[int]] = {'counts': [], 'size': list(binary_mask.shape)}
    counts = rle.get('counts')
    for i, (value, elements) in enumerate(groupby(binary_mask.ravel(order='F'))):
        if i == 0 and value == 1:
            counts.append(0)
        counts.append(len(list(elements)))
    return rle

def rle_to_binary_mask(rle: Dict[str, List[int]]) -> np.ndarray:
    """Converts a COCOs run-length encoding (RLE) to binary mask.
    :param rle: Mask in RLE format
    :return: a 2D binary numpy array where '1's represent the object
    """
    binary_array = np.zeros(np.prod(rle.get('size')), dtype=bool)
    counts: List[int] = rle.get('counts')

    start = 0
    for i in range(len(counts) - 1):
        start += counts[i]
        end = start + counts[i + 1]
        binary_array[start:end] = (i + 1) % 2

    binary_mask = binary_array.reshape(*rle.get('size'), order='F')

    return binary_mask

class _CocoWriterUtility:

    @staticmethod
    def generate_coco_annotations(inst_segmaps, inst_attribute_maps, image_paths, supercategory,
                                  mask_encoding_format, existing_coco_annotations=None,
                                  label_mapping: LabelIdMapping = None):
        """Generates coco annotations for images

        :param inst_segmaps: List of instance segmentation maps
        :param inst_attribute_maps: per-frame mappings with idx, class and optionally supercategory/bop_dataset_name
        :param image_paths: A list of paths which points to the rendered images.
        :param supercategory: name of the dataset/supercategory to filter for, e.g. a specific BOP dataset
        :param mask_encoding_format: Encoding format of the binary mask. Type: string.
        :param existing_coco_annotations: If given, the new coco annotations will be appended to the given
                                          coco annotations dict.
        :param label_mapping: The label mapping which should be used to label the categories based on their ids.
                              If None, is given then the `name` field in the csv files is used or - if not existing -
                              the category id itself is used.
        :return: dict containing coco annotations
        """

        categories = []
        visited_categories = []
        instance_2_category_maps = []

        for inst_attribute_map in inst_attribute_maps:
            instance_2_category_map = {}
            for inst in inst_attribute_map:
                # skip background
                if int(inst["category_id"]) != 0:
                    # take all objects or objects from specified supercategory is defined
                    inst_supercategory = "coco_annotations"
                    if "bop_dataset_name" in inst:
                        inst_supercategory = inst["bop_dataset_name"]
                    elif "supercategory" in inst:
                        inst_supercategory = inst["supercategory"]

                    if supercategory in [inst_supercategory, 'coco_annotations']:
                        if int(inst["category_id"]) not in visited_categories:
                            cat_dict: Dict[str, Union[str, int]] = {'id': int(inst["category_id"]),
                                                                    'supercategory': inst_supercategory}
                            # Determine name of category based on label_mapping, name or category_id
                            if label_mapping is not None:
                                cat_dict["name"] = label_mapping.label_from_id(cat_dict['id'])
                            elif "name" in inst:
                                cat_dict["name"] = inst["name"]
                            else:
                                cat_dict["name"] = inst["category_id"]

                            categories.append(cat_dict)
                            visited_categories.append(cat_dict['id'])
                        instance_2_category_map[int(inst["idx"])] = int(inst["category_id"])
            instance_2_category_maps.append(instance_2_category_map)

        licenses = [{
            "id": 1,
            "name": "Attribution-NonCommercial-ShareAlike License",
            "url": "http://creativecommons.org/licenses/by-nc-sa/2.0/"
        }]
        info = {
            "description": supercategory,
            "url": "https://github.com/waspinator/pycococreator",
            "version": "0.1.0",
            "year": 2020,
            "contributor": "Unknown",
            "date_created": datetime.datetime.utcnow().isoformat(' ')
        }

        images: List[Dict[str, Union[str, int]]] = []
        annotations: List[Dict[str, Union[str, int]]] = []

        for inst_segmap, image_path, instance_2_category_map in zip(inst_segmaps, image_paths,
                                                                    instance_2_category_maps):

            # Add coco info for image
            image_id = len(images)
            images.append(_CocoWriterUtility.create_image_info(image_id, image_path, inst_segmap.shape))

            # Go through all objects visible in this image
            instances = np.unique(inst_segmap)
            # Remove background
            instances = np.delete(instances, np.where(instances == 0))
            for inst in instances:
                if inst in instance_2_category_map:
                    # Calc object mask
                    binary_inst_mask = np.where(inst_segmap == inst, 1, 0)
                    # Add coco info for object in this image
                    annotation = _CocoWriterUtility.create_annotation_info(len(annotations) + 1,
                                                                           image_id,
                                                                           instance_2_category_map[inst],
                                                                           binary_inst_mask,
                                                                           mask_encoding_format)
                    if annotation is not None:
                        annotations.append(annotation)

        new_coco_annotations = {
            "info": info,
            "licenses": licenses,
            "categories": categories,
            "images": images,
            "annotations": annotations
        }

        if existing_coco_annotations is not None:
            new_coco_annotations = _CocoWriterUtility.merge_coco_annotations(existing_coco_annotations,
                                                                             new_coco_annotations)

        return new_coco_annotations

    @staticmethod
    def merge_coco_annotations(existing_coco_annotations, new_coco_annotations):
        """ Merges the two given coco annotation dicts into one.

        Currently, this requires both coco annotations to have the exact same categories/objects.
        The "images" and "annotations" sections are concatenated and respective ids are adjusted.

        :param existing_coco_annotations: A dict describing the first coco annotations.
        :param new_coco_annotations: A dict describing the second coco annotations.
        :return: A dict containing the merged coco annotations.
        """

        # Concatenate category sections
        for cat_dict in new_coco_annotations["categories"]:
            if cat_dict not in existing_coco_annotations["categories"]:
                existing_coco_annotations["categories"].append(cat_dict)

        # Concatenate images sections
        image_id_offset = max(image["id"] for image in existing_coco_annotations["images"]) + 1
        for image in new_coco_annotations["images"]:
            image["id"] += image_id_offset
        existing_coco_annotations["images"].extend(new_coco_annotations["images"])

        # Concatenate annotations sections
        if len(existing_coco_annotations["annotations"]) > 0:
            annotation_id_offset = max(annotation["id"] for annotation in existing_coco_annotations["annotations"]) + 1
        else:
            annotation_id_offset = 0
        for annotation in new_coco_annotations["annotations"]:
            annotation["id"] += annotation_id_offset
            annotation["image_id"] += image_id_offset
        existing_coco_annotations["annotations"].extend(new_coco_annotations["annotations"])

        return existing_coco_annotations

    @staticmethod
    def create_image_info(image_id: int, file_name: str, image_size: Tuple[int, int]) -> Dict[str, Union[str, int]]:
        """Creates image info section of coco annotation

        :param image_id: integer to uniquly identify image
        :param file_name: filename for image
        :param image_size: The size of the image, given as [W, H]
        """
        image_info: Dict[str, Union[str, int]] = {
            "id": image_id,
            "file_name": file_name,
            "width": image_size[1],
            "height": image_size[0],
            "date_captured": datetime.datetime.utcnow().isoformat(' '),
            "license": 1,
            "coco_url": "",
            "flickr_url": ""
        }

        return image_info

    @staticmethod
    def create_annotation_info(annotation_id: int, image_id: int, category_id: int, binary_mask: np.ndarray,
                               mask_encoding_format: str, tolerance: int = 2) -> Optional[Dict[str, Union[str, int]]]:
        """Creates info section of coco annotation

        :param annotation_id: integer to uniquly identify the annotation
        :param image_id: integer to uniquly identify image
        :param category_id: Id of the category
        :param binary_mask: A binary image mask of the object with the shape [H, W].
        :param mask_encoding_format: Encoding format of the mask. Type: string.
        :param tolerance: The tolerance for fitting polygons to the objects mask.
        """

        area = _CocoWriterUtility.calc_binary_mask_area(binary_mask)
        if area < 1:
            return None

        bounding_box = _CocoWriterUtility.bbox_from_binary_mask(binary_mask)

        if mask_encoding_format == 'rle':
            segmentation = binary_mask_to_rle(binary_mask)
        elif mask_encoding_format == 'polygon':
            segmentation = _CocoWriterUtility.binary_mask_to_polygon(binary_mask, tolerance)
            if not segmentation:
                return None
        else:
            raise RuntimeError(f"Unknown encoding format: {mask_encoding_format}")

        annotation_info: Dict[str, Union[str, int]] = {
            "id": annotation_id,
            "image_id": image_id,
            "category_id": category_id,
            "iscrowd": 0,
            "area": area,
            "bbox": bounding_box,
            "segmentation": segmentation,
            "width": binary_mask.shape[1],
            "height": binary_mask.shape[0],
        }
        return annotation_info

    @staticmethod
    def bbox_from_binary_mask(binary_mask: np.ndarray) -> List[int]:
        """ Returns the smallest bounding box containing all pixels marked "1" in the given image mask.

        :param binary_mask: A binary image mask with the shape [H, W].
        :return: The bounding box represented as [x, y, width, height]
        """
        # Find all columns and rows that contain 1s
        rows = np.any(binary_mask, axis=1)
        cols = np.any(binary_mask, axis=0)
        # Find the min and max col/row index that contain 1s
        rmin, rmax = np.where(rows)[0][[0, -1]]
        cmin, cmax = np.where(cols)[0][[0, -1]]
        # Calc height and width
        h = rmax - rmin + 1
        w = cmax - cmin + 1
        return [int(cmin), int(rmin), int(w), int(h)]

    @staticmethod
    def calc_binary_mask_area(binary_mask: np.ndarray) -> int:
        """ Returns the area of the given binary mask which is defined as the number of 1s in the mask.

        :param binary_mask: A binary image mask with the shape [H, W].
        :return: The computed area
        """
        return binary_mask.sum().tolist()

    @staticmethod
    def close_contour(contour: np.ndarray) -> np.ndarray:
        """ Makes sure the given contour is closed.

        :param contour: The contour to close.
        :return: The closed contour.
        """
        # If first != last point => add first point to end of contour to close it
        if not np.array_equal(contour[0], contour[-1]):
            contour = np.vstack((contour, contour[0]))
        return contour

    @staticmethod
    def binary_mask_to_polygon(binary_mask: np.ndarray, tolerance: int = 0) -> List[np.ndarray]:
        """Converts a binary mask to COCO polygon representation

         :param binary_mask: a 2D binary numpy array where '1's represent the object
         :param tolerance: Maximum distance from original points of polygon to approximated polygonal chain. If
                           tolerance is 0, the original coordinate array is returned.
        """
        polygons = []
        # pad mask to close contours of shapes which start and end at an edge
        padded_binary_mask = np.pad(binary_mask, pad_width=1, mode='constant', constant_values=0)
        contours = np.array(measure.find_contours(padded_binary_mask, 0.5))
        # Reverse padding
        contours -= 1
        for contour in contours:
            # Make sure contour is closed
            contour = _CocoWriterUtility.close_contour(contour)
            # Approximate contour by polygon
            polygon = measure.approximate_polygon(contour, tolerance)
            # Skip invalid polygons
            if len(polygon) < 3:
                continue
            # Flip xy to yx point representation
            polygon = np.flip(polygon, axis=1)
            # Flatten
            polygon = polygon.ravel()
            # after padding and subtracting 1 we may get -0.5 points in our segmentation
            polygon[polygon < 0] = 0
            polygons.append(polygon.tolist())

        return polygons


#!/usr/bin/python3

import os
import sys

"""
Offline analysis by replaying logs
"""

# Import MobileInsight modules
from mobile_insight.monitor import OfflineReplayer
from mobile_insight.analyzer import MsgLogger, LteRrcAnalyzer, WcdmaRrcAnalyzer, LteNasAnalyzer, UmtsNasAnalyzer, LteMacAnalyzer, LteMeasurementAnalyzer

# Initialize a monitor
src = OfflineReplayer()
src.set_input_path("./logs/")
# src.enable_log_all()

src.enable_log("LTE_PHY_Serv_Cell_Measurement")
src.enable_log("5G_NR_RRC_OTA_Packet")
src.enable_log("LTE_RRC_OTA_Packet")
src.enable_log("LTE_NB1_ML1_GM_DCI_Info")

logger = MsgLogger()
logger.set_decode_format(MsgLogger.JSON)
logger.set_dump_type(MsgLogger.FILE_ONLY)
logger.save_decoded_msg_as("./mobileinsight.log")
logger.set_source(src)

# # Analyzers
# nr_rrc_analyzer = NrRrcAnalyzer()
# nr_rrc_analyzer.set_source(src)  # bind with the monitor

lte_rrc_analyzer = LteRrcAnalyzer()
lte_rrc_analyzer.set_source(src)  # bind with the monitor

# wcdma_rrc_analyzer = WcdmaRrcAnalyzer()
# wcdma_rrc_analyzer.set_source(src)  # bind with the monitor

# lte_nas_analyzer = LteNasAnalyzer()
# lte_nas_analyzer.set_source(src)

# umts_nas_analyzer = UmtsNasAnalyzer()
# umts_nas_analyzer.set_source(src)

# lte_mac_analyzer = LteMacAnalyzer()
# lte_mac_analyzer.set_source(src)

# lte_meas_analyzer = LteMeasurementAnalyzer()
# lte_meas_analyzer.set_source(src)

# print lte_meas_analyzer.get_rsrp_list()
# print lte_meas_analyzer.get_rsrq_list()

# Start the monitoring
src.run()

#The code above uses the mobile insight packages to decode the logs into a readable JSON-like format

#These system calls correct the formatting within the log so the python JSON decoder is able to interpret it correctly

os.system("echo \"{\" > log.json")
os.system("echo '\"log\": [' >> log.json")
os.system("sed -i 's/$/,/g' mobileinsight.log")  # add commas to the end of each line
os.system("sed -i '$ s/.$//' mobileinsight.log")  # remove the last comma
os.system("cat mobileinsight.log >> log.json")
os.system("echo \"]\" >> log.json")
os.system("echo \"}\" >> log.json")

import json
from datetime import datetime
from datetime import timedelta

with open("log.json", "r") as f:
    data = json.load(f)

log = data["log"]

def message_type_parsing(log):
    message_types = []
    for (i, entry) in enumerate(log): #iterate over each log entry
        type_id = entry['type_id']
        if type_id != 'LTE_RRC_OTA_Packet':
            message_types.append((i, type_id)) #only interested in parsing the LTE_RRC_OTA_Packets so skip the others
            continue
        cell_id = entry['Physical Cell ID']
        msg = entry['Msg']['msg']['packet']['proto'][5]['field']['field']
        if msg[1]['field'].__class__ != list: # paging messages
            message_types.append((i, type_id, cell_id, "paging"))
            continue
        msg_type = msg[1]['field'][1]['@showname']
        if msg_type == "c1: systemInformation (0)": #Handles SIB Info packets
            sib_types = msg[1]['field'][1]['field']['field'][1]['field']['field'][2]['field']
            if sib_types.__class__ != list:
                sib_types = [sib_type]
            sibs = []
            for sib_type in sib_types:
                sibs.append(sib_type['field'][2]['@showname'])
            message_types.append((i, type_id, cell_id, msg_type, sibs))
        else:
            message_types.append((i, type_id, cell_id, msg_type))
    return message_types

for line in message_type_parsing(log):
    print(line, file=open("message_types.txt", 'a'))

#The message_type_parsing function gives a high level overview of the type of information for each and every log entry
#This information is then stored in message_types.txt

rrcpackts = [entry for entry in log if entry["type_id"] == "LTE_RRC_OTA_Packet"]

releaseNsetup = []
for i, entry in enumerate(log):
    type_id = entry['type_id']
    if type_id != 'LTE_RRC_OTA_Packet':
        continue
    msg = entry["Msg"]["msg"]["packet"]["proto"][5]["field"]["field"]
    if msg[1]["field"].__class__ != list:  # paging messages
        continue
    msg_type = msg[1]["field"][1]["@showname"]
    if ( #want to extract only the connectionRelease and connectionSetupComplete log entries
        msg_type == "c1: rrcConnectionRelease (5)"
        or msg_type == "c1: rrcConnectionSetupComplete (4)"
    ):
        temp = dict(
            (k, entry[k]) for k in ["timestamp", "Physical Cell ID"] if k in entry
        )
        temp["msg_type"] = msg_type
        temp["index"] = i
        releaseNsetup.append(temp)

#releaseNsetup is a list of all the connection setup and releases and their associated timestamps

sum = 0

for i in range(len(releaseNsetup) - 1):
    entry1 = releaseNsetup[i]
    entry2 = releaseNsetup[i + 1]

    ts1 = entry1["timestamp"]
    ts2 = entry2["timestamp"]

    d1 = datetime.strptime(ts1, "%Y-%m-%d %H:%M:%S.%f")
    d2 = datetime.strptime(ts2, "%Y-%m-%d %H:%M:%S.%f")

    difference = timedelta.total_seconds(d2 - d1)

    if i > 0:
        sum += difference

for line in releaseNsetup:
    print(line, file=open("setupAndReleases.txt", 'a'))

average = sum / (len(releaseNsetup) - 2)
s = f'The average time difference between the two events is {average} seconds'

print(s, file=open("setupAndReleases.txt", 'a'))

#The contents of releaseNsetup as well as the average difference in timestamp is stored in setupAndReleases.txt