#!/usr/bin/python3

import numpy as np
from scipy.stats import beta
from scipy.stats import norm
import matplotlib.pyplot as plt
from matplotlib import rc
rc('font',**{'family':'serif','serif':['Cardo']})
rc('text', usetex=True)

def q_of_Z_pi(Z, RHO, mu_sigma2_a_b):
    q = norm.pdf(Z, mu_sigma2_a_b[0], np.sqrt(mu_sigma2_a_b[1])) * beta.pdf(RHO, mu_sigma2_a_b[2], mu_sigma2_a_b[3]);
    return q

def update_filter_vogiatzis(z, tau2, mu_range, mu_sigma2_a_b):
    mu = mu_sigma2_a_b[0]
    sigma2 = mu_sigma2_a_b[1]
    a = mu_sigma2_a_b[2]
    b = mu_sigma2_a_b[3]

    norm_scale = np.sqrt(sigma2 + tau2)

    s2 = 1.0/(1.0/sigma2 + 1.0/tau2)
    m = s2*(mu/sigma2 + z/tau2)
    uniform_x = 1.0/mu_range
    C1 = a/(a+b) * norm.pdf(z, mu, norm_scale)
    C2 = b/(a+b) * uniform_x
    normalization_constant = C1 + C2
    C1 /= normalization_constant
    C2 /= normalization_constant
    f = C1*(a+1.0)/(a+b+1.0) + C2*a/(a+b+1.0)
    e = C1*(a+1.0)*(a+2.0)/((a+b+1.0)*(a+b+2.0)) \
      + C2*a*(a+1.0)/((a+b+1.0)*(a+b+2.0));

    # update parameters
    mu_new = C1*m+C2*mu
    sigma2 = C1*(s2 + m*m) + C2*(sigma2 + mu*mu) - mu_new*mu_new
    mu = mu_new
    a = (e - f) / (f - e / f)
    b = a * (1.0 - f) / f

    mu_sigma2_a_b = np.array([mu, sigma2, a, b])
    return mu_sigma2_a_b

def update_filter_gaussian(z, tau2, mu_sigma2):
    mu = mu_sigma2[0]
    sigma2 = mu_sigma2[1]

    denom = (sigma2 + tau2)
    mu = (sigma2 * z + tau2 * mu) / denom
    sigma2 = sigma2 * tau2 / denom
    mu_sigma2 = np.array([mu, sigma2])
    return mu_sigma2

def plot_distribution_vogiatzis(ax, mu_sigma2_a_b, Z, RHO):
    Q = q_of_Z_pi(Z, RHO, mu_sigma2_a_b)
    ax.imshow(Q, aspect='auto', origin='lower')

def plot_histogram(ax, z_meas_vec, z_min, z_max):
    bins = np.linspace(z_min, z_max, 40)
    ax.hist(z_meas_vec, bins, color='blue', edgecolor='w') # none
    ax.set_xlim([z_min, z_max])

def plot_gaussian(ax, mu_sigma2, z_min, z_max):
    x = np.linspace(z_min, z_max, 1000)
    y = norm.pdf(x, mu_sigma2[0], np.sqrt(mu_sigma2[1]))
    ax.plot(x, y, lw=2.5, color='b')
    ax.set_xlim([z_min, z_max])

def experiment_1(inlier_prob = 0.7):
    # init state
    z_max = 1.0/0.5
    z_min = 0.0
    z_true = 1.0/4.0
    z_init = 1.0/1.0
    tau = 0.1
    a_b_init = 1.5
    N = 30              # num updates
    mu_range = z_max - z_min
    sigma2_init = mu_range * mu_range / 36.0
    tau2 = tau*tau
    mu_sigma2_a_b = [z_init, sigma2_init, a_b_init, a_b_init] # for vogiatzis
    mu_sigma2 = [z_init, sigma2_init]                         # for gaussian

    # prepare plotting
    depth_samples = np.linspace(z_min, z_max, 400)
    inlier_ratios = np.linspace(0, 1, 100)
    Z, RHO = np.meshgrid(depth_samples, inlier_ratios)
    axes_vec = list()
    fig1, axes = plt.subplots(3, 1, figsize=(5, 3))
    axes_vec.append(axes)
    fig2, axes = plt.subplots(3, 1, figsize=(5, 3))
    axes_vec.append(axes)
    fig3, axes = plt.subplots(3, 1, figsize=(5, 3))
    axes_vec.append(axes)

    z_meas_vec = list()

    # perform measurement updates
    for i in range(N):

        # measurement is inlier with certain probability
        t = np.random.choice(['inlier', 'outlier'], p=[inlier_prob, (1.0-inlier_prob)])
        if t == 'inlier':
            z_meas = z_true  +  np.random.normal(0.0, tau)
        elif t == 'outlier':
            z_meas = np.random.uniform(z_min, z_max)
        z_meas_vec.append(z_meas)

        mu_sigma2_a_b = update_filter_vogiatzis(z_meas, tau2, mu_range, mu_sigma2_a_b)
        mu_sigma2 = update_filter_gaussian(z_meas, tau2, mu_sigma2)
        if(i == 2):
            plot_distribution_vogiatzis(axes_vec[0][2], mu_sigma2_a_b, Z, RHO)
            plot_gaussian(axes_vec[0][1], mu_sigma2, z_min, z_max)
            plot_histogram(axes_vec[0][0], z_meas_vec, z_min, z_max)
        if(i == N/2):
            plot_distribution_vogiatzis(axes_vec[1][2], mu_sigma2_a_b, Z, RHO)
            plot_gaussian(axes_vec[1][1], mu_sigma2, z_min, z_max)
            plot_histogram(axes_vec[1][0], z_meas_vec, z_min, z_max)
        if(i == N-1) :
            plot_distribution_vogiatzis(axes_vec[2][2], mu_sigma2_a_b, Z, RHO)
            plot_gaussian(axes_vec[2][1], mu_sigma2, z_min, z_max)
            plot_histogram(axes_vec[2][0], z_meas_vec, z_min, z_max)

    def format_axes(axes, n_iter):

        for i in range(3):
            ax = axes[i]

            ax.xaxis.set_ticks_position('none') # don't display tick marks
            ax.yaxis.set_ticks_position('none')

            if i%3 == 2: # vogiatzis:
              #ax.set_title('After '+str(int(n_iter))+' measurements, inlier probability = '+str(inlier_prob))
              ax.set_ylabel(r"$\gamma$", rotation=0)
              ax.set_yticks([0,50,100])
              ax.set_yticklabels([0,0.5,1])
              ax.set_xlabel('Inverse Depth')
            if i%3 == 1: # gaussian:
              #ax.locator_params(axis='y', nbins=3)
              ax.spines['top'].set_visible(False)
              ax.spines['right'].set_visible(False)
              ax.set_yticks([])
              ax.set_xticks([])
              ax.tick_params('x',top='off')
            if i%3 == 0: # histogram
              ax.spines['top'].set_visible(False)
              ax.spines['right'].set_visible(False)
              ax.set_yticks([])
              ax.set_xticks([])

            #width = 1.0 - 2*h_space
            #height = (1.0 - v_space) / n_plots - v_space
            #ax.set_position([h_space, 1-(i+1)*(height+v_space)+v_space/2, width, height])

    format_axes(axes_vec[0], 3)
    format_axes(axes_vec[1], N/2)
    format_axes(axes_vec[2], N)

    fig1.tight_layout()
    fig1.savefig('depth_estimation_1.pdf')
    fig2.tight_layout()
    fig2.savefig('depth_estimation_2.pdf')
    fig3.tight_layout()
    fig3.savefig('depth_estimation_3.pdf')

    print('error vogiatzis = ' + str(np.abs(mu_sigma2_a_b[0]-z_true)))
    print('error gaussian = ' + str(np.abs(mu_sigma2[0]-z_true)))

experiment_1(0.7)


from server.server_utils.controller import *
from functools import partial

class Router:
    def __init__(self, data, token, transport, connections, users):
        self.data = data
        self.username = data.username if data else None
        self.token = token
        self.transport = transport
        self.connections = connections
        self.users = users

    def probe(self):
        handle_probe(self.transport)

    def login(self):
        task = asyncio.create_task(DB.get_user_by_username(self.username))
        task.add_done_callback(partial(
            handle_login,
            self.username,
            self.data.password,
            self.transport,
            self.users,
            self.connections
            )
        )

    def register(self):
        task = asyncio.create_task(DB.register_user(self.data))
        task.add_done_callback(partial(
            handle_register,
            self.transport))

    def add_contact(self):
        task = asyncio.create_task(DB.add_contact(self.data.id, self.data.contact))
        task.add_done_callback(partial(
            handle_add_contact,
            self.data.id,
            self.username,
            self.transport))

    def send_msg(self):
        msg = self.data.message
        time = self.data.send_time
        send_to = self.data.send_to
        asyncio.create_task(DB.write_msg(
            self.data.id,
            send_to,
            msg,
            time
        ))
        user_send_to = self.users.get(send_to, self.data.send_to)
        handle_send_msg(
            self.data.id,
            self.username,
            self.transport,
            user_send_to,
            msg,
            time)

    def get_contacts(self):
        task = asyncio.create_task(DB.get_contacts(self.data.id))
        task.add_done_callback(partial(handle_get_contacts, self.transport))

    def get_messages(self):
        task = asyncio.create_task(DB.get_all_messages(self.data.id, self.data.contact))
        task.add_done_callback(partial(handle_get_messages, self.transport))

    def rout(self, command):
        auth_required = ["add_contact", "send_msg", "get_contacts", "get_messages"]
        if command not in auth_required:
            getattr(self, command)()
        else:
            @is_authenticated(self.token, self.data.id, self.username, self.transport)
            def run_routing(*args, **kwargs):
                getattr(self, command)()
            run_routing()


from typing import List, Protocol

from adapters.outbound.dynamo_corporate_events_repository import DynamoCorporateEventsRepository
from application.models.earnings_in_assets_event import EarningsInAssetCorporateEvent

class CorporateEventsRepository(Protocol):
    def find_by_emitted_asset(self, emitted_isin: str) -> List[EarningsInAssetCorporateEvent]:
        ...

def get_all_previous_symbols(isin_code: str, corp_events_repository: CorporateEventsRepository) -> List[str]:
    previous_symbols = []
    events = corp_events_repository.find_by_emitted_asset(isin_code)
    for event in events:
        if event.isin_code == isin_code:
            continue
        previous_symbols.append(event.isin_code)
        previous_symbols += get_all_previous_symbols(event.isin_code, corp_events_repository)

    return previous_symbols

if __name__ == '__main__':
    print(get_all_previous_symbols("BREMBRACNOR4", DynamoCorporateEventsRepository()))


from pynput.keyboard import Listener
import logging

# Setup logging
# This will save a list of key presses and key releases as a textfile in the same directory we are in.
for handler in logging.root.handlers[:]:
    logging.root.removeHandler(handler)
logging.basicConfig(filename="filename.txt", level=logging.DEBUG, format='%(asctime)s: %(message)s')

# This logs the key press and stops when the 's' key is pressed.
def on_press(key):  # The function that's called when a key is pressed
    logging.info("Key pressed: {0}".format(key))
    if key.char == 's':
        listener.stop()

# This logs the key release (can probably just put pass so we don't log it)
def on_release(key):  # The function that's called when a key is released
    logging.info("Key released: {0}".format(key))

print("listening!!!")

with Listener(on_press=on_press, on_release=on_release) as listener:  # Create an instance of Listener
    listener.join()  # Join the listener thread to the main thread to keep waiting for keys


from pathlib import Path
from scipy.io import wavfile
import numpy as np
from scipy.fftpack import fft, ifft
from scipy.stats import norm, kurtosis, skew
import pandas as pd
import warnings
import matplotlib.pyplot as plt
import librosa
import librosa.display

## reading in multiple wav files
def read_wav(dataset_names):
    '''
    Reads multiple wav files
    :param dataset_names: filenames
    :return: samplerate and data
    '''

    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        samplerate = []
        df = []
        for data in dataset_names:
            path =  Path.cwd()/"data"
            sf, d = wavfile.read(path/data)
            samplerate.append(sf)
            df.append(d)
        return samplerate , df

### plotting time waveform

def plot_timewave(datapath):
    '''
    plot_timewave
    :param datapath: datapath
    '''

    sr, data = wavfile.read(datapath)
    length = data.shape[0] / sr
    time = np.linspace(0., length, data.shape[0])
    plt.plot(time, data)

    plt.title("Raw Audio")

    plt.xlabel("Time [s]")
    plt.ylabel("Amplitude")

    plt.show()

##spectro librosa

def spectro(data_path):
    '''
    Plots spectrogram
    :param data_path: data_path

    '''

    x, sr = librosa.load(data_path)
    X = librosa.stft(x)
    Xdb = librosa.amplitude_to_db(abs(X))
    plt.figure(figsize=(14, 5))
    librosa.display.specshow(Xdb, sr=sr, x_axis='time', y_axis='hz')
    plt.colorbar()

## windowing samples
def splitting (sr, df, sec):

    '''
    Window sizing of audio files
    :param sr: sample rate
    :param df: data
    :param sec: window size
    :return: splitted data
    '''

    seg_len = int(sr * sec)

    splits = []

    for data in df:
        #get number of sections
        sections = int(np.ceil(len(data) / seg_len))
        for i in range(sections):
            #slice section range
            t = data[i * seg_len: (i + 1) * seg_len]
            splits.append(t)

    return splits

## fourier transform multiple samples
def fourier_trans(sr, data):
    '''
    fourier transform multiple samples
    :param sr: sample rate
    :param df: data
    :return: frequency and transformed data
    '''

    df_ftt = []
    freq = []

    df = pd.DataFrame()
    x = 0
    for i in data:

        length = i.shape[0] / sr
        N = i.shape[0]
        n = np.arange(N)
        freq.append(n/length)
        df_i = pd.DataFrame(fft(i).real, columns = [x])
        x += 1
        df = pd.concat([df,df_i], axis = 1)

    return freq, df

def plot_whole_freq(frequency, spectrum):

    plt.figure(figsize = (12, 6))
    plt.subplot(121)

    plt.stem(frequency[1], np.abs(spectrum), 'b', \
             markerfmt=" ", basefmt="-b")
    plt.xlabel('Freq (Hz)')
    plt.ylabel('FFT Amplitude |X(freq)|')

    plt.show()

def plot_half_freq(frequency, spectrum):

    X = np.fft.fft(spectrum)
    X_mag = np.abs(X)
    f = np.linspace(0, sr[0], len(X_mag))
    half = int(len(X_mag)/2)
    fft_fre = np.fft.rfftfreq(len(X_mag), d=1./frequency)

    abs_spec = abs(X_mag[:half+1])
    plt.figure(figsize=(5, 5))
    plt.plot(fft_fre, abs_spec) # magnitude spectrum
    plt.xlabel('Frequency (Hz)')
    plt.show()

## statistical features

def feature_extraction(df, label, sr):
    '''
    returns features of audio
    :param df: data
    :param label: class label
    :return: features
    '''

    column_name = ["mean", "median", "quartile_25", "quartile_75", "Max", "Min", "quartile", "std", "rms", "sra", "ff", "clf", "cf", "kurtosis", "skew"]
    df_features = pd.DataFrame(columns = column_name) #

    for column in df:
        feature_list = []

           # central trend statistics
        data_mean = np.mean(df[column])
        data_median = np.median(df[column])
        data_quartile_025 = np.quantile(df[column], 0.25)
        data_quartile_075 = np.quantile(df[column], 0.75)

           # dispersion degree statistics
        data_Minimum = np.min(df[column])
        data_Maximum = np.max(df[column])
        data_quartile = data_quartile_075 - data_quartile_025
        data_std = np.std(df[column])
        data_rms = np.sqrt((np.mean(df[column]**2)))
        data_sra = (np.sum(np.sqrt(np.abs(df[column])))/len(df[column]))**2

           # distribution shape statistics
        data_kurtosis = kurtosis(df[column])
        data_skew = skew(df[column])

        data_avg = np.mean(np.abs(df[column]))
        data_ff = data_rms / data_avg

        data_clf = np.max(np.abs(df[column])) / data_sra
        data_cf = np.max(np.abs(df[column])) / data_rms

        # Mel Frequency Cepstral Coefficients (MFCC)
        # mel = librosa.feature.mfcc(y = np.array(df[column]))#, sr = sr, n_mels=128
        # mel_mean = np.mean(mel)
        # mel_var = np.var(mel)

        feature_list = [data_mean, data_median, data_quartile_025, data_quartile_075, data_Maximum, data_Minimum,
                        data_quartile, data_std, data_rms, data_sra, data_ff, data_clf, data_cf , data_kurtosis,
                        data_skew

                        ]
        feature_list = pd.DataFrame(data=feature_list).T #,
        feature_list.columns = column_name

        df_features = pd.concat([df_features,feature_list])

    if label == "Cavitation":
        df_features["Cavitation"] = 1

    if label == "No Cavitation":
        df_features["Cavitation"] = 0

    return df_features

def feature_pipe (path, splits_in_sec):
    '''
    Pipeline for feature transformation
    :param path: data path
    :param splits_in_sec: window size
    :return:  lenght of data , data
    '''

    target_path = Path.cwd()/path
    print(f"Extracting features from {target_path}")
    if not (target_path/"cavitation").is_dir():
        print("Folder named cavitation does not exist!")
        return
    if not (target_path/"no_cavitation").is_dir():
        print("Folder named no_cavitationn does not exist!")
        return
    no_cav_f_names = list((target_path/"no_cavitation").glob('*.wav'))
    cav_f_names = list((target_path/"cavitation").glob('*.wav'))

    sr_cav, df_cav = read_wav(cav_f_names)
    df_split_cav = splitting(sr_cav[0],df_cav, splits_in_sec)
    freq_cav, df_ftt_cav = fourier_trans(sr_cav[0],df_split_cav)
    output_df_cav = feature_extraction(df_ftt_cav, "Cavitation", sr = sr_cav[0]) #cavitation = 1

    sr_no_cav, df_no_cav = read_wav(no_cav_f_names)
    df_split_no_cav = splitting(sr_no_cav[0], df_no_cav, splits_in_sec)
    freq_no_cav, df_ftt_no_cav = fourier_trans(sr_no_cav[0], df_split_no_cav)
    output_df_no_cav = feature_extraction(df_ftt_no_cav, "No Cavitation", sr = sr_no_cav[0]) #no_cavitation = 0

    df = pd.concat([output_df_cav, output_df_no_cav])
    print(df)
    return len(df_split_cav)+len(df_split_no_cav), df

import splitfolders
import shutil
import os
def create_split():

    splitfolders.ratio(Path().resolve()/"data/Training_all/oil/", output=Path().resolve()/"data/oil_split/", seed=1623806, ratio=(.8,0.2))

    splitfolders.ratio(Path().resolve()/"data/Training_all/teflon", output=Path().resolve()/"data/teflon_split/", seed=1623806, ratio=(.8,0.2))
    splitfolders.ratio(Path().resolve()/"data/Training_all/vacuum", output=Path().resolve()/"data/vacuum_split/", seed=1623806, ratio=(.8,0.2))

    # only oil
    if not os.path.exists(Path().resolve()/"data/train_oil"):
        os.makedirs(Path().resolve()/"data/train_oil")

    src_path = Path().resolve()/"data/oil_split/train/cavitation"
    dst_path = Path().resolve()/"data/train_oil/cavitation"
    shutil.copytree(src_path, dst_path, dirs_exist_ok=True)
    src_path = Path().resolve()/"data/oil_split/train/no_cavitation"
    dst_path = Path().resolve()/"data/train_oil/no_cavitatio"
    shutil.copytree(src_path, dst_path, dirs_exist_ok=True)
    ##test
    if not os.path.exists(Path().resolve() / "data/test_oil"):
        os.makedirs(Path().resolve() / "data/test_oil")
    src_path = Path().resolve()/"data/oil_split/val/cavitation"
    dst_path = Path().resolve()/"data/test_oil/cavitation"
    shutil.copytree(src_path, dst_path, dirs_exist_ok=True)
    src_path = Path().resolve()/"data/oil_split/val/no_cavitation"
    dst_path = Path().resolve()/"data/test_oil/no_cavitatio"
    shutil.copytree(src_path, dst_path, dirs_exist_ok=True)
def create_oil_vacuum():

    # oil vacuum train
    if not os.path.exists(Path().resolve() / "data/train_oil_vacuum"):
        os.makedirs(Path().resolve() / "data/train_oil_vacuum")
    src_path = Path().resolve()/"data/oil_split/train/cavitation"
    dst_path = Path().resolve()/"data/train_oil_vacuum/cavitation"
    shutil.copytree(src_path, dst_path, dirs_exist_ok=True)
    src_path = Path().resolve()/"data/oil_split/train/no_cavitation"
    dst_path = Path().resolve()/"data/train_oil_vacuum/no_cavitation"
    shutil.copytree(src_path, dst_path, dirs_exist_ok=True)

    src_path = Path().resolve()/"data/vacuum_split/train/cavitation"
    dst_path = Path().resolve()/"data/train_oil_vacuum/cavitation"

    file_names = os.listdir(src_path)

    for file_name in file_names:
        shutil.copy2(os.path.join(src_path, file_name), dst_path)

    #oil vacuum test
    if not os.path.exists(Path().resolve() / "data/test_oil_vacuum"):
        os.makedirs(Path().resolve() / "data/test_oil_vacuum")
    src_path = Path().resolve() / "data/oil_split/val/cavitation"
    dst_path = Path().resolve() / "data/test_oil_vacuum/cavitation"
    shutil.copytree(src_path, dst_path, dirs_exist_ok=True)
    src_path = Path().resolve() / "data/oil_split/val/no_cavitation"
    dst_path = Path().resolve() / "data/test_oil_vacuum/no_cavitation"
    shutil.copytree(src_path, dst_path, dirs_exist_ok=True)

    src_path = Path().resolve() / "data/vacuum_split/val/cavitation"
    dst_path = Path().resolve() / "data/test_oil_vacuum/cavitation"

    file_names = os.listdir(src_path)

    for file_name in file_names:
        shutil.copy2(os.path.join(src_path, file_name), dst_path)

def create_oil_vacuum_teflon():

    # oil vacuum train teflon
    if not os.path.exists(Path().resolve() / "data/train_oil_vacuum_teflon"):
        os.makedirs(Path().resolve() / "data/train_oil_vacuum_teflon")
    src_path = Path().resolve() / "data/oil_split/train/cavitation"
    dst_path = Path().resolve() / "data/train_oil_vacuum_teflon"
    shutil.copy2(src_path, dst_path)
    src_path = Path().resolve() / "data/oil_split/train/no_cavitation"
    dst_path = Path().resolve() / "data/train_oil_vacuum_teflon"
    shutil.copy2(src_path, dst_path)

    src_path = Path().resolve() / "data/vacuum_split/train/cavitation"
    dst_path = Path().resolve() / "data/train_oil_vacuum_teflon/cavitation"

    file_names = os.listdir(src_path)

    for file_name in file_names:
        shutil.copy2(os.path.join(src_path, file_name), dst_path)

    src_path = Path().resolve() / "data/teflon_split/train/cavitation"
    dst_path = Path().resolve() / "data/train_oil_vacuum_teflon/cavitation"

    file_names = os.listdir(src_path)

    for file_name in file_names:
        shutil.copy2(os.path.join(src_path, file_name), dst_path)

    # oil vacuum  teflon test
    if not os.path.exists(Path().resolve() / "data/test_oil_vacuum_teflon"):
        os.makedirs(Path().resolve() / "data/test_oil_vacuum_teflon")

    src_path = Path().resolve() / "data/oil_split/val/cavitation"
    dst_path = Path().resolve() / "data/test_oil_vacuum_teflon"
    shutil.copy2(src_path, dst_path)

    src_path = Path().resolve() / "data/oil_split/val/no_cavitation"
    dst_path = Path().resolve() / "data/test_oil_vacuum_teflon"
    shutil.copy2(src_path, dst_path)

    src_path = Path().resolve() / "data/vacuum_split/val/cavitation"
    dst_path = Path().resolve() / "data/test_oil_vacuum_teflon/cavitation"

    file_names = os.listdir(src_path)

    for file_name in file_names:
        shutil.copy2(os.path.join(src_path, file_name), dst_path)

    src_path = Path().resolve() / "data/teflon_split/val/cavitation"
    dst_path = Path().resolve() / "data/test_oil_vacuum_teflon/cavitation"

    file_names = os.listdir(src_path)

    for file_name in file_names:
        shutil.copy2(os.path.join(src_path, file_name), dst_path)
def create_split_electro():
    splitfolders.ratio(Path().resolve()/"data/electrochem_20_splits/", output=Path().resolve()/"data/electrochem_20_splits_train_test/", seed=1623806, ratio=(.8,0.2))


'''
En este archivo se implementan las funciones necesarias para obtener los datos de entrada a la red (imágenes/fotogramas)
a partir de archivos de video MKV con subtítulos flotantes.
'''

import os # Para navegar por los directorios del sistema
import subprocess # Para ejecutar comandos de consola
import sys
import numpy as np
import cv2
import multiprocessing as mp

'''
Genera el dataset para la red neuronal a partir de archivos mkv con subtítulos flotantes.
Los mkv deben ser colocados en un directorio llamado videos.
Se debe crear un directorio llamado images dentro de videos.
Se debe crear un directorio llamado dataset dentro de videos.
'''

def conversion_segundos(tiempo_string):
    '''
    Convierte un string con el formato 00:01:07,800 a 67.8 (segundos)
    '''
    hhmmss = tiempo_string.split(":")
    horas = int(hhmmss[0])*60*60 # Conversión de la horas a segundos
    minutos = int(hhmmss[1])*60 # Conversion de los minutos a segundos
    segundos_milisegundos = hhmmss[2].split(",")
    segundos = float(segundos_milisegundos[0] + "." + segundos_milisegundos[1])
    tiempo = horas + minutos + segundos
    return tiempo

def obtener_tiempos(subtitulos_ruta, fps):
    '''
    Crea una lista de listas con el formato: [<tiempo_inicio>, <cantidad_frames>, <etiqueta>]
    a partir de un archivo de subtítulos y los fotogramas por segundo del video de origen.
    '''
    tiempos_frames_etiquetas = list()
    with open(subtitulos_ruta, encoding="utf8") as subtitulos:
        tiempo_0 = "00:00:00,000"
        for linea in subtitulos:
            if "-->" in linea:
                contenido = linea.split() # Separación por espacios " "
                tiempo_1 = contenido[0] # contenido[1] es -->
                tiempo_fin = contenido[2]

                tiempo_0_segundos = conversion_segundos(tiempo_0)
                tiempo_1_segundos = conversion_segundos(tiempo_1)

                if tiempo_1_segundos < tiempo_0_segundos:
                    continue

                if tiempo_1 == tiempo_0:
                    tiempo_1 = tiempo_fin

                    tiempo_0_segundos = conversion_segundos(tiempo_0)
                    tiempo_1_segundos = conversion_segundos(tiempo_1)

                    frames_1 = int((tiempo_1_segundos - tiempo_0_segundos) * fps)
                    temp = tiempo_0.split(",")
                    tiempo_0 = temp[0] + "." + temp[1]
                    tiempos_frames_etiquetas.append([tiempo_0, str(frames_1), "1"])

                    tiempo_0 = tiempo_1
                else:
                    tiempo_fin_segundos = conversion_segundos(tiempo_fin)
                    tiempo_0_segundos = conversion_segundos(tiempo_0)
                    tiempo_1_segundos = conversion_segundos(tiempo_1)

                    frames_0 = int((tiempo_1_segundos - tiempo_0_segundos) * fps)
                    frames_1 = int((tiempo_fin_segundos - tiempo_1_segundos) * fps)

                    temp = tiempo_0.split(",")
                    tiempo_0 = temp[0] + "." + temp[1]
                    temp = tiempo_1.split(",")
                    tiempo_1 = temp[0] + "." + temp[1]

                    tiempos_frames_etiquetas.append([tiempo_0, str(frames_0), "0"])
                    tiempos_frames_etiquetas.append([tiempo_1, str(frames_1), "1"])

                    tiempo_0 = tiempo_fin

    return tiempos_frames_etiquetas

def extraer_subtitulos(videos_directorio):
    archivos_en_directorio = os.listdir(videos_directorio)
    for archivo in archivos_en_directorio:
        archivo_split = archivo.split(".")
        if len(archivo_split) > 1: # Evitar archivos sin extensión o directorios
            archivo_nombre = archivo_split[0]
            archivo_extension = archivo_split[1]
            if archivo_extension == "mkv": # Se analizan solo los archivos con extension mkv
                video_ruta = os.path.join(videos_directorio, archivo)
                if archivo_nombre + ".srt" and archivo_nombre + ".ass" in archivos_en_directorio:
                    print("Los archivos de subtítulos {} y {} ya existen.".format(archivo_nombre + ".srt", archivo_nombre + ".ass"))
                else: # Se extraen los subtítulos en los formatos srt y ass
                    print("Extrayendo subtítulos en formato SRT de {}".format(archivo))
                    subtitulo_salida = os.path.join(videos_directorio, archivo_nombre) + ".srt"
                    subprocess.run(["ffmpeg.exe",
                                    "-loglevel", "quiet",
                                    "-i",
                                    video_ruta,
                                    "-vn", "-an", "-dn",
                                    subtitulo_salida,
                                    "-hide_banner"])
                    print("Extrayendo subtítulos en formato ASS de {}".format(archivo))
                    subtitulo_salida = os.path.join(videos_directorio, archivo_nombre) + ".ass"
                    subprocess.run(["ffmpeg.exe",
                                    "-loglevel", "quiet",
                                    "-i",
                                    video_ruta,
                                    "-vn", "-an", "-dn",
                                    subtitulo_salida,
                                    "-hide_banner"])

def incrustar_subtitulos(videos_directorio):
    archivos_en_directorio = os.listdir(videos_directorio)
    for archivo in archivos_en_directorio:
        archivo_split = archivo.split(".")
        if len(archivo_split) > 1: # Evitar archivos sin extensión o directorios
            archivo_nombre = archivo_split[0]
            archivo_extension = archivo_split[1]
            if archivo_extension == "mkv": # Se analizan solo lor archivos con extension mkv
                video_ruta = os.path.join(videos_directorio, archivo)
                if archivo_nombre + ".mp4" not in archivos_en_directorio:
                    print("Incrustando {} en {}.".format(archivo_nombre + ".ass", archivo))
                    video_salida = os.path.join(videos_directorio, archivo_nombre) + ".mp4"
                    cadena = "subtitles=./videos/" + archivo_nombre + ".ass"
                    subprocess.run(["ffmpeg.exe",
                                    "-loglevel", "level",
                                    "-i",
                                    video_ruta,
                                    "-filter_complex",
                                    cadena,
                                    video_salida,
                                    "-hide_banner"])
                else:
                    print("El archivo con subtítulos incrustados {} ya existe.".format(archivo_nombre + ".mp4"))

def extraer_fotogramas(videos_directorio):
    archivos_en_directorio = os.listdir(videos_directorio)
    for archivo in archivos_en_directorio:
        archivo_split = archivo.split(".")
        if len(archivo_split) > 1: # Evitar archivos sin extensión o directorios
            archivo_nombre = archivo_split[0]
            archivo_extension = archivo_split[1]
            if archivo_extension == "mp4": # Se analizan solo lor archivos con extension mp4
                video_ruta = os.path.join(videos_directorio, archivo)
                cap = cv2.VideoCapture(video_ruta)
                fps = cap.get(cv2.CAP_PROP_FPS)
                subtitulos_ruta = os.path.join(videos_directorio, archivo_nombre) + ".srt"
                tiempos_frames_etiquetas = obtener_tiempos(subtitulos_ruta, fps)
                contador = 0
                for tiempo in tiempos_frames_etiquetas:
                    print("Extrayendo fotogramas {} de {} ({}/{}).".format(tiempo, archivo, contador+1, len(tiempos_frames_etiquetas)))
                    imagen_salida = os.path.join(videos_directorio, "images", archivo_nombre)
                    subprocess.run(["ffmpeg.exe",
                                    "-loglevel", "quiet",
                                    "-i",
                                    video_ruta,
                                    "-ss", tiempo[0],
                                    "-vframes", tiempo[1],
                                    imagen_salida + "_" + str(contador) + "_imagen%04d_" + tiempo[2] + ".jpg",
                                    "-hide_banner"])
                    contador += 1

def convertir_fotogramas(videos_directorio, dimension):
    dataset_directorio = os.path.join(videos_directorio, "dataset")
    imagenes_directorio = os.path.join(videos_directorio, "images")
    imagenes_en_directorio = os.listdir(imagenes_directorio)
    imagenes_cantidad = len(imagenes_en_directorio)
    contador = 1
    for imagen in imagenes_en_directorio:
        print("Convirtiendo: {} | {}/{}".format(imagen, contador, imagenes_cantidad))
        imagen_ruta = os.path.join(imagenes_directorio, imagen)
        img = cv2.imread(imagen_ruta, cv2.IMREAD_COLOR)
        alto, ancho, canales = img.shape # filas, columnas, profundidad
        negro = np.zeros((ancho, ancho, 3), np.uint8)
        inicio = int((ancho-alto)/2)
        negro[inicio:inicio+alto, :] = img
        img = cv2.resize(negro, (dimension, dimension))
        # cv2.imshow("Fotograma",img)
        # cv2.waitKey(0)
        # cv2.destroyAllWindows()
        imagen_salida = os.path.join(dataset_directorio, imagen)
        cv2.imwrite(imagen_salida, img)
        contador += 1

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Modo de uso: {} <videos_directorio>".format(sys.argv[0])) # argv[0] es el nombre de la función
        exit()

    # extraer_subtitulos(sys.argv[1])
    # incrustar_subtitulos(sys.argv[1])
    # extraer_fotogramas(sys.argv[1])
    convertir_fotogramas(sys.argv[1], 128)


#!/usr/bin/env python3

import numpy
from matplotlib import pyplot

#https://en.wikipedia.org/wiki/Logistic_map
#r: 0 to 1 -> die
#r: 1 to 2 -> quick to (r-1)/r
#r: 2 to 3 -> oscillate to (r-1)/r
#r: 3 to 3.449 -> oscillate between 2 values
#r: 3.450 to 3.544 -> oscillate between 4 values
#r: 3.545 to 3.564 -> oscillate between 8 values
#r: 3.570 to 3.999 -> chaos
#r: 4 and up -> diverge

growth_rates = [0.9, 1.3, 2.9, 3.1, 3.8]
labels = ["r = 0.9 (die)", "r = 1.3 (quick)", "r = 2.9 (slow)",
			 "r = 3.1 (bimodal)", "r = 3.8 (chaos)",]
original_population = 0.3
num_generations = 60
population_multiplier = 20000
population_list_list = []

for growth_rate in growth_rates:
	population_list = [original_population, ]
	old_population = original_population
	for generation_number in range(num_generations):
		new_population = growth_rate * old_population * (1.0 - old_population)
		population_list.append(new_population)
		old_population = new_population
	population_list_list.append(population_list)

#make the graph
for i in range(len(population_list_list)):
	population_list = population_list_list[i]
	label = "r = %.3f"%(growth_rates[i])
	nums = numpy.around(population_list, 5)*population_multiplier
	pyplot.plot(list(range(num_generations+1)), nums, label=labels[i])
pyplot.xlabel("Generation Number")
pyplot.ylabel("Population")
pyplot.legend(bbox_to_anchor=(0.65, 0.3), loc=2, borderaxespad=0.)
pyplot.ylim(ymin=0, ymax=20000)
pyplot.xlim(xmin=0, xmax=54)
#pyplot.savefig("outchaos.png", dpi=300)
pyplot.show()


import os
import math
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers

np.random.seed(2)

print(tf.__version__)

# Folder path
folder_path = r'/Users/thesauravs/Documents/GitHub/Deep-Learning-from-Scratch/Sinusoidal'
os.chdir(folder_path)

#Loading the dataset
data = pd.read_csv('sine.csv')
data.head()

# Quick look at the joint distribution of a few pairs of columns
sns.pairplot(data[["x", "sin(x)"]], diag_kind="kde")

# To look at overall statistics
data_stats = data.describe()
data_stats.pop('sin(x)')
data_stats = data_stats.transpose()
print(data_stats)

# Split features from labels
train_labels = data.pop('sin(x)')
print(train_labels)
#print(len(data.keys()))

print(data)
# Normalizing the features to train faster
#data = (data - data_stats['mean']) / data_stats['std']
#print(data)

# Build the model
model = keras.Sequential([
        layers.Dense(128, activation='relu', input_shape=[len(data.keys())]),
        layers.Dense(64, activation='relu'),
        layers.Dense(32, activation='relu'),
        layers.Dense(16, activation='relu'),
#        layers.Dense(8, activation='relu'),
#        layers.Dense(4, activation='relu'),
#        layers.Dense(2, activation='relu'),
        layers.Dense(1)
        ])

# Try learning_rate = 0.1, 0.001, 0.0001, 0.00001 and analyse the loss graph
optimizer = keras.optimizers.Adam(learning_rate=0.0001)

model.compile(loss='mse',
              optimizer=optimizer,
              metrics=['mae','mse'])

model.summary()

# The patience parameter is the amount of epochs to check for improvement
early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)

history = model.fit(data, train_labels,
                    epochs=200,
                    validation_split=0.2,
                    verbose=1,
                    callbacks=[early_stop])

hist = pd.DataFrame(history.history)
hist['epoch'] = history.epoch

plt.figure()
plt.xlabel('Epoch')
plt.ylabel('Mean Squared Error')
plt.ylim([0, hist['loss'][0]])
plt.plot(hist['epoch'], hist['mse'], label='Train error')
plt.plot(hist['epoch'], hist['val_mse'], label='Val error')
plt.legend()
plt.show()

hist.head(7)
hist.tail(7)

predictions = model.predict(data[:])

plt.scatter(list(data['x']), train_labels, color='g', s=0.05, label='Original')
plt.scatter(list(data['x']), predictions[:], color='r', s=0.05, label='Predicted')
plt.xlabel('X')
plt.ylabel('sin(X)')
plt.title('Sine Wave')
plt.legend(loc='best', fontsize='large')
plt.show()

model.save('sine.h5')


import sys
import logging
import threading
import time

def ExceptionCatcher(typeErr, value = None, traceback = None, thread = None):
    # sys.__excepthook__(typeErr, value, traceback)
    logging.info(f"{str(threading.currentThread()).split(',')[0][8:]}: Erreur {str(typeErr.exc_value)}")

threading.excepthook = ExceptionCatcher

def Thread1():
    logging.info("Thread 1: starting")
    import Test
    logging.info("Thread 1: finishing")

def Thread2():
    logging.info("Thread 2: starting")
    import Test2
    logging.info("Thread 2: finishing")

if __name__ == "__main__":
    format = "%(asctime)s: %(message)s"
    logging.basicConfig(format=format, level=logging.INFO,
                        datefmt="%H:%M:%S")
    Programme_1 = threading.Thread(target=Thread1, args=())
    Programme_2 = threading.Thread(target=Thread2, args=())

    Programme_1.start()

    Programme_2.start()


import numpy as np
from PIL import Image

def pad_img_KAR(img, target_w, target_h, pad_value=(124, 116, 104)):
	w, h = img.size
	if w / h == target_w / target_h:
		return img, (0, 0)

	if w < h or w * (target_h / target_w) < h:
		new_w = int(h * (target_w / target_h))
		new_img = Image.new('RGB', (new_w, h), color=pad_value)
		new_img.paste(img, (int((new_w - w) // 2), 0))
		return new_img, (new_w - w, 0)
	else:
		new_h = int(w * (target_h / target_w))
		new_img = Image.new('RGB', (w, new_h), color=pad_value)
		new_img.paste(img, (0, int((new_h - h) // 2)))
		return new_img, (0, new_h - h)

def pad_array_KAR(arr, target_h, target_w, pad_value=np.array([[0]])):
	h, w = arr.shape
	if w / h == target_w / target_h:
		return arr, (0, 0)

	if w < h or w * (target_h / target_w) < h:
		new_w = int(h * (target_w / target_h))
		new_arr = np.ones((h, new_w)) * pad_value
		new_arr[:, int((new_w - w) // 2):int((new_w - w) // 2)+w] = arr
		return new_arr, (0, new_w - w)
	else:
		new_h = int(w * (target_h / target_w))
		new_arr = np.ones((new_h, w)) * pad_value
		new_arr[int((new_h - h) // 2):int((new_h - h) // 2) + h, :] = arr
		return new_arr, (new_h - h, 0)


import pygame
import sys
import math
import time
import random

#TASTELE SUNT K;M si A;Z

def linie(screen, x, y, l,color):
    pygame.draw.line(screen, color, [x, y],[x+1 , y+l])

def dreptunghi(screen, x, y, l, w, color):
    pygame.draw.line(screen, color, [x, y], [x + l, y])
    pygame.draw.line(screen, color, [x+ l,y], [x + l,y + w])
    pygame.draw.line(screen, color, [x + l,y + w], [x ,y + w])
    pygame.draw.line(screen, color, [x ,y + w],[x ,y ])

def minge(screen, x, y, r, color, w):
    pygame.draw.circle(screen, color,[x, y], r, w)

def intersects(mx, my, r, left, top, right, bottom):
   if mx < left:
       closestX = left
   else:
       if mx > right:
           closestX = right
       else:
            closestX = mx

   if my < top:
       closestY = top
   else:
       if my > bottom:
           closestY = bottom
       else:
            closestY = my

   dx = closestX - mx
   dy = closestY - my

   return (dx * dx + dy * dy) <= r * r

 #culori RED,GREEN,BLUE
RED = (255 ,0, 0)
WHITE=(255,255,255)
BLUE=(0,0,255)
GREEN=(0,255,0)
BLACK=(0,0,0)

pygame.init()

size=[500, 350]
screen=pygame.display.set_mode(size)

#MINGE

r=7
my=50
mx=250
mdir=2
ui=random.randint(0,360)* math.pi/180

scoreS=0
scoreR=0

pygame.display.set_caption("PONG ")
font=pygame.font.SysFont('Calibri',25,True,False)

won=False

sx=5
rx=475
sy=125
ry=125
sdir=0
rdir=0
done=False
clock=pygame.time.Clock()

pygame.key.set_repeat(1,10)

while not done:
    for event in pygame.event.get():
        if event.type==pygame.QUIT:
            done=True
        keys = pygame.key.get_pressed()

        if keys[pygame.K_a]:
            if sy>2:
                sy-=2

        if keys[pygame.K_z]:
            if sy<249:
                sy+=2

        if keys[pygame.K_k]:
            if ry>2:
                ry-=2

        if keys[pygame.K_m]:
            if ry<249:
                ry+=2

    screen.fill(WHITE)

    dreptunghi(screen,5,sy,20,100,BLACK)

    textscoreR= font.render("score: " + str(scoreR),True,BLACK)
    screen.blit(textscoreR,[350,325])
    textscoreS= font.render("score: "+ str(scoreS),True,BLACK)
    screen.blit(textscoreS,[87,325])

    if scoreS==5:
        wintextS=font.render("PLAYER1 WINS! ",True,GREEN)
        screen.blit(wintextS,[225,175])
        done=True
        won=True

    if scoreR==5:
        wintextR=font.render("PLAYER2 WINS !",True,GREEN)
        screen.blit(wintextR,[225,175])
        done=True
        won=True

    linie(screen,250,0,350,BLACK)
    dreptunghi(screen,475,ry,20,100,BLACK)
    minge(screen,int(mx),int(my),r,BLUE,0)

    if mx -r <=0:
        scoreR=scoreR+1
        mx=250
        my=50

    if mx + r>=500:
        scoreS=scoreS +1
        mx=250
        my=50

    mx=mx + mdir * math.sin(ui)
    if mx <= 0 + r:
        ui= - ui

    if mx>= 500 - r:
        ui= - ui
    my=my + mdir * math.cos(ui)

    if my <= 0 + r:
        ui= math.pi - ui

    if my >= 350-r:
        ui=math.pi - ui

    if intersects(mx,my,r,sx,sy,sx+20,sy+100):
        ui= math.atan2(my,mx)

    if intersects(mx,my,r,rx,ry,rx+20,ry + 100):
        ui= - math.atan2(my,mx)

        print(scoreS,scoreR)

    pygame.display.flip()
    clock.tick(100)

    if won:
        time.sleep(5)

pygame.quit()
sys.exit()


"""
Read and parse the grammar format as described in

    https://dfns.dyalog.com/s_Binding.htm
    https://dfns.dyalog.com/n_parse.htm
    https://dfns.dyalog.com/s_parse.htm

"""
from functools import reduce
from itertools import product
import re
from string import ascii_letters, digits, whitespace

import numpy as np

class Grammar:
    functions = '+-×÷*=≥>≠∨∧⍒⍋⌽⍉⊖⍟⍱⍲!?∊⍴~↑↓⍳○*⌈⌊∇⍎⍕⊃⊂∩∪⊥⊤|≡≢,⍪⊆⌹'
    hybrids = '/⌿\⍀'
    monadic_operators = '⌸¨⍨'
    dyadic_operators = '⍣⌺@⍥⍤'

    def __init__(self, cats, zmat, bmat, xmat, bkts, bcats, lbs, rbs, blabs):
        self.cats = cats
        self.zmat = zmat
        self.bmat = bmat
        self.xmat = xmat
        self.bkts = bkts
        self.bcats = bcats
        self.lbs = lbs
        self.rbs = rbs
        self.blabs = blabs

    @classmethod
    def compile(cls, grammar: list[str]):
        sep = [r.startswith('⍝⍝⍝⍝⍝⍝⍝⍝') for r in grammar]
        idx = [i for i, s in enumerate(sep) if s]

        categories = grammar[idx[0]:idx[1]]
        macros = grammar[idx[1]+1:idx[2]]
        bindings = grammar[idx[2]+1:]

        # Categories & brackets
        if categories[-1] == '': # Sometimes we have an empty line at the end :/
            categories = categories[:-1]

        cats = [r[:r.index(' ')] for r in categories[1:] if r != ''] # Last are the brackets
        brackets = None
        if cats[-1].startswith('()'):
            b = categories.pop(-1)
            brackets = b[:b.index('⍝')].strip()
            cats.pop(-1)

        lcats = len(cats)

        # Macros
        if macros[-1] == '': # Sometimes we have an empty line at the end :/
            macros = macros[:-1]
        macro_tab = dict([mk[:mk.index('⍝')].strip().split('=') for mk in macros]) # Trim

        # Bindings -- strip comments. Separating lines are sometimes empty, and sometimes
        # they contain a lamp :/
        bb = []
        for b in bindings:
            b = b.strip()
            if b == '':
                bb.append(b)
            else:
                bb.append(b[:b.index('⍝')].strip())

        bindings = bb

        # The bindings section has sub-sections, now delineated by a
        # an empty line

        sep = [r=='' for r in bindings]
        idx = [i for i, s in enumerate(sep) if s]

        # Partition
        pidx = [e.tolist() for e in np.split(np.arange(len(bindings)), idx)]
        sections = []
        for sect in pidx:
            if len(sect) == 1:
                sections.append([bindings[sect[0]]])
            else:
                sections.append(bindings[sect[1]:sect[-1]+1])

        # Expand macros and separate multiple entries on space
        split_sections = []
        for s in sections:
            new_s = []
            for item in s:
                new_s.extend(macro_expand(item, macro_tab).split())
            split_sections.append(new_s)

        # Expand the dots
        dot_sections = []
        for s in split_sections:
            new_s = []
            for item in s:
                new_s.extend(dot_expand(item))
            dot_sections.append(new_s)

        # Build the complete bindings table
        zmat = np.zeros(lcats*lcats, dtype=int).reshape((lcats, lcats)) # Resulting categories
        bmat = np.zeros(lcats*lcats, dtype=int).reshape((lcats, lcats)) # Binding strengths

        for i, s in enumerate(dot_sections):
            for (left, right), target in s:  # type: ignore
                l_index = cats.index(left)   # type: ignore
                r_index = cats.index(right)  # type: ignore
                t_index = cats.index(target) # type: ignore

                zmat[l_index, r_index] = t_index
                bmat[l_index, r_index] = len(dot_sections) - i

        assert brackets
        bkts = [a for a in brackets if a not in ascii_letters + digits + whitespace] # Bracket pairings
        lbs, rbs = np.array(list(zip(bkts[::2], bkts[1::2]))).T.tolist()             # Lefts and rights of brackets
        p = f'[{re.escape("".join(lbs))}](.*?)[{re.escape("".join(rbs))}]'
        blabs = re.findall(p, brackets)                                              # Bracket labels
        bcats = []                                                                   # Bracket categories

        for b in blabs:
            try:
                bcats.append(cats.index(b))
            except ValueError:
                bcats.append(len(cats))

        xmat = np.pad(bmat, ((1, 1), (1, 1)), 'constant') # Extended bmat: pad with zeros in x and y

        return cls(cats, zmat, bmat, xmat, bkts, bcats, lbs, rbs, blabs)

    def classify(self, src: str) -> list[tuple]:
        i = 0
        pairs: list[tuple] = []
        while i<len(src):
            ch = src[i]
            if ch == '⍝':
                i = skip_comment(src, i)
                continue
            elif ch in whitespace:
                pass
            elif ch in self.bkts:
                pairs.append((-1, ch))
            elif ch == "'":
                i, s = getstring(src, i)
                pairs.append((0, s))
                continue
            elif ch == '¯' or ch.isdigit() or ch == '.' and peek(src, i).isdigit():
                i, num = getnum(src, i)
                pairs.append((0, num))
                continue
            elif ch in ascii_letters+"⎕_": # Named arrays, functions or operators
                i, name = getname(src, i)
                pairs.append((self.cats.index('N'), name))
                continue
            elif ch in Grammar.functions:
                pairs.append((self.cats.index('F'), ch))     # Primitive function
            elif ch in Grammar.hybrids:
                pairs.append((self.cats.index('H'), ch))     # Hybrid function/operator
            elif ch in Grammar.monadic_operators:
                pairs.append((self.cats.index('MOP'), ch))   # Primitive monadic operator
            elif ch in Grammar.dyadic_operators:
                pairs.append((self.cats.index('DOP'), ch))   # Primitive dyadic operator
            elif ch == '∘':
                pairs.append((self.cats.index('JOT'), ch))   # Jot: compose / null operand
            elif ch == ';':
                pairs.append((self.cats.index('LST'), ch))   # Subscript list separator
            elif ch == ':':
                pairs.append((self.cats.index('CLN'), ch))   # Expression list separator
            elif ch == '⋄':
                pairs.append((self.cats.index('XL'), ch))    # Colon for guard
            elif ch == '.':
                pairs.append((self.cats.index('DOT'), ch))   # Dot: ref / product
            elif ch == '←':
                pairs.append((self.cats.index('ARO'), ch))   # Assignment arrow
            elif ch in '⍺⍵':
                pairs.append((0, ch))                        # Dfn arg arrays
            i += 1
        return pairs

    def bkt(self, bracket: str, t: tuple) -> tuple:
        """
        bkt ← {                       ⍝ bind of bracketed node [ ⍵ ].
            (cat expr)←⍵              ⍝ category of bracketed expr.
            zcat←(cat,1↓bcats)[lbs⍳⍺] ⍝ resulting category.
            zcat(⍺ expr)              ⍝ ⍺-bracketed node.
        }                             ⍝ :: left_bkt ∇ node → node
        """
        cat, expr = t
        zcat = ([cat]+self.bcats[1:])[self.lbs.index(bracket)]

        return (zcat, (bracket, expr))

    def ebk(self, bracket: str) -> tuple:
        """
        Tag empty brackets with the category of the bracketed node:

            [] ⍝ Empty index list
            {} ⍝ Empty dfn
        """
        return (self.bcats[self.lbs.index(bracket)], bracket)

    def bind(self, stream: tuple) -> tuple:
        Aa, Bb, Cc = stream[1:4]
        D_, L = _unpack(stream[0])
        R, _D = _unpack(stream[4])

        if type(Aa) == int and Aa == Cc == -1: # Single node; we're done
            return Bb,

        if type(Aa) == int and Aa == Bb == -1:
            # Partial parse; we can't reduce further. This is either a
            # syntax error, or we've landed on a name that would need
            # looking up.
            return stream

        # Separate the tuples into categories and tokens
        A, a = _unpack(Aa)
        B, b = _unpack(Bb)
        C, c = _unpack(Cc)

        if a in self.bkts[2:] and b in self.bkts[2:]: # empty brackets []
            return self.bind((rgt((D_, L)), Aa, self.ebk(b), R, _D)) # rgt(∆_ L)Aa(ebk b)R _∆

        if a in self.bkts and c in self.bkts: # bracketed single value Bb
            return self.bind((D_, L, self.bkt(a, Bb), R, _D))  # (⊂a c)∊bkts:∇ rgt ∆_ L(a bkt Bb)R _∆

        if a in self.rbs: # right bracket: skip left.
            return self.bind(lft(lft(stream))) # (⊂a)∊rbs:∇ lft lft ⍵

        if self.xmat[(A+1, B+1)] >= self.xmat[(B+1, C+1)]: # A:B ≥ B:C → skip left.
            return self.bind(lft(stream))

        BbCc = (self.zmat[(B, C)], (b, c)) # BbCc←zmat[B;C],⊂b c ⍝ B bound with C.
        return self.bind(((D_, L), Aa, BbCc, R, _D)) # binds with token to the right?

def rgt(stream: tuple) -> tuple:
    """
    Skip right

    rgt ← {
        ∆_ A B C (R _∆) ← ⍵
        (∆_ A) B C R _∆
    }
    """
    D_, A, B, C = stream[:-1]
    (R, _D) = _unpack(stream[-1])

    return (D_, A), B, C, R, _D

def lft(stream: tuple) -> tuple:
    """
    Skip left

    lft ← {
        (∆_ L) A B C _∆ ← ⍵
        ∆_ L A B (C _∆)
    }
    """
    D_, L = _unpack(stream[0])
    A, B, C, _D = stream[1:]

    return D_, L, A, B, (C, _D)

def macro_expand(str, macros):
    """
    Expand macros in a string.
    """
    s = str
    while True:
        changed = False
        for k, v in macros.items():
            s_r = s.replace(k, v)
            if s_r != s:
                changed = True
                s = s_r
        if not changed:
            break
    return s

def _unpack(item: int|tuple) -> tuple:
    """
    Python doesn't do scalar extension, so we have to do it ourselves.
    """
    if type(item) == int:
        return (item, item)
    else:
        return item # type: ignore

def dot_expand(str):
    """
    A:F.H => AF

    means

    A:F => AF
    A:H => AF
    """
    rule, target = str.split('→')
    left, right = rule.split(':')

    lefts = left.split('.')
    rights = right.split('.')
    targets = target.split('.')

    rules = list(product(lefts, rights))
    if len(targets) == 1: # Scalar extension on the right
        targets = targets * len(rules)
    if len(rules) != len(targets):
        raise SyntaxError('SYNTAX ERROR')
    return list(zip(rules, targets))

def peek(src: str, idx: int) -> str:
    try:
        return src[idx+1]
    except IndexError:
        return ''

def skip_comment(src: str, i: int) -> int:
    while i < len(src) and src[i] != '\n':
        i += 1
    return i

def getstring(src: str, idx: int) -> tuple[int, str]:
    m = re.match(r"'((?:''|[^'])*)'", src[idx:]) # Note: APL escapes single quotes with single quotes
    if not m:
        raise SyntaxError("SYNTAX ERROR: Unpaired quote")
    data = m.group(1)
    return (2+idx+len(data), data.replace("''", "'"))

def get_cmplx(tok: str) -> complex:
    parts = tok.split('J')
    if len(parts) != 2:
        raise SyntaxError('SYNTAX ERROR: malformed complex scalar')
    (re, im) = parts
    try:
        cmplx = complex(float(re), float(im))
    except TypeError:
        raise SyntaxError('SYNTAX ERROR: malformed complex scalar')
    return cmplx

def getnum(src: str, idx: int) -> tuple[int, int|float|complex]:
    tok = ''
    start = idx
    m = re.match(r"[¯0-9eEjJ.]+", src[idx:])
    if not m:
        raise SyntaxError('SYNTAX ERROR: malformed numeric scalar')
    tok = m.group(0).upper()
    tok = tok.replace('¯', '-')
    if 'J' in tok:
        return (idx+len(tok), get_cmplx(tok))
    if '.' in tok:
        val = float(tok)
    else:
        val = int(tok)
    return (idx+len(tok), val)

def getname(src: str, idx: int) -> tuple[int, str]:
    m = re.match(r'(^[⎕_a-z][_a-z0-9]*)', src[idx:], re.IGNORECASE)
    if not m:
        raise SyntaxError('SYNTAX ERROR: malformed name')
    data = m.group(1)
    if data[0] == '⎕':
        return (idx+len(data), data.upper())
    return (idx+len(data), data)

if __name__ == "__main__":
    with open('grammar') as f:
        grammar = f.read().splitlines()
    g = Grammar.compile(grammar)
    # Ok, we have parsed the grammar, yay: cats, zmat, bmat, bkts, blabs

    src = '{⍺+⍵}/⍳5'
    # src = '+/⍳5'
    # src = '⍳5'
    # src = 'x ← ⍳5 ⋄ x×3'
    pairs = g.classify(src)
    eos = -1
    ll = [eos]+pairs[:-2]
    if ll == [-1]:
        D_ = -1
    else:
        # Left cons list: ∆_←↑{⍺ ⍵}⍨/⌽eos,¯2↓pairs
        D_ = tuple(reduce(lambda x, y: (x, y), ll)) # type: ignore
    Aa, Bb, Cc = (eos, eos, *pairs, eos)[-3:] # 3-token window
    stream = (D_, Aa, Bb, Cc, eos)
    tree = g.bind(stream)

    print(tree)


# -*- coding: utf-8 -*-
"""
Created on Wed Nov  4 15:44:25 2020

@author: Jonathan
"""

from tensorflow.keras.optimizers import SGD, Adam
import tensorflow as tf
from sklearn.model_selection import train_test_split
import pandas as pd
from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, f1_score
import matplotlib.pyplot as plt
import numpy as np
from sklearn.utils import compute_class_weight
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences

#read in and prepare reddit data
dfText = pd.read_csv('data/redpolitics30min.csv', index_col=0)
dfText= dfText[130:].reset_index(drop=True)
dfText.afterEpoch = dfText.afterEpoch.astype(int)
dfText.beforeEpoch = dfText.beforeEpoch.astype(int)
dfText = dfText.replace(np.nan,'')
dfText['sentence'] = dfText.art0 + '. ' + dfText.art1+ '. ' + dfText.art2+ '. ' + dfText.art3+ '. ' + dfText.art4+ '. ' + dfText.art5 + '. ' + dfText.art6+ '. ' + dfText.art7+ '. ' + dfText.art8+ '. ' + dfText.art9
dfText['sentence'] = dfText['sentence'].astype('string')

#read in and prepare spy data
dfSpy = pd.read_csv('data/spy_30Min2yrsNum2.csv', index_col=0)
dfSpy = dfSpy[:6103].reset_index(drop=True)

#create up/down target column
dfSpy['up'] = 0
dfSpy['down'] = 0
dfSpy['direction'] = np.log(dfSpy.close/dfSpy.opn)
dfSpy.loc[(dfSpy.direction >= 0), 'up'] = 1
dfSpy.loc[(dfSpy.direction < 0), 'down'] = 1
#one column label
dfSpy['directionTarget1Col'] = 0
dfSpy.loc[(dfSpy.direction >= 0), 'directionTarget1Col'] = 1

#replace missing volumes with median
volMed = dfSpy['volume'].median()
dfSpy['volume'] = dfSpy['volume'].replace(0,volMed)

#shift columns to create features from prior/next day
dfSpy['close_open'] = np.log(dfSpy.close/dfSpy.opn).shift(1)
dfSpy['high_open'] = np.log(dfSpy.high/dfSpy.opn).shift(1)
dfSpy['low_open'] = np.log(dfSpy.opn/dfSpy.low).shift(1)
for i in range(len(dfSpy['volume'])-1):
    dfSpy.loc[i+1,'volPercent'] = np.log(dfSpy.loc[i+1,'volume']/dfSpy.loc[i,'volume'])

#add combined sentence feature and drop first row
dfSpy['sentence']=dfText['sentence']
dfSpy= dfSpy.drop([0]).reset_index(drop=True)

#create final preNormalized feature/target df
FinDf=dfSpy[['close_open', 'high_open', 'low_open', 'volPercent',
             'sentence', 'directionTarget1Col']]

#separate df into train, val, test
dftrainDataWith_y = FinDf[:3900]
dfvalDataWith_y = FinDf[3900:4880]
dftestDataWith_y = FinDf[4880:]

#fit and standardize train data for numerical input
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
Xtrain = scaler.fit_transform(dftrainDataWith_y[['close_open', 'high_open', 'low_open', 'volPercent', ]])

#normalize val and test data for numerical input
Xval = scaler.transform(dfvalDataWith_y[['close_open', 'high_open', 'low_open', 'volPercent', ]])
Xtest = scaler.transform(dftestDataWith_y[['close_open', 'high_open', 'low_open', 'volPercent', ]])

#set Target column and target values
target = 'directionTarget1Col'
ytrain = dftrainDataWith_y[target]
yval = dfvalDataWith_y[target]
ytest = dftestDataWith_y[target]

#tokenize/encode/pad train data for text input
t = Tokenizer()
t.fit_on_texts(dftrainDataWith_y['sentence'])
max_length = 100
vocab_size = len(t.word_index)+1
encodedTrain = t.texts_to_sequences(dftrainDataWith_y['sentence'])
padded_encodedTrain = pad_sequences(encodedTrain, maxlen=max_length, padding='post', truncating='post')

#tokenize/encode/pad val data for text input
encodedVal = t.texts_to_sequences(dfvalDataWith_y['sentence'])
padded_encodedVal = pad_sequences(encodedVal, maxlen=max_length, padding='post', truncating='post')

#tokenize/encode/pad test data for text input
encodedTest = t.texts_to_sequences(dftestDataWith_y['sentence'])
padded_encodedTest = pad_sequences(encodedTest, maxlen=max_length, padding='post', truncating='post')

#shuffle everything
from sklearn.utils import shuffle
Xtrain, padded_encodedTrain, ytrain = shuffle(Xtrain, padded_encodedTrain, ytrain, random_state=37)
Xval, padded_encodedVal, yval = shuffle(Xval, padded_encodedVal, yval, random_state=37)
Xtest, padded_encodedTest, ytest = shuffle(Xtest, padded_encodedTest, ytest, random_state=37)

#weighting target classes for loss function
'''
#create weights for target column
sumUp=sum(dftrainDataWith_y[target])
length = len(dftrainDataWith_y[target])
weight=sumUp/(length-sumUp)
class_weight = {0: weight, 1: 1.0}
'''

#build model architecture
n_features = Xtrain.shape[1]

inputsNum = tf.keras.Input(shape=(n_features,))
inputsText = tf.keras.Input(shape=(max_length,))

x1 = tf.keras.layers.Dense(5, activation='relu')(inputsNum)
x1 = tf.keras.layers.Dropout(.4)(x1)
x1 = tf.keras.layers.BatchNormalization()(x1)
x1 = tf.keras.layers.Dense(2, activation='relu')(x1)

x2 = tf.keras.layers.Embedding(vocab_size, 50)(inputsText)
x2 = tf.keras.layers.Dropout(.4)(x2)
x2 = tf.keras.layers.BatchNormalization()(x2)
x2 = tf.keras.layers.Flatten()(x2)
x2 = tf.keras.layers.Dense(2, activation='relu')(x2)
x2 = tf.keras.layers.Dropout(.4)(x2)

xAll = tf.keras.layers.concatenate([x1,x2])
xAll = tf.keras.layers.Dense(2)(xAll)

out = tf.keras.layers.Dense(1, activation="sigmoid")(xAll)

#hyperparameters
opt = Adam(2e-5)
#opt = SGD(lr=0.001)
#loss = tf.keras.losses.CategoricalCrossentropy()
loss = 'binary_crossentropy'

#compile and fit model
model = tf.keras.Model(inputs=[inputsNum, inputsText], outputs=out)
model.compile(loss=loss, optimizer=opt, metrics=['accuracy'])
history = model.fit([Xtrain, padded_encodedTrain], ytrain, epochs=55, batch_size = 32, shuffle=(True),
                    validation_data = ([Xval, padded_encodedVal], yval))
model.summary()

#evaluate model
def evalModel():
    v = np.round(model.predict([Xtest, padded_encodedTest]))
    w = ytest
    print(confusion_matrix(w,v))
    print("Accuracy:", accuracy_score(w,v))
    print("F1:", f1_score(w,v))
    print("Precision:", precision_score(w,v))
    print("Recall:", recall_score(w,v))

def plotLoss():
    plt.plot(history.history['loss'])
    plt.plot(history.history['val_loss'])
    plt.title('tain loss vs val loss')
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.legend(['train', 'validation'], loc='upper right')
    plt.show()

def plotAcc():
    plt.plot(history.history['accuracy'])
    plt.plot(history.history['val_accuracy'])
    plt.title('train acc vs val acc')
    plt.ylabel('acc')
    plt.xlabel('epoch')
    plt.legend(['train', 'validation'], loc='upper right')
    plt.show()

evalModel()
plotLoss()
plotAcc()


from typing import List

class Solution:
    def singleNumber(self, nums: List[int]) -> List[int]:

        s = set(nums)
        for num in s:
            nums.remove(num)
        return list(s.difference(set(nums)))

print(Solution().singleNumber([1,2,1,3,2,5]))


import cv2
import face_recognition
import pickle

dataset_paths = ['dataset/son/', 'dataset/tedy/']
names = ['Son', 'Tedy']
number_images = 10
image_type = '.jpg'
encoding_file = 'encodings.pickle'
# Either cnn  or hog. The CNN method is more accurate but slower. HOG is faster but less accurate.
model_method = 'cnn'

# initialize the list of known encodings and known names
knownEncodings = []
knownNames = []

# loop over the image paths
for (i, dataset_path) in enumerate(dataset_paths):
    # extract the person name from names
    name = names[i]

    for idx in range(number_images):
        file_name = dataset_path + str(idx+1) + image_type

        # load the input image and convert it from BGR (OpenCV ordering)
        # to dlib ordering (RGB)
        image = cv2.imread(file_name)
        rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

        # detect the (x, y)-coordinates of the bounding boxes
        # corresponding to each face in the input image
        boxes = face_recognition.face_locations(rgb,
            model=model_method)

        # compute the facial embedding for the face
        encodings = face_recognition.face_encodings(rgb, boxes)

        # loop over the encodings
        for encoding in encodings:
            # add each encoding + name to our set of known names and
            # encodings
            print(file_name, name, encoding)
            knownEncodings.append(encoding)
            knownNames.append(name)

# Save the facial encodings + names to disk
data = {"encodings": knownEncodings, "names": knownNames}
f = open(encoding_file, "wb")
f.write(pickle.dumps(data))
f.close()


######
#Date: 07.10.2022
##Version Log:
#
#
#
######

from cProfile import label
from math import sqrt
import re
from socket import EAI_SOCKTYPE
from turtle import color
from typing import Mapping
from cv2 import mean
from matplotlib.pyplot import axis, plot, show
from pip import main
import pymap3d as map
import numpy as np
import pandas as pd
import copy
import matplotlib.pyplot as plt

###Import

##Real Flight Test Data
val_data1 = pd.read_csv('../Flight_Logs/Flight_Test/DJIFlightRecord_2022-08-25_[10-38-43].csv', sep=',')
val_data2 = pd.read_csv('../Flight_Logs/Flight_Test/DJIFlightRecord_2022-08-25_[10-42-05].csv', sep=',')
val_data3 = pd.read_csv('../Flight_Logs/Flight_Test/DJIFlightRecord_2022-08-25_[10-45-30].csv', sep=',')
val_flights = [val_data1,val_data2,val_data3]

##Simulation Data
sim_data1 = pd.read_csv('../Flight_Logs/Simulation/verification_final1.csv', sep=';')
sim_data2 = pd.read_csv('../Flight_Logs/Simulation/verification_final2.csv', sep=';')
sim_data3 = pd.read_csv('../Flight_Logs/Simulation/verification_final3.csv', sep=';')

# sim_datatest = pd.read_csv('Validation/validation_test.csv', sep=';') #Only for test flight data
# sim_datatest= sim_datatest.iloc[:,1:] #Only for test flight data

#Removing first line (Time)
sim_data1 = sim_data1.iloc[:,1:]
sim_data2 = sim_data2.iloc[:,1:]
sim_data3 = sim_data3.iloc[:,1:]
sim_flights = [sim_data1,sim_data2,sim_data3]

# sim_flights = [sim_datatest,sim_datatest,sim_datatest] #Only for test flight data

##Simulation Data before Model Tuning
sim_data_comparison = pd.read_csv('../Flight_Logs/Simulation/verification_no_adjustments.csv', sep=';')
sim_data_comparison2 = pd.read_csv('../Flight_Logs/Simulation/verification_no_adjustments2.csv', sep=';')
sim_data_comparison3 = pd.read_csv('../Flight_Logs/Simulation/verification_no_adjustments3.csv', sep=';')
sim_data_comparison = sim_data_comparison.iloc[:,1:]
sim_data_comparison2 = sim_data_comparison2.iloc[:,1:]
sim_data_comparison3 = sim_data_comparison3.iloc[:,1:]
sim_compare = [sim_data_comparison,sim_data_comparison2,sim_data_comparison3]

##Waypoints
waypoints = pd.read_excel('../Script/Flight_Plans/Flight_Plan.xlsx')
waypoints = waypoints.iloc[0:waypoints.shape[0]-1,:]
waypoints = waypoints.rename(columns={waypoints.columns[1]:'latitude',waypoints.columns[2]:'longitude',waypoints.columns[3]:'height'})

###Data Post Processing

def real_post_process(real_data,logsteps,start,end):
    '''Takes a data frame with 3 single flight logs from the flight testing as input. Extracts the most interesting columns.
    Turns them into correct units if needed, removes the starting and landing procedure and takes the average of the 3 logs.
    The function returns the averaged data set and the shortened flight logs of all 3 flights in one data frame'''

    ## Initial Variables
    real = pd.DataFrame()
    lat_waypoint1 = 46.9765208  #Latitude of Waypoint 1
    long_waypoint1 = 7.179124   #Longitude of Waypoint 1

    ## Post Processing Real Data
    # Iterating through Data Sets
    for i in real_data:

        #Exporting important columns
        real_single = copy.deepcopy(i.iloc[:,4])                    #Add latitude
        real_single = pd.concat([real_single,i.iloc[:,5]],axis=1)   #Add longitude
        real_single = pd.concat([real_single,i.iloc[:,6]],axis=1)   #Add height [ft]
        real_single = pd.concat([real_single,i.iloc[:,9]],axis=1)   #Add altitude [ft]
        real_single = pd.concat([real_single,i.iloc[:,10]],axis=1)  #Add speed [mph]
        real_single = real_single.rename({' OSD.latitude':'latitude',' OSD.longitude':'longitude',' OSD.height [ft]':'height',' OSD.altitude [ft]':'altitude',' OSD.hSpeed [MPH]':'speed'},axis=1)

        # Imperial Unit in SI
        real_single.loc[:,'height'] = real_single.loc[:,'height']/3.33      #ft --> m
        real_single.loc[:,'altitude'] = real_single.loc[:,'altitude']/3.33  #ft --> m
        real_single.loc[:,'speed'] = real_single.loc[:,'speed']*0.44704     #mph --> m/s

        #Removing Start Procedure
        real_start = copy.deepcopy(real_single)
        real_start.loc[:,'latitude'] = abs(real_start.loc[:,'latitude']-lat_waypoint1)
        real_start.loc[:,'longitude'] = abs(real_start.loc[:,'longitude']-long_waypoint1)
        summe = real_start.loc[:,'latitude'] + real_start.loc[:,'longitude']
        summe_start = summe.iloc[start:end]
        idx_start = summe_start.idxmin()
        summe_end = summe.iloc[end+1:]
        idx_end = summe_end.idxmin()

        #Merging all runs in one data sate
        real_add = copy.deepcopy(real_single.iloc[idx_start:idx_end,:])
        new_idx = pd.Index(range(0,idx_end-idx_start,1))
        real_add.set_index(new_idx,inplace=True)
        real = pd.concat([real,real_add], axis=1, ignore_index=True)

    #Averaging Date of the runs
    cols = real_add.shape[1]
    lat_start = 0; long_start = 1; height_start = 2
    alt_start = 3; speed_start = 4

    lat = real[[lat_start,lat_start+cols,lat_start+2*cols]].mean(axis = 1, skipna = True) #Selecting necessary columns for mean
    long = real[[long_start,long_start+cols,long_start+2*cols]].mean(axis = 1, skipna = True)
    height = real[[height_start,height_start+cols,height_start+2*cols]].mean(axis = 1, skipna = True)
    alt = real[[alt_start,alt_start+cols,alt_start+2*cols]].mean(axis = 1, skipna = True)
    speed = real[[speed_start,speed_start+cols,speed_start+2*cols]].mean(axis = 1, skipna = True)

    #Create column with frequency of logged data
    time = pd.Series(np.arange(0,real.shape[0]/logsteps,1/logsteps))
    real = pd.concat([time,real], axis=1, ignore_index=True)     #Add them also to the data sets containing all runs

    real_avg = pd.DataFrame({
        'time': time,
        'latitude':lat,
        'longitude': long,
        'height': height,
        'altitude':alt,
        'speed':speed})

    return real_avg, real

##Post Processing Sim Data
def sim_post_process(sim,logsteps,start,end):
    '''Takes a data frame with 3 single flight logs from the simulation as input.
    Removes the starting and landing procedure and takes the average of the 3 logs.
    The function returns the averaged data set and flight logs of all 3 flights in one data frame'''

    sim_all = pd.DataFrame()
    lat_waypoint1 = 46.9765208  #Latitude of Waypoint 1
    long_waypoint1 = 7.179124   #Longitude of Waypoint 1

    ##Iterating through Sim Runs
    for i in sim:
        #Deleting Start Procedure
        sim_start = copy.deepcopy(i)
        sim_start.loc[:,'latitude'] = abs(sim_start.loc[:,'latitude']-lat_waypoint1)
        sim_start.loc[:,'longitude'] = abs(sim_start.loc[:,'longitude']-long_waypoint1)
        summe = sim_start.loc[:,'latitude'] + sim_start.loc[:,'longitude']
        summe_start = summe.iloc[start:end]
        idx_start = summe_start.idxmin()
        summe_end = summe.iloc[end+1:]
        idx_end = summe_end.idxmin()

        #Merging all runs in one data sate
        sim_add = copy.deepcopy(i.iloc[idx_start:idx_end,:])
        new_idx = pd.Index(range(0,idx_end-idx_start,1))
        sim_add.set_index(new_idx,inplace=True)
        sim_all = pd.concat([sim_all,sim_add], axis=1, ignore_index=True)

    ##Averaging Date of the runs
    cols = sim_add.shape[1]
    lat_start = 1; long_start = 2; height_start = 3
    pitch_start = 5; roll_start = 6; yaw_start = 7

    lat = sim_all[[lat_start,lat_start+cols,lat_start+2*cols]].mean(axis = 1, skipna = True) #Selecting necessary columns for mean
    long = sim_all[[long_start,long_start+cols,long_start+2*cols]].mean(axis = 1, skipna = True)
    height = sim_all[[height_start,height_start+cols,height_start+2*cols]].mean(axis = 1, skipna = True)
    pitch = sim_all[[pitch_start,pitch_start+cols,pitch_start+2*cols]].mean(axis = 1, skipna = True)
    roll = sim_all[[roll_start,roll_start+cols,roll_start+2*cols]].mean(axis = 1, skipna = True)
    yaw = sim_all[[yaw_start,yaw_start+cols,yaw_start+2*cols]].mean(axis = 1, skipna = True)

    #Create column with frequency of logged data
    time = pd.Series(np.arange(0,sim_all.shape[0]/logsteps,1/logsteps))
    sim_all = pd.concat([time,sim_all], axis=1, ignore_index=True)     #Add them also to the data sets containing all runs

    sim_avg = pd.DataFrame({
        'time':time,
        'latitude':lat,
        'longitude': long,
        'height': height,
        'pitch':pitch,
        'roll':roll,
        'yaw':yaw})

    return sim_avg, sim_all

def gps2enu(log):
    '''Turn GPS Coordinates in ENU coordinates'''

    enu_coord=[]
    ##Home Coordinates
    gps_home_lat = 46.9765208
    gps_home_long = 7.179124
    gps_home_alt = 59

    for i in log:
        enu = map.geodetic2enu(i.loc[:,'latitude'],i.loc[:,'longitude'],i.loc[:,'height'],gps_home_lat,gps_home_long,gps_home_alt)
        i.loc[:,'latitude'] = enu[1]  #north
        i.loc[:,'longitude'] = enu[0] #east
        i.loc[:,'height'] = enu[2]    #up
        mapping = {i.columns[1]:'north',i.columns[2]:'east',i.columns[3]:'up'}
        p = i.rename(columns = mapping)
        enu_coord.append(p)

    return enu_coord

###########################################################################

####DATA POSTPROCESSING

###########################################################################

###Apply Post-Process Function
avg_real, real_flights = real_post_process(val_flights,10,440,600)
avg_sim, sim_flights = sim_post_process(sim_flights,5,0,400)
avg_compare, compare_flights = sim_post_process(sim_compare,5,200,290)

###Apply GPS to ENU Function
gps_coord = copy.deepcopy([avg_real,avg_sim,waypoints,avg_compare])
enu_coord = gps2enu(gps_coord)
avg_real_enu = enu_coord[0]
avg_sim_enu = enu_coord[1]
waypoints_enu = enu_coord[2]
avg_compare_enu = enu_coord[3]

###Interpolate Coordiantes

# Creates 3 Data Sets (north, east, up) with real, sim and comparison data and
# Interpolates Coordinates as the real and sim flight logs doesn't contain same logging frequency
# Real Data 10Hz, Sim Data 5Hz

##North
north = pd.DataFrame({'real':pd.Series.to_numpy(avg_real_enu.loc[:,'north'])},index=avg_real_enu.loc[:,'time'])
north_sim = pd.DataFrame({'sim':pd.Series.to_numpy(avg_sim_enu.loc[:,'north'])},index=avg_sim_enu.loc[:,'time'])
north_comp = pd.DataFrame({'comp':pd.Series.to_numpy(avg_compare_enu.loc[:,'north'])},index=avg_compare_enu.loc[:,'time'])
north = pd.concat([north,north_sim,north_comp],axis=1)

north['sim'] = north['sim'].interpolate()
north['real'] = north['real'].interpolate()
north['comp'] = north['comp'].interpolate()

##East
east = pd.DataFrame({'real':pd.Series.to_numpy(avg_real_enu.loc[:,'east'])},index=avg_real_enu.loc[:,'time'])
east_sim = pd.DataFrame({'sim':pd.Series.to_numpy(avg_sim_enu.loc[:,'east'])},index=avg_sim_enu.loc[:,'time'])
east_comp = pd.DataFrame({'comp':pd.Series.to_numpy(avg_compare_enu.loc[:,'east'])},index=avg_compare_enu.loc[:,'time'])
east = pd.concat([east,east_sim,east_comp],axis=1)

east['sim'] = east['sim'].interpolate()
east['real'] = east['real'].interpolate()
east['comp'] = east['comp'].interpolate()

##Up
up = pd.DataFrame({'real':pd.Series.to_numpy(avg_real_enu.loc[:,'up'])},index=avg_real_enu.loc[:,'time'])
up_sim = pd.DataFrame({'sim':pd.Series.to_numpy(avg_sim_enu.loc[:,'up'])},index=avg_sim_enu.loc[:,'time'])
up_comp = pd.DataFrame({'comp':pd.Series.to_numpy(avg_compare_enu.loc[:,'up'])},index=avg_compare_enu.loc[:,'time'])
up = pd.concat([up,up_sim,up_comp],axis=1)

up['sim'] = up['sim'].interpolate()
up['real'] = up['real'].interpolate()
up['comp'] = up['comp'].interpolate()

###########################################################################

####RESULT: FLIGHT PAHT PLOTTINGS

###########################################################################

###Plottings

##North vs. East (2D Plot)

fig = plt.figure()
ax = plt.axes()
ax.plot(avg_real_enu['north'],avg_real_enu['east'],'r--',label = 'Flight Test')
ax.plot(avg_sim_enu['north'],avg_sim_enu['east'],'g',label= 'Simulation')
ax.scatter(waypoints_enu['north'],waypoints_enu['east'],marker='o',color='black')
ax.set_title('Coordinates relative to Waypoint 1')
ax.set_ylabel('East [m]')
ax.set_xlabel('North [m]')
ax.legend(loc='upper right')
annotations=["WP-1","WP-2","WP-3","WP-4"]
for i, label in enumerate(annotations):
    plt.text(waypoints_enu['north'][i], waypoints_enu['east'][i],label)
ax.grid()
plt.show()

##North vs. East vs Up (3D Plot)

fig = plt.figure()
ax = plt.axes(projection='3d')
ax.plot3D(avg_real_enu['north'],avg_real_enu['east'],avg_real['height'],'r--',label='Flight Test')
ax.plot3D(avg_sim_enu['north'],avg_sim_enu['east'],avg_sim['height'],'g',label='Simulation')
ax.scatter(waypoints_enu['north'],waypoints_enu['east'],waypoints['height'],marker='o',color='black')
ax.set_title('Coordinates relative to Waypoint 1')
ax.set_xlabel('North [m]')
ax.set_ylabel('East [m]')
annotations=["WP-1","WP-2","WP-3","WP-4"]
for i, label in enumerate(annotations):
    ax.text(waypoints_enu['north'][i], waypoints_enu['east'][i],waypoints['height'][i],  label, zorder=1)
ax.legend()
plt.show()

##North vs. Time / East vs. Time (1D Plots)

fig, (ax1, ax2, ax3) = plt.subplots(3, 1)

fig.suptitle('ENU coordinates with respect to time')

ax2.plot(north['real'],'r--',label = 'Flight Test')
ax2.plot(north['sim'],'g',label = 'Simulation')
ax2.plot(north['comp'],'b',alpha = 0.2 ,label = 'Unadjusted')
ax2.set_xlabel('Time [s]')
ax2.set_ylabel('North [m]')
ax2.legend()
ax2.grid()

ax3.plot(east['real'],'r--',label = 'Flight Test')
ax3.plot(east['sim'],'g',label = 'Simulation')
ax3.plot(east['comp'],'b',alpha = 0.2 ,label = 'Unadjusted')
ax3.set_xlabel('Time [s]')
ax3.set_ylabel('East [m]')
ax3.grid()

ax1.plot(up['real'],'r--',label = 'Flight Test')
ax1.plot(up['sim'],'g',label = 'Simulation')
ax1.plot(up['comp'],'b',alpha = 0.2 ,label = 'Unadjusted')
ax1.set_xlabel('Time [s]')
ax1.set_ylabel('Height [m]')
ax1.grid()

plt.show()

###########################################################################

####RESULTS: RUN VARIATION INCLUDING PLOTS AND ERROR EVALUATION

###########################################################################

###Run Variation

# Analysis the amount of variation between the 3 runs. Performs post processing of the data and plottings.
# Some Data must be post processed as the post processing before mainly focuses on the averaged data where
# here each run must be post processed as well.

##Turn each run from GPS to ENU coordinates
gps_home_lat = 46.9765208
gps_home_long = 7.179124
gps_home_alt = 59

enu_real1= map.geodetic2enu(real_flights.loc[:,1],real_flights.loc[:,2],real_flights.loc[:,3],gps_home_lat,gps_home_long,gps_home_alt)
enu_real2= map.geodetic2enu(real_flights.loc[:,6],real_flights.loc[:,7],real_flights.loc[:,8],gps_home_lat,gps_home_long,gps_home_alt)
enu_real3= map.geodetic2enu(real_flights.loc[:,11],real_flights.loc[:,12],real_flights.loc[:,13],gps_home_lat,gps_home_long,gps_home_alt)

enu_sim1= map.geodetic2enu(sim_flights.loc[:,2],sim_flights.loc[:,3],sim_flights.loc[:,4],gps_home_lat,gps_home_long,gps_home_alt)
enu_sim2= map.geodetic2enu(sim_flights.loc[:,10],sim_flights.loc[:,11],sim_flights.loc[:,12],gps_home_lat,gps_home_long,gps_home_alt)
enu_sim3= map.geodetic2enu(sim_flights.loc[:,18],sim_flights.loc[:,19],sim_flights.loc[:,20],gps_home_lat,gps_home_long,gps_home_alt)

enu_comp1= map.geodetic2enu(compare_flights.loc[:,2],compare_flights.loc[:,3],compare_flights.loc[:,4],gps_home_lat,gps_home_long,gps_home_alt)
enu_comp2= map.geodetic2enu(compare_flights.loc[:,10],compare_flights.loc[:,11],compare_flights.loc[:,12],gps_home_lat,gps_home_long,gps_home_alt)
enu_comp3= map.geodetic2enu(compare_flights.loc[:,18],compare_flights.loc[:,19],compare_flights.loc[:,20],gps_home_lat,gps_home_long,gps_home_alt)

##Interpolation and Creation of North, East, Up data sets

#North Interpolation
north_sens =pd.DataFrame({'real':pd.Series.to_numpy(avg_real_enu.loc[:,'north'])},index=avg_real_enu.loc[:,'time'])
north1 = pd.DataFrame({'north1':pd.Series.to_numpy(enu_real1[1])},index=real_flights.loc[:,0])
north2 = pd.DataFrame({'north2':pd.Series.to_numpy(enu_real2[1])},index=real_flights.loc[:,0])
north3 = pd.DataFrame({'north3':pd.Series.to_numpy(enu_real3[1])},index=real_flights.loc[:,0])
north_avg_sim = pd.DataFrame({'sim':pd.Series.to_numpy(avg_sim_enu.loc[:,'north'])},index=avg_sim_enu.loc[:,'time'])
north1_sim = pd.DataFrame({'north1_sim':pd.Series.to_numpy(enu_sim1[1])},index=sim_flights.loc[:,0])
north2_sim = pd.DataFrame({'north2_sim':pd.Series.to_numpy(enu_sim2[1])},index=sim_flights.loc[:,0])
north3_sim = pd.DataFrame({'north3_sim':pd.Series.to_numpy(enu_sim3[1])},index=sim_flights.loc[:,0])
north_avg_comp = pd.DataFrame({'comp':pd.Series.to_numpy(avg_compare_enu.loc[:,'north'])},index=avg_compare_enu.loc[:,'time'])
north1_comp = pd.DataFrame({'north1_comp':pd.Series.to_numpy(enu_comp1[1])},index=compare_flights.loc[:,0])
north2_comp = pd.DataFrame({'north2_comp':pd.Series.to_numpy(enu_comp2[1])},index=compare_flights.loc[:,0])
north3_comp = pd.DataFrame({'north3_comp':pd.Series.to_numpy(enu_comp3[1])},index=compare_flights.loc[:,0])
north_sens = pd.concat([north_sens,north1,north2,north3,north_avg_sim,north1_sim,north2_sim,north3_sim,north_avg_comp,north1_comp,north2_comp,north3_comp],axis=1)

north_sens['real'] = north_sens['real'].interpolate()
north_sens['sim'] = north_sens['sim'].interpolate()
north_sens['comp'] = north_sens['comp'].interpolate()
north_sens['north1_sim'] = north_sens['north1_sim'].interpolate()
north_sens['north2_sim'] = north_sens['north2_sim'].interpolate()
north_sens['north3_sim'] = north_sens['north3_sim'].interpolate()
north_sens['north1'] = north_sens['north1'].interpolate()
north_sens['north2'] = north_sens['north2'].interpolate()
north_sens['north3'] = north_sens['north3'].interpolate()
north_sens['north1_comp'] = north_sens['north1_comp'].interpolate()
north_sens['north2_comp'] = north_sens['north2_comp'].interpolate()
north_sens['north3_comp'] = north_sens['north3_comp'].interpolate()

#East Interpolation
east_sens =pd.DataFrame({'real':pd.Series.to_numpy(avg_real_enu.loc[:,'east'])},index=avg_real_enu.loc[:,'time'])
east1 = pd.DataFrame({'east1':pd.Series.to_numpy(enu_real1[0])},index=real_flights.loc[:,0])
east2 = pd.DataFrame({'east2':pd.Series.to_numpy(enu_real2[0])},index=real_flights.loc[:,0])
east3 = pd.DataFrame({'east3':pd.Series.to_numpy(enu_real3[0])},index=real_flights.loc[:,0])
east_avg_sim = pd.DataFrame({'sim':pd.Series.to_numpy(avg_sim_enu.loc[:,'east'])},index=avg_sim_enu.loc[:,'time'])
east1_sim = pd.DataFrame({'east1_sim':pd.Series.to_numpy(enu_sim1[0])},index=sim_flights.loc[:,0])
east2_sim = pd.DataFrame({'east2_sim':pd.Series.to_numpy(enu_sim2[0])},index=sim_flights.loc[:,0])
east3_sim = pd.DataFrame({'east3_sim':pd.Series.to_numpy(enu_sim3[0])},index=sim_flights.loc[:,0])
east_avg_comp = pd.DataFrame({'comp':pd.Series.to_numpy(avg_compare_enu.loc[:,'east'])},index=avg_compare_enu.loc[:,'time'])
east1_comp = pd.DataFrame({'east1_comp':pd.Series.to_numpy(enu_comp1[0])},index=compare_flights.loc[:,0])
east2_comp = pd.DataFrame({'east2_comp':pd.Series.to_numpy(enu_comp2[0])},index=compare_flights.loc[:,0])
east3_comp = pd.DataFrame({'east3_comp':pd.Series.to_numpy(enu_comp3[0])},index=compare_flights.loc[:,0])
east_sens = pd.concat([east_sens,east1,east2,east3,east_avg_sim,east1_sim,east2_sim,east3_sim,east_avg_comp,east1_comp,east2_comp,east3_comp],axis=1)

east_sens['real'] = east_sens['real'].interpolate()
east_sens['sim'] = east_sens['sim'].interpolate()
east_sens['comp'] = east_sens['comp'].interpolate()
east_sens['east1_sim'] = east_sens['east1_sim'].interpolate()
east_sens['east2_sim'] = east_sens['east2_sim'].interpolate()
east_sens['east3_sim'] = east_sens['east3_sim'].interpolate()
east_sens['east1'] = east_sens['east1'].interpolate()
east_sens['east2'] = east_sens['east2'].interpolate()
east_sens['east3'] = east_sens['east3'].interpolate()
east_sens['east1_comp'] = east_sens['east1_comp'].interpolate()
east_sens['east2_comp'] = east_sens['east2_comp'].interpolate()
east_sens['east3_comp'] = east_sens['east3_comp'].interpolate()

#Up Interpolation
up_sens =pd.DataFrame({'real':pd.Series.to_numpy(avg_real_enu.loc[:,'up'])},index=avg_real_enu.loc[:,'time'])
up1 = pd.DataFrame({'up1':pd.Series.to_numpy(enu_real1[2])},index=real_flights.loc[:,0])
up2 = pd.DataFrame({'up2':pd.Series.to_numpy(enu_real2[2])},index=real_flights.loc[:,0])
up3 = pd.DataFrame({'up3':pd.Series.to_numpy(enu_real3[2])},index=real_flights.loc[:,0])
up_avg_sim = pd.DataFrame({'sim':pd.Series.to_numpy(avg_sim_enu.loc[:,'up'])},index=avg_sim_enu.loc[:,'time'])
up1_sim = pd.DataFrame({'up1_sim':pd.Series.to_numpy(enu_sim1[2])},index=sim_flights.loc[:,0])
up2_sim = pd.DataFrame({'up2_sim':pd.Series.to_numpy(enu_sim2[2])},index=sim_flights.loc[:,0])
up3_sim = pd.DataFrame({'up3_sim':pd.Series.to_numpy(enu_sim3[2])},index=sim_flights.loc[:,0])
up_avg_comp = pd.DataFrame({'comp':pd.Series.to_numpy(avg_compare_enu.loc[:,'up'])},index=avg_compare_enu.loc[:,'time'])
up1_comp = pd.DataFrame({'up1_comp':pd.Series.to_numpy(enu_comp1[2])},index=compare_flights.loc[:,0])
up2_comp = pd.DataFrame({'up2_comp':pd.Series.to_numpy(enu_comp2[2])},index=compare_flights.loc[:,0])
up3_comp = pd.DataFrame({'up3_comp':pd.Series.to_numpy(enu_comp3[2])},index=compare_flights.loc[:,0])
up_sens = pd.concat([up_sens,up1,up2,up3,up_avg_sim,up1_sim,up2_sim,up3_sim,up_avg_comp,up1_comp,up2_comp,up3_comp],axis=1)

up_sens['real'] = up_sens['real'].interpolate()
up_sens['sim'] = up_sens['sim'].interpolate()
up_sens['comp'] = up_sens['comp'].interpolate()
up_sens['up1_sim'] = up_sens['up1_sim'].interpolate()
up_sens['up2_sim'] = up_sens['up2_sim'].interpolate()
up_sens['up3_sim'] = up_sens['up3_sim'].interpolate()
up_sens['up1'] = up_sens['up1'].interpolate()
up_sens['up2'] = up_sens['up2'].interpolate()
up_sens['up3'] = up_sens['up3'].interpolate()
up_sens['up1_comp'] = up_sens['up1_comp'].interpolate()
up_sens['up2_comp'] = up_sens['up2_comp'].interpolate()
up_sens['up3_comp'] = up_sens['up3_comp'].interpolate()

##Distance Analysis between Mean and Run (3D Distance Deviation from Mean)

sd1 = np.std(((north_sens['real']-north_sens['north1'])**2+(east_sens['real']-east_sens['east1'])**2+(up_sens['real']-up_sens['up1'])**2)**0.5)
sd2 = np.std(((north_sens['real']-north_sens['north2'])**2+(east_sens['real']-east_sens['east2'])**2+(up_sens['real']-up_sens['up2'])**2)**0.5)
sd3 = np.std(((north_sens['real']-north_sens['north3'])**2+(east_sens['real']-east_sens['east3'])**2+(up_sens['real']-up_sens['up3'])**2)**0.5)
sd_real = (sd1+sd2+sd3)/3

mean1 = np.mean(((north_sens['real']-north_sens['north1'])**2+(east_sens['real']-east_sens['east1'])**2+(up_sens['real']-up_sens['up1'])**2)**0.5)
mean2 = np.mean(((north_sens['real']-north_sens['north2'])**2+(east_sens['real']-east_sens['east2'])**2+(up_sens['real']-up_sens['up2'])**2)**0.5)
mean3 = np.mean(((north_sens['real']-north_sens['north3'])**2+(east_sens['real']-east_sens['east3'])**2+(up_sens['real']-up_sens['up3'])**2)**0.5)

sd1_sim = np.std(((north_sens['sim']-north_sens['north1_sim'])**2+(east_sens['sim']-east_sens['east1_sim'])**2+(up_sens['sim']-up_sens['up1_sim'])**2)**0.5)
sd2_sim = np.std(((north_sens['sim']-north_sens['north2_sim'])**2+(east_sens['sim']-east_sens['east2_sim'])**2+(up_sens['sim']-up_sens['up2_sim'])**2)**0.5)
sd3_sim = np.std(((north_sens['sim']-north_sens['north3_sim'])**2+(east_sens['sim']-east_sens['east3_sim'])**2+(up_sens['sim']-up_sens['up3_sim'])**2)**0.5)
sd_sim = (sd1_sim+sd2_sim+sd3_sim)/3

mean1_sim = np.mean(((north_sens['sim']-north_sens['north1_sim'])**2+(east_sens['sim']-east_sens['east1_sim'])**2+(up_sens['sim']-up_sens['up1_sim'])**2)**0.5)
mean2_sim = np.mean(((north_sens['sim']-north_sens['north2_sim'])**2+(east_sens['sim']-east_sens['east2_sim'])**2+(up_sens['sim']-up_sens['up2_sim'])**2)**0.5)
mean3_sim = np.mean(((north_sens['sim']-north_sens['north3_sim'])**2+(east_sens['sim']-east_sens['east3_sim'])**2+(up_sens['sim']-up_sens['up3_sim'])**2)**0.5)

sd1_comp = np.std(((north_sens['comp']-north_sens['north1_comp'])**2+(east_sens['comp']-east_sens['east1_comp'])**2+(up_sens['comp']-up_sens['up1_comp'])**2)**0.5)
sd2_comp = np.std(((north_sens['comp']-north_sens['north2_comp'])**2+(east_sens['comp']-east_sens['east2_comp'])**2+(up_sens['comp']-up_sens['up2_comp'])**2)**0.5)
sd3_comp = np.std(((north_sens['comp']-north_sens['north3_comp'])**2+(east_sens['comp']-east_sens['east3_comp'])**2+(up_sens['comp']-up_sens['up3_comp'])**2)**0.5)

mean1_comp = np.mean(((north_sens['comp']-north_sens['north1_comp'])**2+(east_sens['comp']-east_sens['east1_comp'])**2+(up_sens['comp']-up_sens['up1_comp'])**2)**0.5)
mean2_comp = np.mean(((north_sens['comp']-north_sens['north2_comp'])**2+(east_sens['comp']-east_sens['east2_comp'])**2+(up_sens['comp']-up_sens['up2_comp'])**2)**0.5)
mean3_comp = np.mean(((north_sens['comp']-north_sens['north3_comp'])**2+(east_sens['comp']-east_sens['east3_comp'])**2+(up_sens['comp']-up_sens['up3_comp'])**2)**0.5)

run_dev = pd.DataFrame(
  {
    'mean_real':    [mean1,mean2,mean3],
    'sd_real':      [sd1,sd2,sd3],
    'mean_sim':     [mean1_sim,mean2_sim,mean3_sim],
    'sd_sim':       [sd1_sim,sd2_sim,sd3_sim],
    'mean_comp':    [mean1_comp,mean2_comp,mean3_comp],
    'sd_comp':      [sd1_comp, sd2_comp, sd3_comp]

  },
  index=['run1','run2','run3']
)

## Plotting of the Standard Deviation
fig, ax1 = plt.subplots()

barwidth = 0.25

r1 = [1,2,3]
r2 = [x + barwidth for x in r1]
r3 = [x - 2*barwidth for x in r2]

ax1.bar(r1, run_dev['sd_real'], width=barwidth, alpha=0.5, color='r',label = 'Flight Test')
ax1.bar(r2, run_dev['sd_sim'], width=barwidth, alpha=0.5, color='g',label = 'Simulation')
ax1.bar(r3, run_dev['sd_comp'], width=barwidth, alpha=0.5, color='b',label = 'Unadjusted')

ax1.set_ylabel('Standard Deviation [m]')
ax1.set_title('3D distance deviation between runs and run mean')
ax1.legend(loc = 'upper right')
ax1.axes.set_xticks(r1, ['Run 1','Run 2','Run 3'])
plt.show()

## Plotting Run Deviation from the mean including the 95% confidence interval over all runs

#Standard Deviation over all 3 runs for ENU coordinates (Sim & Real)
error_sd_east = (np.std(east_sens['real']-east_sens['east1'])+np.std(east_sens['real']-east_sens['east2'])+np.std(east_sens['real']-east_sens['east3']))/3
error_sd_east_sim = (np.std(east_sens['sim']-east_sens['east1_sim'])+np.std(east_sens['sim']-east_sens['east2_sim'])+np.std(east_sens['sim']-east_sens['east3_sim']))/3
error_sd_north = (np.std(north_sens['real']-north_sens['north1'])+np.std(north_sens['real']-north_sens['north2'])+np.std(north_sens['real']-north_sens['north3']))/3
error_sd_north_sim = (np.std(north_sens['sim']-north_sens['north1_sim'])+np.std(north_sens['sim']-north_sens['north2_sim'])+np.std(north_sens['sim']-north_sens['north3_sim']))/3
error_sd_up = (np.std(up_sens['real']-up_sens['up1'])+np.std(up_sens['real']-up_sens['up2'])+np.std(up_sens['real']-up_sens['up3']))/3
error_sd_up_sim = (np.std(up_sens['sim']-up_sens['up1_sim'])+np.std(up_sens['sim']-up_sens['up2_sim'])+np.std(up_sens['sim']-up_sens['up3_sim']))/3

#Plot
fig, (ax1, ax2, ax3) = plt.subplots(3, 1)

fig.suptitle('Deviations from mean, including 95% confidence intervall')

ax2.plot(north_sens['real']-north_sens['north1'],'r',alpha = 0.5, label = 'Flight Test')
ax2.plot(north_sens['real']-north_sens['north2'],'r',alpha = 0.3)
ax2.plot(north_sens['real']-north_sens['north3'],'r',alpha = 0.1)
ax2.fill_between(north_sens.index,(north_sens['real']-north_sens['real']+2*error_sd_north),(north_sens['real']-north_sens['real']-2*error_sd_north),color='r',alpha=0.1)
ax2.plot(north_sens['sim']-north_sens['north1_sim'],'g',alpha = 0.5, label = 'Simulation')
ax2.plot(north_sens['sim']-north_sens['north2_sim'],'g',alpha = 0.3)
ax2.plot(north_sens['sim']-north_sens['north3_sim'],'g',alpha = 0.1)
ax2.fill_between(north_sens.index,(north_sens['sim']-north_sens['sim']+2*error_sd_north_sim),(north_sens['sim']-north_sens['sim']-2*error_sd_north_sim),color='g',alpha=0.2)
ax2.set_xlabel('Time [s]')
ax2.set_ylabel('North [m]')
ax2.set_ylim([-3,3])
ax2.grid()

ax3.plot(east_sens['real']-east_sens['east1'],'r',alpha = 0.5, label = 'Flight Test')
ax3.plot(east_sens['real']-east_sens['east2'],'r',alpha = 0.3)
ax3.plot(east_sens['real']-east_sens['east3'],'r',alpha = 0.1)
ax3.fill_between(east_sens.index,(east_sens['real']-east_sens['real']+2*error_sd_east),(east_sens['real']-east_sens['real']-2*error_sd_east),color='r',alpha=0.1)
ax3.plot(east_sens['sim']-east_sens['east1_sim'],'g',alpha = 0.5, label = 'Simulation')
ax3.plot(east_sens['sim']-east_sens['east2_sim'],'g',alpha = 0.3)
ax3.plot(east_sens['sim']-east_sens['east3_sim'],'g',alpha = 0.1)
ax3.fill_between(east_sens.index,(east_sens['sim']-east_sens['sim']+2*error_sd_east_sim),(east_sens['sim']-east_sens['sim']-2*error_sd_east_sim),color='g',alpha=0.2)
ax3.set_xlabel('Time [s]')
ax3.set_ylabel('East [m]')
ax3.set_ylim([-3,3])
ax3.grid()

ax1.plot(up_sens['real']-up_sens['up1'],'r',alpha = 0.5, label = 'Flight Test')
ax1.plot(up_sens['real']-up_sens['up2'],'r',alpha = 0.3)
ax1.plot(up_sens['real']-up_sens['up3'],'r',alpha = 0.1)
ax1.fill_between(up_sens.index,(up_sens['real']-up_sens['real']+2*error_sd_up),(up_sens['real']-up_sens['real']-2*error_sd_up),color='r',alpha=0.1)
ax1.plot(up_sens['sim']-up_sens['up1_sim'],'g',alpha = 0.5, label = 'Simulation')
ax1.plot(up_sens['sim']-up_sens['up2_sim'],'g',alpha = 0.3)
ax1.plot(up_sens['sim']-up_sens['up3_sim'],'g',alpha = 0.1)
ax1.fill_between(up_sens.index,(up_sens['sim']-up_sens['sim']+2*error_sd_up_sim),(up_sens['sim']-up_sens['sim']-2*error_sd_up_sim),color='g',alpha=0.2)
ax1.set_xlabel('Time [s]')
ax1.set_ylabel('Height [m]')
ax1.legend()
ax1.grid()

plt.show()

##3D Distance Deviation (real vs sim) incl. 95% confidence intervall

err = ((north['real']-north['sim'])**2+(east['real']-east['sim'])**2+(up['real']-up['sim'])**2)**0.5
sd_err = np.sqrt(sd_real**2+(-sd_sim)**2)

#North vs. Time / East vs. Time / Up vs. Tim / Distance Error (1D Plots)

fig, (ax1, ax2, ax3, ax4) = plt.subplots(4, 1)

fig.suptitle('ENU coordinates and spatial difference')

ax2.plot(north['real'],'r--',label = 'Flight Test')
ax2.plot(north['sim'],'g',label = 'Simulation')
ax2.set_xlabel('Time [s]')
ax2.set_ylabel('North [m]')
ax2.grid()

ax3.plot(east['real'],'r--',label = 'Flight Test')
ax3.plot(east['sim'],'g',label = 'Simulation')
ax3.set_xlabel('Time [s]')
ax3.set_ylabel('East [m]')
ax3.grid()

ax1.plot(up['real'],'r--',label = 'Flight Test')
ax1.plot(up['sim'],'g',label = 'Simulation')
ax1.set_xlabel('Time [s]')
ax1.set_ylabel('Height [m]')
ax1.legend()
ax1.grid()

ax4.plot(err.index,err,'r')
ax4.plot(err.index,np.zeros(np.shape(err.index)[0]),color='black')
ax4.fill_between(err.index,(err+2*(sd_err)),(err-2*(sd_err)),color='r',alpha = 0.2)
ax4.set_xlabel('Time [s]')
ax4.set_ylabel('Difference [m]')
ax4.grid()

plt.show()

##Distance Errors

#Mean Error Spacial Distance (MAE)

actual = ((north['real'])**2+(east['real'])**2+(up['real'])**2)**0.5
simulated = ((north['sim'])**2+(east['sim'])**2+(up['sim'])**2)**0.5

mae = np.mean(np.abs(actual-simulated))
print(mae)

#MAPE
per_err = (pd.Series.to_numpy(actual)-pd.Series.to_numpy(simulated))/pd.Series.to_numpy(actual)
mape = (sum(abs(per_err))/len(per_err))*100
print(mape)

#SDMAE

sdmae = (mae/np.std(actual))*100
print(sdmae)

#MSE
sqr_err = (pd.Series.to_numpy(actual)-pd.Series.to_numpy(simulated))**2
mse = sum(sqr_err)/len(sqr_err)
print(mse)

#Summary
print('MAE:',mae, 'MAPE:',mape, 'MSE:',mse, 'SDMAE', sdmae)

sd_literature = np.std(np.array([99.46,99.03,98.78,98.69,99.72,100.25,100.17,98.61,99.84,99.3])/3.33)


import requests
import pandas as pd
from bs4 import BeautifulSoup
from datetime import datetime

keyword = input("Kata Kunci: ")
total_pages = input("Jumlah Artikel [10~100~1000]: ")
articles = []

for page in range(0, int(total_pages)):
    url = f"https://search.okezone.com/searchsphinx/loaddata/article/{keyword}/{page}"

    page = requests.get(url)
    soup = BeautifulSoup(page.text, "html5lib")

    article_scrape = soup.find("div", class_="listnews")

    published_time = article_scrape.find("div", class_="tgl").get_text()
    title = article_scrape.find("div", class_="title").get_text()
    href = article_scrape.a["href"]

    page_more = requests.get(href)
    soup_more = BeautifulSoup(page_more.text, "html5lib")

    content = ""
    paragraphs = soup_more.find("div", {"id": "contentx", "class": "read"}).find_all("p")
    for p in paragraphs:
        paragraph = p.get_text(strip=True)
        if not paragraph.endswith("*"):
            content += paragraph

    journalis = list(soup_more.find("div", class_="namerep").stripped_strings)[0].strip(",")

    articles.append({
        "published_time": published_time,
        "title": title,
        "content": content,
        "journalis": journalis,
        "url": href
    })

df = pd.DataFrame(articles)
# df.style.hide_index() | (this method has been DEPRECATED)
time_scrape = datetime.now().strftime("%m%d%Y_%H%M%S")
df.to_csv(f"result_by-query_{keyword}_{time_scrape}.csv", index=False)


import os
import re
import nltk
import numpy as np
from keras.utils import to_categorical
from keras.models import Sequential, load_model
from keras.layers import Embedding, LSTM, Dense, SimpleRNN, GRU, Masking, TimeDistributed
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer

base_path = os.path.abspath('English Literature.txt')

with open(base_path, encoding='utf-8') as f:
    sample = f.read()

def format_data(input_string):
    clean_string = re.sub(r'\((\d+)\)', r'', input_string)

    clean_string = re.sub(r'\s\s', ' ', clean_string)
    return clean_string

formatted_string = format_data(sample)

# integer encode text
regular_exp = nltk.RegexpTokenizer(r"\w+")
sent = regular_exp.tokenize(formatted_string)
tokenizer = Tokenizer()
tokenizer.fit_on_texts(sent)
encoded = tokenizer.texts_to_sequences([sent])[0]

sentence_tokenize = nltk.tokenize.sent_tokenize(formatted_string)
print(sentence_tokenize)

sent_len = 16
i = 0
input_X = []
output_y = []

X = []
y1 = []
padded_sent = []
for i in range(len(sentence_tokenize)):
    rem_exp = nltk.RegexpTokenizer(r"\w+")
    sentences = rem_exp.tokenize(sentence_tokenize[i])
    encoded = tokenizer.texts_to_sequences([sentences])[0]
    if (len(encoded) < sent_len) or (len(encoded) > sent_len):
        padded_sent = pad_sequences([encoded], maxlen=sent_len, dtype='int32', padding='pre', truncating='pre',
                                    value=0.0)
    input_X = padded_sent[0][0:(sent_len - 1)]
    output_y = padded_sent[0][1:sent_len]
    X.append(input_X)
    y1.append(output_y)

# integer encode text
X = np.array(X)
num_y = np.array(y1)
print(X, num_y)
max_words = 10

vocab_size = len(tokenizer.word_index) + 1
print('Vocabulary Size: %d' % vocab_size)

# one hot encode outputs
y = to_categorical(num_y, num_classes=vocab_size)

# define the network architecture: a embedding followed by LSTM
embedding_size = 100
rnn_size = 50
model1 = Sequential()
model1.add(Embedding(vocab_size, embedding_size, input_length=15))
model1.add(Masking(mask_value=0.0))
model1.add(SimpleRNN(rnn_size, return_sequences=True))
model1.add(TimeDistributed(Dense(vocab_size, activation='softmax')))
model1.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model1.summary())

# fit network
model1.fit(X, y, epochs=10, verbose=2, batch_size=1)
model1.save("my_rnn_model.h5")


import logging

import telegram

__all__ = ("TelegramService")

logger = logging.getLogger(__name__)

class TelegramService:
    def __init__(self, token: str, chat_id: str) -> None:
        self.__bot = telegram.Bot(token=token)
        self.__chat_id = chat_id

    def send_text_message(self, text: str):
        logger.debug(f"Telegram: send  message {text}")
        self.__bot.sendMessage(chat_id=self.__chat_id, text=text)

    def send_image_message(self, image_path: str, text_description: str = ""):
        logger.debug(f"Telegram: send image: {image_path} with description {text_description}")
        self.__bot.send_photo(
            chat_id=self.__chat_id,
            photo=open(image_path, 'rb'),
            caption=text_description
        )