from __future__ import print_function
import keras
from keras import Model, Input
from keras.initializers import glorot_uniform
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization, Add, AveragePooling2D
from keras.layers import Conv2D, MaxPooling2D
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import RMSprop, SGD, Adam
from keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau
from keras.models import Sequential
from keras.layers.normalization import BatchNormalization
from keras.layers.convolutional import Conv2D, MaxPooling2D, ZeroPadding2D
from keras.layers.advanced_activations import ELU
from keras.layers.core import Activation, Flatten, Dropout, Dense
import os

num_classes = 6
img_rows, img_cols = 48, 48
batch_size = 1

train_data_dir = './data/train2'
validation_data_dir = './data/validation'

# Let's use some data augmentaiton
train_datagen = ImageDataGenerator(
    rescale=1. / 255,
    rotation_range=30,
    shear_range=0.3,
    zoom_range=0.3,
    width_shift_range=0.4,
    height_shift_range=0.4,
    horizontal_flip=True,
    fill_mode='nearest')

validation_datagen = ImageDataGenerator(rescale=1. / 255)

train_generator = train_datagen.flow_from_directory(
    train_data_dir,
    color_mode='grayscale',
    target_size=(img_rows, img_cols),
    batch_size=batch_size,
    class_mode='categorical',
    shuffle=True)

validation_generator = validation_datagen.flow_from_directory(
    validation_data_dir,
    color_mode='grayscale',
    target_size=(img_rows, img_cols),
    batch_size=batch_size,
    class_mode='categorical',
    shuffle=True)

# model = Sequential()

def identity_block(X, f, filters, stage, block):
    """
    Implementation of the identity block as defined in Figure 3

    Arguments:
    X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)
    f -- integer, specifying the shape of the middle CONV's window for the main path
    filters -- python list of integers, defining the number of filters in the CONV layers of the main path
    stage -- integer, used to name the layers, depending on their position in the network
    block -- string/character, used to name the layers, depending on their position in the network

    Returns:
    X -- output of the identity block, tensor of shape (n_H, n_W, n_C)
    """

    # defining name basis
    conv_name_base = 'res' + str(stage) + block + '_branch'
    bn_name_base = 'bn' + str(stage) + block + '_branch'

    # Retrieve Filters
    F1, F2, F3 = filters

    # Save the input value. You'll need this later to add back to the main path.
    X_shortcut = X

    # First component of main path
    X = Conv2D(filters=F1, kernel_size=(1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2a',
               kernel_initializer=glorot_uniform(seed=0))(X)
    X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X)
    X = Activation('relu')(X)

    # Second component of main path (≈3 lines)
    X = Conv2D(filters=F2, kernel_size=(f, f), strides=(1, 1), padding='same', name=conv_name_base + '2b',
               kernel_initializer=glorot_uniform(seed=0))(X)
    X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X)
    X = Activation('relu')(X)

    # Third component of main path (≈2 lines)
    X = Conv2D(filters=F3, kernel_size=(1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2c',
               kernel_initializer=glorot_uniform(seed=0))(X)
    X = BatchNormalization(axis=3, name=bn_name_base + '2c')(X)

    # Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines)
    X = Add()([X, X_shortcut])
    X = Activation('relu')(X)

    return X

def convolutional_block(X, f, filters, stage, block, s=2):
    """
    Implementation of the convolutional block as defined in Figure 4

    Arguments:
    X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)
    f -- integer, specifying the shape of the middle CONV's window for the main path
    filters -- python list of integers, defining the number of filters in the CONV layers of the main path
    stage -- integer, used to name the layers, depending on their position in the network
    block -- string/character, used to name the layers, depending on their position in the network
    s -- Integer, specifying the stride to be used

    Returns:
    X -- output of the convolutional block, tensor of shape (n_H, n_W, n_C)
    """

    # defining name basis
    conv_name_base = 'res' + str(stage) + block + '_branch'
    bn_name_base = 'bn' + str(stage) + block + '_branch'

    # Retrieve Filters
    F1, F2, F3 = filters

    # Save the input value
    X_shortcut = X

    ##### MAIN PATH #####
    # First component of main path
    X = Conv2D(F1, (1, 1), strides=(s, s), name=conv_name_base + '2a', kernel_initializer=glorot_uniform(seed=0))(X)
    X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X)
    X = Activation('relu')(X)

    # Second component of main path (≈3 lines)
    X = Conv2D(filters=F2, kernel_size=(f, f), strides=(1, 1), padding='same', name=conv_name_base + '2b',
               kernel_initializer=glorot_uniform(seed=0))(X)
    X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X)
    X = Activation('relu')(X)

    # Third component of main path (≈2 lines)
    X = Conv2D(filters=F3, kernel_size=(1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2c',
               kernel_initializer=glorot_uniform(seed=0))(X)
    X = BatchNormalization(axis=3, name=bn_name_base + '2c')(X)

    ##### SHORTCUT PATH #### (≈2 lines)
    X_shortcut = Conv2D(filters=F3, kernel_size=(1, 1), strides=(s, s), padding='valid', name=conv_name_base + '1',
                        kernel_initializer=glorot_uniform(seed=0))(X_shortcut)
    X_shortcut = BatchNormalization(axis=3, name=bn_name_base + '1')(X_shortcut)

    # Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines)
    X = Add()([X, X_shortcut])
    X = Activation('relu')(X)

    return X

def ResNet50(input_shape=(64, 64, 3), classes=6):
    """
    Implementation of the popular ResNet50 the following architecture:
    CONV2D -> BATCHNORM -> RELU -> MAXPOOL -> CONVBLOCK -> IDBLOCK*2 -> CONVBLOCK -> IDBLOCK*3
    -> CONVBLOCK -> IDBLOCK*5 -> CONVBLOCK -> IDBLOCK*2 -> AVGPOOL -> TOPLAYER

    Arguments:
    input_shape -- shape of the images of the dataset
    classes -- integer, number of classes

    Returns:
    model -- a Model() instance in Keras
    """

    # Define the input as a tensor with shape input_shape
    X_input = Input(input_shape)

    # Zero-Padding
    X = ZeroPadding2D((3, 3))(X_input)

    # Stage 1
    X = Conv2D(64, (7, 7), strides=(2, 2), name='conv1', kernel_initializer=glorot_uniform(seed=0))(X)
    X = BatchNormalization(axis=3, name='bn_conv1')(X)
    X = Activation('relu')(X)
    X = MaxPooling2D((3, 3), strides=(2, 2))(X)

    # Stage 2
    X = convolutional_block(X, f=3, filters=[64, 64, 256], stage=2, block='a', s=1)
    X = identity_block(X, 3, [64, 64, 256], stage=2, block='b')
    X = identity_block(X, 3, [64, 64, 256], stage=2, block='c')

    ### START CODE HERE ###

    # Stage 3 (≈4 lines)
    X = convolutional_block(X, f=3, filters=[128, 128, 512], stage=3, block='a', s=2)
    X = identity_block(X, 3, [128, 128, 512], stage=3, block='b')
    X = identity_block(X, 3, [128, 128, 512], stage=3, block='c')
    X = identity_block(X, 3, [128, 128, 512], stage=3, block='d')

    # Stage 4 (≈6 lines)
    X = convolutional_block(X, f=3, filters=[256, 256, 1024], stage=4, block='a', s=2)
    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='b')
    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='c')
    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='d')
    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='e')
    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='f')

    # Stage 5 (≈3 lines)
    X = convolutional_block(X, f=3, filters=[512, 512, 2048], stage=5, block='a', s=2)
    X = identity_block(X, 3, [512, 512, 2048], stage=5, block='b')
    X = identity_block(X, 3, [512, 512, 2048], stage=5, block='c')

    # AVGPOOL (≈1 line). Use "X = AveragePooling2D(...)(X)"
    X = AveragePooling2D((2, 2), name="avg_pool")(X)

    ### END CODE HERE ###

    # output layer
    X = Flatten()(X)
    X = Dense(classes, activation='softmax', name='fc' + str(classes), kernel_initializer=glorot_uniform(seed=0))(X)

    # Create model
    model = Model(inputs=X_input, outputs=X, name='ResNet50')

    return model

model = ResNet50(input_shape = (img_rows, img_cols, 1), classes = num_classes)

print(model.summary())

# checkpoint = ModelCheckpoint("emotion_ResNet.h5",
#                              monitor="val_loss",
#                              mode="min",
#                              save_best_only = True,
#                              verbose=1)
#
# earlystop = EarlyStopping(monitor = 'val_loss',
#                           min_delta = 0,
#                           patience = 4,
#                           verbose = 1,
#                           restore_best_weights = True)
#
# reduce_lr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, patience = 3, verbose = 1, min_delta = 0.0001)
#
# # we put our call backs into a callback list
# callbacks = [earlystop, checkpoint, reduce_lr]
#
# # We use a very small learning rate
# model.compile(loss = 'categorical_crossentropy',
#               optimizer = Adam(lr=0.001),
#               metrics = ['accuracy'])
#
# nb_train_samples = 29045
# nb_validation_samples = 3534
# epochs = 10
#
# history = model.fit_generator(
#     train_generator,
#     steps_per_epoch = nb_train_samples // batch_size,
#     epochs = epochs,
#     callbacks = callbacks,
#     validation_data = validation_generator,
#     validation_steps = nb_validation_samples // batch_size)
#
# model.save('Resnet50_save.model')


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Project ：ML_project
@Product_name ：PyCharm
@File ：main.py
@Author ：RockJim
@Date ：2023/8/21 22:14
@Description ：测试的主函数
@Version ：1.0
"""
import os

import numpy as np
import torch
from matplotlib import pyplot as plt
from pymoo.algorithms.moo.nsga2 import NSGA2
from pymoo.factory import get_sampling, get_crossover, get_mutation
from pymoo.optimize import minimize
from torch import nn, optim
from torch.utils.tensorboard import SummaryWriter

from config.log_config import log
from config.config import DATASET_NAME, ROOT_PATH
from dataset.data_load import load_data, data_processing
from evaluate.evaluate import evaluate_prediction
from model.ann.ann import Net
from model.ann.genetic_algorithm import GeneticAlogrithm
from model.nsga.nsga_2 import MultiObjectiveProblem
from config.config import log_dir
from pymoo.operators.sampling.rnd import FloatRandomSampling
from pymoo.operators.crossover.sbx import SBX
from pymoo.operators.mutation.pm import PolynomialMutation

logger = log().getLogger(__name__)

def data_load():
    # 加载数据集
    x_data, y_data = load_data(os.path.join(ROOT_PATH, 'resource', DATASET_NAME))
    # 数据归一化处理
    x_train, x_test, y_train, y_test = data_processing(x_data, y_data)

    # 将这些值转为tensor格式
    X_train = torch.tensor(x_train, dtype=torch.float32)
    Y_train = torch.tensor(y_train, dtype=torch.float32)
    X_test = torch.tensor(x_test, dtype=torch.float32)
    Y_test = torch.tensor(y_test, dtype=torch.float32)
    return X_train, Y_train, X_test, Y_test

if __name__ == '__main__':
    input_features = 12  # 单个输入样本的变量个数。
    output_features = 6  # 单个输出的变量个数。
    hidden_layer_range = [2, 128]
    learning_rate = 0.1
    epochs = 2
    population_size = 20
    num_generations = 1
    mutation_rate = 0.1
    # 加载数据
    X_train, Y_train, X_test, Y_test = data_load()

    # ga = GeneticAlogrithm(population_size, num_generations, mutation_rate, X_train, Y_train,
    #                       X_test, Y_test, input_features, hidden_layer_range, output_features)
    # # 选择最佳模型结构
    # best_hidden_dims = ga.forward()
    best_hidden_dims = [2, 16]
    logger.info("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
    logger.info("最佳隐藏层参数为：{}".format(best_hidden_dims))
    writer = SummaryWriter(log_dir=log_dir)
    # 创建模型
    best_model = Net(input_features, best_hidden_dims, output_features)
    logger.info("最佳模型结构如下：\t\n{}".format(best_model))

    mseloss = nn.MSELoss()  # 均方误差对象作为损失函数
    sgd = optim.SGD(best_model.parameters(), lr=learning_rate)  # 用随机梯度下降对象作为优化函数。
    for epoch in range(epochs * 100):
        y_predict = best_model(X_train)  # 前向传播，自动调用Module的forward方法
        loss = mseloss(y_predict, Y_train)  # 计算损失函数
        loss.backward()  # 反向传播
        sgd.step()  # 更新参数(weight和bias)
        sgd.zero_grad()  # 清零梯度数据
        logger.info(
            f"epoch:{epoch}, loss:{loss:.6f}"
        )
        writer.add_scalar('Loss/train', loss.item(), epoch)

    writer.add_graph(best_model, X_train[0])
    writer.close()
    logger.info("在测试集上进行评估………………")
    best_model.eval()
    with torch.no_grad():
        predictions = best_model(X_test)
    pred = predictions.detach().numpy()

    # 计算评估指标
    evaluate_prediction(Y_test.detach().numpy(), pred)

    # 多目标优化参数的约束条件
    constr_list = [np.array([0.5, 0.5, 0.5, 3.0, 0.20, 0.20,
                             0.20, 0.20, 0.2, 0.2, 0.2, 0.2]),
                   np.array([1.0, 1.5, 1.5, 4.0, 0.40, 0.80,
                             0.40, 0.40, 0.3, 0.3, 0.4, 0.4])]

    # 初始化多目标优化问题
    problem = MultiObjectiveProblem(best_model, input_features, output_features, constr_list)

    # 定义优化算法的参数和配置
    algorithm = NSGA2(
        pop_size=100,  # 种群个数
        sampling=FloatRandomSampling(),
        crossover=SBX(prob=0.9, eta=15),
        mutation=PolynomialMutation(eta=20),
        eliminate_duplicates=True
    )

    # 执行多目标优化
    res = minimize(
        problem,
        algorithm,
        ('n_gen', 100),  # 迭代次数
        seed=1,
        verbose=True
    )

    # # 输出优化结果
    # print("Final population:")
    # for solution in res.X:
    #     print(solution)

    # 获得优化结果
    optimal_solutions = res.X
    optimal_objectives = res.F

    # print("Optimal Solutions:")
    # print(optimal_solutions)
    # print("Optimal Objectives:")
    # print(optimal_objectives)
    # 创建雷达图
    fig, ax = plt.subplots(subplot_kw={'projection': 'polar'})

    # 创建角度
    theta = np.linspace(0, 2 * np.pi, problem.n_obj, endpoint=False)

    # 绘制雷达图
    for i in range(len(optimal_objectives)):
        ax.plot(theta, optimal_objectives[i])

    # 设置角度刻度标签
    ax.set_xticks(theta)
    ax.set_xticklabels(["F1", "F2", "F3", "F4", "F5", "F6"])

    # 添加网格线
    ax.grid(True)

    # 添加标题和图例
    plt.title("Pareto Front")
    plt.legend(["Solution " + str(i+1) for i in range(len(optimal_solutions))])

    # 设置图例显示的位置
    plt.legend(loc='lower right')

    # 保存图片
    plt.savefig('my_plot.png', dpi=300, bbox_inches='tight', format='jpg')


from dataclasses import dataclass
from functools import total_ordering
import heapq
import math
from typing import Dict, Union

@dataclass
@total_ordering
class WeightedVertex:
    weight: float
    neighbour: str
    vertex: str

    def __eq__(self, other: object) -> bool:
        return self.weight == other.weight
    def __lt__(self, other: object) -> bool:
        return self.weight == other.weight

def make_link(graph, node1, node2, weight):
    if node1 not in graph:
        graph[node1] = {}
    if node2 not in graph:
        graph[node2] = {}
    graph[node1][node2] = {"weight": weight}
    graph[node2][node1] = {"weight": weight}

def find_smallest_path_from(vertex: str, in_graph: Dict[str, Dict[str, Dict[str, Union[int, bool]]]]) -> Dict[str, int]:
    neighbours = []
    path_map = {node: {"w": math.inf, "p": []} for node in in_graph}
    path_map[vertex]["w"] = 0

    for neighbour in in_graph[vertex]:
        weight = in_graph[vertex][neighbour]["weight"]
        in_graph[vertex][neighbour]["visited"] = True
        heapq.heappush(neighbours, (weight, neighbour, vertex))

    while neighbours:
        weight, to_vertex, from_vertex = heapq.heappop(neighbours)
        if path_map[to_vertex]["w"] > weight + path_map[from_vertex]["w"]:
            path_map[to_vertex]["w"] = weight + path_map[from_vertex]["w"]
            path_map[to_vertex]["p"] = path_map[from_vertex]["p"] + [from_vertex]

        for neighbour in in_graph[to_vertex]:
            if not in_graph[to_vertex][neighbour].get("visited"):
                weight = in_graph[to_vertex][neighbour]["weight"]
                in_graph[to_vertex][neighbour]["visited"] = True
                heapq.heappush(neighbours, (weight, neighbour, to_vertex))

    return path_map

if __name__ == "__main__":
    import sys
    import copy
    graph = {}
    fn = lambda x: int(x) if x.isdigit() else x
    for line in sys.stdin:
        node1, node2, weight = list(map(fn, line.split()))
        make_link(graph, node1, node2, weight)

    for key in graph:
        weight_map = find_smallest_path_from(key, copy.deepcopy(graph))
        print(key, weight_map)


from starlite import MediaType, OpenAPIConfig, Starlite, get

from app.controllers.search import SearchController
from app.controllers.vehicles import VehicleController
from app.services.tile38 import tile38

APP_NAME = "starlite-tile38"

async def shutdown_tile38() -> None:
    """Closes the tile38 connection"""
    await tile38.quit()

@get(path="/healthz", media_type=MediaType.TEXT)
def health_check() -> str:
    return "OK"

app = Starlite(
    route_handlers=[SearchController, VehicleController, health_check],
    openapi_config=OpenAPIConfig(title=f"{APP_NAME}", version="1.0.0"),
    on_shutdown=[shutdown_tile38],
)


import numpy as np
from pathlib import Path
import shutil
from argparse import ArgumentParser
from pyslam.metrics import TrajectoryMetrics
from pyslam.visualizers import TrajectoryVisualizer
import pandas as pd
from liegroups import SE3
import settings
import matplotlib.pyplot as plt
import pdb

def get_metrics(ins_df, ro_df, kf_aux1_df, kf_aux2_df, kf_aux3_df,  output_path):
    # Kalman filtered data only starts from second frame, so need to crop the others
    ins_df = ins_df.iloc[1:].reset_index(drop=True)
    ro_df = ro_df.iloc[1:].reset_index(drop=True)

    gt_se3s, gt_timestamps = get_poses_and_timestamps_from_df(ins_df)
    aux0_se3s, aux0_timestamps = get_poses_and_timestamps_from_df(ro_df)
    aux1_se3s, aux1_timestamps = get_poses_and_timestamps_from_df(kf_aux1_df)
    aux2_se3s, aux2_timestamps = get_poses_and_timestamps_from_df(kf_aux2_df)
    aux3_se3s, aux3_timestamps = get_poses_and_timestamps_from_df(kf_aux3_df)

    # check timestamps all line up before proceeding
    assert(gt_timestamps.iloc[0] == aux0_timestamps.iloc[0])
    assert(gt_timestamps.iloc[0] == aux1_timestamps.iloc[0])
    assert(gt_timestamps.iloc[0] == aux2_timestamps.iloc[0])
    assert(gt_timestamps.iloc[0] == aux3_timestamps.iloc[0])

    # making global poses from the relative poses
    gt_global_se3s = [np.identity(4)]
    for i in range(1, len(gt_se3s)):
        gt_global_se3s.append(gt_global_se3s[i - 1] @ gt_se3s[i])
    gt_global_SE3s = get_se3s_from_raw_se3s(gt_global_se3s)

    aux0_global_se3s = [np.identity(4)]
    for i in range(1, len(aux0_se3s)):
        aux0_global_se3s.append(aux0_global_se3s[i - 1] @ aux0_se3s[i])
    aux0_global_SE3s = get_se3s_from_raw_se3s(aux0_global_se3s)

    aux1_global_se3s = [np.identity(4)]
    for i in range(1, len(aux1_se3s)):
        aux1_global_se3s.append(aux1_global_se3s[i - 1] @ aux1_se3s[i])
    aux1_global_SE3s = get_se3s_from_raw_se3s(aux1_global_se3s)

    aux2_global_se3s = [np.identity(4)]
    for i in range(1, len(aux2_se3s)):
        aux2_global_se3s.append(aux2_global_se3s[i - 1] @ aux2_se3s[i])
    aux2_global_SE3s = get_se3s_from_raw_se3s(aux2_global_se3s)

    aux3_global_se3s = [np.identity(4)]
    for i in range(1, len(aux3_se3s)):
        aux3_global_se3s.append(aux3_global_se3s[i - 1] @ aux3_se3s[i])
    aux3_global_SE3s = get_se3s_from_raw_se3s(aux3_global_se3s)

    segment_lengths = [100, 200, 300, 400, 500, 600, 700, 800]

    tm_gt_aux0 = TrajectoryMetrics(gt_global_SE3s, aux0_global_SE3s)
    tm_gt_aux1 = TrajectoryMetrics(gt_global_SE3s, aux1_global_SE3s)
    tm_gt_aux2 = TrajectoryMetrics(gt_global_SE3s, aux2_global_SE3s)
    tm_gt_aux3 = TrajectoryMetrics(gt_global_SE3s, aux3_global_SE3s)
    # print_trajectory_metrics(tm_gt_aux0, segment_lengths, data_name="RO")

    # Save metrics to text file and make plots
    output_path_for_metrics = output_path / "trajectory_metrics"
    if output_path_for_metrics.exists() and output_path_for_metrics.is_dir():
        shutil.rmtree(output_path_for_metrics)
    output_path_for_metrics.mkdir(parents=True)

    save_trajectory_metrics_to_file(
        output_path_for_metrics, {"RO": tm_gt_aux0, settings.AUX1_NAME: tm_gt_aux1,
                                  settings.AUX2_NAME: tm_gt_aux2, settings.AUX3_NAME: tm_gt_aux3},
        segment_lengths)

    visualiser = TrajectoryVisualizer(
        {"RO": tm_gt_aux0, settings.AUX1_NAME: tm_gt_aux1,
         settings.AUX2_NAME: tm_gt_aux2, settings.AUX3_NAME: tm_gt_aux3})
    visualiser.plot_segment_errors(figsize=(10, 4), segs=segment_lengths, legend_fontsize=8,
                                   outfile="%s%s" % (output_path_for_metrics, "/segment_errors.pdf"))
    visualiser.plot_topdown(which_plane='yx',  # this was yx, a custom flip to conform to MRG convention, instead of xy
                            outfile="%s%s" % (output_path_for_metrics, "/topdown.pdf"), figsize=(6, 6))

def save_trajectory_metrics_to_file(output_path, tm_gt_est_dict, segment_lengths):
    print("Calculating trajectory metrics to save to file...")
    results_file = output_path / "trajectory_metrics.txt"
    with open(results_file, "w") as text_file:
        for data_name, tm_gt_est in tm_gt_est_dict.items():
            print(f"{data_name} metrics:", file=text_file)
            print(
                f"Segment error - lengths (m), translation (m), rotation (deg) \n {tm_gt_est.segment_errors(segment_lengths, rot_unit='deg')[1]}",
                file=text_file)
            print(
                f"Mean segment error: translation (m), rotation (deg) \n {np.mean(tm_gt_est.segment_errors(segment_lengths, rot_unit='deg')[1], axis=0)[1:]} \n",
                file=text_file)

def print_trajectory_metrics(tm_gt_est, segment_lengths, data_name="this"):
    print("\nTrajectory Metrics for", data_name, "set:")
    print("average segment_error:", np.mean(tm_gt_est.segment_errors(
        segment_lengths, rot_unit='deg')[1], axis=0)[1:])
    print("mean_err:", tm_gt_est.mean_err(rot_unit='deg'))
    print("rms_err:", tm_gt_est.rms_err(rot_unit='deg'))

def get_poses_and_timestamps_from_df(df):
    se3s = []
    for i in range(len(df.x)):
        th = df.yaw.iloc[i]
        pose = np.identity(4)
        pose[0, 0] = np.cos(th)
        pose[0, 1] = -np.sin(th)
        pose[1, 0] = np.sin(th)
        pose[1, 1] = np.cos(th)
        pose[0, 3] = df.x.iloc[i]
        pose[1, 3] = df.y.iloc[i]
        se3s.append(pose)
    return se3s, df.timestamp

def get_se3s_from_raw_se3s(raw_se3s):
    """
    Transform from raw se3 matrices into fancier SE3 type
    """
    se3s = []
    for pose in raw_se3s:
        se3s.append(SE3.from_matrix(np.asarray(pose)))
    return se3s

def plot_x_y_yaw(output_file, ins_df, ro_df, kf_aux1_df, kf_aux2_df, kf_aux3_df):
    _, axes = plt.subplots(nrows=3, ncols=1, figsize=(20, 10))

    ax = axes[0]
    ax.plot(ins_df.x, ',',
            color=settings.colours.ins, label="INS")
    ax.plot(ro_df.x, ',',
            color=settings.colours.ro, label="RO")
    ax.plot(kf_aux1_df.x, ',',
            color=settings.colours.kf_aux1, label="KF-aux1")
    ax.plot(kf_aux2_df.x, ',',
            color=settings.colours.kf_aux2, label="KF-aux2")
    ax.plot(kf_aux3_df.x, ',',
            color=settings.colours.kf_aux3, label="KF-aux3")
    ax.set_title("Title")
    ax.set_xlabel("x label")
    ax.set_ylabel("y label")
    ax.legend()
    ax.grid()

    ax = axes[1]
    ax.plot(ins_df.y, ',',
            color=settings.colours.ins, label="INS")
    ax.plot(ro_df.y, ',',
            color=settings.colours.ro, label="RO")
    ax.plot(kf_aux1_df.y, ',',
            color=settings.colours.kf_aux1, label="KF-aux1")
    ax.plot(kf_aux2_df.y, ',',
            color=settings.colours.kf_aux2, label="KF-aux2")
    ax.plot(kf_aux3_df.y, ',',
            color=settings.colours.kf_aux3, label="KF-aux3")
    ax.set_xlabel("x label")
    ax.set_ylabel("y label")
    ax.legend()
    ax.grid()

    ax = axes[2]
    ax.plot(ins_df.yaw, ',',
            color=settings.colours.ins, label="INS")
    ax.plot(ro_df.yaw, ',',
            color=settings.colours.ro, label="RO")
    ax.plot(kf_aux1_df.yaw, ',',
            color=settings.colours.kf_aux1, label="KF-aux1")
    ax.plot(kf_aux2_df.yaw, ',',
            color=settings.colours.kf_aux2, label="KF-aux2")
    ax.plot(kf_aux3_df.yaw, ',',
            color=settings.colours.kf_aux3, label="KF-aux3")
    ax.set_xlabel("x label")
    ax.set_ylabel("y label")
    ax.legend()
    ax.grid()

    plt.savefig(output_file)
    plt.close()

    print("Plots generated and written to:", output_file)

def parse_arguments(args):
    parser = ArgumentParser(description="Plot INS vs RO outputs.")
    parser.add_argument('--ins_csv', default=settings.INS_CSV, type=Path,
                        help="INS relative pose CSV")
    parser.add_argument('--ro_csv', default=settings.RO_CSV, type=Path,
                        help="RO relative pose CSV")
    # parser.add_argument('--kf_csv', default=settings.KF_CSV, type=Path,
    #                     help="Kalman filter relative pose CSV")
    parser.add_argument('--output_dir', default="", type=Path,
                        help="Output directory to store figures")
    parser.add_argument('--overwrite', default=False, action="store_true",
                        help="Overwrite exported plots if this flag is set")

    return parser.parse_args(args)

def main(arg_list=None):
    args = parse_arguments(arg_list)

    output_path = args.output_dir / "tmp-plots"
    output_path.mkdir(parents=True, exist_ok=args.overwrite)

    ins_df = pd.read_csv(args.ins_csv)
    ro_df = pd.read_csv(args.ro_csv)
    kf_aux1_df = pd.read_csv(settings.AUX1_CSV)
    kf_aux2_df = pd.read_csv(settings.AUX2_CSV)
    kf_aux3_df = pd.read_csv(settings.AUX3_CSV)

    plot_x_y_yaw(f"{output_path}/x_y_yaw.pdf", ins_df,
                 ro_df, kf_aux1_df, kf_aux2_df, kf_aux3_df)
    get_metrics(ins_df, ro_df, kf_aux1_df, kf_aux2_df, kf_aux3_df, output_path)

if __name__ == '__main__':
    main()  # pragma: no cover


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import json
import os
from mpesa.daraja import utils

from django.http import HttpResponse, JsonResponse
from django.views.generic import View
from mpesa.daraja.core import MpesaClient
from decouple import config
from datetime import datetime
from django.conf import settings
from mpesa.daraja_v2 import statusQueryHandler,reversalHandler,c2bHandler

cl = MpesaClient()
stk_push_callback_url = settings.MPESA_EXPRESS_CALLBACK_URL
b2c_callback_url = settings.MPESA_B2C_CALLBACK_URL

def index(request):

	return HttpResponse('Welcome to the home of daraja APIs')

def oauth_success(request):
	r = cl.access_token()
	return JsonResponse(r, safe=False)

def stk_push_success(request):
	phone_number = os.environ.get('MPESA_TEST_PHONENUMBER')
	amount = 1
	account_reference = 'ABC001'
	transaction_desc = 'STK Push Description'
	callback_url = stk_push_callback_url
	r = cl.stk_push(phone_number, amount, account_reference, transaction_desc, callback_url)
	return JsonResponse(r.response_description, safe=False)

def business_payment_success(request):
	phone_number = os.environ.get('MPESA_TEST_PHONENUMBER')
	amount = 1
	transaction_desc = 'Business Payment Description'
	occassion = 'Test business payment occassion'
	callback_url = b2c_callback_url
	r = cl.business_payment(phone_number, amount, transaction_desc, callback_url, occassion)
	return JsonResponse(r.response_description, safe=False)

def salary_payment_success(request):
	phone_number = os.environ.get('MPESA_TEST_PHONENUMBER')
	amount = 1
	transaction_desc = 'Salary Payment Description'
	occassion = 'Test salary payment occassion'
	callback_url = b2c_callback_url
	r = cl.salary_payment(phone_number, amount, transaction_desc, callback_url, occassion)
	return JsonResponse(r.response_description, safe=False)

def promotion_payment_success(request):
	phone_number = os.environ.get('MPESA_TEST_PHONENUMBER')
	amount = 1
	transaction_desc = 'Promotion Payment Description'
	occassion = 'Test promotion payment occassion'
	callback_url = b2c_callback_url
	r = cl.promotion_payment(phone_number, amount, transaction_desc, callback_url, occassion)
	return JsonResponse(r.response_description, safe=False)

def transaction_status_query_success(request):
	class StatusQueryInitiator:
		MSISDN = 1
		TILL_NUMBER = 2
		ORGANISATION_SHORT_CODE = 4

	mpesa_transaction_id = 'RGK11RZPTT'
	initiator = StatusQueryInitiator.ORGANISATION_SHORT_CODE

	r = statusQueryHandler().handle(mpesaTransactionID=mpesa_transaction_id,initiator=initiator)
	return JsonResponse(json.loads(r), safe=False)

def transaction_reversal_success(request):
	mpesa_transaction_id = 'RGL9297DVR'
	amount = 1
	remarks = 'Reversal for mistaken payment'

	r = reversalHandler().handle(mpesaTransactionID=mpesa_transaction_id,amount=amount,remarks=remarks)
	return JsonResponse(json.loads(r), safe=False)

def register_c2b_urls_success(request):
	confirmation_url = 'https://example.com/payme/projectmate-django-c2b-confirm.php'
	validation_url = 'https://example.com/payme/projectmate-django-c2b-validate.php'

	r = c2bHandler().handle(confirmation_url=confirmation_url,validation_url=validation_url)
	return JsonResponse(json.loads(r), safe=False)


"""empty message

Revision ID: f6829f6ef2f0
Revises: 24e17715ed8e
Create Date: 2021-04-22 22:54:10.073990

"""
import sqlalchemy as sa

from alembic import op

# revision identifiers, used by Alembic.
revision = "f6829f6ef2f0"
down_revision = "24e17715ed8e"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "stocks",
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("ticker", sa.String(length=10), nullable=True),
        sa.Column("buy", sa.Boolean(), nullable=True),
        sa.Column("price", sa.Float(), nullable=True),
        sa.Column("user_id", sa.Integer(), nullable=True),
        sa.ForeignKeyConstraint(
            ["user_id"],
            ["users.id"],
        ),
        sa.PrimaryKeyConstraint("id"),
        sa.UniqueConstraint("ticker", "buy", "user_id", name="unique_stock"),
    )
    op.create_index(op.f("ix_stocks_id"), "stocks", ["id"], unique=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f("ix_stocks_id"), table_name="stocks")
    op.drop_table("stocks")
    # ### end Alembic commands ###


"""Abstract model describing WorldMap Layers"""
from __future__ import print_function
from abc import abstractmethod
from urlparse import urlparse
import logging

from django.conf import settings
from django.db import models
from django import forms

import jsonfield  # using jsonfield.JSONField

from shared_dataverse_information.dataverse_info.forms_existing_layer\
    import CheckForExistingLayerForm

from shared_dataverse_information.map_layer_metadata.forms import\
    MapLayerMetadataValidationForm,\
    GeoconnectToDataverseMapLayerMetadataValidationForm,\
    GeoconnectToDataverseDeleteMapLayerMetadataForm

from .download_link_formatter import DownloadLinkFormatter

from gc_apps.core.models import TimeStampedModel
from gc_apps.geo_utils.json_field_reader import JSONHelper

LOGGER = logging.getLogger(__name__)

class WorldMapLayerInfo(TimeStampedModel):
    """
    Store the results of a new layer created by mapping a Dataverse file
    Abstract model used as a mix-in
    """
    layer_name = models.CharField(max_length=255, blank=True, help_text='auto-filled on save')

    core_data = jsonfield.JSONField()
    attribute_data = jsonfield.JSONField()
    download_links = jsonfield.JSONField(blank=True)

    # for object identification
    md5 = models.CharField(max_length=40,\
                    blank=True,\
                    db_index=True,\
                    help_text='auto-filled on save')

    class Meta:
        """model meta info"""
        abstract = True
        ordering = ('-modified', '-created')
        verbose_name = 'WorldMapLayerInfo'
        verbose_name_plural = verbose_name

    def __str__(self):
        """string representation"""
        return self.layer_name

    @abstractmethod
    def get_layer_type(self):
        """return type such as:
        TYPE_JOIN_LAYER, TYPE_LAT_LNG_LAYER, TYPE_SHAPEFILE_LAYER, etc"""

    @abstractmethod
    def get_gis_data_info(self):
        """Return the attribute holding gis_data_file
        e.g. return self.tabular_info
        OR   return self.shapefile_info, etc"""

    @abstractmethod
    def get_description_for_core_data(self):
        """Return a description of the map layer source.
        e.g. 'Layer created from tabular file'"""

    @abstractmethod
    def get_failed_rows(self):
        """Return a list of rows which failed to map.
        e.g. 'Layer created from tabular file'"""

    def is_shapefile_layer(self):
        """Is this the result of mapping a zipped shapefile?"""
        return False
        #self.get_layer_type() == DV_MAP_TYPE_SHAPEFILE

    def is_lat_lng_layer(self):
        """Is this the result of mapping Lat/Lng columns?"""
        return False
        #return self.get_layer_type() == TYPE_LAT_LNG_LAYER

    def is_join_layer(self):
        """Is this the result of joining an existing layer?"""
        return False
        #return self.get_layer_type() == TYPE_JOIN_LAYER

    def get_core_data_dict_for_views(self):
        """
        Parameters used for HTML views of map data:
            core_data, attribute_data, download_links
        """
        return dict(worldmap_layerinfo=self,
                    core_data=self.core_data,
                    attribute_data=self.attribute_data,
                    download_links=self.get_formatted_download_links())

    def get_dict_for_classify_form(self):
        """
        Parameters used for populating the classification form
        # Override in concrete class
        """
        return dict(layer_name=self.layer_name,\
                data_source_type=self.get_layer_type(),\
                raw_attribute_info=self.attribute_data)

    def get_download_link(self, link_type='png'):
        """
        See download_link_formatter.DownloadLinkFormatter
        for different format types
        """
        if not self.download_links:
            return None

        return self.download_links.get('png', None)

    @staticmethod
    def build_dict_from_worldmap_json(json_dict):
        """Given json_dict containing WorldMap layer information,
        split out the core_data, attribute_data, and download_link information

        Used for creating subclass objects:
            - WorldMapShapefileLayerInfo
            - WorldMapJoinLayerInfo
            - WorldMapLatLngInfo
        """

        if json_dict is None:
            LOGGER.error('json_dict cannot be None')
            return None

        if not hasattr(json_dict, 'has_key'):
            LOGGER.error('json_dict must be a dict.  not type: [%s]', type(json_dict))
            return None

        # -----------------------------------------
        # Get core data (required)
        # -----------------------------------------
        if not 'data' in json_dict:
            LOGGER.error('The json_dict must have a "data" key')
            return None
        core_data = json_dict['data']

        # -----------------------------------------
        # Get attribute data (required)
        # Note: Currently this is an escaped string within core data...
        # -----------------------------------------
        if not 'attribute_info' in core_data:
            LOGGER.error('The core_data must have a "attribute_info" key')
            return None

        attribute_data = JSONHelper.to_python_or_none(core_data['attribute_info'])
        if attribute_data is None:
            LOGGER.error(('Failed to convert core_data'
                          ' "attribute_info" from string'
                          ' to python object (list)'))
            return None

        # -----------------------------------------
        # Get download_links (optional)
        # Note: Currently this is an escaped string within core data...
        # -----------------------------------------
        if 'download_links' in core_data:
            download_links = JSONHelper.to_python_or_none(core_data['download_links'])

            if download_links is None:
                LOGGER.error(('Failed to convert core_data "download_links"'
                              ' from string to python object (list)'))
                download_links = ''
        else:
            download_links = ''

        # -----------------------------------------
        # Gather initial values
        # -----------------------------------------
        init_data = dict(core_data=core_data,\
                    attribute_data=attribute_data,\
                    download_links=download_links)

        return init_data

    @staticmethod
    def clear_duplicate_worldmapinfo(worldmap_info):
        """
        Remove any duplicate objects of the same subclass.
        Subclass objects include:
            - WorldMapShapefileLayerInfo
            - WorldMapJoinLayerInfo
            - WorldMapLatLngInfo

        worldmap_info - instance of a subclass above
        """
        if worldmap_info is None or not worldmap_info.id:
            # Make sure the object has been saved -- e.g. has an 'id'
            return

        assert isinstance(worldmap_info, WorldMapLayerInfo),\
            ("worldmap_info must be an instance/subclass of"
             " WorldMapJoinLayerInfo")

        WorldMapLayerInfoType = worldmap_info.__class__

        if worldmap_info.is_shapefile_layer():
            filters = dict(shapefile_info=worldmap_info.get_gis_data_info())
        else:
            filters = dict(tabular_info=worldmap_info.get_gis_data_info())

        # Pull objects except the current "worldmap_info"
        #
        older_info_objects = WorldMapLayerInfoType.objects\
                                .filter(**filters)\
                                .exclude(id=worldmap_info.id)

        # Delete the older objects
        older_info_objects.delete()

    def get_layer_url_base(self):
        """
        Return the layer url base. Examples:
            - http://worldmap.harvard.edu
            - http(s)://worldmap.harvard.edu
        """
        if not self.core_data:
            return None

        layer_link = self.core_data.get('layer_link', None)
        if layer_link is None:
            return None

        parsed_url = urlparse(layer_link)

        return '%s://%s' % (parsed_url.scheme, parsed_url.netloc)

    def get_params_to_check_for_existing_layer_metadata(self):
        """Return dict of params used to check WorldMap
        for existing layer metadata"""

        gis_data_info = self.get_gis_data_info()
        assert gis_data_info is not None, "gis_data_info cannot be None"

        f = CheckForExistingLayerForm(gis_data_info.__dict__)
        if not f.is_valid():
            raise forms.ValidationError(\
                'CheckForExistingLayerForm params did not validate: %s'\
                 % f.errors)

        return f.cleaned_data

    def get_params_for_dv_delete_layer_metadata(self):
        """Return dict of params used to delete an
        WorldMap metadata from Dataverse"""

        gis_data_info = self.get_gis_data_info()
        assert gis_data_info is not None, "gis_data_info cannot be None"

        f = GeoconnectToDataverseDeleteMapLayerMetadataForm(\
                {'dv_session_token' : gis_data_info.dv_session_token})

        if not f.is_valid():
            raise forms.ValidationError(\
                'WorldMapLayerInfo DELETE params did not validate: %s' %\
                 f.errors)

        return f.format_for_dataverse_api()

    def get_legend_img_url(self):
        """
        Construct a url that returns a Legend for a Worldmap layer in the form of PNG file
        """
        if not self.core_data:
            return None

        params = (('request', 'GetLegendGraphic'),
                  ('format', 'image/png'),
                  ('width', 20),
                  ('height', 20),
                  ('layer', self.layer_name),
                  ('legend_options', 'fontAntiAliasing:true;fontSize:11;'))

        param_str = '&'.join(['%s=%s' % (k, v) for k, v in params])

        legend_img_url = '%s/geoserver/wms?%s' %\
                                (self.get_layer_url_base(),
                                 param_str)
        if settings.WORLDMAP_EMBED_FORCE_HTTPS:
            return legend_img_url.replace('http://', 'https://', 1)
        else:
            return legend_img_url

        """
        Example of how an image tag is formed:
        <img src="{{ worldmap_layerinfo.get_layer_url_base }}
        /geoserver/wms?request=GetLegendGraphic&format=image/png&width=20&height=20
        &layer={{ worldmap_layerinfo.layer_name }}
        &legend_options=fontAntiAliasing:true;fontSize:12;
        &trefresh={% now "U" %}" id="legend_img" alt="legend" />
        """

    def get_dataverse_server_url(self):
        """
        Retrieve the Dataverse base url to be used
        for using the Dataverse API
        """
        wm_info = self.get_gis_data_info()

        if not wm_info:
            return None

        return wm_info.get_dataverse_server_url()

    def get_data_dict(self, json_format=False):
        """
        Used for processing model data.
        """
        f = MapLayerMetadataValidationForm(self.core_data)
        if not f.is_valid():
            raise forms.ValidationError('WorldMapLayerInfo params did not validate: %s' % f.errors)

        if not json_format:
            return f.cleaned_data

        try:
            return json.dumps(f.cleaned_data)
        except:
            raise ValueError('Failed to convert data to json\ndata: %s' % f.cleaned_data)

    def get_layer_link(self):

        if not self.core_data:
            return None

        layer_link = self.core_data.get('layer_link', None)
        if not layer_link:
            return None

        if layer_link.startswith('/'):
            return settings.WORLDMAP_SERVER_URL + layer_link

        return layer_link

    def get_params_for_dv_update(self):
        """
        Format data to send to the Dataverse
        """
        self.verify_layer_link_format()
        if self.core_data and self.core_data.get('joinDescription') is None:
            self.core_data['joinDescription'] = self.get_description_for_core_data()
            self.save()

        f = GeoconnectToDataverseMapLayerMetadataValidationForm(self.core_data)
        if not f.is_valid():
            raise forms.ValidationError('WorldMapLayerInfo params did not validate: %s' % f.errors)

        gis_data_info = self.get_gis_data_info()
        if gis_data_info is None:
            raise forms.ValidationError(\
                'Serious error!  Could not find gis_data_info: %s' % f.errors)

        return f.format_data_for_dataverse_api(gis_data_info.dv_session_token,\
                        join_description=self.get_description_for_core_data())

    def get_formatted_download_links(self):
        """Format the download links from WorldMap"""
        if not self.download_links:
            return None

        dl = DownloadLinkFormatter(self.download_links)

        return dl.get_formatted_links()

    def get_embed_map_link(self):
        """
        Return the WorldMap embed link.
        By default, make the link 'https'
        """
        if not self.core_data:
            return None

        embed_link = self.core_data.get('embed_map_link', None)
        if not embed_link:
            return None

        if settings.WORLDMAP_EMBED_FORCE_HTTPS and embed_link.startswith('http://'):
            return embed_link.replace('http://', 'https://', 1)

        return embed_link

    def verify_layer_link_format(self):
        """
        Hack to make sure the layer_link is a full url and not just the path

        e.g., If it's just the path, take the rest of the url from the embed_link
        """
        layer_link = self.core_data.get('layer_link', None)

        # Is it just a path?
        if layer_link and layer_link.startswith('/'):
            full_link = self.core_data.get('embed_link', None)
            if not full_link:
                full_link = self.core_data.get('map_image_link', None)

            # Does the embed link a full url
            if full_link and full_link.lower().startswith('http'):
                # Parse the embed link and use it to reformat the layer_link
                url_parts = urlparse(full_link)

                # Full layer link
                layer_link = '%s://%s%s' % (url_parts.scheme,
                                            url_parts.netloc,
                                            layer_link)
                self.core_data['layer_link'] = layer_link
                self.save()


"""test flow with otp stages"""
from base64 import b32decode
from time import sleep
from urllib.parse import parse_qs, urlparse

from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait

from authentik.blueprints.tests import apply_blueprint
from authentik.flows.models import Flow
from authentik.stages.authenticator.oath import TOTP
from authentik.stages.authenticator_static.models import (
    AuthenticatorStaticStage,
    StaticDevice,
    StaticToken,
)
from authentik.stages.authenticator_totp.models import AuthenticatorTOTPStage, TOTPDevice
from tests.e2e.utils import SeleniumTestCase, retry

class TestFlowsAuthenticator(SeleniumTestCase):
    """test flow with otp stages"""

    @retry()
    @apply_blueprint(
        "default/flow-default-authentication-flow.yaml",
        "default/flow-default-invalidation-flow.yaml",
    )
    def test_totp_validate(self):
        """test flow with otp stages"""
        # Setup TOTP Device
        device = TOTPDevice.objects.create(user=self.user, confirmed=True, digits=6)

        flow: Flow = Flow.objects.get(slug="default-authentication-flow")

        self.driver.get(self.url("authentik_core:if-flow", flow_slug=flow.slug))
        self.login()

        # Get expected token
        totp = TOTP(device.bin_key, device.step, device.t0, device.digits, device.drift)

        flow_executor = self.get_shadow_root("ak-flow-executor")
        validation_stage = self.get_shadow_root("ak-stage-authenticator-validate", flow_executor)
        code_stage = self.get_shadow_root("ak-stage-authenticator-validate-code", validation_stage)
        code_stage.find_element(By.CSS_SELECTOR, "input[name=code]").send_keys(totp.token())
        code_stage.find_element(By.CSS_SELECTOR, "input[name=code]").send_keys(Keys.ENTER)
        self.wait_for_url(self.if_user_url("/library"))
        self.assert_user(self.user)

    @retry()
    @apply_blueprint(
        "default/flow-default-authentication-flow.yaml",
        "default/flow-default-invalidation-flow.yaml",
    )
    @apply_blueprint("default/flow-default-authenticator-totp-setup.yaml")
    def test_totp_setup(self):
        """test TOTP Setup stage"""
        flow: Flow = Flow.objects.get(slug="default-authentication-flow")

        self.driver.get(self.url("authentik_core:if-flow", flow_slug=flow.slug))
        self.login()

        self.wait_for_url(self.if_user_url("/library"))
        self.assert_user(self.user)

        self.driver.get(
            self.url(
                "authentik_flows:configure",
                stage_uuid=AuthenticatorTOTPStage.objects.first().stage_uuid,
            )
        )

        flow_executor = self.get_shadow_root("ak-flow-executor")
        totp_stage = self.get_shadow_root("ak-stage-authenticator-totp", flow_executor)
        wait = WebDriverWait(totp_stage, self.wait_timeout)

        wait.until(ec.presence_of_element_located((By.CSS_SELECTOR, "input[name=otp_uri]")))
        otp_uri = totp_stage.find_element(By.CSS_SELECTOR, "input[name=otp_uri]").get_attribute(
            "value"
        )

        # Parse the OTP URI, extract the secret and get the next token
        otp_args = urlparse(otp_uri)
        self.assertEqual(otp_args.scheme, "otpauth")
        otp_qs = parse_qs(otp_args.query)
        secret_key = b32decode(otp_qs["secret"][0])

        totp = TOTP(secret_key)

        totp_stage.find_element(By.CSS_SELECTOR, "input[name=code]").send_keys(totp.token())
        totp_stage.find_element(By.CSS_SELECTOR, "input[name=code]").send_keys(Keys.ENTER)
        sleep(3)

        self.assertTrue(TOTPDevice.objects.filter(user=self.user, confirmed=True).exists())

    @retry()
    @apply_blueprint(
        "default/flow-default-authentication-flow.yaml",
        "default/flow-default-invalidation-flow.yaml",
    )
    @apply_blueprint("default/flow-default-authenticator-static-setup.yaml")
    def test_static_setup(self):
        """test Static OTP Setup stage"""
        flow: Flow = Flow.objects.get(slug="default-authentication-flow")

        self.driver.get(self.url("authentik_core:if-flow", flow_slug=flow.slug))
        self.login()

        self.wait_for_url(self.if_user_url("/library"))
        self.assert_user(self.user)

        self.driver.get(
            self.url(
                "authentik_flows:configure",
                stage_uuid=AuthenticatorStaticStage.objects.first().stage_uuid,
            )
        )

        # Remember the current URL as we should end up back here
        destination_url = self.driver.current_url

        flow_executor = self.get_shadow_root("ak-flow-executor")
        authenticator_stage = self.get_shadow_root("ak-stage-authenticator-static", flow_executor)
        token = authenticator_stage.find_element(By.CSS_SELECTOR, "ul li:nth-child(1)").text

        authenticator_stage.find_element(By.CSS_SELECTOR, "button[type=submit]").click()

        self.wait_for_url(destination_url)
        sleep(1)

        self.assertTrue(StaticDevice.objects.filter(user=self.user, confirmed=True).exists())
        device = StaticDevice.objects.filter(user=self.user, confirmed=True).first()
        self.assertTrue(StaticToken.objects.filter(token=token, device=device).exists())


import requests
from bs4 import BeautifulSoup

url = "https://comic.naver.com/webtoon/weekday.nhn"
res = requests.get(url)
res.raise_for_status()

soup = BeautifulSoup(res.text, "lxml")
cartoons = soup.find_all("a", attrs={"class" : "title"})
# find_all => a의 class="title"인 값들 모두 찾기

for c in cartoons:
	print(c.get_text())


"""
Posts Tweets.
"""
import os
import json
import tweepy
import logging
import argparse
from typing import Dict, Any
from datetime import datetime
from typing import Any

LOG = logging.getLogger(__name__)

parser = argparse.ArgumentParser(description="""
    Posts a all of Andrew Yang's policies as top-level Tweet with formatted text.
    Subsequent statuses are posted in response to the first Tweet containing
    formatted goal text. A final status is posted in the same thread with a
    uniform message.""")
args = parser.parse_args()

if __name__ == '__main__':
    def needs_ellipses(text: str, remaining_chars: int) -> bool:
        """
        Returns True if the length of a given *text* exceeds the value of
        *remaining_chars* minus 3. Otherwise returns False.
        """
        return len(text) >= remaining_chars - 3

    def post_policy(policy: Dict[str, str]) -> Any:
        """
        Posts a status containing formatted text from a given *policy*
        description. Text may be truncated with an ellipses to fit within
        Twitter's character limit.
        """
        title       = f"I #EndorseAndrewYang for this policy: {policy['title']}\n\n"
        url         = f"\n {policy['url']} "

        remaining_chars = 280 - len(title) - len(url)
        end = '...' if needs_ellipses(policy['description'], remaining_chars) else ''

        description = policy['description'][0:remaining_chars - len(end)] + end

        status = title + description + url
        print(status)
        return api.update_status(status=status)

    def post_goals(policy: Dict[str, str], status_id: int):
        """
        Posts one or more statuses in response to the given *status_id*.
        New posts contain the text of individual goals listed within a given
        *policy*.
        """
        title = 'Goal: '
        remaining_chars = 280 - len(title)

        for goal in policy['goals']:
            end = '...' if needs_ellipses(goal, remaining_chars) else ''
            text = goal[0:remaining_chars - len(end)] + end

            status = title + text
            api.update_status(status=status, in_reply_to_status_id=status_id)

    def post_closing_remarks(status_id: int):
        """ Posts a new status in response to the given *status_id*. """
        status = 'Follow @AndrewYang for more messages like these!\n' + \
        'This bot was made by one of the #WomenForYang\n' + \
        '#YangGang #Yang2020'
        api.update_status(status=status, in_reply_to_status_id=status_id)

    auth = tweepy.OAuthHandler(os.environ['TWITTER_CONSUMER_KEY'],
        os.environ['TWITTER_CONSUMER_SECRET'])
    auth.set_access_token(os.environ['TWITTER_ACCESS_TOKEN'],
        os.environ['TWITTER_ACCESS_SECRET'])

    api = tweepy.API(auth, wait_on_rate_limit=True,
        wait_on_rate_limit_notify=True,
        compression=True)

    with open('policies.json', 'r') as f:
        policies = json.load(f)

        for policy in policies:
            status = post_policy(policy)

            post_goals(policy, status.id)
            post_closing_remarks(status.id)


from collections import deque

from gym_sokoban.envs.sokoban_env_fast import SokobanEnvFast

from utils.utils_sokoban import HashableNumpyArray

class BFSGraph:
    class BFSNode:
        def __init__(self, state, depth, parent, parent_action):
            self.state = state
            self.depth = depth
            self.parent = parent
            self.parent_action = parent_action

    def __init__(self, env, initial_state, depth):
        assert isinstance(env, SokobanEnvFast), "Methods used for state clone and restore are sokoban specific."

        hashable_initial_state = HashableNumpyArray(initial_state)
        root = self.BFSNode(hashable_initial_state, depth=0, parent=None, parent_action=None)
        node_queue = deque([root])
        nodes = set([root])
        state2node = {hashable_initial_state: root}

        while node_queue and node_queue[0].depth < depth:
            node = node_queue.popleft()

            if node.depth > depth:
                break

            for action in range(env.action_space.n):
                env.restore_full_state_from_np_array_version(node.state.np_array)
                _, _, done, _ = env.step(action)
                neighbour_state = HashableNumpyArray(env.render(mode="one_hot"))

                # Check if state wasn't visited yet
                if neighbour_state not in state2node:
                    neighbour = self.BFSNode(neighbour_state, node.depth + 1, node, parent_action=action)
                    nodes.add(neighbour)
                    state2node[neighbour_state] = neighbour

                    if not done:
                        node_queue.append(neighbour)

        self.nodes = nodes
        self.state2node = state2node

    def generate_path_to_state(self, state):
        node = self.state2node[state]
        reversed_path = list()
        while node.depth > 0:
            reversed_path.append(node.parent_action)
            node = node.parent
        return list(reversed(reversed_path))

    def is_state_visited(self, state):
        assert isinstance(state, HashableNumpyArray)
        return state in self.state2node


# Standard Library
import json
from typing import List
import re
import os

# Third Party
from nltk import pos_tag
from nltk.stem import WordNetLemmatizer
import numpy as np
import pandas as pd

class CharacterAttributesPipeline:
    def __init__(self, story_id: str, pipeline_dir: str,
                 sentences_df: pd.DataFrame, sentences_characters_df: pd.DataFrame, characters_df: pd.DataFrame,
                 attributes = ['gender']):
        self.story_id = story_id
        self.pipeline_dir = pipeline_dir
        self.attributes = attributes
        self.lemmatizer = WordNetLemmatizer()

        self.sentences_df = sentences_df
        self.sentences_characters_df = sentences_characters_df
        self.characters_df = characters_df
        self.characters =  characters_df['easy_name'].unique()

        self.male_pronouns = {"he", "him", "his"}
        self.female_pronouns = {"she", "her", "hers"}
        self.group_nonbinary_pronouns = {"they", "them", "their", "theirs"}
        self.male_nouns = {"man", "men", "gentleman", "gentlemen", "boy", "husband", "father", "brother", "grandfather", "nephew", "emperor", "king", "prince"}
        self.female_nouns = {"woman", "women", "lady", "girl", "maiden", "mother", "wife", "sister", "grandmother", "niece", "emperess", "queen", "princess"}

        if 'gender' in self.attributes:
            self.set_gender_in_characters_df()
        if 'importance' in self.attributes:
            self.set_character_importance_in_characters_df()
        if 'age' in self.attributes:
            self.set_age_in_characters_df()

        self.save_characters_df_with_attributes()

    @classmethod
    def from_files(cls, story_id: str, pipeline_dir: str, attributes = ['gender']):
        story_dir_with_prefix = os.path.join(pipeline_dir, story_id, story_id)
        sentences_path = story_dir_with_prefix + '.sentences.csv'
        sentences_characters_path = story_dir_with_prefix + '.sentences_characters.csv'
        characters_path = story_dir_with_prefix + '.character_meta_pron'

        sentences_df = pd.read_csv(sentences_path)
        sentences_characters_df = pd.read_csv(sentences_characters_path)
        characters_df = pd.read_csv(characters_path, delimiter = '\t')

        return cls(story_id, pipeline_dir, sentences_df, sentences_characters_df, characters_df, attributes)

    # Character Gender: Female, Male, Mixed Group/Nonbinary
    def get_gender_of_character_name(self, character_name: str):
        gender_flags = []
        tokens = [self.lemmatizer.lemmatize(token.lower()) for token in character_name.split()]

        # Remove possessives
        for token in tokens:
            for possessive_flag in ["'", "his", "her", "their"]:
                if possessive_flag == token:
                    tokens.remove(token)

        if not set(tokens).isdisjoint(self.male_nouns):
            gender_flags.append('male')
        if not set(tokens).isdisjoint(self.female_nouns):
            gender_flags.append('female')

        return gender_flags

    def get_gender_of_pronouns(self, corefs: List):
        gender_flags = []
        for pronoun in corefs:
            if pronoun in self.male_pronouns:
                gender_flags.append('male')
            if pronoun in self.female_pronouns:
                gender_flags.append('female')
            if pronoun in self.group_nonbinary_pronouns:
                gender_flags.append('group/nonbinary')
        return gender_flags

    def get_gender_with_certainty(self, name_gender_flags, pronoun_gender_flags):
        if len(pd.Series(name_gender_flags, dtype = 'object').unique()) == 1:
            name_gender = pd.Series(name_gender_flags, dtype = 'object').unique().tolist()[0]
            name_gender_certainty = 1.0
        elif len(pd.Series(name_gender_flags, dtype = 'object').unique()) > 1:
            name_gender = pd.Series(name_gender_flags, dtype = 'object').mode().values[0]
            name_gender_certainty = (pd.Series(name_gender_flags, dtype = 'object').value_counts()/len(name_gender_flags)).to_dict()[name_gender]
        else:
            name_gender = 'unknown'
            name_gender_certainty = 0.0

        if len(pronoun_gender_flags) > 0:
            pronoun_gender = pd.Series(pronoun_gender_flags, dtype = 'object').mode().values[0]
            pronoun_gender_certainty = (pd.Series(pronoun_gender_flags, dtype = 'object').value_counts()/len(pronoun_gender_flags)).to_dict()[pronoun_gender]
        else:
            pronoun_gender = 'unknown'
            pronoun_gender_certainty = 0.0

        gender_flags = list(np.repeat(name_gender_flags, len(pronoun_gender_flags) + 1)) + pronoun_gender_flags

        if name_gender_certainty == 1.0:
            return name_gender, name_gender_certainty
        elif name_gender == 'unknown' and pronoun_gender == 'unknown':
            return name_gender, name_gender_certainty
        else:
            gender = pd.Series(gender_flags, dtype = 'object').mode().tolist()[0]
            gender_certainty = (pd.Series(gender_flags, dtype = 'object').value_counts()/len(gender_flags)).to_dict()[gender]
            return gender, gender_certainty

    def set_gender_in_characters_df(self):
        for coref in self.characters_df['coref_idx'].unique():
            char_df = self.characters_df[self.characters_df['coref_idx'] == coref]
            character_name = char_df['easy_name'].values[0]
            name_gender_flags = self.get_gender_of_character_name(character_name)
            char_sentences_df = self.sentences_characters_df[self.sentences_characters_df['coref_id'] == coref]
            pronoun_gender_flags = self.get_gender_of_pronouns(char_sentences_df['character_token'].tolist())
            gender, certainty = self.get_gender_with_certainty(name_gender_flags, pronoun_gender_flags)
            self.characters_df.loc[self.characters_df['coref_idx'] == coref, 'gender'] = gender
            self.characters_df.loc[self.characters_df['coref_idx'] == coref, 'gender_certainty'] = certainty

    # Character Importance: Primary, Secondary, Tertiary
    def get_most_important_character(self):
        return self.characters_df.loc[self.characters_df['total'].idxmax(), 'coref_idx']

    def set_character_importance_in_characters_df(self):
        most_important_character = self.get_most_important_character()
        most_important_appearances = self.characters_df.loc[self.characters_df['coref_idx'] == most_important_character, 'total']
        secondary_threshold = int(most_important_appearances * 0.66)
        tertiary_threshold = int(most_important_appearances * 0.33)
        self.characters_df['importance'] = self.characters_df['total'].apply(set_character_importance, args = (secondary_threshold, tertiary_threshold))

    # Character Age
    def set_age_in_characters_df(self):
        pass

    def save_characters_df_with_attributes(self):
        self.characters_df.to_csv(self.pipeline_dir + self.story_id + '/' + self.story_id + '.character_attributes.csv', index = False)

# Pandas Apply Function for Character Importance
def set_character_importance(appearances, secondary_threshold, tertiary_threshold):
    if appearances >= secondary_threshold:
        return 'primary'
    elif appearances >= tertiary_threshold and appearances < secondary_threshold:
        return 'secondary'
    else:
        return 'tertiary'


from rest_framework import serializers
from profiles.models import Profile

class ProfilePostSerializer(serializers.ModelSerializer):
    posts = serializers.HyperlinkedRelatedField(
        many=True,
        read_only=True,
        view_name='post-detail',
    )
    class Meta:
        model = Profile
        fields = ['id', 'name', 'email', 'posts']