text:
/*
 * Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
 */

#include "ifmap/ifmap_agent_table.h"

#include <boost/algorithm/string.hpp>
#include <boost/bind.hpp>
#include <boost/format.hpp>
#include "base/logging.h"
#include "db/db.h"
#include "db/db_graph.h"
#include "db/db_table_partition.h"
#include "ifmap/ifmap_agent_table.h"
#include "ifmap/ifmap_agent_parser.h"
#include "ifmap/ifmap_link_table.h"
#include "ifmap/ifmap_node.h"
#include "ifmap/ifmap_link.h"
#include "sandesh/sandesh.h"
#include "sandesh/sandesh_types.h"
#include "ifmap/ifmap_agent_types.h"

using namespace std;

SandeshTraceBufferPtr
IFMapAgentTraceBuf(SandeshTraceBufferCreate("IFMapAgentTrace", 1000));

IFMapAgentTable::IFMapAgentTable(DB *db, const string &name, DBGraph *graph)
        : IFMapTable(db, name), graph_(graph), pre_filter_(NULL) {
}

auto_ptr<DBEntry> IFMapAgentTable::AllocEntry(const DBRequestKey *key) const {
    auto_ptr<DBEntry> entry(
        new IFMapNode(const_cast<IFMapAgentTable *>(this)));
    entry->SetKey(key);
    return entry;
}

IFMapNode* IFMapAgentTable::TableEntryLookup(DB *db, RequestKey *key) {

    IFMapTable *table = FindTable(db, key->id_type);
    if (!table) {
        return NULL;
    }

    auto_ptr<DBEntry> entry(new IFMapNode(table));
    entry->SetKey(key);
    IFMapNode *node = static_cast<IFMapNode *>(table->Find(entry.get()));
    return node;
}

IFMapAgentTable* IFMapAgentTable::TableFind(const string &node_name) {
    string name = node_name;
    boost::replace_all(name, "-", "_");
    name = "__ifmap__." + name + ".0";
    IFMapAgentTable *table =
            static_cast<IFMapAgentTable *>(database()->FindTable(name));
    return table;
}

IFMapNode *IFMapAgentTable::EntryLookup(RequestKey *request) {
    auto_ptr<DBEntry> key(AllocEntry(request));
    IFMapNode *node = static_cast<IFMapNode *>(Find(key.get()));
    return node;
}

IFMapNode *IFMapAgentTable::EntryLocate(IFMapNode *node, RequestKey *req) {

    IFMapObject *obj;

    if (node != NULL) {
        /* If delete marked, clear it now */
        if (node->IsDeleted()) {
            node->ClearDelete();
            graph_->AddNode(node);
        }

        obj = node->GetObject();
        assert(obj);
        //We dont accept lesser sequence number updates
        assert(obj->sequence_number() <= req->id_seq_num);

        node->Remove(obj);

    } else {
        auto_ptr<DBEntry> key(AllocEntry(req));
        node = const_cast<IFMapNode *>(
            static_cast<const IFMapNode *>(key.release()));
        DBTablePartition *partition =
            static_cast<DBTablePartition *>(GetTablePartition(0));
        partition->Add(node);
        graph_->AddNode(node);
    }

    return node;
}

// A node is deleted. Move all links for the node to defer-list
void IFMapAgentTable::HandlePendingLinks(IFMapNode *node) {

    IFMapNode *right;
    DBGraphEdge *edge;

    IFMapAgentLinkTable *ltable = static_cast<IFMapAgentLinkTable *>
        (database()->FindTable(IFMAP_AGENT_LINK_DB_NAME));
    assert(ltable != NULL);

    DBGraphVertex::adjacency_iterator iter;
    bool origin_exists;
    uint64_t seq;
    for (iter = node->begin(graph_); iter != node->end(graph_);) {

        right = static_cast<IFMapNode *>(iter.operator->());
        iter++;
        edge = graph_->GetEdge(node, right);
        assert(edge);
        IFMapLink *l = static_cast<IFMapLink *>(edge);
        seq = l->sequence_number(IFMapOrigin::UNKNOWN, &origin_exists);
        assert(origin_exists);

        // Create both the request keys
        auto_ptr <IFMapAgentLinkTable::RequestKey> req_key (new IFMapAgentLinkTable::RequestKey);
        req_key->left_key.id_name = node->name();
        req_key->left_key.id_type = node->table()->Typename();
        req_key->left_key.id_seq_num = seq;

        req_key->right_key.id_name = right->name();
        req_key->right_key.id_type = right->table()->Typename();
        req_key->right_key.id_seq_num = seq;
        req_key->metadata = l->metadata();

        DBRequest req;
        req.oper = DBRequest::DB_ENTRY_ADD_CHANGE;
        req.key = req_key;

        //Add it to defer list
        ltable->LinkDefAdd(&req);

        ltable->DelLink(node, right, edge);
    }
}

void IFMapAgentTable::DeleteNode(IFMapNode *node) {

    if ((node->HasAdjacencies(graph_) == true)) {
        HandlePendingLinks(node);
    }

    //Now there should not be any more adjacencies
    assert((node->HasAdjacencies(graph_) == false));

    DBTablePartition *partition =
        static_cast<DBTablePartition *>(GetTablePartition(0));
    graph_->RemoveNode(node);
    partition->Delete(node);
}

void IFMapAgentTable::NotifyNode(IFMapNode *node) {
    DBTablePartition *partition =
        static_cast<DBTablePartition *>(GetTablePartition(0));
    partition->Change(node);
}

// Process link-defer list based for the request.
// If request is add, create left->right and right-left defer nodes
// If request is delete, remove left->right and right->left defer nodes
// The sequence number is valid only in the DeferrendNode entry
void IFMapAgentLinkTable::LinkDefAdd(DBRequest *request) {
    RequestKey *key = static_cast<RequestKey *>(request->key.get());

    std::list<DeferredNode>::iterator it;

    std::list<DeferredNode> *left = NULL;
    LinkDefMap::iterator left_it = link_def_map_.find(key->left_key);
    if (link_def_map_.end() != left_it)
        left = left_it->second;

    std::list<DeferredNode> *right = NULL;
    LinkDefMap::iterator right_it = link_def_map_.find(key->right_key);
    if (link_def_map_.end() != right_it)
        right = right_it->second;

    if (request->oper == DBRequest::DB_ENTRY_DELETE)  {
        //We need to delete the old sequence links as well
        // remove left->right entry
        if (left) {
            for(it = left->begin(); it != left->end(); it++) {
                if (((*it).node_key.id_type == key->right_key.id_type) &&
                    ((*it).node_key.id_name == key->right_key.id_name)) {
                    left->erase(it);
                    break;
                }
            }
            RemoveDefListEntry(&link_def_map_, left_it, NULL);
        }

        // remove right->left entry
        if (right) {
            for(it = right->begin(); it != right->end(); it++) {
                if (((*it).node_key.id_type == key->left_key.id_type) &&
                    ((*it).node_key.id_name == key->left_key.id_name)) {
                    right->erase(it);
                    break;
                }
            }
            RemoveDefListEntry(&link_def_map_, right_it, NULL);
        }

        return;
    }

    bool push_left = true;

    // Add/Update left->right entry
    if (left) {
        // If list already contains, just update the seq number
        for(it = left->begin(); it != left->end(); it++) {
            if (((*it).node_key.id_type == key->right_key.id_type) &&
                    ((*it).node_key.id_name == key->right_key.id_name)) {
                (*it).node_key.id_seq_num = key->right_key.id_seq_num;
                (*it).link_metadata = key->metadata;
                push_left = false;
                break;
            }
        }
    } else {
        left = new std::list<DeferredNode>();
        link_def_map_[key->left_key] = left;
    }

    bool push_right = true;
    // Add/Update right->left entry
    if (right) {
        // If list already contains, just update the seq number
        for(it = right->begin(); it != right->end(); it++) {
            if (((*it).node_key.id_type == key->left_key.id_type) &&
                    ((*it).node_key.id_name == key->left_key.id_name)) {
                (*it).node_key.id_seq_num = key->left_key.id_seq_num;
                (*it).link_metadata = key->metadata;
                push_right = false;
                break;
            }
        }
    } else {
        right = new std::list<DeferredNode>();
        link_def_map_[key->right_key] = right;
    }

    // Add it to the end of the list
    struct DeferredNode dn;
    dn.link_metadata = key->metadata;
    if (push_left) {
        dn.node_key = key->right_key;
        left->push_back(dn);
    }
    if (push_right) {
        dn.node_key = key->left_key;
        right->push_back(dn);
    }
    return;
}

void IFMapAgentTable::Input(DBTablePartition *partition, DBClient *client,
                             DBRequest *request) {
    RequestKey *key = static_cast<RequestKey *>(request->key.get());
    IFMapAgentTable *table = NULL;
    struct IFMapAgentData *req_data;
    IFMapObject *obj;

    table = TableFind(key->id_type);
    if (!table) {
        IFMAP_AGENT_TRACE(Trace, key->id_seq_num,
                "Table " + key->id_type + " not found");
        return;
    }

    IFMapNode *node = EntryLookup(key);
    if (table->pre_filter_) {
        DBRequest::DBOperation old_oper = request->oper;
        if (table->pre_filter_(table, node, request) == false) {
            IFMAP_AGENT_TRACE(Trace, key->id_seq_num,
                    "Node " + key->id_name + " neglected as filter"
                    + "suppressed");
            return;
        }
        if ((old_oper != DBRequest::DB_ENTRY_DELETE) &&
                (request->oper == DBRequest::DB_ENTRY_DELETE)) {
            IFMAP_AGENT_TRACE(Trace, key->id_seq_num,
                    "Node " + key->id_name + "ID_PERMS Null");
        }
    }

    if (request->oper == DBRequest::DB_ENTRY_DELETE) {
        if (node == NULL) {
            IFMAP_AGENT_TRACE(Trace, key->id_seq_num,
                    "Node " + key->id_name + " not found in Delete");
            return;
        }

        if (node->IsDeleted()) {
            IFMAP_AGENT_TRACE(Trace, key->id_seq_num,
                        "Node " + key->id_name + " already deleted");
            return;
        }

        obj = node->GetObject();
        //We dont accept lesser sequence number updates
        assert(obj->sequence_number() <= key->id_seq_num);

        //Upate the sequence number even for deletion of node
        obj->set_sequence_number(key->id_seq_num);
        DeleteNode(node);
        return;
    }

    node = EntryLocate(node, key);
    assert(node);

    //Get the data from request key and notify oper tables
    req_data = static_cast<struct IFMapAgentData *>(request->data.get());
    obj = static_cast<IFMapObject *>(req_data->content.release());

    //Set the sequence number of the object
    obj->set_sequence_number(key->id_seq_num);

    node->Insert(obj);
    NotifyNode(node);

    IFMapAgentLinkTable *link_table = static_cast<IFMapAgentLinkTable *>(
        database()->FindTable(IFMAP_AGENT_LINK_DB_NAME));
    link_table->EvalDefLink(key);
}

void IFMapAgentTable::Clear() {
    assert(!HasListeners());
    DBTablePartition *partition = static_cast<DBTablePartition *>(
        GetTablePartition(0));
    IFMapNode *next = NULL;
    for (IFMapNode *node = static_cast<IFMapNode *>(partition->GetFirst());
         node != NULL; node = next) {
        next = static_cast<IFMapNode *>(partition->GetNext(node));
        if (node->IsDeleted()) {
            continue;
        }
        graph_->RemoveNode(node);
        partition->Delete(node);
    }
}

// Agent link table routines

void IFMapAgentLinkTable::AddLink(DBGraphBase::edge_descriptor edge,
                                  IFMapNode *left, IFMapNode *right,
                                  const std::string &metadata,
                                  uint64_t seq) {

    IFMapLinkTable *table = static_cast<IFMapLinkTable *>(
        database()->FindTable(IFMAP_AGENT_LINK_DB_NAME));
    assert(table != NULL);
    table->AddLink(edge, left, right, metadata, seq,
                   IFMapOrigin(IFMapOrigin::UNKNOWN));
}

void IFMapAgentLinkTable::DelLink(IFMapNode *left, IFMapNode *right, DBGraphEdge *edge) {
    IFMapAgentLinkTable *table = static_cast<IFMapAgentLinkTable *>(
        database()->FindTable(IFMAP_AGENT_LINK_DB_NAME));
    assert(table != NULL);
    table->DeleteLink(edge);
    graph_->Unlink(left, right);
}

IFMapAgentLinkTable::IFMapAgentLinkTable(DB *db, const string &name, DBGraph *graph)
        : IFMapLinkTable(db, name, graph), graph_(graph) {
}

DBTable *IFMapAgentLinkTable::CreateTable(DB *db, const string &name,
                                     DBGraph *graph) {
    IFMapAgentLinkTable *table = new IFMapAgentLinkTable(db, name, graph);
    table->Init();
    return table;
}

void IFMapAgentLinkTable_Init(DB *db, DBGraph *graph) {
    db->RegisterFactory(IFMAP_AGENT_LINK_DB_NAME,
        boost::bind(&IFMapAgentLinkTable::CreateTable, _1, _2, graph));
    db->CreateTable(IFMAP_AGENT_LINK_DB_NAME);
}

void IFMapAgentLinkTable::Input(DBTablePartition *partition, DBClient *client,
                           DBRequest *req) {

    RequestKey *key = static_cast<RequestKey *>(req->key.get());

    IFMapNode *left;
    left = IFMapAgentTable::TableEntryLookup(database(), &key->left_key);
    if (!left) {
        IFMAP_AGENT_TRACE(Trace, key->left_key.id_seq_num,
                key->left_key.id_type + ":" + key->left_key.id_name +
                " not present for link to " + key->right_key.id_type +
                ":" + key->right_key.id_name);
        LinkDefAdd(req);
        return;
    }

    IFMapNode *right;
    right = IFMapAgentTable::TableEntryLookup(database(), &key->right_key);
    if (!right) {
        IFMAP_AGENT_TRACE(Trace, key->left_key.id_seq_num,
                key->right_key.id_type + " : " + key->right_key.id_name +
                " not present for link to " + key->left_key.id_type + " : " +
                key->left_key.id_name);
        LinkDefAdd(req);
        return;
    }

    if (left->IsDeleted()) {
        IFMAP_AGENT_TRACE(Trace, key->left_key.id_seq_num,
            "Adding Link" + key->left_key.id_type + ":" +
            key->left_key.id_name + "->" + key->right_key.id_type +
                            ":" + key->right_key.id_name + " to defer "
                            "list as left is deleted marked");
        LinkDefAdd(req);
        return;
    }

    if (right->IsDeleted()) {
        IFMAP_AGENT_TRACE(Trace, key->left_key.id_seq_num,
            "Adding Link" + key->left_key.id_type + ":" +
            key->left_key.id_name + "->" + key->right_key.id_type +
                            ":" + key->right_key.id_name + " to defer "
                            "list as right is deleted marked");
        LinkDefAdd(req);
        return;
    }

    IFMapObject *obj = left->GetObject();
    if (obj->sequence_number() < key->left_key.id_seq_num) {
        IFMAP_AGENT_TRACE(Trace, key->left_key.id_seq_num,
            "IFMap Link " + left->name() + right->name() +
            " with wrong seq number");
        LinkDefAdd(req);
        return;
    }

    obj = right->GetObject();
    if (obj->sequence_number() < key->left_key.id_seq_num) {
        IFMAP_AGENT_TRACE(Trace, key->left_key.id_seq_num,
            "IFMap Link " + left->name() + right->name() +
            " with wrong seq number");
        LinkDefAdd(req);
        return;
    }

    DBGraph::Edge edge;
    DBGraphEdge *link = graph_->GetEdge(left, right);

    if (req->oper == DBRequest::DB_ENTRY_ADD_CHANGE) {
        if (link == NULL) {
            edge = graph_->Link(left, right);
            AddLink(edge, left, right, key->metadata, key->left_key.id_seq_num);
        } else {
            IFMapOrigin origin(IFMapOrigin::UNKNOWN);
            IFMapLink *l = static_cast<IFMapLink *>(link);
            l->UpdateProperties(origin, key->left_key.id_seq_num);
        }
    } else {
        if (link == NULL) {
            return;
        }
        DelLink(left, right, link);
    }
}

bool IFMapAgentLinkTable::RemoveDefListEntry
    (LinkDefMap *map, LinkDefMap::iterator &map_it,
     std::list<DeferredNode>::iterator *list_it) {

    std::list<DeferredNode> *list = map_it->second;
    if (list_it) {
        list->erase(*list_it);
    }

    if (list->size()) {
        return false;
    }
    map->erase(map_it);
    delete list;
    return true;
}

// For every link there are 2 entries,
//  left->right
//  right->left
//
//  If both left and right node are available, remove the entries and try to
//  add the link
void IFMapAgentLinkTable::EvalDefLink(IFMapTable::RequestKey *key) {
    LinkDefMap::iterator link_defmap_it = link_def_map_.find(*key);
    if (link_def_map_.end() == link_defmap_it)
        return;

    std::list<DeferredNode> *left_list = link_defmap_it->second;
    std::list<DeferredNode>::iterator left_it, left_list_entry;
    for(left_it = left_list->begin(); left_it != left_list->end();) {
        left_list_entry = left_it++;

        // If link seq is older, dont consider the link.
        if ((*left_list_entry).node_key.id_seq_num < key->id_seq_num)
            continue;

        // Skip if right-node is not yet present
        IFMapNode *node = IFMapAgentTable::TableEntryLookup(database(),
                    &((*left_list_entry).node_key));
        if (!node)
            continue;

        //If the other end of the node is not from active control node,
        //dont consider the link
        IFMapObject *obj = node->GetObject();
        if (obj->sequence_number() < key->id_seq_num)
            continue;

        // left->right entry found defer-list. Find the right->left entry
        LinkDefMap::iterator right_defmap_it =
            link_def_map_.find((*left_list_entry).node_key);
        assert(link_def_map_.end() != right_defmap_it);

        std::list<DeferredNode> *right_list = right_defmap_it->second;
        std::list<DeferredNode>::iterator right_it, right_list_entry;
        for(right_it = right_list->begin(); right_it !=
                right_list->end(); right_it++) {

            // If link seq is older, dont consider the link.
            if ((*right_it).node_key.id_seq_num < key->id_seq_num)
                continue;

            if ((*right_it).node_key.id_type == key->id_type &&
                    (*right_it).node_key.id_name == key->id_name) {
                RemoveDefListEntry(&link_def_map_, right_defmap_it,
                               &right_it);
                break;
            }
        }

        //We should have removed something in the above iteration
        assert(right_it != right_list->end());

        //Remove from deferred list before enqueing
        auto_ptr <RequestKey> req_key (new RequestKey);
        req_key->left_key = *key;
        req_key->right_key = (*left_list_entry).node_key;
        req_key->metadata = (*left_list_entry).link_metadata;
        // Dont delete left_list_entry. Its passed in req structure
        left_list->erase(left_list_entry);

        DBRequest req;
        req.key = req_key;
        req.oper = DBRequest::DB_ENTRY_ADD_CHANGE;
        Enqueue(&req);
    }

    // If list does not have any entries, delete the list
    RemoveDefListEntry(&link_def_map_, link_defmap_it, NULL);
}

void IFMapAgentLinkTable::DestroyDefLink(uint64_t seq) {
    std::list<DeferredNode> *ent;
    std::list<DeferredNode>::iterator it, list_entry;
    IFMapAgentLinkTable::LinkDefMap::iterator dlist_it, temp;

    for(dlist_it = link_def_map_.begin(); dlist_it != link_def_map_.end(); ) {
        temp = dlist_it++;
        ent = temp->second;
        for(it = ent->begin(); it != ent->end();) {
            list_entry = it++;

            //Delete the deferred link if it is old seq
            if ((*list_entry).node_key.id_seq_num < seq) {
                if (RemoveDefListEntry(&link_def_map_, temp,
                            &list_entry) == true) {
                    //The list has been deleted. Move to the next map
                    //entry
                    break;
                }
            }
        }
    }
}

//Stale Cleaner functionality
class IFMapAgentStaleCleaner::IFMapAgentStaleCleanerWorker : public Task {
public:

    IFMapAgentStaleCleanerWorker(DB *db, DBGraph *graph, uint64_t seq):
        Task(TaskScheduler::GetInstance()->GetTaskId("db::DBTable"), 0),
        db_(db), graph_(graph), seq_(seq) {
    }

    bool Run() {

        IFMAP_AGENT_TRACE(Trace, seq_,
                "IFMap Config Audit start:");
        //Handle the links first
        DBGraph::edge_iterator e_next(graph_);
        for (DBGraph::edge_iterator e_iter = graph_->edge_list_begin();
            e_iter != graph_->edge_list_end(); e_iter = e_next) {

            const DBGraph::DBVertexPair &tuple = *e_iter;

            e_next = ++e_iter;

            IFMapNode *lhs = static_cast<IFMapNode *>(tuple.first);
            IFMapNode *rhs = static_cast<IFMapNode *>(tuple.second);

            IFMapLink *link =
                static_cast<IFMapLink *>(graph_->GetEdge(lhs, rhs));
            assert(link);

            bool exists = false;
            IFMapLink::LinkOriginInfo origin_info =
                link->GetOriginInfo(IFMapOrigin::UNKNOWN, &exists);
            if (exists && (origin_info.sequence_number < seq_ )) {
                IFMapAgentLinkTable *ltable = static_cast<IFMapAgentLinkTable *>(
                    db_->FindTable(IFMAP_AGENT_LINK_DB_NAME));
                IFMAP_AGENT_TRACE(Trace,
                     origin_info.sequence_number, "Deleting Link between " +
                     lhs->name() + rhs->name());
                ltable->DeleteLink(link, lhs, rhs);
            }
        }

        //Handle the vertices now
        DBGraph::vertex_iterator v_next(graph_);
        for (DBGraph::vertex_iterator v_iter = graph_->vertex_list_begin();
            v_iter != graph_->vertex_list_end(); v_iter = v_next) {

            IFMapNode *node = static_cast<IFMapNode *>(v_iter.operator->());
            v_next = ++v_iter;

            IFMapObject *obj = node->GetObject();
            assert(obj);
            if (obj->sequence_number() < seq_) {
                IFMapAgentTable *table = static_cast<IFMapAgentTable *>(node->table());
                IFMAP_AGENT_TRACE(Trace, obj->sequence_number(),
                        "Deleting node " + node->name());
                table->DeleteNode(node);
            }
        }

        //Handle deferred list
        IFMapAgentLinkTable *table = static_cast<IFMapAgentLinkTable *>(
                    db_->FindTable(IFMAP_AGENT_LINK_DB_NAME));
        table->DestroyDefLink(seq_);

        return true;
    }
    std::string Description() const {
        return "IFMapAgentStaleCleaner::IFMapAgentStaleCleanerWorker";
    }

private:
    DB *db_;
    DBGraph *graph_;
    uint64_t seq_;
};

IFMapAgentStaleCleaner::~IFMapAgentStaleCleaner() {
}

IFMapAgentStaleCleaner::IFMapAgentStaleCleaner(DB *db, DBGraph *graph) :
        db_(db), graph_(graph) {
}

bool IFMapAgentStaleCleaner::StaleTimeout(uint64_t seq) {
    seq_ = seq;
    IFMapAgentStaleCleanerWorker *cleaner = new IFMapAgentStaleCleanerWorker(db_, graph_, seq_);
    TaskScheduler *sch = TaskScheduler::GetInstance();
    sch->Enqueue(cleaner);
    return false;
}

void IFMapAgentStaleCleaner::Clear() {
    IFMapLinkTable *table = static_cast<IFMapLinkTable *>(
        db_->FindTable(IFMAP_AGENT_LINK_DB_NAME));
    table->Clear();
    IFMapTable::ClearTables(db_);
}

max_stars_repo_name: biswajit-mandal/contrail-controller

max_issues_repo_name: biswajit-mandal/contrail-controller

max_forks_repo_name: biswajit-mandal/contrail-controller


text:
/**
 * @file
 * @brief
 *
 * Publishes ~/
 */

#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
#include <thread>
#include <vector>
#include <iostream>
#include <fstream>

#include <pcl/conversions.h>
#include <pcl/common/common_headers.h>
#include <pcl/point_types.h>
#include <pcl_conversions/pcl_conversions.h>
#include <pcl/visualization/pcl_visualizer.h>
#include <pcl/sample_consensus/sac_model_circle3d.h>
#include <pcl_ros/point_cloud.h>
#include <pcl/sample_consensus/ransac.h>

#include <rosbag/bag.h>

#include <ouster/os1.h>
#include <ouster/os1_packet.h>
#include <ouster/os1_util.h>
#include <ouster_ros/OS1ConfigSrv.h>
#include <ouster_ros/os1_ros.h>
#include <ros/ros.h>
#include <sensor_msgs/PointCloud2.h>

#include <visualization_msgs/MarkerArray.h>
#include <visualization_msgs/Marker.h>

#include <unique_id/unique_id.h>
#include <boost/uuid/random_generator.hpp>

#include <opencv2/opencv.hpp>

#include <cartographer_ros_msgs/LandmarkEntry.h>
#include <cartographer_ros_msgs/LandmarkList.h>

#include <rosbag/bag.h>
#include <rosbag/view.h>

#include <boost/foreach.hpp>
#include "ouster_ros/point_os1.h"

#define foreach BOOST_FOREACH

namespace OS1 = ouster::OS1;
using namespace Eigen;

void parseToXYZ (pcl::PointCloud<ouster_ros::OS1::PointOS1> os1cloud,pcl::PointCloud<pcl::PointXYZ>::Ptr &xyzCloud)
{
    foreach (ouster_ros::OS1::PointOS1 pointOs1, os1cloud.points ) {
        pcl::PointXYZ pointXyz;
        pointXyz.x = pointOs1.x;
        pointXyz.y = pointOs1.y;
        pointXyz.z = pointOs1.z;
        xyzCloud->push_back(pointXyz);
    }
}

double areaSumOfAllChildrenOf(const std::vector<std::vector<cv::Point>> contours,const std::vector<cv::Vec4i> hierarchy, int i, cv::RotatedRect *rotRect) {
    double sumOfHoles = 0;
    int idx = i;

    cv::Vec4i currentChild = hierarchy[idx];

    if (currentChild[2] != -1) {
        idx = currentChild[2];
        currentChild = hierarchy[idx];

        sumOfHoles = sumOfHoles + contourArea(contours[idx]);
        if (contours[idx].size() > 5) {
            *rotRect = fitEllipse(contours[idx]);
        }
    }

    return sumOfHoles;
}

pcl::PointXYZ pointAtIndex(const cv::Point2f coords,const cv::Mat indexImage,const ouster_ros::OS1::CloudOS1 ptCloud) {
    size_t index = indexImage.at<int>((int)round(coords.y),(int)round(coords.x)); //due to rotation in prev func
    pcl::PointXYZ point(ptCloud.points[index].x,ptCloud.points[index].y,ptCloud.points[index].z);
    return point;
}

std::vector<cv::Point2f> ptCloudIndicesOfLandMarks (const cv::Mat image, const float binaryThreshhold, const cv::Mat indexImage, const ouster_ros::OS1::CloudOS1 ptCloud) {
    std::vector<cv::Point2f> indices;

    double min_im, max_im;

    cv::minMaxLoc(image, &min_im, &max_im);

    cv::Mat im_norm(image);
    if (min_im == max_im) {
        PCL_INFO("Cannot normalize intensity image...\n");
    } else {
        im_norm = (im_norm - min_im) / (max_im - min_im);

        cv::Mat threshIm;
        threshIm = image > .5171f;
        cv::Mat drawing(threshIm);

        cv::Mat dst = cv::Mat::zeros(64, 2048, CV_8UC3);
        rotate(im_norm, im_norm, cv::ROTATE_90_CLOCKWISE);
        //TODO is everything kosher
        cvtColor(drawing, drawing, CV_GRAY2RGB);

        std::vector<std::vector<cv::Point> > contours;
        std::vector<cv::Vec4i> hierarchy;

        std::vector<std::vector<cv::Point> > contours_poly;
        std::vector<cv::Rect> boundRect;
        std::vector<cv::RotatedRect> outerEllipseRects;
        std::vector<cv::RotatedRect> innerEllipseRects;
        std::vector<int> areaIdx;

        findContours(threshIm, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, cv::Point(0, 0));
        pcl::PointCloud<pcl::PointXYZ>::Ptr contourPtCloud (new pcl::PointCloud<pcl::PointXYZ> ());
        int idx = 0;
        for (size_t i = 0; i < contours.size(); i++) {

            double area = contourArea(contours[i]);
            if (area > 45 && area < 600  && contours[i].size() > 4) {     //to fit ellipse you need 6 points
                double subArea;

                cv::RotatedRect innerRotRect;
                subArea = areaSumOfAllChildrenOf(contours, hierarchy, i, &innerRotRect);

                for (size_t x = 0; x < contours[i].size(); x++) {
                    pcl::PointXYZ contourPoint;
                    contourPoint = pointAtIndex(contours[i][x],indexImage,ptCloud);
                    contourPtCloud->push_back(contourPoint);

                }
                pcl::SampleConsensusModelCircle3D<pcl::PointXYZ>::Ptr
                        modelCircle3D(new pcl::SampleConsensusModelCircle3D<pcl::PointXYZ> (contourPtCloud));

                pcl::RandomSampleConsensus<pcl::PointXYZ> ransac (modelCircle3D);
                ransac.setDistanceThreshold (.01);
                ransac.computeModel();
                float circle_radius = ransac.model_coefficients_[3];
                contourPtCloud->clear();

                if (subArea > 5 && circle_radius >0.07 && circle_radius <0.08) {
                    innerEllipseRects.push_back(innerRotRect);
                    std::vector<cv::Point> polygon;
                    cv::Rect rect;

                    approxPolyDP(cv::Mat(contours[i]), polygon, 3, true);
                    contours_poly.push_back(polygon);

                    rect = boundingRect(cv::Mat(contours_poly[idx]));
                    boundRect.push_back(rect);

                    areaIdx.push_back(i);
                    cv::RotatedRect rotRect = fitEllipse(contours[i]);
                    outerEllipseRects.push_back(rotRect);

                    indices.push_back(rotRect.center);

                    idx = idx + 1;
                }
            }
        }
        for (int i = 0; i < areaIdx.size(); i++) {
            cv::Scalar color = cv::Scalar(0, 0, 255);
            rectangle(drawing, outerEllipseRects[i].boundingRect(), color, 2);
           //cv::Scalar color2 = cv::Scalar(0, 255, 0);
            //rectangle(drawing,innerEllipseRects[i].boundingRect(),color2,2);
        }

            if (areaIdx.size() >0) {
                namedWindow("Targets", cv::WINDOW_AUTOSIZE);
                imshow("Targets", drawing);
                cv::waitKey(0);
            }
    }

    return indices;
}

bool isDefinedPoint(pcl::PointXYZ point) {
    if (point.x == 0 && point.y == 0 && point.z == 0) {
        return false;
    }
    return true;
}

int main(int argc, char** argv) {
    rosbag::Bag bag;
    std::cout << "Loading bag ... " << std::endl;
    bag.open("/mnt/b6ef98c1-c7b0-4b69-87d0-2165d664c748/bags/debug/map_gen_0.bag");
    std::cout << "Finished loading bag." << std::endl;

    ouster_ros::OS1::CloudOS1 cloud{};
    auto H = OS1::pixels_per_column;
    auto W = OS1::n_cols_of_lidar_mode(OS1::lidar_mode::MODE_1024x10);
    const auto px_offset = ouster::OS1::get_px_offset(W);
    int ptCloudCounter = 0;

    for(rosbag::MessageInstance const m: rosbag::View(bag))
    {
        sensor_msgs::PointCloud2::ConstPtr ptCloud = m.instantiate<sensor_msgs::PointCloud2>();
               if (ptCloud != nullptr) {
                   pcl::fromROSMsg(*ptCloud, cloud);
                   ptCloudCounter = ptCloudCounter + 1;

                   cv::Mat intensity_image = cv::Mat(H, W, CV_32F);
                   cv::Mat index_image = cv::Mat(H, W, CV_32SC1);

                   for (int u = 0; u < H; u++) {
                       for (int v = 0; v < W; v++) {
                           const size_t vv = (v + px_offset[u]) % W;
                           const size_t index = vv * H + u;
                           const auto& pt = cloud[index];
                           index_image.at<int>(u, v) = (int)  index;
                           intensity_image.at<float>(u, v) = pt.intensity;

                       }
                   }
                   if (ptCloudCounter> 1750) {
                       std::vector<cv::Point2f> intensityImageIndices;
                       intensityImageIndices = ptCloudIndicesOfLandMarks(intensity_image, 0.5271f,index_image,cloud);
                       std::cout << ptCloudCounter << "th point cloud contains {" << intensityImageIndices.size()
                                 << "} targets" << std::endl;
                   }
               }
    }

    bag.close();

    return EXIT_SUCCESS;
}

ext: cpp


text:
#include <boost/typeof/typeof.hpp>
#include <iostream>
#include <vector>
#include "OccupancyGrid/raytrace.hpp"
#include <gtest/gtest.h>
using namespace occgrid;

int discretize(double x, double origin, double cellsize) {
  return std::floor((x - origin) / cellsize);
}

void slow_raytrace(
    double px,
    double py,
    double dx,
    double dy,
    double origin_x,
    double origin_y,
    double cell_size_x,
    double cell_size_y,
    size_t N,
    std::vector<int>& is,
    std::vector<int>& js)
{
  double step = std::min(cell_size_x, cell_size_y) / 2.;
  double steps_taken = 0;
  int i = 0, j=0;
  for (size_t k = 0; k < N; ++k) {
    i = discretize(px + steps_taken * dx, origin_x, cell_size_x);
    j = discretize(py + steps_taken * dy, origin_y, cell_size_y);
    steps_taken += step;
    while ((k > 0) && ((i == is[k - 1]) && (j == js[k - 1]))) {
      i = discretize(px + steps_taken * dx, origin_x, cell_size_x);
      j = discretize(py + steps_taken * dy, origin_y, cell_size_y);
      steps_taken += step;
    }
    is[k] = i;
    js[k] = j;
  }
}

void test_with_values(double px, double py, double dx, double dy) {
  double cell_size_x = 1, cell_size_y = 1;
  double origin_x = -10, origin_y = 10;
  // std::cout << "pos:(" << px << ", " << py << "),"
  //   << "dir:(" << dx << ", " << dy << ")"
  //   << "cellsize:(" << cell_size_x << ", " << cell_size_y << ")"
  //   << "origin:(" << origin_x << ", " << origin_y << ")" << std::endl;

  size_t N = 10;
  std::vector<int> is(N);
  std::vector<int> js(N);
  slow_raytrace(px, py, dx, dy, origin_x, origin_y, cell_size_x, cell_size_y, N, is, js);

  typedef ray_trace_iterator<double, int> raytrace_it;
  BOOST_AUTO(rt_it, raytrace_it(px, py, dx, dy, origin_x, origin_y, cell_size_x, cell_size_y));
  for (size_t k = 0; k < 10; ++k, ++rt_it) {
    std::stringstream ss;
    ss << "(" << rt_it->first << ", " << rt_it->second << ")"
      << "<=> (" << is[k] << "," << js[k] << ")" << std::endl;
    bool x_test = ((is[k] - 1) <= rt_it->first) && (rt_it->first <= (is[k] + 1));
    ASSERT_TRUE(x_test) ;
    bool y_test = ((js[k] - 1) <= rt_it->second) && (rt_it->second <= (js[k] + 1));
    ASSERT_TRUE(y_test);
  }
}

TEST(test_raytrace, test_random)
{
  double px = (20. * rand() / RAND_MAX - 10);
  double py = (20. * rand() / RAND_MAX - 10);
  double dx = (2. * rand() / RAND_MAX - 1);
  double dy = (2. * rand() / RAND_MAX - 1);
  if ((dx == 0) && (dy == 0))
    dx = 1;
  test_with_values(px, py, dx, dy);
}

TEST(test_raytrace, test_zero_direction)
{
  double px = (20. * rand() / RAND_MAX - 10);
  double py = (20. * rand() / RAND_MAX - 10);
  double dx = (2. * rand() / RAND_MAX - 1);
  double dy = 0;
  if ((dx == 0) && (dy == 0))
    dx = 1;
  test_with_values(px, py, dx, dy);
}

int main(int argc, char *argv[])
{
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}

max_stars_repo_name: wecacuee/modern-occupancy-grid

max_issues_repo_name: wecacuee/modern-occupancy-grid

max_forks_repo_name: wecacuee/modern-occupancy-grid


text:
#include <cassert>
#include <string>
#include <vector>
#include <iostream>
#include <fstream>
#include <boost/algorithm/string.hpp>
#include <numeric>
#include "gurobi_c.h"

using namespace std;

int ReadSolution(string solfile,
        vector<double> &solution,
        vector<string> &names,
        double &objval);

int main(int argc, char* argv[])
{
    int status = 0;

    GRBenv *env = NULL;

    status = GRBloadenv(&env, NULL);
    assert(!status);

    //status = CPXsetintparam(env, CPXPARAM_ScreenOutput, CPX_ON);
    //assert(!status);

    GRBmodel *model;
    status = GRBreadmodel(env, argv[1], &model);
    assert(!status);

    vector<double> solution;
    vector<string> names;
    double objectivevalue;

    status = ReadSolution(argv[2],
            solution,
            names,
            objectivevalue);
    assert(!status);

    //We sort the solution, according to index in problem
    int numvars;
    status = GRBgetintattr(model, "NumVars", &numvars);
    assert(!status);
    vector<double> sortedsolution(numvars);

    for(int i = 0; i < solution.size(); ++i)
    {
        int idx;
        status = GRBgetvarbyname(model,
                names[i].c_str(),
                &idx);
        if(idx != -1)
        {
            sortedsolution[idx] =solution[i];
            if(idx >= sortedsolution.size())
            {
                cout << "Variable index out of bounds!!" <<  endl;
                exit(1);
            }
        }
        else
        {
            cerr << "Error. Name not found" << endl;
            exit(1);
        }
    }

    //vector<int> indices;
    //int beg = 0;
    //int effortlevel = CPX_MIPSTART_CHECKFEAS;
    //for(int i = 0; i < CPXgetnumcols(env, problem); ++i)
        //indices.push_back(i);

    status = GRBsetdblattrarray(model,
            "Start",
            0,
            numvars-1,
            &(sortedsolution[0]));
    assert(!status);

    //status = CPXaddmipstarts(env,
            //problem,
            //1,
            //CPXgetnumcols(env, problem),
            //&beg,
            //&(indices[0]),
            //&(sortedsolution[0]),
            //&(effortlevel),
            //NULL);
    //assert(!status);

    ////check objective function
    //vector<double> obj(CPXgetnumcols(env, problem));

    //status = CPXgetobj(env,
            //problem,
            //&(obj[0]),
            //0,
            //CPXgetnumcols(env, problem)-1);
    //assert(!status);

    //cout << "Number of binary variables: " << CPXgetnumbin(env, problem) << "/" << CPXgetnumcols(env, problem) << endl;
    //cout << "Obj value of solution: " << inner_product(obj.begin(), obj.end(), sortedsolution.begin(), 0.0) << endl;

    //cout << "Sum of nonzero coefficients: " << accumulate(obj.begin(), obj.end(), 0.0) << endl;

    //status = CPXwriteprob(env, problem, "problem.lp", NULL);
    //assert(!status);

    //status = GRBread(model, argv[2]);
    //assert(!status);

    status = GRBsetdblparam(GRBgetenv(model), "TimeLimit", 10.0);
    assert(!status);

    status = GRBoptimize(model);

    string solname = argv[2];
    solname += "_grb.sol";

    status = GRBwrite(model, solname.c_str());
    assert(!status);
    status = GRBfreemodel(model);
    assert(!status);

    GRBfreeenv(env);

}

int ReadSolution(string solfile,
        vector<double> &solution,
        vector<string> &names,
        double &objval)
{
    int status;

    ifstream infile(solfile.c_str());

    string line;
    vector<string> tokens;

    while(getline(infile, line))
    {
        if(line == "")
            continue;
        boost::algorithm::split(tokens, line, boost::algorithm::is_space(), boost::token_compress_on);
        if(tokens[0] == "=obj=")
        {
            objval = atof(tokens[1].c_str());
        }
        else
        {
            solution.push_back(atof(tokens[1].c_str()));
            names.push_back(tokens[0].c_str());
            cout << "Variable " << tokens[0] << "= " << atof(tokens[1].c_str()) << endl;
        }
    }

    infile.close();

    return 0;
}

max_stars_repo_name: rocarvaj/mipstart-example

max_issues_repo_name: rocarvaj/mipstart-example

max_forks_repo_name: rocarvaj/mipstart-example


text:
#include "mex.h"
#include <iostream>

#include "fitPlane.h"
#include <Eigen/Dense>

using namespace std;

void mexFunction (int nlhs,mxArray *plhs[],int nrhs,const mxArray *prhs[]) {

    // check arguments
    if (nrhs < 5) mexErrMsgTxt("At least 5 input arguments required.");

    // input pointers
    double *X        =    (double*)mxGetPr(prhs[0]);
    double *Y        =    (double*)mxGetPr(prhs[1]);
    double *Z        =    (double*)mxGetPr(prhs[2]);
    bool   *MASK     =      (bool*)mxGetPr(prhs[3]);
    double *param    =    (double*)mxGetPr(prhs[4]);

    int dim1 = mxGetM(prhs[1]);       // (height)
    int dim2 = mxGetN(prhs[1]);       // (width)
    int sz = dim1*dim2;

    // Output pointers
    if (dim1 < 1)   return;

    // Compute Geometry
    CFitPlane fP;
    fP.Init(X,Y,Z,MASK, dim1,dim2);
    fP.SetNeighborSize((int)param[0]);
    fP.SetSubsampleStep((int)param[1]);
    fP.fitPlane();
    Eigen::MatrixXd C = fP.GetCoeffs();

    if (nlhs == 1 ){
        plhs[0]         = mxCreateDoubleMatrix(4,sz,mxREAL);
        double *pOut1   = (double*)mxGetPr(plhs[0]);

        for (int k=0;k<dim1*dim2;k++)
        {
          int _4k = 4*k;
          *(pOut1+_4k+0) = C(0,k);
          *(pOut1+_4k+1) = C(1,k);
          *(pOut1+_4k+2) = C(2,k);
          *(pOut1+_4k+3) = C(3,k);
        }
    } else if (nlhs == 2 ){

        plhs[0]         = mxCreateDoubleMatrix(4,sz,mxREAL);
        double *pOut1   = (double*)mxGetPr(plhs[0]);

        plhs[1]         = mxCreateDoubleMatrix(4,sz,mxREAL);
        double *pOut2   = (double*)mxGetPr(plhs[1]);

        Eigen::MatrixXd S = fP.GetSingValues();
        for (int k=0;k<dim1*dim2;k++)
        {
          int _4k = 4*k;
          *(pOut1+_4k+0) = C(0,k);
          *(pOut1+_4k+1) = C(1,k);
          *(pOut1+_4k+2) = C(2,k);
          *(pOut1+_4k+3) = C(3,k);

          *(pOut2+_4k+0) = S(0,k);
          *(pOut2+_4k+1) = S(1,k);
          *(pOut2+_4k+2) = S(2,k);
          *(pOut2+_4k+3) = S(3,k);
        }
    } else if (nlhs == 3)
    {
        plhs[0]         = mxCreateDoubleMatrix(4,sz,mxREAL);
        double *pOut1   = (double*)mxGetPr(plhs[0]);

        plhs[1]         = mxCreateDoubleMatrix(4,sz,mxREAL);
        double *pOut2   = (double*)mxGetPr(plhs[1]);

        plhs[2]         = mxCreateDoubleMatrix(1,sz,mxREAL);
        double *pOut3   = (double*)mxGetPr(plhs[2]);

        Eigen::MatrixXd S = fP.GetSingValues();
        Eigen::VectorXi V = fP.GetValidity();
        for (int k=0;k<dim1*dim2;k++)
        {
          int _4k = 4*k;
          *(pOut1+_4k+0) = C(0,k);
          *(pOut1+_4k+1) = C(1,k);
          *(pOut1+_4k+2) = C(2,k);
          *(pOut1+_4k+3) = C(3,k);

          *(pOut2+_4k+0) = S(0,k);
          *(pOut2+_4k+1) = S(1,k);
          *(pOut2+_4k+2) = S(2,k);
          *(pOut2+_4k+3) = S(3,k);

          *(pOut3+k) = (double)V(k);
        }

    }

    return;
}

max_stars_repo_name: ToyotaResearchInstitute/rad-robot

max_issues_repo_name: ToyotaResearchInstitute/rad-robot

max_forks_repo_name: ToyotaResearchInstitute/rad-robot


text:
/*
    Copyright 2008 Christian Henning
    Use, modification and distribution are subject to the Boost Software License,
    Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
    http://www.boost.org/LICENSE_1_0.txt).
*/

/*************************************************************************************************/

#ifndef BOOST_GIL_EXTENSION_OPENCV_IPL_IMAGE_WRAPPER_HPP_INCLUDED
#define BOOST_GIL_EXTENSION_OPENCV_IPL_IMAGE_WRAPPER_HPP_INCLUDED

////////////////////////////////////////////////////////////////////////////////////////
/// \file
/// \brief
/// \author Christian Henning \n
///
/// \date 2008 \n
///
////////////////////////////////////////////////////////////////////////////////////////

#include <boost\shared_ptr.hpp>
#include <boost\gil\gil_all.hpp>

#include "utilities.hpp"

namespace boost { namespace gil { namespace opencv {

template < typename Channel > struct ipl_channel_type : boost::mpl::false_ {};
template<> struct ipl_channel_type< bits8 >   : boost::mpl::int_< IPL_DEPTH_8U  > {};
template<> struct ipl_channel_type< bits16 >  : boost::mpl::int_< IPL_DEPTH_16U > {};
template<> struct ipl_channel_type< bits32f > : boost::mpl::int_< IPL_DEPTH_32F > {};
template<> struct ipl_channel_type< double >  : boost::mpl::int_< IPL_DEPTH_64F > {};
template<> struct ipl_channel_type< bits8s >  : boost::mpl::int_< IPL_DEPTH_8S  > {};
template<> struct ipl_channel_type< bits16s > : boost::mpl::int_< IPL_DEPTH_16S > {};
template<> struct ipl_channel_type< bits32s > : boost::mpl::int_< IPL_DEPTH_32S > {};

/**
 *
 * ipl_image_wrapper encloses a IplImage pointer. Value semantics, like
 * copying, are supported by using shared_ptr.
 *
 **/
class ipl_image_wrapper
{
public:

    typedef boost::shared_ptr< IplImage > ipl_image_ptr_t;

public:
    ipl_image_wrapper() {}
    ipl_image_wrapper( IplImage* img ) : _img( img, ipl_deleter ) {}

    IplImage*       get()       { return _img.get(); }
    const IplImage* get() const { return _img.get(); }

private:

    static void ipl_deleter( IplImage* ipl_img )
    {
        if( ipl_img )
        {
            cvReleaseImageHeader( &ipl_img );
        }
    }

   ipl_image_ptr_t _img;
};

template< typename View >
inline
ipl_image_wrapper create_ipl_image( View view )
{
    typedef typename channel_type< View >::type channel_t;

    IplImage* img;

    if(( img = cvCreateImageHeader( make_cvSize( view.dimensions() )
                                  , ipl_channel_type<channel_t>::type::value
                                  , num_channels<View>::value
                                  )) == NULL )
    {
        throw std::runtime_error( "Cannot create IPL image." );
    }

    cvSetData( img
             , interleaved_view_get_raw_data( view )
             , num_channels<View>::value * view.width() * sizeof( channel_t ) );

    return ipl_image_wrapper( img );
}

} // namespace opencv
} // namespace gil
} // namespace boost

#endif // BOOST_GIL_EXTENSION_OPENCV_IPL_IMAGE_WRAPPER_HPP_INCLUDED

ext: hpp

max_stars_repo_name: boost-gil/gil-contributions

max_issues_repo_name: boost-gil/gil-contributions-archive

max_forks_repo_name: boost-gil/gil-contributions-archive


text: #include <boost/range/metafunctions.hpp>

max_stars_repo_name: miathedev/BoostForArduino

max_issues_repo_name: miathedev/BoostForArduino

max_forks_repo_name: miathedev/BoostForArduino


text:
/**
 * Copyright Soramitsu Co., Ltd. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0
 */

#ifndef KAGOME_CONSENSUS_GRANDPA_GRANDPAIMPL
#define KAGOME_CONSENSUS_GRANDPA_GRANDPAIMPL

#include "consensus/grandpa/grandpa.hpp"
#include "consensus/grandpa/grandpa_observer.hpp"

#include <boost/operators.hpp>

#include "application/app_state_manager.hpp"
#include "blockchain/block_tree.hpp"
#include "consensus/authority/authority_manager.hpp"
#include "consensus/grandpa/environment.hpp"
#include "consensus/grandpa/impl/voting_round_impl.hpp"
#include "consensus/grandpa/movable_round_state.hpp"
#include "consensus/grandpa/voter_set.hpp"
#include "crypto/ed25519_provider.hpp"
#include "crypto/hasher.hpp"
#include "log/logger.hpp"
#include "metrics/metrics.hpp"
#include "network/synchronizer.hpp"
#include "runtime/runtime_api/grandpa_api.hpp"
#include "storage/buffer_map_types.hpp"

namespace kagome::consensus::grandpa {

  // help struct to correctly compare rounds in different voter sets
  struct FullRound : boost::less_than_comparable<FullRound>,
                     boost::equality_comparable<FullRound> {
    MembershipCounter voter_set_id;
    RoundNumber round_number;
    explicit FullRound(const std::shared_ptr<const VotingRound> &round)
        : voter_set_id(round->voterSetId()),
          round_number(round->roundNumber()) {}
    explicit FullRound(const network::GrandpaNeighborMessage &msg)
        : voter_set_id(msg.voter_set_id), round_number(msg.round_number) {}
    explicit FullRound(const network::CatchUpRequest &msg)
        : voter_set_id(msg.voter_set_id), round_number(msg.round_number) {}
    explicit FullRound(const network::CatchUpResponse &msg)
        : voter_set_id(msg.voter_set_id), round_number(msg.round_number) {}
    explicit FullRound(const VoteMessage &msg)
        : voter_set_id(msg.counter), round_number(msg.round_number) {}
    FullRound &operator=(const FullRound &) = default;

    bool operator<(const FullRound &round) const {
      return voter_set_id < round.voter_set_id
             or (voter_set_id == round.voter_set_id
                 and round_number < round.round_number);
    }

    bool operator==(const FullRound &round) const {
      return voter_set_id == round.voter_set_id
             && round_number == round.round_number;
    }
  };

  class GrandpaImpl : public Grandpa,
                      public GrandpaObserver,
                      public std::enable_shared_from_this<GrandpaImpl> {
   public:
    ~GrandpaImpl() override = default;

    GrandpaImpl(std::shared_ptr<application::AppStateManager> app_state_manager,
                std::shared_ptr<Environment> environment,
                std::shared_ptr<storage::BufferStorage> storage,
                std::shared_ptr<crypto::Ed25519Provider> crypto_provider,
                std::shared_ptr<runtime::GrandpaApi> grandpa_api,
                const std::shared_ptr<crypto::Ed25519Keypair> &keypair,
                std::shared_ptr<Clock> clock,
                std::shared_ptr<boost::asio::io_context> io_context,
                std::shared_ptr<authority::AuthorityManager> authority_manager,
                std::shared_ptr<network::Synchronizer> synchronizer);

    /** @see AppStateManager::takeControl */
    bool prepare();

    /** @see AppStateManager::takeControl */
    bool start();

    /** @see AppStateManager::takeControl */
    void stop();

    // Neighbor

    void onNeighborMessage(const libp2p::peer::PeerId &peer_id,
                           const network::GrandpaNeighborMessage &msg) override;

    // Catch-up methods

    void onCatchUpRequest(const libp2p::peer::PeerId &peer_id,
                          const network::CatchUpRequest &msg) override;

    void onCatchUpResponse(const libp2p::peer::PeerId &peer_id,
                           const network::CatchUpResponse &msg) override;

    // Voting methods

    void onVoteMessage(const libp2p::peer::PeerId &peer_id,
                       const network::VoteMessage &msg) override;

    void onCommitMessage(const libp2p::peer::PeerId &peer_id,
                         const network::FullCommitMessage &msg) override;

    outcome::result<void> applyJustification(
        const BlockInfo &block_info,
        const GrandpaJustification &justification) override;

    // Round processing method

    void executeNextRound() override;

   private:
    std::shared_ptr<VotingRound> selectRound(
        RoundNumber round_number,
        std::optional<MembershipCounter> voter_set_id);
    outcome::result<MovableRoundState> getLastCompletedRound() const;

    std::shared_ptr<VotingRound> makeInitialRound(
        const MovableRoundState &round_state, std::shared_ptr<VoterSet> voters);

    std::shared_ptr<VotingRound> makeNextRound(
        const std::shared_ptr<VotingRound> &previous_round);

    void loadMissingBlocks();

    void onCompletedRound(outcome::result<MovableRoundState> round_state_res);

    void tryCatchUp(const libp2p::peer::PeerId &peer_id,
                    const FullRound &next,
                    const FullRound &curr);

    // Note: Duration value was gotten from substrate
    // https://github.com/paritytech/substrate/blob/efbac7be80c6e8988a25339061078d3e300f132d/bin/node-template/node/src/service.rs#L166
    // Perhaps, 333ms is not enough for normal communication during the round
    const Clock::Duration round_time_factor_ = std::chrono::milliseconds(333);

    std::shared_ptr<VotingRound> previous_round_;
    std::shared_ptr<VotingRound> current_round_;

    std::shared_ptr<Environment> environment_;
    std::shared_ptr<storage::BufferStorage> storage_;
    std::shared_ptr<crypto::Ed25519Provider> crypto_provider_;
    std::shared_ptr<runtime::GrandpaApi> grandpa_api_;
    const std::shared_ptr<crypto::Ed25519Keypair> &keypair_;
    std::shared_ptr<Clock> clock_;
    std::shared_ptr<boost::asio::io_context> io_context_;
    std::shared_ptr<authority::AuthorityManager> authority_manager_;
    std::shared_ptr<network::Synchronizer> synchronizer_;

    std::vector<FullRound> neighbor_msgs_{};

    // Metrics
    metrics::RegistryPtr metrics_registry_ = metrics::createRegistry();
    metrics::Gauge *metric_highest_round_;

    log::Logger logger_ = log::createLogger("Grandpa", "grandpa");
  };

}  // namespace kagome::consensus::grandpa

#endif  // KAGOME_CONSENSUS_GRANDPA_GRANDPAIMPL

max_stars_repo_name: soramitsu/kagome

max_issues_repo_name: soramitsu/kagome

max_forks_repo_name: soramitsu/kagome


text:
#pragma once

#include <boost/filesystem.hpp>

#include <lmdb/libraries/liblmdb/lmdb.h>

#include <btcb/lib/numbers.hpp>
#include <btcb/secure/blockstore.hpp>
#include <btcb/secure/common.hpp>

namespace btcb
{
class mdb_env;
class mdb_txn : public transaction_impl
{
public:
	mdb_txn (btcb::mdb_env const &, bool = false);
	mdb_txn (btcb::mdb_txn const &) = delete;
	mdb_txn (btcb::mdb_txn &&) = default;
	~mdb_txn ();
	btcb::mdb_txn & operator= (btcb::mdb_txn const &) = delete;
	btcb::mdb_txn & operator= (btcb::mdb_txn &&) = default;
	operator MDB_txn * () const;
	MDB_txn * handle;
};
/**
 * RAII wrapper for MDB_env
 */
class mdb_env
{
public:
	mdb_env (bool &, boost::filesystem::path const &, int max_dbs = 128);
	~mdb_env ();
	operator MDB_env * () const;
	btcb::transaction tx_begin (bool = false) const;
	MDB_txn * tx (btcb::transaction const &) const;
	MDB_env * environment;
};

/**
 * Encapsulates MDB_val and provides uint256_union conversion of the data.
 */
class mdb_val
{
public:
	enum class no_value
	{
		dummy
	};
	mdb_val (btcb::epoch = btcb::epoch::unspecified);
	mdb_val (btcb::account_info const &);
	mdb_val (btcb::block_info const &);
	mdb_val (MDB_val const &, btcb::epoch = btcb::epoch::unspecified);
	mdb_val (btcb::pending_info const &);
	mdb_val (btcb::pending_key const &);
	mdb_val (size_t, void *);
	mdb_val (btcb::uint128_union const &);
	mdb_val (btcb::uint256_union const &);
	mdb_val (std::shared_ptr<btcb::block> const &);
	mdb_val (std::shared_ptr<btcb::vote> const &);
	void * data () const;
	size_t size () const;
	explicit operator btcb::account_info () const;
	explicit operator btcb::block_info () const;
	explicit operator btcb::pending_info () const;
	explicit operator btcb::pending_key () const;
	explicit operator btcb::uint128_union () const;
	explicit operator btcb::uint256_union () const;
	explicit operator std::array<char, 64> () const;
	explicit operator no_value () const;
	explicit operator std::shared_ptr<btcb::block> () const;
	explicit operator std::shared_ptr<btcb::send_block> () const;
	explicit operator std::shared_ptr<btcb::receive_block> () const;
	explicit operator std::shared_ptr<btcb::open_block> () const;
	explicit operator std::shared_ptr<btcb::change_block> () const;
	explicit operator std::shared_ptr<btcb::state_block> () const;
	explicit operator std::shared_ptr<btcb::vote> () const;
	explicit operator uint64_t () const;
	operator MDB_val * () const;
	operator MDB_val const & () const;
	MDB_val value;
	std::shared_ptr<std::vector<uint8_t>> buffer;
	btcb::epoch epoch{ btcb::epoch::unspecified };
};
class block_store;

template <typename T, typename U>
class mdb_iterator : public store_iterator_impl<T, U>
{
public:
	mdb_iterator (btcb::transaction const & transaction_a, MDB_dbi db_a, btcb::epoch = btcb::epoch::unspecified);
	mdb_iterator (std::nullptr_t, btcb::epoch = btcb::epoch::unspecified);
	mdb_iterator (btcb::transaction const & transaction_a, MDB_dbi db_a, MDB_val const & val_a, btcb::epoch = btcb::epoch::unspecified);
	mdb_iterator (btcb::mdb_iterator<T, U> && other_a);
	mdb_iterator (btcb::mdb_iterator<T, U> const &) = delete;
	~mdb_iterator ();
	btcb::store_iterator_impl<T, U> & operator++ () override;
	std::pair<btcb::mdb_val, btcb::mdb_val> * operator-> ();
	bool operator== (btcb::store_iterator_impl<T, U> const & other_a) const override;
	bool is_end_sentinal () const override;
	void fill (std::pair<T, U> &) const override;
	void clear ();
	btcb::mdb_iterator<T, U> & operator= (btcb::mdb_iterator<T, U> && other_a);
	btcb::store_iterator_impl<T, U> & operator= (btcb::store_iterator_impl<T, U> const &) = delete;
	MDB_cursor * cursor;
	std::pair<btcb::mdb_val, btcb::mdb_val> current;

private:
	MDB_txn * tx (btcb::transaction const &) const;
};

/**
 * Iterates the key/value pairs of two stores merged together
 */
template <typename T, typename U>
class mdb_merge_iterator : public store_iterator_impl<T, U>
{
public:
	mdb_merge_iterator (btcb::transaction const &, MDB_dbi, MDB_dbi);
	mdb_merge_iterator (std::nullptr_t);
	mdb_merge_iterator (btcb::transaction const &, MDB_dbi, MDB_dbi, MDB_val const &);
	mdb_merge_iterator (btcb::mdb_merge_iterator<T, U> &&);
	mdb_merge_iterator (btcb::mdb_merge_iterator<T, U> const &) = delete;
	~mdb_merge_iterator ();
	btcb::store_iterator_impl<T, U> & operator++ () override;
	std::pair<btcb::mdb_val, btcb::mdb_val> * operator-> ();
	bool operator== (btcb::store_iterator_impl<T, U> const &) const override;
	bool is_end_sentinal () const override;
	void fill (std::pair<T, U> &) const override;
	void clear ();
	btcb::mdb_merge_iterator<T, U> & operator= (btcb::mdb_merge_iterator<T, U> &&) = default;
	btcb::mdb_merge_iterator<T, U> & operator= (btcb::mdb_merge_iterator<T, U> const &) = delete;

private:
	btcb::mdb_iterator<T, U> & least_iterator () const;
	std::unique_ptr<btcb::mdb_iterator<T, U>> impl1;
	std::unique_ptr<btcb::mdb_iterator<T, U>> impl2;
};

/**
 * mdb implementation of the block store
 */
class mdb_store : public block_store
{
	friend class btcb::block_predecessor_set;

public:
	mdb_store (bool &, boost::filesystem::path const &, int lmdb_max_dbs = 128);

	btcb::transaction tx_begin_write () override;
	btcb::transaction tx_begin_read () override;
	btcb::transaction tx_begin (bool write = false) override;

	void initialize (btcb::transaction const &, btcb::genesis const &) override;
	void block_put (btcb::transaction const &, btcb::block_hash const &, btcb::block const &, btcb::block_hash const & = btcb::block_hash (0), btcb::epoch version = btcb::epoch::epoch_0) override;
	btcb::block_hash block_successor (btcb::transaction const &, btcb::block_hash const &) override;
	void block_successor_clear (btcb::transaction const &, btcb::block_hash const &) override;
	std::shared_ptr<btcb::block> block_get (btcb::transaction const &, btcb::block_hash const &) override;
	std::shared_ptr<btcb::block> block_random (btcb::transaction const &) override;
	void block_del (btcb::transaction const &, btcb::block_hash const &) override;
	bool block_exists (btcb::transaction const &, btcb::block_hash const &) override;
	bool block_exists (btcb::transaction const &, btcb::block_type, btcb::block_hash const &) override;
	btcb::block_counts block_count (btcb::transaction const &) override;
	bool root_exists (btcb::transaction const &, btcb::uint256_union const &) override;

	void frontier_put (btcb::transaction const &, btcb::block_hash const &, btcb::account const &) override;
	btcb::account frontier_get (btcb::transaction const &, btcb::block_hash const &) override;
	void frontier_del (btcb::transaction const &, btcb::block_hash const &) override;

	void account_put (btcb::transaction const &, btcb::account const &, btcb::account_info const &) override;
	bool account_get (btcb::transaction const &, btcb::account const &, btcb::account_info &) override;
	void account_del (btcb::transaction const &, btcb::account const &) override;
	bool account_exists (btcb::transaction const &, btcb::account const &) override;
	size_t account_count (btcb::transaction const &) override;
	btcb::store_iterator<btcb::account, btcb::account_info> latest_v0_begin (btcb::transaction const &, btcb::account const &) override;
	btcb::store_iterator<btcb::account, btcb::account_info> latest_v0_begin (btcb::transaction const &) override;
	btcb::store_iterator<btcb::account, btcb::account_info> latest_v0_end () override;
	btcb::store_iterator<btcb::account, btcb::account_info> latest_v1_begin (btcb::transaction const &, btcb::account const &) override;
	btcb::store_iterator<btcb::account, btcb::account_info> latest_v1_begin (btcb::transaction const &) override;
	btcb::store_iterator<btcb::account, btcb::account_info> latest_v1_end () override;
	btcb::store_iterator<btcb::account, btcb::account_info> latest_begin (btcb::transaction const &, btcb::account const &) override;
	btcb::store_iterator<btcb::account, btcb::account_info> latest_begin (btcb::transaction const &) override;
	btcb::store_iterator<btcb::account, btcb::account_info> latest_end () override;

	void pending_put (btcb::transaction const &, btcb::pending_key const &, btcb::pending_info const &) override;
	void pending_del (btcb::transaction const &, btcb::pending_key const &) override;
	bool pending_get (btcb::transaction const &, btcb::pending_key const &, btcb::pending_info &) override;
	bool pending_exists (btcb::transaction const &, btcb::pending_key const &) override;
	btcb::store_iterator<btcb::pending_key, btcb::pending_info> pending_v0_begin (btcb::transaction const &, btcb::pending_key const &) override;
	btcb::store_iterator<btcb::pending_key, btcb::pending_info> pending_v0_begin (btcb::transaction const &) override;
	btcb::store_iterator<btcb::pending_key, btcb::pending_info> pending_v0_end () override;
	btcb::store_iterator<btcb::pending_key, btcb::pending_info> pending_v1_begin (btcb::transaction const &, btcb::pending_key const &) override;
	btcb::store_iterator<btcb::pending_key, btcb::pending_info> pending_v1_begin (btcb::transaction const &) override;
	btcb::store_iterator<btcb::pending_key, btcb::pending_info> pending_v1_end () override;
	btcb::store_iterator<btcb::pending_key, btcb::pending_info> pending_begin (btcb::transaction const &, btcb::pending_key const &) override;
	btcb::store_iterator<btcb::pending_key, btcb::pending_info> pending_begin (btcb::transaction const &) override;
	btcb::store_iterator<btcb::pending_key, btcb::pending_info> pending_end () override;

	void block_info_put (btcb::transaction const &, btcb::block_hash const &, btcb::block_info const &) override;
	void block_info_del (btcb::transaction const &, btcb::block_hash const &) override;
	bool block_info_get (btcb::transaction const &, btcb::block_hash const &, btcb::block_info &) override;
	bool block_info_exists (btcb::transaction const &, btcb::block_hash const &) override;
	btcb::store_iterator<btcb::block_hash, btcb::block_info> block_info_begin (btcb::transaction const &, btcb::block_hash const &) override;
	btcb::store_iterator<btcb::block_hash, btcb::block_info> block_info_begin (btcb::transaction const &) override;
	btcb::store_iterator<btcb::block_hash, btcb::block_info> block_info_end () override;
	btcb::uint128_t block_balance (btcb::transaction const &, btcb::block_hash const &) override;
	btcb::epoch block_version (btcb::transaction const &, btcb::block_hash const &) override;

	btcb::uint128_t representation_get (btcb::transaction const &, btcb::account const &) override;
	void representation_put (btcb::transaction const &, btcb::account const &, btcb::uint128_t const &) override;
	void representation_add (btcb::transaction const &, btcb::account const &, btcb::uint128_t const &) override;
	btcb::store_iterator<btcb::account, btcb::uint128_union> representation_begin (btcb::transaction const &) override;
	btcb::store_iterator<btcb::account, btcb::uint128_union> representation_end () override;

	void unchecked_clear (btcb::transaction const &) override;
	void unchecked_put (btcb::transaction const &, btcb::unchecked_key const &, std::shared_ptr<btcb::block> const &) override;
	void unchecked_put (btcb::transaction const &, btcb::block_hash const &, std::shared_ptr<btcb::block> const &) override;
	std::vector<std::shared_ptr<btcb::block>> unchecked_get (btcb::transaction const &, btcb::block_hash const &) override;
	bool unchecked_exists (btcb::transaction const &, btcb::unchecked_key const &) override;
	void unchecked_del (btcb::transaction const &, btcb::unchecked_key const &) override;
	btcb::store_iterator<btcb::unchecked_key, std::shared_ptr<btcb::block>> unchecked_begin (btcb::transaction const &) override;
	btcb::store_iterator<btcb::unchecked_key, std::shared_ptr<btcb::block>> unchecked_begin (btcb::transaction const &, btcb::unchecked_key const &) override;
	btcb::store_iterator<btcb::unchecked_key, std::shared_ptr<btcb::block>> unchecked_end () override;
	size_t unchecked_count (btcb::transaction const &) override;

	void checksum_put (btcb::transaction const &, uint64_t, uint8_t, btcb::checksum const &) override;
	bool checksum_get (btcb::transaction const &, uint64_t, uint8_t, btcb::checksum &) override;
	void checksum_del (btcb::transaction const &, uint64_t, uint8_t) override;

	// Return latest vote for an account from store
	std::shared_ptr<btcb::vote> vote_get (btcb::transaction const &, btcb::account const &) override;
	// Populate vote with the next sequence number
	std::shared_ptr<btcb::vote> vote_generate (btcb::transaction const &, btcb::account const &, btcb::raw_key const &, std::shared_ptr<btcb::block>) override;
	std::shared_ptr<btcb::vote> vote_generate (btcb::transaction const &, btcb::account const &, btcb::raw_key const &, std::vector<btcb::block_hash>) override;
	// Return either vote or the stored vote with a higher sequence number
	std::shared_ptr<btcb::vote> vote_max (btcb::transaction const &, std::shared_ptr<btcb::vote>) override;
	// Return latest vote for an account considering the vote cache
	std::shared_ptr<btcb::vote> vote_current (btcb::transaction const &, btcb::account const &) override;
	void flush (btcb::transaction const &) override;
	btcb::store_iterator<btcb::account, std::shared_ptr<btcb::vote>> vote_begin (btcb::transaction const &) override;
	btcb::store_iterator<btcb::account, std::shared_ptr<btcb::vote>> vote_end () override;
	std::mutex cache_mutex;
	std::unordered_map<btcb::account, std::shared_ptr<btcb::vote>> vote_cache_l1;
	std::unordered_map<btcb::account, std::shared_ptr<btcb::vote>> vote_cache_l2;

	void version_put (btcb::transaction const &, int) override;
	int version_get (btcb::transaction const &) override;
	void do_upgrades (btcb::transaction const &);
	void upgrade_v1_to_v2 (btcb::transaction const &);
	void upgrade_v2_to_v3 (btcb::transaction const &);
	void upgrade_v3_to_v4 (btcb::transaction const &);
	void upgrade_v4_to_v5 (btcb::transaction const &);
	void upgrade_v5_to_v6 (btcb::transaction const &);
	void upgrade_v6_to_v7 (btcb::transaction const &);
	void upgrade_v7_to_v8 (btcb::transaction const &);
	void upgrade_v8_to_v9 (btcb::transaction const &);
	void upgrade_v9_to_v10 (btcb::transaction const &);
	void upgrade_v10_to_v11 (btcb::transaction const &);
	void upgrade_v11_to_v12 (btcb::transaction const &);

	// Requires a write transaction
	btcb::raw_key get_node_id (btcb::transaction const &) override;

	/** Deletes the node ID from the store */
	void delete_node_id (btcb::transaction const &) override;

	btcb::mdb_env env;

	/**
	 * Maps head block to owning account
	 * btcb::block_hash -> btcb::account
	 */
	MDB_dbi frontiers;

	/**
	 * Maps account v1 to account information, head, rep, open, balance, timestamp and block count.
	 * btcb::account -> btcb::block_hash, btcb::block_hash, btcb::block_hash, btcb::amount, uint64_t, uint64_t
	 */
	MDB_dbi accounts_v0;

	/**
	 * Maps account v0 to account information, head, rep, open, balance, timestamp and block count.
	 * btcb::account -> btcb::block_hash, btcb::block_hash, btcb::block_hash, btcb::amount, uint64_t, uint64_t
	 */
	MDB_dbi accounts_v1;

	/**
	 * Maps block hash to send block.
	 * btcb::block_hash -> btcb::send_block
	 */
	MDB_dbi send_blocks;

	/**
	 * Maps block hash to receive block.
	 * btcb::block_hash -> btcb::receive_block
	 */
	MDB_dbi receive_blocks;

	/**
	 * Maps block hash to open block.
	 * btcb::block_hash -> btcb::open_block
	 */
	MDB_dbi open_blocks;

	/**
	 * Maps block hash to change block.
	 * btcb::block_hash -> btcb::change_block
	 */
	MDB_dbi change_blocks;

	/**
	 * Maps block hash to v0 state block.
	 * btcb::block_hash -> btcb::state_block
	 */
	MDB_dbi state_blocks_v0;

	/**
	 * Maps block hash to v1 state block.
	 * btcb::block_hash -> btcb::state_block
	 */
	MDB_dbi state_blocks_v1;

	/**
	 * Maps min_version 0 (destination account, pending block) to (source account, amount).
	 * btcb::account, btcb::block_hash -> btcb::account, btcb::amount
	 */
	MDB_dbi pending_v0;

	/**
	 * Maps min_version 1 (destination account, pending block) to (source account, amount).
	 * btcb::account, btcb::block_hash -> btcb::account, btcb::amount
	 */
	MDB_dbi pending_v1;

	/**
	 * Maps block hash to account and balance.
	 * block_hash -> btcb::account, btcb::amount
	 */
	MDB_dbi blocks_info;

	/**
	 * Representative weights.
	 * btcb::account -> btcb::uint128_t
	 */
	MDB_dbi representation;

	/**
	 * Unchecked bootstrap blocks.
	 * btcb::block_hash -> btcb::block
	 */
	MDB_dbi unchecked;

	/**
	 * Mapping of region to checksum.
	 * (uint56_t, uint8_t) -> btcb::block_hash
	 */
	MDB_dbi checksum;

	/**
	 * Highest vote observed for account.
	 * btcb::account -> uint64_t
	 */
	MDB_dbi vote;

	/**
	 * Meta information about block store, such as versions.
	 * btcb::uint256_union (arbitrary key) -> blob
	 */
	MDB_dbi meta;

private:
	MDB_dbi block_database (btcb::block_type, btcb::epoch);
	template <typename T>
	std::shared_ptr<btcb::block> block_random (btcb::transaction const &, MDB_dbi);
	MDB_val block_raw_get (btcb::transaction const &, btcb::block_hash const &, btcb::block_type &);
	void block_raw_put (btcb::transaction const &, MDB_dbi, btcb::block_hash const &, MDB_val);
	void clear (MDB_dbi);
};
class wallet_value
{
public:
	wallet_value () = default;
	wallet_value (btcb::mdb_val const &);
	wallet_value (btcb::uint256_union const &, uint64_t);
	btcb::mdb_val val () const;
	btcb::private_key key;
	uint64_t work;
};
}

max_stars_repo_name: melnaquib/btcb

max_issues_repo_name: melnaquib/btcb

max_forks_repo_name: melnaquib/btcb


text:
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include <stdint.h>

#include <algorithm>
#include <memory>
#include <ostream>
#include <string>
#include <vector>

#include <boost/optional/optional.hpp>
#include <gflags/gflags_declare.h>
#include <glog/logging.h>
#include <gtest/gtest.h>

#include "kudu/common/common.pb.h"
#include "kudu/common/schema.h"
#include "kudu/common/wire_protocol.h"
#include "kudu/common/wire_protocol.pb.h"
#include "kudu/gutil/gscoped_ptr.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/gutil/walltime.h"
#include "kudu/master/catalog_manager.h"
#include "kudu/master/master-test-util.h"
#include "kudu/master/master.h"
#include "kudu/master/master.pb.h"
#include "kudu/master/mini_master.h"
#include "kudu/master/sys_catalog.h"
#include "kudu/master/ts_descriptor.h"
#include "kudu/mini-cluster/internal_mini_cluster.h"
#include "kudu/security/test/test_certs.h"
#include "kudu/security/tls_context.h"
#include "kudu/security/token_verifier.h"
#include "kudu/server/webserver_options.h"
#include "kudu/tablet/tablet.h"
#include "kudu/tablet/tablet_replica.h"
#include "kudu/tserver/mini_tablet_server.h"
#include "kudu/tserver/tablet_server.h"
#include "kudu/tserver/tablet_server_options.h"
#include "kudu/util/curl_util.h"
#include "kudu/util/faststring.h"
#include "kudu/util/metrics.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/sockaddr.h"
#include "kudu/util/pb_util.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
#include "kudu/util/version_info.h"

DECLARE_int32(heartbeat_interval_ms);
METRIC_DECLARE_counter(rows_inserted);
METRIC_DECLARE_counter(rows_updated);

using boost::none;
using std::shared_ptr;
using std::string;
using std::vector;

namespace kudu {

using cluster::InternalMiniCluster;
using master::CatalogManager;
using master::CreateTableRequestPB;
using master::CreateTableResponsePB;
using master::GetTableLocationsResponsePB;
using master::IsCreateTableDoneRequestPB;
using master::IsCreateTableDoneResponsePB;
using master::MiniMaster;
using master::TSDescriptor;
using master::TabletLocationsPB;
using kudu::pb_util::SecureShortDebugString;
using tserver::MiniTabletServer;

void CreateTableForTesting(MiniMaster* mini_master,
                           const string& table_name,
                           const Schema& schema,
                           string* tablet_id) {
  {
    CreateTableRequestPB req;
    CreateTableResponsePB resp;

    req.set_name(table_name);
    req.set_num_replicas(1);
    ASSERT_OK(SchemaToPB(schema, req.mutable_schema()));
    CatalogManager* catalog = mini_master->master()->catalog_manager();
    CatalogManager::ScopedLeaderSharedLock l(catalog);
    ASSERT_OK(l.first_failed_status());
    ASSERT_OK(catalog->CreateTable(&req, &resp, nullptr));
  }

  int wait_time = 1000;
  bool is_table_created = false;
  for (int i = 0; i < 80; ++i) {
    IsCreateTableDoneRequestPB req;
    IsCreateTableDoneResponsePB resp;

    req.mutable_table()->set_table_name(table_name);
    CatalogManager* catalog = mini_master->master()->catalog_manager();
    {
      CatalogManager::ScopedLeaderSharedLock l(catalog);
      ASSERT_OK(l.first_failed_status());
      ASSERT_OK(catalog->IsCreateTableDone(&req, &resp, /*user=*/none));
    }
    if (resp.done()) {
      is_table_created = true;
      break;
    }

    VLOG(1) << "Waiting for table '" << table_name << "'to be created";

    SleepFor(MonoDelta::FromMicroseconds(wait_time));
    wait_time = std::min(wait_time * 5 / 4, 1000000);
  }
  ASSERT_TRUE(is_table_created);

  GetTableLocationsResponsePB resp;
  ASSERT_OK(WaitForRunningTabletCount(mini_master, table_name, 1, &resp));
  *tablet_id = resp.tablet_locations(0).tablet_id();
  LOG(INFO) << "Got tablet " << *tablet_id << " for table " << table_name;
}

// Tests for the Tablet Server registering with the Master,
// and the master maintaining the tablet descriptor.
class RegistrationTest : public KuduTest {
 public:
  RegistrationTest()
      : schema_({ ColumnSchema("c1", UINT32) }, 1) {
  }

  void SetUp() override {
    // Make heartbeats faster to speed test runtime.
    FLAGS_heartbeat_interval_ms = 10;
    setup_time_ = WallTime_Now();

    KuduTest::SetUp();

    cluster_.reset(new InternalMiniCluster(env_, {}));
    ASSERT_OK(cluster_->Start());
  }

  void TearDown() override {
    cluster_->Shutdown();
  }

  void CheckTabletServersPage(string* contents = nullptr) {
    EasyCurl c;
    faststring buf;
    string addr = cluster_->mini_master()->bound_http_addr().ToString();
    ASSERT_OK(c.FetchURL(strings::Substitute("http://$0/tablet-servers", addr),
                                &buf));
    string buf_str = buf.ToString();

    // Should include the TS UUID
    string expected_uuid =
      cluster_->mini_tablet_server(0)->server()->instance_pb().permanent_uuid();
    ASSERT_STR_CONTAINS(buf_str, expected_uuid);

    // Should check that the TS software version is included on the page.
    // tserver version should be the same as returned by GetVersionInfo()
    ASSERT_STR_CONTAINS(buf_str, VersionInfo::GetVersionInfo());
    if (contents != nullptr) {
      *contents = std::move(buf_str);
    }
  }

  Status WaitForReplicaCount(const string& tablet_id,
                             int expected_count,
                             TabletLocationsPB* locations = nullptr) {
    while (true) {
      master::CatalogManager* catalog =
          cluster_->mini_master()->master()->catalog_manager();
      Status s;
      TabletLocationsPB loc;
      CatalogManager::TSInfosDict infos_dict;
      do {
        master::CatalogManager::ScopedLeaderSharedLock l(catalog);
        const Status& ls = l.first_failed_status();
        if (ls.IsServiceUnavailable() || ls.IsIllegalState()) {
          // ServiceUnavailable means catalog manager is not yet ready
          // to serve requests; IllegalState means the catalog is not the
          // leader yet. That's an indication of a transient state where it's
          // it's necessary to try again later.
          break;  // exiting out of the 'do {...} while (false)' scope
        }
        RETURN_NOT_OK(ls);
        s = catalog->GetTabletLocations(tablet_id,
                                        master::VOTER_REPLICA,
                                        &loc,
                                        &infos_dict,
                                        /*user=*/none);
      } while (false);
      if (s.ok() && loc.interned_replicas_size() == expected_count) {
        if (locations) {
          *locations = std::move(loc);
        }
        return Status::OK();
      }
      SleepFor(MonoDelta::FromMilliseconds(10));
    }
  }

 protected:
  gscoped_ptr<InternalMiniCluster> cluster_;
  Schema schema_;
  int64_t setup_time_;
};

TEST_F(RegistrationTest, TestTSRegisters) {
  // Wait for the TS to register.
  vector<shared_ptr<TSDescriptor> > descs;
  ASSERT_OK(cluster_->WaitForTabletServerCount(
      1, InternalMiniCluster::MatchMode::MATCH_TSERVERS, &descs));
  ASSERT_EQ(1, descs.size());

  // Verify that the registration is sane.
  ServerRegistrationPB reg;
  descs[0]->GetRegistration(&reg);
  {
    SCOPED_TRACE(SecureShortDebugString(reg));
    ASSERT_EQ(string::npos, SecureShortDebugString(reg).find("0.0.0.0"))
        << "Should not include wildcards in registration";
  }

  NO_FATALS(CheckTabletServersPage());

  // Restart the master, so it loses the descriptor, and ensure that the
  // heartbeater thread handles re-registering.
  cluster_->mini_master()->Shutdown();
  ASSERT_OK(cluster_->mini_master()->Restart());

  int64_t seqno = descs.back()->latest_seqno();
  descs.clear();
  ASSERT_OK(cluster_->WaitForTabletServerCount(
      1, InternalMiniCluster::MatchMode::MATCH_TSERVERS, &descs));
  ASSERT_EQ(descs.back()->latest_seqno(), seqno);

  // Restart the tserver, so it will register, and ensure that it gets the
  // newer sequence number.
  cluster_->mini_tablet_server(0)->Shutdown();
  ASSERT_OK(cluster_->mini_tablet_server(0)->Restart());

  seqno = descs.back()->latest_seqno();
  descs.clear();
  ASSERT_OK(cluster_->WaitForTabletServerCount(
      1, InternalMiniCluster::MatchMode::MATCH_TSERVERS, &descs));
  ASSERT_GT(descs.back()->latest_seqno(), seqno);
}

TEST_F(RegistrationTest, TestMasterSoftwareVersion) {
  // Verify that the master's software version exists.
  ServerRegistrationPB reg;
  cluster_->mini_master()->master()->GetMasterRegistration(&reg);
  {
    SCOPED_TRACE(SecureShortDebugString(reg));
    ASSERT_TRUE(reg.has_software_version());
    ASSERT_STR_CONTAINS(reg.software_version(),
                        VersionInfo::GetVersionInfo());
    ASSERT_LE(setup_time_, reg.start_time());
    ASSERT_LE(reg.start_time(), WallTime_Now());
  }
}

TEST_F(RegistrationTest, TestServerStartTime) {
  ServerRegistrationPB reg;
  cluster_->mini_master()->master()->GetMasterRegistration(&reg);
  ASSERT_LE(setup_time_, reg.start_time());
  ASSERT_LE(reg.start_time(), WallTime_Now());

  for (int i = 0; i < cluster_->num_tablet_servers(); ++i) {
    auto start_time = cluster_->mini_tablet_server(i)->server()->start_time();
    ASSERT_LE(setup_time_, start_time);
    ASSERT_LE(start_time, WallTime_Now());
  }
}

// Test starting multiple tablet servers and ensuring they both register with the master.
TEST_F(RegistrationTest, TestMultipleTS) {
  ASSERT_OK(cluster_->AddTabletServer());
  ASSERT_OK(cluster_->WaitForTabletServerCount(2));
}

// TODO: this doesn't belong under "RegistrationTest" - rename this file
// to something more appropriate - doesn't seem worth having separate
// whole test suites for registration, tablet reports, etc.
TEST_F(RegistrationTest, TestTabletReports) {
  auto GetCatalogMetric = [&](CounterPrototype& prototype) {
    auto metrics = cluster_->mini_master()->master()->catalog_manager()->
        sys_catalog()->tablet_replica()->tablet()->GetMetricEntity();
    return prototype.Instantiate(metrics)->value();
  };
  const int startup_rows_inserted = GetCatalogMetric(METRIC_rows_inserted);

  // Add a table, make sure it reports itself.
  string tablet_id_1;
  NO_FATALS(CreateTableForTesting(
      cluster_->mini_master(), "tablet-reports-1", schema_, &tablet_id_1));
  TabletLocationsPB locs_1;
  ASSERT_OK(WaitForReplicaCount(tablet_id_1, 1, &locs_1));
  ASSERT_EQ(1, locs_1.interned_replicas_size());

  // Check that we inserted the right number of rows for the new single-tablet table
  // (one for the table, one for the tablet).
  const int post_create_rows_inserted = GetCatalogMetric(METRIC_rows_inserted);
  EXPECT_EQ(2, post_create_rows_inserted - startup_rows_inserted)
      << "Should have inserted one row each for the table and tablet";

  // Add another table, make sure it is reported via incremental.
  string tablet_id_2;
  NO_FATALS(CreateTableForTesting(
      cluster_->mini_master(), "tablet-reports-2", schema_, &tablet_id_2));
  ASSERT_OK(WaitForReplicaCount(tablet_id_2, 1));

  // Shut down the whole system, bring it back up, and make sure the tablets
  // are reported.
  cluster_->Shutdown();
  ASSERT_OK(cluster_->Start());

  // After restart, check that the tablet reports produced the expected number of
  // writes to the catalog table:
  // - No inserts, because there are no new tablets.
  // - Two updates, since both replicas increase their term on restart.
  //
  // It can take some time for the TS to re-heartbeat. To avoid flakiness, here
  // it's easier to wait for the target non-zero metric value instead of
  // hard-coding an extra delay.
  AssertEventually([&]() {
    ASSERT_EQ(0, GetCatalogMetric(METRIC_rows_inserted));
    ASSERT_EQ(2, GetCatalogMetric(METRIC_rows_updated));
    });

  // If we restart just the master, it should not write any data to the catalog, since the
  // tablets themselves are not changing term, etc.
  cluster_->mini_master()->Shutdown();
  ASSERT_OK(cluster_->mini_master()->Restart());

  // Sleep for a second to make sure the TS has plenty of time to re-heartbeat.
  // The metrics are updated after processing TS heartbeats, and we want to
  // capture updated metric readings.
  SleepFor(MonoDelta::FromSeconds(1));

  EXPECT_EQ(0, GetCatalogMetric(METRIC_rows_inserted));
  EXPECT_EQ(0, GetCatalogMetric(METRIC_rows_updated));
}

// Check that after the tablet server registers, it gets a signed cert
// from the master.
TEST_F(RegistrationTest, TestTSGetsSignedX509Certificate) {
  MiniTabletServer* ts = cluster_->mini_tablet_server(0);
  ASSERT_EVENTUALLY([&](){
      ASSERT_TRUE(ts->server()->tls_context().has_signed_cert());
    });
}

// Check that after the tablet server registers, it gets the list of valid
// public token signing keys.
TEST_F(RegistrationTest, TestTSGetsTskList) {
  MiniTabletServer* ts = cluster_->mini_tablet_server(0);
  ASSERT_EVENTUALLY([&](){
      ASSERT_FALSE(ts->server()->token_verifier().ExportKeys().empty());
    });
}

// Test that, if the tserver has HTTPS enabled, the master links to it
// via https:// URLs and not http://.
TEST_F(RegistrationTest, TestExposeHttpsURLs) {
  MiniTabletServer* ts = cluster_->mini_tablet_server(0);
  string password;
  WebserverOptions* opts = &ts->options()->webserver_opts;
  ASSERT_OK(security::CreateTestSSLCertWithEncryptedKey(GetTestDataDirectory(),
                                                        &opts->certificate_file,
                                                        &opts->private_key_file,
                                                        &password));
  opts->private_key_password_cmd = strings::Substitute("echo $0", password);
  ts->Shutdown();
  ASSERT_OK(ts->Start());

  // The URL displayed on the page uses a hostname. Rather than
  // dealing with figuring out what the hostname should be, just
  // use a more permissive regex which doesn't check the host.
  string expected_url_regex = strings::Substitute(
      "https://[a-zA-Z0-9.-]+:$0/", opts->port);

  // Need "eventually" here because the tserver may take a few seconds
  // to re-register while starting up.
  ASSERT_EVENTUALLY([&](){
      string contents;
      NO_FATALS(CheckTabletServersPage(&contents));
      ASSERT_STR_MATCHES(contents, expected_url_regex);
    });
}

} // namespace kudu

ext: cc

max_stars_repo_name: xiaokai-wang/kudu

max_issues_repo_name: xiaokai-wang/kudu

max_forks_repo_name: xiaokai-wang/kudu