text:
// Copyright (c) 2014-2016 Daniel Kraft
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "base58.h"
#include "chainparams.h"
#include "init.h"
#include "names/common.h"
#include "names/main.h"
#include "primitives/transaction.h"
#include "rpc/server.h"
#include "script/names.h"
#include "txmempool.h"
#include "utilstrencodings.h"
#include "validation.h"
#ifdef ENABLE_WALLET
# include "wallet/rpcwallet.h"
# include "wallet/wallet.h"
#endif

#include <boost/xpressive/xpressive_dynamic.hpp>

#include <memory>
#include <sstream>

#include <univalue.h>

/**
 * Utility routine to construct a "name info" object to return.  This is used
 * for name_show and also name_list.
 * @param name The name.
 * @param value The name's value.
 * @param outp The last update's outpoint.
 * @param addr The name's address script.
 * @param height The name's last update height.
 * @return A JSON object to return.
 */
UniValue
getNameInfo (const valtype& name, const valtype& value, const COutPoint& outp,
             const CScript& addr, int height)
{
  UniValue obj(UniValue::VOBJ);
  obj.pushKV ("name", ValtypeToString (name));
  obj.pushKV ("value", ValtypeToString (value));
  obj.pushKV ("txid", outp.hash.GetHex ());
  obj.pushKV ("vout", static_cast<int> (outp.n));

  /* Try to extract the address.  May fail if we can't parse the script
     as a "standard" script.  */
  CTxDestination dest;
  CBitcoinAddress addrParsed;
  std::string addrStr;
  if (ExtractDestination (addr, dest) && addrParsed.Set (dest))
    addrStr = addrParsed.ToString ();
  else
    addrStr = "<nonstandard>";
  obj.pushKV ("address", addrStr);

  /* Calculate expiration data.  */
  const int curHeight = chainActive.Height ();
  const Consensus::Params& params = Params ().GetConsensus ();
  const int expireDepth = params.rules->NameExpirationDepth (curHeight);
  const int expireHeight = height + expireDepth;
  const int expiresIn = expireHeight - curHeight;
  const bool expired = (expiresIn <= 0);
  obj.pushKV ("height", height);
  obj.pushKV ("expires_in", expiresIn);
  obj.pushKV ("expired", expired);

  return obj;
}

/**
 * Return name info object for a CNameData object.
 * @param name The name.
 * @param data The name's data.
 * @return A JSON object to return.
 */
UniValue
getNameInfo (const valtype& name, const CNameData& data)
{
  return getNameInfo (name, data.getValue (), data.getUpdateOutpoint (),
                      data.getAddress (), data.getHeight ());
}

/**
 * Return the help string description to use for name info objects.
 * @param indent Indentation at the line starts.
 * @param trailing Trailing string (e. g., comma for an array of these objects).
 * @return The description string.
 */
std::string
getNameInfoHelp (const std::string& indent, const std::string& trailing)
{
  std::ostringstream res;

  res << indent << "{" << std::endl;
  res << indent << "  \"name\": xxxxx,           "
      << "(string) the requested name" << std::endl;
  res << indent << "  \"value\": xxxxx,          "
      << "(string) the name's current value" << std::endl;
  res << indent << "  \"txid\": xxxxx,           "
      << "(string) the name's last update tx" << std::endl;
  res << indent << "  \"address\": xxxxx,        "
      << "(string) the address holding the name" << std::endl;
  res << indent << "  \"height\": xxxxx,         "
      << "(numeric) the name's last update height" << std::endl;
  res << indent << "  \"expires_in\": xxxxx,     "
      << "(numeric) expire counter for the name" << std::endl;
  res << indent << "  \"expired\": xxxxx,        "
      << "(boolean) whether the name is expired" << std::endl;
  res << indent << "}" << trailing << std::endl;

  return res.str ();
}

/**
 * Implement the rawtx name operation feature.  This routine interprets
 * the given JSON object describing the desired name operation and then
 * modifies the transaction accordingly.
 * @param tx The transaction to extend.
 * @param obj The name operation "description" as given to the call.
 */
void
AddRawTxNameOperation (CMutableTransaction& tx, const UniValue& obj)
{
  UniValue val = find_value (obj, "op");
  if (!val.isStr ())
    throw JSONRPCError (RPC_INVALID_PARAMETER, "missing op key");
  const std::string op = val.get_str ();

  if (op != "name_update")
    throw JSONRPCError (RPC_INVALID_PARAMETER,
                        "only name_update is implemented for the rawtx API");

  val = find_value (obj, "name");
  if (!val.isStr ())
    throw JSONRPCError (RPC_INVALID_PARAMETER, "missing name key");
  const valtype name = ValtypeFromString (val.get_str ());

  val = find_value (obj, "value");
  if (!val.isStr ())
    throw JSONRPCError (RPC_INVALID_PARAMETER, "missing value key");
  const valtype value = ValtypeFromString (val.get_str ());

  val = find_value (obj, "address");
  if (!val.isStr ())
    throw JSONRPCError (RPC_INVALID_PARAMETER, "missing address key");
  const CBitcoinAddress toAddress(val.get_str ());
  if (!toAddress.IsValid ())
    throw JSONRPCError (RPC_INVALID_ADDRESS_OR_KEY, "invalid address");
  const CScript addr = GetScriptForDestination (toAddress.Get ());

  tx.SetDenemecoin ();

  /* We do not add the name input.  This has to be done explicitly,
     but is easy from the name_show output.  That way, createrawtransaction
     doesn't depend on the chainstate at all.  */

  const CScript outScript = CNameScript::buildNameUpdate (addr, name, value);
  tx.vout.push_back (CTxOut (NAME_LOCKED_AMOUNT, outScript));
}

/* ************************************************************************** */

UniValue
name_show (const JSONRPCRequest& request)
{
  if (request.fHelp || request.params.size () != 1)
    throw std::runtime_error (
        "name_show \"name\"\n"
        "\nLook up the current data for the given name."
        "  Fails if the name doesn't exist.\n"
        "\nArguments:\n"
        "1. \"name\"          (string, required) the name to query for\n"
        "\nResult:\n"
        + getNameInfoHelp ("", "") +
        "\nExamples:\n"
        + HelpExampleCli ("name_show", "\"myname\"")
        + HelpExampleRpc ("name_show", "\"myname\"")
      );

  if (IsInitialBlockDownload ())
    throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD,
                       "Denemecoin is downloading blocks...");

  const std::string nameStr = request.params[0].get_str ();
  const valtype name = ValtypeFromString (nameStr);

  CNameData data;
  {
    LOCK (cs_main);
    if (!pcoinsTip->GetName (name, data))
      {
        std::ostringstream msg;
        msg << "name not found: '" << nameStr << "'";
        throw JSONRPCError (RPC_WALLET_ERROR, msg.str ());
      }
  }

  return getNameInfo (name, data);
}

/* ************************************************************************** */

UniValue
name_history (const JSONRPCRequest& request)
{
  if (request.fHelp || request.params.size () != 1)
    throw std::runtime_error (
        "name_history \"name\"\n"
        "\nLook up the current and all past data for the given name."
        "  -namehistory must be enabled.\n"
        "\nArguments:\n"
        "1. \"name\"          (string, required) the name to query for\n"
        "\nResult:\n"
        "[\n"
        + getNameInfoHelp ("  ", ",") +
        "  ...\n"
        "]\n"
        "\nExamples:\n"
        + HelpExampleCli ("name_history", "\"myname\"")
        + HelpExampleRpc ("name_history", "\"myname\"")
      );

  if (!fNameHistory)
    throw std::runtime_error ("-namehistory is not enabled");

  if (IsInitialBlockDownload ())
    throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD,
                       "Denemecoin is downloading blocks...");

  const std::string nameStr = request.params[0].get_str ();
  const valtype name = ValtypeFromString (nameStr);

  CNameData data;
  CNameHistory history;

  {
    LOCK (cs_main);

    if (!pcoinsTip->GetName (name, data))
      {
        std::ostringstream msg;
        msg << "name not found: '" << nameStr << "'";
        throw JSONRPCError (RPC_WALLET_ERROR, msg.str ());
      }

    if (!pcoinsTip->GetNameHistory (name, history))
      assert (history.empty ());
  }

  UniValue res(UniValue::VARR);
  BOOST_FOREACH (const CNameData& entry, history.getData ())
    res.push_back (getNameInfo (name, entry));
  res.push_back (getNameInfo (name, data));

  return res;
}

/* ************************************************************************** */

UniValue
name_scan (const JSONRPCRequest& request)
{
  if (request.fHelp || request.params.size () > 2)
    throw std::runtime_error (
        "name_scan (\"start\" (\"count\"))\n"
        "\nList names in the database.\n"
        "\nArguments:\n"
        "1. \"start\"       (string, optional) skip initially to this name\n"
        "2. \"count\"       (numeric, optional, default=500) stop after this many names\n"
        "\nResult:\n"
        "[\n"
        + getNameInfoHelp ("  ", ",") +
        "  ...\n"
        "]\n"
        "\nExamples:\n"
        + HelpExampleCli ("name_scan", "")
        + HelpExampleCli ("name_scan", "\"d/abc\"")
        + HelpExampleCli ("name_scan", "\"d/abc\" 10")
        + HelpExampleRpc ("name_scan", "\"d/abc\"")
      );

  if (IsInitialBlockDownload ())
    throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD,
                       "Denemecoin is downloading blocks...");

  valtype start;
  if (request.params.size () >= 1)
    start = ValtypeFromString (request.params[0].get_str ());

  int count = 500;
  if (request.params.size () >= 2)
    count = request.params[1].get_int ();

  UniValue res(UniValue::VARR);
  if (count <= 0)
    return res;

  LOCK (cs_main);

  valtype name;
  CNameData data;
  std::unique_ptr<CNameIterator> iter(pcoinsTip->IterateNames ());
  for (iter->seek (start); count > 0 && iter->next (name, data); --count)
    res.push_back (getNameInfo (name, data));

  return res;
}

/* ************************************************************************** */

UniValue
name_filter (const JSONRPCRequest& request)
{
  if (request.fHelp || request.params.size () > 5)
    throw std::runtime_error (
        "name_filter (\"regexp\" (\"maxage\" (\"from\" (\"nb\" (\"stat\")))))\n"
        "\nScan and list names matching a regular expression.\n"
        "\nArguments:\n"
        "1. \"regexp\"      (string, optional) filter names with this regexp\n"
        "2. \"maxage\"      (numeric, optional, default=36000) only consider names updated in the last \"maxage\" blocks; 0 means all names\n"
        "3. \"from\"        (numeric, optional, default=0) return from this position onward; index starts at 0\n"
        "4. \"nb\"          (numeric, optional, default=0) return only \"nb\" entries; 0 means all\n"
        "5. \"stat\"        (string, optional) if set to the string \"stat\", print statistics instead of returning the names\n"
        "\nResult:\n"
        "[\n"
        + getNameInfoHelp ("  ", ",") +
        "  ...\n"
        "]\n"
        "\nExamples:\n"
        + HelpExampleCli ("name_filter", "\"\" 5")
        + HelpExampleCli ("name_filter", "\"^id/\"")
        + HelpExampleCli ("name_filter", "\"^id/\" 36000 0 0 \"stat\"")
        + HelpExampleRpc ("name_scan", "\"^d/\"")
      );

  if (IsInitialBlockDownload ())
    throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD,
                       "Denemecoin is downloading blocks...");

  /* ********************** */
  /* Interpret parameters.  */

  bool haveRegexp(false);
  boost::xpressive::sregex regexp;

  int maxage(36000), from(0), nb(0);
  bool stats(false);

  if (request.params.size () >= 1)
    {
      haveRegexp = true;
      regexp = boost::xpressive::sregex::compile (request.params[0].get_str ());
    }

  if (request.params.size () >= 2)
    maxage = request.params[1].get_int ();
  if (maxage < 0)
    throw JSONRPCError (RPC_INVALID_PARAMETER,
                        "'maxage' should be non-negative");
  if (request.params.size () >= 3)
    from = request.params[2].get_int ();
  if (from < 0)
    throw JSONRPCError (RPC_INVALID_PARAMETER, "'from' should be non-negative");

  if (request.params.size () >= 4)
    nb = request.params[3].get_int ();
  if (nb < 0)
    throw JSONRPCError (RPC_INVALID_PARAMETER, "'nb' should be non-negative");

  if (request.params.size () >= 5)
    {
      if (request.params[4].get_str () != "stat")
        throw JSONRPCError (RPC_INVALID_PARAMETER,
                            "fifth argument must be the literal string 'stat'");
      stats = true;
    }

  /* ******************************************* */
  /* Iterate over names to build up the result.  */

  UniValue names(UniValue::VARR);
  unsigned count(0);

  LOCK (cs_main);

  valtype name;
  CNameData data;
  std::unique_ptr<CNameIterator> iter(pcoinsTip->IterateNames ());
  while (iter->next (name, data))
    {
      const int age = chainActive.Height () - data.getHeight ();
      assert (age >= 0);
      if (maxage != 0 && age >= maxage)
        continue;

      if (haveRegexp)
        {
          const std::string nameStr = ValtypeToString (name);
          boost::xpressive::smatch matches;
          if (!boost::xpressive::regex_search (nameStr, matches, regexp))
            continue;
        }

      if (from > 0)
        {
          --from;
          continue;
        }
      assert (from == 0);

      if (stats)
        ++count;
      else
        names.push_back (getNameInfo (name, data));

      if (nb > 0)
        {
          --nb;
          if (nb == 0)
            break;
        }
    }

  /* ********************************************************** */
  /* Return the correct result (take stats mode into account).  */

  if (stats)
    {
      UniValue res(UniValue::VOBJ);
      res.pushKV ("blocks", chainActive.Height ());
      res.pushKV ("count", static_cast<int> (count));

      return res;
    }

  return names;
}

/* ************************************************************************** */

UniValue
name_pending (const JSONRPCRequest& request)
{
  if (request.fHelp || request.params.size () > 1)
    throw std::runtime_error (
        "name_pending (\"name\")\n"
        "\nList unconfirmed name operations in the mempool.\n"
        "\nIf a name is given, only check for operations on this name.\n"
        "\nArguments:\n"
        "1. \"name\"        (string, optional) only look for this name\n"
        "\nResult:\n"
        "[\n"
        "  {\n"
        "    \"op\": xxxx       (string) the operation being performed\n"
        "    \"name\": xxxx     (string) the name operated on\n"
        "    \"value\": xxxx    (string) the name's new value\n"
        "    \"txid\": xxxx     (string) the txid corresponding to the operation\n"
        "    \"ismine\": xxxx   (boolean) whether the name is owned by the wallet\n"
        "  },\n"
        "  ...\n"
        "]\n"
        + HelpExampleCli ("name_pending", "")
        + HelpExampleCli ("name_pending", "\"d/domob\"")
        + HelpExampleRpc ("name_pending", "")
      );

#ifdef ENABLE_WALLET
    CWallet* pwallet = GetWalletForJSONRPCRequest (request);
    LOCK2 (pwallet ? &pwallet->cs_wallet : nullptr, mempool.cs);
#else
    LOCK (mempool.cs);
#endif

  std::vector<uint256> txHashes;
  if (request.params.size () == 0)
    mempool.queryHashes (txHashes);
  else
    {
      const std::string name = request.params[0].get_str ();
      const valtype vchName = ValtypeFromString (name);
      const uint256 txid = mempool.getTxForName (vchName);
      if (!txid.IsNull ())
        txHashes.push_back (txid);
    }

  UniValue arr(UniValue::VARR);
  for (std::vector<uint256>::const_iterator i = txHashes.begin ();
       i != txHashes.end (); ++i)
    {
      std::shared_ptr<const CTransaction> tx = mempool.get (*i);
      if (!tx || !tx->IsDenemecoin ())
        continue;

      for (const auto& txOut : tx->vout)
        {
          const CNameScript op(txOut.scriptPubKey);
          if (!op.isNameOp () || !op.isAnyUpdate ())
            continue;

          const valtype vchName = op.getOpName ();
          const valtype vchValue = op.getOpValue ();

          const std::string name = ValtypeToString (vchName);
          const std::string value = ValtypeToString (vchValue);

          std::string strOp;
          switch (op.getNameOp ())
            {
            case OP_NAME_FIRSTUPDATE:
              strOp = "name_firstupdate";
              break;
            case OP_NAME_UPDATE:
              strOp = "name_update";
              break;
            default:
              assert (false);
            }

          UniValue obj(UniValue::VOBJ);
          obj.pushKV ("op", strOp);
          obj.pushKV ("name", name);
          obj.pushKV ("value", value);
          obj.pushKV ("txid", tx->GetHash ().GetHex ());

#ifdef ENABLE_WALLET
          isminetype mine = ISMINE_NO;
          if (pwallet)
            mine = IsMine (*pwallet, op.getAddress ());
          const bool isMine = (mine & ISMINE_SPENDABLE);
          obj.pushKV ("ismine", isMine);
#endif

          arr.push_back (obj);
        }
    }

  return arr;
}

/* ************************************************************************** */

UniValue
name_checkdb (const JSONRPCRequest& request)
{
  if (request.fHelp || request.params.size () != 0)
    throw std::runtime_error (
        "name_checkdb\n"
        "\nValidate the name DB's consistency.\n"
        "\nRoughly between blocks 139,000 and 180,000, this call is expected\n"
        "to fail due to the historic 'name stealing' bug.\n"
        "\nResult:\n"
        "xxxxx                        (boolean) whether the state is valid\n"
        "\nExamples:\n"
        + HelpExampleCli ("name_checkdb", "")
        + HelpExampleRpc ("name_checkdb", "")
      );

  LOCK (cs_main);
  pcoinsTip->Flush ();
  return pcoinsTip->ValidateNameDB ();
}

/* ************************************************************************** */

static const CRPCCommand commands[] =
{ //  category              name                      actor (function)         okSafeMode
  //  --------------------- ------------------------  -----------------------  ----------
    { "denemecoin",           "name_show",              &name_show,              false,  {"name"} },
    { "denemecoin",           "name_history",           &name_history,           false,  {"name"} },
    { "denemecoin",           "name_scan",              &name_scan,              false,  {"start","count"} },
    { "denemecoin",           "name_filter",            &name_filter,            false,  {"regexp","maxage","from","nb","stat"} },
    { "denemecoin",           "name_pending",           &name_pending,           true,   {"name"} },
    { "denemecoin",           "name_checkdb",           &name_checkdb,           false,  {} },
};

void RegisterNameRPCCommands(CRPCTable &t)
{
    for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++)
        t.appendCommand(commands[vcidx].name, &commands[vcidx]);
}

max_stars_repo_name: Calipson/denemecoin-core

max_issues_repo_name: Calipson/denemecoin-core

max_forks_repo_name: Calipson/denemecoin-core


text:
////////////////////////////////////////////////////////////////////////
//
// This file is part of gmic-8bf, a filter plug-in module that
// interfaces with G'MIC-Qt.
//
// Copyright (c) 2020, 2021 Nicholas Hayes
//
// This file is licensed under the MIT License.
// See LICENSE.txt for complete licensing and attribution information.
//
////////////////////////////////////////////////////////////////////////

#include "stdafx.h"
#include "GmicPlugin.h"
#include "FileUtilWin.h"
#include <objbase.h>
#include <ShlObj.h>
#include <wil/resource.h>
#include <wil/result.h>
#include <wil/win32_helpers.h>
#include <boost/filesystem.hpp>

namespace
{
    boost::filesystem::path GetPluginInstallDirectory()
    {
        boost::filesystem::path path;

        try
        {
            wil::unique_cotaskmem_string dllPath = wil::GetModuleFileNameW(wil::GetModuleInstanceHandle());

            boost::filesystem::path temp(dllPath.get());

            path = temp.parent_path();
        }
        catch (const wil::ResultException& e)
        {
            if (e.GetErrorCode() == E_OUTOFMEMORY)
            {
                throw std::bad_alloc();
            }
            else
            {
                throw std::runtime_error(e.what());
            }
        }

        return path;
    }

    DWORD ReadCore(HANDLE hFile, BYTE* buffer, size_t count)
    {
        try
        {
            DWORD numBytesToRead = 0x80000000UL;

            if (count < numBytesToRead)
            {
                numBytesToRead = static_cast<DWORD>(count);
            }

            DWORD bytesRead = 0;

            THROW_IF_WIN32_BOOL_FALSE(ReadFile(hFile, buffer, numBytesToRead, &bytesRead, nullptr));

            return bytesRead;
        }
        catch (const wil::ResultException& e)
        {
            if (e.GetErrorCode() == E_OUTOFMEMORY)
            {
                throw std::bad_alloc();
            }
            else
            {
                throw std::runtime_error(e.what());
            }
        }
    }

    void ReadCoreBlocking(HANDLE hFile, BYTE* buffer, size_t count)
    {
        size_t offset = 0;
        size_t bytesRemaining = count;

        while (bytesRemaining > 0)
        {
            DWORD bytesRead = ReadCore(hFile, buffer + offset, bytesRemaining);

            if (bytesRead == 0)
            {
                throw std::runtime_error("Attempted to read beyond the end of the file.");
            }

            offset += bytesRead;
            bytesRemaining -= bytesRead;
        }
    }

    void WriteCore(HANDLE hFile, const BYTE* buffer, size_t count)
    {
        try
        {
            size_t bytesRemaining = count;

            while (bytesRemaining > 0)
            {
                DWORD numBytesToWrite = 0x80000000UL;

                if (bytesRemaining < numBytesToWrite)
                {
                    numBytesToWrite = static_cast<DWORD>(bytesRemaining);
                }

                DWORD bytesWritten = 0;

                THROW_IF_WIN32_BOOL_FALSE(WriteFile(hFile, buffer, numBytesToWrite, &bytesWritten, nullptr));

                buffer += bytesWritten;
                bytesRemaining -= bytesWritten;
            }
        }
        catch (const wil::ResultException& e)
        {
            if (e.GetErrorCode() == E_OUTOFMEMORY)
            {
                throw std::bad_alloc();
            }
            else
            {
                throw std::runtime_error(e.what());
            }
        }
    }
}

struct FileBuffer
{
    std::vector<BYTE> buffer;
    size_t readOffset;
    size_t readLength;
    size_t writeOffset;

    FileBuffer()
        : buffer(4096), readOffset(0), readLength(0), writeOffset(0)
    {
    }
};

class FileHandleWin : public FileHandle
{
public:
    FileHandleWin(const boost::filesystem::path& path, FileOpenMode mode)
        : buffer(std::make_unique<FileBuffer>()), uncaughtExceptionCount(std::uncaught_exceptions()), hFile()
    {
        DWORD dwDesiredAccess;
        DWORD dwShareMode;
        DWORD dwCreationDisposition;
        DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL;

        switch (mode)
        {
        case FileOpenMode::Read:
            dwDesiredAccess = GENERIC_READ;
            dwShareMode = FILE_SHARE_READ;
            dwCreationDisposition = OPEN_EXISTING;
            break;
        case FileOpenMode::Write:
            dwDesiredAccess = GENERIC_WRITE;
            dwShareMode = 0;
            dwCreationDisposition = CREATE_ALWAYS;
            break;
        default:
            throw std::invalid_argument("Unknown FileOpenMode");
        }

        hFile.reset(CreateFileW(path.c_str(), dwDesiredAccess, dwShareMode, nullptr, dwCreationDisposition, dwFlagsAndAttributes, nullptr));

        if (!hFile)
        {
            THROW_LAST_ERROR();
        }
    }

    ~FileHandleWin() override
    {
        // Flush the write buffer to disk, unless the stack is being unwound due to an uncaught exception.
        if (uncaughtExceptionCount == std::uncaught_exceptions())
        {
            if (buffer->writeOffset > 0)
            {
                WriteCore(hFile.get(), buffer->buffer.data(), buffer->writeOffset);
                buffer->writeOffset = 0;
            }
        }
    }

    FileHandleWin(const FileHandleWin&) = delete;
    FileHandleWin& operator=(const FileHandleWin&) = delete;

    FileHandleWin(FileHandleWin&&) = delete;
    FileHandleWin& operator=(FileHandleWin&&) = delete;

    HANDLE get_native_handle() const noexcept
    {
        return hFile.get();
    }

    FileBuffer* get_buffer() const noexcept
    {
        return buffer.get();
    }

private:
    std::unique_ptr<FileBuffer> buffer;
    const int uncaughtExceptionCount;
    wil::unique_hfile hFile;
};

boost::filesystem::path GetGmicQtPathNative()
{
    boost::filesystem::path path = GetPluginInstallDirectory();

    path /= "gmic";
    path /= "gmic_8bf_qt.exe";

    return path;
}

boost::filesystem::path GetPluginSettingsDirectoryNative()
{
    boost::filesystem::path path;

    try
    {
        wil::unique_cotaskmem_string appDataPath;

        THROW_IF_FAILED(SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, nullptr, &appDataPath));

        path = appDataPath.get();
        path /= L"Gmic8bfPlugin";
        path /= L"settings";
    }
    catch (const wil::ResultException& e)
    {
        if (e.GetErrorCode() == E_OUTOFMEMORY)
        {
            throw std::bad_alloc();
        }
        else
        {
            throw std::runtime_error(e.what());
        }
    }

    return path;
}

boost::filesystem::path GetSessionDirectoriesRootNative()
{
    boost::filesystem::path path;

    try
    {
        wil::unique_cotaskmem_string programDataPath;

        THROW_IF_FAILED(SHGetKnownFolderPath(FOLDERID_ProgramData, 0, nullptr, &programDataPath));

        path = programDataPath.get();
        path /= L"Gmic8bfPlugin";
        path /= L"SessionData";
    }
    catch (const wil::ResultException& e)
    {
        if (e.GetErrorCode() == E_OUTOFMEMORY)
        {
            throw std::bad_alloc();
        }
        else
        {
            throw std::runtime_error(e.what());
        }
    }

    return path;
}

std::unique_ptr<FileHandle> OpenFileNative(const boost::filesystem::path& path, FileOpenMode mode)
{
    try
    {
        return std::make_unique<FileHandleWin>(path, mode);
    }
    catch (const wil::ResultException& e)
    {
        if (e.GetErrorCode() == E_OUTOFMEMORY)
        {
            throw std::bad_alloc();
        }
        else
        {
            throw std::runtime_error(e.what());
        }
    }
}

void ReadFileNative(FileHandle* fileHandle, void* data, size_t dataSize)
{
    if (!fileHandle)
    {
        throw std::runtime_error("Null file handle");
    }

    FileHandleWin* fileHandleWin = static_cast<FileHandleWin*>(fileHandle);

    HANDLE hFile = fileHandleWin->get_native_handle();
    FileBuffer* buffer = fileHandleWin->get_buffer();

    if (buffer->writeOffset > 0)
    {
        // Flush the buffered data.
        WriteCore(hFile, buffer->buffer.data(), buffer->writeOffset);
        buffer->writeOffset = 0;
    }

    BYTE* output = static_cast<BYTE*>(data);
    size_t outputOffset = 0;
    size_t outputBytesRemaining = dataSize;

    if (buffer->readLength == 0)
    {
        if (dataSize >= buffer->buffer.size())
        {
            ReadCoreBlocking(hFile, output, dataSize);
            // Invalidate the existing buffer.
            buffer->readOffset = 0;
            buffer->readLength = 0;
            return;
        }
        else
        {
            DWORD bytesRead = ReadCore(hFile, buffer->buffer.data(), buffer->buffer.size());

            if (bytesRead == 0)
            {
                throw std::runtime_error("Attempted to read beyond the end of the file.");
            }

            buffer->readOffset = 0;
            buffer->readLength = bytesRead;
        }
    }

    size_t bytesToRead = buffer->readLength - buffer->readOffset;

    if (bytesToRead > outputBytesRemaining)
    {
        bytesToRead = outputBytesRemaining;
    }

    memcpy(output + outputOffset, buffer->buffer.data() + buffer->readOffset, bytesToRead);
    buffer->readOffset += bytesToRead;
    outputOffset += bytesToRead;
    outputBytesRemaining -= bytesToRead;

    if (outputBytesRemaining > 0)
    {
        // If we have read all of the data in the buffer, read the remaining bytes directly from the file.
        ReadCoreBlocking(hFile, output + outputOffset, outputBytesRemaining);
        // Invalidate the existing buffer.
        buffer->readOffset = 0;
        buffer->readLength = 0;
    }
}

void SetFileLengthNative(FileHandle* fileHandle, int64 length)
{
    try
    {
        FILE_END_OF_FILE_INFO endOfFileInfo;
        endOfFileInfo.EndOfFile.QuadPart = length;

        THROW_IF_WIN32_BOOL_FALSE(SetFileInformationByHandle(
            static_cast<const FileHandleWin*>(fileHandle)->get_native_handle(),
            FileEndOfFileInfo,
            &endOfFileInfo,
            sizeof(endOfFileInfo)));
    }
    catch (const wil::ResultException& e)
    {
        if (e.GetErrorCode() == E_OUTOFMEMORY)
        {
            throw std::bad_alloc();
        }
        else
        {
            throw std::runtime_error(e.what());
        }
    }
}

void SetFilePositionNative(FileHandle* fileHandle, int16 posMode, int64 posOffset)
{
    if (!fileHandle)
    {
        throw std::runtime_error("Null file handle");
    }

    FileHandleWin* fileHandleWin = static_cast<FileHandleWin*>(fileHandle);

    HANDLE hFile = fileHandleWin->get_native_handle();
    FileBuffer* buffer = fileHandleWin->get_buffer();

    // Invalidate the existing read buffer and flush the write buffer.
    buffer->readOffset = 0;
    buffer->readLength = 0;

    if (buffer->writeOffset > 0)
    {
        WriteCore(hFile, buffer->buffer.data(), buffer->writeOffset);
        buffer->writeOffset = 0;
    }

    try
    {
        LARGE_INTEGER distanceToMove = {};

        distanceToMove.QuadPart = posOffset;

        THROW_IF_WIN32_BOOL_FALSE(SetFilePointerEx(
            hFile,
            distanceToMove,
            nullptr,
            static_cast<DWORD>(posMode)));
    }
    catch (const wil::ResultException& e)
    {
        if (e.GetErrorCode() == E_OUTOFMEMORY)
        {
            throw std::bad_alloc();
        }
        else
        {
            throw std::runtime_error(e.what());
        }
    }
}

void WriteFileNative(FileHandle* fileHandle, const void* data, size_t dataSize)
{
    if (!fileHandle)
    {
        throw std::runtime_error("Null file handle");
    }

    const BYTE* input = static_cast<const BYTE*>(data);
    size_t inputOffset = 0;
    size_t inputLength = dataSize;

    FileHandleWin* fileHandleWin = static_cast<FileHandleWin*>(fileHandle);

    HANDLE hFile = fileHandleWin->get_native_handle();
    FileBuffer* buffer = fileHandleWin->get_buffer();

    if (buffer->readLength > 0)
    {
        // Invalidate the read buffer.
        buffer->readOffset = buffer->readLength = 0;
    }

    if (buffer->writeOffset > 0)
    {
        size_t remainingSpaceInBuffer = buffer->buffer.size() - buffer->writeOffset;

        if (remainingSpaceInBuffer > 0)
        {
            size_t bytesToCopy = remainingSpaceInBuffer;

            if (inputLength < bytesToCopy)
            {
                bytesToCopy = inputLength;
            }

            memcpy(buffer->buffer.data() + buffer->writeOffset, input, bytesToCopy);
            buffer->writeOffset += bytesToCopy;

            if (bytesToCopy == inputLength)
            {
                return;
            }

            inputOffset += bytesToCopy;
            inputLength -= bytesToCopy;
        }

        WriteCore(hFile, buffer->buffer.data(), buffer->buffer.size());
        buffer->writeOffset = 0;
    }

    if (inputLength >= buffer->buffer.size())
    {
        WriteCore(hFile, input + inputOffset, inputLength);
    }
    else
    {
        memcpy(buffer->buffer.data(), input + inputOffset, inputLength);
        buffer->writeOffset = inputLength;
    }
}

ext: cpp


text:
/*
 * Copyright 2013 Rob Groenendijk
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef JNIREFERENCE_H_
#define JNIREFERENCE_H_

#include <boost/ref_ptr.h>
#include <jni.h>
#include "org_horizonremote_jni_JNIReference.h"

typedef jlong jpointer;

namespace horizonremote {

class JNIReference {
public:

	JNIReference(Referenceable* object);

	Referenceable* get();

	/*
	 * fast implementation to unwrap from a JNI jlong to a given derived class of Referenceable
	 * */
	template<class T>
	static inline T cast(jpointer ptr) {
		return static_cast<T>(static_cast<JNIReference*>((void*) ptr)->get());
	}

	static void dispose(jpointer ptr);

private:

	virtual ~JNIReference();
	ref_ptr<Referenceable> m_refobj;
};

}

typedef horizonremote::JNIReference JNIReference;

#endif /* JNIREFERENCE_H_ */

ext: hpp

max_stars_repo_name: ilya-shknaj/HorizonRemote

max_issues_repo_name: ilya-shknaj/HorizonRemote

max_forks_repo_name: ilya-shknaj/HorizonRemote


text:
#pragma once
//=====================================================================//
/*!	@file
	@brief	collada library_lights のパーサー（ヘッダー）
	@author	平松邦仁 (hira@rvf-rc45.net)
*/
//=====================================================================//
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/xml_parser.hpp>
#include "utils/string_utils.hpp"

#include "utils/verbose.hpp"

namespace collada {

	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief	collada scene のパーサー・クラス
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	class dae_lights {

		int		error_;

	public:

		dae_lights() : error_(0) { }

		//-----------------------------------------------------------------//
		/*!
			@brief	パース
			@return エラー数（「０」なら成功）
		*/
		//-----------------------------------------------------------------//
		int parse(utils::verbose& v, const boost::property_tree::ptree::value_type& element);

	};

}


text:
#include "Application.h"
#include "OpLog.h"

#include <guard/check.h>
#include <pimg/ImageData.h>
#include <pimg/RegularRectCut.h>
#include <pimg/Cropping.h>
#include <gimg_export.h>
#include <gimg_typedef.h>
#include <sx/ResFileHelper.h>
#include <facade/ResPool.h>

#include <boost/filesystem.hpp>

#include <string>
#include <regex>

namespace
{

static const char* TIME_FILEPATH = "auto_crop_grids_time.json";
static const char* LOG_FILEPATH  = "auto_crop_grids_log.json";

template<typename ... Args>
std::string string_format(const std::string& format, Args ... args)
{
	size_t size = std::snprintf(nullptr, 0, format.c_str(), args ...) + 1; // Extra space for '\0'
	std::unique_ptr<char[]> buf(new char[size]);
	std::snprintf(buf.get(), size, format.c_str(), args ...);
	return std::string(buf.get(), buf.get() + size - 1); // We don't want the '\0' inside
}

bool Crop(const std::string& src_filepath, const std::string& dst_filepath, const std::string& src_dir)
{
	if (sx::ResFileHelper::Type(src_filepath.c_str()) != sx::RES_FILE_IMAGE) {
		return false;
	}

	auto dst_dir = boost::filesystem::path(dst_filepath).parent_path().string();

	static const bool PRE_MUL_ALPHA(false);
	auto img = facade::ResPool::Instance().Fetch<pimg::ImageData>(src_filepath, PRE_MUL_ALPHA);

	pimg::RegularRectCut cut(img->GetPixelData(), img->GetWidth(), img->GetHeight());
	cut.AutoCut();

//	std::cout << ee::StringHelper::Format("File: %s, Left: %d, Used: %d", filepath.c_str(), cut.GetLeftArea(), cut.GetUseArea()) << std::endl;

	std::string filename = boost::filesystem::relative(src_filepath, src_dir).string();
	filename = filename.substr(0, filename.find_last_of('.'));
	filename = std::regex_replace(filename, std::regex("\\\\"), "%");

	GD_ASSERT(img->GetFormat() == GPF_RGB || img->GetFormat() == GPF_RGBA8, "err img");
	int channels = img->GetFormat() == GPF_RGB ? 3 : 4;
	pimg::Cropping crop(img->GetPixelData(), img->GetWidth(), img->GetHeight(), channels, true);

	auto& result = cut.GetResult();
	for (int i = 0, n = result.size(); i < n; ++i)
	{
		auto& r = result[i];
		const uint8_t* pixels = crop.Crop(r.x, r.y, r.x + r.w, r.y + r.h);

		std::string out_path = string_format("%s\\%s#%d#%d#%d#%d#", dst_dir.c_str(), filename.c_str(), r.x, r.y, r.w, r.h) + ".png";
		gimg_export(out_path.c_str(), pixels, r.w, r.h, GPF_RGBA8, true);
		delete[] pixels;
	}
}

bool CropSingle(const std::string& src_filepath, const std::string& dst_filepath)
{
	auto src_dir = boost::filesystem::path(src_filepath).parent_path().string();
	return Crop(src_filepath, dst_filepath, src_dir);
}

bool CropMulti(const std::string& filepath, const std::string& src_dir, const std::string& dst_filepath,
	           std::unique_ptr<tc::OpLog>& op_log, rapidjson::MemoryPoolAllocator<>& alloc)
{
	if (!Crop(filepath, dst_filepath, src_dir)) {
		return false;
	}

	// todo log

	return true;
}

}
namespace tc
{
extern bool InitRender();
}

namespace crop
{

void AutoCropGrids(const std::string& src_path, const std::string& dst_path)
{
	tc::InitRender();

	auto modify_time_filepath = boost::filesystem::absolute(TIME_FILEPATH, dst_path);
	auto cfg_filepath = boost::filesystem::absolute(LOG_FILEPATH, dst_path);
	tc::Application app(modify_time_filepath.string(), cfg_filepath.string());
	app.Do(src_path, dst_path, CropSingle, CropMulti, sx::RES_FILE_IMAGE);
}

}

max_stars_repo_name: xzrunner/toolchain

max_issues_repo_name: xzrunner/toolchain

max_forks_repo_name: xzrunner/toolchain


text:
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "runtime/row-batch.h"

#include <stdint.h> // for intptr_t
#include <memory>
#include <boost/scoped_ptr.hpp>

#include "runtime/exec-env.h"
#include "runtime/mem-tracker.h"
#include "runtime/string-value.h"
#include "runtime/tuple-row.h"
#include "util/compress.h"
#include "util/debug-util.h"
#include "util/decompress.h"
#include "util/fixed-size-hash-table.h"
#include "util/scope-exit-trigger.h"

#include "gen-cpp/Results_types.h"
#include "gen-cpp/row_batch.pb.h"

#include "common/names.h"

namespace impala {

const int RowBatch::AT_CAPACITY_MEM_USAGE;
const int RowBatch::FIXED_LEN_BUFFER_LIMIT;

RowBatch::RowBatch(const RowDescriptor* row_desc, int capacity, MemTracker* mem_tracker)
  : num_rows_(0),
    capacity_(capacity),
    flush_(FlushMode::NO_FLUSH_RESOURCES),
    needs_deep_copy_(false),
    num_tuples_per_row_(row_desc->tuple_descriptors().size()),
    tuple_ptrs_size_(capacity * num_tuples_per_row_ * sizeof(Tuple*)),
    attached_buffer_bytes_(0),
    tuple_data_pool_(mem_tracker),
    row_desc_(row_desc),
    mem_tracker_(mem_tracker) {
  DCHECK(mem_tracker_ != NULL);
  DCHECK_GT(capacity, 0);
  DCHECK_GT(tuple_ptrs_size_, 0);
  // TODO: switch to Init() pattern so we can check memory limit and return Status.
  mem_tracker_->Consume(tuple_ptrs_size_);
  tuple_ptrs_ = reinterpret_cast<Tuple**>(malloc(tuple_ptrs_size_));
  DCHECK(tuple_ptrs_ != NULL);
}

// TODO: we want our input_batch's tuple_data to come from our (not yet implemented)
// global runtime memory segment; how do we get thrift to allocate it from there?
// maybe change line (in Data_types.cc generated from Data.thrift)
//              xfer += iprot->readString(this->tuple_data[_i9]);
// to allocated string data in special mempool
// (change via python script that runs over Data_types.cc)
RowBatch::RowBatch(
    const RowDescriptor* row_desc, const TRowBatch& input_batch, MemTracker* mem_tracker)
  : num_rows_(input_batch.num_rows),
    capacity_(input_batch.num_rows),
    flush_(FlushMode::NO_FLUSH_RESOURCES),
    needs_deep_copy_(false),
    num_tuples_per_row_(input_batch.row_tuples.size()),
    tuple_ptrs_size_(capacity_ * num_tuples_per_row_ * sizeof(Tuple*)),
    attached_buffer_bytes_(0),
    tuple_data_pool_(mem_tracker),
    row_desc_(row_desc),
    mem_tracker_(mem_tracker) {
  DCHECK(mem_tracker_ != nullptr);
  DCHECK_EQ(num_tuples_per_row_, row_desc_->tuple_descriptors().size());
  DCHECK_GT(tuple_ptrs_size_, 0);
  kudu::Slice input_tuple_data =
      kudu::Slice(input_batch.tuple_data.c_str(), input_batch.tuple_data.size());
  kudu::Slice input_tuple_offsets = kudu::Slice(
      reinterpret_cast<const char*>(input_batch.tuple_offsets.data()),
      input_batch.tuple_offsets.size() * sizeof(int32_t));
  const THdfsCompression::type& compression_type = input_batch.compression_type;
  DCHECK(compression_type == THdfsCompression::NONE ||
      compression_type == THdfsCompression::LZ4)
      << "Unexpected compression type: " << input_batch.compression_type;

  mem_tracker_->Consume(tuple_ptrs_size_);
  tuple_ptrs_ = reinterpret_cast<Tuple**>(malloc(tuple_ptrs_size_));
  DCHECK(tuple_ptrs_ != nullptr) << "Failed to allocate tuple pointers";

  const uint64_t uncompressed_size = input_batch.uncompressed_size;
  uint8_t* tuple_data = tuple_data_pool_.Allocate(uncompressed_size);
  DCHECK(tuple_data != nullptr) << "Failed to allocate tuple data";

  Deserialize(input_tuple_offsets, input_tuple_data, uncompressed_size,
      compression_type == THdfsCompression::LZ4, tuple_data);
}

RowBatch::RowBatch(const RowDescriptor* row_desc, const RowBatchHeaderPB& header,
    MemTracker* mem_tracker)
  : num_rows_(0),
    capacity_(0),
    flush_(FlushMode::NO_FLUSH_RESOURCES),
    needs_deep_copy_(false),
    num_tuples_per_row_(header.num_tuples_per_row()),
    tuple_ptrs_size_(header.num_rows() * num_tuples_per_row_ * sizeof(Tuple*)),
    attached_buffer_bytes_(0),
    tuple_data_pool_(mem_tracker),
    row_desc_(row_desc),
    mem_tracker_(mem_tracker) {
  DCHECK(mem_tracker_ != nullptr);
  DCHECK_EQ(num_tuples_per_row_, row_desc_->tuple_descriptors().size());
  DCHECK_GT(tuple_ptrs_size_, 0);
}

void RowBatch::Deserialize(const kudu::Slice& input_tuple_offsets,
    const kudu::Slice& input_tuple_data, int64_t uncompressed_size,
    bool is_compressed, uint8_t* tuple_data) {
  DCHECK(tuple_ptrs_ != nullptr);
  DCHECK(tuple_data != nullptr);
  if (is_compressed) {
    // Decompress tuple data into data pool
    const uint8_t* compressed_data = input_tuple_data.data();
    size_t compressed_size = input_tuple_data.size();

    Lz4Decompressor decompressor(nullptr, false);
    Status status = decompressor.Init();
    DCHECK(status.ok()) << status.GetDetail();
    auto compressor_cleanup =
        MakeScopeExitTrigger([&decompressor]() { decompressor.Close(); });

    status = decompressor.ProcessBlock(
        true, compressed_size, compressed_data, &uncompressed_size, &tuple_data);
    DCHECK_NE(uncompressed_size, -1) << "RowBatch decompression failed";
    DCHECK(status.ok()) << "RowBatch decompression failed.";
  } else {
    // Tuple data uncompressed, copy directly into data pool
    DCHECK_EQ(uncompressed_size, input_tuple_data.size());
    memcpy(tuple_data, input_tuple_data.data(), input_tuple_data.size());
  }

  // Convert input_batch.tuple_offsets into pointers
  const int32_t* tuple_offsets =
      reinterpret_cast<const int32_t*>(input_tuple_offsets.data());
  DCHECK_EQ(input_tuple_offsets.size() % sizeof(int32_t), 0);
  int num_tuples = input_tuple_offsets.size() / sizeof(int32_t);
  for (int tuple_idx = 0; tuple_idx < num_tuples; ++tuple_idx) {
    int32_t offset = tuple_offsets[tuple_idx];
    if (offset == -1) {
      tuple_ptrs_[tuple_idx] = nullptr;
    } else {
      tuple_ptrs_[tuple_idx] = reinterpret_cast<Tuple*>(tuple_data + offset);
    }
  }

  // Check whether we have slots that require offset-to-pointer conversion.
  if (!row_desc_->HasVarlenSlots()) return;

  // For every unique tuple, convert string offsets contained in tuple data into
  // pointers. Tuples were serialized in the order we are deserializing them in,
  // so the first occurrence of a tuple will always have a higher offset than any
  // tuple we already converted.
  Tuple* last_converted = nullptr;
  for (int i = 0; i < num_rows_; ++i) {
    for (int j = 0; j < num_tuples_per_row_; ++j) {
      const TupleDescriptor* desc = row_desc_->tuple_descriptors()[j];
      if (!desc->HasVarlenSlots()) continue;
      Tuple* tuple = GetRow(i)->GetTuple(j);
      // Handle NULL or already converted tuples with one check.
      if (tuple <= last_converted) continue;
      last_converted = tuple;
      tuple->ConvertOffsetsToPointers(*desc, tuple_data);
    }
  }
}

Status RowBatch::FromProtobuf(const RowDescriptor* row_desc,
    const RowBatchHeaderPB& header, const kudu::Slice& input_tuple_offsets,
    const kudu::Slice& input_tuple_data, MemTracker* mem_tracker,
    BufferPool::ClientHandle* client, unique_ptr<RowBatch>* row_batch_ptr) {
  unique_ptr<RowBatch> row_batch(new RowBatch(row_desc, header, mem_tracker));

  DCHECK(client != nullptr);
  row_batch->tuple_ptrs_info_.reset(new BufferInfo());
  row_batch->tuple_ptrs_info_->client = client;
  BufferPool::BufferHandle* tuple_ptrs_buffer = &(row_batch->tuple_ptrs_info_->buffer);
  RETURN_IF_ERROR(
      row_batch->AllocateBuffer(client, row_batch->tuple_ptrs_size_, tuple_ptrs_buffer));
  row_batch->tuple_ptrs_ = reinterpret_cast<Tuple**>(tuple_ptrs_buffer->data());

  const int64_t uncompressed_size = header.uncompressed_size();
  BufferPool::BufferHandle tuple_data_buffer;
  RETURN_IF_ERROR(
      row_batch->AllocateBuffer(client, uncompressed_size, &tuple_data_buffer));
  uint8_t* tuple_data = tuple_data_buffer.data();
  row_batch->AddBuffer(client, move(tuple_data_buffer), FlushMode::NO_FLUSH_RESOURCES);

  row_batch->num_rows_ = header.num_rows();
  row_batch->capacity_ = header.num_rows();
  const CompressionType& compression_type = header.compression_type();
  DCHECK(compression_type == CompressionType::NONE ||
      compression_type == CompressionType::LZ4)
      << "Unexpected compression type: " << compression_type;
  row_batch->Deserialize(input_tuple_offsets, input_tuple_data, uncompressed_size,
      compression_type == CompressionType::LZ4, tuple_data);
  *row_batch_ptr = std::move(row_batch);
  return Status::OK();
}

RowBatch::~RowBatch() {
  tuple_data_pool_.FreeAll();
  FreeBuffers();
  if (tuple_ptrs_info_.get() != nullptr) {
    ExecEnv::GetInstance()->buffer_pool()->FreeBuffer(
        tuple_ptrs_info_->client, &(tuple_ptrs_info_->buffer));
  } else {
    DCHECK(tuple_ptrs_ != nullptr);
    free(tuple_ptrs_);
    mem_tracker_->Release(tuple_ptrs_size_);
  }
  tuple_ptrs_ = nullptr;
}

Status RowBatch::Serialize(TRowBatch* output_batch) {
  return Serialize(output_batch, UseFullDedup());
}

Status RowBatch::Serialize(TRowBatch* output_batch, bool full_dedup) {
  // why does Thrift not generate a Clear() function?
  output_batch->row_tuples.clear();
  output_batch->tuple_offsets.clear();
  int64_t uncompressed_size;
  bool is_compressed;
  RETURN_IF_ERROR(Serialize(full_dedup, &output_batch->tuple_offsets,
      &output_batch->tuple_data, &uncompressed_size, &is_compressed));
  // TODO: max_size() is much larger than the amount of memory we could feasibly
  // allocate. Need better way to detect problem.
  DCHECK_LE(uncompressed_size, output_batch->tuple_data.max_size());
  output_batch->__set_num_rows(num_rows_);
  output_batch->__set_uncompressed_size(uncompressed_size);
  output_batch->__set_compression_type(
      is_compressed ? THdfsCompression::LZ4 : THdfsCompression::NONE);
  row_desc_->ToThrift(&output_batch->row_tuples);
  return Status::OK();
}

Status RowBatch::Serialize(OutboundRowBatch* output_batch) {
  int64_t uncompressed_size;
  bool is_compressed;
  output_batch->tuple_offsets_.clear();
  RETURN_IF_ERROR(Serialize(UseFullDedup(), &output_batch->tuple_offsets_,
      &output_batch->tuple_data_, &uncompressed_size, &is_compressed));

  // Initialize the RowBatchHeaderPB
  RowBatchHeaderPB* header = &output_batch->header_;
  header->Clear();
  header->set_num_rows(num_rows_);
  header->set_num_tuples_per_row(row_desc_->tuple_descriptors().size());
  header->set_uncompressed_size(uncompressed_size);
  header->set_compression_type(
      is_compressed ? CompressionType::LZ4 : CompressionType::NONE);
  return Status::OK();
}

Status RowBatch::Serialize(bool full_dedup, vector<int32_t>* tuple_offsets,
    string* tuple_data, int64_t* uncompressed_size, bool* is_compressed) {
  // As part of the serialization process we deduplicate tuples to avoid serializing a
  // Tuple multiple times for the RowBatch. By default we only detect duplicate tuples
  // in adjacent rows only. If full deduplication is enabled, we will build a
  // map to detect non-adjacent duplicates. Building this map comes with significant
  // overhead, so is only worthwhile in the uncommon case of many non-adjacent duplicates.
  int64_t size;
  if (full_dedup) {
    // Maps from tuple to offset of its serialized data in output_batch->tuple_data.
    DedupMap distinct_tuples;
    RETURN_IF_ERROR(distinct_tuples.Init(num_rows_ * num_tuples_per_row_ * 2, 0));
    size = TotalByteSize(&distinct_tuples);
    distinct_tuples.Clear(); // Reuse allocated hash table.
    RETURN_IF_ERROR(SerializeInternal(size, &distinct_tuples, tuple_offsets, tuple_data));
  } else {
    size = TotalByteSize(nullptr);
    RETURN_IF_ERROR(SerializeInternal(size, nullptr, tuple_offsets, tuple_data));
  }
  *uncompressed_size = size;
  *is_compressed = false;

  if (size > 0) {
    // Try compressing tuple_data to compression_scratch_, swap if compressed data is
    // smaller
    Lz4Compressor compressor(nullptr, false);
    RETURN_IF_ERROR(compressor.Init());
    auto compressor_cleanup =
        MakeScopeExitTrigger([&compressor]() { compressor.Close(); });

    // If the input size is too large for LZ4 to compress, MaxOutputLen() will return 0.
    int64_t compressed_size = compressor.MaxOutputLen(size);
    if (compressed_size == 0) {
      return Status(TErrorCode::LZ4_COMPRESSION_INPUT_TOO_LARGE, size);
    }
    DCHECK_GT(compressed_size, 0);
    if (compression_scratch_.size() < compressed_size) {
      compression_scratch_.resize(compressed_size);
    }
    uint8_t* input = (uint8_t*)tuple_data->c_str();
    uint8_t* compressed_output = (uint8_t*)compression_scratch_.c_str();
    RETURN_IF_ERROR(
        compressor.ProcessBlock(true, size, input, &compressed_size, &compressed_output));
    if (LIKELY(compressed_size < size)) {
      compression_scratch_.resize(compressed_size);
      tuple_data->swap(compression_scratch_);
      *is_compressed = true;
    }
    VLOG_ROW << "uncompressed size: " << size << ", compressed size: " << compressed_size;
  }
  return Status::OK();
}

bool RowBatch::UseFullDedup() {
  // Switch to using full deduplication in cases where severe size blow-ups are known to
  // be common: when a row contains tuples with collections and where there are three or
  // more tuples per row so non-adjacent duplicate tuples may have been created when
  // joining tuples from multiple sources into a single row.
  if (row_desc_->tuple_descriptors().size() < 3) return false;
  vector<TupleDescriptor*>::const_iterator tuple_desc =
      row_desc_->tuple_descriptors().begin();
  for (; tuple_desc != row_desc_->tuple_descriptors().end(); ++tuple_desc) {
    if (!(*tuple_desc)->collection_slots().empty()) return true;
  }
  return false;
}

Status RowBatch::SerializeInternal(int64_t size, DedupMap* distinct_tuples,
    vector<int32_t>* tuple_offsets, string* tuple_data_str) {
  DCHECK(distinct_tuples == nullptr || distinct_tuples->size() == 0);

  // The maximum uncompressed RowBatch size that can be serialized is INT_MAX. This
  // is because the tuple offsets are int32s and will overflow for a larger size.
  if (size > numeric_limits<int32_t>::max()) {
    return Status(TErrorCode::ROW_BATCH_TOO_LARGE, size, numeric_limits<int32_t>::max());
  }

  // TODO: track memory usage
  // TODO: detect if serialized size is too large to allocate and return proper error.
  tuple_data_str->resize(size);
  tuple_offsets->reserve(num_rows_ * num_tuples_per_row_);

  // Copy tuple data of unique tuples, including strings, into output_batch (converting
  // string pointers into offsets in the process).
  int offset = 0; // current offset into output_batch->tuple_data
  char* tuple_data = const_cast<char*>(tuple_data_str->c_str());

  for (int i = 0; i < num_rows_; ++i) {
    vector<TupleDescriptor*>::const_iterator desc =
        row_desc_->tuple_descriptors().begin();
    for (int j = 0; desc != row_desc_->tuple_descriptors().end(); ++desc, ++j) {
      Tuple* tuple = GetRow(i)->GetTuple(j);
      if (UNLIKELY(tuple == nullptr)) {
        // NULLs are encoded as -1
        tuple_offsets->push_back(-1);
        continue;
      } else if (LIKELY(i > 0) && UNLIKELY(GetRow(i - 1)->GetTuple(j) == tuple)) {
        // Fast tuple deduplication for adjacent rows.
        int prev_row_idx = tuple_offsets->size() - num_tuples_per_row_;
        tuple_offsets->push_back((*tuple_offsets)[prev_row_idx]);
        continue;
      } else if (UNLIKELY(distinct_tuples != nullptr)) {
        if ((*desc)->byte_size() == 0) {
          // Zero-length tuples can be represented as nullptr.
          tuple_offsets->push_back(-1);
          continue;
        }
        int* dedupd_offset = distinct_tuples->FindOrInsert(tuple, offset);
        if (*dedupd_offset != offset) {
          // Repeat of tuple
          DCHECK_GE(*dedupd_offset, 0);
          tuple_offsets->push_back(*dedupd_offset);
          continue;
        }
      }
      // Record offset before creating copy (which increments offset and tuple_data)
      tuple_offsets->push_back(offset);
      tuple->DeepCopy(**desc, &tuple_data, &offset, /* convert_ptrs */ true);
      DCHECK_LE(offset, size);
    }
  }
  DCHECK_EQ(offset, size);
  return Status::OK();
}

Status RowBatch::AllocateBuffer(BufferPool::ClientHandle* client, int64_t len,
    BufferPool::BufferHandle* buffer_handle) {
  BufferPool* buffer_pool = ExecEnv::GetInstance()->buffer_pool();
  int64_t buffer_len = BitUtil::RoundUpToPowerOfTwo(len);
  buffer_len = max(buffer_pool->min_buffer_len(), buffer_len);
  RETURN_IF_ERROR(
      buffer_pool->AllocateUnreservedBuffer(client, buffer_len, buffer_handle));
  if (UNLIKELY(!buffer_handle->is_open())) {
    return mem_tracker_->MemLimitExceeded(
        nullptr, "Failed to allocate row batch", buffer_len);
  }
  return Status::OK();
}

void RowBatch::AddBuffer(BufferPool::ClientHandle* client,
    BufferPool::BufferHandle&& buffer, FlushMode flush) {
  attached_buffer_bytes_ += buffer.len();
  BufferInfo buffer_info;
  buffer_info.client = client;
  buffer_info.buffer = std::move(buffer);
  buffers_.push_back(std::move(buffer_info));
  if (flush == FlushMode::FLUSH_RESOURCES) MarkFlushResources();
}

void RowBatch::FreeBuffers() {
  for (BufferInfo& buffer_info : buffers_) {
    ExecEnv::GetInstance()->buffer_pool()->FreeBuffer(
        buffer_info.client, &buffer_info.buffer);
  }
  buffers_.clear();
}

void RowBatch::Reset() {
  num_rows_ = 0;
  capacity_ = tuple_ptrs_size_ / (num_tuples_per_row_ * sizeof(Tuple*));
  // TODO: Change this to Clear() and investigate the repercussions.
  tuple_data_pool_.FreeAll();
  FreeBuffers();
  attached_buffer_bytes_ = 0;
  flush_ = FlushMode::NO_FLUSH_RESOURCES;
  needs_deep_copy_ = false;
}

void RowBatch::TransferResourceOwnership(RowBatch* dest) {
  dest->tuple_data_pool_.AcquireData(&tuple_data_pool_, false);
  for (BufferInfo& buffer_info : buffers_) {
    dest->AddBuffer(
        buffer_info.client, std::move(buffer_info.buffer), FlushMode::NO_FLUSH_RESOURCES);
  }
  buffers_.clear();
  if (needs_deep_copy_) {
    dest->MarkNeedsDeepCopy();
  } else if (flush_ == FlushMode::FLUSH_RESOURCES) {
    dest->MarkFlushResources();
  }
  Reset();
}

void RowBatch::SetMemTracker(MemTracker* new_tracker) {
  tuple_data_pool_.SetMemTracker(new_tracker);
  mem_tracker_->TransferTo(new_tracker, tuple_ptrs_size_);
  mem_tracker_ = new_tracker;
}

int64_t RowBatch::GetDeserializedSize(const TRowBatch& batch) {
  return batch.uncompressed_size + batch.tuple_offsets.size() * sizeof(Tuple*);
}

int64_t RowBatch::GetSerializedSize(const TRowBatch& batch) {
  int64_t result = batch.tuple_data.size();
  result += batch.row_tuples.size() * sizeof(TTupleId);
  result += batch.tuple_offsets.size() * sizeof(int32_t);
  return result;
}

int64_t RowBatch::GetDeserializedSize(const RowBatchHeaderPB& header,
    const kudu::Slice& tuple_offsets) {
  DCHECK_EQ(tuple_offsets.size() % sizeof(int32_t), 0);
  return header.uncompressed_size() +
      (tuple_offsets.size() / sizeof(int32_t)) * sizeof(Tuple*);
}

int64_t RowBatch::GetDeserializedSize(const OutboundRowBatch& batch) {
  return batch.header_.uncompressed_size() + batch.tuple_offsets_.size() * sizeof(Tuple*);
}

int64_t RowBatch::GetSerializedSize(const OutboundRowBatch& batch) {
  return batch.tuple_data_.size() + batch.tuple_offsets_.size() * sizeof(int32_t);
}

void RowBatch::AcquireState(RowBatch* src) {
  DCHECK(row_desc_->LayoutEquals(*src->row_desc_));
  DCHECK_EQ(num_tuples_per_row_, src->num_tuples_per_row_);
  DCHECK_EQ(tuple_ptrs_size_, src->tuple_ptrs_size_);

  // The destination row batch should be empty.
  DCHECK(!needs_deep_copy_);
  DCHECK_EQ(num_rows_, 0);
  DCHECK_EQ(attached_buffer_bytes_, 0);

  num_rows_ = src->num_rows_;
  capacity_ = src->capacity_;
  // tuple_ptrs_ were allocated with malloc so can be swapped between batches.
  DCHECK(tuple_ptrs_info_.get() == nullptr);
  std::swap(tuple_ptrs_, src->tuple_ptrs_);
  src->TransferResourceOwnership(this);
}

void RowBatch::DeepCopyTo(RowBatch* dst) {
  DCHECK(dst->row_desc_->Equals(*row_desc_));
  DCHECK_EQ(dst->num_rows_, 0);
  DCHECK_GE(dst->capacity_, num_rows_);
  dst->AddRows(num_rows_);
  for (int i = 0; i < num_rows_; ++i) {
    TupleRow* src_row = GetRow(i);
    TupleRow* dst_row = reinterpret_cast<TupleRow*>(dst->tuple_ptrs_ +
        i * num_tuples_per_row_);
    src_row->DeepCopy(
        dst_row, row_desc_->tuple_descriptors(), &dst->tuple_data_pool_, false);
  }
  dst->CommitRows(num_rows_);
}

// TODO: consider computing size of batches as they are built up
int64_t RowBatch::TotalByteSize(DedupMap* distinct_tuples) {
  DCHECK(distinct_tuples == nullptr || distinct_tuples->size() == 0);
  int64_t result = 0;
  vector<int> tuple_count(row_desc_->tuple_descriptors().size(), 0);

  // Sum total variable length byte sizes.
  for (int i = 0; i < num_rows_; ++i) {
    for (int j = 0; j < num_tuples_per_row_; ++j) {
      Tuple* tuple = GetRow(i)->GetTuple(j);
      if (UNLIKELY(tuple == nullptr)) continue;
      // Only count the data of unique tuples.
      if (LIKELY(i > 0) && UNLIKELY(GetRow(i - 1)->GetTuple(j) == tuple)) {
        // Fast tuple deduplication for adjacent rows.
        continue;
      } else if (UNLIKELY(distinct_tuples != nullptr)) {
        if (row_desc_->tuple_descriptors()[j]->byte_size() == 0) continue;
        bool inserted = distinct_tuples->InsertIfNotPresent(tuple, -1);
        if (!inserted) continue;
      }
      result += tuple->VarlenByteSize(*row_desc_->tuple_descriptors()[j]);
      ++tuple_count[j];
    }
  }
  // Compute sum of fixed component of tuple sizes.
  for (int j = 0; j < num_tuples_per_row_; ++j) {
    result += row_desc_->tuple_descriptors()[j]->byte_size() * tuple_count[j];
  }
  return result;
}

Status RowBatch::ResizeAndAllocateTupleBuffer(
    RuntimeState* state, int64_t* buffer_size, uint8_t** buffer) {
  return ResizeAndAllocateTupleBuffer(
      state, &tuple_data_pool_, row_desc_->GetRowSize(), &capacity_, buffer_size, buffer);
}

Status RowBatch::ResizeAndAllocateTupleBuffer(RuntimeState* state, MemPool* pool,
    int row_size, int* capacity, int64_t* buffer_size, uint8_t** buffer) {
  // Avoid divide-by-zero. Don't need to modify capacity for empty rows anyway.
  if (row_size != 0) {
    *capacity = max(1, min(*capacity, FIXED_LEN_BUFFER_LIMIT / row_size));
  }
  *buffer_size = static_cast<int64_t>(row_size) * *capacity;
  *buffer = pool->TryAllocate(*buffer_size);
  if (*buffer == nullptr) {
    return pool->mem_tracker()->MemLimitExceeded(
        state, "Failed to allocate tuple buffer", *buffer_size);
  }
  return Status::OK();
}

void RowBatch::VLogRows(const string& context) {
  if (!VLOG_ROW_IS_ON) return;
  VLOG_ROW << context << ": #rows=" << num_rows_;
  for (int i = 0; i < num_rows_; ++i) {
    VLOG_ROW << PrintRow(GetRow(i), *row_desc_);
  }
}

}

ext: cc

max_stars_repo_name: caofanCPU/Impala

max_issues_repo_name: Pandinosaurus/Impala

max_forks_repo_name: Pandinosaurus/Impala


text:
#ifndef BOOST_NETWORK_PROTOCOL_HTTP_SERVER_CONNECTION_HPP_20101027
#define BOOST_NETWORK_PROTOCOL_HTTP_SERVER_CONNECTION_HPP_20101027

// Copyright 2010 Dean Michael Berris.
// Copyright 2014 Jelle Van den Driessche.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

#include <boost/throw_exception.hpp>
#include <boost/scope_exit.hpp>
#include <boost/network/protocol/http/algorithms/linearize.hpp>
#include <boost/network/utils/thread_pool.hpp>
#include <boost/range/adaptor/sliced.hpp>
#include <boost/range/algorithm/transform.hpp>
#include <boost/range/algorithm/copy.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/streambuf.hpp>
#include <boost/asio/strand.hpp>
#include <boost/asio/buffer.hpp>
#include <boost/make_shared.hpp>
#include <boost/network/protocol/stream_handler.hpp>
#include <boost/network/protocol/http/server/request_parser.hpp>
#include <boost/range/iterator_range.hpp>
#include <boost/optional.hpp>
#include <boost/utility/typed_in_place_factory.hpp>
#include <boost/thread/locks.hpp>
#include <boost/thread/recursive_mutex.hpp>
#include <boost/utility/enable_if.hpp>
#include <list>
#include <vector>
#include <iterator>
#ifdef BOOST_NETWORK_NO_LIB
#include <boost/network/protocol/http/server/impl/parsers.ipp>
#endif

#ifndef BOOST_NETWORK_HTTP_SERVER_CONNECTION_HEADER_BUFFER_MAX_SIZE
/** Here we define a page's worth of header connection buffer data.
 *  This can be tuned to reduce the memory cost of connections, but this
 *  default size is set to be friendly to typical service applications.
 *  This is the maximum size though and Boost.Asio's internal representation
 *  of a streambuf would make appropriate decisions on how big a buffer
 *  is to begin with.
 *
 *  This kinda assumes that a page is by default 4096. Since we're using
 *  the default allocator with the static buffers, it's not guaranteed that
 *  the static buffers will be page-aligned when they are allocated.
 */
#define BOOST_NETWORK_HTTP_SERVER_CONNECTION_HEADER_BUFFER_MAX_SIZE 4096
#endif /* BOOST_NETWORK_HTTP_SERVER_CONNECTION_HEADER_BUFFER_MAX_SIZE */

namespace boost {
namespace network {
namespace http {

#ifndef BOOST_NETWORK_NO_LIB
extern void parse_version(std::string const& partial_parsed,
                          fusion::tuple<uint8_t, uint8_t>& version_pair);
extern void parse_headers(std::string const& input,
                          std::vector<request_header_narrow>& container);
#endif

template <class Tag, class Handler>
struct async_connection
    : boost::enable_shared_from_this<async_connection<Tag, Handler> > {

  enum status_t {
    ok = 200,
    created = 201,
    accepted = 202,
    no_content = 204,
    partial_content = 206,
    multiple_choices = 300,
    moved_permanently = 301,
    moved_temporarily = 302,
    not_modified = 304,
    bad_request = 400,
    unauthorized = 401,
    forbidden = 403,
    not_found = 404,
    not_supported = 405,
    not_acceptable = 406,
    request_timeout = 408,
    precondition_failed = 412,
    unsatisfiable_range = 416,
    internal_server_error = 500,
    not_implemented = 501,
    bad_gateway = 502,
    service_unavailable = 503,
    space_unavailable = 507
  };

  typedef typename string<Tag>::type string_type;
  typedef basic_request<Tag> request;
  typedef shared_ptr<async_connection> connection_ptr;

 private:
  static char const* status_message(status_t status) {
    static char const ok_[] = "OK", created_[] = "Created",
                      accepted_[] = "Accepted", no_content_[] = "No Content",
                      multiple_choices_[] = "Multiple Choices",
                      moved_permanently_[] = "Moved Permanently",
                      moved_temporarily_[] = "Moved Temporarily",
                      not_modified_[] = "Not Modified",
                      bad_request_[] = "Bad Request",
                      unauthorized_[] = "Unauthorized",
                      forbidden_[] = "Fobidden", not_found_[] = "Not Found",
                      not_supported_[] = "Not Supported",
                      not_acceptable_[] = "Not Acceptable",
                      internal_server_error_[] = "Internal Server Error",
                      not_implemented_[] = "Not Implemented",
                      bad_gateway_[] = "Bad Gateway",
                      service_unavailable_[] = "Service Unavailable",
                      unknown_[] = "Unknown",
                      partial_content_[] = "Partial Content",
                      request_timeout_[] = "Request Timeout",
                      precondition_failed_[] = "Precondition Failed",
                      unsatisfiable_range_[] =
                          "Requested Range Not Satisfiable",
                      space_unavailable_[] =
                          "Insufficient Space to Store Resource";
    switch (status) {
      case ok:
        return ok_;
      case created:
        return created_;
      case accepted:
        return accepted_;
      case no_content:
        return no_content_;
      case multiple_choices:
        return multiple_choices_;
      case moved_permanently:
        return moved_permanently_;
      case moved_temporarily:
        return moved_temporarily_;
      case not_modified:
        return not_modified_;
      case bad_request:
        return bad_request_;
      case unauthorized:
        return unauthorized_;
      case forbidden:
        return forbidden_;
      case not_found:
        return not_found_;
      case not_supported:
        return not_supported_;
      case not_acceptable:
        return not_acceptable_;
      case internal_server_error:
        return internal_server_error_;
      case not_implemented:
        return not_implemented_;
      case bad_gateway:
        return bad_gateway_;
      case service_unavailable:
        return service_unavailable_;
      case partial_content:
        return partial_content_;
      case request_timeout:
        return request_timeout_;
      case precondition_failed:
        return precondition_failed_;
      case unsatisfiable_range:
        return unsatisfiable_range_;
      case space_unavailable:
        return space_unavailable_;
      default:
        return unknown_;
    }
  }

 public:
  async_connection(asio::io_service& io_service, Handler& handler,
                   utils::thread_pool& thread_pool,
                   boost::shared_ptr<ssl_context> ctx =
                       boost::shared_ptr<ssl_context>())
      : strand(io_service),
        handler(handler),
        thread_pool_(thread_pool),
        headers_buffer(
            BOOST_NETWORK_HTTP_SERVER_CONNECTION_HEADER_BUFFER_MAX_SIZE),
#ifdef BOOST_NETWORK_ENABLE_HTTPS
        socket_(io_service, ctx),
#else
        socket_(io_service),
#endif
        handshake_done(false),
        headers_already_sent(false),
        headers_in_progress(false) {
    new_start = read_buffer_.begin();
  }

  ~async_connection() throw() {
    boost::system::error_code ignored;
    socket_.shutdown(asio::ip::tcp::socket::shutdown_receive, ignored);
  }

  /** Function: template <class Range> set_headers(Range headers)
   *  Precondition: headers have not been sent yet
   *  Postcondition: headers have been linearized to a buffer,
   *                 and assumed to have been sent already when the
   *function exits
   *  Throws: std::logic_error in case the headers have already been sent.
   *
   *  A call to set_headers takes a Range where each element models the
   *  Header concept. This Range will be linearized onto a buffer, which
   *is
   *  then sent as soon as the first call to `write` or `flush` commences.
   */
  template <class Range>
  void set_headers(Range headers) {
    lock_guard lock(headers_mutex);
    if (headers_in_progress || headers_already_sent)
      boost::throw_exception(
          std::logic_error("Headers have already been sent."));

    if (error_encountered)
      boost::throw_exception(boost::system::system_error(*error_encountered));

    typedef constants<Tag> consts;
    {
      std::ostream stream(&headers_buffer);
      stream << consts::http_slash() << 1 << consts::dot() << 1
             << consts::space() << status << consts::space()
             << status_message(status) << consts::crlf();
      if (!boost::empty(headers)) {
        typedef typename string<Tag>::type string_type;
        boost::transform(headers, std::ostream_iterator<string_type>(stream),
                         linearize_header<Tag>());
      } else {
        stream << consts::crlf();
      }
      stream << consts::crlf();
    }

    write_headers_only(
        boost::bind(&async_connection<Tag, Handler>::do_nothing,
                    async_connection<Tag, Handler>::shared_from_this()));
  }

  void set_status(status_t new_status) {
    lock_guard lock(headers_mutex);
    if (headers_already_sent)
      boost::throw_exception(std::logic_error(
          "Headers have already been sent, cannot reset status."));
    if (error_encountered)
      boost::throw_exception(boost::system::system_error(*error_encountered));

    status = new_status;
  }

  template <class Range>
  void write(Range const& range) {
    lock_guard lock(headers_mutex);
    if (error_encountered)
      boost::throw_exception(boost::system::system_error(*error_encountered));

    boost::function<void(boost::system::error_code)> f = boost::bind(
        &async_connection<Tag, Handler>::default_error,
        async_connection<Tag, Handler>::shared_from_this(), boost::arg<1>());

    write_impl(boost::make_iterator_range(range), f);
  }

  template <class Range, class Callback>
  typename disable_if<
      is_base_of<asio::const_buffer, typename Range::value_type>, void>::type
  write(Range const& range, Callback const& callback) {
    lock_guard lock(headers_mutex);
    if (error_encountered)
      boost::throw_exception(boost::system::system_error(*error_encountered));
    write_impl(boost::make_iterator_range(range), callback);
  }

  template <class ConstBufferSeq, class Callback>
  typename enable_if<
      is_base_of<asio::const_buffer, typename ConstBufferSeq::value_type>,
      void>::type
  write(ConstBufferSeq const& seq, Callback const& callback) {
    write_vec_impl(seq, callback, shared_array_list(), shared_buffers());
  }

 private:
  typedef boost::array<char, BOOST_NETWORK_HTTP_SERVER_CONNECTION_BUFFER_SIZE>
      buffer_type;

 public:
  typedef iterator_range<buffer_type::const_iterator> input_range;
  typedef boost::function<
      void(input_range, boost::system::error_code, std::size_t, connection_ptr)>
      read_callback_function;

  void read(read_callback_function callback) {
    if (error_encountered)
      boost::throw_exception(boost::system::system_error(*error_encountered));
    if (new_start != read_buffer_.begin()) {
      input_range input =
          boost::make_iterator_range(new_start, read_buffer_.end());
      buffer_type::iterator start_tmp = new_start;
      new_start = read_buffer_.begin();
      thread_pool().post(
          boost::bind(callback, input, boost::system::error_code(),
                      std::distance(start_tmp, data_end),
                      async_connection<Tag, Handler>::shared_from_this()));
      return;
    }

    socket().async_read_some(
        asio::buffer(read_buffer_),
        strand.wrap(boost::bind(
            &async_connection<Tag, Handler>::wrap_read_handler,
            async_connection<Tag, Handler>::shared_from_this(), callback,
            asio::placeholders::error, asio::placeholders::bytes_transferred)));
  }

  boost::network::stream_handler& socket() { return socket_; }
  utils::thread_pool& thread_pool() { return thread_pool_; }
  bool has_error() { return (!!error_encountered); }
  optional<boost::system::system_error> error() { return error_encountered; }

 private:
  void wrap_read_handler(read_callback_function callback,
                         boost::system::error_code const& ec,
                         std::size_t bytes_transferred) {
    if (ec) error_encountered = in_place<boost::system::system_error>(ec);
    buffer_type::const_iterator data_start = read_buffer_.begin(),
                                data_end = read_buffer_.begin();
    std::advance(data_end, bytes_transferred);
    thread_pool().post(boost::bind(
        callback, boost::make_iterator_range(data_start, data_end), ec,
        bytes_transferred, async_connection<Tag, Handler>::shared_from_this()));
  }

  void default_error(boost::system::error_code const& ec) {
    error_encountered = in_place<boost::system::system_error>(ec);
  }

  typedef boost::array<char, BOOST_NETWORK_HTTP_SERVER_CONNECTION_BUFFER_SIZE>
      array;
  typedef std::list<shared_ptr<array> > array_list;
  typedef boost::shared_ptr<array_list> shared_array_list;
  typedef boost::shared_ptr<std::vector<asio::const_buffer> > shared_buffers;
  typedef request_parser<Tag> request_parser_type;
  typedef boost::lock_guard<boost::recursive_mutex> lock_guard;
  typedef std::list<boost::function<void()> > pending_actions_list;

  asio::io_service::strand strand;
  Handler& handler;
  utils::thread_pool& thread_pool_;
  asio::streambuf headers_buffer;
  boost::network::stream_handler socket_;
  bool handshake_done;
  volatile bool headers_already_sent, headers_in_progress;

  boost::recursive_mutex headers_mutex;
  buffer_type read_buffer_;
  status_t status;
  request_parser_type parser;
  request request_;
  buffer_type::iterator new_start, data_end;
  string_type partial_parsed;
  optional<boost::system::system_error> error_encountered;
  pending_actions_list pending_actions;

  template <class, class>
  friend struct async_server_base;

  enum state_t {
    method,
    uri,
    version,
    headers
  };

  void start() {
    typename ostringstream<Tag>::type ip_stream;
    ip_stream << socket_.remote_endpoint().address().to_v4().to_string() << ':'
              << socket_.remote_endpoint().port();
    request_.source = ip_stream.str();
    read_more(method);
  }

  void read_more(state_t state) {
#ifdef BOOST_NETWORK_ENABLE_HTTPS
    if (socket_.is_ssl_enabled() && !handshake_done) {
      socket_.async_handshake(
          boost::asio::ssl::stream_base::server,
          boost::bind(&async_connection::handle_handshake,
                      async_connection<Tag, Handler>::shared_from_this(),
                      boost::asio::placeholders::error, state));
    } else {
#endif
      socket_.async_read_some(
          asio::buffer(read_buffer_),
          strand.wrap(
              boost::bind(&async_connection<Tag, Handler>::handle_read_data,
                          async_connection<Tag, Handler>::shared_from_this(),
                          state, boost::asio::placeholders::error,
                          boost::asio::placeholders::bytes_transferred)));
#ifdef BOOST_NETWORK_ENABLE_HTTPS
    }
#endif
  }

  void handle_read_data(state_t state, boost::system::error_code const& ec,
                        std::size_t bytes_transferred) {
    if (!ec) {
      logic::tribool parsed_ok;
      iterator_range<buffer_type::iterator> result_range, input_range;
      data_end = read_buffer_.begin();
      std::advance(data_end, bytes_transferred);
      switch (state) {
        case method:
          input_range = boost::make_iterator_range(new_start, data_end);
          fusion::tie(parsed_ok, result_range) =
              parser.parse_until(request_parser_type::method_done, input_range);
          if (!parsed_ok) {
            client_error();
            break;
          } else if (parsed_ok == true) {
            swap(partial_parsed, request_.method);
            request_.method.append(boost::begin(result_range),
                                   boost::end(result_range));
            trim(request_.method);
            new_start = boost::end(result_range);
          } else {
            partial_parsed.append(boost::begin(result_range),
                                  boost::end(result_range));
            new_start = read_buffer_.begin();
            read_more(method);
            break;
          }
        case uri:
          input_range = boost::make_iterator_range(new_start, data_end);
          fusion::tie(parsed_ok, result_range) =
              parser.parse_until(request_parser_type::uri_done, input_range);
          if (!parsed_ok) {
            client_error();
            break;
          } else if (parsed_ok == true) {
            swap(partial_parsed, request_.destination);
            request_.destination.append(boost::begin(result_range),
                                        boost::end(result_range));
            trim(request_.destination);
            new_start = boost::end(result_range);
          } else {
            partial_parsed.append(boost::begin(result_range),
                                  boost::end(result_range));
            new_start = read_buffer_.begin();
            read_more(uri);
            break;
          }
        case version:
          input_range = boost::make_iterator_range(new_start, data_end);
          fusion::tie(parsed_ok, result_range) = parser.parse_until(
              request_parser_type::version_done, input_range);
          if (!parsed_ok) {
            client_error();
            break;
          } else if (parsed_ok == true) {
            fusion::tuple<uint8_t, uint8_t> version_pair;
            partial_parsed.append(boost::begin(result_range),
                                  boost::end(result_range));
            parse_version(partial_parsed, version_pair);
            request_.http_version_major = fusion::get<0>(version_pair);
            request_.http_version_minor = fusion::get<1>(version_pair);
            new_start = boost::end(result_range);
            partial_parsed.clear();
          } else {
            partial_parsed.append(boost::begin(result_range),
                                  boost::end(result_range));
            new_start = read_buffer_.begin();
            read_more(version);
            break;
          }
        case headers:
          input_range = boost::make_iterator_range(new_start, data_end);
          fusion::tie(parsed_ok, result_range) = parser.parse_until(
              request_parser_type::headers_done, input_range);
          if (!parsed_ok) {
            client_error();
            break;
          } else if (parsed_ok == true) {
            partial_parsed.append(boost::begin(result_range),
                                  boost::end(result_range));
            try {
              parse_headers(partial_parsed, request_.headers);
            } catch (...) {
              client_error();
              break;
            }
            new_start = boost::end(result_range);
            thread_pool().post(boost::bind(
                &Handler::operator(), &handler, cref(request_),
                async_connection<Tag, Handler>::shared_from_this()));
            return;
          } else {
            partial_parsed.append(boost::begin(result_range),
                                  boost::end(result_range));
            new_start = read_buffer_.begin();
            read_more(headers);
            break;
          }
        default:
          BOOST_ASSERT(false &&
                       "This is a bug, report to the cpp-netlib devel "
                       "mailing list!");
          std::abort();
      }
    } else {
      error_encountered = in_place<boost::system::system_error>(ec);
    }
  }

  void client_error() {
    static char const* bad_request =
        "HTTP/1.0 400 Bad Request\r\nConnection: close\r\nContent-Type: "
        "text/plain\r\nContent-Length: 12\r\n\r\nBad Request.";

    asio::async_write(
        socket(), asio::buffer(bad_request, strlen(bad_request)),
        strand.wrap(boost::bind(
            &async_connection<Tag, Handler>::client_error_sent,
            async_connection<Tag, Handler>::shared_from_this(),
            asio::placeholders::error, asio::placeholders::bytes_transferred)));
  }

  void client_error_sent(boost::system::error_code const& ec,
                         std::size_t bytes_transferred) {
    if (!ec) {
      boost::system::error_code ignored;
      socket().shutdown(asio::ip::tcp::socket::shutdown_both, ignored);
      socket().close(ignored);
    } else {
      error_encountered = in_place<boost::system::system_error>(ec);
    }
  }

  void do_nothing() {}

  void write_headers_only(boost::function<void()> callback) {
    if (headers_in_progress) return;
    headers_in_progress = true;
    asio::async_write(
        socket(), headers_buffer,
        strand.wrap(boost::bind(
            &async_connection<Tag, Handler>::handle_write_headers,
            async_connection<Tag, Handler>::shared_from_this(), callback,
            asio::placeholders::error, asio::placeholders::bytes_transferred)));
  }

  void handle_write_headers(boost::function<void()> callback,
                            boost::system::error_code const& ec,
                            std::size_t bytes_transferred) {
    lock_guard lock(headers_mutex);
    if (!ec) {
      headers_buffer.consume(headers_buffer.size());
      headers_already_sent = true;
      thread_pool().post(callback);
      pending_actions_list::iterator start = pending_actions.begin(),
                                     end = pending_actions.end();
      while (start != end) {
        thread_pool().post(*start++);
      }
      pending_actions_list().swap(pending_actions);
    } else {
      error_encountered = in_place<boost::system::system_error>(ec);
    }
  }

  void handle_write(
      boost::function<void(boost::system::error_code const&)> callback,
      shared_array_list temporaries, shared_buffers buffers,
      boost::system::error_code const& ec, std::size_t bytes_transferred) {
    // we want to forget the temporaries and buffers
    thread_pool().post(boost::bind(callback, ec));
  }

  template <class Range>
  void write_impl(Range range,
                  boost::function<void(boost::system::error_code)> callback) {
    // linearize the whole range into a vector
    // of fixed-sized buffers, then schedule an asynchronous
    // write of these buffers -- make sure they are live
    // by making these linearized buffers shared and made
    // part of the completion handler.
    //
    // once the range has been linearized and sent, schedule
    // a wrapper to be called in the io_service's thread, that
    // will re-schedule the given callback into the thread pool
    // referred to here so that the io_service's thread can concentrate
    // on doing I/O.
    //

    static std::size_t const connection_buffer_size =
        BOOST_NETWORK_HTTP_SERVER_CONNECTION_BUFFER_SIZE;
    shared_array_list temporaries = boost::make_shared<array_list>();
    shared_buffers buffers =
        boost::make_shared<std::vector<asio::const_buffer> >(0);

    std::size_t range_size = boost::distance(range);
    buffers->reserve((range_size / connection_buffer_size) +
                     ((range_size % connection_buffer_size) ? 1 : 0));
    std::size_t slice_size = std::min(range_size, connection_buffer_size);
    typename boost::range_iterator<Range>::type start = boost::begin(range),
                                                end = boost::end(range);
    while (slice_size != 0) {
      using boost::adaptors::sliced;
      shared_ptr<array> new_array = make_shared<array>();
      boost::copy(range | sliced(0, slice_size), new_array->begin());
      temporaries->push_back(new_array);
      buffers->push_back(asio::buffer(new_array->data(), slice_size));
      std::advance(start, slice_size);
      range = boost::make_iterator_range(start, end);
      range_size = boost::distance(range);
      slice_size = std::min(range_size, connection_buffer_size);
    }

    if (!buffers->empty()) {
      write_vec_impl(*buffers, callback, temporaries, buffers);
    }
  }

  template <class ConstBufferSeq, class Callback>
  void write_vec_impl(ConstBufferSeq const& seq, Callback const& callback,
                      shared_array_list temporaries, shared_buffers buffers) {
    lock_guard lock(headers_mutex);
    if (error_encountered)
      boost::throw_exception(boost::system::system_error(*error_encountered));

    boost::function<void(boost::system::error_code)> callback_function =
        callback;

    boost::function<void()> continuation = boost::bind(
        &async_connection<Tag, Handler>::template write_vec_impl<
             ConstBufferSeq, boost::function<void(boost::system::error_code)> >,
        async_connection<Tag, Handler>::shared_from_this(), seq,
        callback_function, temporaries, buffers);

    if (!headers_already_sent && !headers_in_progress) {
      write_headers_only(continuation);
      return;
    } else if (headers_in_progress && !headers_already_sent) {
      pending_actions.push_back(continuation);
      return;
    }

    asio::async_write(
        socket_, seq,
        boost::bind(&async_connection<Tag, Handler>::handle_write,
                    async_connection<Tag, Handler>::shared_from_this(),
                    callback_function, temporaries, buffers,
                    asio::placeholders::error,
                    asio::placeholders::bytes_transferred));
  }
  void handle_handshake(const boost::system::error_code& ec, state_t state) {
    if (!ec) {
      handshake_done = true;
      read_more(state);
    } else {
      error_encountered = in_place<boost::system::system_error>(ec);
    }
  }
};

} /* http */

} /* network */

} /* boost */

#endif /* BOOST_NETWORK_PROTOCOL_HTTP_SERVER_CONNECTION_HPP_20101027 */

max_stars_repo_name: TyRoXx/cdm

max_issues_repo_name: TyRoXx/cdm

max_forks_repo_name: TyRoXx/cdm


text:
/*
 * Copyright 2010,
 * François Bleibel,
 * Olivier Stasse,
 *
 * CNRS/AIST
 *
 */

/* --------------------------------------------------------------------- */
/* --- INCLUDE --------------------------------------------------------- */
/* --------------------------------------------------------------------- */

/* SOT */
#include <sot/core/debug.hh>
#include <sot/core/reader.hh>

#include <boost/bind.hpp>
#include <dynamic-graph/all-commands.h>
#include <dynamic-graph/factory.h>
#include <sstream>

using namespace dynamicgraph;
using namespace dynamicgraph::sot;
using namespace std;

DYNAMICGRAPH_FACTORY_ENTITY_PLUGIN(sotReader, "Reader");

/* --------------------------------------------------------------------- */
/* --- CLASS ----------------------------------------------------------- */
/* --------------------------------------------------------------------- */

sotReader::sotReader(const std::string n)
    : Entity(n), selectionSIN(NULL, "Reader(" + n + ")::input(flag)::selec"),
      vectorSOUT(boost::bind(&sotReader::getNextData, this, _1, _2),
                 sotNOSIGNAL, "Reader(" + n + ")::vector"),
      matrixSOUT(boost::bind(&sotReader::getNextMatrix, this, _1, _2),
                 vectorSOUT, "Reader(" + n + ")::matrix"),
      dataSet(), currentData(), iteratorSet(false), rows(0), cols(0) {
  signalRegistration(selectionSIN << vectorSOUT << matrixSOUT);
  selectionSIN = true;
  vectorSOUT.setNeedUpdateFromAllChildren(true);

  initCommands();
}

/* --------------------------------------------------------------------- */
/* --------------------------------------------------------------------- */
/* --------------------------------------------------------------------- */

void sotReader::load(const string &filename) {
  sotDEBUGIN(15);

  std::ifstream datafile(filename.c_str());
  const unsigned int SIZE = 1024;
  char buffer[SIZE];
  std::vector<double> newline;
  while (datafile.good()) {
    datafile.getline(buffer, SIZE);
    const unsigned int gcount = (unsigned int)(datafile.gcount());
    if (gcount >= SIZE) { /* TODO read error, line to long. */
    }
    std::istringstream iss(buffer);
    newline.clear();
    sotDEBUG(25) << "Get line = '" << buffer << "'" << std::endl;
    while (1) {
      double x;
      iss >> x;
      if (!iss.fail())
        newline.push_back(x);
      else
        break;
      sotDEBUG(45) << "New data = " << x << std::endl;
    }
    if (newline.size() > 0)
      dataSet.push_back(newline);
  }

  sotDEBUGOUT(15);
}

void sotReader::clear(void) {
  sotDEBUGIN(15);

  dataSet.clear();
  iteratorSet = false;

  sotDEBUGOUT(15);
}

void sotReader::rewind(void) {
  sotDEBUGIN(15);
  iteratorSet = false;
  sotDEBUGOUT(15);
}

dynamicgraph::Vector &sotReader::getNextData(dynamicgraph::Vector &res,
                                             const unsigned int time) {
  sotDEBUGIN(15);

  if (!iteratorSet) {
    sotDEBUG(15) << "Start the list" << std::endl;
    currentData = dataSet.begin();
    iteratorSet = true;
  } else if (currentData != dataSet.end()) {
    ++currentData;
  }

  if (currentData == dataSet.end()) {
    sotDEBUGOUT(15);
    return res;
  }

  const Flags &selection = selectionSIN(time);
  const std::vector<double> &curr = *currentData;

  unsigned int dim = 0;
  for (unsigned int i = 0; i < curr.size(); ++i)
    if (selection(i))
      dim++;

  res.resize(dim);
  int cursor = 0;
  for (unsigned int i = 0; i < curr.size(); ++i)
    if (selection(i))
      res(cursor++) = curr[i];

  sotDEBUGOUT(15);
  return res;
}

dynamicgraph::Matrix &sotReader::getNextMatrix(dynamicgraph::Matrix &res,
                                               const unsigned int time) {
  sotDEBUGIN(15);
  const dynamicgraph::Vector &vect = vectorSOUT(time);
  if (vect.size() < rows * cols)
    return res;

  res.resize(rows, cols);
  for (int i = 0; i < rows; ++i)
    for (int j = 0; j < cols; ++j)
      res(i, j) = vect(i * cols + j);

  sotDEBUGOUT(15);
  return res;
}
/* --------------------------------------------------------------------- */
/* --------------------------------------------------------------------- */
/* --------------------------------------------------------------------- */

void sotReader::display(std::ostream &os) const {
  os << CLASS_NAME << " " << name << endl;
}

std::ostream &operator<<(std::ostream &os, const sotReader &t) {
  t.display(os);
  return os;
}

/* --- Command line interface
 * ------------------------------------------------------ */
void sotReader::initCommands() {
  namespace dc = ::dynamicgraph::command;
  addCommand("clear", dc::makeCommandVoid0(*this, &sotReader::clear,
                                           "Clear the data loaded"));
  addCommand("rewind",
             dc::makeCommandVoid0(
                 *this, &sotReader::rewind,
                 "Reset the iterator to the beginning of the data set"));
  addCommand("load",
             dc::makeCommandVoid1(*this, &sotReader::load, "load file"));
  addCommand("resize", dc::makeCommandVoid2(*this, &sotReader::resize, " "));
}

void sotReader::resize(const int &row, const int &col) {
  rows = row;
  cols = col;
}

max_stars_repo_name: Rascof/sot-core

max_issues_repo_name: Rascof/sot-core

max_forks_repo_name: Rascof/sot-core