folly::bser Namespace Reference

Classes
class	BserDecodeError
struct	serialization_opts

Enumerations
enum	BserType : int8_t {   BserType::Array = 0, BserType::Object, BserType::String, BserType::Int8,   BserType::Int16, BserType::Int32, BserType::Int64, BserType::Real,   BserType::True, BserType::False, BserType::Null, BserType::Template,   BserType::Skip }

Functions
folly::dynamic	parseBser (folly::StringPiece)
folly::dynamic	parseBser (folly::ByteRange)
folly::dynamic	parseBser (const folly::IOBuf *)
size_t	decodePduLength (const folly::IOBuf *)
folly::fbstring	toBser (folly::dynamic const &, const serialization_opts &)
std::unique_ptr< folly::IOBuf >	toBserIOBuf (folly::dynamic const &, const serialization_opts &)
static void	bserEncode (dynamic const &dyn, QueueAppender &appender, const serialization_opts &opts)
static const dynamic *	getTemplate (const serialization_opts &opts, dynamic const &dynArray)
static void	bserEncodeInt (int64_t ival, QueueAppender &appender)
static void	bserEncodeString (folly::StringPiece str, QueueAppender &appender)
static void	bserEncodeArraySimple (dynamic const &dyn, QueueAppender &appender, const serialization_opts &opts)
static void	bserEncodeArray (dynamic const &dyn, QueueAppender &appender, const serialization_opts &opts)
static void	bserEncodeObject (dynamic const &dyn, QueueAppender &appender, const serialization_opts &opts)
static dynamic	parseBser (Cursor &curs)
template<typename... ARGS>
static void	throwDecodeError (Cursor &curs, ARGS &&...args)
static int64_t	decodeInt (Cursor &curs)
static std::string	decodeString (Cursor &curs)
static dynamic	decodeArray (Cursor &curs)
static dynamic	decodeObject (Cursor &curs)
static dynamic	decodeTemplate (Cursor &curs)
static size_t	decodeHeader (Cursor &curs)

Variables
const uint8_t	kMagic [2] = {0, 1}

Enumeration Type Documentation

enum folly::bser::BserType : int8_t

strong

Enumerator
Array
Object
String
Int8
Int16
Int32
Int64
Real
True
False
Null
Template
Skip

Definition at line 41 of file Bser.h.

                    : int8_t {
  Array = 0,
  Object,
  String,
  Int8,
  Int16,
  Int32,
  Int64,
  Real,
  True,
  False,
  Null,
  Template,
  Skip,
};

Function Documentation

static void folly::bser::bserEncode ( dynamic const &  dyn, QueueAppender &  appender, const serialization_opts &  opts  )

static

Definition at line 158 of file Dump.cpp.

References bserEncodeArray(), bserEncodeInt(), bserEncodeObject(), bserEncodeString(), False, folly::dynamic::getBool(), folly::dynamic::getDouble(), folly::dynamic::getInt(), folly::dynamic::getString(), int8_t, Null, Real, STRING, True, folly::dynamic::type(), and folly::io::QueueAppender::write().

Referenced by bserEncodeArray(), bserEncodeArraySimple(), bserEncodeObject(), and toBserIOBuf().

                                    {
  switch (dyn.type()) {
    case dynamic::Type::NULLT:
      appender.write((int8_t)BserType::Null);
      return;
    case dynamic::Type::BOOL:
      appender.write(
          (int8_t)(dyn.getBool() ? BserType::True : BserType::False));
      return;
    case dynamic::Type::DOUBLE: {
      double dval = dyn.getDouble();
      appender.write((int8_t)BserType::Real);
      appender.write(dval);
      return;
    }
    case dynamic::Type::INT64:
      bserEncodeInt(dyn.getInt(), appender);
      return;
    case dynamic::Type::OBJECT:
      bserEncodeObject(dyn, appender, opts);
      return;
    case dynamic::Type::ARRAY:
      bserEncodeArray(dyn, appender, opts);
      return;
    case dynamic::Type::STRING:
      bserEncodeString(dyn.getString(), appender);
      return;
  }
}

static void folly::bser::bserEncodeArray ( dynamic const &  dyn, QueueAppender &  appender, const serialization_opts &  opts  )

static

Definition at line 98 of file Dump.cpp.

References bserEncode(), bserEncodeArraySimple(), bserEncodeInt(), getTemplate(), int64_t, int8_t, name, folly::dynamic::size(), Skip, Template, UNLIKELY, and folly::io::QueueAppender::write().

Referenced by bserEncode().

                                    {
  auto templ = getTemplate(opts, dyn);
  if (UNLIKELY(templ != nullptr)) {
    appender.write((int8_t)BserType::Template);

    // Emit the list of property names
    bserEncodeArraySimple(*templ, appender, opts);

    // The number of objects in the array
    bserEncodeInt(int64_t(dyn.size()), appender);

    // For each object in the array
    for (const auto& ele : dyn) {
      // For each key in the template
      for (const auto& name : *templ) {
        if (auto found = ele.get_ptr(name)) {
          if (found->isNull()) {
            // Prefer to Skip rather than encode a null value for
            // compatibility with the other bser implementations
            appender.write((int8_t)BserType::Skip);
          } else {
            bserEncode(*found, appender, opts);
          }
        } else {
          appender.write((int8_t)BserType::Skip);
        }
      }
    }
    return;
  }

  bserEncodeArraySimple(dyn, appender, opts);
}

static void folly::bser::bserEncodeArraySimple ( dynamic const &  dyn, QueueAppender &  appender, const serialization_opts &  opts  )

static

Definition at line 87 of file Dump.cpp.

References Array, bserEncode(), bserEncodeInt(), int64_t, int8_t, folly::dynamic::size(), and folly::io::QueueAppender::write().

Referenced by bserEncodeArray().

                                    {
  appender.write((int8_t)BserType::Array);
  bserEncodeInt(int64_t(dyn.size()), appender);
  for (const auto& ele : dyn) {
    bserEncode(ele, appender, opts);
  }
}

static void folly::bser::bserEncodeInt ( int64_t  ival, QueueAppender &  appender  )

static

Definition at line 52 of file Dump.cpp.

References Int16, int16_t, Int32, int32_t, Int64, Int8, int8_t, folly::size(), and folly::io::QueueAppender::write().

Referenced by bserEncode(), bserEncodeArray(), bserEncodeArraySimple(), bserEncodeObject(), and bserEncodeString().

                                                                 {
  /* Return the smallest size int that can store the value */
  auto size =
      ((ival == ((int8_t)ival))
           ? 1
           : (ival == ((int16_t)ival)) ? 2 : (ival == ((int32_t)ival)) ? 4 : 8);

  switch (size) {
    case 1:
      appender.write((int8_t)BserType::Int8);
      appender.write(int8_t(ival));
      return;
    case 2:
      appender.write((int8_t)BserType::Int16);
      appender.write(int16_t(ival));
      return;
    case 4:
      appender.write((int8_t)BserType::Int32);
      appender.write(int32_t(ival));
      return;
    case 8:
      appender.write((int8_t)BserType::Int64);
      appender.write(ival);
      return;
    default:
      throw std::runtime_error("impossible integer size");
  }
}

static void folly::bser::bserEncodeObject ( dynamic const &  dyn, QueueAppender &  appender, const serialization_opts &  opts  )

static

Definition at line 135 of file Dump.cpp.

References bserEncode(), bserEncodeInt(), int64_t, int8_t, folly::dynamic::items(), Object, folly::dynamic::size(), folly::bser::serialization_opts::sort_keys, and folly::io::QueueAppender::write().

Referenced by bserEncode().

                                    {
  appender.write((int8_t)BserType::Object);
  bserEncodeInt(int64_t(dyn.size()), appender);

  if (opts.sort_keys) {
    std::vector<std::pair<dynamic, dynamic>> sorted(
        dyn.items().begin(), dyn.items().end());
    std::sort(sorted.begin(), sorted.end());
    for (const auto& item : sorted) {
      bserEncode(item.first, appender, opts);
      bserEncode(item.second, appender, opts);
    }
  } else {
    for (const auto& item : dyn.items()) {
      bserEncode(item.first, appender, opts);
      bserEncode(item.second, appender, opts);
    }
  }
}

static void folly::bser::bserEncodeString ( folly::StringPiece  str, QueueAppender &  appender  )

static

Definition at line 81 of file Dump.cpp.

References bserEncodeInt(), folly::Range< Iter >::data(), int64_t, int8_t, folly::io::detail::Writable< Derived >::push(), folly::Range< Iter >::size(), String, uint8_t, and folly::io::QueueAppender::write().

Referenced by bserEncode().

                                                                            {
  appender.write((int8_t)BserType::String);
  bserEncodeInt(int64_t(str.size()), appender);
  appender.push((uint8_t*)str.data(), str.size());
}

static dynamic folly::bser::decodeArray ( Cursor &  curs )

static

Definition at line 83 of file Load.cpp.

References folly::dynamic::array(), decodeInt(), parseBser(), folly::dynamic::push_back(), and folly::size().

Referenced by decodeTemplate(), and parseBser().

                                         {
  dynamic arr = dynamic::array();
  auto size = decodeInt(curs);
  while (size-- > 0) {
    arr.push_back(parseBser(curs));
  }
  return arr;
}

static size_t folly::bser::decodeHeader ( Cursor &  curs )

static

Definition at line 173 of file Load.cpp.

References folly::Range< Iter >::at(), decodeInt(), Int16, Int32, Int64, Int8, kMagic, folly::io::detail::CursorBase< Derived, BufType >::peekBytes(), and folly::io::detail::CursorBase< Derived, BufType >::pull().

Referenced by decodePduLength(), and parseBser().

                                         {
  char header[sizeof(kMagic)];
  curs.pull(header, sizeof(header));
  if (memcmp(header, kMagic, sizeof(kMagic))) {
    throw std::runtime_error("invalid BSER magic header");
  }

  auto enc = (BserType)curs.peekBytes().at(0);
  size_t int_size;
  switch (enc) {
    case BserType::Int8:
      int_size = 1;
      break;
    case BserType::Int16:
      int_size = 2;
      break;
    case BserType::Int32:
      int_size = 4;
      break;
    case BserType::Int64:
      int_size = 8;
      break;
    default:
      int_size = 0;
  }

  return int_size + 3 /* magic + int type */ + decodeInt(curs);
}

static int64_t folly::bser::decodeInt ( Cursor &  curs )

static

Definition at line 38 of file Load.cpp.

References Int16, int16_t, Int32, int32_t, Int64, int64_t, Int8, int8_t, folly::io::detail::CursorBase< Derived, BufType >::read(), and throwDecodeError().

Referenced by decodeArray(), decodeHeader(), decodeObject(), decodeString(), and decodeTemplate().

                                       {
  auto enc = (BserType)curs.read<int8_t>();
  switch (enc) {
    case BserType::Int8:
      return curs.read<int8_t>();
    case BserType::Int16:
      return curs.read<int16_t>();
    case BserType::Int32:
      return curs.read<int32_t>();
    case BserType::Int64:
      return curs.read<int64_t>();
    default:
      throwDecodeError(
          curs, "invalid integer encoding detected (", (int8_t)enc, ")");
  }
}

static dynamic folly::bser::decodeObject ( Cursor &  curs )

static

Definition at line 92 of file Load.cpp.

References decodeInt(), decodeString(), int8_t, folly::dynamic::object(), parseBser(), folly::io::detail::CursorBase< Derived, BufType >::read(), folly::size(), String, and throwDecodeError().

Referenced by parseBser().

                                          {
  dynamic obj = dynamic::object;
  auto size = decodeInt(curs);
  while (size-- > 0) {
    if ((BserType)curs.read<int8_t>() != BserType::String) {
      throwDecodeError(curs, "expected String");
    }
    auto key = decodeString(curs);
    obj[key] = parseBser(curs);
  }
  return obj;
}

size_t folly::bser::decodePduLength

(

const folly::IOBuf *

buf

)

Definition at line 202 of file Load.cpp.

References decodeHeader().

Referenced by TEST().

                                              {
  Cursor curs(buf);
  return decodeHeader(curs);
}

static std::string folly::bser::decodeString ( Cursor &  curs )

static

Definition at line 55 of file Load.cpp.

References decodeInt(), upload::dest, folly::io::detail::CursorBase< Derived, BufType >::pullAtMost(), string, and throwDecodeError().

Referenced by decodeObject(), and parseBser().

                                            {
  auto len = decodeInt(curs);
  std::string str;

  if (len < 0) {
    throw std::range_error("string length must not be negative");
  }

  // We could use Cursor::readFixedString() here, but we'd like
  // to throw our own exception with some increased diagnostics.
  str.resize(len);

  // The start of the string data, mutable.
  auto* dest = &str[0];

  auto pulled = curs.pullAtMost(dest, len);
  if (pulled != size_t(len)) {
    // Saw this case when decodeHeader was returning the incorrect length
    // and we were splitting off too few bytes from the IOBufQueue
    throwDecodeError(
        curs,
        "no data available while decoding a string, header was "
        "not decoded properly");
  }

  return str;
}

static dynamic folly::bser::decodeTemplate ( Cursor &  curs )

static

Definition at line 105 of file Load.cpp.

References Array, folly::dynamic::array(), decodeArray(), decodeInt(), int8_t, folly::gen::move, name, folly::dynamic::object(), parseBser(), folly::io::detail::CursorBase< Derived, BufType >::peekBytes(), folly::dynamic::push_back(), folly::io::detail::CursorBase< Derived, BufType >::read(), folly::size(), Skip, and folly::io::detail::CursorBase< Derived, BufType >::skipAtMost().

Referenced by parseBser().

                                            {
  dynamic arr = folly::dynamic::array;

  // List of property names
  if ((BserType)curs.read<int8_t>() != BserType::Array) {
    throw std::runtime_error("Expected array encoding for property names");
  }
  auto names = decodeArray(curs);

  auto size = decodeInt(curs);

  while (size-- > 0) {
    dynamic obj = dynamic::object;

    for (auto& name : names) {
      auto bytes = curs.peekBytes();
      if ((BserType)bytes.at(0) == BserType::Skip) {
        obj[name.getString()] = nullptr;
        curs.skipAtMost(1);
        continue;
      }

      obj[name.getString()] = parseBser(curs);
    }

    arr.push_back(std::move(obj));
  }

  return arr;
}

static const dynamic* folly::bser::getTemplate ( const serialization_opts &  opts, dynamic const &  dynArray  )

static

Definition at line 38 of file Dump.cpp.

References folly::Optional< Value >::hasValue(), folly::bser::serialization_opts::templates, and folly::Optional< Value >::value().

Referenced by bserEncodeArray().

                             {
  if (!opts.templates.hasValue()) {
    return nullptr;
  }
  const auto& templates = opts.templates.value();
  const auto it = templates.find(&dynArray);
  if (it == templates.end()) {
    return nullptr;
  }
  return &it->second;
}

static dynamic folly::bser::parseBser ( Cursor &  curs )

static

Definition at line 136 of file Load.cpp.

References Array, decodeArray(), decodeObject(), decodeString(), decodeTemplate(), False, Int16, int16_t, Int32, int32_t, Int64, int64_t, Int8, int8_t, Null, Object, folly::io::detail::CursorBase< Derived, BufType >::pull(), folly::io::detail::CursorBase< Derived, BufType >::read(), Real, Skip, String, Template, and True.

                                       {
  switch ((BserType)curs.read<int8_t>()) {
    case BserType::Int8:
      return curs.read<int8_t>();
    case BserType::Int16:
      return curs.read<int16_t>();
    case BserType::Int32:
      return curs.read<int32_t>();
    case BserType::Int64:
      return curs.read<int64_t>();
    case BserType::Real: {
      double dval;
      curs.pull((void*)&dval, sizeof(dval));
      return dval;
    }
    case BserType::Null:
      return nullptr;
    case BserType::True:
      return (bool)true;
    case BserType::False:
      return (bool)false;
    case BserType::String:
      return decodeString(curs);
    case BserType::Array:
      return decodeArray(curs);
    case BserType::Object:
      return decodeObject(curs);
    case BserType::Template:
      return decodeTemplate(curs);
    case BserType::Skip:
      throw std::runtime_error(
          "Skip not valid at this location in the bser stream");
    default:
      throw std::runtime_error("invalid bser encoding");
  }
}

folly::dynamic folly::bser::parseBser

(

folly::StringPiece

str

)

Definition at line 219 of file Load.cpp.

References folly::Range< Iter >::data(), folly::Range< Iter >::size(), and uint8_t.

Referenced by decodeArray(), decodeObject(), decodeTemplate(), parseBser(), and TEST().

                                        {
  return parseBser(ByteRange((uint8_t*)str.data(), str.size()));
}

folly::dynamic folly::bser::parseBser

(

folly::ByteRange

str

)

Definition at line 214 of file Load.cpp.

References folly::Range< Iter >::data(), parseBser(), folly::Range< Iter >::size(), and folly::IOBuf::wrapBuffer().

                                      {
  auto buf = IOBuf::wrapBuffer(str.data(), str.size());
  return parseBser(&*buf);
}

folly::dynamic folly::bser::parseBser

(

const folly::IOBuf *

buf

)

Definition at line 207 of file Load.cpp.

References decodeHeader(), and parseBser().

                                         {
  Cursor curs(buf);

  decodeHeader(curs);
  return parseBser(curs);
}

template<typename... ARGS>

static void folly::bser::throwDecodeError ( Cursor &  curs, ARGS &&...  args  )

static

Definition at line 30 of file Load.cpp.

References folly::io::detail::CursorBase< Derived, BufType >::length().

Referenced by decodeInt(), decodeObject(), and decodeString().

                                                                        {
  throw BserDecodeError(folly::to<std::string>(
      std::forward<ARGS>(args)...,
      " with ",
      curs.length(),
      " bytes remaining in cursor"));
}

fbstring folly::bser::toBser	(	folly::dynamic const &	dyn,
		const serialization_opts &	opts
	)

Definition at line 246 of file Dump.cpp.

References toBserIOBuf().

Referenced by TEST().

                                                                    {
  auto buf = toBserIOBuf(dyn, opts);
  return buf->moveToFbString();
}

std::unique_ptr< folly::IOBuf > folly::bser::toBserIOBuf	(	folly::dynamic const &	dyn,
		const serialization_opts &	opts
	)

Definition at line 191 of file Dump.cpp.

References folly::IOBufQueue::append(), bserEncode(), folly::IOBufQueue::cacheChainLength(), folly::IOBufQueue::chainLength(), folly::IOBuf::create(), folly::bser::serialization_opts::growth_increment, Int16, int16_t, Int32, int32_t, Int64, int64_t, Int8, int8_t, kMagic, max, folly::IOBufQueue::move(), folly::gen::move, folly::IOBufQueue::prepend(), uint64_t, and uint8_t.

Referenced by toBser().

                                    {
  IOBufQueue q(IOBufQueue::cacheChainLength());
  uint8_t hdrbuf[sizeof(kMagic) + 1 + sizeof(int64_t)];

  // Reserve some headroom for the overall PDU size; we'll fill this in
  // after we've serialized the data and know the length
  auto firstbuf = IOBuf::create(opts.growth_increment);
  firstbuf->advance(sizeof(hdrbuf));
  q.append(std::move(firstbuf));

  // encode the value
  QueueAppender appender(&q, opts.growth_increment);
  bserEncode(dyn, appender, opts);

  // compute the length
  auto len = q.chainLength();
  if (len > uint64_t(std::numeric_limits<int64_t>::max())) {
    throw std::range_error(folly::to<std::string>(
        "serialized data size ", len, " is too large to represent as BSER"));
  }

  // This is a bit verbose, but it computes a header that is appropriate
  // to the size of the serialized data

  memcpy(hdrbuf, kMagic, sizeof(kMagic));
  size_t hdrlen = sizeof(kMagic) + 1;
  auto magicptr = hdrbuf + sizeof(kMagic);
  auto lenptr = hdrbuf + hdrlen;

  if (len > uint64_t(std::numeric_limits<int32_t>::max())) {
    *magicptr = (int8_t)BserType::Int64;
    *(int64_t*)lenptr = (int64_t)len;
    hdrlen += sizeof(int64_t);
  } else if (len > uint64_t(std::numeric_limits<int16_t>::max())) {
    *magicptr = (int8_t)BserType::Int32;
    *(int32_t*)lenptr = (int32_t)len;
    hdrlen += sizeof(int32_t);
  } else if (len > uint64_t(std::numeric_limits<int8_t>::max())) {
    *magicptr = (int8_t)BserType::Int16;
    *(int16_t*)lenptr = (int16_t)len;
    hdrlen += sizeof(int16_t);
  } else {
    *magicptr = (int8_t)BserType::Int8;
    *(int8_t*)lenptr = (int8_t)len;
    hdrlen += sizeof(int8_t);
  }

  // and place the data in the headroom
  q.prepend(hdrbuf, hdrlen);

  return q.move();
}

Variable Documentation

const uint8_t folly::bser::kMagic = {0, 1}

Definition at line 28 of file Dump.cpp.

Referenced by decodeHeader(), folly::recordio_helpers::findRecord(), folly::recordio_helpers::prependHeader(), toBserIOBuf(), and folly::recordio_helpers::validateRecord().