folly::json Namespace Reference

Classes
struct	serialization_opts

Functions
std::array< uint64_t, 2 >	buildExtraAsciiToEscapeBitmap (StringPiece chars)
std::string	serialize (dynamic const &dyn, serialization_opts const &opts)
template<bool EnableExtraAsciiEscapes, class T >
size_t	firstEscapableInWord (T s, const serialization_opts &opts)
template<bool EnableExtraAsciiEscapes>
void	escapeStringImpl (StringPiece input, std::string &out, const serialization_opts &opts)
void	escapeString (StringPiece input, std::string &out, const serialization_opts &opts)
std::string	stripComments (StringPiece jsonC)

Function Documentation

std::array< uint64_t, 2 > folly::json::buildExtraAsciiToEscapeBitmap

(

StringPiece

chars

)

Definition at line 611 of file json.cpp.

References b, and uint64_t.

Referenced by BENCHMARK(), and TEST().

                                                                       {
  std::array<uint64_t, 2> escapes{{0, 0}};
  for (auto b : ByteRange(chars)) {
    if (b >= 0x20 && b < 0x80) {
      escapes[b / 64] |= uint64_t(1) << (b % 64);
    }
  }
  return escapes;
}

void folly::json::escapeString	(	StringPiece	input,
		std::string &	out,
		const serialization_opts &	opts
	)

Definition at line 824 of file json.cpp.

References folly::json::serialization_opts::extra_ascii_to_escape_bitmap, and FOLLY_UNLIKELY.

                                    {
  if (FOLLY_UNLIKELY(
          opts.extra_ascii_to_escape_bitmap[0] ||
          opts.extra_ascii_to_escape_bitmap[1])) {
    escapeStringImpl<true>(input, out, opts);
  } else {
    escapeStringImpl<false>(input, out, opts);
  }
}

template<bool EnableExtraAsciiEscapes>

void folly::json::escapeStringImpl	(	StringPiece	input,
		std::string &	out,
		const serialization_opts &	opts
	)

Definition at line 688 of file json.cpp.

References folly::Range< Iter >::begin(), c, folly::json::serialization_opts::encode_non_ascii, folly::Range< Iter >::end(), folly::json::serialization_opts::extra_ascii_to_escape_bitmap, prefix(), folly::json::serialization_opts::skip_invalid_utf8, uint64_t, uint8_t, folly::utf8ToCodePoint(), v, and folly::json::serialization_opts::validate_utf8.

                                    {
  auto hexDigit = [](uint8_t c) -> char {
    return c < 10 ? c + '0' : c - 10 + 'a';
  };

  out.push_back('\"');

  auto* p = reinterpret_cast<const unsigned char*>(input.begin());
  auto* q = reinterpret_cast<const unsigned char*>(input.begin());
  auto* e = reinterpret_cast<const unsigned char*>(input.end());

  while (p < e) {
    // Find the longest prefix that does not need escaping, and copy
    // it literally into the output string.
    auto firstEsc = p;
    while (firstEsc < e) {
      auto avail = e - firstEsc;
      uint64_t word = 0;
      if (avail >= 8) {
        word = folly::loadUnaligned<uint64_t>(firstEsc);
      } else {
        word = folly::partialLoadUnaligned<uint64_t>(firstEsc, avail);
      }
      auto prefix = firstEscapableInWord<EnableExtraAsciiEscapes>(word, opts);
      DCHECK_LE(prefix, avail);
      firstEsc += prefix;
      if (prefix < 8) {
        break;
      }
    }
    if (firstEsc > p) {
      out.append(reinterpret_cast<const char*>(p), firstEsc - p);
      p = firstEsc;
      // We can't be in the middle of a multibyte sequence, so we can reset q.
      q = p;
      if (p == e) {
        break;
      }
    }

    // Handle the next byte that may need escaping.

    // Since non-ascii encoding inherently does utf8 validation
    // we explicitly validate utf8 only if non-ascii encoding is disabled.
    if ((opts.validate_utf8 || opts.skip_invalid_utf8) &&
        !opts.encode_non_ascii) {
      // To achieve better spatial and temporal coherence
      // we do utf8 validation progressively along with the
      // string-escaping instead of two separate passes.

      // As the encoding progresses, q will stay at or ahead of p.
      CHECK_GE(q, p);

      // As p catches up with q, move q forward.
      if (q == p) {
        // calling utf8_decode has the side effect of
        // checking that utf8 encodings are valid
        char32_t v = utf8ToCodePoint(q, e, opts.skip_invalid_utf8);
        if (opts.skip_invalid_utf8 && v == U'\ufffd') {
          out.append(u8"\ufffd");
          p = q;
          continue;
        }
      }
    }

    auto encodeUnicode = opts.encode_non_ascii && (*p & 0x80);
    if /* constexpr */ (EnableExtraAsciiEscapes) {
      encodeUnicode = encodeUnicode ||
          (*p >= 0x20 && *p < 0x80 &&
           (opts.extra_ascii_to_escape_bitmap[*p / 64] &
            (uint64_t(1) << (*p % 64))));
    }

    if (encodeUnicode) {
      // note that this if condition captures utf8 chars
      // with value > 127, so size > 1 byte (or they are whitelisted for
      // Unicode encoding).
      // NOTE: char32_t / char16_t are both unsigned.
      char32_t cp = utf8ToCodePoint(p, e, opts.skip_invalid_utf8);
      auto writeHex = [&](char16_t v) {
        char buf[] = "\\u\0\0\0\0";
        buf[2] = hexDigit((v >> 12) & 0x0f);
        buf[3] = hexDigit((v >> 8) & 0x0f);
        buf[4] = hexDigit((v >> 4) & 0x0f);
        buf[5] = hexDigit(v & 0x0f);
        out.append(buf, 6);
      };
      // From the ECMA-404 The JSON Data Interchange Syntax 2nd Edition Dec 2017
      if (cp < 0x10000u) {
        // If the code point is in the Basic Multilingual Plane (U+0000 through
        // U+FFFF), then it may be represented as a six-character sequence:
        // a reverse solidus, followed by the lowercase letter u, followed by
        // four hexadecimal digits that encode the code point.
        writeHex(static_cast<char16_t>(cp));
      } else {
        // To escape a code point that is not in the Basic Multilingual Plane,
        // the character may be represented as a twelve-character sequence,
        // encoding the UTF-16 surrogate pair corresponding to the code point.
        writeHex(static_cast<char16_t>(
            0xd800u + (((cp - 0x10000u) >> 10) & 0x3ffu)));
        writeHex(static_cast<char16_t>(0xdc00u + ((cp - 0x10000u) & 0x3ffu)));
      }
    } else if (*p == '\\' || *p == '\"') {
      char buf[] = "\\\0";
      buf[1] = char(*p++);
      out.append(buf, 2);
    } else if (*p <= 0x1f) {
      switch (*p) {
        // clang-format off
        case '\b': out.append("\\b"); p++; break;
        case '\f': out.append("\\f"); p++; break;
        case '\n': out.append("\\n"); p++; break;
        case '\r': out.append("\\r"); p++; break;
        case '\t': out.append("\\t"); p++; break;
        // clang-format on
        default:
          // Note that this if condition captures non readable chars
          // with value < 32, so size = 1 byte (e.g control chars).
          char buf[] = "\\u00\0\0";
          buf[4] = hexDigit(uint8_t((*p & 0xf0) >> 4));
          buf[5] = hexDigit(uint8_t(*p & 0xf));
          out.append(buf, 6);
          p++;
      }
    } else {
      out.push_back(char(*p++));
    }
  }

  out.push_back('\"');
}

template<bool EnableExtraAsciiEscapes, class T >

size_t folly::json::firstEscapableInWord	(	T	s,
		const serialization_opts &	opts
	)

Definition at line 633 of file json.cpp.

References b, c, folly::json::serialization_opts::extra_ascii_to_escape_bitmap, folly::findFirstSet(), folly::findLastSet(), i, folly::kIsLittleEndian, folly::T, uint64_t, uint8_t, and value.

                                                                 {
  static_assert(std::is_unsigned<T>::value, "Unsigned integer required");
  static constexpr T kOnes = ~T() / 255; // 0x...0101
  static constexpr T kMsbs = kOnes * 0x80; // 0x...8080

  // Sets the MSB of bytes < b. Precondition: b < 128.
  auto isLess = [](T w, uint8_t b) {
    // A byte is < b iff subtracting b underflows, so we check that
    // the MSB wasn't set before and it's set after the subtraction.
    return (w - kOnes * b) & ~w & kMsbs;
  };

  auto isChar = [&](uint8_t c) {
    // A byte is == c iff it is 0 if xored with c.
    return isLess(s ^ (kOnes * c), 1);
  };

  // The following masks have the MSB set for each byte of the word
  // that satisfies the corresponding condition.
  auto isHigh = s & kMsbs; // >= 128
  auto isLow = isLess(s, 0x20); // <= 0x1f
  auto needsEscape = isHigh | isLow | isChar('\\') | isChar('"');

  if /* constexpr */ (EnableExtraAsciiEscapes) {
    // Deal with optional bitmap for unicode escapes. Escapes can optionally be
    // set for ascii characters 32 - 127, so the inner loop may run up to 96
    // times. However, for the case where 0 or a handful of bits are set,
    // looping will be minimal through use of findFirstSet.
    for (size_t i = 0; i < opts.extra_ascii_to_escape_bitmap.size(); ++i) {
      const auto offset = i * 64;
      // Clear first 32 characters if this is the first index, since those are
      // always escaped.
      auto bitmap = opts.extra_ascii_to_escape_bitmap[i] &
          (i == 0 ? uint64_t(-1) << 32 : ~0UL);
      while (bitmap) {
        auto bit = folly::findFirstSet(bitmap);
        needsEscape |= isChar(offset + bit - 1);
        bitmap &= bitmap - 1;
      }
    }
  }

  if (!needsEscape) {
    return sizeof(T);
  }

  if (folly::kIsLittleEndian) {
    return folly::findFirstSet(needsEscape) / 8 - 1;
  } else {
    return sizeof(T) - folly::findLastSet(needsEscape) / 8;
  }
}

std::string folly::json::serialize	(	dynamic const &	dyn,
		serialization_opts const &	opts
	)

Definition at line 621 of file json.cpp.

References folly::json::serialization_opts::pretty_formatting, and string.

Referenced by BENCHMARK(), wangle::FilePersistenceLayer< K, V >::persist(), folly::dynamic::print_as_pseudo_json(), folly::PrintTo(), TEST(), folly::toJson(), folly::toPrettyJson(), ClientSerializeHandler::write(), and ServerSerializeHandler::write().

                                                                        {
  std::string ret;
  unsigned indentLevel = 0;
  Printer p(ret, opts.pretty_formatting ? &indentLevel : nullptr, &opts);
  p(dyn);
  return ret;
}

std::string folly::json::stripComments

(

StringPiece

jsonC

)

Definition at line 837 of file json.cpp.

References i, s, folly::Range< Iter >::size(), string, folly::Range< Iter >::subpiece(), and UNLIKELY.

Referenced by folly::parseLogConfigJson(), and TEST().

                                           {
  std::string result;
  enum class State {
    None,
    InString,
    InlineComment,
    LineComment
  } state = State::None;

  for (size_t i = 0; i < jsonC.size(); ++i) {
    auto s = jsonC.subpiece(i);
    switch (state) {
      case State::None:
        if (s.startsWith("/*")) {
          state = State::InlineComment;
          ++i;
          continue;
        } else if (s.startsWith("//")) {
          state = State::LineComment;
          ++i;
          continue;
        } else if (s[0] == '\"') {
          state = State::InString;
        }
        result.push_back(s[0]);
        break;
      case State::InString:
        if (s[0] == '\\') {
          if (UNLIKELY(s.size() == 1)) {
            throw std::logic_error("Invalid JSONC: string is not terminated");
          }
          result.push_back(s[0]);
          result.push_back(s[1]);
          ++i;
          continue;
        } else if (s[0] == '\"') {
          state = State::None;
        }
        result.push_back(s[0]);
        break;
      case State::InlineComment:
        if (s.startsWith("*/")) {
          state = State::None;
          ++i;
        }
        break;
      case State::LineComment:
        if (s[0] == '\n') {
          // skip the line break. It doesn't matter.
          state = State::None;
        }
        break;
      default:
        throw std::logic_error("Unknown comment state");
    }
  }
  return result;
}