folly::recordio_helpers Namespace Reference

Namespaces
	recordio_detail

Classes
struct	RecordInfo

Functions
constexpr size_t	headerSize ()
RecordInfo	findRecord (ByteRange range, uint32_t fileId)
size_t	prependHeader (std::unique_ptr< IOBuf > &buf, uint32_t fileId)
RecordInfo	validateRecord (ByteRange range, uint32_t fileId)
RecordInfo	findRecord (ByteRange searchRange, ByteRange wholeRange, uint32_t fileId)

Function Documentation

RecordInfo folly::recordio_helpers::findRecord ( ByteRange  range, uint32_t  fileId  )

inline

Search for the first valid record in range.

Definition at line 105 of file RecordIO-inl.h.

                                                               {
  return findRecord(range, range, fileId);
}

RecordInfo folly::recordio_helpers::findRecord	(	ByteRange	searchRange,
		ByteRange	wholeRange,
		uint32_t	fileId
	)

Definition at line 193 of file RecordIO.cpp.

References folly::Range< Iter >::begin(), folly::RecordIOReader::end(), folly::Range< Iter >::end(), folly::Range< Iter >::find(), folly::bser::kMagic, min, folly::Range< const unsigned char * >::npos, start, uint32_t, uint8_t, and validateRecord().

Referenced by folly::RecordIOReader::Iterator::advanceToValid().

                                                                         {
  static const uint32_t magic = Header::kMagic;
  static const ByteRange magicRange(
      reinterpret_cast<const uint8_t*>(&magic), sizeof(magic));

  DCHECK_GE(searchRange.begin(), wholeRange.begin());
  DCHECK_LE(searchRange.end(), wholeRange.end());

  const uint8_t* start = searchRange.begin();
  const uint8_t* end =
      std::min(searchRange.end(), wholeRange.end() - sizeof(Header));
  // end-1: the last place where a Header could start
  while (start < end) {
    auto p = ByteRange(start, end + sizeof(magic)).find(magicRange);
    if (p == ByteRange::npos) {
      break;
    }

    start += p;
    auto r = validateRecord(ByteRange(start, wholeRange.end()), fileId);
    if (!r.record.empty()) {
      return r;
    }

    // No repeated prefix in magic, so we can do better than start++
    start += sizeof(magic);
  }

  return {0, {}};
}

constexpr size_t folly::recordio_helpers::headerSize

(

)

Header size.

Definition at line 101 of file RecordIO-inl.h.

Referenced by folly::RecordIOReader::Iterator::advanceToValid(), getTracepointArguments(), folly::RecordIOReader::Iterator::increment(), HTTP1xCodecCallback::onHeadersComplete(), prependHeader(), folly::test::TEST(), and validateRecord().

                              {
  return sizeof(recordio_detail::Header);
}

size_t folly::recordio_helpers::prependHeader	(	std::unique_ptr< IOBuf > &	buf,
		uint32_t	fileId = 1
	)

Write a header in the buffer. We will prepend the header to the front of the chain. Do not write the buffer if empty (we don't allow empty records). Returns the total length, including header (0 if empty) (same as buf->computeChainDataLength(), but likely faster)

The fileId should be unique per stream and allows you to have RecordIO headers stored inside the data (for example, have an entire RecordIO file stored as a record inside another RecordIO file). The fileId may not be 0.

Definition at line 139 of file RecordIO.cpp.

References b, folly::IOBuf::create(), headerSize(), folly::bser::kMagic, folly::gen::move, and uint32_t.

Referenced by folly::RecordIOWriter::write().

                                                                 {
  if (fileId == 0) {
    throw std::invalid_argument("invalid file id");
  }
  auto lengthAndHash = dataLengthAndHash(buf.get());
  if (lengthAndHash.first == 0) {
    return 0; // empty, nothing to do, no zero-length records
  }

  // Prepend to the first buffer in the chain if we have room, otherwise
  // prepend a new buffer.
  if (buf->headroom() >= headerSize()) {
    buf->unshareOne();
    buf->prepend(headerSize());
  } else {
    auto b = IOBuf::create(headerSize());
    b->append(headerSize());
    b->appendChain(std::move(buf));
    buf = std::move(b);
  }
  Header* header = reinterpret_cast<Header*>(buf->writableData());
  memset(header, 0, sizeof(Header));
  header->magic = Header::kMagic;
  header->fileId = fileId;
  header->dataLength = uint32_t(lengthAndHash.first);
  header->dataHash = lengthAndHash.second;
  header->headerHash = headerHash(*header);

  return lengthAndHash.first + headerSize();
}

RecordInfo folly::recordio_helpers::validateRecord	(	ByteRange	range,
		uint32_t	fileId
	)

Check if there is a valid record at the beginning of range. Returns the record data (not the header) if the record is valid, ByteRange() otherwise.

Definition at line 170 of file RecordIO.cpp.

References folly::Range< Iter >::advance(), folly::Range< Iter >::begin(), headerSize(), folly::bser::kMagic, folly::Range< Iter >::reset(), and folly::Range< Iter >::size().

Referenced by findRecord().

                                                            {
  if (range.size() <= headerSize()) { // records may not be empty
    return {0, {}};
  }
  const Header* header = reinterpret_cast<const Header*>(range.begin());
  range.advance(sizeof(Header));
  if (header->magic != Header::kMagic || header->version != 0 ||
      header->hashFunction != 0 || header->flags != 0 ||
      (fileId != 0 && header->fileId != fileId) ||
      header->dataLength > range.size()) {
    return {0, {}};
  }
  if (headerHash(*header) != header->headerHash) {
    return {0, {}};
  }
  range.reset(range.begin(), header->dataLength);
  if (dataHash(range) != header->dataHash) {
    return {0, {}};
  }
  return {header->fileId, range};
}