folly::io::Codec Class Reference

#include <Compression.h>

Inheritance diagram for folly::io::Codec:

Public Member Functions
virtual	~Codec ()
uint64_t	maxUncompressedLength () const
CodecType	type () const
bool	needsUncompressedLength () const
std::unique_ptr< IOBuf >	compress (const folly::IOBuf *data)
std::string	compress (StringPiece data)
std::unique_ptr< IOBuf >	uncompress (const IOBuf *data, folly::Optional< uint64_t > uncompressedLength=folly::none)
std::string	uncompress (StringPiece data, folly::Optional< uint64_t > uncompressedLength=folly::none)
uint64_t	maxCompressedLength (uint64_t uncompressedLength) const
folly::Optional< uint64_t >	getUncompressedLength (const folly::IOBuf *data, folly::Optional< uint64_t > uncompressedLength=folly::none) const
virtual std::vector< std::string >	validPrefixes () const
virtual bool	canUncompress (const folly::IOBuf *data, folly::Optional< uint64_t > uncompressedLength=folly::none) const

Static Public Attributes
static constexpr uint64_t	UNLIMITED_UNCOMPRESSED_LENGTH = uint64_t(-1)

Protected Member Functions
	Codec (CodecType type, folly::Optional< int > level=folly::none, folly::StringPiece name={}, bool counters=true)

Private Member Functions
virtual uint64_t	doMaxUncompressedLength () const
virtual bool	doNeedsUncompressedLength () const
virtual std::unique_ptr< IOBuf >	doCompress (const folly::IOBuf *data)=0
virtual std::unique_ptr< IOBuf >	doUncompress (const folly::IOBuf *data, folly::Optional< uint64_t > uncompressedLength)=0
virtual std::string	doCompressString (StringPiece data)
virtual std::string	doUncompressString (StringPiece data, folly::Optional< uint64_t > uncompressedLength)
virtual uint64_t	doMaxCompressedLength (uint64_t uncompressedLength) const =0
virtual folly::Optional< uint64_t >	doGetUncompressedLength (const folly::IOBuf *data, folly::Optional< uint64_t > uncompressedLength) const

Private Attributes
CodecType	type_
folly::detail::CompressionCounter	bytesBeforeCompression_
folly::detail::CompressionCounter	bytesAfterCompression_
folly::detail::CompressionCounter	bytesBeforeDecompression_
folly::detail::CompressionCounter	bytesAfterDecompression_
folly::detail::CompressionCounter	compressions_
folly::detail::CompressionCounter	decompressions_
folly::detail::CompressionCounter	compressionMilliseconds_
folly::detail::CompressionCounter	decompressionMilliseconds_

Detailed Description

Definition at line 131 of file Compression.h.

Constructor & Destructor Documentation

virtual folly::io::Codec::~Codec ( )

inlinevirtual

Definition at line 133 of file Compression.h.

{}

folly::io::Codec::Codec ( CodecType  type, folly::Optional< int >  level = folly::none, folly::StringPiece  name = {}, bool  counters = true  )

protected

Definition at line 71 of file Compression.cpp.

References folly::BYTES_AFTER_COMPRESSION, folly::BYTES_AFTER_DECOMPRESSION, folly::BYTES_BEFORE_COMPRESSION, folly::BYTES_BEFORE_DECOMPRESSION, bytesAfterCompression_, bytesAfterDecompression_, bytesBeforeCompression_, bytesBeforeDecompression_, folly::COMPRESSION_MILLISECONDS, compressionMilliseconds_, folly::COMPRESSIONS, compressions_, counter, counter_, folly::DECOMPRESSION_MILLISECONDS, decompressionMilliseconds_, folly::DECOMPRESSIONS, decompressions_, name, folly::SUM, timer_, type(), and uint32_t.

Referenced by folly::io::StreamCodec::doUncompress().

    : type_(type) {
  if (counters) {
    bytesBeforeCompression_ = {type,
                               name,
                               level,
                               CompressionCounterKey::BYTES_BEFORE_COMPRESSION,
                               CompressionCounterType::SUM};
    bytesAfterCompression_ = {type,
                              name,
                              level,
                              CompressionCounterKey::BYTES_AFTER_COMPRESSION,
                              CompressionCounterType::SUM};
    bytesBeforeDecompression_ = {
        type,
        name,
        level,
        CompressionCounterKey::BYTES_BEFORE_DECOMPRESSION,
        CompressionCounterType::SUM};
    bytesAfterDecompression_ = {
        type,
        name,
        level,
        CompressionCounterKey::BYTES_AFTER_DECOMPRESSION,
        CompressionCounterType::SUM};
    compressions_ = {type,
                     name,
                     level,
                     CompressionCounterKey::COMPRESSIONS,
                     CompressionCounterType::SUM};
    decompressions_ = {type,
                       name,
                       level,
                       CompressionCounterKey::DECOMPRESSIONS,
                       CompressionCounterType::SUM};
    compressionMilliseconds_ = {type,
                                name,
                                level,
                                CompressionCounterKey::COMPRESSION_MILLISECONDS,
                                CompressionCounterType::SUM};
    decompressionMilliseconds_ = {
        type,
        name,
        level,
        CompressionCounterKey::DECOMPRESSION_MILLISECONDS,
        CompressionCounterType::SUM};
  }
}

Member Function Documentation

bool folly::io::Codec::canUncompress ( const folly::IOBuf *  data, folly::Optional< uint64_t >  uncompressedLength = folly::none  ) const

virtual

Returns true if the codec thinks it can uncompress the data. If a codec doesn't have magic bytes at the beginning, like LZ4 and Snappy, it can always return false. default: Returns false.

Definition at line 263 of file Compression.cpp.

Referenced by folly::io::StreamCodec::doUncompress().

                                                                {
  return false;
}

std::unique_ptr< IOBuf > folly::io::Codec::compress

(

const folly::IOBuf *

data

)

Compress data, returning an IOBuf (which may share storage with data). Throws std::invalid_argument if data is larger than maxUncompressedLength().

Definition at line 143 of file Compression.cpp.

References bytesAfterCompression_, bytesBeforeCompression_, compressionMilliseconds_, compressions_, folly::IOBuf::computeChainDataLength(), doCompress(), maxUncompressedLength(), folly::Random::oneIn(), and uint64_t.

                                                      {
  if (data == nullptr) {
    throw std::invalid_argument("Codec: data must not be nullptr");
  }
  const uint64_t len = data->computeChainDataLength();
  if (len > maxUncompressedLength()) {
    throw std::runtime_error("Codec: uncompressed length too large");
  }
  bool const logging = folly::Random::oneIn(kLoggingRate);
  folly::Optional<Timer> const timer =
      logging ? Timer(compressionMilliseconds_) : folly::Optional<Timer>();
  auto result = doCompress(data);
  if (logging) {
    compressions_++;
    bytesBeforeCompression_ += len;
    bytesAfterCompression_ += result->computeChainDataLength();
  }
  return result;
}

std::string folly::io::Codec::compress

(

StringPiece

data

)

Compresses data. May involve additional copies compared to the overload that takes and returns IOBufs. Has the same error semantics as the IOBuf version.

Definition at line 163 of file Compression.cpp.

References bytesAfterCompression_, bytesBeforeCompression_, compressionMilliseconds_, compressions_, doCompressString(), maxUncompressedLength(), folly::Random::oneIn(), folly::Range< Iter >::size(), and uint64_t.

                                                {
  const uint64_t len = data.size();
  if (len > maxUncompressedLength()) {
    throw std::runtime_error("Codec: uncompressed length too large");
  }
  bool const logging = folly::Random::oneIn(kLoggingRate);
  folly::Optional<Timer> const timer =
      logging ? Timer(compressionMilliseconds_) : folly::Optional<Timer>();
  auto result = doCompressString(data);
  if (logging) {
    compressions_++;
    bytesBeforeCompression_ += len;
    bytesAfterCompression_ += result.size();
  }
  return result;
}

virtual std::unique_ptr<IOBuf> folly::io::Codec::doCompress ( const folly::IOBuf *  data )

privatepure virtual

Implemented in folly::io::StreamCodec.

Referenced by compress(), doCompressString(), and folly::io::StreamCodec::doUncompress().

std::string folly::io::Codec::doCompressString ( StringPiece  data )

privatevirtual

Definition at line 267 of file Compression.cpp.

References doCompress(), gmock_output_test::output, folly::range(), string, and folly::IOBuf::WRAP_BUFFER.

Referenced by compress().

                                                        {
  const IOBuf inputBuffer{IOBuf::WRAP_BUFFER, data};
  auto outputBuffer = doCompress(&inputBuffer);
  std::string output;
  output.reserve(outputBuffer->computeChainDataLength());
  for (auto range : *outputBuffer) {
    output.append(reinterpret_cast<const char*>(range.data()), range.size());
  }
  return output;
}

Optional< uint64_t > folly::io::Codec::doGetUncompressedLength ( const folly::IOBuf *  data, folly::Optional< uint64_t >  uncompressedLength  ) const

privatevirtual

Definition at line 308 of file Compression.cpp.

Referenced by getUncompressedLength().

                                                 {
  return uncompressedLength;
}

virtual uint64_t folly::io::Codec::doMaxCompressedLength ( uint64_t  uncompressedLength ) const

privatepure virtual

Referenced by folly::io::StreamCodec::doUncompress(), and maxCompressedLength().

uint64_t folly::io::Codec::doMaxUncompressedLength ( ) const

privatevirtual

Definition at line 255 of file Compression.cpp.

References UNLIMITED_UNCOMPRESSED_LENGTH.

Referenced by folly::io::StreamCodec::doUncompress(), and maxUncompressedLength().

                                              {
  return UNLIMITED_UNCOMPRESSED_LENGTH;
}

bool folly::io::Codec::doNeedsUncompressedLength ( ) const

privatevirtual

Definition at line 251 of file Compression.cpp.

Referenced by folly::io::StreamCodec::doUncompress(), and needsUncompressedLength().

                                            {
  return false;
}

virtual std::unique_ptr<IOBuf> folly::io::Codec::doUncompress ( const folly::IOBuf *  data, folly::Optional< uint64_t >  uncompressedLength  )

privatepure virtual

Implemented in folly::io::StreamCodec.

Referenced by folly::io::StreamCodec::doUncompress(), doUncompressString(), and uncompress().

std::string folly::io::Codec::doUncompressString ( StringPiece  data, folly::Optional< uint64_t >  uncompressedLength  )

privatevirtual

Definition at line 278 of file Compression.cpp.

References doUncompress(), gmock_output_test::output, folly::range(), string, and folly::IOBuf::WRAP_BUFFER.

Referenced by uncompress().

                                           {
  const IOBuf inputBuffer{IOBuf::WRAP_BUFFER, data};
  auto outputBuffer = doUncompress(&inputBuffer, uncompressedLength);
  std::string output;
  output.reserve(outputBuffer->computeChainDataLength());
  for (auto range : *outputBuffer) {
    output.append(reinterpret_cast<const char*>(range.data()), range.size());
  }
  return output;
}

Optional< uint64_t > folly::io::Codec::getUncompressedLength	(	const folly::IOBuf *	data,
		folly::Optional< uint64_t >	uncompressedLength = folly::none
	)		const

Extracts the uncompressed length from the compressed data if possible. If the codec doesn't store the uncompressed length, or the data is corrupted it returns the given uncompressedLength. If the uncompressed length is stored in the compressed data and uncompressedLength is not none and they do not match a std::runtime_error is thrown.

Definition at line 295 of file Compression.cpp.

References folly::IOBuf::computeChainDataLength(), doGetUncompressedLength(), and folly::Optional< Value >::value_or().

Referenced by folly::io::StreamCodec::doUncompress().

                                                 {
  auto const compressedLength = data->computeChainDataLength();
  if (compressedLength == 0) {
    if (uncompressedLength.value_or(0) != 0) {
      throw std::runtime_error("Invalid uncompressed length");
    }
    return 0;
  }
  return doGetUncompressedLength(data, uncompressedLength);
}

uint64_t folly::io::Codec::maxCompressedLength

(

uint64_t

uncompressedLength

)

const

Returns a bound on the maximum compressed length when compressing data with the given uncompressed length.

Definition at line 291 of file Compression.cpp.

References doMaxCompressedLength().

Referenced by folly::io::StreamCodec::doCompress(), and folly::io::StreamCodec::doUncompress().

                                                                     {
  return doMaxCompressedLength(uncompressedLength);
}

uint64_t folly::io::Codec::maxUncompressedLength

(

)

const

Return the maximum length of data that may be compressed with this codec. NO_COMPRESSION and ZLIB support arbitrary lengths; LZ4 supports up to 1.9GiB; SNAPPY supports up to 4GiB. May return UNLIMITED_UNCOMPRESSED_LENGTH if unlimited.

Definition at line 247 of file Compression.cpp.

References doMaxUncompressedLength().

Referenced by compress(), folly::io::StreamCodec::doUncompress(), and uncompress().

                                            {
  return doMaxUncompressedLength();
}

bool folly::io::Codec::needsUncompressedLength

(

)

const

Does this codec need the exact uncompressed length on decompression?

Definition at line 243 of file Compression.cpp.

References doNeedsUncompressedLength().

Referenced by uncompress().

                                          {
  return doNeedsUncompressedLength();
}

CodecType folly::io::Codec::type ( ) const

inline

Return the codec's type.

Definition at line 147 of file Compression.h.

References folly::data(), name, folly::none, string, type, type_, and uint64_t.

Referenced by Codec(), folly::io::StreamCodec::doUncompress(), folly::io::getCodec(), and folly::io::getStreamCodec().

                         {
    return type_;
  }

std::unique_ptr< IOBuf > folly::io::Codec::uncompress	(	const IOBuf *	data,
		folly::Optional< uint64_t >	uncompressedLength = folly::none
	)

Uncompress data. Throws std::runtime_error on decompression error.

Some codecs (LZ4) require the exact uncompressed length; this is indicated by needsUncompressedLength().

For other codes (zlib), knowing the exact uncompressed length ahead of time might be faster.

Regardless of the behavior of the underlying compressor, uncompressing an empty IOBuf chain will return an empty IOBuf chain.

Definition at line 180 of file Compression.cpp.

References bytesAfterDecompression_, bytesBeforeDecompression_, folly::IOBuf::computeChainDataLength(), folly::IOBuf::create(), decompressionMilliseconds_, decompressions_, doUncompress(), folly::IOBuf::empty(), maxUncompressedLength(), needsUncompressedLength(), folly::Random::oneIn(), and folly::Optional< Value >::value_or().

                                           {
  if (data == nullptr) {
    throw std::invalid_argument("Codec: data must not be nullptr");
  }
  if (!uncompressedLength) {
    if (needsUncompressedLength()) {
      throw std::invalid_argument("Codec: uncompressed length required");
    }
  } else if (*uncompressedLength > maxUncompressedLength()) {
    throw std::runtime_error("Codec: uncompressed length too large");
  }

  if (data->empty()) {
    if (uncompressedLength.value_or(0) != 0) {
      throw std::runtime_error("Codec: invalid uncompressed length");
    }
    return IOBuf::create(0);
  }

  bool const logging = folly::Random::oneIn(kLoggingRate);
  folly::Optional<Timer> const timer =
      logging ? Timer(decompressionMilliseconds_) : folly::Optional<Timer>();
  auto result = doUncompress(data, uncompressedLength);
  if (logging) {
    decompressions_++;
    bytesBeforeDecompression_ += data->computeChainDataLength();
    bytesAfterDecompression_ += result->computeChainDataLength();
  }
  return result;
}

std::string folly::io::Codec::uncompress	(	StringPiece	data,
		folly::Optional< uint64_t >	uncompressedLength = folly::none
	)

Uncompresses data. May involve additional copies compared to the overload that takes and returns IOBufs. Has the same error semantics as the IOBuf version.

Definition at line 213 of file Compression.cpp.

References bytesAfterDecompression_, bytesBeforeDecompression_, decompressionMilliseconds_, decompressions_, doUncompressString(), folly::Range< Iter >::empty(), maxUncompressedLength(), needsUncompressedLength(), folly::Random::oneIn(), folly::Range< Iter >::size(), and folly::Optional< Value >::value_or().

                                           {
  if (!uncompressedLength) {
    if (needsUncompressedLength()) {
      throw std::invalid_argument("Codec: uncompressed length required");
    }
  } else if (*uncompressedLength > maxUncompressedLength()) {
    throw std::runtime_error("Codec: uncompressed length too large");
  }

  if (data.empty()) {
    if (uncompressedLength.value_or(0) != 0) {
      throw std::runtime_error("Codec: invalid uncompressed length");
    }
    return "";
  }

  bool const logging = folly::Random::oneIn(kLoggingRate);
  folly::Optional<Timer> const timer =
      logging ? Timer(decompressionMilliseconds_) : folly::Optional<Timer>();
  auto result = doUncompressString(data, uncompressedLength);
  if (logging) {
    decompressions_++;
    bytesBeforeDecompression_ += data.size();
    bytesAfterDecompression_ += result.size();
  }
  return result;
}

std::vector< std::string > folly::io::Codec::validPrefixes ( ) const

virtual

Returns a superset of the set of prefixes for which canUncompress() will return true. A superset is allowed for optimizations in canUncompress() based on other knowledge such as length. None of the prefixes may be empty. default: No prefixes.

Definition at line 259 of file Compression.cpp.

Referenced by folly::io::StreamCodec::doUncompress().

                                                {
  return {};
}

Member Data Documentation

folly::detail::CompressionCounter folly::io::Codec::bytesAfterCompression_

private

Definition at line 265 of file Compression.h.

Referenced by Codec(), and compress().

folly::detail::CompressionCounter folly::io::Codec::bytesAfterDecompression_

private

Definition at line 267 of file Compression.h.

Referenced by Codec(), and uncompress().

folly::detail::CompressionCounter folly::io::Codec::bytesBeforeCompression_

private

Definition at line 264 of file Compression.h.

Referenced by Codec(), and compress().

folly::detail::CompressionCounter folly::io::Codec::bytesBeforeDecompression_

private

Definition at line 266 of file Compression.h.

Referenced by Codec(), and uncompress().

folly::detail::CompressionCounter folly::io::Codec::compressionMilliseconds_

private

Definition at line 270 of file Compression.h.

Referenced by Codec(), and compress().

folly::detail::CompressionCounter folly::io::Codec::compressions_

private

Definition at line 268 of file Compression.h.

Referenced by Codec(), and compress().

folly::detail::CompressionCounter folly::io::Codec::decompressionMilliseconds_

private

Definition at line 271 of file Compression.h.

Referenced by Codec(), and uncompress().

folly::detail::CompressionCounter folly::io::Codec::decompressions_

private

Definition at line 269 of file Compression.h.

Referenced by Codec(), and uncompress().

CodecType folly::io::Codec::type_

private

Definition at line 263 of file Compression.h.

constexpr uint64_t folly::io::Codec::UNLIMITED_UNCOMPRESSED_LENGTH = uint64_t(-1)

static

Definition at line 135 of file Compression.h.

Referenced by doMaxUncompressedLength().

---

The documentation for this class was generated from the following files:

• proxygen/folly/folly/compression/Compression.h
• proxygen/folly/folly/compression/Compression.cpp