proxygen: folly::io Namespace Reference

Namespaces
	compression

	detail

	test

Classes
class	Appender

class	Codec

class	Cursor

class	QueueAppender

class	RWCursor

class	StreamCodec

Typedefs
typedef RWCursor< CursorAccess::PRIVATE >	RWPrivateCursor

typedef RWCursor< CursorAccess::UNSHARE >	RWUnshareCursor

Enumerations
enum	CodecType { CodecType::USER_DEFINED = 0, CodecType::NO_COMPRESSION = 1, CodecType::LZ4 = 2, CodecType::SNAPPY = 3, CodecType::ZLIB = 4, CodecType::LZ4_VARINT_SIZE = 5, CodecType::LZMA2 = 6, CodecType::LZMA2_VARINT_SIZE = 7, CodecType::ZSTD = 8, CodecType::GZIP = 9, CodecType::LZ4_FRAME = 10, CodecType::BZIP2 = 11, CodecType::ZSTD_FAST = 12, CodecType::NUM_CODEC_TYPES = 13 }

enum	CursorAccess { CursorAccess::PRIVATE, CursorAccess::UNSHARE }

Functions
static std::unique_ptr< IOBuf >	addOutputBuffer (MutableByteRange &output, uint64_t size)

static uint64_t	computeBufferLength (uint64_t const compressedLength, uint64_t const blockSize)

bool	hasCodec (CodecType type)

std::unique_ptr< Codec >	getCodec (CodecType type, int level)

bool	hasStreamCodec (CodecType type)

std::unique_ptr< StreamCodec >	getStreamCodec (CodecType type, int level)

std::unique_ptr< Codec >	getAutoUncompressionCodec (std::vector< std::unique_ptr< Codec >> customCodecs, std::unique_ptr< Codec > terminalCodec)

Variables
constexpr int	COMPRESSION_LEVEL_FASTEST = -1

constexpr int	COMPRESSION_LEVEL_DEFAULT = -2

constexpr int	COMPRESSION_LEVEL_BEST = -3

Typedef Documentation

typedef RWCursor<CursorAccess::PRIVATE> folly::io::RWPrivateCursor

Definition at line 958 of file Cursor.h.

typedef RWCursor<CursorAccess::UNSHARE> folly::io::RWUnshareCursor

Definition at line 959 of file Cursor.h.

Enumeration Type Documentation

enum folly::io::CodecType

strong

Enumerator
USER_DEFINED	This codec type is not defined; getCodec() will throw an exception if used. Useful if deriving your own classes from Codec without going through the getCodec() interface.
NO_COMPRESSION	Use no compression. Levels supported: 0
LZ4	Use LZ4 compression. Levels supported: 1 = fast, 2 = best; default = 1
SNAPPY	Use Snappy compression. Levels supported: 1
ZLIB	Use zlib compression. Levels supported: 0 = no compression, 1 = fast, ..., 9 = best; default = 6 Streaming compression is supported.
LZ4_VARINT_SIZE	Use LZ4 compression, prefixed with size (as Varint).
LZMA2	Use LZMA2 compression. Levels supported: 0 = no compression, 1 = fast, ..., 9 = best; default = 6 Streaming compression is supported.
LZMA2_VARINT_SIZE
ZSTD	Use ZSTD compression. Levels supported: 1 = fast, ..., 19 = best; default = 3 Use ZSTD_FAST for the fastest zstd compression (negative levels). Streaming compression is supported.
GZIP	Use gzip compression. This is the same compression algorithm as ZLIB but gzip-compressed files tend to be easier to work with from the command line. Levels supported: 0 = no compression, 1 = fast, ..., 9 = best; default = 6 Streaming compression is supported.
LZ4_FRAME	Use LZ4 frame compression. Levels supported: 0 = fast, 16 = best; default = 0
BZIP2	Use bzip2 compression. Levels supported: 1 = fast, 9 = best; default = 9 Streaming compression is supported BUT FlushOp::FLUSH does NOT ensure that the decompressor can read all the data up to that point, due to a bug in the bzip2 library.
ZSTD_FAST	Use ZSTD compression with a negative compression level (1=-1, 2=-2, ...). Higher compression levels mean faster. Level 1 is around the same speed as Snappy with better compression. Level 5 is around the same speed as LZ4 with slightly worse compression. Each level gains about 6-15% speed and loses 3-7% compression. Decompression speed improves for each level, and level 1 decompression speed is around 25% faster than ZSTD. This codec is fully compatible with ZSTD. Levels supported: 1 = best, ..., 5 = fast; default = 1 Streaming compression is supported.
NUM_CODEC_TYPES

Definition at line 37 of file Compression.h.

                      {
   USER_DEFINED = 0,
 
   NO_COMPRESSION = 1,
 
   LZ4 = 2,
 
   SNAPPY = 3,
 
   ZLIB = 4,
 
   LZ4_VARINT_SIZE = 5,
 
   LZMA2 = 6,
   LZMA2_VARINT_SIZE = 7,
 
   ZSTD = 8,
 
   GZIP = 9,
 
   LZ4_FRAME = 10,
 
   BZIP2 = 11,
 
   ZSTD_FAST = 12,
 
   NUM_CODEC_TYPES = 13,
 };

enum folly::io::CursorAccess

strong

Enumerator
PRIVATE
UNSHARE

Definition at line 812 of file Cursor.h.

812 { PRIVATE, UNSHARE };

folly::io::CursorAccess::UNSHARE

folly::io::CursorAccess::PRIVATE

Function Documentation

static std::unique_ptr<IOBuf> folly::io::addOutputBuffer	(	MutableByteRange &	output,
		uint64_t	size
	)

static

Definition at line 434 of file Compression.cpp.

References buffer(), folly::IOBuf::create(), and folly::Range< Iter >::empty().

Referenced by proxygen::ZlibStreamCompressor::compress(), folly::io::StreamCodec::doCompress(), and folly::io::StreamCodec::doUncompress().

                    {
   DCHECK(output.empty());
   auto buffer = IOBuf::create(size);
   buffer->append(buffer->capacity());
   output = {buffer->writableData(), buffer->length()};
   return buffer;
 }

static uint64_t folly::io::computeBufferLength	(	uint64_t const	compressedLength,
		uint64_t const	blockSize
	)

static

Definition at line 486 of file Compression.cpp.

References max, min, and uint64_t.

Referenced by folly::io::StreamCodec::doUncompress().

                               {
   uint64_t constexpr kMaxBufferLength = uint64_t(4) << 20; // 4 MiB
   uint64_t const goodBufferSize = 4 * std::max(blockSize, compressedLength);
   return std::min(goodBufferSize, kMaxBufferLength);
 }

std::unique_ptr< Codec > folly::io::getAutoUncompressionCodec	(	std::vector< std::unique_ptr< Codec >>	customCodecs = `{}`,
		std::unique_ptr< Codec >	terminalCodec = `{}`
	)

Returns a codec that can uncompress any of the given codec types as well as {LZ4_FRAME, ZSTD, ZLIB, GZIP, LZMA2, BZIP2}. Appends each default codec to customCodecs in order, so long as a codec with the same type() isn't already present in customCodecs or as the terminalCodec. When uncompress() is called, each codec's canUncompress() is called in the order that they are given. Appended default codecs are checked last. uncompress() is called on the first codec whose canUncompress() returns true.

In addition, an optional terminalCodec can be provided. This codec's uncompress() will be called either when no other codec canUncompress() the data or the chosen codec throws an exception on the data. The terminalCodec is intended for ambiguous headers, when canUncompress() is false for some data it can actually uncompress. The terminalCodec does not need to override validPrefixes() or canUncompress() and overriding these functions will have no effect on the returned codec's validPrefixes() or canUncompress() functions. The terminalCodec's needsUncompressedLength() and maxUncompressedLength() will affect the returned codec's respective functions. The terminalCodec must not be duplicated in customCodecs.

An exception is thrown if no codec canUncompress() the data and either no terminal codec was provided or a terminal codec was provided and it throws on the data. An exception is thrown if the chosen codec's uncompress() throws on the data and either no terminal codec was provided or a terminal codec was provided and it also throws on the data. An exception is thrown if compress() is called on the returned codec.

Requirements are checked in debug mode and are as follows: Let headers be the concatenation of every codec's validPrefixes().

Each codec must override validPrefixes() and canUncompress().
No codec's validPrefixes() may be empty.
No header in headers may be empty.
headers must not contain any duplicate elements.
No strict non-empty prefix of any header in headers may be in headers.
The terminalCodec's type must not be the same as any other codec's type (with USER_DEFINED being the exception).

Definition at line 2130 of file Compression.cpp.

References folly::gen::move.

Referenced by folly::io::test::AutomaticCodecTest::SetUp(), folly::io::test::TerminalCodecTest::SetUp(), folly::io::test::TEST(), and folly::io::test::TEST_P().

                                         {
   return AutomaticCodec::create(
       std::move(customCodecs), std::move(terminalCodec));
 }

std::unique_ptr< Codec > folly::io::getCodec	(	CodecType	type,
		int	level = `COMPRESSION_LEVEL_DEFAULT`
	)

Return a codec for the given type. Throws on error. The level is a non-negative codec-dependent integer indicating the level of compression desired, or one of the following constants:

COMPRESSION_LEVEL_FASTEST is fastest (uses least CPU / memory, worst compression) COMPRESSION_LEVEL_DEFAULT is the default (likely a tradeoff between FASTEST and BEST) COMPRESSION_LEVEL_BEST is the best compression (uses most CPU / memory, best compression)

When decompressing, the compression level is ignored. All codecs will decompress all data compressed with the a codec of the same type, regardless of compression level.

Definition at line 2104 of file Compression.cpp.

References codec, and folly::io::Codec::type().

                                                          {
   auto const factory = getFactory(type).codec;
   if (!factory) {
     throw std::invalid_argument(
         to<std::string>("Compression type ", type, " not supported"));
   }
   auto codec = (*factory)(level, type);
   DCHECK(codec->type() == type);
   return codec;
 }

std::unique_ptr< StreamCodec > folly::io::getStreamCodec	(	CodecType	type,
		int	level = `COMPRESSION_LEVEL_DEFAULT`
	)

Return a codec for the given type. Throws on error. The level is a non-negative codec-dependent integer indicating the level of compression desired, or one of the following constants:

When decompressing, the compression level is ignored. All codecs will decompress all data compressed with the a codec of the same type, regardless of compression level.

Definition at line 2119 of file Compression.cpp.

References codec, and folly::io::Codec::type().

Referenced by folly::io::StreamCodec::doUncompress(), folly::io::test::StreamingCompressionTest::runFlushTest(), folly::io::test::StreamingUnitTest::SetUp(), folly::io::test::StreamingCompressionTest::SetUp(), and folly::io::test::TEST().

                                                                      {
   auto const factory = getFactory(type).stream;
   if (!factory) {
     throw std::invalid_argument(
         to<std::string>("Compression type ", type, " not supported"));
   }
   auto codec = (*factory)(level, type);
   DCHECK(codec->type() == type);
   return codec;
 }

bool folly::io::hasCodec ( CodecType type )

Check if a specified codec is supported.

Definition at line 2100 of file Compression.cpp.

Referenced by folly::io::test::availableCodecs(), folly::io::test::supportedCodecs(), and folly::io::test::TEST().

                               {
   return getFactory(type).codec != nullptr;
 }

bool folly::io::hasStreamCodec ( CodecType type )

Check if a specified codec is supported and supports streaming.

Definition at line 2115 of file Compression.cpp.

Referenced by folly::io::test::availableStreamCodecs(), and folly::io::test::TEST().

                                     {
   return getFactory(type).stream != nullptr;
 }

Variable Documentation

constexpr int folly::io::COMPRESSION_LEVEL_BEST = -3

Definition at line 442 of file Compression.h.

Referenced by folly::io::StreamCodec::doUncompress().

constexpr int folly::io::COMPRESSION_LEVEL_DEFAULT = -2

Definition at line 441 of file Compression.h.

Referenced by folly::io::StreamCodec::doUncompress().

constexpr int folly::io::COMPRESSION_LEVEL_FASTEST = -1

Definition at line 440 of file Compression.h.

Referenced by folly::io::StreamCodec::doUncompress().

Namespaces

Classes

Typedefs

Enumerations

Functions

Variables

Typedef Documentation

Enumeration Type Documentation

Function Documentation

Variable Documentation