proxygen: proxygen::compress::QMINScheme Class Reference

#include <QMINScheme.h>

Inheritance diagram for proxygen::compress::QMINScheme:

Classes
struct	QMINAck

Public Member Functions
	QMINScheme (CompressionSimulator *sim, uint32_t)

	~QMINScheme ()

std::unique_ptr< Ack >	getAck (uint16_t) override

void	recvAck (std::unique_ptr< Ack > generic_ack) override

std::pair< FrameFlags, std::unique_ptr< folly::IOBuf > >	encode (bool, std::vector< compress::Header > allHeaders, SimStats &stats) override

void	decode (FrameFlags, std::unique_ptr< folly::IOBuf > encodedReq, SimStats &, SimStreamingCallback &callback) override

uint32_t	getHolBlockCount () const override

void	runLoopCallback () noexceptoverride

void	write_ctl_msg (const void *buf, size_t sz, unsigned idx)

Public Member Functions inherited from proxygen::compress::CompressionScheme
	CompressionScheme (CompressionSimulator *sim)

virtual	~CompressionScheme ()

Public Member Functions inherited from folly::EventBase::LoopCallback
virtual	~LoopCallback ()=default

void	cancelLoopCallback ()

bool	isLoopCallbackScheduled () const

Static Public Member Functions
static struct stream_chunk *	stream_chunk_new (size_t off, const void *buf, size_t bufsz)

static void	insert_chunk (struct stream stream, struct stream_chunk new_chunk)

static struct stream_chunk *	maybe_pop_chunk (struct stream *stream)

static void	write_enc2dec (void ctx, const void buf, size_t sz)

static void	write_dec2enc (void ctx, const void buf, size_t sz)

Public Attributes
char *	qms_idstr

struct qmin_enc *	qms_enc

struct qmin_dec *	qms_dec

unsigned	qms_next_stream_id_to_encode

struct stream *	qms_streams

struct {
struct qmin_ctl_out out

size_t write_off

size_t sz

unsigned char buf [0x1000]

}	qms_ctl [2]

Public Attributes inherited from proxygen::compress::CompressionScheme
std::list< BlockInfo >	encodedBlocks

std::list< BlockInfo >	packetBlocks

size_t	index {0}

std::chrono::milliseconds	prev

size_t	packetBytes {0}

std::chrono::milliseconds	decodeDelay

std::list< uint16_t >	packetIndices

Additional Inherited Members
Public Types inherited from proxygen::compress::CompressionScheme
using	BlockInfo = std::tuple< FrameFlags, bool, std::unique_ptr< folly::IOBuf >, SimStreamingCallback * >

Detailed Description

Definition at line 124 of file QMINScheme.h.

Constructor & Destructor Documentation

proxygen::compress::QMINScheme::QMINScheme	(	CompressionSimulator *	sim,
		uint32_t
	)

inlineexplicit

Definition at line 159 of file QMINScheme.h.

References qmin_dec_new(), qmin_enc_new(), QSIDE_CLIENT, QSIDE_SERVER, s_seq, and stream::sm_chunks.

       : CompressionScheme(sim) {
     // TODO: set table size?
     qms_ctl[0].out.qco_write = write_enc2dec;
     qms_ctl[0].write_off = 0;
     qms_ctl[0].sz = 0;
     qms_ctl[1].out.qco_write = write_dec2enc;
     qms_ctl[1].write_off = 0;
     qms_ctl[1].sz = 0;
 
     qms_idstr = (char *)malloc(8);
     sprintf(qms_idstr, "%u", s_seq++);
 
     qms_enc = qmin_enc_new(QSIDE_CLIENT, 4 * 1024, &qms_ctl[0].out, qms_idstr);
     qms_dec = qmin_dec_new(QSIDE_SERVER, 4 * 1024, &qms_ctl[1].out, qms_idstr);
 
     qms_streams = (struct stream *)calloc(2, sizeof(qms_streams[0]));
     TAILQ_INIT(&qms_streams[0].sm_chunks);
     TAILQ_INIT(&qms_streams[1].sm_chunks);
 
     qms_next_stream_id_to_encode = 1;
   }

proxygen::compress::QMINScheme::~QMINScheme ( )

inline

Definition at line 182 of file QMINScheme.h.

References bm::free(), qmin_dec_destroy(), and qmin_enc_destroy().

                 {
     free(qms_streams);
     qmin_enc_destroy(qms_enc);
     qmin_dec_destroy(qms_dec);
     free(qms_idstr);
   }

Member Function Documentation

void proxygen::compress::QMINScheme::decode	(	FrameFlags	,
		std::unique_ptr< folly::IOBuf >	encodedReq,
		SimStats &	,
		SimStreamingCallback &	callback
	)

inlineoverridevirtual

Implements proxygen::compress::CompressionScheme.

Definition at line 338 of file QMINScheme.h.

References wangle::HTTPHeaderSize::compressed, folly::IOBuf::computeChainDataLength(), folly::test::end(), bm::free(), name, proxygen::compress::SimStreamingCallback::onHeader(), proxygen::compress::SimStreamingCallback::onHeadersComplete(), qmin_dec_cmds_in(), qmin_dec_decode(), qmin_dec_stream_done(), stream_chunk::sc_buf, stream_chunk::sc_sz, string, folly::IOBuf::trimStart(), uint32_t, wangle::HTTPHeaderSize::uncompressed, and value.

                                                        {
     folly::io::Cursor cursor(encodedReq.get());
     const unsigned char *buf;
     ssize_t nread;
     size_t ctl_sz, stream_off;
     char outbuf[0x1000];
     unsigned name_len, val_len;
     unsigned decoded_size = 0;
     uint32_t stream_id;
 
     qms_ctl[1].out.qco_ctx = this;
 
     /* Read Stream ID: */
     buf = cursor.data();
     memcpy(&stream_id, buf, sizeof(uint32_t));
     encodedReq->trimStart(sizeof(uint32_t));
 
     /* Read size of control messages */
     buf = cursor.data();
     memcpy(&ctl_sz, buf, sizeof(ctl_sz));
     encodedReq->trimStart(sizeof(ctl_sz));
 
     /* Feed control messages to the decoder: */
     if (ctl_sz) {
       struct stream_chunk *chunk;
 
       /* Read stream offset: */
       buf = cursor.data();
       memcpy(&stream_off, buf, sizeof(stream_off));
       encodedReq->trimStart(sizeof(stream_off));
 
       buf = cursor.data();
       chunk = stream_chunk_new(stream_off, buf, ctl_sz);
       encodedReq->trimStart(ctl_sz);
 
       insert_chunk(&qms_streams[1], chunk);
 
       while ((chunk = maybe_pop_chunk(&qms_streams[1]))) {
         nread = qmin_dec_cmds_in(qms_dec, chunk->sc_buf, chunk->sc_sz);
         if (nread < 0 || (size_t)nread != chunk->sc_sz) {
           VLOG(1) << "error: qmin_dec_cmds_in failed";
           assert(0);
         }
         free(chunk);
       }
     }
 
     buf = cursor.data();
     const unsigned char *const end = buf + cursor.length();
 
     while (buf < end) {
       nread = qmin_dec_decode(
           qms_dec, buf, end - buf, outbuf, sizeof(outbuf), &name_len, &val_len);
       if (nread < 0) {
         VLOG(1) << "error: decoder failed!";
         assert(0);
         return;
       }
       assert(nread);
       buf += nread;
       decoded_size += name_len + val_len;
       std::string name{outbuf, name_len};
       std::string value{outbuf + name_len, val_len};
       callback.onHeader(name, value);
     }
 
     if (0 != qmin_dec_stream_done(qms_dec, stream_id)) {
       assert(0);
       VLOG(1) << "error: qmin_dec_stream_done failed";
     }
 
     proxygen::HTTPHeaderSize sz;
     sz.compressed = encodedReq->computeChainDataLength();
     sz.uncompressed = decoded_size;
     callback.onHeadersComplete(sz);
   }

std::pair<FrameFlags, std::unique_ptr<folly::IOBuf> > proxygen::compress::QMINScheme::encode	(	bool	,
		std::vector< compress::Header >	allHeaders,
		SimStats &	stats
	)

inlineoverridevirtual

Implements proxygen::compress::CompressionScheme.

Definition at line 243 of file QMINScheme.h.

References proxygen::compress::FrameFlags::allowOOO, proxygen::compress::SimStats::compressed, folly::IOBuf::copyBuffer(), bm::free(), name, QES_ERR, QES_NOBUFS, QES_OK, QIT_YES, qmin_enc_encode(), qmin_enc_end_stream_headers(), qmin_enc_to_str(), string, folly::pushmi::operators::transform, uint32_t, and proxygen::compress::SimStats::uncompressed.

                                 {
     const size_t max_ctl = 0x1000;
     const size_t max_comp = 0x1000;
     unsigned char outbuf[max_ctl + max_comp];
     unsigned char *const comp = outbuf + max_ctl;
     size_t nw, comp_sz;
     enum qmin_encode_status qes;
     FrameFlags flags;
 
     qms_ctl[0].out.qco_ctx = this;
     comp_sz = 0;
 
     for (const auto header : allHeaders) {
       std::string name{header.name->c_str()};
       std::transform(name.begin(), name.end(), name.begin(), ::tolower);
       qes = qmin_enc_encode(qms_enc,
                             qms_next_stream_id_to_encode,
                             name.c_str(),
                             name.length(),
                             header.value->c_str(),
                             header.value->length(),
                             QIT_YES,
                             comp + comp_sz,
                             max_comp - comp_sz,
                             &nw);
       switch (qes) {
         case QES_OK:
           /* 2 is a magic number added to the uncompressed size by the other
            * encoder.  We follow suit to make the numbers match.
            */
           stats.uncompressed += name.length() + header.value->length() + 2;
           stats.compressed += nw;
           comp_sz += nw;
           break;
         case QES_NOBUFS:
           VLOG(1) << "compressed header does not fit into temporary "
                      "output buffer";
           return {flags, nullptr};
         case QES_ERR:
           VLOG(1) << "error: " << strerror(errno);
           assert(0);
           return {flags, nullptr};
       }
     }
 
     {
       size_t sz;
       char *state = qmin_enc_to_str(qms_enc, &sz);
       VLOG(4) << "encoder state: " << state;
       free(state);
     }
 
     if (0 != qmin_enc_end_stream_headers(qms_enc)) {
       VLOG(1) << "error: qmin_enc_end_stream_headers failed";
       assert(0);
     }
 
     /* Prepend control message and its size: */
     size_t ctl_msg_sz = qms_ctl[0].sz;
     qms_ctl[0].sz = 0;
     size_t ctl_msg_sz_with_off;
     if (ctl_msg_sz) {
       memcpy(outbuf + max_ctl - ctl_msg_sz, qms_ctl[0].buf, ctl_msg_sz);
       memcpy(outbuf + max_ctl - ctl_msg_sz - sizeof(qms_ctl[0].write_off),
              &qms_ctl[0].write_off,
              sizeof(qms_ctl[0].write_off));
       qms_ctl[0].write_off += ctl_msg_sz;
       ctl_msg_sz_with_off = ctl_msg_sz + sizeof(qms_ctl[0].write_off);
     } else
       ctl_msg_sz_with_off = 0;
     memcpy(outbuf + max_ctl - ctl_msg_sz_with_off - sizeof(ctl_msg_sz),
            &ctl_msg_sz,
            sizeof(ctl_msg_sz));
 
     stats.compressed += ctl_msg_sz;
 
     /* Prepend Stream ID: */
     memcpy(outbuf + max_ctl - ctl_msg_sz_with_off - sizeof(ctl_msg_sz) -
                sizeof(uint32_t),
            &qms_next_stream_id_to_encode,
            sizeof(qms_next_stream_id_to_encode));
 
     qms_next_stream_id_to_encode += 2;
     flags.allowOOO = true;
     return {
         flags,
         folly::IOBuf::copyBuffer(outbuf + max_ctl - ctl_msg_sz_with_off -
                                      sizeof(ctl_msg_sz) - sizeof(uint32_t),
                                  comp_sz + ctl_msg_sz_with_off +
                                      sizeof(ctl_msg_sz) + sizeof(uint32_t))};
   }

std::unique_ptr<Ack> proxygen::compress::QMINScheme::getAck ( uint16_t )

inlineoverridevirtual

Implements proxygen::compress::CompressionScheme.

Definition at line 204 of file QMINScheme.h.

References folly::gen::move.

                                                  {
     if (qms_ctl[1].sz) {
       auto ack = std::make_unique<QMINAck>(
           qms_ctl[1].write_off, qms_ctl[1].buf, qms_ctl[1].sz);
       VLOG(4) << "sent ACK for instance " << qms_idstr
               << " off: " << qms_ctl[1].write_off << "; sz: " << qms_ctl[1].sz;
       qms_ctl[1].write_off += qms_ctl[1].sz;
       qms_ctl[1].sz = 0;
       return std::move(ack);
     } else {
       assert(0);
       return nullptr;
     }
   }

uint32_t proxygen::compress::QMINScheme::getHolBlockCount ( ) const

inlineoverridevirtual

Implements proxygen::compress::CompressionScheme.

Definition at line 418 of file QMINScheme.h.

                                              {
     return 0;
   }

static void proxygen::compress::QMINScheme::insert_chunk	(	struct stream *	stream,
		struct stream_chunk *	new_chunk
	)

inlinestatic

Definition at line 138 of file QMINScheme.h.

References stream_chunk::sc_off, and stream::sm_chunks.

                                                            {
     struct stream_chunk *chunk;
     TAILQ_FOREACH(chunk, &stream->sm_chunks, sc_next)
     if (chunk->sc_off > new_chunk->sc_off) {
       TAILQ_INSERT_BEFORE(chunk, new_chunk, sc_next);
       return;
     }
     TAILQ_INSERT_TAIL(&stream->sm_chunks, new_chunk, sc_next);
   }

static struct stream_chunk* proxygen::compress::QMINScheme::maybe_pop_chunk ( struct stream * stream )

inlinestatic

Definition at line 149 of file QMINScheme.h.

References stream_chunk::sc_off, stream_chunk::sc_sz, stream::sm_chunks, and stream::sm_read_off.

                                                                      {
     struct stream_chunk *chunk = TAILQ_FIRST(&stream->sm_chunks);
     if (chunk && chunk->sc_off == stream->sm_read_off) {
       TAILQ_REMOVE(&stream->sm_chunks, chunk, sc_next);
       stream->sm_read_off += chunk->sc_sz;
       return chunk;
     } else
       return NULL;
   }

void proxygen::compress::QMINScheme::recvAck ( std::unique_ptr< Ack > generic_ack )

inlineoverridevirtual

Implements proxygen::compress::CompressionScheme.

Definition at line 219 of file QMINScheme.h.

References bm::free(), qmin_enc_cmds_in(), stream_chunk::sc_buf, and stream_chunk::sc_sz.

                                                         {
     struct stream_chunk *chunk;
 
     CHECK(generic_ack);
     auto ack = dynamic_cast<QMINAck *>(generic_ack.get());
     CHECK_NOTNULL(ack);
 
     VLOG(4) << "received ACK for instance " << qms_idstr
             << " off: " << ack->qma_off << "; sz: " << ack->qma_sz;
 
     chunk = stream_chunk_new(ack->qma_off, ack->qma_buf, ack->qma_sz);
     insert_chunk(&qms_streams[0], chunk);
 
     while ((chunk = maybe_pop_chunk(&qms_streams[0]))) {
       ssize_t nread;
       nread = qmin_enc_cmds_in(qms_enc, chunk->sc_buf, chunk->sc_sz);
       if (nread < 0 || (size_t)nread != chunk->sc_sz) {
         VLOG(1) << "error: qmin_enc_cmds_in failed";
         assert(0);
       }
       free(chunk);
     }
   }

void proxygen::compress::QMINScheme::runLoopCallback ( )

inlineoverridevirtualnoexcept

Reimplemented from proxygen::compress::CompressionScheme.

Definition at line 422 of file QMINScheme.h.

                                            {
     CompressionScheme::runLoopCallback();
   }

static struct stream_chunk* proxygen::compress::QMINScheme::stream_chunk_new	(	size_t	off,
		const void *	buf,
		size_t	bufsz
	)

inlinestatic

Definition at line 126 of file QMINScheme.h.

References stream_chunk::sc_buf, stream_chunk::sc_off, and stream_chunk::sc_sz.

                                                              {
     struct stream_chunk *chunk =
         (struct stream_chunk *)malloc(sizeof(*chunk) + bufsz);
     assert(chunk);
     chunk->sc_off = off;
     chunk->sc_sz = bufsz;
     memcpy(chunk->sc_buf, buf, bufsz);
     return chunk;
   }

void proxygen::compress::QMINScheme::write_ctl_msg	(	const void *	buf,
		size_t	sz,
		unsigned	idx
	)

inline

Definition at line 426 of file QMINScheme.h.

Referenced by write_dec2enc(), and write_enc2dec().

                                                                {
     size_t avail = sizeof(qms_ctl[idx].buf) - qms_ctl[idx].sz;
     assert(avail >= sz);
     if (avail < sz) {
       VLOG(1) << "Truncating control message from " << sz << " to " << avail
               << "bytes";
       sz = avail;
     }
     memcpy(qms_ctl[idx].buf + qms_ctl[idx].sz, buf, sz);
     qms_ctl[idx].sz += sz;
     VLOG(4) << "Wrote " << sz << " bytes to control channel";
   }

static void proxygen::compress::QMINScheme::write_dec2enc	(	void *	ctx,
		const void *	buf,
		size_t	sz
	)

inlinestatic

Definition at line 444 of file QMINScheme.h.

References write_ctl_msg().

                                                                    {
     QMINScheme *const qms = (QMINScheme *)ctx;
     qms->write_ctl_msg(buf, sz, 1);
   }

static void proxygen::compress::QMINScheme::write_enc2dec	(	void *	ctx,
		const void *	buf,
		size_t	sz
	)

inlinestatic

Definition at line 439 of file QMINScheme.h.

References write_ctl_msg().

                                                                    {
     QMINScheme *const qms = (QMINScheme *)ctx;
     qms->write_ctl_msg(buf, sz, 0);
   }

Member Data Documentation

unsigned char proxygen::compress::QMINScheme::buf[0x1000]

Definition at line 466 of file QMINScheme.h.

struct qmin_ctl_out proxygen::compress::QMINScheme::out

Definition at line 463 of file QMINScheme.h.

struct { ... } proxygen::compress::QMINScheme::qms_ctl[2]

struct qmin_dec* proxygen::compress::QMINScheme::qms_dec

Definition at line 452 of file QMINScheme.h.

struct qmin_enc* proxygen::compress::QMINScheme::qms_enc

Definition at line 451 of file QMINScheme.h.

char* proxygen::compress::QMINScheme::qms_idstr

Definition at line 449 of file QMINScheme.h.

unsigned proxygen::compress::QMINScheme::qms_next_stream_id_to_encode

Definition at line 457 of file QMINScheme.h.

struct stream* proxygen::compress::QMINScheme::qms_streams

Definition at line 460 of file QMINScheme.h.

size_t proxygen::compress::QMINScheme::sz

Definition at line 465 of file QMINScheme.h.

size_t proxygen::compress::QMINScheme::write_off

Definition at line 464 of file QMINScheme.h.

The documentation for this class was generated from the following file:

proxygen/lib/http/codec/compress/experimental/simulator/QMINScheme.h

Classes

Public Member Functions

Static Public Member Functions

Public Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation