proxygen: proxygen::compress::CompressionSimulator Class Reference

#include <CompressionSimulator.h>

Public Member Functions
	CompressionSimulator (SimParams p)

bool	readInputFromFileAndSchedule (const std::string &filename)

void	run ()

void	flushSchemePackets (CompressionScheme *scheme)

void	flushPacket (CompressionScheme *scheme)

Private Member Functions
void	flushRequests (CompressionScheme *scheme)

void	setupRequest (uint16_t seqn, HTTPMessage &&msg, std::chrono::milliseconds encodeDelay)

CompressionScheme *	getScheme (folly::StringPiece host)

std::unique_ptr< CompressionScheme >	makeScheme ()

std::pair< FrameFlags, std::unique_ptr< folly::IOBuf > >	encode (CompressionScheme *scheme, bool newPacket, uint16_t seqn)

void	decodePacket (CompressionScheme *scheme, std::list< CompressionScheme::BlockInfo > &blocks)

void	decode (CompressionScheme *scheme, FrameFlags flags, std::unique_ptr< folly::IOBuf > encodedReq, SimStreamingCallback &cb)

void	scheduleEvent (folly::Function< void()> f, std::chrono::milliseconds ms)

void	sendAck (CompressionScheme *scheme, std::unique_ptr< CompressionScheme::Ack > ack)

void	recvAck (CompressionScheme *scheme, std::unique_ptr< CompressionScheme::Ack > ack)

std::chrono::milliseconds	deliveryDelay ()

std::chrono::milliseconds	rtt ()

std::chrono::milliseconds	one_half_rtt ()

std::chrono::milliseconds	rxmitDelay ()

bool	loss ()

bool	delayed ()

std::chrono::milliseconds	extraDelay ()

uint32_t	minOOOThresh ()

Private Attributes
SimParams	params_

std::vector< proxygen::HTTPMessage >	requests_

folly::EventBase	eventBase_

std::unordered_map< std::string, std::unique_ptr< CompressionScheme > >	domains_

std::vector< SimStreamingCallback >	callbacks_

folly::Random::DefaultGenerator	rng_

SimStats	stats_

Detailed Description

Definition at line 23 of file CompressionSimulator.h.

Constructor & Destructor Documentation

proxygen::compress::CompressionSimulator::CompressionSimulator ( SimParams p )

inlineexplicit

Definition at line 25 of file CompressionSimulator.h.

References decode(), decodePacket(), delayed(), deliveryDelay(), encode(), extraDelay(), f, flushPacket(), flushRequests(), flushSchemePackets(), getScheme(), loss(), makeScheme(), minOOOThresh(), one_half_rtt(), readInputFromFileAndSchedule(), recvAck(), rtt(), run(), rxmitDelay(), scheduleEvent(), sendAck(), setupRequest(), string, uint16_t, and uint32_t.

25 : params_(p) {

26 }

proxygen::compress::CompressionSimulator::params_

SimParams params_

Definition: CompressionSimulator.h:65

Member Function Documentation

void proxygen::compress::CompressionSimulator::decode	(	CompressionScheme *	scheme,
		FrameFlags	flags,
		std::unique_ptr< folly::IOBuf >	encodedReq,
		SimStreamingCallback &	cb
	)

private

Definition at line 304 of file CompressionSimulator.cpp.

References proxygen::compress::CompressionScheme::decode(), and folly::gen::move.

Referenced by CompressionSimulator().

                                                             {
   scheme->decode(flags, std::move(encodedReq), stats_, cb);
 }

void proxygen::compress::CompressionSimulator::decodePacket	(	CompressionScheme *	scheme,
		std::list< CompressionScheme::BlockInfo > &	blocks
	)

private

Definition at line 311 of file CompressionSimulator.cpp.

References fizz::decode(), and folly::gen::move.

Referenced by CompressionSimulator().

                                                  {
   VLOG(1) << "decode packet with " << blocks.size() << " blocks";
   while (!blocks.empty()) {
     auto encodeRes = &blocks.front();
     // TODO(ckrasic) - to get packet coordination correct, could plumb
     // through "start of packet" flag here.  Probably not worth it,
     // since it seems to make only a very small difference (about a
     // 0.1% compressiondifference on my facebook har).
     decode(scheme,
            std::get<0>(*encodeRes),
            std::move(std::get<2>(*encodeRes)),
            *std::get<3>(*encodeRes));
     blocks.pop_front();
   }
 }

bool proxygen::compress::CompressionSimulator::delayed ( )

private

Definition at line 390 of file CompressionSimulator.cpp.

Referenced by CompressionSimulator().

                                    {
   return Random::randDouble01(rng_) < params_.delayProbability;
 }

std::chrono::milliseconds proxygen::compress::CompressionSimulator::deliveryDelay ( )

private

Definition at line 352 of file CompressionSimulator.cpp.

Referenced by CompressionSimulator().

                                                           {
   std::chrono::milliseconds delay = one_half_rtt();
   while (loss()) {
     stats_.packetLosses++;
     scheduleEvent([] { VLOG(4) << "Packet lost!"; }, delay);
     std::chrono::milliseconds rxmit = rxmitDelay();
     delay += rxmit;
     scheduleEvent(
         [rxmit] {
           VLOG(4) << "Packet loss detected, retransmitting with additional "
                   << rxmit.count();
         },
         delay - one_half_rtt());
   }
   if (delayed()) {
     scheduleEvent([] { VLOG(4) << "Packet delayed in network"; }, delay);
     delay += extraDelay();
   }
   return delay;
 }

std::pair< FrameFlags, unique_ptr< IOBuf > > proxygen::compress::CompressionSimulator::encode	(	CompressionScheme *	scheme,
		bool	newPacket,
		uint16_t	seqn
	)

private

Definition at line 284 of file CompressionSimulator.cpp.

References proxygen::compress::CompressionScheme::encode(), proxygen::HTTP_HEADER_HOST, folly::gen::move, and proxygen::compress::prepareMessageForCompression().

Referenced by CompressionSimulator().

                                                                {
   VLOG(1) << "Start encoding request=" << index;
   // vector to hold cookie crumbs
   vector<string> cookies;
   vector<compress::Header> allHeaders = prepareMessageForCompression(
       requests_[index], cookies);
 
   auto before = stats_.uncompressed;
   auto res = scheme->encode(newPacket, std::move(allHeaders), stats_);
   VLOG(1) << "Encoded request=" << index << " for host="
           << requests_[index].getHeaders().getSingleOrEmpty(HTTP_HEADER_HOST)
           << " orig size=" << (stats_.uncompressed - before)
           << " block size=" << res.second->computeChainDataLength()
           << " cumulative bytes=" << stats_.compressed
           << " cumulative compression ratio="
           << int(100 - double(100 * stats_.compressed) / stats_.uncompressed);
   return res;
 }

std::chrono::milliseconds proxygen::compress::CompressionSimulator::extraDelay ( )

private

Definition at line 394 of file CompressionSimulator.cpp.

References uint32_t.

Referenced by CompressionSimulator().

                                                        {
   uint32_t ms = params_.maxDelay.count() * Random::randDouble01(rng_);
   return std::chrono::milliseconds(ms);
 }

void proxygen::compress::CompressionSimulator::flushPacket ( CompressionScheme * scheme )

Definition at line 184 of file CompressionSimulator.cpp.

References proxygen::compress::CompressionScheme::decodeDelay, folly::gen::move, proxygen::compress::CompressionScheme::packetBlocks, and proxygen::compress::CompressionScheme::packetBytes.

Referenced by CompressionSimulator().

                                                                 {
   if (scheme->packetBlocks.empty()) {
     return;
   }
 
   stats_.packets++;
   VLOG(1) << "schedule decode for " << scheme->packetBlocks.size()
           << " blocks at " << scheme->decodeDelay.count();
   scheduleEvent(
       {[this, scheme, blocks = std::move(scheme->packetBlocks)]() mutable {
         decodePacket(scheme, blocks);
       }},
       scheme->decodeDelay);
   scheme->packetBytes = 0;
 }

void proxygen::compress::CompressionSimulator::flushRequests ( CompressionScheme * scheme )

private

Definition at line 103 of file CompressionSimulator.cpp.

References folly::gen::move, proxygen::compress::CompressionScheme::packetIndices, and proxygen::compress::CompressionScheme::prev.

Referenced by CompressionSimulator().

                                                                   {
   VLOG(5) << "schedule encode for " << scheme->packetIndices.size()
           << " blocks at " << scheme->prev.count();
   // Flush previous train
   scheduleEvent(
       [this, scheme, indices = std::move(scheme->packetIndices)]() mutable {
         bool newPacket = true;
         while (!indices.empty()) {
           int16_t index = indices.front();
           indices.pop_front();
           auto schemeIndex = scheme->index;
           auto encodeRes = encode(scheme, newPacket, index);
           FrameFlags flags = encodeRes.first;
           bool allowOOO = flags.allowOOO;
           if (schemeIndex < minOOOThresh()) {
             allowOOO = false;
             auto ack = scheme->getAck(schemeIndex);
             if (ack) {
               scheme->recvAck(std::move(ack));
             }
           }
           stats_.allowedOOO += (allowOOO) ? 1 : 0;
           flags.allowOOO = allowOOO;
           scheme->encodedBlocks.emplace_back(flags,
                                              newPacket,
                                              std::move(encodeRes.second),
                                              &callbacks_[index]);
           newPacket = false;
         }
         eventBase_.runInLoop(scheme, true);
       },
       scheme->prev);
 }

void proxygen::compress::CompressionSimulator::flushSchemePackets ( CompressionScheme * scheme )

Definition at line 200 of file CompressionSimulator.cpp.

References proxygen::compress::CompressionScheme::decodeDelay, proxygen::compress::CompressionScheme::encodedBlocks, max, min, proxygen::compress::CompressionScheme::packetBlocks, and proxygen::compress::CompressionScheme::packetBytes.

Referenced by CompressionSimulator().

                                                                        {
   CHECK(!scheme->encodedBlocks.empty());
   VLOG(2) << "Flushing " << scheme->encodedBlocks.size() << " requests";
   // tracks the number of bytes in the current simulated packet
   auto encodeRes = &scheme->encodedBlocks.front();
   bool newPacket = std::get<1>(*encodeRes);
   size_t headerBlockBytesRemaining =
       std::get<2>(*encodeRes)->computeChainDataLength();
   std::chrono::milliseconds packetDelay = deliveryDelay();
   scheme->decodeDelay = packetDelay;
   while (true) {
     if (newPacket) {
       flushPacket(scheme);
     }
     newPacket = false;
     // precondition packetBytes < kMTU
     if (scheme->packetBytes + headerBlockBytesRemaining >= kMTU) {
       // Header block filled current packet, triggering a flush
       VLOG(2) << "Request(s) spanned multiple packets";
       newPacket = true;
     } else {
       scheme->packetBytes += headerBlockBytesRemaining;
     }
     headerBlockBytesRemaining -=
         std::min(headerBlockBytesRemaining, kMTU - scheme->packetBytes);
     if (headerBlockBytesRemaining == 0) {
       // Move from the first element of encodedBlocks to the last
       // element of packetBlocks.
       scheme->packetBlocks.splice(scheme->packetBlocks.end(),
                                   scheme->encodedBlocks,
                                   scheme->encodedBlocks.begin());
       if (scheme->encodedBlocks.empty()) {
         // All done
         break;
       }
       // Grab the next request
       encodeRes = &scheme->encodedBlocks.front();
       newPacket = std::get<1>(*encodeRes);
       headerBlockBytesRemaining =
           std::get<2>(*encodeRes)->computeChainDataLength();
     }
     if (newPacket) {
       packetDelay = deliveryDelay();
       scheme->decodeDelay = std::max(scheme->decodeDelay, packetDelay);
     }
   }
   flushPacket(scheme);
   CHECK(scheme->encodedBlocks.empty());
 }

CompressionScheme * proxygen::compress::CompressionSimulator::getScheme ( folly::StringPiece host )

private

Definition at line 250 of file CompressionSimulator.cpp.

References folly::Range< Iter >::endsWith(), folly::Range< Iter >::find(), folly::INFO, folly::gen::move, and folly::Range< Iter >::str().

Referenced by CompressionSimulator().

                                                                      {
   static string blended("\"Facebook\"");
   if (params_.blend &&
       (domain.endsWith("facebook.com") || domain.endsWith("fbcdn.net"))) {
     domain = blended;
   }
 
   auto it = domains_.find(domain.str());
   CompressionScheme* scheme = nullptr;
   if (it == domains_.end()) {
     LOG(INFO) << "Creating scheme for domain=" << domain;
     auto schemePtr = makeScheme();
     scheme = schemePtr.get();
     domains_.emplace(domain.str(), std::move(schemePtr));
   } else {
     scheme = it->second.get();
   }
   return scheme;
 }

bool proxygen::compress::CompressionSimulator::loss ( )

private

Definition at line 386 of file CompressionSimulator.cpp.

Referenced by CompressionSimulator().

                                 {
   return Random::randDouble01(rng_) < params_.lossProbability;
 }

unique_ptr< CompressionScheme > proxygen::compress::CompressionSimulator::makeScheme ( )

private

Definition at line 270 of file CompressionSimulator.cpp.

References folly::FATAL.

Referenced by CompressionSimulator().

                                                                {
   switch (params_.type) {
     case SchemeType::QPACK:
       return make_unique<QPACKScheme>(this, params_.tableSize,
                                       params_.maxBlocking);
     case SchemeType::QMIN:
       return make_unique<QMINScheme>(this, params_.tableSize);
     case SchemeType::HPACK:
       return make_unique<HPACKScheme>(this, params_.tableSize);
   }
   LOG(FATAL) << "Bad scheme";
   return nullptr;
 }

uint32_t proxygen::compress::CompressionSimulator::minOOOThresh ( )

private

Definition at line 399 of file CompressionSimulator.cpp.

Referenced by CompressionSimulator().

                                             {
   return params_.minOOOThresh;
 }

std::chrono::milliseconds proxygen::compress::CompressionSimulator::one_half_rtt ( )

private

Definition at line 377 of file CompressionSimulator.cpp.

Referenced by CompressionSimulator().

                                                          {
   return params_.rtt / 2;
 }

bool proxygen::compress::CompressionSimulator::readInputFromFileAndSchedule ( const std::string & filename )

Definition at line 36 of file CompressionSimulator.cpp.

References a, b, proxygen::ERROR, folly::exceptionStr(), folly::gen::fromFile(), proxygen::HTTPMessage::getStartTime(), proxygen::millisecondsBetween(), folly::gen::move, proxygen::HTTPArchive::requests, and uint16_t.

Referenced by CompressionSimulator(), and main().

                             {
   unique_ptr<HTTPArchive> har;
   try {
     har = HTTPArchive::fromFile(kTestDir + filename);
   } catch (const std::exception& ex) {
     LOG(ERROR) << folly::exceptionStr(ex);
   }
   if (!har || har->requests.size() == 0) {
     return false;
   }
   // Sort by start time (har ordered by finish time?)
   std::sort(har->requests.begin(),
             har->requests.end(),
             [](const HTTPMessage& a, const HTTPMessage& b) {
               return a.getStartTime() < b.getStartTime();
             });
   TimePoint last = har->requests[0].getStartTime();
   std::chrono::milliseconds cumulativeDelay(0);
   uint16_t index = 0;
   for (HTTPMessage& msg : har->requests) {
     auto delayFromPrevious = millisecondsBetween(msg.getStartTime(), last);
     // If there was a quiescent gap in the HAR of at least some value, shrink
     // it so the test doesn't last forever
     if (delayFromPrevious > std::chrono::milliseconds(1000)) {
       delayFromPrevious = std::chrono::milliseconds(1000);
     }
     last = msg.getStartTime();
     cumulativeDelay += delayFromPrevious;
     setupRequest(index++, std::move(msg), cumulativeDelay);
   }
   for (auto& kv : domains_) {
     flushRequests(kv.second.get());
   }
   return true;
 }

void proxygen::compress::CompressionSimulator::recvAck	(	CompressionScheme *	scheme,
		std::unique_ptr< CompressionScheme::Ack >	ack
	)

private

Definition at line 347 of file CompressionSimulator.cpp.

References folly::gen::move, and proxygen::compress::CompressionScheme::recvAck().

Referenced by CompressionSimulator().

                                                                          {
   scheme->recvAck(std::move(ack));
 }

std::chrono::milliseconds proxygen::compress::CompressionSimulator::rtt ( )

private

Definition at line 373 of file CompressionSimulator.cpp.

Referenced by CompressionSimulator().

                                                 {
   return params_.rtt;
 }

void proxygen::compress::CompressionSimulator::run ( )

Definition at line 73 of file CompressionSimulator.cpp.

References folly::INFO, and uint32_t.

Referenced by CompressionSimulator(), and main().

                                {
 #ifndef HAVE_REAL_QMIN
   if (params_.type == SchemeType::QMIN) {
     LOG(INFO) << "QMIN not available";
     return;
   }
 #endif
 
   LOG(INFO) << "Starting run";
   eventBase_.loop();
   uint32_t holBlockCount = 0;
   for (auto& scheme : domains_) {
     holBlockCount += scheme.second->getHolBlockCount();
   }
   LOG(INFO) << "Complete"
             << "\nStats:"
                "\nSeed: "
             << params_.seed << "\nBlocks sent: " << requests_.size()
             << "\nAllowed OOO: " << stats_.allowedOOO
             << "\nPackets: " << stats_.packets
             << "\nPacket Losses: " << stats_.packetLosses
             << "\nHOL Block Count: " << holBlockCount
             << "\nHOL Delay (ms): " << stats_.holDelay.count()
             << "\nMax Queue Buffer Bytes: " << stats_.maxQueueBufferBytes
             << "\nUncompressed Bytes: " << stats_.uncompressed
             << "\nCompressed Bytes: " << stats_.compressed
             << "\nCompression Ratio: "
             << int(100 - double(100 * stats_.compressed) / stats_.uncompressed);
 }

std::chrono::milliseconds proxygen::compress::CompressionSimulator::rxmitDelay ( )

private

Definition at line 381 of file CompressionSimulator.cpp.

References uint32_t.

Referenced by CompressionSimulator().

                                                        {
   uint32_t ms = rtt().count() * Random::randDouble(1.1, 2, rng_);
   return std::chrono::milliseconds(ms);
 }

void proxygen::compress::CompressionSimulator::scheduleEvent	(	folly::Function< void()>	f,
		std::chrono::milliseconds	ms
	)

private

Definition at line 329 of file CompressionSimulator.cpp.

References f, and folly::gen::move.

Referenced by CompressionSimulator().

                                                                    {
   eventBase_.runAfterDelay(std::move(f), ms.count());
 }

void proxygen::compress::CompressionSimulator::sendAck	(	CompressionScheme *	scheme,
		std::unique_ptr< CompressionScheme::Ack >	ack
	)

private

Definition at line 334 of file CompressionSimulator.cpp.

References a, and folly::gen::move.

Referenced by CompressionSimulator().

                                                                          {
   if (!ack) {
     return;
   }
   // An ack is a packet
   stats_.packets++;
   scheduleEvent([a = std::move(ack),
                  this,
                  scheme]() mutable { recvAck(scheme, std::move(a)); },
                 deliveryDelay());
 }

void proxygen::compress::CompressionSimulator::setupRequest	(	uint16_t	seqn,
		HTTPMessage &&	msg,
		std::chrono::milliseconds	encodeDelay
	)

private

Definition at line 137 of file CompressionSimulator.cpp.

References proxygen::HTTP_HEADER_HOST, and value.

Referenced by CompressionSimulator().

                                                                            {
   // Normalize to relative paths
   const auto& query = msg.getQueryString();
   if (query.empty()) {
     msg.setURL(msg.getPath());
   } else {
     msg.setURL(folly::to<string>(msg.getPath(), "?", query));
   }
 
   auto scheme = getScheme(msg.getHeaders().getSingleOrEmpty(HTTP_HEADER_HOST));
   requests_.emplace_back(msg);
   auto decodeCompleteCB =
       [index, this, scheme](std::chrono::milliseconds holDelay) {
         // record processed timestamp
         CHECK(!callbacks_[index].getResult().hasError());
         DCHECK_EQ(requests_[index], *callbacks_[index].getResult().value());
         stats_.holDelay += holDelay;
         VLOG(1) << "Finished decoding request=" << index
                 << " with holDelay=" << holDelay.count()
                 << " cumulative HoL delay=" << stats_.holDelay.count();
         sendAck(scheme, scheme->getAck(callbacks_[index].seqn));
       };
   callbacks_.emplace_back(index, decodeCompleteCB);
 
   // Assume that all packets with same encodeDelay will form a packet
   // train.  Encode them as a group, so can we can emulate packet
   // boundaries more realistically, telling the encoder which blocks
   // start such trains.
   if (scheme->packetIndices.size() > 0) {
     auto delayFromPrevious = encodeDelay - scheme->prev;
     VLOG(1) << "request " << index << " delay " << delayFromPrevious.count();
     if (delayFromPrevious > std::chrono::milliseconds(1)) {
       flushRequests(scheme);
     }
   }
   scheme->prev = encodeDelay;
   scheme->packetIndices.push_back(index);
 }

Member Data Documentation

std::vector<SimStreamingCallback> proxygen::compress::CompressionSimulator::callbacks_

private

Definition at line 70 of file CompressionSimulator.h.

std::unordered_map<std::string, std::unique_ptr<CompressionScheme> > proxygen::compress::CompressionSimulator::domains_

private

Definition at line 69 of file CompressionSimulator.h.

folly::EventBase proxygen::compress::CompressionSimulator::eventBase_

private

Definition at line 67 of file CompressionSimulator.h.

SimParams proxygen::compress::CompressionSimulator::params_

private

Definition at line 65 of file CompressionSimulator.h.

std::vector<proxygen::HTTPMessage> proxygen::compress::CompressionSimulator::requests_

private

Definition at line 66 of file CompressionSimulator.h.

folly::Random::DefaultGenerator proxygen::compress::CompressionSimulator::rng_

private

Initial value:

{

static_cast<folly::Random::DefaultGenerator::result_type>(params_.seed)}

Definition at line 71 of file CompressionSimulator.h.

SimStats proxygen::compress::CompressionSimulator::stats_

private

Definition at line 73 of file CompressionSimulator.h.

The documentation for this class was generated from the following files:

proxygen/lib/http/codec/compress/experimental/simulator/CompressionSimulator.h
proxygen/lib/http/codec/compress/experimental/simulator/CompressionSimulator.cpp

Public Member Functions

Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation