folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer > Class Template Reference

#include <SharedMutex.h>

Classes
class	ReadHolder
class	UpgradeHolder
struct	WaitForDuration
struct	WaitForever
struct	WaitNever
struct	WaitUntilDeadline
class	WriteHolder

Public Types
typedef Tag_	Tag
typedef SharedMutexToken	Token
typedef Atom< uintptr_t >	DeferredReaderSlot

Public Member Functions
constexpr	SharedMutexImpl () noexcept
	SharedMutexImpl (const SharedMutexImpl &)=delete
	SharedMutexImpl (SharedMutexImpl &&)=delete
SharedMutexImpl &	operator= (const SharedMutexImpl &)=delete
SharedMutexImpl &	operator= (SharedMutexImpl &&)=delete
	~SharedMutexImpl ()
void	lock ()
bool	try_lock ()
template<class Rep , class Period >
bool	try_lock_for (const std::chrono::duration< Rep, Period > &duration)
template<class Clock , class Duration >
bool	try_lock_until (const std::chrono::time_point< Clock, Duration > &absDeadline)
void	unlock ()
void	lock_shared ()
void	lock_shared (Token &token)
bool	try_lock_shared ()
bool	try_lock_shared (Token &token)
template<class Rep , class Period >
bool	try_lock_shared_for (const std::chrono::duration< Rep, Period > &duration)
template<class Rep , class Period >
bool	try_lock_shared_for (const std::chrono::duration< Rep, Period > &duration, Token &token)
template<class Clock , class Duration >
bool	try_lock_shared_until (const std::chrono::time_point< Clock, Duration > &absDeadline)
template<class Clock , class Duration >
bool	try_lock_shared_until (const std::chrono::time_point< Clock, Duration > &absDeadline, Token &token)
void	unlock_shared ()
void	unlock_shared (Token &token)
void	unlock_and_lock_shared ()
void	unlock_and_lock_shared (Token &token)
void	lock_upgrade ()
bool	try_lock_upgrade ()
template<class Rep , class Period >
bool	try_lock_upgrade_for (const std::chrono::duration< Rep, Period > &duration)
template<class Clock , class Duration >
bool	try_lock_upgrade_until (const std::chrono::time_point< Clock, Duration > &absDeadline)
void	unlock_upgrade ()
void	unlock_upgrade_and_lock ()
void	unlock_upgrade_and_lock_shared ()
void	unlock_upgrade_and_lock_shared (Token &token)
void	unlock_and_lock_upgrade ()

Static Public Attributes
static constexpr bool	kReaderPriority = ReaderPriority
static constexpr uint32_t	kMaxDeferredReaders = 64
static constexpr uint32_t	kDeferredSearchDistance = 2
static constexpr uint32_t	kDeferredSeparationFactor = 4

Private Types
typedef folly::detail::Futex< Atom >	Futex

Private Member Functions
void	annotateLazyCreate ()
void	annotateDestroy ()
void	annotateAcquired (annotate_rwlock_level w)
void	annotateTryAcquired (bool result, annotate_rwlock_level w)
void	annotateReleased (annotate_rwlock_level w)
template<class WaitContext >
bool	lockExclusiveImpl (uint32_t preconditionGoalMask, WaitContext &ctx)
template<class WaitContext >
bool	lockExclusiveImpl (uint32_t &state, uint32_t preconditionGoalMask, WaitContext &ctx)
template<class WaitContext >
bool	waitForZeroBits (uint32_t &state, uint32_t goal, uint32_t waitMask, WaitContext &ctx)
template<class WaitContext >
bool	yieldWaitForZeroBits (uint32_t &state, uint32_t goal, uint32_t waitMask, WaitContext &ctx)
template<class WaitContext >
bool	futexWaitForZeroBits (uint32_t &state, uint32_t goal, uint32_t waitMask, WaitContext &ctx)
void	wakeRegisteredWaiters (uint32_t &state, uint32_t wakeMask)
void	wakeRegisteredWaitersImpl (uint32_t &state, uint32_t wakeMask)
void	futexWakeAll (uint32_t wakeMask)
DeferredReaderSlot *	deferredReader (uint32_t slot)
uintptr_t	tokenfulSlotValue ()
uintptr_t	tokenlessSlotValue ()
bool	slotValueIsThis (uintptr_t slotValue)
template<class WaitContext >
void	applyDeferredReaders (uint32_t &state, WaitContext &ctx)
template<class WaitContext >
void	applyDeferredReaders (uint32_t &state, WaitContext &ctx, uint32_t slot)
template<class WaitContext >
bool	lockSharedImpl (Token *token, WaitContext &ctx)
template<class WaitContext >
bool	lockSharedImpl (uint32_t &state, Token *token, WaitContext &ctx)
void	cleanupTokenlessSharedDeferred (uint32_t &state)
bool	tryUnlockTokenlessSharedDeferred ()
bool	tryUnlockSharedDeferred (uint32_t slot)
uint32_t	unlockSharedInline ()
template<class WaitContext >
bool	lockUpgradeImpl (WaitContext &ctx)

Private Attributes
Futex	state_ {}

Static Private Attributes
static constexpr uint32_t	kIncrHasS = 1 << 11
static constexpr uint32_t	kHasS = ~(kIncrHasS - 1)
static constexpr uint32_t	kAnnotationCreated = 1 << 10
static constexpr uint32_t	kMayDefer = 1 << 9
static constexpr uint32_t	kPrevDefer = 1 << 8
static constexpr uint32_t	kHasE = 1 << 7
static constexpr uint32_t	kBegunE = 1 << 6
static constexpr uint32_t	kHasU = 1 << 5
static constexpr uint32_t	kHasSolo = kHasE | kBegunE | kHasU
static constexpr uint32_t	kWaitingNotS = 1 << 4
static constexpr uint32_t	kWaitingESingle = 1 << 2
static constexpr uint32_t	kWaitingEMultiple = 1 << 3
static constexpr uint32_t	kWaitingE = kWaitingESingle | kWaitingEMultiple
static constexpr uint32_t	kWaitingU = 1 << 1
static constexpr uint32_t	kWaitingS = 1 << 0
static constexpr uint32_t	kWaitingAny
static constexpr uint32_t	kNumSharedToStartDeferring = 2
static constexpr uint32_t	kMaxSpinCount = !BlockImmediately ? 1000 : 2
static constexpr uint32_t	kMaxSoftYieldCount = !BlockImmediately ? 1000 : 0
static constexpr uint32_t	kTokenStackTLSCapacity = 2
static constexpr uintptr_t	kTokenless = 0x1
static FOLLY_TLS uint32_t	tls_lastTokenlessSlot = 0
static FOLLY_TLS uint32_t	tls_lastDeferredReaderSlot = 0
static DeferredReaderSlot	deferredReaders [kMaxDeferredReaders *kDeferredSeparationFactor] = {}

Friends
void	acquireRead (SharedMutexImpl &lock)
void	acquireReadWrite (SharedMutexImpl &lock)
void	releaseRead (SharedMutexImpl &lock)
void	releaseReadWrite (SharedMutexImpl &lock)
bool	acquireRead (SharedMutexImpl &lock, unsigned int ms)
bool	acquireReadWrite (SharedMutexImpl &lock, unsigned int ms)

Detailed Description

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>
class folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >

Definition at line 273 of file SharedMutex.h.

Member Typedef Documentation

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

typedef Atom<uintptr_t> folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::DeferredReaderSlot

Definition at line 892 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

typedef folly::detail::Futex<Atom> folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::Futex

private

Definition at line 572 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

typedef Tag_ folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::Tag

Definition at line 277 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

typedef SharedMutexToken folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::Token

Definition at line 279 of file SharedMutex.h.

Constructor & Destructor Documentation

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::SharedMutexImpl ( )

inlinenoexcept

Definition at line 285 of file SharedMutex.h.

: state_(0) {}

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::SharedMutexImpl ( const SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer > &  )

delete

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::SharedMutexImpl ( SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer > &&  )

delete

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::~SharedMutexImpl ( )

inline

Definition at line 300 of file SharedMutex.h.

                     {
    auto state = state_.load(std::memory_order_relaxed);
    if (UNLIKELY((state & kHasS) != 0)) {
      cleanupTokenlessSharedDeferred(state);
    }

#ifndef NDEBUG
    // These asserts check that everybody has released the lock before it
    // is destroyed.  If you arrive here while debugging that is likely
    // the problem.  (You could also have general heap corruption.)

    // if a futexWait fails to go to sleep because the value has been
    // changed, we don't necessarily clean up the wait bits, so it is
    // possible they will be set here in a correct system
    assert((state & ~(kWaitingAny | kMayDefer | kAnnotationCreated)) == 0);
    if ((state & kMayDefer) != 0) {
      for (uint32_t slot = 0; slot < kMaxDeferredReaders; ++slot) {
        auto slotValue = deferredReader(slot)->load(std::memory_order_relaxed);
        assert(!slotValueIsThis(slotValue));
      }
    }
#endif
    annotateDestroy();
  }

Member Function Documentation

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::annotateAcquired ( annotate_rwlock_level  w )

inlineprivate

Definition at line 693 of file SharedMutex.h.

                                                 {
    if (AnnotateForThreadSanitizer) {
      annotateLazyCreate();
      annotate_rwlock_acquired(this, w, __FILE__, __LINE__);
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::annotateDestroy ( )

inlineprivate

Definition at line 686 of file SharedMutex.h.

                         {
    if (AnnotateForThreadSanitizer) {
      annotateLazyCreate();
      annotate_rwlock_destroy(this, __FILE__, __LINE__);
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::annotateLazyCreate ( )

inlineprivate

Definition at line 672 of file SharedMutex.h.

                            {
    if (AnnotateForThreadSanitizer &&
        (state_.load() & kAnnotationCreated) == 0) {
      auto guard = detail::sharedMutexAnnotationGuard(this);
      // check again
      if ((state_.load() & kAnnotationCreated) == 0) {
        state_.fetch_or(kAnnotationCreated);
        annotate_benign_race_sized(
            &state_, sizeof(state_), "init TSAN", __FILE__, __LINE__);
        annotate_rwlock_create(this, __FILE__, __LINE__);
      }
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::annotateReleased ( annotate_rwlock_level  w )

inlineprivate

Definition at line 707 of file SharedMutex.h.

                                                 {
    if (AnnotateForThreadSanitizer) {
      assert((state_.load() & kAnnotationCreated) != 0);
      annotate_rwlock_released(this, w, __FILE__, __LINE__);
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::annotateTryAcquired ( bool  result, annotate_rwlock_level  w  )

inlineprivate

Definition at line 700 of file SharedMutex.h.

                                                                 {
    if (AnnotateForThreadSanitizer) {
      annotateLazyCreate();
      annotate_rwlock_try_acquired(this, w, result, __FILE__, __LINE__);
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class WaitContext >

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::applyDeferredReaders ( uint32_t &  state, WaitContext &  ctx  )

inlineprivate

Definition at line 1147 of file SharedMutex.h.

                                                               {
    uint32_t slot = 0;

    uint32_t spinCount = 0;
    while (true) {
      while (!slotValueIsThis(
          deferredReader(slot)->load(std::memory_order_acquire))) {
        if (++slot == kMaxDeferredReaders) {
          return;
        }
      }
      asm_volatile_pause();
      if (UNLIKELY(++spinCount >= kMaxSpinCount)) {
        applyDeferredReaders(state, ctx, slot);
        return;
      }
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class WaitContext >

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::applyDeferredReaders ( uint32_t &  state, WaitContext &  ctx, uint32_t  slot  )

inlineprivate

Definition at line 1167 of file SharedMutex.h.

                                                                              {
#ifdef RUSAGE_THREAD
    struct rusage usage;
    std::memset(&usage, 0, sizeof(usage));
    long before = -1;
#endif
    for (uint32_t yieldCount = 0; yieldCount < kMaxSoftYieldCount;
         ++yieldCount) {
      for (int softState = 0; softState < 3; ++softState) {
        if (softState < 2) {
          std::this_thread::yield();
        } else {
#ifdef RUSAGE_THREAD
          getrusage(RUSAGE_THREAD, &usage);
#endif
        }
        while (!slotValueIsThis(
            deferredReader(slot)->load(std::memory_order_acquire))) {
          if (++slot == kMaxDeferredReaders) {
            return;
          }
        }
        if (ctx.shouldTimeOut()) {
          // finish applying immediately on timeout
          break;
        }
      }
#ifdef RUSAGE_THREAD
      if (before >= 0 && usage.ru_nivcsw >= before + 2) {
        // heuristic says run queue is not empty
        break;
      }
      before = usage.ru_nivcsw;
#endif
    }

    uint32_t movedSlotCount = 0;
    for (; slot < kMaxDeferredReaders; ++slot) {
      auto slotPtr = deferredReader(slot);
      auto slotValue = slotPtr->load(std::memory_order_acquire);
      if (slotValueIsThis(slotValue) &&
          slotPtr->compare_exchange_strong(slotValue, 0)) {
        ++movedSlotCount;
      }
    }

    if (movedSlotCount > 0) {
      state = (state_ += movedSlotCount * kIncrHasS);
    }
    assert((state & (kHasE | kBegunE)) != 0);

    // if state + kIncrHasS overflows (off the end of state) then either
    // we have 2^(32-9) readers (almost certainly an application bug)
    // or we had an underflow (also a bug)
    assert(state < state + kIncrHasS);
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::cleanupTokenlessSharedDeferred ( uint32_t &  state )

inlineprivate

Definition at line 1255 of file SharedMutex.h.

                                                       {
    for (uint32_t i = 0; i < kMaxDeferredReaders; ++i) {
      auto slotPtr = deferredReader(i);
      auto slotValue = slotPtr->load(std::memory_order_relaxed);
      if (slotValue == tokenlessSlotValue()) {
        slotPtr->store(0, std::memory_order_relaxed);
        state += kIncrHasS;
        if ((state & kHasS) == 0) {
          break;
        }
      }
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

DeferredReaderSlot* folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::deferredReader ( uint32_t  slot )

inlineprivate

Definition at line 1126 of file SharedMutex.h.

                                                    {
    return &deferredReaders[slot * kDeferredSeparationFactor];
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class WaitContext >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::futexWaitForZeroBits ( uint32_t &  state, uint32_t  goal, uint32_t  waitMask, WaitContext &  ctx  )

inlineprivate

Definition at line 1043 of file SharedMutex.h.

                        {
    assert(
        waitMask == kWaitingNotS || waitMask == kWaitingE ||
        waitMask == kWaitingU || waitMask == kWaitingS);

    while (true) {
      state = state_.load(std::memory_order_acquire);
      if ((state & goal) == 0) {
        return true;
      }

      auto after = state;
      if (waitMask == kWaitingE) {
        if ((state & kWaitingESingle) != 0) {
          after |= kWaitingEMultiple;
        } else {
          after |= kWaitingESingle;
        }
      } else {
        after |= waitMask;
      }

      // CAS is better than atomic |= here, because it lets us avoid
      // setting the wait flag when the goal is concurrently achieved
      if (after != state && !state_.compare_exchange_strong(state, after)) {
        continue;
      }

      if (!ctx.doWait(state_, after, waitMask)) {
        // timed out
        return false;
      }
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::futexWakeAll ( uint32_t  wakeMask )

inlineprivate

Definition at line 1122 of file SharedMutex.h.

                                       {
    detail::futexWake(&state_, std::numeric_limits<int>::max(), wakeMask);
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::lock ( )

inline

Definition at line 325 of file SharedMutex.h.

Referenced by folly::ThreadLocalPtr< T, Tag, AccessMode >::Accessor::Accessor(), BENCHMARK(), and TEST().

              {
    WaitForever ctx;
    (void)lockExclusiveImpl(kHasSolo, ctx);
    annotateAcquired(annotate_rwlock_level::wrlock);
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::lock_shared ( )

inline

Definition at line 367 of file SharedMutex.h.

Referenced by BENCHMARK(), and TEST().

                     {
    WaitForever ctx;
    (void)lockSharedImpl(nullptr, ctx);
    annotateAcquired(annotate_rwlock_level::rdlock);
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::lock_shared ( Token &  token )

inline

Definition at line 373 of file SharedMutex.h.

                                 {
    WaitForever ctx;
    (void)lockSharedImpl(&token, ctx);
    annotateAcquired(annotate_rwlock_level::rdlock);
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::lock_upgrade ( )

inline

Definition at line 490 of file SharedMutex.h.

                      {
    WaitForever ctx;
    (void)lockUpgradeImpl(ctx);
    // For TSAN: treat upgrade locks as equivalent to read locks
    annotateAcquired(annotate_rwlock_level::rdlock);
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class WaitContext >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::lockExclusiveImpl ( uint32_t  preconditionGoalMask, WaitContext &  ctx  )

inlineprivate

Definition at line 901 of file SharedMutex.h.

                                                                          {
    uint32_t state = state_.load(std::memory_order_acquire);
    if (LIKELY(
            (state & (preconditionGoalMask | kMayDefer | kHasS)) == 0 &&
            state_.compare_exchange_strong(state, (state | kHasE) & ~kHasU))) {
      return true;
    } else {
      return lockExclusiveImpl(state, preconditionGoalMask, ctx);
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class WaitContext >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::lockExclusiveImpl ( uint32_t &  state, uint32_t  preconditionGoalMask, WaitContext &  ctx  )

inlineprivate

Definition at line 913 of file SharedMutex.h.

                        {
    while (true) {
      if (UNLIKELY((state & preconditionGoalMask) != 0) &&
          !waitForZeroBits(state, preconditionGoalMask, kWaitingE, ctx) &&
          ctx.canTimeOut()) {
        return false;
      }

      uint32_t after = (state & kMayDefer) == 0 ? 0 : kPrevDefer;
      if (!kReaderPriority || (state & (kMayDefer | kHasS)) == 0) {
        // Block readers immediately, either because we are in write
        // priority mode or because we can acquire the lock in one
        // step.  Note that if state has kHasU, then we are doing an
        // unlock_upgrade_and_lock() and we should clear it (reader
        // priority branch also does this).
        after |= (state | kHasE) & ~(kHasU | kMayDefer);
      } else {
        after |= (state | kBegunE) & ~(kHasU | kMayDefer);
      }
      if (state_.compare_exchange_strong(state, after)) {
        auto before = state;
        state = after;

        // If we set kHasE (writer priority) then no new readers can
        // arrive.  If we set kBegunE then they can still enter, but
        // they must be inline.  Either way we need to either spin on
        // deferredReaders[] slots, or inline them so that we can wait on
        // kHasS to zero itself.  deferredReaders[] is pointers, which on
        // x86_64 are bigger than futex() can handle, so we inline the
        // deferred locks instead of trying to futexWait on each slot.
        // Readers are responsible for rechecking state_ after recording
        // a deferred read to avoid atomicity problems between the state_
        // CAS and applyDeferredReader's reads of deferredReaders[].
        if (UNLIKELY((before & kMayDefer) != 0)) {
          applyDeferredReaders(state, ctx);
        }
        while (true) {
          assert((state & (kHasE | kBegunE)) != 0 && (state & kHasU) == 0);
          if (UNLIKELY((state & kHasS) != 0) &&
              !waitForZeroBits(state, kHasS, kWaitingNotS, ctx) &&
              ctx.canTimeOut()) {
            // Ugh.  We blocked new readers and other writers for a while,
            // but were unable to complete.  Move on.  On the plus side
            // we can clear kWaitingNotS because nobody else can piggyback
            // on it.
            state = (state_ &= ~(kPrevDefer | kHasE | kBegunE | kWaitingNotS));
            wakeRegisteredWaiters(state, kWaitingE | kWaitingU | kWaitingS);
            return false;
          }

          if (kReaderPriority && (state & kHasE) == 0) {
            assert((state & kBegunE) != 0);
            if (!state_.compare_exchange_strong(
                    state, (state & ~kBegunE) | kHasE)) {
              continue;
            }
          }

          return true;
        }
      }
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class WaitContext >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::lockSharedImpl ( Token *  token, WaitContext &  ctx  )

inlineprivate

Definition at line 1238 of file SharedMutex.h.

                                                      {
    uint32_t state = state_.load(std::memory_order_relaxed);
    if ((state & (kHasS | kMayDefer | kHasE)) == 0 &&
        state_.compare_exchange_strong(state, state + kIncrHasS)) {
      if (token != nullptr) {
        token->type_ = Token::Type::INLINE_SHARED;
      }
      return true;
    }
    return lockSharedImpl(state, token, ctx);
  }

template<bool ReaderPriority, typename Tag_ , template< typename > class Atom, bool BlockImmediately, bool AnnotateForThreadSanitizer>

template<class WaitContext >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::lockSharedImpl ( uint32_t &  state, Token *  token, WaitContext &  ctx  )

private

Definition at line 1604 of file SharedMutex.h.

References i, folly::SharedMutexToken::slot_, folly::SharedMutexToken::type_, uint16_t, uint32_t, and UNLIKELY.

                                                                    {
  while (true) {
    if (UNLIKELY((state & kHasE) != 0) &&
        !waitForZeroBits(state, kHasE, kWaitingS, ctx) && ctx.canTimeOut()) {
      return false;
    }

    uint32_t slot = tls_lastDeferredReaderSlot;
    uintptr_t slotValue = 1; // any non-zero value will do

    bool canAlreadyDefer = (state & kMayDefer) != 0;
    bool aboveDeferThreshold =
        (state & kHasS) >= (kNumSharedToStartDeferring - 1) * kIncrHasS;
    bool drainInProgress = ReaderPriority && (state & kBegunE) != 0;
    if (canAlreadyDefer || (aboveDeferThreshold && !drainInProgress)) {
      /* Try using the most recent slot first. */
      slotValue = deferredReader(slot)->load(std::memory_order_relaxed);
      if (slotValue != 0) {
        // starting point for our empty-slot search, can change after
        // calling waitForZeroBits
        uint32_t bestSlot =
            (uint32_t)folly::AccessSpreader<Atom>::current(kMaxDeferredReaders);

        // deferred readers are already enabled, or it is time to
        // enable them if we can find a slot
        for (uint32_t i = 0; i < kDeferredSearchDistance; ++i) {
          slot = bestSlot ^ i;
          assert(slot < kMaxDeferredReaders);
          slotValue = deferredReader(slot)->load(std::memory_order_relaxed);
          if (slotValue == 0) {
            // found empty slot
            tls_lastDeferredReaderSlot = slot;
            break;
          }
        }
      }
    }

    if (slotValue != 0) {
      // not yet deferred, or no empty slots
      if (state_.compare_exchange_strong(state, state + kIncrHasS)) {
        // successfully recorded the read lock inline
        if (token != nullptr) {
          token->type_ = Token::Type::INLINE_SHARED;
        }
        return true;
      }
      // state is updated, try again
      continue;
    }

    // record that deferred readers might be in use if necessary
    if ((state & kMayDefer) == 0) {
      if (!state_.compare_exchange_strong(state, state | kMayDefer)) {
        // keep going if CAS failed because somebody else set the bit
        // for us
        if ((state & (kHasE | kMayDefer)) != kMayDefer) {
          continue;
        }
      }
      // state = state | kMayDefer;
    }

    // try to use the slot
    bool gotSlot = deferredReader(slot)->compare_exchange_strong(
        slotValue,
        token == nullptr ? tokenlessSlotValue() : tokenfulSlotValue());

    // If we got the slot, we need to verify that an exclusive lock
    // didn't happen since we last checked.  If we didn't get the slot we
    // need to recheck state_ anyway to make sure we don't waste too much
    // work.  It is also possible that since we checked state_ someone
    // has acquired and released the write lock, clearing kMayDefer.
    // Both cases are covered by looking for the readers-possible bit,
    // because it is off when the exclusive lock bit is set.
    state = state_.load(std::memory_order_acquire);

    if (!gotSlot) {
      continue;
    }

    if (token == nullptr) {
      tls_lastTokenlessSlot = slot;
    }

    if ((state & kMayDefer) != 0) {
      assert((state & kHasE) == 0);
      // success
      if (token != nullptr) {
        token->type_ = Token::Type::DEFERRED_SHARED;
        token->slot_ = (uint16_t)slot;
      }
      return true;
    }

    // release the slot before retrying
    if (token == nullptr) {
      // We can't rely on slot.  Token-less slot values can be freed by
      // any unlock_shared(), so we need to do the full deferredReader
      // search during unlock.  Unlike unlock_shared(), we can't trust
      // kPrevDefer here.  This deferred lock isn't visible to lock()
      // (that's the whole reason we're undoing it) so there might have
      // subsequently been an unlock() and lock() with no intervening
      // transition to deferred mode.
      if (!tryUnlockTokenlessSharedDeferred()) {
        unlockSharedInline();
      }
    } else {
      if (!tryUnlockSharedDeferred(slot)) {
        unlockSharedInline();
      }
    }

    // We got here not because the lock was unavailable, but because
    // we lost a compare-and-swap.  Try-lock is typically allowed to
    // have spurious failures, but there is no lock efficiency gain
    // from exploiting that freedom here.
  }
}

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class WaitContext >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::lockUpgradeImpl ( WaitContext &  ctx )

inlineprivate

Definition at line 1291 of file SharedMutex.h.

                                         {
    uint32_t state;
    do {
      if (!waitForZeroBits(state, kHasSolo, kWaitingU, ctx)) {
        return false;
      }
    } while (!state_.compare_exchange_strong(state, state | kHasU));
    return true;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

SharedMutexImpl& folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::operator= ( const SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer > &  )

delete

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

SharedMutexImpl& folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::operator= ( SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer > &&  )

delete

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::slotValueIsThis ( uintptr_t  slotValue )

inlineprivate

Definition at line 1138 of file SharedMutex.h.

                                            {
    return (slotValue & ~kTokenless) == tokenfulSlotValue();
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

uintptr_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::tokenfulSlotValue ( )

inlineprivate

Definition at line 1130 of file SharedMutex.h.

                                {
    return reinterpret_cast<uintptr_t>(this);
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

uintptr_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::tokenlessSlotValue ( )

inlineprivate

Definition at line 1134 of file SharedMutex.h.

                                 {
    return tokenfulSlotValue() | kTokenless;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::try_lock ( )

inline

Definition at line 331 of file SharedMutex.h.

                  {
    WaitNever ctx;
    auto result = lockExclusiveImpl(kHasSolo, ctx);
    annotateTryAcquired(result, annotate_rwlock_level::wrlock);
    return result;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class Rep , class Period >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::try_lock_for ( const std::chrono::duration< Rep, Period > &  duration )

inline

Definition at line 339 of file SharedMutex.h.

                                                                    {
    WaitForDuration<Rep, Period> ctx(duration);
    auto result = lockExclusiveImpl(kHasSolo, ctx);
    annotateTryAcquired(result, annotate_rwlock_level::wrlock);
    return result;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::try_lock_shared ( )

inline

Definition at line 379 of file SharedMutex.h.

Referenced by TEST().

                         {
    WaitNever ctx;
    auto result = lockSharedImpl(nullptr, ctx);
    annotateTryAcquired(result, annotate_rwlock_level::rdlock);
    return result;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::try_lock_shared ( Token &  token )

inline

Definition at line 386 of file SharedMutex.h.

                                     {
    WaitNever ctx;
    auto result = lockSharedImpl(&token, ctx);
    annotateTryAcquired(result, annotate_rwlock_level::rdlock);
    return result;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class Rep , class Period >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::try_lock_shared_for ( const std::chrono::duration< Rep, Period > &  duration )

inline

Definition at line 394 of file SharedMutex.h.

                                                                           {
    WaitForDuration<Rep, Period> ctx(duration);
    auto result = lockSharedImpl(nullptr, ctx);
    annotateTryAcquired(result, annotate_rwlock_level::rdlock);
    return result;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class Rep , class Period >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::try_lock_shared_for ( const std::chrono::duration< Rep, Period > &  duration, Token &  token  )

inline

Definition at line 402 of file SharedMutex.h.

                    {
    WaitForDuration<Rep, Period> ctx(duration);
    auto result = lockSharedImpl(&token, ctx);
    annotateTryAcquired(result, annotate_rwlock_level::rdlock);
    return result;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class Clock , class Duration >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::try_lock_shared_until ( const std::chrono::time_point< Clock, Duration > &  absDeadline )

inline

Definition at line 412 of file SharedMutex.h.

                                                               {
    WaitUntilDeadline<Clock, Duration> ctx{absDeadline};
    auto result = lockSharedImpl(nullptr, ctx);
    annotateTryAcquired(result, annotate_rwlock_level::rdlock);
    return result;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class Clock , class Duration >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::try_lock_shared_until ( const std::chrono::time_point< Clock, Duration > &  absDeadline, Token &  token  )

inline

Definition at line 421 of file SharedMutex.h.

                    {
    WaitUntilDeadline<Clock, Duration> ctx{absDeadline};
    auto result = lockSharedImpl(&token, ctx);
    annotateTryAcquired(result, annotate_rwlock_level::rdlock);
    return result;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class Clock , class Duration >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::try_lock_until ( const std::chrono::time_point< Clock, Duration > &  absDeadline )

inline

Definition at line 347 of file SharedMutex.h.

                                                               {
    WaitUntilDeadline<Clock, Duration> ctx{absDeadline};
    auto result = lockExclusiveImpl(kHasSolo, ctx);
    annotateTryAcquired(result, annotate_rwlock_level::wrlock);
    return result;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::try_lock_upgrade ( )

inline

Definition at line 497 of file SharedMutex.h.

Referenced by TEST().

                          {
    WaitNever ctx;
    auto result = lockUpgradeImpl(ctx);
    annotateTryAcquired(result, annotate_rwlock_level::rdlock);
    return result;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class Rep , class Period >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::try_lock_upgrade_for ( const std::chrono::duration< Rep, Period > &  duration )

inline

Definition at line 505 of file SharedMutex.h.

                                                      {
    WaitForDuration<Rep, Period> ctx(duration);
    auto result = lockUpgradeImpl(ctx);
    annotateTryAcquired(result, annotate_rwlock_level::rdlock);
    return result;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class Clock , class Duration >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::try_lock_upgrade_until ( const std::chrono::time_point< Clock, Duration > &  absDeadline )

inline

Definition at line 514 of file SharedMutex.h.

                                                               {
    WaitUntilDeadline<Clock, Duration> ctx{absDeadline};
    auto result = lockUpgradeImpl(ctx);
    annotateTryAcquired(result, annotate_rwlock_level::rdlock);
    return result;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::tryUnlockSharedDeferred ( uint32_t  slot )

inlineprivate

Definition at line 1271 of file SharedMutex.h.

                                              {
    assert(slot < kMaxDeferredReaders);
    auto slotValue = tokenfulSlotValue();
    return deferredReader(slot)->compare_exchange_strong(slotValue, 0);
  }

template<bool ReaderPriority, typename Tag_ , template< typename > class Atom, bool BlockImmediately, bool AnnotateForThreadSanitizer>

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::tryUnlockTokenlessSharedDeferred ( )

private

Definition at line 1577 of file SharedMutex.h.

References Atom, i, and uint32_t.

                                                                  {
  auto bestSlot = tls_lastTokenlessSlot;
  for (uint32_t i = 0; i < kMaxDeferredReaders; ++i) {
    auto slotPtr = deferredReader(bestSlot ^ i);
    auto slotValue = slotPtr->load(std::memory_order_relaxed);
    if (slotValue == tokenlessSlotValue() &&
        slotPtr->compare_exchange_strong(slotValue, 0)) {
      tls_lastTokenlessSlot = bestSlot ^ i;
      return true;
    }
  }
  return false;
}

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::unlock ( )

inline

Definition at line 355 of file SharedMutex.h.

Referenced by BENCHMARK(), folly::ThreadLocalPtr< T, Tag, AccessMode >::Accessor::release(), and TEST().

                {
    annotateReleased(annotate_rwlock_level::wrlock);
    // It is possible that we have a left-over kWaitingNotS if the last
    // unlock_shared() that let our matching lock() complete finished
    // releasing before lock()'s futexWait went to sleep.  Clean it up now
    auto state = (state_ &= ~(kWaitingNotS | kPrevDefer | kHasE));
    assert((state & ~(kWaitingAny | kAnnotationCreated)) == 0);
    wakeRegisteredWaiters(state, kWaitingE | kWaitingU | kWaitingS);
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::unlock_and_lock_shared ( )

inline

Definition at line 465 of file SharedMutex.h.

                                {
    annotateReleased(annotate_rwlock_level::wrlock);
    annotateAcquired(annotate_rwlock_level::rdlock);
    // We can't use state_ -=, because we need to clear 2 bits (1 of which
    // has an uncertain initial state) and set 1 other.  We might as well
    // clear the relevant wake bits at the same time.  Note that since S
    // doesn't block the beginning of a transition to E (writer priority
    // can cut off new S, reader priority grabs BegunE and blocks deferred
    // S) we need to wake E as well.
    auto state = state_.load(std::memory_order_acquire);
    do {
      assert(
          (state & ~(kWaitingAny | kPrevDefer | kAnnotationCreated)) == kHasE);
    } while (!state_.compare_exchange_strong(
        state, (state & ~(kWaitingAny | kPrevDefer | kHasE)) + kIncrHasS));
    if ((state & (kWaitingE | kWaitingU | kWaitingS)) != 0) {
      futexWakeAll(kWaitingE | kWaitingU | kWaitingS);
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::unlock_and_lock_shared ( Token &  token )

inline

Definition at line 485 of file SharedMutex.h.

                                            {
    unlock_and_lock_shared();
    token.type_ = Token::Type::INLINE_SHARED;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::unlock_and_lock_upgrade ( )

inline

Definition at line 550 of file SharedMutex.h.

                                 {
    annotateReleased(annotate_rwlock_level::wrlock);
    annotateAcquired(annotate_rwlock_level::rdlock);
    // We can't use state_ -=, because we need to clear 2 bits (1 of
    // which has an uncertain initial state) and set 1 other.  We might
    // as well clear the relevant wake bits at the same time.
    auto state = state_.load(std::memory_order_acquire);
    while (true) {
      assert(
          (state & ~(kWaitingAny | kPrevDefer | kAnnotationCreated)) == kHasE);
      auto after =
          (state & ~(kWaitingNotS | kWaitingS | kPrevDefer | kHasE)) + kHasU;
      if (state_.compare_exchange_strong(state, after)) {
        if ((state & kWaitingS) != 0) {
          futexWakeAll(kWaitingS);
        }
        return;
      }
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::unlock_shared ( )

inline

Definition at line 430 of file SharedMutex.h.

Referenced by BENCHMARK(), and TEST().

                       {
    annotateReleased(annotate_rwlock_level::rdlock);

    auto state = state_.load(std::memory_order_acquire);

    // kPrevDefer can only be set if HasE or BegunE is set
    assert((state & (kPrevDefer | kHasE | kBegunE)) != kPrevDefer);

    // lock() strips kMayDefer immediately, but then copies it to
    // kPrevDefer so we can tell if the pre-lock() lock_shared() might
    // have deferred
    if ((state & (kMayDefer | kPrevDefer)) == 0 ||
        !tryUnlockTokenlessSharedDeferred()) {
      // Matching lock_shared() couldn't have deferred, or the deferred
      // lock has already been inlined by applyDeferredReaders()
      unlockSharedInline();
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::unlock_shared ( Token &  token )

inline

Definition at line 449 of file SharedMutex.h.

                                   {
    annotateReleased(annotate_rwlock_level::rdlock);

    assert(
        token.type_ == Token::Type::INLINE_SHARED ||
        token.type_ == Token::Type::DEFERRED_SHARED);

    if (token.type_ != Token::Type::DEFERRED_SHARED ||
        !tryUnlockSharedDeferred(token.slot_)) {
      unlockSharedInline();
    }
#ifndef NDEBUG
    token.type_ = Token::Type::INVALID;
#endif
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::unlock_upgrade ( )

inline

Definition at line 522 of file SharedMutex.h.

Referenced by TEST().

                        {
    annotateReleased(annotate_rwlock_level::rdlock);
    auto state = (state_ -= kHasU);
    assert((state & (kWaitingNotS | kHasSolo)) == 0);
    wakeRegisteredWaiters(state, kWaitingE | kWaitingU);
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::unlock_upgrade_and_lock ( )

inline

Definition at line 529 of file SharedMutex.h.

                                 {
    // no waiting necessary, so waitMask is empty
    WaitForever ctx;
    (void)lockExclusiveImpl(0, ctx);
    annotateReleased(annotate_rwlock_level::rdlock);
    annotateAcquired(annotate_rwlock_level::wrlock);
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::unlock_upgrade_and_lock_shared ( )

inline

Definition at line 537 of file SharedMutex.h.

                                        {
    // No need to annotate for TSAN here because we model upgrade and shared
    // locks as the same.
    auto state = (state_ -= kHasU - kIncrHasS);
    assert((state & (kWaitingNotS | kHasSolo)) == 0);
    wakeRegisteredWaiters(state, kWaitingE | kWaitingU);
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::unlock_upgrade_and_lock_shared ( Token &  token )

inline

Definition at line 545 of file SharedMutex.h.

                                                    {
    unlock_upgrade_and_lock_shared();
    token.type_ = Token::Type::INLINE_SHARED;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::unlockSharedInline ( )

inlineprivate

Definition at line 1277 of file SharedMutex.h.

                                {
    uint32_t state = (state_ -= kIncrHasS);
    assert(
        (state & (kHasE | kBegunE | kMayDefer)) != 0 ||
        state < state + kIncrHasS);
    if ((state & kHasS) == 0) {
      // Only the second half of lock() can be blocked by a non-zero
      // reader count, so that's the only thing we need to wake
      wakeRegisteredWaiters(state, kWaitingNotS);
    }
    return state;
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class WaitContext >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::waitForZeroBits ( uint32_t &  state, uint32_t  goal, uint32_t  waitMask, WaitContext &  ctx  )

inlineprivate

Definition at line 981 of file SharedMutex.h.

                        {
    uint32_t spinCount = 0;
    while (true) {
      state = state_.load(std::memory_order_acquire);
      if ((state & goal) == 0) {
        return true;
      }
      asm_volatile_pause();
      ++spinCount;
      if (UNLIKELY(spinCount >= kMaxSpinCount)) {
        return ctx.canBlock() &&
            yieldWaitForZeroBits(state, goal, waitMask, ctx);
      }
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::wakeRegisteredWaiters ( uint32_t &  state, uint32_t  wakeMask  )

inlineprivate

Definition at line 1085 of file SharedMutex.h.

                                                                 {
    if (UNLIKELY((state & wakeMask) != 0)) {
      wakeRegisteredWaitersImpl(state, wakeMask);
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::wakeRegisteredWaitersImpl ( uint32_t &  state, uint32_t  wakeMask  )

inlineprivate

Definition at line 1091 of file SharedMutex.h.

                                                                     {
    // If there are multiple lock() pending only one of them will actually
    // get to wake up, so issuing futexWakeAll will make a thundering herd.
    // There's nothing stopping us from issuing futexWake(1) instead,
    // so long as the wait bits are still an accurate reflection of
    // the waiters.  If we notice (via futexWake's return value) that
    // nobody woke up then we can try again with the normal wake-all path.
    // Note that we can't just clear the bits at that point; we need to
    // clear the bits and then issue another wakeup.
    //
    // It is possible that we wake an E waiter but an outside S grabs the
    // lock instead, at which point we should wake pending U and S waiters.
    // Rather than tracking state to make the failing E regenerate the
    // wakeup, we just disable the optimization in the case that there
    // are waiting U or S that we are eligible to wake.
    if ((wakeMask & kWaitingE) == kWaitingE &&
        (state & wakeMask) == kWaitingE &&
        detail::futexWake(&state_, 1, kWaitingE) > 0) {
      // somebody woke up, so leave state_ as is and clear it later
      return;
    }

    if ((state & wakeMask) != 0) {
      auto prev = state_.fetch_and(~wakeMask);
      if ((prev & wakeMask) != 0) {
        futexWakeAll(wakeMask);
      }
      state = prev & ~wakeMask;
    }
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

template<class WaitContext >

bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::yieldWaitForZeroBits ( uint32_t &  state, uint32_t  goal, uint32_t  waitMask, WaitContext &  ctx  )

inlineprivate

Definition at line 1002 of file SharedMutex.h.

                        {
#ifdef RUSAGE_THREAD
    struct rusage usage;
    std::memset(&usage, 0, sizeof(usage));
    long before = -1;
#endif
    for (uint32_t yieldCount = 0; yieldCount < kMaxSoftYieldCount;
         ++yieldCount) {
      for (int softState = 0; softState < 3; ++softState) {
        if (softState < 2) {
          std::this_thread::yield();
        } else {
#ifdef RUSAGE_THREAD
          getrusage(RUSAGE_THREAD, &usage);
#endif
        }
        if (((state = state_.load(std::memory_order_acquire)) & goal) == 0) {
          return true;
        }
        if (ctx.shouldTimeOut()) {
          return false;
        }
      }
#ifdef RUSAGE_THREAD
      if (before >= 0 && usage.ru_nivcsw >= before + 2) {
        // One involuntary csw might just be occasional background work,
        // but if we get two in a row then we guess that there is someone
        // else who can profitably use this CPU.  Fall back to futex
        break;
      }
      before = usage.ru_nivcsw;
#endif
    }
    return futexWaitForZeroBits(state, goal, waitMask, ctx);
  }

Friends And Related Function Documentation

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void acquireRead ( SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer > &  lock )

friend

Definition at line 1489 of file SharedMutex.h.

                                                 {
    lock.lock_shared();
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

bool acquireRead ( SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer > &  lock, unsigned int  ms  )

friend

Definition at line 1501 of file SharedMutex.h.

                                                                  {
    return lock.try_lock_shared_for(std::chrono::milliseconds(ms));
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void acquireReadWrite ( SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer > &  lock )

friend

Definition at line 1492 of file SharedMutex.h.

                                                      {
    lock.lock();
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

bool acquireReadWrite ( SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer > &  lock, unsigned int  ms  )

friend

Definition at line 1504 of file SharedMutex.h.

                                                                       {
    return lock.try_lock_for(std::chrono::milliseconds(ms));
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void releaseRead ( SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer > &  lock )

friend

Definition at line 1495 of file SharedMutex.h.

                                                 {
    lock.unlock_shared();
  }

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

void releaseReadWrite ( SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer > &  lock )

friend

Definition at line 1498 of file SharedMutex.h.

                                                      {
    lock.unlock();
  }

Member Data Documentation

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::DeferredReaderSlot folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::deferredReaders = {}

staticprivate

Definition at line 896 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kAnnotationCreated = 1 << 10

staticprivate

Definition at line 731 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kBegunE = 1 << 6

staticprivate

Definition at line 755 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kDeferredSearchDistance = 2

static

Definition at line 852 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kDeferredSeparationFactor = 4

static

Definition at line 853 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kHasE = 1 << 7

staticprivate

Definition at line 748 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kHasS = ~(kIncrHasS - 1)

staticprivate

Definition at line 726 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kHasSolo = kHasE | kBegunE | kHasU

staticprivate

Definition at line 773 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kHasU = 1 << 5

staticprivate

Definition at line 765 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kIncrHasS = 1 << 11

staticprivate

Definition at line 725 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kMaxDeferredReaders = 64

static

Definition at line 851 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kMaxSoftYieldCount = !BlockImmediately ? 1000 : 0

staticprivate

Definition at line 828 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kMaxSpinCount = !BlockImmediately ? 1000 : 2

staticprivate

Definition at line 820 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kMayDefer = 1 << 9

staticprivate

Definition at line 735 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kNumSharedToStartDeferring = 2

staticprivate

Definition at line 813 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kPrevDefer = 1 << 8

staticprivate

Definition at line 742 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr bool folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kReaderPriority = ReaderPriority

static

Definition at line 275 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uintptr_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kTokenless = 0x1

staticprivate

Definition at line 879 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kTokenStackTLSCapacity = 2

staticprivate

Definition at line 868 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kWaitingAny

staticprivate

Initial value:

=
      kWaitingNotS | kWaitingE | kWaitingU | kWaitingS

Definition at line 805 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kWaitingE = kWaitingESingle | kWaitingEMultiple

staticprivate

Definition at line 792 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kWaitingEMultiple = 1 << 3

staticprivate

Definition at line 791 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kWaitingESingle = 1 << 2

staticprivate

Definition at line 790 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kWaitingNotS = 1 << 4

staticprivate

Definition at line 779 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kWaitingS = 1 << 0

staticprivate

Definition at line 800 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

constexpr uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::kWaitingU = 1 << 1

staticprivate

Definition at line 796 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

Futex folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::state_ {}

private

Definition at line 715 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

FOLLY_TLS uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::tls_lastDeferredReaderSlot = 0

staticprivate

Definition at line 885 of file SharedMutex.h.

template<bool ReaderPriority, typename Tag_ = void, template< typename > class Atom = std::atomic, bool BlockImmediately = false, bool AnnotateForThreadSanitizer = kIsSanitizeThread && !ReaderPriority>

FOLLY_TLS uint32_t folly::SharedMutexImpl< ReaderPriority, Tag_, Atom, BlockImmediately, AnnotateForThreadSanitizer >::tls_lastTokenlessSlot = 0

staticprivate

Definition at line 882 of file SharedMutex.h.

---

The documentation for this class was generated from the following file:

• proxygen/folly/folly/SharedMutex.h