25 #include <type_traits> 240 #ifndef FOLLY_SHAREDMUTEX_TLS 242 #define FOLLY_SHAREDMUTEX_TLS FOLLY_TLS 244 #define FOLLY_SHAREDMUTEX_TLS 269 typename Tag_ = void,
270 template <
typename>
class Atom = std::atomic,
271 bool BlockImmediately =
false,
275 static constexpr
bool kReaderPriority = ReaderPriority;
301 auto state = state_.load(std::memory_order_relaxed);
303 cleanupTokenlessSharedDeferred(
state);
314 assert((
state & ~(kWaitingAny | kMayDefer | kAnnotationCreated)) == 0);
315 if ((
state & kMayDefer) != 0) {
316 for (
uint32_t slot = 0; slot < kMaxDeferredReaders; ++slot) {
317 auto slotValue = deferredReader(slot)->load(std::memory_order_relaxed);
318 assert(!slotValueIsThis(slotValue));
327 (void)lockExclusiveImpl(kHasSolo, ctx);
333 auto result = lockExclusiveImpl(kHasSolo, ctx);
338 template <
class Rep,
class Period>
339 bool try_lock_for(
const std::chrono::duration<Rep, Period>& duration) {
340 WaitForDuration<Rep, Period> ctx(duration);
341 auto result = lockExclusiveImpl(kHasSolo, ctx);
346 template <
class Clock,
class Duration>
348 const std::chrono::time_point<Clock, Duration>& absDeadline) {
349 WaitUntilDeadline<Clock, Duration> ctx{absDeadline};
350 auto result = lockExclusiveImpl(kHasSolo, ctx);
360 auto state = (state_ &= ~(kWaitingNotS | kPrevDefer | kHasE));
361 assert((
state & ~(kWaitingAny | kAnnotationCreated)) == 0);
362 wakeRegisteredWaiters(
state, kWaitingE | kWaitingU | kWaitingS);
369 (void)lockSharedImpl(
nullptr, ctx);
375 (void)lockSharedImpl(&token, ctx);
381 auto result = lockSharedImpl(
nullptr, ctx);
388 auto result = lockSharedImpl(&token, ctx);
393 template <
class Rep,
class Period>
395 WaitForDuration<Rep, Period> ctx(duration);
396 auto result = lockSharedImpl(
nullptr, ctx);
401 template <
class Rep,
class Period>
403 const std::chrono::duration<Rep, Period>& duration,
405 WaitForDuration<Rep, Period> ctx(duration);
406 auto result = lockSharedImpl(&token, ctx);
411 template <
class Clock,
class Duration>
413 const std::chrono::time_point<Clock, Duration>& absDeadline) {
414 WaitUntilDeadline<Clock, Duration> ctx{absDeadline};
415 auto result = lockSharedImpl(
nullptr, ctx);
420 template <
class Clock,
class Duration>
422 const std::chrono::time_point<Clock, Duration>& absDeadline,
424 WaitUntilDeadline<Clock, Duration> ctx{absDeadline};
425 auto result = lockSharedImpl(&token, ctx);
433 auto state = state_.load(std::memory_order_acquire);
436 assert((
state & (kPrevDefer | kHasE | kBegunE)) != kPrevDefer);
441 if ((
state & (kMayDefer | kPrevDefer)) == 0 ||
442 !tryUnlockTokenlessSharedDeferred()) {
445 unlockSharedInline();
453 token.
type_ == Token::Type::INLINE_SHARED ||
454 token.
type_ == Token::Type::DEFERRED_SHARED);
456 if (token.
type_ != Token::Type::DEFERRED_SHARED ||
457 !tryUnlockSharedDeferred(token.
slot_)) {
458 unlockSharedInline();
461 token.
type_ = Token::Type::INVALID;
474 auto state = state_.load(std::memory_order_acquire);
477 (
state & ~(kWaitingAny | kPrevDefer | kAnnotationCreated)) == kHasE);
478 }
while (!state_.compare_exchange_strong(
479 state, (
state & ~(kWaitingAny | kPrevDefer | kHasE)) + kIncrHasS));
480 if ((
state & (kWaitingE | kWaitingU | kWaitingS)) != 0) {
481 futexWakeAll(kWaitingE | kWaitingU | kWaitingS);
486 unlock_and_lock_shared();
487 token.
type_ = Token::Type::INLINE_SHARED;
492 (void)lockUpgradeImpl(ctx);
499 auto result = lockUpgradeImpl(ctx);
504 template <
class Rep,
class Period>
506 const std::chrono::duration<Rep, Period>& duration) {
507 WaitForDuration<Rep, Period> ctx(duration);
508 auto result = lockUpgradeImpl(ctx);
513 template <
class Clock,
class Duration>
515 const std::chrono::time_point<Clock, Duration>& absDeadline) {
516 WaitUntilDeadline<Clock, Duration> ctx{absDeadline};
517 auto result = lockUpgradeImpl(ctx);
524 auto state = (state_ -= kHasU);
525 assert((
state & (kWaitingNotS | kHasSolo)) == 0);
526 wakeRegisteredWaiters(
state, kWaitingE | kWaitingU);
532 (void)lockExclusiveImpl(0, ctx);
540 auto state = (state_ -= kHasU - kIncrHasS);
541 assert((
state & (kWaitingNotS | kHasSolo)) == 0);
542 wakeRegisteredWaiters(
state, kWaitingE | kWaitingU);
546 unlock_upgrade_and_lock_shared();
547 token.
type_ = Token::Type::INLINE_SHARED;
556 auto state = state_.load(std::memory_order_acquire);
559 (
state & ~(kWaitingAny | kPrevDefer | kAnnotationCreated)) == kHasE);
561 (
state & ~(kWaitingNotS | kWaitingS | kPrevDefer | kHasE)) + kHasU;
562 if (state_.compare_exchange_strong(
state, after)) {
563 if ((
state & kWaitingS) != 0) {
564 futexWakeAll(kWaitingS);
616 template <
class Rep,
class Period>
623 : duration_(duration), deadlineComputed_(false) {}
626 if (!deadlineComputed_) {
628 deadlineComputed_ =
true;
634 return duration_.count() > 0;
651 template <
class Clock,
class Duration>
673 if (AnnotateForThreadSanitizer &&
674 (state_.load() & kAnnotationCreated) == 0) {
677 if ((state_.load() & kAnnotationCreated) == 0) {
678 state_.fetch_or(kAnnotationCreated);
680 &state_,
sizeof(state_),
"init TSAN", __FILE__, __LINE__);
687 if (AnnotateForThreadSanitizer) {
688 annotateLazyCreate();
694 if (AnnotateForThreadSanitizer) {
695 annotateLazyCreate();
701 if (AnnotateForThreadSanitizer) {
702 annotateLazyCreate();
708 if (AnnotateForThreadSanitizer) {
709 assert((state_.load() & kAnnotationCreated) != 0);
726 static constexpr
uint32_t kHasS = ~(kIncrHasS - 1);
731 static constexpr
uint32_t kAnnotationCreated = 1 << 10;
773 static constexpr
uint32_t kHasSolo = kHasE | kBegunE | kHasU;
790 static constexpr
uint32_t kWaitingESingle = 1 << 2;
791 static constexpr
uint32_t kWaitingEMultiple = 1 << 3;
792 static constexpr
uint32_t kWaitingE = kWaitingESingle | kWaitingEMultiple;
806 kWaitingNotS | kWaitingE | kWaitingU | kWaitingS;
813 static constexpr
uint32_t kNumSharedToStartDeferring = 2;
820 static constexpr
uint32_t kMaxSpinCount = !BlockImmediately ? 1000 : 2;
828 static constexpr
uint32_t kMaxSoftYieldCount = !BlockImmediately ? 1000 : 0;
851 static constexpr
uint32_t kMaxDeferredReaders = 64;
852 static constexpr
uint32_t kDeferredSearchDistance = 2;
853 static constexpr
uint32_t kDeferredSeparationFactor = 4;
857 !(kMaxDeferredReaders & (kMaxDeferredReaders - 1)),
858 "kMaxDeferredReaders must be a power of 2");
860 !(kDeferredSearchDistance & (kDeferredSearchDistance - 1)),
861 "kDeferredSearchDistance must be a power of 2");
868 static constexpr
uint32_t kTokenStackTLSCapacity = 2;
879 static constexpr uintptr_t kTokenless = 0x1;
896 deferredReaders[kMaxDeferredReaders * kDeferredSeparationFactor];
900 template <
class WaitContext>
904 (state & (preconditionGoalMask | kMayDefer | kHasS)) == 0 &&
905 state_.compare_exchange_strong(state, (state | kHasE) & ~kHasU))) {
908 return lockExclusiveImpl(state, preconditionGoalMask, ctx);
912 template <
class WaitContext>
918 if (
UNLIKELY((state & preconditionGoalMask) != 0) &&
919 !waitForZeroBits(state, preconditionGoalMask, kWaitingE, ctx) &&
924 uint32_t after = (state & kMayDefer) == 0 ? 0 : kPrevDefer;
925 if (!kReaderPriority || (state & (kMayDefer | kHasS)) == 0) {
931 after |= (state | kHasE) & ~(kHasU | kMayDefer);
933 after |= (state | kBegunE) & ~(kHasU | kMayDefer);
935 if (state_.compare_exchange_strong(state, after)) {
949 if (
UNLIKELY((before & kMayDefer) != 0)) {
950 applyDeferredReaders(state, ctx);
953 assert((state & (kHasE | kBegunE)) != 0 && (state & kHasU) == 0);
954 if (
UNLIKELY((state & kHasS) != 0) &&
955 !waitForZeroBits(state, kHasS, kWaitingNotS, ctx) &&
961 state = (state_ &= ~(kPrevDefer | kHasE | kBegunE | kWaitingNotS));
962 wakeRegisteredWaiters(state, kWaitingE | kWaitingU | kWaitingS);
966 if (kReaderPriority && (state & kHasE) == 0) {
967 assert((state & kBegunE) != 0);
968 if (!state_.compare_exchange_strong(
969 state, (state & ~kBegunE) | kHasE)) {
980 template <
class WaitContext>
988 state = state_.load(std::memory_order_acquire);
989 if ((state & goal) == 0) {
994 if (
UNLIKELY(spinCount >= kMaxSpinCount)) {
995 return ctx.canBlock() &&
996 yieldWaitForZeroBits(state, goal, waitMask, ctx);
1001 template <
class WaitContext>
1007 #ifdef RUSAGE_THREAD 1008 struct rusage usage;
1009 std::memset(&usage, 0,
sizeof(usage));
1012 for (
uint32_t yieldCount = 0; yieldCount < kMaxSoftYieldCount;
1014 for (
int softState = 0; softState < 3; ++softState) {
1015 if (softState < 2) {
1018 #ifdef RUSAGE_THREAD 1019 getrusage(RUSAGE_THREAD, &usage);
1022 if (((state = state_.load(std::memory_order_acquire)) & goal) == 0) {
1025 if (ctx.shouldTimeOut()) {
1029 #ifdef RUSAGE_THREAD 1030 if (before >= 0 && usage.ru_nivcsw >= before + 2) {
1036 before = usage.ru_nivcsw;
1039 return futexWaitForZeroBits(state, goal, waitMask, ctx);
1042 template <
class WaitContext>
1049 waitMask == kWaitingNotS || waitMask == kWaitingE ||
1050 waitMask == kWaitingU || waitMask == kWaitingS);
1053 state = state_.load(std::memory_order_acquire);
1054 if ((state & goal) == 0) {
1059 if (waitMask == kWaitingE) {
1060 if ((state & kWaitingESingle) != 0) {
1061 after |= kWaitingEMultiple;
1063 after |= kWaitingESingle;
1071 if (after != state && !state_.compare_exchange_strong(state, after)) {
1075 if (!ctx.doWait(state_, after, waitMask)) {
1086 if (
UNLIKELY((state & wakeMask) != 0)) {
1087 wakeRegisteredWaitersImpl(state, wakeMask);
1106 if ((wakeMask & kWaitingE) == kWaitingE &&
1107 (state & wakeMask) == kWaitingE &&
1113 if ((state & wakeMask) != 0) {
1114 auto prev = state_.fetch_and(~wakeMask);
1115 if ((prev & wakeMask) != 0) {
1116 futexWakeAll(wakeMask);
1118 state = prev & ~wakeMask;
1127 return &deferredReaders[slot * kDeferredSeparationFactor];
1131 return reinterpret_cast<uintptr_t
>(
this);
1135 return tokenfulSlotValue() | kTokenless;
1139 return (slotValue & ~kTokenless) == tokenfulSlotValue();
1146 template <
class WaitContext>
1152 while (!slotValueIsThis(
1153 deferredReader(slot)->
load(std::memory_order_acquire))) {
1154 if (++slot == kMaxDeferredReaders) {
1159 if (
UNLIKELY(++spinCount >= kMaxSpinCount)) {
1160 applyDeferredReaders(state, ctx, slot);
1166 template <
class WaitContext>
1168 #ifdef RUSAGE_THREAD 1169 struct rusage usage;
1170 std::memset(&usage, 0,
sizeof(usage));
1173 for (
uint32_t yieldCount = 0; yieldCount < kMaxSoftYieldCount;
1175 for (
int softState = 0; softState < 3; ++softState) {
1176 if (softState < 2) {
1179 #ifdef RUSAGE_THREAD 1180 getrusage(RUSAGE_THREAD, &usage);
1183 while (!slotValueIsThis(
1184 deferredReader(slot)->
load(std::memory_order_acquire))) {
1185 if (++slot == kMaxDeferredReaders) {
1189 if (ctx.shouldTimeOut()) {
1194 #ifdef RUSAGE_THREAD 1195 if (before >= 0 && usage.ru_nivcsw >= before + 2) {
1199 before = usage.ru_nivcsw;
1204 for (; slot < kMaxDeferredReaders; ++slot) {
1205 auto slotPtr = deferredReader(slot);
1206 auto slotValue = slotPtr->load(std::memory_order_acquire);
1207 if (slotValueIsThis(slotValue) &&
1208 slotPtr->compare_exchange_strong(slotValue, 0)) {
1213 if (movedSlotCount > 0) {
1214 state = (state_ += movedSlotCount * kIncrHasS);
1216 assert((state & (kHasE | kBegunE)) != 0);
1221 assert(state < state + kIncrHasS);
1237 template <
class WaitContext>
1240 if ((state & (kHasS | kMayDefer | kHasE)) == 0 &&
1241 state_.compare_exchange_strong(state, state + kIncrHasS)) {
1242 if (token !=
nullptr) {
1243 token->
type_ = Token::Type::INLINE_SHARED;
1247 return lockSharedImpl(state, token, ctx);
1250 template <
class WaitContext>
1251 bool lockSharedImpl(
uint32_t&
state, Token* token, WaitContext& ctx);
1256 for (
uint32_t i = 0;
i < kMaxDeferredReaders; ++
i) {
1257 auto slotPtr = deferredReader(
i);
1258 auto slotValue = slotPtr->load(std::memory_order_relaxed);
1259 if (slotValue == tokenlessSlotValue()) {
1260 slotPtr->store(0, std::memory_order_relaxed);
1262 if ((state & kHasS) == 0) {
1269 bool tryUnlockTokenlessSharedDeferred();
1272 assert(slot < kMaxDeferredReaders);
1273 auto slotValue = tokenfulSlotValue();
1274 return deferredReader(slot)->compare_exchange_strong(slotValue, 0);
1278 uint32_t state = (state_ -= kIncrHasS);
1280 (state & (kHasE | kBegunE | kMayDefer)) != 0 ||
1281 state < state + kIncrHasS);
1282 if ((state & kHasS) == 0) {
1285 wakeRegisteredWaiters(state, kWaitingNotS);
1290 template <
class WaitContext>
1294 if (!waitForZeroBits(state, kHasSolo, kWaitingU, ctx)) {
1297 }
while (!state_.compare_exchange_strong(state, state | kHasU));
1309 lock_->lock_shared(token_);
1315 lock_->lock_shared(token_);
1319 : lock_(
rhs.lock_), token_(
rhs.token_) {
1320 rhs.lock_ =
nullptr;
1325 assert(upgraded.lock_ !=
nullptr);
1326 upgraded.lock_ =
nullptr;
1327 lock_->unlock_upgrade_and_lock_shared(token_);
1332 assert(writer.lock_ !=
nullptr);
1333 writer.lock_ =
nullptr;
1334 lock_->unlock_and_lock_shared(token_);
1352 lock_->unlock_shared(token_);
1370 lock_->lock_upgrade();
1375 lock_->lock_upgrade();
1380 assert(writer.lock_ !=
nullptr);
1381 writer.lock_ =
nullptr;
1382 lock_->unlock_and_lock_upgrade();
1386 rhs.lock_ =
nullptr;
1403 lock_->unlock_upgrade();
1430 assert(upgrade.lock_ !=
nullptr);
1431 upgrade.lock_ =
nullptr;
1432 lock_->unlock_upgrade_and_lock();
1460 rhs.lock_ =
nullptr;
1505 return lock.
try_lock_for(std::chrono::milliseconds(ms));
1521 bool ReaderPriority,
1523 template <
typename>
class Atom,
1524 bool BlockImmediately,
1525 bool AnnotateForThreadSanitizer>
1531 AnnotateForThreadSanitizer>::DeferredReaderSlot
1537 AnnotateForThreadSanitizer>::deferredReaders
1538 [kMaxDeferredReaders * kDeferredSeparationFactor] = {};
1541 bool ReaderPriority,
1543 template <
typename>
class Atom,
1544 bool BlockImmediately,
1545 bool AnnotateForThreadSanitizer>
1551 AnnotateForThreadSanitizer>::tls_lastTokenlessSlot = 0;
1554 bool ReaderPriority,
1556 template <
typename>
class Atom,
1557 bool BlockImmediately,
1558 bool AnnotateForThreadSanitizer>
1564 AnnotateForThreadSanitizer>::tls_lastDeferredReaderSlot = 0;
1567 bool ReaderPriority,
1569 template <
typename>
class Atom,
1570 bool BlockImmediately,
1571 bool AnnotateForThreadSanitizer>
1577 AnnotateForThreadSanitizer>::tryUnlockTokenlessSharedDeferred() {
1578 auto bestSlot = tls_lastTokenlessSlot;
1579 for (
uint32_t i = 0;
i < kMaxDeferredReaders; ++
i) {
1580 auto slotPtr = deferredReader(bestSlot ^
i);
1581 auto slotValue = slotPtr->load(std::memory_order_relaxed);
1582 if (slotValue == tokenlessSlotValue() &&
1583 slotPtr->compare_exchange_strong(slotValue, 0)) {
1584 tls_lastTokenlessSlot = bestSlot ^
i;
1592 bool ReaderPriority,
1594 template <
typename>
class Atom,
1595 bool BlockImmediately,
1596 bool AnnotateForThreadSanitizer>
1597 template <
class WaitContext>
1603 AnnotateForThreadSanitizer>::
1606 if (
UNLIKELY((state & kHasE) != 0) &&
1607 !waitForZeroBits(state, kHasE, kWaitingS, ctx) && ctx.canTimeOut()) {
1611 uint32_t slot = tls_lastDeferredReaderSlot;
1612 uintptr_t slotValue = 1;
1614 bool canAlreadyDefer = (state & kMayDefer) != 0;
1615 bool aboveDeferThreshold =
1616 (state & kHasS) >= (kNumSharedToStartDeferring - 1) * kIncrHasS;
1617 bool drainInProgress = ReaderPriority && (state & kBegunE) != 0;
1618 if (canAlreadyDefer || (aboveDeferThreshold && !drainInProgress)) {
1620 slotValue = deferredReader(slot)->load(std::memory_order_relaxed);
1621 if (slotValue != 0) {
1629 for (
uint32_t i = 0;
i < kDeferredSearchDistance; ++
i) {
1630 slot = bestSlot ^
i;
1631 assert(slot < kMaxDeferredReaders);
1632 slotValue = deferredReader(slot)->load(std::memory_order_relaxed);
1633 if (slotValue == 0) {
1635 tls_lastDeferredReaderSlot = slot;
1642 if (slotValue != 0) {
1644 if (state_.compare_exchange_strong(state, state + kIncrHasS)) {
1646 if (token !=
nullptr) {
1647 token->
type_ = Token::Type::INLINE_SHARED;
1656 if ((state & kMayDefer) == 0) {
1657 if (!state_.compare_exchange_strong(state, state | kMayDefer)) {
1660 if ((state & (kHasE | kMayDefer)) != kMayDefer) {
1668 bool gotSlot = deferredReader(slot)->compare_exchange_strong(
1670 token ==
nullptr ? tokenlessSlotValue() : tokenfulSlotValue());
1679 state = state_.load(std::memory_order_acquire);
1685 if (token ==
nullptr) {
1686 tls_lastTokenlessSlot = slot;
1689 if ((state & kMayDefer) != 0) {
1690 assert((state & kHasE) == 0);
1692 if (token !=
nullptr) {
1693 token->
type_ = Token::Type::DEFERRED_SHARED;
1700 if (token ==
nullptr) {
1708 if (!tryUnlockTokenlessSharedDeferred()) {
1709 unlockSharedInline();
1712 if (!tryUnlockSharedDeferred(slot)) {
1713 unlockSharedInline();
bool lockExclusiveImpl(uint32_t preconditionGoalMask, WaitContext &ctx)
bool lockUpgradeImpl(WaitContext &ctx)
bool slotValueIsThis(uintptr_t slotValue)
bool try_lock_shared_for(const std::chrono::duration< Rep, Period > &duration)
void annotateReleased(annotate_rwlock_level w)
void wakeRegisteredWaiters(uint32_t &state, uint32_t wakeMask)
bool try_lock_upgrade_for(const std::chrono::duration< Rep, Period > &duration)
friend void acquireRead(SharedMutexImpl &lock)
bool lockSharedImpl(Token *token, WaitContext &ctx)
folly::detail::Futex< Atom > Futex
bool doWait(Futex &futex, uint32_t expected, uint32_t waitMask)
SharedMutexImpl< true > SharedMutexReadPriority
static FOLLY_ALWAYS_INLINE void annotate_rwlock_released(void const volatile *const addr, annotate_rwlock_level const w, char const *const f, int const l)
constexpr bool kIsSanitizeThread
static FOLLY_ALWAYS_INLINE void annotate_rwlock_destroy(void const volatile *const addr, char const *const f, int const l)
bool try_lock_for(const std::chrono::duration< Rep, Period > &duration)
static FOLLY_TLS uint32_t tls_lastTokenlessSlot
std::chrono::steady_clock::time_point now()
Atom< std::uint32_t > Futex
uintptr_t tokenlessSlotValue()
void unlock_and_lock_shared()
WriteHolder & operator=(WriteHolder &&rhs) noexcept
UpgradeHolder(SharedMutexImpl *lock)
ReadHolder(WriteHolder &&writer)
—— Concurrent Priority Queue Implementation ——
static FOLLY_ALWAYS_INLINE void annotate_benign_race_sized(void const volatile *const addr, long const size, char const *const desc, char const *const f, int const l)
bool try_lock_shared_until(const std::chrono::time_point< Clock, Duration > &absDeadline)
requires E e noexcept(noexcept(s.error(std::move(e))))
FutexResult futexWait(const Futex *futex, uint32_t expected, uint32_t waitMask)
std::chrono::time_point< Clock, Duration > absDeadline_
UpgradeHolder(UpgradeHolder &&rhs) noexcept
#define FOLLY_SHAREDMUTEX_TLS
std::chrono::duration< Rep, Period > duration_
std::chrono::steady_clock::time_point deadline()
friend void releaseRead(SharedMutexImpl &lock)
FOLLY_PUSH_WARNING RHS rhs
bool doWait(Futex &futex, uint32_t expected, uint32_t waitMask)
ReadHolder(const SharedMutexImpl *lock)
ReadHolder(UpgradeHolder &&upgraded)
void applyDeferredReaders(uint32_t &state, WaitContext &ctx)
friend bool acquireReadWrite(SharedMutexImpl &lock, unsigned int ms)
constexpr std::size_t hardware_destructive_interference_size
bool try_lock_shared_for(const std::chrono::duration< Rep, Period > &duration, Token &token)
void unlock_and_lock_shared(Token &token)
void cleanupTokenlessSharedDeferred(uint32_t &state)
WaitForDuration(const std::chrono::duration< Rep, Period > &duration)
WriteHolder(SharedMutexImpl &lock)
bool try_lock_shared(Token &token)
WriteHolder(SharedMutexImpl *lock)
void unlock_upgrade_and_lock()
static FOLLY_TLS uint32_t tls_lastDeferredReaderSlot
static FOLLY_ALWAYS_INLINE void annotate_rwlock_try_acquired(void const volatile *const addr, annotate_rwlock_level const w, bool const result, char const *const f, int const l)
GuardImpl guard(ErrorHandler &&handler)
auto lock(Synchronized< D, M > &synchronized, Args &&...args)
WriteHolder(WriteHolder &&rhs) noexcept
uintptr_t tokenfulSlotValue()
void unlock_upgrade_and_lock_shared()
void applyDeferredReaders(uint32_t &state, WaitContext &ctx, uint32_t slot)
void annotateAcquired(annotate_rwlock_level w)
SharedMutexWritePriority SharedMutex
std::chrono::steady_clock::time_point deadline_
FutexResult futexWaitUntil(const Futex *futex, uint32_t expected, std::chrono::time_point< Clock, Duration > const &deadline, uint32_t waitMask)
bool lockExclusiveImpl(uint32_t &state, uint32_t preconditionGoalMask, WaitContext &ctx)
Atom< uintptr_t > DeferredReaderSlot
void unlock_and_lock_upgrade()
bool yieldWaitForZeroBits(uint32_t &state, uint32_t goal, uint32_t waitMask, WaitContext &ctx)
void annotateTryAcquired(bool result, annotate_rwlock_level w)
void unlock_shared(Token &token)
bool try_lock_upgrade_until(const std::chrono::time_point< Clock, Duration > &absDeadline)
bool futexWaitForZeroBits(uint32_t &state, uint32_t goal, uint32_t waitMask, WaitContext &ctx)
bool try_lock_until(const std::chrono::time_point< Clock, Duration > &absDeadline)
UpgradeHolder(SharedMutexImpl &lock)
ReadHolder(ReadHolder &&rhs) noexcept
bool doWait(Futex &futex, uint32_t expected, uint32_t waitMask)
uint32_t unlockSharedInline()
SharedMutexImpl< false, void, std::atomic, false, false > SharedMutexSuppressTSAN
ReadHolder(const SharedMutexImpl &lock)
friend bool acquireRead(SharedMutexImpl &lock, unsigned int ms)
bool waitForZeroBits(uint32_t &state, uint32_t goal, uint32_t waitMask, WaitContext &ctx)
void futexWakeAll(uint32_t wakeMask)
static FOLLY_ALWAYS_INLINE void annotate_rwlock_acquired(void const volatile *const addr, annotate_rwlock_level const w, char const *const f, int const l)
WriteHolder(UpgradeHolder &&upgrade)
SharedMutexImpl< false > SharedMutexWritePriority
ReadHolder & operator=(ReadHolder &&rhs) noexcept
void swap(SwapTrackingAlloc< T > &, SwapTrackingAlloc< T > &)
constexpr SharedMutexImpl() noexcept
std::unique_lock< std::mutex > sharedMutexAnnotationGuard(void *ptr)
UpgradeHolder & operator=(UpgradeHolder &&rhs) noexcept
void lock_shared(Token &token)
bool try_lock_shared_until(const std::chrono::time_point< Clock, Duration > &absDeadline, Token &token)
void wakeRegisteredWaitersImpl(uint32_t &state, uint32_t wakeMask)
bool doWait(Futex &, uint32_t, uint32_t)
void annotateLazyCreate()
void unlock_upgrade_and_lock_shared(Token &token)
friend void releaseReadWrite(SharedMutexImpl &lock)
bool tryUnlockSharedDeferred(uint32_t slot)
int futexWake(const Futex *futex, int count, uint32_t wakeMask)
DeferredReaderSlot * deferredReader(uint32_t slot)
void asm_volatile_pause()
UpgradeHolder(WriteHolder &&writer)
friend void acquireReadWrite(SharedMutexImpl &lock)
static FOLLY_ALWAYS_INLINE void annotate_rwlock_create(void const volatile *const addr, char const *const f, int const l)