folly::threadlocal_detail::StaticMetaBase Struct Reference

#include <ThreadLocalDetail.h>

Inheritance diagram for folly::threadlocal_detail::StaticMetaBase:

Classes
class	EntryID

Public Member Functions
	StaticMetaBase (ThreadEntry *(*threadEntry)(), bool strict)
void	push_back (ThreadEntry *t)
void	erase (ThreadEntry *t)
uint32_t	elementsCapacity () const
uint32_t	allocate (EntryID *ent)
void	destroy (EntryID *ent)
void	reserve (EntryID *id)
ElementWrapper &	getElement (EntryID *ent)
void	reserveHeadUnlocked (uint32_t id)
void	pushBackLocked (ThreadEntry *t, uint32_t id)
void	pushBackUnlocked (ThreadEntry *t, uint32_t id)

Static Public Member Functions
static FOLLY_EXPORT ThreadEntryList *	getThreadEntryList ()
static void	onThreadExit (void *ptr)
static ElementWrapper *	reallocate (ThreadEntry *threadEntry, uint32_t idval, size_t &newCapacity)

Public Attributes
uint32_t	nextId_
std::vector< uint32_t >	freeIds_
std::mutex	lock_
SharedMutex	accessAllThreadsLock_
pthread_key_t	pthreadKey_
ThreadEntry	head_
ThreadEntry *(*	threadEntry_ )()
bool	strict_

Protected Member Functions
	~StaticMetaBase ()

Detailed Description

Definition at line 299 of file ThreadLocalDetail.h.

Constructor & Destructor Documentation

folly::threadlocal_detail::StaticMetaBase::StaticMetaBase	(	ThreadEntry *(*)()	threadEntry,
		bool	strict
	)

Definition at line 67 of file ThreadLocalDetail.cpp.

References folly::checkPosixError(), head_, folly::threadlocal_detail::ThreadEntry::next, onThreadExit(), folly::threadlocal_detail::ThreadEntry::prev, pthreadKey_, and folly::threadlocal_detail::PthreadKeyUnregister::registerKey().

    : nextId_(1), threadEntry_(threadEntry), strict_(strict) {
  head_.next = head_.prev = &head_;
  int ret = pthread_key_create(&pthreadKey_, &onThreadExit);
  checkPosixError(ret, "pthread_key_create failed");
  PthreadKeyUnregister::registerKey(pthreadKey_);
}

folly::threadlocal_detail::StaticMetaBase::~StaticMetaBase ( )

inlineprotected

Definition at line 403 of file ThreadLocalDetail.h.

{}

Member Function Documentation

uint32_t folly::threadlocal_detail::StaticMetaBase::allocate

(

EntryID *

ent

)

Definition at line 217 of file ThreadLocalDetail.cpp.

References g(), folly::threadlocal_detail::kEntryIDInvalid, reserveHeadUnlocked(), uint32_t, and folly::threadlocal_detail::StaticMetaBase::EntryID::value.

Referenced by folly::threadlocal_detail::StaticMetaBase::EntryID::getOrAllocate().

                                              {
  uint32_t id;
  auto& meta = *this;
  std::lock_guard<std::mutex> g(meta.lock_);

  id = ent->value.load();
  if (id != kEntryIDInvalid) {
    return id;
  }

  if (!meta.freeIds_.empty()) {
    id = meta.freeIds_.back();
    meta.freeIds_.pop_back();
  } else {
    id = meta.nextId_++;
  }

  uint32_t old_id = ent->value.exchange(id);
  DCHECK_EQ(old_id, kEntryIDInvalid);

  reserveHeadUnlocked(id);

  return id;
}

void folly::threadlocal_detail::StaticMetaBase::destroy

(

EntryID *

ent

)

Definition at line 242 of file ThreadLocalDetail.cpp.

References folly::ALL_THREADS, folly::threadlocal_detail::ElementWrapper::deleter1, folly::threadlocal_detail::ThreadEntry::elements, elementsCapacity(), g(), folly::threadlocal_detail::ThreadEntry::getElementsCapacity(), folly::threadlocal_detail::kEntryIDInvalid, cpp.ast::next(), folly::threadlocal_detail::ElementWrapper::ownsDeleter, ptr, folly::threadlocal_detail::ElementWrapper::ptr, uint32_t, folly::threadlocal_detail::StaticMetaBase::EntryID::value, folly::WARNING, and folly::wlock().

                                         {
  try {
    auto& meta = *this;

    // Elements in other threads that use this id.
    std::vector<ElementWrapper> elements;

    {
      SharedMutex::WriteHolder wlock(nullptr);
      if (meta.strict_) {
        /*
         * In strict mode, the logic guarantees per-thread instances are
         * destroyed by the moment ThreadLocal<> dtor returns.
         * In order to achieve that, we should wait until concurrent
         * onThreadExit() calls (that might acquire ownership over per-thread
         * instances in order to destroy them) are finished.
         */
        wlock = SharedMutex::WriteHolder(meta.accessAllThreadsLock_);
      }

      {
        std::lock_guard<std::mutex> g(meta.lock_);
        uint32_t id = ent->value.exchange(kEntryIDInvalid);
        if (id == kEntryIDInvalid) {
          return;
        }

        auto& node = meta.head_.elements[id].node;
        while (!node.empty()) {
          auto* next = node.getNext();
          next->eraseZero();

          ThreadEntry* e = next->parent;
          auto elementsCapacity = e->getElementsCapacity();
          if (id < elementsCapacity && e->elements[id].ptr) {
            elements.push_back(e->elements[id]);

            /*
             * Writing another thread's ThreadEntry from here is fine;
             * the only other potential reader is the owning thread --
             * from onThreadExit (which grabs the lock, so is properly
             * synchronized with us) or from get(), which also grabs
             * the lock if it needs to resize the elements vector.
             *
             * We can't conflict with reads for a get(id), because
             * it's illegal to call get on a thread local that's
             * destructing.
             */
            e->elements[id].ptr = nullptr;
            e->elements[id].deleter1 = nullptr;
            e->elements[id].ownsDeleter = false;
          }
        }
        meta.freeIds_.push_back(id);
      }
    }
    // Delete elements outside the locks.
    for (ElementWrapper& elem : elements) {
      if (elem.dispose(TLPDestructionMode::ALL_THREADS)) {
        elem.cleanup();
      }
    }
  } catch (...) { // Just in case we get a lock error or something anyway...
    LOG(WARNING) << "Destructor discarding an exception that was thrown.";
  }
}

uint32_t folly::threadlocal_detail::StaticMetaBase::elementsCapacity

(

)

const

Definition at line 211 of file ThreadLocalDetail.cpp.

References FOLLY_LIKELY, and folly::threadlocal_detail::ThreadEntry::getElementsCapacity().

Referenced by destroy(), and onThreadExit().

                                                {
  ThreadEntry* threadEntry = (*threadEntry_)();

  return FOLLY_LIKELY(!!threadEntry) ? threadEntry->getElementsCapacity() : 0;
}

void folly::threadlocal_detail::StaticMetaBase::erase ( ThreadEntry *  t )

inline

Definition at line 351 of file ThreadLocalDetail.h.

References destroy(), FOLLY_EXPORT, folly::threadlocal_detail::ThreadEntry::next, folly::threadlocal_detail::ThreadEntry::prev, ptr, folly::reserve, folly::pushmi::detail::t, and uint32_t.

                             {
    t->next->prev = t->prev;
    t->prev->next = t->next;
    t->next = t->prev = t;
  }

ElementWrapper& folly::threadlocal_detail::StaticMetaBase::getElement

(

EntryID *

ent

)

ThreadEntryList * folly::threadlocal_detail::StaticMetaBase::getThreadEntryList ( )

static

Definition at line 75 of file ThreadLocalDetail.cpp.

References folly::checkPosixError(), FOLLY_ALWAYS_INLINE, pthreadKey_, folly::threadlocal_detail::PthreadKeyUnregister::registerKey(), and UNLIKELY.

                                                    {
#ifdef FOLLY_TLD_USE_FOLLY_TLS
  static FOLLY_TLS ThreadEntryList threadEntryListSingleton;
  return &threadEntryListSingleton;
#else
  class PthreadKey {
   public:
    PthreadKey() {
      int ret = pthread_key_create(&pthreadKey_, nullptr);
      checkPosixError(ret, "pthread_key_create failed");
      PthreadKeyUnregister::registerKey(pthreadKey_);
    }

    FOLLY_ALWAYS_INLINE pthread_key_t get() const {
      return pthreadKey_;
    }

   private:
    pthread_key_t pthreadKey_;
  };

  static auto instance = detail::createGlobal<PthreadKey, void>();

  ThreadEntryList* threadEntryList =
      static_cast<ThreadEntryList*>(pthread_getspecific(instance->get()));

  if (UNLIKELY(!threadEntryList)) {
    threadEntryList = new ThreadEntryList();
    int ret = pthread_setspecific(instance->get(), threadEntryList);
    checkPosixError(ret, "pthread_setspecific failed");
  }

  return threadEntryList;
#endif
}

void folly::threadlocal_detail::StaticMetaBase::onThreadExit ( void *  ptr )

static

Definition at line 111 of file ThreadLocalDetail.cpp.

References elementsCapacity(), FOR_EACH_RANGE, bm::free(), g(), i, folly::threadlocal_detail::ThreadEntry::meta, ptr, folly::rlock(), and folly::THIS_THREAD.

Referenced by StaticMetaBase().

                                           {
  auto threadEntry = static_cast<ThreadEntry*>(ptr);

  {
    auto& meta = *threadEntry->meta;

    // Make sure this ThreadEntry is available if ThreadLocal A is accessed in
    // ThreadLocal B destructor.
    pthread_setspecific(meta.pthreadKey_, threadEntry);
    SharedMutex::ReadHolder rlock(nullptr);
    if (meta.strict_) {
      rlock = SharedMutex::ReadHolder(meta.accessAllThreadsLock_);
    }
    {
      std::lock_guard<std::mutex> g(meta.lock_);
      // mark it as removed
      threadEntry->removed_ = true;
      meta.erase(&(*threadEntry));
      auto elementsCapacity = threadEntry->getElementsCapacity();
      for (size_t i = 0u; i < elementsCapacity; ++i) {
        threadEntry->elements[i].node.eraseZero();
      }
      // No need to hold the lock any longer; the ThreadEntry is private to this
      // thread now that it's been removed from meta.
    }
    // NOTE: User-provided deleter / object dtor itself may be using ThreadLocal
    // with the same Tag, so dispose() calls below may (re)create some of the
    // elements or even increase elementsCapacity, thus multiple cleanup rounds
    // may be required.
    for (bool shouldRun = true; shouldRun;) {
      shouldRun = false;
      auto elementsCapacity = threadEntry->getElementsCapacity();
      FOR_EACH_RANGE (i, 0, elementsCapacity) {
        if (threadEntry->elements[i].dispose(TLPDestructionMode::THIS_THREAD)) {
          threadEntry->elements[i].cleanup();
          shouldRun = true;
        }
      }
    }
    pthread_setspecific(meta.pthreadKey_, nullptr);
  }

  auto threadEntryList = threadEntry->list;
  DCHECK_GT(threadEntryList->count, 0u);

  --threadEntryList->count;

  if (threadEntryList->count) {
    return;
  }

  // dispose all the elements
  for (bool shouldRunOuter = true; shouldRunOuter;) {
    shouldRunOuter = false;
    auto tmp = threadEntryList->head;
    while (tmp) {
      auto& meta = *tmp->meta;
      pthread_setspecific(meta.pthreadKey_, tmp);
      SharedMutex::ReadHolder rlock(nullptr);
      if (meta.strict_) {
        rlock = SharedMutex::ReadHolder(meta.accessAllThreadsLock_);
      }
      for (bool shouldRunInner = true; shouldRunInner;) {
        shouldRunInner = false;
        auto elementsCapacity = tmp->getElementsCapacity();
        FOR_EACH_RANGE (i, 0, elementsCapacity) {
          if (tmp->elements[i].dispose(TLPDestructionMode::THIS_THREAD)) {
            tmp->elements[i].cleanup();
            shouldRunInner = true;
            shouldRunOuter = true;
          }
        }
      }
      pthread_setspecific(meta.pthreadKey_, nullptr);
      tmp = tmp->listNext;
    }
  }

  // free the entry list
  auto head = threadEntryList->head;
  threadEntryList->head = nullptr;
  while (head) {
    auto tmp = head;
    head = head->listNext;
    if (tmp->elements) {
      free(tmp->elements);
      tmp->elements = nullptr;
      tmp->setElementsCapacity(0);
    }

#ifndef FOLLY_TLD_USE_FOLLY_TLS
    delete tmp;
#endif
  }

#ifndef FOLLY_TLD_USE_FOLLY_TLS
  delete threadEntryList;
#endif
}

void folly::threadlocal_detail::StaticMetaBase::push_back ( ThreadEntry *  t )

inline

Definition at line 344 of file ThreadLocalDetail.h.

References folly::threadlocal_detail::ThreadEntry::next, folly::threadlocal_detail::ThreadEntry::prev, and folly::pushmi::detail::t.

                                 {
    t->next = &head_;
    t->prev = head_.prev;
    head_.prev->next = t;
    head_.prev = t;
  }

void folly::threadlocal_detail::StaticMetaBase::pushBackLocked	(	ThreadEntry *	t,
		uint32_t	id
	)

Definition at line 452 of file ThreadLocalDetail.cpp.

References folly::threadlocal_detail::ThreadEntry::elements, g(), head_, LIKELY, lock_, folly::threadlocal_detail::ElementWrapper::node, folly::threadlocal_detail::ThreadEntryNode::push_back(), and folly::threadlocal_detail::ThreadEntry::removed_.

Referenced by folly::threadlocal_detail::ThreadEntryNode::initIfZero().

                                                               {
  if (LIKELY(!t->removed_)) {
    std::lock_guard<std::mutex> g(lock_);
    auto* node = &t->elements[id].node;
    node->push_back(&head_);
  }
}

void folly::threadlocal_detail::StaticMetaBase::pushBackUnlocked	(	ThreadEntry *	t,
		uint32_t	id
	)

Definition at line 460 of file ThreadLocalDetail.cpp.

References folly::threadlocal_detail::ThreadEntry::elements, FOLLY_STATIC_CTOR_PRIORITY_MAX, head_, folly::threadlocal_detail::PthreadKeyUnregister::instance_, LIKELY, folly::threadlocal_detail::ElementWrapper::node, folly::threadlocal_detail::ThreadEntryNode::push_back(), and folly::threadlocal_detail::ThreadEntry::removed_.

Referenced by folly::threadlocal_detail::ThreadEntryNode::initIfZero().

                                                                 {
  if (LIKELY(!t->removed_)) {
    auto* node = &t->elements[id].node;
    node->push_back(&head_);
  }
}

ElementWrapper * folly::threadlocal_detail::StaticMetaBase::reallocate ( ThreadEntry *  threadEntry, uint32_t  idval, size_t &  newCapacity  )

static

Definition at line 309 of file ThreadLocalDetail.cpp.

References folly::threadlocal_detail::ThreadEntry::elements, folly::threadlocal_detail::ThreadEntry::getElementsCapacity(), head_, folly::jemallocMinInPlaceExpandable, kBigGrowthFactor, kSmallGrowthFactor, mallocx, MALLOCX_ZERO, folly::threadlocal_detail::ThreadEntry::meta, nallocx, folly::detail::success, folly::usingJEMalloc(), and xallocx.

Referenced by reserve(), and reserveHeadUnlocked().

                         {
  size_t prevCapacity = threadEntry->getElementsCapacity();

  // Growth factor < 2, see folly/docs/FBVector.md; + 5 to prevent
  // very slow start.
  auto smallCapacity = static_cast<size_t>((idval + 5) * kSmallGrowthFactor);
  auto bigCapacity = static_cast<size_t>((idval + 5) * kBigGrowthFactor);

  newCapacity =
      (threadEntry->meta &&
       (bigCapacity <= threadEntry->meta->head_.getElementsCapacity()))
      ? bigCapacity
      : smallCapacity;

  assert(newCapacity > prevCapacity);
  ElementWrapper* reallocated = nullptr;

  // Need to grow. Note that we can't call realloc, as elements is
  // still linked in meta, so another thread might access invalid memory
  // after realloc succeeds. We'll copy by hand and update our ThreadEntry
  // under the lock.
  if (usingJEMalloc()) {
    bool success = false;
    size_t newByteSize = nallocx(newCapacity * sizeof(ElementWrapper), 0);

    // Try to grow in place.
    //
    // Note that xallocx(MALLOCX_ZERO) will only zero newly allocated memory,
    // even if a previous allocation allocated more than we requested.
    // This is fine; we always use MALLOCX_ZERO with jemalloc and we
    // always expand our allocation to the real size.
    if (prevCapacity * sizeof(ElementWrapper) >= jemallocMinInPlaceExpandable) {
      success =
          (xallocx(threadEntry->elements, newByteSize, 0, MALLOCX_ZERO) ==
           newByteSize);
    }

    // In-place growth failed.
    if (!success) {
      success =
          ((reallocated = static_cast<ElementWrapper*>(
                mallocx(newByteSize, MALLOCX_ZERO))) != nullptr);
    }

    if (success) {
      // Expand to real size
      assert(newByteSize / sizeof(ElementWrapper) >= newCapacity);
      newCapacity = newByteSize / sizeof(ElementWrapper);
    } else {
      throw std::bad_alloc();
    }
  } else { // no jemalloc
    // calloc() is simpler than malloc() followed by memset(), and
    // potentially faster when dealing with a lot of memory, as it can get
    // already-zeroed pages from the kernel.
    reallocated = static_cast<ElementWrapper*>(
        calloc(newCapacity, sizeof(ElementWrapper)));
    if (!reallocated) {
      throw std::bad_alloc();
    }
  }

  return reallocated;
}

void folly::threadlocal_detail::StaticMetaBase::reserve

(

EntryID *

id

)

Reserve enough space in the ThreadEntry::elements for the item to fit in.

Definition at line 382 of file ThreadLocalDetail.cpp.

References folly::threadlocal_detail::ThreadEntry::elements, bm::free(), g(), folly::threadlocal_detail::ThreadEntry::getElementsCapacity(), i, folly::threadlocal_detail::ThreadEntryNode::initZero(), folly::threadlocal_detail::ElementWrapper::node, reallocate(), folly::threadlocal_detail::ThreadEntry::setElementsCapacity(), folly::f14::swap(), and uint32_t.

                                        {
  auto& meta = *this;
  ThreadEntry* threadEntry = (*threadEntry_)();
  size_t prevCapacity = threadEntry->getElementsCapacity();

  uint32_t idval = id->getOrAllocate(meta);
  if (prevCapacity > idval) {
    return;
  }

  size_t newCapacity;
  ElementWrapper* reallocated = reallocate(threadEntry, idval, newCapacity);

  // Success, update the entry
  {
    std::lock_guard<std::mutex> g(meta.lock_);

    if (prevCapacity == 0) {
      meta.push_back(threadEntry);
    }

    if (reallocated) {
      /*
       * Note: we need to hold the meta lock when copying data out of
       * the old vector, because some other thread might be
       * destructing a ThreadLocal and writing to the elements vector
       * of this thread.
       */
      if (prevCapacity != 0) {
        memcpy(
            reallocated,
            threadEntry->elements,
            sizeof(*reallocated) * prevCapacity);
      }
      std::swap(reallocated, threadEntry->elements);
    }

    for (size_t i = prevCapacity; i < newCapacity; i++) {
      threadEntry->elements[i].node.initZero(threadEntry, i);
    }

    threadEntry->setElementsCapacity(newCapacity);
  }

  free(reallocated);
}

void folly::threadlocal_detail::StaticMetaBase::reserveHeadUnlocked

(

uint32_t

id

)

Definition at line 429 of file ThreadLocalDetail.cpp.

References folly::threadlocal_detail::ThreadEntry::elements, bm::free(), folly::threadlocal_detail::ThreadEntry::getElementsCapacity(), head_, i, folly::threadlocal_detail::ThreadEntryNode::init(), folly::threadlocal_detail::ElementWrapper::node, reallocate(), folly::threadlocal_detail::ThreadEntry::setElementsCapacity(), and folly::f14::swap().

Referenced by allocate().

                                                    {
  if (head_.getElementsCapacity() <= id) {
    size_t prevCapacity = head_.getElementsCapacity();
    size_t newCapacity;
    ElementWrapper* reallocated = reallocate(&head_, id, newCapacity);

    if (reallocated) {
      if (prevCapacity != 0) {
        memcpy(
            reallocated, head_.elements, sizeof(*reallocated) * prevCapacity);
      }
      std::swap(reallocated, head_.elements);
    }

    for (size_t i = prevCapacity; i < newCapacity; i++) {
      head_.elements[i].node.init(&head_, i);
    }

    head_.setElementsCapacity(newCapacity);
    free(reallocated);
  }
}

Member Data Documentation

SharedMutex folly::threadlocal_detail::StaticMetaBase::accessAllThreadsLock_

Definition at line 396 of file ThreadLocalDetail.h.

std::vector<uint32_t> folly::threadlocal_detail::StaticMetaBase::freeIds_

Definition at line 394 of file ThreadLocalDetail.h.

ThreadEntry folly::threadlocal_detail::StaticMetaBase::head_

Definition at line 398 of file ThreadLocalDetail.h.

Referenced by folly::ThreadLocalPtr< T, Tag, AccessMode >::Accessor::Iterator::decrementToValid(), folly::ThreadLocalPtr< T, Tag, AccessMode >::Accessor::Iterator::incrementToValid(), pushBackLocked(), pushBackUnlocked(), reallocate(), reserveHeadUnlocked(), and StaticMetaBase().

std::mutex folly::threadlocal_detail::StaticMetaBase::lock_

Definition at line 395 of file ThreadLocalDetail.h.

Referenced by pushBackLocked().

uint32_t folly::threadlocal_detail::StaticMetaBase::nextId_

Definition at line 393 of file ThreadLocalDetail.h.

pthread_key_t folly::threadlocal_detail::StaticMetaBase::pthreadKey_

Definition at line 397 of file ThreadLocalDetail.h.

Referenced by getThreadEntryList(), and StaticMetaBase().

bool folly::threadlocal_detail::StaticMetaBase::strict_

Definition at line 400 of file ThreadLocalDetail.h.

ThreadEntry*(* folly::threadlocal_detail::StaticMetaBase::threadEntry_) ()

Definition at line 399 of file ThreadLocalDetail.h.

---

The documentation for this struct was generated from the following files:

• proxygen/folly/folly/detail/ThreadLocalDetail.h
• proxygen/folly/folly/detail/ThreadLocalDetail.cpp