FiberManagerInternal.h

Go to the documentation of this file.

/*
 * Copyright 2014-present Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#pragma once

#include <functional>
#include <memory>
#include <queue>
#include <thread>
#include <type_traits>
#include <typeindex>
#include <unordered_set>
#include <vector>

#include <folly/AtomicIntrusiveLinkedList.h>
#include <folly/CPortability.h>
#include <folly/Executor.h>
#include <folly/IntrusiveList.h>
#include <folly/Likely.h>
#include <folly/Try.h>
#include <folly/functional/Invoke.h>
#include <folly/io/async/Request.h>

#include <folly/experimental/ExecutionObserver.h>
#include <folly/fibers/BoostContextCompatibility.h>
#include <folly/fibers/Fiber.h>
#include <folly/fibers/GuardPageAllocator.h>
#include <folly/fibers/TimeoutController.h>
#include <folly/fibers/traits.h>

namespace folly {

template <class T>
class Future;

namespace fibers {

class Baton;
class Fiber;
class LoopController;
class TimeoutController;

template <typename T>
class LocalType {};

class InlineFunctionRunner {
 public:
  virtual ~InlineFunctionRunner() {}

  virtual void run(folly::Function<void()> func) = 0;
};

class FiberManager : public ::folly::Executor {
 public:
  struct Options {
    static constexpr size_t kDefaultStackSize{16 * 1024};

    size_t stackSize{kDefaultStackSize};

    size_t stackSizeMultiplier{kIsSanitize ? 16 : 1};

    size_t recordStackEvery{0};

    size_t maxFibersPoolSize{1000};

    bool useGuardPages{true};

    uint32_t fibersPoolResizePeriodMs{0};

    constexpr Options() {}
  };

  using ExceptionCallback =
      folly::Function<void(std::exception_ptr, std::string)>;

  FiberManager(const FiberManager&) = delete;
  FiberManager& operator=(const FiberManager&) = delete;

  explicit FiberManager(
      std::unique_ptr<LoopController> loopController,
      Options options = Options());

  template <typename LocalT>
  FiberManager(
      LocalType<LocalT>,
      std::unique_ptr<LoopController> loopController,
      Options options = Options());

  ~FiberManager() override;

  LoopController& loopController();
  const LoopController& loopController() const;

  void loopUntilNoReady();

  void loopUntilNoReadyImpl();

  bool shouldRunLoopRemote();

  bool hasTasks() const;

  bool hasReadyTasks() const;

  void setExceptionCallback(ExceptionCallback ec);

  template <typename F>
  void addTask(F&& func);

  template <typename F>
  auto addTaskFuture(F&& func)
      -> folly::Future<typename folly::lift_unit<invoke_result_t<F>>::type>;
  template <typename F>
  void addTaskRemote(F&& func);

  template <typename F>
  auto addTaskRemoteFuture(F&& func)
      -> folly::Future<typename folly::lift_unit<invoke_result_t<F>>::type>;

  // Executor interface calls addTaskRemote
  void add(folly::Func f) override {
    addTaskRemote(std::move(f));
  }

  template <typename F, typename G>
  void addTaskFinally(F&& func, G&& finally);

  template <typename F>
  invoke_result_t<F> runInMainContext(F&& func);

  template <typename T>
  T& local();

  template <typename T>
  FOLLY_EXPORT static T& localThread();

  size_t fibersAllocated() const;

  size_t fibersPoolSize() const;

  bool hasActiveFiber() const;

  Fiber* currentFiber() const {
    return currentFiber_;
  }

  size_t stackHighWatermark() const;

  void yield();

  void setObserver(ExecutionObserver* observer);

  ExecutionObserver* getObserver();

  void setPreemptRunner(InlineFunctionRunner* preemptRunner);

  size_t runQueueSize() const {
    return readyFibers_.size() + yieldedFibers_.size();
  }

  static FiberManager& getFiberManager();
  static FiberManager* getFiberManagerUnsafe();

 private:
  friend class Baton;
  friend class Fiber;
  template <typename F>
  struct AddTaskHelper;
  template <typename F, typename G>
  struct AddTaskFinallyHelper;

  struct RemoteTask {
    template <typename F>
    explicit RemoteTask(F&& f)
        : func(std::forward<F>(f)), rcontext(RequestContext::saveContext()) {}
    template <typename F>
    RemoteTask(F&& f, const Fiber::LocalData& localData_)
        : func(std::forward<F>(f)),
          localData(std::make_unique<Fiber::LocalData>(localData_)),
          rcontext(RequestContext::saveContext()) {}
    folly::Function<void()> func;
    std::unique_ptr<Fiber::LocalData> localData;
    std::shared_ptr<RequestContext> rcontext;
    AtomicIntrusiveLinkedListHook<RemoteTask> nextRemoteTask;
  };

  void activateFiber(Fiber* fiber);
  void deactivateFiber(Fiber* fiber);

  typedef folly::IntrusiveList<Fiber, &Fiber::listHook_> FiberTailQueue;
  typedef folly::IntrusiveList<Fiber, &Fiber::globalListHook_>
      GlobalFiberTailQueue;

  Fiber* activeFiber_{nullptr}; 
  Fiber* currentFiber_{nullptr};

  FiberTailQueue readyFibers_; 
  FiberTailQueue yieldedFibers_; 
  FiberTailQueue fibersPool_; 
  GlobalFiberTailQueue allFibers_; 
  size_t fibersAllocated_{0}; 
  size_t fibersPoolSize_{0}; 
  size_t fibersActive_{0}; 
  size_t fiberId_{0}; 
  size_t maxFibersActiveLastPeriod_{0};

  std::unique_ptr<LoopController> loopController_;
  bool isLoopScheduled_{false}; 
  static FOLLY_TLS FiberManager* currentFiberManager_;

  GuardPageAllocator stackAllocator_;

  const Options options_; 
  size_t stackHighWatermark_{0};

  void ensureLoopScheduled();

  Fiber* getFiber();

  void initLocalData(Fiber& fiber);

  folly::Function<void(Fiber&)> awaitFunc_;

  folly::Function<void()> immediateFunc_;

  InlineFunctionRunner* preemptRunner_{nullptr};

  ExecutionObserver* observer_{nullptr};

  ExceptionCallback exceptionCallback_; 
  folly::AtomicIntrusiveLinkedList<Fiber, &Fiber::nextRemoteReady_>
      remoteReadyQueue_;

  folly::AtomicIntrusiveLinkedList<RemoteTask, &RemoteTask::nextRemoteTask>
      remoteTaskQueue_;

  ssize_t remoteCount_{0};

  std::shared_ptr<TimeoutController> timeoutManager_;

  struct FibersPoolResizer {
    explicit FibersPoolResizer(FiberManager& fm) : fiberManager_(fm) {}
    void operator()();

   private:
    FiberManager& fiberManager_;
  };

  FibersPoolResizer fibersPoolResizer_;
  bool fibersPoolResizerScheduled_{false};

  void doFibersPoolResizing();

  std::type_index localType_;

  void runReadyFiber(Fiber* fiber);
  void remoteReadyInsert(Fiber* fiber);

#ifdef FOLLY_SANITIZE_ADDRESS

  // These methods notify ASAN when a fiber is entered/exited so that ASAN can
  // find the right stack extents when it needs to poison/unpoison the stack.

  void registerStartSwitchStackWithAsan(
      void** saveFakeStack,
      const void* stackBase,
      size_t stackSize);
  void registerFinishSwitchStackWithAsan(
      void* fakeStack,
      const void** saveStackBase,
      size_t* saveStackSize);
  void freeFakeStack(void* fakeStack);
  void unpoisonFiberStack(const Fiber* fiber);

#endif // FOLLY_SANITIZE_ADDRESS

#ifndef _WIN32
  bool alternateSignalStackRegistered_{false};

  void registerAlternateSignalStack();
#endif
};

inline bool onFiber() {
  auto fm = FiberManager::getFiberManagerUnsafe();
  return fm ? fm->hasActiveFiber() : false;
}

template <typename F>
inline void addTask(F&& func) {
  return FiberManager::getFiberManager().addTask(std::forward<F>(func));
}

template <typename F, typename G>
inline void addTaskFinally(F&& func, G&& finally) {
  return FiberManager::getFiberManager().addTaskFinally(
      std::forward<F>(func), std::forward<G>(finally));
}

template <typename F>
typename FirstArgOf<F>::type::value_type inline await(F&& func);

template <typename F>
invoke_result_t<F> inline runInMainContext(F&& func) {
  auto fm = FiberManager::getFiberManagerUnsafe();
  if (UNLIKELY(fm == nullptr)) {
    return func();
  }
  return fm->runInMainContext(std::forward<F>(func));
}

template <typename T>
T& local() {
  auto fm = FiberManager::getFiberManagerUnsafe();
  if (fm) {
    return fm->local<T>();
  }
  return FiberManager::localThread<T>();
}

inline void yield() {
  auto fm = FiberManager::getFiberManagerUnsafe();
  if (fm) {
    fm->yield();
  } else {
    std::this_thread::yield();
  }
}
} // namespace fibers
} // namespace folly

#include <folly/fibers/FiberManagerInternal-inl.h>