// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. #include "queueService.h" #include "taskBatch.h" #include "taskContext.h" namespace Mso { //============================================================================= // QueueService implementation. //============================================================================= QueueService::QueueService(Mso::CntPtr &&scheduler) noexcept : m_scheduler{std::move(scheduler)} { m_scheduler->InitializeScheduler(this); } QueueService::~QueueService() noexcept { AwaitTermination(); } void QueueService::Post(DispatchTask &&task) noexcept { VerifyElseCrashSz(task, "The task is empty"); bool isShutdown = false; // Once m_mutex is released, another thread can run the task, and the task can drop the last outside // reference to this queue (e.g. ActiveObjectDeleter posts a task that destroys the queue's owner); the // worker then destroys the queue. Don't touch members after the unlock: keep the scheduler alive in a local. Mso::CntPtr scheduler; { std::lock_guard lock{m_mutex}; auto it = m_taskBatches.find(std::this_thread::get_id()); if (it != m_taskBatches.end()) { it->second->AddTask(std::move(task)); } else { isShutdown = m_shutdownAction.has_value(); if (!isShutdown) { m_queue.Enqueue(std::move(task)); if (m_suspendCounter == 0) { scheduler = m_scheduler; } } } } if (scheduler) { scheduler->Post(); } else if (isShutdown) { CancelTask(std::move(task)); } } bool QueueService::ShouldYield(TaskYieldReason *yieldReason) noexcept { auto setReason = [&](TaskYieldReason reason) noexcept { return yieldReason ? *yieldReason = reason : reason, true; }; std::lock_guard lock{m_mutex}; return (m_shutdownAction.has_value() && setReason(TaskYieldReason::QueueShutdown)) || (m_suspendCounter > 0 && setReason(TaskYieldReason::QueueSuspended)); } bool QueueService::IsCurrentQueue() noexcept { return TaskContext::CurrentQueue() == this; } bool QueueService::IsSerial() noexcept { return m_scheduler->IsSerial(); } bool QueueService::HasThreadAccess() noexcept { return m_scheduler->HasThreadAccess(); } void QueueService::InvokeElsePost(DispatchTask &&task) noexcept { if (HasThreadAccess()) { if (TaskContext::CurrentQueue() == this) { task.Get()->Invoke(); task = nullptr; } else { InvokeTask(std::move(task), std::nullopt); } } else { Post(std::move(task)); } } void QueueService::DeferElsePost(DispatchTask &&task) noexcept { if (TaskContext::CurrentQueue() == this) { TaskContext::CurrentContext()->Defer(std::move(task)); } else { Post(std::move(task)); } } void QueueService::BeginTaskBatching() noexcept { auto taskBatch{Mso::Make()}; std::lock_guard lock{m_mutex}; auto result = m_taskBatches.try_emplace(std::this_thread::get_id(), std::move(taskBatch)); if (result.second) { taskBatch->SetEnclosingBatch(std::move(result.first->second)); result.first->second = std::move(taskBatch); } } DispatchTask QueueService::EndTaskBatching() noexcept { Mso::CntPtr taskBatch; std::lock_guard lock{m_mutex}; auto it = m_taskBatches.find(std::this_thread::get_id()); if (it != m_taskBatches.end()) { taskBatch = std::move(it->second); if (auto enclosingBatch = taskBatch->TakeEnclosingBatch()) { it->second = std::move(enclosingBatch); } else { m_taskBatches.erase(it); } } else { taskBatch = Mso::Make(); } return {std::move(taskBatch)}; } bool QueueService::HasTaskBatching() noexcept { std::lock_guard lock{m_mutex}; return m_taskBatches.find(std::this_thread::get_id()) != m_taskBatches.end(); } bool QueueService::TryLockQueueLocalValue(SwapDispatchLocalValueCallback swapLocalValue, void **tlsValue) noexcept { return TrySwapLocalValue(swapLocalValue, tlsValue, LocalValueSwapAction::Lock); } void QueueService::UnlockLocalValue(void **tlsValue) noexcept { TrySwapLocalValue(nullptr, tlsValue, LocalValueSwapAction::Unlock); } bool QueueService::TrySwapLocalValue( SwapDispatchLocalValueCallback swapLocalValue, void **tlsValue, LocalValueSwapAction action) noexcept { std::lock_guard lock{m_mutex}; // TLS value has a different address in each thread, but all TLS values must have the same // layout between threads. We use address delta between the provided TLS value and // our internal localValueAnchor. It must be the same for a TLS value between threads. static thread_local uint8_t localValueAnchor{0}; ptrdiff_t key = static_cast(static_cast(tlsValue)) - &localValueAnchor; auto [it, added] = m_localValues.try_emplace(key, swapLocalValue); return it->second.TrySwapLocalValue(tlsValue, action); } void QueueService::Suspend() noexcept { std::lock_guard lock{m_mutex}; ++m_suspendCounter; } void QueueService::Resume() noexcept { size_t postCount{0}; Mso::CntPtr scheduler; // See Post: no member access after the unlock. { std::lock_guard lock{m_mutex}; VerifyElseCrashSz(m_suspendCounter > 0, "m_suspendCounter must not be negative"); if (--m_suspendCounter == 0) { postCount = m_queue.Size(); scheduler = m_scheduler; } } for (size_t i = 0; i < postCount; ++i) { scheduler->Post(); } } void QueueService::Shutdown(PendingTaskAction pendingTaskAction) noexcept { std::vector tasksToCancel; { std::lock_guard lock{m_mutex}; m_shutdownAction = pendingTaskAction; if (pendingTaskAction == PendingTaskAction::Cancel) { m_queue.DequeueAll(/*out*/ tasksToCancel); } } for (auto &task : tasksToCancel) { CancelTask(std::move(task)); } m_scheduler->Shutdown(); } void QueueService::AwaitTermination() noexcept { Shutdown(PendingTaskAction::Complete); m_scheduler->AwaitTermination(); } bool QueueService::HasTasks() noexcept { std::lock_guard lock{m_mutex}; return m_suspendCounter == 0 && !m_queue.IsEmpty(); } bool QueueService::TryDequeTask(/*out*/ DispatchTask &task) noexcept { std::lock_guard lock{m_mutex}; return m_suspendCounter == 0 && m_queue.TryDequeue(/*out*/ task); } void QueueService::InvokeTask( DispatchTask &&task, std::optional endTime) noexcept { TaskContext context{this, endTime}; DispatchTask taskToInvoke{std::move(task)}; taskToInvoke.Get()->Invoke(); // Call Get()->Invoke instead of operator() to flatten call stack taskToInvoke = context.TakeNextDeferredTask(); while (taskToInvoke) { taskToInvoke.Get()->Invoke(); taskToInvoke = context.TakeNextDeferredTask(); } } void QueueService::CancelTask(DispatchTask &&task) noexcept { DispatchTask taskToCancel{std::move(task)}; if (auto cancellation = query_cast(taskToCancel.Get())) { cancellation->OnCancel(); } } //============================================================================= // LocalValueEntry implementation. //============================================================================= QueueLocalValueEntry::QueueLocalValueEntry(SwapDispatchLocalValueCallback swapLocalValue) noexcept : m_swapLocalValue{swapLocalValue} {} QueueLocalValueEntry::~QueueLocalValueEntry() noexcept { m_swapLocalValue(&m_data, nullptr); } bool QueueLocalValueEntry::TrySwapLocalValue(void **tlsValue, LocalValueSwapAction action) noexcept { if (action == LocalValueSwapAction::Lock && !m_isLocked) { m_isLocked = true; } else if (action == LocalValueSwapAction::Unlock && m_isLocked) { m_isLocked = false; } else { return false; } m_swapLocalValue(&m_data, tlsValue); return true; } //============================================================================= // IDispatchQueueStatic implementation. //============================================================================= /*static*/ IDispatchQueueStatic *IDispatchQueueStatic::Instance() noexcept { return DispatchQueueStatic::Instance(); } //============================================================================= // DispatchQueueStatic implementation. //============================================================================= /*static*/ DispatchQueueStatic *DispatchQueueStatic::Instance() noexcept { static std::unique_ptr instance{new DispatchQueueStatic()}; return instance.get(); } DispatchQueue DispatchQueueStatic::CurrentQueue() noexcept { return Mso::CntPtr{TaskContext::CurrentQueue()}; } DispatchQueue const &DispatchQueueStatic::ConcurrentQueue() noexcept { static auto concurrentQueue{Mso::Make(MakeThreadPoolScheduler(0))}; return *static_cast(static_cast(&concurrentQueue)); } DispatchQueue DispatchQueueStatic::MakeSerialQueue() noexcept { return Mso::Make(MakeThreadPoolScheduler(/*maxThreads:*/ 1)); } DispatchQueue DispatchQueueStatic::MakeLooperQueue(DispatchQueueSettings const &settings) noexcept { return Mso::Make(MakeLooperScheduler(settings)); } DispatchQueue DispatchQueueStatic::MakeConcurrentQueue(uint32_t maxThreads) noexcept { return Mso::Make(MakeThreadPoolScheduler(maxThreads)); } DispatchQueue DispatchQueueStatic::MakeCustomQueue(Mso::CntPtr &&scheduler) noexcept { return Mso::Make(std::move(scheduler)); } } // namespace Mso