#include "processor/processor_task.h" #include "common/task_system/progress_bar.h" #include "main/client_context.h" #include "main/database.h" #include "main/settings.h" #include "processor/execution_context.h" #include "processor/operator/order_by/order_by_merge.h" #include "processor/operator/scan/scan_node_table.h" #include "processor/operator/table_function_call.h" #include "processor/operator/table_scan/ftable_scan_function.h" #include "processor/processor.h" #include "processor/result/result_set.h" #include "storage/buffer_manager/memory_manager.h" using namespace lbug::common; namespace lbug { namespace processor { ProcessorTask::ProcessorTask(Sink* sink, ExecutionContext* executionContext) : Task{executionContext->clientContext->getCurrentSetting(main::ThreadsSetting::name) .getValue()}, sharedStateInitialized{false}, sink{sink}, executionContext{executionContext} {} void ProcessorTask::narrowToSingleMorselSource() { // A pipeline is the child-0 chain from its sink to its source, as in initLocalState. Every // other child, e.g. a hash join's build side, is a sink that runs as a child task. PhysicalOperator* op = sink; while (!op->isSource()) { op = op->getChild(0); } if (op->getOperatorType() == PhysicalOperatorType::SCAN_NODE_TABLE) { // The scan counts its morsels when its shared state is set up. No thread runs this task // yet, so the state is set up here instead of by the first run(). sink->initGlobalState(executionContext); sharedStateInitialized = true; if (op->ptrCast()->getNumMorsels() <= 1) { setSingleThreadedTask(); } return; } if (op->getOperatorType() == PhysicalOperatorType::ORDER_BY_MERGE) { if (op->ptrCast()->getNumSortedKeyBlocks() <= 1) { setSingleThreadedTask(); } return; } if (op->getOperatorType() != PhysicalOperatorType::TABLE_FUNCTION_CALL) { return; } auto* call = op->ptrCast(); if (call->getInfo().function.name != FTableScan::name || call->getInfo().bindData == nullptr) { return; } auto* bindData = call->getInfo().bindData->constPtrCast(); if (bindData->table->getNumTuples() <= bindData->morselSize) { setSingleThreadedTask(); } } void ProcessorTask::run() { // We need the lock when cloning because multiple threads can be accessing to clone, // which is not thread safe lock_t lck{taskMtx}; if (!sharedStateInitialized) { sink->initGlobalState(executionContext); sharedStateInitialized = true; } auto taskRoot = sink->copy(); lck.unlock(); // Reuse the DataChunk / ValueVector / value-buffer allocations across // executions of the same prepared statement. The old code allocated a // fresh ResultSet per ProcessorTask::run() call; for a loop like // for i in range(n): conn.execute("RETURN $i", {"i": i}) // that's a 16KB+ calloc on every iteration. // // The reusable ResultSet is cached per executing thread in the database-owned // ResultSetPool (see QueryProcessor), never in a process-wide thread_local: the cached // vectors' buffers belong to this database's MemoryManager, and a thread_local slot would // outlive Database::~Database and free (or reuse) them through a dangling MemoryManager the // next time any database runs a query on this thread. Holding the shared_ptr here keeps the // ResultSet alive even if a nested query on this thread replaces the slot. ResultSet* resultSetPtr = nullptr; std::unique_ptr ownedResultSet; std::shared_ptr pooledResultSet; auto* clientContext = executionContext->clientContext; if (auto* desc = sink->getDescriptor()) { pooledResultSet = clientContext->getDatabase()->getQueryProcessor()->getResultSetPool().getOrCreate(desc, storage::MemoryManager::Get(*clientContext)); resultSetPtr = pooledResultSet.get(); } else { // No descriptor (e.g. OrderByMerge): fall back to per-call allocation. ownedResultSet = sink->getResultSet(storage::MemoryManager::Get(*executionContext->clientContext)); resultSetPtr = ownedResultSet.get(); } taskRoot->ptrCast()->execute(resultSetPtr, executionContext); } void ProcessorTask::finalize() { ProgressBar::Get(*executionContext->clientContext)->finishPipeline(executionContext->queryID); sink->finalize(executionContext); } bool ProcessorTask::terminate() { return sink->terminate(); } } // namespace processor } // namespace lbug