diff --git a/vendor/magento/module-catalog/Model/Indexer/Product/Full.php b/vendor/magento/module-catalog/Model/Indexer/Product/Full.php index bb696c5cab4..aed594ee414 100644 --- a/vendor/magento/module-catalog/Model/Indexer/Product/Full.php +++ b/vendor/magento/module-catalog/Model/Indexer/Product/Full.php @@ -6,7 +6,9 @@ namespace Magento\Catalog\Model\Indexer\Product; +use Magento\Framework\App\ObjectManager; use Magento\Framework\Indexer\ActionInterface; +use Magento\Framework\Indexer\ConfigInterface; use Magento\Framework\Indexer\IndexerRegistry; /** @@ -24,26 +26,35 @@ class Full implements ActionInterface */ private $indexerList; + /** + * @var ConfigInterface + */ + private $config; + /** * Initialize dependencies * * @param IndexerRegistry $indexerRegistry * @param string[] $indexerList + * @param ConfigInterface|null $config */ public function __construct( IndexerRegistry $indexerRegistry, - array $indexerList + array $indexerList, + ?ConfigInterface $config = null ) { $this->indexerRegistry = $indexerRegistry; $this->indexerList = $indexerList; + $this->config = $config + ?? ObjectManager::getInstance()->get(ConfigInterface::class); } /** - * {@inheritdoc} + * @inheritdoc */ public function executeFull() { - foreach ($this->indexerList as $indexerName) { + foreach ($this->getIndexerList() as $indexerName) { $indexer = $this->indexerRegistry->get($indexerName); if (!$indexer->isScheduled()) { $indexer->reindexAll(); @@ -52,12 +63,12 @@ class Full implements ActionInterface } /** - * {@inheritdoc} + * @inheritdoc */ public function executeList(array $ids) { if (!empty($ids)) { - foreach ($this->indexerList as $indexerName) { + foreach ($this->getIndexerList() as $indexerName) { $indexer = $this->indexerRegistry->get($indexerName); if (!$indexer->isScheduled()) { $indexer->reindexList($ids); @@ -67,12 +78,12 @@ class Full implements ActionInterface } /** - * {@inheritDoc} + * @inheritDoc */ public function executeRow($id) { if (!empty($id)) { - foreach ($this->indexerList as $indexerName) { + foreach ($this->getIndexerList() as $indexerName) { $indexer = $this->indexerRegistry->get($indexerName); if (!$indexer->isScheduled()) { $indexer->reindexRow($id); @@ -80,4 +91,21 @@ class Full implements ActionInterface } } } + + /** + * Returns indexers in the order according to dependency tree + * + * @return array + */ + private function getIndexerList(): array + { + $indexers = []; + foreach (array_keys($this->config->getIndexers()) as $indexerId) { + if (in_array($indexerId, $this->indexerList, true)) { + $indexers[] = $indexerId; + } + } + + return $indexers; + } } diff --git a/vendor/magento/module-catalog-rule/Model/Indexer/CatalogRuleInsertBatchSizeCalculator.php b/vendor/magento/module-catalog-rule/Model/Indexer/CatalogRuleInsertBatchSizeCalculator.php new file mode 100644 index 00000000000..c82129a954d --- /dev/null +++ b/vendor/magento/module-catalog-rule/Model/Indexer/CatalogRuleInsertBatchSizeCalculator.php @@ -0,0 +1,59 @@ +batchSizeManagement = $batchSizeManagement; + $this->defaultBatchSize = $defaultBatchSize; + } + + /** + * Retrieve validated batch size for insert operations + * + * @param AdapterInterface $connection + * @return int + */ + public function getInsertBatchSize(AdapterInterface $connection): int + { + $batchSize = $this->defaultBatchSize; + + $this->batchSizeManagement->ensureBatchSize($connection, $batchSize); + + return (int)$batchSize; + } +} diff --git a/vendor/magento/module-catalog-rule/Model/Indexer/CatalogRuleProductPriceRowSizeEstimator.php b/vendor/magento/module-catalog-rule/Model/Indexer/CatalogRuleProductPriceRowSizeEstimator.php new file mode 100644 index 00000000000..13347aaf62c --- /dev/null +++ b/vendor/magento/module-catalog-rule/Model/Indexer/CatalogRuleProductPriceRowSizeEstimator.php @@ -0,0 +1,82 @@ +resourceConnection = $resourceConnection; + $this->customerGroupCollectionFactory = $customerGroupCollectionFactory; + $this->storeManager = $storeManager; + } + + /** + * @inheritdoc + */ + public function estimateRowSize() + { + $customerGroupCount = $this->customerGroupCollectionFactory->create()->getSize(); + + $websiteCount = count($this->storeManager->getWebsites()); + + $estimatedRowsPerProduct = $customerGroupCount * $websiteCount * 2; + + $memoryPerProduct = $estimatedRowsPerProduct * self::APPROXIMATE_ROW_SIZE_BYTES; + + return (int)ceil($memoryPerProduct); + } +} diff --git a/vendor/magento/module-catalog-rule/Model/Indexer/DynamicBatchSizeCalculator.php b/vendor/magento/module-catalog-rule/Model/Indexer/DynamicBatchSizeCalculator.php new file mode 100644 index 00000000000..684db1dc384 --- /dev/null +++ b/vendor/magento/module-catalog-rule/Model/Indexer/DynamicBatchSizeCalculator.php @@ -0,0 +1,164 @@ +memoryLimit === null) { + $memoryLimit = ini_get('memory_limit'); + + if ($memoryLimit === '-1') { + $this->memoryLimit = 2 * 1024 * 1024 * 1024; + } else { + $this->memoryLimit = $this->convertToBytes($memoryLimit); + } + } + + return $this->memoryLimit; + } + + /** + * Convert PHP memory limit notation to bytes + * + * @param string $value + * @return int + */ + private function convertToBytes(string $value): int + { + $value = trim($value); + $unit = strtolower(substr($value, -1)); + $number = (int)substr($value, 0, -1); + + switch ($unit) { + case 'g': + return $number * 1024 * 1024 * 1024; + case 'm': + return $number * 1024 * 1024; + case 'k': + return $number * 1024; + default: + return (int)$value; + } + } + + /** + * Get available memory for operations (excluding Magento base usage) + * + * @return int + */ + private function getAvailableMemory(): int + { + $totalMemory = $this->getMemoryLimit(); + $currentUsage = memory_get_usage(true); + $magentoBaseOverhead = 400 * 1024 * 1024; + + $available = $totalMemory - $currentUsage - $magentoBaseOverhead; + + return max($available, 100 * 1024 * 1024); + } + + /** + * Calculate optimal batch size for attribute loading + * + * @return int + */ + public function getAttributeBatchSize(): int + { + if (isset($this->calculatedSizes['attribute_batch_size'])) { + return $this->calculatedSizes['attribute_batch_size']; + } + + $availableMemory = $this->getAvailableMemory(); + $memoryForAttributes = $availableMemory * self::ATTRIBUTE_CACHE_MEMORY_PERCENTAGE; + + $maxBatchesInMemory = $this->getMaxBatchesInMemory(); + $memoryPerBatch = $memoryForAttributes / $maxBatchesInMemory; + + $batchSize = (int)($memoryPerBatch / self::MEMORY_PER_PRODUCT_ATTRIBUTE); + + $batchSize = max(self::MIN_BATCH_SIZE, min(self::MAX_BATCH_SIZE, $batchSize)); + + $this->calculatedSizes['attribute_batch_size'] = $batchSize; + + return $batchSize; + } + + /** + * Calculate maximum number of batches to keep in memory + * + * @return int + */ + public function getMaxBatchesInMemory(): int + { + if (isset($this->calculatedSizes['max_batches'])) { + return $this->calculatedSizes['max_batches']; + } + + $availableMemory = $this->getAvailableMemory(); + $memoryForAttributes = $availableMemory * self::ATTRIBUTE_CACHE_MEMORY_PERCENTAGE; + + $estimatedBatchMemory = self::MIN_BATCH_SIZE * self::MEMORY_PER_PRODUCT_ATTRIBUTE; + $maxBatches = (int)($memoryForAttributes / $estimatedBatchMemory); + + $maxBatches = max(self::MIN_BATCHES_IN_MEMORY, min(self::MAX_BATCHES_IN_MEMORY, $maxBatches)); + + $this->calculatedSizes['max_batches'] = $maxBatches; + + return $maxBatches; + } +} diff --git a/vendor/magento/module-catalog-rule/Model/Indexer/IndexBuilder.php b/vendor/magento/module-catalog-rule/Model/Indexer/IndexBuilder.php index 3f2e039d0d6..ff94a9eada5 100644 --- a/vendor/magento/module-catalog-rule/Model/Indexer/IndexBuilder.php +++ b/vendor/magento/module-catalog-rule/Model/Indexer/IndexBuilder.php @@ -17,7 +17,10 @@ use Magento\CatalogRule\Model\Indexer\IndexBuilder\ProductLoader; use Magento\CatalogRule\Model\Indexer\IndexerTableSwapperInterface as TableSwapper; use Magento\CatalogRule\Model\ResourceModel\Rule\Collection as RuleCollection; use Magento\CatalogRule\Model\ResourceModel\Rule\CollectionFactory as RuleCollectionFactory; +use Magento\CatalogRule\Model\ResourceModel\Rule\RuleIdProvider; use Magento\CatalogRule\Model\Rule; +use Magento\CatalogRule\Model\RuleFactory; +use Magento\Customer\Api\GroupExcludedWebsiteRepositoryInterface; use Magento\Eav\Model\Config; use Magento\Framework\App\ObjectManager; use Magento\Framework\App\ResourceConnection; @@ -192,6 +195,30 @@ class IndexBuilder * @var int */ private $productBatchSize; + /** + * @var DynamicBatchSizeCalculator + */ + private $batchSizeCalculator; + + /** + * @var CatalogRuleInsertBatchSizeCalculator + */ + private $insertBatchSizeCalculator; + + /** + * @var RuleIdProvider + */ + private $ruleIdProvider; + + /** + * @var RuleFactory + */ + private $ruleFactory; + + /** + * @var GroupExcludedWebsiteRepositoryInterface + */ + private $groupExcludedWebsiteRepository; /** * @param RuleCollectionFactory $ruleCollectionFactory @@ -218,6 +245,11 @@ class IndexBuilder * @param IndexerRegistry|null $indexerRegistry * @param ReindexRuleProductsPrice|null $reindexRuleProductsPrice * @param int $productBatchSize + * @param DynamicBatchSizeCalculator|null $batchSizeCalculator + * @param CatalogRuleInsertBatchSizeCalculator|null $insertBatchSizeCalculator + * @param RuleIdProvider|null $ruleIdProvider + * @param RuleFactory|null $ruleFactory + * @param GroupExcludedWebsiteRepositoryInterface|null $groupExcludedWebsiteRepository * @SuppressWarnings(PHPMD.ExcessiveParameterList) * @SuppressWarnings(PHPMD.UnusedFormalParameter) */ @@ -245,7 +277,12 @@ class IndexBuilder ProductCollectionFactory $productCollectionFactory = null, IndexerRegistry $indexerRegistry = null, ReindexRuleProductsPrice $reindexRuleProductsPrice = null, - int $productBatchSize = 1000 + int $productBatchSize = 1000, + ?DynamicBatchSizeCalculator $batchSizeCalculator = null, + ?CatalogRuleInsertBatchSizeCalculator $insertBatchSizeCalculator = null, + ?RuleIdProvider $ruleIdProvider = null, + ?RuleFactory $ruleFactory = null, + ?GroupExcludedWebsiteRepositoryInterface $groupExcludedWebsiteRepository = null ) { $this->resource = $resource; $this->connection = $resource->getConnection(); @@ -294,6 +331,16 @@ class IndexBuilder ObjectManager::getInstance()->get(ProductCollectionFactory::class); $this->reindexRuleProductsPrice = $reindexRuleProductsPrice ?? ObjectManager::getInstance()->get(ReindexRuleProductsPrice::class); + $this->batchSizeCalculator = $batchSizeCalculator ?? + ObjectManager::getInstance()->get(DynamicBatchSizeCalculator::class); + $this->insertBatchSizeCalculator = $insertBatchSizeCalculator ?? + ObjectManager::getInstance()->get(CatalogRuleInsertBatchSizeCalculator::class); + $this->ruleIdProvider = $ruleIdProvider ?? + ObjectManager::getInstance()->get(RuleIdProvider::class); + $this->ruleFactory = $ruleFactory ?? + ObjectManager::getInstance()->get(RuleFactory::class); + $this->groupExcludedWebsiteRepository = $groupExcludedWebsiteRepository ?? + ObjectManager::getInstance()->get(GroupExcludedWebsiteRepositoryInterface::class); } /** @@ -404,11 +451,32 @@ class IndexBuilder */ protected function doReindexFull() { - foreach ($this->getAllRules() as $rule) { - $this->reindexRuleProduct->execute($rule, $this->batchCount, true); + $dynamicBatchCount = $this->insertBatchSizeCalculator->getInsertBatchSize($this->connection); + $ruleIds = $this->getActiveRuleIds(); + + $allExcludedWebsites = $this->groupExcludedWebsiteRepository->getAllExcludedWebsites(); + + foreach ($ruleIds as $ruleId) { + + $rule = $this->loadRuleById($ruleId); + if (!$rule) { + $this->logger->warning("Rule ID {$ruleId} not found, skipping"); + continue; + } + + $ruleExcludedWebsites = $this->filterExcludedWebsitesForRule($rule, $allExcludedWebsites); + $rule->setData('excluded_website_ids', $ruleExcludedWebsites); + + $this->reindexRuleProduct->execute($rule, $dynamicBatchCount, true); + + $rule->clearInstance(); + unset($rule); } - $this->reindexRuleProductPrice->execute($this->batchCount, null, true); + $priceBatchSize = $this->insertBatchSizeCalculator->getInsertBatchSize($this->connection); + + $this->reindexRuleProductPrice->execute($priceBatchSize, null, true); + $this->reindexRuleGroupWebsite->execute(true); $this->tableSwapper->swapIndexTables( @@ -705,6 +773,55 @@ class IndexBuilder return $this->ruleCollectionFactory->create()->addFieldToFilter('is_active', 1); } + /** + * Get active rule IDs only (lightweight) + * + * @return array + */ + protected function getActiveRuleIds() + { + return $this->ruleIdProvider->getActiveRuleIds(); + } + + /** + * Load a single rule by ID + * + * @param int $ruleId + * @return Rule|null + */ + protected function loadRuleById($ruleId) + { + $rule = $this->ruleFactory->create(); + $rule->load($ruleId); + + return $rule->getId() ? $rule : null; + } + + /** + * Filter excluded websites for a specific rule based on its customer groups + * + * @param Rule $rule + * @param array $allExcludedWebsites + * @return array + */ + private function filterExcludedWebsitesForRule(Rule $rule, array $allExcludedWebsites): array + { + $ruleExcludedWebsites = []; + $customerGroupIds = $rule->getCustomerGroupIds(); + + if (empty($customerGroupIds) || empty($allExcludedWebsites)) { + return $ruleExcludedWebsites; + } + + foreach ($customerGroupIds as $customerGroupId) { + if (isset($allExcludedWebsites[$customerGroupId])) { + $ruleExcludedWebsites[$customerGroupId] = $allExcludedWebsites[$customerGroupId]; + } + } + + return $ruleExcludedWebsites; + } + /** * Get active rules * diff --git a/vendor/magento/module-catalog-rule/Model/Indexer/ReindexRuleProduct.php b/vendor/magento/module-catalog-rule/Model/Indexer/ReindexRuleProduct.php index 320eb8a38ba..c8dae7186b0 100644 --- a/vendor/magento/module-catalog-rule/Model/Indexer/ReindexRuleProduct.php +++ b/vendor/magento/module-catalog-rule/Model/Indexer/ReindexRuleProduct.php @@ -1,7 +1,7 @@ resource = $resource; $this->activeTableSwitcher = $activeTableSwitcher; $this->tableSwapper = $tableSwapper; $this->localeDate = $localeDate; $this->useWebsiteTimezone = $useWebsiteTimezone; + $this->batchSizeCalculator = $batchSizeCalculator ?? + ObjectManager::getInstance()->get(DynamicBatchSizeCalculator::class); } /** @@ -84,81 +96,290 @@ class ReindexRuleProduct } $connection = $this->resource->getConnection(); - $websiteIds = $rule->getWebsiteIds(); - if (!is_array($websiteIds)) { - $websiteIds = explode(',', $websiteIds); - } + $websiteIds = $this->getWebsiteIdsAsArray($rule->getWebsiteIds()); \Magento\Framework\Profiler::start('__MATCH_PRODUCTS__'); $productIds = $rule->getMatchingProductIds(); \Magento\Framework\Profiler::stop('__MATCH_PRODUCTS__'); - $indexTable = $this->resource->getTableName('catalogrule_product'); + $indexTable = $this->getIndexTableName($useAdditionalTable); + $ruleData = $this->prepareRuleData($rule); + + $productBatchSize = $this->batchSizeCalculator->getAttributeBatchSize(); + $totalBatches = $this->calculateTotalBatches(count($productIds), $productBatchSize); + + $rows = []; + for ($batchIndex = 0; $batchIndex < $totalBatches; $batchIndex++) { + $productBatch = array_slice($productIds, $batchIndex * $productBatchSize, $productBatchSize, true); + $rows = $this->processBatch( + $productBatch, + $websiteIds, + $rule, + $ruleData, + $indexTable, + $connection, + (int)$batchCount, + $rows + ); + unset($productBatch); + } + + unset($productIds); + + if (!empty($rows)) { + $connection->insertMultiple($indexTable, $rows); + } + + $rule->_resetState(); + return true; + } + + /** + * Get website IDs as array + * + * @param string|array $websiteIds + * @return array + */ + private function getWebsiteIdsAsArray($websiteIds): array + { + return is_array($websiteIds) ? $websiteIds : explode(',', $websiteIds); + } + + /** + * Get index table name + * + * @param bool $useAdditionalTable + * @return string + */ + private function getIndexTableName(bool $useAdditionalTable): string + { if ($useAdditionalTable) { - $indexTable = $this->resource->getTableName( + return $this->resource->getTableName( $this->tableSwapper->getWorkingTableName('catalogrule_product') ); } + return $this->resource->getTableName('catalogrule_product'); + } - $ruleId = $rule->getId(); - $customerGroupIds = $rule->getCustomerGroupIds(); - $sortOrder = (int)$rule->getSortOrder(); - $actionOperator = $rule->getSimpleAction(); - $actionAmount = $rule->getDiscountAmount(); - $actionStop = $rule->getStopRulesProcessing(); + /** + * Prepare rule data for indexing + * + * @param Rule $rule + * @return array + */ + private function prepareRuleData(Rule $rule): array + { $fromTimeInAdminTz = $this->parseDateByWebsiteTz((string)$rule->getFromDate(), self::ADMIN_WEBSITE_ID); $toTimeInAdminTz = $this->parseDateByWebsiteTz((string)$rule->getToDate(), self::ADMIN_WEBSITE_ID); + $excludedWebsites = []; + $ruleExtensionAttributes = $rule->getExtensionAttributes(); if ($ruleExtensionAttributes && $ruleExtensionAttributes->getExcludeWebsiteIds()) { $excludedWebsites = $ruleExtensionAttributes->getExcludeWebsiteIds(); + } elseif ($rule->hasData('excluded_website_ids')) { + $excludedWebsites = $rule->getData('excluded_website_ids') ?: []; } - $rows = []; + return [ + 'rule_id' => (int)$rule->getId(), + 'customer_group_ids' => $rule->getCustomerGroupIds(), + 'sort_order' => (int)$rule->getSortOrder(), + 'action_operator' => $rule->getSimpleAction(), + 'action_amount' => $rule->getDiscountAmount(), + 'action_stop' => $rule->getStopRulesProcessing(), + 'from_time_admin_tz' => $fromTimeInAdminTz, + 'to_time_admin_tz' => $toTimeInAdminTz, + 'excluded_websites' => $excludedWebsites, + ]; + } + + /** + * Calculate total batches + * + * @param int $productCount + * @param int $productBatchSize + * @return int + */ + private function calculateTotalBatches(int $productCount, int $productBatchSize): int + { + return $productCount > $productBatchSize ? (int)ceil($productCount / $productBatchSize) : 1; + } + + /** + * Process product batch + * + * @param array $productBatch + * @param array $websiteIds + * @param Rule $rule + * @param array $ruleData + * @param string $indexTable + * @param \Magento\Framework\DB\Adapter\AdapterInterface $connection + * @param int $batchCount + * @param array $rows + * @return array + * @SuppressWarnings(PHPMD.CyclomaticComplexity) + */ + private function processBatch( + array $productBatch, + array $websiteIds, + Rule $rule, + array $ruleData, + string $indexTable, + $connection, + int $batchCount, + array $rows + ): array { foreach ($websiteIds as $websiteId) { - $fromTime = $this->useWebsiteTimezone - ? $this->parseDateByWebsiteTz((string)$rule->getFromDate(), (int)$websiteId) - : $fromTimeInAdminTz; - $toTime = $this->useWebsiteTimezone - ? $this->parseDateByWebsiteTz((string)$rule->getToDate(), (int)$websiteId) - + ($rule->getToDate() ? IndexBuilder::SECONDS_IN_DAY - 1 : 0) - : $toTimeInAdminTz; - - foreach ($productIds as $productId => $validationByWebsite) { + $websiteTimeData = $this->getWebsiteTimeData($rule, (int)$websiteId, $ruleData); + + foreach ($productBatch as $productId => $validationByWebsite) { if (empty($validationByWebsite[$websiteId])) { continue; } - foreach ($customerGroupIds as $customerGroupId) { - if (!array_key_exists($customerGroupId, $excludedWebsites) - || !in_array((int)$websiteId, array_values($excludedWebsites[$customerGroupId]), true) - ) { - $rows[] = [ - 'rule_id' => $ruleId, - 'from_time' => $fromTime, - 'to_time' => $toTime, - 'website_id' => $websiteId, - 'customer_group_id' => $customerGroupId, - 'product_id' => $productId, - 'action_operator' => $actionOperator, - 'action_amount' => $actionAmount, - 'action_stop' => $actionStop, - 'sort_order' => $sortOrder, - ]; - - if (count($rows) === $batchCount) { - $connection->insertMultiple($indexTable, $rows); - $rows = []; - } - } - } + $this->handleAntecedentRules( + $validationByWebsite, + (int)$productId, + $indexTable, + $connection, + (int)$ruleData['rule_id'], + $ruleData['sort_order'] + ); + + $rows = $this->addCustomerGroupRows( + $rows, + $ruleData, + $websiteTimeData, + (int)$websiteId, + (int)$productId, + $indexTable, + $connection, + $batchCount + ); } } - if (!empty($rows)) { - $connection->insertMultiple($indexTable, $rows); + return $rows; + } + + /** + * Get website time data + * + * @param Rule $rule + * @param int $websiteId + * @param array $ruleData + * @return array + */ + private function getWebsiteTimeData(Rule $rule, int $websiteId, array $ruleData): array + { + if ($this->useWebsiteTimezone) { + $fromTime = $this->parseDateByWebsiteTz((string)$rule->getFromDate(), $websiteId); + $toTime = $this->parseDateByWebsiteTz((string)$rule->getToDate(), $websiteId) + + ($rule->getToDate() ? IndexBuilder::SECONDS_IN_DAY - 1 : 0); + } else { + $fromTime = $ruleData['from_time_admin_tz']; + $toTime = $ruleData['to_time_admin_tz']; } - return true; + return ['from_time' => $fromTime, 'to_time' => $toTime]; + } + + /** + * Handle antecedent rules + * + * @param array $validationByWebsite + * @param int $productId + * @param string $indexTable + * @param \Magento\Framework\DB\Adapter\AdapterInterface $connection + * @param int $ruleId + * @param int $sortOrder + * @return void + */ + private function handleAntecedentRules( + array $validationByWebsite, + int $productId, + string $indexTable, + $connection, + int $ruleId, + int $sortOrder + ): void { + if (!isset($validationByWebsite['has_antecedent_rule'])) { + return; + } + + $antecedentRuleProductList = array_keys( + $connection->fetchAssoc( + $connection->select()->from($indexTable) + ->where('product_id = ?', $productId) + ->where('rule_id NOT IN (?)', $ruleId) + ->where('sort_order = ?', $sortOrder) + ) + ); + $connection->delete($indexTable, ['rule_product_id IN (?)' => $antecedentRuleProductList]); + } + + /** + * Add customer group rows + * + * @param array $rows + * @param array $ruleData + * @param array $websiteTimeData + * @param int $websiteId + * @param int $productId + * @param string $indexTable + * @param \Magento\Framework\DB\Adapter\AdapterInterface $connection + * @param int $batchCount + * @return array + */ + private function addCustomerGroupRows( + array $rows, + array $ruleData, + array $websiteTimeData, + int $websiteId, + int $productId, + string $indexTable, + $connection, + int $batchCount + ): array { + foreach ($ruleData['customer_group_ids'] as $customerGroupId) { + $customerGroupId = (int)$customerGroupId; + if ($this->isWebsiteExcluded($customerGroupId, $websiteId, $ruleData['excluded_websites'])) { + continue; + } + + $rows[] = [ + 'rule_id' => $ruleData['rule_id'], + 'from_time' => $websiteTimeData['from_time'], + 'to_time' => $websiteTimeData['to_time'], + 'website_id' => $websiteId, + 'customer_group_id' => $customerGroupId, + 'product_id' => $productId, + 'action_operator' => $ruleData['action_operator'], + 'action_amount' => $ruleData['action_amount'], + 'action_stop' => $ruleData['action_stop'], + 'sort_order' => $ruleData['sort_order'], + ]; + + if (count($rows) === $batchCount) { + $connection->insertMultiple($indexTable, $rows); + $rows = []; + } + } + return $rows; + } + + /** + * Check if website is excluded for customer group + * + * @param int $customerGroupId + * @param int $websiteId + * @param array $excludedWebsites + * @return bool + */ + private function isWebsiteExcluded(int $customerGroupId, int $websiteId, array $excludedWebsites): bool + { + return array_key_exists($customerGroupId, $excludedWebsites) + && in_array($websiteId, array_values($excludedWebsites[$customerGroupId]), true); } /** diff --git a/vendor/magento/module-catalog-rule/Model/Indexer/Rule/GetAffectedProductIds.php b/vendor/magento/module-catalog-rule/Model/Indexer/Rule/GetAffectedProductIds.php new file mode 100644 index 00000000000..a7e36dcbb66 --- /dev/null +++ b/vendor/magento/module-catalog-rule/Model/Indexer/Rule/GetAffectedProductIds.php @@ -0,0 +1,54 @@ +ruleResourceModel->getProductIdsByRuleIds($ids); + $rules = $this->ruleCollectionFactory->create() + ->addFieldToFilter('rule_id', ['in' => array_map('intval', $ids)]); + foreach ($rules as $rule) { + /** @var Rule $rule */ + array_push($productIds, ...array_keys($rule->getMatchingProductIds())); + } + return array_values(array_unique($productIds)); + } +} diff --git a/vendor/magento/module-catalog-rule/Model/Indexer/Rule/RuleProductIndexer.php b/vendor/magento/module-catalog-rule/Model/Indexer/Rule/RuleProductIndexer.php index 3e978ffe5d3..f01fc873bf3 100644 --- a/vendor/magento/module-catalog-rule/Model/Indexer/Rule/RuleProductIndexer.php +++ b/vendor/magento/module-catalog-rule/Model/Indexer/Rule/RuleProductIndexer.php @@ -3,30 +3,63 @@ * Copyright © Magento, Inc. All rights reserved. * See COPYING.txt for license details. */ + namespace Magento\CatalogRule\Model\Indexer\Rule; use Magento\CatalogRule\Model\Indexer\AbstractIndexer; +use Magento\CatalogRule\Model\Indexer\IndexBuilder; +use Magento\CatalogRule\Model\Indexer\Product\ProductRuleProcessor; +use Magento\Framework\App\ObjectManager; +use Magento\Framework\Event\ManagerInterface; class RuleProductIndexer extends AbstractIndexer { /** - * {@inheritdoc} - * - * @SuppressWarnings(PHPMD.UnusedFormalParameter) + * @var ProductRuleProcessor + */ + private $productRuleProcessor; + + /** + * @var GetAffectedProductIds + */ + private $getAffectedProductIds; + + /** + * @param IndexBuilder $indexBuilder + * @param ManagerInterface $eventManager + * @param ProductRuleProcessor|null $productRuleProcessor + * @param GetAffectedProductIds|null $getAffectedProductIds + */ + public function __construct( + IndexBuilder $indexBuilder, + ManagerInterface $eventManager, + ?ProductRuleProcessor $productRuleProcessor = null, + ?GetAffectedProductIds $getAffectedProductIds = null + ) { + $this->productRuleProcessor = $productRuleProcessor + ?? ObjectManager::getInstance()->get(ProductRuleProcessor::class); + $this->getAffectedProductIds = $getAffectedProductIds + ?? ObjectManager::getInstance()->get(GetAffectedProductIds::class); + parent::__construct($indexBuilder, $eventManager); + } + + /** + * @inheritdoc */ protected function doExecuteList($ids) { - $this->indexBuilder->reindexFull(); - $this->getCacheContext()->registerTags($this->getIdentities()); + $affectedProductIds = $this->getAffectedProductIds->execute($ids); + if (!$affectedProductIds) { + return; + } + $this->productRuleProcessor->reindexList($affectedProductIds, true); } /** - * {@inheritdoc} - * - * @SuppressWarnings(PHPMD.UnusedFormalParameter) + * @inheritdoc */ protected function doExecuteRow($id) { - $this->indexBuilder->reindexFull(); + $this->doExecuteList([$id]); } } diff --git a/vendor/magento/module-catalog-rule/Model/Indexer/Rule/RuleProductProcessor.php b/vendor/magento/module-catalog-rule/Model/Indexer/Rule/RuleProductProcessor.php index 260b119cb29..062d54325e6 100644 --- a/vendor/magento/module-catalog-rule/Model/Indexer/Rule/RuleProductProcessor.php +++ b/vendor/magento/module-catalog-rule/Model/Indexer/Rule/RuleProductProcessor.php @@ -5,12 +5,79 @@ */ namespace Magento\CatalogRule\Model\Indexer\Rule; +use Magento\Catalog\Model\Indexer\Product\Price\Processor as ProductPriceProcessor; +use Magento\CatalogRule\Model\Indexer\Product\ProductRuleProcessor; +use Magento\Framework\App\ObjectManager; use Magento\Framework\Indexer\AbstractProcessor; +use Magento\Framework\Indexer\IndexerRegistry; class RuleProductProcessor extends AbstractProcessor { /** * Indexer id */ - const INDEXER_ID = 'catalogrule_rule'; + public const INDEXER_ID = 'catalogrule_rule'; + + /** + * @var ProductRuleProcessor + */ + private $productRuleProcessor; + + /** + * @var ProductPriceProcessor + */ + private $productPriceProcessor; + + /** + * @var GetAffectedProductIds + */ + private $getAffectedProductIds; + + /** + * @param IndexerRegistry $indexerRegistry + * @param ProductRuleProcessor|null $productRuleProcessor + * @param ProductPriceProcessor|null $productPriceProcessor + * @param GetAffectedProductIds|null $getAffectedProductIds + */ + public function __construct( + IndexerRegistry $indexerRegistry, + ?ProductRuleProcessor $productRuleProcessor = null, + ?ProductPriceProcessor $productPriceProcessor = null, + ?GetAffectedProductIds $getAffectedProductIds = null + ) { + $this->productRuleProcessor = $productRuleProcessor + ?? ObjectManager::getInstance()->get(ProductRuleProcessor::class); + $this->productPriceProcessor = $productPriceProcessor + ?? ObjectManager::getInstance()->get(ProductPriceProcessor::class); + $this->getAffectedProductIds = $getAffectedProductIds + ?? ObjectManager::getInstance()->get(GetAffectedProductIds::class); + parent::__construct($indexerRegistry); + } + + /** + * @inheritdoc + */ + public function reindexRow($id, $forceReindex = false) + { + $this->reindexList([$id], $forceReindex); + } + + /** + * @inheritdoc + */ + public function reindexList($ids, $forceReindex = false) + { + if (empty($ids) || !$forceReindex && $this->isIndexerScheduled()) { + return; + } + $affectedProductIds = $this->getAffectedProductIds->execute($ids); + if (!$affectedProductIds) { + return; + } + // catalog_product_price depends on catalogrule_rule. However, their interfaces are not compatible, + // thus the rule is indexed using catalogrule_product + // and price indexer is triggered to update dependent indexes. + $this->productRuleProcessor->reindexList($affectedProductIds); + $this->productPriceProcessor->reindexList($affectedProductIds); + } } diff --git a/vendor/magento/module-catalog-rule/Model/ResourceModel/Product/AttributeValuesLoader.php b/vendor/magento/module-catalog-rule/Model/ResourceModel/Product/AttributeValuesLoader.php new file mode 100644 index 00000000000..515756848e1 --- /dev/null +++ b/vendor/magento/module-catalog-rule/Model/ResourceModel/Product/AttributeValuesLoader.php @@ -0,0 +1,248 @@ +> + * @implements \Countable + */ +class AttributeValuesLoader implements \ArrayAccess, \Countable +{ + /** + * @var Collection + */ + private Collection $collection; + + /** + * @var AbstractAttribute + */ + private AbstractAttribute $attribute; + + /** + * @var DynamicBatchSizeCalculator + */ + private $batchSizeCalculator; + + /** + * @var int + */ + private $batchSize; + + /** + * @var int + */ + private $maxBatchesInMemory; + + /** + * @var array Loaded data: entity_id => [store_id => value] + */ + private $loadedData = []; + + /** + * @var array Track loaded entity IDs in batches + */ + private $loadedBatches = []; + + /** + * @var array Queue of loaded batch start IDs for LRU eviction + */ + private $batchQueue = []; + + /** + * @var int|null Cached count + */ + private $totalCount = null; + + /** + * @param Collection $collection + * @param AbstractAttribute $attribute + * @param DynamicBatchSizeCalculator $batchSizeCalculator + */ + public function __construct( + Collection $collection, + AbstractAttribute $attribute, + DynamicBatchSizeCalculator $batchSizeCalculator + ) { + $this->collection = $collection; + $this->attribute = $attribute; + $this->batchSizeCalculator = $batchSizeCalculator; + $this->batchSize = $batchSizeCalculator->getAttributeBatchSize(); + $this->maxBatchesInMemory = $batchSizeCalculator->getMaxBatchesInMemory(); + } + + /** + * Check if entity has attribute values + * + * @param mixed $offset + * @return bool + */ + public function offsetExists($offset): bool + { + $entityId = (int)$offset; + + if (isset($this->loadedData[$entityId])) { + return true; + } + + $this->loadBatchForEntity($entityId); + + return isset($this->loadedData[$entityId]); + } + + /** + * Get attribute values for entity + * + * @param mixed $offset + * @return array|null + */ + public function offsetGet($offset): ?array + { + $entityId = (int)$offset; + + if (!isset($this->loadedData[$entityId])) { + $this->loadBatchForEntity($entityId); + } + + return $this->loadedData[$entityId] ?? null; + } + + /** + * Set offset not supported + * + * @param mixed $offset + * @param mixed $value + * @return void + * @SuppressWarnings(PHPMD.UnusedFormalParameter) + */ + public function offsetSet($offset, $value): void + { + throw new \LogicException('AttributeValuesLoader is read-only'); + } + + /** + * Offset unset not supported + * + * @param mixed $offset + * @return void + * @SuppressWarnings(PHPMD.UnusedFormalParameter) + */ + public function offsetUnset($offset): void + { + throw new \LogicException('AttributeValuesLoader is read-only'); + } + + /** + * Get total count of entities + * + * @return int + */ + public function count(): int + { + if ($this->totalCount === null) { + $this->totalCount = (int)$this->collection->getSize(); + } + return $this->totalCount; + } + + /** + * Load batch containing the requested entity + * + * @param int $entityId + * @return void + */ + private function loadBatchForEntity(int $entityId): void + { + $batchStartId = (int)(floor($entityId / $this->batchSize) * $this->batchSize); + + if (isset($this->loadedBatches[$batchStartId])) { + return; + } + + $this->loadBatch($batchStartId); + $this->evictOldBatchesIfNeeded(); + } + + /** + * Load a specific batch of attribute values + * + * @param int $batchStartId + * @return void + * @throws LocalizedException|\Zend_Db_Statement_Exception + */ + private function loadBatch(int $batchStartId): void + { + $attributeId = (int)$this->attribute->getId(); + $fieldMainTable = $this->collection->getConnection()->getAutoIncrementField( + $this->collection->getMainTable() + ); + $fieldJoinTable = $this->attribute->getEntity()->getLinkField(); + + $select = $this->collection->getConnection()->select() + ->from(['cpe' => $this->collection->getMainTable()], ['entity_id']) + ->join( + ['cpa' => $this->attribute->getBackend()->getTable()], + 'cpe.' . $fieldMainTable . ' = cpa.' . $fieldJoinTable, + ['store_id', 'value'] + ) + ->where('attribute_id = ?', $attributeId) + ->where('cpe.entity_id >= ?', $batchStartId) + ->where('cpe.entity_id < ?', $batchStartId + $this->batchSize) + ->order(['cpe.entity_id ASC', 'cpa.store_id ASC']); + + $stmt = $this->collection->getConnection()->query($select); + + while ($row = $stmt->fetch()) { + $entityId = (int)$row['entity_id']; + if (!isset($this->loadedData[$entityId])) { + $this->loadedData[$entityId] = []; + } + $this->loadedData[$entityId][(int)$row['store_id']] = $row['value']; + } + + unset($stmt); + + $this->loadedBatches[$batchStartId] = true; + $this->batchQueue[] = $batchStartId; + } + + /** + * Evict old batches if memory limit exceeded + * + * @return void + */ + private function evictOldBatchesIfNeeded(): void + { + while (count($this->loadedBatches) > $this->maxBatchesInMemory) { + $oldestBatchStart = array_shift($this->batchQueue); + unset($this->loadedBatches[$oldestBatchStart]); + + for ($id = $oldestBatchStart; $id < $oldestBatchStart + $this->batchSize; $id++) { + unset($this->loadedData[$id]); + } + } + } + + /** + * Reset all loaded data and clear cache + * + * @return void + */ + public function resetCache(): void + { + $this->loadedData = []; + $this->loadedBatches = []; + $this->batchQueue = []; + $this->totalCount = null; + } +} diff --git a/vendor/magento/module-catalog-rule/Model/ResourceModel/Product/Collection.php b/vendor/magento/module-catalog-rule/Model/ResourceModel/Product/Collection.php new file mode 100644 index 00000000000..11c029a6938 --- /dev/null +++ b/vendor/magento/module-catalog-rule/Model/ResourceModel/Product/Collection.php @@ -0,0 +1,147 @@ +batchSizeCalculator = $batchSizeCalculator ?? + ObjectManager::getInstance()->get(DynamicBatchSizeCalculator::class); + } + + /** + * Get all attribute values for products in collection + * + * @param string|AbstractAttribute $attribute + * @return AttributeValuesLoader + * @throws LocalizedException + */ + public function getAllAttributeValues($attribute) + { + if (!$attribute instanceof AbstractAttribute) { + $attribute = $this->getEntity()->getAttribute($attribute); + } + + $attributeId = (int)$attribute->getId(); + + if (!isset($this->loaderCache[$attributeId])) { + $this->loaderCache[$attributeId] = new AttributeValuesLoader( + $this, + $attribute, + $this->batchSizeCalculator + ); + } + + return $this->loaderCache[$attributeId]; + } + + /** + * Clear attribute loader cache + * + * @return void + */ + public function __destruct() + { + foreach ($this->loaderCache as $loader) { + $loader->resetCache(); + } + $this->loaderCache = []; + } +} diff --git a/vendor/magento/module-catalog-rule/Model/ResourceModel/Rule.php b/vendor/magento/module-catalog-rule/Model/ResourceModel/Rule.php index dd4f3306b86..55761873792 100644 --- a/vendor/magento/module-catalog-rule/Model/ResourceModel/Rule.php +++ b/vendor/magento/module-catalog-rule/Model/ResourceModel/Rule.php @@ -12,6 +12,8 @@ namespace Magento\CatalogRule\Model\ResourceModel; use Magento\Catalog\Model\Product; +use Magento\Framework\App\ObjectManager; +use Magento\Framework\EntityManager\EntityManager; use Magento\Framework\Model\AbstractModel; use Magento\Framework\Pricing\PriceCurrencyInterface; @@ -23,7 +25,7 @@ class Rule extends \Magento\Rule\Model\ResourceModel\AbstractResource /** * Store number of seconds in a day */ - const SECONDS_IN_DAY = 86400; + public const SECONDS_IN_DAY = 86400; /** * @var \Psr\Log\LoggerInterface @@ -31,8 +33,6 @@ class Rule extends \Magento\Rule\Model\ResourceModel\AbstractResource protected $_logger; /** - * Catalog rule data - * * @var \Magento\CatalogRule\Helper\Data */ protected $_catalogRuleData = null; @@ -75,7 +75,7 @@ class Rule extends \Magento\Rule\Model\ResourceModel\AbstractResource protected $priceCurrency; /** - * @var \Magento\Framework\EntityManager\EntityManager + * @var EntityManager */ protected $entityManager; @@ -91,7 +91,8 @@ class Rule extends \Magento\Rule\Model\ResourceModel\AbstractResource * @param \Psr\Log\LoggerInterface $logger * @param \Magento\Framework\Stdlib\DateTime $dateTime * @param PriceCurrencyInterface $priceCurrency - * @param null $connectionName + * @param string|null $connectionName + * @param EntityManager|null $entityManager * @SuppressWarnings(PHPMD.ExcessiveParameterList) */ public function __construct( @@ -105,7 +106,8 @@ class Rule extends \Magento\Rule\Model\ResourceModel\AbstractResource \Psr\Log\LoggerInterface $logger, \Magento\Framework\Stdlib\DateTime $dateTime, PriceCurrencyInterface $priceCurrency, - $connectionName = null + $connectionName = null, + ?EntityManager $entityManager = null ) { $this->_storeManager = $storeManager; $this->_conditionFactory = $conditionFactory; @@ -116,7 +118,11 @@ class Rule extends \Magento\Rule\Model\ResourceModel\AbstractResource $this->_logger = $logger; $this->dateTime = $dateTime; $this->priceCurrency = $priceCurrency; - $this->_associatedEntitiesMap = $this->getAssociatedEntitiesMap(); + $this->entityManager = $entityManager ?? ObjectManager::getInstance()->get(EntityManager::class); + $this->_associatedEntitiesMap = ObjectManager::getInstance() + // phpstan:ignore this is a virtual class + ->get(\Magento\CatalogRule\Model\ResourceModel\Rule\AssociatedEntityMap::class) + ->getData(); parent::__construct($context, $connectionName); } @@ -132,26 +138,7 @@ class Rule extends \Magento\Rule\Model\ResourceModel\AbstractResource } /** - * @param \Magento\Framework\Model\AbstractModel $rule - * @return $this - */ - protected function _afterDelete(\Magento\Framework\Model\AbstractModel $rule) - { - $connection = $this->getConnection(); - $connection->delete( - $this->getTable('catalogrule_product'), - ['rule_id=?' => $rule->getId()] - ); - $connection->delete( - $this->getTable('catalogrule_group_website'), - ['rule_id=?' => $rule->getId()] - ); - return parent::_afterDelete($rule); - } - - /** - * Get catalog rules product price for specific date, website and - * customer group + * Get catalog rules product price for specific date, website and customer group * * @param \DateTimeInterface $date * @param int $wId @@ -171,6 +158,7 @@ class Rule extends \Magento\Rule\Model\ResourceModel\AbstractResource /** * Retrieve product prices by catalog rule for specific date, website and customer group + * * Collect data with product Id => price pairs * * @param \DateTimeInterface $date @@ -219,6 +207,8 @@ class Rule extends \Magento\Rule\Model\ResourceModel\AbstractResource } /** + * Load an object + * * @param \Magento\Framework\Model\AbstractModel $object * @param mixed $value * @param string $field @@ -227,18 +217,16 @@ class Rule extends \Magento\Rule\Model\ResourceModel\AbstractResource */ public function load(\Magento\Framework\Model\AbstractModel $object, $value, $field = null) { - $this->getEntityManager()->load($object, $value); + $this->entityManager->load($object, $value); return $this; } /** - * @param AbstractModel $object - * @return $this - * @throws \Exception + * @inheritdoc */ public function save(\Magento\Framework\Model\AbstractModel $object) { - $this->getEntityManager()->save($object); + $this->entityManager->save($object); return $this; } @@ -251,34 +239,31 @@ class Rule extends \Magento\Rule\Model\ResourceModel\AbstractResource */ public function delete(AbstractModel $object) { - $this->getEntityManager()->delete($object); + $this->entityManager->delete($object); return $this; } /** + * Get product ids matching specified rules + * + * @param array $ruleIds * @return array - * @deprecated 100.1.0 */ - private function getAssociatedEntitiesMap() + public function getProductIdsByRuleIds(array $ruleIds): array { - if (!$this->_associatedEntitiesMap) { - $this->_associatedEntitiesMap = \Magento\Framework\App\ObjectManager::getInstance() - ->get(\Magento\CatalogRule\Model\ResourceModel\Rule\AssociatedEntityMap::class) - ->getData(); - } - return $this->_associatedEntitiesMap; - } - - /** - * @return \Magento\Framework\EntityManager\EntityManager - * @deprecated 100.1.0 - */ - private function getEntityManager() - { - if (null === $this->entityManager) { - $this->entityManager = \Magento\Framework\App\ObjectManager::getInstance() - ->get(\Magento\Framework\EntityManager\EntityManager::class); - } - return $this->entityManager; + $connection = $this->getConnection(); + $select = $connection->select() + ->from( + $this->getTable('catalogrule_product'), + ['product_id'] + ) + ->where( + 'rule_id IN (?)', + array_map('intval', $ruleIds) + ) + ->distinct( + true + ); + return array_map('intval', $connection->fetchCol($select)); } } diff --git a/vendor/magento/module-catalog-rule/Model/ResourceModel/Rule/RuleIdProvider.php b/vendor/magento/module-catalog-rule/Model/ResourceModel/Rule/RuleIdProvider.php new file mode 100644 index 00000000000..11005b249f5 --- /dev/null +++ b/vendor/magento/module-catalog-rule/Model/ResourceModel/Rule/RuleIdProvider.php @@ -0,0 +1,64 @@ +resourceConnection = $resourceConnection; + } + + /** + * Get active rule IDs (lightweight query - only IDs) + * + * @return array + */ + public function getActiveRuleIds(): array + { + $connection = $this->resourceConnection->getConnection(); + $tableName = $this->resourceConnection->getTableName('catalogrule'); + + $select = $connection->select() + ->from($tableName, ['rule_id']) + ->where('is_active = ?', 1) + ->order('sort_order ASC'); + + return $connection->fetchCol($select); + } + + /** + * Get all rule IDs + * + * @return array + */ + public function getAllRuleIds(): array + { + $connection = $this->resourceConnection->getConnection(); + $tableName = $this->resourceConnection->getTableName('catalogrule'); + + $select = $connection->select() + ->from($tableName, ['rule_id']) + ->order('sort_order ASC'); + + return $connection->fetchCol($select); + } +} diff --git a/vendor/magento/module-catalog-rule/Model/Rule.php b/vendor/magento/module-catalog-rule/Model/Rule.php index 1eca8469db1..9827dd98289 100644 --- a/vendor/magento/module-catalog-rule/Model/Rule.php +++ b/vendor/magento/module-catalog-rule/Model/Rule.php @@ -84,6 +84,35 @@ class Rule extends AbstractModel implements RuleInterface, IdentityInterface, Re */ protected $_productsFilter = null; + /** + * Counter for products processed during validation + * + * @var int + */ + private $productsProcessedCount = 0; + + /** + * Total time spent in condition validation + * + * @var float + */ + private $totalValidationTime = 0.0; + + /** + * @var array|null + */ + private $cachedWebsitesMap = null; + + /** + * @var array|null + */ + private $cachedWebsiteIdsArray = null; + + /** + * @var \Magento\Rule\Model\Condition\Combine|null + */ + private $cachedConditions = null; + /** * Store current date at "Y-m-d H:i:s" format * @@ -348,7 +377,11 @@ class Rule extends AbstractModel implements RuleInterface, IdentityInterface, Re if ($this->_productIds === null) { $this->_productIds = []; $this->setCollectedAttributes([]); - + $this->productsProcessedCount = 0; + $this->totalValidationTime = 0.0; + $this->cachedWebsitesMap = null; + $this->cachedWebsiteIdsArray = null; + $this->cachedConditions = null; if ($this->getWebsiteIds()) { /** @var $productCollection \Magento\Catalog\Model\ResourceModel\Product\Collection */ $productCollection = $this->_productCollectionFactory->create(); @@ -358,12 +391,17 @@ class Rule extends AbstractModel implements RuleInterface, IdentityInterface, Re $productCollection->addIdFilter($this->_productsFilter); } $this->getConditions()->collectValidatedAttributes($productCollection); - if ($this->canPreMapProducts()) { - $productCollection = $this->conditionsToCollectionApplier - ->applyConditionsToCollection($this->getConditions(), $productCollection); + $productCollection = $this->conditionsToCollectionApplier->applyConditionsToCollection( + $this->getConditions(), + $productCollection + ); } + $this->cachedWebsitesMap = $this->_getWebsitesMap(); + $websiteIds = $this->getWebsiteIds(); + $this->cachedWebsiteIdsArray = is_array($websiteIds) ? $websiteIds : explode(',', $websiteIds); + $this->cachedConditions = $this->getConditions(); $this->_resourceIterator->walk( $productCollection->getSelect(), [[$this, 'callbackValidateProduct']], @@ -406,21 +444,21 @@ class Rule extends AbstractModel implements RuleInterface, IdentityInterface, Re $product = clone $args['product']; $product->setData($args['row']); - $websites = $this->_getWebsitesMap(); - $websiteIds = $this->getWebsiteIds(); - if (!is_array($websiteIds)) { - $websiteIds = explode(',', $websiteIds); - } $results = []; - foreach ($websites as $websiteId => $defaultStoreId) { - if (!in_array($websiteId, $websiteIds)) { + $validationStart = microtime(true); + foreach ($this->cachedWebsitesMap as $websiteId => $defaultStoreId) { + if (!in_array($websiteId, $this->cachedWebsiteIdsArray)) { continue; } $product->setStoreId($defaultStoreId); - $results[$websiteId] = $this->getConditions()->validate($product); + $results[$websiteId] = $this->cachedConditions->validate($product); } + $this->totalValidationTime += (microtime(true) - $validationStart); + $this->_productIds[$product->getId()] = $results; + + $this->productsProcessedCount++; } /** @@ -605,13 +643,8 @@ class Rule extends AbstractModel implements RuleInterface, IdentityInterface, Re return parent::afterSave(); } - if ($this->isObjectNew() && !$this->_ruleProductProcessor->isIndexerScheduled()) { - $productIds = $this->getMatchingProductIds(); - if (!empty($productIds) && is_array($productIds)) { - $this->ruleResourceModel->addCommitCallback([$this, 'reindex']); - } - } else { - $this->_ruleProductProcessor->getIndexer()->invalidate(); + if (!$this->_ruleProductProcessor->isIndexerScheduled()) { + $this->ruleResourceModel->addCommitCallback([$this, 'reindex']); } return parent::afterSave(); @@ -624,15 +657,7 @@ class Rule extends AbstractModel implements RuleInterface, IdentityInterface, Re */ public function reindex() { - $productIds = $this->_productIds ? array_keys( - array_filter( - $this->_productIds, - function (array $data) { - return array_filter($data); - } - ) - ) : []; - $this->_ruleProductProcessor->reindexList($productIds); + $this->_ruleProductProcessor->reindexRow($this->getRuleId()); } /** @@ -642,7 +667,9 @@ class Rule extends AbstractModel implements RuleInterface, IdentityInterface, Re */ public function afterDelete() { - $this->_ruleProductProcessor->getIndexer()->invalidate(); + if ($this->getIsActive() && !$this->_ruleProductProcessor->isIndexerScheduled()) { + $this->ruleResourceModel->addCommitCallback([$this, 'reindex']); + } return parent::afterDelete(); } @@ -926,5 +953,11 @@ class Rule extends AbstractModel implements RuleInterface, IdentityInterface, Re public function _resetState(): void { self::$_priceRulesData = []; + $this->_productIds = null; + $this->productsProcessedCount = 0; + $this->totalValidationTime = 0.0; + $this->cachedWebsitesMap = null; + $this->cachedWebsiteIdsArray = null; + $this->cachedConditions = null; } } diff --git a/vendor/magento/module-catalog-rule/etc/di.xml b/vendor/magento/module-catalog-rule/etc/di.xml index e0d91db5423..6373548ac3a 100644 --- a/vendor/magento/module-catalog-rule/etc/di.xml +++ b/vendor/magento/module-catalog-rule/etc/di.xml @@ -164,4 +164,27 @@ CatalogRuleCustomConditionProvider + + + Magento\CatalogRule\Model\ResourceModel\Product\Collection + + + + + Magento\CatalogRule\Model\ResourceModel\Product\OptimizedCollectionFactory + + + + + + Magento\CatalogRule\Model\Indexer\CatalogRuleProductPriceRowSizeEstimator + + + + + + Magento\CatalogRule\Model\Indexer\CatalogRuleBatchSizeManagement + 5000 + + diff --git a/vendor/magento/module-catalog-rule-configurable/Plugin/CatalogRule/Model/Rule/ConfigurableProductHandler.php b/vendor/magento/module-catalog-rule-configurable/Plugin/CatalogRule/Model/Rule/ConfigurableProductHandler.php index 231696f259f..555683eabe1 100644 --- a/vendor/magento/module-catalog-rule-configurable/Plugin/CatalogRule/Model/Rule/ConfigurableProductHandler.php +++ b/vendor/magento/module-catalog-rule-configurable/Plugin/CatalogRule/Model/Rule/ConfigurableProductHandler.php @@ -1,13 +1,14 @@ configurable = $configurable; $this->configurableProductsProvider = $configurableProductsProvider; @@ -49,40 +50,71 @@ class ConfigurableProductHandler * @return array * @SuppressWarnings(PHPMD.UnusedFormalParameter) * @SuppressWarnings(PHPMD.CyclomaticComplexity) + * @SuppressWarnings(PHPMD.NPathComplexity) + * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ public function aroundGetMatchingProductIds( \Magento\CatalogRule\Model\Rule $rule, \Closure $proceed - ) { + ): array { $productsFilter = $rule->getProductsFilter() ? (array) $rule->getProductsFilter() : []; if ($productsFilter) { - $parentProductIds = $this->configurable->getParentIdsByChild($productsFilter); - $rule->setProductsFilter(array_unique(array_merge($productsFilter, $parentProductIds))); + $rule->setProductsFilter( + array_unique( + array_merge( + $productsFilter, + $this->configurable->getParentIdsByChild($productsFilter) + ) + ) + ); } $productIds = $proceed(); + foreach ($productIds as $productId => $productData) { + if ($this->hasAntecedentRule((int) $productId)) { + $productIds[$productId]['has_antecedent_rule'] = true; + } + } - $configurableProductIds = $this->configurableProductsProvider->getIds(array_keys($productIds)); - foreach ($configurableProductIds as $productId) { - if (!isset($this->childrenProducts[$productId])) { - $this->childrenProducts[$productId] = $this->configurable->getChildrenIds($productId)[0]; + foreach ($this->configurableProductsProvider->getIds(array_keys($productIds)) as $configurableProductId) { + if (!isset($this->childrenProducts[$configurableProductId])) { + $this->childrenProducts[$configurableProductId] = + $this->configurable->getChildrenIds($configurableProductId)[0]; } - $subProductIds = $this->childrenProducts[$productId]; - $parentValidationResult = isset($productIds[$productId]) - ? array_filter($productIds[$productId]) + + $parentValidationResult = isset($productIds[$configurableProductId]) + ? array_filter($productIds[$configurableProductId]) : []; - $processAllChildren = !$productsFilter || in_array($productId, $productsFilter); - foreach ($subProductIds as $subProductId) { - if ($processAllChildren || in_array($subProductId, $productsFilter)) { - $childValidationResult = isset($productIds[$subProductId]) - ? array_filter($productIds[$subProductId]) + $processAllChildren = !$productsFilter || in_array($configurableProductId, $productsFilter); + foreach ($this->childrenProducts[$configurableProductId] as $childrenProductId) { + if ($processAllChildren || in_array($childrenProductId, $productsFilter)) { + $childValidationResult = isset($productIds[$childrenProductId]) + ? array_filter($productIds[$childrenProductId]) : []; - $productIds[$subProductId] = $parentValidationResult + $childValidationResult; + $productIds[$childrenProductId] = $parentValidationResult + $childValidationResult; } - } - unset($productIds[$productId]); + unset($productIds[$configurableProductId]); } + return $productIds; } + + /** + * Check if simple product has previously applied rule. + * + * @param int $productId + * @return bool + * @SuppressWarnings(PHPMD.UnusedLocalVariable) + */ + private function hasAntecedentRule(int $productId): bool + { + foreach ($this->childrenProducts as $parent => $children) { + if (in_array($productId, $children)) { + return true; + } + } + + return false; + } } diff --git a/vendor/magento/module-catalog-search/etc/mview.xml b/vendor/magento/module-catalog-search/etc/mview.xml index 28737ce23b1..833954f993f 100644 --- a/vendor/magento/module-catalog-search/etc/mview.xml +++ b/vendor/magento/module-catalog-search/etc/mview.xml @@ -19,6 +19,7 @@
+
diff --git a/vendor/magento/module-indexer/Model/Indexer.php b/vendor/magento/module-indexer/Model/Indexer.php index c09f5110028..d8a63f59b10 100644 --- a/vendor/magento/module-indexer/Model/Indexer.php +++ b/vendor/magento/module-indexer/Model/Indexer.php @@ -6,9 +6,11 @@ namespace Magento\Indexer\Model; +use Magento\Framework\App\ObjectManager; use Magento\Framework\DataObject; use Magento\Framework\Indexer\ActionFactory; use Magento\Framework\Indexer\ActionInterface; +use Magento\Framework\Indexer\Config\DependencyInfoProviderInterface; use Magento\Framework\Indexer\ConfigInterface; use Magento\Framework\Indexer\IndexerInterface; use Magento\Framework\Indexer\IndexStructureInterface; @@ -16,6 +18,10 @@ use Magento\Framework\Indexer\StateInterface; use Magento\Framework\Indexer\StructureFactory; use Magento\Framework\Indexer\IndexerInterfaceFactory; use Magento\Framework\Indexer\SuspendableIndexerInterface; +use Magento\Framework\Mview\View\ChangelogTableNotExistsException; +use Magento\Framework\Mview\ViewInterface; +use Magento\Indexer\Model\Indexer\CollectionFactory; +use Magento\Indexer\Model\Indexer\StateFactory; /** * Indexer model. @@ -74,16 +80,23 @@ class Indexer extends DataObject implements IndexerInterface, SuspendableIndexer */ private $indexerFactory; + /** + * @var DependencyInfoProviderInterface + */ + private $dependencyInfoProvider; + /** * @param ConfigInterface $config * @param ActionFactory $actionFactory * @param StructureFactory $structureFactory - * @param \Magento\Framework\Mview\ViewInterface $view - * @param Indexer\StateFactory $stateFactory - * @param Indexer\CollectionFactory $indexersFactory + * @param ViewInterface $view + * @param StateFactory $stateFactory + * @param CollectionFactory $indexersFactory * @param WorkingStateProvider $workingStateProvider * @param IndexerInterfaceFactory $indexerFactory * @param array $data + * @param DependencyInfoProviderInterface|null $dependencyInfoProvider + * @SuppressWarnings(PHPMD.ExcessiveParameterList) */ public function __construct( ConfigInterface $config, @@ -94,7 +107,8 @@ class Indexer extends DataObject implements IndexerInterface, SuspendableIndexer Indexer\CollectionFactory $indexersFactory, WorkingStateProvider $workingStateProvider, IndexerInterfaceFactory $indexerFactory, - array $data = [] + array $data = [], + ?DependencyInfoProviderInterface $dependencyInfoProvider = null ) { $this->config = $config; $this->actionFactory = $actionFactory; @@ -104,6 +118,8 @@ class Indexer extends DataObject implements IndexerInterface, SuspendableIndexer $this->indexersFactory = $indexersFactory; $this->workingStateProvider = $workingStateProvider; $this->indexerFactory = $indexerFactory; + $this->dependencyInfoProvider = $dependencyInfoProvider + ?? ObjectManager::getInstance()->get(DependencyInfoProviderInterface::class); parent::__construct($data); } @@ -439,18 +455,16 @@ class Indexer extends DataObject implements IndexerInterface, SuspendableIndexer $state->setStatus(StateInterface::STATUS_WORKING); $state->save(); + $resetViewVersion = $this->shouldResetViewVersion(); + $sharedIndexers = []; $indexerConfig = $this->config->getIndexer($this->getId()); if ($indexerConfig['shared_index'] !== null) { $sharedIndexers = $this->getSharedIndexers($indexerConfig['shared_index']); } - if (!empty($sharedIndexers)) { - $this->suspendSharedViews($sharedIndexers); - } - if ($this->getView()->isEnabled()) { - $this->getView()->suspend(); - } + $this->suspendViews(array_merge($sharedIndexers, [$this]), $resetViewVersion); + try { $this->getActionInstance()->executeFull(); if ($this->workingStateProvider->isWorking($this->getId())) { @@ -497,16 +511,25 @@ class Indexer extends DataObject implements IndexerInterface, SuspendableIndexer } /** - * Suspend views of shared indexers + * Suspend views * - * @param array $sharedIndexers + * @param IndexerInterface[] $indexers + * @param bool $reset * @return void + * @throws \Exception */ - private function suspendSharedViews(array $sharedIndexers) : void + private function suspendViews(array $indexers, bool $reset = true) : void { - foreach ($sharedIndexers as $indexer) { + foreach ($indexers as $indexer) { if ($indexer->getView()->isEnabled()) { - $indexer->getView()->suspend(); + if ($reset) { + // this method also resets the mview version to the current one + $indexer->getView()->suspend(); + } else { + $state = $indexer->getView()->getState(); + $state->setStatus(\Magento\Framework\Mview\View\StateInterface::STATUS_SUSPENDED); + $state->save(); + } } } } @@ -547,4 +570,71 @@ class Indexer extends DataObject implements IndexerInterface, SuspendableIndexer $this->getActionInstance()->executeList($ids); $this->getState()->save(); } + + /** + * Return all indexer Ids on which the current indexer depends (directly or indirectly). + * + * @param string $indexerId + * @return array + */ + private function getIndexerIdsToRunBefore(string $indexerId): array + { + $relatedIndexerIds = []; + foreach ($this->dependencyInfoProvider->getIndexerIdsToRunBefore($indexerId) as $relatedIndexerId) { + if ($relatedIndexerId !== $indexerId) { + $relatedIndexerIds[] = [$relatedIndexerId]; + $relatedIndexerIds[] = $this->getIndexerIdsToRunBefore($relatedIndexerId); + } + } + + return array_unique(array_merge([], ...$relatedIndexerIds)); + } + + /** + * Check whether view is up to date + * + * @param ViewInterface $view + * @return bool + */ + private function isViewUpToDate(\Magento\Framework\Mview\ViewInterface $view): bool + { + if (!$view->isEnabled()) { + return true; + } + + try { + $currentVersionId = $view->getChangelog()->getVersion(); + } catch (ChangelogTableNotExistsException $e) { + return true; + } + + $lastVersionId = (int)$view->getState()->getVersionId(); + if ($lastVersionId >= $currentVersionId) { + return true; + } + + return false; + } + + /** + * Check whether indexer view version should be reset + * + * @return bool + */ + private function shouldResetViewVersion(): bool + { + $resetViewVersion = true; + foreach ($this->getIndexerIdsToRunBefore($this->getId()) as $indexerId) { + if ($indexerId === $this->getId()) { + continue; + } + $indexer = $this->indexerFactory->create(); + $indexer->load($indexerId); + if ($indexer->isValid() && !$this->isViewUpToDate($indexer->getView())) { + $resetViewVersion = false; + break; + } + } + return $resetViewVersion; + } } diff --git a/vendor/magento/module-indexer/Model/Indexer/DependencyDecorator.php b/vendor/magento/module-indexer/Model/Indexer/DependencyDecorator.php index 7306405319e..f75dea6210e 100644 --- a/vendor/magento/module-indexer/Model/Indexer/DependencyDecorator.php +++ b/vendor/magento/module-indexer/Model/Indexer/DependencyDecorator.php @@ -232,10 +232,9 @@ class DependencyDecorator implements IndexerInterface { $this->indexer->invalidate(); $currentIndexerId = $this->indexer->getId(); - $idsToRunBefore = $this->dependencyInfoProvider->getIndexerIdsToRunBefore($currentIndexerId); $idsToRunAfter = $this->dependencyInfoProvider->getIndexerIdsToRunAfter($currentIndexerId); - $indexersToInvalidate = array_unique(array_merge($idsToRunBefore, $idsToRunAfter)); + $indexersToInvalidate = array_unique($idsToRunAfter); foreach ($indexersToInvalidate as $indexerId) { $indexer = $this->indexerRegistry->get($indexerId); if (!$indexer->isInvalid()) {