Merge pull request #18 from fredden/large-data-set

sprankhub · web-flow · commit 6147ee4de4cb · 2022-07-28T15:17:12.000+02:00
Replace use of fetchAll() for database queries
diff --git a/Console/Command/CleanUpAttributesAndValuesWithoutParentCommand.php b/Console/Command/CleanUpAttributesAndValuesWithoutParentCommand.php
@@ -85,12 +85,12 @@ public function execute(InputInterface $input, OutputInterface $output): int
             foreach ($types as $type) {
                 $eavTable         = $this->resourceConnection->getTableName('eav_attribute');
                 $entityValueTable = $this->resourceConnection->getTableName($code . '_entity_' . $type);
-                $query            = "SELECT * FROM $entityValueTable WHERE `attribute_id` NOT IN(SELECT attribute_id"
+                $query = "SELECT COUNT(*) FROM $entityValueTable WHERE `attribute_id` NOT IN(SELECT attribute_id"
                     . " FROM `$eavTable` WHERE entity_type_id = " . $entityType->getEntityTypeId() . " AND backend_type = '$type')";
-                $results          = $db->fetchAll($query);
-                $output->writeln("Clean up " . count($results) . " rows in $entityValueTable");
+                $count = (int) $db->fetchOne($query);
+                $output->writeln("Clean up $count rows in $entityValueTable");
 
-                if (!$isDryRun && count($results) > 0) {
+                if (!$isDryRun && $count > 0) {
                     $db->query("DELETE FROM $entityValueTable WHERE `attribute_id` NOT IN(SELECT attribute_id"
                         . " FROM `$eavTable` WHERE entity_type_id = " . $entityType->getEntityTypeId() . " AND backend_type = '$type')");
                 }
diff --git a/Console/Command/RestoreUseDefaultConfigValueCommand.php b/Console/Command/RestoreUseDefaultConfigValueCommand.php
@@ -4,6 +4,7 @@
 
 use Magento\Framework\App\Config\Initial\Reader;
 use Magento\Framework\App\ResourceConnection;
+use Magento\Framework\Model\ResourceModel\IteratorFactory;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -14,6 +15,9 @@ class RestoreUseDefaultConfigValueCommand extends Command
     /** @var Reader */
     private $configReader;
 
+    /** @var IteratorFactory */
+    private $iteratorFactory;
+
     /**
      * @var ResourceConnection
      */
@@ -24,12 +28,14 @@ class RestoreUseDefaultConfigValueCommand extends Command
 
     public function __construct(
         Reader $configReader,
+        IteratorFactory $iteratorFactory,
         ResourceConnection $resourceConnection,
         string $name = null
     ) {
         parent::__construct($name);
 
         $this->configReader = $configReader;
+        $this->iteratorFactory = $iteratorFactory;
         $this->resourceConnection = $resourceConnection;
     }
 
@@ -65,21 +71,25 @@ public function execute(InputInterface $input, OutputInterface $output): int
 
         $removedConfigValues = 0;
 
-        $db         = $this->resourceConnection->getConnection();
+        $dbRead = $this->resourceConnection->getConnection('core_read');
+        $dbWrite = $this->resourceConnection->getConnection('core_write');
         $tableName = $this->resourceConnection->getTableName('core_config_data');
-        $configData = $db->fetchAll('SELECT DISTINCT path, value FROM ' . $tableName
-            . ' WHERE scope_id = 0');
 
-        foreach ($configData as $config) {
-            $count = $db->fetchOne('SELECT COUNT(*) FROM ' . $tableName
+        $query = $dbRead->query("SELECT DISTINCT path, value FROM $tableName WHERE scope_id = 0");
+
+        $iterator = $this->iteratorFactory->create();
+        $iterator->walk($query, [function (array $result) use ($dbRead, $dbWrite, $isDryRun, $output, &$removedConfigValues, $tableName): void {
+            $config = $result['row'];
+
+            $count = (int) $dbRead->fetchOne('SELECT COUNT(*) FROM ' . $tableName
                 . ' WHERE path = ? AND BINARY value = ?', [$config['path'], $config['value']]);
 
             if ($count > 1) {
                 $output->writeln('Config path ' . $config['path'] . ' with value ' . $config['value'] . ' has ' . $count
                     . ' values; deleting non-default values');
 
                 if (!$isDryRun) {
-                    $db->query(
+                    $dbWrite->query(
                         'DELETE FROM ' . $tableName . ' WHERE path = ? AND BINARY value = ? AND scope_id != ?',
                         [$config['path'], $config['value'], 0]
                     );
@@ -93,12 +103,12 @@ public function execute(InputInterface $input, OutputInterface $output): int
 
                 if (!$isDryRun) {
                     if ($config['value'] === null) {
-                        $db->query(
+                        $dbWrite->query(
                             "DELETE FROM $tableName WHERE path = ? AND value IS NULL AND scope_id = 0",
                             [$config['path']]
                         );
                     } else {
-                        $db->query(
+                        $dbWrite->query(
                             "DELETE FROM $tableName WHERE path = ? AND BINARY value = ? AND scope_id = 0",
                             [$config['path'], $config['value']]
                         );
@@ -107,7 +117,7 @@ public function execute(InputInterface $input, OutputInterface $output): int
 
                 $removedConfigValues++;
             }
-        }
+        }]);
 
         $output->writeln('Removed ' . $removedConfigValues . ' values from core_config_data table.');
 
diff --git a/Console/Command/RestoreUseDefaultValueCommand.php b/Console/Command/RestoreUseDefaultValueCommand.php
@@ -4,6 +4,7 @@
 
 use Magento\Framework\App\ProductMetadataInterface;
 use Magento\Framework\App\ResourceConnection;
+use Magento\Framework\Model\ResourceModel\IteratorFactory;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
@@ -12,6 +13,9 @@
 
 class RestoreUseDefaultValueCommand extends Command
 {
+    /** @var IteratorFactory */
+    protected $iteratorFactory;
+
     /**
      * @var ProductMetaDataInterface
      */
@@ -23,11 +27,14 @@ class RestoreUseDefaultValueCommand extends Command
     private $resourceConnection;
 
     public function __construct(
+        IteratorFactory $iteratorFactory,
         ProductMetaDataInterface $productMetaData,
         ResourceConnection $resourceConnection,
         string $name = null
     ) {
         parent::__construct($name);
+
+        $this->iteratorFactory = $iteratorFactory;
         $this->productMetaData    = $productMetaData;
         $this->resourceConnection = $resourceConnection;
     }
@@ -58,7 +65,7 @@ public function execute(InputInterface $input, OutputInterface $output): int
         if (!in_array($entity, ['product', 'category'])) {
             $output->writeln('Please specify the entity with --entity. Possible options are product or category');
 
-            return 1;
+            return 1; // error.
         }
 
         if (!$isDryRun && !$isForce) {
@@ -76,57 +83,64 @@ public function execute(InputInterface $input, OutputInterface $output): int
             }
         }
 
-        $db     = $this->resourceConnection->getConnection();
+        $dbRead = $this->resourceConnection->getConnection('core_read');
+        $dbWrite = $this->resourceConnection->getConnection('core_write');
         $counts = [];
         $tables = ['varchar', 'int', 'decimal', 'text', 'datetime'];
         $column = $this->productMetaData->getEdition() === 'Enterprise' ? 'row_id' : 'entity_id';
 
         foreach ($tables as $table) {
             // Select all non-global values
             $fullTableName = $this->resourceConnection->getTableName('catalog_' . $entity . '_entity_' . $table);
-            $rows          = $db->fetchAll('SELECT * FROM ' . $fullTableName . ' WHERE store_id != 0');
 
-            foreach ($rows as $row) {
+            // NULL values are handled separately
+            $query = $dbRead->query("SELECT * FROM $fullTableName WHERE store_id != 0 AND value IS NOT NULL");
+
+            $iterator = $this->iteratorFactory->create();
+            $iterator->walk($query, [function (array $result) use ($column, &$counts, $dbRead, $dbWrite, $fullTableName, $isDryRun, $output): void {
+                $row = $result['row'];
+
                 // Select the global value if it's the same as the non-global value
-                $results = $db->fetchAll(
+                $query = $dbRead->query(
                     'SELECT * FROM ' . $fullTableName
                     . ' WHERE attribute_id = ? AND store_id = ? AND ' . $column . ' = ? AND BINARY value = ?',
                     [$row['attribute_id'], 0, $row[$column], $row['value']]
                 );
 
-                if (count($results) > 0) {
-                    foreach ($results as $result) {
-                        if (!$isDryRun) {
-                            // Remove the non-global value
-                            $db->query(
-                                'DELETE FROM ' . $fullTableName . ' WHERE value_id = ?',
-                                $row['value_id']
-                            );
-                        }
-
-                        $output->writeln(
-                            'Deleting value ' . $row['value_id'] . ' "' . $row['value'] . '" in favor of '
-                            . $result['value_id']
-                            . ' for attribute ' . $row['attribute_id'] . ' in table ' . $fullTableName
+                $iterator = $this->iteratorFactory->create();
+                $iterator->walk($query, [function (array $result) use (&$counts, $dbWrite, $fullTableName, $isDryRun, $output, $row): void {
+                    $result = $result['row'];
+
+                    if (!$isDryRun) {
+                        // Remove the non-global value
+                        $dbWrite->query(
+                            'DELETE FROM ' . $fullTableName . ' WHERE value_id = ?',
+                            $row['value_id']
                         );
+                    }
 
-                        if (!isset($counts[$row['attribute_id']])) {
-                            $counts[$row['attribute_id']] = 0;
-                        }
+                    $output->writeln(
+                        'Deleting value ' . $row['value_id'] . ' "' . $row['value'] . '" in favor of '
+                        . $result['value_id']
+                        . ' for attribute ' . $row['attribute_id'] . ' in table ' . $fullTableName
+                    );
 
-                        $counts[$row['attribute_id']]++;
+                    if (!isset($counts[$row['attribute_id']])) {
+                        $counts[$row['attribute_id']] = 0;
                     }
-                }
-            }
 
-            $nullValues = $db->fetchOne(
+                    $counts[$row['attribute_id']]++;
+                }]);
+            }]);
+
+            $nullCount = (int) $dbRead->fetchOne(
                 'SELECT COUNT(*) FROM ' . $fullTableName . ' WHERE store_id != 0 AND value IS NULL'
             );
 
-            if (!$isDryRun && $nullValues > 0) {
-                $output->writeln("Deleting " . $nullValues . " NULL value(s) from " . $fullTableName);
+            if (!$isDryRun && $nullCount > 0) {
+                $output->writeln("Deleting $nullCount NULL value(s) from $fullTableName");
                 // Remove all non-global null values
-                $db->query(
+                $dbWrite->query(
                     'DELETE FROM ' . $fullTableName . ' WHERE store_id != 0 AND value IS NULL'
                 );
             }
