Rework GoogleSheetExtractor to use batch get functionality

src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/GoogleSheetExtractor.php

@@ -37,48 +37,60 @@ public function __construct(
 
     public function extract(FlowContext $context) : \Generator
     {
-        $cellsRange = new SheetRange($this->columnRange, 1, $this->rowsPerPage);
-        $headers = [];
-
-        $response = $this->service->spreadsheets_values->get(
+        $spreadsheet = $this->service->spreadsheets->get(
             $this->spreadsheetId,
-            $cellsRange->toString(),
-            $this->options
+            ['ranges' => [], 'includeGridData' => false]
         );
 
-        /**
-         * @var array<array<null|string>> $values
-         */
-        $values = $response->getValues() ?? [];
+        $maxRows = 0;
 
-        $totalRows = 0;
-
-        if ($this->withHeader && [] !== $values) {
-            foreach ($values as $index => $row) {
-                if ([] === $row) {
-                    // Remove empty rows at the beginning of a sheet
-                    unset($values[$index]);
-
-                    continue;
-                }
-
-                /** @var array<string> $headers */
-                $headers = $row;
-                unset($values[$index]);
-                $totalRows = 1;
+        foreach ($spreadsheet->getSheets() as $sheet) {
+            if ($sheet->getProperties()->title === $this->columnRange->sheetName) {
+                $maxRows = $sheet->getProperties()->getGridProperties()->getRowCount();
 
                 break;
             }
         }
 
-        $headersCount = \count($headers);
+        $cellsRange = new SheetRange($this->columnRange, 1, $this->rowsPerPage, $maxRows);
+
+        $ranges = [];
+
+        for ($totalRows = 0; $totalRows < $cellsRange->endRow; $totalRows += $this->rowsPerPage) {
+            $ranges[] = $cellsRange->toString();
+
+            $cellsRange = $cellsRange->nextRows($this->rowsPerPage);
+        }
 
         $shouldPutInputIntoRows = $context->config->shouldPutInputIntoRows();
 
-        while ([] !== $values) {
-            foreach ($values as $rowData) {
+        $headers = [];
+        $headersCount = 0;
+
+        $response = $this->service->spreadsheets_values->batchGet($this->spreadsheetId, array_merge($this->options, ['ranges' => $ranges]));
+
+        foreach ($response->getValueRanges() as $valueRange) {
+            foreach ($valueRange->getValues() as $rowData) {
                 $rowDataCount = \count($rowData);
 
+                if ($this->withHeader) {
+                    if ([] === $headers) {
+                        // Skip empty rows at the beginning of a sheet
+                        if ([] === $rowData) {
+                            continue;
+                        }
+
+                        /** @var array<string> $headers */
+                        $headers = $rowData;
+
+                        $headersCount = $rowDataCount;
+
+                        continue;
+                    }
+                } elseif (0 === $headersCount) {
+                    $headersCount = $rowDataCount;
+                }
+
                 // Expand columns to the size of the previous row
                 for ($i = $rowDataCount; $i < $headersCount; $i++) {
                     $rowData[$i] = null;
@@ -92,35 +104,23 @@ public function extract(FlowContext $context) : \Generator
                     $rowData = \array_slice($rowData, 0, $headersCount);
                 }
 
-                $row = \array_combine($headers, $rowData);
+                if ($this->withHeader) {
+                    $rowData = \array_combine($headers, $rowData);
+                }
 
                 if ($shouldPutInputIntoRows) {
-                    $row['_spread_sheet_id'] = $this->spreadsheetId;
-                    $row['_sheet_name'] = $this->columnRange->sheetName;
+                    $rowData['_spread_sheet_id'] = $this->spreadsheetId;
+                    $rowData['_sheet_name'] = $this->columnRange->sheetName;
                 }
 
-                $totalRows++;
-
-                $signal = yield array_to_rows($row, $context->entryFactory(), schema: $this->schema);
+                $signal = yield array_to_rows($rowData, $context->entryFactory(), schema: $this->schema);
 
                 $this->incrementReturnedRows();
 
                 if ($signal === Signal::STOP || $this->reachedLimit()) {
                     return;
                 }
             }
-
-            if ($totalRows < $cellsRange->endRow) {
-                return;
-            }
-
-            $cellsRange = $cellsRange->nextRows($this->rowsPerPage);
-
-            $response = $this->service->spreadsheets_values->get($this->spreadsheetId, $cellsRange->toString(), $this->options);
-            /**
-             * @var array<array<null|string>> $values
-             */
-            $values = $response->getValues() ?? [];
         }
     }
 

src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/SheetRange.php

@@ -8,22 +8,31 @@
 
 final readonly class SheetRange
 {
+    public int $endRow;
+
     public function __construct(
         public Columns $columnRange,
         public int $startRow,
-        public int $endRow,
+        int $endRow,
+        private int $maxRows,
     ) {
         if ($this->startRow < 1) {
             throw new InvalidArgumentException(\sprintf('Start row "%d" must be greater than 0', $this->startRow));
         }
 
-        if ($this->endRow < 1) {
-            throw new InvalidArgumentException(\sprintf('End row "%d" must be greater than 0', $this->endRow));
+        if ($endRow < 1) {
+            throw new InvalidArgumentException(\sprintf('End row "%d" must be greater than 0', $endRow));
+        }
+
+        if ($endRow < $this->startRow) {
+            throw new InvalidArgumentException(\sprintf('End row "%d" must be greater or equal to start row "%d"', $endRow, $this->startRow));
         }
 
-        if ($this->endRow < $this->startRow) {
-            throw new InvalidArgumentException(\sprintf('End row "%d" must be greater or equal to start row "%d"', $this->endRow, $this->startRow));
+        if ($this->maxRows < 1) {
+            throw new InvalidArgumentException(\sprintf('Max rows "%d" must be greater than 0', $this->maxRows));
         }
+
+        $this->endRow = min($endRow, $this->maxRows);
     }
 
     public function nextRows(int $count) : self
@@ -34,8 +43,9 @@ public function nextRows(int $count) : self
 
         return new self(
             $this->columnRange,
-            $this->endRow + 1,
-            $this->endRow + $count,
+            min($this->endRow + 1, $this->maxRows),
+            min($this->endRow + $count, $this->maxRows),
+            $this->maxRows,
         );
     }
 
@@ -47,7 +57,7 @@ public function toString() : string
             $this->columnRange->startColumn,
             $this->startRow,
             $this->columnRange->endColumn,
-            $this->endRow
+            $this->endRow,
         );
     }
 }

src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/batch.json

@@ -0,0 +1,37 @@
+{
+  "spreadsheetId": "eae3913a86de8d5ccfe841fd3c7ec50dba3902045232",
+  "valueRanges": [
+    {
+      "range": "Sheet!A1:C10",
+      "majorDimension": "ROWS",
+      "values": [
+        ["Header 1", "Header 2", "Header 3"],
+        ["A2", "B2", "C2"],
+        ["A3", "B3", "C3"],
+        ["A4", "B4", "C4"],
+        ["A5", "B5", "C5"],
+        ["A6", "B6", "C6"],
+        ["A7", "B7", "C7"],
+        ["A8", "B8", "C8"],
+        ["A9", "B9", "C9"],
+        ["A10", "B10", "C10"]
+      ]
+    },
+    {
+      "range": "Sheet!A11:C20",
+      "majorDimension": "ROWS",
+      "values": [
+        ["A11", "B11", "C11"],
+        ["A12", "B12", "C12"],
+        ["A13", "B13", "C13"],
+        ["A14", "B14", "C14"],
+        ["A15", "B15", "C15"],
+        ["A16", "B16", "C16"],
+        ["A17", "B17", "C17"],
+        ["A18", "B18", "C18"],
+        ["A19", "B19", "C19"],
+        ["A20", "B20", "C20"]
+      ]
+    }
+  ]
+}

src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/extra-columns.json

@@ -1,17 +1,22 @@
 {
-  "range": "Sheet!A1:D11",
-  "majorDimension": "ROWS",
-  "values": [
-    ["Header 1", "Header 2", "Header 3"],
-    ["A2", "B2", "C2"],
-    ["A3", "B3", "C3"],
-    ["A4", "B4", "C4"],
-    ["A5", "B5", "C5"],
-    ["A6", "B6", "C6"],
-    ["A7", "B7", "C7"],
-    ["A8", "B8", "C8"],
-    ["A9", "B9", "C9"],
-    ["A10", "B10", "C10"],
-    ["A11", "B11", "C11", "D11"]
+  "spreadsheetId": "eae3913a86de8d5ccfe841fd3c7ec50dba3902045232",
+  "valueRanges": [
+    {
+      "range": "Sheet!A1:D11",
+      "majorDimension": "ROWS",
+      "values": [
+        ["Header 1", "Header 2", "Header 3"],
+        ["A2", "B2", "C2"],
+        ["A3", "B3", "C3"],
+        ["A4", "B4", "C4"],
+        ["A5", "B5", "C5"],
+        ["A6", "B6", "C6"],
+        ["A7", "B7", "C7"],
+        ["A8", "B8", "C8"],
+        ["A9", "B9", "C9"],
+        ["A10", "B10", "C10"],
+        ["A11", "B11", "C11", "D11"]
+      ]
+    }
   ]
 }

src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/extra-empty-rows.json

@@ -1,18 +1,23 @@
 {
-  "range": "Sheet!A1:D11",
-  "majorDimension": "ROWS",
-  "values": [
-    [],
-    ["Header 1", "Header 2", "Header 3"],
-    ["A2", "B2", "C2"],
-    ["A3", "B3", "C3"],
-    ["A4", "B4", "C4"],
-    ["A5", "B5", "C5"],
-    ["A6", "B6", "C6"],
-    ["A7", "B7", "C7"],
-    ["A8", "B8", "C8"],
-    ["A9", "B9", "C9"],
-    ["A10", "B10", "C10"],
-    ["A11", "B11", "C11", "D11"]
+  "spreadsheetId": "eae3913a86de8d5ccfe841fd3c7ec50dba3902045232",
+  "valueRanges": [
+    {
+      "range": "Sheet!A1:D11",
+      "majorDimension": "ROWS",
+      "values": [
+        [],
+        ["Header 1", "Header 2", "Header 3"],
+        ["A2", "B2", "C2"],
+        ["A3", "B3", "C3"],
+        ["A4", "B4", "C4"],
+        ["A5", "B5", "C5"],
+        ["A6", "B6", "C6"],
+        ["A7", "B7", "C7"],
+        ["A8", "B8", "C8"],
+        ["A9", "B9", "C9"],
+        ["A10", "B10", "C10"],
+        ["A11", "B11", "C11", "D11"]
+      ]
+    }
   ]
 }

src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/missing-columns.json

@@ -1,11 +1,16 @@
 {
-  "range": "Sheet!A1:C5",
-  "majorDimension": "ROWS",
-  "values": [
-    ["Header 1", "Header 2", "Header 3"],
-    ["A2", "B2", "C2"],
-    ["A3", "B3"],
-    ["A4", "B4"],
-    ["A5", "B5", "C5"]
+  "spreadsheetId": "eae3913a86de8d5ccfe841fd3c7ec50dba3902045232",
+  "valueRanges": [
+    {
+      "range": "Sheet!A1:C5",
+      "majorDimension": "ROWS",
+      "values": [
+        ["Header 1", "Header 2", "Header 3"],
+        ["A2", "B2", "C2"],
+        ["A3", "B3"],
+        ["A4", "B4"],
+        ["A5", "B5", "C5"]
+      ]
+    }
   ]
 }

src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Fixtures/spreadsheet.json

@@ -0,0 +1,69 @@
+{
+  "spreadsheetId": "eae3913a86de8d5ccfe841fd3c7ec50dba3902045232",
+  "properties": {
+    "title": "Spreadsheet 1",
+    "locale": "en_US",
+    "autoRecalc": "ON_CHANGE",
+    "timeZone": "Europe/Warsaw",
+    "defaultFormat": {
+      "backgroundColor": {
+        "red": 1,
+        "green": 1,
+        "blue": 1
+      },
+      "padding": {
+        "top": 2,
+        "right": 3,
+        "bottom": 2,
+        "left": 3
+      },
+      "verticalAlignment": "BOTTOM",
+      "wrapStrategy": "OVERFLOW_CELL",
+      "textFormat": {
+        "foregroundColor": {},
+        "fontFamily": "arial,sans,sans-serif",
+        "fontSize": 10,
+        "bold": false,
+        "italic": false,
+        "strikethrough": false,
+        "underline": false,
+        "foregroundColorStyle": {
+          "rgbColor": {}
+        }
+      },
+      "backgroundColorStyle": {
+        "rgbColor": {
+          "red": 1,
+          "green": 1,
+          "blue": 1
+        }
+      }
+    },
+    "spreadsheetTheme": {
+      "primaryFontFamily": "Arial",
+      "themeColors": [
+        {
+          "colorType": "TEXT",
+          "color": {
+            "rgbColor": {}
+          }
+        }
+      ]
+    }
+  },
+  "sheets": [
+    {
+      "properties": {
+        "sheetId": 0,
+        "title": "Sheet",
+        "index": 0,
+        "sheetType": "GRID",
+        "gridProperties": {
+          "rowCount": 20,
+          "columnCount": 6
+        }
+      }
+    }
+  ],
+  "spreadsheetUrl": "https://docs.google.com/spreadsheets/d/eae3913a86de8d5ccfe841fd3c7ec50dba3902045232/edit"
+}