Share scrape/crawl pagination helpers across managers

cursoragent · shrisukhani · cursoragent · commit c00a15f62a62 · 2026-02-14T10:30:23.000Z
Co-authored-by: Shri Sukhani &lt;shrisukhani@users.noreply.github.com&gt;
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -89,6 +89,7 @@ This runs lint, format checks, compile checks, tests, and package build.
   - `tests/test_examples_naming_convention.py` (example sync/async prefix naming enforcement),
   - `tests/test_examples_syntax.py` (example script syntax guardrail),
   - `tests/test_guardrail_ast_utils.py` (shared AST guard utility contract),
+  - `tests/test_job_pagination_helper_usage.py` (shared scrape/crawl pagination helper usage enforcement),
   - `tests/test_makefile_quality_targets.py` (Makefile quality-gate target enforcement),
   - `tests/test_manager_model_dump_usage.py` (manager serialization centralization),
   - `tests/test_mapping_keys_access_usage.py` (centralized key-iteration boundaries),
diff --git a/hyperbrowser/client/managers/async_manager/crawl.py b/hyperbrowser/client/managers/async_manager/crawl.py
@@ -9,6 +9,10 @@
     poll_until_terminal_status_async,
     retry_operation_async,
 )
+from ..job_pagination_utils import (
+    initialize_job_paginated_response,
+    merge_job_paginated_page_response,
+)
 from ..serialization_utils import (
     serialize_model_dump_or_default,
     serialize_model_dump_to_dict,
@@ -102,24 +106,19 @@ async def start_and_wait(
                 retry_delay_seconds=0.5,
             )
 
-        job_response = CrawlJobResponse(
-            jobId=job_id,
+        job_response = initialize_job_paginated_response(
+            model=CrawlJobResponse,
+            job_id=job_id,
             status=job_status,
-            data=[],
-            currentPageBatch=0,
-            totalPageBatches=0,
-            totalCrawledPages=0,
-            batchSize=100,
+            total_counter_alias="totalCrawledPages",
         )
 
         def merge_page_response(page_response: CrawlJobResponse) -> None:
-            if page_response.data:
-                job_response.data.extend(page_response.data)
-            job_response.current_page_batch = page_response.current_page_batch
-            job_response.total_crawled_pages = page_response.total_crawled_pages
-            job_response.total_page_batches = page_response.total_page_batches
-            job_response.batch_size = page_response.batch_size
-            job_response.error = page_response.error
+            merge_job_paginated_page_response(
+                job_response,
+                page_response,
+                total_counter_attr="total_crawled_pages",
+            )
 
         await collect_paginated_results_async(
             operation_name=operation_name,
diff --git a/hyperbrowser/client/managers/async_manager/scrape.py b/hyperbrowser/client/managers/async_manager/scrape.py
@@ -10,6 +10,10 @@
     retry_operation_async,
     wait_for_job_result_async,
 )
+from ..job_pagination_utils import (
+    initialize_job_paginated_response,
+    merge_job_paginated_page_response,
+)
 from ..serialization_utils import (
     serialize_model_dump_or_default,
     serialize_model_dump_to_dict,
@@ -109,24 +113,19 @@ async def start_and_wait(
                 retry_delay_seconds=0.5,
             )
 
-        job_response = BatchScrapeJobResponse(
-            jobId=job_id,
+        job_response = initialize_job_paginated_response(
+            model=BatchScrapeJobResponse,
+            job_id=job_id,
             status=job_status,
-            data=[],
-            currentPageBatch=0,
-            totalPageBatches=0,
-            totalScrapedPages=0,
-            batchSize=100,
+            total_counter_alias="totalScrapedPages",
         )
 
         def merge_page_response(page_response: BatchScrapeJobResponse) -> None:
-            if page_response.data:
-                job_response.data.extend(page_response.data)
-            job_response.current_page_batch = page_response.current_page_batch
-            job_response.total_scraped_pages = page_response.total_scraped_pages
-            job_response.total_page_batches = page_response.total_page_batches
-            job_response.batch_size = page_response.batch_size
-            job_response.error = page_response.error
+            merge_job_paginated_page_response(
+                job_response,
+                page_response,
+                total_counter_attr="total_scraped_pages",
+            )
 
         await collect_paginated_results_async(
             operation_name=operation_name,
diff --git a/hyperbrowser/client/managers/job_pagination_utils.py b/hyperbrowser/client/managers/job_pagination_utils.py
@@ -0,0 +1,41 @@
+from typing import Any, Type, TypeVar
+
+T = TypeVar("T")
+
+
+def initialize_job_paginated_response(
+    *,
+    model: Type[T],
+    job_id: str,
+    status: str,
+    total_counter_alias: str,
+    batch_size: int = 100,
+) -> T:
+    return model(
+        jobId=job_id,
+        status=status,
+        data=[],
+        currentPageBatch=0,
+        totalPageBatches=0,
+        batchSize=batch_size,
+        **{total_counter_alias: 0},
+    )
+
+
+def merge_job_paginated_page_response(
+    job_response: Any,
+    page_response: Any,
+    *,
+    total_counter_attr: str,
+) -> None:
+    if page_response.data:
+        job_response.data.extend(page_response.data)
+    job_response.current_page_batch = page_response.current_page_batch
+    setattr(
+        job_response,
+        total_counter_attr,
+        getattr(page_response, total_counter_attr),
+    )
+    job_response.total_page_batches = page_response.total_page_batches
+    job_response.batch_size = page_response.batch_size
+    job_response.error = page_response.error
diff --git a/hyperbrowser/client/managers/sync_manager/crawl.py b/hyperbrowser/client/managers/sync_manager/crawl.py
@@ -9,6 +9,10 @@
     poll_until_terminal_status,
     retry_operation,
 )
+from ..job_pagination_utils import (
+    initialize_job_paginated_response,
+    merge_job_paginated_page_response,
+)
 from ..serialization_utils import (
     serialize_model_dump_or_default,
     serialize_model_dump_to_dict,
@@ -102,24 +106,19 @@ def start_and_wait(
                 retry_delay_seconds=0.5,
             )
 
-        job_response = CrawlJobResponse(
-            jobId=job_id,
+        job_response = initialize_job_paginated_response(
+            model=CrawlJobResponse,
+            job_id=job_id,
             status=job_status,
-            data=[],
-            currentPageBatch=0,
-            totalPageBatches=0,
-            totalCrawledPages=0,
-            batchSize=100,
+            total_counter_alias="totalCrawledPages",
         )
 
         def merge_page_response(page_response: CrawlJobResponse) -> None:
-            if page_response.data:
-                job_response.data.extend(page_response.data)
-            job_response.current_page_batch = page_response.current_page_batch
-            job_response.total_crawled_pages = page_response.total_crawled_pages
-            job_response.total_page_batches = page_response.total_page_batches
-            job_response.batch_size = page_response.batch_size
-            job_response.error = page_response.error
+            merge_job_paginated_page_response(
+                job_response,
+                page_response,
+                total_counter_attr="total_crawled_pages",
+            )
 
         collect_paginated_results(
             operation_name=operation_name,
diff --git a/hyperbrowser/client/managers/sync_manager/scrape.py b/hyperbrowser/client/managers/sync_manager/scrape.py
@@ -10,6 +10,10 @@
     retry_operation,
     wait_for_job_result,
 )
+from ..job_pagination_utils import (
+    initialize_job_paginated_response,
+    merge_job_paginated_page_response,
+)
 from ..serialization_utils import (
     serialize_model_dump_or_default,
     serialize_model_dump_to_dict,
@@ -107,24 +111,19 @@ def start_and_wait(
                 retry_delay_seconds=0.5,
             )
 
-        job_response = BatchScrapeJobResponse(
-            jobId=job_id,
+        job_response = initialize_job_paginated_response(
+            model=BatchScrapeJobResponse,
+            job_id=job_id,
             status=job_status,
-            data=[],
-            currentPageBatch=0,
-            totalPageBatches=0,
-            totalScrapedPages=0,
-            batchSize=100,
+            total_counter_alias="totalScrapedPages",
         )
 
         def merge_page_response(page_response: BatchScrapeJobResponse) -> None:
-            if page_response.data:
-                job_response.data.extend(page_response.data)
-            job_response.current_page_batch = page_response.current_page_batch
-            job_response.total_scraped_pages = page_response.total_scraped_pages
-            job_response.total_page_batches = page_response.total_page_batches
-            job_response.batch_size = page_response.batch_size
-            job_response.error = page_response.error
+            merge_job_paginated_page_response(
+                job_response,
+                page_response,
+                total_counter_attr="total_scraped_pages",
+            )
 
         collect_paginated_results(
             operation_name=operation_name,
diff --git a/tests/test_architecture_marker_usage.py b/tests/test_architecture_marker_usage.py
@@ -29,6 +29,7 @@
     "tests/test_examples_syntax.py",
     "tests/test_docs_python3_commands.py",
     "tests/test_examples_naming_convention.py",
+    "tests/test_job_pagination_helper_usage.py",
     "tests/test_example_sync_async_parity.py",
     "tests/test_example_run_instructions.py",
     "tests/test_computer_action_endpoint_helper_usage.py",
diff --git a/tests/test_job_pagination_helper_usage.py b/tests/test_job_pagination_helper_usage.py
@@ -0,0 +1,23 @@
+from pathlib import Path
+
+import pytest
+
+pytestmark = pytest.mark.architecture
+
+
+BATCH_JOB_MANAGER_MODULES = (
+    "hyperbrowser/client/managers/sync_manager/scrape.py",
+    "hyperbrowser/client/managers/async_manager/scrape.py",
+    "hyperbrowser/client/managers/sync_manager/crawl.py",
+    "hyperbrowser/client/managers/async_manager/crawl.py",
+)
+
+
+def test_job_managers_use_shared_pagination_helpers():
+    for module_path in BATCH_JOB_MANAGER_MODULES:
+        module_text = Path(module_path).read_text(encoding="utf-8")
+        assert "initialize_job_paginated_response(" in module_text
+        assert "merge_job_paginated_page_response(" in module_text
+        assert "total_page_batches = page_response.total_page_batches" not in module_text
+        assert "job_response = BatchScrapeJobResponse(" not in module_text
+        assert "job_response = CrawlJobResponse(" not in module_text
diff --git a/tests/test_job_pagination_utils.py b/tests/test_job_pagination_utils.py
@@ -0,0 +1,72 @@
+from hyperbrowser.client.managers.job_pagination_utils import (
+    initialize_job_paginated_response,
+    merge_job_paginated_page_response,
+)
+from hyperbrowser.models.crawl import CrawlJobResponse
+from hyperbrowser.models.scrape import BatchScrapeJobResponse
+
+
+def test_initialize_job_paginated_response_for_batch_scrape():
+    response = initialize_job_paginated_response(
+        model=BatchScrapeJobResponse,
+        job_id="job-1",
+        status="completed",
+        total_counter_alias="totalScrapedPages",
+    )
+
+    assert response.job_id == "job-1"
+    assert response.status == "completed"
+    assert response.data == []
+    assert response.current_page_batch == 0
+    assert response.total_page_batches == 0
+    assert response.total_scraped_pages == 0
+    assert response.batch_size == 100
+
+
+def test_initialize_job_paginated_response_for_crawl_with_custom_batch_size():
+    response = initialize_job_paginated_response(
+        model=CrawlJobResponse,
+        job_id="job-2",
+        status="running",
+        total_counter_alias="totalCrawledPages",
+        batch_size=25,
+    )
+
+    assert response.job_id == "job-2"
+    assert response.status == "running"
+    assert response.data == []
+    assert response.current_page_batch == 0
+    assert response.total_page_batches == 0
+    assert response.total_crawled_pages == 0
+    assert response.batch_size == 25
+
+
+def test_merge_job_paginated_page_response_updates_totals_and_error():
+    job_response = initialize_job_paginated_response(
+        model=CrawlJobResponse,
+        job_id="job-2",
+        status="running",
+        total_counter_alias="totalCrawledPages",
+    )
+    page_response = CrawlJobResponse(
+        jobId="job-2",
+        status="running",
+        data=[],
+        currentPageBatch=3,
+        totalPageBatches=9,
+        totalCrawledPages=21,
+        batchSize=50,
+        error="partial failure",
+    )
+
+    merge_job_paginated_page_response(
+        job_response,
+        page_response,
+        total_counter_attr="total_crawled_pages",
+    )
+
+    assert job_response.current_page_batch == 3
+    assert job_response.total_page_batches == 9
+    assert job_response.total_crawled_pages == 21
+    assert job_response.batch_size == 50
+    assert job_response.error == "partial failure"
