Reuse pagination merge callbacks across job managers

cursoragent · shrisukhani · cursoragent · commit ddf37ac7d8f3 · 2026-02-14T10:33:58.000Z
Co-authored-by: Shri Sukhani &lt;shrisukhani@users.noreply.github.com&gt;
diff --git a/hyperbrowser/client/managers/async_manager/crawl.py b/hyperbrowser/client/managers/async_manager/crawl.py
@@ -10,8 +10,8 @@
     retry_operation_async,
 )
 from ..job_pagination_utils import (
+    build_job_paginated_page_merge_callback,
     initialize_job_paginated_response,
-    merge_job_paginated_page_response,
 )
 from ..serialization_utils import (
     serialize_model_dump_or_default,
@@ -113,13 +113,6 @@ async def start_and_wait(
             total_counter_alias="totalCrawledPages",
         )
 
-        def merge_page_response(page_response: CrawlJobResponse) -> None:
-            merge_job_paginated_page_response(
-                job_response,
-                page_response,
-                total_counter_attr="total_crawled_pages",
-            )
-
         await collect_paginated_results_async(
             operation_name=operation_name,
             get_next_page=lambda page: self.get(
@@ -132,7 +125,10 @@ def merge_page_response(page_response: CrawlJobResponse) -> None:
             get_total_page_batches=lambda page_response: (
                 page_response.total_page_batches
             ),
-            on_page_success=merge_page_response,
+            on_page_success=build_job_paginated_page_merge_callback(
+                job_response=job_response,
+                total_counter_attr="total_crawled_pages",
+            ),
             max_wait_seconds=max_wait_seconds,
             max_attempts=POLLING_ATTEMPTS,
             retry_delay_seconds=0.5,
diff --git a/hyperbrowser/client/managers/async_manager/scrape.py b/hyperbrowser/client/managers/async_manager/scrape.py
@@ -11,8 +11,8 @@
     wait_for_job_result_async,
 )
 from ..job_pagination_utils import (
+    build_job_paginated_page_merge_callback,
     initialize_job_paginated_response,
-    merge_job_paginated_page_response,
 )
 from ..serialization_utils import (
     serialize_model_dump_or_default,
@@ -120,13 +120,6 @@ async def start_and_wait(
             total_counter_alias="totalScrapedPages",
         )
 
-        def merge_page_response(page_response: BatchScrapeJobResponse) -> None:
-            merge_job_paginated_page_response(
-                job_response,
-                page_response,
-                total_counter_attr="total_scraped_pages",
-            )
-
         await collect_paginated_results_async(
             operation_name=operation_name,
             get_next_page=lambda page: self.get(
@@ -139,7 +132,10 @@ def merge_page_response(page_response: BatchScrapeJobResponse) -> None:
             get_total_page_batches=lambda page_response: (
                 page_response.total_page_batches
             ),
-            on_page_success=merge_page_response,
+            on_page_success=build_job_paginated_page_merge_callback(
+                job_response=job_response,
+                total_counter_attr="total_scraped_pages",
+            ),
             max_wait_seconds=max_wait_seconds,
             max_attempts=POLLING_ATTEMPTS,
             retry_delay_seconds=0.5,
diff --git a/hyperbrowser/client/managers/async_manager/web/batch_fetch.py b/hyperbrowser/client/managers/async_manager/web/batch_fetch.py
@@ -11,8 +11,8 @@
 from ...web_payload_utils import build_batch_fetch_start_payload
 from ...web_payload_utils import build_batch_fetch_get_params
 from ...web_pagination_utils import (
+    build_paginated_page_merge_callback,
     initialize_paginated_job_response,
-    merge_paginated_page_response,
 )
 from ....polling import (
     build_fetch_operation_name,
@@ -106,9 +106,6 @@ async def start_and_wait(
             status=job_status,
         )
 
-        def merge_page_response(page_response: BatchFetchJobResponse) -> None:
-            merge_paginated_page_response(job_response, page_response)
-
         await collect_paginated_results_async(
             operation_name=operation_name,
             get_next_page=lambda page: self.get(
@@ -121,7 +118,9 @@ def merge_page_response(page_response: BatchFetchJobResponse) -> None:
             get_total_page_batches=lambda page_response: (
                 page_response.total_page_batches
             ),
-            on_page_success=merge_page_response,
+            on_page_success=build_paginated_page_merge_callback(
+                job_response=job_response,
+            ),
             max_wait_seconds=max_wait_seconds,
             max_attempts=POLLING_ATTEMPTS,
             retry_delay_seconds=0.5,
diff --git a/hyperbrowser/client/managers/async_manager/web/crawl.py b/hyperbrowser/client/managers/async_manager/web/crawl.py
@@ -11,8 +11,8 @@
 from ...web_payload_utils import build_web_crawl_start_payload
 from ...web_payload_utils import build_web_crawl_get_params
 from ...web_pagination_utils import (
+    build_paginated_page_merge_callback,
     initialize_paginated_job_response,
-    merge_paginated_page_response,
 )
 from ....polling import (
     build_fetch_operation_name,
@@ -104,9 +104,6 @@ async def start_and_wait(
             status=job_status,
         )
 
-        def merge_page_response(page_response: WebCrawlJobResponse) -> None:
-            merge_paginated_page_response(job_response, page_response)
-
         await collect_paginated_results_async(
             operation_name=operation_name,
             get_next_page=lambda page: self.get(
@@ -119,7 +116,9 @@ def merge_page_response(page_response: WebCrawlJobResponse) -> None:
             get_total_page_batches=lambda page_response: (
                 page_response.total_page_batches
             ),
-            on_page_success=merge_page_response,
+            on_page_success=build_paginated_page_merge_callback(
+                job_response=job_response,
+            ),
             max_wait_seconds=max_wait_seconds,
             max_attempts=POLLING_ATTEMPTS,
             retry_delay_seconds=0.5,
diff --git a/hyperbrowser/client/managers/job_pagination_utils.py b/hyperbrowser/client/managers/job_pagination_utils.py
@@ -1,4 +1,4 @@
-from typing import Any, Type, TypeVar
+from typing import Any, Callable, Type, TypeVar
 
 T = TypeVar("T")
 
@@ -39,3 +39,18 @@ def merge_job_paginated_page_response(
     job_response.total_page_batches = page_response.total_page_batches
     job_response.batch_size = page_response.batch_size
     job_response.error = page_response.error
+
+
+def build_job_paginated_page_merge_callback(
+    *,
+    job_response: Any,
+    total_counter_attr: str,
+) -> Callable[[Any], None]:
+    def _merge_callback(page_response: Any) -> None:
+        merge_job_paginated_page_response(
+            job_response,
+            page_response,
+            total_counter_attr=total_counter_attr,
+        )
+
+    return _merge_callback
diff --git a/hyperbrowser/client/managers/sync_manager/crawl.py b/hyperbrowser/client/managers/sync_manager/crawl.py
@@ -10,8 +10,8 @@
     retry_operation,
 )
 from ..job_pagination_utils import (
+    build_job_paginated_page_merge_callback,
     initialize_job_paginated_response,
-    merge_job_paginated_page_response,
 )
 from ..serialization_utils import (
     serialize_model_dump_or_default,
@@ -113,13 +113,6 @@ def start_and_wait(
             total_counter_alias="totalCrawledPages",
         )
 
-        def merge_page_response(page_response: CrawlJobResponse) -> None:
-            merge_job_paginated_page_response(
-                job_response,
-                page_response,
-                total_counter_attr="total_crawled_pages",
-            )
-
         collect_paginated_results(
             operation_name=operation_name,
             get_next_page=lambda page: self.get(
@@ -132,7 +125,10 @@ def merge_page_response(page_response: CrawlJobResponse) -> None:
             get_total_page_batches=lambda page_response: (
                 page_response.total_page_batches
             ),
-            on_page_success=merge_page_response,
+            on_page_success=build_job_paginated_page_merge_callback(
+                job_response=job_response,
+                total_counter_attr="total_crawled_pages",
+            ),
             max_wait_seconds=max_wait_seconds,
             max_attempts=POLLING_ATTEMPTS,
             retry_delay_seconds=0.5,
diff --git a/hyperbrowser/client/managers/sync_manager/scrape.py b/hyperbrowser/client/managers/sync_manager/scrape.py
@@ -11,8 +11,8 @@
     wait_for_job_result,
 )
 from ..job_pagination_utils import (
+    build_job_paginated_page_merge_callback,
     initialize_job_paginated_response,
-    merge_job_paginated_page_response,
 )
 from ..serialization_utils import (
     serialize_model_dump_or_default,
@@ -118,13 +118,6 @@ def start_and_wait(
             total_counter_alias="totalScrapedPages",
         )
 
-        def merge_page_response(page_response: BatchScrapeJobResponse) -> None:
-            merge_job_paginated_page_response(
-                job_response,
-                page_response,
-                total_counter_attr="total_scraped_pages",
-            )
-
         collect_paginated_results(
             operation_name=operation_name,
             get_next_page=lambda page: self.get(
@@ -137,7 +130,10 @@ def merge_page_response(page_response: BatchScrapeJobResponse) -> None:
             get_total_page_batches=lambda page_response: (
                 page_response.total_page_batches
             ),
-            on_page_success=merge_page_response,
+            on_page_success=build_job_paginated_page_merge_callback(
+                job_response=job_response,
+                total_counter_attr="total_scraped_pages",
+            ),
             max_wait_seconds=max_wait_seconds,
             max_attempts=POLLING_ATTEMPTS,
             retry_delay_seconds=0.5,
diff --git a/hyperbrowser/client/managers/sync_manager/web/batch_fetch.py b/hyperbrowser/client/managers/sync_manager/web/batch_fetch.py
@@ -11,8 +11,8 @@
 from ...web_payload_utils import build_batch_fetch_start_payload
 from ...web_payload_utils import build_batch_fetch_get_params
 from ...web_pagination_utils import (
+    build_paginated_page_merge_callback,
     initialize_paginated_job_response,
-    merge_paginated_page_response,
 )
 from ....polling import (
     build_fetch_operation_name,
@@ -104,9 +104,6 @@ def start_and_wait(
             status=job_status,
         )
 
-        def merge_page_response(page_response: BatchFetchJobResponse) -> None:
-            merge_paginated_page_response(job_response, page_response)
-
         collect_paginated_results(
             operation_name=operation_name,
             get_next_page=lambda page: self.get(
@@ -119,7 +116,9 @@ def merge_page_response(page_response: BatchFetchJobResponse) -> None:
             get_total_page_batches=lambda page_response: (
                 page_response.total_page_batches
             ),
-            on_page_success=merge_page_response,
+            on_page_success=build_paginated_page_merge_callback(
+                job_response=job_response,
+            ),
             max_wait_seconds=max_wait_seconds,
             max_attempts=POLLING_ATTEMPTS,
             retry_delay_seconds=0.5,
diff --git a/hyperbrowser/client/managers/sync_manager/web/crawl.py b/hyperbrowser/client/managers/sync_manager/web/crawl.py
@@ -11,8 +11,8 @@
 from ...web_payload_utils import build_web_crawl_start_payload
 from ...web_payload_utils import build_web_crawl_get_params
 from ...web_pagination_utils import (
+    build_paginated_page_merge_callback,
     initialize_paginated_job_response,
-    merge_paginated_page_response,
 )
 from ....polling import (
     build_fetch_operation_name,
@@ -104,9 +104,6 @@ def start_and_wait(
             status=job_status,
         )
 
-        def merge_page_response(page_response: WebCrawlJobResponse) -> None:
-            merge_paginated_page_response(job_response, page_response)
-
         collect_paginated_results(
             operation_name=operation_name,
             get_next_page=lambda page: self.get(
@@ -119,7 +116,9 @@ def merge_page_response(page_response: WebCrawlJobResponse) -> None:
             get_total_page_batches=lambda page_response: (
                 page_response.total_page_batches
             ),
-            on_page_success=merge_page_response,
+            on_page_success=build_paginated_page_merge_callback(
+                job_response=job_response,
+            ),
             max_wait_seconds=max_wait_seconds,
             max_attempts=POLLING_ATTEMPTS,
             retry_delay_seconds=0.5,
diff --git a/hyperbrowser/client/managers/web_pagination_utils.py b/hyperbrowser/client/managers/web_pagination_utils.py
@@ -1,4 +1,4 @@
-from typing import Any, Type, TypeVar
+from typing import Any, Callable, Type, TypeVar
 
 T = TypeVar("T")
 
@@ -29,3 +29,10 @@ def merge_paginated_page_response(job_response: Any, page_response: Any) -> None
     job_response.total_page_batches = page_response.total_page_batches
     job_response.batch_size = page_response.batch_size
     job_response.error = page_response.error
+
+
+def build_paginated_page_merge_callback(*, job_response: Any) -> Callable[[Any], None]:
+    def _merge_callback(page_response: Any) -> None:
+        merge_paginated_page_response(job_response, page_response)
+
+    return _merge_callback
diff --git a/tests/test_job_pagination_helper_usage.py b/tests/test_job_pagination_helper_usage.py
@@ -17,7 +17,8 @@ def test_job_managers_use_shared_pagination_helpers():
     for module_path in BATCH_JOB_MANAGER_MODULES:
         module_text = Path(module_path).read_text(encoding="utf-8")
         assert "initialize_job_paginated_response(" in module_text
-        assert "merge_job_paginated_page_response(" in module_text
+        assert "build_job_paginated_page_merge_callback(" in module_text
         assert "total_page_batches = page_response.total_page_batches" not in module_text
         assert "job_response = BatchScrapeJobResponse(" not in module_text
         assert "job_response = CrawlJobResponse(" not in module_text
+        assert "def merge_page_response(" not in module_text
diff --git a/tests/test_job_pagination_utils.py b/tests/test_job_pagination_utils.py
@@ -1,4 +1,5 @@
 from hyperbrowser.client.managers.job_pagination_utils import (
+    build_job_paginated_page_merge_callback,
     initialize_job_paginated_response,
     merge_job_paginated_page_response,
 )
@@ -70,3 +71,33 @@ def test_merge_job_paginated_page_response_updates_totals_and_error():
     assert job_response.total_crawled_pages == 21
     assert job_response.batch_size == 50
     assert job_response.error == "partial failure"
+
+
+def test_build_job_paginated_page_merge_callback_merges_values():
+    job_response = initialize_job_paginated_response(
+        model=BatchScrapeJobResponse,
+        job_id="job-3",
+        status="running",
+        total_counter_alias="totalScrapedPages",
+    )
+    page_response = BatchScrapeJobResponse(
+        jobId="job-3",
+        status="running",
+        data=[],
+        currentPageBatch=1,
+        totalPageBatches=5,
+        totalScrapedPages=6,
+        batchSize=30,
+        error=None,
+    )
+    merge_callback = build_job_paginated_page_merge_callback(
+        job_response=job_response,
+        total_counter_attr="total_scraped_pages",
+    )
+
+    merge_callback(page_response)
+
+    assert job_response.current_page_batch == 1
+    assert job_response.total_page_batches == 5
+    assert job_response.total_scraped_pages == 6
+    assert job_response.batch_size == 30
diff --git a/tests/test_web_pagination_utils.py b/tests/test_web_pagination_utils.py
@@ -1,4 +1,5 @@
 from hyperbrowser.client.managers.web_pagination_utils import (
+    build_paginated_page_merge_callback,
     initialize_paginated_job_response,
     merge_paginated_page_response,
 )
@@ -62,3 +63,30 @@ def test_merge_paginated_page_response_merges_page_data_and_metadata():
     assert job_response.total_pages == 10
     assert job_response.batch_size == 100
     assert job_response.error == "partial error"
+
+
+def test_build_paginated_page_merge_callback_merges_values():
+    job_response = initialize_paginated_job_response(
+        model=WebCrawlJobResponse,
+        job_id="job-4",
+        status="running",
+    )
+    page_response = WebCrawlJobResponse(
+        jobId="job-4",
+        status="running",
+        data=[],
+        currentPageBatch=4,
+        totalPageBatches=8,
+        totalPages=19,
+        batchSize=40,
+        error="crawl page error",
+    )
+    merge_callback = build_paginated_page_merge_callback(job_response=job_response)
+
+    merge_callback(page_response)
+
+    assert job_response.current_page_batch == 4
+    assert job_response.total_page_batches == 8
+    assert job_response.total_pages == 19
+    assert job_response.batch_size == 40
+    assert job_response.error == "crawl page error"
diff --git a/tests/test_web_payload_helper_usage.py b/tests/test_web_payload_helper_usage.py
