Centralize web batch/crawl get payload serialization

Co-authored-by: Shri Sukhani <shrisukhani@users.noreply.github.com>

Author: cursoragent

hyperbrowser/client/managers/async_manager/web/batch_fetch.py

@@ -8,8 +8,8 @@
     BatchFetchJobResponse,
     POLLING_ATTEMPTS,
 )
-from ...serialization_utils import serialize_model_dump_to_dict
 from ...web_payload_utils import build_batch_fetch_start_payload
+from ...web_payload_utils import build_batch_fetch_get_params
 from ....polling import (
     build_fetch_operation_name,
     build_operation_name,
@@ -53,11 +53,7 @@ async def get_status(self, job_id: str) -> BatchFetchJobStatusResponse:
     async def get(
         self, job_id: str, params: Optional[GetBatchFetchJobParams] = None
     ) -> BatchFetchJobResponse:
-        params_obj = params or GetBatchFetchJobParams()
-        query_params = serialize_model_dump_to_dict(
-            params_obj,
-            error_message="Failed to serialize batch fetch get params",
-        )
+        query_params = build_batch_fetch_get_params(params)
         response = await self._client.transport.get(
             self._client._build_url(f"/web/batch-fetch/{job_id}"),
             params=query_params,

hyperbrowser/client/managers/async_manager/web/crawl.py

@@ -8,8 +8,8 @@
     WebCrawlJobResponse,
     POLLING_ATTEMPTS,
 )
-from ...serialization_utils import serialize_model_dump_to_dict
 from ...web_payload_utils import build_web_crawl_start_payload
+from ...web_payload_utils import build_web_crawl_get_params
 from ....polling import (
     build_fetch_operation_name,
     build_operation_name,
@@ -51,11 +51,7 @@ async def get_status(self, job_id: str) -> WebCrawlJobStatusResponse:
     async def get(
         self, job_id: str, params: Optional[GetWebCrawlJobParams] = None
     ) -> WebCrawlJobResponse:
-        params_obj = params or GetWebCrawlJobParams()
-        query_params = serialize_model_dump_to_dict(
-            params_obj,
-            error_message="Failed to serialize web crawl get params",
-        )
+        query_params = build_web_crawl_get_params(params)
         response = await self._client.transport.get(
             self._client._build_url(f"/web/crawl/{job_id}"),
             params=query_params,

hyperbrowser/client/managers/sync_manager/web/batch_fetch.py

@@ -8,8 +8,8 @@
     BatchFetchJobResponse,
     POLLING_ATTEMPTS,
 )
-from ...serialization_utils import serialize_model_dump_to_dict
 from ...web_payload_utils import build_batch_fetch_start_payload
+from ...web_payload_utils import build_batch_fetch_get_params
 from ....polling import (
     build_fetch_operation_name,
     build_operation_name,
@@ -51,11 +51,7 @@ def get_status(self, job_id: str) -> BatchFetchJobStatusResponse:
     def get(
         self, job_id: str, params: Optional[GetBatchFetchJobParams] = None
     ) -> BatchFetchJobResponse:
-        params_obj = params or GetBatchFetchJobParams()
-        query_params = serialize_model_dump_to_dict(
-            params_obj,
-            error_message="Failed to serialize batch fetch get params",
-        )
+        query_params = build_batch_fetch_get_params(params)
         response = self._client.transport.get(
             self._client._build_url(f"/web/batch-fetch/{job_id}"),
             params=query_params,

hyperbrowser/client/managers/sync_manager/web/crawl.py

@@ -8,8 +8,8 @@
     WebCrawlJobResponse,
     POLLING_ATTEMPTS,
 )
-from ...serialization_utils import serialize_model_dump_to_dict
 from ...web_payload_utils import build_web_crawl_start_payload
+from ...web_payload_utils import build_web_crawl_get_params
 from ....polling import (
     build_fetch_operation_name,
     build_operation_name,
@@ -51,11 +51,7 @@ def get_status(self, job_id: str) -> WebCrawlJobStatusResponse:
     def get(
         self, job_id: str, params: Optional[GetWebCrawlJobParams] = None
     ) -> WebCrawlJobResponse:
-        params_obj = params or GetWebCrawlJobParams()
-        query_params = serialize_model_dump_to_dict(
-            params_obj,
-            error_message="Failed to serialize web crawl get params",
-        )
+        query_params = build_web_crawl_get_params(params)
         response = self._client.transport.get(
             self._client._build_url(f"/web/crawl/{job_id}"),
             params=query_params,

hyperbrowser/client/managers/web_payload_utils.py

@@ -1,7 +1,9 @@
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 from hyperbrowser.models import (
     FetchParams,
+    GetBatchFetchJobParams,
+    GetWebCrawlJobParams,
     StartBatchFetchJobParams,
     StartWebCrawlJobParams,
     WebSearchParams,
@@ -43,3 +45,23 @@ def build_web_crawl_start_payload(params: StartWebCrawlJobParams) -> Dict[str, A
     )
     inject_web_output_schemas(payload, params.outputs.formats if params.outputs else None)
     return payload
+
+
+def build_batch_fetch_get_params(
+    params: Optional[GetBatchFetchJobParams] = None,
+) -> Dict[str, Any]:
+    params_obj = params or GetBatchFetchJobParams()
+    return serialize_model_dump_to_dict(
+        params_obj,
+        error_message="Failed to serialize batch fetch get params",
+    )
+
+
+def build_web_crawl_get_params(
+    params: Optional[GetWebCrawlJobParams] = None,
+) -> Dict[str, Any]:
+    params_obj = params or GetWebCrawlJobParams()
+    return serialize_model_dump_to_dict(
+        params_obj,
+        error_message="Failed to serialize web crawl get params",
+    )

tests/test_web_payload_helper_usage.py

@@ -34,11 +34,15 @@ def test_batch_fetch_managers_use_shared_start_payload_helper():
     for module_path in BATCH_FETCH_MANAGER_MODULES:
         module_text = Path(module_path).read_text(encoding="utf-8")
         assert "build_batch_fetch_start_payload(" in module_text
+        assert "build_batch_fetch_get_params(" in module_text
         assert "inject_web_output_schemas(" not in module_text
+        assert "serialize_model_dump_to_dict(" not in module_text
 
 
 def test_web_crawl_managers_use_shared_start_payload_helper():
     for module_path in WEB_CRAWL_MANAGER_MODULES:
         module_text = Path(module_path).read_text(encoding="utf-8")
         assert "build_web_crawl_start_payload(" in module_text
+        assert "build_web_crawl_get_params(" in module_text
         assert "inject_web_output_schemas(" not in module_text
+        assert "serialize_model_dump_to_dict(" not in module_text

tests/test_web_payload_utils.py

@@ -5,6 +5,8 @@
 from hyperbrowser.models import (
     FetchOutputOptions,
     FetchParams,
+    GetBatchFetchJobParams,
+    GetWebCrawlJobParams,
     StartBatchFetchJobParams,
     StartWebCrawlJobParams,
     WebSearchParams,
@@ -85,6 +87,34 @@ def test_build_web_crawl_start_payload_returns_serialized_payload():
     assert payload["url"] == "https://example.com"
 
 
+def test_build_batch_fetch_get_params_returns_serialized_payload():
+    payload = web_payload_utils.build_batch_fetch_get_params(
+        GetBatchFetchJobParams(page=2, batch_size=50)
+    )
+
+    assert payload == {"page": 2, "batchSize": 50}
+
+
+def test_build_batch_fetch_get_params_uses_default_params():
+    payload = web_payload_utils.build_batch_fetch_get_params()
+
+    assert payload == {}
+
+
+def test_build_web_crawl_get_params_returns_serialized_payload():
+    payload = web_payload_utils.build_web_crawl_get_params(
+        GetWebCrawlJobParams(page=3, batch_size=25)
+    )
+
+    assert payload == {"page": 3, "batchSize": 25}
+
+
+def test_build_web_crawl_get_params_uses_default_params():
+    payload = web_payload_utils.build_web_crawl_get_params()
+
+    assert payload == {}
+
+
 def test_build_web_crawl_start_payload_invokes_schema_injection(
     monkeypatch: pytest.MonkeyPatch,
 ):
@@ -155,3 +185,27 @@ def model_dump(self, **kwargs):  # noqa: ARG002
         web_payload_utils.build_web_crawl_start_payload(_BrokenWebCrawlParams())  # type: ignore[arg-type]
 
     assert exc_info.value.original_error is None
+
+
+def test_build_batch_fetch_get_params_wraps_runtime_model_dump_failures():
+    class _BrokenBatchFetchGetParams:
+        def model_dump(self, **kwargs):  # noqa: ARG002
+            raise RuntimeError("boom")
+
+    with pytest.raises(
+        HyperbrowserError, match="Failed to serialize batch fetch get params"
+    ) as exc_info:
+        web_payload_utils.build_batch_fetch_get_params(_BrokenBatchFetchGetParams())  # type: ignore[arg-type]
+
+    assert isinstance(exc_info.value.original_error, RuntimeError)
+
+
+def test_build_web_crawl_get_params_preserves_hyperbrowser_model_dump_failures():
+    class _BrokenWebCrawlGetParams:
+        def model_dump(self, **kwargs):  # noqa: ARG002
+            raise HyperbrowserError("custom dump failure")
+
+    with pytest.raises(HyperbrowserError, match="custom dump failure") as exc_info:
+        web_payload_utils.build_web_crawl_get_params(_BrokenWebCrawlGetParams())  # type: ignore[arg-type]
+
+    assert exc_info.value.original_error is None