update of utils module

Author: vdusek

src/apify_client/_utils.py

@@ -2,15 +2,11 @@
 
 import asyncio
 import base64
-import contextlib
 import io
 import json
-import json as jsonlib
 import random
 import re
 import time
-from collections.abc import Callable
-from datetime import datetime, timezone
 from enum import Enum
 from http import HTTPStatus
 from typing import TYPE_CHECKING, Any, TypeVar, cast
@@ -20,141 +16,119 @@
 from apify_client.errors import InvalidResponseBodyError
 
 if TYPE_CHECKING:
-    from collections.abc import Awaitable
+    from collections.abc import Awaitable, Callable
 
     from impit import Response
 
     from apify_client.errors import ApifyApiError
 
-PARSE_DATE_FIELDS_MAX_DEPTH = 3
-PARSE_DATE_FIELDS_KEY_SUFFIX = 'At'
-RECORD_NOT_FOUND_EXCEPTION_TYPES = ['record-not-found', 'record-or-token-not-found']
-
 T = TypeVar('T')
-StopRetryingType = Callable[[], None]
-ListOrDict = TypeVar('ListOrDict', list, dict)
-
 
-def filter_out_none_values_recursively(dictionary: dict) -> dict:
-    """Return copy of the dictionary, recursively omitting all keys for which values are None."""
-    return cast('dict', filter_out_none_values_recursively_internal(dictionary))
 
-
-def filter_out_none_values_recursively_internal(
+def filter_out_none_values_recursively(
     dictionary: dict,
     *,
     remove_empty_dicts: bool | None = None,
-) -> dict | None:
-    """Recursively filters out None values from a dictionary.
-
-    Unfortunately, it's necessary to have an internal function for the correct result typing,
-    without having to create complicated overloads
-    """
-    result = {}
-    for k, v in dictionary.items():
-        if isinstance(v, dict):
-            v = filter_out_none_values_recursively_internal(  # noqa: PLW2901
-                v, remove_empty_dicts=remove_empty_dicts is True or remove_empty_dicts is None
-            )
-        if v is not None:
-            result[k] = v
-    if not result and remove_empty_dicts:
-        return None
-    return result
-
-
-def parse_date_fields(data: ListOrDict, max_depth: int = PARSE_DATE_FIELDS_MAX_DEPTH) -> ListOrDict:
-    """Recursively parse date fields in a list or dictionary up to the specified depth."""
-    if max_depth < 0:
-        return data
-
-    if isinstance(data, list):
-        return [parse_date_fields(item, max_depth - 1) for item in data]
-
-    if isinstance(data, dict):
-
-        def parse(key: str, value: object) -> object:
-            parsed_value = value
-            if key.endswith(PARSE_DATE_FIELDS_KEY_SUFFIX) and isinstance(value, str):
-                with contextlib.suppress(ValueError):
-                    parsed_value = datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=timezone.utc)
-            elif isinstance(value, dict):
-                parsed_value = parse_date_fields(value, max_depth - 1)
-            elif isinstance(value, list):
-                parsed_value = parse_date_fields(value, max_depth)
-            return parsed_value
-
-        return {key: parse(key, value) for (key, value) in data.items()}
-
-    return data
-
+) -> dict:
+    """Return a copy of the dictionary with all None values recursively removed.
 
-def is_content_type_json(content_type: str) -> bool:
-    """Check if the given content type is JSON."""
-    return bool(re.search(r'^application/json', content_type, flags=re.IGNORECASE))
+    Args:
+        dictionary: The dictionary to filter.
+        remove_empty_dicts: If True, also remove empty dictionaries after filtering.
 
-
-def is_content_type_xml(content_type: str) -> bool:
-    """Check if the given content type is XML."""
-    return bool(re.search(r'^application/.*xml$', content_type, flags=re.IGNORECASE))
-
-
-def is_content_type_text(content_type: str) -> bool:
-    """Check if the given content type is text."""
-    return bool(re.search(r'^text/', content_type, flags=re.IGNORECASE))
-
-
-def is_file_or_bytes(value: Any) -> bool:
-    """Check if the input value is a file-like object or bytes.
-
-    The check for IOBase is not ideal, it would be better to use duck typing,
-    but then the check would be super complex, judging from how the 'requests' library does it.
-    This way should be good enough for the vast majority of use cases, if it causes issues, we can improve it later.
+    Returns:
+        A new dictionary without None values.
     """
-    return isinstance(value, (bytes, bytearray, io.IOBase))
 
+    def _internal(dictionary: dict, *, remove_empty: bool | None = None) -> dict | None:
+        result = {}
+        for key, val in dictionary.items():
+            if isinstance(val, dict):
+                val = _internal(val, remove_empty=remove_empty)  # noqa: PLW2901
+            if val is not None:
+                result[key] = val
+        if not result and remove_empty:
+            return None
+        return result
 
-def json_dumps(obj: Any) -> str:
-    """Dump JSON to a string with the correct settings and serializer."""
-    return json.dumps(obj, ensure_ascii=False, indent=2, default=str)
+    return cast('dict', _internal(dictionary, remove_empty=remove_empty_dicts))
 
 
 def maybe_extract_enum_member_value(maybe_enum_member: Any) -> Any:
-    """Extract the value of an enumeration member if it is an Enum, otherwise return the original value."""
+    """Extract the value from an Enum member, or return the input unchanged if not an Enum."""
     if isinstance(maybe_enum_member, Enum):
         return maybe_enum_member.value
     return maybe_enum_member
 
 
 def to_safe_id(id: str) -> str:
-    # Identificators of resources in the API are either in the format `resource_id` or `username/resource_id`.
-    # Since the `/` character has a special meaning in URL paths,
-    # we replace it with `~` for proper route parsing on the API, where after parsing the URL it's replaced back to `/`.
+    """Convert a resource ID to URL-safe format by replacing `/` with `~`.
+
+    Args:
+        id: The resource identifier (format: `resource_id` or `username/resource_id`).
+
+    Returns:
+        The resource identifier with `/` replaced by `~`.
+    """
     return id.replace('/', '~')
 
 
 def pluck_data(parsed_response: Any) -> dict:
+    """Extract the "data" field from an API response.
+
+    Args:
+        parsed_response: The parsed API response.
+
+    Returns:
+        The value of the "data" field.
+
+    Raises:
+        ValueError: If the "data" field is missing.
+    """
     if isinstance(parsed_response, dict) and 'data' in parsed_response:
         return cast('dict', parsed_response['data'])
 
     raise ValueError('The "data" property is missing in the response.')
 
 
 def pluck_data_as_list(parsed_response: Any) -> list:
+    """Extract the "data" field from an API response as a list.
+
+    Args:
+        parsed_response: The parsed API response.
+
+    Returns:
+        The value of the "data" field as a list.
+
+    Raises:
+        ValueError: If the "data" field is missing.
+    """
     if isinstance(parsed_response, dict) and 'data' in parsed_response:
         return cast('list', parsed_response['data'])
 
     raise ValueError('The "data" property is missing in the response.')
 
 
 def retry_with_exp_backoff(
-    func: Callable[[StopRetryingType, int], T],
+    func: Callable[[Callable[[], None], int], T],
     *,
     max_retries: int = 8,
     backoff_base_millis: int = 500,
     backoff_factor: float = 2,
     random_factor: float = 1,
 ) -> T:
+    """Retry a function with exponential backoff.
+
+    Args:
+        func: Function to retry. Receives a stop_retrying callback and attempt number.
+        max_retries: Maximum number of retry attempts.
+        backoff_base_millis: Base backoff delay in milliseconds.
+        backoff_factor: Exponential backoff multiplier (1-10).
+        random_factor: Random jitter factor (0-1).
+
+    Returns:
+        The return value of the function.
+    """
     random_factor = min(max(0, random_factor), 1)
     backoff_factor = min(max(1, backoff_factor), 10)
     swallow = True
@@ -181,13 +155,25 @@ def stop_retrying() -> None:
 
 
 async def retry_with_exp_backoff_async(
-    async_func: Callable[[StopRetryingType, int], Awaitable[T]],
+    async_func: Callable[[Callable[[], None], int], Awaitable[T]],
     *,
     max_retries: int = 8,
     backoff_base_millis: int = 500,
     backoff_factor: float = 2,
     random_factor: float = 1,
 ) -> T:
+    """Retry an async function with exponential backoff.
+
+    Args:
+        async_func: Async function to retry. Receives a stop_retrying callback and attempt number.
+        max_retries: Maximum number of retry attempts.
+        backoff_base_millis: Base backoff delay in milliseconds.
+        backoff_factor: Exponential backoff multiplier (1-10).
+        random_factor: Random jitter factor (0-1).
+
+    Returns:
+        The return value of the async function.
+    """
     random_factor = min(max(0, random_factor), 1)
     backoff_factor = min(max(1, backoff_factor), 10)
     swallow = True
@@ -214,14 +200,29 @@ def stop_retrying() -> None:
 
 
 def catch_not_found_or_throw(exc: ApifyApiError) -> None:
+    """Suppress 404 Not Found errors, re-raise all other exceptions.
+
+    Args:
+        exc: The API error to check.
+
+    Raises:
+        ApifyApiError: If the error is not a 404 Not Found error.
+    """
     is_not_found_status = exc.status_code == HTTPStatus.NOT_FOUND
-    is_not_found_type = exc.type in RECORD_NOT_FOUND_EXCEPTION_TYPES
+    is_not_found_type = exc.type in ['record-not-found', 'record-or-token-not-found']
     if not (is_not_found_status and is_not_found_type):
         raise exc
 
 
 def encode_webhook_list_to_base64(webhooks: list[dict]) -> str:
-    """Encode a list of dictionaries representing webhooks to their base64-encoded representation for the API."""
+    """Encode a list of webhook dictionaries to base64 for API transmission.
+
+    Args:
+        webhooks: List of webhook dictionaries with keys like "event_types", "request_url", etc.
+
+    Returns:
+        Base64-encoded JSON string.
+    """
     data = []
     for webhook in webhooks:
         webhook_representation = {
@@ -234,25 +235,49 @@ def encode_webhook_list_to_base64(webhooks: list[dict]) -> str:
             webhook_representation['headersTemplate'] = webhook['headers_template']
         data.append(webhook_representation)
 
-    return base64.b64encode(jsonlib.dumps(data).encode('utf-8')).decode('ascii')
+    return base64.b64encode(json.dumps(data).encode('utf-8')).decode('ascii')
 
 
 def encode_key_value_store_record_value(value: Any, content_type: str | None = None) -> tuple[Any, str]:
+    """Encode a value for storage in a key-value store record.
+
+    Args:
+        value: The value to encode (can be dict, str, bytes, or file-like object).
+        content_type: The content type. If None, it's inferred from the value type.
+
+    Returns:
+        A tuple of (encoded_value, content_type).
+    """
     if not content_type:
-        if is_file_or_bytes(value):
+        if isinstance(value, (bytes, bytearray, io.IOBase)):
             content_type = 'application/octet-stream'
         elif isinstance(value, str):
             content_type = 'text/plain; charset=utf-8'
         else:
             content_type = 'application/json; charset=utf-8'
 
-    if 'application/json' in content_type and not is_file_or_bytes(value) and not isinstance(value, str):
-        value = jsonlib.dumps(value, ensure_ascii=False, indent=2, allow_nan=False, default=str).encode('utf-8')
+    if (
+        'application/json' in content_type
+        and not isinstance(value, (bytes, bytearray, io.IOBase))
+        and not isinstance(value, str)
+    ):
+        value = json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False, default=str).encode('utf-8')
 
     return (value, content_type)
 
 
 def maybe_parse_response(response: Response) -> Any:
+    """Parse an HTTP response based on its content type.
+
+    Args:
+        response: The HTTP response to parse.
+
+    Returns:
+        Parsed response data (JSON dict/list, text string, or raw bytes).
+
+    Raises:
+        InvalidResponseBodyError: If the response body cannot be parsed.
+    """
     if response.status_code == HTTPStatus.NO_CONTENT:
         return None
 
@@ -261,9 +286,11 @@ def maybe_parse_response(response: Response) -> Any:
         content_type = response.headers['content-type'].split(';')[0].strip()
 
     try:
-        if is_content_type_json(content_type):
+        if re.search(r'^application/json', content_type, flags=re.IGNORECASE):
             response_data = response.json()
-        elif is_content_type_xml(content_type) or is_content_type_text(content_type):
+        elif re.search(r'^application/.*xml$', content_type, flags=re.IGNORECASE) or re.search(
+            r'^text/', content_type, flags=re.IGNORECASE
+        ):
             response_data = response.text
         else:
             response_data = response.content
@@ -274,7 +301,14 @@ def maybe_parse_response(response: Response) -> Any:
 
 
 def is_retryable_error(exc: Exception) -> bool:
-    """Check if the given error is retryable."""
+    """Check if an exception should be retried.
+
+    Args:
+        exc: The exception to check.
+
+    Returns:
+        True if the exception is retryable (network errors, timeouts, etc.).
+    """
     return isinstance(
         exc,
         (