chore: mock tests to succeed if envidat is unavailable

Author: olevski

components/renku_data_services/data_connectors/core.py

@@ -478,9 +478,19 @@ async def convert_envidat_v1_data_connector_to_s3(
     config = payload.configuration
     doi = config.get("doi")
     if not isinstance(doi, str):
-        raise errors.ValidationError()
+        if doi is None:
+            raise errors.ValidationError(
+                message="Cannot get configuration for Envidat data connector because "
+                "the doi is missing from the payload."
+            )
+        raise errors.ValidationError(
+            message=f"Cannot get configuration for Envidat data connector because the doi '{doi}' "
+            "in the payload is not a string."
+        )
     if len(doi) == 0:
-        raise errors.ValidationError()
+        raise errors.ValidationError(
+            message="Cannot get configuration for Envidat data connector because the doi is a string with zero length."
+        )
     doi = DOI(doi)
 
     new_config = payload.model_copy(deep=True)
@@ -494,7 +504,11 @@ async def convert_envidat_v1_data_connector_to_s3(
     async with clnt:
         res = await clnt.get(envidat_url, params=query_params, headers=headers)
         if res.status_code != 200:
-            raise errors.ProgrammingError()
+            raise errors.ValidationError(
+                message="Cannot get configuration for Envidat data connector because Envidat responded "
+                f"with an unexpected {res.status_code} status code at {res.url}.",
+                detail=f"Response from envidat: {res.text}",
+            )
     dataset = SchemaOrgDataset.model_validate_json(res.text)
     s3_config = schema_org.get_rclone_config(
         dataset,

components/renku_data_services/data_connectors/doi/schema_org.py

@@ -2,6 +2,7 @@
 
 from dataclasses import dataclass
 from enum import StrEnum
+from pathlib import PurePosixPath
 from urllib.parse import parse_qs, urlparse
 
 from renku_data_services.data_connectors.doi.models import SchemaOrgDataset
@@ -25,7 +26,8 @@ class S3Config:
     @property
     def path(self) -> str:
         """Return the path including the bucket name and the prefix."""
-        return f"/{self.bucket}{self.prefix}"
+        # NOTE: PurePosixPath("/test") / "/subfolder" == /subfolder, so we still have to strip /
+        return (PurePosixPath("/") / PurePosixPath(self.bucket) / self.prefix.lstrip("/")).as_posix()
 
 
 def get_rclone_config(dataset: SchemaOrgDataset, provider: DatasetProvider) -> S3Config:

components/renku_data_services/storage/rclone_patches.py

@@ -354,9 +354,6 @@ def apply_patches(spec: list[dict[str, Any]]) -> None:
         __patch_schema_remove_oauth_propeties,
         __patch_polybox_storage,
         __patch_switchdrive_storage,
-        # __add_custom_doi_s3_provider("Envidat", "Envidat data provider", ENVIDAT_V1_PROVIDER),
-        # TODO: Enable Scicat when it is ready in production
-        # __add_custom_doi_s3_provider("SciCat", "SciCat data provider", SCICAT_V1_PROVIDER),
         __patch_schema_remove_banned_sftp_options,
         __patch_schema_add_openbis_type,
     ]

test/bases/renku_data_services/data_api/test_data_connectors.py

@@ -2480,7 +2480,40 @@ async def _mock(*args, **kwargs) -> DOIMetadata | None:
     )
 
 
-async def test_validate_envidat_data_connector() -> None:
+def _mock_get_envidat_metadata(metadata: DOIMetadata, sanic_client: SanicASGITestClient, monkeypatch: "MonkeyPatch"):
+    """Mock the _get_envidat_metadata method."""
+
+    # The Evnidat API may be unresponsive, so we mock its response
+    from renku_data_services.data_connectors.doi import metadata as metadata_mod
+
+    _orig_get_envidat_metadata = metadata_mod._get_envidat_metadata
+
+    def _mock_get_envidat_metadata(original_fn):
+        async def _mock(*args, **kwargs) -> DOIMetadata | None:
+            fetched_metadata = await original_fn(*args, **kwargs)
+            if fetched_metadata is not None:
+                assert fetched_metadata == metadata
+                return fetched_metadata
+
+            warnings.warn("Could not retrieve DOI metadata, returning saved one", stacklevel=2)
+            return metadata
+
+        return _mock
+
+    monkeypatch.setattr(metadata_mod, "_get_envidat_metadata", _mock_get_envidat_metadata(_orig_get_envidat_metadata))
+    monkeypatch.setattr(
+        metadata_mod,
+        "_get_envidat_metadata",
+        _mock_get_envidat_metadata(_orig_get_envidat_metadata),
+    )
+
+
+async def test_validate_envidat_data_connector(
+    envidat_metadata: DOIMetadata,
+    sanic_client: SanicASGITestClient,
+    monkeypatch: "MonkeyPatch",
+) -> None:
+    _mock_get_envidat_metadata(envidat_metadata, sanic_client, monkeypatch)
     body = GlobalDataConnectorPost(
         storage=CloudStorageCorePost(
             storage_type="doi",
@@ -2510,10 +2543,16 @@ async def test_validate_envidat_data_connector() -> None:
     assert res.publisher_name is not None
 
 
-async def test_add_envidat_data_connector(sanic_client: SanicASGITestClient, user_headers) -> None:
+async def test_add_envidat_data_connector(
+    sanic_client: SanicASGITestClient,
+    user_headers: dict[str, str],
+    envidat_metadata: DOIMetadata,
+    monkeypatch: "MonkeyPatch",
+) -> None:
+    _mock_get_envidat_metadata(envidat_metadata, sanic_client, monkeypatch)
     payload = {
         "storage": {
-            "configuration": {"type": "doi", "doi": "10.16904/envidat.716"},
+            "configuration": {"type": "doi", "doi": "10.16904/12"},
             "source_path": "/",
             "target_path": "/",
             "readonly": True,

test/conftest.py

@@ -29,12 +29,14 @@
 from renku_data_services.app_config import logging
 from renku_data_services.authz.config import AuthzConfig
 from renku_data_services.data_api.dependencies import DependencyManager
+from renku_data_services.data_connectors.doi.models import DOIMetadata, SchemaOrgDataset
 from renku_data_services.db_config.config import DBConfig
 from renku_data_services.secrets_storage_api.dependencies import DependencyManager as SecretsDependencyManager
 from renku_data_services.solr import entity_schema
 from renku_data_services.solr.solr_client import SolrClientConfig
 from renku_data_services.solr.solr_migrate import SchemaMigrator
 from renku_data_services.users import models as user_preferences_models
+from test.constants import envidat_sample_response
 from test.utils import TestDependencyManager
 
 
@@ -489,3 +491,9 @@ def pytest_runtest_setup(item):
             ),
             append=False,
         )
+
+
+@pytest.fixture()
+def envidat_metadata() -> DOIMetadata:
+    md = SchemaOrgDataset.model_validate_json(envidat_sample_response)
+    return DOIMetadata(name=md.name, description=md.description or "", keywords=md.keywords)

…ervices/data_connectors/test_metadata.py test/constants.pytest/components/renku_data_services/data_connectors/test_metadata.py renamed to test/constants.py test/components/renku_data_services/data_connectors/test_metadata.py renamed to test/constants.py

@@ -1,6 +1,6 @@
-from renku_data_services.data_connectors.doi.models import SchemaOrgDataset
+from typing import Final
 
-envidat_sample_response = """
+envidat_sample_response: Final[str] = """
 {
     "@context": "http://schema.org",
     "@type": "https://schema.org/Dataset",
@@ -180,7 +180,3 @@
     "license": "https://opendefinition.org/licenses/odc-odbl"
 }
 """  # noqa: E501
-
-
-def test_envidat_metadata_parsing() -> None:
-    SchemaOrgDataset.model_validate_json(envidat_sample_response)