squashme: cleanup

Author: olevski

components/renku_data_services/data_connectors/core.py

@@ -374,15 +374,8 @@ def handle_data(self, data: str) -> None:
 async def convert_envidat_v1_data_connector_to_s3(
     payload: apispec.CloudStorageCorePost,
 ) -> apispec.CloudStorageCorePost:
-    """Converts a doi-like configuration for Envidat to S3.
-
-    If the paylaod that is passed in is not of the expected type nothing is changed
-    and the same payload that was passed in is returned.
-    """
+    """Converts a doi-like configuration for Envidat to S3."""
     config = payload.configuration
-    if config.get("type") != ENVIDAT_V1_PROVIDER:
-        return payload
-
     doi = config.get("doi")
     if not isinstance(doi, str):
         raise errors.ValidationError()
@@ -402,7 +395,7 @@ async def convert_envidat_v1_data_connector_to_s3(
         res = await clnt.get(envidat_url, params=query_params, headers=headers)
         if res.status_code != 200:
             raise errors.ProgrammingError()
-    dataset = SchemaOrgDataset.model_validate_strings(res.text)
+    dataset = SchemaOrgDataset.model_validate_json(res.text)
     s3_config = schema_org.get_rclone_config(
         dataset,
         schema_org.DatasetProvider.envidat,

components/renku_data_services/data_connectors/doi/schema_org.py

@@ -25,7 +25,7 @@ class S3Config:
     @property
     def path(self) -> str:
         """Return the path including the bucket name and the prefix."""
-        return f"{self.bucket}/{self.prefix}"
+        return f"{self.bucket}{self.prefix}"
 
 
 def get_rclone_config(dataset: SchemaOrgDataset, provider: DatasetProvider) -> S3Config:
@@ -62,14 +62,14 @@ def __get_rclone_s3_config_envidat(dataset: SchemaOrgDataset) -> S3Config:
             message="The envidat s3 url is expected to have a host name with at least two parts."
         )
     s3_host = ".".join(host_split[1:])
-    bucket = host_split[0]
+    bucket = host_split[0].strip("/")
     prefix = "/" + prefix.strip("/")
     return S3Config(
         {
             "type": "s3",
             "provider": "Other",
             "endpoint": f"{url_parsed.scheme}://{s3_host}",
         },
-        bucket.strip("/"),
+        bucket,
         prefix,
     )

test/bases/renku_data_services/data_api/test_data_connectors.py

@@ -4,14 +4,16 @@
 
 import pytest
 from httpx import Response
+from renku_data_services.data_connectors.apispec import CloudStorageCorePost, GlobalDataConnectorPost
+from renku_data_services.data_connectors.models import GlobalDataConnector
 from sanic_testing.testing import SanicASGITestClient
 
 from renku_data_services.authz.models import Visibility
 from renku_data_services.base_models.core import NamespacePath, ProjectPath
 from renku_data_services.data_connectors import core
 from renku_data_services.data_connectors.doi.models import DOIMetadata
 from renku_data_services.namespace.models import NamespaceKind
-from renku_data_services.storage.rclone import RCloneDOIMetadata
+from renku_data_services.storage.rclone import RCloneDOIMetadata, RCloneValidator
 from renku_data_services.users.models import UserInfo
 from test.bases.renku_data_services.data_api.utils import merge_headers
 
@@ -2431,3 +2433,22 @@ async def _mock(*args, **kwargs) -> DOIMetadata | None:
         "_get_dataset_metadata_dataverse",
         _mock_get_dataset_metadata(_orig_get_dataset_metadata_dataverse),
     )
+
+
+async def test_prevalidate_envidat_data_connector() -> None:
+    body = GlobalDataConnectorPost(
+        storage=CloudStorageCorePost(
+            storage_type="doi",
+            configuration={"type": "doi", "doi": "10.16904/12"},
+            source_path="/",
+            target_path="/",
+            readonly=True,
+        )
+    )
+    validator = RCloneValidator()
+    res = await core.prevalidate_unsaved_global_data_connector(body, validator)
+    config = res.storage.configuration
+    assert config["type"] == "s3"
+    assert config["provider"] == "Other"
+    assert config["endpoint"].find("zhdk.cloud.switch.ch") >= 0
+    assert res.storage.source_path == "envidat-doi/10.16904_12"