refactor: move global data connectors to the _global namespace

components/renku_data_services/authz/authz.py

@@ -51,7 +51,7 @@
     DataConnectorToProjectLink,
     DataConnectorUpdate,
     DeletedDataConnector,
-    GlobalDataConnector,
+    # GlobalDataConnector,
 )
 from renku_data_services.errors import errors
 from renku_data_services.namespace.models import (
@@ -91,7 +91,7 @@ def authz(self) -> "Authz":
     | UserInfo
     | DeletedUser
     | DataConnector
-    | GlobalDataConnector
+    # | GlobalDataConnector
     | DataConnectorUpdate
     | DeletedDataConnector
     | DataConnectorToProjectLink
@@ -270,7 +270,7 @@ async def decorated_function(
                 | DeletedGroup
                 | Namespace
                 | DataConnector
-                | GlobalDataConnector
+                # | GlobalDataConnector
                 | DeletedDataConnector
                 | None
             ) = None
@@ -282,7 +282,9 @@ async def decorated_function(
                 case ResourceType.user_namespace if isinstance(potential_resource, Namespace):
                     resource = potential_resource
                 case ResourceType.data_connector if isinstance(
-                    potential_resource, (DataConnector, GlobalDataConnector, DeletedDataConnector)
+                    # potential_resource, (DataConnector, GlobalDataConnector, DeletedDataConnector)
+                    potential_resource,
+                    (DataConnector, DeletedDataConnector),
                 ):
                     resource = potential_resource
                 case _:
@@ -673,9 +675,12 @@ async def _get_authz_change(
                                 )
                             authz_change.extend(db_repo.authz._add_user_namespace(res.namespace))
                     case AuthzOperation.create, ResourceType.data_connector if isinstance(result, DataConnector):
-                        authz_change = db_repo.authz._add_data_connector(result)
-                    case AuthzOperation.create, ResourceType.data_connector if isinstance(result, GlobalDataConnector):
-                        authz_change = db_repo.authz._add_global_data_connector(result)
+                        if result.is_global():
+                            authz_change = db_repo.authz._add_global_data_connector(result)
+                        else:
+                            authz_change = db_repo.authz._add_data_connector(result)
+                    # case AuthzOperation.create, ResourceType.data_connector if isinstance(result, GlobalDataConnector):  # noqa E501
+                    #     authz_change = db_repo.authz._add_global_data_connector(result)
                     case AuthzOperation.delete, ResourceType.data_connector if result is None:
                         # NOTE: This means that the dc does not exist in the first place so nothing was deleted
                         pass
@@ -689,7 +694,8 @@ async def _get_authz_change(
                             authz_change.extend(await db_repo.authz._update_data_connector_visibility(user, result.new))
                         if result.old.namespace != result.new.namespace:
                             user = _extract_user_from_args(*func_args, **func_kwargs)
-                            if isinstance(result.new, GlobalDataConnector):
+                            # if isinstance(result.new, GlobalDataConnector):
+                            if result.new.is_global():
                                 raise errors.ValidationError(
                                     message=f"Updating the namespace of a global data connector is not supported ('{result.new.id}')"  # noqa E501
                                 )
@@ -1618,7 +1624,11 @@ def _add_data_connector(self, data_connector: DataConnector) -> _AuthzChange:
         )
         return _AuthzChange(apply=apply, undo=undo)
 
-    def _add_global_data_connector(self, data_connector: GlobalDataConnector) -> _AuthzChange:
+    def _add_global_data_connector(
+        self,
+        # data_connector: GlobalDataConnector
+        data_connector: DataConnector,
+    ) -> _AuthzChange:
         """Create the new global data connector and associated resources and relations in the DB."""
         data_connector_res = _AuthzConverter.data_connector(data_connector.id)
 
@@ -1714,7 +1724,7 @@ async def _remove_user(
     async def _update_data_connector_visibility(
         self,
         user: base_models.APIUser,
-        data_connector: DataConnector | GlobalDataConnector,
+        data_connector: DataConnector,  # | GlobalDataConnector,
         *,
         zed_token: ZedToken | None = None,
     ) -> _AuthzChange:

components/renku_data_services/base_models/core.py

@@ -7,7 +7,7 @@
 from dataclasses import dataclass, field
 from datetime import datetime
 from enum import Enum, StrEnum
-from typing import ClassVar, Never, NewType, Optional, Protocol, Self, TypeVar, overload
+from typing import ClassVar, Final, Never, NewType, Optional, Protocol, Self, TypeVar, overload
 
 from sanic import Request
 
@@ -206,6 +206,13 @@ class NamespaceSlug(Slug):
     """The slug for a group or user namespace."""
 
 
+class GlobalNamespaceSlug(NamespaceSlug):
+    """The slug for the global namespace."""
+
+    def __init__(self) -> None:
+        object.__setattr__(self, "value", GLOBAL_NAMESPACE_SLUG_STR)
+
+
 class ProjectSlug(Slug):
     """The slug for a project."""
 
@@ -381,6 +388,13 @@ def from_strings(cls, *slugs: str) -> Self:
         return cls(NamespaceSlug(slugs[0]), ProjectSlug(slugs[1]), DataConnectorSlug(slugs[2]))
 
 
+GLOBAL_NAMESPACE_SLUG_STR: Final[str] = "_global"
+"""The string value of the slug of the global namespace."""
+GLOBAL_NAMESPACE_SLUG: Final[GlobalNamespaceSlug] = GlobalNamespaceSlug()
+"""The value of the slug of the global namespace."""
+GLOBAL_NAMESPACE_PATH: Final[NamespacePath] = NamespacePath(first=GLOBAL_NAMESPACE_SLUG)
+"""The value of the path of the global namespace."""
+
 AnyAPIUser = TypeVar("AnyAPIUser", bound=APIUser, covariant=True)
 
 

components/renku_data_services/data_connectors/blueprints.py

@@ -465,11 +465,13 @@ async def _delete_secrets(
 
     @staticmethod
     def _dump_data_connector(
-        data_connector: models.DataConnector | models.GlobalDataConnector, validator: RCloneValidator
+        data_connector: models.DataConnector,  # | models.GlobalDataConnector,
+        validator: RCloneValidator,
     ) -> dict[str, Any]:
         """Dumps a data connector for API responses."""
         storage = dump_storage_with_sensitive_fields(data_connector.storage, validator=validator)
-        if data_connector.namespace is None:
+        # if data_connector.namespace is None:
+        if data_connector.is_global():
             return dict(
                 id=str(data_connector.id),
                 name=data_connector.name,
@@ -488,7 +490,6 @@ def _dump_data_connector(
             namespace=data_connector.namespace.path.serialize(),
             slug=data_connector.slug,
             storage=storage,
-            # secrets=,
             creation_date=data_connector.creation_date,
             created_by=data_connector.created_by,
             visibility=data_connector.visibility.value,

components/renku_data_services/data_connectors/core.py

@@ -110,7 +110,7 @@ def validate_unsaved_data_connector(
 
 async def prevalidate_unsaved_global_data_connector(
     body: apispec.GlobalDataConnectorPost, validator: RCloneValidator
-) -> models.UnsavedGlobalDataConnector:
+) -> models.UnsavedDataConnector:  # models.UnsavedGlobalDataConnector:
     """Pre-validate an unsaved data connector."""
 
     storage = validate_unsaved_storage(body.storage, validator=validator)
@@ -128,8 +128,9 @@ async def prevalidate_unsaved_global_data_connector(
     # Override provider in storage config
     storage.configuration["provider"] = rclone_metadata.provider
 
-    return models.UnsavedGlobalDataConnector(
+    return models.UnsavedDataConnector(
         name=doi_uri,
+        namespace=base_models.GLOBAL_NAMESPACE_PATH,
         slug=slug,
         visibility=Visibility.PUBLIC,
         created_by="",
@@ -140,9 +141,9 @@ async def prevalidate_unsaved_global_data_connector(
 
 
 async def validate_unsaved_global_data_connector(
-    data_connector: models.UnsavedGlobalDataConnector,
+    data_connector: models.UnsavedDataConnector,  # models.UnsavedGlobalDataConnector,
     validator: RCloneValidator,
-) -> models.UnsavedGlobalDataConnector:
+) -> models.UnsavedDataConnector:  # models.UnsavedGlobalDataConnector:
     """Validate an unsaved data connector."""
 
     # Check that we can list the files in the DOI
@@ -197,8 +198,9 @@ async def validate_unsaved_global_data_connector(
         readonly=data_connector.storage.readonly,
     )
 
-    return models.UnsavedGlobalDataConnector(
+    return models.UnsavedDataConnector(
         name=name,
+        namespace=base_models.GLOBAL_NAMESPACE_PATH,
         slug=data_connector.slug,
         visibility=Visibility.PUBLIC,
         created_by="",
@@ -233,18 +235,15 @@ def validate_storage_patch(
 
 
 def validate_data_connector_patch(
-    data_connector: models.DataConnector | models.GlobalDataConnector,
+    data_connector: models.DataConnector,  # | models.GlobalDataConnector,
     patch: apispec.DataConnectorPatch,
     validator: RCloneValidator,
 ) -> models.DataConnectorPatch:
     """Validate the update to a data connector."""
-    if isinstance(data_connector, models.GlobalDataConnector) and patch.namespace is not None:
+    # if isinstance(data_connector, models.GlobalDataConnector) and patch.namespace is not None:
+    if data_connector.is_global() and patch.namespace is not None:
         raise errors.ValidationError(message="Assigning a namespace to a global data connector is not supported")
-    if (
-        isinstance(data_connector, models.GlobalDataConnector)
-        and patch.slug is not None
-        and patch.slug != data_connector.slug
-    ):
+    if data_connector.is_global() and patch.slug is not None and patch.slug != data_connector.slug:
         raise errors.ValidationError(message="Updating the slug of a global data connector is not supported")
 
     slugs = patch.namespace.split("/") if patch.namespace else []

components/renku_data_services/data_connectors/db.py

@@ -66,7 +66,7 @@ async def get_data_connectors(
         user: base_models.APIUser,
         pagination: PaginationRequest,
         namespace: ProjectPath | NamespacePath | None = None,
-    ) -> tuple[list[models.DataConnector | models.GlobalDataConnector], int]:
+    ) -> tuple[list[models.DataConnector], int]:
         """Get multiple data connectors from the database."""
         data_connector_ids = await self.authz.resources_with_permission(
             user, user.id, ResourceType.data_connector, Scope.READ
@@ -102,7 +102,7 @@ async def get_data_connector(
         self,
         user: base_models.APIUser,
         data_connector_id: ULID,
-    ) -> models.DataConnector | models.GlobalDataConnector:
+    ) -> models.DataConnector:
         """Get one data connector from the database."""
         not_found_msg = f"Data connector with id '{data_connector_id}' does not exist or you do not have access to it."
 
@@ -152,7 +152,7 @@ async def get_data_connector_by_slug(
         self,
         user: base_models.APIUser,
         path: DataConnectorInProjectPath | DataConnectorPath,
-    ) -> models.DataConnector | models.GlobalDataConnector:
+    ) -> models.DataConnector:
         """Get one data connector from the database by slug.
 
         This will not return or find data connectors owned by projects.
@@ -199,7 +199,7 @@ async def get_global_data_connector_by_slug(
         self,
         user: base_models.APIUser,
         slug: base_models.Slug,
-    ) -> models.DataConnector | models.GlobalDataConnector:
+    ) -> models.DataConnector:
         """Get one global data connector from the database by slug."""
         not_found_msg = f"Data connector with identifier '{slug}' does not exist or you do not have access to it."
 
@@ -232,7 +232,7 @@ async def _insert_data_connector(
         data_connector: models.UnsavedDataConnector | models.UnsavedGlobalDataConnector,
         *,
         session: AsyncSession | None = None,
-    ) -> models.DataConnector | models.GlobalDataConnector:
+    ) -> models.DataConnector:
         """Insert a new data connector entry."""
         if not session:
             raise errors.ProgrammingError(message="A database session is required.")
@@ -371,24 +371,33 @@ async def insert_namespaced_data_connector(
     async def insert_global_data_connector(
         self,
         user: base_models.APIUser,
-        data_connector: models.UnsavedGlobalDataConnector,
+        # data_connector: models.UnsavedGlobalDataConnector,
+        data_connector: models.UnsavedDataConnector,
         validator: RCloneValidator | None,
         *,
         session: AsyncSession | None = None,
-    ) -> tuple[models.GlobalDataConnector, bool]:
+    ) -> tuple[models.DataConnector, bool]:
         """Insert a new global data connector entry."""
+        if not data_connector.is_global():
+            raise errors.ValidationError(message="The data connector is not global.")
         if not session:
             raise errors.ProgrammingError(message="A database session is required.")
         if user.id is None:
             raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.")
 
         slug = data_connector.slug or base_models.Slug.from_name(data_connector.name).value
 
-        existing_global_dc_stmt = select(schemas.DataConnectorORM).where(schemas.DataConnectorORM.global_slug == slug)
+        existing_global_dc_stmt = select(schemas.DataConnectorORM).where(
+            schemas.DataConnectorORM.slug.has(
+                ns_schemas.EntitySlugORM.slug == slug,
+            )
+        )  # .where(schemas.DataConnectorORM.slug)
+        existing_global_dc_stmt = _filter_by_namespace_slug(existing_global_dc_stmt, base_models.GLOBAL_NAMESPACE_PATH)
         existing_global_dc = await session.scalar(existing_global_dc_stmt)
         if existing_global_dc is not None:
             dc = existing_global_dc.dump()
-            if not isinstance(dc, models.GlobalDataConnector):
+            # if not isinstance(dc, models.GlobalDataConnector):
+            if not dc.is_global():
                 raise errors.ProgrammingError(message=f"Expected to get a global data connector ('{dc.id}')")
             authorized = await self.authz.has_permission(user, ResourceType.data_connector, dc.id, Scope.READ)
             if not authorized:
@@ -398,15 +407,21 @@ async def insert_global_data_connector(
             return dc, False
 
         # Fully validate a global data connector before inserting
-        if isinstance(data_connector, models.UnsavedGlobalDataConnector):
-            if validator is None:
-                raise RuntimeError("Could not validate global data connector")
-            data_connector = await validate_unsaved_global_data_connector(
-                data_connector=data_connector, validator=validator
-            )
+        if validator is None:
+            raise RuntimeError("Could not validate global data connector")
+        data_connector = await validate_unsaved_global_data_connector(
+            data_connector=data_connector, validator=validator
+        )
+        # if isinstance(data_connector, models.UnsavedGlobalDataConnector):
+        #     if validator is None:
+        #         raise RuntimeError("Could not validate global data connector")
+        #     data_connector = await validate_unsaved_global_data_connector(
+        #         data_connector=data_connector, validator=validator
+        #     )
 
         dc = await self._insert_data_connector(user=user, data_connector=data_connector, session=session)
-        if not isinstance(dc, models.GlobalDataConnector):
+        # if not isinstance(dc, models.GlobalDataConnector):
+        if not dc.is_global():
             raise errors.ProgrammingError(message=f"Expected to get a global data connector ('{dc.id}')")
         return dc, True