From f8d17b14a8ac156c6a493ec12b7a2453067d5ace Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 14 May 2025 14:25:48 +0200 Subject: [PATCH 1/4] refactor: move global data connectors to the _global namespace Closes #830. --- .../47e51c42e391_add_global_namespace.py | 109 ++++++++++++++++++ .../renku_data_services/namespace/orm.py | 6 +- 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 components/renku_data_services/migrations/versions/47e51c42e391_add_global_namespace.py diff --git a/components/renku_data_services/migrations/versions/47e51c42e391_add_global_namespace.py b/components/renku_data_services/migrations/versions/47e51c42e391_add_global_namespace.py new file mode 100644 index 000000000..5835c6f9e --- /dev/null +++ b/components/renku_data_services/migrations/versions/47e51c42e391_add_global_namespace.py @@ -0,0 +1,109 @@ +"""Add global namespace + +Revision ID: 47e51c42e391 +Revises: dcb9648c3c15 +Create Date: 2025-05-14 07:20:22.778969 + +""" + +from alembic import op +from dataclasses import dataclass +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "47e51c42e391" +down_revision = "dcb9648c3c15" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + connection = op.get_bind() + with connection.begin_nested() as tx: + op.execute(sa.text("LOCK TABLE common.namespaces IN EXCLUSIVE MODE")) + op.drop_constraint( + "either_group_id_or_user_id_is_set", + "namespaces", + schema="common", + type_="check", + ) + op.create_check_constraint( + "either_group_id_or_user_id_is_set", + "namespaces", + "(user_id IS NOT NULL) OR (group_id IS NOT NULL) OR (slug = '_global')", + schema="common", + ) + + insert_global_namespace_stmt = ( + sa.insert( + sa.table( + "namespaces", + sa.column("id", type_=sa.VARCHAR), + sa.column("slug", type_=sa.VARCHAR), + schema="common", + ) + ) + .values(id=sa.text("generate_ulid()"), slug="_global") + .returning(sa.column("id", type_=sa.VARCHAR)) + ) + namespace_id = connection.execute(insert_global_namespace_stmt).scalar() + if not namespace_id: + raise RuntimeError("Failed to insert the _global namespace") + + print(f"namespace_id={namespace_id}") + + op.execute(sa.text("LOCK TABLE storage.data_connectors IN EXCLUSIVE MODE")) + select_global_data_connectors_stmt = ( + sa.select(sa.column("id", type_=sa.VARCHAR), sa.column("global_slug", type_=sa.VARCHAR)) + .select_from(sa.table("data_connectors", schema="storage")) + .where(sa.column("global_slug", type_=sa.VARCHAR).is_not(sa.null())) + ) + data_connectors = connection.execute(select_global_data_connectors_stmt).scalars().all() + print(f"data_connectors={data_connectors}") + + for dc_id, dc_global_slug in data_connectors: + insert_entity_slug_stmt = ( + sa.insert( + sa.table( + "entity_slugs", + sa.column("id", type_=sa.VARCHAR), + sa.column("slug", type_=sa.VARCHAR), + sa.column("data_connector_id", type_=sa.VARCHAR), + sa.column("namespace_id", type_=sa.VARCHAR), + schema="common", + ) + ) + .values( + id=sa.text("generate_ulid()"), + slug=dc_global_slug, + data_connector_id=dc_id, + namespace_id=namespace_id, + ) + .returning(sa.column("id", type_=sa.VARCHAR)) + ) + slug_id = connection.execute(insert_entity_slug_stmt).scalar() + if not slug_id: + raise RuntimeError(f"Failed to insert the entity slug for data connector '{dc_id}'") + + tx.commit() + + +def downgrade() -> None: + connection = op.get_bind() + with connection.begin_nested() as tx: + op.execute(sa.text("LOCK TABLE common.namespaces IN EXCLUSIVE MODE")) + op.drop_constraint( + "either_group_id_or_user_id_is_set", + "namespaces", + schema="common", + type_="check", + ) + op.create_check_constraint( + "either_group_id_or_user_id_is_set", + "namespaces", + "(user_id IS NULL) <> (group_id IS NULL)", + schema="common", + ) + tx.commit() + # CheckConstraint("(user_id IS NULL) <> (group_id IS NULL)", name="either_group_id_or_user_id_is_set"), diff --git a/components/renku_data_services/namespace/orm.py b/components/renku_data_services/namespace/orm.py index 8830f527e..52758850f 100644 --- a/components/renku_data_services/namespace/orm.py +++ b/components/renku_data_services/namespace/orm.py @@ -55,7 +55,11 @@ class NamespaceORM(BaseORM): __tablename__ = "namespaces" __table_args__ = ( - CheckConstraint("(user_id IS NULL) <> (group_id IS NULL)", name="either_group_id_or_user_id_is_set"), + # CheckConstraint("(user_id IS NULL) <> (group_id IS NULL)", name="either_group_id_or_user_id_is_set"), + CheckConstraint( + "(user_id IS NOT NULL) OR (group_id IS NOT NULL) OR (slug = '_global')", + name="either_group_id_or_user_id_is_set", + ), ) id: Mapped[ULID] = mapped_column("id", ULIDType, primary_key=True, default_factory=lambda: str(ULID()), init=False) From d0424b4a04fbeded052246156dd81ccc7d928a23 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 14 May 2025 14:54:11 +0200 Subject: [PATCH 2/4] wip --- .../data_connectors/orm.py | 4 +- .../47e51c42e391_add_global_namespace.py | 45 ++++++++++++++++--- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/components/renku_data_services/data_connectors/orm.py b/components/renku_data_services/data_connectors/orm.py index f51d54d5c..80f838fdc 100644 --- a/components/renku_data_services/data_connectors/orm.py +++ b/components/renku_data_services/data_connectors/orm.py @@ -71,8 +71,8 @@ class DataConnectorORM(BaseORM): ) """Slug of the data connector.""" - global_slug: Mapped[str | None] = mapped_column(String(99), index=True, nullable=True, default=None, unique=True) - """Slug for global data connectors.""" + # global_slug: Mapped[str | None] = mapped_column(String(99), index=True, nullable=True, default=None, unique=True) + # """Slug for global data connectors.""" readonly: Mapped[bool] = mapped_column("readonly", Boolean(), default=True) """Whether this storage should be mounted readonly or not """ diff --git a/components/renku_data_services/migrations/versions/47e51c42e391_add_global_namespace.py b/components/renku_data_services/migrations/versions/47e51c42e391_add_global_namespace.py index 5835c6f9e..5ef4103bd 100644 --- a/components/renku_data_services/migrations/versions/47e51c42e391_add_global_namespace.py +++ b/components/renku_data_services/migrations/versions/47e51c42e391_add_global_namespace.py @@ -6,10 +6,8 @@ """ -from alembic import op -from dataclasses import dataclass import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "47e51c42e391" @@ -22,6 +20,10 @@ def upgrade() -> None: connection = op.get_bind() with connection.begin_nested() as tx: op.execute(sa.text("LOCK TABLE common.namespaces IN EXCLUSIVE MODE")) + op.execute(sa.text("LOCK TABLE common.entity_slugs IN EXCLUSIVE MODE")) + op.execute(sa.text("LOCK TABLE storage.data_connectors IN EXCLUSIVE MODE")) + + # Step 1: update the namespaces table and add the _global namespace op.drop_constraint( "either_group_id_or_user_id_is_set", "namespaces", @@ -34,7 +36,6 @@ def upgrade() -> None: "(user_id IS NOT NULL) OR (group_id IS NOT NULL) OR (slug = '_global')", schema="common", ) - insert_global_namespace_stmt = ( sa.insert( sa.table( @@ -50,10 +51,9 @@ def upgrade() -> None: namespace_id = connection.execute(insert_global_namespace_stmt).scalar() if not namespace_id: raise RuntimeError("Failed to insert the _global namespace") - print(f"namespace_id={namespace_id}") - op.execute(sa.text("LOCK TABLE storage.data_connectors IN EXCLUSIVE MODE")) + # Step 2: create a row in the entity_slug table for each global data connector select_global_data_connectors_stmt = ( sa.select(sa.column("id", type_=sa.VARCHAR), sa.column("global_slug", type_=sa.VARCHAR)) .select_from(sa.table("data_connectors", schema="storage")) @@ -86,6 +86,11 @@ def upgrade() -> None: if not slug_id: raise RuntimeError(f"Failed to insert the entity slug for data connector '{dc_id}'") + # Step 3: update the data_connectors table + op.drop_index("ix_storage_data_connectors_global_slug", table_name="data_connectors", schema="storage") + op.drop_column("data_connectors", "global_slug", schema="storage") + # TODO: + tx.commit() @@ -93,6 +98,33 @@ def downgrade() -> None: connection = op.get_bind() with connection.begin_nested() as tx: op.execute(sa.text("LOCK TABLE common.namespaces IN EXCLUSIVE MODE")) + op.execute(sa.text("LOCK TABLE common.entity_slugs IN EXCLUSIVE MODE")) + op.execute(sa.text("LOCK TABLE storage.data_connectors IN EXCLUSIVE MODE")) + + # Step 1: update the data_connectors table + op.add_column( + "data_connectors", + sa.Column("global_slug", sa.VARCHAR(length=99), autoincrement=False, nullable=True), + schema="storage", + ) + op.create_index( + "ix_storage_data_connectors_global_slug", "data_connectors", ["global_slug"], unique=True, schema="storage" + ) + # TODO: + + # Step 2: create a row in the entity_slug table for each global data connector + # TODO: + + # Step 3: update the namespaces table and add the _global namespace + delete_global_namespace_stmt = sa.delete( + sa.table( + "namespaces", + sa.column("id", type_=sa.VARCHAR), + sa.column("slug", type_=sa.VARCHAR), + schema="common", + ) + ).where(sa.column("slug", type_=sa.VARCHAR) == sa.literal("_global")) + op.execute(delete_global_namespace_stmt) op.drop_constraint( "either_group_id_or_user_id_is_set", "namespaces", @@ -106,4 +138,3 @@ def downgrade() -> None: schema="common", ) tx.commit() - # CheckConstraint("(user_id IS NULL) <> (group_id IS NULL)", name="either_group_id_or_user_id_is_set"), From 28a7d46d7aa97546173861bcc8952e24c6880c39 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 15 May 2025 10:32:07 +0200 Subject: [PATCH 3/4] wip --- .../renku_data_services/base_models/core.py | 9 +++- .../data_connectors/core.py | 23 +++++----- .../renku_data_services/data_connectors/db.py | 35 ++++++++++----- .../data_connectors/models.py | 45 +++++++++++-------- .../data_connectors/orm.py | 36 +++++++-------- 5 files changed, 87 insertions(+), 61 deletions(-) diff --git a/components/renku_data_services/base_models/core.py b/components/renku_data_services/base_models/core.py index 341031441..7731fd6f8 100644 --- a/components/renku_data_services/base_models/core.py +++ b/components/renku_data_services/base_models/core.py @@ -7,7 +7,7 @@ from dataclasses import dataclass, field from datetime import datetime from enum import Enum, StrEnum -from typing import ClassVar, Never, NewType, Optional, Protocol, Self, TypeVar, overload +from typing import ClassVar, Final, Never, NewType, Optional, Protocol, Self, TypeVar, overload from sanic import Request @@ -381,6 +381,13 @@ def from_strings(cls, *slugs: str) -> Self: return cls(NamespaceSlug(slugs[0]), ProjectSlug(slugs[1]), DataConnectorSlug(slugs[2])) +GLOBAL_NAMESPACE_SLUG_STR: Final[str] = "_global" +"""The string value of the slug of the global namespace.""" +GLOBAL_NAMESPACE_SLUG: Final[NamespaceSlug] = NamespaceSlug(value=GLOBAL_NAMESPACE_SLUG_STR) +"""The value of the slug of the global namespace.""" +GLOBAL_NAMESPACE_PATH: Final[NamespacePath] = NamespacePath(first=GLOBAL_NAMESPACE_SLUG) +"""The value of the path of the global namespace.""" + AnyAPIUser = TypeVar("AnyAPIUser", bound=APIUser, covariant=True) diff --git a/components/renku_data_services/data_connectors/core.py b/components/renku_data_services/data_connectors/core.py index 193a883af..019281110 100644 --- a/components/renku_data_services/data_connectors/core.py +++ b/components/renku_data_services/data_connectors/core.py @@ -110,7 +110,7 @@ def validate_unsaved_data_connector( async def prevalidate_unsaved_global_data_connector( body: apispec.GlobalDataConnectorPost, validator: RCloneValidator -) -> models.UnsavedGlobalDataConnector: +) -> models.UnsavedDataConnector: # models.UnsavedGlobalDataConnector: """Pre-validate an unsaved data connector.""" storage = validate_unsaved_storage(body.storage, validator=validator) @@ -128,8 +128,9 @@ async def prevalidate_unsaved_global_data_connector( # Override provider in storage config storage.configuration["provider"] = rclone_metadata.provider - return models.UnsavedGlobalDataConnector( + return models.UnsavedDataConnector( name=doi_uri, + namespace=base_models.GLOBAL_NAMESPACE_PATH, slug=slug, visibility=Visibility.PUBLIC, created_by="", @@ -140,9 +141,9 @@ async def prevalidate_unsaved_global_data_connector( async def validate_unsaved_global_data_connector( - data_connector: models.UnsavedGlobalDataConnector, + data_connector: models.UnsavedDataConnector, # models.UnsavedGlobalDataConnector, validator: RCloneValidator, -) -> models.UnsavedGlobalDataConnector: +) -> models.UnsavedDataConnector: # models.UnsavedGlobalDataConnector: """Validate an unsaved data connector.""" # Check that we can list the files in the DOI @@ -197,8 +198,9 @@ async def validate_unsaved_global_data_connector( readonly=data_connector.storage.readonly, ) - return models.UnsavedGlobalDataConnector( + return models.UnsavedDataConnector( name=name, + namespace=base_models.GLOBAL_NAMESPACE_PATH, slug=data_connector.slug, visibility=Visibility.PUBLIC, created_by="", @@ -233,18 +235,15 @@ def validate_storage_patch( def validate_data_connector_patch( - data_connector: models.DataConnector | models.GlobalDataConnector, + data_connector: models.DataConnector, # | models.GlobalDataConnector, patch: apispec.DataConnectorPatch, validator: RCloneValidator, ) -> models.DataConnectorPatch: """Validate the update to a data connector.""" - if isinstance(data_connector, models.GlobalDataConnector) and patch.namespace is not None: + # if isinstance(data_connector, models.GlobalDataConnector) and patch.namespace is not None: + if data_connector.is_global() and patch.namespace is not None: raise errors.ValidationError(message="Assigning a namespace to a global data connector is not supported") - if ( - isinstance(data_connector, models.GlobalDataConnector) - and patch.slug is not None - and patch.slug != data_connector.slug - ): + if data_connector.is_global() and patch.slug is not None and patch.slug != data_connector.slug: raise errors.ValidationError(message="Updating the slug of a global data connector is not supported") slugs = patch.namespace.split("/") if patch.namespace else [] diff --git a/components/renku_data_services/data_connectors/db.py b/components/renku_data_services/data_connectors/db.py index 4f86b1454..ce8b4b03d 100644 --- a/components/renku_data_services/data_connectors/db.py +++ b/components/renku_data_services/data_connectors/db.py @@ -371,12 +371,15 @@ async def insert_namespaced_data_connector( async def insert_global_data_connector( self, user: base_models.APIUser, - data_connector: models.UnsavedGlobalDataConnector, + # data_connector: models.UnsavedGlobalDataConnector, + data_connector: models.UnsavedDataConnector, validator: RCloneValidator | None, *, session: AsyncSession | None = None, - ) -> tuple[models.GlobalDataConnector, bool]: + ) -> tuple[models.DataConnector, bool]: """Insert a new global data connector entry.""" + if not data_connector.is_global(): + raise errors.ValidationError(message="The data connector is not global.") if not session: raise errors.ProgrammingError(message="A database session is required.") if user.id is None: @@ -384,11 +387,17 @@ async def insert_global_data_connector( slug = data_connector.slug or base_models.Slug.from_name(data_connector.name).value - existing_global_dc_stmt = select(schemas.DataConnectorORM).where(schemas.DataConnectorORM.global_slug == slug) + existing_global_dc_stmt = select(schemas.DataConnectorORM).where( + schemas.DataConnectorORM.slug.has( + ns_schemas.EntitySlugORM.slug == slug, + ) + ) # .where(schemas.DataConnectorORM.slug) + existing_global_dc_stmt = _filter_by_namespace_slug(existing_global_dc_stmt, base_models.GLOBAL_NAMESPACE_PATH) existing_global_dc = await session.scalar(existing_global_dc_stmt) if existing_global_dc is not None: dc = existing_global_dc.dump() - if not isinstance(dc, models.GlobalDataConnector): + # if not isinstance(dc, models.GlobalDataConnector): + if not dc.is_global(): raise errors.ProgrammingError(message=f"Expected to get a global data connector ('{dc.id}')") authorized = await self.authz.has_permission(user, ResourceType.data_connector, dc.id, Scope.READ) if not authorized: @@ -398,15 +407,19 @@ async def insert_global_data_connector( return dc, False # Fully validate a global data connector before inserting - if isinstance(data_connector, models.UnsavedGlobalDataConnector): - if validator is None: - raise RuntimeError("Could not validate global data connector") - data_connector = await validate_unsaved_global_data_connector( - data_connector=data_connector, validator=validator - ) + if validator is None: + raise RuntimeError("Could not validate global data connector") + data_connector = await validate_unsaved_global_data_connector(data_connector=data_connector, validator=validator) + # if isinstance(data_connector, models.UnsavedGlobalDataConnector): + # if validator is None: + # raise RuntimeError("Could not validate global data connector") + # data_connector = await validate_unsaved_global_data_connector( + # data_connector=data_connector, validator=validator + # ) dc = await self._insert_data_connector(user=user, data_connector=data_connector, session=session) - if not isinstance(dc, models.GlobalDataConnector): + # if not isinstance(dc, models.GlobalDataConnector): + if not dc.is_global(): raise errors.ProgrammingError(message=f"Expected to get a global data connector ('{dc.id}')") return dc, True diff --git a/components/renku_data_services/data_connectors/models.py b/components/renku_data_services/data_connectors/models.py index df0bcd880..af493b15a 100644 --- a/components/renku_data_services/data_connectors/models.py +++ b/components/renku_data_services/data_connectors/models.py @@ -2,12 +2,13 @@ from dataclasses import dataclass, field from datetime import UTC, datetime -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any from ulid import ULID from renku_data_services.authz.models import Visibility from renku_data_services.base_models.core import ( + GLOBAL_NAMESPACE_PATH, DataConnectorInProjectPath, DataConnectorPath, DataConnectorSlug, @@ -65,6 +66,10 @@ def path(self) -> DataConnectorPath | DataConnectorInProjectPath: """The full path (i.e. sequence of slugs) for the data connector including group or user and/or project.""" return self.namespace.path / DataConnectorSlug(self.slug) + def is_global(self) -> bool: + """Whether this data connector is global.""" + return self.namespace.path == GLOBAL_NAMESPACE_PATH + @dataclass(frozen=True, eq=True, kw_only=True) class UnsavedDataConnector(BaseDataConnector): @@ -77,26 +82,30 @@ def path(self) -> DataConnectorPath | DataConnectorInProjectPath: """The full path (i.e. sequence of slugs) for the data connector including group or user and/or project.""" return self.namespace / DataConnectorSlug(self.slug) + def is_global(self) -> bool: + """Whether this data connector is global.""" + return self.namespace == GLOBAL_NAMESPACE_PATH -@dataclass(frozen=True, eq=True, kw_only=True) -class GlobalDataConnector(BaseDataConnector): - """Global data connector model.""" - id: ULID - namespace: Final[None] = field(default=None, init=False) - updated_at: datetime +# @dataclass(frozen=True, eq=True, kw_only=True) +# class GlobalDataConnector(BaseDataConnector): +# """Global data connector model.""" - @property - def etag(self) -> str: - """Entity tag value for this data connector object.""" - return compute_etag_from_fields(self.updated_at) +# id: ULID +# namespace: Final[None] = field(default=None, init=False) +# updated_at: datetime +# @property +# def etag(self) -> str: +# """Entity tag value for this data connector object.""" +# return compute_etag_from_fields(self.updated_at) -@dataclass(frozen=True, eq=True, kw_only=True) -class UnsavedGlobalDataConnector(BaseDataConnector): - """Global data connector model.""" - namespace: None = None +# @dataclass(frozen=True, eq=True, kw_only=True) +# class UnsavedGlobalDataConnector(BaseDataConnector): +# """Global data connector model.""" + +# namespace: None = None @dataclass(frozen=True, eq=True, kw_only=True) @@ -141,8 +150,8 @@ class CloudStorageCoreWithSensitiveFields(CloudStorageCore): class DataConnectorUpdate: """Information about the update of a data connector.""" - old: DataConnector | GlobalDataConnector - new: DataConnector | GlobalDataConnector + old: DataConnector # | GlobalDataConnector + new: DataConnector # | GlobalDataConnector @dataclass(frozen=True, eq=True, kw_only=True) @@ -194,5 +203,5 @@ class DataConnectorPermissions: class DataConnectorWithSecrets: """A data connector with its secrets.""" - data_connector: DataConnector | GlobalDataConnector + data_connector: DataConnector # | GlobalDataConnector secrets: list[DataConnectorSecret] = field(default_factory=list) diff --git a/components/renku_data_services/data_connectors/orm.py b/components/renku_data_services/data_connectors/orm.py index 80f838fdc..e3f469345 100644 --- a/components/renku_data_services/data_connectors/orm.py +++ b/components/renku_data_services/data_connectors/orm.py @@ -66,7 +66,7 @@ class DataConnectorORM(BaseORM): keywords: Mapped[list[str] | None] = mapped_column("keywords", ARRAY(String(99)), nullable=True) """Keywords for the data connector.""" - slug: Mapped["EntitySlugORM | None"] = relationship( + slug: Mapped["EntitySlugORM"] = relationship( lazy="joined", init=False, repr=False, viewonly=True, back_populates="data_connector" ) """Slug of the data connector.""" @@ -98,25 +98,23 @@ class DataConnectorORM(BaseORM): viewonly=True, ) - def dump(self) -> models.DataConnector | models.GlobalDataConnector: + def dump(self) -> models.DataConnector: # | models.GlobalDataConnector: """Create a data connector model from the DataConnectorORM.""" - if self.global_slug: - return models.GlobalDataConnector( - id=self.id, - name=self.name, - slug=self.global_slug, - visibility=self._dump_visibility(), - created_by=self.created_by_id, # TODO: should we use an admin id? Or drop it? - creation_date=self.creation_date, - updated_at=self.updated_at, - storage=self._dump_storage(), - description=self.description, - keywords=self.keywords, - ) - - elif self.slug is None: - raise ValueError("Either the slug or the global slug must be set.") - + # if self.global_slug: + # return models.GlobalDataConnector( + # id=self.id, + # name=self.name, + # slug=self.global_slug, + # visibility=self._dump_visibility(), + # created_by=self.created_by_id, # TODO: should we use an admin id? Or drop it? + # creation_date=self.creation_date, + # updated_at=self.updated_at, + # storage=self._dump_storage(), + # description=self.description, + # keywords=self.keywords, + # ) + # elif self.slug is None: + # raise ValueError("Either the slug or the global slug must be set.") return models.DataConnector( id=self.id, name=self.name, From 8c805f4ae4379ad001d30d0262ea116691d2f0af Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Thu, 22 May 2025 14:37:36 +0200 Subject: [PATCH 4/4] wip --- components/renku_data_services/authz/authz.py | 30 ++++++++++++------- .../renku_data_services/base_models/core.py | 9 +++++- .../data_connectors/blueprints.py | 7 +++-- .../renku_data_services/data_connectors/db.py | 22 +++++++------- .../renku_data_services/search/reprovision.py | 10 +++---- 5 files changed, 48 insertions(+), 30 deletions(-) diff --git a/components/renku_data_services/authz/authz.py b/components/renku_data_services/authz/authz.py index 32d89bfc4..35466f71d 100644 --- a/components/renku_data_services/authz/authz.py +++ b/components/renku_data_services/authz/authz.py @@ -51,7 +51,7 @@ DataConnectorToProjectLink, DataConnectorUpdate, DeletedDataConnector, - GlobalDataConnector, + # GlobalDataConnector, ) from renku_data_services.errors import errors from renku_data_services.namespace.models import ( @@ -91,7 +91,7 @@ def authz(self) -> "Authz": | UserInfo | DeletedUser | DataConnector - | GlobalDataConnector + # | GlobalDataConnector | DataConnectorUpdate | DeletedDataConnector | DataConnectorToProjectLink @@ -270,7 +270,7 @@ async def decorated_function( | DeletedGroup | Namespace | DataConnector - | GlobalDataConnector + # | GlobalDataConnector | DeletedDataConnector | None ) = None @@ -282,7 +282,9 @@ async def decorated_function( case ResourceType.user_namespace if isinstance(potential_resource, Namespace): resource = potential_resource case ResourceType.data_connector if isinstance( - potential_resource, (DataConnector, GlobalDataConnector, DeletedDataConnector) + # potential_resource, (DataConnector, GlobalDataConnector, DeletedDataConnector) + potential_resource, + (DataConnector, DeletedDataConnector), ): resource = potential_resource case _: @@ -673,9 +675,12 @@ async def _get_authz_change( ) authz_change.extend(db_repo.authz._add_user_namespace(res.namespace)) case AuthzOperation.create, ResourceType.data_connector if isinstance(result, DataConnector): - authz_change = db_repo.authz._add_data_connector(result) - case AuthzOperation.create, ResourceType.data_connector if isinstance(result, GlobalDataConnector): - authz_change = db_repo.authz._add_global_data_connector(result) + if result.is_global(): + authz_change = db_repo.authz._add_global_data_connector(result) + else: + authz_change = db_repo.authz._add_data_connector(result) + # case AuthzOperation.create, ResourceType.data_connector if isinstance(result, GlobalDataConnector): # noqa E501 + # authz_change = db_repo.authz._add_global_data_connector(result) case AuthzOperation.delete, ResourceType.data_connector if result is None: # NOTE: This means that the dc does not exist in the first place so nothing was deleted pass @@ -689,7 +694,8 @@ async def _get_authz_change( authz_change.extend(await db_repo.authz._update_data_connector_visibility(user, result.new)) if result.old.namespace != result.new.namespace: user = _extract_user_from_args(*func_args, **func_kwargs) - if isinstance(result.new, GlobalDataConnector): + # if isinstance(result.new, GlobalDataConnector): + if result.new.is_global(): raise errors.ValidationError( message=f"Updating the namespace of a global data connector is not supported ('{result.new.id}')" # noqa E501 ) @@ -1618,7 +1624,11 @@ def _add_data_connector(self, data_connector: DataConnector) -> _AuthzChange: ) return _AuthzChange(apply=apply, undo=undo) - def _add_global_data_connector(self, data_connector: GlobalDataConnector) -> _AuthzChange: + def _add_global_data_connector( + self, + # data_connector: GlobalDataConnector + data_connector: DataConnector, + ) -> _AuthzChange: """Create the new global data connector and associated resources and relations in the DB.""" data_connector_res = _AuthzConverter.data_connector(data_connector.id) @@ -1714,7 +1724,7 @@ async def _remove_user( async def _update_data_connector_visibility( self, user: base_models.APIUser, - data_connector: DataConnector | GlobalDataConnector, + data_connector: DataConnector, # | GlobalDataConnector, *, zed_token: ZedToken | None = None, ) -> _AuthzChange: diff --git a/components/renku_data_services/base_models/core.py b/components/renku_data_services/base_models/core.py index 7731fd6f8..81ee5bd85 100644 --- a/components/renku_data_services/base_models/core.py +++ b/components/renku_data_services/base_models/core.py @@ -206,6 +206,13 @@ class NamespaceSlug(Slug): """The slug for a group or user namespace.""" +class GlobalNamespaceSlug(NamespaceSlug): + """The slug for the global namespace.""" + + def __init__(self) -> None: + object.__setattr__(self, "value", GLOBAL_NAMESPACE_SLUG_STR) + + class ProjectSlug(Slug): """The slug for a project.""" @@ -383,7 +390,7 @@ def from_strings(cls, *slugs: str) -> Self: GLOBAL_NAMESPACE_SLUG_STR: Final[str] = "_global" """The string value of the slug of the global namespace.""" -GLOBAL_NAMESPACE_SLUG: Final[NamespaceSlug] = NamespaceSlug(value=GLOBAL_NAMESPACE_SLUG_STR) +GLOBAL_NAMESPACE_SLUG: Final[GlobalNamespaceSlug] = GlobalNamespaceSlug() """The value of the slug of the global namespace.""" GLOBAL_NAMESPACE_PATH: Final[NamespacePath] = NamespacePath(first=GLOBAL_NAMESPACE_SLUG) """The value of the path of the global namespace.""" diff --git a/components/renku_data_services/data_connectors/blueprints.py b/components/renku_data_services/data_connectors/blueprints.py index 4ea2913cf..1152fd5ec 100644 --- a/components/renku_data_services/data_connectors/blueprints.py +++ b/components/renku_data_services/data_connectors/blueprints.py @@ -465,11 +465,13 @@ async def _delete_secrets( @staticmethod def _dump_data_connector( - data_connector: models.DataConnector | models.GlobalDataConnector, validator: RCloneValidator + data_connector: models.DataConnector, # | models.GlobalDataConnector, + validator: RCloneValidator, ) -> dict[str, Any]: """Dumps a data connector for API responses.""" storage = dump_storage_with_sensitive_fields(data_connector.storage, validator=validator) - if data_connector.namespace is None: + # if data_connector.namespace is None: + if data_connector.is_global(): return dict( id=str(data_connector.id), name=data_connector.name, @@ -488,7 +490,6 @@ def _dump_data_connector( namespace=data_connector.namespace.path.serialize(), slug=data_connector.slug, storage=storage, - # secrets=, creation_date=data_connector.creation_date, created_by=data_connector.created_by, visibility=data_connector.visibility.value, diff --git a/components/renku_data_services/data_connectors/db.py b/components/renku_data_services/data_connectors/db.py index ce8b4b03d..b46e78085 100644 --- a/components/renku_data_services/data_connectors/db.py +++ b/components/renku_data_services/data_connectors/db.py @@ -66,7 +66,7 @@ async def get_data_connectors( user: base_models.APIUser, pagination: PaginationRequest, namespace: ProjectPath | NamespacePath | None = None, - ) -> tuple[list[models.DataConnector | models.GlobalDataConnector], int]: + ) -> tuple[list[models.DataConnector], int]: """Get multiple data connectors from the database.""" data_connector_ids = await self.authz.resources_with_permission( user, user.id, ResourceType.data_connector, Scope.READ @@ -102,7 +102,7 @@ async def get_data_connector( self, user: base_models.APIUser, data_connector_id: ULID, - ) -> models.DataConnector | models.GlobalDataConnector: + ) -> models.DataConnector: """Get one data connector from the database.""" not_found_msg = f"Data connector with id '{data_connector_id}' does not exist or you do not have access to it." @@ -152,7 +152,7 @@ async def get_data_connector_by_slug( self, user: base_models.APIUser, path: DataConnectorInProjectPath | DataConnectorPath, - ) -> models.DataConnector | models.GlobalDataConnector: + ) -> models.DataConnector: """Get one data connector from the database by slug. This will not return or find data connectors owned by projects. @@ -199,7 +199,7 @@ async def get_global_data_connector_by_slug( self, user: base_models.APIUser, slug: base_models.Slug, - ) -> models.DataConnector | models.GlobalDataConnector: + ) -> models.DataConnector: """Get one global data connector from the database by slug.""" not_found_msg = f"Data connector with identifier '{slug}' does not exist or you do not have access to it." @@ -232,7 +232,7 @@ async def _insert_data_connector( data_connector: models.UnsavedDataConnector | models.UnsavedGlobalDataConnector, *, session: AsyncSession | None = None, - ) -> models.DataConnector | models.GlobalDataConnector: + ) -> models.DataConnector: """Insert a new data connector entry.""" if not session: raise errors.ProgrammingError(message="A database session is required.") @@ -388,10 +388,10 @@ async def insert_global_data_connector( slug = data_connector.slug or base_models.Slug.from_name(data_connector.name).value existing_global_dc_stmt = select(schemas.DataConnectorORM).where( - schemas.DataConnectorORM.slug.has( - ns_schemas.EntitySlugORM.slug == slug, - ) - ) # .where(schemas.DataConnectorORM.slug) + schemas.DataConnectorORM.slug.has( + ns_schemas.EntitySlugORM.slug == slug, + ) + ) # .where(schemas.DataConnectorORM.slug) existing_global_dc_stmt = _filter_by_namespace_slug(existing_global_dc_stmt, base_models.GLOBAL_NAMESPACE_PATH) existing_global_dc = await session.scalar(existing_global_dc_stmt) if existing_global_dc is not None: @@ -409,7 +409,9 @@ async def insert_global_data_connector( # Fully validate a global data connector before inserting if validator is None: raise RuntimeError("Could not validate global data connector") - data_connector = await validate_unsaved_global_data_connector(data_connector=data_connector, validator=validator) + data_connector = await validate_unsaved_global_data_connector( + data_connector=data_connector, validator=validator + ) # if isinstance(data_connector, models.UnsavedGlobalDataConnector): # if validator is None: # raise RuntimeError("Could not validate global data connector") diff --git a/components/renku_data_services/search/reprovision.py b/components/renku_data_services/search/reprovision.py index 7213a2348..ff1a4d4d8 100644 --- a/components/renku_data_services/search/reprovision.py +++ b/components/renku_data_services/search/reprovision.py @@ -8,7 +8,7 @@ from renku_data_services.base_api.pagination import PaginationRequest from renku_data_services.base_models.core import APIUser from renku_data_services.data_connectors.db import DataConnectorRepository -from renku_data_services.data_connectors.models import DataConnector, GlobalDataConnector +from renku_data_services.data_connectors.models import DataConnector # , GlobalDataConnector from renku_data_services.message_queue.db import ReprovisioningRepository from renku_data_services.message_queue.models import Reprovisioning from renku_data_services.namespace.db import GroupRepository @@ -59,12 +59,10 @@ async def get_current_reprovision(self) -> Reprovisioning | None: """Return the current reprovisioning lock.""" return await self._reprovisioning_repo.get_active_reprovisioning() - async def _get_all_data_connectors( - self, user: APIUser, per_page: int = 20 - ) -> AsyncGenerator[DataConnector | GlobalDataConnector, None]: + async def _get_all_data_connectors(self, user: APIUser, per_page: int = 20) -> AsyncGenerator[DataConnector, None]: """Get all data connectors, retrieving `per_page` each time.""" preq = PaginationRequest(page=1, per_page=per_page) - result: tuple[list[DataConnector | GlobalDataConnector], int] | None = None + result: tuple[list[DataConnector], int] | None = None count: int = 0 while result is None or result[1] > count: result = await self._data_connector_repo.get_data_connectors(user=user, pagination=preq) @@ -121,7 +119,7 @@ def log_counter(c: int) -> None: async def __update_entities( self, - iter: AsyncGenerator[Project | Group | UserInfo | DataConnector | GlobalDataConnector, None], + iter: AsyncGenerator[Project | Group | UserInfo | DataConnector, None], name: str, started: datetime, counter: int,