Skip to content

Commit 28a7d46

Browse files
leaftyleafty
committed
wip
1 parent d0424b4 commit 28a7d46

File tree

5 files changed

+87
-61
lines changed

5 files changed

+87
-61
lines changed

‎components/renku_data_services/base_models/core.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from dataclasses import dataclass, field
88
from datetime import datetime
99
from enum import Enum, StrEnum
10-
from typing import ClassVar, Never, NewType, Optional, Protocol, Self, TypeVar, overload
10+
from typing import ClassVar, Final, Never, NewType, Optional, Protocol, Self, TypeVar, overload
1111

1212
from sanic import Request
1313

@@ -381,6 +381,13 @@ def from_strings(cls, *slugs: str) -> Self:
381381
return cls(NamespaceSlug(slugs[0]), ProjectSlug(slugs[1]), DataConnectorSlug(slugs[2]))
382382

383383

384+
GLOBAL_NAMESPACE_SLUG_STR: Final[str] = "_global"
385+
"""The string value of the slug of the global namespace."""
386+
GLOBAL_NAMESPACE_SLUG: Final[NamespaceSlug] = NamespaceSlug(value=GLOBAL_NAMESPACE_SLUG_STR)
387+
"""The value of the slug of the global namespace."""
388+
GLOBAL_NAMESPACE_PATH: Final[NamespacePath] = NamespacePath(first=GLOBAL_NAMESPACE_SLUG)
389+
"""The value of the path of the global namespace."""
390+
384391
AnyAPIUser = TypeVar("AnyAPIUser", bound=APIUser, covariant=True)
385392

386393

‎components/renku_data_services/data_connectors/core.py‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ def validate_unsaved_data_connector(
110110

111111
async def prevalidate_unsaved_global_data_connector(
112112
body: apispec.GlobalDataConnectorPost, validator: RCloneValidator
113-
) -> models.UnsavedGlobalDataConnector:
113+
) -> models.UnsavedDataConnector: # models.UnsavedGlobalDataConnector:
114114
"""Pre-validate an unsaved data connector."""
115115

116116
storage = validate_unsaved_storage(body.storage, validator=validator)
@@ -128,8 +128,9 @@ async def prevalidate_unsaved_global_data_connector(
128128
# Override provider in storage config
129129
storage.configuration["provider"] = rclone_metadata.provider
130130

131-
return models.UnsavedGlobalDataConnector(
131+
return models.UnsavedDataConnector(
132132
name=doi_uri,
133+
namespace=base_models.GLOBAL_NAMESPACE_PATH,
133134
slug=slug,
134135
visibility=Visibility.PUBLIC,
135136
created_by="",
@@ -140,9 +141,9 @@ async def prevalidate_unsaved_global_data_connector(
140141

141142

142143
async def validate_unsaved_global_data_connector(
143-
data_connector: models.UnsavedGlobalDataConnector,
144+
data_connector: models.UnsavedDataConnector, # models.UnsavedGlobalDataConnector,
144145
validator: RCloneValidator,
145-
) -> models.UnsavedGlobalDataConnector:
146+
) -> models.UnsavedDataConnector: # models.UnsavedGlobalDataConnector:
146147
"""Validate an unsaved data connector."""
147148

148149
# Check that we can list the files in the DOI
@@ -197,8 +198,9 @@ async def validate_unsaved_global_data_connector(
197198
readonly=data_connector.storage.readonly,
198199
)
199200

200-
return models.UnsavedGlobalDataConnector(
201+
return models.UnsavedDataConnector(
201202
name=name,
203+
namespace=base_models.GLOBAL_NAMESPACE_PATH,
202204
slug=data_connector.slug,
203205
visibility=Visibility.PUBLIC,
204206
created_by="",
@@ -233,18 +235,15 @@ def validate_storage_patch(
233235

234236

235237
def validate_data_connector_patch(
236-
data_connector: models.DataConnector | models.GlobalDataConnector,
238+
data_connector: models.DataConnector, # | models.GlobalDataConnector,
237239
patch: apispec.DataConnectorPatch,
238240
validator: RCloneValidator,
239241
) -> models.DataConnectorPatch:
240242
"""Validate the update to a data connector."""
241-
if isinstance(data_connector, models.GlobalDataConnector) and patch.namespace is not None:
243+
# if isinstance(data_connector, models.GlobalDataConnector) and patch.namespace is not None:
244+
if data_connector.is_global() and patch.namespace is not None:
242245
raise errors.ValidationError(message="Assigning a namespace to a global data connector is not supported")
243-
if (
244-
isinstance(data_connector, models.GlobalDataConnector)
245-
and patch.slug is not None
246-
and patch.slug != data_connector.slug
247-
):
246+
if data_connector.is_global() and patch.slug is not None and patch.slug != data_connector.slug:
248247
raise errors.ValidationError(message="Updating the slug of a global data connector is not supported")
249248

250249
slugs = patch.namespace.split("/") if patch.namespace else []

‎components/renku_data_services/data_connectors/db.py‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -371,24 +371,33 @@ async def insert_namespaced_data_connector(
371371
async def insert_global_data_connector(
372372
self,
373373
user: base_models.APIUser,
374-
data_connector: models.UnsavedGlobalDataConnector,
374+
# data_connector: models.UnsavedGlobalDataConnector,
375+
data_connector: models.UnsavedDataConnector,
375376
validator: RCloneValidator | None,
376377
*,
377378
session: AsyncSession | None = None,
378-
) -> tuple[models.GlobalDataConnector, bool]:
379+
) -> tuple[models.DataConnector, bool]:
379380
"""Insert a new global data connector entry."""
381+
if not data_connector.is_global():
382+
raise errors.ValidationError(message="The data connector is not global.")
380383
if not session:
381384
raise errors.ProgrammingError(message="A database session is required.")
382385
if user.id is None:
383386
raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.")
384387

385388
slug = data_connector.slug or base_models.Slug.from_name(data_connector.name).value
386389

387-
existing_global_dc_stmt = select(schemas.DataConnectorORM).where(schemas.DataConnectorORM.global_slug == slug)
390+
existing_global_dc_stmt = select(schemas.DataConnectorORM).where(
391+
schemas.DataConnectorORM.slug.has(
392+
ns_schemas.EntitySlugORM.slug == slug,
393+
)
394+
) # .where(schemas.DataConnectorORM.slug)
395+
existing_global_dc_stmt = _filter_by_namespace_slug(existing_global_dc_stmt, base_models.GLOBAL_NAMESPACE_PATH)
388396
existing_global_dc = await session.scalar(existing_global_dc_stmt)
389397
if existing_global_dc is not None:
390398
dc = existing_global_dc.dump()
391-
if not isinstance(dc, models.GlobalDataConnector):
399+
# if not isinstance(dc, models.GlobalDataConnector):
400+
if not dc.is_global():
392401
raise errors.ProgrammingError(message=f"Expected to get a global data connector ('{dc.id}')")
393402
authorized = await self.authz.has_permission(user, ResourceType.data_connector, dc.id, Scope.READ)
394403
if not authorized:
@@ -398,15 +407,19 @@ async def insert_global_data_connector(
398407
return dc, False
399408

400409
# Fully validate a global data connector before inserting
401-
if isinstance(data_connector, models.UnsavedGlobalDataConnector):
402-
if validator is None:
403-
raise RuntimeError("Could not validate global data connector")
404-
data_connector = await validate_unsaved_global_data_connector(
405-
data_connector=data_connector, validator=validator
406-
)
410+
if validator is None:
411+
raise RuntimeError("Could not validate global data connector")
412+
data_connector = await validate_unsaved_global_data_connector(data_connector=data_connector, validator=validator)
413+
# if isinstance(data_connector, models.UnsavedGlobalDataConnector):
414+
# if validator is None:
415+
# raise RuntimeError("Could not validate global data connector")
416+
# data_connector = await validate_unsaved_global_data_connector(
417+
# data_connector=data_connector, validator=validator
418+
# )
407419

408420
dc = await self._insert_data_connector(user=user, data_connector=data_connector, session=session)
409-
if not isinstance(dc, models.GlobalDataConnector):
421+
# if not isinstance(dc, models.GlobalDataConnector):
422+
if not dc.is_global():
410423
raise errors.ProgrammingError(message=f"Expected to get a global data connector ('{dc.id}')")
411424
return dc, True
412425

‎components/renku_data_services/data_connectors/models.py‎

Lines changed: 27 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@
22

33
from dataclasses import dataclass, field
44
from datetime import UTC, datetime
5-
from typing import TYPE_CHECKING, Any, Final
5+
from typing import TYPE_CHECKING, Any
66

77
from ulid import ULID
88

99
from renku_data_services.authz.models import Visibility
1010
from renku_data_services.base_models.core import (
11+
GLOBAL_NAMESPACE_PATH,
1112
DataConnectorInProjectPath,
1213
DataConnectorPath,
1314
DataConnectorSlug,
@@ -65,6 +66,10 @@ def path(self) -> DataConnectorPath | DataConnectorInProjectPath:
6566
"""The full path (i.e. sequence of slugs) for the data connector including group or user and/or project."""
6667
return self.namespace.path / DataConnectorSlug(self.slug)
6768

69+
def is_global(self) -> bool:
70+
"""Whether this data connector is global."""
71+
return self.namespace.path == GLOBAL_NAMESPACE_PATH
72+
6873

6974
@dataclass(frozen=True, eq=True, kw_only=True)
7075
class UnsavedDataConnector(BaseDataConnector):
@@ -77,26 +82,30 @@ def path(self) -> DataConnectorPath | DataConnectorInProjectPath:
7782
"""The full path (i.e. sequence of slugs) for the data connector including group or user and/or project."""
7883
return self.namespace / DataConnectorSlug(self.slug)
7984

85+
def is_global(self) -> bool:
86+
"""Whether this data connector is global."""
87+
return self.namespace == GLOBAL_NAMESPACE_PATH
8088

81-
@dataclass(frozen=True, eq=True, kw_only=True)
82-
class GlobalDataConnector(BaseDataConnector):
83-
"""Global data connector model."""
8489

85-
id: ULID
86-
namespace: Final[None] = field(default=None, init=False)
87-
updated_at: datetime
90+
# @dataclass(frozen=True, eq=True, kw_only=True)
91+
# class GlobalDataConnector(BaseDataConnector):
92+
# """Global data connector model."""
8893

89-
@property
90-
def etag(self) -> str:
91-
"""Entity tag value for this data connector object."""
92-
return compute_etag_from_fields(self.updated_at)
94+
# id: ULID
95+
# namespace: Final[None] = field(default=None, init=False)
96+
# updated_at: datetime
9397

98+
# @property
99+
# def etag(self) -> str:
100+
# """Entity tag value for this data connector object."""
101+
# return compute_etag_from_fields(self.updated_at)
94102

95-
@dataclass(frozen=True, eq=True, kw_only=True)
96-
class UnsavedGlobalDataConnector(BaseDataConnector):
97-
"""Global data connector model."""
98103

99-
namespace: None = None
104+
# @dataclass(frozen=True, eq=True, kw_only=True)
105+
# class UnsavedGlobalDataConnector(BaseDataConnector):
106+
# """Global data connector model."""
107+
108+
# namespace: None = None
100109

101110

102111
@dataclass(frozen=True, eq=True, kw_only=True)
@@ -141,8 +150,8 @@ class CloudStorageCoreWithSensitiveFields(CloudStorageCore):
141150
class DataConnectorUpdate:
142151
"""Information about the update of a data connector."""
143152

144-
old: DataConnector | GlobalDataConnector
145-
new: DataConnector | GlobalDataConnector
153+
old: DataConnector # | GlobalDataConnector
154+
new: DataConnector # | GlobalDataConnector
146155

147156

148157
@dataclass(frozen=True, eq=True, kw_only=True)
@@ -194,5 +203,5 @@ class DataConnectorPermissions:
194203
class DataConnectorWithSecrets:
195204
"""A data connector with its secrets."""
196205

197-
data_connector: DataConnector | GlobalDataConnector
206+
data_connector: DataConnector # | GlobalDataConnector
198207
secrets: list[DataConnectorSecret] = field(default_factory=list)

‎components/renku_data_services/data_connectors/orm.py‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ class DataConnectorORM(BaseORM):
6666
keywords: Mapped[list[str] | None] = mapped_column("keywords", ARRAY(String(99)), nullable=True)
6767
"""Keywords for the data connector."""
6868

69-
slug: Mapped["EntitySlugORM | None"] = relationship(
69+
slug: Mapped["EntitySlugORM"] = relationship(
7070
lazy="joined", init=False, repr=False, viewonly=True, back_populates="data_connector"
7171
)
7272
"""Slug of the data connector."""
@@ -98,25 +98,23 @@ class DataConnectorORM(BaseORM):
9898
viewonly=True,
9999
)
100100

101-
def dump(self) -> models.DataConnector | models.GlobalDataConnector:
101+
def dump(self) -> models.DataConnector: # | models.GlobalDataConnector:
102102
"""Create a data connector model from the DataConnectorORM."""
103-
if self.global_slug:
104-
return models.GlobalDataConnector(
105-
id=self.id,
106-
name=self.name,
107-
slug=self.global_slug,
108-
visibility=self._dump_visibility(),
109-
created_by=self.created_by_id, # TODO: should we use an admin id? Or drop it?
110-
creation_date=self.creation_date,
111-
updated_at=self.updated_at,
112-
storage=self._dump_storage(),
113-
description=self.description,
114-
keywords=self.keywords,
115-
)
116-
117-
elif self.slug is None:
118-
raise ValueError("Either the slug or the global slug must be set.")
119-
103+
# if self.global_slug:
104+
# return models.GlobalDataConnector(
105+
# id=self.id,
106+
# name=self.name,
107+
# slug=self.global_slug,
108+
# visibility=self._dump_visibility(),
109+
# created_by=self.created_by_id, # TODO: should we use an admin id? Or drop it?
110+
# creation_date=self.creation_date,
111+
# updated_at=self.updated_at,
112+
# storage=self._dump_storage(),
113+
# description=self.description,
114+
# keywords=self.keywords,
115+
# )
116+
# elif self.slug is None:
117+
# raise ValueError("Either the slug or the global slug must be set.")
120118
return models.DataConnector(
121119
id=self.id,
122120
name=self.name,

0 commit comments

Comments
 (0)