From cb68c2153f84273bc5f497ffda91c2de492b8dd8 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 26 Feb 2024 10:46:47 +0000 Subject: [PATCH 01/28] exp: concurrency and etags --- .../user_preferences/api.spec.yaml | 70 +++++++++++++++++++ .../user_preferences/apispec.py | 13 +++- 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/components/renku_data_services/user_preferences/api.spec.yaml b/components/renku_data_services/user_preferences/api.spec.yaml index 1af36eb57..c57ca63d1 100644 --- a/components/renku_data_services/user_preferences/api.spec.yaml +++ b/components/renku_data_services/user_preferences/api.spec.yaml @@ -12,9 +12,14 @@ paths: "/user/preferences": get: summary: Get user preferences for the currently logged in user + parameters: + - $ref: "#/components/parameters/If-None-Match" responses: "200": description: The user preferences + headers: + ETag: + $ref: "#/components/headers/ETag" content: "application/json": schema: @@ -29,6 +34,35 @@ paths: $ref: "#/components/responses/Error" tags: - user_preferences + patch: + summary: Update specific fields of a user preferences object + parameters: + - $ref: "#/components/parameters/If-Match" + requestBody: + required: true + content: + "application/json": + schema: + $ref: "#/components/schemas/UserPreferencesPatch" + responses: + "200": + description: The updated user preferences + content: + "application/json": + schema: + $ref: "#/components/schemas/UserPreferences" + "409": + description: The update conflicts with the current state on the server + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + default: + $ref: "#/components/responses/Error" + tags: + - user_preferences + + "/user/preferences/pinned_projects": post: summary: Add a pinned project @@ -80,6 +114,13 @@ components: pinned_projects: $ref: "#/components/schemas/PinnedProjects" required: ["user_id", "pinned_projects"] + UserPreferencesPatch: + type: object + description: Update user preferences + additionalProperties: false + properties: + pinned_projects: + $ref: "#/components/schemas/PinnedProjects" UserId: type: string description: The unique identifier for a user @@ -127,6 +168,10 @@ components: example: "Something went wrong - please try again later" required: ["code", "message"] required: ["error"] + ETag: + type: string + description: Entity Tag + example: "abcd" responses: Error: @@ -135,6 +180,31 @@ components: "application/json": schema: $ref: "#/components/schemas/ErrorResponse" + + headers: + ETag: + description: Entity Tag header + required: true + schema: + $ref: "#/components/schemas/ETag" + + parameters: + If-Match: + in: header + name: If-Match header + description: If-Match header, for avoiding mid-air collisions + required: false + schema: + $ref: "#/components/schemas/ETag" + If-None-Match: + in: header + name: If-None-Match header + description: If-None-Match header, for caching entities + required: false + schema: + $ref: "#/components/schemas/ETag" + + securitySchemes: oidc: type: openIdConnect diff --git a/components/renku_data_services/user_preferences/apispec.py b/components/renku_data_services/user_preferences/apispec.py index e584202ed..bfc7c169f 100644 --- a/components/renku_data_services/user_preferences/apispec.py +++ b/components/renku_data_services/user_preferences/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2024-01-12T10:11:36+00:00 +# timestamp: 2024-02-26T10:46:20+00:00 from __future__ import annotations @@ -44,6 +44,10 @@ class ErrorResponse(BaseAPISpec): error: Error +class ETag(RootModel[str]): + root: str = Field(..., description="Entity Tag", example="abcd") + + class PinnedProjects(BaseAPISpec): project_slugs: Optional[List[ProjectSlug]] = None @@ -54,3 +58,10 @@ class UserPreferences(BaseAPISpec): ) user_id: str = Field(..., description="The unique identifier for a user", example="user-id-example", min_length=3) pinned_projects: PinnedProjects + + +class UserPreferencesPatch(BaseAPISpec): + model_config = ConfigDict( + extra="forbid", + ) + pinned_projects: Optional[PinnedProjects] = None From 137dc7644ccaa5843f83c36086d8e9b1073966e5 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 26 Feb 2024 12:26:22 +0000 Subject: [PATCH 02/28] add datetime fields --- .../user_preferences/models.py | 8 +++-- .../user_preferences/orm.py | 12 +++++++- .../15c2cebaa8ec_add_datetime_fields.py | 30 +++++++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 components/renku_data_services/user_preferences_migrations/versions/15c2cebaa8ec_add_datetime_fields.py diff --git a/components/renku_data_services/user_preferences/models.py b/components/renku_data_services/user_preferences/models.py index 365f4fdf7..e3cd856ba 100644 --- a/components/renku_data_services/user_preferences/models.py +++ b/components/renku_data_services/user_preferences/models.py @@ -1,5 +1,7 @@ """Models for user preferences.""" -from typing import List, Optional + +from datetime import datetime +from typing import List from pydantic import BaseModel, Field @@ -7,7 +9,7 @@ class PinnedProjects(BaseModel): """Pinned projects model.""" - project_slugs: Optional[List[str]] = None + project_slugs: List[str] | None = None @classmethod def from_dict(cls, data: dict) -> "PinnedProjects": @@ -20,3 +22,5 @@ class UserPreferences(BaseModel): user_id: str = Field(min_length=3) pinned_projects: PinnedProjects + created_at: datetime | None = Field(default=None) + updated_at: datetime | None = Field(default=None) diff --git a/components/renku_data_services/user_preferences/orm.py b/components/renku_data_services/user_preferences/orm.py index 9bca938b6..f0f5a1df9 100644 --- a/components/renku_data_services/user_preferences/orm.py +++ b/components/renku_data_services/user_preferences/orm.py @@ -1,7 +1,9 @@ """SQLAlchemy schemas for the user preferences database.""" + +from datetime import datetime from typing import Any -from sqlalchemy import JSON, Integer, MetaData, String +from sqlalchemy import JSON, DateTime, Integer, MetaData, String, func from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import DeclarativeBase, Mapped, MappedAsDataclass, mapped_column @@ -32,6 +34,14 @@ class UserPreferencesORM(BaseORM): pinned_projects: Mapped[dict[str, Any]] = mapped_column("pinned_projects", JSONVariant) """Pinned projects.""" + created_at: Mapped[datetime | None] = mapped_column( + "created_at", DateTime(timezone=True), default=None, server_default=func.now() + ) + + updated_at: Mapped[datetime | None] = mapped_column( + "updated_at", DateTime(timezone=True), default=None, server_default=func.now(), onupdate=func.now() + ) + @classmethod def load(cls, user_preferences: models.UserPreferences): """Create UserPreferencesORM from the user preferences model.""" diff --git a/components/renku_data_services/user_preferences_migrations/versions/15c2cebaa8ec_add_datetime_fields.py b/components/renku_data_services/user_preferences_migrations/versions/15c2cebaa8ec_add_datetime_fields.py new file mode 100644 index 000000000..3e33b39de --- /dev/null +++ b/components/renku_data_services/user_preferences_migrations/versions/15c2cebaa8ec_add_datetime_fields.py @@ -0,0 +1,30 @@ +"""add_datetime_fields + +Revision ID: 15c2cebaa8ec +Revises: 6eccd7d4e3ed +Create Date: 2024-02-26 12:24:32.708388 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '15c2cebaa8ec' +down_revision = '6eccd7d4e3ed' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('user_preferences', sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), schema='user_preferences') + op.add_column('user_preferences', sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), schema='user_preferences') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('user_preferences', 'updated_at', schema='user_preferences') + op.drop_column('user_preferences', 'created_at', schema='user_preferences') + # ### end Alembic commands ### From 7100bb250e887855a95eb488bbc465c3c4eb2e9e Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 26 Feb 2024 13:05:44 +0000 Subject: [PATCH 03/28] add ETag header --- .../renku_data_services/user_preferences/blueprints.py | 7 ++++++- components/renku_data_services/user_preferences/models.py | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/components/renku_data_services/user_preferences/blueprints.py b/components/renku_data_services/user_preferences/blueprints.py index b3596ee9d..22e8867f6 100644 --- a/components/renku_data_services/user_preferences/blueprints.py +++ b/components/renku_data_services/user_preferences/blueprints.py @@ -1,4 +1,5 @@ """User preferences app.""" + from dataclasses import dataclass from sanic import Request, json @@ -26,7 +27,11 @@ def get(self) -> BlueprintFactoryResponse: async def _get(_: Request, user: base_models.APIUser): user_preferences: models.UserPreferences | None user_preferences = await self.user_preferences_repo.get_user_preferences(user=user) - return json(apispec.UserPreferences.model_validate(user_preferences).model_dump()) + headers = {"ETag": user_preferences.etag} if user_preferences.etag is not None else None + return json( + apispec.UserPreferences.model_validate(user_preferences).model_dump(), + headers=headers, + ) return "/user/preferences", ["GET"], _get diff --git a/components/renku_data_services/user_preferences/models.py b/components/renku_data_services/user_preferences/models.py index e3cd856ba..8370941fb 100644 --- a/components/renku_data_services/user_preferences/models.py +++ b/components/renku_data_services/user_preferences/models.py @@ -1,6 +1,7 @@ """Models for user preferences.""" from datetime import datetime +from hashlib import md5 from typing import List from pydantic import BaseModel, Field @@ -24,3 +25,10 @@ class UserPreferences(BaseModel): pinned_projects: PinnedProjects created_at: datetime | None = Field(default=None) updated_at: datetime | None = Field(default=None) + + @property + def etag(self) -> str | None: + """Entity tag value for this user preferences object.""" + if self.updated_at is None: + return None + return md5(self.updated_at.isoformat().encode(), usedforsecurity=False).hexdigest().upper() From 6637cc7306480d526d2268dbc586b96036eb0469 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 26 Feb 2024 13:19:08 +0000 Subject: [PATCH 04/28] add datetime fields to dump() --- components/renku_data_services/user_preferences/orm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/renku_data_services/user_preferences/orm.py b/components/renku_data_services/user_preferences/orm.py index f0f5a1df9..851b548b9 100644 --- a/components/renku_data_services/user_preferences/orm.py +++ b/components/renku_data_services/user_preferences/orm.py @@ -53,5 +53,8 @@ def load(cls, user_preferences: models.UserPreferences): def dump(self): """Create a user preferences model from the ORM object.""" return models.UserPreferences( - user_id=self.user_id, pinned_projects=models.PinnedProjects.from_dict(self.pinned_projects) + user_id=self.user_id, + pinned_projects=models.PinnedProjects.from_dict(self.pinned_projects), + created_at=self.created_at, + updated_at=self.updated_at, ) From f258aaf0c0a7415923deb2ecc8d54d55ae387027 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 26 Feb 2024 13:33:10 +0000 Subject: [PATCH 05/28] return 304 if etag matches --- .../renku_data_services/user_preferences/api.spec.yaml | 7 +++---- .../renku_data_services/user_preferences/apispec.py | 4 ++-- .../renku_data_services/user_preferences/blueprints.py | 9 +++++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/components/renku_data_services/user_preferences/api.spec.yaml b/components/renku_data_services/user_preferences/api.spec.yaml index c57ca63d1..45dcf9f50 100644 --- a/components/renku_data_services/user_preferences/api.spec.yaml +++ b/components/renku_data_services/user_preferences/api.spec.yaml @@ -171,7 +171,7 @@ components: ETag: type: string description: Entity Tag - example: "abcd" + example: "9EE498F9D565D0C41E511377425F32F3" responses: Error: @@ -191,20 +191,19 @@ components: parameters: If-Match: in: header - name: If-Match header + name: If-Match description: If-Match header, for avoiding mid-air collisions required: false schema: $ref: "#/components/schemas/ETag" If-None-Match: in: header - name: If-None-Match header + name: If-None-Match description: If-None-Match header, for caching entities required: false schema: $ref: "#/components/schemas/ETag" - securitySchemes: oidc: type: openIdConnect diff --git a/components/renku_data_services/user_preferences/apispec.py b/components/renku_data_services/user_preferences/apispec.py index bfc7c169f..38664b149 100644 --- a/components/renku_data_services/user_preferences/apispec.py +++ b/components/renku_data_services/user_preferences/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2024-02-26T10:46:20+00:00 +# timestamp: 2024-02-26T13:33:02+00:00 from __future__ import annotations @@ -45,7 +45,7 @@ class ErrorResponse(BaseAPISpec): class ETag(RootModel[str]): - root: str = Field(..., description="Entity Tag", example="abcd") + root: str = Field(..., description="Entity Tag", example="9EE498F9D565D0C41E511377425F32F3") class PinnedProjects(BaseAPISpec): diff --git a/components/renku_data_services/user_preferences/blueprints.py b/components/renku_data_services/user_preferences/blueprints.py index 22e8867f6..b6a506932 100644 --- a/components/renku_data_services/user_preferences/blueprints.py +++ b/components/renku_data_services/user_preferences/blueprints.py @@ -4,7 +4,7 @@ from sanic import Request, json from sanic_ext import validate - +from sanic import HTTPResponse import renku_data_services.base_models as base_models from renku_data_services.base_api.auth import authenticate from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint @@ -24,9 +24,14 @@ def get(self) -> BlueprintFactoryResponse: """Get user preferences for the logged in user.""" @authenticate(self.authenticator) - async def _get(_: Request, user: base_models.APIUser): + async def _get(request: Request, user: base_models.APIUser): user_preferences: models.UserPreferences | None user_preferences = await self.user_preferences_repo.get_user_preferences(user=user) + + etag = request.headers.get("If-None-Match") + if user_preferences.etag is not None and user_preferences.etag == etag: + return HTTPResponse(status=304) + headers = {"ETag": user_preferences.etag} if user_preferences.etag is not None else None return json( apispec.UserPreferences.model_validate(user_preferences).model_dump(), From eea9ec350ef7ecdee93fd9f31c117c5b1ccbbde8 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 26 Feb 2024 16:24:14 +0000 Subject: [PATCH 06/28] implement PATCH --- .../renku_data_services/errors/errors.py | 10 +++++ .../user_preferences/blueprints.py | 23 ++++++++++-- .../user_preferences/db.py | 37 +++++++++++++++++++ .../user_preferences/models.py | 14 ++++++- 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/components/renku_data_services/errors/errors.py b/components/renku_data_services/errors/errors.py index 55b044f34..8df58b382 100644 --- a/components/renku_data_services/errors/errors.py +++ b/components/renku_data_services/errors/errors.py @@ -1,4 +1,5 @@ """Exceptions for the server.""" + from dataclasses import dataclass from typing import Optional @@ -71,3 +72,12 @@ class ProgrammingError(BaseError): code: int = 1500 message: str = "An unexpected error occurred." status_code: int = 500 + + +@dataclass +class ConflictError(BaseError): + """Rased when a conflicting update occurs.""" + + code: int = 1409 + message: str = "Conflicting update detected." + status_code: int = 409 diff --git a/components/renku_data_services/user_preferences/blueprints.py b/components/renku_data_services/user_preferences/blueprints.py index b6a506932..770c0d41a 100644 --- a/components/renku_data_services/user_preferences/blueprints.py +++ b/components/renku_data_services/user_preferences/blueprints.py @@ -2,13 +2,13 @@ from dataclasses import dataclass -from sanic import Request, json +from sanic import HTTPResponse, Request, json from sanic_ext import validate -from sanic import HTTPResponse + import renku_data_services.base_models as base_models from renku_data_services.base_api.auth import authenticate from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint -from renku_data_services.user_preferences import apispec, models +from renku_data_services.user_preferences import apispec from renku_data_services.user_preferences.apispec_base import PinnedProjectFilter from renku_data_services.user_preferences.db import UserPreferencesRepository @@ -25,7 +25,6 @@ def get(self) -> BlueprintFactoryResponse: @authenticate(self.authenticator) async def _get(request: Request, user: base_models.APIUser): - user_preferences: models.UserPreferences | None user_preferences = await self.user_preferences_repo.get_user_preferences(user=user) etag = request.headers.get("If-None-Match") @@ -40,6 +39,22 @@ async def _get(request: Request, user: base_models.APIUser): return "/user/preferences", ["GET"], _get + def patch(self) -> BlueprintFactoryResponse: + """Partially update the user preferences for the logged in user.""" + + @authenticate(self.authenticator) + @validate(json=apispec.UserPreferencesPatch) + async def _patch(request: Request, body: apispec.UserPreferencesPatch, user: base_models.APIUser): + etag = request.headers.get("If-None-Match") + body_dict = body.model_dump(exclude_none=True) + user_preferences = await self.user_preferences_repo.update_user_preferences( + user=user, etag=etag, **body_dict + ) + + return json(apispec.UserPreferences.model_validate(user_preferences).model_dump()) + + return "/user/preferences", ["PATCH"], _patch + def post_pinned_projects(self) -> BlueprintFactoryResponse: """Add a pinned project to user preferences for the logged in user.""" diff --git a/components/renku_data_services/user_preferences/db.py b/components/renku_data_services/user_preferences/db.py index 04a545746..664e380ca 100644 --- a/components/renku_data_services/user_preferences/db.py +++ b/components/renku_data_services/user_preferences/db.py @@ -1,4 +1,5 @@ """Adapters for user preferences database classes.""" + from typing import Callable, List, cast from sqlalchemy import select @@ -43,6 +44,42 @@ async def get_user_preferences( raise errors.MissingResourceError(message="Preferences not found for user.") return user_preferences.dump() + async def update_user_preferences( + self, user: base_models.APIUser, etag: str | None = None, **kwargs + ) -> models.UserPreferences: + """Update user preferences.""" + if not user.is_authenticated or user.id is None: + raise errors.Unauthorized(message="Anonymous users cannot have user preferences.") + + async with self.session_maker() as session: + async with session.begin(): + res = await session.scalars( + select(schemas.UserPreferencesORM).where(schemas.UserPreferencesORM.user_id == user.id) + ) + user_preferences = res.one_or_none() + + if user_preferences is None: + project_slugs = kwargs.get("project_slugs", []) + new_preferences = models.UserPreferences( + user_id=user.id, pinned_projects=models.PinnedProjects(project_slugs=project_slugs) + ) + user_preferences = schemas.UserPreferencesORM.load(new_preferences) + session.add(user_preferences) + return user_preferences.dump() + + current_etag = user_preferences.dump().etag + if etag is not None and current_etag != etag: + raise errors.ConflictError(message=f"Current ETag is {current_etag}, not {etag}.") + + if "pinned_projects" in kwargs: + kwargs["pinned_projects"] = models.PinnedProjects.from_dict(kwargs["pinned_projects"]) + + for key, value in kwargs.items(): + if key in ["pinned_projects"]: + setattr(user_preferences, key, value) + + return user_preferences.dump() + async def delete_user_preferences(self, user: base_models.APIUser) -> None: """Delete user preferences from the database.""" async with self.session_maker() as session: diff --git a/components/renku_data_services/user_preferences/models.py b/components/renku_data_services/user_preferences/models.py index 8370941fb..d0630c4d9 100644 --- a/components/renku_data_services/user_preferences/models.py +++ b/components/renku_data_services/user_preferences/models.py @@ -2,9 +2,9 @@ from datetime import datetime from hashlib import md5 -from typing import List +from typing import List, MutableSet -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator class PinnedProjects(BaseModel): @@ -17,6 +17,16 @@ def from_dict(cls, data: dict) -> "PinnedProjects": """Create model from a dict object.""" return cls(project_slugs=data.get("project_slugs")) + @field_validator("project_slugs") + @classmethod + def pinned_projects_are_unique(cls, project_slugs: List[str] | None) -> List[str] | None: + """Validates that pinned projects are unique.""" + seen: MutableSet[str] = set() + if project_slugs is None: + return None + project_slugs = [(seen.add(project), project)[1] for project in project_slugs if project not in seen] + return project_slugs + class UserPreferences(BaseModel): """User preferences model.""" From e841d4d31c148c1f15c2a420ee12caf8dd53ed02 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 26 Feb 2024 16:37:50 +0000 Subject: [PATCH 07/28] fix update() --- components/renku_data_services/user_preferences/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/renku_data_services/user_preferences/db.py b/components/renku_data_services/user_preferences/db.py index 664e380ca..638160e4f 100644 --- a/components/renku_data_services/user_preferences/db.py +++ b/components/renku_data_services/user_preferences/db.py @@ -72,7 +72,7 @@ async def update_user_preferences( raise errors.ConflictError(message=f"Current ETag is {current_etag}, not {etag}.") if "pinned_projects" in kwargs: - kwargs["pinned_projects"] = models.PinnedProjects.from_dict(kwargs["pinned_projects"]) + kwargs["pinned_projects"] = models.PinnedProjects.from_dict(kwargs["pinned_projects"]).model_dump() for key, value in kwargs.items(): if key in ["pinned_projects"]: From 354222c4789f08f5d1810136b144e37728ce767a Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 26 Feb 2024 16:44:46 +0000 Subject: [PATCH 08/28] fix typo for patch --- components/renku_data_services/user_preferences/blueprints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/renku_data_services/user_preferences/blueprints.py b/components/renku_data_services/user_preferences/blueprints.py index 770c0d41a..b346534d2 100644 --- a/components/renku_data_services/user_preferences/blueprints.py +++ b/components/renku_data_services/user_preferences/blueprints.py @@ -45,7 +45,7 @@ def patch(self) -> BlueprintFactoryResponse: @authenticate(self.authenticator) @validate(json=apispec.UserPreferencesPatch) async def _patch(request: Request, body: apispec.UserPreferencesPatch, user: base_models.APIUser): - etag = request.headers.get("If-None-Match") + etag = request.headers.get("If-Match") body_dict = body.model_dump(exclude_none=True) user_preferences = await self.user_preferences_repo.update_user_preferences( user=user, etag=etag, **body_dict From 05393f26a75400bc2248babe3bebb211a52117ac Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 28 Feb 2024 13:51:27 +0000 Subject: [PATCH 09/28] add etag to projects --- .../renku_data_services/project/blueprints.py | 5 +- components/renku_data_services/project/db.py | 17 ++++--- .../renku_data_services/project/models.py | 21 ++++++--- components/renku_data_services/project/orm.py | 15 ++++-- .../versions/7c08ed2fb79d_generate_tables.py | 3 ++ .../7f74e3591ffd_update_datetime_fields.py | 47 +++++++++++++++++++ 6 files changed, 89 insertions(+), 19 deletions(-) create mode 100644 components/renku_data_services/project_migrations/versions/7f74e3591ffd_update_datetime_fields.py diff --git a/components/renku_data_services/project/blueprints.py b/components/renku_data_services/project/blueprints.py index 9845dc281..a0ca07199 100644 --- a/components/renku_data_services/project/blueprints.py +++ b/components/renku_data_services/project/blueprints.py @@ -79,7 +79,10 @@ def get_one(self) -> BlueprintFactoryResponse: @authenticate(self.authenticator) async def _get_one(_: Request, *, user: base_models.APIUser, project_id: str): project = await self.project_repo.get_project(user=user, project_id=project_id) - return json(apispec.Project.model_validate(project).model_dump(exclude_none=True, mode="json")) + headers = {"ETag": project.etag} if project.etag is not None else None + return json( + apispec.Project.model_validate(project).model_dump(exclude_none=True, mode="json"), headers=headers + ) return "/projects/", ["GET"], _get_one diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index 853bf2f33..b9264b9cc 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -62,7 +62,7 @@ async def get_projects( stmt = select(schemas.ProjectORM) stmt = stmt.where(schemas.ProjectORM.id.in_(project_ids)) stmt = stmt.limit(per_page).offset((page - 1) * per_page) - stmt = stmt.order_by(schemas.ProjectORM.creation_date.desc()) + stmt = stmt.order_by(schemas.ProjectORM.created_at.desc()) result = await session.execute(stmt) projects_orm = result.scalars().all() @@ -115,7 +115,9 @@ async def insert_project(self, user: base_models.APIUser, project: models.Projec return project_orm.dump() - async def update_project(self, user: base_models.APIUser, project_id: str, **payload) -> models.Project: + async def update_project( + self, user: base_models.APIUser, project_id: str, etag: str | None = None, **payload + ) -> models.Project: """Update a project entry.""" authorized = await self.project_authz.has_permission(user=user, project_id=project_id, scope=Scope.WRITE) if not authorized: @@ -125,13 +127,16 @@ async def update_project(self, user: base_models.APIUser, project_id: str, **pay async with self.session_maker() as session: async with session.begin(): - result = await session.execute(select(schemas.ProjectORM).where(schemas.ProjectORM.id == project_id)) - projects = result.one_or_none() + result = await session.scalars(select(schemas.ProjectORM).where(schemas.ProjectORM.id == project_id)) + project = result.one_or_none() - if projects is None: + if project is None: raise errors.MissingResourceError(message=f"The project with id '{project_id}' cannot be found") - project = projects[0] + current_etag = project.dump().etag + if etag is not None and current_etag != etag: + raise errors.ConflictError(message=f"Current ETag is {current_etag}, not {etag}.") + visibility_before = project.visibility if "repositories" in payload: diff --git a/components/renku_data_services/project/models.py b/components/renku_data_services/project/models.py index 278dcbd14..b4ad1c215 100644 --- a/components/renku_data_services/project/models.py +++ b/components/renku_data_services/project/models.py @@ -2,11 +2,12 @@ import re import unicodedata -from dataclasses import dataclass, field -from datetime import datetime, timezone +from dataclasses import dataclass +from datetime import datetime +from hashlib import md5 from typing import Dict, List, Optional -from pydantic import BaseModel +from pydantic import BaseModel, Field from renku_data_services import errors from renku_data_services.project.apispec import Role, Visibility @@ -55,10 +56,18 @@ class Project(BaseModel): slug: str visibility: Visibility created_by: Member - creation_date: Optional[datetime] = None - repositories: List[Repository] = field(default_factory=list) + created_at: datetime | None = Field(default=None) + updated_at: datetime | None = Field(default=None) + repositories: List[Repository] = Field(default_factory=list) description: Optional[str] = None + @property + def etag(self) -> str | None: + """Entity tag value for this project object.""" + if self.updated_at is None: + return None + return md5(self.updated_at.isoformat().encode(), usedforsecurity=False).hexdigest().upper() + @classmethod def from_dict(cls, data: Dict) -> "Project": """Create the model from a plain dictionary.""" @@ -73,7 +82,6 @@ def from_dict(cls, data: Dict) -> "Project": name = data["name"] slug = data.get("slug") or get_slug(name) created_by = data["created_by"] - creation_date = data.get("creation_date") or datetime.now(timezone.utc).replace(microsecond=0) return cls( id=project_id, @@ -81,7 +89,6 @@ def from_dict(cls, data: Dict) -> "Project": slug=slug, created_by=created_by, visibility=data.get("visibility", Visibility.private), - creation_date=creation_date, repositories=[Repository(r) for r in data.get("repositories", [])], description=data.get("description"), ) diff --git a/components/renku_data_services/project/orm.py b/components/renku_data_services/project/orm.py index fb7f09832..47bbea64c 100644 --- a/components/renku_data_services/project/orm.py +++ b/components/renku_data_services/project/orm.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import List, Optional -from sqlalchemy import DateTime, Integer, MetaData, String +from sqlalchemy import DateTime, Integer, MetaData, String, func from sqlalchemy.orm import DeclarativeBase, Mapped, MappedAsDataclass, mapped_column, relationship from sqlalchemy.schema import ForeignKey from ulid import ULID @@ -30,8 +30,13 @@ class ProjectORM(BaseORM): slug: Mapped[str] = mapped_column("slug", String(99)) visibility: Mapped[Visibility] created_by_id: Mapped[str] = mapped_column("created_by_id", String()) - creation_date: Mapped[Optional[datetime]] = mapped_column("creation_date", DateTime(timezone=True)) - description: Mapped[Optional[str]] = mapped_column("description", String(500)) + description: Mapped[str | None] = mapped_column("description", String(500)) + created_at: Mapped[datetime | None] = mapped_column( + "created_at", DateTime(timezone=True), default=None, server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime | None] = mapped_column( + "updated_at", DateTime(timezone=True), default=None, server_default=func.now(), onupdate=func.now() + ) repositories: Mapped[List["ProjectRepositoryORM"]] = relationship( back_populates="project", default_factory=list, @@ -47,7 +52,6 @@ def load(cls, project: models.Project): slug=project.slug, visibility=project.visibility, created_by_id=project.created_by.id, - creation_date=project.creation_date, repositories=[ProjectRepositoryORM(url=r) for r in project.repositories], description=project.description, ) @@ -60,7 +64,8 @@ def dump(self) -> models.Project: slug=self.slug, visibility=self.visibility, created_by=models.Member(id=self.created_by_id), - creation_date=self.creation_date, + created_at=self.created_at, + updated_at=self.updated_at, repositories=[models.Repository(r.url) for r in self.repositories], description=self.description, ) diff --git a/components/renku_data_services/project_migrations/versions/7c08ed2fb79d_generate_tables.py b/components/renku_data_services/project_migrations/versions/7c08ed2fb79d_generate_tables.py index fbccdd88b..42257c89f 100644 --- a/components/renku_data_services/project_migrations/versions/7c08ed2fb79d_generate_tables.py +++ b/components/renku_data_services/project_migrations/versions/7c08ed2fb79d_generate_tables.py @@ -56,9 +56,12 @@ def upgrade() -> None: def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### + from renku_data_services.project.apispec import Visibility + op.drop_index( op.f("ix_projects_projects_repositories_project_id"), table_name="projects_repositories", schema="projects" ) op.drop_table("projects_repositories", schema="projects") op.drop_table("projects", schema="projects") + sa.Enum(Visibility).drop(op.get_bind()) # ### end Alembic commands ### diff --git a/components/renku_data_services/project_migrations/versions/7f74e3591ffd_update_datetime_fields.py b/components/renku_data_services/project_migrations/versions/7f74e3591ffd_update_datetime_fields.py new file mode 100644 index 000000000..411be7fc6 --- /dev/null +++ b/components/renku_data_services/project_migrations/versions/7f74e3591ffd_update_datetime_fields.py @@ -0,0 +1,47 @@ +"""update datetime fields + +Revision ID: 7f74e3591ffd +Revises: 7c08ed2fb79d +Create Date: 2024-02-28 13:57:36.667354 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "7f74e3591ffd" +down_revision = "7c08ed2fb79d" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "projects", + column_name="creation_date", + new_column_name="created_at", + server_default=sa.text("now()"), + schema="projects", + ) + op.add_column( + "projects", + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=True), + schema="projects", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "projects", + column_name="created_at", + new_column_name="creation_date", + server_default=None, + schema="projects", + ) + op.drop_column("projects", "updated_at", schema="projects") + # ### end Alembic commands ### From 11e9b2c260c915ac99c8e4de009479543aa00bfa Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 28 Feb 2024 15:36:10 +0100 Subject: [PATCH 10/28] update action --- .github/workflows/acceptance-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 59c4d00d4..911d7292f 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -30,7 +30,7 @@ jobs: extra-values: ${{ steps.deploy-comment.outputs.extra-values}} steps: - id: deploy-comment - uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.8.4 + uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.10.0 with: string: /deploy pr_ref: ${{ github.event.number }} @@ -60,7 +60,7 @@ jobs: body: | You can access the deployment of this PR at https://renku-ci-ds-${{ github.event.number }}.dev.renku.ch - name: Build and deploy - uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.8.4 + uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.10.0 env: RANCHER_PROJECT_ID: ${{ secrets.CI_RANCHER_PROJECT }} DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }} @@ -90,7 +90,7 @@ jobs: if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true' runs-on: ubuntu-22.04 steps: - - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.8.4 + - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.10.0 with: kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }} renku-release: renku-ci-ds-${{ github.event.number }} @@ -117,7 +117,7 @@ jobs: steps: - name: Extract Renku repository reference run: echo "RENKU_REFERENCE=`echo '${{ needs.check-deploy.outputs.renku }}' | cut -d'@' -f2`" >> $GITHUB_ENV - - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.8.4 + - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.10.0 with: e2e-target: ${{ matrix.tests }} renku-reference: ${{ env.RENKU_REFERENCE }} @@ -145,7 +145,7 @@ jobs: body: | Tearing down the temporary RenkuLab deplyoment for this PR. - name: renku teardown - uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.8.4 + uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.10.0 env: HELM_RELEASE_REGEX: "^renku-ci-ds-${{ github.event.number }}$" GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }} From e9ef0d28e763119770719c2f09c78e6ea180c931 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 28 Feb 2024 15:37:34 +0100 Subject: [PATCH 11/28] add renku-ui --- .github/workflows/acceptance-tests.yml | 1 + components/renku_data_services/crc/apispec.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 911d7292f..d2a6889e7 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -24,6 +24,7 @@ jobs: renku-gateway: ${{ steps.deploy-comment.outputs.renku-gateway}} renku-graph: ${{ steps.deploy-comment.outputs.renku-graph}} renku-notebooks: ${{ steps.deploy-comment.outputs.renku-notebooks}} + renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}} test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}} test-cypress-enabled: ${{ steps.deploy-comment.outputs.test-cypress-enabled}} persist: ${{ steps.deploy-comment.outputs.persist}} diff --git a/components/renku_data_services/crc/apispec.py b/components/renku_data_services/crc/apispec.py index bf5bb34cc..d0eacde4f 100644 --- a/components/renku_data_services/crc/apispec.py +++ b/components/renku_data_services/crc/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2024-01-12T10:11:35+00:00 +# timestamp: 2024-02-28T14:37:28+00:00 from __future__ import annotations @@ -33,11 +33,11 @@ class Version(BaseAPISpec): version: str -class IntegerIds(RootModel[List[int]]): +class IntegerIds(RootModel): root: List[int] = Field(..., example=[1, 3, 5], min_length=1) -class K8sLabel(RootModel[str]): +class K8sLabel(RootModel): root: str = Field( ..., description="A valid K8s label", @@ -85,7 +85,7 @@ class UserWithId(BaseAPISpec): ) -class UsersWithId(RootModel[List[UserWithId]]): +class UsersWithId(RootModel): root: List[UserWithId] @@ -354,9 +354,9 @@ class ResourcePoolWithIdFiltered(BaseAPISpec): default: bool = Field(..., description="A default selection for resource classes or resource pools", example=False) -class ResourcePoolsWithId(RootModel[List[ResourcePoolWithId]]): +class ResourcePoolsWithId(RootModel): root: List[ResourcePoolWithId] -class ResourcePoolsWithIdFiltered(RootModel[List[ResourcePoolWithIdFiltered]]): +class ResourcePoolsWithIdFiltered(RootModel): root: List[ResourcePoolWithIdFiltered] From 7fb97b750c9d6ff62a971b32a9c9059da1aaf86b Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 28 Feb 2024 15:02:14 +0000 Subject: [PATCH 12/28] fix projects --- components/renku_data_services/crc/apispec.py | 12 +++++----- .../renku_data_services/project/blueprints.py | 16 ++++++------- components/renku_data_services/project/db.py | 23 +++++++++---------- .../renku_data_services/project/models.py | 4 +++- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/components/renku_data_services/crc/apispec.py b/components/renku_data_services/crc/apispec.py index d0eacde4f..3ec85099b 100644 --- a/components/renku_data_services/crc/apispec.py +++ b/components/renku_data_services/crc/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2024-02-28T14:37:28+00:00 +# timestamp: 2024-02-28T15:00:52+00:00 from __future__ import annotations @@ -33,11 +33,11 @@ class Version(BaseAPISpec): version: str -class IntegerIds(RootModel): +class IntegerIds(RootModel[List[int]]): root: List[int] = Field(..., example=[1, 3, 5], min_length=1) -class K8sLabel(RootModel): +class K8sLabel(RootModel[str]): root: str = Field( ..., description="A valid K8s label", @@ -85,7 +85,7 @@ class UserWithId(BaseAPISpec): ) -class UsersWithId(RootModel): +class UsersWithId(RootModel[List[UserWithId]]): root: List[UserWithId] @@ -354,9 +354,9 @@ class ResourcePoolWithIdFiltered(BaseAPISpec): default: bool = Field(..., description="A default selection for resource classes or resource pools", example=False) -class ResourcePoolsWithId(RootModel): +class ResourcePoolsWithId(RootModel[List[ResourcePoolWithId]]): root: List[ResourcePoolWithId] -class ResourcePoolsWithIdFiltered(RootModel): +class ResourcePoolsWithIdFiltered(RootModel[List[ResourcePoolWithIdFiltered]]): root: List[ResourcePoolWithIdFiltered] diff --git a/components/renku_data_services/project/blueprints.py b/components/renku_data_services/project/blueprints.py index fa30528b5..bb3551022 100644 --- a/components/renku_data_services/project/blueprints.py +++ b/components/renku_data_services/project/blueprints.py @@ -1,8 +1,6 @@ """Project blueprint.""" from dataclasses import dataclass -from datetime import datetime, timezone -from typing import cast from sanic import HTTPResponse, Request, json from sanic_ext import validate @@ -11,7 +9,7 @@ from renku_data_services.base_api.auth import authenticate, only_authenticated from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint from renku_data_services.errors import errors -from renku_data_services.project import apispec, models +from renku_data_services.project import apispec from renku_data_services.project.apispec import FullUserWithRole, UserWithId from renku_data_services.project.db import ProjectMemberRepository, ProjectRepository from renku_data_services.users.db import UserRepo @@ -66,13 +64,13 @@ def post(self) -> BlueprintFactoryResponse: @only_authenticated @validate(json=apispec.ProjectPost) async def _post(_: Request, *, user: base_models.APIUser, body: apispec.ProjectPost): - data = body.model_dump(exclude_none=True) - user_id: str = cast(str, user.id) - data["created_by"] = models.Member(id=user_id) + # body_dict = body.model_dump(exclude_none=True) + # user_id: str = cast(str, user.id) + # data["created_by"] = models.Member(id=user_id) # NOTE: Set ``creation_date`` to override possible value set by users - data["creation_date"] = datetime.now(timezone.utc).replace(microsecond=0) - project = models.Project.from_dict(data) - result = await self.project_repo.insert_project(user=user, project=project) + # data["creation_date"] = datetime.now(timezone.utc).replace(microsecond=0) + # project = models.Project.from_dict(data) + result = await self.project_repo.insert_project(user=user, new_project=body) return json(apispec.Project.model_validate(result).model_dump(exclude_none=True, mode="json"), 201) return "/projects", ["POST"], _post diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index 2a8ffc166..4793cf37c 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -2,7 +2,6 @@ from __future__ import annotations -from datetime import datetime, timezone from typing import Any, Callable, Dict, List, NamedTuple, Tuple, cast from sqlalchemy import func, select @@ -13,7 +12,7 @@ from renku_data_services.authz import models as authz_models from renku_data_services.authz.authz import IProjectAuthorizer from renku_data_services.authz.models import MemberQualifier, Scope -from renku_data_services.project import models +from renku_data_services.project import apispec, models from renku_data_services.project import orm as schemas from renku_data_services.project.apispec import Role, Visibility @@ -98,25 +97,25 @@ async def get_project(self, user: base_models.APIUser, project_id: str) -> model return project_orm.dump() - async def insert_project(self, user: base_models.APIUser, project: models.Project) -> models.Project: + async def insert_project(self, user: base_models.APIUser, new_project=apispec.ProjectPost) -> models.Project: """Insert a new project entry.""" - project_orm = schemas.ProjectORM.load(project) - project_orm.creation_date = datetime.now(timezone.utc).replace(microsecond=0) - project_orm.created_by = user.id + + project_model = models.Project.from_dict(**new_project) + project = schemas.ProjectORM.load(project_model) async with self.session_maker() as session: async with session.begin(): - session.add(project_orm) + session.add(project) - project = project_orm.dump() - public_project = project.visibility == Visibility.public - if project.id is None: + project_model = project.dump() + public_project = project_model.visibility == Visibility.public + if project_model.id is None: raise errors.BaseError(detail="The created project does not have an ID but it should.") await self.project_authz.create_project( - requested_by=user, project_id=project.id, public_project=public_project + requested_by=user, project_id=project_model.id, public_project=public_project ) - return project_orm.dump() + return project.dump() async def update_project( self, user: base_models.APIUser, project_id: str, etag: str | None = None, **payload diff --git a/components/renku_data_services/project/models.py b/components/renku_data_services/project/models.py index b4ad1c215..a9c1173c3 100644 --- a/components/renku_data_services/project/models.py +++ b/components/renku_data_services/project/models.py @@ -87,8 +87,10 @@ def from_dict(cls, data: Dict) -> "Project": id=project_id, name=name, slug=slug, - created_by=created_by, visibility=data.get("visibility", Visibility.private), + created_by=created_by, + created_at=data.get("created_at"), + updated_at=data.get("updated_at"), repositories=[Repository(r) for r in data.get("repositories", [])], description=data.get("description"), ) From 5e0618a95faeb1449b9096555446c12a8a99eacb Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 28 Feb 2024 15:14:49 +0000 Subject: [PATCH 13/28] fix projects --- components/renku_data_services/project/db.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index 4793cf37c..539625934 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -100,7 +100,10 @@ async def get_project(self, user: base_models.APIUser, project_id: str) -> model async def insert_project(self, user: base_models.APIUser, new_project=apispec.ProjectPost) -> models.Project: """Insert a new project entry.""" - project_model = models.Project.from_dict(**new_project) + project_dict = new_project.model_dump(exclude_none=True) + user_id: str = cast(str, user.id) + project_dict['created_by'] = models.Member(id=user_id) + project_model = models.Project.from_dict(**project_dict) project = schemas.ProjectORM.load(project_model) async with self.session_maker() as session: From 0bf47bb062e3f2dd1ffbbdfbc186fc3d0719b0e7 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 28 Feb 2024 15:21:42 +0000 Subject: [PATCH 14/28] fix projects --- components/renku_data_services/project/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index 539625934..c06270377 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -103,7 +103,7 @@ async def insert_project(self, user: base_models.APIUser, new_project=apispec.Pr project_dict = new_project.model_dump(exclude_none=True) user_id: str = cast(str, user.id) project_dict['created_by'] = models.Member(id=user_id) - project_model = models.Project.from_dict(**project_dict) + project_model = models.Project.from_dict(project_dict) project = schemas.ProjectORM.load(project_model) async with self.session_maker() as session: From ecc6e96cd40baf17184d84a5fdf190c0d5c47b00 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 28 Feb 2024 15:30:51 +0000 Subject: [PATCH 15/28] fix projects --- components/renku_data_services/project/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index c06270377..19db3bc6e 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -118,7 +118,7 @@ async def insert_project(self, user: base_models.APIUser, new_project=apispec.Pr requested_by=user, project_id=project_model.id, public_project=public_project ) - return project.dump() + return project.dump() async def update_project( self, user: base_models.APIUser, project_id: str, etag: str | None = None, **payload From 1d7689aa6d3bc36377e0e15815410cd8f04df374 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 1 Mar 2024 12:35:56 +0000 Subject: [PATCH 16/28] do not rename column --- .../renku_data_services/errors/errors.py | 11 +++++++++- components/renku_data_services/project/db.py | 2 +- .../renku_data_services/project/models.py | 4 ++-- components/renku_data_services/project/orm.py | 6 +++--- ...=> e0ce196a6022_update_datetime_fields.py} | 21 +++---------------- 5 files changed, 19 insertions(+), 25 deletions(-) rename components/renku_data_services/project_migrations/versions/{7f74e3591ffd_update_datetime_fields.py => e0ce196a6022_update_datetime_fields.py} (58%) diff --git a/components/renku_data_services/errors/errors.py b/components/renku_data_services/errors/errors.py index 8df58b382..31e182543 100644 --- a/components/renku_data_services/errors/errors.py +++ b/components/renku_data_services/errors/errors.py @@ -76,8 +76,17 @@ class ProgrammingError(BaseError): @dataclass class ConflictError(BaseError): - """Rased when a conflicting update occurs.""" + """Raised when a conflicting update occurs.""" code: int = 1409 message: str = "Conflicting update detected." status_code: int = 409 + + +@dataclass +class PreconditionRequiredError(BaseError): + """Raised when a precondition is not met.""" + + code: int = 1428 + message: str = "Conflicting update detected." + status_code: int = 428 diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index 19db3bc6e..b9ca9f583 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -64,7 +64,7 @@ async def get_projects( stmt = select(schemas.ProjectORM) stmt = stmt.where(schemas.ProjectORM.id.in_(project_ids)) stmt = stmt.limit(per_page).offset(offset) - stmt = stmt.order_by(schemas.ProjectORM.created_at.desc()) + stmt = stmt.order_by(schemas.ProjectORM.creation_date.desc()) result = await session.execute(stmt) projects_orm = result.scalars().all() diff --git a/components/renku_data_services/project/models.py b/components/renku_data_services/project/models.py index a9c1173c3..8159c05a2 100644 --- a/components/renku_data_services/project/models.py +++ b/components/renku_data_services/project/models.py @@ -56,7 +56,7 @@ class Project(BaseModel): slug: str visibility: Visibility created_by: Member - created_at: datetime | None = Field(default=None) + creation_date: datetime | None = Field(default=None) updated_at: datetime | None = Field(default=None) repositories: List[Repository] = Field(default_factory=list) description: Optional[str] = None @@ -89,7 +89,7 @@ def from_dict(cls, data: Dict) -> "Project": slug=slug, visibility=data.get("visibility", Visibility.private), created_by=created_by, - created_at=data.get("created_at"), + creation_date=data.get("creation_date"), updated_at=data.get("updated_at"), repositories=[Repository(r) for r in data.get("repositories", [])], description=data.get("description"), diff --git a/components/renku_data_services/project/orm.py b/components/renku_data_services/project/orm.py index 47bbea64c..22debc979 100644 --- a/components/renku_data_services/project/orm.py +++ b/components/renku_data_services/project/orm.py @@ -31,8 +31,8 @@ class ProjectORM(BaseORM): visibility: Mapped[Visibility] created_by_id: Mapped[str] = mapped_column("created_by_id", String()) description: Mapped[str | None] = mapped_column("description", String(500)) - created_at: Mapped[datetime | None] = mapped_column( - "created_at", DateTime(timezone=True), default=None, server_default=func.now(), nullable=False + creation_date: Mapped[datetime | None] = mapped_column( + "creation_date", DateTime(timezone=True), default=None, server_default=func.now(), nullable=False ) updated_at: Mapped[datetime | None] = mapped_column( "updated_at", DateTime(timezone=True), default=None, server_default=func.now(), onupdate=func.now() @@ -64,7 +64,7 @@ def dump(self) -> models.Project: slug=self.slug, visibility=self.visibility, created_by=models.Member(id=self.created_by_id), - created_at=self.created_at, + creation_date=self.creation_date, updated_at=self.updated_at, repositories=[models.Repository(r.url) for r in self.repositories], description=self.description, diff --git a/components/renku_data_services/project_migrations/versions/7f74e3591ffd_update_datetime_fields.py b/components/renku_data_services/project_migrations/versions/e0ce196a6022_update_datetime_fields.py similarity index 58% rename from components/renku_data_services/project_migrations/versions/7f74e3591ffd_update_datetime_fields.py rename to components/renku_data_services/project_migrations/versions/e0ce196a6022_update_datetime_fields.py index 411be7fc6..5020b607c 100644 --- a/components/renku_data_services/project_migrations/versions/7f74e3591ffd_update_datetime_fields.py +++ b/components/renku_data_services/project_migrations/versions/e0ce196a6022_update_datetime_fields.py @@ -1,17 +1,16 @@ """update datetime fields -Revision ID: 7f74e3591ffd +Revision ID: e0ce196a6022 Revises: 7c08ed2fb79d -Create Date: 2024-02-28 13:57:36.667354 +Create Date: 2024-03-01 12:33:51.068133 """ import sqlalchemy as sa from alembic import op -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = "7f74e3591ffd" +revision = "e0ce196a6022" down_revision = "7c08ed2fb79d" branch_labels = None depends_on = None @@ -19,13 +18,6 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.alter_column( - "projects", - column_name="creation_date", - new_column_name="created_at", - server_default=sa.text("now()"), - schema="projects", - ) op.add_column( "projects", sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=True), @@ -36,12 +28,5 @@ def upgrade() -> None: def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.alter_column( - "projects", - column_name="created_at", - new_column_name="creation_date", - server_default=None, - schema="projects", - ) op.drop_column("projects", "updated_at", schema="projects") # ### end Alembic commands ### From 6264535f3f3182c274a8d593372d06724736ee33 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 1 Mar 2024 13:04:28 +0000 Subject: [PATCH 17/28] fix? --- .../6aff84383f42_update_datetime_fields.py | 40 +++++++++++++++++++ .../e0ce196a6022_update_datetime_fields.py | 12 ++++++ 2 files changed, 52 insertions(+) create mode 100644 components/renku_data_services/project_migrations/versions/6aff84383f42_update_datetime_fields.py diff --git a/components/renku_data_services/project_migrations/versions/6aff84383f42_update_datetime_fields.py b/components/renku_data_services/project_migrations/versions/6aff84383f42_update_datetime_fields.py new file mode 100644 index 000000000..c82e8dcf4 --- /dev/null +++ b/components/renku_data_services/project_migrations/versions/6aff84383f42_update_datetime_fields.py @@ -0,0 +1,40 @@ +"""update datetime fields + +Revision ID: e0ce196a6022 +Revises: 7c08ed2fb79d +Create Date: 2024-03-01 12:33:51.068133 + +""" + +import sqlalchemy as sa +from alembic import op + +# TODO: squash with previous revision + +# revision identifiers, used by Alembic. +revision = "6aff84383f42" +down_revision = "e0ce196a6022" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "projects", + column_name="creation_date", + server_default=sa.text("now()"), + schema="projects", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "projects", + column_name="creation_date", + server_default=None, + schema="projects", + ) + # ### end Alembic commands ### diff --git a/components/renku_data_services/project_migrations/versions/e0ce196a6022_update_datetime_fields.py b/components/renku_data_services/project_migrations/versions/e0ce196a6022_update_datetime_fields.py index 5020b607c..420b4e727 100644 --- a/components/renku_data_services/project_migrations/versions/e0ce196a6022_update_datetime_fields.py +++ b/components/renku_data_services/project_migrations/versions/e0ce196a6022_update_datetime_fields.py @@ -18,6 +18,12 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "projects", + column_name="creation_date", + server_default=sa.text("now()"), + schema="projects", + ) op.add_column( "projects", sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=True), @@ -28,5 +34,11 @@ def upgrade() -> None: def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "projects", + column_name="creation_date", + server_default=None, + schema="projects", + ) op.drop_column("projects", "updated_at", schema="projects") # ### end Alembic commands ### From 5a9b0007c3c69d56794fb0da8d5929152aad729e Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 1 Mar 2024 13:18:16 +0000 Subject: [PATCH 18/28] add logs --- components/renku_data_services/project/blueprints.py | 12 ++++-------- components/renku_data_services/project/db.py | 6 ++++-- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/components/renku_data_services/project/blueprints.py b/components/renku_data_services/project/blueprints.py index bb3551022..273dfab27 100644 --- a/components/renku_data_services/project/blueprints.py +++ b/components/renku_data_services/project/blueprints.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from sanic import HTTPResponse, Request, json +from sanic.log import logger from sanic_ext import validate import renku_data_services.base_models as base_models @@ -64,14 +65,9 @@ def post(self) -> BlueprintFactoryResponse: @only_authenticated @validate(json=apispec.ProjectPost) async def _post(_: Request, *, user: base_models.APIUser, body: apispec.ProjectPost): - # body_dict = body.model_dump(exclude_none=True) - # user_id: str = cast(str, user.id) - # data["created_by"] = models.Member(id=user_id) - # NOTE: Set ``creation_date`` to override possible value set by users - # data["creation_date"] = datetime.now(timezone.utc).replace(microsecond=0) - # project = models.Project.from_dict(data) - result = await self.project_repo.insert_project(user=user, new_project=body) - return json(apispec.Project.model_validate(result).model_dump(exclude_none=True, mode="json"), 201) + project = await self.project_repo.insert_project(user=user, new_project=body) + logger.info(f"creation_date = {project.creation_date}") + return json(apispec.Project.model_validate(project).model_dump(exclude_none=True, mode="json"), 201) return "/projects", ["POST"], _post diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index b9ca9f583..6e1399ab2 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -4,6 +4,7 @@ from typing import Any, Callable, Dict, List, NamedTuple, Tuple, cast +from sanic.log import logger from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession @@ -102,7 +103,7 @@ async def insert_project(self, user: base_models.APIUser, new_project=apispec.Pr project_dict = new_project.model_dump(exclude_none=True) user_id: str = cast(str, user.id) - project_dict['created_by'] = models.Member(id=user_id) + project_dict["created_by"] = models.Member(id=user_id) project_model = models.Project.from_dict(project_dict) project = schemas.ProjectORM.load(project_model) @@ -111,6 +112,7 @@ async def insert_project(self, user: base_models.APIUser, new_project=apispec.Pr session.add(project) project_model = project.dump() + logger.info(f"creation_date = {project_model.creation_date}") public_project = project_model.visibility == Visibility.public if project_model.id is None: raise errors.BaseError(detail="The created project does not have an ID but it should.") @@ -118,7 +120,7 @@ async def insert_project(self, user: base_models.APIUser, new_project=apispec.Pr requested_by=user, project_id=project_model.id, public_project=public_project ) - return project.dump() + return project_model async def update_project( self, user: base_models.APIUser, project_id: str, etag: str | None = None, **payload From e6c6cd717919a789bc7a17de411b04dcc405e7d0 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 1 Mar 2024 13:26:35 +0000 Subject: [PATCH 19/28] try to load() in transaction --- components/renku_data_services/project/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index 6e1399ab2..a3e1307e0 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -105,10 +105,10 @@ async def insert_project(self, user: base_models.APIUser, new_project=apispec.Pr user_id: str = cast(str, user.id) project_dict["created_by"] = models.Member(id=user_id) project_model = models.Project.from_dict(project_dict) - project = schemas.ProjectORM.load(project_model) async with self.session_maker() as session: async with session.begin(): + project = schemas.ProjectORM.load(project_model) session.add(project) project_model = project.dump() From 47413beb76b97a7ccf409170214ed39c388e6d16 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 1 Mar 2024 13:37:31 +0000 Subject: [PATCH 20/28] more loggign --- components/renku_data_services/project/db.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index a3e1307e0..257896abb 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -105,14 +105,17 @@ async def insert_project(self, user: base_models.APIUser, new_project=apispec.Pr user_id: str = cast(str, user.id) project_dict["created_by"] = models.Member(id=user_id) project_model = models.Project.from_dict(project_dict) + project = schemas.ProjectORM.load(project_model) async with self.session_maker() as session: async with session.begin(): - project = schemas.ProjectORM.load(project_model) session.add(project) + logger.info(f"orm creation_date = {project.creation_date}") + logger.info(f"orm updated_at = {project.updated_at}") project_model = project.dump() - logger.info(f"creation_date = {project_model.creation_date}") + logger.info(f"model creation_date = {project_model.creation_date}") + logger.info(f"model updated_at = {project_model.updated_at}") public_project = project_model.visibility == Visibility.public if project_model.id is None: raise errors.BaseError(detail="The created project does not have an ID but it should.") From 95d51255ac0c190586b2d0b1fc579be563dc5385 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 1 Mar 2024 13:47:25 +0000 Subject: [PATCH 21/28] logs in user_preferences --- components/renku_data_services/user_preferences/blueprints.py | 3 +++ components/renku_data_services/user_preferences/db.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/components/renku_data_services/user_preferences/blueprints.py b/components/renku_data_services/user_preferences/blueprints.py index b346534d2..092469bbd 100644 --- a/components/renku_data_services/user_preferences/blueprints.py +++ b/components/renku_data_services/user_preferences/blueprints.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from sanic import HTTPResponse, Request, json +from sanic.log import logger from sanic_ext import validate import renku_data_services.base_models as base_models @@ -62,6 +63,8 @@ def post_pinned_projects(self) -> BlueprintFactoryResponse: @validate(json=apispec.AddPinnedProject) async def _post(_: Request, body: apispec.AddPinnedProject, user: base_models.APIUser): res = await self.user_preferences_repo.add_pinned_project(user=user, project_slug=body.project_slug) + logger.info(f"model created_at = {res.created_at}") + logger.info(f"model updated_at = {res.updated_at}") return json(apispec.UserPreferences.model_validate(res).model_dump()) return "/user/preferences/pinned_projects", ["POST"], _post diff --git a/components/renku_data_services/user_preferences/db.py b/components/renku_data_services/user_preferences/db.py index 638160e4f..7d0f7e476 100644 --- a/components/renku_data_services/user_preferences/db.py +++ b/components/renku_data_services/user_preferences/db.py @@ -2,6 +2,7 @@ from typing import Callable, List, cast +from sanic.log import logger from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -115,6 +116,8 @@ async def add_pinned_project(self, user: base_models.APIUser, project_slug: str) ) user_preferences = schemas.UserPreferencesORM.load(new_preferences) session.add(user_preferences) + logger.info(f"orm created_at = {user_preferences.created_at}") + logger.info(f"orm updated_at = {user_preferences.updated_at}") return user_preferences.dump() project_slugs: List[str] From 53f146c85e5e669bf7625814383042b8411c5f5e Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 1 Mar 2024 14:05:34 +0000 Subject: [PATCH 22/28] try with re-assign --- components/renku_data_services/project/db.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index 257896abb..70dd21bc0 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -111,6 +111,10 @@ async def insert_project(self, user: base_models.APIUser, new_project=apispec.Pr async with session.begin(): session.add(project) + logger.info(f"orm creation_date = {project.creation_date}") + logger.info(f"orm updated_at = {project.updated_at}") + project.creation_date = project.creation_date + project.updated_at = project.updated_at logger.info(f"orm creation_date = {project.creation_date}") logger.info(f"orm updated_at = {project.updated_at}") project_model = project.dump() From 350b22fb3c3271c2f2f2e9e118dbaa65cdff8412 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 1 Mar 2024 14:24:45 +0000 Subject: [PATCH 23/28] try with commit() --- components/renku_data_services/project/db.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index 70dd21bc0..bcabb8154 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -111,10 +111,6 @@ async def insert_project(self, user: base_models.APIUser, new_project=apispec.Pr async with session.begin(): session.add(project) - logger.info(f"orm creation_date = {project.creation_date}") - logger.info(f"orm updated_at = {project.updated_at}") - project.creation_date = project.creation_date - project.updated_at = project.updated_at logger.info(f"orm creation_date = {project.creation_date}") logger.info(f"orm updated_at = {project.updated_at}") project_model = project.dump() @@ -127,7 +123,11 @@ async def insert_project(self, user: base_models.APIUser, new_project=apispec.Pr requested_by=user, project_id=project_model.id, public_project=public_project ) - return project_model + # Need to commit() to get the timestamps(?) + await session.commit() + logger.info(f"orm creation_date = {project.creation_date}") + logger.info(f"orm updated_at = {project.updated_at}") + return project.dump() async def update_project( self, user: base_models.APIUser, project_id: str, etag: str | None = None, **payload From 3d5e75a6f368f112eaea86700989d95606ec2163 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Fri, 1 Mar 2024 15:30:04 +0100 Subject: [PATCH 24/28] empty commit From eb554977ab5c24c97993f80ec3f88c3f15c8e81c Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 4 Mar 2024 13:54:18 +0000 Subject: [PATCH 25/28] Use entity tags on projects --- .../renku_data_services/project/blueprints.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/components/renku_data_services/project/blueprints.py b/components/renku_data_services/project/blueprints.py index 273dfab27..13826afa2 100644 --- a/components/renku_data_services/project/blueprints.py +++ b/components/renku_data_services/project/blueprints.py @@ -75,8 +75,13 @@ def get_one(self) -> BlueprintFactoryResponse: """Get a specific project.""" @authenticate(self.authenticator) - async def _get_one(_: Request, *, user: base_models.APIUser, project_id: str): + async def _get_one(request: Request, *, user: base_models.APIUser, project_id: str): project = await self.project_repo.get_project(user=user, project_id=project_id) + + etag = request.headers.get("If-None-Match") + if project.etag is not None and project.etag == etag: + return HTTPResponse(status=304) + headers = {"ETag": project.etag} if project.etag is not None else None return json( apispec.Project.model_validate(project).model_dump(exclude_none=True, mode="json"), headers=headers @@ -101,10 +106,17 @@ def patch(self) -> BlueprintFactoryResponse: @authenticate(self.authenticator) @only_authenticated @validate(json=apispec.ProjectPatch) - async def _patch(_: Request, *, user: base_models.APIUser, project_id: str, body: apispec.ProjectPatch): + async def _patch(request: Request, *, user: base_models.APIUser, project_id: str, body: apispec.ProjectPatch): + etag = request.headers.get("If-Match") + + if etag is None: + raise errors.PreconditionRequiredError(message="If-Match header not provided.") + body_dict = body.model_dump(exclude_none=True) - updated_project = await self.project_repo.update_project(user=user, project_id=project_id, **body_dict) + updated_project = await self.project_repo.update_project( + user=user, project_id=project_id, etag=etag, **body_dict + ) return json(apispec.Project.model_validate(updated_project).model_dump(exclude_none=True, mode="json")) From 2b123ed75c47ee71ac26e40a4debcd628d955da5 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Mon, 4 Mar 2024 14:37:17 +0000 Subject: [PATCH 26/28] experiment with adding etag as a field --- components/renku_data_services/project/api.spec.yaml | 6 ++++++ components/renku_data_services/project/apispec.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/components/renku_data_services/project/api.spec.yaml b/components/renku_data_services/project/api.spec.yaml index 620df7b22..81d533917 100644 --- a/components/renku_data_services/project/api.spec.yaml +++ b/components/renku_data_services/project/api.spec.yaml @@ -265,6 +265,8 @@ components: $ref: "#/components/schemas/Visibility" description: $ref: "#/components/schemas/Description" + etag: + $ref: "#/components/schemas/ETag" required: - "id" - "name" @@ -461,6 +463,10 @@ components: format: email description: User email example: some-user@gmail.com + ETag: + type: string + description: Entity Tag + example: "9EE498F9D565D0C41E511377425F32F3" ErrorResponse: type: object properties: diff --git a/components/renku_data_services/project/apispec.py b/components/renku_data_services/project/apispec.py index 89eae5f54..1b8366003 100644 --- a/components/renku_data_services/project/apispec.py +++ b/components/renku_data_services/project/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2024-01-19T00:27:33+00:00 +# timestamp: 2024-03-04T14:36:08+00:00 from __future__ import annotations @@ -96,6 +96,7 @@ class Project(BaseAPISpec): ) visibility: Visibility description: Optional[str] = Field(None, description="A description for project", max_length=500) + etag: Optional[str] = Field(None, description="Entity Tag", example="9EE498F9D565D0C41E511377425F32F3") class ProjectPost(BaseAPISpec): From da778e86af63e1aa5168a422e379e76c5b8b8f50 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 5 Mar 2024 12:05:51 +0000 Subject: [PATCH 27/28] update api spec --- .../renku_data_services/project/api.spec.yaml | 19 ++++++++++++++----- .../renku_data_services/project/apispec.py | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/components/renku_data_services/project/api.spec.yaml b/components/renku_data_services/project/api.spec.yaml index 81d533917..e1774b325 100644 --- a/components/renku_data_services/project/api.spec.yaml +++ b/components/renku_data_services/project/api.spec.yaml @@ -115,6 +115,7 @@ paths: required: true schema: type: string + - $ref: "#/components/parameters/If-Match" requestBody: required: true content: @@ -295,7 +296,7 @@ components: repositories: $ref: "#/components/schemas/RepositoriesList" visibility: - $ref: "#/components/schemas/Visibility" # Visibility is ``private`` if not passed at this point + $ref: "#/components/schemas/Visibility" # Visibility is ``private`` if not passed at this point default: "private" description: $ref: "#/components/schemas/Description" @@ -319,7 +320,7 @@ components: type: string minLength: 26 maxLength: 26 - pattern: "^[A-Z0-9]{26}$" # This is case-insensitive + pattern: "^[A-Z0-9]{26}$" # This is case-insensitive format: ulid Name: description: Renku project name @@ -332,7 +333,7 @@ components: type: string minLength: 1 maxLength: 99 - pattern: "^[a-z0-9]+[a-z0-9._-]*$" # Cannot contain consecutive special characters or end in .git and .atom + pattern: "^[a-z0-9]+[a-z0-9._-]*$" # Cannot contain consecutive special characters or end in .git and .atom example: "my-renku-project" CreationDate: description: The date and time the project was created (time is always in UTC) @@ -372,8 +373,7 @@ components: Repository: description: A project's repository type: string - example: - git@github.com:SwissDataScienceCenter/project-1.git + example: git@github.com:SwissDataScienceCenter/project-1.git Visibility: description: Project's visibility levels type: string @@ -496,3 +496,12 @@ components: application/json: schema: $ref: "#/components/schemas/ErrorResponse" + + parameters: + If-Match: + in: header + name: If-Match + description: If-Match header, for avoiding mid-air collisions + required: true + schema: + $ref: "#/components/schemas/ETag" diff --git a/components/renku_data_services/project/apispec.py b/components/renku_data_services/project/apispec.py index 1b8366003..d97d1993a 100644 --- a/components/renku_data_services/project/apispec.py +++ b/components/renku_data_services/project/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2024-03-04T14:36:08+00:00 +# timestamp: 2024-03-05T12:05:38+00:00 from __future__ import annotations From 0818b96bc7663a03953b298c8a85b975ead874b6 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Tue, 5 Mar 2024 12:32:43 +0000 Subject: [PATCH 28/28] update when repos are updated --- components/renku_data_services/project/db.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index bcabb8154..f01f80fb8 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, List, NamedTuple, Tuple, cast from sanic.log import logger -from sqlalchemy import func, select +from sqlalchemy import func, select, update from sqlalchemy.ext.asyncio import AsyncSession import renku_data_services.base_models as base_models @@ -158,6 +158,10 @@ async def update_project( schemas.ProjectRepositoryORM(url=r, project_id=project_id, project=project) for r in payload["repositories"] ] + # Trigger update for ``updated_at`` column + await session.execute( + update(schemas.ProjectORM).where(schemas.ProjectORM.id == project_id).values() + ) for key, value in payload.items(): # NOTE: ``slug``, ``id``, ``created_by``, and ``creation_date`` cannot be edited