WIP: add ETags

.github/workflows/acceptance-tests.yml

@@ -24,13 +24,14 @@ jobs:
       renku-gateway: ${{ steps.deploy-comment.outputs.renku-gateway}}
       renku-graph: ${{ steps.deploy-comment.outputs.renku-graph}}
       renku-notebooks: ${{ steps.deploy-comment.outputs.renku-notebooks}}
+      renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
       test-cypress-enabled: ${{ steps.deploy-comment.outputs.test-cypress-enabled}}
       persist: ${{ steps.deploy-comment.outputs.persist}}
       extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
     steps:
       - id: deploy-comment
-        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.8.4
+        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.10.0
         with:
           string: /deploy
           pr_ref: ${{ github.event.number }}
@@ -60,7 +61,7 @@ jobs:
           body: |
             You can access the deployment of this PR at https://renku-ci-ds-${{ github.event.number }}.dev.renku.ch
       - name: Build and deploy
-        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.8.4
+        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.10.0
         env:
           RANCHER_PROJECT_ID: ${{ secrets.CI_RANCHER_PROJECT }}
           DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
@@ -90,7 +91,7 @@ jobs:
     if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true'
     runs-on: ubuntu-22.04
     steps:
-      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.8.4
+      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.10.0
         with:
           kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
           renku-release: renku-ci-ds-${{ github.event.number }}
@@ -117,7 +118,7 @@ jobs:
     steps:
       - name: Extract Renku repository reference
         run: echo "RENKU_REFERENCE=`echo '${{ needs.check-deploy.outputs.renku }}' | cut -d'@' -f2`" >> $GITHUB_ENV
-      - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.8.4
+      - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.10.0
         with:
           e2e-target: ${{ matrix.tests }}
           renku-reference: ${{ env.RENKU_REFERENCE }}
@@ -145,7 +146,7 @@ jobs:
           body: |
             Tearing down the temporary RenkuLab deplyoment for this PR.
       - name: renku teardown
-        uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.8.4
+        uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.10.0
         env:
           HELM_RELEASE_REGEX: "^renku-ci-ds-${{ github.event.number }}$"
           GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }}

components/renku_data_services/crc/apispec.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2024-01-12T10:11:35+00:00
+#   timestamp: 2024-02-28T15:00:52+00:00
 
 from __future__ import annotations
 

components/renku_data_services/errors/errors.py

@@ -1,4 +1,5 @@
 """Exceptions for the server."""
+
 from dataclasses import dataclass
 from typing import Optional
 
@@ -71,3 +72,21 @@ class ProgrammingError(BaseError):
     code: int = 1500
     message: str = "An unexpected error occurred."
     status_code: int = 500
+
+
+@dataclass
+class ConflictError(BaseError):
+    """Raised when a conflicting update occurs."""
+
+    code: int = 1409
+    message: str = "Conflicting update detected."
+    status_code: int = 409
+
+
+@dataclass
+class PreconditionRequiredError(BaseError):
+    """Raised when a precondition is not met."""
+
+    code: int = 1428
+    message: str = "Conflicting update detected."
+    status_code: int = 428

components/renku_data_services/project/api.spec.yaml

@@ -115,6 +115,7 @@ paths:
           required: true
           schema:
             type: string
+        - $ref: "#/components/parameters/If-Match"
       requestBody:
         required: true
         content:
@@ -265,6 +266,8 @@ components:
           $ref: "#/components/schemas/Visibility"
         description:
           $ref: "#/components/schemas/Description"
+        etag:
+          $ref: "#/components/schemas/ETag"
       required:
         - "id"
         - "name"
@@ -293,7 +296,7 @@ components:
         repositories:
           $ref: "#/components/schemas/RepositoriesList"
         visibility:
-          $ref: "#/components/schemas/Visibility"  # Visibility is ``private`` if not passed at this point
+          $ref: "#/components/schemas/Visibility" # Visibility is ``private`` if not passed at this point
           default: "private"
         description:
           $ref: "#/components/schemas/Description"
@@ -317,7 +320,7 @@ components:
       type: string
       minLength: 26
       maxLength: 26
-      pattern: "^[A-Z0-9]{26}$"  # This is case-insensitive
+      pattern: "^[A-Z0-9]{26}$" # This is case-insensitive
       format: ulid
     Name:
       description: Renku project name
@@ -330,7 +333,7 @@ components:
       type: string
       minLength: 1
       maxLength: 99
-      pattern: "^[a-z0-9]+[a-z0-9._-]*$"  # Cannot contain consecutive special characters or end in .git and .atom
+      pattern: "^[a-z0-9]+[a-z0-9._-]*$" # Cannot contain consecutive special characters or end in .git and .atom
       example: "my-renku-project"
     CreationDate:
       description: The date and time the project was created (time is always in UTC)
@@ -370,8 +373,7 @@ components:
     Repository:
       description: A project's repository
       type: string
-      example:
-        git@github.com:SwissDataScienceCenter/project-1.git
+      example: git@github.com:SwissDataScienceCenter/project-1.git
     Visibility:
       description: Project's visibility levels
       type: string
@@ -461,6 +463,10 @@ components:
       format: email
       description: User email
       example: some-user@gmail.com
+    ETag:
+      type: string
+      description: Entity Tag
+      example: "9EE498F9D565D0C41E511377425F32F3"
     ErrorResponse:
       type: object
       properties:
@@ -490,3 +496,12 @@ components:
         application/json:
           schema:
             $ref: "#/components/schemas/ErrorResponse"
+
+  parameters:
+    If-Match:
+      in: header
+      name: If-Match
+      description: If-Match header, for avoiding mid-air collisions
+      required: true
+      schema:
+        $ref: "#/components/schemas/ETag"

components/renku_data_services/project/apispec.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2024-01-19T00:27:33+00:00
+#   timestamp: 2024-03-05T12:05:38+00:00
 
 from __future__ import annotations
 
@@ -96,6 +96,7 @@ class Project(BaseAPISpec):
     )
     visibility: Visibility
     description: Optional[str] = Field(None, description="A description for project", max_length=500)
+    etag: Optional[str] = Field(None, description="Entity Tag", example="9EE498F9D565D0C41E511377425F32F3")
 
 
 class ProjectPost(BaseAPISpec):

components/renku_data_services/project/blueprints.py

@@ -1,17 +1,16 @@
 """Project blueprint."""
 
 from dataclasses import dataclass
-from datetime import datetime, timezone
-from typing import cast
 
 from sanic import HTTPResponse, Request, json
+from sanic.log import logger
 from sanic_ext import validate
 
 import renku_data_services.base_models as base_models
 from renku_data_services.base_api.auth import authenticate, only_authenticated
 from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint
 from renku_data_services.errors import errors
-from renku_data_services.project import apispec, models
+from renku_data_services.project import apispec
 from renku_data_services.project.apispec import FullUserWithRole, UserWithId
 from renku_data_services.project.db import ProjectMemberRepository, ProjectRepository
 from renku_data_services.users.db import UserRepo
@@ -66,24 +65,27 @@ def post(self) -> BlueprintFactoryResponse:
         @only_authenticated
         @validate(json=apispec.ProjectPost)
         async def _post(_: Request, *, user: base_models.APIUser, body: apispec.ProjectPost):
-            data = body.model_dump(exclude_none=True)
-            user_id: str = cast(str, user.id)
-            data["created_by"] = models.Member(id=user_id)
-            # NOTE: Set ``creation_date`` to override possible value set by users
-            data["creation_date"] = datetime.now(timezone.utc).replace(microsecond=0)
-            project = models.Project.from_dict(data)
-            result = await self.project_repo.insert_project(user=user, project=project)
-            return json(apispec.Project.model_validate(result).model_dump(exclude_none=True, mode="json"), 201)
+            project = await self.project_repo.insert_project(user=user, new_project=body)
+            logger.info(f"creation_date = {project.creation_date}")
+            return json(apispec.Project.model_validate(project).model_dump(exclude_none=True, mode="json"), 201)
 
         return "/projects", ["POST"], _post
 
     def get_one(self) -> BlueprintFactoryResponse:
         """Get a specific project."""
 
         @authenticate(self.authenticator)
-        async def _get_one(_: Request, *, user: base_models.APIUser, project_id: str):
+        async def _get_one(request: Request, *, user: base_models.APIUser, project_id: str):
             project = await self.project_repo.get_project(user=user, project_id=project_id)
-            return json(apispec.Project.model_validate(project).model_dump(exclude_none=True, mode="json"))
+
+            etag = request.headers.get("If-None-Match")
+            if project.etag is not None and project.etag == etag:
+                return HTTPResponse(status=304)
+
+            headers = {"ETag": project.etag} if project.etag is not None else None
+            return json(
+                apispec.Project.model_validate(project).model_dump(exclude_none=True, mode="json"), headers=headers
+            )
 
         return "/projects/<project_id>", ["GET"], _get_one
 
@@ -104,10 +106,17 @@ def patch(self) -> BlueprintFactoryResponse:
         @authenticate(self.authenticator)
         @only_authenticated
         @validate(json=apispec.ProjectPatch)
-        async def _patch(_: Request, *, user: base_models.APIUser, project_id: str, body: apispec.ProjectPatch):
+        async def _patch(request: Request, *, user: base_models.APIUser, project_id: str, body: apispec.ProjectPatch):
+            etag = request.headers.get("If-Match")
+
+            if etag is None:
+                raise errors.PreconditionRequiredError(message="If-Match header not provided.")
+
             body_dict = body.model_dump(exclude_none=True)
 
-            updated_project = await self.project_repo.update_project(user=user, project_id=project_id, **body_dict)
+            updated_project = await self.project_repo.update_project(
+                user=user, project_id=project_id, etag=etag, **body_dict
+            )
 
             return json(apispec.Project.model_validate(updated_project).model_dump(exclude_none=True, mode="json"))
 

components/renku_data_services/project/db.py

@@ -2,18 +2,18 @@
 
 from __future__ import annotations
 
-from datetime import datetime, timezone
 from typing import Any, Callable, Dict, List, NamedTuple, Tuple, cast
 
-from sqlalchemy import func, select
+from sanic.log import logger
+from sqlalchemy import func, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 import renku_data_services.base_models as base_models
 from renku_data_services import errors
 from renku_data_services.authz import models as authz_models
 from renku_data_services.authz.authz import IProjectAuthorizer
 from renku_data_services.authz.models import MemberQualifier, Scope
-from renku_data_services.project import models
+from renku_data_services.project import apispec, models
 from renku_data_services.project import orm as schemas
 from renku_data_services.project.apispec import Role, Visibility
 
@@ -98,27 +98,40 @@ async def get_project(self, user: base_models.APIUser, project_id: str) -> model
 
             return project_orm.dump()
 
-    async def insert_project(self, user: base_models.APIUser, project: models.Project) -> models.Project:
+    async def insert_project(self, user: base_models.APIUser, new_project=apispec.ProjectPost) -> models.Project:
         """Insert a new project entry."""
-        project_orm = schemas.ProjectORM.load(project)
-        project_orm.creation_date = datetime.now(timezone.utc).replace(microsecond=0)
-        project_orm.created_by = user.id
+
+        project_dict = new_project.model_dump(exclude_none=True)
+        user_id: str = cast(str, user.id)
+        project_dict["created_by"] = models.Member(id=user_id)
+        project_model = models.Project.from_dict(project_dict)
+        project = schemas.ProjectORM.load(project_model)
 
         async with self.session_maker() as session:
             async with session.begin():
-                session.add(project_orm)
-
-                project = project_orm.dump()
-                public_project = project.visibility == Visibility.public
-                if project.id is None:
+                session.add(project)
+
+                logger.info(f"orm creation_date = {project.creation_date}")
+                logger.info(f"orm updated_at = {project.updated_at}")
+                project_model = project.dump()
+                logger.info(f"model creation_date = {project_model.creation_date}")
+                logger.info(f"model updated_at = {project_model.updated_at}")
+                public_project = project_model.visibility == Visibility.public
+                if project_model.id is None:
                     raise errors.BaseError(detail="The created project does not have an ID but it should.")
                 await self.project_authz.create_project(
-                    requested_by=user, project_id=project.id, public_project=public_project
+                    requested_by=user, project_id=project_model.id, public_project=public_project
                 )
 
-        return project_orm.dump()
+                # Need to commit() to get the timestamps(?)
+                await session.commit()
+                logger.info(f"orm creation_date = {project.creation_date}")
+                logger.info(f"orm updated_at = {project.updated_at}")
+                return project.dump()
 
-    async def update_project(self, user: base_models.APIUser, project_id: str, **payload) -> models.Project:
+    async def update_project(
+        self, user: base_models.APIUser, project_id: str, etag: str | None = None, **payload
+    ) -> models.Project:
         """Update a project entry."""
         authorized = await self.project_authz.has_permission(user=user, project_id=project_id, scope=Scope.WRITE)
         if not authorized:
@@ -128,20 +141,27 @@ async def update_project(self, user: base_models.APIUser, project_id: str, **pay
 
         async with self.session_maker() as session:
             async with session.begin():
-                result = await session.execute(select(schemas.ProjectORM).where(schemas.ProjectORM.id == project_id))
-                projects = result.one_or_none()
+                result = await session.scalars(select(schemas.ProjectORM).where(schemas.ProjectORM.id == project_id))
+                project = result.one_or_none()
 
-                if projects is None:
+                if project is None:
                     raise errors.MissingResourceError(message=f"The project with id '{project_id}' cannot be found")
 
-                project = projects[0]
+                current_etag = project.dump().etag
+                if etag is not None and current_etag != etag:
+                    raise errors.ConflictError(message=f"Current ETag is {current_etag}, not {etag}.")
+
                 visibility_before = project.visibility
 
                 if "repositories" in payload:
                     payload["repositories"] = [
                         schemas.ProjectRepositoryORM(url=r, project_id=project_id, project=project)
                         for r in payload["repositories"]
                     ]
+                    # Trigger update for ``updated_at`` column
+                    await session.execute(
+                        update(schemas.ProjectORM).where(schemas.ProjectORM.id == project_id).values()
+                    )
 
                 for key, value in payload.items():
                     # NOTE: ``slug``, ``id``, ``created_by``, and ``creation_date`` cannot be edited