Adding version control to samples, starting materials and cells

pydatalab/pyproject.toml

@@ -27,6 +27,7 @@ dependencies = [
     "pint ~= 0.24",
     "pandas[excel] ~= 2.2",
     "pymongo ~= 4.7",
+    "deepdiff ~= 8.1",
 ]
 
 [project.urls]

pydatalab/schemas/cell.json

@@ -36,6 +36,11 @@
       "title": "Revisions",
       "type": "object"
     },
+    "version": {
+      "title": "Version",
+      "default": 1,
+      "type": "integer"
+    },
     "creator_ids": {
       "title": "Creator Ids",
       "default": [],
@@ -622,6 +627,11 @@
           "title": "Revisions",
           "type": "object"
         },
+        "version": {
+          "title": "Version",
+          "default": 1,
+          "type": "integer"
+        },
         "creator_ids": {
           "title": "Creator Ids",
           "default": [],

pydatalab/schemas/equipment.json

@@ -36,6 +36,11 @@
       "title": "Revisions",
       "type": "object"
     },
+    "version": {
+      "title": "Version",
+      "default": 1,
+      "type": "integer"
+    },
     "creator_ids": {
       "title": "Creator Ids",
       "default": [],
@@ -586,6 +591,11 @@
           "title": "Revisions",
           "type": "object"
         },
+        "version": {
+          "title": "Version",
+          "default": 1,
+          "type": "integer"
+        },
         "creator_ids": {
           "title": "Creator Ids",
           "default": [],

pydatalab/schemas/sample.json

@@ -48,6 +48,11 @@
       "title": "Revisions",
       "type": "object"
     },
+    "version": {
+      "title": "Version",
+      "default": 1,
+      "type": "integer"
+    },
     "creator_ids": {
       "title": "Creator Ids",
       "default": [],
@@ -675,6 +680,11 @@
           "title": "Revisions",
           "type": "object"
         },
+        "version": {
+          "title": "Version",
+          "default": 1,
+          "type": "integer"
+        },
         "creator_ids": {
           "title": "Creator Ids",
           "default": [],

pydatalab/schemas/startingmaterial.json

@@ -48,6 +48,11 @@
       "title": "Revisions",
       "type": "object"
     },
+    "version": {
+      "title": "Version",
+      "default": 1,
+      "type": "integer"
+    },
     "creator_ids": {
       "title": "Creator Ids",
       "default": [],
@@ -728,6 +733,11 @@
           "title": "Revisions",
           "type": "object"
         },
+        "version": {
+          "title": "Version",
+          "default": 1,
+          "type": "integer"
+        },
         "creator_ids": {
           "title": "Creator Ids",
           "default": [],

pydatalab/src/pydatalab/models/__init__.py

@@ -7,6 +7,13 @@
 from pydatalab.models.people import Person
 from pydatalab.models.samples import Sample
 from pydatalab.models.starting_materials import StartingMaterial
+from pydatalab.models.versions import (
+    CompareVersionsQuery,
+    ItemVersion,
+    RestoreVersionRequest,
+    VersionAction,
+    VersionCounter,
+)
 
 ITEM_MODELS: dict[str, type[BaseModel]] = {
     "samples": Sample,
@@ -24,4 +31,10 @@
     "Collection",
     "Equipment",
     "ITEM_MODELS",
+    "ItemVersion",
+    "VersionCounter",
+    "UserSnapshot",
+    "VersionAction",
+    "RestoreVersionRequest",
+    "CompareVersionsQuery",
 )

pydatalab/src/pydatalab/models/traits.py

@@ -22,6 +22,9 @@ class HasRevisionControl(BaseModel):
     revisions: dict[int, Any] | None = None
     """An optional mapping from old revision numbers to the model state at that revision."""
 
+    version: int = 1
+    """The version number used by the version control system for tracking snapshots."""
+
 
 class HasBlocks(BaseModel):
     blocks_obj: dict[str, DataBlockResponse] = Field({})

pydatalab/src/pydatalab/models/versions.py

@@ -0,0 +1,117 @@
+"""Pydantic models for version control system."""
+
+from datetime import datetime
+from enum import Enum
+
+from pydantic import BaseModel, Field, validator
+
+from pydatalab.models.utils import PyObjectId, Refcode
+
+
+class VersionAction(str, Enum):
+    """Valid actions that can create a version snapshot."""
+
+    CREATED = "created"
+    MANUAL_SAVE = "manual_save"
+    AUTO_SAVE = "auto_save"
+    RESTORED = "restored"
+
+
+class ItemVersion(BaseModel):
+    """A complete snapshot of an item at a specific point in time.
+
+    This model represents a version entry in the `item_versions` collection.
+    Each version captures the complete state of an item, allowing users to
+    view history and restore previous states.
+    """
+
+    refcode: Refcode = Field(..., description="The refcode of the item this version belongs to")
+    version: int = Field(..., ge=1, description="Sequential version number (1-indexed)")
+    timestamp: datetime = Field(
+        ..., description="When this version was created (ISO format with timezone)"
+    )
+    action: VersionAction = Field(
+        ...,
+        description="The action that triggered this version: 'created' (item creation), "
+        "'manual_save' (user save), 'auto_save' (system save), or 'restored' (version restore)",
+    )
+    user_id: PyObjectId | None = Field(
+        None, description="User's ObjectId for efficient querying and indexing"
+    )
+    datalab_version: str = Field(
+        ..., description="Version of datalab-server that created this snapshot"
+    )
+    data: dict = Field(..., description="Complete snapshot of the item data at this version")
+    restored_from_version: PyObjectId | None = Field(
+        None,
+        description="ObjectId of the version that was restored from (only present if action='restored')",
+    )
+
+    @validator("restored_from_version")
+    def validate_restored_from_version(cls, v, values):
+        """Ensure restored_from_version is only present when action='restored'."""
+        action = values.get("action")
+        if action == VersionAction.RESTORED and v is None:
+            raise ValueError("restored_from_version must be provided when action='restored'")
+        if action != VersionAction.RESTORED and v is not None:
+            raise ValueError(
+                f"restored_from_version should only be present when action='restored', got action='{action}'"
+            )
+        return v
+
+
+class VersionCounter(BaseModel):
+    """Atomic counter for tracking version numbers per item.
+
+    This model represents a document in the `version_counters` collection.
+    It ensures atomic increment of version numbers to prevent race conditions.
+    """
+
+    refcode: Refcode = Field(..., description="The refcode this counter belongs to")
+    counter: int = Field(
+        1, ge=1, description="Current version counter value (1-indexed, matches version numbers)"
+    )
+
+    class Config:
+        extra = "ignore"  # Allow MongoDB's _id field and other internal fields
+
+
+class RestoreVersionRequest(BaseModel):
+    """Request body for restoring a version."""
+
+    version_id: str = Field(..., description="ObjectId string of the version to restore to")
+
+    @validator("version_id")
+    def validate_version_id_format(cls, v):
+        """Validate that version_id is a valid ObjectId string."""
+        try:
+            from bson import ObjectId
+
+            ObjectId(v)
+        except Exception as e:
+            raise ValueError(f"version_id must be a valid ObjectId string: {e}")
+        return v
+
+    class Config:
+        extra = "forbid"
+
+
+class CompareVersionsQuery(BaseModel):
+    """Query parameters for comparing two versions."""
+
+    v1: str = Field(..., description="ObjectId string of the first version")
+    v2: str = Field(..., description="ObjectId string of the second version")
+
+    @validator("v1", "v2")
+    def validate_version_ids(cls, v):
+        """Validate that version IDs are valid ObjectId strings."""
+        try:
+            from bson import ObjectId
+
+            ObjectId(v)
+        except Exception as e:
+            raise ValueError(f"Version ID must be a valid ObjectId string: {e}")
+        return v
+
+    class Config:
+        extra = "forbid"

pydatalab/src/pydatalab/mongo.py

@@ -116,6 +116,11 @@ def create_default_indices(
         - An index over item type,
         - A unique index over `item_id` and `refcode`.
         - A text index over user names and identities.
+        - Version control indexes:
+            - Index on item_versions.refcode for fast version history lookup
+            - Index on item_versions.user_id for fast user contribution queries
+            - Compound index on (refcode, version) for sorted version history
+            - Unique index on version_counters.refcode for atomic version numbering
 
     Parameters:
         background: If true, indexes will be created as background jobs.
@@ -238,4 +243,16 @@ def create_group_fts():
         db.users.drop_index(group_fts_name)
         ret += create_group_fts()
 
+    # Version control indexes
+    ret += db.item_versions.create_index("refcode", name="version refcode", background=background)
+    ret += db.item_versions.create_index("user_id", name="version user_id", background=background)
+    ret += db.item_versions.create_index(
+        [("refcode", pymongo.ASCENDING), ("version", pymongo.DESCENDING)],
+        name="refcode and version",
+        background=background,
+    )
+    ret += db.version_counters.create_index(
+        "refcode", unique=True, name="unique refcode counter", background=background
+    )
+
     return ret