✨ Adding rclount mount capability to dynamic-sidecar

.env-devel

@@ -244,6 +244,7 @@ RESOURCE_USAGE_TRACKER_TRACING={}
 R_CLONE_OPTION_BUFFER_SIZE=16M
 R_CLONE_OPTION_RETRIES=3
 R_CLONE_OPTION_TRANSFERS=5
+R_CLONE_MOUNT_SETTINGS={}
 R_CLONE_PROVIDER=MINIO
 
 # simcore-user used in docker images

.github/workflows/ci-testing-deploy.yml

@@ -1551,8 +1551,6 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
           cache-dependency-glob: "**/director-v2/requirements/ci.txt"
-      - name: setup rclone docker volume plugin
-        run: sudo ./ci/github/helpers/install_rclone_docker_volume_plugin.bash
       - name: Download and load Docker images
         uses: ./.github/actions/download-load-docker-images
         with:

packages/postgres-database/src/simcore_postgres_database/migration/versions/e4db35fe8054_added_use_r_clone_mounting_field.py

@@ -0,0 +1,36 @@
+"""added use_r_clone_mounting field
+
+Revision ID: e4db35fe8054
+Revises: ce69cc44246a
+Create Date: 2025-12-16 11:43:36.941571+00:00
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "e4db35fe8054"
+down_revision = "ce69cc44246a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "groups_extra_properties",
+        sa.Column(
+            "use_r_clone_mounting",
+            sa.Boolean(),
+            server_default=sa.text("false"),
+            nullable=False,
+        ),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("groups_extra_properties", "use_r_clone_mounting")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/groups_extra_properties.py

@@ -68,6 +68,13 @@
         server_default=sa.sql.expression.false(),
         doc="If true, will mount efs distributed file system when dynamic services starts",
     ),
+    sa.Column(
+        "use_r_clone_mounting",
+        sa.Boolean(),
+        nullable=False,
+        server_default=sa.sql.expression.false(),
+        doc="If true, will mount efs distributed file system when dynamic services starts",
+    ),
     sa.UniqueConstraint(
         "group_id", "product_name", name="group_id_product_name_uniqueness"
     ),

packages/postgres-database/src/simcore_postgres_database/utils_groups_extra_properties.py

@@ -40,6 +40,7 @@ class GroupExtraProperties(FromRowMixin):
     created: datetime.datetime
     modified: datetime.datetime
     enable_efs: bool
+    use_r_clone_mounting: bool
 
 
 def _list_table_entries_ordered_by_group_type_stmt(user_id: int, product_name: str):

packages/service-library/src/servicelib/r_clone_utils.py

@@ -0,0 +1,13 @@
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from aiofiles import tempfile
+
+
+@asynccontextmanager
+async def config_file(config: str) -> AsyncIterator[str]:
+    async with tempfile.NamedTemporaryFile("w") as f:
+        await f.write(config)
+        await f.flush()
+        assert isinstance(f.name, str)  # nosec
+        yield f.name

packages/service-library/tests/tests_r_clone_utils.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+
+from faker import Faker
+from servicelib.r_clone_utils import config_file
+
+
+async def test_config_file(faker: Faker) -> None:
+    text_to_write = faker.text()
+    async with config_file(text_to_write) as file_name:
+        assert text_to_write == Path(file_name).read_text()
+    assert Path(file_name).exists() is False

packages/settings-library/src/settings_library/r_clone.py

@@ -1,38 +1,268 @@
+from datetime import timedelta
 from enum import StrEnum
-from typing import Annotated
+from pathlib import Path
+from typing import Annotated, Final
 
-from pydantic import Field, NonNegativeInt
+from common_library.pydantic_validators import validate_numeric_string_as_timedelta
+from pydantic import ByteSize, Field, NonNegativeInt, TypeAdapter
 
 from .base import BaseCustomSettings
 from .s3 import S3Settings
 
+DEFAULT_VFS_CACHE_PATH: Final[Path] = Path("/vfs-cache")
+DEFAULT_VFS_CACHE_MAX_SIZE: Final[str] = "500G"
+
+
+_TRANSFER_COUNT: Final[NonNegativeInt] = 15
+_TPSLIMIT: Final[NonNegativeInt] = 2000
+
+_ONE_NANO_CPU: Final[NonNegativeInt] = int(1e9)
+
 
 class S3Provider(StrEnum):
     AWS = "AWS"
+    AWS_MOTO = "AWS_MOTO"
     CEPH = "CEPH"
     MINIO = "MINIO"
 
 
+class RCloneMountSettings(BaseCustomSettings):
+    R_CLONE_MOUNT_TRANSFERS_COMPLETED_TIMEOUT: Annotated[
+        timedelta,
+        Field(
+            description="max amount of time to wait for rclone mount command to finish",
+        ),
+    ] = timedelta(minutes=60)
+
+    _validate_r_clone_mount_transfers_completed_timeout = (
+        validate_numeric_string_as_timedelta(
+            "R_CLONE_MOUNT_TRANSFERS_COMPLETED_TIMEOUT"
+        )
+    )
+
+    # CONTAINER
+    R_CLONE_MOUNT_CONTAINER_CONFIG_FILE_PATH: Annotated[
+        Path,
+        Field(
+            description="path inside the container where the rclone config file is located",
+        ),
+    ] = Path(
+        "/tmp/rclone.conf"  # noqa: S108
+    )
+
+    R_CLONE_MOUNT_CONTAINER_SHOW_DEBUG_LOGS: Annotated[
+        bool,
+        Field(
+            description="whether to enable debug logs for the rclone mount command",
+        ),
+    ] = False
+
+    R_CLONE_MOUNT_CONTAINER_MEMORY_LIMIT: Annotated[
+        ByteSize, Field(description="memory limit for the rclone mount container")
+    ] = TypeAdapter(ByteSize).validate_python("2GiB")
+
+    R_CLONE_MOUNT_CONTAINER_NANO_CPUS: Annotated[
+        NonNegativeInt, Field(description="CPU limit for the rclone mount container")
+    ] = (1 * _ONE_NANO_CPU)
+
+    # CLI command `rclone mount`
+
+    R_CLONE_MOUNT_VFS_CACHE_PATH: Annotated[
+        Path,
+        Field(
+            description="`--cache-dir X`: sets the path to use for vfs cache",
+        ),
+    ] = DEFAULT_VFS_CACHE_PATH
+
+    R_CLONE_MOUNT_VFS_READ_AHEAD: Annotated[
+        str,
+        Field(
+            description="`--vfs-read-ahead X`: sets the read ahead buffer size",
+        ),
+    ] = "16M"
+
+    R_CLONE_MOUNT_VFS_CACHE_MAX_SIZE: Annotated[
+        str,
+        Field(
+            description="`--vfs-cache-max-size X`: sets the maximum size of the vfs cache",
+        ),
+    ] = DEFAULT_VFS_CACHE_MAX_SIZE
+
+    R_CLONE_MOUNT_VFS_CACHE_MIN_FREE_SPACE: Annotated[
+        str,
+        Field(
+            description="`--vfs-cache-min-free-space X`: sets the minimum free space to keep on disk",
+        ),
+    ] = "5G"
+
+    R_CLONE_MOUNT_CACHE_POLL_INTERVAL: Annotated[
+        str,
+        Field(
+            description="`--vfs-cache-poll-interval X`: sets the interval to poll the vfs cache",
+        ),
+    ] = "1m"
+
+    R_CLONE_MOUNT_VFS_WRITE_BACK: Annotated[
+        str,
+        Field(
+            description="`--vfs-write-back X`: sets the time to wait before writing back data to the remote",
+        ),
+    ] = "10s"
+
+    R_CLONE_MOUNT_DIR_CACHE_TIME: Annotated[
+        str,
+        Field(
+            description="`--dir-cache-time X`: time before directory is uploaded from remote if changed",
+        ),
+    ] = "10m"
+
+    R_CLONE_MOUNT_ATTR_TIMEOUT: Annotated[
+        str,
+        Field(
+            description="`--attr-timeout X`: sets the time to cache file attributes",
+        ),
+    ] = "1m"
+
+    R_CLONE_MOUNT_TPSLIMIT: Annotated[
+        NonNegativeInt,
+        Field(
+            description="`--tpslimit X`: sets the transactions per second limit",
+        ),
+    ] = _TPSLIMIT
+    R_CLONE_MOUNT_TPSLIMIT_BURST: Annotated[
+        NonNegativeInt,
+        Field(
+            description="`--tpslimit-burst X`: sets the burst limit for transactions per second",
+        ),
+    ] = (
+        _TPSLIMIT * 2
+    )
+
+    R_CLONE_MOUNT_MAX_BUFFER_MEMORY: Annotated[
+        str,
+        Field(
+            description="`--max-buffer-memory X`: sets the maximum buffer memory for rclone",
+        ),
+    ] = "16M"
+
+    R_CLONE_MOUNT_RETRIES: Annotated[
+        NonNegativeInt,
+        Field(
+            description="`--retries X`: sets the number of retries for rclone mount command",
+        ),
+    ] = 3
+
+    R_CLONE_MOUNT_RETRIES_SLEEP: Annotated[
+        str,
+        Field(
+            description="`--retries-sleep X`: sets the maximum sleep time between retries",
+        ),
+    ] = "30s"
+    R_CLONE_MOUNT_TRANSFERS: Annotated[
+        NonNegativeInt,
+        Field(
+            description="`--transfers X`: sets the number of parallel transfers for rclone mount command",
+        ),
+    ] = 15
+    R_CLONE_MOUNT_BUFFER_SIZE: Annotated[
+        str,
+        Field(
+            description="`--buffer-size X`: sets the buffer size for rclone mount command",
+        ),
+    ] = "16M"
+    R_CLONE_MOUNT_CHECKERS: Annotated[
+        NonNegativeInt,
+        Field(
+            description="`--checkers X`: sets the number of checkers for rclone mount command",
+        ),
+    ] = 8
+    R_CLONE_MOUNT_S3_UPLOAD_CONCURRENCY: Annotated[
+        NonNegativeInt,
+        Field(
+            description="`--s3-upload-concurrency X`: sets the number of concurrent uploads to S3",
+        ),
+    ] = 5
+    R_CLONE_MOUNT_S3_CHUNK_SIZE: Annotated[
+        str,
+        Field(
+            description="`--s3-chunk-size X`: sets the chunk size for S3",
+        ),
+    ] = "16M"
+    R_CLONE_MOUNT_ORDER_BY: Annotated[
+        str,
+        Field(
+            description="`--order-by X`: sets the order of file upload, e.g., 'size,mixed'",
+        ),
+    ] = "size,mixed"
+
+
 class RCloneSettings(BaseCustomSettings):
     R_CLONE_S3: Annotated[
         S3Settings, Field(json_schema_extra={"auto_default_from_env": True})
     ]
     R_CLONE_PROVIDER: S3Provider
 
+    R_CLONE_VERSION: Annotated[
+        str | None,
+        Field(
+            pattern=r"^\d+\.\d+\.\d+$",
+            description="version of rclone for the container image",
+        ),
+    ] = None
+
+    R_CLONE_MOUNT_SETTINGS: RCloneMountSettings = Field(
+        json_schema_extra={"auto_default_from_env": True}
+    )
+
     R_CLONE_OPTION_TRANSFERS: Annotated[
         # SEE https://rclone.org/docs/#transfers-n
         NonNegativeInt,
         Field(description="`--transfers X`: sets the amount of parallel transfers"),
     ] = 5
+
     R_CLONE_OPTION_RETRIES: Annotated[
         # SEE https://rclone.org/docs/#retries-int
         NonNegativeInt,
         Field(description="`--retries X`: times to retry each individual transfer"),
     ] = 3
+
+    R_CLONE_RETRIES_SLEEP: Annotated[
+        str,
+        Field(
+            description="`--retries-sleep X`: max time to sleep between retries (caps exponential backoff)"
+        ),
+    ] = "30s"
+
     R_CLONE_OPTION_BUFFER_SIZE: Annotated[
         # SEE https://rclone.org/docs/#buffer-size-size
         str,
         Field(
             description="`--buffer-size X`: sets the amount of RAM to use for each individual transfer",
         ),
     ] = "16M"
+
+    R_CLONE_CHECKERS: Annotated[
+        NonNegativeInt,
+        Field(
+            description="`--checkers X`: sets the number checkers",
+        ),
+    ] = 8
+
+    R_CLONE_S3_UPLOAD_CONCURRENCY: Annotated[
+        NonNegativeInt,
+        Field(
+            description="`--s3-upload-concurrency X`: sets the number of concurrent uploads to S3",
+        ),
+    ] = 5
+
+    R_CLONE_CHUNK_SIZE: Annotated[
+        str,
+        Field(description="`--s3-chunk-size X`: sets the chunk size for S3"),
+    ] = "16M"
+
+    R_CLONE_ORDER_BY: Annotated[
+        str,
+        Field(
+            description="`--order-by X`: sets the order of file upload, e.g., 'size,mixed'",
+        ),
+    ] = "size,mixed"