From 9c22aa6919ad18b3afe0d10f285336357ac4bd3e Mon Sep 17 00:00:00 2001 From: Benjamin Milan Date: Wed, 15 Jul 2026 10:54:57 +0100 Subject: [PATCH 1/5] BWDO-809 Implement clone cache lock --- setup.py | 1 + src/sc/clone/cloners/repo_cloner.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7d0526d..8693e81 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ def read_version(): 'pyyaml~=6.0', 'rich>=14', 'requests==2.31.0', # Docker SDK breaks on 2.32.0 + 'filelock==3.29.7', 'repo_library @ git+https://github.com/rdkcentral/sc-repo-library.git@master', 'git_flow_library @ git+https://github.com/rdkcentral/sc-git-flow-library.git@master', 'sc_manifest_parser @ git+https://github.com/rdkcentral/sc-manifest-parser.git@main' diff --git a/src/sc/clone/cloners/repo_cloner.py b/src/sc/clone/cloners/repo_cloner.py index f531212..c4e1089 100644 --- a/src/sc/clone/cloners/repo_cloner.py +++ b/src/sc/clone/cloners/repo_cloner.py @@ -19,6 +19,7 @@ import subprocess import sys +from filelock import FileLock, Timeout from pydantic import BaseModel from .cloner import Cloner, RefType @@ -29,6 +30,7 @@ logger = logging.getLogger(__name__) REPO_CACHE_DIR = Path.home() / ".caches" +LOCK_FILE_PATH = REPO_CACHE_DIR / ".lock" class RepoClonerConfig(BaseModel): """ @@ -74,8 +76,19 @@ def clone(self, directory: Path): - Parses the manifest to retrieve projects. - Initializes GitFlow for all unlocked projects. """ - reference = self._cache() if self.config.cache else None + if self.config.cache: + REPO_CACHE_DIR.mkdir(exist_ok=True) + try: + with FileLock(LOCK_FILE_PATH, timeout=600): + reference = self._cache() + self._clone(directory, reference) + except Timeout: + logger.error(f"The cache lock file {LOCK_FILE_PATH} remained 10 minutes.") + sys.exit(1) + else: + self._clone(directory, None) + def _clone(self, directory: Path, reference: Path | None): self._init_repo(directory=directory, reference=reference) RepoLibrary.sync( directory, @@ -95,7 +108,6 @@ def _cache(self) -> Path: Returns: Path: The directory of the mirrored cache. """ - REPO_CACHE_DIR.mkdir(exist_ok=True) manifest_hostname = self._get_manifest_hostname(self.config.uri) host_cache_dir = Path(REPO_CACHE_DIR / manifest_hostname) host_cache_dir.mkdir(exist_ok=True) From 5272667a2cc8d77e4a1b438425ee2758381040d5 Mon Sep 17 00:00:00 2001 From: Benjamin Milan Date: Wed, 15 Jul 2026 11:46:37 +0100 Subject: [PATCH 2/5] BWDO-809 Implement cache lock --- src/sc/clone/cloners/repo_cloner.py | 43 ++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/sc/clone/cloners/repo_cloner.py b/src/sc/clone/cloners/repo_cloner.py index c4e1089..f92c4b6 100644 --- a/src/sc/clone/cloners/repo_cloner.py +++ b/src/sc/clone/cloners/repo_cloner.py @@ -18,6 +18,7 @@ import shutil import subprocess import sys +import time from filelock import FileLock, Timeout from pydantic import BaseModel @@ -30,7 +31,8 @@ logger = logging.getLogger(__name__) REPO_CACHE_DIR = Path.home() / ".caches" -LOCK_FILE_PATH = REPO_CACHE_DIR / ".lock" +CACHE_LOCK_PATH = REPO_CACHE_DIR / ".lock" +CACHE_MAX_WAIT = 600 class RepoClonerConfig(BaseModel): """ @@ -77,18 +79,39 @@ def clone(self, directory: Path): - Initializes GitFlow for all unlocked projects. """ if self.config.cache: - REPO_CACHE_DIR.mkdir(exist_ok=True) + self._clone_with_cache(directory) + else: + self._clone(directory) + + def _clone_with_cache(self, directory: Path): + REPO_CACHE_DIR.mkdir(exist_ok=True) + + started = time.monotonic() + lock = FileLock(CACHE_LOCK_PATH) + + while True: try: - with FileLock(LOCK_FILE_PATH, timeout=600): - reference = self._cache() - self._clone(directory, reference) + lock.acquire(timeout=5) + break except Timeout: - logger.error(f"The cache lock file {LOCK_FILE_PATH} remained 10 minutes.") - sys.exit(1) - else: - self._clone(directory, None) + waited = int(time.monotonic() - started) + if waited < CACHE_MAX_WAIT: + logger.info( + f"Cache is in use by another process, waited {waited} seconds..." + ) + else: + logger.error( + f"Cache remained lock past the set wait time of {CACHE_MAX_WAIT} seconds." + ) + sys.exit(1) + + try: + reference = self._cache() + self._clone(directory, reference) + finally: + lock.release() - def _clone(self, directory: Path, reference: Path | None): + def _clone(self, directory: Path, reference: Path | None = None): self._init_repo(directory=directory, reference=reference) RepoLibrary.sync( directory, From aab5d4bb5702b57f84646bd9baffd9583ee6b13a Mon Sep 17 00:00:00 2001 From: Benjamin Milan Date: Wed, 15 Jul 2026 13:18:09 +0100 Subject: [PATCH 3/5] BWDO-809 increased timeout to 20s --- src/sc/clone/cloners/repo_cloner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sc/clone/cloners/repo_cloner.py b/src/sc/clone/cloners/repo_cloner.py index f92c4b6..e3577bc 100644 --- a/src/sc/clone/cloners/repo_cloner.py +++ b/src/sc/clone/cloners/repo_cloner.py @@ -89,9 +89,10 @@ def _clone_with_cache(self, directory: Path): started = time.monotonic() lock = FileLock(CACHE_LOCK_PATH) + logger.info("Acquiring cache lock.") while True: try: - lock.acquire(timeout=5) + lock.acquire(timeout=20) break except Timeout: waited = int(time.monotonic() - started) @@ -101,9 +102,10 @@ def _clone_with_cache(self, directory: Path): ) else: logger.error( - f"Cache remained lock past the set wait time of {CACHE_MAX_WAIT} seconds." + f"Cache remained locked past the set wait time of {CACHE_MAX_WAIT} seconds." ) sys.exit(1) + logger.info("Cache lock acquired.") try: reference = self._cache() From 246b7bc8bc9ae6c13ec007d11ccd650566cb4546 Mon Sep 17 00:00:00 2001 From: Benjamin Milan Date: Mon, 3 Aug 2026 15:35:03 +0100 Subject: [PATCH 4/5] BWDO-809 - Allow bypassing clone cache lock after 5 minutes --- src/sc/clone/cloners/repo_cloner.py | 62 +++++++++++++++++++---------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/src/sc/clone/cloners/repo_cloner.py b/src/sc/clone/cloners/repo_cloner.py index e3577bc..81225e2 100644 --- a/src/sc/clone/cloners/repo_cloner.py +++ b/src/sc/clone/cloners/repo_cloner.py @@ -31,8 +31,8 @@ logger = logging.getLogger(__name__) REPO_CACHE_DIR = Path.home() / ".caches" -CACHE_LOCK_PATH = REPO_CACHE_DIR / ".lock" -CACHE_MAX_WAIT = 600 +CACHE_LOCK_PATH = REPO_CACHE_DIR / ".sc_lock" +CACHE_MAX_WAIT = 300 class RepoClonerConfig(BaseModel): """ @@ -86,32 +86,15 @@ def clone(self, directory: Path): def _clone_with_cache(self, directory: Path): REPO_CACHE_DIR.mkdir(exist_ok=True) - started = time.monotonic() lock = FileLock(CACHE_LOCK_PATH) - - logger.info("Acquiring cache lock.") - while True: - try: - lock.acquire(timeout=20) - break - except Timeout: - waited = int(time.monotonic() - started) - if waited < CACHE_MAX_WAIT: - logger.info( - f"Cache is in use by another process, waited {waited} seconds..." - ) - else: - logger.error( - f"Cache remained locked past the set wait time of {CACHE_MAX_WAIT} seconds." - ) - sys.exit(1) - logger.info("Cache lock acquired.") + lock_acquired = self._acquire_cache_lock(lock) try: reference = self._cache() self._clone(directory, reference) finally: - lock.release() + if lock_acquired: + lock.release() def _clone(self, directory: Path, reference: Path | None = None): self._init_repo(directory=directory, reference=reference) @@ -176,6 +159,41 @@ def _init_repo(self, directory: Path, mirror: bool = False, reference: Path | No logger.error(f"repo init error: {e}") sys.exit(1) + def _acquire_cache_lock(self, lock: FileLock) -> bool: + """Try to acquire cache lock. After a certain time bypass it. + + Return True if lock acquired or False if bypassed. + """ + started = time.monotonic() + first_warning = False + + logger.info("Acquiring cache lock.") + while True: + try: + lock.acquire(timeout=20) + logger.info("Cache lock acquired.") + return True + + except Timeout: + waited = int(time.monotonic() - started) + if waited < CACHE_MAX_WAIT: + logger.info( + f"Cache is in use by another process, waited {waited} seconds..." + ) + + if first_warning == False: + logger.info(f"SC will wait for {CACHE_MAX_WAIT} seconds before bypassing.") + logger.info( + "The cache is user specific, if you believe there is no way your cache " + f"should be in use you could try deleting the lock: {CACHE_LOCK_PATH}") + first_warning = True + else: + logger.warning( + f"Cache remained locked past the set wait time of {CACHE_MAX_WAIT} seconds. " + "Force proceeding without acquiring cache lock..." + ) + return False + def _get_manifest_hostname(self, url: str) -> str: """Extracts the hostname from a given URL. From 162c8d8e264f121495b18d2e6fbe52571bf6cf86 Mon Sep 17 00:00:00 2001 From: Benjamin Milan Date: Mon, 3 Aug 2026 16:34:27 +0100 Subject: [PATCH 5/5] BWDO-809 switch to SoftLockFile for cache locking --- src/sc/clone/cloners/repo_cloner.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/sc/clone/cloners/repo_cloner.py b/src/sc/clone/cloners/repo_cloner.py index 81225e2..5f85416 100644 --- a/src/sc/clone/cloners/repo_cloner.py +++ b/src/sc/clone/cloners/repo_cloner.py @@ -20,7 +20,7 @@ import sys import time -from filelock import FileLock, Timeout +from filelock import SoftFileLock, Timeout from pydantic import BaseModel from .cloner import Cloner, RefType @@ -86,15 +86,14 @@ def clone(self, directory: Path): def _clone_with_cache(self, directory: Path): REPO_CACHE_DIR.mkdir(exist_ok=True) - lock = FileLock(CACHE_LOCK_PATH) - lock_acquired = self._acquire_cache_lock(lock) + lock = SoftFileLock(CACHE_LOCK_PATH) + self._acquire_cache_lock(lock) try: reference = self._cache() self._clone(directory, reference) finally: - if lock_acquired: - lock.release() + lock.release() def _clone(self, directory: Path, reference: Path | None = None): self._init_repo(directory=directory, reference=reference) @@ -159,11 +158,8 @@ def _init_repo(self, directory: Path, mirror: bool = False, reference: Path | No logger.error(f"repo init error: {e}") sys.exit(1) - def _acquire_cache_lock(self, lock: FileLock) -> bool: - """Try to acquire cache lock. After a certain time bypass it. - - Return True if lock acquired or False if bypassed. - """ + def _acquire_cache_lock(self, lock: SoftFileLock): + """Try to acquire cache lock. After a certain time bypass it.""" started = time.monotonic() first_warning = False @@ -172,7 +168,7 @@ def _acquire_cache_lock(self, lock: FileLock) -> bool: try: lock.acquire(timeout=20) logger.info("Cache lock acquired.") - return True + return except Timeout: waited = int(time.monotonic() - started) @@ -192,7 +188,9 @@ def _acquire_cache_lock(self, lock: FileLock) -> bool: f"Cache remained locked past the set wait time of {CACHE_MAX_WAIT} seconds. " "Force proceeding without acquiring cache lock..." ) - return False + lock.break_lock() + lock.acquire() + return def _get_manifest_hostname(self, url: str) -> str: """Extracts the hostname from a given URL.