From a0e28b86ded24361057e5bdbbaf2bab2f859150b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rnar=20Snoksrud?= Date: Wed, 3 Jun 2026 15:06:40 +0200 Subject: [PATCH] Add `novem --load` tree sync with `--dry-run` `--load` previously walked a dumped folder and blindly PUT+POSTed every file, leaving stale remote files in place, rewriting unchanged ones, and reporting success regardless of the server response. Make it a real sync that treats the local folder as the desired state of the resource: * create files that don't exist remotely * overwrite only files whose content differs (single-line values log the old -> new change instead of a byte count) * delete remote files no longer present locally * leave unchanged files untouched `--dry-run` prints the planned actions without sending any request. Only real file_content-backed files are synced. The directory listing flattens every leaf to type="file", so selection keys on the DELETE verb in each node's `actions`: read-only files (no DELETE) and virtual/computed files like `notifications` (POST but no DELETE) are excluded. Virtual paths that exist remotely are tracked so a stale local copy is ignored rather than re-created every run. Shares (`/shared/`) round-trip as links created via PUT (POST 405s on them); tags (`/tags/`) are excluded entirely as they're managed via the -t flag. Each create/overwrite/delete is gated on its POST/DELETE status: failures print a FAILED line and are tallied under a `N failed` summary suffix instead of being counted as done. The dump/load logic is shared by the vis and job APIs, so it lives in a NovemTreeSync mixin in novem/sync.py; each class supplies a base-path and a label hook. Covered by tests/test_load_sync.py. --- .gitignore | 3 + novem/cli/common.py | 4 +- novem/cli/setup.py | 9 ++ novem/job/__init__.py | 118 +--------------- novem/sync.py | 274 ++++++++++++++++++++++++++++++++++++++ novem/vis/__init__.py | 124 ++--------------- tests/test_load_sync.py | 288 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 591 insertions(+), 229 deletions(-) create mode 100644 novem/sync.py create mode 100644 tests/test_load_sync.py diff --git a/.gitignore b/.gitignore index 5d41342..1fee0a9 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,6 @@ CLAUDE.md # Direnv .envrc .direnv + +# Local scratch / experiment dirs +scratch/ diff --git a/novem/cli/common.py b/novem/cli/common.py index 3c8ef72..f0f25f0 100644 --- a/novem/cli/common.py +++ b/novem/cli/common.py @@ -155,7 +155,7 @@ def __call__(self, args: Dict[str, Any]) -> None: path = args["load"] print(f'Loading api tree structure from "{path}"') - vis.api_load(inpath=path) + vis.api_load(inpath=path, dry_run=args.get("dry_run", False)) return # if we detect a tree query then we'll discard all other IO @@ -405,7 +405,7 @@ def job(args: Dict[str, Any]) -> None: if "load" in args and args["load"]: path = args["load"] print(f'Loading api tree structure from "{path}"') - j.api_load(inpath=path) + j.api_load(inpath=path, dry_run=args.get("dry_run", False)) return # --tree: print API tree structure diff --git a/novem/cli/setup.py b/novem/cli/setup.py index fbd5866..34ad00c 100644 --- a/novem/cli/setup.py +++ b/novem/cli/setup.py @@ -113,6 +113,15 @@ def setup(raw_args: Any = None) -> Tuple[Any, Dict[str, str]]: help=ap.SUPPRESS, ) + parser.add_argument( + "--dry-run", + dest="dry_run", + action="store_true", + required=False, + default=False, + help=ap.SUPPRESS, + ) + parser.add_argument( "--version", dest="version", diff --git a/novem/job/__init__.py b/novem/job/__init__.py index cb6828a..6a59b18 100644 --- a/novem/job/__init__.py +++ b/novem/job/__init__.py @@ -7,6 +7,7 @@ from ..api_ref import NovemAPI from ..shared import NovemShare +from ..sync import NovemTreeSync from ..tags import NovemTags from ..utils import cl from ..utils import colors as clrs @@ -17,7 +18,7 @@ """ -class NovemJobAPI(NovemAPI): +class NovemJobAPI(NovemTreeSync, NovemAPI): config: Optional[NovemJobConfig] shared: Optional[NovemShare] tags: Optional[NovemTags] @@ -453,117 +454,12 @@ def run( if self._debug: print(" files out: 0") - def api_dump(self, outpath: str) -> None: - """ - Iterate over current job and dump output to supplied path - """ - - # Base path without trailing slash - qpath = self._path() - - # create util function - def rec_tree(path: str) -> None: - qp = f"{qpath}{path}" - fp = f"{outpath}{path}" - req = self._session.get(qp) - - if not req.ok: - return None - - headers = req.headers - tp = headers.get("X-NVM-Type", headers.get("X-NS-Type", "file")) - - # if i am a file, write to disc - if tp == "file": - # skip files with default values - if headers.get("x-nvm-default", "").lower() == "true": - print(f"Skipping default: {fp}") - return None - # ensure parent directory exists before writing - parent_dir = os.path.dirname(fp) - if parent_dir and not os.path.exists(parent_dir): - os.makedirs(parent_dir, exist_ok=True) - print(f"Creating folder: {parent_dir}") - with open(fp, "w") as f: - f.write(req.text) - print(f"Writing file: {fp}") - return None - - # if I am a folder, recurse without creating yet - nodes: List[Dict[str, str]] = req.json() - - # Recurse relevant structure (skip system entries and read-only files) - for r in nodes: - if r["type"] in ["system_file", "system_dir"]: - continue - child_path = f'{path}/{r["name"]}' - child_fp = f"{outpath}{child_path}" - - # /shared/ and /tags/ are special markers - create empty files from listing - if r["type"] in ["file", "link"] and ( - child_path.startswith("/shared/") or child_path.startswith("/tags/") - ): - parent_dir = os.path.dirname(child_fp) - if parent_dir and not os.path.exists(parent_dir): - os.makedirs(parent_dir, exist_ok=True) - print(f"Creating folder: {parent_dir}") - with open(child_fp, "w") as f: - f.write("") - print(f"Writing file: {child_fp}") - continue - - # skip read-only files/links - if r["type"] in ["file", "link"] and "w" not in r.get("permissions", []): - continue - rec_tree(child_path) - - # start recursion - rec_tree("/") - - def api_load(self, inpath: str) -> None: - """ - Load a dumped folder structure back into the API. - Walks the folder and for each file: PUT to create, then POST content. - """ - if self.user: - print("You cannot modify another user's job") - return - - qpath = self._path() - - def load_tree(local_path: str, api_path: str) -> None: - full_local = os.path.join(inpath, local_path.lstrip("/")) if local_path else inpath - - if os.path.isfile(full_local): - # Read file content - with open(full_local, "r") as f: - content = f.read() - - full_api = f"{qpath}{api_path}" - - # Try PUT first to create the resource - r = self._session.put(full_api) - put_status = r.status_code - - # POST the content - r = self._session.post( - full_api, - headers={"Content-type": "text/plain"}, - data=content.encode("utf-8"), - ) - print(f"Loaded file: {api_path} (PUT: {put_status}, POST: {r.status_code}, {len(content)} bytes)") - - elif os.path.isdir(full_local): - print(f"Processing dir: {api_path or '/'}") - - # Iterate over directory contents - for entry in sorted(os.listdir(full_local)): - entry_local = os.path.join(local_path, entry) if local_path else entry - entry_api = f"{api_path}/{entry}" - load_tree(entry_local, entry_api) + def _sync_base(self, user_aware: bool) -> str: + # _path() is already user-aware + return self._path() - # Start loading from root - load_tree("", "") + def _sync_label(self) -> str: + return "job" def api_tree(self, colors: bool = False, relpath: str = "/") -> str: """ diff --git a/novem/sync.py b/novem/sync.py new file mode 100644 index 0000000..482aaf3 --- /dev/null +++ b/novem/sync.py @@ -0,0 +1,274 @@ +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple + +import requests + + +class NovemTreeSync: + """ + Shared `--dump` / `--load` tree-sync logic for the resource APIs. + + A resource exposes its state as a tree of files under an HTTP path. Dump + walks that tree to disk; load treats a dumped folder as the desired state + and syncs the remote tree to match it (create / overwrite / delete). + + Concrete classes mix this in and provide two hooks plus the usual + ``self._session`` / ``self.user`` attributes: + + * ``_sync_base(user_aware)`` - the resource's base URL WITHOUT a trailing + slash (e.g. ``.../vis/plots/myplot``). When ``user_aware`` is true and + the resource belongs to another user, return that user-scoped path so + dump can read it. + * ``_sync_label()`` - a noun for the "cannot modify another user's X" + message (e.g. ``"plots"`` / ``"job"``). + + Only real, round-trippable files are synced. The directory listing reports + every leaf as ``type="file"``, so selection keys on the DELETE verb in each + node's ``actions``: read-only files (no DELETE) and virtual/computed files + like ``notifications`` (POST but no DELETE) are excluded. Shares + (``/shared/``) round-trip as links created via PUT; tags (``/tags/``) are + excluded entirely (managed via the -t CLI flag). + """ + + # supplied by the concrete NovemAPI subclass + _session: requests.Session + user: Optional[str] + + def _sync_base(self, user_aware: bool) -> str: # pragma: no cover - hook + raise NotImplementedError + + def _sync_label(self) -> str: # pragma: no cover - hook + raise NotImplementedError + + def api_dump(self, outpath: str) -> None: + """ + Walk the remote tree and write every round-trippable file to disk. + """ + read_root = f"{self._sync_base(user_aware=True)}/" + out = Path(outpath) + + def write_file(api_path: str, content: str) -> None: + fp = out / api_path.lstrip("/") + if not fp.parent.exists(): + fp.parent.mkdir(parents=True, exist_ok=True) + print(f"Creating folder: {fp.parent}") + fp.write_text(content) + print(f"Writing file: {fp}") + + def rec_tree(path: str) -> None: + req = self._session.get(f"{read_root}{path}") + if not req.ok: + return + + headers = req.headers + tp = headers.get("X-NVM-Type", headers.get("X-NS-Type", "file")) + + if tp == "file": + # skip files with default values + if headers.get("x-nvm-default", "").lower() == "true": + print(f"Skipping default: {out / path.lstrip('/')}") + return + write_file(path, req.text) + return + + nodes: List[Dict[str, str]] = req.json() + for r in nodes: + if r["type"] in ["system_file", "system_dir"]: + continue + child_path = f'{path}/{r["name"]}' + + # tags are managed via the -t CLI flag, not the load/dump sync + if child_path == "/tags" or child_path.startswith("/tags/"): + continue + + # /shared/ markers are links dumped as empty files + if r["type"] in ["file", "link"] and child_path.startswith("/shared/"): + write_file(child_path, "") + continue + + # Only round-trip real file_content-backed files (those with a + # DELETE verb); skip read-only and virtual/computed files. + if r["type"] in ["file", "link"] and "DELETE" not in r.get("actions", []): + continue + rec_tree(child_path) + + rec_tree("") + + def _collect_local_files(self, inpath: str) -> Dict[str, str]: + """ + Walk a dumped folder and return {api_path: content} for every file, + keyed by the api path it maps to (e.g. "/config/type"). + """ + files: Dict[str, str] = {} + + def walk(full: Path, api_path: str) -> None: + # tags are managed via the -t CLI flag, not the load/dump sync + if api_path == "/tags" or api_path.startswith("/tags/"): + return + + if full.is_file(): + files[api_path] = full.read_text() + elif full.is_dir(): + for name in sorted(p.name for p in full.iterdir()): + walk(full / name, f"{api_path}/{name}") + + walk(Path(inpath), "") + return files + + def _collect_remote_files(self, read_root: str) -> Tuple[Dict[str, str], Set[str]]: + """ + Walk the current remote tree and return (files, skipped): + * files - {api_path: content} for every real, round-trippable file + * skipped - api paths that exist remotely but the sync does not manage + (virtual/computed or read-only files, e.g. `notifications`) + + `skipped` lets api_load ignore local copies of those paths rather than + trying to (re)create them on every run. Mirrors api_dump's filtering. + """ + files: Dict[str, str] = {} + skipped: Set[str] = set() + + def rec_tree(path: str) -> None: + req = self._session.get(f"{read_root}{path}") + if not req.ok: + return + + headers = req.headers + tp = headers.get("X-NVM-Type", headers.get("X-NS-Type", "file")) + + if tp == "file": + # skip files with default values + if headers.get("x-nvm-default", "").lower() == "true": + return + files[path] = req.text + return + + nodes: List[Dict[str, str]] = req.json() + for r in nodes: + if r["type"] in ["system_file", "system_dir"]: + continue + child_path = f'{path}/{r["name"]}' + + # tags are managed via the -t CLI flag, not the sync + if child_path == "/tags" or child_path.startswith("/tags/"): + continue + + # /shared/ markers are links round-tripped as empty files + if r["type"] in ["file", "link"] and child_path.startswith("/shared/"): + files[child_path] = "" + continue + + # Only round-trip real file_content-backed files. Read-only + # files expose no DELETE verb, and virtual/computed files like + # `notifications` accept POST but no DELETE. Record them as + # skipped so api_load ignores any stale local copy. + if r["type"] in ["file", "link"] and "DELETE" not in r.get("actions", []): + skipped.add(child_path) + continue + rec_tree(child_path) + + rec_tree("") + return files, skipped + + def api_load(self, inpath: str, dry_run: bool = False) -> None: + """ + Sync a dumped folder into the API, treating the local folder as the + desired state of the resource: + + * files that don't exist remotely are created + * files whose content differs are overwritten + * remote files no longer present locally are deleted + * unchanged files are left untouched + + With dry_run=True no state-changing requests are sent; the actions that + would be taken are printed instead. + """ + if self.user: + print(f"You cannot modify another user's {self._sync_label()}") + return + + base = self._sync_base(user_aware=False) + read_root = f"{base}/" + prefix = "[dry-run] " if dry_run else "" + + local = self._collect_local_files(inpath) + remote, skipped = self._collect_remote_files(read_root) + + # don't (re)create local copies of paths the sync doesn't manage + # (virtual/read-only files like `notifications`) + to_create = sorted(p for p in local if p not in remote and p not in skipped) + to_overwrite = sorted(p for p in local if p in remote and local[p] != remote[p]) + # delete deepest paths first so we don't strand children + to_delete = sorted((p for p in remote if p not in local), reverse=True) + unchanged = sum(1 for p in local if p in remote and local[p] == remote[p]) + + created = overwritten = deleted = failed = 0 + + for api_path in to_create: + content = local[api_path] + if dry_run: + print(f"{prefix}create: {api_path} ({len(content)} bytes)") + created += 1 + continue + full_api = f"{base}{api_path}" + if api_path.startswith("/shared/"): + # shares are links created via PUT with no body + r = self._session.put(full_api) + else: + # PUT is best-effort (file leaves have no PUT route); the POST + # is what actually creates and writes, so gate success on it. + self._session.put(full_api) + r = self._session.post( + full_api, + headers={"Content-type": "text/plain"}, + data=content.encode("utf-8"), + ) + if r.ok: + print(f"create: {api_path} ({len(content)} bytes)") + created += 1 + else: + print(f"FAILED create: {api_path} (HTTP {r.status_code})") + failed += 1 + + for api_path in to_overwrite: + content = local[api_path] + old = remote[api_path] + # for single-line values show the actual change, not just a size + if "\n" not in content.strip() and "\n" not in old.strip(): + detail = f'"{old.strip()}" -> "{content.strip()}"' + else: + detail = f"{len(content)} bytes" + if dry_run: + print(f"{prefix}overwrite: {api_path} ({detail})") + overwritten += 1 + continue + full_api = f"{base}{api_path}" + r = self._session.post( + full_api, + headers={"Content-type": "text/plain"}, + data=content.encode("utf-8"), + ) + if r.ok: + print(f"overwrite: {api_path} ({detail})") + overwritten += 1 + else: + print(f"FAILED overwrite: {api_path} (HTTP {r.status_code})") + failed += 1 + + for api_path in to_delete: + if dry_run: + print(f"{prefix}delete: {api_path}") + deleted += 1 + continue + r = self._session.delete(f"{base}{api_path}") + if r.ok: + print(f"delete: {api_path}") + deleted += 1 + else: + print(f"FAILED delete: {api_path} (HTTP {r.status_code})") + failed += 1 + + summary = f"{prefix}{created} created, {overwritten} overwritten, " f"{deleted} deleted, {unchanged} unchanged" + if failed: + summary += f", {failed} failed" + print(summary) diff --git a/novem/vis/__init__.py b/novem/vis/__init__.py index 61bd2ad..3f080a8 100644 --- a/novem/vis/__init__.py +++ b/novem/vis/__init__.py @@ -1,4 +1,3 @@ -import os import sys from typing import Any, Dict, List, Optional, Tuple @@ -6,13 +5,14 @@ from ..api_ref import NovemAPI from ..shared import NovemShare +from ..sync import NovemTreeSync from ..tags import NovemTags from ..utils import cl from ..utils import colors as clrs from .files import NovemFiles -class NovemVisAPI(NovemAPI): +class NovemVisAPI(NovemTreeSync, NovemAPI): shared: NovemShare tags: NovemTags files: Optional[NovemFiles] = None @@ -55,121 +55,13 @@ def __setattr__(self, name: str, value: Any) -> None: else: super().__setattr__(name, value) - def api_dump(self, outpath: str) -> None: - """ - Iterate over current id and dump output to supplied path - """ - - qpath = f"{self._api_root}vis/{self._vispath}/{self.id}/" - - if self.user: - qpath = f"{self._api_root}users/{self.user}/vis/" f"{self._vispath}/{self.id}/" - - # create util function - def rec_tree(path: str) -> None: - qp = f"{qpath}{path}" - fp = f"{outpath}{path}" - # print(f"QP: {qp}") - req = self._session.get(qp) - - if not req.ok: - return None - - headers = req.headers - tp = headers.get("X-NVM-Type", headers.get("X-NS-Type", "file")) - - # if i am a file, write to disc - if tp == "file": - # skip files with default values - if headers.get("x-nvm-default", "").lower() == "true": - print(f"Skipping default: {fp}") - return None - # ensure parent directory exists before writing - parent_dir = os.path.dirname(fp) - if parent_dir and not os.path.exists(parent_dir): - os.makedirs(parent_dir, exist_ok=True) - print(f"Creating folder: {parent_dir}") - with open(fp, "w") as f: - f.write(req.text) - print(f"Writing file: {fp}") - return None - - # if I am a folder, recurse without creating yet - nodes: List[Dict[str, str]] = req.json() - - # Recurse relevant structure (skip system entries and read-only files) - for r in nodes: - if r["type"] in ["system_file", "system_dir"]: - continue - child_path = f'{path}/{r["name"]}' - child_fp = f"{outpath}{child_path}" - - # /shared/ and /tags/ are special markers - create empty files from listing - if r["type"] in ["file", "link"] and ( - child_path.startswith("/shared/") or child_path.startswith("/tags/") - ): - parent_dir = os.path.dirname(child_fp) - if parent_dir and not os.path.exists(parent_dir): - os.makedirs(parent_dir, exist_ok=True) - print(f"Creating folder: {parent_dir}") - with open(child_fp, "w") as f: - f.write("") - print(f"Writing file: {child_fp}") - continue - - # skip read-only files/links - if r["type"] in ["file", "link"] and "w" not in r.get("permissions", []): - continue - rec_tree(child_path) - - # start recurison - rec_tree("") - - def api_load(self, inpath: str) -> None: - """ - Load a dumped folder structure back into the API. - Walks the folder and for each file: PUT to create, then POST content. - """ - - qpath = f"{self._api_root}vis/{self._vispath}/{self.id}" - - if self.user: - print(f"You cannot modify another user's {self._vispath}") - return - - def load_tree(local_path: str, api_path: str) -> None: - full_local = os.path.join(inpath, local_path.lstrip("/")) if local_path else inpath - - if os.path.isfile(full_local): - # Read file content - with open(full_local, "r") as f: - content = f.read() - - full_api = f"{qpath}{api_path}" - - # Try PUT first to create the resource - r = self._session.put(full_api) - put_status = r.status_code - - # POST the content - r = self._session.post( - full_api, - headers={"Content-type": "text/plain"}, - data=content.encode("utf-8"), - ) - print(f"Loaded file: {api_path} (PUT: {put_status}, POST: {r.status_code}, {len(content)} bytes)") - - elif os.path.isdir(full_local): - print(f"Processing dir: {api_path or '/'}") - - # Iterate over directory contents - for entry in sorted(os.listdir(full_local)): - entry_local = os.path.join(local_path, entry) if local_path else entry - entry_api = f"{api_path}/{entry}" - load_tree(entry_local, entry_api) + def _sync_base(self, user_aware: bool) -> str: + if user_aware and self.user: + return f"{self._api_root}users/{self.user}/vis/{self._vispath}/{self.id}" + return f"{self._api_root}vis/{self._vispath}/{self.id}" - # Start loading from root - load_tree("", "") + def _sync_label(self) -> str: + return self._vispath or "vis" def api_tree(self, colors: bool = False, relpath: str = "/") -> str: """ diff --git a/tests/test_load_sync.py b/tests/test_load_sync.py new file mode 100644 index 0000000..8b341e3 --- /dev/null +++ b/tests/test_load_sync.py @@ -0,0 +1,288 @@ +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path + +from novem import Plot + +config_file = str(Path(__file__).resolve().parent / "test.conf") + + +class FakeResp: + def __init__(self, status_code=200, text="", headers=None, json_data=None): + self.status_code = status_code + self.text = text + self.headers = headers or {} + self._json = json_data + self.ok = 200 <= status_code < 300 + + def json(self): + return self._json + + +class FakeSession: + """ + Minimal session that serves a canned remote tree for GET and records + every state-changing call so the sync behaviour can be asserted. + + `tree` is keyed by api path ("" for root, "/config/type" for a file) and + maps to either ("file", content) or ("dir", [node, ...]). + """ + + def __init__(self, tree, read_prefix, write_prefix, fail_paths=None): + self.tree = tree + self.read_prefix = read_prefix + self.write_prefix = write_prefix + # api paths whose POST/DELETE should return a non-ok status + self.fail_paths = set(fail_paths or []) + self.puts = [] + self.posts = [] + self.deletes = [] + + def get(self, url, **kwargs): + api_path = url[len(self.read_prefix) :] + node = self.tree.get(api_path) + if node is None: + return FakeResp(status_code=404) + kind, payload = node + if kind == "file": + return FakeResp(text=payload, headers={"X-NVM-Type": "file"}) + return FakeResp(headers={"X-NVM-Type": "dir"}, json_data=payload) + + def put(self, url, **kwargs): + self.puts.append(url[len(self.write_prefix) :]) + return FakeResp(status_code=201) + + def post(self, url, **kwargs): + path = url[len(self.write_prefix) :] + self.posts.append(path) + return FakeResp(status_code=500 if path in self.fail_paths else 200) + + def delete(self, url, **kwargs): + path = url[len(self.write_prefix) :] + self.deletes.append(path) + return FakeResp(status_code=404 if path in self.fail_paths else 200) + + +def _make_plot_with_remote(tree, fail_paths=None): + p = Plot(id="test_plot", create=False, config_path=config_file) + read_prefix = f"{p._api_root}vis/plots/{p.id}/" + write_prefix = f"{p._api_root}vis/plots/{p.id}" + session = FakeSession(tree, read_prefix, write_prefix, fail_paths=fail_paths) + p._session = session + return p, session + + +def _write_local(root, files): + for rel, content in files.items(): + full = Path(root) / rel + full.parent.mkdir(parents=True, exist_ok=True) + full.write_text(content) + + +# Verbs as the directory listing reports them. A real file_content-backed +# file carries DELETE; virtual/computed files (e.g. `notifications`) accept +# POST but no DELETE; read-only files only GET. +FILE_ACTIONS = ["POST", "GET", "DELETE", "OPTIONS"] +VIRTUAL_ACTIONS = ["POST", "GET", "OPTIONS"] +RO_ACTIONS = ["GET", "OPTIONS"] + +REMOTE_TREE = { + "": ( + "dir", + [ + {"name": "config", "type": "dir", "permissions": "r", "actions": RO_ACTIONS}, + {"name": "name", "type": "file", "permissions": "rw", "actions": FILE_ACTIONS}, + {"name": "stale", "type": "file", "permissions": "rw", "actions": FILE_ACTIONS}, + # virtual file: writable (POST) but no DELETE -> must be ignored + {"name": "notifications", "type": "file", "permissions": "rw", "actions": VIRTUAL_ACTIONS}, + # read-only file: no DELETE -> must be ignored + {"name": "url", "type": "file", "permissions": "r", "actions": RO_ACTIONS}, + ], + ), + "/config": ("dir", [{"name": "type", "type": "file", "permissions": "rw", "actions": FILE_ACTIONS}]), + "/config/type": ("file", "bar"), + "/name": ("file", "old name"), + "/stale": ("file", "remove me"), + # present remotely, but never reached because the walk skips them above + "/notifications": ("file", "info"), + "/url": ("file", "https://example/x"), +} + + +def test_load_sync_creates_overwrites_and_deletes(tmp_path): + p, session = _make_plot_with_remote(REMOTE_TREE) + + _write_local( + str(tmp_path), + { + "config/type": "bar", # unchanged + "name": "new name", # changed -> overwrite + "fresh": "brand new", # new -> create + # "stale" absent -> delete + }, + ) + + out = StringIO() + with redirect_stdout(out): + p.api_load(inpath=str(tmp_path)) + + assert session.puts == ["/fresh"] + assert sorted(session.posts) == ["/fresh", "/name"] + assert session.deletes == ["/stale"] + + # virtual (notifications) and read-only (url) files are never managed: + # not created, overwritten, deleted, nor counted as unchanged + all_calls = session.puts + session.posts + session.deletes + assert not any("notifications" in c for c in all_calls) + assert not any("url" in c for c in all_calls) + + summary = out.getvalue() + assert "1 created, 1 overwritten, 1 deleted, 1 unchanged" in summary + # single-line values log the actual change rather than a byte count + assert 'overwrite: /name ("old name" -> "new name")' in summary + + +def test_load_sync_overwrite_multiline_shows_byte_count(tmp_path): + # a multi-line body has no useful one-line diff, so fall back to size + tree = { + "": ("dir", [{"name": "data", "type": "file", "permissions": "rw", "actions": FILE_ACTIONS}]), + "/data": ("file", "a,b\n1,2\n"), + } + p, session = _make_plot_with_remote(tree) + _write_local(str(tmp_path), {"data": "a,b\n9,9\n"}) + + out = StringIO() + with redirect_stdout(out): + p.api_load(inpath=str(tmp_path)) + + text = out.getvalue() + assert "overwrite: /data (8 bytes)" in text + assert "->" not in text + + +def test_load_sync_dry_run_sends_nothing(tmp_path): + p, session = _make_plot_with_remote(REMOTE_TREE) + + _write_local( + str(tmp_path), + { + "config/type": "bar", + "name": "new name", + "fresh": "brand new", + }, + ) + + out = StringIO() + with redirect_stdout(out): + p.api_load(inpath=str(tmp_path), dry_run=True) + + assert session.puts == [] + assert session.posts == [] + assert session.deletes == [] + + text = out.getvalue() + assert "[dry-run] create: /fresh" in text + assert "[dry-run] overwrite: /name" in text + assert "[dry-run] delete: /stale" in text + assert "[dry-run] 1 created, 1 overwritten, 1 deleted, 1 unchanged" in text + + +def test_load_sync_reports_failed_operations(tmp_path): + # the server rejects the DELETE of /stale (e.g. no DELETE route) -> the + # sync must report it as failed, not silently count it as deleted + p, session = _make_plot_with_remote(REMOTE_TREE, fail_paths={"/stale"}) + + _write_local( + str(tmp_path), + { + "config/type": "bar", + "name": "new name", + "fresh": "brand new", + }, + ) + + out = StringIO() + with redirect_stdout(out): + p.api_load(inpath=str(tmp_path)) + + assert session.deletes == ["/stale"] # the attempt was made + + text = out.getvalue() + assert "FAILED delete: /stale (HTTP 404)" in text + # /stale is counted as failed, not deleted + assert "1 created, 1 overwritten, 0 deleted, 1 unchanged, 1 failed" in text + + +def test_load_sync_ignores_local_virtual_file(tmp_path): + # a stale dump may still contain a `notifications` file on disk; since the + # remote walk reports it as skipped (virtual, no DELETE), load must ignore + # the local copy rather than "creating" it on every run + tree = { + "": ( + "dir", + [ + {"name": "name", "type": "file", "permissions": "rw", "actions": FILE_ACTIONS}, + {"name": "notifications", "type": "file", "permissions": "rw", "actions": VIRTUAL_ACTIONS}, + ], + ), + "/name": ("file", "hi"), + "/notifications": ("file", "info"), + } + p, session = _make_plot_with_remote(tree) + _write_local(str(tmp_path), {"name": "hi", "notifications": "info"}) + + out = StringIO() + with redirect_stdout(out): + p.api_load(inpath=str(tmp_path)) + + # notifications is never touched, and there are no phantom actions + all_calls = session.puts + session.posts + session.deletes + assert all_calls == [] + assert "0 created, 0 overwritten, 0 deleted, 1 unchanged" in out.getvalue() + + +def test_load_sync_creates_shares_via_put(tmp_path): + # shares are links created via PUT (no body); POST 405s. A new share must + # be created with PUT alone and reported as a success, not a failed POST. + tree = { + "": ("dir", [{"name": "shared", "type": "dir", "permissions": "r", "actions": RO_ACTIONS}]), + "/shared": ("dir", []), # no shares yet + } + p, session = _make_plot_with_remote(tree) + _write_local(str(tmp_path), {"shared/public": ""}) + + out = StringIO() + with redirect_stdout(out): + p.api_load(inpath=str(tmp_path)) + + assert session.puts == ["/shared/public"] + assert session.posts == [] # never POST to a share + text = out.getvalue() + assert "FAILED" not in text + assert "1 created, 0 overwritten, 0 deleted, 0 unchanged" in text + + +def test_load_sync_ignores_tags(tmp_path): + # tags are managed via the -t flag; the sync must ignore /tags entirely, + # both remotely (not fetched) and locally (stale dump entries dropped) + tree = { + "": ( + "dir", + [ + {"name": "tags", "type": "dir", "permissions": "rw", "actions": FILE_ACTIONS}, + {"name": "name", "type": "file", "permissions": "rw", "actions": FILE_ACTIONS}, + ], + ), + "/name": ("file", "x"), + } + p, session = _make_plot_with_remote(tree) + _write_local(str(tmp_path), {"name": "x", "tags/mytag": ""}) + + out = StringIO() + with redirect_stdout(out): + p.api_load(inpath=str(tmp_path)) + + all_calls = session.puts + session.posts + session.deletes + assert not any("tags" in c for c in all_calls) + assert all_calls == [] + assert "0 created, 0 overwritten, 0 deleted, 1 unchanged" in out.getvalue()