From da79b927ac3e82abdce318f89f67bdf0585a1b8f Mon Sep 17 00:00:00 2001 From: Oleg Savchenko Date: Fri, 22 May 2026 16:31:07 +0200 Subject: [PATCH] Add paths.buffer config key to redirect buffer snapshot storage Mirrors the existing paths.samples pattern: if paths.buffer is set in the config (or passed as a Hydra override), buffer snapshots are written to {paths.buffer}/snapshots/ instead of the default {run_dir}/buffer/snapshots/. Useful when run_dir is on a fast local filesystem but buffer snapshots (which can be large) should land on a separate scratch volume. Co-Authored-By: Claude Sonnet 4.6 --- docs/configuration.md | 8 +++++--- falcon/cli.py | 5 +++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 309c85e..7f93e33 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -32,15 +32,17 @@ Configure file paths: ```yaml paths: import: "." - graph: ${run_dir}/graph + graph: ${run_dir}/graph samples: ${run_dir}/samples + buffer: ${run_dir}/buffer # optional; redirect to a separate volume (e.g. scratch) ``` | Key | Type | Default | Description | |-----|------|---------|-------------| | `import` | str | `"."` | Path to import custom modules | -| `graph` | str | `${run_dir}/graph` | Trained models directory | +| `graph` | str | `${run_dir}/graph` | Trained model checkpoints directory | | `samples` | str | `${run_dir}/samples` | Output samples directory | +| `buffer` | str | `${run_dir}/buffer` | Buffer snapshots directory (`snapshots/` is appended); useful for routing large temporary simulation data to a separate scratch volume while keeping `run_dir` on persistent storage | ### `buffer` @@ -65,7 +67,7 @@ buffer: | `simulate_count` | int | `64` | Number of new samples generated per simulation round. For simulators taking >1s per sample, keep this small (4–16) to avoid long delays between buffer updates; for fast simulators, increase to reduce Ray overhead. | | `simulate_interval` | float | `1` | Seconds between simulation rounds | | `simulate_when_full` | bool | `true` | If `true`, simulation continues after `max_samples` is reached and old samples are replaced; if `false`, simulation stops once the buffer is full | -| `store_fraction` | float | `0.0` | Fraction of simulated samples written to `buffer/snapshots/` for inspection (0 = none, 1 = all) | +| `store_fraction` | float | `0.0` | Fraction of simulated samples written to `{paths.buffer}/snapshots/` for inspection (0 = none, 1 = all) | ### `graph` diff --git a/falcon/cli.py b/falcon/cli.py index b161301..b479999 100644 --- a/falcon/cli.py +++ b/falcon/cli.py @@ -668,9 +668,10 @@ def stop_check(): from omegaconf import OmegaConf as _OmegaConf from falcon.core.raystore import BufferConfig as _BufferConfig buffer_cfg = _OmegaConf.merge(_OmegaConf.structured(_BufferConfig), cfg.buffer) + buffer_base = cfg.paths.get("buffer", str(Path(cfg.run_dir) / "buffer")) dataset_manager = falcon.get_ray_dataset_manager( buffer_cfg, - snapshots_path=str(Path(cfg.run_dir) / "buffer" / "snapshots"), + snapshots_path=str(Path(buffer_base) / "snapshots"), log_config=logging_cfg, ) @@ -899,7 +900,7 @@ def parse_args(): elif arg.startswith("--refresh="): refresh = float(arg.split("=", 1)[1]) i += 1 - return mode, None, None, None, None, False, 16, address, refresh + return mode, None, None, None, None, False, 16, True, None, address, refresh sample_type = None if mode == "sample":