feat(config): index-dir config_source pointer for sibling-config discovery

docs/CONFIGURATION.md

@@ -39,6 +39,8 @@ The tool automatically walks up the directory tree from the current working dire
 4. Walk-up discovery result (config directory itself)
 5. Current working directory (fallback)
 
+**Config separate from the index — `.java-codebase-rag/config_source`.** Walk-up only checks *ancestor* directories (like Git), so a config living in a *sibling* directory of the Java tree is invisible when your working directory is inside a microservice — e.g. with `project-context/.java-codebase-rag.yml` beside `microservice-a/`, `microservice-b/`, and `.java-codebase-rag/`, an agent working from `microservice-a/` cannot reach the config by walking up. To bridge this, `init` / `reprocess` / `increment` (and `install` / `update`) write a one-line pointer `.java-codebase-rag/config_source` inside the index dir recording the absolute path of the YAML they used. When discovery anchors on the index dir but finds no YAML beside it, it follows that pointer and resolves the config — and its relative `source_root` / `index_dir` / `embedding.model` — as if you were in the config's own directory. A direct YAML at the anchor always wins; a stale pointer (target moved or deleted) is ignored with a one-line stderr hint and falls back to defaults. The pointer holds a machine-local absolute path and is not meant to be committed (the index dir is gitignored anyway).
+
 This walk-up behavior means you no longer need to set environment variables or pass flags when working from within a project — the tool finds the config automatically.
 
 | Variable | Purpose |
@@ -233,7 +235,7 @@ async_producer_overrides:
 
 **Tips & gotchas:**
 
-- **The config file may live anywhere under your project, including a subdirectory of the Java tree.** Both the CLI (`init` / `increment` / `reprocess`) and the MCP server walk up from cwd to find `.java-codebase-rag.yml`, then resolve `source_root` and `index_dir` relative to the config file's directory. So a config living in `my-context/` next to `source_root: ../` and `index_dir: ../.java-codebase-rag` resolves identically for the CLI and the MCP server. Keep the file under your project (not `$HOME`); set `JAVA_CODEBASE_RAG_SOURCE_ROOT` (MCP) or `--source-root` (CLI) only to override the discovered location.
+- **The config file may live anywhere under your project, including a subdirectory of the Java tree.** Both the CLI (`init` / `increment` / `reprocess`) and the MCP server walk up from cwd to find `.java-codebase-rag.yml`, then resolve `source_root` and `index_dir` relative to the config file's directory. So a config living in `my-context/` next to `source_root: ../` and `index_dir: ../.java-codebase-rag` resolves identically for the CLI and the MCP server. If the config lives in a *sibling* dir of the Java tree (not an ancestor of where the agent works), the index remembers its location via `.java-codebase-rag/config_source` — see [Config file discovery (walk-up)](#config-file-discovery-walk-up). Keep the file under your project (not `$HOME`); set `JAVA_CODEBASE_RAG_SOURCE_ROOT` (MCP) or `--source-root` (CLI) only to override the discovered location.
 - **Don't commit secrets** into this YAML — it sits next to your source tree and is read by every operator who clones it.
 - **Rebuild after editing brownfield overrides.** Run a full `java-codebase-rag reprocess` (no flags) so Lance and LadybugDB stay coherent, or use `--graph-only` / `--vectors-only` when you know only one store needs invalidation. Editing `embedding.model` requires a vector rebuild (`reprocess` or `--vectors-only`).
 - **Diagnose what's loaded.** `java-codebase-rag meta` prints the resolved config and each value's `*_source` (`cli` / `env` / `yaml` / `default`) — see `embedding_model_source`, `embedding_device_source`, `index_dir_source`.

java_codebase_rag/cli.py

@@ -15,12 +15,14 @@
 from typing import Any, Callable
 
 from java_codebase_rag.config import (
+    OPERATOR_OWNED_INDEX_FILES,
     ResolvedOperatorConfig,
     describe_path_sizes,
     emit_legacy_env_hints_if_present,
     emit_legacy_yaml_hint_if_needed,
     index_dir_has_existing_artifacts,
     resolve_operator_config,
+    write_config_source_pointer,
 )
 from java_codebase_rag._fdlimit import raise_fd_limit
 from java_codebase_rag.pipeline import clip, run_build_ast_graph, run_cocoindex_drop, run_cocoindex_update, run_incremental_graph
@@ -126,6 +128,21 @@ def _pipeline_footer(subcommand: str, started: float, exit_code: int) -> None:
     )
 
 
+# Subcommands that build/refresh an index and therefore should record which YAML
+# was used (so a later discovery run from a sibling/cwd can relocate it).
+_CONFIG_SOURCE_RECORDING_SUBCOMMANDS = frozenset({"init", "increment", "reprocess"})
+
+
+def _maybe_record_config_source(
+    subcommand: str, cfg: ResolvedOperatorConfig, code: int
+) -> None:
+    """On a successful index build, remember the YAML path inside the index dir."""
+    if code == 0 and subcommand in _CONFIG_SOURCE_RECORDING_SUBCOMMANDS:
+        write_config_source_pointer(
+            index_dir=cfg.index_dir, yaml_config_path=cfg.yaml_config_path
+        )
+
+
 def _run_with_pipeline_progress(
     subcommand: str,
     cfg: ResolvedOperatorConfig,
@@ -143,7 +160,9 @@ def _run_with_pipeline_progress(
     raw-relays subprocess output).
     """
     if quiet or verbose:
-        return int(work(None))
+        code = int(work(None))
+        _maybe_record_config_source(subcommand, cfg, code)
+        return code
     from java_codebase_rag.progress import build_index_progress_context
 
     # PR-3 owns all three tasks in order: Vectors → Optimize → Graph. The vectors
@@ -183,6 +202,7 @@ def _run_with_pipeline_progress(
     finally:
         renderer.stop()
         _pipeline_footer(subcommand, t0, code)
+        _maybe_record_config_source(subcommand, cfg, code)
 
 
 class PipelineProgress:
@@ -612,7 +632,13 @@ def _cmd_erase(args: argparse.Namespace) -> int:
     # every other command) stay fast -- see the lazy-import invariant atop this file.
     from build_ast_graph import BUILDER_OWNED_INDEX_FILES
     builder_paths = [cfg.ladybug_path.parent / name for name in BUILDER_OWNED_INDEX_FILES]
-    to_describe: list[Path] = [cfg.ladybug_path, cfg.cocoindex_db, *builder_paths]
+    operator_paths = [cfg.index_dir / name for name in OPERATOR_OWNED_INDEX_FILES]
+    to_describe: list[Path] = [
+        cfg.ladybug_path,
+        cfg.cocoindex_db,
+        *builder_paths,
+        *operator_paths,
+    ]
     if cfg.index_dir.is_dir():
         try:
             import lancedb
@@ -671,6 +697,11 @@ def work(progress: "PipelineProgress | None") -> int:
         _rm_any(cfg.cocoindex_db)
         for builder_path in builder_paths:
             _rm_any(builder_path)
+        # Operator-owned index files (the config_source pointer recording which
+        # YAML built this index) — owned by the CLI/installer, not the graph
+        # builder, so removed here rather than via BUILDER_OWNED_INDEX_FILES.
+        for operator_path in operator_paths:
+            _rm_any(operator_path)
         if cfg.index_dir.is_dir():
             try:
                 import lancedb

java_codebase_rag/config.py

@@ -18,6 +18,18 @@
 YAML_CONFIG_FILENAMES = (".java-codebase-rag.yml", ".java-codebase-rag.yaml")
 LEGACY_YAML_FILENAMES = (".lancedb-mcp.yml", ".lancedb-mcp.yaml")
 
+# Pointer file written into the index dir at index time so discovery can locate
+# a YAML that does not sit beside the index-dir anchor — e.g. a config living in
+# a sibling ``project-context/`` dir when the agent's cwd is inside a
+# microservice (a descendant of the index anchor, a sibling of the config).
+# Contains one line: the absolute path of the YAML used to build the index. A
+# direct YAML at the anchor always wins; the pointer only fires when the anchor
+# has no YAML beside it (see ``_effective_config_dir``).
+CONFIG_SOURCE_FILENAME = "config_source"
+# Operator-owned files inside the index dir that ``erase`` removes. Kept separate
+# from ``build_ast_graph.BUILDER_OWNED_INDEX_FILES`` (builder-owned artifacts).
+OPERATOR_OWNED_INDEX_FILES = (CONFIG_SOURCE_FILENAME,)
+
 ENV_INDEX_DIR = "JAVA_CODEBASE_RAG_INDEX_DIR"
 # Public operator contract is six names: INDEX_DIR, DEBUG_CONTEXT, RUN_HEAVY, SBERT_MODEL, SBERT_DEVICE, HINTS_ENABLED.
 # SOURCE_ROOT is still required for MCP / subprocess Java tree resolution (see mcp.json.example); it is not folded into the headline "5".
@@ -234,6 +246,61 @@ def discover_project_root(start: Path) -> Path | None:
         current = parent
 
 
+_stale_pointer_seen: set[str] = set()
+
+
+def _config_dir_from_pointer(anchor: Path) -> Path | None:
+    """Return the YAML config dir recorded in the index dir's ``config_source`` pointer.
+
+    Reads ``<anchor>/.java-codebase-rag/config_source`` (one absolute path). If it
+    names an existing ``.java-codebase-rag.yml`` / ``.yaml``, returns that file's
+    parent directory; otherwise (missing/blank/stale) returns ``None``. Used only
+    when the anchor has no direct YAML — see :func:`_effective_config_dir`.
+    """
+    pointer = anchor / ".java-codebase-rag" / CONFIG_SOURCE_FILENAME
+    if not pointer.is_file():
+        return None
+    try:
+        raw = pointer.read_text(encoding="utf-8").strip()
+    except OSError:
+        return None
+    if not raw:
+        return None
+    target = Path(raw).expanduser()
+    if not target.is_absolute():
+        # Relative to the anchor (the index-dir parent), not the pointer file.
+        target = (anchor / target).resolve()
+    if not target.is_file() or target.name not in YAML_CONFIG_FILENAMES:
+        key = str(pointer.resolve())
+        if key not in _stale_pointer_seen:
+            _stale_pointer_seen.add(key)
+            print(
+                "java-codebase-rag: ignoring stale index pointer "
+                f"{pointer} -> {raw} (target missing or not a config file).",
+                file=sys.stderr,
+            )
+        return None
+    return target.parent
+
+
+def _effective_config_dir(config_dir: Path) -> Path:
+    """Resolve the directory YAML config fields are relative to.
+
+    A direct ``.java-codebase-rag.yml`` / ``.yaml`` in ``config_dir`` always wins.
+    Otherwise, if ``config_dir`` hosts the ``.java-codebase-rag/`` index dir and
+    that index remembers its config via a ``config_source`` pointer, follow it to
+    the YAML's directory. This lets a config in a sibling dir (e.g.
+    ``project-context/`` beside the Java tree) be found when discovery anchors on
+    the index dir from inside a microservice — without an env var or flag, and
+    with YAML-relative fields (``index_dir``, ``source_root``, ``embedding.model``)
+    resolving against the YAML's home rather than the index anchor. Falls back to
+    ``config_dir`` unchanged when neither applies.
+    """
+    if find_yaml_config_file(config_dir) is not None:
+        return config_dir
+    return _config_dir_from_pointer(config_dir) or config_dir
+
+
 def load_yaml_mapping(source_root: Path) -> dict[str, Any]:
     path = find_yaml_config_file(source_root)
     if path is None:
@@ -272,6 +339,10 @@ class ResolvedOperatorConfig:
     embedding_model_source: SettingSource
     embedding_device_source: SettingSource
     hints_enabled_source: SettingSource
+    # Absolute path of the YAML actually loaded (None when built-in defaults were
+    # used with no config file). Recorded into the index dir at index time so a
+    # later discovery run from a sibling/cwd can relocate this config.
+    yaml_config_path: Path | None = None
 
     def apply_to_os_environ(self) -> None:
         """Make downstream modules (server, ladybug_queries, flows) see a consistent environment.
@@ -411,21 +482,26 @@ def resolve_operator_config(
     # Phase 1: Find the config file directory
     if source_root is not None:
         # CLI flag provided: use it as both config_dir and effective source_root
-        # (skip YAML source_root check - CLI wins)
+        # (skip YAML source_root check - CLI wins). ``_effective_config_dir`` may
+        # rebase config_dir to a YAML reached via the index-dir pointer; root is
+        # untouched (explicit source_root wins).
         root = source_root.expanduser().resolve()
-        config_dir = root
+        config_dir = _effective_config_dir(root)
         yaml_dict = load_yaml_mapping(config_dir)
     else:
         # Check env var first
         env_raw = os.environ.get(ENV_SOURCE_ROOT, "").strip()
         if env_raw:
             root = Path(env_raw).expanduser().resolve()
-            config_dir = root
+            config_dir = _effective_config_dir(root)
             yaml_dict = load_yaml_mapping(config_dir)
         else:
             # Walk up to find config dir
             discovered = discover_project_root(Path.cwd())
             config_dir = discovered if discovered is not None else Path.cwd().resolve()
+            # Follow an index-dir pointer to the real config dir when the anchor
+            # has no YAML beside it (e.g. config in a sibling dir).
+            config_dir = _effective_config_dir(config_dir)
             # Load YAML from config dir
             yaml_dict = load_yaml_mapping(config_dir)
 
@@ -479,9 +555,35 @@ def resolve_operator_config(
         embedding_model_source=model_src,
         embedding_device_source=device_src,
         hints_enabled_source=hints_src,
+        yaml_config_path=find_yaml_config_file(config_dir),
     )
 
 
+def write_config_source_pointer(
+    *, index_dir: Path, yaml_config_path: Path | None
+) -> None:
+    """Record the YAML config path inside the index dir (best-effort).
+
+    Writes ``<index_dir>/config_source`` with the YAML's absolute path so a later
+    discovery run that anchors on the index dir (but has no YAML beside it) can
+    relocate the config via :func:`_effective_config_dir`. No-op when
+    ``yaml_config_path`` is None (pure-default build — nothing to remember). Never
+    raises: the pointer is an optimization, not a correctness requirement — a
+    missing/unreadable pointer just falls back to built-in defaults.
+    """
+    if yaml_config_path is None:
+        return
+    try:
+        index_dir.mkdir(parents=True, exist_ok=True)
+        content = str(yaml_config_path.resolve()) + "\n"
+        target = index_dir / CONFIG_SOURCE_FILENAME
+        tmp = index_dir / (CONFIG_SOURCE_FILENAME + ".tmp")
+        tmp.write_text(content, encoding="utf-8")
+        os.replace(tmp, target)
+    except OSError:
+        pass
+
+
 def index_dir_has_existing_artifacts(index_dir: Path) -> tuple[bool, list[str]]:
     """True if graph dir or any Lance table already exists under index_dir."""
     paths: list[str] = []

java_codebase_rag/installer.py

@@ -1021,6 +1021,7 @@ def run_init_if_needed(
     from java_codebase_rag.config import (
         index_dir_has_existing_artifacts,
         resolve_operator_config,
+        write_config_source_pointer,
     )
     from java_codebase_rag.pipeline import run_build_ast_graph, run_cocoindex_update
 
@@ -1093,6 +1094,12 @@ def run_init_if_needed(
         if renderer is not None:
             renderer.stop()
             _index_progress_footer("install", started, ok=index_ok)
+    if index_ok:
+        # Remember which YAML built this index so discovery from a sibling/cwd
+        # can relocate the config (e.g. a config beside, not inside, the tree).
+        write_config_source_pointer(
+            index_dir=cfg.index_dir, yaml_config_path=cfg.yaml_config_path
+        )
     return index_ok
 
 
@@ -1595,6 +1602,7 @@ def run_update(
         discover_project_root,
         index_dir_has_existing_artifacts,
         resolve_operator_config,
+        write_config_source_pointer,
     )
     from java_codebase_rag.pipeline import run_cocoindex_update, run_incremental_graph
 
@@ -1699,6 +1707,11 @@ def run_update(
                 _index_progress_footer("update", started, ok=index_ok)
         if not index_ok:
             return 1
+        # Refresh the config pointer so a config moved/renamed since the last
+        # index is relocated correctly by discovery from a sibling/cwd.
+        write_config_source_pointer(
+            index_dir=cfg.index_dir, yaml_config_path=cfg.yaml_config_path
+        )
     else:
         print("\nWould run incremental index update (Lance + graph).")
 

java_codebase_rag/jrag.py

@@ -1101,6 +1101,10 @@ def _resolve_cfg(args: argparse.Namespace):  # type: ignore[no-untyped-def]
     applies CLI ``--index-dir`` if given, and calls ``apply_to_os_environ`` so
     downstream modules see a consistent env (critically: SBERT_MODEL for
     ``jrag search`` in PR-JRAG-4).
+
+    When the anchor is an index dir with no YAML beside it, resolution follows
+    that index's ``config_source`` pointer (see ``config._effective_config_dir``)
+    so a config living in a sibling dir is still found from inside a microservice.
     """
     from java_codebase_rag.config import discover_project_root, resolve_operator_config