From e85c7d69f0d2a8f694f75d9cc97af0bd72041307 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 4 Jul 2026 18:38:58 +0200 Subject: [PATCH] fix(index): bound + source-cap parallel retention with re-read fallback (low-RAM peak RSS) Distilled from #685 (nguyentamdat) rebased onto current main, plus two research-driven refinements and a genuine reproduce-first guard. The parallel extract retains each file's source text so the fused cross-file LSP resolve can re-parse it. That retention is transient but a peak-RSS driver. On main the caps are flat (100 MiB/file, 2 GiB total) and a file over the cap is silently unretained AND its cross-file resolution is skipped -- a graph-quality gap. This change bounds retention AND keeps every cross-file edge. - Source-text cap as a FLOOR, not just a ceiling: retention total defaults to min(cbm_mem_budget()/8, 1 GiB), per-file min(32 MiB, total). Following the rust-analyzer memory model, the RAM-derived default is clamped to a small absolute ceiling so a huge-RAM host does not hold tens of GB it would re-read cheaply. Both caps env-overridable via CBM_RETAIN_TOTAL_MB / CBM_RETAIN_PER_FILE_MB (limits.c convention); ceilings bound only the auto-derived default, never an explicit operator/caller choice. A dropped file emits one index.retain_capped WARN per run. - Bounded re-read fallback (the correctness guarantee): resolve_worker re-reads an unretained file's source on demand (bounded, freed immediately) instead of skipping resolution, wired at every cross-LSP site that consumes source. The cap now only trades retained RAM for a bounded re-read, never a lost edge. - cbm_parallel_extract_ex + opts struct (cbm_parallel_extract is now a wrapper passing NULL -> env-derived defaults); malloc/calloc NULL-check hardening. Reproduce-first: parallel_cross_file_reread_preserves_unretained_edges uses a Java<->Kotlin pair whose cross-file lsp edges are genuinely source-dependent; the edges are lost when the caller is unretained and the fallback is absent (RED), present with it (GREEN), with a retained CONTROL scenario proving non-vacuity. #685's original Python red test was a false guard (per-file py_lsp already resolves those calls) and is replaced. Peak-bound guards (retained_bytes <= total_cap; retain_sources=false retains nothing) in test_mem.c. Verify: make -f Makefile.cbm cbm && make -f Makefile.cbm lint-ci; test-runner parallel pipeline incremental py_lsp ts_lsp java_lsp kotlin_lsp c_lsp cs_lsp go_lsp rust_lsp mem -> 2323 passed. Co-authored-by: nguyentamdat Signed-off-by: Martin Vogel --- src/pipeline/pass_parallel.c | 295 ++++++++++++++++++++++++------- src/pipeline/pipeline_internal.h | 15 ++ tests/test_mem.c | 163 +++++++++++++++++ tests/test_parallel.c | 148 +++++++++++++++- 4 files changed, 552 insertions(+), 69 deletions(-) diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 8724d47c8..4006cb022 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -40,15 +40,32 @@ enum { #define PP_FIELD_HINT_CONF 0.85 enum { PP_CSHARP_M_PREFIX_LEN = 2 }; -/* Source-retention caps for the parallel pipeline. The extract worker - * copies source bytes into result->arena so the fused cross-file LSP - * step in resolve_worker can run without re-reading from disk. Bound - * peak RSS with a per-file cap (skip retention for pathological huge - * generated files) and a total project-wide cap (skip when budget - * exhausted — cross-file LSP becomes a no-op for those late files, - * defs/calls already extracted are unaffected). */ -#define PP_RETAIN_PER_FILE_MAX_BYTES (100LL * 1024 * 1024) -#define PP_RETAIN_TOTAL_BUDGET_BYTES (2LL * 1024 * 1024 * 1024) +/* Absolute source-retention ceilings for the parallel extract pipeline. + * + * The extract worker copies each file's source bytes into result->arena so + * the fused cross-file LSP step in resolve_worker can re-parse without + * re-opening the file. That retention is TRANSIENT (freed at run end) but it + * is a PEAK-RSS driver: every retained byte is resident at once across the + * extract→resolve handoff. + * + * A cap here is a FLOOR as much as a ceiling: whatever total we allow, we + * WILL hold that much resident at peak on a large repo. rust-analyzer bounds + * retained file *text* to a small fixed budget and re-reads source on a miss + * rather than scaling the retained set with host RAM — because the re-read is + * cheap relative to holding tens of GB resident. We follow the same model: + * - derive the total budget from the process memory budget (budget/8), + * - BUT clamp the RAM-derived DEFAULT to a small absolute ceiling (1 GiB) + * so a 512 GiB host does not retain 64 GiB of source it would re-read + * cheaply anyway, and keep the per-file cap modest (32 MiB) so one + * pathological generated blob cannot monopolise the budget. + * A file dropped from retention is NOT lost to cross-file resolution: + * resolve_worker re-reads it on demand, bounded and freed immediately + * (source_reread fallback). Both caps are env-overridable — CBM_RETAIN_TOTAL_MB + * / CBM_RETAIN_PER_FILE_MB raise or lower the auto-derived defaults directly + * (the hard ceilings bound only the RAM-derived default, never a deliberate + * operator/caller choice). */ +#define PP_RETAIN_TOTAL_HARD_MAX_BYTES (1024ULL * 1024 * 1024) /* 1 GiB default ceiling */ +#define PP_RETAIN_PER_FILE_HARD_MAX_BYTES (32ULL * 1024 * 1024) /* 32 MiB per file */ #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" #include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */ @@ -73,6 +90,7 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 }; #include "simhash/minhash.h" #include "semantic/ast_profile.h" +#include #include #include #include @@ -80,6 +98,95 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 }; #include #include +/* Parse a positive MB-valued retention env knob (CBM_RETAIN_*_MB) into bytes. + * Follows the limits.c strtol convention: unset / unparseable / non-positive + * → return 0 so the caller keeps its derived default. */ +static size_t cbm_retain_env_bytes(const char *name) { + const char *raw = getenv(name); + if (!raw || !raw[0]) { + return 0; + } + errno = 0; + char *end = NULL; + long v = strtol(raw, &end, 10); + if (errno != 0 || end == raw || *end != '\0' || v <= 0) { + return 0; + } + return (size_t)v * 1024 * 1024; +} + +/* Auto-derived TOTAL retention budget: a fraction of the process memory + * budget, clamped to the absolute ceiling. The clamp bounds ONLY this + * RAM-derived default — a huge-RAM host must not retain tens of GB it would + * re-read cheaply. When the budget is unset (tests / no cgroup) fall back to + * the ceiling. */ +static size_t cbm_parallel_extract_default_total_cap(void) { + size_t cap = cbm_mem_budget(); + if (cap > 0) { + cap /= 8; + } else { + cap = PP_RETAIN_TOTAL_HARD_MAX_BYTES; + } + if (cap > PP_RETAIN_TOTAL_HARD_MAX_BYTES) { + cap = PP_RETAIN_TOTAL_HARD_MAX_BYTES; + } + return cap; +} + +/* Auto-derived PER-FILE cap: modest absolute ceiling, never above the total. */ +static size_t cbm_parallel_extract_default_per_file_cap(size_t total_cap) { + size_t cap = PP_RETAIN_PER_FILE_HARD_MAX_BYTES; + if (cap > total_cap) { + cap = total_cap; + } + return cap; +} + +/* Resolve the effective retention options from (in precedence order): + * explicit caller opts > CBM_RETAIN_*_MB env knobs > RAM-derived defaults. + * The absolute hard ceilings apply only to the RAM-derived defaults; an + * operator/caller that sets a value explicitly is trusted. The only invariant + * enforced unconditionally is per-file ≤ total. */ +static cbm_parallel_extract_opts_t cbm_parallel_extract_resolve_opts( + const cbm_parallel_extract_opts_t *opts) { + cbm_parallel_extract_opts_t resolved = { + .retain_sources = true, + .retain_sources_set = true, + .retain_total_budget_bytes = cbm_parallel_extract_default_total_cap(), + .retain_per_file_max_bytes = 0, + }; + + size_t env_total = cbm_retain_env_bytes("CBM_RETAIN_TOTAL_MB"); + if (env_total > 0) { + resolved.retain_total_budget_bytes = env_total; + } + + resolved.retain_per_file_max_bytes = + cbm_parallel_extract_default_per_file_cap(resolved.retain_total_budget_bytes); + size_t env_per_file = cbm_retain_env_bytes("CBM_RETAIN_PER_FILE_MB"); + if (env_per_file > 0) { + resolved.retain_per_file_max_bytes = env_per_file; + } + + if (opts) { + if (opts->retain_sources_set) { + resolved.retain_sources = opts->retain_sources; + } + if (opts->retain_total_budget_bytes > 0) { + resolved.retain_total_budget_bytes = opts->retain_total_budget_bytes; + } + if (opts->retain_per_file_max_bytes > 0) { + resolved.retain_per_file_max_bytes = opts->retain_per_file_max_bytes; + } + } + + /* Correctness invariant: a single file can never exceed the total budget. */ + if (resolved.retain_per_file_max_bytes > resolved.retain_total_budget_bytes) { + resolved.retain_per_file_max_bytes = resolved.retain_total_budget_bytes; + } + return resolved; +} + static uint64_t extract_now_ns(void) { struct timespec ts; cbm_clock_gettime(CLOCK_MONOTONIC, &ts); @@ -569,7 +676,12 @@ typedef struct { _Atomic int next_file_idx; cbm_pkg_entries_t *pkg_entries; /* per-worker manifest arrays (separate allocation) */ - _Atomic int64_t retained_bytes; /* total source bytes copied into result arenas */ + + bool retain_sources; /* copy source into result->arena for cross-file LSP */ + size_t retain_total_budget_bytes; /* project-wide retention cap (peak-RSS bound) */ + size_t retain_per_file_max_bytes; /* per-file retention cap */ + _Atomic int64_t retained_bytes; /* total source bytes copied into result arenas */ + _Atomic int retain_cap_warned; /* WARN index.retain_capped emitted once per run */ /* Per-worker skip lists (separate allocation, indexed by worker_id — no hot- * path lock). Merged into the pipeline in the sequential merge loop. */ @@ -768,29 +880,44 @@ static void extract_worker(int worker_id, void *ctx_ptr) { source_len, &ec->pkg_entries[worker_id]); } - /* Retain source bytes in result->arena so the fused cross-file - * LSP step in resolve_worker can run without re-reading from - * disk. Capped per-file (PP_RETAIN_PER_FILE_MAX_BYTES) and - * globally (PP_RETAIN_TOTAL_BUDGET_BYTES) to bound peak RSS. - * Skipping retention just means cross-file LSP no-ops for this - * file — defs/calls already extracted are unaffected. */ - if (source_len > 0 && (int64_t)source_len <= PP_RETAIN_PER_FILE_MAX_BYTES) { - int64_t prior = atomic_fetch_add_explicit(&ec->retained_bytes, (int64_t)source_len, - memory_order_relaxed); - if (prior + (int64_t)source_len <= PP_RETAIN_TOTAL_BUDGET_BYTES) { - char *copy = (char *)cbm_arena_alloc(&result->arena, (size_t)source_len + 1); - if (copy) { - memcpy(copy, source, (size_t)source_len); - copy[source_len] = '\0'; - result->source = copy; - result->source_len = source_len; + /* Retain source bytes in result->arena so the fused cross-file LSP + * step in resolve_worker can re-parse without re-reading from disk. + * Bounded per-file (retain_per_file_max_bytes) and by a project-wide + * budget (retain_total_budget_bytes) to bound peak RSS — see the + * retention-cap comment at the top of this file. A file DROPPED here + * is NOT lost to cross-file resolution: resolve_worker re-reads it on + * demand (bounded, freed immediately). WARN once per run so the + * operator knows retention was capped (index.retain_capped). */ + if (ec->retain_sources && source_len > 0) { + bool dropped = false; + if ((size_t)source_len > ec->retain_per_file_max_bytes) { + dropped = true; /* over the per-file cap */ + } else { + int64_t prior = atomic_fetch_add_explicit(&ec->retained_bytes, (int64_t)source_len, + memory_order_relaxed); + if ((size_t)(prior + (int64_t)source_len) <= ec->retain_total_budget_bytes) { + char *copy = (char *)cbm_arena_alloc(&result->arena, (size_t)source_len + 1); + if (copy) { + memcpy(copy, source, (size_t)source_len); + copy[source_len] = '\0'; + result->source = copy; + result->source_len = source_len; + } else { + /* Arena OOM — not a cap drop; re-read still covers + * cross-file resolution, so don't emit the WARN. */ + atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len, + memory_order_relaxed); + } } else { atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len, memory_order_relaxed); + dropped = true; /* project-wide budget exhausted */ } - } else { - atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len, - memory_order_relaxed); + } + if (dropped && + atomic_exchange_explicit(&ec->retain_cap_warned, 1, memory_order_relaxed) == 0) { + cbm_log_warn("index.retain_capped", "path", fi->rel_path ? fi->rel_path : "?", + "bytes", itoa_log((int)source_len)); } } @@ -856,15 +983,24 @@ static void log_extract_mem_stats(int worker_count) { } } -int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, - CBMFileResult **result_cache, _Atomic int64_t *shared_ids, - int worker_count) { +int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, + CBMFileResult **result_cache, _Atomic int64_t *shared_ids, + int worker_count, const cbm_parallel_extract_opts_t *opts) { + cbm_parallel_extract_opts_t resolved_opts = cbm_parallel_extract_resolve_opts(opts); + if (file_count == 0) { return 0; } cbm_log_info("parallel.extract.start", "files", itoa_log(file_count), "workers", itoa_log(worker_count)); + { + size_t mb = (size_t)CBM_SZ_1K * CBM_SZ_1K; + cbm_log_info("parallel.extract.retention", "retain_sources", + resolved_opts.retain_sources ? "true" : "false", "total_mb", + itoa_log((int)(resolved_opts.retain_total_budget_bytes / mb)), "per_file_mb", + itoa_log((int)(resolved_opts.retain_per_file_max_bytes / mb))); + } /* Log per-worker memory budget */ if (cbm_mem_budget() > 0) { @@ -885,7 +1021,10 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, /* Sub-phase: Sort files by descending size for tail-latency reduction */ CBM_PROF_START(t_sort); - file_sort_entry_t *sorted = malloc(file_count * sizeof(file_sort_entry_t)); + file_sort_entry_t *sorted = malloc((size_t)file_count * sizeof(file_sort_entry_t)); + if (!sorted) { + return CBM_NOT_FOUND; + } for (int i = 0; i < file_count; i++) { sorted[i].idx = i; sorted[i].size = files[i].size; @@ -903,11 +1042,16 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, memset(workers, 0, (size_t)worker_count * sizeof(extract_worker_state_t)); /* Per-worker manifest entry arrays (separate from cache-line-aligned worker state) */ - cbm_pkg_entries_t *pkg_entries = calloc(worker_count, sizeof(cbm_pkg_entries_t)); + cbm_pkg_entries_t *pkg_entries = calloc((size_t)worker_count, sizeof(cbm_pkg_entries_t)); + if (!pkg_entries) { + cbm_aligned_free(workers); + free(sorted); + return CBM_NOT_FOUND; + } /* Per-worker skip lists (separate allocation; merged into the pipeline in the * sequential merge loop below). */ - pp_err_list_t *err_lists = calloc(worker_count, sizeof(pp_err_list_t)); + pp_err_list_t *err_lists = calloc((size_t)worker_count, sizeof(pp_err_list_t)); extract_ctx_t ec = { .files = files, @@ -922,16 +1066,20 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, .cancelled = ctx->cancelled, .pkg_entries = pkg_entries, .err_lists = err_lists, + .retain_sources = resolved_opts.retain_sources, + .retain_total_budget_bytes = resolved_opts.retain_total_budget_bytes, + .retain_per_file_max_bytes = resolved_opts.retain_per_file_max_bytes, }; atomic_init(&ec.next_worker_id, 0); atomic_init(&ec.next_file_idx, 0); atomic_init(&ec.retained_bytes, 0); + atomic_init(&ec.retain_cap_warned, 0); atomic_init(&ec.oversized_warned, 0); /* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */ CBM_PROF_START(t_dispatch); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, extract_worker, &ec, opts); + cbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count, .force_pthreads = false}; + cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts); CBM_PROF_END_N("parallel_extract", "3_dispatch_workers_parallel", t_dispatch, file_count); /* Sub-phase: Merge all local gbufs into main gbuf (SEQUENTIAL, gbuf not thread-safe) */ @@ -982,6 +1130,13 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, return 0; } +int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, + CBMFileResult **result_cache, _Atomic int64_t *shared_ids, + int worker_count) { + return cbm_parallel_extract_ex(ctx, files, file_count, result_cache, shared_ids, worker_count, + NULL); +} + /* ── Phase 3B: Serial Registry Build ─────────────────────────────── */ /* Register one definition and create DEFINES + DEFINES_METHOD edges. Returns edge count. */ @@ -2466,18 +2621,31 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { * Runs BEFORE resolve_file_calls so its additions to * result->resolved_calls are picked up by * cbm_pipeline_find_lsp_resolution when calls become CALLS - * edges. Requires result->source to have been retained in - * result->arena during extract (PP_RETAIN_*); files over the - * cap or past the budget have result->source==NULL and are - * counted as skipped_no_source — defs/calls already in the - * extract are unaffected. + * edges. Prefers source bytes retained in result->arena during + * extract; when the low-RAM retention cap dropped this file + * (result->source==NULL) it FALLS BACK to a bounded per-file + * read from disk, freed immediately after the LSP call. This is + * the correctness guarantee: lowering the retention cap trades + * retained RAM for a bounded re-read, it NEVER drops a cross-file + * edge. Only a genuine read failure (deleted/unreadable/oversized) + * leaves source NULL and is counted as skipped_no_source; defs/calls + * already in the extract are unaffected either way. * * Slab reclaim afterward: the LSP re-parses via tree-sitter, * which allocates through this worker's TLS slab. Reclaiming * here keeps the slab high-water bounded as the resolve phase * walks across thousands of files in a single worker thread. */ if (cross_lsp_eligible) { - if (result->source && result->source_len > 0) { + char *lsp_source_owned = NULL; + const char *lsp_source = result->source; + int lsp_source_len = result->source_len; + if ((!lsp_source || lsp_source_len <= 0) && rc->files[file_idx].path) { + /* Retention cap skipped this file — re-read on demand (bounded + * by read_file's cbm_max_file_bytes cap), freed below. */ + lsp_source_owned = read_file(rc->files[file_idx].path, &lsp_source_len, NULL, NULL); + lsp_source = lsp_source_owned; + } + if (lsp_source && lsp_source_len > 0) { const char *def_module = rc->def_modules ? rc->def_modules[file_idx] : module_qn; uint64_t lsp_t0 = extract_now_ns(); @@ -2511,8 +2679,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { } case CBM_LANG_PYTHON: cbm_run_py_lsp_cross_with_registry( - &result->arena, result->source, result->source_len, def_module, - prebuilt, imp_keys, imp_vals, imp_count, result->cached_tree, + &result->arena, lsp_source, lsp_source_len, def_module, prebuilt, + imp_keys, imp_vals, imp_count, result->cached_tree, &result->resolved_calls); used_prebuilt = true; break; @@ -2520,16 +2688,15 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { case CBM_LANG_CPP: case CBM_LANG_CUDA: cbm_run_c_lsp_cross_with_registry( - &result->arena, result->source, result->source_len, def_module, + &result->arena, lsp_source, lsp_source_len, def_module, (lang != CBM_LANG_C), prebuilt, imp_keys, imp_vals, imp_count, result->cached_tree, &result->resolved_calls); used_prebuilt = true; break; case CBM_LANG_CSHARP: - cbm_run_cs_lsp_cross_with_registry(&result->arena, result->source, - result->source_len, def_module, prebuilt, - imp_vals, imp_count, result->cached_tree, - &result->resolved_calls); + cbm_run_cs_lsp_cross_with_registry( + &result->arena, lsp_source, lsp_source_len, def_module, prebuilt, + imp_vals, imp_count, result->cached_tree, &result->resolved_calls); used_prebuilt = true; break; case CBM_LANG_JAVASCRIPT: @@ -2557,8 +2724,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { } } cbm_run_ts_lsp_cross_with_registry( - &result->arena, result->source, result->source_len, def_module, js, jsx, - dts, prebuilt, ts_defs, ts_def_count, imp_keys, imp_vals, imp_count, + &result->arena, lsp_source, lsp_source_len, def_module, js, jsx, dts, + prebuilt, ts_defs, ts_def_count, imp_keys, imp_vals, imp_count, result->cached_tree, &result->resolved_calls); free(ts_filtered); used_prebuilt = true; @@ -2590,16 +2757,17 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { lang == CBM_LANG_TSX) { bool js, jsx, dts; cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts); - cbm_pxc_run_one_ts(result, result->source, result->source_len, def_module, + cbm_pxc_run_one_ts(result, lsp_source, lsp_source_len, def_module, file_defs, file_def_count, imp_keys, imp_vals, imp_count, js, jsx, dts); } else { - cbm_pxc_run_one(lang, result, result->source, result->source_len, - def_module, file_defs, file_def_count, imp_keys, imp_vals, - imp_count); + cbm_pxc_run_one(lang, result, lsp_source, lsp_source_len, def_module, + file_defs, file_def_count, imp_keys, imp_vals, imp_count); } } free(filtered); + /* Free the on-demand re-read (no-op when source was retained). */ + free_source(lsp_source_owned); /* Contract: cbm_slab_reclaim() requires the thread parser to be * destroyed first; otherwise its lexer holds slab pointers * (lexer.included_ranges) that get freed underneath it, causing @@ -2617,13 +2785,14 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { } atomic_fetch_add_explicit(&rc->lsp_cross_processed, SKIP_ONE, memory_order_relaxed); } else { - /* No retained source → the cross-file LSP refinement no-ops for - * this file. This is a bounded OPTIMIZATION skip, NOT a file - * failure: defs/calls were already extracted and are unaffected. - * Deliberately NOT recorded as a cbm_file_error — doing so would - * flood skipped[] with false positives (itself a false-guard - * bug). The "cross_lsp" phase string is reserved for Track C's - * real crash-attribution signal; leave it unwired here. */ + /* Source unavailable even after the re-read fallback (file + * deleted / unreadable / oversized) → the cross-file LSP + * refinement no-ops for this file. This is a bounded skip, NOT + * a file failure: defs/calls were already extracted and are + * unaffected. Deliberately NOT recorded as a cbm_file_error — + * doing so would flood skipped[] with false positives (itself a + * false-guard bug). The "cross_lsp" phase string is reserved for + * Track C's real crash-attribution signal; leave it unwired. */ atomic_fetch_add_explicit(&rc->lsp_cross_skipped_no_source, SKIP_ONE, memory_order_relaxed); } diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 0beaf1b39..af8bf4b84 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -444,6 +444,21 @@ char *cbm_infra_qn(const char *project_name, const char *rel_path, const char *i * Each worker creates nodes in a per-worker gbuf, then merges into ctx->gbuf. * Caches CBMFileResult* in result_cache[file_idx] for reuse in Phase 3B/4. * shared_ids provides globally unique node/edge IDs across workers. */ + +/* Source-retention tuning for cbm_parallel_extract_ex. Zero-valued byte caps + * mean "use the derived default" (RAM-fraction total, clamped to an absolute + * ceiling; modest per-file cap); CBM_RETAIN_TOTAL_MB / CBM_RETAIN_PER_FILE_MB + * override those. retain_sources_set=false keeps the default retain policy. */ +typedef struct { + bool retain_sources; + bool retain_sources_set; /* false keeps the default retain_sources policy */ + size_t retain_total_budget_bytes; + size_t retain_per_file_max_bytes; +} cbm_parallel_extract_opts_t; + +int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, + CBMFileResult **result_cache, _Atomic int64_t *shared_ids, + int worker_count, const cbm_parallel_extract_opts_t *opts); int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, CBMFileResult **result_cache, _Atomic int64_t *shared_ids, int worker_count); diff --git a/tests/test_mem.c b/tests/test_mem.c index debb9b505..26bf04f1b 100644 --- a/tests/test_mem.c +++ b/tests/test_mem.c @@ -565,6 +565,167 @@ static void teardown_mem_test_repo(void) { } } +static size_t count_retained_source_bytes(CBMFileResult **result_cache, int file_count, + int *retained_count) { + size_t retained_bytes = 0; + int count = 0; + + for (int i = 0; i < file_count; i++) { + CBMFileResult *result = result_cache[i]; + if (result && result->source) { + retained_bytes += (size_t)result->source_len; + count++; + } + } + + if (retained_count) { + *retained_count = count; + } + return retained_bytes; +} + +/* retain_sources=false disables source retention entirely: no result->source is + * kept, yet extraction still produces defs/nodes. Guards the low-RAM opt-out. */ +TEST(parallel_extract_without_source_retention) { + if (setup_mem_test_repo() != 0) { + FAIL("tmpdir setup failed"); + } + + cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; + cbm_file_info_t *files = NULL; + int file_count = 0; + if (cbm_discover(g_mem_tmpdir, &opts, &files, &file_count) != 0) { + teardown_mem_test_repo(); + FAIL("discover failed"); + } + + cbm_gbuf_t *gbuf = cbm_gbuf_new("mem-test", g_mem_tmpdir); + cbm_registry_t *reg = cbm_registry_new(); + atomic_int cancelled; + atomic_init(&cancelled, 0); + + cbm_pipeline_ctx_t ctx = { + .project_name = "mem-test", + .repo_path = g_mem_tmpdir, + .gbuf = gbuf, + .registry = reg, + .cancelled = &cancelled, + }; + + _Atomic int64_t shared_ids; + atomic_init(&shared_ids, cbm_gbuf_next_id(gbuf)); + + CBMFileResult **result_cache = calloc((size_t)file_count, sizeof(CBMFileResult *)); + ASSERT_NOT_NULL(result_cache); + + cbm_parallel_extract_opts_t extract_opts = { + .retain_sources = false, + .retain_sources_set = true, + .retain_total_budget_bytes = 0, + .retain_per_file_max_bytes = 0, + }; + int rc = cbm_parallel_extract_ex(&ctx, files, file_count, result_cache, &shared_ids, 2, + &extract_opts); + ASSERT_EQ(rc, 0); + + int defs_seen = 0; + for (int i = 0; i < file_count; i++) { + if (result_cache[i]) { + ASSERT_EQ(result_cache[i]->source, NULL); + defs_seen += result_cache[i]->defs.count; + } + } + ASSERT_GT(defs_seen, 0); + ASSERT_GT(cbm_gbuf_node_count(gbuf), 0); + + for (int i = 0; i < file_count; i++) { + if (result_cache[i]) { + cbm_free_result(result_cache[i]); + } + } + free(result_cache); + cbm_registry_free(reg); + cbm_gbuf_free(gbuf); + cbm_discover_free(files, file_count); + teardown_mem_test_repo(); + PASS(); +} + +/* Guard B (peak bound): a tiny total retention budget must actually bound the + * retained source bytes — retained_bytes <= budget — while extraction still + * produces defs/nodes. Over-budget files fall back to a bounded re-read during + * cross-file resolution (exercised in test_parallel.c), so the cap trades + * retained RAM, never correctness. */ +TEST(parallel_extract_tiny_source_retention_budget) { + if (setup_mem_test_repo() != 0) { + FAIL("tmpdir setup failed"); + } + + cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; + cbm_file_info_t *files = NULL; + int file_count = 0; + if (cbm_discover(g_mem_tmpdir, &opts, &files, &file_count) != 0) { + teardown_mem_test_repo(); + FAIL("discover failed"); + } + + cbm_gbuf_t *gbuf = cbm_gbuf_new("mem-test", g_mem_tmpdir); + cbm_registry_t *reg = cbm_registry_new(); + atomic_int cancelled; + atomic_init(&cancelled, 0); + + cbm_pipeline_ctx_t ctx = { + .project_name = "mem-test", + .repo_path = g_mem_tmpdir, + .gbuf = gbuf, + .registry = reg, + .cancelled = &cancelled, + }; + + _Atomic int64_t shared_ids; + atomic_init(&shared_ids, cbm_gbuf_next_id(gbuf)); + + CBMFileResult **result_cache = calloc((size_t)file_count, sizeof(CBMFileResult *)); + ASSERT_NOT_NULL(result_cache); + + const size_t retain_total_budget_bytes = 256; + cbm_parallel_extract_opts_t extract_opts = { + .retain_sources = true, + .retain_sources_set = true, + .retain_total_budget_bytes = retain_total_budget_bytes, + .retain_per_file_max_bytes = 100U * 1024U * 1024U, + }; + int rc = cbm_parallel_extract_ex(&ctx, files, file_count, result_cache, &shared_ids, 2, + &extract_opts); + ASSERT_EQ(rc, 0); + + int retained_count = 0; + size_t retained_bytes = count_retained_source_bytes(result_cache, file_count, &retained_count); + int defs_seen = 0; + for (int i = 0; i < file_count; i++) { + if (result_cache[i]) { + defs_seen += result_cache[i]->defs.count; + } + } + + ASSERT_GT(defs_seen, 0); + ASSERT_GT(retained_count, 0); + ASSERT_LTE(retained_bytes, retain_total_budget_bytes); + ASSERT_GT(cbm_gbuf_node_count(gbuf), 0); + + for (int i = 0; i < file_count; i++) { + if (result_cache[i]) { + cbm_free_result(result_cache[i]); + } + } + free(result_cache); + cbm_registry_free(reg); + cbm_gbuf_free(gbuf); + cbm_discover_free(files, file_count); + teardown_mem_test_repo(); + PASS(); +} + TEST(parallel_extract_with_slab) { cbm_mem_init(0.5); @@ -666,5 +827,7 @@ SUITE(mem) { RUN_TEST(slab_calloc_zeroed); RUN_TEST(slab_mixed_alloc_free_stress); /* Integration */ + RUN_TEST(parallel_extract_without_source_retention); + RUN_TEST(parallel_extract_tiny_source_retention_budget); RUN_TEST(parallel_extract_with_slab); } diff --git a/tests/test_parallel.c b/tests/test_parallel.c index 680345d7d..919351af2 100644 --- a/tests/test_parallel.c +++ b/tests/test_parallel.c @@ -112,8 +112,10 @@ static cbm_gbuf_t *run_sequential(const char *project, const char *repo_path, /* ── Run parallel pipeline on files, returning gbuf ───────────────── */ -static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_file_info_t *files, - int file_count, int worker_count) { +static cbm_gbuf_t *run_parallel_with_extract_opts(const char *project, const char *repo_path, + cbm_file_info_t *files, int file_count, + int worker_count, + const cbm_parallel_extract_opts_t *extract_opts) { cbm_gbuf_t *gbuf = cbm_gbuf_new(project, repo_path); cbm_registry_t *reg = cbm_registry_new(); atomic_int cancelled; @@ -131,10 +133,15 @@ static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_ int64_t gbuf_next = cbm_gbuf_next_id(gbuf); atomic_init(&shared_ids, gbuf_next); - CBMFileResult **result_cache = calloc(file_count, sizeof(CBMFileResult *)); + CBMFileResult **result_cache = calloc((size_t)file_count, sizeof(CBMFileResult *)); cbm_init(); - cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, worker_count); + if (extract_opts) { + cbm_parallel_extract_ex(&ctx, files, file_count, result_cache, &shared_ids, worker_count, + extract_opts); + } else { + cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, worker_count); + } cbm_gbuf_set_next_id(gbuf, atomic_load(&shared_ids)); cbm_build_registry_from_cache(&ctx, files, file_count, result_cache); @@ -175,6 +182,12 @@ static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_ return gbuf; } +static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_file_info_t *files, + int file_count, int worker_count) { + return run_parallel_with_extract_opts(project, repo_path, files, file_count, worker_count, + NULL); +} + /* ── Parity Tests ─────────────────────────────────────────────────── */ static cbm_gbuf_t *g_seq_gbuf = NULL; @@ -741,8 +754,7 @@ TEST(parallel_lsp_tail_match_fallbacks_gated_to_jvm) { int64_t nid = cbm_gbuf_upsert_node(tgbuf, "Method", "run", "proj.zeta.Helper.run", "zeta/helper.py", 1, 3, NULL); ASSERT_TRUE(nid != 0); - ASSERT_TRUE(cbm_pipeline_lsp_target_node(tgbuf, "proj", "com.other.Helper.run", false) == - NULL); + ASSERT_TRUE(cbm_pipeline_lsp_target_node(tgbuf, "proj", "com.other.Helper.run", false) == NULL); const cbm_gbuf_node_t *jvm_hit = cbm_pipeline_lsp_target_node(tgbuf, "proj", "com.other.Helper.run", true); ASSERT_NOT_NULL(jvm_hit); @@ -884,6 +896,129 @@ TEST(parallel_python_lsp_override_cross_file_emits_lsp_strategy_edges) { PASS(); } +/* RED/GREEN A — the graph-quality guarantee behind the low-RAM retention cap. + * + * The fused cross-file LSP step re-parses each file's source to resolve calls + * whose receiver type lives in ANOTHER file (the per-file pass cannot). When + * the retention cap drops a file's source, that resolution MUST still happen + * via a bounded on-demand re-read; otherwise the cross-file CALLS edge is LOST. + * + * Fixture: a Java<->Kotlin pair with genuinely cross-language calls that only + * the cross-file LSP resolves — JavaCaller.call -> KotlinService.ping (Java -> + * Kotlin) and KotlinService.ping -> JavaService.pong (Kotlin -> Java). These + * carry the "lsp" strategy and do NOT exist without the cross-file source, + * unlike same-file or import-local Python calls which the per-file pass already + * resolves (so counting lsp edges on those cannot detect the fallback). + * + * Three scenarios asserted GREEN with the re-read fallback in place: + * 1. CONTROL — default retention: both cross-file edges present. Proves the + * fixture genuinely produces them (non-vacuity guard). + * 2. NO-RETAIN — retain_sources=false: nothing retained -> edges survive only + * via the re-read fallback. + * 3. OVER-CAP — per-file cap = 1 byte: every file dropped by the SIZE cap -> + * edges survive only via the re-read fallback. + * On main (no fallback) scenarios 2 and 3 LOSE both edges = RED; scenario 1 + * stays present = the non-vacuity control. */ +TEST(parallel_cross_file_reread_preserves_unretained_edges) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_par_xf_reread_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) { + FAIL("mkdtemp failed"); + } + + char jpath[512]; + snprintf(jpath, sizeof(jpath), "%s/src/main/java/com/example/Example.java", tmpdir); + char jdir[512]; + snprintf(jdir, sizeof(jdir), "%s/src/main/java/com/example", tmpdir); + cbm_mkdir_p(jdir, 0755); + FILE *jf = fopen(jpath, "w"); + if (!jf) { + FAIL("fopen Example.java failed"); + } + fprintf(jf, "package com.example;\n" + "\n" + "class JavaCaller {\n" + " String call(KotlinService kotlinService) {\n" + " return kotlinService.ping(new JavaService());\n" + " }\n" + "}\n" + "\n" + "class JavaService {\n" + " String pong() {\n" + " return \"pong\";\n" + " }\n" + "}\n"); + fclose(jf); + + char kpath[512]; + snprintf(kpath, sizeof(kpath), "%s/src/main/kotlin/com/example/KotlinService.kt", tmpdir); + char kdir[512]; + snprintf(kdir, sizeof(kdir), "%s/src/main/kotlin/com/example", tmpdir); + cbm_mkdir_p(kdir, 0755); + FILE *kf = fopen(kpath, "w"); + if (!kf) { + FAIL("fopen KotlinService.kt failed"); + } + fprintf(kf, "package com.example\n" + "\n" + "class KotlinService {\n" + " fun ping(javaService: JavaService): String {\n" + " return javaService.pong()\n" + " }\n" + "}\n"); + fclose(kf); + + cbm_file_info_t files[2] = {0}; + files[0].path = jpath; + files[0].rel_path = (char *)"src/main/java/com/example/Example.java"; + files[0].language = CBM_LANG_JAVA; + files[1].path = kpath; + files[1].rel_path = (char *)"src/main/kotlin/com/example/KotlinService.kt"; + files[1].language = CBM_LANG_KOTLIN; + + /* CONTROL (retained) + two drop scenarios that reach the cross-file edge + * only via the on-demand re-read: NO-RETAIN disables retention entirely; + * OVER-CAP sets a 1-byte per-file cap so every file is dropped by size. */ + const cbm_parallel_extract_opts_t no_retain = { + .retain_sources = false, + .retain_sources_set = true, + }; + const cbm_parallel_extract_opts_t over_cap = { + .retain_sources = true, + .retain_sources_set = true, + .retain_per_file_max_bytes = 1, /* 1 byte → every file dropped by the size cap */ + }; + const cbm_parallel_extract_opts_t *scenarios[3] = {NULL, &no_retain, &over_cap}; + + for (int s = 0; s < 3; s++) { + cbm_gbuf_t *gbuf = run_parallel_with_extract_opts("com", tmpdir, files, 2, 2, scenarios[s]); + ASSERT_NOT_NULL(gbuf); + + const cbm_gbuf_edge_t *java_to_kotlin = + find_calls_edge_by_tails(gbuf, "JavaCaller.call", "KotlinService.ping"); + const cbm_gbuf_edge_t *kotlin_to_java = + find_calls_edge_by_tails(gbuf, "KotlinService.ping", "JavaService.pong"); + + /* Both cross-file (Java↔Kotlin) CALLS edges must be present in EVERY + * scenario. In the drop scenarios (s=1,2) the caller's source is NOT + * retained, so these edges exist ONLY because resolve_worker re-reads + * the source on demand. Without that fallback (main) they are LOST. */ + ASSERT_NOT_NULL(java_to_kotlin); + ASSERT_NOT_NULL(kotlin_to_java); + /* And they must come from the source-dependent cross-file LSP, not a + * source-free suffix heuristic — proving the re-read actually ran. */ + ASSERT_NOT_NULL(java_to_kotlin->properties_json); + ASSERT_NOT_NULL(strstr(java_to_kotlin->properties_json, "\"strategy\":\"lsp")); + ASSERT_NOT_NULL(kotlin_to_java->properties_json); + ASSERT_NOT_NULL(strstr(kotlin_to_java->properties_json, "\"strategy\":\"lsp")); + + cbm_gbuf_free(gbuf); + } + + th_rmtree(tmpdir); + PASS(); +} + /* issue #294: gRPC service-name extraction must (a) preserve the canonical * proto service name (FooServiceClient → FooService, not Foo) and (b) only * match real stub/client types — ordinary receiver vars must NOT produce @@ -941,6 +1076,7 @@ SUITE(parallel) { RUN_TEST(parallel_node_count); RUN_TEST(parallel_python_lsp_override_emits_lsp_strategy_edges); RUN_TEST(parallel_python_lsp_override_cross_file_emits_lsp_strategy_edges); + RUN_TEST(parallel_cross_file_reread_preserves_unretained_edges); RUN_TEST(parallel_java_kotlin_lsp_override_cross_file_emits_lsp_strategy_edges); RUN_TEST(parallel_lsp_tail_match_fallbacks_gated_to_jvm); RUN_TEST(parallel_calls_parity);