From 4f41ae1e4ceda61634cb5ac908f8b9f9b1c6a115 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Wed, 18 Mar 2026 13:48:12 -0400 Subject: [PATCH 1/3] Improve indexing performance Signed-off-by: Juan Cruz Viotti --- src/build/delta.cc | 64 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/src/build/delta.cc b/src/build/delta.cc index bc81e91f..c490cf98 100644 --- a/src/build/delta.cc +++ b/src/build/delta.cc @@ -317,6 +317,49 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type, plan.type = build_type; if (!affected_schemas.empty()) { + // Build a reverse index: for each schema relative path, collect + // the dependencies.metapack keys that reference it. This is a + // single O(keys) pass instead of O(affected × keys). + std::unordered_map> + reverse_dep_index; + for (const auto dep_key : entries.keys()) { + if (!dep_key.ends_with("/%/dependencies.metapack")) { + continue; + } + + const auto *dep_entry{entries.entry(std::string{dep_key})}; + if (dep_entry == nullptr) { + continue; + } + + for (const auto &dependency : dep_entry->dependencies) { + const auto &dep_path{dependency.native()}; + if (!dep_path.starts_with(schemas_prefix)) { + continue; + } + + const auto sentinel_pos{dep_path.find("/%/", owner_start)}; + if (sentinel_pos == std::string::npos) { + continue; + } + + auto referenced_schema{ + dep_path.substr(owner_start, sentinel_pos - owner_start)}; + if (affected_schemas.contains(referenced_schema)) { + reverse_dep_index[std::move(referenced_schema)].emplace_back( + dep_key); + } + } + } + + // Deduplicate: a single dep_key may reference the same schema + // through multiple dependency paths + for (auto &[schema, dep_keys] : reverse_dep_index) { + std::ranges::sort(dep_keys); + const auto [first, last] = std::ranges::unique(dep_keys); + dep_keys.erase(first, last); + } + std::vector dependents_wave; for (const auto &[uri, info] : schemas) { const auto &relative_string{info.relative_path.native()}; @@ -330,22 +373,11 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type, append_filename(schema_base, "dependents.metapack")}}; BuildPlan::Action::Dependencies action_dependencies; - for (const auto dep_key : entries.keys()) { - if (!dep_key.ends_with("/%/dependencies.metapack")) { - continue; - } - - const auto *dep_entry{entries.entry(std::string{dep_key})}; - if (dep_entry == nullptr) { - continue; - } - - const auto target_prefix{schemas_prefix + relative_string + "/%/"}; - for (const auto &dependency : dep_entry->dependencies) { - if (dependency.native().starts_with(target_prefix)) { - action_dependencies.emplace_back(std::string{dep_key}); - break; - } + const auto reverse_it{reverse_dep_index.find(relative_string)}; + if (reverse_it != reverse_dep_index.end()) { + action_dependencies.reserve(reverse_it->second.size()); + for (const auto &dep_key : reverse_it->second) { + action_dependencies.emplace_back(dep_key); } } From dc1e2071d994a1f46005d23609e3fd0d8afe3a27 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Wed, 18 Mar 2026 14:02:25 -0400 Subject: [PATCH 2/3] More Signed-off-by: Juan Cruz Viotti --- src/build/delta.cc | 70 ++++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/src/build/delta.cc b/src/build/delta.cc index c490cf98..e7f2fb21 100644 --- a/src/build/delta.cc +++ b/src/build/delta.cc @@ -679,41 +679,51 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type, } } - bool propagation_changed{true}; - while (propagation_changed) { - propagation_changed = false; - for (const auto &[target_path, target] : targets) { - if (dirty_set.contains(target_path)) { - continue; - } + // Phase 1: Single pass over all targets to find state-based dirtiness + // and build a reverse adjacency from declared dependencies. + std::unordered_map> + reverse_dag; + for (const auto &[target_path, target] : targets) { + for (const auto &dep : target.dependencies) { + reverse_dag[dep].push_back(target_path); + } - for (const auto &dep : target.dependencies) { - if (dirty_set.contains(dep)) { - dirty_set.insert(target_path); - propagation_changed = true; - goto next_target; - } - } + if (dirty_set.contains(target_path)) { + continue; + } - { - const auto *state_entry{entries.entry(target_path)}; - if (state_entry == nullptr) { - dirty_set.insert(target_path); - propagation_changed = true; - continue; - } + const auto *state_entry{entries.entry(target_path)}; + if (state_entry == nullptr) { + dirty_set.insert(target_path); + continue; + } - for (const auto &dep : state_entry->dependencies) { - if (dirty_set.contains(dep.native()) || - removed_entries.contains(dep.native())) { - dirty_set.insert(target_path); - propagation_changed = true; - break; - } - } + for (const auto &dep : state_entry->dependencies) { + if (dirty_set.contains(dep.native()) || + removed_entries.contains(dep.native())) { + dirty_set.insert(target_path); + break; } + } + } - next_target:; + // Phase 2: BFS from dirty targets through the declared DAG. + std::vector worklist; + worklist.reserve(dirty_set.size()); + for (const auto &dirty_path : dirty_set) { + worklist.push_back(dirty_path); + } + + for (std::size_t i{0}; i < worklist.size(); i++) { + const auto reverse_it{reverse_dag.find(worklist[i])}; + if (reverse_it == reverse_dag.end()) { + continue; + } + + for (const auto &dependent : reverse_it->second) { + if (dirty_set.insert(std::string{dependent}).second) { + worklist.push_back(dependent); + } } } } From 012c5bb12f31777883129345c4fafa9838086779 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Wed, 18 Mar 2026 14:06:30 -0400 Subject: [PATCH 3/3] More Signed-off-by: Juan Cruz Viotti --- src/build/delta.cc | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/build/delta.cc b/src/build/delta.cc index e7f2fb21..f0f0be79 100644 --- a/src/build/delta.cc +++ b/src/build/delta.cc @@ -373,10 +373,10 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type, append_filename(schema_base, "dependents.metapack")}}; BuildPlan::Action::Dependencies action_dependencies; - const auto reverse_it{reverse_dep_index.find(relative_string)}; - if (reverse_it != reverse_dep_index.end()) { - action_dependencies.reserve(reverse_it->second.size()); - for (const auto &dep_key : reverse_it->second) { + const auto reverse_iterator{reverse_dep_index.find(relative_string)}; + if (reverse_iterator != reverse_dep_index.end()) { + action_dependencies.reserve(reverse_iterator->second.size()); + for (const auto &dep_key : reverse_iterator->second) { action_dependencies.emplace_back(dep_key); } } @@ -682,10 +682,10 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type, // Phase 1: Single pass over all targets to find state-based dirtiness // and build a reverse adjacency from declared dependencies. std::unordered_map> - reverse_dag; + reverse_adjacency; for (const auto &[target_path, target] : targets) { - for (const auto &dep : target.dependencies) { - reverse_dag[dep].push_back(target_path); + for (const auto &dependency : target.dependencies) { + reverse_adjacency[dependency].push_back(target_path); } if (dirty_set.contains(target_path)) { @@ -698,9 +698,9 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type, continue; } - for (const auto &dep : state_entry->dependencies) { - if (dirty_set.contains(dep.native()) || - removed_entries.contains(dep.native())) { + for (const auto &dependency : state_entry->dependencies) { + if (dirty_set.contains(dependency.native()) || + removed_entries.contains(dependency.native())) { dirty_set.insert(target_path); break; } @@ -714,13 +714,13 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type, worklist.push_back(dirty_path); } - for (std::size_t i{0}; i < worklist.size(); i++) { - const auto reverse_it{reverse_dag.find(worklist[i])}; - if (reverse_it == reverse_dag.end()) { + for (std::size_t index{0}; index < worklist.size(); index++) { + const auto match{reverse_adjacency.find(worklist[index])}; + if (match == reverse_adjacency.end()) { continue; } - for (const auto &dependent : reverse_it->second) { + for (const auto &dependent : match->second) { if (dirty_set.insert(std::string{dependent}).second) { worklist.push_back(dependent); }