Skip to content

[WIP] Assert on redundant indexing JSON writes #745

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jviotti wants to merge 2 commits into from
Closed

[WIP] Assert on redundant indexing JSON writes #745

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3108bcd
[WIP] Assert on redundant indexing JSON writes
jviotti Mar 17, 2026
a5152a6
Incremental flag
jviotti Mar 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 36 additions & 22 deletions src/build/delta.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,9 +349,9 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
}
}

dependents_wave.push_back({BuildPlan::Action::Type::Dependents,
std::move(destination),
std::move(action_dependencies), uri});
dependents_wave.push_back(
{BuildPlan::Action::Type::Dependents, std::move(destination),
std::move(action_dependencies), uri, incremental});
}

std::ranges::sort(dependents_wave, [](const BuildPlan::Action &left,
Expand Down Expand Up @@ -506,7 +506,8 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
plan.waves.push_back({{.type = BuildPlan::Action::Type::Comment,
.destination = comment_path,
.dependencies = {},
.data = comment}});
.data = comment,
.incremental = incremental}});
plan.size = 1;
return plan;
}
Expand All @@ -518,7 +519,8 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
plan.waves.push_back({{.type = BuildPlan::Action::Type::Remove,
.destination = comment_path,
.dependencies = {},
.data = {}}});
.data = {},
.incremental = incremental}});
plan.size = 1;
return plan;
}
Expand Down Expand Up @@ -758,7 +760,8 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
plan.waves.push_back({{.type = BuildPlan::Action::Type::Comment,
.destination = comment_path,
.dependencies = {},
.data = comment}});
.data = comment,
.incremental = incremental}});
plan.size = 1;
return plan;
}
Expand All @@ -770,7 +773,8 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
plan.waves.push_back({{.type = BuildPlan::Action::Type::Remove,
.destination = comment_path,
.dependencies = {},
.data = {}}});
.data = {},
.incremental = incremental}});
plan.size = 1;
return plan;
}
Expand Down Expand Up @@ -994,9 +998,9 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
action_deps.emplace_back(std::move(dep));
}

dag_waves[wave].push_back({target.action,
std::filesystem::path{target_path},
std::move(action_deps), target.data});
dag_waves[wave].push_back(
{target.action, std::filesystem::path{target_path},
std::move(action_deps), target.data, incremental});
}
}

Expand All @@ -1014,13 +1018,15 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
std::filesystem::path{make_base_string(
output_string, SCHEMAS_DIRECTORY, relative_string)},
{},
{}});
{},
incremental});
remove_wave.push_back(
{BuildPlan::Action::Type::Remove,
std::filesystem::path{make_base_string(
output_string, EXPLORER_DIRECTORY, relative_string)},
{},
{}});
{},
incremental});
}

for (const auto &schema : active_schemas) {
Expand All @@ -1043,7 +1049,8 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
remove_wave.push_back({BuildPlan::Action::Type::Remove,
std::filesystem::path{std::move(target_path)},
{},
{}});
{},
incremental});
}
}
}
Expand Down Expand Up @@ -1138,7 +1145,8 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
remove_wave.push_back({BuildPlan::Action::Type::Remove,
std::filesystem::path{root},
{},
{}});
{},
incremental});
}
}

Expand Down Expand Up @@ -1173,7 +1181,8 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
remove_wave.push_back({BuildPlan::Action::Type::Remove,
std::filesystem::path{entry_path},
{},
{}});
{},
incremental});
web_removed = true;
}
}
Expand Down Expand Up @@ -1210,29 +1219,33 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
init_wave.push_back({.type = BuildPlan::Action::Type::Version,
.destination = version_path,
.dependencies = {},
.data = version});
.data = version,
.incremental = incremental});
init_wave.push_back({.type = BuildPlan::Action::Type::Configuration,
.destination = configuration_path,
.dependencies = {},
.data = {}});
.data = {},
.incremental = incremental});
if (!comment.empty()) {
init_wave.push_back({.type = BuildPlan::Action::Type::Comment,
.destination = comment_path,
.dependencies = {},
.data = comment});
.data = comment,
.incremental = incremental});
} else if (entries.contains(comment_string)) {
remove_wave.push_back(
{BuildPlan::Action::Type::Remove, comment_path, {}, {}});
{BuildPlan::Action::Type::Remove, comment_path, {}, {}, incremental});
}
plan.waves.push_back(std::move(init_wave));
} else if (!comment.empty()) {
plan.waves.push_back({{.type = BuildPlan::Action::Type::Comment,
.destination = comment_path,
.dependencies = {},
.data = comment}});
.data = comment,
.incremental = incremental}});
} else if (entries.contains(comment_string)) {
remove_wave.push_back(
{BuildPlan::Action::Type::Remove, comment_path, {}, {}});
{BuildPlan::Action::Type::Remove, comment_path, {}, {}, incremental});
}

for (auto &wave : dag_waves) {
Expand All @@ -1252,7 +1265,8 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
{rule.action,
output / rule.filename,
{configuration_path},
build_type == BuildPlan::Type::Full ? "Full" : "Headless"});
build_type == BuildPlan::Type::Full ? "Full" : "Headless",
incremental});
}

if (!global_wave.empty()) {
Expand Down
1 change: 1 addition & 0 deletions src/build/include/sourcemeta/one/build.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ struct BuildPlan {
std::filesystem::path destination;
Dependencies dependencies;
std::string_view data;
bool incremental;
};

std::filesystem::path output;
Expand Down
6 changes: 4 additions & 2 deletions src/index/explorer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,8 @@ struct GENERATE_EXPLORER_SCHEMA_METADATA {
action.destination, result, "application/json",
sourcemeta::one::Encoding::GZIP, sourcemeta::core::JSON{nullptr},
std::chrono::duration_cast<std::chrono::milliseconds>(timestamp_end -
timestamp_start));
timestamp_start),
action.incremental);
return true;
}
};
Expand Down Expand Up @@ -440,7 +441,8 @@ struct GENERATE_EXPLORER_DIRECTORY_LIST {
action.destination, meta, "application/json",
sourcemeta::one::Encoding::GZIP, sourcemeta::core::JSON{nullptr},
std::chrono::duration_cast<std::chrono::milliseconds>(timestamp_end -
timestamp_start));
timestamp_start),
action.incremental);
return true;
}
};
Expand Down
39 changes: 26 additions & 13 deletions src/index/generators.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,8 @@ struct GENERATE_MATERIALISED_SCHEMA {
sourcemeta::one::Encoding::GZIP,
sourcemeta::core::JSON{std::string{dialect_identifier}},
std::chrono::duration_cast<std::chrono::milliseconds>(timestamp_end -
timestamp_start));
timestamp_start),
action.incremental);
resolver.cache_path(action.data, action.destination);
return true;
}
Expand Down Expand Up @@ -169,7 +170,8 @@ struct GENERATE_POINTER_POSITIONS {
action.destination, result, "application/json",
sourcemeta::one::Encoding::GZIP, sourcemeta::core::JSON{nullptr},
std::chrono::duration_cast<std::chrono::milliseconds>(timestamp_end -
timestamp_start));
timestamp_start),
action.incremental);
return true;
}
};
Expand Down Expand Up @@ -198,7 +200,8 @@ struct GENERATE_FRAME_LOCATIONS {
action.destination, result, "application/json",
sourcemeta::one::Encoding::GZIP, sourcemeta::core::JSON{nullptr},
std::chrono::duration_cast<std::chrono::milliseconds>(timestamp_end -
timestamp_start));
timestamp_start),
action.incremental);
return true;
}
};
Expand Down Expand Up @@ -236,7 +239,8 @@ struct GENERATE_DEPENDENCIES {
action.destination, result, "application/json",
sourcemeta::one::Encoding::GZIP, sourcemeta::core::JSON{nullptr},
std::chrono::duration_cast<std::chrono::milliseconds>(timestamp_end -
timestamp_start));
timestamp_start),
action.incremental);
return true;
}

Expand Down Expand Up @@ -321,7 +325,8 @@ struct GENERATE_DEPENDENTS {
action.destination, result, "application/json",
sourcemeta::one::Encoding::GZIP, sourcemeta::core::JSON{nullptr},
std::chrono::duration_cast<std::chrono::milliseconds>(timestamp_end -
timestamp_start));
timestamp_start),
action.incremental);
return true;
}
};
Expand Down Expand Up @@ -379,7 +384,8 @@ struct GENERATE_HEALTH {
action.destination, report, "application/json",
sourcemeta::one::Encoding::GZIP, sourcemeta::core::JSON{nullptr},
std::chrono::duration_cast<std::chrono::milliseconds>(timestamp_end -
timestamp_start));
timestamp_start),
action.incremental);
return true;
}

Expand Down Expand Up @@ -454,7 +460,8 @@ struct GENERATE_BUNDLE {
sourcemeta::one::Encoding::GZIP,
sourcemeta::core::JSON{std::string{dialect_identifier}},
std::chrono::duration_cast<std::chrono::milliseconds>(timestamp_end -
timestamp_start));
timestamp_start),
action.incremental);
return true;
}
};
Expand Down Expand Up @@ -488,15 +495,17 @@ struct GENERATE_EDITOR {
sourcemeta::one::Encoding::GZIP,
sourcemeta::core::JSON{std::string{dialect_identifier}},
std::chrono::duration_cast<std::chrono::milliseconds>(timestamp_end -
timestamp_start));
timestamp_start),
action.incremental);
return true;
}
};

static auto generate_blaze_template(
const std::filesystem::path &destination,
const sourcemeta::one::BuildPlan::Action::Dependencies &dependencies,
const sourcemeta::blaze::Mode mode) -> void {
const sourcemeta::blaze::Mode mode, const bool enable_debug_checks)
-> void {
const auto timestamp_start{std::chrono::steady_clock::now()};
const auto contents{sourcemeta::one::read_json(dependencies.front())};
sourcemeta::core::SchemaFrame frame{
Expand All @@ -514,7 +523,8 @@ static auto generate_blaze_template(
destination, result, "application/json", sourcemeta::one::Encoding::GZIP,
sourcemeta::core::JSON{nullptr},
std::chrono::duration_cast<std::chrono::milliseconds>(timestamp_end -
timestamp_start));
timestamp_start),
enable_debug_checks);
}

struct GENERATE_BLAZE_TEMPLATE_EXHAUSTIVE {
Expand All @@ -525,7 +535,8 @@ struct GENERATE_BLAZE_TEMPLATE_EXHAUSTIVE {
const sourcemeta::one::Configuration &,
const sourcemeta::core::JSON &) -> bool {
generate_blaze_template(action.destination, action.dependencies,
sourcemeta::blaze::Mode::Exhaustive);
sourcemeta::blaze::Mode::Exhaustive,
action.incremental);
return true;
}
};
Expand All @@ -538,7 +549,8 @@ struct GENERATE_BLAZE_TEMPLATE_FAST {
const sourcemeta::one::Configuration &,
const sourcemeta::core::JSON &) -> bool {
generate_blaze_template(action.destination, action.dependencies,
sourcemeta::blaze::Mode::FastValidation);
sourcemeta::blaze::Mode::FastValidation,
action.incremental);
return true;
}
};
Expand Down Expand Up @@ -583,7 +595,8 @@ struct GENERATE_STATS {
"application/json", sourcemeta::one::Encoding::GZIP,
sourcemeta::core::JSON{nullptr},
std::chrono::duration_cast<std::chrono::milliseconds>(timestamp_end -
timestamp_start));
timestamp_start),
action.incremental);
return true;
}
};
Expand Down
6 changes: 4 additions & 2 deletions src/shared/include/sourcemeta/one/shared_metapack.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,16 @@ auto write_json(const std::filesystem::path &destination,
const sourcemeta::core::JSON::String &mime,
const Encoding encoding,
const sourcemeta::core::JSON &extension,
const std::chrono::milliseconds duration) -> void;
const std::chrono::milliseconds duration,
const bool enable_debug_checks) -> void;

auto write_pretty_json(const std::filesystem::path &destination,
const sourcemeta::core::JSON &document,
const sourcemeta::core::JSON::String &mime,
const Encoding encoding,
const sourcemeta::core::JSON &extension,
const std::chrono::milliseconds duration) -> void;
const std::chrono::milliseconds duration,
const bool enable_debug_checks) -> void;

auto write_text(const std::filesystem::path &destination,
const std::string_view contents,
Expand Down
14 changes: 12 additions & 2 deletions src/shared/metapack.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@
#include <sourcemeta/core/io.h>
#include <sourcemeta/core/time.h>

// TODO: Add this macro to all other file types too
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define ASSERT_NOT_REDUNDANT_WRITE(destination, document, debug_checks) \
assert(!(debug_checks) || !std::filesystem::exists(destination) || \
read_json(destination) != (document))

#include <cassert> // assert
#include <chrono> // std::chrono::system_clock::time_point
#include <functional> // std::functional
Expand Down Expand Up @@ -162,7 +168,9 @@ auto write_json(const std::filesystem::path &destination,
const sourcemeta::core::JSON::String &mime,
const Encoding encoding,
const sourcemeta::core::JSON &extension,
const std::chrono::milliseconds duration) -> void {
const std::chrono::milliseconds duration,
const bool enable_debug_checks) -> void {
ASSERT_NOT_REDUNDANT_WRITE(destination, document, enable_debug_checks);
write_stream(destination, mime, encoding, extension, duration,
[&document](auto &stream) {
sourcemeta::core::stringify(document, stream);
Expand All @@ -174,7 +182,9 @@ auto write_pretty_json(const std::filesystem::path &destination,
const sourcemeta::core::JSON::String &mime,
const Encoding encoding,
const sourcemeta::core::JSON &extension,
const std::chrono::milliseconds duration) -> void {
const std::chrono::milliseconds duration,
const bool enable_debug_checks) -> void {
ASSERT_NOT_REDUNDANT_WRITE(destination, document, enable_debug_checks);
write_stream(destination, mime, encoding, extension, duration,
[&document](auto &stream) {
sourcemeta::core::prettify(document, stream);
Expand Down
Loading