diff --git a/CMakeLists.txt b/CMakeLists.txt index c60c9065..4c0417b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.16) -project(one VERSION 5.1.0 LANGUAGES C CXX) +project(one VERSION 5.1.1 LANGUAGES C CXX) list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") # Options diff --git a/src/configuration/include/sourcemeta/one/configuration.h b/src/configuration/include/sourcemeta/one/configuration.h index f139f50a..df8d32ab 100644 --- a/src/configuration/include/sourcemeta/one/configuration.h +++ b/src/configuration/include/sourcemeta/one/configuration.h @@ -9,12 +9,18 @@ #include // std::filesystem::path #include // std::optional +#include // std::string #include // std::unordered_map +#include // std::unordered_set #include // std::variant namespace sourcemeta::one { struct Configuration { + static auto read(const std::filesystem::path &configuration_path, + const std::filesystem::path &collections_path, + std::unordered_set &configuration_files) + -> sourcemeta::core::JSON; static auto read(const std::filesystem::path &configuration_path, const std::filesystem::path &collections_path) -> sourcemeta::core::JSON; diff --git a/src/configuration/read.cc b/src/configuration/read.cc index 1faa75b5..a7c68044 100644 --- a/src/configuration/read.cc +++ b/src/configuration/read.cc @@ -52,7 +52,8 @@ auto dereference(const std::filesystem::path &collections_path, const std::filesystem::path &base, sourcemeta::core::JSON &input, const sourcemeta::core::Pointer &location, - std::unordered_set &visited) -> void { + std::unordered_set &visited, + std::unordered_set &all_files) -> void { assert(base.is_absolute()); if (!input.is_object()) { return; @@ -71,11 +72,12 @@ auto dereference(const std::filesystem::path &collections_path, throw sourcemeta::one::ConfigurationCyclicReferenceError( base, new_location, target_path); } + all_files.emplace(target_path.native()); auto extension{ read_file(base, new_location, target_path, entry.to_string())}; if (extension.is_object()) { dereference(collections_path, target_path, extension, new_location, - visited); + visited, all_files); accumulator.merge(std::move(extension).as_object()); } @@ -87,7 +89,7 @@ auto dereference(const std::filesystem::path &collections_path, accumulator.merge(input.as_object()); input = std::move(accumulator); assert(!input.defines("extends")); - dereference(collections_path, base, input, location, visited); + dereference(collections_path, base, input, location, visited, all_files); // Read included files } else if (!location.empty() && input.defines("include") && @@ -103,9 +105,11 @@ auto dereference(const std::filesystem::path &collections_path, throw sourcemeta::one::ConfigurationCyclicReferenceError( base, new_location, target_path); } + all_files.emplace(target_path.native()); input.into(read_file(base, new_location, target_path, input.at("include").to_string())); - dereference(collections_path, target_path, input, new_location, visited); + dereference(collections_path, target_path, input, new_location, visited, + all_files); visited.erase(target_path.native()); // Revisit and relativize paths @@ -130,7 +134,7 @@ auto dereference(const std::filesystem::path &collections_path, [](const auto &entry) { return entry.first; }); for (const auto &key : keys) { dereference(collections_path, base, input.at("contents").at(key), - location.concat({"contents", key}), visited); + location.concat({"contents", key}), visited, all_files); } } } @@ -163,7 +167,8 @@ auto default_base_uri(sourcemeta::core::JSON &contents, namespace sourcemeta::one { auto Configuration::read(const std::filesystem::path &configuration_path, - const std::filesystem::path &collections_path) + const std::filesystem::path &collections_path, + std::unordered_set &configuration_files) -> sourcemeta::core::JSON { auto data{sourcemeta::core::read_json(configuration_path)}; @@ -182,10 +187,13 @@ auto Configuration::read(const std::filesystem::path &configuration_path, sourcemeta::core::JSON{"The next-generation JSON Schema platform"}); } + const auto canonical_config{ + std::filesystem::weakly_canonical(configuration_path).native()}; + configuration_files.emplace(canonical_config); std::unordered_set visited; - visited.emplace( - std::filesystem::weakly_canonical(configuration_path).native()); - dereference(collections_path, configuration_path, data, {}, visited); + visited.emplace(canonical_config); + dereference(collections_path, configuration_path, data, {}, visited, + configuration_files); if (data.is_object() && data.defines("url") && data.defines("contents") && data.at("contents").is_object()) { @@ -195,4 +203,11 @@ auto Configuration::read(const std::filesystem::path &configuration_path, return data; } +auto Configuration::read(const std::filesystem::path &configuration_path, + const std::filesystem::path &collections_path) + -> sourcemeta::core::JSON { + std::unordered_set configuration_files; + return read(configuration_path, collections_path, configuration_files); +} + } // namespace sourcemeta::one diff --git a/src/index/index.cc b/src/index/index.cc index 4a73610d..9f3b32dc 100644 --- a/src/index/index.cc +++ b/src/index/index.cc @@ -18,22 +18,23 @@ #include "explorer.h" #include "generators.h" -#include // std::sort -#include // std::array -#include // std::atomic -#include // assert -#include // std::chrono -#include // std::uint8_t -#include // EXIT_FAILURE, EXIT_SUCCESS -#include // std::exception -#include // std::filesystem -#include // std::reference_wrapper, std::cref -#include // std::setw, std::setfill -#include // std::cerr, std::cout -#include // std::mutex, std::lock_guard -#include // std::string -#include // std::string_view -#include // std::vector +#include // std::sort +#include // std::array +#include // std::atomic +#include // assert +#include // std::chrono +#include // std::uint8_t +#include // EXIT_FAILURE, EXIT_SUCCESS +#include // std::exception +#include // std::filesystem +#include // std::reference_wrapper, std::cref +#include // std::setw, std::setfill +#include // std::cerr, std::cout +#include // std::mutex, std::lock_guard +#include // std::string +#include // std::string_view +#include // std::unordered_set +#include // std::vector // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) #define PROFILE_INIT(state) \ @@ -230,8 +231,9 @@ static auto index_main(const std::string_view &program, const auto configuration_path{ std::filesystem::canonical(app.positional().at(0))}; std::cerr << "Using configuration: " << configuration_path.string() << "\n"; + std::unordered_set configuration_files; const auto raw_configuration{sourcemeta::one::Configuration::read( - configuration_path, SOURCEMETA_ONE_COLLECTIONS)}; + configuration_path, SOURCEMETA_ONE_COLLECTIONS, configuration_files)}; if (app.contains("configuration")) { sourcemeta::core::prettify(raw_configuration, std::cout); @@ -341,6 +343,11 @@ static auto index_main(const std::string_view &program, continue; } + if (configuration_files.contains( + std::filesystem::weakly_canonical(entry.path()).native())) { + continue; + } + const auto extension{entry.path().extension()}; // TODO: Allow the configuration file to override this if (extension != ".yaml" && extension != ".yml" && extension != ".json") { diff --git a/test/cli/CMakeLists.txt b/test/cli/CMakeLists.txt index f161412c..83e79dd5 100644 --- a/test/cli/CMakeLists.txt +++ b/test/cli/CMakeLists.txt @@ -16,8 +16,9 @@ if(ONE_INDEX) sourcemeta_one_test_cli(common index fail-circular-config-include) sourcemeta_one_test_cli(common index fail-config-include-chain-cycle) sourcemeta_one_test_cli(common index fail-config-extends-itself) - sourcemeta_one_test_cli(common index fail-collection-path-dot) - sourcemeta_one_test_cli(common index fail-collection-path-empty) + sourcemeta_one_test_cli(common index collection-path-dot) + sourcemeta_one_test_cli(common index collection-path-empty) + sourcemeta_one_test_cli(common index collection-path-skips-config-files) sourcemeta_one_test_cli(common index fail-config-file-not-found) sourcemeta_one_test_cli(common index fail-config-include-itself) sourcemeta_one_test_cli(common index fail-duplicate-id) diff --git a/test/cli/index/common/fail-collection-path-dot.sh b/test/cli/index/common/collection-path-dot.sh similarity index 67% rename from test/cli/index/common/fail-collection-path-dot.sh rename to test/cli/index/common/collection-path-dot.sh index 7d417e03..6c8bac2b 100755 --- a/test/cli/index/common/fail-collection-path-dot.sh +++ b/test/cli/index/common/collection-path-dot.sh @@ -23,8 +23,9 @@ cat << EOF > "$TMP/one.json" } EOF -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" && CODE="$?" || CODE="$?" -test "$CODE" = "1" || exit 1 +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 \ + 2> "$TMP/output.txt" && CODE="$?" || CODE="$?" +test "$CODE" = "0" || exit 1 # Remove thread information if [ "$(uname)" = "Darwin" ] @@ -37,10 +38,13 @@ fi cat << EOF > "$TMP/expected.txt" Writing output to: $(realpath "$TMP")/output Using configuration: $(realpath "$TMP")/one.json -Detecting: $(realpath "$TMP")/one.json (#1) -(100%) Resolving: one.json -error: Could not determine the dialect of the schema - at path $(realpath "$TMP")/one.json +( 14%) Producing: configuration.json +( 28%) Producing: version.json +( 42%) Producing: explorer/%/404.metapack +( 57%) Producing: explorer/%/directory.metapack +( 71%) Producing: explorer/%/directory-html.metapack +( 85%) Producing: explorer/%/search.metapack +(100%) Producing: routes.bin EOF diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/test/cli/index/common/fail-collection-path-empty.sh b/test/cli/index/common/collection-path-empty.sh similarity index 67% rename from test/cli/index/common/fail-collection-path-empty.sh rename to test/cli/index/common/collection-path-empty.sh index 8ee68acd..2c946726 100755 --- a/test/cli/index/common/fail-collection-path-empty.sh +++ b/test/cli/index/common/collection-path-empty.sh @@ -23,8 +23,9 @@ cat << EOF > "$TMP/one.json" } EOF -"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" && CODE="$?" || CODE="$?" -test "$CODE" = "1" || exit 1 +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 \ + 2> "$TMP/output.txt" && CODE="$?" || CODE="$?" +test "$CODE" = "0" || exit 1 # Remove thread information if [ "$(uname)" = "Darwin" ] @@ -37,10 +38,13 @@ fi cat << EOF > "$TMP/expected.txt" Writing output to: $(realpath "$TMP")/output Using configuration: $(realpath "$TMP")/one.json -Detecting: $(realpath "$TMP")/one.json (#1) -(100%) Resolving: one.json -error: Could not determine the dialect of the schema - at path $(realpath "$TMP")/one.json +( 14%) Producing: configuration.json +( 28%) Producing: version.json +( 42%) Producing: explorer/%/404.metapack +( 57%) Producing: explorer/%/directory.metapack +( 71%) Producing: explorer/%/directory-html.metapack +( 85%) Producing: explorer/%/search.metapack +(100%) Producing: routes.bin EOF diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/test/cli/index/common/collection-path-skips-config-files.sh b/test/cli/index/common/collection-path-skips-config-files.sh new file mode 100755 index 00000000..68b13fe6 --- /dev/null +++ b/test/cli/index/common/collection-path-skips-config-files.sh @@ -0,0 +1,90 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/defaults.json" +{ + "contents": { + "example": { + "contents": { + "schemas": { + "include": "./jsonschema.json" + } + } + } + } +} +EOF + +cat << 'EOF' > "$TMP/jsonschema.json" +{ + "baseUri": "https://example.com/", + "defaultDialect": "http://json-schema.org/draft-07/schema#", + "path": "." +} +EOF + +cat << EOF > "$TMP/one.json" +{ + "url": "https://sourcemeta.com/", + "extends": [ "./defaults.json" ] +} +EOF + +cat << 'EOF' > "$TMP/test.json" +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/test", + "type": "string" +} +EOF + +"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 \ + 2> "$TMP/output.txt" && CODE="$?" || CODE="$?" +test "$CODE" = "0" || exit 1 + +# Remove thread information +if [ "$(uname)" = "Darwin" ] +then + sed -i '' 's/ \[.*\]//g' "$TMP/output.txt" +else + sed -i 's/ \[.*\]//g' "$TMP/output.txt" +fi + +cat << EOF > "$TMP/expected.txt" +Writing output to: $(realpath "$TMP")/output +Using configuration: $(realpath "$TMP")/one.json +Detecting: $(realpath "$TMP")/test.json (#1) +(100%) Resolving: test.json +( 4%) Producing: configuration.json +( 8%) Producing: version.json +( 13%) Producing: explorer/%/404.metapack +( 17%) Producing: schemas/example/schemas/test/%/schema.metapack +( 21%) Producing: schemas/example/schemas/test/%/dependencies.metapack +( 26%) Producing: schemas/example/schemas/test/%/locations.metapack +( 30%) Producing: schemas/example/schemas/test/%/positions.metapack +( 34%) Producing: schemas/example/schemas/test/%/stats.metapack +( 39%) Producing: schemas/example/schemas/test/%/bundle.metapack +( 43%) Producing: schemas/example/schemas/test/%/health.metapack +( 47%) Producing: explorer/example/schemas/test/%/schema.metapack +( 52%) Producing: schemas/example/schemas/test/%/blaze-exhaustive.metapack +( 56%) Producing: schemas/example/schemas/test/%/blaze-fast.metapack +( 60%) Producing: schemas/example/schemas/test/%/editor.metapack +( 65%) Producing: explorer/example/schemas/%/directory.metapack +( 69%) Producing: explorer/example/schemas/test/%/schema-html.metapack +( 73%) Producing: explorer/example/%/directory.metapack +( 78%) Producing: explorer/example/schemas/%/directory-html.metapack +( 82%) Producing: explorer/%/directory.metapack +( 86%) Producing: explorer/example/%/directory-html.metapack +( 91%) Producing: explorer/%/directory-html.metapack +( 95%) Producing: explorer/%/search.metapack +(100%) Producing: routes.bin +(100%) Combining: schemas/example/schemas/test/%/dependents.metapack +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/test/cli/index/common/fail-self-referencing-metaschema.sh b/test/cli/index/common/fail-self-referencing-metaschema.sh index 2990738e..029f0bbd 100755 --- a/test/cli/index/common/fail-self-referencing-metaschema.sh +++ b/test/cli/index/common/fail-self-referencing-metaschema.sh @@ -80,9 +80,10 @@ Detecting: $(realpath "$TMP")/schemas/test.json (#2) ( 8%) Producing: explorer/%/404.metapack ( 11%) Producing: schemas/example/schemas/my-metaschema/%/schema.metapack error: Could not resolve the metaschema of the schema - https://sourcemeta.com/example/schemas/my-metaschema + at identifier https://sourcemeta.com/example/schemas/my-metaschema + at path $(realpath "$TMP")/schemas/my-metaschema.json -Did you forget to register a schema with such URI in the one? +Did you forget to register a schema with such URI? EOF diff "$TMP/output.txt" "$TMP/expected1.txt" || diff "$TMP/output.txt" "$TMP/expected2.txt" diff --git a/test/unit/configuration/configuration_read_test.cc b/test/unit/configuration/configuration_read_test.cc index 6a823011..20f8f50f 100644 --- a/test/unit/configuration/configuration_read_test.cc +++ b/test/unit/configuration/configuration_read_test.cc @@ -2,6 +2,9 @@ #include +#include // std::string +#include // std::unordered_set + auto replace_all(std::string &text, const std::string &from, const std::string &to) -> void { assert(!from.empty()); @@ -560,3 +563,59 @@ TEST(Configuration_read, read_invalid_006_circular_include) { FAIL(); } } + +TEST(Configuration_read, read_configuration_files_no_extends_no_includes) { + const auto configuration_path{std::filesystem::path{STUB_DIRECTORY} / + "read_valid_001.json"}; + std::unordered_set configuration_files; + sourcemeta::one::Configuration::read( + configuration_path, COLLECTIONS_DIRECTORY, configuration_files); + + EXPECT_EQ(configuration_files.size(), 1); + EXPECT_TRUE(configuration_files.contains( + std::filesystem::weakly_canonical(configuration_path).native())); +} + +TEST(Configuration_read, read_configuration_files_with_extends) { + const auto configuration_path{std::filesystem::path{STUB_DIRECTORY} / + "read_valid_016.json"}; + std::unordered_set configuration_files; + sourcemeta::one::Configuration::read( + configuration_path, COLLECTIONS_DIRECTORY, configuration_files); + + EXPECT_EQ(configuration_files.size(), 4); + EXPECT_TRUE(configuration_files.contains( + std::filesystem::weakly_canonical(configuration_path).native())); + EXPECT_TRUE(configuration_files.contains( + std::filesystem::weakly_canonical(std::filesystem::path{STUB_DIRECTORY} / + "read_valid_016_b.json") + .native())); + EXPECT_TRUE(configuration_files.contains( + std::filesystem::weakly_canonical(std::filesystem::path{STUB_DIRECTORY} / + "read_valid_016_c.json") + .native())); + EXPECT_TRUE(configuration_files.contains( + std::filesystem::weakly_canonical(std::filesystem::path{STUB_DIRECTORY} / + "read_valid_016_d.json") + .native())); +} + +TEST(Configuration_read, read_configuration_files_with_include_chain) { + const auto configuration_path{std::filesystem::path{STUB_DIRECTORY} / + "read_valid_002.json"}; + std::unordered_set configuration_files; + sourcemeta::one::Configuration::read( + configuration_path, COLLECTIONS_DIRECTORY, configuration_files); + + EXPECT_EQ(configuration_files.size(), 3); + EXPECT_TRUE(configuration_files.contains( + std::filesystem::weakly_canonical(configuration_path).native())); + EXPECT_TRUE(configuration_files.contains( + std::filesystem::weakly_canonical(std::filesystem::path{STUB_DIRECTORY} / + "read_partial_001.json") + .native())); + EXPECT_TRUE(configuration_files.contains( + std::filesystem::weakly_canonical(std::filesystem::path{STUB_DIRECTORY} / + "folder" / "jsonschema.json") + .native())); +}