Automatically ignore configuration files in collection paths

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.16)
-project(one VERSION 5.1.0 LANGUAGES C CXX)
+project(one VERSION 5.1.1 LANGUAGES C CXX)
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
 
 # Options

src/configuration/include/sourcemeta/one/configuration.h

@@ -9,12 +9,18 @@
 
 #include <filesystem>    // std::filesystem::path
 #include <optional>      // std::optional
+#include <string>        // std::string
 #include <unordered_map> // std::unordered_map
+#include <unordered_set> // std::unordered_set
 #include <variant>       // std::variant
 
 namespace sourcemeta::one {
 
 struct Configuration {
+  static auto read(const std::filesystem::path &configuration_path,
+                   const std::filesystem::path &collections_path,
+                   std::unordered_set<std::string> &configuration_files)
+      -> sourcemeta::core::JSON;
   static auto read(const std::filesystem::path &configuration_path,
                    const std::filesystem::path &collections_path)
       -> sourcemeta::core::JSON;

src/configuration/read.cc

@@ -52,7 +52,8 @@ auto dereference(const std::filesystem::path &collections_path,
                  const std::filesystem::path &base,
                  sourcemeta::core::JSON &input,
                  const sourcemeta::core::Pointer &location,
-                 std::unordered_set<std::string> &visited) -> void {
+                 std::unordered_set<std::string> &visited,
+                 std::unordered_set<std::string> &all_files) -> void {
   assert(base.is_absolute());
   if (!input.is_object()) {
     return;
@@ -71,11 +72,12 @@ auto dereference(const std::filesystem::path &collections_path,
           throw sourcemeta::one::ConfigurationCyclicReferenceError(
               base, new_location, target_path);
         }
+        all_files.emplace(target_path.native());
         auto extension{
             read_file(base, new_location, target_path, entry.to_string())};
         if (extension.is_object()) {
           dereference(collections_path, target_path, extension, new_location,
-                      visited);
+                      visited, all_files);
           accumulator.merge(std::move(extension).as_object());
         }
 
@@ -87,7 +89,7 @@ auto dereference(const std::filesystem::path &collections_path,
     accumulator.merge(input.as_object());
     input = std::move(accumulator);
     assert(!input.defines("extends"));
-    dereference(collections_path, base, input, location, visited);
+    dereference(collections_path, base, input, location, visited, all_files);
 
     // Read included files
   } else if (!location.empty() && input.defines("include") &&
@@ -103,9 +105,11 @@ auto dereference(const std::filesystem::path &collections_path,
       throw sourcemeta::one::ConfigurationCyclicReferenceError(
           base, new_location, target_path);
     }
+    all_files.emplace(target_path.native());
     input.into(read_file(base, new_location, target_path,
                          input.at("include").to_string()));
-    dereference(collections_path, target_path, input, new_location, visited);
+    dereference(collections_path, target_path, input, new_location, visited,
+                all_files);
     visited.erase(target_path.native());
 
     // Revisit and relativize paths
@@ -130,7 +134,7 @@ auto dereference(const std::filesystem::path &collections_path,
                            [](const auto &entry) { return entry.first; });
     for (const auto &key : keys) {
       dereference(collections_path, base, input.at("contents").at(key),
-                  location.concat({"contents", key}), visited);
+                  location.concat({"contents", key}), visited, all_files);
     }
   }
 }
@@ -163,7 +167,8 @@ auto default_base_uri(sourcemeta::core::JSON &contents,
 namespace sourcemeta::one {
 
 auto Configuration::read(const std::filesystem::path &configuration_path,
-                         const std::filesystem::path &collections_path)
+                         const std::filesystem::path &collections_path,
+                         std::unordered_set<std::string> &configuration_files)
     -> sourcemeta::core::JSON {
   auto data{sourcemeta::core::read_json(configuration_path)};
 
@@ -182,10 +187,13 @@ auto Configuration::read(const std::filesystem::path &configuration_path,
         sourcemeta::core::JSON{"The next-generation JSON Schema platform"});
   }
 
+  const auto canonical_config{
+      std::filesystem::weakly_canonical(configuration_path).native()};
+  configuration_files.emplace(canonical_config);
   std::unordered_set<std::string> visited;
-  visited.emplace(
-      std::filesystem::weakly_canonical(configuration_path).native());
-  dereference(collections_path, configuration_path, data, {}, visited);
+  visited.emplace(canonical_config);
+  dereference(collections_path, configuration_path, data, {}, visited,
+              configuration_files);
 
   if (data.is_object() && data.defines("url") && data.defines("contents") &&
       data.at("contents").is_object()) {
@@ -195,4 +203,11 @@ auto Configuration::read(const std::filesystem::path &configuration_path,
   return data;
 }
 
+auto Configuration::read(const std::filesystem::path &configuration_path,
+                         const std::filesystem::path &collections_path)
+    -> sourcemeta::core::JSON {
+  std::unordered_set<std::string> configuration_files;
+  return read(configuration_path, collections_path, configuration_files);
+}
+
 } // namespace sourcemeta::one

src/index/index.cc

@@ -18,22 +18,23 @@
 #include "explorer.h"
 #include "generators.h"
 
-#include <algorithm>   // std::sort
-#include <array>       // std::array
-#include <atomic>      // std::atomic
-#include <cassert>     // assert
-#include <chrono>      // std::chrono
-#include <cstdint>     // std::uint8_t
-#include <cstdlib>     // EXIT_FAILURE, EXIT_SUCCESS
-#include <exception>   // std::exception
-#include <filesystem>  // std::filesystem
-#include <functional>  // std::reference_wrapper, std::cref
-#include <iomanip>     // std::setw, std::setfill
-#include <iostream>    // std::cerr, std::cout
-#include <mutex>       // std::mutex, std::lock_guard
-#include <string>      // std::string
-#include <string_view> // std::string_view
-#include <vector>      // std::vector
+#include <algorithm>     // std::sort
+#include <array>         // std::array
+#include <atomic>        // std::atomic
+#include <cassert>       // assert
+#include <chrono>        // std::chrono
+#include <cstdint>       // std::uint8_t
+#include <cstdlib>       // EXIT_FAILURE, EXIT_SUCCESS
+#include <exception>     // std::exception
+#include <filesystem>    // std::filesystem
+#include <functional>    // std::reference_wrapper, std::cref
+#include <iomanip>       // std::setw, std::setfill
+#include <iostream>      // std::cerr, std::cout
+#include <mutex>         // std::mutex, std::lock_guard
+#include <string>        // std::string
+#include <string_view>   // std::string_view
+#include <unordered_set> // std::unordered_set
+#include <vector>        // std::vector
 
 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
 #define PROFILE_INIT(state)                                                    \
@@ -230,8 +231,9 @@ static auto index_main(const std::string_view &program,
   const auto configuration_path{
       std::filesystem::canonical(app.positional().at(0))};
   std::cerr << "Using configuration: " << configuration_path.string() << "\n";
+  std::unordered_set<std::string> configuration_files;
   const auto raw_configuration{sourcemeta::one::Configuration::read(
-      configuration_path, SOURCEMETA_ONE_COLLECTIONS)};
+      configuration_path, SOURCEMETA_ONE_COLLECTIONS, configuration_files)};
 
   if (app.contains("configuration")) {
     sourcemeta::core::prettify(raw_configuration, std::cout);
@@ -341,6 +343,11 @@ static auto index_main(const std::string_view &program,
         continue;
       }
 
+      if (configuration_files.contains(
+              std::filesystem::weakly_canonical(entry.path()).native())) {
+        continue;
+      }
+
       const auto extension{entry.path().extension()};
       // TODO: Allow the configuration file to override this
       if (extension != ".yaml" && extension != ".yml" && extension != ".json") {

test/cli/CMakeLists.txt

@@ -16,8 +16,9 @@ if(ONE_INDEX)
   sourcemeta_one_test_cli(common index fail-circular-config-include)
   sourcemeta_one_test_cli(common index fail-config-include-chain-cycle)
   sourcemeta_one_test_cli(common index fail-config-extends-itself)
-  sourcemeta_one_test_cli(common index fail-collection-path-dot)
-  sourcemeta_one_test_cli(common index fail-collection-path-empty)
+  sourcemeta_one_test_cli(common index collection-path-dot)
+  sourcemeta_one_test_cli(common index collection-path-empty)
+  sourcemeta_one_test_cli(common index collection-path-skips-config-files)
   sourcemeta_one_test_cli(common index fail-config-file-not-found)
   sourcemeta_one_test_cli(common index fail-config-include-itself)
   sourcemeta_one_test_cli(common index fail-duplicate-id)

test/cli/index/common/fail-collection-path-dot.sh → test/cli/index/common/collection-path-dot.sh

@@ -23,8 +23,9 @@ cat << EOF > "$TMP/one.json"
 }
 EOF
 
-"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" && CODE="$?" || CODE="$?"
-test "$CODE" = "1" || exit 1
+"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 \
+  2> "$TMP/output.txt" && CODE="$?" || CODE="$?"
+test "$CODE" = "0" || exit 1
 
 # Remove thread information
 if [ "$(uname)" = "Darwin" ]
@@ -37,10 +38,13 @@ fi
 cat << EOF > "$TMP/expected.txt"
 Writing output to: $(realpath "$TMP")/output
 Using configuration: $(realpath "$TMP")/one.json
-Detecting: $(realpath "$TMP")/one.json (#1)
-(100%) Resolving: one.json
-error: Could not determine the dialect of the schema
-  at path $(realpath "$TMP")/one.json
+( 14%) Producing: configuration.json
+( 28%) Producing: version.json
+( 42%) Producing: explorer/%/404.metapack
+( 57%) Producing: explorer/%/directory.metapack
+( 71%) Producing: explorer/%/directory-html.metapack
+( 85%) Producing: explorer/%/search.metapack
+(100%) Producing: routes.bin
 EOF
 
 diff "$TMP/output.txt" "$TMP/expected.txt"

test/cli/index/common/fail-collection-path-empty.sh → test/cli/index/common/collection-path-empty.sh

@@ -23,8 +23,9 @@ cat << EOF > "$TMP/one.json"
 }
 EOF
 
-"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 2> "$TMP/output.txt" && CODE="$?" || CODE="$?"
-test "$CODE" = "1" || exit 1
+"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 \
+  2> "$TMP/output.txt" && CODE="$?" || CODE="$?"
+test "$CODE" = "0" || exit 1
 
 # Remove thread information
 if [ "$(uname)" = "Darwin" ]
@@ -37,10 +38,13 @@ fi
 cat << EOF > "$TMP/expected.txt"
 Writing output to: $(realpath "$TMP")/output
 Using configuration: $(realpath "$TMP")/one.json
-Detecting: $(realpath "$TMP")/one.json (#1)
-(100%) Resolving: one.json
-error: Could not determine the dialect of the schema
-  at path $(realpath "$TMP")/one.json
+( 14%) Producing: configuration.json
+( 28%) Producing: version.json
+( 42%) Producing: explorer/%/404.metapack
+( 57%) Producing: explorer/%/directory.metapack
+( 71%) Producing: explorer/%/directory-html.metapack
+( 85%) Producing: explorer/%/search.metapack
+(100%) Producing: routes.bin
 EOF
 
 diff "$TMP/output.txt" "$TMP/expected.txt"

test/cli/index/common/collection-path-skips-config-files.sh

@@ -0,0 +1,90 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/defaults.json"
+{
+  "contents": {
+    "example": {
+      "contents": {
+        "schemas": {
+          "include": "./jsonschema.json"
+        }
+      }
+    }
+  }
+}
+EOF
+
+cat << 'EOF' > "$TMP/jsonschema.json"
+{
+  "baseUri": "https://example.com/",
+  "defaultDialect": "http://json-schema.org/draft-07/schema#",
+  "path": "."
+}
+EOF
+
+cat << EOF > "$TMP/one.json"
+{
+  "url": "https://sourcemeta.com/",
+  "extends": [ "./defaults.json" ]
+}
+EOF
+
+cat << 'EOF' > "$TMP/test.json"
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://example.com/test",
+  "type": "string"
+}
+EOF
+
+"$1" --skip-banner "$TMP/one.json" "$TMP/output" --concurrency 1 \
+  2> "$TMP/output.txt" && CODE="$?" || CODE="$?"
+test "$CODE" = "0" || exit 1
+
+# Remove thread information
+if [ "$(uname)" = "Darwin" ]
+then
+  sed -i '' 's/ \[.*\]//g' "$TMP/output.txt"
+else
+  sed -i 's/ \[.*\]//g' "$TMP/output.txt"
+fi
+
+cat << EOF > "$TMP/expected.txt"
+Writing output to: $(realpath "$TMP")/output
+Using configuration: $(realpath "$TMP")/one.json
+Detecting: $(realpath "$TMP")/test.json (#1)
+(100%) Resolving: test.json
+(  4%) Producing: configuration.json
+(  8%) Producing: version.json
+( 13%) Producing: explorer/%/404.metapack
+( 17%) Producing: schemas/example/schemas/test/%/schema.metapack
+( 21%) Producing: schemas/example/schemas/test/%/dependencies.metapack
+( 26%) Producing: schemas/example/schemas/test/%/locations.metapack
+( 30%) Producing: schemas/example/schemas/test/%/positions.metapack
+( 34%) Producing: schemas/example/schemas/test/%/stats.metapack
+( 39%) Producing: schemas/example/schemas/test/%/bundle.metapack
+( 43%) Producing: schemas/example/schemas/test/%/health.metapack
+( 47%) Producing: explorer/example/schemas/test/%/schema.metapack
+( 52%) Producing: schemas/example/schemas/test/%/blaze-exhaustive.metapack
+( 56%) Producing: schemas/example/schemas/test/%/blaze-fast.metapack
+( 60%) Producing: schemas/example/schemas/test/%/editor.metapack
+( 65%) Producing: explorer/example/schemas/%/directory.metapack
+( 69%) Producing: explorer/example/schemas/test/%/schema-html.metapack
+( 73%) Producing: explorer/example/%/directory.metapack
+( 78%) Producing: explorer/example/schemas/%/directory-html.metapack
+( 82%) Producing: explorer/%/directory.metapack
+( 86%) Producing: explorer/example/%/directory-html.metapack
+( 91%) Producing: explorer/%/directory-html.metapack
+( 95%) Producing: explorer/%/search.metapack
+(100%) Producing: routes.bin
+(100%) Combining: schemas/example/schemas/test/%/dependents.metapack
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"

test/cli/index/common/fail-self-referencing-metaschema.sh

@@ -80,9 +80,10 @@ Detecting: $(realpath "$TMP")/schemas/test.json (#2)
 (  8%) Producing: explorer/%/404.metapack
 ( 11%) Producing: schemas/example/schemas/my-metaschema/%/schema.metapack
 error: Could not resolve the metaschema of the schema
-  https://sourcemeta.com/example/schemas/my-metaschema
+  at identifier https://sourcemeta.com/example/schemas/my-metaschema
+  at path $(realpath "$TMP")/schemas/my-metaschema.json
 
-Did you forget to register a schema with such URI in the one?
+Did you forget to register a schema with such URI?
 EOF
 
 diff "$TMP/output.txt" "$TMP/expected1.txt" || diff "$TMP/output.txt" "$TMP/expected2.txt"