Compute search index out of the directory metadata

src/build/delta.cc

@@ -560,8 +560,7 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
   }
 
   TargetMap targets;
-  targets.reserve(schemas.size() * PER_SCHEMA_RULES.size() +
-                  AGGREGATE_RULES.size());
+  targets.reserve(schemas.size() * PER_SCHEMA_RULES.size());
   const auto &output_string{output.native()};
   const auto configuration_string{configuration_path.string()};
   const auto explorer_string{explorer_path.string()};
@@ -837,42 +836,6 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
       }
     }
 
-    for (std::size_t index{0}; index < AGGREGATE_RULES.size(); index++) {
-      const auto &rule{AGGREGATE_RULES[index]};
-
-      std::vector<std::string> all_collected;
-      all_collected.reserve(active_schemas.size());
-      for (const auto &schema : active_schemas) {
-        const auto &base{rule.collector_base == TargetBase::Schema
-                             ? schema.schema_base
-                             : schema.explorer_base};
-        all_collected.push_back(append_filename(base, rule.collector_filename));
-      }
-
-      std::string destination;
-      if (rule.output_base == AggregateOutputBase::Explorer) {
-        destination.reserve(
-            explorer_string.size() + 4 +
-            std::char_traits<char>::length(rule.output_filename));
-        destination += explorer_string;
-        destination += '/';
-        destination += SENTINEL;
-        destination += '/';
-        destination += rule.output_filename;
-      } else {
-        destination.reserve(
-            output_string.size() + 1 +
-            std::char_traits<char>::length(rule.output_filename));
-        destination += output_string;
-        destination += '/';
-        destination += rule.output_filename;
-      }
-
-      declare_target(targets, rule.action, destination,
-                     std::move(all_collected));
-      dirty_set.insert(std::move(destination));
-    }
-
     auto has_graph_change{!removed_uris.empty()};
     if (!has_graph_change) {
       for (const auto &schema : active_schemas) {
@@ -918,6 +881,8 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
 
     const auto affected_dirs{
         collect_affected_directories(schemas_path, affected_relative_paths)};
+    const auto all_dirs{
+        collect_affected_directories(schemas_path, all_relative_paths)};
 
     for (const auto &rule : DIRECTORY_RULES) {
       if (rule.gate == TargetGate::FullOnly &&
@@ -978,12 +943,26 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
                 if (other_parent == relative) {
                   rule_dependencies.push_back(
                       (explorer_path / other_relative / SENTINEL /
-                       DIRECTORY_RULES[0].filename)
+                       DIRECTORY_LIST_RULE.filename)
                           .lexically_normal()
                           .string());
                 }
               }
               break;
+            case DirectoryDependencyKind::AllDirectoryListings:
+              for (const auto &any_directory : all_dirs) {
+                const auto dir_relative{
+                    std::filesystem::relative(any_directory, schemas_path)};
+                rule_dependencies.push_back(
+                    (dir_relative == "."
+                         ? explorer_path / SENTINEL /
+                               DIRECTORY_LIST_RULE.filename
+                         : explorer_path / dir_relative / SENTINEL /
+                               DIRECTORY_LIST_RULE.filename)
+                        .lexically_normal()
+                        .string());
+              }
+              break;
             case DirectoryDependencyKind::SameDirectoryTarget:
               rule_dependencies.push_back(
                   (explorer_path / relative / SENTINEL / dependency.filename)
@@ -1122,14 +1101,6 @@ auto delta(const BuildPhase phase, const BuildPlan::Type build_type,
     const auto output_prefix{output_string + "/"};
 
     std::unordered_set<std::string> global_skip_paths;
-    for (const auto &rule : AGGREGATE_RULES) {
-      if (rule.output_base == AggregateOutputBase::Explorer) {
-        global_skip_paths.insert(explorer_string + '/' + SENTINEL + '/' +
-                                 rule.output_filename);
-      } else {
-        global_skip_paths.insert(output_string + '/' + rule.output_filename);
-      }
-    }
     for (const auto &rule : GLOBAL_RULES) {
       global_skip_paths.insert(output_string + '/' + rule.filename);
     }

src/build/rules.h

@@ -190,29 +190,12 @@ static constexpr std::array<PerSchemaRule, 13> PER_SCHEMA_RULES{{
      .dependency_count = 2},
 }};
 
-enum class AggregateOutputBase : std::uint8_t { Output, Explorer };
-
-struct AggregateRule {
-  BuildPlan::Action::Type action;
-  const char *output_filename;
-  AggregateOutputBase output_base;
-  TargetBase collector_base;
-  const char *collector_filename;
-};
-
-static constexpr std::array<AggregateRule, 1> AGGREGATE_RULES{{
-    {.action = BuildPlan::Action::Type::SearchIndex,
-     .output_filename = "search.metapack",
-     .output_base = AggregateOutputBase::Explorer,
-     .collector_base = TargetBase::Explorer,
-     .collector_filename = "schema.metapack"},
-}};
-
 enum class DirectoryScope : std::uint8_t { AllDirectories, NonRoot, RootOnly };
 
 enum class DirectoryDependencyKind : std::uint8_t {
   SchemaMetadata,
   ChildDirectories,
+  AllDirectoryListings,
   SameDirectoryTarget,
   ExternalConfig
 };
@@ -234,7 +217,7 @@ struct DirectoryRule {
   std::uint8_t dependency_count;
 };
 
-static constexpr std::array<DirectoryRule, 4> DIRECTORY_RULES{{
+static constexpr std::array<DirectoryRule, 5> DIRECTORY_RULES{{
     {.action = BuildPlan::Action::Type::DirectoryList,
      .filename = "directory.metapack",
      .gate = TargetGate::Always,
@@ -246,6 +229,15 @@ static constexpr std::array<DirectoryRule, 4> DIRECTORY_RULES{{
                         .filename = nullptr}}},
      .dependency_count = 2},
 
+    {.action = BuildPlan::Action::Type::SearchIndex,
+     .filename = "search.metapack",
+     .gate = TargetGate::Always,
+     .scope = DirectoryScope::RootOnly,
+     .only_full_rebuild = false,
+     .dependencies = {{{.kind = DirectoryDependencyKind::AllDirectoryListings,
+                        .filename = nullptr}}},
+     .dependency_count = 1},
+
     {.action = BuildPlan::Action::Type::WebIndex,
      .filename = "directory-html.metapack",
      .gate = TargetGate::FullOnly,
@@ -345,6 +337,21 @@ static constexpr auto find_rule_by_action(BuildPlan::Action::Type action)
 static constexpr const auto &SCHEMA_METADATA_RULE =
     find_rule_by_action(BuildPlan::Action::Type::SchemaMetadata);
 
+static constexpr auto
+find_directory_rule_by_action(BuildPlan::Action::Type action)
+    -> const DirectoryRule & {
+  for (const auto &rule : DIRECTORY_RULES) {
+    if (rule.action == action) {
+      return rule;
+    }
+  }
+
+  return DIRECTORY_RULES[0];
+}
+
+static constexpr const auto &DIRECTORY_LIST_RULE =
+    find_directory_rule_by_action(BuildPlan::Action::Type::DirectoryList);
+
 } // namespace sourcemeta::one
 
 #endif

src/index/explorer.h

@@ -419,35 +419,32 @@ struct GENERATE_EXPLORER_SEARCH_INDEX {
                       const sourcemeta::core::JSON &) -> bool {
     const auto timestamp_start{std::chrono::steady_clock::now()};
     std::vector<sourcemeta::core::JSON> result;
-    result.reserve(action.dependencies.size());
 
     for (const auto &dependency : action.dependencies) {
-      sourcemeta::core::FileView dependency_view{dependency};
-      const auto extension_offset{
-          sourcemeta::one::metapack_extension_offset(dependency_view)};
-      if (extension_offset == 0) {
-        continue;
-      }
+      const auto directory_option{
+          sourcemeta::one::metapack_read_json(dependency)};
+      assert(directory_option.has_value());
+      const auto &directory{directory_option.value()};
+      assert(directory.is_object());
+      assert(directory.defines("entries"));
+
+      for (const auto &directory_entry : directory.at("entries").as_array()) {
+        if (!directory_entry.defines("type") ||
+            directory_entry.at("type").to_string() != "schema") {
+          continue;
+        }
 
-      const auto *extension{
-          sourcemeta::one::metapack_extension<MetapackExplorerSchemaExtension>(
-              dependency_view)};
-      if (extension == nullptr) {
-        continue;
+        auto entry{sourcemeta::core::JSON::make_array()};
+        entry.push_back(
+            sourcemeta::core::JSON{directory_entry.at("path").to_string()});
+        entry.push_back(directory_entry.defines("title")
+                            ? directory_entry.at("title")
+                            : sourcemeta::core::JSON{""});
+        entry.push_back(directory_entry.defines("description")
+                            ? directory_entry.at("description")
+                            : sourcemeta::core::JSON{""});
+        result.push_back(std::move(entry));
       }
-
-      const auto *extension_base{
-          dependency_view.as<std::uint8_t>(extension_offset)};
-      const auto path{explorer_extension_path(extension, extension_base)};
-      const auto title{explorer_extension_title(extension, extension_base)};
-      const auto description{
-          explorer_extension_description(extension, extension_base)};
-
-      auto entry{sourcemeta::core::JSON::make_array()};
-      entry.push_back(sourcemeta::core::JSON{std::string{path}});
-      entry.push_back(sourcemeta::core::JSON{std::string{title}});
-      entry.push_back(sourcemeta::core::JSON{std::string{description}});
-      result.push_back(std::move(entry));
     }
 
     std::sort(result.begin(), result.end(),

test/cli/CMakeLists.txt

@@ -35,6 +35,7 @@ if(ONE_INDEX)
   sourcemeta_one_test_cli(common index rebuild-cache-config-change)
   sourcemeta_one_test_cli(common index rebuild-deleted-deps)
   sourcemeta_one_test_cli(common index rebuild-nested-directories)
+  sourcemeta_one_test_cli(common index search-index-nested-rebuild)
   sourcemeta_one_test_cli(common index rebuild-to-empty)
   sourcemeta_one_test_cli(common index verbose-long)
   sourcemeta_one_test_cli(common index verbose-short)

test/cli/index/common/comment-removed-on-rebuild.sh

@@ -35,8 +35,8 @@ Using configuration: $(realpath "$TMP")/one.json
 ( 37%) Producing: version.json
 ( 50%) Producing: explorer/%/404.metapack
 ( 62%) Producing: explorer/%/directory.metapack
-( 75%) Producing: explorer/%/search.metapack
-( 87%) Producing: explorer/%/directory-html.metapack
+( 75%) Producing: explorer/%/directory-html.metapack
+( 87%) Producing: explorer/%/search.metapack
 (100%) Producing: routes.bin
 EOF
 

test/cli/index/common/comment-updated-on-rebuild.sh

@@ -35,8 +35,8 @@ Using configuration: $(realpath "$TMP")/one.json
 ( 37%) Producing: version.json
 ( 50%) Producing: explorer/%/404.metapack
 ( 62%) Producing: explorer/%/directory.metapack
-( 75%) Producing: explorer/%/search.metapack
-( 87%) Producing: explorer/%/directory-html.metapack
+( 75%) Producing: explorer/%/directory-html.metapack
+( 87%) Producing: explorer/%/search.metapack
 (100%) Producing: routes.bin
 EOF
 

test/cli/index/common/dependents-remove-referencing-schema.sh

@@ -71,8 +71,8 @@ Writing output to: $(realpath "$TMP")/output
 Using configuration: $(realpath "$TMP")/one.json
 Detecting: $(realpath "$TMP")/schemas/a.json (#1)
 ( 20%) Producing: explorer/%/directory.metapack
-( 40%) Producing: explorer/%/search.metapack
-( 60%) Producing: explorer/%/directory-html.metapack
+( 40%) Producing: explorer/%/directory-html.metapack
+( 60%) Producing: explorer/%/search.metapack
 ( 80%) Disposing: explorer/example/schemas/b
 (100%) Disposing: schemas/example/schemas/b
 (100%) Combining: schemas/example/schemas/a/%/dependents.metapack

test/cli/index/common/extra-files-on-rebuild.sh

@@ -63,14 +63,14 @@ Detecting: $(realpath "$TMP")/schemas/test.json (#1)
 ( 52%) Producing: schemas/example/schemas/old/%/blaze-exhaustive.metapack
 ( 56%) Producing: schemas/example/schemas/old/%/blaze-fast.metapack
 ( 60%) Producing: schemas/example/schemas/old/%/editor.metapack
-( 65%) Producing: explorer/%/search.metapack
-( 69%) Producing: explorer/example/schemas/%/directory.metapack
-( 73%) Producing: explorer/example/schemas/old/%/schema-html.metapack
-( 78%) Producing: explorer/example/%/directory.metapack
-( 82%) Producing: explorer/example/schemas/%/directory-html.metapack
-( 86%) Producing: explorer/%/directory.metapack
-( 91%) Producing: explorer/example/%/directory-html.metapack
-( 95%) Producing: explorer/%/directory-html.metapack
+( 65%) Producing: explorer/example/schemas/%/directory.metapack
+( 69%) Producing: explorer/example/schemas/old/%/schema-html.metapack
+( 73%) Producing: explorer/example/%/directory.metapack
+( 78%) Producing: explorer/example/schemas/%/directory-html.metapack
+( 82%) Producing: explorer/%/directory.metapack
+( 86%) Producing: explorer/example/%/directory-html.metapack
+( 91%) Producing: explorer/%/directory-html.metapack
+( 95%) Producing: explorer/%/search.metapack
 (100%) Producing: routes.bin
 (100%) Combining: schemas/example/schemas/old/%/dependents.metapack
 EOF
@@ -116,14 +116,14 @@ Detecting: $(realpath "$TMP")/schemas/test.json (#1)
 ( 42%) Producing: schemas/example/schemas/new/%/blaze-exhaustive.metapack
 ( 47%) Producing: schemas/example/schemas/new/%/blaze-fast.metapack
 ( 52%) Producing: schemas/example/schemas/new/%/editor.metapack
-( 57%) Producing: explorer/%/search.metapack
-( 61%) Producing: explorer/example/schemas/%/directory.metapack
-( 66%) Producing: explorer/example/schemas/new/%/schema-html.metapack
-( 71%) Producing: explorer/example/%/directory.metapack
-( 76%) Producing: explorer/example/schemas/%/directory-html.metapack
-( 80%) Producing: explorer/%/directory.metapack
-( 85%) Producing: explorer/example/%/directory-html.metapack
-( 90%) Producing: explorer/%/directory-html.metapack
+( 57%) Producing: explorer/example/schemas/%/directory.metapack
+( 61%) Producing: explorer/example/schemas/new/%/schema-html.metapack
+( 66%) Producing: explorer/example/%/directory.metapack
+( 71%) Producing: explorer/example/schemas/%/directory-html.metapack
+( 76%) Producing: explorer/%/directory.metapack
+( 80%) Producing: explorer/example/%/directory-html.metapack
+( 85%) Producing: explorer/%/directory-html.metapack
+( 90%) Producing: explorer/%/search.metapack
 ( 95%) Disposing: explorer/example/schemas/old
 (100%) Disposing: schemas/example/schemas/old
 (100%) Combining: schemas/example/schemas/new/%/dependents.metapack

test/cli/index/common/rebuild-cache-config-change.sh

@@ -93,14 +93,14 @@ Detecting: $(realpath "$TMP")/schemas/foo.json (#1)
 ( 52%) Producing: schemas/example/schemas/foo/%/blaze-exhaustive.metapack
 ( 56%) Producing: schemas/example/schemas/foo/%/blaze-fast.metapack
 ( 60%) Producing: schemas/example/schemas/foo/%/editor.metapack
-( 65%) Producing: explorer/%/search.metapack
-( 69%) Producing: explorer/example/schemas/%/directory.metapack
-( 73%) Producing: explorer/example/schemas/foo/%/schema-html.metapack
-( 78%) Producing: explorer/example/%/directory.metapack
-( 82%) Producing: explorer/example/schemas/%/directory-html.metapack
-( 86%) Producing: explorer/%/directory.metapack
-( 91%) Producing: explorer/example/%/directory-html.metapack
-( 95%) Producing: explorer/%/directory-html.metapack
+( 65%) Producing: explorer/example/schemas/%/directory.metapack
+( 69%) Producing: explorer/example/schemas/foo/%/schema-html.metapack
+( 73%) Producing: explorer/example/%/directory.metapack
+( 78%) Producing: explorer/example/schemas/%/directory-html.metapack
+( 82%) Producing: explorer/%/directory.metapack
+( 86%) Producing: explorer/example/%/directory-html.metapack
+( 91%) Producing: explorer/%/directory-html.metapack
+( 95%) Producing: explorer/%/search.metapack
 (100%) Producing: routes.bin
 EOF
 diff "$TMP/output.txt" "$TMP/expected.txt"

test/cli/index/common/rebuild-cache.sh

@@ -62,14 +62,14 @@ Detecting: $(realpath "$TMP")/schemas/foo.json (#1)
 ( 52%) Producing: schemas/example/schemas/foo/%/blaze-exhaustive.metapack
 ( 56%) Producing: schemas/example/schemas/foo/%/blaze-fast.metapack
 ( 60%) Producing: schemas/example/schemas/foo/%/editor.metapack
-( 65%) Producing: explorer/%/search.metapack
-( 69%) Producing: explorer/example/schemas/%/directory.metapack
-( 73%) Producing: explorer/example/schemas/foo/%/schema-html.metapack
-( 78%) Producing: explorer/example/%/directory.metapack
-( 82%) Producing: explorer/example/schemas/%/directory-html.metapack
-( 86%) Producing: explorer/%/directory.metapack
-( 91%) Producing: explorer/example/%/directory-html.metapack
-( 95%) Producing: explorer/%/directory-html.metapack
+( 65%) Producing: explorer/example/schemas/%/directory.metapack
+( 69%) Producing: explorer/example/schemas/foo/%/schema-html.metapack
+( 73%) Producing: explorer/example/%/directory.metapack
+( 78%) Producing: explorer/example/schemas/%/directory-html.metapack
+( 82%) Producing: explorer/%/directory.metapack
+( 86%) Producing: explorer/example/%/directory-html.metapack
+( 91%) Producing: explorer/%/directory-html.metapack
+( 95%) Producing: explorer/%/search.metapack
 (100%) Producing: routes.bin
 (100%) Combining: schemas/example/schemas/foo/%/dependents.metapack
 EOF
@@ -112,14 +112,14 @@ Detecting: $(realpath "$TMP")/schemas/foo.json (#1)
 ( 47%) Producing: schemas/example/schemas/foo/%/blaze-exhaustive.metapack
 ( 52%) Producing: schemas/example/schemas/foo/%/blaze-fast.metapack
 ( 57%) Producing: schemas/example/schemas/foo/%/editor.metapack
-( 63%) Producing: explorer/%/search.metapack
-( 68%) Producing: explorer/example/schemas/%/directory.metapack
-( 73%) Producing: explorer/example/schemas/foo/%/schema-html.metapack
-( 78%) Producing: explorer/example/%/directory.metapack
-( 84%) Producing: explorer/example/schemas/%/directory-html.metapack
-( 89%) Producing: explorer/%/directory.metapack
-( 94%) Producing: explorer/example/%/directory-html.metapack
-(100%) Producing: explorer/%/directory-html.metapack
+( 63%) Producing: explorer/example/schemas/%/directory.metapack
+( 68%) Producing: explorer/example/schemas/foo/%/schema-html.metapack
+( 73%) Producing: explorer/example/%/directory.metapack
+( 78%) Producing: explorer/example/schemas/%/directory-html.metapack
+( 84%) Producing: explorer/%/directory.metapack
+( 89%) Producing: explorer/example/%/directory-html.metapack
+( 94%) Producing: explorer/%/directory-html.metapack
+(100%) Producing: explorer/%/search.metapack
 EOF
 diff "$TMP/output.txt" "$TMP/expected.txt"