Install Faux Cross-Module Incremental Infrastructure In Driver

Treat any incremental external depends like normal external depends. This will eventually become the fallback behavior for cross-module incremental builds.

Author: CodaFi

include/swift/Driver/FineGrainedDependencyDriverGraph.h

@@ -168,6 +168,7 @@ class ModuleDepGraph {
 
   // Supports requests from the driver to getExternalDependencies.
   std::unordered_set<std::string> externalDependencies;
+  std::unordered_set<std::string> incrementalExternalDependencies;
 
   /// Keyed by swiftdeps filename, so we can get back to Jobs.
   std::unordered_map<std::string, const driver::Job *> jobsBySwiftDeps;
@@ -512,15 +513,28 @@ class ModuleDepGraph {
   std::vector<const driver::Job *>
   findExternallyDependentUntracedJobs(StringRef externalDependency);
 
+  /// Find jobs that were previously not known to need compilation but that
+  /// depend on \c incrementalExternalDependency.
+  ///
+  /// This code path should only act as a fallback to the status-quo behavior.
+  /// Otherwise it acts to pessimize the behavior of cross-module incremental
+  /// builds.
+  std::vector<const driver::Job *>
+  findIncrementalExternallyDependentUntracedJobs(StringRef externalDependency);
+
   //============================================================================
   // MARK: ModuleDepGraph - External dependencies
   //============================================================================
 
 public:
   std::vector<StringRef> getExternalDependencies() const;
+  std::vector<StringRef> getIncrementalExternalDependencies() const;
 
   void forEachUntracedJobDirectlyDependentOnExternalSwiftDeps(
       StringRef externalDependency, function_ref<void(const driver::Job *)> fn);
+  void forEachUntracedJobDirectlyDependentOnExternalIncrementalSwiftDeps(
+      StringRef externalDependency, function_ref<void(const driver::Job *)> fn);
+
   //============================================================================
   // MARK: ModuleDepGraph - verification
   //============================================================================

lib/Driver/Compilation.cpp

@@ -935,6 +935,10 @@ namespace driver {
       for (const auto cmd :
            collectExternallyDependentJobsFromDependencyGraph(forRanges))
         jobsToSchedule.insert(cmd);
+      for (const auto cmd :
+           collectIncrementalExternallyDependentJobsFromDependencyGraph(
+               forRanges))
+        jobsToSchedule.insert(cmd);
       return jobsToSchedule;
     }
 
@@ -1109,6 +1113,21 @@ namespace driver {
       }
     }
 
+    void forEachOutOfDateIncrementalExternalDependency(
+        const bool forRanges,
+        function_ref<void(StringRef)> consumeExternalSwiftDeps) {
+      for (StringRef dependency :
+           getIncrementalExternalDependencies(forRanges)) {
+        // If the dependency has been modified since the oldest built file,
+        // or if we can't stat it for some reason (perhaps it's been
+        // deleted?), trigger rebuilds through the dependency graph.
+        llvm::sys::fs::file_status depStatus;
+        if (llvm::sys::fs::status(dependency, depStatus) ||
+            Comp.getLastBuildTime() < depStatus.getLastModificationTime())
+          consumeExternalSwiftDeps(dependency);
+      }
+    }
+
     CommandSet collectCascadedJobsFromDependencyGraph(
         const CommandSet &InitialCascadingCommands, const bool forRanges) {
       CommandSet CascadedJobs;
@@ -1141,6 +1160,25 @@ namespace driver {
       return ExternallyDependentJobs;
     }
 
+    SmallVector<const Job *, 16>
+    collectIncrementalExternallyDependentJobsFromDependencyGraph(
+        const bool forRanges) {
+      SmallVector<const Job *, 16> ExternallyDependentJobs;
+      // Check all cross-module dependencies as well.
+      forEachOutOfDateIncrementalExternalDependency(
+          forRanges, [&](StringRef dependency) {
+            // If the dependency has been modified since the oldest built file,
+            // or if we can't stat it for some reason (perhaps it's been
+            // deleted?), trigger rebuilds through the dependency graph.
+            for (const Job *marked :
+                 markExternalInDepGraph(dependency, forRanges))
+              ExternallyDependentJobs.push_back(marked);
+          });
+      noteBuildingJobs(ExternallyDependentJobs, forRanges,
+                       "because of external dependencies");
+      return ExternallyDependentJobs;
+    }
+
     /// Insert all jobs in \p Cmds (of descriptive name \p Kind) to the \c
     /// TaskQueue, and clear \p Cmds.
     template <typename Container>
@@ -1594,6 +1632,12 @@ namespace driver {
       return getFineGrainedDepGraph(forRanges).getExternalDependencies();
     }
 
+    std::vector<StringRef>
+    getIncrementalExternalDependencies(const bool forRanges) const {
+      return getFineGrainedDepGraph(forRanges)
+          .getIncrementalExternalDependencies();
+    }
+
     std::vector<const Job*>
     markExternalInDepGraph(StringRef externalDependency,
                                 const bool forRanges) {

lib/Driver/FineGrainedDependencyDriverGraph.cpp

@@ -195,6 +195,12 @@ std::vector<StringRef> ModuleDepGraph::getExternalDependencies() const {
                                 externalDependencies.end());
 }
 
+std::vector<StringRef>
+ModuleDepGraph::getIncrementalExternalDependencies() const {
+  return std::vector<StringRef>(incrementalExternalDependencies.begin(),
+                                incrementalExternalDependencies.end());
+}
+
 // Add every (swiftdeps) use of the external dependency to foundJobs.
 // Can return duplicates, but it doesn't break anything, and they will be
 // canonicalized later.
@@ -216,6 +222,26 @@ std::vector<const Job *> ModuleDepGraph::findExternallyDependentUntracedJobs(
   return foundJobs;
 }
 
+std::vector<const Job *>
+ModuleDepGraph::findIncrementalExternallyDependentUntracedJobs(
+    StringRef externalDependency) {
+  FrontendStatsTracer tracer(stats,
+                             "fine-grained-dependencies-"
+                             "findIncrementalExternallyDependentUntracedJobs");
+  std::vector<const Job *> foundJobs;
+  forEachUntracedJobDirectlyDependentOnExternalIncrementalSwiftDeps(
+      externalDependency, [&](const Job *job) {
+        foundJobs.push_back(job);
+        for (const Job *marked : findJobsToRecompileWhenWholeJobChanges(job)) {
+          // findJobsToRecompileWhenWholeJobChanges is reflexive
+          // Don't return job twice.
+          if (marked != job)
+            foundJobs.push_back(marked);
+        }
+      });
+  return foundJobs;
+}
+
 void ModuleDepGraph::forEachUntracedJobDirectlyDependentOnExternalSwiftDeps(
     StringRef externalSwiftDeps, function_ref<void(const Job *)> fn) {
   // TODO move nameForDep into key
@@ -228,6 +254,19 @@ void ModuleDepGraph::forEachUntracedJobDirectlyDependentOnExternalSwiftDeps(
   }
 }
 
+void ModuleDepGraph::
+    forEachUntracedJobDirectlyDependentOnExternalIncrementalSwiftDeps(
+        StringRef externalSwiftDeps, function_ref<void(const Job *)> fn) {
+  // TODO move nameForDep into key
+  // These nodes will depend on the *interface* of the external Decl.
+  DependencyKey key(NodeKind::incrementalExternalDepend, DeclAspect::interface,
+                    "", externalSwiftDeps.str());
+  for (const ModuleDepGraphNode *useNode : usesByDef[key]) {
+    if (!useNode->getHasBeenTraced())
+      fn(getJob(useNode->getSwiftDepsOfProvides()));
+  }
+}
+
 //==============================================================================
 // MARK: Integrating SourceFileDepGraph into ModuleDepGraph
 //==============================================================================
@@ -387,9 +426,14 @@ bool ModuleDepGraph::recordWhatUseDependsUpon(
       sourceFileUseNode, [&](const SourceFileDepGraphNode *def) {
         const bool isNewUse =
             usesByDef[def->getKey()].insert(moduleUseNode).second;
-        if (isNewUse && def->getKey().getKind() == NodeKind::externalDepend) {
+        if (isNewUse) {
           StringRef externalSwiftDeps = def->getKey().getName();
-          externalDependencies.insert(externalSwiftDeps.str());
+          if (def->getKey().getKind() == NodeKind::externalDepend) {
+            externalDependencies.insert(externalSwiftDeps.str());
+          } else if (def->getKey().getKind() ==
+                     NodeKind::incrementalExternalDepend) {
+            incrementalExternalDependencies.insert(externalSwiftDeps.str());
+          }
           useHasNewExternalDependency = true;
         }
       });
@@ -641,6 +685,9 @@ void ModuleDepGraph::verifyExternalDependencyUniqueness(
   assert((key.getKind() != NodeKind::externalDepend ||
           externalDependencies.count(key.getName().str()) == 1) &&
          "Ensure each external dependency is tracked exactly once");
+  assert((key.getKind() != NodeKind::incrementalExternalDepend ||
+          incrementalExternalDependencies.count(key.getName().str()) == 1) &&
+         "Ensure each incremental external dependency is tracked exactly once");
 }
 
 void ModuleDepGraph::verifyCanFindEachJob() const {
@@ -725,6 +772,10 @@ void ModuleDepGraph::printOneNodeOfPath(raw_ostream &out,
     out << filename << " depends on " << key.aspectName() << " of module '"
         << key.humanReadableName() << "'";
     break;
+  case NodeKind::incrementalExternalDepend:
+    out << filename << " depends on " << key.aspectName()
+        << " of incremental module '" << key.humanReadableName() << "'";
+    break;
   case NodeKind::sourceFileProvide:
     out << key.aspectName() << " of source file " << key.humanReadableName();
     break;