Merge pull request swiftlang#34075 from CodaFi/crosstabs

Differentiate Cross-Module Incremental Dependencies

Author: CodaFi

include/swift/AST/FineGrainedDependencies.h

@@ -346,6 +346,10 @@ class BiIndexedTwoStageMap {
 // MARK: Start of fine-grained-dependency-specific code
 //==============================================================================
 
+/// Uses the provided module or source file to construct a dependency graph,
+/// which is provided back to the caller in the continuation callback.
+///
+/// \Note The returned graph should not be escaped from the callback.
 bool withReferenceDependencies(
     llvm::PointerUnion<ModuleDecl *, SourceFile *> MSF,
     const DependencyTracker &depTracker, StringRef outputPath,

include/swift/AST/ModuleLoader.h

@@ -75,6 +75,9 @@ enum class IntermoduleDepTrackingMode {
 /// implemented in terms of a wrapped clang::DependencyCollector.
 class DependencyTracker {
   std::shared_ptr<clang::DependencyCollector> clangCollector;
+  SmallVector<std::string, 8> incrementalDeps;
+  llvm::StringSet<> incrementalDepsUniquer;
+
 public:
   explicit DependencyTracker(
       IntermoduleDepTrackingMode Mode,
@@ -87,9 +90,19 @@ class DependencyTracker {
   /// No path canonicalization is done.
   void addDependency(StringRef File, bool IsSystem);
 
+  /// Adds a file as an incremental dependency.
+  ///
+  /// No additional canonicalization or adulteration of the file path in
+  /// \p File is performed.
+  void addIncrementalDependency(StringRef File);
+
   /// Fetches the list of dependencies.
   ArrayRef<std::string> getDependencies() const;
 
+  /// Fetches the list of dependencies that are known to have incremental swift
+  /// dependency information embedded inside of them.
+  ArrayRef<std::string> getIncrementalDependencies() const;
+
   /// Return the underlying clang::DependencyCollector that this
   /// class wraps.
   std::shared_ptr<clang::DependencyCollector> getClangCollector();

include/swift/Basic/ReferenceDependencyKeys.h

@@ -53,15 +53,18 @@ enum class NodeKind {
   dynamicLookup,
   externalDepend,
   sourceFileProvide,
+  incrementalExternalDepend,
   /// For iterating through the NodeKinds.
   kindCount
 };
 
 /// Used for printing out NodeKinds to dot files, and dumping nodes for
 /// debugging.
 const std::string NodeKindNames[]{
-    "topLevel",      "nominal",        "potentialMember",  "member",
-    "dynamicLookup", "externalDepend", "sourceFileProvide"};
+    "topLevel",          "nominal",
+    "potentialMember",   "member",
+    "dynamicLookup",     "externalDepend",
+    "sourceFileProvide", "incrementalExternalDepend"};
 } // end namespace fine_grained_dependencies
 } // end namespace swift
 

include/swift/Basic/Statistics.def

@@ -114,6 +114,7 @@ FRONTEND_STATISTIC(AST, NumASTBytesAllocated)
 /// Number of file-level dependencies of this frontend job, as tracked in the
 /// AST context's dependency collector.
 FRONTEND_STATISTIC(AST, NumDependencies)
+FRONTEND_STATISTIC(AST, NumIncrementalDependencies)
 
 /// Number of top-level, dynamic, and member names referenced in this frontend
 /// job's source file, as tracked by the AST context's referenced-name tracker.

include/swift/Driver/FineGrainedDependencyDriverGraph.h

@@ -168,6 +168,7 @@ class ModuleDepGraph {
 
   // Supports requests from the driver to getExternalDependencies.
   std::unordered_set<std::string> externalDependencies;
+  std::unordered_set<std::string> incrementalExternalDependencies;
 
   /// Keyed by swiftdeps filename, so we can get back to Jobs.
   std::unordered_map<std::string, const driver::Job *> jobsBySwiftDeps;
@@ -512,15 +513,28 @@ class ModuleDepGraph {
   std::vector<const driver::Job *>
   findExternallyDependentUntracedJobs(StringRef externalDependency);
 
+  /// Find jobs that were previously not known to need compilation but that
+  /// depend on \c incrementalExternalDependency.
+  ///
+  /// This code path should only act as a fallback to the status-quo behavior.
+  /// Otherwise it acts to pessimize the behavior of cross-module incremental
+  /// builds.
+  std::vector<const driver::Job *>
+  findIncrementalExternallyDependentUntracedJobs(StringRef externalDependency);
+
   //============================================================================
   // MARK: ModuleDepGraph - External dependencies
   //============================================================================
 
 public:
   std::vector<StringRef> getExternalDependencies() const;
+  std::vector<StringRef> getIncrementalExternalDependencies() const;
 
   void forEachUntracedJobDirectlyDependentOnExternalSwiftDeps(
       StringRef externalDependency, function_ref<void(const driver::Job *)> fn);
+  void forEachUntracedJobDirectlyDependentOnExternalIncrementalSwiftDeps(
+      StringRef externalDependency, function_ref<void(const driver::Job *)> fn);
+
   //============================================================================
   // MARK: ModuleDepGraph - verification
   //============================================================================

lib/AST/FineGrainedDependencies.cpp

@@ -231,6 +231,7 @@ std::string DependencyKey::humanReadableName() const {
   switch (kind) {
   case NodeKind::member:
     return demangleTypeAsContext(context) + "." + name;
+  case NodeKind::incrementalExternalDepend:
   case NodeKind::externalDepend:
   case NodeKind::sourceFileProvide:
     return llvm::sys::path::filename(name).str();
@@ -261,9 +262,12 @@ raw_ostream &fine_grained_dependencies::operator<<(raw_ostream &out,
 bool DependencyKey::verify() const {
   assert((getKind() != NodeKind::externalDepend || isInterface()) &&
          "All external dependencies must be interfaces.");
+  assert((getKind() != NodeKind::incrementalExternalDepend || isInterface()) &&
+         "All incremental external dependencies must be interfaces.");
   switch (getKind()) {
   case NodeKind::topLevel:
   case NodeKind::dynamicLookup:
+  case NodeKind::incrementalExternalDepend:
   case NodeKind::externalDepend:
   case NodeKind::sourceFileProvide:
     assert(context.empty() && !name.empty() && "Must only have a name");
@@ -294,6 +298,7 @@ void DependencyKey::verifyNodeKindNames() {
       CHECK_NAME(potentialMember)
       CHECK_NAME(member)
       CHECK_NAME(dynamicLookup)
+      CHECK_NAME(incrementalExternalDepend)
       CHECK_NAME(externalDepend)
       CHECK_NAME(sourceFileProvide)
     case NodeKind::kindCount:

lib/AST/FrontendSourceFileDepGraphFactory.cpp

@@ -241,8 +241,10 @@ bool fine_grained_dependencies::withReferenceDependencies(
     const DependencyTracker &depTracker, StringRef outputPath,
     bool alsoEmitDotFile,
     llvm::function_ref<bool(SourceFileDepGraph &&)> cont) {
-  if (MSF.dyn_cast<ModuleDecl *>()) {
-    llvm_unreachable("Cannot construct dependency graph for modules!");
+  if (auto *MD = MSF.dyn_cast<ModuleDecl *>()) {
+    SourceFileDepGraph g =
+        ModuleDepGraphFactory(MD, alsoEmitDotFile).construct();
+    return cont(std::move(g));
   } else {
     auto *SF = MSF.get<SourceFile *>();
     SourceFileDepGraph g = FrontendSourceFileDepGraphFactory(
@@ -592,6 +594,9 @@ class UsedDeclEnumerator {
   }
 
   void enumerateExternalUses() {
+    for (StringRef s : depTracker.getIncrementalDependencies())
+      enumerateUse<NodeKind::incrementalExternalDepend>("", s);
+
     for (StringRef s : depTracker.getDependencies())
       enumerateUse<NodeKind::externalDepend>("", s);
   }
@@ -632,3 +637,75 @@ FrontendSourceFileDepGraphFactory::getFingerprintIfAny(const Decl *d) {
   }
   return None;
 }
+
+//==============================================================================
+// MARK: ModuleDepGraphFactory
+//==============================================================================
+
+ModuleDepGraphFactory::ModuleDepGraphFactory(ModuleDecl *Mod, bool emitDot)
+    : AbstractSourceFileDepGraphFactory(
+          /*include private*/ true, Mod->getASTContext().hadError(),
+          Mod->getNameStr(), "0xBADBEEF", emitDot, Mod->getASTContext().Diags),
+      Mod(Mod) {}
+
+void ModuleDepGraphFactory::addAllDefinedDecls() {
+  // TODO: express the multiple provides and depends streams with variadic
+  // templates
+
+  // Many kinds of Decls become top-level depends.
+
+  SmallVector<Decl *, 32> TopLevelDecls;
+  Mod->getTopLevelDecls(TopLevelDecls);
+  DeclFinder declFinder(TopLevelDecls, includePrivateDeps,
+                        [this](VisibleDeclConsumer &consumer) {
+                          return Mod->lookupClassMembers({}, consumer);
+                        });
+
+  addAllDefinedDeclsOfAGivenType<NodeKind::topLevel>(
+      declFinder.precedenceGroups);
+  addAllDefinedDeclsOfAGivenType<NodeKind::topLevel>(
+      declFinder.memberOperatorDecls);
+  addAllDefinedDeclsOfAGivenType<NodeKind::topLevel>(declFinder.operators);
+  addAllDefinedDeclsOfAGivenType<NodeKind::topLevel>(declFinder.topNominals);
+  addAllDefinedDeclsOfAGivenType<NodeKind::topLevel>(declFinder.topValues);
+  addAllDefinedDeclsOfAGivenType<NodeKind::nominal>(declFinder.allNominals);
+  addAllDefinedDeclsOfAGivenType<NodeKind::potentialMember>(
+      declFinder.potentialMemberHolders);
+  addAllDefinedDeclsOfAGivenType<NodeKind::member>(
+      declFinder.valuesInExtensions);
+  addAllDefinedDeclsOfAGivenType<NodeKind::dynamicLookup>(
+      declFinder.classMembers);
+}
+
+/// Given an array of Decls or pairs of them in \p declsOrPairs
+/// create node pairs for context and name
+template <NodeKind kind, typename ContentsT>
+void ModuleDepGraphFactory::addAllDefinedDeclsOfAGivenType(
+    std::vector<ContentsT> &contentsVec) {
+  for (const auto &declOrPair : contentsVec) {
+    Optional<std::string> fp = getFingerprintIfAny(declOrPair);
+    addADefinedDecl(
+        DependencyKey::createForProvidedEntityInterface<kind>(declOrPair),
+        fp ? StringRef(fp.getValue()) : Optional<StringRef>());
+  }
+}
+
+//==============================================================================
+// MARK: ModuleDepGraphFactory - adding individual defined Decls
+//==============================================================================
+
+/// At present, only \c NominalTypeDecls have (body) fingerprints
+Optional<std::string> ModuleDepGraphFactory::getFingerprintIfAny(
+    std::pair<const NominalTypeDecl *, const ValueDecl *>) {
+  return None;
+}
+Optional<std::string>
+ModuleDepGraphFactory::getFingerprintIfAny(const Decl *d) {
+  if (const auto *idc = dyn_cast<IterableDeclContext>(d)) {
+    auto result = idc->getBodyFingerprint();
+    assert((!result || !result->empty()) &&
+           "Fingerprint should never be empty");
+    return result;
+  }
+  return None;
+}

lib/AST/FrontendSourceFileDepGraphFactory.h

@@ -54,6 +54,30 @@ class FrontendSourceFileDepGraphFactory
   static Optional<std::string> getFingerprintIfAny(const Decl *d);
 };
 
+class ModuleDepGraphFactory : public AbstractSourceFileDepGraphFactory {
+  ModuleDecl *const Mod;
+
+public:
+  ModuleDepGraphFactory(ModuleDecl *Mod, bool emitDot);
+
+  ~ModuleDepGraphFactory() override = default;
+
+private:
+  void addAllDefinedDecls() override;
+  void addAllUsedDecls() override {}
+
+  /// Given an array of Decls or pairs of them in \p declsOrPairs
+  /// create node pairs for context and name
+  template <NodeKind kind, typename ContentsT>
+  void addAllDefinedDeclsOfAGivenType(std::vector<ContentsT> &contentsVec);
+
+  /// At present, only nominals, protocols, and extensions have (body)
+  /// fingerprints
+  static Optional<std::string> getFingerprintIfAny(
+      std::pair<const NominalTypeDecl *, const ValueDecl *>);
+  static Optional<std::string> getFingerprintIfAny(const Decl *d);
+};
+
 } // namespace fine_grained_dependencies
 } // namespace swift
 

lib/AST/ModuleLoader.cpp

@@ -50,11 +50,22 @@ DependencyTracker::addDependency(StringRef File, bool IsSystem) {
                                      /*IsMissing=*/false);
 }
 
+void DependencyTracker::addIncrementalDependency(StringRef File) {
+  if (incrementalDepsUniquer.insert(File).second) {
+    incrementalDeps.emplace_back(File.str());
+  }
+}
+
 ArrayRef<std::string>
 DependencyTracker::getDependencies() const {
   return clangCollector->getDependencies();
 }
 
+ArrayRef<std::string>
+DependencyTracker::getIncrementalDependencies() const {
+  return incrementalDeps;
+}
+
 std::shared_ptr<clang::DependencyCollector>
 DependencyTracker::getClangCollector() {
   return clangCollector;

lib/Driver/Compilation.cpp

@@ -935,6 +935,10 @@ namespace driver {
       for (const auto cmd :
            collectExternallyDependentJobsFromDependencyGraph(forRanges))
         jobsToSchedule.insert(cmd);
+      for (const auto cmd :
+           collectIncrementalExternallyDependentJobsFromDependencyGraph(
+               forRanges))
+        jobsToSchedule.insert(cmd);
       return jobsToSchedule;
     }
 
@@ -1109,6 +1113,21 @@ namespace driver {
       }
     }
 
+    void forEachOutOfDateIncrementalExternalDependency(
+        const bool forRanges,
+        function_ref<void(StringRef)> consumeExternalSwiftDeps) {
+      for (StringRef dependency :
+           getIncrementalExternalDependencies(forRanges)) {
+        // If the dependency has been modified since the oldest built file,
+        // or if we can't stat it for some reason (perhaps it's been
+        // deleted?), trigger rebuilds through the dependency graph.
+        llvm::sys::fs::file_status depStatus;
+        if (llvm::sys::fs::status(dependency, depStatus) ||
+            Comp.getLastBuildTime() < depStatus.getLastModificationTime())
+          consumeExternalSwiftDeps(dependency);
+      }
+    }
+
     CommandSet collectCascadedJobsFromDependencyGraph(
         const CommandSet &InitialCascadingCommands, const bool forRanges) {
       CommandSet CascadedJobs;
@@ -1141,6 +1160,25 @@ namespace driver {
       return ExternallyDependentJobs;
     }
 
+    SmallVector<const Job *, 16>
+    collectIncrementalExternallyDependentJobsFromDependencyGraph(
+        const bool forRanges) {
+      SmallVector<const Job *, 16> ExternallyDependentJobs;
+      // Check all cross-module dependencies as well.
+      forEachOutOfDateIncrementalExternalDependency(
+          forRanges, [&](StringRef dependency) {
+            // If the dependency has been modified since the oldest built file,
+            // or if we can't stat it for some reason (perhaps it's been
+            // deleted?), trigger rebuilds through the dependency graph.
+            for (const Job *marked :
+                 markExternalInDepGraph(dependency, forRanges))
+              ExternallyDependentJobs.push_back(marked);
+          });
+      noteBuildingJobs(ExternallyDependentJobs, forRanges,
+                       "because of external dependencies");
+      return ExternallyDependentJobs;
+    }
+
     /// Insert all jobs in \p Cmds (of descriptive name \p Kind) to the \c
     /// TaskQueue, and clear \p Cmds.
     template <typename Container>
@@ -1594,6 +1632,12 @@ namespace driver {
       return getFineGrainedDepGraph(forRanges).getExternalDependencies();
     }
 
+    std::vector<StringRef>
+    getIncrementalExternalDependencies(const bool forRanges) const {
+      return getFineGrainedDepGraph(forRanges)
+          .getIncrementalExternalDependencies();
+    }
+
     std::vector<const Job*>
     markExternalInDepGraph(StringRef externalDependency,
                                 const bool forRanges) {