[Dependency Scanning] Refactor primary scan operations into 'ModuleDependencyScanner' class

From being a scattered collection of 'static' methods in ScanDependencies.cpp
and member methods of ASTContext. This makes 'ScanDependencies.cpp' much easier
to read, and abstracts the actual scanning logic away to a place with common
state which will make it easier to reason about in the future.

Author: artemcm

include/swift/AST/ASTContext.h

@@ -92,7 +92,6 @@ namespace swift {
   class LazyIterableDeclContextData;
   class LazyMemberLoader;
   struct MacroDiscriminatorContext;
-  class ModuleDependencyInfo;
   class PatternBindingDecl;
   class PatternBindingInitializer;
   class PluginLoader;
@@ -1045,27 +1044,6 @@ class ASTContext final {
   /// Retrieve the module interface checker associated with this AST context.
   ModuleInterfaceChecker *getModuleInterfaceChecker() const;
 
-  /// Retrieve the module dependencies for the module with the given name.
-  ///
-  llvm::Optional<const ModuleDependencyInfo *> getModuleDependencies(
-      StringRef moduleName, ModuleDependenciesCache &cache,
-      InterfaceSubContextDelegate &delegate,
-      bool optionalDependencyLookup = false, bool isTestableImport = false,
-      llvm::Optional<std::pair<std::string, swift::ModuleDependencyKind>>
-          dependencyOf = llvm::None);
-
-  /// Retrieve the module dependencies for the Clang module with the given name.
-  llvm::Optional<const ModuleDependencyInfo *>
-  getClangModuleDependencies(StringRef moduleName,
-                             ModuleDependenciesCache &cache,
-                             InterfaceSubContextDelegate &delegate);
-
-  /// Retrieve the module dependencies for the Swift module with the given name.
-  llvm::Optional<const ModuleDependencyInfo *>
-  getSwiftModuleDependencies(StringRef moduleName,
-                             ModuleDependenciesCache &cache,
-                             InterfaceSubContextDelegate &delegate);
-
   /// Compute the extra implicit framework search paths on Apple platforms:
   /// $SDKROOT/System/Library/Frameworks/ and $SDKROOT/Library/Frameworks/.
   std::vector<std::string> getDarwinImplicitFrameworkSearchPaths() const;

include/swift/AST/ModuleDependencies.h

@@ -39,6 +39,7 @@
 #include "llvm/Support/VirtualFileSystem.h"
 #include <string>
 #include <unordered_map>
+#include <unordered_set>
 #include <vector>
 
 namespace swift {
@@ -86,17 +87,38 @@ enum class ModuleDependencyKind : int8_t {
   LastKind = SwiftPlaceholder + 1
 };
 
-using ModuleDependencyID = std::pair<std::string, ModuleDependencyKind>;
-using ModuleDependencyIDSetVector =
-    llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
-                    std::set<ModuleDependencyID>>;
+/// This is used to identify a specific module.
+struct ModuleDependencyID {
+  std::string ModuleName;
+  ModuleDependencyKind Kind;
+  bool operator==(const ModuleDependencyID &Other) const {
+    return std::tie(ModuleName, Kind) ==
+           std::tie(Other.ModuleName, Other.Kind);
+  }
+  bool operator<(const ModuleDependencyID& Other) const {
+    return std::tie(ModuleName, Kind) <
+           std::tie(Other.ModuleName, Other.Kind);
+  }
+};
 
 struct ModuleDependencyKindHash {
   std::size_t operator()(ModuleDependencyKind k) const {
     using UnderlyingType = std::underlying_type<ModuleDependencyKind>::type;
     return std::hash<UnderlyingType>{}(static_cast<UnderlyingType>(k));
   }
 };
+struct ModuleDependencyIDHash {
+  std::size_t operator()(ModuleDependencyID id) const {
+    return llvm::hash_combine(id.ModuleName, id.Kind);
+  }
+};
+
+using ModuleDependencyIDSet =
+    std::unordered_set<ModuleDependencyID,
+                       ModuleDependencyIDHash>;
+using ModuleDependencyIDSetVector =
+    llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
+                    std::set<ModuleDependencyID>>;
 
 namespace dependencies {
   std::string createEncodedModuleKindAndName(ModuleDependencyID id);
@@ -750,6 +772,7 @@ class ModuleDependencyInfo {
   collectCrossImportOverlayNames(ASTContext &ctx, StringRef moduleName) const;
 };
 
+using ModuleDependencyVector = llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>;
 using ModuleNameToDependencyMap = llvm::StringMap<ModuleDependencyInfo>;
 using ModuleDependenciesKindMap =
     std::unordered_map<ModuleDependencyKind,
@@ -861,6 +884,7 @@ class SwiftDependencyScanningService {
   }
 
   bool usingCachingFS() const { return !UseClangIncludeTree && (bool)CacheFS; }
+  llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> getCachingFS() const { return CacheFS; }
 
   llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const {
     assert(CacheFS && "Expect CachingOnDiskFileSystem");
@@ -895,9 +919,10 @@ class SwiftDependencyScanningService {
   /// Enforce clients not being allowed to query this cache directly, it must be
   /// wrapped in an instance of `ModuleDependenciesCache`.
   friend class ModuleDependenciesCache;
+  friend class ModuleDependencyScanner;
+  friend class ModuleDependencyScanningWorker;
   friend class ModuleDependenciesCacheDeserializer;
   friend class ModuleDependenciesCacheSerializer;
-  friend class DependencyScanningTool;
 
   /// Configure the current state of the cache to respond to queries
   /// for the specified scanning context hash.
@@ -964,8 +989,6 @@ class ModuleDependenciesCache {
   std::string scannerContextHash;
   /// The location of where the built modules will be output to
   std::string moduleOutputPath;
-  /// The Clang dependency scanner tool
-  clang::tooling::dependencies::DependencyScanningTool clangScanningTool;
 
   /// Retrieve the dependencies map that corresponds to the given dependency
   /// kind.
@@ -987,23 +1010,29 @@ class ModuleDependenciesCache {
   bool hasDependency(StringRef moduleName,
                      llvm::Optional<ModuleDependencyKind> kind) const;
 
-  /// Produce a reference to the Clang scanner tool associated with this cache
-  clang::tooling::dependencies::DependencyScanningTool& getClangScannerTool() {
-    return clangScanningTool;
-  }
   SwiftDependencyScanningService &getScanService() {
     return globalScanningService;
   }
-  llvm::DenseSet<clang::tooling::dependencies::ModuleID>& getAlreadySeenClangModules() {
+  const llvm::DenseSet<clang::tooling::dependencies::ModuleID>& getAlreadySeenClangModules() const {
     return alreadySeenClangModules;
   }
   void addSeenClangModule(clang::tooling::dependencies::ModuleID newModule) {
     alreadySeenClangModules.insert(newModule);
   }
-  std::string getModuleOutputPath() {
+  std::string getModuleOutputPath() const {
     return moduleOutputPath;
   }
 
+  /// Query all dependencies, direct and Swift overlay.
+  std::vector<ModuleDependencyID>
+  getAllDependencies(const ModuleDependencyID &moduleID) const;
+
+  /// Look for module dependencies for a module with the given ID
+  ///
+  /// \returns the cached result, or \c None if there is no cached entry.
+  llvm::Optional<const ModuleDependencyInfo *>
+  findDependency(const ModuleDependencyID moduleID) const;
+
   /// Look for module dependencies for a module with the given name
   ///
   /// \returns the cached result, or \c None if there is no cached entry.
@@ -1015,6 +1044,9 @@ class ModuleDependenciesCache {
   void recordDependency(StringRef moduleName,
                         ModuleDependencyInfo dependencies);
 
+  /// Record dependencies for the given module collection.
+  void recordDependencies(ModuleDependencyVector moduleDependencies);
+
   /// Update stored dependencies for the given module.
   void updateDependency(ModuleDependencyID moduleID,
                         ModuleDependencyInfo dependencies);
@@ -1039,12 +1071,8 @@ class ModuleDependenciesCache {
 namespace std {
 template <>
 struct hash<swift::ModuleDependencyID> {
-  using UnderlyingKindType = std::underlying_type<swift::ModuleDependencyKind>::type;
   std::size_t operator()(const swift::ModuleDependencyID &id) const {
-    auto underlyingKindValue = static_cast<UnderlyingKindType>(id.second);
-
-    return (hash<string>()(id.first) ^
-            (hash<UnderlyingKindType>()(underlyingKindValue)));
+    return llvm::hash_combine(id.ModuleName, id.Kind);
   }
 };
 } // namespace std

include/swift/AST/ModuleLoader.h

@@ -28,6 +28,7 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TinyPtrVector.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/Support/VersionTuple.h"
 #include <system_error>
 
@@ -36,10 +37,19 @@ class FileCollectorBase;
 namespace vfs {
 class OutputBackend;
 }
+namespace cas {
+class CachingOnDiskFileSystem;
+}
 }
 
 namespace clang {
 class DependencyCollector;
+namespace tooling {
+namespace dependencies {
+struct ModuleID;
+class DependencyScanningTool;
+}
+}
 }
 
 namespace swift {
@@ -51,6 +61,7 @@ class ClassDecl;
 class FileUnit;
 class ModuleDecl;
 class ModuleDependencyInfo;
+struct ModuleDependencyID;
 class ModuleDependenciesCache;
 class NominalTypeDecl;
 class SourceFile;
@@ -327,8 +338,12 @@ class ModuleLoader {
 
   /// Retrieve the dependencies for the given, named module, or \c None
   /// if no such module exists.
-  virtual llvm::Optional<const ModuleDependencyInfo *>
-  getModuleDependencies(StringRef moduleName, ModuleDependenciesCache &cache,
+  virtual llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
+  getModuleDependencies(StringRef moduleName,
+                        StringRef moduleOutputPath,
+                        llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS,
+                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
+                        clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                         InterfaceSubContextDelegate &delegate,
                         bool isTestableImport = false) = 0;
 };

include/swift/ClangImporter/ClangImporter.h

@@ -75,6 +75,7 @@ class FuncDecl;
 class ImportDecl;
 class IRGenOptions;
 class ModuleDecl;
+struct ModuleDependencyID;
 class NominalTypeDecl;
 class StructDecl;
 class SwiftLookupTable;
@@ -433,32 +434,37 @@ class ClangImporter final : public ClangModuleLoader {
 
   void verifyAllModules() override;
 
-  void recordModuleDependencies(
-      ModuleDependenciesCache &cache,
-      const clang::tooling::dependencies::ModuleDepsGraph &clangDependencies);
+  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1> bridgeClangModuleDependencies(
+      const clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
+      StringRef moduleOutputPath);
+
+  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
+  getModuleDependencies(StringRef moduleName, StringRef moduleOutputPath,
+                        llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS,
+                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
+                        clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
+                        InterfaceSubContextDelegate &delegate,
+                        bool isTestableImport = false) override;
 
   void recordBridgingHeaderOptions(
       ModuleDependencyInfo &MDI,
       const clang::tooling::dependencies::TranslationUnitDeps &deps);
 
-  llvm::Optional<const ModuleDependencyInfo *>
-  getModuleDependencies(StringRef moduleName, ModuleDependenciesCache &cache,
-                        InterfaceSubContextDelegate &delegate,
-                        bool isTestableImport = false) override;
-
   /// Add dependency information for the bridging header.
   ///
-  /// \param moduleName the name of the Swift module whose dependency
+  /// \param moduleID the name of the Swift module whose dependency
   /// information will be augmented with information about the given
   /// bridging header.
   ///
+  /// \param clangScanningTool The clang dependency scanner.
+  ///
   /// \param cache The module dependencies cache to update, with information
   /// about new Clang modules discovered along the way.
   ///
   /// \returns \c true if an error occurred, \c false otherwise
   bool addBridgingHeaderDependencies(
-      StringRef moduleName,
-      ModuleDependencyKind moduleKind,
+      ModuleDependencyID moduleID,
+      clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
       ModuleDependenciesCache &cache);
   clang::TargetInfo &getModuleAvailabilityTarget() const override;
   clang::ASTContext &getClangASTContext() const override;

include/swift/DependencyScan/ScanDependencies.h

@@ -55,19 +55,20 @@ bool prescanDependencies(CompilerInstance &instance);
 bool batchScanDependencies(CompilerInstance &instance,
                            llvm::StringRef batchInputFile);
 
-/// Batch prescan the imports of modules specified in \c batchInputFile.
-bool batchPrescanDependencies(CompilerInstance &instance,
-                              llvm::StringRef batchInputFile);
-
 // MARK: Dependency scanning execution
 /// Scans the dependencies of the main module of \c instance.
 llvm::ErrorOr<swiftscan_dependency_graph_t>
 performModuleScan(CompilerInstance &instance,
                   ModuleDependenciesCache &cache);
 
+llvm::ErrorOr<swiftscan_dependency_graph_t>
+performParallelModuleScan(CompilerInstance &instance,
+                          ModuleDependenciesCache &cache);
+
 /// Scans the main module of \c instance for all direct module imports
 llvm::ErrorOr<swiftscan_import_set_t>
-performModulePrescan(CompilerInstance &instance);
+performModulePrescan(CompilerInstance &instance,
+                     ModuleDependenciesCache &cache);
 
 /// Batch scan the dependencies for modules specified in \c batchInputFile.
 std::vector<llvm::ErrorOr<swiftscan_dependency_graph_t>>
@@ -76,14 +77,6 @@ performBatchModuleScan(CompilerInstance &invocationInstance,
                        CompilerArgInstanceCacheMap *versionedPCMInstanceCache,
                        llvm::StringSaver &saver,
                        const std::vector<BatchScanInput> &BatchInput);
-
-/// Batch prescan the imports of modules specified in \c batchInputFile.
-std::vector<llvm::ErrorOr<swiftscan_import_set_t>>
-performBatchModulePrescan(CompilerInstance &invocationInstance,
-                          ModuleDependenciesCache &cache,
-                          llvm::StringSaver &saver,
-                          const std::vector<BatchScanInput> &BatchInput);
-
 } // end namespace dependencies
 } // end namespace swift
 

include/swift/Frontend/FrontendOptions.h

@@ -363,9 +363,12 @@ class FrontendOptions {
   /// The path at which to either serialize or deserialize the dependency scanner cache.
   std::string SerializedDependencyScannerCachePath;
 
-  /// Emit remarks indicating use of the serialized module dependency scanning cache
+  /// Emit remarks indicating use of the serialized module dependency scanning cache.
   bool EmitDependencyScannerCacheRemarks = false;
 
+  /// Whether the dependency scanner invocation should use multiple threads.
+  bool ParallelDependencyScan = false;
+
   /// When performing an incremental build, ensure that cross-module incremental
   /// build metadata is available in any swift modules emitted by this frontend
   /// job.

include/swift/Option/FrontendOptions.td

@@ -255,6 +255,9 @@ def dependency_scan_cache_path : Separate<["-"], "dependency-scan-cache-path">,
 def dependency_scan_cache_remarks : Flag<["-"], "Rdependency-scan-cache">,
   HelpText<"Emit remarks indicating use of the serialized module dependency scanning cache.">;
 
+def parallel_scan : Flag<["-"], "parallel-scan">,
+   HelpText<"Perform dependency scanning in-parallel.">;
+
 def enable_copy_propagation : Flag<["-"], "enable-copy-propagation">,
   HelpText<"Run SIL copy propagation with lexical lifetimes to shorten object "
            "lifetimes while preserving variable lifetimes.">;

include/swift/Sema/SourceLoader.h

@@ -97,8 +97,11 @@ class SourceLoader : public ModuleLoader {
     // Parsing populates the Objective-C method tables.
   }
 
-  llvm::Optional<const ModuleDependencyInfo *>
-  getModuleDependencies(StringRef moduleName, ModuleDependenciesCache &cache,
+  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
+  getModuleDependencies(StringRef moduleName, StringRef moduleOutputPath,
+                        llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS,
+                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
+                        clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                         InterfaceSubContextDelegate &delegate,
                         bool isTestableImport) override;
 };