[ParseableInterface] Don't serialize resource directory deps and stop adding cached modules to the dependency tracker

This patch modifies ParseableInterfaceBuilder::CollectDepsForSerialization to
avoid serializing dependencies from the runtime resource path into the
swiftmodules generated from .swiftinterface files. This means the module cache
should now be relocatable across machines.

It also modifies ParseableInterfaceModuleLoader to never add any dependencies
from the module cache and prebuilt cache to the dependency tracker (in addition
to the existing behaviour of not serializing them in the generated
swiftmodules). As a result, CollectDepsForSerialization no longer checks if the
dependencies it is given come from the cache as they are provided by the
dependency tracker. It now asserts that's the case instead.

Author: Nathan Hawes

include/swift/Frontend/ParseableInterfaceModuleLoader.h

@@ -139,6 +139,8 @@ class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase {
     std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer) override;
 
+  bool isCached(StringRef DepPath) override;
+
 public:
   static std::unique_ptr<ParseableInterfaceModuleLoader>
   create(ASTContext &ctx, StringRef cacheDir, StringRef prebuiltCacheDir,

include/swift/Serialization/SerializedModuleLoader.h

@@ -80,6 +80,12 @@ class SerializedModuleLoaderBase : public ModuleLoader {
     return false;
   }
 
+  /// Determines if the provided path is a cached artifact for dependency
+  /// tracking purposes.
+  virtual bool isCached(StringRef DepPath) {
+    return false;
+  }
+
 public:
   virtual ~SerializedModuleLoaderBase();
   SerializedModuleLoaderBase(const SerializedModuleLoaderBase &) = delete;

lib/Frontend/ParseableInterfaceModuleLoader.cpp

@@ -356,19 +356,6 @@ class swift::ParseableInterfaceBuilder {
     return false;
   }
 
-  /// Determines if the dependency with the provided path is a swiftmodule in
-  /// either the module cache or prebuilt module cache
-  bool isCachedModule(StringRef DepName) const {
-    auto Ext = llvm::sys::path::extension(DepName);
-    auto Ty = file_types::lookupTypeForExtension(Ext);
-
-    if (Ty != file_types::TY_SwiftModuleFile)
-      return false;
-    if (!moduleCachePath.empty() && DepName.startswith(moduleCachePath))
-      return true;
-    return !prebuiltCachePath.empty() && DepName.startswith(prebuiltCachePath);
-  }
-
   /// Populate the provided \p Deps with \c FileDependency entries for all
   /// dependencies \p SubInstance's DependencyTracker recorded while compiling
   /// the module, excepting .swiftmodules in \p moduleCachePath or
@@ -379,12 +366,31 @@ class swift::ParseableInterfaceBuilder {
                                    SmallVectorImpl<FileDependency> &Deps,
                                    bool IsHashBased) {
     StringRef SDKPath = SubInstance.getASTContext().SearchPathOpts.SDKPath;
+    StringRef ResourcePath =
+        SubInstance.getASTContext().SearchPathOpts.RuntimeResourcePath;
     auto DTDeps = SubInstance.getDependencyTracker()->getDependencies();
     SmallVector<StringRef, 16> InitialDepNames(DTDeps.begin(), DTDeps.end());
     InitialDepNames.push_back(interfacePath);
     llvm::StringSet<> AllDepNames;
 
     for (auto const &DepName : InitialDepNames) {
+      assert(moduleCachePath.empty() || !DepName.startswith(moduleCachePath));
+      assert(prebuiltCachePath.empty() || !DepName.startswith(prebuiltCachePath));
+
+      /// Lazily load the dependency buffer if we need it. If we're not
+      /// dealing with a hash-based dependencies, and if the dependency is
+      /// not a .swiftmodule, we can avoid opening the buffer.
+      std::unique_ptr<llvm::MemoryBuffer> DepBuf = nullptr;
+      auto getDepBuf = [&]() -> llvm::MemoryBuffer * {
+        if (DepBuf) return DepBuf.get();
+        if (auto Buf = getBufferOfDependency(fs, DepName, interfacePath,
+                                             diags, diagnosticLoc)) {
+          DepBuf = std::move(Buf);
+          return DepBuf.get();
+        }
+        return nullptr;
+      };
+
       // Adjust the paths of dependences in the SDK to be relative to it
       bool IsSDKRelative = false;
       StringRef DepNameToStore = DepName;
@@ -410,48 +416,10 @@ class swift::ParseableInterfaceBuilder {
         dependencyTracker->addDependency(DepName, /*isSystem*/IsSDKRelative);
       }
 
-      /// Lazily load the dependency buffer if we need it. If we're not
-      /// dealing with a hash-based dependencies, and if the dependency is
-      /// not a .swiftmodule, we can avoid opening the buffer.
-      std::unique_ptr<llvm::MemoryBuffer> DepBuf = nullptr;
-      auto getDepBuf = [&]() -> llvm::MemoryBuffer * {
-        if (DepBuf) return DepBuf.get();
-        if (auto Buf = getBufferOfDependency(fs, DepName, interfacePath,
-                                             diags, diagnosticLoc)) {
-          DepBuf = std::move(Buf);
-          return DepBuf.get();
-        }
-        return nullptr;
-      };
-
-      // If Dep is itself a cached .swiftmodule, pull out its deps and include
-      // them in our own, so we have a single-file view of transitive deps:
-      // removes redundancies, makes the cache more relocatable, and avoids
-      // opening and reading multiple swiftmodules during future loads.
-      if (isCachedModule(DepName)) {
-        auto buf = getDepBuf();
-        if (!buf) return true;
-        SmallVector<FileDependency, 16> SubDeps;
-        auto VI = serialization::validateSerializedAST(buf->getBuffer(),
-          /*ExtendedValidationInfo=*/nullptr, &SubDeps);
-        if (VI.status != serialization::Status::Valid) {
-          diags.diagnose(diagnosticLoc,
-                         diag::error_extracting_dependencies_from_cached_module,
-                         DepName);
-          return true;
-        }
-        for (auto const &SubDep : SubDeps) {
-          if (AllDepNames.insert(SubDep.getPath()).second) {
-            Deps.push_back(SubDep);
-            if (dependencyTracker)
-              dependencyTracker->addDependency(
-                  SubDep.getPath(), /*IsSystem=*/SubDep.isSDKRelative());
-          }
-        }
+      // Don't serialize compiler-relative deps so the cache is relocatable.
+      if (DepName.startswith(ResourcePath))
         continue;
-      }
 
-      // Otherwise, include this dependency directly
       auto Status = getStatusOfDependency(fs, DepName, interfacePath,
                                           diags, diagnosticLoc);
       if (!Status)
@@ -761,11 +729,11 @@ class ParseableInterfaceModuleLoaderImpl {
       if (dependencyTracker)
         dependencyTracker->addDependency(fullPath, /*isSystem*/in.isSDKRelative());
       if (!dependencyIsUpToDate(in, fullPath)) {
-        LLVM_DEBUG(llvm::dbgs() << "Dep " << in.getPath()
+        LLVM_DEBUG(llvm::dbgs() << "Dep " << fullPath
                    << " is directly out of date\n");
         return false;
       }
-      LLVM_DEBUG(llvm::dbgs() << "Dep " << in.getPath() << " is up to date\n");
+      LLVM_DEBUG(llvm::dbgs() << "Dep " << fullPath << " is up to date\n");
     }
     return true;
   }
@@ -1047,6 +1015,12 @@ class ParseableInterfaceModuleLoaderImpl {
 
 } // end anonymous namespace
 
+bool ParseableInterfaceModuleLoader::isCached(StringRef DepPath) {
+  if (!CacheDir.empty() && DepPath.startswith(CacheDir))
+    return true;
+  return !PrebuiltCacheDir.empty() && DepPath.startswith(PrebuiltCacheDir);
+}
+
 /// Load a .swiftmodule associated with a .swiftinterface either from a
 /// cache or by converting it in a subordinate \c CompilerInstance, caching
 /// the results.

lib/Serialization/SerializedModuleLoader.cpp

@@ -645,9 +645,13 @@ ModuleDecl *SerializedModuleLoaderBase::loadModule(SourceLoc importLoc,
                     isFramework)) {
       return nullptr;
     }
-    if (dependencyTracker)
-      dependencyTracker->addDependency(moduleInputBuffer->getBufferIdentifier(),
-                                       /*isSystem=*/false);
+    if (dependencyTracker) {
+      // Don't record cached artifacts as dependencies.
+      StringRef DepPath = moduleInputBuffer->getBufferIdentifier();
+      if (!isCached(DepPath)) {
+        dependencyTracker->addDependency(DepPath, /*isSystem=*/false);
+      }
+    }
   }
 
   assert(moduleInputBuffer);

test/ParseableInterface/ModuleCache/SDKDependencies.swift

@@ -7,13 +7,14 @@
 // RUN: %target-swift-frontend -build-module-from-parseable-interface -serialize-parseable-module-interface-dependency-hashes -sdk %t/my-sdk -prebuilt-module-cache-path %t/prebuilt-cache -I %t/my-sdk -module-cache-path %t/MCP -o %t/prebuilt-cache/ExportedLib.swiftmodule -track-system-dependencies -module-name ExportedLib %t/my-sdk/ExportedLib.swiftinterface
 // RUN: %target-swift-frontend -build-module-from-parseable-interface -serialize-parseable-module-interface-dependency-hashes -sdk %t/my-sdk -prebuilt-module-cache-path %t/prebuilt-cache -I %t/my-sdk -module-cache-path %t/MCP -o %t/prebuilt-cache/SdkLib.swiftmodule -track-system-dependencies -module-name SdkLib %t/my-sdk/SdkLib.swiftinterface
 //
-// Check the prebuilt modules don't contain dependencies in the module cache or prebuilt cache
+// Check the prebuilt modules don't contain dependencies in the module cache, prebuilt cache, or resource dir
 // RUN: llvm-bcanalyzer -dump %t/prebuilt-cache/ExportedLib.swiftmodule | %FileCheck %s -check-prefix=PREBUILT
 // RUN: llvm-bcanalyzer -dump %t/prebuilt-cache/SdkLib.swiftmodule | %FileCheck %s -check-prefix=PREBUILT
 //
 // PREBUILT: MODULE_BLOCK
 // PREBUILT-NOT: FILE_DEPENDENCY {{.*}}/MCP/{{.*}}
 // PREBUILT-NOT: FILE_DEPENDENCY {{.*}}/prebuilt-cache/{{.*}}
+// PREBUILD-NOT: FILE_DEPENDENCY {{.*}}/lib/swift/{{.*}}
 //
 // Re-build them in the opposite order
 // RUN: %empty-directory(%t/prebuilt-cache)
@@ -45,6 +46,12 @@
 // RUN: llvm-bcanalyzer -dump %t/MCP/SdkLib-*.swiftmodule | %FileCheck %s -check-prefix=PREBUILT
 // RUN: llvm-bcanalyzer -dump %t/MCP/ExportedLib-*.swiftmodule | %FileCheck %s -check-prefix=PREBUILT
 //
+// Check we didn't emit anything from the cache in the .d file either
+// RUN: cat %t/dummy.d | %FileCheck %s -check-prefix=DEPFILE
+//
+// DEPFILE-NOT: /MCP/
+// DEPFILE-NOT: /prebuilt-cache/
+//
 // RUN: %empty-directory(%t/MCP)
 // RUN: echo '2: PASSED'
 
@@ -82,6 +89,9 @@
 // NOCACHE-NOT: /prebuilt-cache/
 // NOCACHE-NOT: /MCP/
 //
+// Check we didn't emit anything from the cache in the .d file either
+// RUN: cat %t/dummy.d | %FileCheck %s -check-prefix=DEPFILE
+//
 // RUN: %empty-directory(%t/MCP)
 // RUN: echo '3: PASSED'
 
@@ -115,6 +125,9 @@
 // RUN: cat %t/MCP/ExportedLib-*.swiftmodule | %FileCheck %s -check-prefix=NOCACHE -DLIB_NAME=ExportedLib
 // RUN: cat %t/MCP/SdkLib-*.swiftmodule | %FileCheck %s -check-prefix=NOCACHE -DLIB_NAME=SdkLib
 //
+// Check we didn't emit anything from the cache in the .d file either
+// RUN: cat %t/dummy.d | %FileCheck %s -check-prefix=DEPFILE
+//
 // RUN: %empty-directory(%t/MCP)
 // RUN: echo '4: PASSED'
 
@@ -156,6 +169,9 @@
 // RUN: llvm-bcanalyzer -dump %t/MCP/ExportedLib-*.swiftmodule | %FileCheck %s -check-prefix=PREBUILT
 // RUN: llvm-bcanalyzer -dump %t/MCP/SdkLib-*.swiftmodule | %FileCheck %s -check-prefix=PREBUILT
 //
+// Check we didn't emit anything from the cache in the .d file either
+// RUN: cat %t/dummy.d | %FileCheck %s -check-prefix=DEPFILE
+//
 // RUN: echo '5: PASSED'
 
 import SdkLib

test/ParseableInterface/ModuleCache/module-cache-init.swift

@@ -24,9 +24,10 @@
 // RUN: test -f %t/modulecache/LeafModule-*.swiftmodule
 // RUN: llvm-bcanalyzer -dump %t/modulecache/LeafModule-*.swiftmodule | %FileCheck %s -check-prefix=CHECK-LEAFMODULE
 // CHECK-LEAFMODULE: {{MODULE_NAME.*blob data = 'LeafModule'}}
-// CHECK-LEAFMODULE: {{FILE_DEPENDENCY.*Swift.swiftmodule([/\\].+[.]swiftmodule)?'}}
 // CHECK-LEAFMODULE: {{FILE_DEPENDENCY.*LeafModule.swiftinterface'}}
 // CHECK-LEAFMODULE: FUNC_DECL
+// RUN: llvm-bcanalyzer -dump %t/modulecache/LeafModule-*.swiftmodule | %FileCheck %s -check-prefix=CHECK-LEAFMODULE-NEGATIVE
+// CHECK-LEAFMODULE-NEGATIVE-NOT: {{FILE_DEPENDENCY.*Swift.swiftmodule([/\\].+[.]swiftmodule)?'}}
 //
 //
 // Phase 3: build TestModule into a .swiftmodule explicitly us OtherModule via OtherModule.swiftinterface, creating OtherModule-*.swiftmodule along the way.
@@ -37,7 +38,7 @@
 // RUN: test -f %t/TestModule.d
 // RUN: llvm-bcanalyzer -dump %t/modulecache/OtherModule-*.swiftmodule | %FileCheck %s -check-prefix=CHECK-OTHERMODULE
 // CHECK-OTHERMODULE: {{MODULE_NAME.*blob data = 'OtherModule'}}
-// CHECK-OTHERMODULE: {{FILE_DEPENDENCY.*Swift.swiftmodule([/\\].+[.]swiftmodule)?'}}
+// CHECK-OTHERMODULE-NOT: {{FILE_DEPENDENCY.*Swift.swiftmodule([/\\].+[.]swiftmodule)?'}}
 // CHECK-OTHERMODULE: {{FILE_DEPENDENCY.*LeafModule.swiftinterface'}}
 // CHECK-OTHERMODULE: {{FILE_DEPENDENCY.*OtherModule.swiftinterface'}}
 // CHECK-OTHERMODULE: FUNC_DECL
@@ -56,10 +57,13 @@
 // CHECK-DEPENDS: TestModule.swiftmodule :
 // CHECK-DEPENDS-DAG: LeafModule.swiftinterface
 // CHECK-DEPENDS-DAG: OtherModule.swiftinterface
-// CHECK-DEPENDS-DAG: {{OtherModule-[^ ]+.swiftmodule}}
 // CHECK-DEPENDS-DAG: Swift.swiftmodule
 // CHECK-DEPENDS-DAG: SwiftOnoneSupport.swiftmodule
-// CHECK-DEPENDS-DAG: {{LeafModule-[^ ]+.swiftmodule}}
+//
+// RUN: %FileCheck %s -check-prefix=CHECK-DEPENDS-NEGATIVE <%t/TestModule.d
+//
+// CHECK-DEPENDS-NEGATIVE-NOT: {{LeafModule-[^ ]+.swiftmodule}}
+// CHECK-DEPENDS-NEGATIVE-NOT: {{OtherModule-[^ ]+.swiftmodule}}
 
 import OtherModule