Merge pull request swiftlang#21278 from jrose-apple/build-parseable-interface-to-module

Add frontend mode -build-module-from-parseable-interface

Author: jrose-apple

include/swift/Frontend/FrontendOptions.h

@@ -126,6 +126,9 @@ class FrontendOptions {
     EmitModuleOnly, ///< Emit module only
     MergeModules,   ///< Merge modules only
 
+    /// Build from a swiftinterface, as close to `import` as possible
+    BuildModuleFromParseableInterface,
+
     EmitSIBGen, ///< Emit serialized AST + raw SIL
     EmitSIB,    ///< Emit serialized AST + canonical SIL
 

include/swift/Frontend/ParseableInterfaceSupport.h

@@ -69,10 +69,13 @@ class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase {
 
   std::string CacheDir;
 
-  void
-  configureSubInvocationAndOutputPaths(CompilerInvocation &SubInvocation,
-                                       Identifier ModuleName, StringRef InPath,
-                                       llvm::SmallString<128> &OutPath);
+  /// Wire up the SubInvocation's InputsAndOutputs to contain both input and
+  /// output filenames.
+  ///
+  /// This is a method rather than a helper function in the implementation file
+  /// because it accesses non-public bits of FrontendInputsAndOutputs.
+  static void configureSubInvocationInputsAndOutputs(
+    CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath);
 
   std::error_code
   openModuleFiles(AccessPathElem ModuleID, StringRef DirName,
@@ -89,6 +92,17 @@ class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase {
     return std::unique_ptr<ParseableInterfaceModuleLoader>(
         new ParseableInterfaceModuleLoader(ctx, cacheDir, tracker, loadMode));
   }
+
+  /// Unconditionally build \p InPath (a swiftinterface file) to \p OutPath (as
+  /// a swiftmodule file).
+  ///
+  /// A simplified version of the core logic in #openModuleFiles, mostly for
+  /// testing purposes.
+  static bool buildSwiftModuleFromSwiftInterface(ASTContext &Ctx,
+                                                 StringRef CacheDir,
+                                                 StringRef ModuleName,
+                                                 StringRef InPath,
+                                                 StringRef OutPath);
 };
 
 

include/swift/Option/FrontendOptions.td

@@ -513,6 +513,11 @@ def dump_interface_hash : Flag<["-"], "dump-interface-hash">,
    HelpText<"Parse input file(s) and dump interface token hash(es)">,
    ModeOpt;
 
+def build_module_from_parseable_interface :
+  Flag<["-"], "build-module-from-parseable-interface">,
+  HelpText<"Treat the (single) input as a swiftinterface and produce a module">,
+  ModeOpt;
+
 def enable_resilience_bypass : Flag<["-"], "enable-resilience-bypass">,
    HelpText<"Completely bypass resilience when accessing types in resilient frameworks">;
 

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -370,6 +370,8 @@ ArgsToFrontendOptionsConverter::determineRequestedAction(const ArgList &args) {
     return FrontendOptions::ActionType::REPL;
   if (Opt.matches(OPT_interpret))
     return FrontendOptions::ActionType::Immediate;
+  if (Opt.matches(OPT_build_module_from_parseable_interface))
+    return FrontendOptions::ActionType::BuildModuleFromParseableInterface;
 
   llvm_unreachable("Unhandled mode option");
 }

lib/Frontend/FrontendOptions.cpp

@@ -48,6 +48,7 @@ bool FrontendOptions::needsProperModuleName(ActionType action) {
   case ActionType::EmitSIB:
   case ActionType::EmitModuleOnly:
   case ActionType::MergeModules:
+  case ActionType::BuildModuleFromParseableInterface:
     return true;
   case ActionType::Immediate:
   case ActionType::REPL:
@@ -83,6 +84,7 @@ bool FrontendOptions::isActionImmediate(ActionType action) {
   case ActionType::EmitSIB:
   case ActionType::EmitModuleOnly:
   case ActionType::MergeModules:
+  case ActionType::BuildModuleFromParseableInterface:
     return false;
   case ActionType::Immediate:
   case ActionType::REPL:
@@ -170,6 +172,7 @@ FrontendOptions::formatForPrincipalOutputFileForAction(ActionType action) {
 
   case ActionType::MergeModules:
   case ActionType::EmitModuleOnly:
+  case ActionType::BuildModuleFromParseableInterface:
     return TY_SwiftModuleFile;
 
   case ActionType::Immediate:
@@ -207,6 +210,7 @@ bool FrontendOptions::canActionEmitDependencies(ActionType action) {
   case ActionType::DumpScopeMaps:
   case ActionType::DumpTypeRefinementContexts:
   case ActionType::DumpTypeInfo:
+  case ActionType::BuildModuleFromParseableInterface:
   case ActionType::Immediate:
   case ActionType::REPL:
     return false;
@@ -242,6 +246,7 @@ bool FrontendOptions::canActionEmitReferenceDependencies(ActionType action) {
   case ActionType::DumpScopeMaps:
   case ActionType::DumpTypeRefinementContexts:
   case ActionType::DumpTypeInfo:
+  case ActionType::BuildModuleFromParseableInterface:
   case ActionType::Immediate:
   case ActionType::REPL:
     return false;
@@ -277,6 +282,7 @@ bool FrontendOptions::canActionEmitObjCHeader(ActionType action) {
   case ActionType::DumpScopeMaps:
   case ActionType::DumpTypeRefinementContexts:
   case ActionType::DumpTypeInfo:
+  case ActionType::BuildModuleFromParseableInterface:
   case ActionType::Immediate:
   case ActionType::REPL:
     return false;
@@ -309,6 +315,7 @@ bool FrontendOptions::canActionEmitLoadedModuleTrace(ActionType action) {
   case ActionType::DumpScopeMaps:
   case ActionType::DumpTypeRefinementContexts:
   case ActionType::DumpTypeInfo:
+  case ActionType::BuildModuleFromParseableInterface:
   case ActionType::Immediate:
   case ActionType::REPL:
     return false;
@@ -347,6 +354,7 @@ bool FrontendOptions::canActionEmitModule(ActionType action) {
   case ActionType::DumpTypeRefinementContexts:
   case ActionType::DumpTypeInfo:
   case ActionType::EmitSILGen:
+  case ActionType::BuildModuleFromParseableInterface:
   case ActionType::Immediate:
   case ActionType::REPL:
     return false;
@@ -379,15 +387,16 @@ bool FrontendOptions::canActionEmitInterface(ActionType action) {
   case ActionType::DumpAST:
   case ActionType::EmitSyntax:
   case ActionType::PrintAST:
+  case ActionType::EmitImportedModules:
   case ActionType::EmitPCH:
   case ActionType::DumpScopeMaps:
   case ActionType::DumpTypeRefinementContexts:
   case ActionType::DumpTypeInfo:
   case ActionType::EmitSILGen:
   case ActionType::EmitSIBGen:
+  case ActionType::BuildModuleFromParseableInterface:
   case ActionType::Immediate:
   case ActionType::REPL:
-  case ActionType::EmitImportedModules:
     return false;
   case ActionType::Typecheck:
   case ActionType::MergeModules:
@@ -427,6 +436,7 @@ bool FrontendOptions::doesActionProduceOutput(ActionType action) {
   case ActionType::EmitObject:
   case ActionType::EmitImportedModules:
   case ActionType::MergeModules:
+  case ActionType::BuildModuleFromParseableInterface:
   case ActionType::DumpTypeInfo:
     return true;
 
@@ -446,6 +456,7 @@ bool FrontendOptions::doesActionProduceTextualOutput(ActionType action) {
   case ActionType::EmitSIB:
   case ActionType::MergeModules:
   case ActionType::EmitModuleOnly:
+  case ActionType::BuildModuleFromParseableInterface:
   case ActionType::EmitBC:
   case ActionType::EmitObject:
   case ActionType::Immediate:
@@ -488,6 +499,7 @@ bool FrontendOptions::doesActionGenerateSIL(ActionType action) {
   case ActionType::DumpTypeRefinementContexts:
   case ActionType::EmitImportedModules:
   case ActionType::EmitPCH:
+  case ActionType::BuildModuleFromParseableInterface:
     return false;
   case ActionType::EmitSILGen:
   case ActionType::EmitSIBGen:

lib/Frontend/ParseableInterfaceSupport.cpp

@@ -107,7 +107,7 @@ getBufferOfDependency(clang::vfs::FileSystem &FS,
 /// with dead entries -- when other factors change, such as the contents of
 /// the .swiftinterface input or its dependencies.
 static std::string getCacheHash(ASTContext &Ctx,
-                                CompilerInvocation &SubInvocation,
+                                const CompilerInvocation &SubInvocation,
                                 StringRef InPath) {
   // Start with the compiler version (which will be either tag names or revs).
   std::string vers = swift::version::getSwiftFullVersion(
@@ -128,16 +128,14 @@ static std::string getCacheHash(ASTContext &Ctx,
   return llvm::APInt(64, H).toString(36, /*Signed=*/false);
 }
 
-void
-ParseableInterfaceModuleLoader::configureSubInvocationAndOutputPaths(
-    CompilerInvocation &SubInvocation,
-    Identifier ModuleName,
-    StringRef InPath,
-    llvm::SmallString<128> &OutPath) {
-
+static CompilerInvocation
+createInvocationForBuildingFromInterface(ASTContext &Ctx, StringRef ModuleName,
+                                         StringRef CacheDir) {
   auto &SearchPathOpts = Ctx.SearchPathOpts;
   auto &LangOpts = Ctx.LangOpts;
 
+  CompilerInvocation SubInvocation;
+
   // Start with a SubInvocation that copies various state from our
   // invoking ASTContext.
   SubInvocation.setImportSearchPaths(SearchPathOpts.ImportSearchPaths);
@@ -147,7 +145,7 @@ ParseableInterfaceModuleLoader::configureSubInvocationAndOutputPaths(
   SubInvocation.setRuntimeResourcePath(SearchPathOpts.RuntimeResourcePath);
   SubInvocation.setTargetTriple(LangOpts.Target);
   SubInvocation.setClangModuleCachePath(CacheDir);
-  SubInvocation.setModuleName(ModuleName.str());
+  SubInvocation.setModuleName(ModuleName);
 
   // Inhibit warnings from the SubInvocation since we are assuming the user
   // is not in a position to fix them.
@@ -161,24 +159,35 @@ ParseableInterfaceModuleLoader::configureSubInvocationAndOutputPaths(
   // modules that don't import Foundation.
   SubInvocation.getLangOptions().EnableObjCAttrRequiresFoundation = false;
 
-  // Calculate an output filename that includes a hash of relevant key data, and
-  // wire up the SubInvocation's InputsAndOutputs to contain both input and
-  // output filenames.
-  OutPath = CacheDir;
-  llvm::sys::path::append(OutPath, ModuleName.str());
+  return SubInvocation;
+}
+
+/// Calculate an output filename in \p SubInvocation's cache path that includes
+/// a hash of relevant key data.
+static void computeCachedOutputPath(ASTContext &Ctx,
+                                    const CompilerInvocation &SubInvocation,
+                                    StringRef InPath,
+                                    llvm::SmallString<128> &OutPath) {
+  OutPath = SubInvocation.getClangModuleCachePath();
+  llvm::sys::path::append(OutPath, SubInvocation.getModuleName());
   OutPath.append("-");
   OutPath.append(getCacheHash(Ctx, SubInvocation, InPath));
   OutPath.append(".");
   auto OutExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
   OutPath.append(OutExt);
+}
 
+void ParseableInterfaceModuleLoader::configureSubInvocationInputsAndOutputs(
+    CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath) {
   auto &SubFEOpts = SubInvocation.getFrontendOptions();
   SubFEOpts.RequestedAction = FrontendOptions::ActionType::EmitModuleOnly;
   SubFEOpts.EnableParseableModuleInterface = true;
   SubFEOpts.InputsAndOutputs.addPrimaryInputFile(InPath);
   SupplementaryOutputPaths SOPs;
   SOPs.ModuleOutputPath = OutPath.str();
-  StringRef MainOut = "/dev/null";
+
+  // Pick a primary output path that will cause problems to use.
+  StringRef MainOut = "/<unused>";
   SubFEOpts.InputsAndOutputs.setMainAndSupplementaryOutputs({MainOut}, {SOPs});
 }
 
@@ -258,6 +267,9 @@ collectDepsForSerialization(clang::vfs::FileSystem &FS,
     uint64_t Hash = xxHash64(DepBuf->getBuffer());
     Deps.push_back(FileDependency{Size, Hash, DepName});
 
+    if (ModuleCachePath.empty())
+      continue;
+
     // If Dep is itself a .swiftmodule in the cache dir, pull out its deps
     // and include them in our own, so we have a single-file view of
     // transitive deps: removes redundancies, and avoids opening and reading
@@ -290,11 +302,21 @@ collectDepsForSerialization(clang::vfs::FileSystem &FS,
 
 static bool buildSwiftModuleFromSwiftInterface(
     clang::vfs::FileSystem &FS, DiagnosticEngine &Diags, SourceLoc DiagLoc,
-    CompilerInvocation &SubInvocation, StringRef InPath, StringRef OutPath,
-    StringRef ModuleCachePath, DependencyTracker *OuterTracker) {
+    CompilerInvocation &SubInvocation, StringRef ModuleCachePath,
+    DependencyTracker *OuterTracker) {
   bool SubError = false;
   bool RunSuccess = llvm::CrashRecoveryContext().RunSafelyOnThread([&] {
-    (void)llvm::sys::fs::create_directory(ModuleCachePath);
+    // Note that we don't assume ModuleCachePath is the same as the Clang
+    // module cache path at this point.
+    if (!ModuleCachePath.empty())
+      (void)llvm::sys::fs::create_directory(ModuleCachePath);
+
+    FrontendOptions &FEOpts = SubInvocation.getFrontendOptions();
+    const auto &InputInfo = FEOpts.InputsAndOutputs.firstInput();
+    StringRef InPath = InputInfo.file();
+    const auto &OutputInfo =
+        InputInfo.getPrimarySpecificPaths().SupplementaryOutputs;
+    StringRef OutPath = OutputInfo.ModuleOutputPath;
 
     llvm::BumpPtrAllocator SubArgsAlloc;
     llvm::StringSaver SubArgSaver(SubArgsAlloc);
@@ -343,6 +365,7 @@ static bool buildSwiftModuleFromSwiftInterface(
     LLVM_DEBUG(llvm::dbgs() << "Setting up instance to compile "
                << InPath << " to " << OutPath << "\n");
     CompilerInstance SubInstance;
+    SubInstance.getSourceMgr().setFileSystem(&FS);
 
     ForwardingDiagnosticConsumer FDC(Diags);
     SubInstance.addDiagnosticConsumer(&FDC);
@@ -372,7 +395,6 @@ static bool buildSwiftModuleFromSwiftInterface(
 
     // Setup the callbacks for serialization, which can occur during the
     // optimization pipeline.
-    FrontendOptions &FEOpts = SubInvocation.getFrontendOptions();
     SerializationOptions SerializationOpts;
     std::string OutPathStr = OutPath;
     SerializationOpts.OutputPath = OutPathStr.c_str();
@@ -456,15 +478,16 @@ std::error_code ParseableInterfaceModuleLoader::openModuleFiles(
 
   // Set up a _potential_ sub-invocation to consume the .swiftinterface and emit
   // the .swiftmodule.
-  CompilerInvocation SubInvocation;
-  configureSubInvocationAndOutputPaths(SubInvocation, ModuleID.first, InPath,
-                                       OutPath);
+  CompilerInvocation SubInvocation =
+      createInvocationForBuildingFromInterface(Ctx, ModuleID.first.str(), CacheDir);
+  computeCachedOutputPath(Ctx, SubInvocation, InPath, OutPath);
+  configureSubInvocationInputsAndOutputs(SubInvocation, InPath, OutPath);
 
   // Evaluate if we need to run this sub-invocation, and if so run it.
   if (!swiftModuleIsUpToDate(FS, ModuleID, OutPath, Diags, dependencyTracker)) {
-    if (buildSwiftModuleFromSwiftInterface(FS, Diags, ModuleID.second,
-                                           SubInvocation, InPath, OutPath,
-                                           CacheDir, dependencyTracker))
+    if (::buildSwiftModuleFromSwiftInterface(FS, Diags, ModuleID.second,
+                                             SubInvocation, CacheDir,
+                                             dependencyTracker))
       return std::make_error_code(std::errc::invalid_argument);
   }
 
@@ -484,6 +507,21 @@ std::error_code ParseableInterfaceModuleLoader::openModuleFiles(
   return ErrorCode;
 }
 
+bool
+ParseableInterfaceModuleLoader::buildSwiftModuleFromSwiftInterface(
+    ASTContext &Ctx, StringRef CacheDir, StringRef ModuleName,
+    StringRef InPath, StringRef OutPath) {
+  CompilerInvocation SubInvocation =
+      createInvocationForBuildingFromInterface(Ctx, ModuleName, CacheDir);
+  configureSubInvocationInputsAndOutputs(SubInvocation, InPath, OutPath);
+
+  auto &FS = *Ctx.SourceMgr.getFileSystem();
+  auto &Diags = Ctx.Diags;
+  return ::buildSwiftModuleFromSwiftInterface(FS, Diags, /*DiagLoc*/SourceLoc(),
+                                              SubInvocation, /*CachePath*/"",
+                                              /*OuterTracker*/nullptr);
+}
+
 /// Diagnose any scoped imports in \p imports, i.e. those with a non-empty
 /// access path. These are not yet supported by parseable interfaces, since the
 /// information about the declaration kind is not preserved through the binary

lib/FrontendTool/FrontendTool.cpp

@@ -559,6 +559,17 @@ static bool precompileBridgingHeader(CompilerInvocation &Invocation,
           .InputsAndOutputs.getSingleOutputFilename());
 }
 
+static bool buildModuleFromParseableInterface(CompilerInvocation &Invocation,
+                                              CompilerInstance &Instance) {
+  const auto &InputsAndOutputs =
+      Invocation.getFrontendOptions().InputsAndOutputs;
+  assert(InputsAndOutputs.hasSingleInput());
+  StringRef InputPath = InputsAndOutputs.getFilenameOfFirstInput();
+  return ParseableInterfaceModuleLoader::buildSwiftModuleFromSwiftInterface(
+      Instance.getASTContext(), Invocation.getClangModuleCachePath(),
+      Invocation.getModuleName(), InputPath, Invocation.getOutputFilename());
+}
+
 static bool compileLLVMIR(CompilerInvocation &Invocation,
                           CompilerInstance &Instance,
                           UnifiedStatsReporter *Stats) {
@@ -923,6 +934,9 @@ static bool performCompile(CompilerInstance &Instance,
   if (Action == FrontendOptions::ActionType::EmitPCH)
     return precompileBridgingHeader(Invocation, Instance);
 
+  if (Action == FrontendOptions::ActionType::BuildModuleFromParseableInterface)
+    return buildModuleFromParseableInterface(Invocation, Instance);
+
   if (Invocation.getInputKind() == InputFileKind::LLVM)
     return compileLLVMIR(Invocation, Instance, Stats);
 

test/ParseableInterface/ParseStdlib.swiftinterface

@@ -0,0 +1,12 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -parse-stdlib
+
+// Note that the "-parse-stdlib" is picked up from the module flags. It should
+// not be written in any of the invocations below.
+
+// RUN: %empty-directory(%t)
+// RUN: %target-swift-frontend -build-module-from-parseable-interface -o %t/ParseStdlib.swiftmodule %s
+// RUN: %target-swift-ide-test -print-module -module-to-print ParseStdlib -I %t -source-filename x -print-interface | %FileCheck %s
+
+// CHECK: func test(_: Int42)
+public func test(_: Builtin.Int42) {}