Correct the Serialization of Embedded Swift Dependencies

llvm-bcanalyzer does *not* like it when there are multiple block info metadata blocks in the same bitstream file. This patch will skip the emission of that and just jump straight to the metadata block when we're not reading a "standalone" incremental dependency file. While I'm here, also add the right block abbreviation info so we can get a decent dump from llvm-bcanalyzer

Author: CodaFi

include/swift/AST/FineGrainedDependencyFormat.h

@@ -136,7 +136,8 @@ bool writeFineGrainedDependencyGraphToPath(DiagnosticEngine &diags,
                                            const SourceFileDepGraph &g);
 
 void writeFineGrainedDependencyGraph(llvm::BitstreamWriter &Out,
-                                     const SourceFileDepGraph &g);
+                                     const SourceFileDepGraph &g,
+                                     bool standalone);
 
 } // namespace fine_grained_dependencies
 } // namespace swift

lib/AST/FineGrainedDependencies.cpp

@@ -61,7 +61,7 @@ SourceFileDepGraph::loadFromBuffer(llvm::MemoryBuffer &buffer) {
 Optional<SourceFileDepGraph>
 SourceFileDepGraph::loadFromSwiftModuleBuffer(llvm::MemoryBuffer &buffer) {
   SourceFileDepGraph fg;
-  if (swift::fine_grained_dependencies::
+  if (!swift::fine_grained_dependencies::
           readFineGrainedDependencyGraphFromSwiftModule(buffer, fg))
     return None;
   return Optional<SourceFileDepGraph>(std::move(fg));

lib/AST/FineGrainedDependencyFormat.cpp

@@ -44,7 +44,7 @@ class Deserializer {
 
 public:
   Deserializer(llvm::MemoryBufferRef Data) : Cursor(Data) {}
-  bool readFineGrainedDependencyGraph(SourceFileDepGraph &g);
+  bool readFineGrainedDependencyGraph(SourceFileDepGraph &g, bool standalone);
   bool readFineGrainedDependencyGraphFromSwiftModule(SourceFileDepGraph &g);
 };
 
@@ -151,13 +151,14 @@ static llvm::Optional<DeclAspect> getDeclAspect(unsigned declAspect) {
   return None;
 }
 
-bool Deserializer::readFineGrainedDependencyGraph(SourceFileDepGraph &g) {
+bool Deserializer::readFineGrainedDependencyGraph(SourceFileDepGraph &g,
+                                                  bool standalone) {
   using namespace record_block;
 
-  if (readSignature())
+  if (standalone && readSignature())
     return true;
 
-  if (enterTopLevelBlock())
+  if (standalone && enterTopLevelBlock())
     return true;
 
   if (readMetadata())
@@ -260,7 +261,7 @@ bool Deserializer::readFineGrainedDependencyGraph(SourceFileDepGraph &g) {
 bool swift::fine_grained_dependencies::readFineGrainedDependencyGraph(
     llvm::MemoryBuffer &buffer, SourceFileDepGraph &g) {
   Deserializer deserializer(buffer.getMemBufferRef());
-  return deserializer.readFineGrainedDependencyGraph(g);
+  return deserializer.readFineGrainedDependencyGraph(g, /*standalone*/true);
 }
 
 bool swift::fine_grained_dependencies::readFineGrainedDependencyGraph(
@@ -322,7 +323,8 @@ class Serializer {
   Serializer(llvm::BitstreamWriter &ExistingOut) : Out(ExistingOut) {}
 
 public:
-  void writeFineGrainedDependencyGraph(const SourceFileDepGraph &g);
+  void writeFineGrainedDependencyGraph(const SourceFileDepGraph &g,
+                                       bool standalone);
 };
 
 } // end namespace
@@ -382,11 +384,15 @@ void Serializer::writeMetadata() {
 }
 
 void
-Serializer::writeFineGrainedDependencyGraph(const SourceFileDepGraph &g) {
-  writeSignature();
-  writeBlockInfoBlock();
-
-  llvm::BCBlockRAII restoreBlock(Out, RECORD_BLOCK_ID, 8);
+Serializer::writeFineGrainedDependencyGraph(const SourceFileDepGraph &g,
+                                            bool standalone) {
+  auto blockID = INCREMENTAL_INFORMATION_BLOCK_ID;
+  if (standalone) {
+    writeSignature();
+    writeBlockInfoBlock();
+    blockID = RECORD_BLOCK_ID;
+  }
+  llvm::BCBlockRAII restoreBlock(Out, blockID, 8);
 
   using namespace record_block;
 
@@ -468,9 +474,9 @@ unsigned Serializer::getIdentifier(StringRef str) {
 }
 
 void swift::fine_grained_dependencies::writeFineGrainedDependencyGraph(
-    llvm::BitstreamWriter &Out, const SourceFileDepGraph &g) {
+    llvm::BitstreamWriter &Out, const SourceFileDepGraph &g, bool standalone) {
   Serializer serializer{Out};
-  serializer.writeFineGrainedDependencyGraph(g);
+  serializer.writeFineGrainedDependencyGraph(g, standalone);
 }
 
 bool swift::fine_grained_dependencies::writeFineGrainedDependencyGraphToPath(
@@ -480,7 +486,7 @@ bool swift::fine_grained_dependencies::writeFineGrainedDependencyGraphToPath(
   return withOutputFile(diags, path, [&](llvm::raw_ostream &out) {
     SmallVector<char, 0> Buffer;
     llvm::BitstreamWriter Writer{Buffer};
-    writeFineGrainedDependencyGraph(Writer, g);
+    writeFineGrainedDependencyGraph(Writer, g, /*standalone*/ true);
     out.write(Buffer.data(), Buffer.size());
     out.flush();
     return false;
@@ -516,7 +522,7 @@ static bool enterTopLevelModuleBlock(llvm::BitstreamCursor &cursor, unsigned ID,
   if (next.Kind != llvm::BitstreamEntry::SubBlock)
     return false;
 
-  if (next.ID == RECORD_BLOCK_ID) {
+  if (next.ID == llvm::bitc::BLOCKINFO_BLOCK_ID) {
     if (shouldReadBlockInfo) {
       if (!cursor.ReadBlockInfoBlock())
         return false;
@@ -531,7 +537,6 @@ static bool enterTopLevelModuleBlock(llvm::BitstreamCursor &cursor, unsigned ID,
     return false;
 
   if (llvm::Error Err = cursor.EnterSubBlock(ID)) {
-    // FIXME this drops the error on the floor.
     consumeError(std::move(Err));
     return false;
   }
@@ -549,12 +554,12 @@ bool swift::fine_grained_dependencies::
 bool Deserializer::readFineGrainedDependencyGraphFromSwiftModule(
     SourceFileDepGraph &g) {
   if (!checkModuleSignature(Cursor, {0xE2, 0x9C, 0xA8, 0x0E}) ||
-      !enterTopLevelModuleBlock(Cursor, RECORD_BLOCK_ID, false)) {
+      !enterTopLevelModuleBlock(Cursor, llvm::bitc::FIRST_APPLICATION_BLOCKID, false)) {
     return false;
   }
 
   llvm::BitstreamEntry topLevelEntry;
-
+  bool ReadFineGrainedDependencies = false;
   while (!Cursor.AtEndOfStream()) {
     llvm::Expected<llvm::BitstreamEntry> maybeEntry =
         Cursor.advance(llvm::BitstreamCursor::AF_DontPopBlockAtEnd);
@@ -573,7 +578,11 @@ bool Deserializer::readFineGrainedDependencyGraphFromSwiftModule(
         consumeError(std::move(Err));
         return false;
       }
-      readFineGrainedDependencyGraph(g);
+      if (readFineGrainedDependencyGraph(g, /*standalone*/ false)) {
+        break;
+      }
+
+      ReadFineGrainedDependencies = true;
       break;
     }
 
@@ -587,5 +596,5 @@ bool Deserializer::readFineGrainedDependencyGraphFromSwiftModule(
     }
   }
 
-  return false;
+  return ReadFineGrainedDependencies;
 }

lib/Serialization/CMakeLists.txt

@@ -9,7 +9,6 @@ add_swift_host_library(swiftSerialization STATIC
   SerializedModuleLoader.cpp
   SerializedSILLoader.cpp
   SerializeDoc.cpp
-  SerializeIncremental.cpp
   SerializeSIL.cpp
 
   LLVM_LINK_COMPONENTS

lib/Serialization/Serialization.cpp

@@ -861,6 +861,13 @@ void Serializer::writeBlockInfoBlock() {
   BLOCK_RECORD(sil_index_block, SIL_DIFFERENTIABILITY_WITNESS_NAMES);
   BLOCK_RECORD(sil_index_block, SIL_DIFFERENTIABILITY_WITNESS_OFFSETS);
 
+  BLOCK(INCREMENTAL_INFORMATION_BLOCK);
+  BLOCK_RECORD(fine_grained_dependencies::record_block, METADATA);
+  BLOCK_RECORD(fine_grained_dependencies::record_block, SOURCE_FILE_DEP_GRAPH_NODE);
+  BLOCK_RECORD(fine_grained_dependencies::record_block, FINGERPRINT_NODE);
+  BLOCK_RECORD(fine_grained_dependencies::record_block, DEPENDS_ON_DEFINITION_NODE);
+  BLOCK_RECORD(fine_grained_dependencies::record_block, IDENTIFIER_NODE);
+
 #undef BLOCK
 #undef BLOCK_RECORD
 }
@@ -5231,8 +5238,9 @@ void Serializer::writeToStream(
     S.writeInputBlock(options);
     S.writeSIL(SILMod, options.SerializeAllSIL);
     S.writeAST(DC);
-    if (options.ExperimentalCrossModuleIncrementalInfo) {
-      S.writeIncrementalInfo(DepGraph);
+    if (options.ExperimentalCrossModuleIncrementalInfo && DepGraph) {
+      fine_grained_dependencies::writeFineGrainedDependencyGraph(
+          S.Out, *DepGraph, /*standalone*/ false);
     }
   }
 

lib/Serialization/SerializeIncremental.cpp

@@ -0,0 +1,9 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-swift-frontend -emit-module -module-name EmbeddedIncremental -o %t/EmbeddedIncremental~partial.swiftmodule -primary-file %s
+// RUN: %target-swift-frontend -enable-experimental-cross-module-incremental-build -emit-module -module-name EmbeddedIncremental -o %t/EmbeddedIncremental.swiftmodule %t/EmbeddedIncremental~partial.swiftmodule
+// RUN: llvm-bcanalyzer -dump %t/EmbeddedIncremental.swiftmodule | %FileCheck %s --dump-input=always
+
+public struct AnyWindows {}
+
+// CHECK: <INCREMENTAL_INFORMATION_BLOCK
+// CHECK: </INCREMENTAL_INFORMATION_BLOCK>