Merge pull request #66344 from nate-chandler/cherrypick/release/5.9/dealloc-on-stack-packs

5.9: [IRGen] Dealloc on-stack metadata/wtable packs on the dominance frontier.

Author: nate-chandler

docs/SIL.rst

@@ -3797,7 +3797,21 @@ address of the allocated memory.
 
 ``alloc_pack`` is a stack allocation instruction.  See the section above
 on stack discipline.  The corresponding stack deallocation instruction is
-``dealloc_stack``.
+``dealloc_pack``.
+
+alloc_pack_metadata
+```````````````````
+
+::
+
+  sil-instruction ::= 'alloc_pack_metadata' $()
+
+Inserted as the last SIL lowering pass of IRGen, indicates that the next instruction
+may have on-stack pack metadata allocated on its behalf.
+
+Notionally, ``alloc_pack_metadata`` is a stack allocation instruction.  See the
+section above on stack discipline.  The corresponding stack deallocation
+instruction is ``dealloc_pack_metadata``.
 
 alloc_ref
 `````````
@@ -4056,6 +4070,23 @@ prior to being deallocated.
 on Stack Discipline above.  The operand must be an ``alloc_pack``
 instruction.
 
+dealloc_pack_metadata
+`````````````````````
+
+::
+
+  sil-instruction ::= 'dealloc_pack_metadata' sil-operand
+
+  dealloc_pack_metadata $0 : $*()
+
+Inserted as the last SIL lowering pass of IRGen, indicates that the on-stack
+pack metadata emitted on behalf of its operand (actually on behalf of the
+instruction after its operand) must be cleaned up here.
+
+``dealloc_pack_metadata`` is a stack deallocation instruction.  See the section
+on Stack Discipline above.  The operand must be an ``alloc_pack_metadata``
+instruction.
+
 dealloc_box
 ```````````
 ::

include/swift/AST/SILOptions.h

@@ -176,6 +176,12 @@ class SILOptions {
   /// Require linear OSSA lifetimes after SILGen
   bool OSSACompleteLifetimes = false;
 
+  /// Enable pack metadata stack "promotion".
+  ///
+  /// More accurately, enable skipping mandatory heapification of pack metadata
+  /// when possible.
+  bool EnablePackMetadataStackPromotion = true;
+
   // The kind of function bodies to skip emitting.
   FunctionBodySkipping SkipFunctionBodies = FunctionBodySkipping::None;
 

include/swift/AST/Types.h

@@ -173,7 +173,10 @@ class RecursiveTypeProperties {
     /// This type contains an ElementArchetype.
     HasElementArchetype = 0x4000,
 
-    Last_Property = HasElementArchetype
+    /// This type contains a concrete pack.
+    HasConcretePack = 0x8000,
+
+    Last_Property = HasConcretePack
   };
   enum { BitWidth = countBitsUsed(Property::Last_Property) };
 
@@ -243,6 +246,8 @@ class RecursiveTypeProperties {
 
   bool hasParameterPack() const { return Bits & HasParameterPack; }
 
+  bool hasConcretePack() const { return Bits & HasConcretePack; }
+
   /// Does a type with these properties structurally contain a
   /// parameterized existential type?
   bool hasParameterizedExistential() const {
@@ -670,6 +675,13 @@ class alignas(1 << TypeAlignInBits) TypeBase
     return getRecursiveProperties().hasParameterPack();
   }
 
+  bool hasConcretePack() const {
+    return getRecursiveProperties().hasConcretePack();
+  }
+
+  /// Whether the type has some flavor of pack.
+  bool hasPack() const { return hasParameterPack() || hasConcretePack(); }
+
   /// Determine whether the type involves a parameterized existential type.
   bool hasParameterizedExistential() const {
     return getRecursiveProperties().hasParameterizedExistential();

include/swift/IRGen/IRGenSILPasses.h

@@ -19,6 +19,7 @@ namespace irgen {
 /// Create a pass to hoist alloc_stack instructions with non-fixed size.
 SILTransform *createAllocStackHoisting();
 SILTransform *createLoadableByAddress();
+SILTransform *createPackMetadataMarkerInserter();
 
 } // end namespace irgen
 } // end namespace swift

include/swift/Option/FrontendOptions.td

@@ -1021,6 +1021,16 @@ def force_struct_type_layouts : Flag<["-"], "force-struct-type-layouts">,
 def enable_layout_string_value_witnesses : Flag<["-"], "enable-layout-string-value-witnesses">,
   HelpText<"Enable layout string based value witnesses">;
 
+let Flags = [FrontendOption, NoDriverOption, HelpHidden, ModuleInterfaceOptionIgnorable] in {
+  def enable_pack_metadata_stack_promotion :
+    Joined<["-"], "enable-pack-metadata-stack-promotion=">,
+    HelpText<"Whether to skip heapifying stack metadata packs when possible.">,
+    MetaVarName<"true|false">;
+  def enable_pack_metadata_stack_promotion_noArg :
+    Flag<["-"], "enable-pack-metadata-stack-promotion">,
+    HelpText<"Skip heapifying stack metadata packs when possible.">;
+}
+
 def disable_layout_string_value_witnesses : Flag<["-"], "disable-layout-string-value-witnesses">,
   HelpText<"Disable layout string based value witnesses">;
 

include/swift/SIL/Dominance.h

@@ -94,7 +94,7 @@ class DominanceInfo : public DominatorTreeBase {
 
 /// Compute a single block's dominance frontier.
 ///
-/// Precondition: no critical edges (OSSA)
+/// Precondition: no critical edges
 ///
 /// Postcondition: each block in \p boundary is dominated by \p root and either
 /// exits the function or has a single successor which has a predecessor that is

include/swift/SIL/SILBuilder.h

@@ -443,6 +443,15 @@ class SILBuilder {
     return insert(AllocPackInst::create(getSILDebugLocation(loc), packType,
                                         getFunction()));
   }
+  AllocPackMetadataInst *
+  createAllocPackMetadata(SILLocation loc,
+                          Optional<SILType> elementType = llvm::None) {
+    return insert(new (getModule()) AllocPackMetadataInst(
+        getSILDebugLocation(loc),
+        elementType.value_or(
+            SILType::getEmptyTupleType(getModule().getASTContext())
+                .getAddressType())));
+  }
 
   AllocRefInst *createAllocRef(SILLocation Loc, SILType ObjectType,
                                bool objc, bool canAllocOnStack,
@@ -2215,6 +2224,11 @@ class SILBuilder {
     return insert(new (getModule())
                       DeallocPackInst(getSILDebugLocation(loc), operand));
   }
+  DeallocPackMetadataInst *createDeallocPackMetadata(SILLocation loc,
+                                                     SILValue alloc) {
+    return insert(new (getModule())
+                      DeallocPackMetadataInst(getSILDebugLocation(loc), alloc));
+  }
   DeallocStackRefInst *createDeallocStackRef(SILLocation Loc,
                                                      SILValue operand) {
     return insert(new (getModule())

include/swift/SIL/SILCloner.h

@@ -891,6 +891,14 @@ SILCloner<ImplClass>::visitAllocStackInst(AllocStackInst *Inst) {
   recordClonedInstruction(Inst, NewInst);
 }
 
+template <typename ImplClass>
+void SILCloner<ImplClass>::visitAllocPackMetadataInst(
+    AllocPackMetadataInst *Inst) {
+  getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  recordClonedInstruction(Inst, getBuilder().createAllocPackMetadata(
+                                    getOpLocation(Inst->getLoc())));
+}
+
 template<typename ImplClass>
 void
 SILCloner<ImplClass>::visitAllocPackInst(AllocPackInst *Inst) {
@@ -2844,6 +2852,15 @@ SILCloner<ImplClass>::visitDeallocPackInst(DeallocPackInst *Inst) {
                                            getOpValue(Inst->getOperand())));
 }
 
+template <typename ImplClass>
+void SILCloner<ImplClass>::visitDeallocPackMetadataInst(
+    DeallocPackMetadataInst *Inst) {
+  getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  recordClonedInstruction(
+      Inst, getBuilder().createDeallocPackMetadata(
+                getOpLocation(Inst->getLoc()), Inst->getOperand()));
+}
+
 template<typename ImplClass>
 void
 SILCloner<ImplClass>::visitDeallocRefInst(DeallocRefInst *Inst) {

include/swift/SIL/SILFunction.h

@@ -75,6 +75,11 @@ enum ForceEnableLexicalLifetimes_t {
   DoForceEnableLexicalLifetimes
 };
 
+enum UseStackForPackMetadata_t {
+  DoNotUseStackForPackMetadata,
+  DoUseStackForPackMetadata,
+};
+
 enum class PerformanceConstraints : uint8_t {
   None = 0,
   NoAllocation = 1,
@@ -411,6 +416,10 @@ class SILFunction
   /// If true, the function has lexical lifetimes even if the module does not.
   unsigned ForceEnableLexicalLifetimes : 1;
 
+  /// If true, the function contains an instruction that prevents stack nesting
+  /// from running with pack metadata markers in place.
+  unsigned UseStackForPackMetadata : 1;
+
   static void
   validateSubclassScope(SubclassScope scope, IsThunk_t isThunk,
                         const GenericSpecializationInformation *genericInfo) {
@@ -686,6 +695,14 @@ class SILFunction
     ForceEnableLexicalLifetimes = value;
   }
 
+  UseStackForPackMetadata_t useStackForPackMetadata() const {
+    return UseStackForPackMetadata_t(UseStackForPackMetadata);
+  }
+
+  void setUseStackForPackMetadata(UseStackForPackMetadata_t value) {
+    UseStackForPackMetadata = value;
+  }
+
   /// Returns true if this is a reabstraction thunk of escaping function type
   /// whose single argument is a potentially non-escaping closure. i.e. the
   /// thunks' function argument may itself have @inout_aliasable parameters.

include/swift/SIL/SILInstruction.h

@@ -821,6 +821,10 @@ class SILInstruction : public llvm::ilist_node<SILInstruction> {
   /// The first operand must be the allocating instruction.
   bool isDeallocatingStack() const;
 
+  /// Whether IRGen lowering of this instruction may result in emitting packs of
+  /// metadata or witness tables.
+  bool mayRequirePackMetadata() const;
+
   /// Create a new copy of this instruction, which retains all of the operands
   /// and other information of this one.  If an insertion point is specified,
   /// then the new instruction is inserted before the specified point, otherwise
@@ -2275,6 +2279,27 @@ class AllocPackInst final
   }
 };
 
+/// AllocPackMetadataInst - Marker instruction indicating that the next
+///                         instruction might allocate on-stack pack metadata
+///                         during IRGen.
+///
+/// Only valid in lowered SIL.
+class AllocPackMetadataInst final
+    : public NullaryInstructionWithTypeDependentOperandsBase<
+          SILInstructionKind::AllocPackMetadataInst, AllocPackMetadataInst,
+          AllocationInst> {
+  friend SILBuilder;
+
+  AllocPackMetadataInst(SILDebugLocation loc, SILType elementType)
+      : NullaryInstructionWithTypeDependentOperandsBase(
+            loc, {}, elementType.getAddressType()) {}
+
+public:
+  /// The instruction which may trigger on-stack pack metadata when IRGen
+  /// lowering.
+  SILInstruction *getIntroducer() { return getNextInstruction(); }
+};
+
 /// The base class for AllocRefInst and AllocRefDynamicInst.
 ///
 /// The first NumTailTypes operands are counts for the tail allocated
@@ -8642,6 +8667,27 @@ class DeallocPackInst :
       : UnaryInstructionBase(debugLoc, operand) {}
 };
 
+/// DeallocPackMetadataInst - Deallocate stack memory allocated on behalf of the
+///                           operand by IRGen.
+///
+/// Only valid in lowered SIL.
+class DeallocPackMetadataInst final
+    : public UnaryInstructionBase<SILInstructionKind::DeallocPackMetadataInst,
+                                  DeallocationInst> {
+  friend SILBuilder;
+
+  DeallocPackMetadataInst(SILDebugLocation debugLoc, SILValue alloc)
+      : UnaryInstructionBase(debugLoc, alloc) {}
+
+public:
+  AllocPackMetadataInst *getAllocation() {
+    return cast<AllocPackMetadataInst>(getOperand().getDefiningInstruction());
+  }
+  /// The instruction which may trigger on-stack pack metadata when IRGen
+  /// lowering.
+  SILInstruction *getIntroducer() { return getAllocation()->getIntroducer(); }
+};
+
 /// Like DeallocStackInst, but for `alloc_ref [stack]`.
 class DeallocStackRefInst
     : public UnaryInstructionBase<SILInstructionKind::DeallocStackRefInst,