diff --git a/clang/cmake/caches/Fuchsia.cmake b/clang/cmake/caches/Fuchsia.cmake
index f790acacb8e95..87b8a9f109d3e 100644
--- a/clang/cmake/caches/Fuchsia.cmake
+++ b/clang/cmake/caches/Fuchsia.cmake
@@ -108,6 +108,8 @@ if(APPLE)
   set(COMPILER_RT_ENABLE_IOS OFF CACHE BOOL "")
   set(COMPILER_RT_ENABLE_TVOS OFF CACHE BOOL "")
   set(COMPILER_RT_ENABLE_WATCHOS OFF CACHE BOOL "")
+
+  set(RUNTIMES_CMAKE_ARGS "-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0;-DCMAKE_OSX_ARCHITECTURES=arm64|x86_64;-DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" CACHE STRING "")
 endif()
 
 if(WIN32)
@@ -142,7 +144,6 @@ else()
   set(SANITIZER_TEST_CXX "libc++" CACHE STRING "")
   set(SANITIZER_TEST_CXX_INTREE ON CACHE BOOL "")
   set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
-  set(RUNTIMES_CMAKE_ARGS "-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0;-DCMAKE_OSX_ARCHITECTURES=arm64|x86_64" CACHE STRING "")
 endif()
 
 if(BOOTSTRAP_CMAKE_SYSTEM_NAME)
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 75067f6ab3265..668a03bd08649 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -2202,9 +2202,9 @@ void TypePrinter::printHLSLAttributedResourceAfter(
     const HLSLAttributedResourceType *T, raw_ostream &OS) {
   printAfter(T->getWrappedType(), OS);
   const HLSLAttributedResourceType::Attributes &Attrs = T->getAttrs();
-  OS << " [[hlsl::resource_class("
+  OS << " [[hlsl::resource_class(\""
      << HLSLResourceClassAttr::ConvertResourceClassToStr(Attrs.ResourceClass)
-     << ")]]";
+     << "\")]]";
   if (Attrs.IsROV)
     OS << " [[hlsl::is_rov]]";
   if (Attrs.RawBuffer)
@@ -2225,10 +2225,10 @@ void TypePrinter::printHLSLAttributedResourceAfter(
   }
 
   if (Attrs.ResourceDimension != llvm::dxil::ResourceDimension::Unknown)
-    OS << " [[hlsl::resource_dimension("
+    OS << " [[hlsl::dimension(\""
        << HLSLResourceDimensionAttr::ConvertResourceDimensionToStr(
               Attrs.ResourceDimension)
-       << ")]]";
+       << "\")]]";
 }
 
 void TypePrinter::printHLSLInlineSpirvBefore(const HLSLInlineSpirvType *T,
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp
index 0be493fe9084a..cbea215de83a2 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp
@@ -303,6 +303,17 @@ deriveNeonSISDIntrinsicOperandTypes(CIRGenFunction &cgf, unsigned modifier,
     vecArgTy = cir::VectorType::get(arg0Ty, resVecTy.getSize());
   }
 
+  // True if `ty` is arg0's type, or an integer of the same width that only
+  // differs in signedness. vsqadd/vuqadd mix the two on purpose: vsqaddb_u8
+  // takes a uint8_t and an int8_t, but both become the same vector type.
+  auto matchesArg0Ty = [&](mlir::Type ty) {
+    if (ty == arg0Ty)
+      return true;
+    auto intTy = mlir::dyn_cast<cir::IntType>(ty);
+    auto arg0IntTy = mlir::dyn_cast<cir::IntType>(arg0Ty);
+    return intTy && arg0IntTy && intTy.getWidth() == arg0IntTy.getWidth();
+  };
+
   // `vecArgTy` is populated by `VectorizeArgTypes` or
   // `ArgAsWidenedRetType`. When set, wrap every non-immediate data operand
   // that has the same scalar type as arg0. Checking the ICE bitmap prevents
@@ -312,7 +323,7 @@ deriveNeonSISDIntrinsicOperandTypes(CIRGenFunction &cgf, unsigned modifier,
   argTypes.reserve(ops.size());
   for (unsigned i = 0, e = ops.size(); i != e; ++i) {
     bool isImmediate = iceArguments & (1U << i);
-    if (vecArgTy && !isImmediate && ops[i].getType() == arg0Ty)
+    if (vecArgTy && !isImmediate && matchesArg0Ty(ops[i].getType()))
       argTypes.push_back(vecArgTy);
     else
       argTypes.push_back(ops[i].getType());
@@ -509,6 +520,22 @@ emitCommonNeonSISDBuiltinExpr(CIRGenFunction &cgf,
   case NEON::BI__builtin_neon_vqsubs_u32:
   case NEON::BI__builtin_neon_vqsubd_s64:
   case NEON::BI__builtin_neon_vqsubd_u64:
+  case NEON::BI__builtin_neon_vqaddb_s8:
+  case NEON::BI__builtin_neon_vqaddb_u8:
+  case NEON::BI__builtin_neon_vqaddh_s16:
+  case NEON::BI__builtin_neon_vqaddh_u16:
+  case NEON::BI__builtin_neon_vqadds_s32:
+  case NEON::BI__builtin_neon_vqadds_u32:
+  case NEON::BI__builtin_neon_vqaddd_s64:
+  case NEON::BI__builtin_neon_vqaddd_u64:
+  case NEON::BI__builtin_neon_vsqaddb_u8:
+  case NEON::BI__builtin_neon_vsqaddh_u16:
+  case NEON::BI__builtin_neon_vsqadds_u32:
+  case NEON::BI__builtin_neon_vsqaddd_u64:
+  case NEON::BI__builtin_neon_vuqaddb_s8:
+  case NEON::BI__builtin_neon_vuqaddh_s16:
+  case NEON::BI__builtin_neon_vuqadds_s32:
+  case NEON::BI__builtin_neon_vuqaddd_s64:
     break;
   }
 
@@ -1237,6 +1264,8 @@ static mlir::Value emitCommonNeonBuiltinExpr(
   case NEON::BI__builtin_neon_vhaddq_v:
   case NEON::BI__builtin_neon_vhsub_v:
   case NEON::BI__builtin_neon_vhsubq_v:
+  case NEON::BI__builtin_neon_vqadd_v:
+  case NEON::BI__builtin_neon_vqaddq_v:
   case NEON::BI__builtin_neon_vrhadd_v:
   case NEON::BI__builtin_neon_vrhaddq_v:
   case NEON::BI__builtin_neon_vshl_v:
@@ -3515,10 +3544,18 @@ CIRGenFunction::emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr,
   case NEON::BI__builtin_neon_vqtbx2q_v:
   case NEON::BI__builtin_neon_vqtbx3q_v:
   case NEON::BI__builtin_neon_vqtbx4q_v:
+    cgm.errorNYI(expr->getSourceRange(),
+                 std::string("unimplemented AArch64 builtin call: ") +
+                     getContext().BuiltinInfo.getName(builtinID));
+    return mlir::Value{};
   case NEON::BI__builtin_neon_vsqadd_v:
   case NEON::BI__builtin_neon_vsqaddq_v:
+    return emitNeonCall(cgm, builder, {ty, ty}, ops, "aarch64.neon.usqadd", ty,
+                        loc);
   case NEON::BI__builtin_neon_vuqadd_v:
   case NEON::BI__builtin_neon_vuqaddq_v:
+    return emitNeonCall(cgm, builder, {ty, ty}, ops, "aarch64.neon.suqadd", ty,
+                        loc);
   case NEON::BI__builtin_neon_vluti2_laneq_mf8:
   case NEON::BI__builtin_neon_vluti2_laneq_bf16:
   case NEON::BI__builtin_neon_vluti2_laneq_f16:
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 6c9bccf50b360..d318338187f12 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -48,6 +48,10 @@ class LoopOp;
 } // namespace acc
 } // namespace mlir
 
+namespace clang {
+class SYCLKernelCallStmt;
+} // namespace clang
+
 namespace clang::CIRGen {
 
 struct CGCoroData;
@@ -2295,6 +2299,8 @@ class CIRGenFunction : public CIRGenTypeCache {
                                      bool buildingTopLevelCase);
   mlir::LogicalResult emitSwitchStmt(const clang::SwitchStmt &s);
 
+  mlir::LogicalResult emitSYCLKernelCallStmt(const SYCLKernelCallStmt &s);
+
   std::optional<mlir::Value>
   emitTargetBuiltinExpr(unsigned builtinID, const clang::CallExpr *e,
                         ReturnValueSlot &returnValue);
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
index a7143408c9bee..fa1bdd267fc8f 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
@@ -432,6 +432,23 @@ void CIRGenModule::emitDeferred() {
   curDeclsToEmit.swap(deferredDeclsToEmit);
 
   for (const GlobalDecl &d : curDeclsToEmit) {
+    // Functions declared with the sycl_kernel_entry_point attribute are
+    // emitted normally during host compilation. During device compilation, a
+    // SYCL kernel caller offload entry point function is generated and emitted
+    // in place of each of these functions.
+    if (const auto *fd = d.getDecl()->getAsFunction()) {
+      if (langOpts.SYCLIsDevice && fd->hasAttr<SYCLKernelEntryPointAttr>() &&
+          fd->isDefined()) {
+        // Functions with an invalid sycl_kernel_entry_point attribute are
+        // ignored during device compilation.
+        if (!fd->getAttr<SYCLKernelEntryPointAttr>()->isInvalidAttr())
+          errorNYI(fd->getSourceRange(),
+                   "SYCL kernel caller offload entry point");
+        // Do not emit the sycl_kernel_entry_point attributed function.
+        continue;
+      }
+    }
+
     emitGlobalDecl(d);
 
     // If we found out that we need to emit more decls, do that recursively.
diff --git a/clang/lib/CIR/CodeGen/CIRGenSYCL.cpp b/clang/lib/CIR/CodeGen/CIRGenSYCL.cpp
new file mode 100644
index 0000000000000..9308b1fe4189f
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenSYCL.cpp
@@ -0,0 +1,35 @@
+//===--------- CIRGenSYCL.cpp - Emit CIR for SYCL kernels -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code required for the generation of SYCL kernel code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenFunction.h"
+
+#include "clang/AST/StmtSYCL.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+mlir::LogicalResult
+CIRGenFunction::emitSYCLKernelCallStmt(const SYCLKernelCallStmt &s) {
+  // SYCLKernelCallStmt nodes are only present in the bodies of functions
+  // declared with the sycl_kernel_entry_point attribute. ODR-use of such a
+  // function in code emitted during device compilation should be diagnosed.
+  // During device compilation, the offload kernel entry point is emitted in
+  // place of such a function (see CIRGenModule::emitDeferred), so this
+  // function is only reached during host compilation.
+  assert(!getLangOpts().SYCLIsDevice &&
+         "Attempt to emit a SYCL kernel call statement during device "
+         "compilation");
+
+  // During host compilation, the kernel launch statement is emitted in place
+  // of the original function body.
+  return emitStmt(s.getKernelLaunchStmt(), /*useCurrentScope=*/true);
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
index 628daacb88950..ceda5811cd065 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
@@ -20,6 +20,7 @@
 #include "clang/AST/Stmt.h"
 #include "clang/AST/StmtOpenACC.h"
 #include "clang/AST/StmtOpenMP.h"
+#include "clang/AST/StmtSYCL.h"
 #include "clang/CIR/MissingFeatures.h"
 #include "llvm/Support/SaveAndRestore.h"
 
@@ -210,6 +211,8 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
     return emitIndirectGotoStmt(cast<IndirectGotoStmt>(*s));
   case Stmt::CoreturnStmtClass:
     return emitCoreturnStmt(cast<CoreturnStmt>(*s));
+  case Stmt::SYCLKernelCallStmtClass:
+    return emitSYCLKernelCallStmt(cast<SYCLKernelCallStmt>(*s));
   case Stmt::OpenACCComputeConstructClass:
     return emitOpenACCComputeConstruct(cast<OpenACCComputeConstruct>(*s));
   case Stmt::OpenACCLoopConstructClass:
@@ -431,7 +434,6 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
   case Stmt::DefaultStmtClass:
   case Stmt::CaseStmtClass:
   case Stmt::SEHLeaveStmtClass:
-  case Stmt::SYCLKernelCallStmtClass:
   case Stmt::ObjCAtTryStmtClass:
   case Stmt::ObjCAtThrowStmtClass:
   case Stmt::ObjCAtSynchronizedStmtClass:
diff --git a/clang/lib/CIR/CodeGen/CMakeLists.txt b/clang/lib/CIR/CodeGen/CMakeLists.txt
index 78569b11651e0..1fe9b25dd786c 100644
--- a/clang/lib/CIR/CodeGen/CMakeLists.txt
+++ b/clang/lib/CIR/CodeGen/CMakeLists.txt
@@ -52,6 +52,7 @@ add_clang_library(clangCIR
   CIRGenStmtOpenACC.cpp
   CIRGenStmtOpenACCLoop.cpp
   CIRGenStmtOpenMP.cpp
+  CIRGenSYCL.cpp
   CIRGenTypes.cpp
   CIRGenVTables.cpp
   TargetInfo.cpp
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index c353c3fec3f62..3fda2dbc0ffc6 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -2238,15 +2238,10 @@ bool SemaHLSL::handleResourceTypeAttr(QualType T, const ParsedAttr &AL) {
 
   switch (AL.getKind()) {
   case ParsedAttr::AT_HLSLResourceClass: {
-    if (!AL.isArgIdent(0)) {
-      Diag(AL.getLoc(), diag::err_attribute_argument_type)
-          << AL << AANT_ArgumentIdentifier;
+    StringRef Identifier;
+    SourceLocation ArgLoc;
+    if (!SemaRef.checkStringLiteralArgumentAttr(AL, 0, Identifier, &ArgLoc))
       return false;
-    }
-
-    IdentifierLoc *Loc = AL.getArgAsIdent(0);
-    StringRef Identifier = Loc->getIdentifierInfo()->getName();
-    SourceLocation ArgLoc = Loc->getLoc();
 
     // Validate resource class value
     ResourceClass RC;
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 05ace69a4d999..0ad7cc47f858c 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -3910,7 +3910,7 @@ void ASTDeclReader::attachPreviousDecl(ASTReader &Reader, Decl *D,
 
   if (PreviousNonLocal) {
     if (Sema *S = Reader.getSema()) {
-      if (auto *ND = dyn_cast<NamedDecl>(D))
+      if (auto *ND = dyn_cast<NamedDecl>(D); ND && !isa<NamespaceDecl>(ND))
         S->mergeDeclAttributes(ND, PreviousNonLocal);
     }
   }
diff --git a/clang/test/AST/HLSL/ByteAddressBuffers-AST.hlsl b/clang/test/AST/HLSL/ByteAddressBuffers-AST.hlsl
index 61585d7770d7c..55db9a936ae15 100644
--- a/clang/test/AST/HLSL/ByteAddressBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/ByteAddressBuffers-AST.hlsl
@@ -37,8 +37,8 @@ RESOURCE Buffer;
 // CHECK: CXXRecordDecl {{.*}} implicit referenced class [[RESOURCE]] definition
 // CHECK: FinalAttr {{.*}}  Implicit final
 // CHECK-NEXT: FieldDecl {{.*}} implicit __handle '__hlsl_resource_t
-// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
-// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
+// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(char8_t)]]
 
diff --git a/clang/test/AST/HLSL/ConstantBuffers-AST.hlsl b/clang/test/AST/HLSL/ConstantBuffers-AST.hlsl
index 378be2e9ffb2a..82fc8dc5aeb28 100644
--- a/clang/test/AST/HLSL/ConstantBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/ConstantBuffers-AST.hlsl
@@ -5,22 +5,22 @@
 // CHECK: CXXRecordDecl {{.*}} ConstantBuffer definition
 // CHECK: FinalAttr {{.*}} Implicit final
 // CHECK-NEXT: FieldDecl {{.*}} implicit __handle '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(CBuffer)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("CBuffer")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 
 // CHECK: CXXConstructorDecl {{.*}} ConstantBuffer<element_type> 'void ()' inline
 // CHECK-NEXT: CompoundStmt
 // CHECK-NEXT: BinaryOperator {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(CBuffer)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("CBuffer")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SAME: ' '='
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(CBuffer)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("CBuffer")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::ConstantBuffer<element_type>' lvalue implicit this
 // CHECK-NEXT: CStyleCastExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(CBuffer)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("CBuffer")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SAME: ' <Dependent>
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
@@ -29,16 +29,16 @@
 // CHECK-NEXT: ParmVarDecl {{.*}} other 'const hlsl::ConstantBuffer<element_type> &'
 // CHECK-NEXT: CompoundStmt
 // CHECK-NEXT: BinaryOperator {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(CBuffer)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("CBuffer")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SAME: ' '='
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(CBuffer)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("CBuffer")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::ConstantBuffer<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(CBuffer)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("CBuffer")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'const hlsl::ConstantBuffer<element_type>' lvalue ParmVar {{.*}} 'other' 'const hlsl::ConstantBuffer<element_type> &'
@@ -47,16 +47,16 @@
 // CHECK-NEXT: ParmVarDecl {{.*}} other 'const hlsl::ConstantBuffer<element_type> &'
 // CHECK-NEXT: CompoundStmt
 // CHECK-NEXT: BinaryOperator {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(CBuffer)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("CBuffer")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SAME: ' '='
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(CBuffer)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("CBuffer")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::ConstantBuffer<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(CBuffer)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("CBuffer")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'const hlsl::ConstantBuffer<element_type>' lvalue ParmVar {{.*}} 'other' 'const hlsl::ConstantBuffer<element_type> &'
@@ -159,6 +159,6 @@ float main() {
   // CHECK-NEXT: ImplicitCastExpr {{.*}} 'const hlsl::ConstantBuffer<Nested>' lvalue <NoOp>
   // CHECK-NEXT: DeclRefExpr {{.*}} 'ConstantBuffer<Nested>':'hlsl::ConstantBuffer<Nested>' lvalue Var {{.*}} 'cb_nested' 'ConstantBuffer<Nested>':'hlsl::ConstantBuffer<Nested>'
   float a = cb_nested.getA();
-   
+
   return f1 + f2 + f3 + a;
 }
diff --git a/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl b/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
index 24dc8b87f9316..6b38b73d82321 100644
--- a/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
@@ -84,8 +84,8 @@ RESOURCE<float> Buffer;
 
 // CHECK: FinalAttr {{.*}} Implicit final
 // CHECK-NEXT: FieldDecl {{.*}} implicit __handle '__hlsl_resource_t
-// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
-// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
+// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 
@@ -338,8 +338,8 @@ RESOURCE<float> Buffer;
 // CHECK-LOAD-NEXT: CallExpr
 // CHECK-LOAD-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_load_with_status' 'void (...) noexcept'
 // CHECK-LOAD-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-LOAD-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-LOAD-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-LOAD-UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
+// CHECK-LOAD-SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // CHECK-LOAD-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-LOAD-NEXT: CXXThisExpr {{.*}} 'const hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-LOAD-NEXT: DeclRefExpr {{.*}} 'unsigned int' lvalue ParmVar {{.*}} 'Index' 'unsigned int'
@@ -355,7 +355,7 @@ RESOURCE<float> Buffer;
 // CHECK-COUNTER-NEXT: CallExpr
 // CHECK-COUNTER-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (__hlsl_resource_t, int) noexcept'
 // CHECK-COUNTER-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__counter_handle
 // CHECK-COUNTER-NEXT: CXXThisExpr {{.*}} 'hlsl::RWStructuredBuffer<element_type>' lvalue implicit this
@@ -371,7 +371,7 @@ RESOURCE<float> Buffer;
 // CHECK-COUNTER-NEXT: CallExpr
 // CHECK-COUNTER-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (__hlsl_resource_t, int) noexcept'
 // CHECK-COUNTER-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-COUNTER-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__counter_handle
 // CHECK-COUNTER-NEXT: CXXThisExpr {{.*}} 'hlsl::RWStructuredBuffer<element_type>' lvalue implicit this
@@ -389,7 +389,7 @@ RESOURCE<float> Buffer;
 // CHECK-APPEND-NEXT: CallExpr
 // CHECK-APPEND-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
 // CHECK-APPEND-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-APPEND-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-APPEND-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-APPEND-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
 // CHECK-APPEND-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
@@ -397,7 +397,7 @@ RESOURCE<float> Buffer;
 // CHECK-APPEND-NEXT: CallExpr
 // CHECK-APPEND-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (__hlsl_resource_t, int) noexcept'
 // CHECK-APPEND-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-APPEND-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-APPEND-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-APPEND-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-APPEND-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__counter_handle
 // CHECK-APPEND-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
@@ -414,7 +414,7 @@ RESOURCE<float> Buffer;
 // CHECK-CONSUME-NEXT: CallExpr
 // CHECK-CONSUME-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
 // CHECK-CONSUME-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-CONSUME-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-CONSUME-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
 // CHECK-CONSUME-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
@@ -422,7 +422,7 @@ RESOURCE<float> Buffer;
 // CHECK-CONSUME-NEXT: CallExpr
 // CHECK-CONSUME-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (__hlsl_resource_t, int) noexcept'
 // CHECK-CONSUME-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-CONSUME-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-CONSUME-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-CONSUME-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__counter_handle
 // CHECK-CONSUME-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
@@ -453,13 +453,13 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: BuiltinType {{.*}} 'float'
 // CHECK-NEXT: FinalAttr {{.*}} Implicit final
 // CHECK-NEXT: FieldDecl {{.*}} implicit referenced __handle '__hlsl_resource_t
-// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
-// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
+// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-ROV-SAME{LITERAL}: [[hlsl::is_rov]]
 // CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(float)]]
 // CHECK-COUNTER-HANDLE: FieldDecl {{.*}} implicit referenced __counter_handle '__hlsl_resource_t
-// CHECK-COUNTER-HANDLE-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-COUNTER-HANDLE-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-COUNTER-HANDLE-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-COUNTER-HANDLE-SAME{LITERAL}: [[hlsl::is_counter]]
 // CHECK-COUNTER-HANDLE-SAME{LITERAL}: [[hlsl::contained_type(float)]]
diff --git a/clang/test/AST/HLSL/Textures-AST.hlsl b/clang/test/AST/HLSL/Textures-AST.hlsl
index 86ddb0a0308f7..5d52a0f71b979 100644
--- a/clang/test/AST/HLSL/Textures-AST.hlsl
+++ b/clang/test/AST/HLSL/Textures-AST.hlsl
@@ -8,12 +8,12 @@
 // CHECK: CXXRecordDecl {{.*}} [[TEXTURE]] definition
 // CHECK: FinalAttr {{.*}} Implicit final
 // CHECK-NEXT: FieldDecl {{.*}} implicit __handle '__hlsl_resource_t
-// SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
-// UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
+// UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // SRV-ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // UAV-ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 
 // SRV: CXXMethodDecl {{.*}} operator[] 'const hlsl_device element_type &(vector<unsigned int, [[INDEX_SIZE]]>) const' inline
 // SRV-NEXT: ParmVarDecl {{.*}} Index 'vector<unsigned int, [[INDEX_SIZE]]>'
@@ -24,10 +24,10 @@
 // SRV-NEXT: CallExpr {{.*}} '<dependent type>'
 // SRV-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
 // SRV-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // SRV-ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // SRV-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// SRV-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// SRV-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // SRV-SAME: ' lvalue .__handle
 // SRV-NEXT: CXXThisExpr {{.*}} 'const hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // SRV-NEXT: DeclRefExpr {{.*}} 'vector<unsigned int, [[INDEX_SIZE]]>' lvalue ParmVar {{.*}} 'Index' 'vector<unsigned int, [[INDEX_SIZE]]>'
@@ -42,10 +42,10 @@
 // UAV-NEXT: CallExpr {{.*}} '<dependent type>'
 // UAV-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
 // UAV-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // UAV-ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // UAV-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// UAV-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// UAV-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // UAV-SAME: ' lvalue .__handle
 // UAV-NEXT: CXXThisExpr {{.*}} 'const hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // UAV-NEXT: DeclRefExpr {{.*}} 'vector<unsigned int, [[INDEX_SIZE]]>' lvalue ParmVar {{.*}} 'Index' 'vector<unsigned int, [[INDEX_SIZE]]>'
@@ -60,12 +60,12 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getdimensions_xy' 'void (__hlsl_resource_t, unsigned int &, unsigned int &) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
-// UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
+// UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // SRV-ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // UAV-ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' lvalue ParmVar {{.*}} 'width' 'unsigned int &__restrict'
@@ -84,12 +84,12 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getdimensions_levels_xy' 'void (__hlsl_resource_t, unsigned int, unsigned int &, unsigned int &, unsigned int &) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
-// UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
+// UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // SRV-ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // UAV-ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' lvalue ParmVar {{.*}} 'mipLevel' 'unsigned int'
@@ -107,12 +107,12 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getdimensions_xy_float' 'void (__hlsl_resource_t, float &, float &) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
-// UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
+// UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // SRV-ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // UAV-ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'float' lvalue ParmVar {{.*}} 'width' 'float &__restrict'
@@ -131,12 +131,12 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getdimensions_levels_xy_float' 'void (__hlsl_resource_t, unsigned int, float &, float &, float &) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
-// UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
+// UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // SRV-ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // UAV-ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' lvalue ParmVar {{.*}} 'mipLevel' 'unsigned int'
diff --git a/clang/test/AST/HLSL/Textures-scalar-AST.hlsl b/clang/test/AST/HLSL/Textures-scalar-AST.hlsl
index 43b700ec6c7f4..54506cdb6adff 100644
--- a/clang/test/AST/HLSL/Textures-scalar-AST.hlsl
+++ b/clang/test/AST/HLSL/Textures-scalar-AST.hlsl
@@ -4,22 +4,22 @@
 // CHECK: CXXRecordDecl {{.*}} SamplerState definition
 // CHECK: FinalAttr {{.*}} Implicit final
 // CHECK-NEXT: FieldDecl {{.*}} implicit {{.*}} __handle '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 
 // CHECK: CXXRecordDecl {{.*}} SamplerComparisonState definition
 // CHECK: FinalAttr {{.*}} Implicit final
 // CHECK-NEXT: FieldDecl {{.*}} implicit {{.*}} __handle '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 
 // CHECK: ClassTemplateDecl {{.*}} [[TEXTURE]]
 // CHECK: TemplateTypeParmDecl {{.*}} element_type
 // CHECK: CXXRecordDecl {{.*}} [[TEXTURE]] definition
 // CHECK: FinalAttr {{.*}} Implicit final
 // CHECK-NEXT: FieldDecl {{.*}} implicit __handle '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 
 // CHECK: CXXMethodDecl {{.*}} Load 'element_type (vector<int, [[LOCATION_DIM]]>)'
 // CHECK-NEXT: ParmVarDecl {{.*}} Location 'vector<int, [[LOCATION_DIM]]>'
@@ -29,10 +29,10 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_load_level' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<int, [[LOCATION_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<int, [[LOCATION_DIM]]>'
@@ -47,10 +47,10 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_load_level' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<int, [[LOCATION_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<int, [[LOCATION_DIM]]>'
@@ -66,10 +66,10 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'const hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<unsigned int, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Index' 'vector<unsigned int, [[COORD_DIM]]>'
@@ -84,14 +84,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -107,14 +107,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -132,14 +132,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -157,14 +157,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_bias' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -182,14 +182,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_bias' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -209,14 +209,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_bias' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -236,14 +236,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_grad' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -263,14 +263,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_grad' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -292,14 +292,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_grad' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -319,14 +319,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_level' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -344,14 +344,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_level' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -369,14 +369,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_cmp' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerComparisonState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerComparisonState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -394,14 +394,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_cmp' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerComparisonState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerComparisonState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -421,14 +421,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_cmp' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerComparisonState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerComparisonState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -447,14 +447,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_cmp_level_zero' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerComparisonState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerComparisonState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -472,14 +472,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_cmp_level_zero' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerComparisonState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerComparisonState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -496,14 +496,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_calculate_lod' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, 2>' lvalue ParmVar {{.*}} 'Location' 'vector<float, 2>'
@@ -518,14 +518,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_calculate_lod_unclamped' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, 2>' lvalue ParmVar {{.*}} 'Location' 'vector<float, 2>'
@@ -540,10 +540,10 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getdimensions_xy' 'void (__hlsl_resource_t, unsigned int &, unsigned int &) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' lvalue ParmVar {{.*}} 'width' 'unsigned int &__restrict'
@@ -562,10 +562,10 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getdimensions_levels_xy' 'void (__hlsl_resource_t, unsigned int, unsigned int &, unsigned int &, unsigned int &) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' lvalue ParmVar {{.*}} 'mipLevel' 'unsigned int'
@@ -583,10 +583,10 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getdimensions_xy_float' 'void (__hlsl_resource_t, float &, float &) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'float' lvalue ParmVar {{.*}} 'width' 'float &__restrict'
@@ -605,10 +605,10 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getdimensions_levels_xy_float' 'void (__hlsl_resource_t, unsigned int, float &, float &, float &) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' lvalue ParmVar {{.*}} 'mipLevel' 'unsigned int'
diff --git a/clang/test/AST/HLSL/Textures-vector-AST.hlsl b/clang/test/AST/HLSL/Textures-vector-AST.hlsl
index f9b347aebaa02..e37779881a0f8 100644
--- a/clang/test/AST/HLSL/Textures-vector-AST.hlsl
+++ b/clang/test/AST/HLSL/Textures-vector-AST.hlsl
@@ -4,12 +4,12 @@
 // CHECK: CXXRecordDecl {{.*}} SamplerState definition
 // CHECK: FinalAttr {{.*}} Implicit final
 // CHECK-NEXT: FieldDecl {{.*}} implicit {{.*}} __handle '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 
 // CHECK: CXXRecordDecl {{.*}} SamplerComparisonState definition
 // CHECK: FinalAttr {{.*}} Implicit final
 // CHECK-NEXT: FieldDecl {{.*}} implicit {{.*}} __handle '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 
 // CHECK: ClassTemplateDecl {{.*}} [[TEXTURE]]
 // CHECK: TemplateTypeParmDecl {{.*}} element_type
@@ -20,10 +20,10 @@
 // CHECK: TemplateTypeParmDecl {{.*}} element_type
 // CHECK: NonTypeTemplateParmDecl {{.*}} element_count
 // CHECK-NEXT: FieldDecl {{.*}} implicit __handle '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 
 // CHECK: CXXMethodDecl {{.*}} Load 'vector<element_type, element_count> (vector<int, [[LOCATION_DIM]]>)'
 // CHECK-NEXT: ParmVarDecl {{.*}} Location 'vector<int, [[LOCATION_DIM]]>'
@@ -33,10 +33,10 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_load_level' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<int, [[LOCATION_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<int, [[LOCATION_DIM]]>'
@@ -51,10 +51,10 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_load_level' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<int, [[LOCATION_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<int, [[LOCATION_DIM]]>'
@@ -70,10 +70,10 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'const hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<unsigned int, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Index' 'vector<unsigned int, [[COORD_DIM]]>'
@@ -88,14 +88,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -111,14 +111,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -136,14 +136,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -161,14 +161,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_bias' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -186,14 +186,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_bias' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -213,14 +213,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_bias' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -240,14 +240,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_grad' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -267,14 +267,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_grad' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -296,14 +296,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_grad' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -323,14 +323,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_level' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -348,14 +348,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_level' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -373,14 +373,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_cmp' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerComparisonState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerComparisonState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -398,14 +398,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_cmp' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerComparisonState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerComparisonState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -425,14 +425,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_cmp' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerComparisonState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerComparisonState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -451,14 +451,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_cmp_level_zero' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerComparisonState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerComparisonState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -476,14 +476,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_sample_cmp_level_zero' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerComparisonState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerComparisonState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, [[COORD_DIM]]>' lvalue ParmVar {{.*}} 'Location' 'vector<float, [[COORD_DIM]]>'
@@ -500,14 +500,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_calculate_lod' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, 2>' lvalue ParmVar {{.*}} 'Location' 'vector<float, 2>'
@@ -522,14 +522,14 @@
 // CHECK-NEXT: CallExpr {{.*}} '<dependent type>'
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_calculate_lod_unclamped' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: DeclRefExpr {{.*}} 'hlsl::SamplerState' lvalue ParmVar {{.*}} 'Sampler' 'hlsl::SamplerState'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'vector<float, 2>' lvalue ParmVar {{.*}} 'Location' 'vector<float, 2>'
@@ -544,10 +544,10 @@
 // CHECK-NEXT: CallExpr {{.*}}
 // CHECK-NEXT: DeclRefExpr {{.*}} '__builtin_hlsl_resource_getdimensions_xy' 'void (__hlsl_resource_t, unsigned int &, unsigned int &) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' lvalue ParmVar {{.*}} 'width' 'unsigned int &__restrict'
@@ -565,10 +565,10 @@
 // CHECK-NEXT: CallExpr {{.*}}
 // CHECK-NEXT: DeclRefExpr {{.*}} '__builtin_hlsl_resource_getdimensions_levels_xy' 'void (__hlsl_resource_t, unsigned int, unsigned int &, unsigned int &, unsigned int &) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' lvalue ParmVar {{.*}} 'mipLevel' 'unsigned int'
@@ -585,10 +585,10 @@
 // CHECK-NEXT: CallExpr {{.*}}
 // CHECK-NEXT: DeclRefExpr {{.*}} '__builtin_hlsl_resource_getdimensions_xy_float' 'void (__hlsl_resource_t, float &, float &) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'float' lvalue ParmVar {{.*}} 'width' 'float &__restrict'
@@ -606,10 +606,10 @@
 // CHECK-NEXT: CallExpr {{.*}}
 // CHECK-NEXT: DeclRefExpr {{.*}} '__builtin_hlsl_resource_getdimensions_levels_xy_float' 'void (__hlsl_resource_t, unsigned int, float &, float &, float &) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // ARRAY-SAME{LITERAL}: [[hlsl::is_array]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<element_type, element_count>)]]
-// CHECK-SAME{LITERAL}: [[hlsl::resource_dimension(2D)]]
+// CHECK-SAME{LITERAL}: [[hlsl::dimension("2D")]]
 // CHECK-SAME: ' lvalue .__handle
 // CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[TEXTURE]]<vector<element_type, element_count>>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' lvalue ParmVar {{.*}} 'mipLevel' 'unsigned int'
diff --git a/clang/test/AST/HLSL/TypedBuffers-AST.hlsl b/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
index 855c2958a0ce3..a2708e0782bef 100644
--- a/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
@@ -60,8 +60,8 @@ RESOURCE<float> Buffer;
 
 // CHECK: FinalAttr {{.*}} Implicit final
 // CHECK-NEXT: FieldDecl {{.*}} implicit __handle '__hlsl_resource_t
-// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
+// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 
 // Default constructor
@@ -171,7 +171,7 @@ RESOURCE<float> Buffer;
 // CHECK-SRV-NEXT: CallExpr
 // CHECK-SRV-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}}  '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
 // CHECK-SRV-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // CHECK-SRV-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SRV-SAME: ' lvalue .__handle {{.*}}
 // CHECK-SRV-NEXT: CXXThisExpr {{.*}} 'const hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
@@ -187,7 +187,7 @@ RESOURCE<float> Buffer;
 // CHECK-UAV-NEXT: CallExpr
 // CHECK-UAV-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}}  '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
 // CHECK-UAV-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-UAV-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-UAV-SAME: ' lvalue .__handle {{.*}}
 // CHECK-UAV-NEXT: CXXThisExpr {{.*}} 'const hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
@@ -205,8 +205,8 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: CallExpr
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}}  '__builtin_hlsl_resource_getpointer' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
+// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SAME: ' lvalue .__handle {{.*}}
 // CHECK-NEXT: CXXThisExpr {{.*}} 'const hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
@@ -224,8 +224,8 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: CallExpr
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_load_with_status' 'void (...) noexcept'
 // CHECK-NEXT: MemberExpr {{.*}} '__hlsl_resource_t
-// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
+// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-NEXT: CXXThisExpr {{.*}} 'const hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' lvalue ParmVar {{.*}} 'Index' 'unsigned int'
@@ -251,6 +251,6 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: BuiltinType {{.*}}  'float'
 // CHECK-NEXT: FinalAttr {{.*}} Implicit final
 // CHECK-NEXT: FieldDecl {{.*}} implicit referenced __handle '__hlsl_resource_t
-// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
-// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
+// CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(float)]]
diff --git a/clang/test/CIR/CodeGenSYCL/kernel-call-stmt.cpp b/clang/test/CIR/CodeGenSYCL/kernel-call-stmt.cpp
new file mode 100644
index 0000000000000..3f42bad95dc14
--- /dev/null
+++ b/clang/test/CIR/CodeGenSYCL/kernel-call-stmt.cpp
@@ -0,0 +1,45 @@
+// RUN: %clang_cc1 -std=c++20 -fsycl-is-host -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -std=c++20 -fsycl-is-host -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -std=c++20 -fsycl-is-host -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+// Verify that, during host compilation, the body of a function declared with
+// the sycl_kernel_entry_point attribute is lowered to its kernel launch
+// statement rather than reporting a not-yet-implemented error. The kernel
+// entry point body must be replaced by the launch call; the original kernel
+// functor invocation must not be emitted on the host.
+
+// Required by sycl_kernel_entry_point semantics.
+template <typename KernelName, typename... Ts>
+void sycl_kernel_launch(const char *, Ts...) {}
+
+template <typename KernelName, typename KernelType>
+[[clang::sycl_kernel_entry_point(KernelName)]]
+void kernel_single_task(KernelType kf) { kf(); }
+
+struct KN;
+struct K {
+  void operator()() const {}
+};
+
+void test() { kernel_single_task<KN>(K{}); }
+
+// The kernel entry point body is replaced by a call to the sycl_kernel_launch
+// specialization, and does not invoke the kernel functor's operator() on the
+// host.
+// CIR-LABEL: cir.func {{.*}}@_Z18kernel_single_taskI2KN1KEvT0_
+// CIR-NOT:     cir.call @_ZNK1KclEv
+// CIR:         cir.call @_Z18sycl_kernel_launchI2KNJ1KEEvPKcDpT0_
+// CIR:         cir.return
+
+// LLVM-LABEL: define {{.*}}void @_Z18kernel_single_taskI2KN1KEvT0_
+// LLVM-NOT:     call {{.*}}@_ZNK1KclEv
+// LLVM:         call void @_Z18sycl_kernel_launchI2KNJ1KEEvPKcDpT0_
+// LLVM:         ret void
+
+// OGCG-LABEL: define {{.*}}void @_Z18kernel_single_taskI2KN1KEvT0_
+// OGCG-NOT:     call {{.*}}@_ZNK1KclEv
+// OGCG:         call void @_Z18sycl_kernel_launchI2KNJ1KEEvPKcDpT0_
+// OGCG:         ret void
diff --git a/clang/test/CIR/CodeGenSYCL/kernel-caller-entry-point.cpp b/clang/test/CIR/CodeGenSYCL/kernel-caller-entry-point.cpp
new file mode 100644
index 0000000000000..c6522064f2fdc
--- /dev/null
+++ b/clang/test/CIR/CodeGenSYCL/kernel-caller-entry-point.cpp
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -std=c++20 -fsycl-is-device -triple spir64-unknown-unknown \
+// RUN:   -fclangir -emit-cir -verify %s
+
+// During device compilation, a SYCL kernel caller offload entry point is
+// emitted in place of each sycl_kernel_entry_point attributed function. That
+// lowering is not yet implemented in CIR, so it must be reported as a clean
+// "Not Yet Implemented" diagnostic rather than crashing.
+
+// Required by sycl_kernel_entry_point semantics.
+template <typename KernelName, typename... Ts>
+void sycl_kernel_launch(const char *, Ts...) {}
+
+template <typename KernelName, typename KernelType>
+[[clang::sycl_kernel_entry_point(KernelName)]]
+// expected-error@+1 {{ClangIR code gen Not Yet Implemented: SYCL kernel caller offload entry point}}
+void kernel_single_task(KernelType kf) { kf(); }
+
+struct KN;
+struct K {
+  void operator()() const {}
+};
+
+void test() { kernel_single_task<KN>(K{}); }
diff --git a/clang/test/CodeGen/AArch64/neon-intrinsics.c b/clang/test/CodeGen/AArch64/neon-intrinsics.c
index 05f563a5331e3..ee91751fed634 100644
--- a/clang/test/CodeGen/AArch64/neon-intrinsics.c
+++ b/clang/test/CodeGen/AArch64/neon-intrinsics.c
@@ -2325,240 +2325,6 @@ uint64x2_t test_vcltq_f64(float64x2_t v1, float64x2_t v2) {
   return vcltq_f64(v1, v2);
 }
 
-// CHECK-LABEL: define dso_local <8 x i8> @test_vqadd_s8(
-// CHECK-SAME: <8 x i8> noundef [[A:%.*]], <8 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VQADD_V_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.sqadd.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
-// CHECK-NEXT:    ret <8 x i8> [[VQADD_V_I]]
-//
-int8x8_t test_vqadd_s8(int8x8_t a, int8x8_t b) {
-  return vqadd_s8(a, b);
-}
-
-// CHECK-LABEL: define dso_local <4 x i16> @test_vqadd_s16(
-// CHECK-SAME: <4 x i16> noundef [[A:%.*]], <4 x i16> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i16> [[A]] to <8 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[B]] to <8 x i8>
-// CHECK-NEXT:    [[VQADD_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x i16>
-// CHECK-NEXT:    [[VQADD_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x i16>
-// CHECK-NEXT:    [[VQADD_V2_I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> [[VQADD_V_I]], <4 x i16> [[VQADD_V1_I]])
-// CHECK-NEXT:    [[VQADD_V3_I:%.*]] = bitcast <4 x i16> [[VQADD_V2_I]] to <8 x i8>
-// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VQADD_V3_I]] to <4 x i16>
-// CHECK-NEXT:    ret <4 x i16> [[TMP2]]
-//
-int16x4_t test_vqadd_s16(int16x4_t a, int16x4_t b) {
-  return vqadd_s16(a, b);
-}
-
-// CHECK-LABEL: define dso_local <2 x i32> @test_vqadd_s32(
-// CHECK-SAME: <2 x i32> noundef [[A:%.*]], <2 x i32> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[A]] to <8 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[B]] to <8 x i8>
-// CHECK-NEXT:    [[VQADD_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
-// CHECK-NEXT:    [[VQADD_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
-// CHECK-NEXT:    [[VQADD_V2_I:%.*]] = call <2 x i32> @llvm.aarch64.neon.sqadd.v2i32(<2 x i32> [[VQADD_V_I]], <2 x i32> [[VQADD_V1_I]])
-// CHECK-NEXT:    [[VQADD_V3_I:%.*]] = bitcast <2 x i32> [[VQADD_V2_I]] to <8 x i8>
-// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VQADD_V3_I]] to <2 x i32>
-// CHECK-NEXT:    ret <2 x i32> [[TMP2]]
-//
-int32x2_t test_vqadd_s32(int32x2_t a, int32x2_t b) {
-  return vqadd_s32(a, b);
-}
-
-// CHECK-LABEL: define dso_local <1 x i64> @test_vqadd_s64(
-// CHECK-SAME: <1 x i64> noundef [[A:%.*]], <1 x i64> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
-// CHECK-NEXT:    [[VQADD_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
-// CHECK-NEXT:    [[VQADD_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
-// CHECK-NEXT:    [[VQADD_V2_I:%.*]] = call <1 x i64> @llvm.aarch64.neon.sqadd.v1i64(<1 x i64> [[VQADD_V_I]], <1 x i64> [[VQADD_V1_I]])
-// CHECK-NEXT:    [[VQADD_V3_I:%.*]] = bitcast <1 x i64> [[VQADD_V2_I]] to <8 x i8>
-// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VQADD_V3_I]] to i64
-// CHECK-NEXT:    [[REF_TMP_I_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP2]], i64 0
-// CHECK-NEXT:    ret <1 x i64> [[REF_TMP_I_SROA_0_0_VEC_INSERT]]
-//
-int64x1_t test_vqadd_s64(int64x1_t a, int64x1_t b) {
-  return vqadd_s64(a, b);
-}
-
-// CHECK-LABEL: define dso_local <8 x i8> @test_vqadd_u8(
-// CHECK-SAME: <8 x i8> noundef [[A:%.*]], <8 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VQADD_V_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.uqadd.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
-// CHECK-NEXT:    ret <8 x i8> [[VQADD_V_I]]
-//
-uint8x8_t test_vqadd_u8(uint8x8_t a, uint8x8_t b) {
-  return vqadd_u8(a, b);
-}
-
-// CHECK-LABEL: define dso_local <4 x i16> @test_vqadd_u16(
-// CHECK-SAME: <4 x i16> noundef [[A:%.*]], <4 x i16> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i16> [[A]] to <8 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[B]] to <8 x i8>
-// CHECK-NEXT:    [[VQADD_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x i16>
-// CHECK-NEXT:    [[VQADD_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x i16>
-// CHECK-NEXT:    [[VQADD_V2_I:%.*]] = call <4 x i16> @llvm.aarch64.neon.uqadd.v4i16(<4 x i16> [[VQADD_V_I]], <4 x i16> [[VQADD_V1_I]])
-// CHECK-NEXT:    [[VQADD_V3_I:%.*]] = bitcast <4 x i16> [[VQADD_V2_I]] to <8 x i8>
-// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VQADD_V3_I]] to <4 x i16>
-// CHECK-NEXT:    ret <4 x i16> [[TMP2]]
-//
-uint16x4_t test_vqadd_u16(uint16x4_t a, uint16x4_t b) {
-  return vqadd_u16(a, b);
-}
-
-// CHECK-LABEL: define dso_local <2 x i32> @test_vqadd_u32(
-// CHECK-SAME: <2 x i32> noundef [[A:%.*]], <2 x i32> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[A]] to <8 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[B]] to <8 x i8>
-// CHECK-NEXT:    [[VQADD_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
-// CHECK-NEXT:    [[VQADD_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
-// CHECK-NEXT:    [[VQADD_V2_I:%.*]] = call <2 x i32> @llvm.aarch64.neon.uqadd.v2i32(<2 x i32> [[VQADD_V_I]], <2 x i32> [[VQADD_V1_I]])
-// CHECK-NEXT:    [[VQADD_V3_I:%.*]] = bitcast <2 x i32> [[VQADD_V2_I]] to <8 x i8>
-// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VQADD_V3_I]] to <2 x i32>
-// CHECK-NEXT:    ret <2 x i32> [[TMP2]]
-//
-uint32x2_t test_vqadd_u32(uint32x2_t a, uint32x2_t b) {
-  return vqadd_u32(a, b);
-}
-
-// CHECK-LABEL: define dso_local <1 x i64> @test_vqadd_u64(
-// CHECK-SAME: <1 x i64> noundef [[A:%.*]], <1 x i64> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
-// CHECK-NEXT:    [[VQADD_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
-// CHECK-NEXT:    [[VQADD_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
-// CHECK-NEXT:    [[VQADD_V2_I:%.*]] = call <1 x i64> @llvm.aarch64.neon.uqadd.v1i64(<1 x i64> [[VQADD_V_I]], <1 x i64> [[VQADD_V1_I]])
-// CHECK-NEXT:    [[VQADD_V3_I:%.*]] = bitcast <1 x i64> [[VQADD_V2_I]] to <8 x i8>
-// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VQADD_V3_I]] to i64
-// CHECK-NEXT:    [[REF_TMP_I_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP2]], i64 0
-// CHECK-NEXT:    ret <1 x i64> [[REF_TMP_I_SROA_0_0_VEC_INSERT]]
-//
-uint64x1_t test_vqadd_u64(uint64x1_t a, uint64x1_t b) {
-  return vqadd_u64(a, b);
-}
-
-// CHECK-LABEL: define dso_local <16 x i8> @test_vqaddq_s8(
-// CHECK-SAME: <16 x i8> noundef [[A:%.*]], <16 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VQADDQ_V_I:%.*]] = call <16 x i8> @llvm.aarch64.neon.sqadd.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
-// CHECK-NEXT:    ret <16 x i8> [[VQADDQ_V_I]]
-//
-int8x16_t test_vqaddq_s8(int8x16_t a, int8x16_t b) {
-  return vqaddq_s8(a, b);
-}
-
-// CHECK-LABEL: define dso_local <8 x i16> @test_vqaddq_s16(
-// CHECK-SAME: <8 x i16> noundef [[A:%.*]], <8 x i16> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[A]] to <16 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[B]] to <16 x i8>
-// CHECK-NEXT:    [[VQADDQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x i16>
-// CHECK-NEXT:    [[VQADDQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x i16>
-// CHECK-NEXT:    [[VQADDQ_V2_I:%.*]] = call <8 x i16> @llvm.aarch64.neon.sqadd.v8i16(<8 x i16> [[VQADDQ_V_I]], <8 x i16> [[VQADDQ_V1_I]])
-// CHECK-NEXT:    [[VQADDQ_V3_I:%.*]] = bitcast <8 x i16> [[VQADDQ_V2_I]] to <16 x i8>
-// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VQADDQ_V3_I]] to <8 x i16>
-// CHECK-NEXT:    ret <8 x i16> [[TMP2]]
-//
-int16x8_t test_vqaddq_s16(int16x8_t a, int16x8_t b) {
-  return vqaddq_s16(a, b);
-}
-
-// CHECK-LABEL: define dso_local <4 x i32> @test_vqaddq_s32(
-// CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[A]] to <16 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[B]] to <16 x i8>
-// CHECK-NEXT:    [[VQADDQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
-// CHECK-NEXT:    [[VQADDQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32>
-// CHECK-NEXT:    [[VQADDQ_V2_I:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> [[VQADDQ_V_I]], <4 x i32> [[VQADDQ_V1_I]])
-// CHECK-NEXT:    [[VQADDQ_V3_I:%.*]] = bitcast <4 x i32> [[VQADDQ_V2_I]] to <16 x i8>
-// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VQADDQ_V3_I]] to <4 x i32>
-// CHECK-NEXT:    ret <4 x i32> [[TMP2]]
-//
-int32x4_t test_vqaddq_s32(int32x4_t a, int32x4_t b) {
-  return vqaddq_s32(a, b);
-}
-
-// CHECK-LABEL: define dso_local <2 x i64> @test_vqaddq_s64(
-// CHECK-SAME: <2 x i64> noundef [[A:%.*]], <2 x i64> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i64> [[A]] to <16 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[B]] to <16 x i8>
-// CHECK-NEXT:    [[VQADDQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <2 x i64>
-// CHECK-NEXT:    [[VQADDQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <2 x i64>
-// CHECK-NEXT:    [[VQADDQ_V2_I:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> [[VQADDQ_V_I]], <2 x i64> [[VQADDQ_V1_I]])
-// CHECK-NEXT:    [[VQADDQ_V3_I:%.*]] = bitcast <2 x i64> [[VQADDQ_V2_I]] to <16 x i8>
-// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VQADDQ_V3_I]] to <2 x i64>
-// CHECK-NEXT:    ret <2 x i64> [[TMP2]]
-//
-int64x2_t test_vqaddq_s64(int64x2_t a, int64x2_t b) {
-  return vqaddq_s64(a, b);
-}
-
-// CHECK-LABEL: define dso_local <16 x i8> @test_vqaddq_u8(
-// CHECK-SAME: <16 x i8> noundef [[A:%.*]], <16 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VQADDQ_V_I:%.*]] = call <16 x i8> @llvm.aarch64.neon.uqadd.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
-// CHECK-NEXT:    ret <16 x i8> [[VQADDQ_V_I]]
-//
-uint8x16_t test_vqaddq_u8(uint8x16_t a, uint8x16_t b) {
-  return vqaddq_u8(a, b);
-}
-
-// CHECK-LABEL: define dso_local <8 x i16> @test_vqaddq_u16(
-// CHECK-SAME: <8 x i16> noundef [[A:%.*]], <8 x i16> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[A]] to <16 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[B]] to <16 x i8>
-// CHECK-NEXT:    [[VQADDQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x i16>
-// CHECK-NEXT:    [[VQADDQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x i16>
-// CHECK-NEXT:    [[VQADDQ_V2_I:%.*]] = call <8 x i16> @llvm.aarch64.neon.uqadd.v8i16(<8 x i16> [[VQADDQ_V_I]], <8 x i16> [[VQADDQ_V1_I]])
-// CHECK-NEXT:    [[VQADDQ_V3_I:%.*]] = bitcast <8 x i16> [[VQADDQ_V2_I]] to <16 x i8>
-// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VQADDQ_V3_I]] to <8 x i16>
-// CHECK-NEXT:    ret <8 x i16> [[TMP2]]
-//
-uint16x8_t test_vqaddq_u16(uint16x8_t a, uint16x8_t b) {
-  return vqaddq_u16(a, b);
-}
-
-// CHECK-LABEL: define dso_local <4 x i32> @test_vqaddq_u32(
-// CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[A]] to <16 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[B]] to <16 x i8>
-// CHECK-NEXT:    [[VQADDQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
-// CHECK-NEXT:    [[VQADDQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32>
-// CHECK-NEXT:    [[VQADDQ_V2_I:%.*]] = call <4 x i32> @llvm.aarch64.neon.uqadd.v4i32(<4 x i32> [[VQADDQ_V_I]], <4 x i32> [[VQADDQ_V1_I]])
-// CHECK-NEXT:    [[VQADDQ_V3_I:%.*]] = bitcast <4 x i32> [[VQADDQ_V2_I]] to <16 x i8>
-// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VQADDQ_V3_I]] to <4 x i32>
-// CHECK-NEXT:    ret <4 x i32> [[TMP2]]
-//
-uint32x4_t test_vqaddq_u32(uint32x4_t a, uint32x4_t b) {
-  return vqaddq_u32(a, b);
-}
-
-// CHECK-LABEL: define dso_local <2 x i64> @test_vqaddq_u64(
-// CHECK-SAME: <2 x i64> noundef [[A:%.*]], <2 x i64> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i64> [[A]] to <16 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[B]] to <16 x i8>
-// CHECK-NEXT:    [[VQADDQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <2 x i64>
-// CHECK-NEXT:    [[VQADDQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <2 x i64>
-// CHECK-NEXT:    [[VQADDQ_V2_I:%.*]] = call <2 x i64> @llvm.aarch64.neon.uqadd.v2i64(<2 x i64> [[VQADDQ_V_I]], <2 x i64> [[VQADDQ_V1_I]])
-// CHECK-NEXT:    [[VQADDQ_V3_I:%.*]] = bitcast <2 x i64> [[VQADDQ_V2_I]] to <16 x i8>
-// CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VQADDQ_V3_I]] to <2 x i64>
-// CHECK-NEXT:    ret <2 x i64> [[TMP2]]
-//
-uint64x2_t test_vqaddq_u64(uint64x2_t a, uint64x2_t b) {
-  return vqaddq_u64(a, b);
-}
-
 // CHECK-LABEL: define dso_local <8 x i8> @test_vqsub_s8(
 // CHECK-SAME: <8 x i8> noundef [[A:%.*]], <8 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -5370,98 +5136,6 @@ int64x2_t test_vqdmlsl_high_s32(int64x2_t a, int32x4_t b, int32x4_t c) {
   return vqdmlsl_high_s32(a, b, c);
 }
 
-// CHECK-LABEL: define dso_local i8 @test_vqaddb_s8(
-// CHECK-SAME: i8 noundef [[A:%.*]], i8 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i8> poison, i8 [[A]], i64 0
-// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i8> poison, i8 [[B]], i64 0
-// CHECK-NEXT:    [[VQADDB_S8_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.sqadd.v8i8(<8 x i8> [[TMP0]], <8 x i8> [[TMP1]])
-// CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i8> [[VQADDB_S8_I]], i64 0
-// CHECK-NEXT:    ret i8 [[TMP2]]
-//
-int8_t test_vqaddb_s8(int8_t a, int8_t b) {
-  return vqaddb_s8(a, b);
-}
-
-// CHECK-LABEL: define dso_local i16 @test_vqaddh_s16(
-// CHECK-SAME: i16 noundef [[A:%.*]], i16 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i16> poison, i16 [[A]], i64 0
-// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i16> poison, i16 [[B]], i64 0
-// CHECK-NEXT:    [[VQADDH_S16_I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> [[TMP0]], <4 x i16> [[TMP1]])
-// CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i16> [[VQADDH_S16_I]], i64 0
-// CHECK-NEXT:    ret i16 [[TMP2]]
-//
-int16_t test_vqaddh_s16(int16_t a, int16_t b) {
-  return vqaddh_s16(a, b);
-}
-
-// CHECK-LABEL: define dso_local i32 @test_vqadds_s32(
-// CHECK-SAME: i32 noundef [[A:%.*]], i32 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VQADDS_S32_I:%.*]] = call i32 @llvm.aarch64.neon.sqadd.i32(i32 [[A]], i32 [[B]])
-// CHECK-NEXT:    ret i32 [[VQADDS_S32_I]]
-//
-int32_t test_vqadds_s32(int32_t a, int32_t b) {
-  return vqadds_s32(a, b);
-}
-
-// CHECK-LABEL: define dso_local i64 @test_vqaddd_s64(
-// CHECK-SAME: i64 noundef [[A:%.*]], i64 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VQADDD_S64_I:%.*]] = call i64 @llvm.aarch64.neon.sqadd.i64(i64 [[A]], i64 [[B]])
-// CHECK-NEXT:    ret i64 [[VQADDD_S64_I]]
-//
-int64_t test_vqaddd_s64(int64_t a, int64_t b) {
-  return vqaddd_s64(a, b);
-}
-
-// CHECK-LABEL: define dso_local i8 @test_vqaddb_u8(
-// CHECK-SAME: i8 noundef [[A:%.*]], i8 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i8> poison, i8 [[A]], i64 0
-// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i8> poison, i8 [[B]], i64 0
-// CHECK-NEXT:    [[VQADDB_U8_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.uqadd.v8i8(<8 x i8> [[TMP0]], <8 x i8> [[TMP1]])
-// CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i8> [[VQADDB_U8_I]], i64 0
-// CHECK-NEXT:    ret i8 [[TMP2]]
-//
-uint8_t test_vqaddb_u8(uint8_t a, uint8_t b) {
-  return vqaddb_u8(a, b);
-}
-
-// CHECK-LABEL: define dso_local i16 @test_vqaddh_u16(
-// CHECK-SAME: i16 noundef [[A:%.*]], i16 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i16> poison, i16 [[A]], i64 0
-// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i16> poison, i16 [[B]], i64 0
-// CHECK-NEXT:    [[VQADDH_U16_I:%.*]] = call <4 x i16> @llvm.aarch64.neon.uqadd.v4i16(<4 x i16> [[TMP0]], <4 x i16> [[TMP1]])
-// CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i16> [[VQADDH_U16_I]], i64 0
-// CHECK-NEXT:    ret i16 [[TMP2]]
-//
-uint16_t test_vqaddh_u16(uint16_t a, uint16_t b) {
-  return vqaddh_u16(a, b);
-}
-
-// CHECK-LABEL: define dso_local i32 @test_vqadds_u32(
-// CHECK-SAME: i32 noundef [[A:%.*]], i32 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VQADDS_U32_I:%.*]] = call i32 @llvm.aarch64.neon.uqadd.i32(i32 [[A]], i32 [[B]])
-// CHECK-NEXT:    ret i32 [[VQADDS_U32_I]]
-//
-uint32_t test_vqadds_u32(uint32_t a, uint32_t b) {
-  return vqadds_u32(a, b);
-}
-
-// CHECK-LABEL: define dso_local i64 @test_vqaddd_u64(
-// CHECK-SAME: i64 noundef [[A:%.*]], i64 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VQADDD_U64_I:%.*]] = call i64 @llvm.aarch64.neon.uqadd.i64(i64 [[A]], i64 [[B]])
-// CHECK-NEXT:    ret i64 [[VQADDD_U64_I]]
-//
-uint64_t test_vqaddd_u64(uint64_t a, uint64_t b) {
-  return vqaddd_u64(a, b);
-}
-
 // CHECK-LABEL: define dso_local i8 @test_vqshlb_s8(
 // CHECK-SAME: i8 noundef [[A:%.*]], i8 noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -8266,98 +7940,6 @@ int64_t test_vqnegd_s64(int64_t a) {
   return (int64_t)vqnegd_s64(a);
 }
 
-// CHECK-LABEL: define dso_local i8 @test_vuqaddb_s8(
-// CHECK-SAME: i8 noundef [[A:%.*]], i8 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i8> poison, i8 [[A]], i64 0
-// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i8> poison, i8 [[B]], i64 0
-// CHECK-NEXT:    [[VUQADDB_S8_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.suqadd.v8i8(<8 x i8> [[TMP0]], <8 x i8> [[TMP1]])
-// CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i8> [[VUQADDB_S8_I]], i64 0
-// CHECK-NEXT:    ret i8 [[TMP2]]
-//
-int8_t test_vuqaddb_s8(int8_t a, uint8_t b) {
-  return (int8_t)vuqaddb_s8(a, b);
-}
-
-// CHECK-LABEL: define dso_local i16 @test_vuqaddh_s16(
-// CHECK-SAME: i16 noundef [[A:%.*]], i16 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i16> poison, i16 [[A]], i64 0
-// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i16> poison, i16 [[B]], i64 0
-// CHECK-NEXT:    [[VUQADDH_S16_I:%.*]] = call <4 x i16> @llvm.aarch64.neon.suqadd.v4i16(<4 x i16> [[TMP0]], <4 x i16> [[TMP1]])
-// CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i16> [[VUQADDH_S16_I]], i64 0
-// CHECK-NEXT:    ret i16 [[TMP2]]
-//
-int16_t test_vuqaddh_s16(int16_t a, uint16_t b) {
-  return (int16_t)vuqaddh_s16(a, b);
-}
-
-// CHECK-LABEL: define dso_local i32 @test_vuqadds_s32(
-// CHECK-SAME: i32 noundef [[A:%.*]], i32 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VUQADDS_S32_I:%.*]] = call i32 @llvm.aarch64.neon.suqadd.i32(i32 [[A]], i32 [[B]])
-// CHECK-NEXT:    ret i32 [[VUQADDS_S32_I]]
-//
-int32_t test_vuqadds_s32(int32_t a, uint32_t b) {
-  return (int32_t)vuqadds_s32(a, b);
-}
-
-// CHECK-LABEL: define dso_local i64 @test_vuqaddd_s64(
-// CHECK-SAME: i64 noundef [[A:%.*]], i64 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VUQADDD_S64_I:%.*]] = call i64 @llvm.aarch64.neon.suqadd.i64(i64 [[A]], i64 [[B]])
-// CHECK-NEXT:    ret i64 [[VUQADDD_S64_I]]
-//
-int64_t test_vuqaddd_s64(int64_t a, uint64_t b) {
-  return (int64_t)vuqaddd_s64(a, b);
-}
-
-// CHECK-LABEL: define dso_local i8 @test_vsqaddb_u8(
-// CHECK-SAME: i8 noundef [[A:%.*]], i8 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i8> poison, i8 [[A]], i64 0
-// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i8> poison, i8 [[B]], i64 0
-// CHECK-NEXT:    [[VSQADDB_U8_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.usqadd.v8i8(<8 x i8> [[TMP0]], <8 x i8> [[TMP1]])
-// CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i8> [[VSQADDB_U8_I]], i64 0
-// CHECK-NEXT:    ret i8 [[TMP2]]
-//
-uint8_t test_vsqaddb_u8(uint8_t a, int8_t b) {
-  return (uint8_t)vsqaddb_u8(a, b);
-}
-
-// CHECK-LABEL: define dso_local i16 @test_vsqaddh_u16(
-// CHECK-SAME: i16 noundef [[A:%.*]], i16 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i16> poison, i16 [[A]], i64 0
-// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i16> poison, i16 [[B]], i64 0
-// CHECK-NEXT:    [[VSQADDH_U16_I:%.*]] = call <4 x i16> @llvm.aarch64.neon.usqadd.v4i16(<4 x i16> [[TMP0]], <4 x i16> [[TMP1]])
-// CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i16> [[VSQADDH_U16_I]], i64 0
-// CHECK-NEXT:    ret i16 [[TMP2]]
-//
-uint16_t test_vsqaddh_u16(uint16_t a, int16_t b) {
-  return (uint16_t)vsqaddh_u16(a, b);
-}
-
-// CHECK-LABEL: define dso_local i32 @test_vsqadds_u32(
-// CHECK-SAME: i32 noundef [[A:%.*]], i32 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VSQADDS_U32_I:%.*]] = call i32 @llvm.aarch64.neon.usqadd.i32(i32 [[A]], i32 [[B]])
-// CHECK-NEXT:    ret i32 [[VSQADDS_U32_I]]
-//
-uint32_t test_vsqadds_u32(uint32_t a, int32_t b) {
-  return (uint32_t)vsqadds_u32(a, b);
-}
-
-// CHECK-LABEL: define dso_local i64 @test_vsqaddd_u64(
-// CHECK-SAME: i64 noundef [[A:%.*]], i64 noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VSQADDD_U64_I:%.*]] = call i64 @llvm.aarch64.neon.usqadd.i64(i64 [[A]], i64 [[B]])
-// CHECK-NEXT:    ret i64 [[VSQADDD_U64_I]]
-//
-uint64_t test_vsqaddd_u64(uint64_t a, int64_t b) {
-  return (uint64_t)vsqaddd_u64(a, b);
-}
-
 // CHECK-LABEL: define dso_local i32 @test_vqdmlalh_s16(
 // CHECK-SAME: i32 noundef [[A:%.*]], i16 noundef [[B:%.*]], i16 noundef [[C:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -12914,214 +12496,6 @@ poly64x2_t test_vreinterpretq_p64_p16(poly16x8_t a) {
   return vreinterpretq_p64_p16(a);
 }
 
-// CHECK-LABEL: define dso_local <16 x i8> @test_vuqaddq_s8(
-// CHECK-SAME: <16 x i8> noundef [[A:%.*]], <16 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VUQADD_I:%.*]] = call <16 x i8> @llvm.aarch64.neon.suqadd.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
-// CHECK-NEXT:    ret <16 x i8> [[VUQADD_I]]
-//
-int8x16_t test_vuqaddq_s8(int8x16_t a, uint8x16_t b) {
-  return vuqaddq_s8(a, b);
-}
-
-// CHECK-LABEL: define dso_local <4 x i32> @test_vuqaddq_s32(
-// CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[A]] to <16 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[B]] to <16 x i8>
-// CHECK-NEXT:    [[VUQADD_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
-// CHECK-NEXT:    [[VUQADD1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32>
-// CHECK-NEXT:    [[VUQADD2_I:%.*]] = call <4 x i32> @llvm.aarch64.neon.suqadd.v4i32(<4 x i32> [[VUQADD_I]], <4 x i32> [[VUQADD1_I]])
-// CHECK-NEXT:    ret <4 x i32> [[VUQADD2_I]]
-//
-int32x4_t test_vuqaddq_s32(int32x4_t a, uint32x4_t b) {
-  return vuqaddq_s32(a, b);
-}
-
-// CHECK-LABEL: define dso_local <2 x i64> @test_vuqaddq_s64(
-// CHECK-SAME: <2 x i64> noundef [[A:%.*]], <2 x i64> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i64> [[A]] to <16 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[B]] to <16 x i8>
-// CHECK-NEXT:    [[VUQADD_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <2 x i64>
-// CHECK-NEXT:    [[VUQADD1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <2 x i64>
-// CHECK-NEXT:    [[VUQADD2_I:%.*]] = call <2 x i64> @llvm.aarch64.neon.suqadd.v2i64(<2 x i64> [[VUQADD_I]], <2 x i64> [[VUQADD1_I]])
-// CHECK-NEXT:    ret <2 x i64> [[VUQADD2_I]]
-//
-int64x2_t test_vuqaddq_s64(int64x2_t a, uint64x2_t b) {
-  return vuqaddq_s64(a, b);
-}
-
-// CHECK-LABEL: define dso_local <8 x i16> @test_vuqaddq_s16(
-// CHECK-SAME: <8 x i16> noundef [[A:%.*]], <8 x i16> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[A]] to <16 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[B]] to <16 x i8>
-// CHECK-NEXT:    [[VUQADD_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x i16>
-// CHECK-NEXT:    [[VUQADD1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x i16>
-// CHECK-NEXT:    [[VUQADD2_I:%.*]] = call <8 x i16> @llvm.aarch64.neon.suqadd.v8i16(<8 x i16> [[VUQADD_I]], <8 x i16> [[VUQADD1_I]])
-// CHECK-NEXT:    ret <8 x i16> [[VUQADD2_I]]
-//
-int16x8_t test_vuqaddq_s16(int16x8_t a, uint16x8_t b) {
-  return vuqaddq_s16(a, b);
-}
-
-// CHECK-LABEL: define dso_local <8 x i8> @test_vuqadd_s8(
-// CHECK-SAME: <8 x i8> noundef [[A:%.*]], <8 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VUQADD_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.suqadd.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
-// CHECK-NEXT:    ret <8 x i8> [[VUQADD_I]]
-//
-int8x8_t test_vuqadd_s8(int8x8_t a, uint8x8_t b) {
-  return vuqadd_s8(a, b);
-}
-
-// CHECK-LABEL: define dso_local <2 x i32> @test_vuqadd_s32(
-// CHECK-SAME: <2 x i32> noundef [[A:%.*]], <2 x i32> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[A]] to <8 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[B]] to <8 x i8>
-// CHECK-NEXT:    [[VUQADD_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
-// CHECK-NEXT:    [[VUQADD1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
-// CHECK-NEXT:    [[VUQADD2_I:%.*]] = call <2 x i32> @llvm.aarch64.neon.suqadd.v2i32(<2 x i32> [[VUQADD_I]], <2 x i32> [[VUQADD1_I]])
-// CHECK-NEXT:    ret <2 x i32> [[VUQADD2_I]]
-//
-int32x2_t test_vuqadd_s32(int32x2_t a, uint32x2_t b) {
-  return vuqadd_s32(a, b);
-}
-
-// CHECK-LABEL: define dso_local <1 x i64> @test_vuqadd_s64(
-// CHECK-SAME: <1 x i64> noundef [[A:%.*]], <1 x i64> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
-// CHECK-NEXT:    [[VUQADD_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
-// CHECK-NEXT:    [[VUQADD1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
-// CHECK-NEXT:    [[VUQADD2_I:%.*]] = call <1 x i64> @llvm.aarch64.neon.suqadd.v1i64(<1 x i64> [[VUQADD_I]], <1 x i64> [[VUQADD1_I]])
-// CHECK-NEXT:    ret <1 x i64> [[VUQADD2_I]]
-//
-int64x1_t test_vuqadd_s64(int64x1_t a, uint64x1_t b) {
-  return vuqadd_s64(a, b);
-}
-
-// CHECK-LABEL: define dso_local <4 x i16> @test_vuqadd_s16(
-// CHECK-SAME: <4 x i16> noundef [[A:%.*]], <4 x i16> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i16> [[A]] to <8 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[B]] to <8 x i8>
-// CHECK-NEXT:    [[VUQADD_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x i16>
-// CHECK-NEXT:    [[VUQADD1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x i16>
-// CHECK-NEXT:    [[VUQADD2_I:%.*]] = call <4 x i16> @llvm.aarch64.neon.suqadd.v4i16(<4 x i16> [[VUQADD_I]], <4 x i16> [[VUQADD1_I]])
-// CHECK-NEXT:    ret <4 x i16> [[VUQADD2_I]]
-//
-int16x4_t test_vuqadd_s16(int16x4_t a, uint16x4_t b) {
-  return vuqadd_s16(a, b);
-}
-
-// CHECK-LABEL: define dso_local <1 x i64> @test_vsqadd_u64(
-// CHECK-SAME: <1 x i64> noundef [[A:%.*]], <1 x i64> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
-// CHECK-NEXT:    [[VSQADD_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
-// CHECK-NEXT:    [[VSQADD1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
-// CHECK-NEXT:    [[VSQADD2_I:%.*]] = call <1 x i64> @llvm.aarch64.neon.usqadd.v1i64(<1 x i64> [[VSQADD_I]], <1 x i64> [[VSQADD1_I]])
-// CHECK-NEXT:    ret <1 x i64> [[VSQADD2_I]]
-//
-uint64x1_t test_vsqadd_u64(uint64x1_t a, int64x1_t b) {
-  return vsqadd_u64(a, b);
-}
-
-// CHECK-LABEL: define dso_local <8 x i8> @test_vsqadd_u8(
-// CHECK-SAME: <8 x i8> noundef [[A:%.*]], <8 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VSQADD_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.usqadd.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
-// CHECK-NEXT:    ret <8 x i8> [[VSQADD_I]]
-//
-uint8x8_t test_vsqadd_u8(uint8x8_t a, int8x8_t b) {
-  return vsqadd_u8(a, b);
-}
-
-// CHECK-LABEL: define dso_local <16 x i8> @test_vsqaddq_u8(
-// CHECK-SAME: <16 x i8> noundef [[A:%.*]], <16 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[VSQADD_I:%.*]] = call <16 x i8> @llvm.aarch64.neon.usqadd.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
-// CHECK-NEXT:    ret <16 x i8> [[VSQADD_I]]
-//
-uint8x16_t test_vsqaddq_u8(uint8x16_t a, int8x16_t b) {
-  return vsqaddq_u8(a, b);
-}
-
-// CHECK-LABEL: define dso_local <4 x i16> @test_vsqadd_u16(
-// CHECK-SAME: <4 x i16> noundef [[A:%.*]], <4 x i16> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i16> [[A]] to <8 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[B]] to <8 x i8>
-// CHECK-NEXT:    [[VSQADD_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x i16>
-// CHECK-NEXT:    [[VSQADD1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x i16>
-// CHECK-NEXT:    [[VSQADD2_I:%.*]] = call <4 x i16> @llvm.aarch64.neon.usqadd.v4i16(<4 x i16> [[VSQADD_I]], <4 x i16> [[VSQADD1_I]])
-// CHECK-NEXT:    ret <4 x i16> [[VSQADD2_I]]
-//
-uint16x4_t test_vsqadd_u16(uint16x4_t a, int16x4_t b) {
-  return vsqadd_u16(a, b);
-}
-
-// CHECK-LABEL: define dso_local <8 x i16> @test_vsqaddq_u16(
-// CHECK-SAME: <8 x i16> noundef [[A:%.*]], <8 x i16> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i16> [[A]] to <16 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[B]] to <16 x i8>
-// CHECK-NEXT:    [[VSQADD_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x i16>
-// CHECK-NEXT:    [[VSQADD1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x i16>
-// CHECK-NEXT:    [[VSQADD2_I:%.*]] = call <8 x i16> @llvm.aarch64.neon.usqadd.v8i16(<8 x i16> [[VSQADD_I]], <8 x i16> [[VSQADD1_I]])
-// CHECK-NEXT:    ret <8 x i16> [[VSQADD2_I]]
-//
-uint16x8_t test_vsqaddq_u16(uint16x8_t a, int16x8_t b) {
-  return vsqaddq_u16(a, b);
-}
-
-// CHECK-LABEL: define dso_local <2 x i32> @test_vsqadd_u32(
-// CHECK-SAME: <2 x i32> noundef [[A:%.*]], <2 x i32> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[A]] to <8 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[B]] to <8 x i8>
-// CHECK-NEXT:    [[VSQADD_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
-// CHECK-NEXT:    [[VSQADD1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
-// CHECK-NEXT:    [[VSQADD2_I:%.*]] = call <2 x i32> @llvm.aarch64.neon.usqadd.v2i32(<2 x i32> [[VSQADD_I]], <2 x i32> [[VSQADD1_I]])
-// CHECK-NEXT:    ret <2 x i32> [[VSQADD2_I]]
-//
-uint32x2_t test_vsqadd_u32(uint32x2_t a, int32x2_t b) {
-  return vsqadd_u32(a, b);
-}
-
-// CHECK-LABEL: define dso_local <4 x i32> @test_vsqaddq_u32(
-// CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[A]] to <16 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[B]] to <16 x i8>
-// CHECK-NEXT:    [[VSQADD_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
-// CHECK-NEXT:    [[VSQADD1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32>
-// CHECK-NEXT:    [[VSQADD2_I:%.*]] = call <4 x i32> @llvm.aarch64.neon.usqadd.v4i32(<4 x i32> [[VSQADD_I]], <4 x i32> [[VSQADD1_I]])
-// CHECK-NEXT:    ret <4 x i32> [[VSQADD2_I]]
-//
-uint32x4_t test_vsqaddq_u32(uint32x4_t a, int32x4_t b) {
-  return vsqaddq_u32(a, b);
-}
-
-// CHECK-LABEL: define dso_local <2 x i64> @test_vsqaddq_u64(
-// CHECK-SAME: <2 x i64> noundef [[A:%.*]], <2 x i64> noundef [[B:%.*]]) #[[ATTR0]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x i64> [[A]] to <16 x i8>
-// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[B]] to <16 x i8>
-// CHECK-NEXT:    [[VSQADD_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <2 x i64>
-// CHECK-NEXT:    [[VSQADD1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <2 x i64>
-// CHECK-NEXT:    [[VSQADD2_I:%.*]] = call <2 x i64> @llvm.aarch64.neon.usqadd.v2i64(<2 x i64> [[VSQADD_I]], <2 x i64> [[VSQADD1_I]])
-// CHECK-NEXT:    ret <2 x i64> [[VSQADD2_I]]
-//
-uint64x2_t test_vsqaddq_u64(uint64x2_t a, int64x2_t b) {
-  return vsqaddq_u64(a, b);
-}
-
 // CHECK-LABEL: define dso_local <1 x i64> @test_vabs_s64(
 // CHECK-SAME: <1 x i64> noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
diff --git a/clang/test/CodeGen/AArch64/neon/add.c b/clang/test/CodeGen/AArch64/neon/add.c
index d462d9bc60a6e..5508f4df7fdfc 100644
--- a/clang/test/CodeGen/AArch64/neon/add.c
+++ b/clang/test/CodeGen/AArch64/neon/add.c
@@ -1250,3 +1250,560 @@ uint32x4_t test_vraddhn_high_u64(uint32x2_t r, uint64x2_t a, uint64x2_t b) {
   // LLVM: ret <4 x i32> [[RES]]
   return vraddhn_high_u64(r, a, b);
 }
+
+//===------------------------------------------------------===//
+// 2.1.1.1.4. Saturating addition
+// https://arm-software.github.io/acle/neon_intrinsics/advsimd.html#saturating-addition
+//===------------------------------------------------------===//
+
+// LLVM-LABEL: @test_vqadd_s8(
+// CIR-LABEL: @vqadd_s8(
+int8x8_t test_vqadd_s8(int8x8_t a, int8x8_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.sqadd"
+
+// LLVM-SAME: <8 x i8> {{.*}}[[A:%.*]], <8 x i8> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <8 x i8> @llvm.aarch64.neon.sqadd.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
+// LLVM: ret <8 x i8> [[RES]]
+  return vqadd_s8(a, b);
+}
+
+// LLVM-LABEL: @test_vqadd_s16(
+// CIR-LABEL: @vqadd_s16(
+int16x4_t test_vqadd_s16(int16x4_t a, int16x4_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.sqadd"
+
+// LLVM-SAME: <4 x i16> {{.*}}[[A:%.*]], <4 x i16> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> [[A]], <4 x i16> [[B]])
+// LLVM: ret <4 x i16> [[RES]]
+  return vqadd_s16(a, b);
+}
+
+// LLVM-LABEL: @test_vqadd_s32(
+// CIR-LABEL: @vqadd_s32(
+int32x2_t test_vqadd_s32(int32x2_t a, int32x2_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.sqadd"
+
+// LLVM-SAME: <2 x i32> {{.*}}[[A:%.*]], <2 x i32> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <2 x i32> @llvm.aarch64.neon.sqadd.v2i32(<2 x i32> [[A]], <2 x i32> [[B]])
+// LLVM: ret <2 x i32> [[RES]]
+  return vqadd_s32(a, b);
+}
+
+// LLVM-LABEL: @test_vqadd_s64(
+// CIR-LABEL: @vqadd_s64(
+int64x1_t test_vqadd_s64(int64x1_t a, int64x1_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.sqadd"
+
+// LLVM-SAME: <1 x i64> {{.*}}[[A:%.*]], <1 x i64> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <1 x i64> @llvm.aarch64.neon.sqadd.v1i64(<1 x i64> [[A]], <1 x i64> [[B]])
+// LLVM: ret <1 x i64> [[RES]]
+  return vqadd_s64(a, b);
+}
+
+// LLVM-LABEL: @test_vqadd_u8(
+// CIR-LABEL: @vqadd_u8(
+uint8x8_t test_vqadd_u8(uint8x8_t a, uint8x8_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.uqadd"
+
+// LLVM-SAME: <8 x i8> {{.*}}[[A:%.*]], <8 x i8> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <8 x i8> @llvm.aarch64.neon.uqadd.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
+// LLVM: ret <8 x i8> [[RES]]
+  return vqadd_u8(a, b);
+}
+
+// LLVM-LABEL: @test_vqadd_u16(
+// CIR-LABEL: @vqadd_u16(
+uint16x4_t test_vqadd_u16(uint16x4_t a, uint16x4_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.uqadd"
+
+// LLVM-SAME: <4 x i16> {{.*}}[[A:%.*]], <4 x i16> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <4 x i16> @llvm.aarch64.neon.uqadd.v4i16(<4 x i16> [[A]], <4 x i16> [[B]])
+// LLVM: ret <4 x i16> [[RES]]
+  return vqadd_u16(a, b);
+}
+
+// LLVM-LABEL: @test_vqadd_u32(
+// CIR-LABEL: @vqadd_u32(
+uint32x2_t test_vqadd_u32(uint32x2_t a, uint32x2_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.uqadd"
+
+// LLVM-SAME: <2 x i32> {{.*}}[[A:%.*]], <2 x i32> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <2 x i32> @llvm.aarch64.neon.uqadd.v2i32(<2 x i32> [[A]], <2 x i32> [[B]])
+// LLVM: ret <2 x i32> [[RES]]
+  return vqadd_u32(a, b);
+}
+
+// LLVM-LABEL: @test_vqadd_u64(
+// CIR-LABEL: @vqadd_u64(
+uint64x1_t test_vqadd_u64(uint64x1_t a, uint64x1_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.uqadd"
+
+// LLVM-SAME: <1 x i64> {{.*}}[[A:%.*]], <1 x i64> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <1 x i64> @llvm.aarch64.neon.uqadd.v1i64(<1 x i64> [[A]], <1 x i64> [[B]])
+// LLVM: ret <1 x i64> [[RES]]
+  return vqadd_u64(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddq_s8(
+// CIR-LABEL: @vqaddq_s8(
+int8x16_t test_vqaddq_s8(int8x16_t a, int8x16_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.sqadd"
+
+// LLVM-SAME: <16 x i8> {{.*}}[[A:%.*]], <16 x i8> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <16 x i8> @llvm.aarch64.neon.sqadd.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
+// LLVM: ret <16 x i8> [[RES]]
+  return vqaddq_s8(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddq_s16(
+// CIR-LABEL: @vqaddq_s16(
+int16x8_t test_vqaddq_s16(int16x8_t a, int16x8_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.sqadd"
+
+// LLVM-SAME: <8 x i16> {{.*}}[[A:%.*]], <8 x i16> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <8 x i16> @llvm.aarch64.neon.sqadd.v8i16(<8 x i16> [[A]], <8 x i16> [[B]])
+// LLVM: ret <8 x i16> [[RES]]
+  return vqaddq_s16(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddq_s32(
+// CIR-LABEL: @vqaddq_s32(
+int32x4_t test_vqaddq_s32(int32x4_t a, int32x4_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.sqadd"
+
+// LLVM-SAME: <4 x i32> {{.*}}[[A:%.*]], <4 x i32> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> [[A]], <4 x i32> [[B]])
+// LLVM: ret <4 x i32> [[RES]]
+  return vqaddq_s32(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddq_s64(
+// CIR-LABEL: @vqaddq_s64(
+int64x2_t test_vqaddq_s64(int64x2_t a, int64x2_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.sqadd"
+
+// LLVM-SAME: <2 x i64> {{.*}}[[A:%.*]], <2 x i64> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> [[A]], <2 x i64> [[B]])
+// LLVM: ret <2 x i64> [[RES]]
+  return vqaddq_s64(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddq_u8(
+// CIR-LABEL: @vqaddq_u8(
+uint8x16_t test_vqaddq_u8(uint8x16_t a, uint8x16_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.uqadd"
+
+// LLVM-SAME: <16 x i8> {{.*}}[[A:%.*]], <16 x i8> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <16 x i8> @llvm.aarch64.neon.uqadd.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
+// LLVM: ret <16 x i8> [[RES]]
+  return vqaddq_u8(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddq_u16(
+// CIR-LABEL: @vqaddq_u16(
+uint16x8_t test_vqaddq_u16(uint16x8_t a, uint16x8_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.uqadd"
+
+// LLVM-SAME: <8 x i16> {{.*}}[[A:%.*]], <8 x i16> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <8 x i16> @llvm.aarch64.neon.uqadd.v8i16(<8 x i16> [[A]], <8 x i16> [[B]])
+// LLVM: ret <8 x i16> [[RES]]
+  return vqaddq_u16(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddq_u32(
+// CIR-LABEL: @vqaddq_u32(
+uint32x4_t test_vqaddq_u32(uint32x4_t a, uint32x4_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.uqadd"
+
+// LLVM-SAME: <4 x i32> {{.*}}[[A:%.*]], <4 x i32> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <4 x i32> @llvm.aarch64.neon.uqadd.v4i32(<4 x i32> [[A]], <4 x i32> [[B]])
+// LLVM: ret <4 x i32> [[RES]]
+  return vqaddq_u32(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddq_u64(
+// CIR-LABEL: @vqaddq_u64(
+uint64x2_t test_vqaddq_u64(uint64x2_t a, uint64x2_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.uqadd"
+
+// LLVM-SAME: <2 x i64> {{.*}}[[A:%.*]], <2 x i64> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <2 x i64> @llvm.aarch64.neon.uqadd.v2i64(<2 x i64> [[A]], <2 x i64> [[B]])
+// LLVM: ret <2 x i64> [[RES]]
+  return vqaddq_u64(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddb_s8(
+// CIR-LABEL: @vqaddb_s8(
+int8_t test_vqaddb_s8(int8_t a, int8_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.sqadd"
+
+// LLVM-SAME: i8 {{.*}}[[A:%.*]], i8 {{.*}}[[B:%.*]])
+// LLVM: [[V0:%.*]] = insertelement <8 x i8> poison, i8 [[A]], i64 0
+// LLVM: [[V1:%.*]] = insertelement <8 x i8> poison, i8 [[B]], i64 0
+// LLVM: [[RES:%.*]] = call <8 x i8> @llvm.aarch64.neon.sqadd.v8i8(<8 x i8> [[V0]], <8 x i8> [[V1]])
+// LLVM: [[EXT:%.*]] = extractelement <8 x i8> [[RES]], i64 0
+// LLVM: ret i8 [[EXT]]
+  return vqaddb_s8(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddh_s16(
+// CIR-LABEL: @vqaddh_s16(
+int16_t test_vqaddh_s16(int16_t a, int16_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.sqadd"
+
+// LLVM-SAME: i16 {{.*}}[[A:%.*]], i16 {{.*}}[[B:%.*]])
+// LLVM: [[V0:%.*]] = insertelement <4 x i16> poison, i16 [[A]], i64 0
+// LLVM: [[V1:%.*]] = insertelement <4 x i16> poison, i16 [[B]], i64 0
+// LLVM: [[RES:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> [[V0]], <4 x i16> [[V1]])
+// LLVM: [[EXT:%.*]] = extractelement <4 x i16> [[RES]], i64 0
+// LLVM: ret i16 [[EXT]]
+  return vqaddh_s16(a, b);
+}
+
+// LLVM-LABEL: @test_vqadds_s32(
+// CIR-LABEL: @vqadds_s32(
+int32_t test_vqadds_s32(int32_t a, int32_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.sqadd"
+
+// LLVM-SAME: i32 {{.*}}[[A:%.*]], i32 {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call i32 @llvm.aarch64.neon.sqadd.i32(i32 [[A]], i32 [[B]])
+// LLVM: ret i32 [[RES]]
+  return vqadds_s32(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddd_s64(
+// CIR-LABEL: @vqaddd_s64(
+int64_t test_vqaddd_s64(int64_t a, int64_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.sqadd"
+
+// LLVM-SAME: i64 {{.*}}[[A:%.*]], i64 {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call i64 @llvm.aarch64.neon.sqadd.i64(i64 [[A]], i64 [[B]])
+// LLVM: ret i64 [[RES]]
+  return vqaddd_s64(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddb_u8(
+// CIR-LABEL: @vqaddb_u8(
+uint8_t test_vqaddb_u8(uint8_t a, uint8_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.uqadd"
+
+// LLVM-SAME: i8 {{.*}}[[A:%.*]], i8 {{.*}}[[B:%.*]])
+// LLVM: [[V0:%.*]] = insertelement <8 x i8> poison, i8 [[A]], i64 0
+// LLVM: [[V1:%.*]] = insertelement <8 x i8> poison, i8 [[B]], i64 0
+// LLVM: [[RES:%.*]] = call <8 x i8> @llvm.aarch64.neon.uqadd.v8i8(<8 x i8> [[V0]], <8 x i8> [[V1]])
+// LLVM: [[EXT:%.*]] = extractelement <8 x i8> [[RES]], i64 0
+// LLVM: ret i8 [[EXT]]
+  return vqaddb_u8(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddh_u16(
+// CIR-LABEL: @vqaddh_u16(
+uint16_t test_vqaddh_u16(uint16_t a, uint16_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.uqadd"
+
+// LLVM-SAME: i16 {{.*}}[[A:%.*]], i16 {{.*}}[[B:%.*]])
+// LLVM: [[V0:%.*]] = insertelement <4 x i16> poison, i16 [[A]], i64 0
+// LLVM: [[V1:%.*]] = insertelement <4 x i16> poison, i16 [[B]], i64 0
+// LLVM: [[RES:%.*]] = call <4 x i16> @llvm.aarch64.neon.uqadd.v4i16(<4 x i16> [[V0]], <4 x i16> [[V1]])
+// LLVM: [[EXT:%.*]] = extractelement <4 x i16> [[RES]], i64 0
+// LLVM: ret i16 [[EXT]]
+  return vqaddh_u16(a, b);
+}
+
+// LLVM-LABEL: @test_vqadds_u32(
+// CIR-LABEL: @vqadds_u32(
+uint32_t test_vqadds_u32(uint32_t a, uint32_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.uqadd"
+
+// LLVM-SAME: i32 {{.*}}[[A:%.*]], i32 {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call i32 @llvm.aarch64.neon.uqadd.i32(i32 [[A]], i32 [[B]])
+// LLVM: ret i32 [[RES]]
+  return vqadds_u32(a, b);
+}
+
+// LLVM-LABEL: @test_vqaddd_u64(
+// CIR-LABEL: @vqaddd_u64(
+uint64_t test_vqaddd_u64(uint64_t a, uint64_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.uqadd"
+
+// LLVM-SAME: i64 {{.*}}[[A:%.*]], i64 {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call i64 @llvm.aarch64.neon.uqadd.i64(i64 [[A]], i64 [[B]])
+// LLVM: ret i64 [[RES]]
+  return vqaddd_u64(a, b);
+}
+
+// LLVM-LABEL: @test_vsqadd_u8(
+// CIR-LABEL: @vsqadd_u8(
+uint8x8_t test_vsqadd_u8(uint8x8_t a, int8x8_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.usqadd"
+
+// LLVM-SAME: <8 x i8> {{.*}}[[A:%.*]], <8 x i8> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <8 x i8> @llvm.aarch64.neon.usqadd.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
+// LLVM: ret <8 x i8> [[RES]]
+  return vsqadd_u8(a, b);
+}
+
+// LLVM-LABEL: @test_vsqadd_u16(
+// CIR-LABEL: @vsqadd_u16(
+uint16x4_t test_vsqadd_u16(uint16x4_t a, int16x4_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.usqadd"
+
+// LLVM-SAME: <4 x i16> {{.*}}[[A:%.*]], <4 x i16> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <4 x i16> @llvm.aarch64.neon.usqadd.v4i16(<4 x i16> [[A]], <4 x i16> [[B]])
+// LLVM: ret <4 x i16> [[RES]]
+  return vsqadd_u16(a, b);
+}
+
+// LLVM-LABEL: @test_vsqadd_u32(
+// CIR-LABEL: @vsqadd_u32(
+uint32x2_t test_vsqadd_u32(uint32x2_t a, int32x2_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.usqadd"
+
+// LLVM-SAME: <2 x i32> {{.*}}[[A:%.*]], <2 x i32> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <2 x i32> @llvm.aarch64.neon.usqadd.v2i32(<2 x i32> [[A]], <2 x i32> [[B]])
+// LLVM: ret <2 x i32> [[RES]]
+  return vsqadd_u32(a, b);
+}
+
+// LLVM-LABEL: @test_vsqadd_u64(
+// CIR-LABEL: @vsqadd_u64(
+uint64x1_t test_vsqadd_u64(uint64x1_t a, int64x1_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.usqadd"
+
+// LLVM-SAME: <1 x i64> {{.*}}[[A:%.*]], <1 x i64> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <1 x i64> @llvm.aarch64.neon.usqadd.v1i64(<1 x i64> [[A]], <1 x i64> [[B]])
+// LLVM: ret <1 x i64> [[RES]]
+  return vsqadd_u64(a, b);
+}
+
+// LLVM-LABEL: @test_vsqaddq_u8(
+// CIR-LABEL: @vsqaddq_u8(
+uint8x16_t test_vsqaddq_u8(uint8x16_t a, int8x16_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.usqadd"
+
+// LLVM-SAME: <16 x i8> {{.*}}[[A:%.*]], <16 x i8> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <16 x i8> @llvm.aarch64.neon.usqadd.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
+// LLVM: ret <16 x i8> [[RES]]
+  return vsqaddq_u8(a, b);
+}
+
+// LLVM-LABEL: @test_vsqaddq_u16(
+// CIR-LABEL: @vsqaddq_u16(
+uint16x8_t test_vsqaddq_u16(uint16x8_t a, int16x8_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.usqadd"
+
+// LLVM-SAME: <8 x i16> {{.*}}[[A:%.*]], <8 x i16> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <8 x i16> @llvm.aarch64.neon.usqadd.v8i16(<8 x i16> [[A]], <8 x i16> [[B]])
+// LLVM: ret <8 x i16> [[RES]]
+  return vsqaddq_u16(a, b);
+}
+
+// LLVM-LABEL: @test_vsqaddq_u32(
+// CIR-LABEL: @vsqaddq_u32(
+uint32x4_t test_vsqaddq_u32(uint32x4_t a, int32x4_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.usqadd"
+
+// LLVM-SAME: <4 x i32> {{.*}}[[A:%.*]], <4 x i32> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <4 x i32> @llvm.aarch64.neon.usqadd.v4i32(<4 x i32> [[A]], <4 x i32> [[B]])
+// LLVM: ret <4 x i32> [[RES]]
+  return vsqaddq_u32(a, b);
+}
+
+// LLVM-LABEL: @test_vsqaddq_u64(
+// CIR-LABEL: @vsqaddq_u64(
+uint64x2_t test_vsqaddq_u64(uint64x2_t a, int64x2_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.usqadd"
+
+// LLVM-SAME: <2 x i64> {{.*}}[[A:%.*]], <2 x i64> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <2 x i64> @llvm.aarch64.neon.usqadd.v2i64(<2 x i64> [[A]], <2 x i64> [[B]])
+// LLVM: ret <2 x i64> [[RES]]
+  return vsqaddq_u64(a, b);
+}
+
+// LLVM-LABEL: @test_vsqaddb_u8(
+// CIR-LABEL: @vsqaddb_u8(
+uint8_t test_vsqaddb_u8(uint8_t a, int8_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.usqadd"
+
+// LLVM-SAME: i8 {{.*}}[[A:%.*]], i8 {{.*}}[[B:%.*]])
+// LLVM: [[V0:%.*]] = insertelement <8 x i8> poison, i8 [[A]], i64 0
+// LLVM: [[V1:%.*]] = insertelement <8 x i8> poison, i8 [[B]], i64 0
+// LLVM: [[RES:%.*]] = call <8 x i8> @llvm.aarch64.neon.usqadd.v8i8(<8 x i8> [[V0]], <8 x i8> [[V1]])
+// LLVM: [[EXT:%.*]] = extractelement <8 x i8> [[RES]], i64 0
+// LLVM: ret i8 [[EXT]]
+  return (uint8_t)vsqaddb_u8(a, b);
+}
+
+// LLVM-LABEL: @test_vsqaddh_u16(
+// CIR-LABEL: @vsqaddh_u16(
+uint16_t test_vsqaddh_u16(uint16_t a, int16_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.usqadd"
+
+// LLVM-SAME: i16 {{.*}}[[A:%.*]], i16 {{.*}}[[B:%.*]])
+// LLVM: [[V0:%.*]] = insertelement <4 x i16> poison, i16 [[A]], i64 0
+// LLVM: [[V1:%.*]] = insertelement <4 x i16> poison, i16 [[B]], i64 0
+// LLVM: [[RES:%.*]] = call <4 x i16> @llvm.aarch64.neon.usqadd.v4i16(<4 x i16> [[V0]], <4 x i16> [[V1]])
+// LLVM: [[EXT:%.*]] = extractelement <4 x i16> [[RES]], i64 0
+// LLVM: ret i16 [[EXT]]
+  return (uint16_t)vsqaddh_u16(a, b);
+}
+
+// LLVM-LABEL: @test_vsqadds_u32(
+// CIR-LABEL: @vsqadds_u32(
+uint32_t test_vsqadds_u32(uint32_t a, int32_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.usqadd"
+
+// LLVM-SAME: i32 {{.*}}[[A:%.*]], i32 {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call i32 @llvm.aarch64.neon.usqadd.i32(i32 [[A]], i32 [[B]])
+// LLVM: ret i32 [[RES]]
+  return (uint32_t)vsqadds_u32(a, b);
+}
+
+// LLVM-LABEL: @test_vsqaddd_u64(
+// CIR-LABEL: @vsqaddd_u64(
+uint64_t test_vsqaddd_u64(uint64_t a, int64_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.usqadd"
+
+// LLVM-SAME: i64 {{.*}}[[A:%.*]], i64 {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call i64 @llvm.aarch64.neon.usqadd.i64(i64 [[A]], i64 [[B]])
+// LLVM: ret i64 [[RES]]
+  return (uint64_t)vsqaddd_u64(a, b);
+}
+
+// LLVM-LABEL: @test_vuqadd_s8(
+// CIR-LABEL: @vuqadd_s8(
+int8x8_t test_vuqadd_s8(int8x8_t a, uint8x8_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.suqadd"
+
+// LLVM-SAME: <8 x i8> {{.*}}[[A:%.*]], <8 x i8> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <8 x i8> @llvm.aarch64.neon.suqadd.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
+// LLVM: ret <8 x i8> [[RES]]
+  return vuqadd_s8(a, b);
+}
+
+// LLVM-LABEL: @test_vuqadd_s16(
+// CIR-LABEL: @vuqadd_s16(
+int16x4_t test_vuqadd_s16(int16x4_t a, uint16x4_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.suqadd"
+
+// LLVM-SAME: <4 x i16> {{.*}}[[A:%.*]], <4 x i16> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <4 x i16> @llvm.aarch64.neon.suqadd.v4i16(<4 x i16> [[A]], <4 x i16> [[B]])
+// LLVM: ret <4 x i16> [[RES]]
+  return vuqadd_s16(a, b);
+}
+
+// LLVM-LABEL: @test_vuqadd_s32(
+// CIR-LABEL: @vuqadd_s32(
+int32x2_t test_vuqadd_s32(int32x2_t a, uint32x2_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.suqadd"
+
+// LLVM-SAME: <2 x i32> {{.*}}[[A:%.*]], <2 x i32> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <2 x i32> @llvm.aarch64.neon.suqadd.v2i32(<2 x i32> [[A]], <2 x i32> [[B]])
+// LLVM: ret <2 x i32> [[RES]]
+  return vuqadd_s32(a, b);
+}
+
+// LLVM-LABEL: @test_vuqadd_s64(
+// CIR-LABEL: @vuqadd_s64(
+int64x1_t test_vuqadd_s64(int64x1_t a, uint64x1_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.suqadd"
+
+// LLVM-SAME: <1 x i64> {{.*}}[[A:%.*]], <1 x i64> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <1 x i64> @llvm.aarch64.neon.suqadd.v1i64(<1 x i64> [[A]], <1 x i64> [[B]])
+// LLVM: ret <1 x i64> [[RES]]
+  return vuqadd_s64(a, b);
+}
+
+// LLVM-LABEL: @test_vuqaddq_s8(
+// CIR-LABEL: @vuqaddq_s8(
+int8x16_t test_vuqaddq_s8(int8x16_t a, uint8x16_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.suqadd"
+
+// LLVM-SAME: <16 x i8> {{.*}}[[A:%.*]], <16 x i8> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <16 x i8> @llvm.aarch64.neon.suqadd.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
+// LLVM: ret <16 x i8> [[RES]]
+  return vuqaddq_s8(a, b);
+}
+
+// LLVM-LABEL: @test_vuqaddq_s16(
+// CIR-LABEL: @vuqaddq_s16(
+int16x8_t test_vuqaddq_s16(int16x8_t a, uint16x8_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.suqadd"
+
+// LLVM-SAME: <8 x i16> {{.*}}[[A:%.*]], <8 x i16> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <8 x i16> @llvm.aarch64.neon.suqadd.v8i16(<8 x i16> [[A]], <8 x i16> [[B]])
+// LLVM: ret <8 x i16> [[RES]]
+  return vuqaddq_s16(a, b);
+}
+
+// LLVM-LABEL: @test_vuqaddq_s32(
+// CIR-LABEL: @vuqaddq_s32(
+int32x4_t test_vuqaddq_s32(int32x4_t a, uint32x4_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.suqadd"
+
+// LLVM-SAME: <4 x i32> {{.*}}[[A:%.*]], <4 x i32> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <4 x i32> @llvm.aarch64.neon.suqadd.v4i32(<4 x i32> [[A]], <4 x i32> [[B]])
+// LLVM: ret <4 x i32> [[RES]]
+  return vuqaddq_s32(a, b);
+}
+
+// LLVM-LABEL: @test_vuqaddq_s64(
+// CIR-LABEL: @vuqaddq_s64(
+int64x2_t test_vuqaddq_s64(int64x2_t a, uint64x2_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.suqadd"
+
+// LLVM-SAME: <2 x i64> {{.*}}[[A:%.*]], <2 x i64> {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call <2 x i64> @llvm.aarch64.neon.suqadd.v2i64(<2 x i64> [[A]], <2 x i64> [[B]])
+// LLVM: ret <2 x i64> [[RES]]
+  return vuqaddq_s64(a, b);
+}
+
+// LLVM-LABEL: @test_vuqaddb_s8(
+// CIR-LABEL: @vuqaddb_s8(
+int8_t test_vuqaddb_s8(int8_t a, uint8_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.suqadd"
+
+// LLVM-SAME: i8 {{.*}}[[A:%.*]], i8 {{.*}}[[B:%.*]])
+// LLVM: [[V0:%.*]] = insertelement <8 x i8> poison, i8 [[A]], i64 0
+// LLVM: [[V1:%.*]] = insertelement <8 x i8> poison, i8 [[B]], i64 0
+// LLVM: [[RES:%.*]] = call <8 x i8> @llvm.aarch64.neon.suqadd.v8i8(<8 x i8> [[V0]], <8 x i8> [[V1]])
+// LLVM: [[EXT:%.*]] = extractelement <8 x i8> [[RES]], i64 0
+// LLVM: ret i8 [[EXT]]
+  return (int8_t)vuqaddb_s8(a, b);
+}
+
+// LLVM-LABEL: @test_vuqaddh_s16(
+// CIR-LABEL: @vuqaddh_s16(
+int16_t test_vuqaddh_s16(int16_t a, uint16_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.suqadd"
+
+// LLVM-SAME: i16 {{.*}}[[A:%.*]], i16 {{.*}}[[B:%.*]])
+// LLVM: [[V0:%.*]] = insertelement <4 x i16> poison, i16 [[A]], i64 0
+// LLVM: [[V1:%.*]] = insertelement <4 x i16> poison, i16 [[B]], i64 0
+// LLVM: [[RES:%.*]] = call <4 x i16> @llvm.aarch64.neon.suqadd.v4i16(<4 x i16> [[V0]], <4 x i16> [[V1]])
+// LLVM: [[EXT:%.*]] = extractelement <4 x i16> [[RES]], i64 0
+// LLVM: ret i16 [[EXT]]
+  return (int16_t)vuqaddh_s16(a, b);
+}
+
+// LLVM-LABEL: @test_vuqadds_s32(
+// CIR-LABEL: @vuqadds_s32(
+int32_t test_vuqadds_s32(int32_t a, uint32_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.suqadd"
+
+// LLVM-SAME: i32 {{.*}}[[A:%.*]], i32 {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call i32 @llvm.aarch64.neon.suqadd.i32(i32 [[A]], i32 [[B]])
+// LLVM: ret i32 [[RES]]
+  return (int32_t)vuqadds_s32(a, b);
+}
+
+// LLVM-LABEL: @test_vuqaddd_s64(
+// CIR-LABEL: @vuqaddd_s64(
+int64_t test_vuqaddd_s64(int64_t a, uint64_t b) {
+// CIR: cir.call_llvm_intrinsic "aarch64.neon.suqadd"
+
+// LLVM-SAME: i64 {{.*}}[[A:%.*]], i64 {{.*}}[[B:%.*]])
+// LLVM: [[RES:%.*]] = call i64 @llvm.aarch64.neon.suqadd.i64(i64 [[A]], i64 [[B]])
+// LLVM: ret i64 [[RES]]
+  return (int64_t)vuqaddd_s64(a, b);
+}
diff --git a/clang/test/CodeGenHLSL/BasicFeatures/InitLists.hlsl b/clang/test/CodeGenHLSL/BasicFeatures/InitLists.hlsl
index 1fd8a3fd51f1f..1e0dcd78f51da 100644
--- a/clang/test/CodeGenHLSL/BasicFeatures/InitLists.hlsl
+++ b/clang/test/CodeGenHLSL/BasicFeatures/InitLists.hlsl
@@ -1231,7 +1231,7 @@ void case26(TwoInts TI) {
   float3 F2 = float3(3, TI);
 }
 
-using handle_float_t = __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]];
+using handle_float_t = __hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(float)]];
 
 struct CustomResource {
   handle_float_t h;
diff --git a/clang/test/CodeGenHLSL/builtins/hlsl_resource_t.hlsl b/clang/test/CodeGenHLSL/builtins/hlsl_resource_t.hlsl
index 7096322c96cdc..e2a7a60ac13b5 100644
--- a/clang/test/CodeGenHLSL/builtins/hlsl_resource_t.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/hlsl_resource_t.hlsl
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -finclude-default-header -x hlsl -emit-llvm -o - %s | FileCheck %s
 
-using handle_float_t = __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]];
+using handle_float_t = __hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(float)]];
 
 struct CustomResource {
   handle_float_t h;
diff --git a/clang/test/Modules/decl-attr-merge2.c b/clang/test/Modules/decl-attr-merge2.c
index fc84b9df70171..6c23fb08f0c9a 100644
--- a/clang/test/Modules/decl-attr-merge2.c
+++ b/clang/test/Modules/decl-attr-merge2.c
@@ -2,7 +2,7 @@
 // RUN: split-file %s %t
 // RUN: %clang_cc1 -fmodules -fimplicit-module-maps \
 // RUN:   -fmodules-cache-path=%t/mcache -triple arm64-apple-macosx10.7.0 \
-// RUN:   -I%t/headers -fsyntax-only %t/test.c -verify
+// RUN:   -I%t/headers -fsyntax-only %t/test.cpp -verify
 
 // Check more cases of attribute merging across multiple modules.
 
@@ -17,11 +17,25 @@ module Second {
 void additiveAttr(void) __attribute__((availability(macos,unavailable)));
 void exclusiveAttr(void) __attribute__((hot));
 
+namespace N {
+inline namespace with_tag __attribute__((__abi_tag__("a"))) {
+  struct First {};
+}
+inline namespace with_tag {
+}
+}
+
 //--- headers/second.h
 void additiveAttr(void) __attribute__((availability(ios,introduced=4.0)));
 void exclusiveAttr(void) __attribute__((cold));
 
-//--- test.c
+namespace N {
+inline namespace with_tag {
+  struct Second {};
+}
+}
+
+//--- test.cpp
 #include <first.h>
 #include <second.h>
 
@@ -35,4 +49,6 @@ void test(void) {
   exclusiveAttr();
   // expected-error@second.h:* {{'cold' and 'hot' attributes are not compatible}}
   // expected-note@first.h:* {{conflicting attribute is here}}
+
+  N::Second second;
 }
diff --git a/clang/test/ParserHLSL/hlsl_contained_type_attr.hlsl b/clang/test/ParserHLSL/hlsl_contained_type_attr.hlsl
index 5f3ff03e96cb5..cf7f9510f1abb 100644
--- a/clang/test/ParserHLSL/hlsl_contained_type_attr.hlsl
+++ b/clang/test/ParserHLSL/hlsl_contained_type_attr.hlsl
@@ -2,24 +2,24 @@
 
 typedef vector<float, 4> float4;
 
-// CHECK: -TypeAliasDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:1, col:83>
+// CHECK: -TypeAliasDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:1, col:85>
 // CHECK: -HLSLAttributedResourceType 0x{{[0-9a-f]+}} '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(int)]]
-using ResourceIntAliasT = __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(int)]];
+using ResourceIntAliasT = __hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(int)]];
 ResourceIntAliasT h1;
 
-// CHECK: -VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:1, col:82> col:82 h2 '__hlsl_resource_t 
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK: -VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:1, col:84> col:84 h2 '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(float4)]]
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float4)]] h2;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(float4)]] h2;
 
 // CHECK: ClassTemplateDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 6]]:1, line:[[# @LINE + 8]]:1> line:[[# @LINE + 6]]:30 S
 // CHECK: TemplateTypeParmDecl 0x{{[0-9a-f]+}} <col:11, col:20> col:20 referenced typename depth 0 index 0 T
 // CHECK: CXXRecordDecl 0x{{[0-9a-f]+}} <col:23, line:[[# @LINE + 6]]:1> line:[[# @LINE + 4]]:30 struct S definition
-// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:3, col:79> col:79 h '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:3, col:81> col:81 h '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(T)]]
 template <typename T> struct S {
-  __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(T)]] h;
+  __hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(T)]] h;
 };
diff --git a/clang/test/ParserHLSL/hlsl_is_array_attr.hlsl b/clang/test/ParserHLSL/hlsl_is_array_attr.hlsl
index e561cba62edd9..5293cbf3ac6e8 100644
--- a/clang/test/ParserHLSL/hlsl_is_array_attr.hlsl
+++ b/clang/test/ParserHLSL/hlsl_is_array_attr.hlsl
@@ -1,22 +1,22 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -o - %s | FileCheck %s
 
 // CHECK: CXXRecordDecl 0x{{[0-9a-f]+}} {{.*}} struct MyBuffer definition
-// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:3, col:70> col:70 h '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:3, col:72> col:72 h '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::is_array]]
 struct MyBuffer {
-  __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_array]] h;
+  __hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::is_array]] h;
 };
 
-// CHECK: VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:1, col:68> col:68 res '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK: VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:1, col:70> col:70 res '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::is_array]]
-__hlsl_resource_t [[hlsl::is_array]] [[hlsl::resource_class(SRV)]] res;
+__hlsl_resource_t [[hlsl::is_array]] [[hlsl::resource_class("SRV")]] res;
 
 // CHECK: FunctionDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:1, line:[[# @LINE + 6]]:1> line:[[# @LINE + 4]]:6 f 'void ()
-// CHECK: VarDecl 0x{{[0-9a-f]+}} <col:3, col:74> col:74 r '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK: VarDecl 0x{{[0-9a-f]+}} <col:3, col:76> col:76 r '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME{LITERAL}: [[hlsl::is_array]]
 void f() {
-  __hlsl_resource_t [[hlsl::resource_class(Sampler)]] [[hlsl::is_array]] r;
+  __hlsl_resource_t [[hlsl::resource_class("Sampler")]] [[hlsl::is_array]] r;
 }
diff --git a/clang/test/ParserHLSL/hlsl_is_ms_attr.hlsl b/clang/test/ParserHLSL/hlsl_is_ms_attr.hlsl
index f99913d0d792b..60c4226216118 100644
--- a/clang/test/ParserHLSL/hlsl_is_ms_attr.hlsl
+++ b/clang/test/ParserHLSL/hlsl_is_ms_attr.hlsl
@@ -1,22 +1,22 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -o - %s | FileCheck %s
 
 // CHECK: CXXRecordDecl 0x{{[0-9a-f]+}} {{.*}} struct MyBuffer definition
-// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:3, col:67> col:67 h '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:3, col:69> col:69 h '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::is_ms]]
 struct MyBuffer {
-  __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_ms]] h;
+  __hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::is_ms]] h;
 };
 
-// CHECK: VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:1, col:65> col:65 res '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK: VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:1, col:67> col:67 res '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::is_ms]]
-__hlsl_resource_t [[hlsl::is_ms]] [[hlsl::resource_class(SRV)]] res;
+__hlsl_resource_t [[hlsl::is_ms]] [[hlsl::resource_class("SRV")]] res;
 
 // CHECK: FunctionDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:1, line:[[# @LINE + 6]]:1> line:[[# @LINE + 4]]:6 f 'void ()
-// CHECK: VarDecl 0x{{[0-9a-f]+}} <col:3, col:71> col:71 r '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK: VarDecl 0x{{[0-9a-f]+}} <col:3, col:73> col:73 r '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME{LITERAL}: [[hlsl::is_ms]]
 void f() {
-  __hlsl_resource_t [[hlsl::resource_class(Sampler)]] [[hlsl::is_ms]] r;
+  __hlsl_resource_t [[hlsl::resource_class("Sampler")]] [[hlsl::is_ms]] r;
 }
diff --git a/clang/test/ParserHLSL/hlsl_is_rov_attr.hlsl b/clang/test/ParserHLSL/hlsl_is_rov_attr.hlsl
index f7c3230ede061..49ac38b45a935 100644
--- a/clang/test/ParserHLSL/hlsl_is_rov_attr.hlsl
+++ b/clang/test/ParserHLSL/hlsl_is_rov_attr.hlsl
@@ -1,22 +1,22 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -o - %s | FileCheck %s
 
 // CHECK: CXXRecordDecl 0x{{[0-9a-f]+}} {{.*}} struct MyBuffer definition
-// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:3, col:68> col:68 h '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:3, col:70> col:70 h '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::is_rov]]
 struct MyBuffer {
-  __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_rov]] h;
+  __hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::is_rov]] h;
 };
 
-// CHECK: VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:1, col:66> col:66 res '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK: VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:1, col:68> col:68 res '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::is_rov]]
-__hlsl_resource_t [[hlsl::is_rov]] [[hlsl::resource_class(SRV)]] res;
+__hlsl_resource_t [[hlsl::is_rov]] [[hlsl::resource_class("SRV")]] res;
 
 // CHECK: FunctionDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:1, line:[[# @LINE + 6]]:1> line:[[# @LINE + 4]]:6 f 'void ()
-// CHECK: VarDecl 0x{{[0-9a-f]+}} <col:3, col:72> col:72 r '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK: VarDecl 0x{{[0-9a-f]+}} <col:3, col:74> col:74 r '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 // CHECK-SAME{LITERAL}: [[hlsl::is_rov]]
 void f() {
-  __hlsl_resource_t [[hlsl::resource_class(Sampler)]] [[hlsl::is_rov]] r;
+  __hlsl_resource_t [[hlsl::resource_class("Sampler")]] [[hlsl::is_rov]] r;
 }
diff --git a/clang/test/ParserHLSL/hlsl_raw_buffer_attr.hlsl b/clang/test/ParserHLSL/hlsl_raw_buffer_attr.hlsl
index 4f7ae455adc62..60bd9ad555728 100644
--- a/clang/test/ParserHLSL/hlsl_raw_buffer_attr.hlsl
+++ b/clang/test/ParserHLSL/hlsl_raw_buffer_attr.hlsl
@@ -1,22 +1,22 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -o - %s | FileCheck %s
 
 // CHECK: CXXRecordDecl 0x{{[0-9a-f]+}} {{.*}} struct MyBuffer definition
-// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:3, col:72> col:72 h1 '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:3, col:74> col:74 h1 '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
 struct MyBuffer {
-  __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer]] h1;
+  __hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::raw_buffer]] h1;
 };
 
-// CHECK: VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:1, col:70> col:70 h2 '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
+// CHECK: VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:1, col:72> col:72 h2 '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
-__hlsl_resource_t [[hlsl::raw_buffer]] [[hlsl::resource_class(SRV)]] h2;
+__hlsl_resource_t [[hlsl::raw_buffer]] [[hlsl::resource_class("SRV")]] h2;
 
 // CHECK: FunctionDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 4]]:1, line:[[# @LINE + 6]]:1> line:[[# @LINE + 4]]:6 f 'void ()
-// CHECK: VarDecl 0x{{[0-9a-f]+}} <col:3, col:72> col:72 h3 '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK: VarDecl 0x{{[0-9a-f]+}} <col:3, col:74> col:74 h3 '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
 void f() {
-  __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer]] h3;
+  __hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::raw_buffer]] h3;
 }
diff --git a/clang/test/ParserHLSL/hlsl_resource_class_attr.hlsl b/clang/test/ParserHLSL/hlsl_resource_class_attr.hlsl
index dad726b9a5b7c..13236a9e5cdcc 100644
--- a/clang/test/ParserHLSL/hlsl_resource_class_attr.hlsl
+++ b/clang/test/ParserHLSL/hlsl_resource_class_attr.hlsl
@@ -1,37 +1,37 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -o - %s | FileCheck %s
 
 // CHECK: CXXRecordDecl 0x{{[0-9a-f]+}} {{.*}} struct MyBuffer definition
-// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:3, col:51> col:51 h '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:3, col:53> col:53 h '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 struct MyBuffer {
-  __hlsl_resource_t [[hlsl::resource_class(UAV)]] h;
+  __hlsl_resource_t [[hlsl::resource_class("UAV")]] h;
 };
 
-// CHECK: VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 2]]:1, col:49> col:49 res '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
-__hlsl_resource_t [[hlsl::resource_class(SRV)]] res;
+// CHECK: VarDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 2]]:1, col:51> col:51 res '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]]
+__hlsl_resource_t [[hlsl::resource_class("SRV")]] res;
 
 // CHECK: FunctionDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:1, line:[[# @LINE + 5]]:1> line:[[# @LINE + 3]]:6 f 'void ()
-// CHECK: VarDecl 0x{{[0-9a-f]+}} <col:3, col:55> col:55 r '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(Sampler)]]
+// CHECK: VarDecl 0x{{[0-9a-f]+}} <col:3, col:57> col:57 r '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("Sampler")]]
 void f() {
-  __hlsl_resource_t [[hlsl::resource_class(Sampler)]] r;
+  __hlsl_resource_t [[hlsl::resource_class("Sampler")]] r;
 }
 
 // CHECK: ClassTemplateDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 6]]:1, line:[[# @LINE + 8]]:1> line:[[# @LINE + 6]]:29 referenced MyBuffer2
 // CHECK: TemplateTypeParmDecl 0x{{[0-9a-f]+}} <col:10, col:19> col:19 typename depth 0 index 0 T
 // CHECK: CXXRecordDecl 0x{{[0-9a-f]+}} <col:22, line:[[# @LINE + 6]]:1> line:[[# @LINE + 4]]:29 struct MyBuffer2 definition
 // CHECK: CXXRecordDecl 0x{{[0-9a-f]+}} <col:22, col:29> col:29 implicit struct MyBuffer2
-// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:3, col:51> col:51 h '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE + 3]]:3, col:53> col:53 h '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 template<typename T> struct MyBuffer2 {
-  __hlsl_resource_t [[hlsl::resource_class(UAV)]] h;
+  __hlsl_resource_t [[hlsl::resource_class("UAV")]] h;
 };
 
 // CHECK: ClassTemplateSpecializationDecl 0x{{[0-9a-f]+}} <line:[[# @LINE - 4]]:1, line:[[# @LINE - 2]]:1> line:[[# @LINE - 4]]:29 referenced struct MyBuffer2 definition external-linkage instantiated_from 0x{{.+}} implicit_instantiation
 // CHECK: TemplateArgument type 'float'
 // CHECK: BuiltinType 0x{{[0-9a-f]+}} 'float'
 // CHECK: CXXRecordDecl 0x{{[0-9a-f]+}} <col:22, col:29> col:29 implicit struct MyBuffer2
-// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE - 7]]:3, col:51> col:51 h '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK: FieldDecl 0x{{[0-9a-f]+}} <line:[[# @LINE - 7]]:3, col:53> col:53 h '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 MyBuffer2<float> myBuffer2;
diff --git a/clang/test/ParserHLSL/hlsl_resource_dimension_attr.hlsl b/clang/test/ParserHLSL/hlsl_resource_dimension_attr.hlsl
index c8b0f967ab03e..efec81bec771e 100644
--- a/clang/test/ParserHLSL/hlsl_resource_dimension_attr.hlsl
+++ b/clang/test/ParserHLSL/hlsl_resource_dimension_attr.hlsl
@@ -1,17 +1,17 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump -o - %s | FileCheck %s
 
-// CHECK: VarDecl {{.*}} res1D '__hlsl_resource_t 
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]] [[hlsl::resource_dimension(1D)]]
-__hlsl_resource_t [[hlsl::resource_class(SRV)]] [[hlsl::dimension("1D")]] res1D;
+// CHECK: VarDecl {{.*}} res1D '__hlsl_resource_t
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]] [[hlsl::dimension("1D")]]
+__hlsl_resource_t [[hlsl::resource_class("SRV")]] [[hlsl::dimension("1D")]] res1D;
 
 // CHECK: VarDecl 0x{{[0-9a-f]+}} {{.*}} res2D '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]] [[hlsl::resource_dimension(2D)]]
-__hlsl_resource_t [[hlsl::resource_class(SRV)]] [[hlsl::dimension("2D")]] res2D;
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]] [[hlsl::dimension("2D")]]
+__hlsl_resource_t [[hlsl::resource_class("SRV")]] [[hlsl::dimension("2D")]] res2D;
 
 // CHECK: VarDecl 0x{{[0-9a-f]+}} {{.*}} res3D '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]] [[hlsl::resource_dimension(3D)]]
-__hlsl_resource_t [[hlsl::resource_class(SRV)]] [[hlsl::dimension("3D")]] res3D;
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]] [[hlsl::dimension("3D")]]
+__hlsl_resource_t [[hlsl::resource_class("SRV")]] [[hlsl::dimension("3D")]] res3D;
 
 // CHECK: VarDecl 0x{{[0-9a-f]+}} {{.*}} resCube '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(SRV)]] [[hlsl::resource_dimension(Cube)]]
-__hlsl_resource_t [[hlsl::resource_class(SRV)]] [[hlsl::dimension("Cube")]] resCube;
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("SRV")]] [[hlsl::dimension("Cube")]]
+__hlsl_resource_t [[hlsl::resource_class("SRV")]] [[hlsl::dimension("Cube")]] resCube;
diff --git a/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl b/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl
index 816d3fc5a94a5..5fd7c6dd77fab 100644
--- a/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl
+++ b/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl
@@ -4,7 +4,7 @@
 // CHECK: TemplateArgument type 'float'
 // CHECK: BuiltinType {{.*}} 'float'
 // CHECK: FieldDecl {{.*}} implicit{{.*}} __handle '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(float)]]
 RWBuffer<float> Buffer1;
 
@@ -13,7 +13,7 @@ RWBuffer<float> Buffer1;
 // CHECK: ExtVectorType {{.*}} 'vector<float, 4>' 4
 // CHECK: BuiltinType {{.*}} 'float'
 // CHECK: FieldDecl {{.*}} implicit{{.*}} __handle '__hlsl_resource_t
-// CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]
+// CHECK-SAME{LITERAL}: [[hlsl::resource_class("UAV")]
 // CHECK-SAME{LITERAL}: [[hlsl::is_rov]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<float, 4>)]]
 RasterizerOrderedBuffer<vector<float, 4> > BufferArray3[4];
diff --git a/clang/test/SemaHLSL/Attributes/hlsl_contained_type_attr_error.hlsl b/clang/test/SemaHLSL/Attributes/hlsl_contained_type_attr_error.hlsl
index 5d8984340a4c2..d27280490bffe 100644
--- a/clang/test/SemaHLSL/Attributes/hlsl_contained_type_attr_error.hlsl
+++ b/clang/test/SemaHLSL/Attributes/hlsl_contained_type_attr_error.hlsl
@@ -6,23 +6,23 @@ typedef vector<float, 4> float4;
 [[hlsl::contained_type(float4)]] __hlsl_resource_t h1;
 
 // expected-error@+1{{'hlsl::contained_type' attribute takes one argument}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type()]] h3;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type()]] h3;
 
 // expected-error@+1{{expected a type}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(0)]] h4;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(0)]] h4;
 
 // expected-error@+1{{unknown type name 'a'}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(a)]] h5;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(a)]] h5;
 
 // expected-error@+1{{expected a type}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type("b", c)]] h6;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type("b", c)]] h6;
 
 // expected-warning@+1{{attribute 'hlsl::contained_type' is already applied}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]] [[hlsl::contained_type(float)]] h7;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(float)]] [[hlsl::contained_type(float)]] h7;
 
 // expected-warning@+1{{attribute 'hlsl::contained_type' is already applied with different arguments}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]] [[hlsl::contained_type(int)]] h8;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(float)]] [[hlsl::contained_type(int)]] h8;
 
 // expected-error@+2{{attribute 'hlsl::resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
 // expected-error@+1{{attribute 'hlsl::contained_type' can be used only on HLSL intangible type '__hlsl_resource_t'}}
-float [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]] res5;
+float [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(float)]] res5;
diff --git a/clang/test/SemaHLSL/Attributes/hlsl_is_array_attr_error.hlsl b/clang/test/SemaHLSL/Attributes/hlsl_is_array_attr_error.hlsl
index e67161715aeb2..3845241ac947f 100644
--- a/clang/test/SemaHLSL/Attributes/hlsl_is_array_attr_error.hlsl
+++ b/clang/test/SemaHLSL/Attributes/hlsl_is_array_attr_error.hlsl
@@ -7,14 +7,14 @@
 __hlsl_resource_t [[hlsl::is_array]] res1;
 
 // expected-error@+1{{'hlsl::is_array' attribute takes no arguments}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_array(3)]] res2;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::is_array(3)]] res2;
 
 // expected-error@+1{{use of undeclared identifier 'gibberish'}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_array(gibberish)]] res3;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::is_array(gibberish)]] res3;
 
 // expected-warning@+1{{attribute 'hlsl::is_array' is already applied}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_array]] [[hlsl::is_array]] res4;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::is_array]] [[hlsl::is_array]] res4;
 
 // expected-error@+2{{attribute 'hlsl::resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
 // expected-error@+1{{attribute 'hlsl::is_array' can be used only on HLSL intangible type '__hlsl_resource_t'}}
-float [[hlsl::resource_class(UAV)]] [[hlsl::is_array]] res5;
+float [[hlsl::resource_class("UAV")]] [[hlsl::is_array]] res5;
diff --git a/clang/test/SemaHLSL/Attributes/hlsl_is_ms_attr_error.hlsl b/clang/test/SemaHLSL/Attributes/hlsl_is_ms_attr_error.hlsl
index 187161d48564f..fd89d466b770a 100644
--- a/clang/test/SemaHLSL/Attributes/hlsl_is_ms_attr_error.hlsl
+++ b/clang/test/SemaHLSL/Attributes/hlsl_is_ms_attr_error.hlsl
@@ -7,14 +7,14 @@
 __hlsl_resource_t [[hlsl::is_ms]] res1;
 
 // expected-error@+1{{'hlsl::is_ms' attribute takes no arguments}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_ms(3)]] res2;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::is_ms(3)]] res2;
 
 // expected-error@+1{{use of undeclared identifier 'gibberish'}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_ms(gibberish)]] res3;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::is_ms(gibberish)]] res3;
 
 // expected-warning@+1{{attribute 'hlsl::is_ms' is already applied}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_ms]] [[hlsl::is_ms]] res4;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::is_ms]] [[hlsl::is_ms]] res4;
 
 // expected-error@+2{{attribute 'hlsl::resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
 // expected-error@+1{{attribute 'hlsl::is_ms' can be used only on HLSL intangible type '__hlsl_resource_t'}}
-float [[hlsl::resource_class(UAV)]] [[hlsl::is_ms]] res5;
+float [[hlsl::resource_class("UAV")]] [[hlsl::is_ms]] res5;
diff --git a/clang/test/SemaHLSL/Attributes/hlsl_is_rov_attr_error.hlsl b/clang/test/SemaHLSL/Attributes/hlsl_is_rov_attr_error.hlsl
index 1b75228be83cc..25e87a51237be 100644
--- a/clang/test/SemaHLSL/Attributes/hlsl_is_rov_attr_error.hlsl
+++ b/clang/test/SemaHLSL/Attributes/hlsl_is_rov_attr_error.hlsl
@@ -7,14 +7,14 @@
 __hlsl_resource_t [[hlsl::is_rov]] res1;
 
 // expected-error@+1{{'hlsl::is_rov' attribute takes no arguments}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_rov(3)]] res2;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::is_rov(3)]] res2;
 
 // expected-error@+1{{use of undeclared identifier 'gibberish'}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_rov(gibberish)]] res3;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::is_rov(gibberish)]] res3;
 
 // expected-warning@+1{{attribute 'hlsl::is_rov' is already applied}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_rov]] [[hlsl::is_rov]] res4;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::is_rov]] [[hlsl::is_rov]] res4;
 
 // expected-error@+2{{attribute 'hlsl::resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
 // expected-error@+1{{attribute 'hlsl::is_rov' can be used only on HLSL intangible type '__hlsl_resource_t'}}
-float [[hlsl::resource_class(UAV)]] [[hlsl::is_rov]] res5;
+float [[hlsl::resource_class("UAV")]] [[hlsl::is_rov]] res5;
diff --git a/clang/test/SemaHLSL/Attributes/hlsl_raw_buffer_attr_error.hlsl b/clang/test/SemaHLSL/Attributes/hlsl_raw_buffer_attr_error.hlsl
index a6e45838d285b..1d696f5499792 100644
--- a/clang/test/SemaHLSL/Attributes/hlsl_raw_buffer_attr_error.hlsl
+++ b/clang/test/SemaHLSL/Attributes/hlsl_raw_buffer_attr_error.hlsl
@@ -4,14 +4,14 @@
 [[hlsl::raw_buffer]] __hlsl_resource_t res0;
 
 // expected-error@+1{{'hlsl::raw_buffer' attribute takes no arguments}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer(3)]] res2;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::raw_buffer(3)]] res2;
 
 // expected-error@+1{{use of undeclared identifier 'gibberish'}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer(gibberish)]] res3;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::raw_buffer(gibberish)]] res3;
 
 // expected-warning@+1{{attribute 'hlsl::raw_buffer' is already applied}}
-__hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer]] [[hlsl::raw_buffer]] res4;
+__hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::raw_buffer]] [[hlsl::raw_buffer]] res4;
 
 // expected-error@+2{{attribute 'hlsl::resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
 // expected-error@+1{{attribute 'hlsl::raw_buffer' can be used only on HLSL intangible type '__hlsl_resource_t'}}
-float [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer]] res5;
+float [[hlsl::resource_class("UAV")]] [[hlsl::raw_buffer]] res5;
diff --git a/clang/test/SemaHLSL/Attributes/hlsl_resource_class_attr_error.hlsl b/clang/test/SemaHLSL/Attributes/hlsl_resource_class_attr_error.hlsl
index 55b8197376006..10e226babd5b3 100644
--- a/clang/test/SemaHLSL/Attributes/hlsl_resource_class_attr_error.hlsl
+++ b/clang/test/SemaHLSL/Attributes/hlsl_resource_class_attr_error.hlsl
@@ -1,22 +1,22 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -o - %s -verify
 
 // expected-error@+1{{'hlsl::resource_class' attribute cannot be applied to a declaration}}
-[[hlsl::resource_class(UAV)]] __hlsl_resource_t e0;
+[[hlsl::resource_class("UAV")]] __hlsl_resource_t e0;
 
 // expected-error@+1{{'hlsl::resource_class' attribute takes one argument}}
 __hlsl_resource_t [[hlsl::resource_class()]] e1;
 
 // expected-warning@+1{{ResourceClass attribute argument not supported: gibberish}}
-__hlsl_resource_t [[hlsl::resource_class(gibberish)]] e2;
+__hlsl_resource_t [[hlsl::resource_class("gibberish")]] e2;
 
 // expected-warning@+1{{attribute 'hlsl::resource_class' is already applied with different arguments}}
-__hlsl_resource_t [[hlsl::resource_class(SRV)]] [[hlsl::resource_class(UAV)]] e3;
+__hlsl_resource_t [[hlsl::resource_class("SRV")]] [[hlsl::resource_class("UAV")]] e3;
 
 // expected-warning@+1{{attribute 'hlsl::resource_class' is already applied}}
-__hlsl_resource_t [[hlsl::resource_class(SRV)]] [[hlsl::resource_class(SRV)]] e4;
+__hlsl_resource_t [[hlsl::resource_class("SRV")]] [[hlsl::resource_class("SRV")]] e4;
 
 // expected-error@+1{{'hlsl::resource_class' attribute takes one argument}}
-__hlsl_resource_t [[hlsl::resource_class(SRV, "aa")]] e5;
+__hlsl_resource_t [[hlsl::resource_class("SRV", "aa")]] e5;
 
 // expected-error@+1{{attribute 'hlsl::resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
-float [[hlsl::resource_class(UAV)]] e6;
+float [[hlsl::resource_class("UAV")]] e6;
diff --git a/clang/test/SemaHLSL/BuiltIns/buffer_update_counter-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/buffer_update_counter-errors.hlsl
index faac8f07b240a..a3f9b8131a085 100644
--- a/clang/test/SemaHLSL/BuiltIns/buffer_update_counter-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/buffer_update_counter-errors.hlsl
@@ -1,13 +1,13 @@
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify
 
 // RWStructuredBuffer<int>
-using handle_t = __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(int)]] [[hlsl::raw_buffer]];
+using handle_t = __hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(int)]] [[hlsl::raw_buffer]];
 // RWBuffer<int>
-using bad_handle_not_raw_t = __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(int)]];
+using bad_handle_not_raw_t = __hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(int)]];
 // RWByteAddressBuffer
-using bad_handle_no_type_t = __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer]];
+using bad_handle_no_type_t = __hlsl_resource_t [[hlsl::resource_class("UAV")]] [[hlsl::raw_buffer]];
 // StructuredBuffer
-using bad_handle_not_uav_t = __hlsl_resource_t [[hlsl::resource_class(SRV)]] [[hlsl::contained_type(int)]] [[hlsl::raw_buffer]];
+using bad_handle_not_uav_t = __hlsl_resource_t [[hlsl::resource_class("SRV")]] [[hlsl::contained_type(int)]] [[hlsl::raw_buffer]];
 
 void test_args(int x, bool b) {
   // expected-error@+1 {{too few arguments to function call, expected 2, have 1}}
@@ -39,7 +39,7 @@ void test_args(int x, bool b) {
 
   // expected-error@+1 {{cannot initialize a parameter of type 'int' with an lvalue of type 'const char[2]'}}
   __builtin_hlsl_buffer_update_counter(res, "1");
-  
+
   // expected-error@+1 {{argument 1 must be constant integer 1 or -1}}
   __builtin_hlsl_buffer_update_counter(res, 10);
 
diff --git a/clang/test/SemaHLSL/BuiltIns/resource_getpointer-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/resource_getpointer-errors.hlsl
index 7dde4dcf6c149..217cf9a55a1ad 100644
--- a/clang/test/SemaHLSL/BuiltIns/resource_getpointer-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/resource_getpointer-errors.hlsl
@@ -2,7 +2,7 @@
 
 // RWBuffer<int>
 using handle_t = __hlsl_resource_t
-    [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(int)]];
+    [[hlsl::resource_class("UAV")]] [[hlsl::contained_type(int)]];
 
 void test_args(unsigned int x) {
   // expected-error@+1 {{used type 'unsigned int' where __hlsl_resource_t is required}}
@@ -30,10 +30,10 @@ void test_args(unsigned int x) {
 }
 
 using tex2d_handle_t = __hlsl_resource_t
-    [[hlsl::resource_class(SRV)]] [[hlsl::dimension("2D")]] [[hlsl::contained_type(float4)]];
+    [[hlsl::resource_class("SRV")]] [[hlsl::dimension("2D")]] [[hlsl::contained_type(float4)]];
 
 using tex3d_handle_t = __hlsl_resource_t
-    [[hlsl::resource_class(SRV)]] [[hlsl::dimension("3D")]] [[hlsl::contained_type(float4)]];
+    [[hlsl::resource_class("SRV")]] [[hlsl::dimension("3D")]] [[hlsl::contained_type(float4)]];
 
 void test_tex_handles(tex2d_handle_t tex2d, tex3d_handle_t tex3d) {
   // expected-error@+1 {{builtin '__builtin_hlsl_resource_getpointer' resource coordinate dimension mismatch: expected 2, found 1}}
diff --git a/clang/test/SemaHLSL/Resources/resource_binding_attr_error.hlsl b/clang/test/SemaHLSL/Resources/resource_binding_attr_error.hlsl
index e2d72197ef602..0f33ae44aa7ad 100644
--- a/clang/test/SemaHLSL/Resources/resource_binding_attr_error.hlsl
+++ b/clang/test/SemaHLSL/Resources/resource_binding_attr_error.hlsl
@@ -2,7 +2,7 @@
 
 template<typename T>
 struct MyTemplatedSRV {
-  __hlsl_resource_t [[hlsl::resource_class(SRV)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("SRV")]] x;
 };
 
 // valid, The register keyword in this statement isn't binding a resource, rather it is
@@ -35,10 +35,10 @@ cbuffer D : register(b 2, space3) {}
 cbuffer E : register(u-1) {};
 
 // expected-error@+1 {{expected <numeric_constant>}}
-cbuffer F : register(u) {}; 
+cbuffer F : register(u) {};
 
 // expected-error@+1 {{binding type 'u' only applies to UAV resources}}
-cbuffer G : register(u13) {}; 
+cbuffer G : register(u13) {};
 
 // expected-error@+1 {{binding type 'c' only applies to numeric variables in the global scope}}
 cbuffer H : register(c0) {};
diff --git a/clang/test/SemaHLSL/Resources/resource_binding_attr_error_resource.hlsl b/clang/test/SemaHLSL/Resources/resource_binding_attr_error_resource.hlsl
index ea43e27b5b5ac..00b622169726a 100644
--- a/clang/test/SemaHLSL/Resources/resource_binding_attr_error_resource.hlsl
+++ b/clang/test/SemaHLSL/Resources/resource_binding_attr_error_resource.hlsl
@@ -6,23 +6,23 @@
 
 template<typename T>
 struct MyTemplatedSRV {
-  __hlsl_resource_t [[hlsl::resource_class(SRV)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("SRV")]] x;
 };
 
 struct MySRV {
-  __hlsl_resource_t [[hlsl::resource_class(SRV)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("SRV")]] x;
 };
 
 struct MySampler {
-  __hlsl_resource_t [[hlsl::resource_class(Sampler)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("Sampler")]] x;
 };
 
 struct MyUAV {
-  __hlsl_resource_t [[hlsl::resource_class(UAV)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("UAV")]] x;
 };
 
 struct MyCBuffer {
-  __hlsl_resource_t [[hlsl::resource_class(CBuffer)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("CBuffer")]] x;
 };
 
 
diff --git a/clang/test/SemaHLSL/Resources/resource_binding_attr_error_udt.hlsl b/clang/test/SemaHLSL/Resources/resource_binding_attr_error_udt.hlsl
index 235004102a539..e7c354724533b 100644
--- a/clang/test/SemaHLSL/Resources/resource_binding_attr_error_udt.hlsl
+++ b/clang/test/SemaHLSL/Resources/resource_binding_attr_error_udt.hlsl
@@ -2,23 +2,23 @@
 
 template<typename T>
 struct MyTemplatedUAV {
-  __hlsl_resource_t [[hlsl::resource_class(UAV)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("UAV")]] x;
 };
 
 struct MySRV {
-  __hlsl_resource_t [[hlsl::resource_class(SRV)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("SRV")]] x;
 };
 
 struct MySampler {
-  __hlsl_resource_t [[hlsl::resource_class(Sampler)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("Sampler")]] x;
 };
 
 struct MyUAV {
-  __hlsl_resource_t [[hlsl::resource_class(UAV)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("UAV")]] x;
 };
 
 struct MyCBuffer {
-  __hlsl_resource_t [[hlsl::resource_class(CBuffer)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("CBuffer")]] x;
 };
 
 // Valid: f is skipped, SRVBuf is bound to t0, UAVBuf is bound to u0
diff --git a/clang/test/SemaHLSL/Resources/resource_binding_attr_error_uint32_max.hlsl b/clang/test/SemaHLSL/Resources/resource_binding_attr_error_uint32_max.hlsl
index 464eb12669a9c..edbcb84405f0e 100644
--- a/clang/test/SemaHLSL/Resources/resource_binding_attr_error_uint32_max.hlsl
+++ b/clang/test/SemaHLSL/Resources/resource_binding_attr_error_uint32_max.hlsl
@@ -30,7 +30,7 @@ RWBuffer<float> Buf[10][10] : register(u4294967234);
 
 // test a standard resource array
 // expected-error@+1 {{register number should not exceed 4294967295}}
-RWBuffer<float> Buf2[10] : register(u4294967294); 
+RWBuffer<float> Buf2[10] : register(u4294967294);
 
 // test directly an excessively high register number.
 // expected-error@+1 {{register number should not exceed 4294967295}}
@@ -44,15 +44,15 @@ cbuffer MyCB : register(b9995294967294) {
 };
 
 struct MySRV {
-  __hlsl_resource_t [[hlsl::resource_class(SRV)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("SRV")]] x;
 };
 
 struct MySampler {
-  __hlsl_resource_t [[hlsl::resource_class(Sampler)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("Sampler")]] x;
 };
 
 struct MyUAV {
-  __hlsl_resource_t [[hlsl::resource_class(UAV)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("UAV")]] x;
 };
 
 // test that different resource classes don't contribute to the
@@ -79,12 +79,10 @@ MyResources M3 : register(t2) : register(s3) : register(u4294967280);
 
 
 // expected-error@+1 {{register number should be an integer}}
-RWBuffer<float> Buf3[10][10] : register(ud); 
+RWBuffer<float> Buf3[10][10] : register(ud);
 
 // this should work
 RWBuffer<float> GoodBuf : register(u4294967295);
 
 // no errors expected, all 100 register numbers are occupied here
-RWBuffer<float> GoodBufArray[10][10] : register(u4294967194); 
-
-
+RWBuffer<float> GoodBufArray[10][10] : register(u4294967194);
diff --git a/clang/test/SemaHLSL/Resources/resource_binding_implicit.hlsl b/clang/test/SemaHLSL/Resources/resource_binding_implicit.hlsl
index 68577d974fddb..0bba3adffa131 100644
--- a/clang/test/SemaHLSL/Resources/resource_binding_implicit.hlsl
+++ b/clang/test/SemaHLSL/Resources/resource_binding_implicit.hlsl
@@ -25,10 +25,10 @@ struct S { int x; };
 StructuredBuffer<S> e;
 
 // No warning - __hlsl_resource_t isn't itself a resource object.
-__hlsl_resource_t [[hlsl::resource_class(SRV)]] f;
+__hlsl_resource_t [[hlsl::resource_class("SRV")]] f;
 
 struct CustomSRV {
-  __hlsl_resource_t [[hlsl::resource_class(SRV)]] x;
+  __hlsl_resource_t [[hlsl::resource_class("SRV")]] x;
 };
 // expected-warning@+1 {{resource has implicit register binding}}
 CustomSRV g;
diff --git a/flang/lib/Optimizer/Transforms/CUDA/CUFAllocDelay.cpp b/flang/lib/Optimizer/Transforms/CUDA/CUFAllocDelay.cpp
index f9e62cccdc6e2..1ee7f96bfc9f3 100644
--- a/flang/lib/Optimizer/Transforms/CUDA/CUFAllocDelay.cpp
+++ b/flang/lib/Optimizer/Transforms/CUDA/CUFAllocDelay.cpp
@@ -7,8 +7,9 @@
 //===----------------------------------------------------------------------===//
 //
 // Delay cuf.alloc of descriptor (box) types from function entry to just before
-// their first use. This defers cudaMallocManaged calls so that users can call
-// cudaSetDevice before any CUDA context is created.
+// their first use, possibly in a later block that dominates every use. This
+// defers cudaMallocManaged calls so that users can call cudaSetDevice before
+// any CUDA context is created.
 //
 //===----------------------------------------------------------------------===//
 
@@ -17,8 +18,10 @@
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "mlir/IR/Block.h"
+#include "mlir/IR/Dominance.h"
 #include "mlir/Pass/Pass.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 
 namespace fir {
@@ -28,56 +31,90 @@ namespace fir {
 
 namespace {
 
-/// Find the earliest use of the descriptor and return the op before which the
-/// cuf.alloc group should be placed. Uses in nested regions (fir.if,
-/// fir.do_loop, ...) resolve to the enclosing entry-block op; uses confined to
-/// a single successor block resolve to that block.
-static mlir::Operation *findDelayTarget(fir::DeclareOp declareOp,
-                                        mlir::Block *entryBlock) {
-  mlir::Operation *earliest = nullptr;
+/// Return the coordinate_of producing the host association tuple slot that
+/// \p storeOp writes \p descriptor into, or null if this is not such a capture.
+static fir::CoordinateOp getHostAssocTupleSlot(fir::StoreOp storeOp,
+                                               mlir::Value descriptor) {
+  if (storeOp.getValue() != descriptor ||
+      !mlir::isa<fir::LLVMPointerType>(storeOp.getMemref().getType()))
+    return nullptr;
+  auto coord = storeOp.getMemref().getDefiningOp<fir::CoordinateOp>();
+  if (!coord ||
+      !mlir::isa<mlir::TupleType>(fir::unwrapRefType(coord.getRef().getType())))
+    return nullptr;
+  return coord;
+}
+
+/// Return true if \p coord's result is only stored into, so it writes the tuple
+/// rather than reading it.
+static bool onlyPopulatesSlot(fir::CoordinateOp coord) {
+  return llvm::all_of(coord->getUsers(), [&](mlir::Operation *user) {
+    auto storeOp = mlir::dyn_cast<fir::StoreOp>(user);
+    return storeOp && storeOp.getMemref() == coord.getResult();
+  });
+}
+
+/// Find the point before which the cuf.alloc group should be placed: the
+/// earliest use in the block that dominates all uses, or that block's
+/// terminator if it holds no use itself. Uses in nested regions resolve to
+/// their enclosing top-level op.
+///
+/// Host association stores go to \p hostAssocStores and sink with the group
+/// instead of constraining it; the tuple's readers constrain it instead.
+static mlir::Operation *
+findDelayTarget(fir::DeclareOp declareOp, mlir::Block *entryBlock,
+                mlir::DominanceInfo &domInfo,
+                llvm::SmallVectorImpl<fir::StoreOp> &hostAssocStores) {
   mlir::Region *funcRegion = entryBlock->getParent();
 
-  // Uses per successor block, with the earliest op in each.
-  llvm::SmallDenseMap<mlir::Block *, mlir::Operation *> successorEarliest;
+  // Uses resolved to an op that sits directly in a block of the function.
+  llvm::SmallVector<mlir::Operation *> uses;
 
-  // Resolve a use in a nested region or successor block to a target in/after
-  // the entry block.
   auto recordRealUse = [&](mlir::Operation *user) {
-    mlir::Operation *target = user;
-    while (target->getBlock() != entryBlock) {
-      // User in another block of the same function.
-      if (target->getBlock() && target->getBlock()->getParent() == funcRegion) {
-        mlir::Block *blk = target->getBlock();
-        auto it = successorEarliest.find(blk);
-        if (it == successorEarliest.end() ||
-            target->isBeforeInBlock(it->second))
-          successorEarliest[blk] = target;
-        return;
-      }
-      target = target->getParentOp();
-      if (!target)
-        return;
-    }
-    if (!earliest || target->isBeforeInBlock(earliest))
-      earliest = target;
+    mlir::Operation *op = user;
+    while (op && op->getBlock() && op->getBlock()->getParent() != funcRegion)
+      op = op->getParentOp();
+    if (op && op->getBlock())
+      uses.push_back(op);
   };
 
   for (mlir::Value result : declareOp->getResults()) {
-    for (mlir::Operation *user : result.getUsers())
-      recordRealUse(user);
+    for (mlir::Operation *user : result.getUsers()) {
+      auto storeOp = mlir::dyn_cast<fir::StoreOp>(user);
+      fir::CoordinateOp slot =
+          storeOp ? getHostAssocTupleSlot(storeOp, result) : nullptr;
+      if (!slot) {
+        recordRealUse(user);
+        continue;
+      }
+      // Whoever reads the tuple must still see a populated slot.
+      hostAssocStores.push_back(storeOp);
+      for (mlir::Operation *tupleUser : slot.getRef().getUsers()) {
+        auto coord = mlir::dyn_cast<fir::CoordinateOp>(tupleUser);
+        if (coord && onlyPopulatesSlot(coord))
+          continue;
+        recordRealUse(tupleUser);
+      }
+    }
   }
 
-  if (earliest)
-    return earliest;
-
-  // No entry-block uses.  If all successor uses are in a single block,
-  // delay directly into that block (before the earliest use there).
-  // Otherwise fall back to the entry block's terminator.
-  if (successorEarliest.size() == 1)
-    return successorEarliest.begin()->second;
-  if (!successorEarliest.empty())
-    return entryBlock->getTerminator();
-  return nullptr;
+  if (uses.empty())
+    return nullptr;
+
+  mlir::Block *common = uses.front()->getBlock();
+  for (mlir::Operation *use : uses) {
+    common = domInfo.findNearestCommonDominator(common, use->getBlock());
+    if (!common)
+      return nullptr;
+  }
+
+  mlir::Operation *earliest = nullptr;
+  for (mlir::Operation *use : uses)
+    if (use->getBlock() == common &&
+        (!earliest || use->isBeforeInBlock(earliest)))
+      earliest = use;
+
+  return earliest ? earliest : common->getTerminator();
 }
 
 struct CUFAllocDelay : public fir::impl::CUFAllocDelayBase<CUFAllocDelay> {
@@ -88,6 +125,7 @@ struct CUFAllocDelay : public fir::impl::CUFAllocDelayBase<CUFAllocDelay> {
       return;
 
     mlir::Block &entryBlock = func.front();
+    mlir::DominanceInfo domInfo(func);
 
     // Collect box-type cuf.alloc ops in the entry block.
     llvm::SmallVector<cuf::AllocOp> boxAllocOps;
@@ -113,7 +151,9 @@ struct CUFAllocDelay : public fir::impl::CUFAllocDelayBase<CUFAllocDelay> {
       if (!declareOp || hasUnknownUser)
         continue;
 
-      mlir::Operation *delayTarget = findDelayTarget(declareOp, &entryBlock);
+      llvm::SmallVector<fir::StoreOp> hostAssocStores;
+      mlir::Operation *delayTarget =
+          findDelayTarget(declareOp, &entryBlock, domInfo, hostAssocStores);
       if (!delayTarget)
         continue;
 
@@ -124,15 +164,36 @@ struct CUFAllocDelay : public fir::impl::CUFAllocDelayBase<CUFAllocDelay> {
       if (delayTarget == declareOp)
         continue;
 
-      // Sink {cuf.alloc, fir.store, fir.declare} before the target; the
-      // embox/shape/constants stay put and still dominate the new position.
-      allocOp->moveBefore(delayTarget);
-      if (storeOp)
-        storeOp->moveAfter(allocOp);
+      // Ops that move together, keeping their relative order.
+      llvm::SmallVector<mlir::Operation *> group;
+      group.push_back(allocOp);
       if (storeOp)
-        declareOp->moveAfter(storeOp);
-      else
-        declareOp->moveAfter(allocOp);
+        group.push_back(storeOp);
+      group.push_back(declareOp);
+      for (fir::StoreOp hostAssocStore : hostAssocStores)
+        group.push_back(hostAssocStore);
+
+      // Whatever the group reads from outside itself stays put, so it must
+      // already dominate the new position.
+      llvm::SmallPtrSet<mlir::Operation *, 8> groupSet(group.begin(),
+                                                       group.end());
+      auto readsDominateTarget = [&](mlir::Operation *op) {
+        return llvm::all_of(op->getOperands(), [&](mlir::Value operand) {
+          mlir::Operation *def = operand.getDefiningOp();
+          return (def && groupSet.contains(def)) ||
+                 domInfo.properlyDominates(operand, delayTarget);
+        });
+      };
+      if (!llvm::all_of(group, readsDominateTarget))
+        continue;
+
+      // Sink the group before the target, preserving its relative order.
+      group.front()->moveBefore(delayTarget);
+      mlir::Operation *last = group.front();
+      for (mlir::Operation *op : llvm::drop_begin(group)) {
+        op->moveAfter(last);
+        last = op;
+      }
     }
   }
 };
diff --git a/flang/test/Transforms/CUF/cuf-alloc-delay.fir b/flang/test/Transforms/CUF/cuf-alloc-delay.fir
index d4e9a6a4479f3..99e8f184b96c4 100644
--- a/flang/test/Transforms/CUF/cuf-alloc-delay.fir
+++ b/flang/test/Transforms/CUF/cuf-alloc-delay.fir
@@ -326,8 +326,8 @@ func.func @_QPuse_in_multi_successor(%arg0: !fir.ref<i32>) {
 
 // -----
 
-// Test 10: A host-association store (fir.store to fir.llvm_ptr) is a use, so
-// the group sinks to just before it and the store is not moved.
+// Test 10: A host association store sinks with the group, so the allocation is
+// still delayed to the first real use.
 func.func @_QPhost_assoc() {
   %tuple = fir.alloca tuple<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
   %c0_i32 = arith.constant 0 : i32
@@ -353,11 +353,11 @@ func.func private @_QFPcontained(!fir.ref<tuple<!fir.ref<!fir.box<!fir.heap<!fir
 // CHECK:         fir.coordinate_of
 // CHECK:         fir.zero_bits
 // CHECK:         fir.embox
+// CHECK:         arith.constant 99
 // CHECK:         cuf.alloc
 // CHECK:         fir.store {{.*}} : !fir.ref<!fir.box
 // CHECK:         fir.declare
 // CHECK:         fir.store {{.*}} : !fir.llvm_ptr
-// CHECK:         arith.constant 99
 // CHECK:         cuf.allocate
 // CHECK:         fir.call @_QFPcontained
 // CHECK:         cuf.free
@@ -365,9 +365,8 @@ func.func private @_QFPcontained(!fir.ref<tuple<!fir.ref<!fir.box<!fir.heap<!fir
 
 // -----
 
-// Test 11: The descriptor's only entry-block use is its host-association store
-// (the allocate happens inside the callee). The group and store stay before the
-// call, so the callee sees an initialized descriptor.
+// Test 11: The descriptor is only captured, never used in the host, so the group
+// and its store must still land before the call.
 func.func @_QPhost_assoc_call() {
   %tuple = fir.alloca tuple<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
   %c0_i32 = arith.constant 0 : i32
@@ -397,9 +396,8 @@ func.func private @_QFPgo(!fir.ref<tuple<!fir.ref<!fir.box<!fir.heap<!fir.array<
 
 // -----
 
-// Test 12: A shared tuple whose slots fill in order, so the descriptor's own
-// slot (index 1) is defined after an earlier coordinate_of of the same tuple
-// (index 0). The group must sink to before its host-association store.
+// Test 12: A shared tuple filled in slot order, so the descriptor's own slot is
+// defined after an earlier coordinate_of of the same tuple.
 func.func @_QPhost_assoc_late_slot() {
   %tuple = fir.alloca tuple<!fir.ref<i32>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
   %c0_i32 = arith.constant 0 : i32
@@ -433,3 +431,156 @@ func.func private @_QFPgo2(!fir.ref<tuple<!fir.ref<i32>, !fir.ref<!fir.box<!fir.
 // CHECK:         fir.call @_QFPgo2
 // CHECK:         cuf.free
 // CHECK:         return
+
+// -----
+
+// Test 13: The tuple is populated in the prologue but the descriptor is not used
+// until later, so the group and its store sink past the intervening call.
+func.func @_QPhost_assoc_prologue() {
+  %tuple = fir.alloca tuple<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
+  %c0_i32 = arith.constant 0 : i32
+  %slot = fir.coordinate_of %tuple, %c0_i32 : (!fir.ref<tuple<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>>, i32) -> !fir.llvm_ptr<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
+  %0 = cuf.alloc !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", data_attr = #cuf.cuda<device>, uniq_name = "_QFhost_assoc_prologueEa"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %1 = fir.zero_bits !fir.heap<!fir.array<?xf32>>
+  %c0 = arith.constant 0 : index
+  %2 = fir.shape %c0 : (index) -> !fir.shape<1>
+  %3 = fir.embox %1(%2) {allocator_idx = 2 : i32} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+  fir.store %3 to %0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %4 = fir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFhost_assoc_prologueEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  fir.store %4 to %slot : !fir.llvm_ptr<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
+  fir.call @_QPsetup() : () -> ()
+  %5 = cuf.allocate %4 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>} -> i32
+  fir.call @_QFPcontained2(%tuple) : (!fir.ref<tuple<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>>) -> ()
+  cuf.free %4 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>}
+  return
+}
+func.func private @_QPsetup()
+func.func private @_QFPcontained2(!fir.ref<tuple<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>>)
+
+// CHECK-LABEL: func.func @_QPhost_assoc_prologue
+// CHECK:         fir.coordinate_of
+// CHECK:         fir.zero_bits
+// CHECK:         fir.embox
+// CHECK:         fir.call @_QPsetup
+// CHECK:         cuf.alloc
+// CHECK:         fir.store {{.*}} : !fir.ref<!fir.box
+// CHECK:         fir.declare
+// CHECK:         fir.store {{.*}} : !fir.llvm_ptr
+// CHECK:         cuf.allocate
+// CHECK:         fir.call @_QFPcontained2
+// CHECK:         cuf.free
+// CHECK:         return
+
+// -----
+
+// Test 14: Uses span several successor blocks, but ^bb2 dominates them all, so
+// the group sinks into ^bb2 instead of stopping at the entry terminator.
+func.func @_QPuse_in_dominating_successor(%arg0: !fir.ref<i32>) {
+  %0 = cuf.alloc !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", data_attr = #cuf.cuda<device>, uniq_name = "_QFuse_in_dominating_successorEa"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %1 = fir.zero_bits !fir.heap<!fir.array<?xf32>>
+  %c0 = arith.constant 0 : index
+  %2 = fir.shape %c0 : (index) -> !fir.shape<1>
+  %3 = fir.embox %1(%2) {allocator_idx = 2 : i32} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+  fir.store %3 to %0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %4 = fir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFuse_in_dominating_successorEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %c0_i32 = arith.constant 0 : i32
+  %5 = fir.load %arg0 : !fir.ref<i32>
+  %6 = arith.cmpi slt, %5, %c0_i32 : i32
+  cf.cond_br %6, ^bb1, ^bb2
+^bb1:
+  fir.call @_FortranAStopStatementText(%c0_i32, %c0_i32) : (i32, i32) -> none
+  fir.unreachable
+^bb2:
+  fir.call @_QPinit_runtime() : () -> ()
+  %7 = cuf.allocate %4 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>} -> i32
+  %8 = fir.load %arg0 : !fir.ref<i32>
+  %9 = arith.cmpi slt, %8, %c0_i32 : i32
+  cf.cond_br %9, ^bb3, ^bb4
+^bb3:
+  cuf.free %4 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>}
+  return
+^bb4:
+  cuf.free %4 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>}
+  return
+}
+func.func private @_FortranAStopStatementText(i32, i32) -> none
+func.func private @_QPinit_runtime()
+
+// CHECK-LABEL: func.func @_QPuse_in_dominating_successor
+// CHECK:         cf.cond_br
+// CHECK:       ^bb1:
+// CHECK:         fir.unreachable
+// CHECK:       ^bb2:
+// The allocation happens after the call, not before the branch.
+// CHECK:         fir.call @_QPinit_runtime
+// CHECK:         cuf.alloc
+// CHECK:         fir.store
+// CHECK:         fir.declare
+// CHECK:         cuf.allocate
+
+// -----
+
+// Test 15: A store into an !fir.llvm_ptr that is not a tuple slot goes to an
+// unknown destination, so it is an ordinary use and pins the group.
+func.func @_QPllvm_ptr_not_tuple(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) {
+  %0 = cuf.alloc !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", data_attr = #cuf.cuda<device>, uniq_name = "_QFllvm_ptr_not_tupleEa"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %1 = fir.zero_bits !fir.heap<!fir.array<?xf32>>
+  %c0 = arith.constant 0 : index
+  %2 = fir.shape %c0 : (index) -> !fir.shape<1>
+  %3 = fir.embox %1(%2) {allocator_idx = 2 : i32} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+  fir.store %3 to %0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %4 = fir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFllvm_ptr_not_tupleEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %opaque = fir.convert %arg0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
+  fir.store %4 to %opaque : !fir.llvm_ptr<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
+  fir.call @_QPsetup3() : () -> ()
+  cuf.free %4 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>}
+  return
+}
+func.func private @_QPsetup3()
+
+// CHECK-LABEL: func.func @_QPllvm_ptr_not_tuple
+// CHECK:         fir.convert
+// CHECK:         cuf.alloc
+// CHECK:         fir.store {{.*}} : !fir.ref<!fir.box
+// CHECK:         fir.declare
+// CHECK:         fir.store {{.*}} : !fir.llvm_ptr
+// CHECK:         fir.call @_QPsetup3
+// CHECK:         cuf.free
+
+// -----
+
+// Test 16: The tuple slot is read back in the host, so the group sinks past the
+// call but stays before the coordinate_of feeding that read.
+func.func @_QPtuple_slot_read() {
+  %tuple = fir.alloca tuple<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
+  %c0_i32 = arith.constant 0 : i32
+  %slot = fir.coordinate_of %tuple, %c0_i32 : (!fir.ref<tuple<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>>, i32) -> !fir.llvm_ptr<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
+  %0 = cuf.alloc !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", data_attr = #cuf.cuda<device>, uniq_name = "_QFtuple_slot_readEa"} -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %1 = fir.zero_bits !fir.heap<!fir.array<?xf32>>
+  %c0 = arith.constant 0 : index
+  %2 = fir.shape %c0 : (index) -> !fir.shape<1>
+  %3 = fir.embox %1(%2) {allocator_idx = 2 : i32} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+  fir.store %3 to %0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  %4 = fir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtuple_slot_readEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+  fir.store %4 to %slot : !fir.llvm_ptr<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
+  fir.call @_QPsetup4() : () -> ()
+  %reread = fir.coordinate_of %tuple, %c0_i32 : (!fir.ref<tuple<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>>, i32) -> !fir.llvm_ptr<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
+  %5 = fir.load %reread : !fir.llvm_ptr<!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>>
+  %6 = cuf.allocate %5 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>} -> i32
+  cuf.free %4 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>}
+  return
+}
+func.func private @_QPsetup4()
+
+// CHECK-LABEL: func.func @_QPtuple_slot_read
+// CHECK:         fir.coordinate_of
+// CHECK:         fir.embox
+// CHECK:         fir.call @_QPsetup4
+// CHECK:         cuf.alloc
+// CHECK:         fir.store {{.*}} : !fir.ref<!fir.box
+// CHECK:         fir.declare
+// CHECK:         fir.store {{.*}} : !fir.llvm_ptr
+// CHECK:         fir.coordinate_of
+// CHECK:         fir.load
+// CHECK:         cuf.allocate
+// CHECK:         cuf.free
diff --git a/libcxx/docs/TestingLibcxx.rst b/libcxx/docs/TestingLibcxx.rst
index aaeceda29fe15..8b902518a204b 100644
--- a/libcxx/docs/TestingLibcxx.rst
+++ b/libcxx/docs/TestingLibcxx.rst
@@ -556,39 +556,36 @@ Testing hardening assertions
 ============================
 
 Each hardening assertion should be tested using death tests (via the
-``TEST_LIBCPP_ASSERT_FAILURE`` macro). Use the ``libcpp-hardening-mode`` Lit
-feature to make sure the assertion is enabled in (and only in) the intended
-modes. The convention is to use `assert.` in the name of the test file to make
-it easier to identify as a hardening test, e.g. ``assert.my_func.pass.cpp``.
+``TEST_LIBCPP_ASSERT_FAILURE`` macro). The convention is to use ``assert.`` in
+the name of the test file to make it easier to identify as a hardening test, e.g.
+``assert.my_func.pass.cpp``.
+
+These tests only make sense in configurations where the death test machinery in
+``check_assertion.h`` is usable, where a failing assertion is observable, and
+where the assertion being tested is enabled in the first place. Use the various
+``can-test-hardening-assertions-<mode>`` Lit features to guard the tests accordingly.
+The bare ``can-test-hardening-assertions`` Lit feature only encodes whether the death
+test machinery is usable; it is meant for tests that select a hardening mode or an
+assertion semantic themselves (see the tests under ``libcxx/test/libcxx/assertions/``).
+
 A toy example:
 
 .. code-block:: cpp
 
-  // Note: the following three annotations are currently needed to use the
-  // `TEST_LIBCPP_ASSERT_FAILURE`.
-  // REQUIRES: has-unix-headers
-  // UNSUPPORTED: c++03
-  // XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
-  // Example: only run this test in `fast`/`extensive`/`debug` modes.
-  // UNSUPPORTED: libcpp-hardening-mode=none
-  // Example: only run this test in the `debug` mode.
-  // REQUIRES: libcpp-hardening-mode=debug
-  // Example: only run this test in `extensive`/`debug` modes.
-  // REQUIRES: libcpp-hardening-mode={{extensive|debug}}
+  // Example: `std::foo(...)` uses `_LIBCPP_ASSERT_NON_NULL`, which is
+  // enabled in the `extensive` and `debug` modes.
+  // REQUIRES: can-test-hardening-assertions-extensive
 
-  #include <header_being_tested>
+  #include <stdfoo>
 
   #include "check_assertion.h" // Contains the `TEST_LIBCPP_ASSERT_FAILURE` macro
 
   int main(int, char**) {
-    std::type_being_tested foo;
     int bad_input = -1;
-    TEST_LIBCPP_ASSERT_FAILURE(foo.some_function_that_asserts(bad_input),
-        "The expected assertion message");
+    TEST_LIBCPP_ASSERT_FAILURE(std::foo(bad_input), "The expected assertion message");
 
     return 0;
   }
 
-Note that error messages are only tested (matched) if the ``debug``
-hardening mode is used.
+Note that error messages are only tested (matched) when the assertion semantic in
+effect logs one, i.e. ``enforce`` or ``observe``.
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 95d371c092983..f6911b8ff05c5 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -294,6 +294,7 @@ set(files
   __chrono/year_month_day.h
   __chrono/year_month_weekday.h
   __chrono/zoned_time.h
+  __cmath/lerp.h
   __cmath/special_functions.h
   __compare/common_comparison_category.h
   __compare/compare_partial_order_fallback.h
diff --git a/libcxx/include/__cmath/lerp.h b/libcxx/include/__cmath/lerp.h
new file mode 100644
index 0000000000000..4cbbcc570fb83
--- /dev/null
+++ b/libcxx/include/__cmath/lerp.h
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___CMATH_LERP_H
+#define _LIBCPP___CMATH_LERP_H
+
+#include <__config>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/promote.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 20
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <typename _Fp>
+_LIBCPP_HIDE_FROM_ABI constexpr _Fp __lerp(_Fp __a, _Fp __b, _Fp __t) noexcept {
+  if ((__a <= 0 && __b >= 0) || (__a >= 0 && __b <= 0))
+    return __t * __b + (1 - __t) * __a;
+
+  if (__t == 1)
+    return __b;
+  const _Fp __x = __a + __t * (__b - __a);
+  if ((__t > 1) == (__b > __a))
+    return __b < __x ? __x : __b;
+  else
+    return __x < __b ? __x : __b;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr float lerp(float __a, float __b, float __t) _NOEXCEPT {
+  return __lerp(__a, __b, __t);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr double lerp(double __a, double __b, double __t) _NOEXCEPT {
+  return __lerp(__a, __b, __t);
+}
+
+_LIBCPP_HIDE_FROM_ABI inline constexpr long double lerp(long double __a, long double __b, long double __t) _NOEXCEPT {
+  return __lerp(__a, __b, __t);
+}
+
+template <class _A1, class _A2, class _A3>
+  requires(is_arithmetic_v<_A1> && is_arithmetic_v<_A2> && is_arithmetic_v<_A3>)
+_LIBCPP_HIDE_FROM_ABI inline constexpr __promote_t<_A1, _A2, _A3> lerp(_A1 __a, _A2 __b, _A3 __t) noexcept {
+  using __result_type = __promote_t<_A1, _A2, _A3>;
+  static_assert(!(
+      _IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value && _IsSame<_A3, __result_type>::value));
+  return std::__lerp((__result_type)__a, (__result_type)__b, (__result_type)__t);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 20
+
+#endif // _LIBCPP___CMATH_LERP_H
diff --git a/libcxx/include/cmath b/libcxx/include/cmath
index a4139fb8181f8..c4f4030c2049b 100644
--- a/libcxx/include/cmath
+++ b/libcxx/include/cmath
@@ -317,12 +317,10 @@ constexpr long double lerp(long double a, long double b, long double t) noexcept
 #else
 #  include <__config>
 #  include <__type_traits/enable_if.h>
-#  include <__type_traits/is_arithmetic.h>
 #  include <__type_traits/is_floating_point.h>
-#  include <__type_traits/is_same.h>
-#  include <__type_traits/promote.h>
 #  include <version>
 
+#  include <__cmath/lerp.h>
 #  include <__cmath/special_functions.h>
 #  include <math.h>
 
@@ -567,43 +565,6 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __constexpr_isinf(_A1 __lcpp_x) _NO
   return std::isinf(__lcpp_x);
 }
 
-#  if _LIBCPP_STD_VER >= 20
-template <typename _Fp>
-_LIBCPP_HIDE_FROM_ABI constexpr _Fp __lerp(_Fp __a, _Fp __b, _Fp __t) noexcept {
-  if ((__a <= 0 && __b >= 0) || (__a >= 0 && __b <= 0))
-    return __t * __b + (1 - __t) * __a;
-
-  if (__t == 1)
-    return __b;
-  const _Fp __x = __a + __t * (__b - __a);
-  if ((__t > 1) == (__b > __a))
-    return __b < __x ? __x : __b;
-  else
-    return __x < __b ? __x : __b;
-}
-
-_LIBCPP_HIDE_FROM_ABI inline constexpr float lerp(float __a, float __b, float __t) _NOEXCEPT {
-  return __lerp(__a, __b, __t);
-}
-
-_LIBCPP_HIDE_FROM_ABI inline constexpr double lerp(double __a, double __b, double __t) _NOEXCEPT {
-  return __lerp(__a, __b, __t);
-}
-
-_LIBCPP_HIDE_FROM_ABI inline constexpr long double lerp(long double __a, long double __b, long double __t) _NOEXCEPT {
-  return __lerp(__a, __b, __t);
-}
-
-template <class _A1, class _A2, class _A3>
-  requires(is_arithmetic_v<_A1> && is_arithmetic_v<_A2> && is_arithmetic_v<_A3>)
-_LIBCPP_HIDE_FROM_ABI inline constexpr __promote_t<_A1, _A2, _A3> lerp(_A1 __a, _A2 __b, _A3 __t) noexcept {
-  using __result_type = __promote_t<_A1, _A2, _A3>;
-  static_assert(!(
-      _IsSame<_A1, __result_type>::value && _IsSame<_A2, __result_type>::value && _IsSame<_A3, __result_type>::value));
-  return std::__lerp((__result_type)__a, (__result_type)__b, (__result_type)__t);
-}
-#  endif // _LIBCPP_STD_VER >= 20
-
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
diff --git a/libcxx/include/complex b/libcxx/include/complex
index 03c3bb4686e81..b407bb70156e3 100644
--- a/libcxx/include/complex
+++ b/libcxx/include/complex
@@ -266,6 +266,11 @@ template<class T> complex<T> tanh (const complex<T>&);
 #  include <__tuple/tuple_element.h>
 #  include <__tuple/tuple_size.h>
 #  include <__type_traits/conditional.h>
+#  include <__type_traits/is_arithmetic.h>
+#  include <__type_traits/is_floating_point.h>
+#  include <__type_traits/is_integral.h>
+#  include <__type_traits/is_same.h>
+#  include <__type_traits/promote.h>
 #  include <__utility/move.h>
 #  include <cmath>
 #  include <version>
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 8e568e04f4d69..a5eac9fc149f0 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -20,6 +20,7 @@ module std_config {
 
 module std_core {
   module cmath {
+    module lerp              { header "__cmath/lerp.h" }
     module special_functions { header "__cmath/special_functions.h" }
   }
 
diff --git a/libcxx/test/benchmarks/function.bench.cpp b/libcxx/test/benchmarks/function.bench.cpp
index e607162d23b72..eed071b1fbd64 100644
--- a/libcxx/test/benchmarks/function.bench.cpp
+++ b/libcxx/test/benchmarks/function.bench.cpp
@@ -224,4 +224,6 @@ int main(int argc, char** argv) {
   makeCartesianProductBenchmark<Invoke, AllFunctionTypes>();
   makeCartesianProductBenchmark<InvokeInlined, AllFunctionTypes>();
   benchmark::RunSpecifiedBenchmarks();
+
+  return 0;
 }
diff --git a/libcxx/test/benchmarks/utc_clock.bench.cpp b/libcxx/test/benchmarks/utc_clock.bench.cpp
index c44652a8f7ae0..6ff0ec59e4b7c 100644
--- a/libcxx/test/benchmarks/utc_clock.bench.cpp
+++ b/libcxx/test/benchmarks/utc_clock.bench.cpp
@@ -57,4 +57,6 @@ int main(int argc, char** argv) {
     return 1;
 
   benchmark::RunSpecifiedBenchmarks();
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx-03/containers/sequences/deque/asan.pass.cpp b/libcxx/test/libcxx-03/containers/sequences/deque/asan.pass.cpp
index 46ca62dda7b20..f2a0f4b8a9f07 100644
--- a/libcxx/test/libcxx-03/containers/sequences/deque/asan.pass.cpp
+++ b/libcxx/test/libcxx-03/containers/sequences/deque/asan.pass.cpp
@@ -65,4 +65,6 @@ int main(int, char**) {
     assert(false);                // if we got here, ASAN didn't trigger
     ((void)foo);
   }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx-03/containers/sequences/vector/asan.pass.cpp b/libcxx/test/libcxx-03/containers/sequences/vector/asan.pass.cpp
index 72875a52246c4..5fb31c5f1088e 100644
--- a/libcxx/test/libcxx-03/containers/sequences/vector/asan.pass.cpp
+++ b/libcxx/test/libcxx-03/containers/sequences/vector/asan.pass.cpp
@@ -50,4 +50,6 @@ int main(int, char**) {
     assert(false);                // if we got here, ASAN didn't trigger
     ((void)foo);
   }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx-03/memory/uninitialized_allocator_copy.pass.cpp b/libcxx/test/libcxx-03/memory/uninitialized_allocator_copy.pass.cpp
index 679ee86844687..545319050c6b7 100644
--- a/libcxx/test/libcxx-03/memory/uninitialized_allocator_copy.pass.cpp
+++ b/libcxx/test/libcxx-03/memory/uninitialized_allocator_copy.pass.cpp
@@ -64,4 +64,6 @@ int main(int, char**) {
 
   assert(constructed_count == 0);
   assert(max_constructed_count == 14);
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx-03/strings/string.view/string.view.iterators/assert.iterator-indexing.pass.cpp b/libcxx/test/libcxx-03/strings/string.view/string.view.iterators/assert.iterator-indexing.pass.cpp
index 5043a88cbc3da..c0b1c5073e63f 100644
--- a/libcxx/test/libcxx-03/strings/string.view/string.view.iterators/assert.iterator-indexing.pass.cpp
+++ b/libcxx/test/libcxx-03/strings/string.view/string.view.iterators/assert.iterator-indexing.pass.cpp
@@ -8,8 +8,8 @@
 
 // Make sure that std::string_view's iterators check for OOB accesses when the debug mode is enabled.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators
-// UNSUPPORTED: libcpp-hardening-mode=none
+// REQUIRES: can-test-hardening-assertions-fast
+// REQUIRES: libcpp-has-abi-bounded-iterators
 
 #include <iterator>
 #include <string_view>
diff --git a/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.shift/assert.shift_left.pass.cpp b/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.shift/assert.shift_left.pass.cpp
index 0d29ebc1b3390..da075e6249f11 100644
--- a/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.shift/assert.shift_left.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.shift/assert.shift_left.pass.cpp
@@ -8,10 +8,8 @@
 
 // <algorithm>
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <algorithm>
 #include <array>
diff --git a/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.shift/assert.shift_right.pass.cpp b/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.shift/assert.shift_right.pass.cpp
index 2c6f54304eacf..d964ad3a645c4 100644
--- a/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.shift/assert.shift_right.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.shift/assert.shift_right.pass.cpp
@@ -8,10 +8,8 @@
 
 // <algorithm>
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <algorithm>
 #include <array>
diff --git a/libcxx/test/libcxx/algorithms/alg.nonmodifying/alg.foreach/assert.for_each_n.pass.cpp b/libcxx/test/libcxx/algorithms/alg.nonmodifying/alg.foreach/assert.for_each_n.pass.cpp
index 5ae0dbc865b46..926b310dd21a7 100644
--- a/libcxx/test/libcxx/algorithms/alg.nonmodifying/alg.foreach/assert.for_each_n.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/alg.nonmodifying/alg.foreach/assert.for_each_n.pass.cpp
@@ -14,10 +14,8 @@
 //
 // [alg.foreach] requires `n >= 0`; passing a negative count is a precondition violation.
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <algorithm>
 
diff --git a/libcxx/test/libcxx/algorithms/alg.nonmodifying/alg.foreach/assert.ranges.for_each_n.pass.cpp b/libcxx/test/libcxx/algorithms/alg.nonmodifying/alg.foreach/assert.ranges.for_each_n.pass.cpp
index e57f342e760a6..3b4c3fc399cae 100644
--- a/libcxx/test/libcxx/algorithms/alg.nonmodifying/alg.foreach/assert.ranges.for_each_n.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/alg.nonmodifying/alg.foreach/assert.ranges.for_each_n.pass.cpp
@@ -15,10 +15,8 @@
 //
 // [alg.foreach] requires `n >= 0`; passing a negative count is a precondition violation.
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <algorithm>
 #include <array>
diff --git a/libcxx/test/libcxx/algorithms/alg.sorting/assert.min.max.pass.cpp b/libcxx/test/libcxx/algorithms/alg.sorting/assert.min.max.pass.cpp
index 7e765d7e84683..f8aa1b3135743 100644
--- a/libcxx/test/libcxx/algorithms/alg.sorting/assert.min.max.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/alg.sorting/assert.min.max.pass.cpp
@@ -8,10 +8,8 @@
 
 // <algorithm>
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <algorithm>
 #include <array>
diff --git a/libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator/assert.sort.invalid_comparator.oob.pass.cpp b/libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator/assert.sort.invalid_comparator.oob.pass.cpp
index 6ddee1b2aabe0..0250f84294d7b 100644
--- a/libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator/assert.sort.invalid_comparator.oob.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator/assert.sort.invalid_comparator.oob.pass.cpp
@@ -6,13 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // In the debug mode, the comparator validations will notice that it doesn't satisfy strict weak ordering before the
 // algorithm actually runs and goes out of bounds, so the test will terminate before the tested assertions are
 // triggered.
-// UNSUPPORTED: libcpp-hardening-mode=none, libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// UNSUPPORTED: libcpp-hardening-mode=debug
 
 #include <algorithm>
 #include <cassert>
diff --git a/libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator/assert.sort.invalid_comparator.pass.cpp b/libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator/assert.sort.invalid_comparator.pass.cpp
index 92671617eb032..5f0710140aa61 100644
--- a/libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator/assert.sort.invalid_comparator.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator/assert.sort.invalid_comparator.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// REQUIRES: libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // This test uses a specific combination of an invalid comparator and sequence of values to
 // ensure that our sorting functions do not go out-of-bounds and satisfy strict weak ordering in that case.
diff --git a/libcxx/test/libcxx/algorithms/debug_less.inconsistent.pass.cpp b/libcxx/test/libcxx/algorithms/debug_less.inconsistent.pass.cpp
index 701edd1335afa..3661484c73c97 100644
--- a/libcxx/test/libcxx/algorithms/debug_less.inconsistent.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/debug_less.inconsistent.pass.cpp
@@ -12,8 +12,7 @@
 
 // Make sure __debug_less asserts when the comparator is not consistent.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode=debug
-// UNSUPPORTED: c++03
+// REQUIRES: can-test-hardening-assertions-debug
 
 #include <algorithm>
 #include <iterator>
diff --git a/libcxx/test/libcxx/algorithms/debug_less.pass.cpp b/libcxx/test/libcxx/algorithms/debug_less.pass.cpp
index 4889f94d170a1..84cde29bffe84 100644
--- a/libcxx/test/libcxx/algorithms/debug_less.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/debug_less.pass.cpp
@@ -12,8 +12,7 @@
 
 // __debug_less checks that a comparator actually provides a strict-weak ordering.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode=debug
-// UNSUPPORTED: c++03
+// REQUIRES: can-test-hardening-assertions-debug
 
 #include <algorithm>
 #include <cassert>
diff --git a/libcxx/test/libcxx/algorithms/debug_three_way_comp.inconsistent.pass.cpp b/libcxx/test/libcxx/algorithms/debug_three_way_comp.inconsistent.pass.cpp
index b9bb1c9136f65..cd184573d2ef4 100644
--- a/libcxx/test/libcxx/algorithms/debug_three_way_comp.inconsistent.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/debug_three_way_comp.inconsistent.pass.cpp
@@ -12,7 +12,7 @@
 
 // Make sure __debug_three_way_comp asserts when the comparator is not consistent.
 
-// REQUIRES: libcpp-hardening-mode=debug
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 #include <algorithm>
diff --git a/libcxx/test/libcxx/assertions/modes/debug.pass.cpp b/libcxx/test/libcxx/assertions/modes/debug.pass.cpp
index ea9770b0b2fbc..6b4b7aba66f47 100644
--- a/libcxx/test/libcxx/assertions/modes/debug.pass.cpp
+++ b/libcxx/test/libcxx/assertions/modes/debug.pass.cpp
@@ -10,10 +10,7 @@
 // by default.
 
 // REQUIRES: libcpp-hardening-mode=debug
-// `check_assertion.h` is only available starting from C++11.
-// UNSUPPORTED: c++03
-// `check_assertion.h` requires Unix headers.
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions
 
 #include <cassert>
 #include "check_assertion.h"
diff --git a/libcxx/test/libcxx/assertions/modes/extensive.pass.cpp b/libcxx/test/libcxx/assertions/modes/extensive.pass.cpp
index 5743f95e472d7..24fd734528d45 100644
--- a/libcxx/test/libcxx/assertions/modes/extensive.pass.cpp
+++ b/libcxx/test/libcxx/assertions/modes/extensive.pass.cpp
@@ -10,10 +10,7 @@
 // has been enabled by default.
 
 // REQUIRES: libcpp-hardening-mode=extensive
-// `check_assertion.h` is only available starting from C++11.
-// UNSUPPORTED: c++03
-// `check_assertion.h` requires Unix headers.
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions
 
 #include <cassert>
 #include "check_assertion.h"
diff --git a/libcxx/test/libcxx/assertions/modes/fast.pass.cpp b/libcxx/test/libcxx/assertions/modes/fast.pass.cpp
index 85181859fdad0..8af86c36e34f9 100644
--- a/libcxx/test/libcxx/assertions/modes/fast.pass.cpp
+++ b/libcxx/test/libcxx/assertions/modes/fast.pass.cpp
@@ -10,10 +10,7 @@
 // been enabled by default.
 
 // REQUIRES: libcpp-hardening-mode=fast
-// `check_assertion.h` is only available starting from C++11.
-// UNSUPPORTED: c++03
-// `check_assertion.h` requires Unix headers.
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions
 
 #include <cassert>
 #include "check_assertion.h"
diff --git a/libcxx/test/libcxx/assertions/modes/override_with_debug_mode.pass.cpp b/libcxx/test/libcxx/assertions/modes/override_with_debug_mode.pass.cpp
index 02565d0b6a176..ec52a03689bd5 100644
--- a/libcxx/test/libcxx/assertions/modes/override_with_debug_mode.pass.cpp
+++ b/libcxx/test/libcxx/assertions/modes/override_with_debug_mode.pass.cpp
@@ -8,9 +8,7 @@
 
 // This test ensures that we can override any hardening mode with the debug mode on a per-TU basis.
 
-// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, no-localization
+// REQUIRES: can-test-hardening-assertions
 // The ability to set a custom abort message is required to compare the assertion message.
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_HARDENING_MODE -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG
diff --git a/libcxx/test/libcxx/assertions/modes/override_with_extensive_mode.pass.cpp b/libcxx/test/libcxx/assertions/modes/override_with_extensive_mode.pass.cpp
index 74fe70feb077c..c6c2891576eda 100644
--- a/libcxx/test/libcxx/assertions/modes/override_with_extensive_mode.pass.cpp
+++ b/libcxx/test/libcxx/assertions/modes/override_with_extensive_mode.pass.cpp
@@ -8,12 +8,7 @@
 
 // This test ensures that we can override any hardening mode with the extensive hardening mode on a per-TU basis.
 
-// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, no-localization
-// The ability to set a custom abort message is required to compare the assertion message (which only happens in the
-// debug mode).
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions
 // HWASAN replaces TRAP with abort or error exit code.
 // XFAIL: hwasan
 // ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_HARDENING_MODE -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE
diff --git a/libcxx/test/libcxx/assertions/modes/override_with_fast_mode.pass.cpp b/libcxx/test/libcxx/assertions/modes/override_with_fast_mode.pass.cpp
index f243897a986b0..eff7680cab063 100644
--- a/libcxx/test/libcxx/assertions/modes/override_with_fast_mode.pass.cpp
+++ b/libcxx/test/libcxx/assertions/modes/override_with_fast_mode.pass.cpp
@@ -8,12 +8,7 @@
 
 // This test ensures that we can override any hardening mode with the fast mode on a per-TU basis.
 
-// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, no-localization
-// The ability to set a custom abort message is required to compare the assertion message (which only happens in the
-// debug mode).
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions
 // HWASAN replaces TRAP with abort or error exit code.
 // XFAIL: hwasan
 // ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_HARDENING_MODE -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_FAST
diff --git a/libcxx/test/libcxx/assertions/modes/override_with_unchecked_mode.pass.cpp b/libcxx/test/libcxx/assertions/modes/override_with_unchecked_mode.pass.cpp
index 0922556c8dc01..7800d93acdb01 100644
--- a/libcxx/test/libcxx/assertions/modes/override_with_unchecked_mode.pass.cpp
+++ b/libcxx/test/libcxx/assertions/modes/override_with_unchecked_mode.pass.cpp
@@ -8,13 +8,10 @@
 
 // This test ensures that we can override any hardening mode with the unchecked mode on a per-TU basis.
 
-// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, no-localization
+// REQUIRES: stdlib=libc++
 // ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_HARDENING_MODE -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_NONE
 
-#include <cassert>
-#include "check_assertion.h"
+#include <__assert>
 
 int main(int, char**) {
   _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(true, "Should not fire");
diff --git a/libcxx/test/libcxx/assertions/semantics/override_with_enforce_semantic.pass.cpp b/libcxx/test/libcxx/assertions/semantics/override_with_enforce_semantic.pass.cpp
index 056864e1aea71..6ff03892f9cbc 100644
--- a/libcxx/test/libcxx/assertions/semantics/override_with_enforce_semantic.pass.cpp
+++ b/libcxx/test/libcxx/assertions/semantics/override_with_enforce_semantic.pass.cpp
@@ -9,10 +9,8 @@
 // This test ensures that we can override the assertion semantic used by any checked hardening mode with `enforce` on
 // a per-TU basis (this is valid for the `debug` mode as well, though a no-op).
 
-// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, no-localization
-// UNSUPPORTED: libcpp-hardening-mode=none, libcpp-has-no-experimental-hardening-observe-semantic
+// REQUIRES: can-test-hardening-assertions-fast
+// UNSUPPORTED: libcpp-has-no-experimental-hardening-observe-semantic
 // The ability to set a custom abort message is required to compare the assertion message.
 // XFAIL: availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_ASSERTION_SEMANTIC -D_LIBCPP_ASSERTION_SEMANTIC=_LIBCPP_ASSERTION_SEMANTIC_ENFORCE
diff --git a/libcxx/test/libcxx/assertions/semantics/override_with_ignore_semantic.pass.cpp b/libcxx/test/libcxx/assertions/semantics/override_with_ignore_semantic.pass.cpp
index b8c9028fe2e5c..d3f861960ab5c 100644
--- a/libcxx/test/libcxx/assertions/semantics/override_with_ignore_semantic.pass.cpp
+++ b/libcxx/test/libcxx/assertions/semantics/override_with_ignore_semantic.pass.cpp
@@ -9,14 +9,13 @@
 // This test ensures that we can override the assertion semantic used by any hardening mode with `ignore` on a per-TU
 // basis (this is valid for the `none` mode as well, though a no-op).
 
-// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, no-localization
 // UNSUPPORTED: libcpp-has-no-experimental-hardening-observe-semantic
+// assertion semantics require libc++ and C++11
+// UNSUPPORTED: c++03
+// REQUIRES: stdlib=libc++
 // ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_ASSERTION_SEMANTIC -D_LIBCPP_ASSERTION_SEMANTIC=_LIBCPP_ASSERTION_SEMANTIC_IGNORE
 
-#include <cassert>
-#include "check_assertion.h"
+#include <__assert>
 
 int main(int, char**) {
   _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(true, "Should not fire");
diff --git a/libcxx/test/libcxx/assertions/semantics/override_with_observe_semantic.pass.cpp b/libcxx/test/libcxx/assertions/semantics/override_with_observe_semantic.pass.cpp
index a14c44f5a8e73..63e255ed9b449 100644
--- a/libcxx/test/libcxx/assertions/semantics/override_with_observe_semantic.pass.cpp
+++ b/libcxx/test/libcxx/assertions/semantics/override_with_observe_semantic.pass.cpp
@@ -9,14 +9,13 @@
 // This test ensures that we can override the assertion semantic used by any checked hardening mode with `observe` on
 // a per-TU basis.
 
-// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, no-localization
-// UNSUPPORTED: libcpp-hardening-mode=none, libcpp-has-no-experimental-hardening-observe-semantic
+// UNSUPPORTED: libcpp-has-no-experimental-hardening-observe-semantic
+// assertion semantics require libc++ and C++11
+// UNSUPPORTED: c++03
+// REQUIRES: stdlib=libc++
 // ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_ASSERTION_SEMANTIC -D_LIBCPP_ASSERTION_SEMANTIC=_LIBCPP_ASSERTION_SEMANTIC_OBSERVE
 
-#include <cassert>
-#include "check_assertion.h"
+#include <__assert>
 
 int main(int, char**) {
   _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(true, "Should not fire");
diff --git a/libcxx/test/libcxx/assertions/semantics/override_with_quick_enforce_semantic.pass.cpp b/libcxx/test/libcxx/assertions/semantics/override_with_quick_enforce_semantic.pass.cpp
index be5038c4bb4ff..baa0cb5209201 100644
--- a/libcxx/test/libcxx/assertions/semantics/override_with_quick_enforce_semantic.pass.cpp
+++ b/libcxx/test/libcxx/assertions/semantics/override_with_quick_enforce_semantic.pass.cpp
@@ -9,10 +9,8 @@
 // This test ensures that we can override the assertion semantic used by any checked hardening mode with `quick-enforce`
 // on a per-TU basis (this is valid for the `fast` and `extensive` modes as well, though a no-op).
 
-// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, no-localization
-// UNSUPPORTED: libcpp-hardening-mode=none, libcpp-has-no-experimental-hardening-observe-semantic
+// REQUIRES: can-test-hardening-assertions-fast
+// UNSUPPORTED: libcpp-has-no-experimental-hardening-observe-semantic
 // ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_ASSERTION_SEMANTIC -D_LIBCPP_ASSERTION_SEMANTIC=_LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE
 
 #include <cassert>
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
index 92f6a622c329c..4eff6021fd900 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none || libcpp-hardening-mode=fast
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
 // <atomic>
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
index 3bee003b2143f..6597d8cd1d26f 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none || libcpp-hardening-mode=fast
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
 // <atomic>
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
index 3d4700406984c..31d0b296b546a 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none || libcpp-hardening-mode=fast
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <atomic>
 
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
index 504d135c4f3d7..b9ae0966939a6 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none || libcpp-hardening-mode=fast
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
 // <atomic>
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
index 1afa42528e14f..a3aa393652d39 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none || libcpp-hardening-mode=fast
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
 // <atomic>
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
index 39178d2393be2..c7e0fa94090b9 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none || libcpp-hardening-mode=fast
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-user-defined-warnings
 
 // <atomic>
diff --git a/libcxx/test/libcxx/containers/associative/debug.non-strict-weak-ordering.pass.cpp b/libcxx/test/libcxx/containers/associative/debug.non-strict-weak-ordering.pass.cpp
index 00a3a0730545d..81eb344dcbab8 100644
--- a/libcxx/test/libcxx/containers/associative/debug.non-strict-weak-ordering.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/debug.non-strict-weak-ordering.pass.cpp
@@ -12,7 +12,7 @@
 // This test ensures that libc++ detects when std::set or std::map are used with a
 // predicate that is not a strict weak ordering when the debug mode is enabled.
 
-// REQUIRES: libcpp-hardening-mode=debug
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14
 
 #include <map>
@@ -127,4 +127,6 @@ int main() {
     }
 #endif
   }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/associative/lower_upper_bound_non_strict_weak_order.pass.cpp b/libcxx/test/libcxx/containers/associative/lower_upper_bound_non_strict_weak_order.pass.cpp
index a3e2f3bb631b3..bebdabe3c8067 100644
--- a/libcxx/test/libcxx/containers/associative/lower_upper_bound_non_strict_weak_order.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/lower_upper_bound_non_strict_weak_order.pass.cpp
@@ -148,4 +148,6 @@ int main() {
       }
     }
   }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/container.adaptors/flat.map/assert.input_range.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.map/assert.input_range.pass.cpp
index 2db803b53441f..88feba3f94cb2 100644
--- a/libcxx/test/libcxx/containers/container.adaptors/flat.map/assert.input_range.pass.cpp
+++ b/libcxx/test/libcxx/containers/container.adaptors/flat.map/assert.input_range.pass.cpp
@@ -5,10 +5,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <flat_map>
 
diff --git a/libcxx/test/libcxx/containers/container.adaptors/flat.map/assert.sorted_unique.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.map/assert.sorted_unique.pass.cpp
index e6bd3f385af9c..34a234e5d5f44 100644
--- a/libcxx/test/libcxx/containers/container.adaptors/flat.map/assert.sorted_unique.pass.cpp
+++ b/libcxx/test/libcxx/containers/container.adaptors/flat.map/assert.sorted_unique.pass.cpp
@@ -5,11 +5,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// REQUIRES: libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <flat_map>
 
diff --git a/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.input_range.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.input_range.pass.cpp
index 504f36fcd00b8..2dc7c7521a1c1 100644
--- a/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.input_range.pass.cpp
+++ b/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.input_range.pass.cpp
@@ -5,10 +5,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <flat_map>
 
diff --git a/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.sorted_equivalent.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.sorted_equivalent.pass.cpp
index 6b8ad3c7ac9aa..892da60aedda5 100644
--- a/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.sorted_equivalent.pass.cpp
+++ b/libcxx/test/libcxx/containers/container.adaptors/flat.multimap/assert.sorted_equivalent.pass.cpp
@@ -5,11 +5,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// REQUIRES: libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <flat_map>
 
diff --git a/libcxx/test/libcxx/containers/container.adaptors/flat.multiset/assert.sorted_unique.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.multiset/assert.sorted_unique.pass.cpp
index 54b07baaff27a..b57f572872fe6 100644
--- a/libcxx/test/libcxx/containers/container.adaptors/flat.multiset/assert.sorted_unique.pass.cpp
+++ b/libcxx/test/libcxx/containers/container.adaptors/flat.multiset/assert.sorted_unique.pass.cpp
@@ -5,11 +5,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// REQUIRES: libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <flat_set>
 
diff --git a/libcxx/test/libcxx/containers/container.adaptors/flat.set/assert.sorted_unique.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.set/assert.sorted_unique.pass.cpp
index 62903af7f4e47..75d68d05a4c93 100644
--- a/libcxx/test/libcxx/containers/container.adaptors/flat.set/assert.sorted_unique.pass.cpp
+++ b/libcxx/test/libcxx/containers/container.adaptors/flat.set/assert.sorted_unique.pass.cpp
@@ -5,11 +5,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// REQUIRES: libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <flat_set>
 
diff --git a/libcxx/test/libcxx/containers/sequences/deque/asan.pass.cpp b/libcxx/test/libcxx/containers/sequences/deque/asan.pass.cpp
index 46ca62dda7b20..f2a0f4b8a9f07 100644
--- a/libcxx/test/libcxx/containers/sequences/deque/asan.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/deque/asan.pass.cpp
@@ -65,4 +65,6 @@ int main(int, char**) {
     assert(false);                // if we got here, ASAN didn't trigger
     ((void)foo);
   }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/deque/assert.pass.cpp b/libcxx/test/libcxx/containers/sequences/deque/assert.pass.cpp
index 375a4cdcd58fe..d2caf7e5b7674 100644
--- a/libcxx/test/libcxx/containers/sequences/deque/assert.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/deque/assert.pass.cpp
@@ -10,10 +10,7 @@
 
 // Test hardening assertions for std::deque.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <deque>
 
diff --git a/libcxx/test/libcxx/containers/sequences/deque/assert.pop_back.empty.pass.cpp b/libcxx/test/libcxx/containers/sequences/deque/assert.pop_back.empty.pass.cpp
index 6bdb117485609..b823410f21e81 100644
--- a/libcxx/test/libcxx/containers/sequences/deque/assert.pop_back.empty.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/deque/assert.pop_back.empty.pass.cpp
@@ -10,10 +10,7 @@
 
 // pop_back() more than the number of elements in a deque
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <deque>
 
diff --git a/libcxx/test/libcxx/containers/sequences/forwardlist/assert.pass.cpp b/libcxx/test/libcxx/containers/sequences/forwardlist/assert.pass.cpp
index 6d1748e645025..a83350655b49f 100644
--- a/libcxx/test/libcxx/containers/sequences/forwardlist/assert.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/forwardlist/assert.pass.cpp
@@ -10,10 +10,7 @@
 
 // Test hardening assertions for std::forward_list.
 
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <forward_list>
 
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.erase_iter.end.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.erase_iter.end.pass.cpp
index 6441d533cfa29..89684d518853e 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.erase_iter.end.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.erase_iter.end.pass.cpp
@@ -10,10 +10,7 @@
 
 // Call erase(const_iterator position) with end()
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <list>
 
diff --git a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.pop_back.empty.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.pop_back.empty.pass.cpp
index c344264852995..4ce50b6690c5c 100644
--- a/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.pop_back.empty.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/list.modifiers/assert.pop_back.empty.pass.cpp
@@ -10,10 +10,7 @@
 
 // void pop_back();
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <list>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/sequences/vector.bool/assert.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector.bool/assert.pass.cpp
index 41badad8f569d..1e987e22aba2c 100644
--- a/libcxx/test/libcxx/containers/sequences/vector.bool/assert.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector.bool/assert.pass.cpp
@@ -10,10 +10,7 @@
 
 // Test hardening assertions for std::vector<bool>.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <vector>
 
diff --git a/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp
index 03d2b3e6ce9b9..627a9e19f8127 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp
@@ -73,4 +73,6 @@ int main(int, char**) {
     assert(false);                // if we got here, ASAN didn't trigger
     ((void)foo);
   }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.back.empty.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.back.empty.pass.cpp
index 169ad1def9e6f..376e80fdb288a 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.back.empty.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.back.empty.pass.cpp
@@ -10,10 +10,7 @@
 
 // Call back() on empty container.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <vector>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.cback.empty.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.cback.empty.pass.cpp
index 5ceb4a16b9340..4423b6a9abe16 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.cback.empty.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.cback.empty.pass.cpp
@@ -10,10 +10,7 @@
 
 // Call back() on empty const container.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <vector>
 
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.cfront.empty.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.cfront.empty.pass.cpp
index 20f94f1d3f0fa..c036bb47948ba 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.cfront.empty.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.cfront.empty.pass.cpp
@@ -10,10 +10,7 @@
 
 // Call front() on empty const container.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <vector>
 
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.cindex.oob.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.cindex.oob.pass.cpp
index 3a9a7add3e30d..dae0d3127cdd5 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.cindex.oob.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.cindex.oob.pass.cpp
@@ -10,10 +10,7 @@
 
 // Index const vector out of bounds.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <vector>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.front.empty.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.front.empty.pass.cpp
index 85364c778ad64..ab6b343ea01cb 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.front.empty.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.front.empty.pass.cpp
@@ -10,10 +10,7 @@
 
 // Call front() on empty container.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <vector>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.index.oob.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.index.oob.pass.cpp
index 14cb89625f064..8cc6d0cf9ea02 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.index.oob.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.index.oob.pass.cpp
@@ -10,10 +10,7 @@
 
 // Index vector out of bounds.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <vector>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.add.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.add.pass.cpp
index a066ad30ebd71..e9e211e401af9 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.add.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.add.pass.cpp
@@ -10,8 +10,8 @@
 
 // Add to iterator out of bounds.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators-in-vector
-// UNSUPPORTED: libcpp-hardening-mode=none, c++03
+// REQUIRES: can-test-hardening-assertions-fast
+// REQUIRES: libcpp-has-abi-bounded-iterators-in-vector
 
 #include <vector>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.decrement.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.decrement.pass.cpp
index 59b9c16a6aa0e..f2887ba9a5fab 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.decrement.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.decrement.pass.cpp
@@ -10,8 +10,8 @@
 
 // Decrement iterator prior to begin.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators-in-vector
-// UNSUPPORTED: libcpp-hardening-mode=none, c++03
+// REQUIRES: can-test-hardening-assertions-fast
+// REQUIRES: libcpp-has-abi-bounded-iterators-in-vector
 
 #include <vector>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.dereference.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.dereference.pass.cpp
index 877d3655fbe2e..308f01b795dc3 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.dereference.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.dereference.pass.cpp
@@ -10,8 +10,8 @@
 
 // Dereference non-dereferenceable iterator.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators-in-vector
-// UNSUPPORTED: libcpp-hardening-mode=none, c++03
+// REQUIRES: can-test-hardening-assertions-fast
+// REQUIRES: libcpp-has-abi-bounded-iterators-in-vector
 
 #include <vector>
 
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.increment.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.increment.pass.cpp
index e540f40f8c476..073c1704e6a25 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.increment.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.increment.pass.cpp
@@ -10,8 +10,8 @@
 
 // Increment iterator past end.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators-in-vector
-// UNSUPPORTED: libcpp-hardening-mode=none, c++03
+// REQUIRES: can-test-hardening-assertions-fast
+// REQUIRES: libcpp-has-abi-bounded-iterators-in-vector
 
 #include <vector>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.index.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.index.pass.cpp
index 63354af5af022..3fc7022a7c684 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.index.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.iterator.index.pass.cpp
@@ -10,8 +10,8 @@
 
 // Index iterator out of bounds.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators-in-vector
-// UNSUPPORTED: libcpp-hardening-mode=none, c++03
+// REQUIRES: can-test-hardening-assertions-fast
+// REQUIRES: libcpp-has-abi-bounded-iterators-in-vector
 
 #include <vector>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/sequences/vector/assert.pop_back.empty.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/assert.pop_back.empty.pass.cpp
index 6734f25b9db10..bf94385f22140 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/assert.pop_back.empty.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/assert.pop_back.empty.pass.cpp
@@ -10,10 +10,7 @@
 
 // pop_back() more than the number of elements in a vector
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <vector>
 
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket.pass.cpp
index 26621bce52187..035f075f3a99a 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket.pass.cpp
@@ -10,10 +10,7 @@
 
 // size_type bucket(const key_type& __k) const;
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <string>
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket_size.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket_size.pass.cpp
index ce6534f95f8dc..50d97380ff4b0 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket_size.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/assert.bucket_size.pass.cpp
@@ -14,10 +14,7 @@
 
 // size_type bucket_size(size_type n) const
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <string>
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/assert.iterator.dereference.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/assert.iterator.dereference.pass.cpp
index f57341d64ff39..b349df9299a7c 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/assert.iterator.dereference.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/assert.iterator.dereference.pass.cpp
@@ -10,9 +10,7 @@
 
 // Dereference non-dereferenceable iterator.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <string>
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/assert.iterator.increment.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/assert.iterator.increment.pass.cpp
index 3f4d1c2d3bdbb..912a011a8f2c3 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/assert.iterator.increment.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/assert.iterator.increment.pass.cpp
@@ -10,9 +10,7 @@
 
 // Increment iterator past end.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/assert.local_iterator.dereference.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/assert.local_iterator.dereference.pass.cpp
index 8b47f54895560..2b77ff8c52d71 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/assert.local_iterator.dereference.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/assert.local_iterator.dereference.pass.cpp
@@ -10,9 +10,7 @@
 
 // Dereference non-dereferenceable iterator.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <string>
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/assert.local_iterator.increment.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/assert.local_iterator.increment.pass.cpp
index 8f8305833e077..64c121b4f007c 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/assert.local_iterator.increment.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/assert.local_iterator.increment.pass.cpp
@@ -10,9 +10,7 @@
 
 // Increment local_iterator past end.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <string>
diff --git a/libcxx/test/libcxx/containers/unord/unord.map/assert.max_load_factor.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.map/assert.max_load_factor.pass.cpp
index 117758c5385e0..be3063edebcd2 100644
--- a/libcxx/test/libcxx/containers/unord/unord.map/assert.max_load_factor.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.map/assert.max_load_factor.pass.cpp
@@ -15,10 +15,7 @@
 // float max_load_factor() const;
 // void max_load_factor(float mlf);
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <string>
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket.pass.cpp
index 61c1651fdd956..3cbf9d6bb5741 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket.pass.cpp
@@ -14,10 +14,7 @@
 
 // size_type bucket(const key_type& __k) const;
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <string>
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket_size.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket_size.pass.cpp
index 8f2efdb6e56e6..5efa862f1e458 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket_size.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.bucket_size.pass.cpp
@@ -14,10 +14,7 @@
 
 // size_type bucket_size(size_type n) const
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <string>
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.iterator.dereference.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.iterator.dereference.pass.cpp
index d295a82a8a1f5..d731d82e8fecf 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.iterator.dereference.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.iterator.dereference.pass.cpp
@@ -10,9 +10,7 @@
 
 // Dereference non-dereferenceable iterator.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <string>
 #include <unordered_map>
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.iterator.increment.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.iterator.increment.pass.cpp
index 4247edc8def97..bdb2bafa4a075 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.iterator.increment.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.iterator.increment.pass.cpp
@@ -10,9 +10,7 @@
 
 // Increment iterator past end.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.local_iterator.dereference.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.local_iterator.dereference.pass.cpp
index 7ea87964e05f0..81c07bd5b60a0 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.local_iterator.dereference.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.local_iterator.dereference.pass.cpp
@@ -10,9 +10,7 @@
 
 // Dereference non-dereferenceable iterator.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <string>
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.local_iterator.increment.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.local_iterator.increment.pass.cpp
index ffa3fec0ca1f1..097a6e4dbfeec 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.local_iterator.increment.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.local_iterator.increment.pass.cpp
@@ -10,9 +10,7 @@
 
 // Increment local_iterator past end.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.max_load_factor.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.max_load_factor.pass.cpp
index dda5fe8b632bd..4c378e98eaa42 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multimap/assert.max_load_factor.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multimap/assert.max_load_factor.pass.cpp
@@ -15,10 +15,7 @@
 // float max_load_factor() const;
 // void max_load_factor(float mlf);
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_map>
 #include <string>
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket.pass.cpp
index eebf11c27bb17..f36f7dd00d877 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket.pass.cpp
@@ -14,10 +14,7 @@
 
 // size_type bucket(const key_type& __k) const;
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket_size.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket_size.pass.cpp
index 1c107ca11193c..bb5bba3a196c2 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket_size.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.bucket_size.pass.cpp
@@ -14,10 +14,7 @@
 
 // size_type bucket_size(size_type n) const
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.iterator.dereference.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.iterator.dereference.pass.cpp
index 31edd6099c965..e1b6f4d9757c4 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.iterator.dereference.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.iterator.dereference.pass.cpp
@@ -10,9 +10,7 @@
 
 // Dereference non-dereferenceable iterator.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.iterator.increment.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.iterator.increment.pass.cpp
index 0e0e4aab303cd..aae10ea967386 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.iterator.increment.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.iterator.increment.pass.cpp
@@ -10,9 +10,7 @@
 
 // Increment iterator past end.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.local_iterator.dereference.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.local_iterator.dereference.pass.cpp
index fe833c40bc351..4b64cd2352212 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.local_iterator.dereference.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.local_iterator.dereference.pass.cpp
@@ -10,9 +10,7 @@
 
 // Dereference non-dereferenceable iterator.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.local_iterator.increment.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.local_iterator.increment.pass.cpp
index 142c07f83c066..40dec5818af85 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.local_iterator.increment.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.local_iterator.increment.pass.cpp
@@ -10,9 +10,7 @@
 
 // Increment local_iterator past end.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.max_load_factor.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.max_load_factor.pass.cpp
index a38849482cee9..6f0cabebe6ba9 100644
--- a/libcxx/test/libcxx/containers/unord/unord.multiset/assert.max_load_factor.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.multiset/assert.max_load_factor.pass.cpp
@@ -15,10 +15,7 @@
 // float max_load_factor() const;
 // void max_load_factor(float mlf);
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket.pass.cpp
index 6504a88a2e1f6..234fdd64b2635 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket.pass.cpp
@@ -14,10 +14,7 @@
 
 // size_type bucket(const key_type& __k) const;
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket_size.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket_size.pass.cpp
index 18440548d76c2..d07243054303f 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket_size.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/assert.bucket_size.pass.cpp
@@ -14,10 +14,7 @@
 
 // size_type bucket_size(size_type n) const
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/assert.iterator.dereference.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/assert.iterator.dereference.pass.cpp
index 8464601f61046..33fc2f19bf2eb 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/assert.iterator.dereference.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/assert.iterator.dereference.pass.cpp
@@ -10,9 +10,7 @@
 
 // Dereference non-dereferenceable iterator.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/assert.iterator.increment.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/assert.iterator.increment.pass.cpp
index 29446880900bc..2e3957e62dc7a 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/assert.iterator.increment.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/assert.iterator.increment.pass.cpp
@@ -10,9 +10,7 @@
 
 // Increment iterator past end.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/assert.local_iterator.dereference.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/assert.local_iterator.dereference.pass.cpp
index 7163e3735cee0..50a19edeb98a1 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/assert.local_iterator.dereference.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/assert.local_iterator.dereference.pass.cpp
@@ -10,9 +10,7 @@
 
 // Dereference non-dereferenceable iterator.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/assert.local_iterator.increment.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/assert.local_iterator.increment.pass.cpp
index c9fe5afd09702..eab305a28c42a 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/assert.local_iterator.increment.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/assert.local_iterator.increment.pass.cpp
@@ -10,9 +10,7 @@
 
 // Increment local_iterator past end.
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 #include <cassert>
diff --git a/libcxx/test/libcxx/containers/unord/unord.set/assert.max_load_factor.pass.cpp b/libcxx/test/libcxx/containers/unord/unord.set/assert.max_load_factor.pass.cpp
index 80e89411d99c3..639cbd00e35e6 100644
--- a/libcxx/test/libcxx/containers/unord/unord.set/assert.max_load_factor.pass.cpp
+++ b/libcxx/test/libcxx/containers/unord/unord.set/assert.max_load_factor.pass.cpp
@@ -15,10 +15,7 @@
 // float max_load_factor() const;
 // void max_load_factor(float mlf);
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <unordered_set>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/extents/assert.conversion.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/extents/assert.conversion.pass.cpp
index 31766e4c51c3b..8f7f24b78ce84 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/extents/assert.conversion.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/extents/assert.conversion.pass.cpp
@@ -4,10 +4,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_array.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_array.pass.cpp
index 90cb0c84a063b..14bd855293695 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_array.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_array.pass.cpp
@@ -4,10 +4,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_integral.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_integral.pass.cpp
index 37e79aabf8532..6fc846dddd5a3 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_integral.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_integral.pass.cpp
@@ -4,10 +4,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_span.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_span.pass.cpp
index 650fecf62128c..810827d556efd 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_span.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/extents/assert.ctor_from_span.pass.cpp
@@ -4,10 +4,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // Test construction from span:
 //
diff --git a/libcxx/test/libcxx/containers/views/mdspan/extents/assert.obs.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/extents/assert.obs.pass.cpp
index e32c0a96c1261..28dd554fc5031 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/extents/assert.obs.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/extents/assert.obs.pass.cpp
@@ -4,10 +4,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.conversion.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.conversion.pass.cpp
index 7b6616f19d724..be7951015512e 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.conversion.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.conversion.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.extents.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.extents.pass.cpp
index 7c96f8ec9353f..5a82f28b62583 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.extents.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.extents.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.layout_right.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.layout_right.pass.cpp
index e578bac2103b0..bfd3241ebce14 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.layout_right.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.layout_right.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.layout_stride.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.layout_stride.pass.cpp
index 0dcd6bd1c0312..cc3e018b76f9c 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.layout_stride.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.layout_stride.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-ctad-maybe-unsupported
 
 // FIXME: https://llvm.org/PR64719
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.index_operator.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.index_operator.pass.cpp
index 79e424fbb52cb..b8b37ea891bb7 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.index_operator.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.index_operator.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.stride.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.stride.pass.cpp
index 61a5149c6881a..c3ae57fbdd365 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.stride.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.stride.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.conversion.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.conversion.pass.cpp
index df16edb925407..88e07e396a2a1 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.conversion.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.conversion.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.extents.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.extents.pass.cpp
index 52095691f6d24..89f6f6463a7ef 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.extents.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.extents.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.layout_left.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.layout_left.pass.cpp
index 1757ddb286b9c..a9e1bd2aa904c 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.layout_left.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.layout_left.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.layout_stride.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.layout_stride.pass.cpp
index 05b3349b7249a..40e866fc2f658 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.layout_stride.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.layout_stride.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // ADDITIONAL_COMPILE_FLAGS: -Wno-ctad-maybe-unsupported
 
 // FIXME: https://llvm.org/PR64719
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.index_operator.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.index_operator.pass.cpp
index 7fae6f87caf7c..8bf36186f3b2d 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.index_operator.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.index_operator.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.stride.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.stride.pass.cpp
index 8e3049c4736f0..7d7b99c879eee 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.stride.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.stride.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.conversion.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.conversion.pass.cpp
index 7deb1215de0de..8ce017dee45d6 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.conversion.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.conversion.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 // XFAIL: libcpp-hardening-mode=debug && target=powerpc{{.*}}le-unknown-linux-gnu
 
 // <mdspan>
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.non_unique.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.non_unique.pass.cpp
index 97a6d56e4f839..9b7485a5a1310 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.non_unique.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.non_unique.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: !libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.pass.cpp
index 860849ded2de2..c5c6a7b066acb 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.non_unique.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.non_unique.pass.cpp
index ed2e475ccf022..30b3c893ad026 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.non_unique.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.non_unique.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: !libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.pass.cpp
index 70ab0616ec01b..4aa3ff868d144 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.index_operator.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.index_operator.pass.cpp
index b5244a60af0f7..b116dd2922a4e 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.index_operator.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.index_operator.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: !libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.stride.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.stride.pass.cpp
index ee2b731da203b..f9307044a2d13 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.stride.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.stride.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.conversion.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.conversion.pass.cpp
index 53aec7bb714ea..97c716cf58bd9 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.conversion.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.conversion.pass.cpp
@@ -4,10 +4,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.index_operator.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.index_operator.pass.cpp
index fa3429dc4c4ff..85c8fde5dd5db 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.index_operator.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.index_operator.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.size.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.size.pass.cpp
index 739da8ece81d7..4654e00cc80ed 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.size.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.size.pass.cpp
@@ -4,10 +4,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <mdspan>
 
diff --git a/libcxx/test/libcxx/containers/views/views.span/assert.iterator-indexing.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/assert.iterator-indexing.pass.cpp
index d4dacb1f2f1c7..f86a4250bbe5f 100644
--- a/libcxx/test/libcxx/containers/views/views.span/assert.iterator-indexing.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/assert.iterator-indexing.pass.cpp
@@ -5,12 +5,12 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // Make sure that std::span's iterators check for OOB accesses when the debug mode is enabled.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators
-// UNSUPPORTED: libcpp-hardening-mode=none
+// REQUIRES: libcpp-has-abi-bounded-iterators
 
 #include <span>
 
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_sent.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_sent.pass.cpp
index 64e3cc8750cd1..ce4480cdfa92e 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_sent.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_sent.pass.cpp
@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <span>
@@ -18,10 +19,6 @@
 //
 // Check that we ensure that `[it, sent)` is a valid range.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 #include <array>
 #include <span>
 
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_size.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_size.pass.cpp
index c8c6e3743bd21..a65a7ea65881c 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_size.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.iter_size.pass.cpp
@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <span>
@@ -15,10 +16,6 @@
 // Note that it doesn't make sense to validate the incoming size in the
 // dynamic_extent version.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 #include <array>
 #include <span>
 
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.other_span.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.other_span.pass.cpp
index 9a4e65b0af903..ea3cb39e2b0d8 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.other_span.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.other_span.pass.cpp
@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <span>
@@ -13,10 +14,6 @@
 //
 // Check that we ensure `other.size() == Extent`.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 #include <array>
 #include <span>
 
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.range.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.range.pass.cpp
index 5e36e9a0418ae..527a660cb1980 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.range.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.cons/assert.range.pass.cpp
@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <span>
@@ -13,10 +14,6 @@
 //
 // Check that we ensure `size(r) == Extent`.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 #include <span>
 #include <vector>
 
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.back.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.back.pass.cpp
index 7e656b7f0b6e9..9282069b42119 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.back.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.back.pass.cpp
@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <span>
@@ -13,10 +14,6 @@
 
 // Make sure that accessing a span out-of-bounds triggers an assertion.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 #include <array>
 #include <span>
 
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.front.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.front.pass.cpp
index 0068aad8cc346..0ccdbb7696dd1 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.front.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.front.pass.cpp
@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <span>
@@ -13,10 +14,6 @@
 
 // Make sure that accessing a span out-of-bounds triggers an assertion.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 #include <array>
 #include <span>
 
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.op_idx.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.op_idx.pass.cpp
index 501067f740788..b582a92061954 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.op_idx.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.elem/assert.op_idx.pass.cpp
@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <span>
@@ -13,10 +14,6 @@
 
 // Make sure that accessing a span out-of-bounds triggers an assertion.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 #include <array>
 #include <span>
 
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.first.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.first.pass.cpp
index f241f2a5aadf2..3e13104c632cc 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.first.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.first.pass.cpp
@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <span>
@@ -13,10 +14,6 @@
 
 // Make sure that creating a sub-span with an incorrect number of elements triggers an assertion.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 #include <array>
 #include <span>
 
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.last.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.last.pass.cpp
index 032df689b1b88..d7550942f3382 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.last.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.last.pass.cpp
@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <span>
@@ -13,10 +14,6 @@
 
 // Make sure that creating a sub-span with an incorrect number of elements triggers an assertion.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 #include <array>
 #include <span>
 
diff --git a/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.subspan.pass.cpp b/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.subspan.pass.cpp
index 5dd0fa0530189..55758033442ec 100644
--- a/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.subspan.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/views.span/span.sub/assert.subspan.pass.cpp
@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <span>
@@ -21,10 +22,6 @@
 
 // Make sure that creating a sub-span with an incorrect number of elements triggers an assertion.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 #include <array>
 #include <span>
 #include <cstddef>
diff --git a/libcxx/test/libcxx/input.output/filesystems/class.path/path.itr/assert.iterator.pass.cpp b/libcxx/test/libcxx/input.output/filesystems/class.path/path.itr/assert.iterator.pass.cpp
index 3ca97ae8e9a4f..d15f71166731e 100644
--- a/libcxx/test/libcxx/input.output/filesystems/class.path/path.itr/assert.iterator.pass.cpp
+++ b/libcxx/test/libcxx/input.output/filesystems/class.path/path.itr/assert.iterator.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14
-// UNSUPPORTED: !libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <filesystem>
 
diff --git a/libcxx/test/libcxx/input.output/iostream.format/print.fun/output_unicode_windows.pass.cpp b/libcxx/test/libcxx/input.output/iostream.format/print.fun/output_unicode_windows.pass.cpp
index 160b0d5dc11b4..bcaecafe70199 100644
--- a/libcxx/test/libcxx/input.output/iostream.format/print.fun/output_unicode_windows.pass.cpp
+++ b/libcxx/test/libcxx/input.output/iostream.format/print.fun/output_unicode_windows.pass.cpp
@@ -108,4 +108,6 @@ static void test() {
 int main(int, char**) {
   test_basics();
   test();
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/iterators/assert.advance.pass.cpp b/libcxx/test/libcxx/iterators/assert.advance.pass.cpp
index a7e8878b933b2..692aca1d511eb 100644
--- a/libcxx/test/libcxx/iterators/assert.advance.pass.cpp
+++ b/libcxx/test/libcxx/iterators/assert.advance.pass.cpp
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 // <list>
 
diff --git a/libcxx/test/libcxx/iterators/assert.next.pass.cpp b/libcxx/test/libcxx/iterators/assert.next.pass.cpp
index 2e0296b72d124..b8fae3c326d86 100644
--- a/libcxx/test/libcxx/iterators/assert.next.pass.cpp
+++ b/libcxx/test/libcxx/iterators/assert.next.pass.cpp
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 // <list>
 
diff --git a/libcxx/test/libcxx/iterators/assert.prev.pass.cpp b/libcxx/test/libcxx/iterators/assert.prev.pass.cpp
index deac1edf59e06..e6f6290cc2888 100644
--- a/libcxx/test/libcxx/iterators/assert.prev.pass.cpp
+++ b/libcxx/test/libcxx/iterators/assert.prev.pass.cpp
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 // <list>
 
diff --git a/libcxx/test/libcxx/iterators/bounded_iter/dereference.pass.cpp b/libcxx/test/libcxx/iterators/bounded_iter/dereference.pass.cpp
index 7e3a59a49ffd4..7db62dbeb2fa8 100644
--- a/libcxx/test/libcxx/iterators/bounded_iter/dereference.pass.cpp
+++ b/libcxx/test/libcxx/iterators/bounded_iter/dereference.pass.cpp
@@ -11,10 +11,7 @@
 //
 // Dereference and indexing operators
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <__iterator/bounded_iter.h>
 
diff --git a/libcxx/test/libcxx/iterators/capacity_aware_iter/assert.pass.cpp b/libcxx/test/libcxx/iterators/capacity_aware_iter/assert.pass.cpp
index ceac20d549c34..d7e077360e26a 100644
--- a/libcxx/test/libcxx/iterators/capacity_aware_iter/assert.pass.cpp
+++ b/libcxx/test/libcxx/iterators/capacity_aware_iter/assert.pass.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: std-at-least-c++26
-// UNSUPPORTED: libcpp-hardening-mode=none
+// REQUIRES: can-test-hardening-assertions-fast
 
 // template <class _Iterator, class _Container, class _ContainerMaxElements>
 // struct __capacity_aware_iterator;
diff --git a/libcxx/test/libcxx/iterators/predef.iterators/counted.iterator/assert.pass.cpp b/libcxx/test/libcxx/iterators/predef.iterators/counted.iterator/assert.pass.cpp
index 2fafe4727185d..7436ea7bd87fc 100644
--- a/libcxx/test/libcxx/iterators/predef.iterators/counted.iterator/assert.pass.cpp
+++ b/libcxx/test/libcxx/iterators/predef.iterators/counted.iterator/assert.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <iterator>
 
diff --git a/libcxx/test/libcxx/iterators/predef.iterators/iterators.common/assert.pass.cpp b/libcxx/test/libcxx/iterators/predef.iterators/iterators.common/assert.pass.cpp
index 01c0fb4048320..6d840c7a14651 100644
--- a/libcxx/test/libcxx/iterators/predef.iterators/iterators.common/assert.pass.cpp
+++ b/libcxx/test/libcxx/iterators/predef.iterators/iterators.common/assert.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <iterator>
 
diff --git a/libcxx/test/libcxx/mem/mem.res/ctor.nullptr.assert.pass.cpp b/libcxx/test/libcxx/mem/mem.res/ctor.nullptr.assert.pass.cpp
index 831985f2ce37f..c14eb85323183 100644
--- a/libcxx/test/libcxx/mem/mem.res/ctor.nullptr.assert.pass.cpp
+++ b/libcxx/test/libcxx/mem/mem.res/ctor.nullptr.assert.pass.cpp
@@ -10,10 +10,8 @@
 
 // Test hardening assertions for std::pmr::polymorphic_allocator.
 
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // We're testing nullptr assertions
 // ADDITIONAL_COMPILE_FLAGS: -Wno-nonnull
diff --git a/libcxx/test/libcxx/memory/uninitialized_allocator_copy.pass.cpp b/libcxx/test/libcxx/memory/uninitialized_allocator_copy.pass.cpp
index 1d127f947c1da..83f9973f9489c 100644
--- a/libcxx/test/libcxx/memory/uninitialized_allocator_copy.pass.cpp
+++ b/libcxx/test/libcxx/memory/uninitialized_allocator_copy.pass.cpp
@@ -65,4 +65,6 @@ int main(int, char**) {
 
   assert(constructed_count == 0);
   assert(max_constructed_count == 14);
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/numerics/numarray/assert.pass.cpp b/libcxx/test/libcxx/numerics/numarray/assert.pass.cpp
index 2bdf52340abfc..a89e85f2612a5 100644
--- a/libcxx/test/libcxx/numerics/numarray/assert.pass.cpp
+++ b/libcxx/test/libcxx/numerics/numarray/assert.pass.cpp
@@ -10,10 +10,7 @@
 
 // Test hardening assertions for std::valarray.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <valarray>
 
diff --git a/libcxx/test/libcxx/numerics/numarray/class.gslice.array/assert.get.pass.cpp b/libcxx/test/libcxx/numerics/numarray/class.gslice.array/assert.get.pass.cpp
index f883f87b05df0..e3610af9e2dcf 100644
--- a/libcxx/test/libcxx/numerics/numarray/class.gslice.array/assert.get.pass.cpp
+++ b/libcxx/test/libcxx/numerics/numarray/class.gslice.array/assert.get.pass.cpp
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 // <valarray>
 
diff --git a/libcxx/test/libcxx/numerics/numarray/class.indirect.array/assert.get.pass.cpp b/libcxx/test/libcxx/numerics/numarray/class.indirect.array/assert.get.pass.cpp
index 8bf747e6aa7f6..cc709d8805e95 100644
--- a/libcxx/test/libcxx/numerics/numarray/class.indirect.array/assert.get.pass.cpp
+++ b/libcxx/test/libcxx/numerics/numarray/class.indirect.array/assert.get.pass.cpp
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 // <valarray>
 
diff --git a/libcxx/test/libcxx/numerics/numarray/class.mask.array/assert.get.pass.cpp b/libcxx/test/libcxx/numerics/numarray/class.mask.array/assert.get.pass.cpp
index a2d284830c1a5..625cdda298538 100644
--- a/libcxx/test/libcxx/numerics/numarray/class.mask.array/assert.get.pass.cpp
+++ b/libcxx/test/libcxx/numerics/numarray/class.mask.array/assert.get.pass.cpp
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 // <valarray>
 
diff --git a/libcxx/test/libcxx/numerics/numarray/class.slice.array/assert.get.pass.cpp b/libcxx/test/libcxx/numerics/numarray/class.slice.array/assert.get.pass.cpp
index 0dbad84486976..bf384c426126f 100644
--- a/libcxx/test/libcxx/numerics/numarray/class.slice.array/assert.get.pass.cpp
+++ b/libcxx/test/libcxx/numerics/numarray/class.slice.array/assert.get.pass.cpp
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 // <valarray>
 
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/assert.begin.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/assert.begin.pass.cpp
index 57af366317091..1912bfff70cdb 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/assert.begin.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/assert.begin.pass.cpp
@@ -6,11 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 // UNSUPPORTED: no-exceptions
-// UNSUPPORTED: !libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <ranges>
 
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/assert.find-next.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/assert.find-next.pass.cpp
index 1c91ee7198952..ed82c72c4b9fb 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/assert.find-next.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/assert.find-next.pass.cpp
@@ -6,11 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 // UNSUPPORTED: no-exceptions
-// UNSUPPORTED: !libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <ranges>
 
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/assert.find-prev.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/assert.find-prev.pass.cpp
index 2605bf6dde074..4f43efab350f0 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/assert.find-prev.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/assert.find-prev.pass.cpp
@@ -6,11 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 // UNSUPPORTED: no-exceptions
-// UNSUPPORTED: !libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <ranges>
 
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/range.chunk.by.iter/assert.deref.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/range.chunk.by.iter/assert.deref.pass.cpp
index 8ed84ca8b56a1..550e79d0f869e 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/range.chunk.by.iter/assert.deref.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/range.chunk.by.iter/assert.deref.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: !libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <ranges>
 
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/range.chunk.by.iter/assert.increment.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/range.chunk.by.iter/assert.increment.pass.cpp
index 1a804b71b5e5e..d1e1584fa5d62 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/range.chunk.by.iter/assert.increment.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.chunk.by/range.chunk.by.iter/assert.increment.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: !libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <ranges>
 
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.concat/iterator.valueless_by_exception.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.concat/iterator.valueless_by_exception.pass.cpp
index 8e0ff79e16a27..afc8b0dcc6e04 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.concat/iterator.valueless_by_exception.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.concat/iterator.valueless_by_exception.pass.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
+// REQUIRES: can-test-hardening-assertions-extensive
 // REQUIRES: std-at-least-c++26
-// UNSUPPORTED: libcpp-hardening-mode=none
 
 #include <iostream>
 #include <ranges>
@@ -656,4 +655,6 @@ int main() {
           [&] { [[maybe_unused]] CIter it3(iter1); }(), "Trying to convert from a valueless iterator of concat_view.");
     }
   }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.drop.while/assert.begin.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.drop.while/assert.begin.pass.cpp
index 205cf40746207..1ed7c0cb3041d 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.drop.while/assert.begin.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.drop.while/assert.begin.pass.cpp
@@ -10,11 +10,9 @@
 
 // Call begin() on drop_while_view with empty predicate
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // UNSUPPORTED: no-exceptions
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <ranges>
 
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.lazy.split/range.lazy.split.inner/assert.equal.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.lazy.split/range.lazy.split.inner/assert.equal.pass.cpp
index 22ede4143ffa4..5d047914299f0 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.lazy.split/range.lazy.split.inner/assert.equal.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.lazy.split/range.lazy.split.inner/assert.equal.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <ranges>
 
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.lazy.split/range.lazy.split.outer/assert.equal.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.lazy.split/range.lazy.split.outer/assert.equal.pass.cpp
index b6cbf5241f744..7b68fef4c3ccd 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.lazy.split/range.lazy.split.outer/assert.equal.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.lazy.split/range.lazy.split.outer/assert.equal.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <ranges>
 
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/ctor.assert.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/ctor.assert.pass.cpp
index e564891b15a15..8ed155dac34f8 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/ctor.assert.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/ctor.assert.pass.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL:libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // constexpr explicit stride_view(_View, range_difference_t<_View>)
 
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/iterator/dereference.assert.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/iterator/dereference.assert.pass.cpp
index 6884c27f5197c..8e0dfe905d7f0 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/iterator/dereference.assert.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/iterator/dereference.assert.pass.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// REQUIRES: libcpp-hardening-mode={{fast|extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // constexpr decltype(auto) operator*() const
 
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/iterator/increment.assert.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/iterator/increment.assert.pass.cpp
index 1c580fea4a310..e600c0bf93a5a 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/iterator/increment.assert.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/iterator/increment.assert.pass.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// REQUIRES: libcpp-hardening-mode={{fast|extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // constexpr __iterator& operator++()
 // constexpr void operator++(int)
diff --git a/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/iterator/operator_plus_equal.assert.pass.cpp b/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/iterator/operator_plus_equal.assert.pass.cpp
index 0a84436444cd5..4dfd268874a18 100644
--- a/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/iterator/operator_plus_equal.assert.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.adaptors/range.stride.view/iterator/operator_plus_equal.assert.pass.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// REQUIRES: libcpp-hardening-mode={{fast|extensive|debug}}
-// XFAIL:libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // constexpr __iterator& operator+=(difference_type __n)
 
diff --git a/libcxx/test/libcxx/ranges/range.factories/range.repeat.view/ctor.piecewise.pass.cpp b/libcxx/test/libcxx/ranges/range.factories/range.repeat.view/ctor.piecewise.pass.cpp
index 6d4e541d98906..0bed88c342a77 100644
--- a/libcxx/test/libcxx/ranges/range.factories/range.repeat.view/ctor.piecewise.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.factories/range.repeat.view/ctor.piecewise.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// REQUIRES: has-unix-headers
-// XFAIL: availability-verbose_abort-missing
 
 // template<class... TArgs, class... BoundArgs>
 //       requires constructible_from<T, TArgs...> &&
diff --git a/libcxx/test/libcxx/ranges/range.factories/range.repeat.view/ctor.value.bound.pass.cpp b/libcxx/test/libcxx/ranges/range.factories/range.repeat.view/ctor.value.bound.pass.cpp
index 2e9c74bc35978..9371892e16a42 100644
--- a/libcxx/test/libcxx/ranges/range.factories/range.repeat.view/ctor.value.bound.pass.cpp
+++ b/libcxx/test/libcxx/ranges/range.factories/range.repeat.view/ctor.value.bound.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// REQUIRES: has-unix-headers
-// XFAIL: availability-verbose_abort-missing
 
 // constexpr explicit repeat_view(W&& value, Bound bound = Bound());
 // constexpr explicit repeat_view(const W& value, Bound bound = Bound());
diff --git a/libcxx/test/libcxx/strings/basic.string/string.access/assert.back.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.access/assert.back.pass.cpp
index 36a485a1e4d00..5e3f8545b565b 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.access/assert.back.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.access/assert.back.pass.cpp
@@ -10,10 +10,7 @@
 
 // Call back() on empty container.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <string>
 
diff --git a/libcxx/test/libcxx/strings/basic.string/string.access/assert.cback.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.access/assert.cback.pass.cpp
index d810acd67e7e7..36969a99b79fb 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.access/assert.cback.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.access/assert.cback.pass.cpp
@@ -10,10 +10,7 @@
 
 // Call back() on empty const container.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <string>
 
diff --git a/libcxx/test/libcxx/strings/basic.string/string.access/assert.cfront.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.access/assert.cfront.pass.cpp
index 12e7ef3328b04..d1673fa765e7b 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.access/assert.cfront.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.access/assert.cfront.pass.cpp
@@ -10,10 +10,7 @@
 
 // Call front() on empty const container.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <string>
 
diff --git a/libcxx/test/libcxx/strings/basic.string/string.access/assert.cindex.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.access/assert.cindex.pass.cpp
index 3983352712963..22705d483bd98 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.access/assert.cindex.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.access/assert.cindex.pass.cpp
@@ -10,10 +10,7 @@
 
 // Index const string out of bounds.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <string>
 #include <cassert>
diff --git a/libcxx/test/libcxx/strings/basic.string/string.access/assert.front.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.access/assert.front.pass.cpp
index 24df3fcad0c5c..950cba5820cd7 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.access/assert.front.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.access/assert.front.pass.cpp
@@ -10,10 +10,7 @@
 
 // Call front() on empty container.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <string>
 #include <cassert>
diff --git a/libcxx/test/libcxx/strings/basic.string/string.access/assert.index.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.access/assert.index.pass.cpp
index d26997d8d24c2..b8b1be075b186 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.access/assert.index.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.access/assert.index.pass.cpp
@@ -10,10 +10,7 @@
 
 // Index string out of bounds.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <string>
 #include <cassert>
diff --git a/libcxx/test/libcxx/strings/basic.string/string.cons/debug.iterator.substr.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.cons/debug.iterator.substr.pass.cpp
index b237a8e91336a..fedf8ebeb6147 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.cons/debug.iterator.substr.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.cons/debug.iterator.substr.pass.cpp
@@ -46,4 +46,6 @@ int main(int, char**) {
     assert(i[0] == 'l');
     TEST_LIBCPP_ASSERT_FAILURE(i[5], "Attempted to subscript an iterator outside its valid range");
   }
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.add.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.add.pass.cpp
index 56c9d63d0dbaf..5bfb376bcacf0 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.add.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.add.pass.cpp
@@ -10,8 +10,8 @@
 
 // Add to iterator out of bounds.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators-in-string
-// UNSUPPORTED: libcpp-hardening-mode=none, c++03
+// REQUIRES: can-test-hardening-assertions-fast
+// REQUIRES: libcpp-has-abi-bounded-iterators-in-string
 
 #include <string>
 #include <cassert>
diff --git a/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.decrement.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.decrement.pass.cpp
index 43a9739bf936f..447105f04a255 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.decrement.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.decrement.pass.cpp
@@ -10,8 +10,8 @@
 
 // Decrement iterator prior to begin.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators-in-string
-// UNSUPPORTED: libcpp-hardening-mode=none, c++03
+// REQUIRES: can-test-hardening-assertions-fast
+// REQUIRES: libcpp-has-abi-bounded-iterators-in-string
 
 #include <string>
 #include <cassert>
diff --git a/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.dereference.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.dereference.pass.cpp
index e2326be021033..f0fce12580664 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.dereference.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.dereference.pass.cpp
@@ -10,8 +10,8 @@
 
 // Dereference non-dereferenceable iterator.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators-in-string
-// UNSUPPORTED: libcpp-hardening-mode=none, c++03
+// REQUIRES: can-test-hardening-assertions-fast
+// REQUIRES: libcpp-has-abi-bounded-iterators-in-string
 
 #include <string>
 
diff --git a/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.increment.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.increment.pass.cpp
index a7453f3115197..5ba020b87d2c1 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.increment.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.increment.pass.cpp
@@ -10,8 +10,8 @@
 
 // Increment iterator past end.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators-in-string
-// UNSUPPORTED: libcpp-hardening-mode=none, c++03
+// REQUIRES: can-test-hardening-assertions-fast
+// REQUIRES: libcpp-has-abi-bounded-iterators-in-string
 
 #include <string>
 #include <cassert>
diff --git a/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.index.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.index.pass.cpp
index e7d384413b589..9688edc9bdcbf 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.index.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.iterators/assert.iterator.index.pass.cpp
@@ -10,8 +10,8 @@
 
 // Index iterator out of bounds.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators-in-string
-// UNSUPPORTED: libcpp-hardening-mode=none, c++03
+// REQUIRES: can-test-hardening-assertions-fast
+// REQUIRES: libcpp-has-abi-bounded-iterators-in-string
 
 #include <string>
 #include <cassert>
diff --git a/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.append.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.append.pass.cpp
index c3745ca0a762d..13add1ee6483a 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.append.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.append.pass.cpp
@@ -12,11 +12,8 @@
 // basic_string& append(const value_type* s);
 // basic_string& append(const value_type* s, size_type pos, size_type n);
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode={{none|fast}}
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: libcpp-assertion-semantic={{ignore|observe}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <string>
 
diff --git a/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.assign.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.assign.pass.cpp
index 80abd15929735..75717d200e49b 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.assign.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.assign.pass.cpp
@@ -12,11 +12,8 @@
 // basic_string& assign(const value_type* s);
 // basic_string& assign(const value_type* s, size_type pos, size_type n);
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode={{none|fast}}
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: libcpp-assertion-semantic={{ignore|observe}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <string>
 
diff --git a/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.erase_iter.null.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.erase_iter.null.pass.cpp
index 036e75965c488..161f9cf7be52d 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.erase_iter.null.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.erase_iter.null.pass.cpp
@@ -10,10 +10,7 @@
 
 // Call erase(const_iterator position) with end()
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <string>
 
diff --git a/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.pop_back.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.pop_back.pass.cpp
index 54c011c4d54a0..b7863b990227a 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.pop_back.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.modifiers/assert.pop_back.pass.cpp
@@ -10,10 +10,7 @@
 
 // void pop_back();
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <string>
 
diff --git a/libcxx/test/libcxx/strings/string.view/assert.ctor.length.pass.cpp b/libcxx/test/libcxx/strings/string.view/assert.ctor.length.pass.cpp
index e47b5f5963109..df898cefa6b38 100644
--- a/libcxx/test/libcxx/strings/string.view/assert.ctor.length.pass.cpp
+++ b/libcxx/test/libcxx/strings/string.view/assert.ctor.length.pass.cpp
@@ -6,11 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++11 && gcc
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // Construct a string_view from an invalid length
 // constexpr basic_string_view( const _CharT* s, size_type len )
diff --git a/libcxx/test/libcxx/strings/string.view/assert.ctor.pointer.pass.cpp b/libcxx/test/libcxx/strings/string.view/assert.ctor.pointer.pass.cpp
index f358b5efd0df2..23e73006265ab 100644
--- a/libcxx/test/libcxx/strings/string.view/assert.ctor.pointer.pass.cpp
+++ b/libcxx/test/libcxx/strings/string.view/assert.ctor.pointer.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // Construct a string_view from a null pointer
 // constexpr basic_string_view( const CharT* s );
diff --git a/libcxx/test/libcxx/strings/string.view/string.view.iterators/assert.iterator-indexing.pass.cpp b/libcxx/test/libcxx/strings/string.view/string.view.iterators/assert.iterator-indexing.pass.cpp
index 5043a88cbc3da..c0b1c5073e63f 100644
--- a/libcxx/test/libcxx/strings/string.view/string.view.iterators/assert.iterator-indexing.pass.cpp
+++ b/libcxx/test/libcxx/strings/string.view/string.view.iterators/assert.iterator-indexing.pass.cpp
@@ -8,8 +8,8 @@
 
 // Make sure that std::string_view's iterators check for OOB accesses when the debug mode is enabled.
 
-// REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators
-// UNSUPPORTED: libcpp-hardening-mode=none
+// REQUIRES: can-test-hardening-assertions-fast
+// REQUIRES: libcpp-has-abi-bounded-iterators
 
 #include <iterator>
 #include <string_view>
diff --git a/libcxx/test/libcxx/text/text_encoding/text_encoding.ctor/assert.id.pass.cpp b/libcxx/test/libcxx/text/text_encoding/text_encoding.ctor/assert.id.pass.cpp
index f23629d5020ab..c182d17b18d34 100644
--- a/libcxx/test/libcxx/text/text_encoding/text_encoding.ctor/assert.id.pass.cpp
+++ b/libcxx/test/libcxx/text/text_encoding/text_encoding.ctor/assert.id.pass.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // REQUIRES: std-at-least-c++26
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
 
 // <text_encoding>
 
diff --git a/libcxx/test/libcxx/text/text_encoding/text_encoding.ctor/assert.string_view.pass.cpp b/libcxx/test/libcxx/text/text_encoding/text_encoding.ctor/assert.string_view.pass.cpp
index ddeef710c62f4..5cd04b81148a8 100644
--- a/libcxx/test/libcxx/text/text_encoding/text_encoding.ctor/assert.string_view.pass.cpp
+++ b/libcxx/test/libcxx/text/text_encoding/text_encoding.ctor/assert.string_view.pass.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // REQUIRES: std-at-least-c++26
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
 
 // <text_encoding>
 
diff --git a/libcxx/test/libcxx/thread/futures/futures.promise/assert.set_exception.pass.cpp b/libcxx/test/libcxx/thread/futures/futures.promise/assert.set_exception.pass.cpp
index 6d5eb5ef9931f..e9f66e79febd5 100644
--- a/libcxx/test/libcxx/thread/futures/futures.promise/assert.set_exception.pass.cpp
+++ b/libcxx/test/libcxx/thread/futures/futures.promise/assert.set_exception.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, no-threads
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
+// UNSUPPORTED: no-threads
 
 // <future>
 
diff --git a/libcxx/test/libcxx/thread/futures/futures.promise/assert.set_exception_at_thread_exit.pass.cpp b/libcxx/test/libcxx/thread/futures/futures.promise/assert.set_exception_at_thread_exit.pass.cpp
index 1bffde5e3ebd1..6a698b2a3b848 100644
--- a/libcxx/test/libcxx/thread/futures/futures.promise/assert.set_exception_at_thread_exit.pass.cpp
+++ b/libcxx/test/libcxx/thread/futures/futures.promise/assert.set_exception_at_thread_exit.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, no-threads
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
+// UNSUPPORTED: no-threads
 
 // <future>
 
diff --git a/libcxx/test/libcxx/thread/thread.barrier/assert.arrive.pass.cpp b/libcxx/test/libcxx/thread/thread.barrier/assert.arrive.pass.cpp
index 2bc4648878f8e..4c4f439a10080 100644
--- a/libcxx/test/libcxx/thread/thread.barrier/assert.arrive.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.barrier/assert.arrive.pass.cpp
@@ -6,15 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 // UNSUPPORTED: no-threads
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
 // Without the assertion, the test will most likely time out.
 // UNSUPPORTED: libcpp-assertion-semantic={{ignore|observe}}
 
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
-// REQUIRES: has-unix-headers
-
 // <barrier>
 
 // class barrier;
diff --git a/libcxx/test/libcxx/thread/thread.barrier/assert.ctor.pass.cpp b/libcxx/test/libcxx/thread/thread.barrier/assert.ctor.pass.cpp
index 0b4fb1d675eaa..cb8c60a0e3637 100644
--- a/libcxx/test/libcxx/thread/thread.barrier/assert.ctor.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.barrier/assert.ctor.pass.cpp
@@ -6,12 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 // UNSUPPORTED: no-threads
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
-// REQUIRES: has-unix-headers
 
 // <barrier>
 
diff --git a/libcxx/test/libcxx/thread/thread.latch/assert.arrive_and_wait.pass.cpp b/libcxx/test/libcxx/thread/thread.latch/assert.arrive_and_wait.pass.cpp
index 30d36b5f6d7b5..a3ab5b2f6d89d 100644
--- a/libcxx/test/libcxx/thread/thread.latch/assert.arrive_and_wait.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.latch/assert.arrive_and_wait.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 // UNSUPPORTED: no-threads
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <latch>
@@ -16,11 +17,8 @@
 
 // Make sure that calling arrive_and_wait with a negative value triggers an assertion.
 
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
 // Without the assertion, the test will most likely time out.
 // UNSUPPORTED: libcpp-assertion-semantic={{ignore|observe}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <latch>
 
diff --git a/libcxx/test/libcxx/thread/thread.latch/assert.count_down.pass.cpp b/libcxx/test/libcxx/thread/thread.latch/assert.count_down.pass.cpp
index 6220cba02af19..3660287cd0ae1 100644
--- a/libcxx/test/libcxx/thread/thread.latch/assert.count_down.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.latch/assert.count_down.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 // UNSUPPORTED: no-threads
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <latch>
@@ -17,10 +18,6 @@
 // Make sure that calling count_down with a negative value or a value
 // higher than the internal counter triggers an assertion.
 
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 #include <latch>
 
 #include "check_assertion.h"
diff --git a/libcxx/test/libcxx/thread/thread.latch/assert.ctor.pass.cpp b/libcxx/test/libcxx/thread/thread.latch/assert.ctor.pass.cpp
index 5f1ea19d82a50..d5f72105c3c98 100644
--- a/libcxx/test/libcxx/thread/thread.latch/assert.ctor.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.latch/assert.ctor.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 // UNSUPPORTED: no-threads
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <latch>
@@ -16,10 +17,6 @@
 
 // Make sure that calling latch with a negative value triggers an assertion
 
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 #include <latch>
 
 #include "check_assertion.h"
diff --git a/libcxx/test/libcxx/thread/thread.semaphore/assert.ctor.pass.cpp b/libcxx/test/libcxx/thread/thread.semaphore/assert.ctor.pass.cpp
index 1e33add779496..c5d7c0222f34f 100644
--- a/libcxx/test/libcxx/thread/thread.semaphore/assert.ctor.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.semaphore/assert.ctor.pass.cpp
@@ -6,12 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 // UNSUPPORTED: no-threads
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
-// REQUIRES: has-unix-headers
 
 // <semaphore>
 
diff --git a/libcxx/test/libcxx/thread/thread.semaphore/assert.release.pass.cpp b/libcxx/test/libcxx/thread/thread.semaphore/assert.release.pass.cpp
index a5a01a3847878..912a3d08e0d80 100644
--- a/libcxx/test/libcxx/thread/thread.semaphore/assert.release.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.semaphore/assert.release.pass.cpp
@@ -6,12 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 // UNSUPPORTED: no-threads
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
-// REQUIRES: has-unix-headers
 
 // <semaphore>
 
diff --git a/libcxx/test/libcxx/time/time.clock/time.clock.gps/time.clock.gps.members/assert.from_utc.pass.cpp b/libcxx/test/libcxx/time/time.clock/time.clock.gps/time.clock.gps.members/assert.from_utc.pass.cpp
index d8200439d9737..34d1dec5c1138 100644
--- a/libcxx/test/libcxx/time/time.clock/time.clock.gps/time.clock.gps.members/assert.from_utc.pass.cpp
+++ b/libcxx/test/libcxx/time/time.clock/time.clock.gps/time.clock.gps.members/assert.from_utc.pass.cpp
@@ -7,15 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: std-at-least-c++20
-// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+// REQUIRES: can-test-hardening-assertions-extensive
+// UNSUPPORTED: no-filesystem, no-tzdb
 
 // XFAIL: libcpp-has-no-experimental-tzdb
 // XFAIL: availability-tzdb-missing
 
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// REQUIRES: has-unix-headers
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 // <chrono>
 //
 // class gps_clock;
diff --git a/libcxx/test/libcxx/time/time.clock/time.clock.gps/time.clock.gps.members/assert.to_utc.pass.cpp b/libcxx/test/libcxx/time/time.clock/time.clock.gps/time.clock.gps.members/assert.to_utc.pass.cpp
index d61b3374f661f..486f58914a84b 100644
--- a/libcxx/test/libcxx/time/time.clock/time.clock.gps/time.clock.gps.members/assert.to_utc.pass.cpp
+++ b/libcxx/test/libcxx/time/time.clock/time.clock.gps/time.clock.gps.members/assert.to_utc.pass.cpp
@@ -7,15 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: std-at-least-c++20
-// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+// REQUIRES: can-test-hardening-assertions-extensive
+// UNSUPPORTED: no-filesystem, no-tzdb
 
 // XFAIL: libcpp-has-no-experimental-tzdb
 // XFAIL: availability-tzdb-missing
 
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// REQUIRES: has-unix-headers
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 // <chrono>
 //
 // class gps_clock;
diff --git a/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.from_utc.pass.cpp b/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.from_utc.pass.cpp
index f04595ce34ceb..9678deedbbaba 100644
--- a/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.from_utc.pass.cpp
+++ b/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.from_utc.pass.cpp
@@ -7,15 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: std-at-least-c++20
-// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+// REQUIRES: can-test-hardening-assertions-extensive
+// UNSUPPORTED: no-filesystem, no-tzdb
 
 // XFAIL: libcpp-has-no-experimental-tzdb
 // XFAIL: availability-tzdb-missing
 
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// REQUIRES: has-unix-headers
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 // <chrono>
 //
 // class tai_clock;
diff --git a/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.to_utc.pass.cpp b/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.to_utc.pass.cpp
index 624c6c4bce4ca..24cba55a06e2d 100644
--- a/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.to_utc.pass.cpp
+++ b/libcxx/test/libcxx/time/time.clock/time.clock.tai/time.clock.tai.members/assert.to_utc.pass.cpp
@@ -7,15 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: std-at-least-c++20
-// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
+// REQUIRES: can-test-hardening-assertions-extensive
+// UNSUPPORTED: no-filesystem, no-tzdb
 
 // XFAIL: libcpp-has-no-experimental-tzdb
 // XFAIL: availability-tzdb-missing
 
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// REQUIRES: has-unix-headers
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 // <chrono>
 //
 // class tai_clock;
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.db/time.zone.db.list/erase_after.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.db/time.zone.db.list/erase_after.pass.cpp
index 92842800f6bbd..dcc386c6b3de2 100644
--- a/libcxx/test/libcxx/time/time.zone/time.zone.db/time.zone.db.list/erase_after.pass.cpp
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.db/time.zone.db.list/erase_after.pass.cpp
@@ -68,4 +68,6 @@ int main(int, const char**) {
   assert(std::distance(list.begin(), list.end()) == 2);
   assert(list.front().version == "4");
   assert(it->version == "3");
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.ambig/assert.ctor.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.ambig/assert.ctor.pass.cpp
index ee2370f5bce25..7a31e689ec743 100644
--- a/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.ambig/assert.ctor.pass.cpp
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.ambig/assert.ctor.pass.cpp
@@ -6,12 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
-
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// UNSUPPORTED: no-filesystem, no-tzdb
 
 // XFAIL: libcpp-has-no-experimental-tzdb
 // XFAIL: availability-tzdb-missing
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.nonexist/assert.ctor.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.nonexist/assert.ctor.pass.cpp
index 5d896c34e4ccd..3bffe11481c2b 100644
--- a/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.nonexist/assert.ctor.pass.cpp
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.exception/time.zone.exception.nonexist/assert.ctor.pass.cpp
@@ -6,12 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
-
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// UNSUPPORTED: no-filesystem, no-tzdb
 
 // XFAIL: libcpp-has-no-experimental-tzdb
 // XFAIL: availability-tzdb-missing
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_local.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_local.pass.cpp
index eb0ae4cf4b187..e85d8a18e3bec 100644
--- a/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_local.pass.cpp
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_local.pass.cpp
@@ -6,12 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
-
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// UNSUPPORTED: no-filesystem, no-tzdb
 
 // XFAIL: libcpp-has-no-experimental-tzdb
 // XFAIL: availability-tzdb-missing
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys.pass.cpp
index 57b7f8d0f30a0..2f835cfb84dbf 100644
--- a/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys.pass.cpp
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys.pass.cpp
@@ -6,12 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
-
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// UNSUPPORTED: no-filesystem, no-tzdb
 
 // XFAIL: libcpp-has-no-experimental-tzdb
 // XFAIL: availability-tzdb-missing
diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys_choose.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys_choose.pass.cpp
index 85ce6019fcd55..8a358856f6a39 100644
--- a/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys_choose.pass.cpp
+++ b/libcxx/test/libcxx/time/time.zone/time.zone.timezone/time.zone.members/assert.to_sys_choose.pass.cpp
@@ -6,12 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: no-filesystem, no-localization, no-tzdb
-
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// UNSUPPORTED: no-filesystem, no-tzdb
 
 // XFAIL: libcpp-has-no-experimental-tzdb
 // XFAIL: availability-tzdb-missing
diff --git a/libcxx/test/libcxx/utilities/assert.exception_guard.no_exceptions.pass.cpp b/libcxx/test/libcxx/utilities/assert.exception_guard.no_exceptions.pass.cpp
index c6ddb8bd252a4..a41c6ea079bc8 100644
--- a/libcxx/test/libcxx/utilities/assert.exception_guard.no_exceptions.pass.cpp
+++ b/libcxx/test/libcxx/utilities/assert.exception_guard.no_exceptions.pass.cpp
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode=debug
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-debug
 // ADDITIONAL_COMPILE_FLAGS: -fno-exceptions
 
 #include <__utility/exception_guard.h>
@@ -19,4 +16,6 @@
 int main(int, char**) {
   TEST_LIBCPP_ASSERT_FAILURE(
       std::__make_exception_guard([] {}), "__exception_guard not completed with exceptions disabled");
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/utilities/expected/expected.expected/assert.arrow.pass.cpp b/libcxx/test/libcxx/utilities/expected/expected.expected/assert.arrow.pass.cpp
index 47481bcbef8a8..1fcc9901e2a6d 100644
--- a/libcxx/test/libcxx/utilities/expected/expected.expected/assert.arrow.pass.cpp
+++ b/libcxx/test/libcxx/utilities/expected/expected.expected/assert.arrow.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // constexpr const T* operator->() const noexcept;
 // constexpr T* operator->() noexcept;
diff --git a/libcxx/test/libcxx/utilities/expected/expected.expected/assert.deref.pass.cpp b/libcxx/test/libcxx/utilities/expected/expected.expected/assert.deref.pass.cpp
index 5ab43d38ccb15..f04b3c8473930 100644
--- a/libcxx/test/libcxx/utilities/expected/expected.expected/assert.deref.pass.cpp
+++ b/libcxx/test/libcxx/utilities/expected/expected.expected/assert.deref.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // constexpr const T& operator*() const & noexcept;
 // constexpr T& operator*() & noexcept;
diff --git a/libcxx/test/libcxx/utilities/expected/expected.expected/assert.error.pass.cpp b/libcxx/test/libcxx/utilities/expected/expected.expected/assert.error.pass.cpp
index 92bf305994c18..61be7893d6c2a 100644
--- a/libcxx/test/libcxx/utilities/expected/expected.expected/assert.error.pass.cpp
+++ b/libcxx/test/libcxx/utilities/expected/expected.expected/assert.error.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // constexpr const E& error() const & noexcept;
 // constexpr E& error() & noexcept;
diff --git a/libcxx/test/libcxx/utilities/expected/expected.void/assert.deref.pass.cpp b/libcxx/test/libcxx/utilities/expected/expected.void/assert.deref.pass.cpp
index 6f1ba075b3245..77a4947b8c1ec 100644
--- a/libcxx/test/libcxx/utilities/expected/expected.void/assert.deref.pass.cpp
+++ b/libcxx/test/libcxx/utilities/expected/expected.void/assert.deref.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // constexpr void operator*() const noexcept;
 //
diff --git a/libcxx/test/libcxx/utilities/expected/expected.void/assert.error.pass.cpp b/libcxx/test/libcxx/utilities/expected/expected.void/assert.error.pass.cpp
index a1c92ff85f33a..891abeb070ccd 100644
--- a/libcxx/test/libcxx/utilities/expected/expected.void/assert.error.pass.cpp
+++ b/libcxx/test/libcxx/utilities/expected/expected.void/assert.error.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // constexpr const E& error() const & noexcept;
 // constexpr E& error() & noexcept;
diff --git a/libcxx/test/libcxx/utilities/format/format.arguments/format.arg/assert.array.pass.cpp b/libcxx/test/libcxx/utilities/format/format.arguments/format.arg/assert.array.pass.cpp
index 1e9b1d93eb06f..83006b37d32a0 100644
--- a/libcxx/test/libcxx/utilities/format/format.arguments/format.arg/assert.array.pass.cpp
+++ b/libcxx/test/libcxx/utilities/format/format.arguments/format.arg/assert.array.pass.cpp
@@ -10,8 +10,8 @@
 
 // Formatting non-null-terminated character arrays.
 
-// REQUIRES: std-at-least-c++20, has-unix-headers, libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
+// REQUIRES: std-at-least-c++20
 
 #include <format>
 
diff --git a/libcxx/test/libcxx/utilities/function.objects/func.wrap/func.wrap.ref/func.wrap.ref.ctor/assert.constant_arg_ptr.pass.cpp b/libcxx/test/libcxx/utilities/function.objects/func.wrap/func.wrap.ref/func.wrap.ref.ctor/assert.constant_arg_ptr.pass.cpp
index 9703de0a59c2c..fc9c2dfa01f49 100644
--- a/libcxx/test/libcxx/utilities/function.objects/func.wrap/func.wrap.ref/func.wrap.ref.ctor/assert.constant_arg_ptr.pass.cpp
+++ b/libcxx/test/libcxx/utilities/function.objects/func.wrap/func.wrap.ref/func.wrap.ref.ctor/assert.constant_arg_ptr.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // REQUIRES: std-at-least-c++26
-// UNSUPPORTED: libcpp-hardening-mode=none || libcpp-hardening-mode=fast
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // template<auto f, class T>
 //   constexpr function_ref(constant_wrapper<f>, cv T* obj) noexcept;
diff --git a/libcxx/test/libcxx/utilities/function.objects/func.wrap/func.wrap.ref/func.wrap.ref.ctor/assert.function_ptr.pass.cpp b/libcxx/test/libcxx/utilities/function.objects/func.wrap/func.wrap.ref/func.wrap.ref.ctor/assert.function_ptr.pass.cpp
index e69776d0e3c90..25bc24c157529 100644
--- a/libcxx/test/libcxx/utilities/function.objects/func.wrap/func.wrap.ref/func.wrap.ref.ctor/assert.function_ptr.pass.cpp
+++ b/libcxx/test/libcxx/utilities/function.objects/func.wrap/func.wrap.ref/func.wrap.ref.ctor/assert.function_ptr.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // REQUIRES: std-at-least-c++26
-// UNSUPPORTED: libcpp-hardening-mode=none || libcpp-hardening-mode=fast
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // template<class F> function_ref(F* f) noexcept;
 // Preconditions: f is not a null pointer.
diff --git a/libcxx/test/libcxx/utilities/optional/optional.iterator/assert.arithmetic.pass.cpp b/libcxx/test/libcxx/utilities/optional/optional.iterator/assert.arithmetic.pass.cpp
index cd73d67b39038..fe348a12962ba 100644
--- a/libcxx/test/libcxx/utilities/optional/optional.iterator/assert.arithmetic.pass.cpp
+++ b/libcxx/test/libcxx/utilities/optional/optional.iterator/assert.arithmetic.pass.cpp
@@ -11,7 +11,8 @@
 // Add to iterator out of bounds.
 
 // REQUIRES: std-at-least-c++26
-// UNSUPPORTED: libcpp-hardening-mode=none, libcpp-has-abi-bounded-iterators-in-optional
+// REQUIRES: can-test-hardening-assertions-fast
+// UNSUPPORTED: libcpp-has-abi-bounded-iterators-in-optional
 
 #include <optional>
 
diff --git a/libcxx/test/libcxx/utilities/optional/optional.object/optional.iterator/assert.bounded_iterator.pass.cpp b/libcxx/test/libcxx/utilities/optional/optional.object/optional.iterator/assert.bounded_iterator.pass.cpp
index 393b9e5f06476..a7aecabd75a01 100644
--- a/libcxx/test/libcxx/utilities/optional/optional.object/optional.iterator/assert.bounded_iterator.pass.cpp
+++ b/libcxx/test/libcxx/utilities/optional/optional.object/optional.iterator/assert.bounded_iterator.pass.cpp
@@ -9,7 +9,7 @@
 // <optional>
 
 // REQUIRES: std-at-least-c++26, libcpp-has-abi-bounded-iterators-in-optional
-// UNSUPPORTED: libcpp-hardening-mode=none
+// REQUIRES: can-test-hardening-assertions-fast
 
 // Test that an assertion fires for invalid uses of the following operators on a bounded iterator:
 
diff --git a/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.observe/assert.dereference.pass.cpp b/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.observe/assert.dereference.pass.cpp
index 31938b3f8fbaa..29e5cf74d7f8d 100644
--- a/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.observe/assert.dereference.pass.cpp
+++ b/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.observe/assert.dereference.pass.cpp
@@ -13,10 +13,8 @@
 // constexpr const T& optional<T>::operator*() const &;
 // constexpr T&& optional<T>::operator*() const &&;
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <optional>
 
diff --git a/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.observe/assert.op_arrow.pass.cpp b/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.observe/assert.op_arrow.pass.cpp
index 52009628327db..f33b7c51fe5a8 100644
--- a/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.observe/assert.op_arrow.pass.cpp
+++ b/libcxx/test/libcxx/utilities/optional/optional.object/optional.object.observe/assert.op_arrow.pass.cpp
@@ -11,10 +11,8 @@
 // constexpr T* optional<T>::operator->();
 // constexpr const T* optional<T>::operator->() const;
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <optional>
 
diff --git a/libcxx/test/libcxx/utilities/template.bitset/assert.pass.cpp b/libcxx/test/libcxx/utilities/template.bitset/assert.pass.cpp
index 4019bdf1318eb..8f2fbc735f3bc 100644
--- a/libcxx/test/libcxx/utilities/template.bitset/assert.pass.cpp
+++ b/libcxx/test/libcxx/utilities/template.bitset/assert.pass.cpp
@@ -10,10 +10,7 @@
 
 // Test hardening assertions for std::bitset.
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// UNSUPPORTED: c++03
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 #include <bitset>
 
diff --git a/libcxx/test/libcxx/utilities/utility/mem.res/mem.poly.allocator.class/mem.poly.allocator.mem/assert.deallocate.pass.cpp b/libcxx/test/libcxx/utilities/utility/mem.res/mem.poly.allocator.class/mem.poly.allocator.mem/assert.deallocate.pass.cpp
index 5a9813a232b85..6e5e5095df704 100644
--- a/libcxx/test/libcxx/utilities/utility/mem.res/mem.poly.allocator.class/mem.poly.allocator.mem/assert.deallocate.pass.cpp
+++ b/libcxx/test/libcxx/utilities/utility/mem.res/mem.poly.allocator.class/mem.poly.allocator.mem/assert.deallocate.pass.cpp
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14
-// REQUIRES: has-unix-headers, libcpp-hardening-mode=debug
 
 // <memory_resource>
 
diff --git a/libcxx/test/libcxx/utilities/utility/mem.res/mem.res.monotonic.buffer/mem.res.monotonic.buffer.ctor/assert.initial_size.pass.cpp b/libcxx/test/libcxx/utilities/utility/mem.res/mem.res.monotonic.buffer/mem.res.monotonic.buffer.ctor/assert.initial_size.pass.cpp
index 61c331d9158ce..7cdd7fe27e9f4 100644
--- a/libcxx/test/libcxx/utilities/utility/mem.res/mem.res.monotonic.buffer/mem.res.monotonic.buffer.ctor/assert.initial_size.pass.cpp
+++ b/libcxx/test/libcxx/utilities/utility/mem.res/mem.res.monotonic.buffer/mem.res.monotonic.buffer.ctor/assert.initial_size.pass.cpp
@@ -18,10 +18,7 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 // UNSUPPORTED: availability-pmr-missing
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// UNSUPPORTED: libcpp-assertion-semantic={{ignore|observe}}
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 #include <cstddef>
 #include <memory_resource>
diff --git a/libcxx/test/selftest/pass.cpp/werror.pass.cpp b/libcxx/test/selftest/pass.cpp/werror.pass.cpp
index 590785fc1774d..16039abc4c001 100644
--- a/libcxx/test/selftest/pass.cpp/werror.pass.cpp
+++ b/libcxx/test/selftest/pass.cpp/werror.pass.cpp
@@ -19,4 +19,5 @@
 
 int main(int, char**) {
     int foo;
+    return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.reverse/pstl.reverse_copy.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.reverse/pstl.reverse_copy.pass.cpp
index d81dde163cf02..cf9b65607e647 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.reverse/pstl.reverse_copy.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.reverse/pstl.reverse_copy.pass.cpp
@@ -113,4 +113,6 @@ int main(int, char**) {
         types::for_each(types::forward_iterator_list<int*>{},
                         TestIteratorWithPolicies<types::partial_instantiation<Test, Iter>::template apply>{});
       }});
+
+  return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.clamp/assert.ranges_clamp.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.clamp/assert.ranges_clamp.pass.cpp
index 07e56e73b6f34..b90fbe2043979 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.clamp/assert.ranges_clamp.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.clamp/assert.ranges_clamp.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: availability-verbose_abort-missing
 
 // <algorithm>
 
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/pop.heap/assert.pop_heap.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/pop.heap/assert.pop_heap.pass.cpp
index b6b6babafba1b..a6133040ae346 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/pop.heap/assert.pop_heap.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/pop.heap/assert.pop_heap.pass.cpp
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 // <algorithm>
 
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/pop.heap/assert.ranges_pop_heap.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/pop.heap/assert.ranges_pop_heap.pass.cpp
index ba55c14bf5590..768212cefae60 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/pop.heap/assert.ranges_pop_heap.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/pop.heap/assert.ranges_pop_heap.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: availability-verbose_abort-missing
 
 // <algorithm>
 
diff --git a/libcxx/test/std/algorithms/pstl.exception_handling.pass.cpp b/libcxx/test/std/algorithms/pstl.exception_handling.pass.cpp
index 19e8cb9f8b0d1..4998b02fd0650 100644
--- a/libcxx/test/std/algorithms/pstl.exception_handling.pass.cpp
+++ b/libcxx/test/std/algorithms/pstl.exception_handling.pass.cpp
@@ -456,4 +456,6 @@ int main(int, char**) {
       }
     }
   });
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/count1.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/count1.compile.fail.cpp
index 0c9d51edccd34..2ab9d5eec2ddb 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/count1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/count1.compile.fail.cpp
@@ -30,4 +30,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().count(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/count2.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/count2.compile.fail.cpp
index 0cc81b3339930..54d1a56030f50 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/count2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/count2.compile.fail.cpp
@@ -30,4 +30,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().count(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/count3.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/count3.compile.fail.cpp
index 1c6a4cd3db095..17ffc8f35cdc7 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/count3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/count3.compile.fail.cpp
@@ -30,4 +30,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().count(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/equal_range1.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/equal_range1.compile.fail.cpp
index d846a58899174..cefe6ca138b58 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/equal_range1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/equal_range1.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/equal_range2.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/equal_range2.compile.fail.cpp
index e341b3cb0ee45..bace64b595a50 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/equal_range2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/equal_range2.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/equal_range3.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/equal_range3.compile.fail.cpp
index f99fc888ff52a..f83d11255a3b5 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/equal_range3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/equal_range3.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/find1.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/find1.compile.fail.cpp
index b67b3b33b38cc..a569b46f18038 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/find1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/find1.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().find(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/find2.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/find2.compile.fail.cpp
index 49ded6659a501..5f702aadf1169 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/find2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/find2.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().find(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/find3.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/find3.compile.fail.cpp
index ba978dde21a76..14b78d2ce27a5 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/find3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/find3.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().find(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/lower_bound1.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/lower_bound1.compile.fail.cpp
index e07177b7a6e86..44e9e16922909 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/lower_bound1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/lower_bound1.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().lower_bound(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/lower_bound2.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/lower_bound2.compile.fail.cpp
index 72dc1a5eb2b37..844118a569b8f 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/lower_bound2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/lower_bound2.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().lower_bound(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/lower_bound3.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/lower_bound3.compile.fail.cpp
index 10da1aad25f29..9cbfe90deb959 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/lower_bound3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/lower_bound3.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().lower_bound(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/upper_bound1.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/upper_bound1.compile.fail.cpp
index 14d69988f7016..a6bb3dabadad6 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/upper_bound1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/upper_bound1.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().upper_bound(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/upper_bound2.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/upper_bound2.compile.fail.cpp
index a66536a9d1f9b..c19ad959bb0ed 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/upper_bound2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/upper_bound2.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().upper_bound(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/upper_bound3.compile.fail.cpp b/libcxx/test/std/containers/associative/map/map.ops/upper_bound3.compile.fail.cpp
index ae6eab1249284..78229a3dca452 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/upper_bound3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/upper_bound3.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().upper_bound(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/count1.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/count1.compile.fail.cpp
index 48b7cee7b0132..56491e1766171 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/count1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/count1.compile.fail.cpp
@@ -28,4 +28,6 @@ int main(int, char**) {
   typedef std::multimap<int, double, transparent_less_no_type> M;
 
   TEST_IGNORE_NODISCARD M().count(C2Int{5});
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/count2.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/count2.compile.fail.cpp
index dd8b3278ae765..e3affbc6aefa4 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/count2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/count2.compile.fail.cpp
@@ -28,4 +28,6 @@ int main(int, char**) {
   typedef std::multimap<int, double, transparent_less_private> M;
 
   TEST_IGNORE_NODISCARD M().count(C2Int{5});
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/count3.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/count3.compile.fail.cpp
index 1cb7c48fc1087..1fe61d30a3cda 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/count3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/count3.compile.fail.cpp
@@ -28,4 +28,6 @@ int main(int, char**) {
   typedef std::multimap<int, double, transparent_less_not_a_type> M;
 
   TEST_IGNORE_NODISCARD M().count(C2Int{5});
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range1.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range1.compile.fail.cpp
index 31ab8071553de..86b4b5dafe97f 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range1.compile.fail.cpp
@@ -29,4 +29,6 @@ int main(int, char**) {
   typedef std::multimap<int, double, transparent_less_no_type> M;
 
   TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range2.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range2.compile.fail.cpp
index 7b92baaed355b..311e68bf65c5c 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range2.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range3.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range3.compile.fail.cpp
index a5ce74a8175c8..5e1a3bbb14b34 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range3.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().equal_range(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/find1.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/find1.compile.fail.cpp
index 170bd8b1aaba3..f61a5a1f5786b 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/find1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/find1.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().find(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/find2.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/find2.compile.fail.cpp
index 18eaa45ddd741..faf4ddb057c3c 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/find2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/find2.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().find(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/find3.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/find3.compile.fail.cpp
index 66263aa07a5b0..2b714d54e5b69 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/find3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/find3.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().find(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound1.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound1.compile.fail.cpp
index 4622cb00da94f..cfef33d20be73 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound1.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().lower_bound(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound2.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound2.compile.fail.cpp
index 146e202b7e332..36fb3d377777a 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound2.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().lower_bound(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound3.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound3.compile.fail.cpp
index 9fe4e0e097328..0a2bfd39a231a 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound3.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().lower_bound(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound1.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound1.compile.fail.cpp
index d28143f7cab9d..b8dea425b5ad8 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound1.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound1.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().upper_bound(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound2.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound2.compile.fail.cpp
index 0f9897996bcca..d78d5e4df8cae 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound2.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound2.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().upper_bound(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound3.compile.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound3.compile.fail.cpp
index 32e8983c8ce3a..ff223e7ad43c3 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound3.compile.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound3.compile.fail.cpp
@@ -31,4 +31,6 @@ int main(int, char**) {
 
     TEST_IGNORE_NODISCARD M().upper_bound(C2Int{5});
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/array/assert.back.pass.cpp b/libcxx/test/std/containers/sequences/array/assert.back.pass.cpp
index b9dec01033334..bd8cf745681c2 100644
--- a/libcxx/test/std/containers/sequences/array/assert.back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/assert.back.pass.cpp
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 // test that array<T, 0>::back() triggers an assertion
 
diff --git a/libcxx/test/std/containers/sequences/array/assert.front.pass.cpp b/libcxx/test/std/containers/sequences/array/assert.front.pass.cpp
index 67a70a1d63c52..d5c02e827914d 100644
--- a/libcxx/test/std/containers/sequences/array/assert.front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/assert.front.pass.cpp
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 // test that array<T, 0>::front() triggers an assertion
 
diff --git a/libcxx/test/std/containers/sequences/array/assert.indexing.pass.cpp b/libcxx/test/std/containers/sequences/array/assert.indexing.pass.cpp
index ea0ffce3b3007..ee3a391e20dad 100644
--- a/libcxx/test/std/containers/sequences/array/assert.indexing.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/assert.indexing.pass.cpp
@@ -6,10 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 // <array>
 
diff --git a/libcxx/test/std/containers/sequences/array/assert.iterators.pass.cpp b/libcxx/test/std/containers/sequences/array/assert.iterators.pass.cpp
index 85e2df1eba981..f9f9091fcd83c 100644
--- a/libcxx/test/std/containers/sequences/array/assert.iterators.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/assert.iterators.pass.cpp
@@ -6,11 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // REQUIRES: libcpp-has-abi-bounded-iterators-in-std-array
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <array>
 
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/reserve_exceptions.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/reserve_exceptions.pass.cpp
index c381e23a04d02..d9252b2c636e0 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/reserve_exceptions.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/reserve_exceptions.pass.cpp
@@ -303,4 +303,6 @@ int main(int, char**) {
 #if TEST_STD_VER >= 11
   test_move_ctor_exceptions();
 #endif
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size_exceptions.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size_exceptions.pass.cpp
index b5e12ea8e5d0a..cf76e7d7313fb 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size_exceptions.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size_exceptions.pass.cpp
@@ -391,4 +391,6 @@ int main(int, char**) {
 #if TEST_STD_VER >= 11
   test_move_ctor_exceptions();
 #endif
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size_value_exceptions.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size_value_exceptions.pass.cpp
index 7217b47d69e4d..7a757b41d9ea3 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size_value_exceptions.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/resize_size_value_exceptions.pass.cpp
@@ -228,4 +228,6 @@ void test_copy_ctor_exceptions() {
 int main(int, char**) {
   test_allocation_exceptions();
   test_copy_ctor_exceptions();
+
+  return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/assert.push_back.invalidation.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/assert.push_back.invalidation.pass.cpp
index 4a899139d5f68..6446211369471 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/assert.push_back.invalidation.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/assert.push_back.invalidation.pass.cpp
@@ -14,11 +14,8 @@
 // the insertion point remain valid but those at or after the insertion point,
 // including the past-the-end iterator, are invalidated.
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // REQUIRES: libcpp-has-abi-bounded-iterators-in-vector
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 #include <vector>
 #include <cassert>
diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.assert.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.assert.pass.cpp
index 9e4d88642d49e..47c96137bc0b0 100644
--- a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.assert.pass.cpp
+++ b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.assert.pass.cpp
@@ -6,12 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
 
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: availability-verbose_abort-missing
-
 // <fstream>
 
 // class basic_filebuf;
diff --git a/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.assert.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.assert.pass.cpp
index 8fd37f68fc97a..201af69d49afc 100644
--- a/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.assert.pass.cpp
+++ b/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.assert.pass.cpp
@@ -6,12 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
 
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: availability-verbose_abort-missing
-
 // <fstream>
 
 // class basic_fstream;
diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.assert.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.assert.pass.cpp
index c17b778c4afad..c4c479bd8dfce 100644
--- a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.assert.pass.cpp
+++ b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.assert.pass.cpp
@@ -6,12 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
 
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: availability-verbose_abort-missing
-
 // <fstream>
 
 // class basic_ifstream;
diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.assert.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.assert.pass.cpp
index 243be70ec9452..f6b3f5045f000 100644
--- a/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.assert.pass.cpp
+++ b/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.assert.pass.cpp
@@ -6,12 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-extensive
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
 
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: availability-verbose_abort-missing
-
 // <fstream>
 
 // class basic_ofstream;
diff --git a/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/streambuf.get.area/setg.assert.pass.cpp b/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/streambuf.get.area/setg.assert.pass.cpp
index 7d1cc8acf7266..c488a3679d804 100644
--- a/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/streambuf.get.area/setg.assert.pass.cpp
+++ b/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/streambuf.get.area/setg.assert.pass.cpp
@@ -6,9 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 // <streambuf>
 
diff --git a/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/streambuf.put.area/setp.assert.pass.cpp b/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/streambuf.put.area/setp.assert.pass.cpp
index e0cd6438d2b44..e3cb6641210b7 100644
--- a/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/streambuf.put.area/setp.assert.pass.cpp
+++ b/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/streambuf.put.area/setp.assert.pass.cpp
@@ -6,9 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-fast
 
 // <streambuf>
 
diff --git a/libcxx/test/std/input.output/syncstream/osyncstream/members/emit.pass.cpp b/libcxx/test/std/input.output/syncstream/osyncstream/members/emit.pass.cpp
index bf0c221b63ac3..d0809f2f7c0ba 100644
--- a/libcxx/test/std/input.output/syncstream/osyncstream/members/emit.pass.cpp
+++ b/libcxx/test/std/input.output/syncstream/osyncstream/members/emit.pass.cpp
@@ -42,4 +42,6 @@ int main(int, char**) {
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
   test<wchar_t>();
 #endif
+
+  return 0;
 }
diff --git a/libcxx/test/std/numerics/numeric.ops/numeric.ops.sat/saturating_div.assert.pass.cpp b/libcxx/test/std/numerics/numeric.ops/numeric.ops.sat/saturating_div.assert.pass.cpp
index 79cbd8124b2de..9e0a79f6f5295 100644
--- a/libcxx/test/std/numerics/numeric.ops/numeric.ops.sat/saturating_div.assert.pass.cpp
+++ b/libcxx/test/std/numerics/numeric.ops/numeric.ops.sat/saturating_div.assert.pass.cpp
@@ -8,9 +8,7 @@
 
 // REQUIRES: std-at-least-c++26
 
-// REQUIRES: has-unix-headers
-// REQUIRES: libcpp-hardening-mode={{extensive|debug}}
-// XFAIL: availability-verbose_abort-missing
+// REQUIRES: can-test-hardening-assertions-extensive
 
 // <numeric>
 
diff --git a/libcxx/test/std/ranges/range.factories/range.iota.view/assert.ctor.value.bound.pass.cpp b/libcxx/test/std/ranges/range.factories/range.iota.view/assert.ctor.value.bound.pass.cpp
index da21a32bf8298..5f1169575a915 100644
--- a/libcxx/test/std/ranges/range.factories/range.iota.view/assert.ctor.value.bound.pass.cpp
+++ b/libcxx/test/std/ranges/range.factories/range.iota.view/assert.ctor.value.bound.pass.cpp
@@ -6,12 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
-
 // Test the precondition check in iota_view(value, bound) that `bound` is reachable from `value`.
 
 #include <ranges>
diff --git a/libcxx/test/std/re/re.submatch/re.submatch.members/swap.pass.cpp b/libcxx/test/std/re/re.submatch/re.submatch.members/swap.pass.cpp
index 9e9337010ef2e..1c7b6338881c5 100644
--- a/libcxx/test/std/re/re.submatch/re.submatch.members/swap.pass.cpp
+++ b/libcxx/test/std/re/re.submatch/re.submatch.members/swap.pass.cpp
@@ -73,4 +73,6 @@ int main(int, char**) {
     ASSERT_NOEXCEPT(sm1.swap(sm2));
   }
 #endif
+
+  return 0;
 }
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.explicit_instantiation.sh.cpp b/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.explicit_instantiation.sh.cpp
index 12a0b4c1f532c..d02a560039560 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.explicit_instantiation.sh.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.explicit_instantiation.sh.cpp
@@ -55,5 +55,7 @@ extern template class std::basic_string<char16, string16_char_traits>;
 int main(int, char**) {
   std::basic_string<char16, string16_char_traits> s;
   s.shrink_to_fit();
+
+  return 0;
 }
 #endif
diff --git a/libcxx/test/std/text/text_encoding/text_encoding.members/text_encoding.aliases_view/index.pass.cpp b/libcxx/test/std/text/text_encoding/text_encoding.members/text_encoding.aliases_view/index.pass.cpp
index d6376dbfa958f..3d68382c6abaa 100644
--- a/libcxx/test/std/text/text_encoding/text_encoding.members/text_encoding.aliases_view/index.pass.cpp
+++ b/libcxx/test/std/text/text_encoding/text_encoding.members/text_encoding.aliases_view/index.pass.cpp
@@ -34,4 +34,6 @@ constexpr bool test() {
 int main(int, char**) {
   test();
   static_assert(test());
+
+  return 0;
 }
diff --git a/libcxx/test/std/utilities/charconv/charconv.msvc/test.cpp b/libcxx/test/std/utilities/charconv/charconv.msvc/test.cpp
index ace6d46b879b0..fb3316884df58 100644
--- a/libcxx/test/std/utilities/charconv/charconv.msvc/test.cpp
+++ b/libcxx/test/std/utilities/charconv/charconv.msvc/test.cpp
@@ -1080,4 +1080,6 @@ int main(int argc, char** argv) {
     } else if (ms > 30'000) {
         puts("That was slow. Consider tuning PrefixesToTest and FractionBits to test fewer cases.");
     }
+
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/allocator.uses/allocator.uses.construction/make_obj_using_allocator.pass.cpp b/libcxx/test/std/utilities/memory/allocator.uses/allocator.uses.construction/make_obj_using_allocator.pass.cpp
index 744e530191cdc..66ff151ceda36 100644
--- a/libcxx/test/std/utilities/memory/allocator.uses/allocator.uses.construction/make_obj_using_allocator.pass.cpp
+++ b/libcxx/test/std/utilities/memory/allocator.uses/allocator.uses.construction/make_obj_using_allocator.pass.cpp
@@ -140,4 +140,6 @@ constexpr bool test() {
 int main(int, char**) {
   test();
   static_assert(test());
+
+  return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/allocator.uses/allocator.uses.construction/uninitialized_construct_using_allocator.pass.cpp b/libcxx/test/std/utilities/memory/allocator.uses/allocator.uses.construction/uninitialized_construct_using_allocator.pass.cpp
index 329698c4371c0..713f897216e2b 100644
--- a/libcxx/test/std/utilities/memory/allocator.uses/allocator.uses.construction/uninitialized_construct_using_allocator.pass.cpp
+++ b/libcxx/test/std/utilities/memory/allocator.uses/allocator.uses.construction/uninitialized_construct_using_allocator.pass.cpp
@@ -186,4 +186,6 @@ constexpr bool test() {
 int main(int, char**) {
   test();
   static_assert(test());
+
+  return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/allocator.uses/allocator.uses.construction/uses_allocator_construction_args.pass.cpp b/libcxx/test/std/utilities/memory/allocator.uses/allocator.uses.construction/uses_allocator_construction_args.pass.cpp
index aa3a5e8a28685..43010a65da3de 100644
--- a/libcxx/test/std/utilities/memory/allocator.uses/allocator.uses.construction/uses_allocator_construction_args.pass.cpp
+++ b/libcxx/test/std/utilities/memory/allocator.uses/allocator.uses.construction/uses_allocator_construction_args.pass.cpp
@@ -253,4 +253,6 @@ constexpr bool test() {
 int main(int, char**) {
   test();
   static_assert(test());
+
+  return 0;
 }
diff --git a/libcxx/test/std/utilities/optional/optional.object/optional.object.ctor/ref_t.pass.cpp b/libcxx/test/std/utilities/optional/optional.object/optional.object.ctor/ref_t.pass.cpp
index 57552743af138..d099edfcb7238 100644
--- a/libcxx/test/std/utilities/optional/optional.object/optional.object.ctor/ref_t.pass.cpp
+++ b/libcxx/test/std/utilities/optional/optional.object/optional.object.ctor/ref_t.pass.cpp
@@ -72,4 +72,6 @@ constexpr bool tests() {
 int main(int, char**) {
   static_assert(tests());
   tests();
+
+  return 0;
 }
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp
index 43c89a84663e7..3f656bcd8fb88 100644
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-fast
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-hardening-mode=none
-// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
 // <memory>
 //
diff --git a/libcxx/test/std/utilities/utility/utility.unreachable/assert.unreachable.pass.cpp b/libcxx/test/std/utilities/utility/utility.unreachable/assert.unreachable.pass.cpp
index f95b83ff4eb3c..76665c8e3b44d 100644
--- a/libcxx/test/std/utilities/utility/utility.unreachable/assert.unreachable.pass.cpp
+++ b/libcxx/test/std/utilities/utility/utility.unreachable/assert.unreachable.pass.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
+// REQUIRES: can-test-hardening-assertions-debug
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
-// REQUIRES: libcpp-hardening-mode=debug
-// XFAIL: availability-verbose_abort-missing
 
 // Make sure that reaching std::unreachable() with assertions enabled triggers an assertion.
 
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/copy.verify.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/copy.verify.cpp
index fe4961b4c157a..04fb0eaeeacaa 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/copy.verify.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/copy.verify.cpp
@@ -29,4 +29,6 @@ int main(int, char**)
     std::variant<NotCopyConstructible> v1;
     std::variant<NotCopyConstructible> v2(v); // expected-error {{call to implicitly-deleted copy constructor of 'std::variant<NotCopyConstructible>'}}
     v1 = v; // expected-error-re {{object of type 'std:{{.*}}:variant<NotCopyConstructible>' cannot be assigned because its copy assignment operator is implicitly deleted}}
+
+    return 0;
 }
diff --git a/libcxx/test/support/min_allocator.h b/libcxx/test/support/min_allocator.h
index 07603425f0668..193fc5b4d81c4 100644
--- a/libcxx/test/support/min_allocator.h
+++ b/libcxx/test/support/min_allocator.h
@@ -35,8 +35,14 @@ class bare_allocator {
 
   void deallocate(T* p, std::size_t) { return ::operator delete(static_cast<void*>(p)); }
 
-  friend bool operator==(bare_allocator, bare_allocator) { return true; }
-  friend bool operator!=(bare_allocator x, bare_allocator y) { return !(x == y); }
+  template <class U>
+  friend bool operator==(bare_allocator, bare_allocator<U>) {
+    return true;
+  }
+  template <class U>
+  friend bool operator!=(bare_allocator x, bare_allocator<U> y) {
+    return !(x == y);
+  }
 };
 
 template <class T>
@@ -65,8 +71,14 @@ class no_default_allocator {
 
   TEST_CONSTEXPR_CXX20 void deallocate(T* p, std::size_t n) { std::allocator<T>().deallocate(p, n); }
 
-  friend TEST_CONSTEXPR bool operator==(no_default_allocator, no_default_allocator) { return true; }
-  friend TEST_CONSTEXPR bool operator!=(no_default_allocator x, no_default_allocator y) { return !(x == y); }
+  template <class U>
+  friend TEST_CONSTEXPR bool operator==(no_default_allocator, no_default_allocator<U>) {
+    return true;
+  }
+  template <class U>
+  friend TEST_CONSTEXPR bool operator!=(no_default_allocator x, no_default_allocator<U> y) {
+    return !(x == y);
+  }
 };
 
 struct malloc_allocator_base {
@@ -118,8 +130,14 @@ class malloc_allocator : public malloc_allocator_base {
     std::free(static_cast<void*>(p));
   }
 
-  friend bool operator==(malloc_allocator, malloc_allocator) { return true; }
-  friend bool operator!=(malloc_allocator x, malloc_allocator y) { return !(x == y); }
+  template <class U>
+  friend bool operator==(malloc_allocator, malloc_allocator<U>) {
+    return true;
+  }
+  template <class U>
+  friend bool operator!=(malloc_allocator x, malloc_allocator<U> y) {
+    return !(x == y);
+  }
 };
 
 template <class T>
@@ -129,6 +147,10 @@ struct cpp03_allocator : bare_allocator<T> {
 
   static bool construct_called;
 
+  cpp03_allocator() TEST_NOEXCEPT {}
+  template <class U>
+  explicit cpp03_allocator(const cpp03_allocator<U>&) TEST_NOEXCEPT {}
+
   // Returned value is not used but it's not prohibited.
   pointer construct(pointer p, const value_type& val) {
     ::new (p) value_type(val);
@@ -148,6 +170,10 @@ struct cpp03_overload_allocator : bare_allocator<T> {
 
   static bool construct_called;
 
+  cpp03_overload_allocator() TEST_NOEXCEPT {}
+  template <class U>
+  explicit cpp03_overload_allocator(const cpp03_overload_allocator<U>&) TEST_NOEXCEPT {}
+
   void construct(pointer p, const value_type& val) { construct(p, val, std::is_class<T>()); }
   void construct(pointer p, const value_type& val, std::true_type) {
     ::new (p) value_type(val);
@@ -395,8 +421,14 @@ class min_allocator {
 
   TEST_CONSTEXPR_CXX20 void deallocate(pointer p, std::size_t n) { std::allocator<T>().deallocate(p.ptr_, n); }
 
-  TEST_CONSTEXPR_CXX20 friend bool operator==(min_allocator, min_allocator) { return true; }
-  TEST_CONSTEXPR_CXX20 friend bool operator!=(min_allocator x, min_allocator y) { return !(x == y); }
+  template <class U>
+  TEST_CONSTEXPR_CXX20 friend bool operator==(min_allocator, min_allocator<U>) {
+    return true;
+  }
+  template <class U>
+  TEST_CONSTEXPR_CXX20 friend bool operator!=(min_allocator x, min_allocator<U> y) {
+    return !(x == y);
+  }
 };
 
 template <class T>
@@ -416,8 +448,14 @@ class complete_type_allocator {
 
   TEST_CONSTEXPR_CXX20 void deallocate(T* p, std::size_t n) { std::allocator<T>().deallocate(p, n); }
 
-  TEST_CONSTEXPR_CXX20 friend bool operator==(complete_type_allocator, complete_type_allocator) { return true; }
-  TEST_CONSTEXPR_CXX20 friend bool operator!=(complete_type_allocator, complete_type_allocator) { return false; }
+  template <class U>
+  TEST_CONSTEXPR_CXX20 friend bool operator==(complete_type_allocator, complete_type_allocator<U>) {
+    return true;
+  }
+  template <class U>
+  TEST_CONSTEXPR_CXX20 friend bool operator!=(complete_type_allocator, complete_type_allocator<U>) {
+    return false;
+  }
 };
 
 template <class T>
@@ -435,8 +473,14 @@ class explicit_allocator
 
   TEST_CONSTEXPR_CXX20 void deallocate(T* p, std::size_t n) { std::allocator<T>().deallocate(p, n); }
 
-  TEST_CONSTEXPR_CXX20 friend bool operator==(explicit_allocator, explicit_allocator) { return true; }
-  TEST_CONSTEXPR_CXX20 friend bool operator!=(explicit_allocator x, explicit_allocator y) { return !(x == y); }
+  template <class U>
+  TEST_CONSTEXPR_CXX20 friend bool operator==(explicit_allocator, explicit_allocator<U>) {
+    return true;
+  }
+  template <class U>
+  TEST_CONSTEXPR_CXX20 friend bool operator!=(explicit_allocator x, explicit_allocator<U> y) {
+    return !(x == y);
+  }
 };
 
 template <class T>
@@ -454,8 +498,14 @@ class unaligned_allocator {
 
   TEST_CONSTEXPR_CXX20 void deallocate(T* p, std::size_t n) { std::allocator<T>().deallocate(p - 1, n + 1); }
 
-  TEST_CONSTEXPR_CXX20 friend bool operator==(unaligned_allocator, unaligned_allocator) { return true; }
-  TEST_CONSTEXPR_CXX20 friend bool operator!=(unaligned_allocator x, unaligned_allocator y) { return !(x == y); }
+  template <class U>
+  TEST_CONSTEXPR_CXX20 friend bool operator==(unaligned_allocator, unaligned_allocator<U>) {
+    return true;
+  }
+  template <class U>
+  TEST_CONSTEXPR_CXX20 friend bool operator!=(unaligned_allocator x, unaligned_allocator<U> y) {
+    return !(x == y);
+  }
 };
 
 template <class T>
@@ -482,8 +532,14 @@ class safe_allocator {
     std::allocator<T>().deallocate(p, n);
   }
 
-  TEST_CONSTEXPR_CXX20 friend bool operator==(safe_allocator, safe_allocator) { return true; }
-  TEST_CONSTEXPR_CXX20 friend bool operator!=(safe_allocator x, safe_allocator y) { return !(x == y); }
+  template <class U>
+  TEST_CONSTEXPR_CXX20 friend bool operator==(safe_allocator, safe_allocator<U>) {
+    return true;
+  }
+  template <class U>
+  TEST_CONSTEXPR_CXX20 friend bool operator!=(safe_allocator x, safe_allocator<U> y) {
+    return !(x == y);
+  }
 };
 
 template <std::size_t MaxSize, class T>
@@ -510,8 +566,14 @@ struct tiny_size_allocator {
 
   TEST_CONSTEXPR_CXX20 size_type max_size() const { return MaxSize; }
 
-  friend bool operator==(tiny_size_allocator, tiny_size_allocator) { return true; }
-  friend bool operator!=(tiny_size_allocator, tiny_size_allocator) { return false; }
+  template <class U>
+  friend TEST_CONSTEXPR_CXX20 bool operator==(tiny_size_allocator, tiny_size_allocator<MaxSize, U>) {
+    return true;
+  }
+  template <class U>
+  friend TEST_CONSTEXPR_CXX20 bool operator!=(tiny_size_allocator, tiny_size_allocator<MaxSize, U>) {
+    return false;
+  }
 };
 
 #endif // MIN_ALLOCATOR_H
diff --git a/libcxx/test/support/test.support/test_allocators.pass.cpp b/libcxx/test/support/test.support/test_allocators.pass.cpp
new file mode 100644
index 0000000000000..574bfc0be1e92
--- /dev/null
+++ b/libcxx/test/support/test.support/test_allocators.pass.cpp
@@ -0,0 +1,224 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Makes sure that test allocators in "min_allocator.h" and "test_allocator.h" properly support
+// heterogeneous construction and comparison.
+
+#include <cassert>
+#include <memory>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+#if TEST_STD_VER >= 11
+template <class A, class U>
+struct rebind_alloc {
+  using type = typename std::allocator_traits<A>::template rebind_alloc<U>;
+};
+#else
+template <class A, class U>
+struct rebind_alloc {
+  typedef typename std::allocator_traits<A>::template rebind_alloc<U>::other type;
+};
+#endif
+
+TEST_CONSTEXPR_CXX20 bool test() {
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<min_allocator<int>, char>::type, min_allocator<char>);
+    min_allocator<int> a1;
+    min_allocator<char> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<complete_type_allocator<int>, char>::type, complete_type_allocator<char>);
+    complete_type_allocator<int> a1;
+    complete_type_allocator<char> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<explicit_allocator<int>, char>::type, explicit_allocator<char>);
+    explicit_allocator<int> a1;
+    explicit_allocator<char> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<unaligned_allocator<unsigned char>, char>::type, unaligned_allocator<char>);
+    unaligned_allocator<unsigned char> a1;
+    unaligned_allocator<char> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<safe_allocator<int>, char>::type, safe_allocator<char>);
+    safe_allocator<int> a1;
+    safe_allocator<char> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<tiny_size_allocator<128, int>, char>::type, tiny_size_allocator<128, char>);
+    tiny_size_allocator<128, int> a1;
+    tiny_size_allocator<128, char> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<test_allocator<int>, char>::type, test_allocator<char>);
+    test_allocator<int> a1(17);
+    test_allocator<char> a2(a1);
+    test_allocator<int> a3(29);
+    test_allocator<char> a4(a3);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+    assert(!(a1 == a3));
+    assert(!(a1 == a4));
+    assert(a1 != a3);
+    assert(a1 != a4);
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<other_allocator<int>, char>::type, other_allocator<char>);
+    other_allocator<int> a1(17);
+    other_allocator<char> a2(a1);
+    other_allocator<int> a3(29);
+    other_allocator<char> a4(a3);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+    assert(!(a1 == a3));
+    assert(!(a1 == a4));
+    assert(a1 != a3);
+    assert(a1 != a4);
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<TaggingAllocator<int>, Tag_X>::type, TaggingAllocator<Tag_X>);
+    TaggingAllocator<int> a1;
+    TaggingAllocator<Tag_X> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<limited_allocator<int, 42>, char>::type, limited_allocator<char, 42>);
+    limited_allocator<int, 42> a1;
+    limited_allocator<char, 42> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  return true;
+}
+
+int main(int, char**) {
+  test();
+#if TEST_STD_VER >= 20
+  static_assert(test());
+#endif
+
+  // constexpr-unfriendly allocators
+
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<bare_allocator<int>, char>::type, bare_allocator<char>);
+    bare_allocator<int> a1;
+    bare_allocator<char> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<no_default_allocator<int>, char>::type, no_default_allocator<char>);
+    no_default_allocator<int> a1 = no_default_allocator<int>::create();
+    no_default_allocator<char> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<malloc_allocator<int>, char>::type, malloc_allocator<char>);
+    malloc_allocator_base::disable_default_constructor = false;
+    malloc_allocator<int> a1;
+    malloc_allocator_base::disable_default_constructor = true;
+    malloc_allocator<char> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<cpp03_allocator<int>, char>::type, cpp03_allocator<char>);
+    cpp03_allocator<int> a1;
+    cpp03_allocator<char> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<cpp03_overload_allocator<int>, char>::type, cpp03_overload_allocator<char>);
+    cpp03_overload_allocator<int> a1;
+    cpp03_overload_allocator<char> a2(a1);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+  }
+  {
+    ASSERT_SAME_TYPE(rebind_alloc<SocccAllocator<int>, char>::type, SocccAllocator<char>);
+    SocccAllocator<int> a1(17);
+    SocccAllocator<char> a2(a1);
+    SocccAllocator<int> a3(29);
+    SocccAllocator<char> a4(a3);
+
+    assert(a1 == a1);
+    assert(a1 == a2);
+    assert(!(a1 != a1));
+    assert(!(a1 != a2));
+    assert(a1 == a3);
+    assert(a1 == a4);
+    assert(!(a1 != a3));
+    assert(!(a1 != a4));
+  }
+  return 0;
+}
diff --git a/libcxx/test/support/test.support/test_check_assertion.pass.cpp b/libcxx/test/support/test.support/test_check_assertion.pass.cpp
index 9d356ef30a501..9d0f847dee6b5 100644
--- a/libcxx/test/support/test.support/test_check_assertion.pass.cpp
+++ b/libcxx/test/support/test.support/test_check_assertion.pass.cpp
@@ -6,9 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03
-// UNSUPPORTED: libcpp-hardening-mode=none
+// REQUIRES: can-test-hardening-assertions-fast
 // XFAIL: availability-verbose_abort-missing
 
 #include <cassert>
diff --git a/libcxx/test/support/test_allocator.h b/libcxx/test/support/test_allocator.h
index f8b622d7f9520..60845fa35a086 100644
--- a/libcxx/test/support/test_allocator.h
+++ b/libcxx/test/support/test_allocator.h
@@ -195,8 +195,14 @@ class test_allocator {
       ++stats_->destroy_count;
     p->~T();
   }
-  TEST_CONSTEXPR friend bool operator==(const test_allocator& x, const test_allocator& y) { return x.data_ == y.data_; }
-  TEST_CONSTEXPR friend bool operator!=(const test_allocator& x, const test_allocator& y) { return !(x == y); }
+  template <class U>
+  TEST_CONSTEXPR friend bool operator==(const test_allocator& x, const test_allocator<U>& y) {
+    return x.data_ == static_cast<const test_allocator&>(y).data_;
+  }
+  template <class U>
+  TEST_CONSTEXPR friend bool operator!=(const test_allocator& x, const test_allocator<U>& y) {
+    return !(x == y);
+  }
 
   TEST_CONSTEXPR int get_data() const { return data_; }
   TEST_CONSTEXPR int get_id() const { return id_; }
@@ -259,8 +265,14 @@ class test_allocator<void> {
   TEST_CONSTEXPR int get_id() const { return id_; }
   TEST_CONSTEXPR int get_data() const { return data_; }
 
-  TEST_CONSTEXPR friend bool operator==(const test_allocator& x, const test_allocator& y) { return x.data_ == y.data_; }
-  TEST_CONSTEXPR friend bool operator!=(const test_allocator& x, const test_allocator& y) { return !(x == y); }
+  template <class U>
+  TEST_CONSTEXPR friend bool operator==(const test_allocator& x, const test_allocator<U>& y) {
+    return x.data_ == static_cast<const test_allocator&>(y).data_;
+  }
+  template <class U>
+  TEST_CONSTEXPR friend bool operator!=(const test_allocator& x, const test_allocator<U>& y) {
+    return !(x == y);
+  }
 };
 
 template <class T>
@@ -284,11 +296,15 @@ class other_allocator {
 
   TEST_CONSTEXPR_CXX14 other_allocator select_on_container_copy_construction() const { return other_allocator(-2); }
 
-  TEST_CONSTEXPR_CXX14 friend bool operator==(const other_allocator& x, const other_allocator& y) {
-    return x.data_ == y.data_;
+  template <class U>
+  TEST_CONSTEXPR_CXX14 friend bool operator==(const other_allocator& x, const other_allocator<U>& y) {
+    return x.data_ == y.get_data();
   }
 
-  TEST_CONSTEXPR_CXX14 friend bool operator!=(const other_allocator& x, const other_allocator& y) { return !(x == y); }
+  template <class U>
+  TEST_CONSTEXPR_CXX14 friend bool operator!=(const other_allocator& x, const other_allocator<U>& y) {
+    return !(x == y);
+  }
   TEST_CONSTEXPR int get_data() const { return data_; }
 
   typedef std::true_type propagate_on_container_copy_assignment;
@@ -361,6 +377,18 @@ class TaggingAllocator {
 
   TEST_CONSTEXPR_CXX20 T* allocate(std::size_t n) { return std::allocator<T>().allocate(n); }
   TEST_CONSTEXPR_CXX20 void deallocate(T* p, std::size_t n) { std::allocator<T>().deallocate(p, n); }
+
+  template <class U>
+  TEST_CONSTEXPR friend bool operator==(const TaggingAllocator&, const TaggingAllocator<U>&) {
+    return true;
+  }
+
+#if TEST_STD_VER < 20
+  template <class U>
+  TEST_CONSTEXPR friend bool operator!=(const TaggingAllocator&, const TaggingAllocator<U>&) {
+    return false;
+  }
+#endif
 };
 
 template <std::size_t MaxAllocs>
@@ -508,7 +536,17 @@ struct SocccAllocator {
 
   SocccAllocator select_on_container_copy_construction() const { return SocccAllocator(count_ + 1); }
 
-  bool operator==(const SocccAllocator&) const { return true; }
+  template <class U>
+  bool operator==(const SocccAllocator<U>&) const {
+    return true;
+  }
+
+#if TEST_STD_VER < 20
+  template <class U>
+  bool operator!=(const SocccAllocator<U>&) const {
+    return false;
+  }
+#endif
 
   using propagate_on_container_copy_assignment = std::false_type;
   using propagate_on_container_move_assignment = std::false_type;
diff --git a/libcxx/utils/libcxx/test/features/__init__.py b/libcxx/utils/libcxx/test/features/__init__.py
index 5c0d1f3aaafc6..8e9bc0e97b2d6 100644
--- a/libcxx/utils/libcxx/test/features/__init__.py
+++ b/libcxx/utils/libcxx/test/features/__init__.py
@@ -6,7 +6,7 @@
 #
 # ===----------------------------------------------------------------------===##
 
-from . import availability, compiler, gdb, libcxx_macros, localization, misc, platform
+from . import availability, compiler, gdb, hardening, libcxx_macros, localization, misc, platform
 
 # Lit features are evaluated in order. Some features depend on other features, so
 # we are careful to define them in the correct order. For example, several features
@@ -19,3 +19,4 @@
 DEFAULT_FEATURES += gdb.features
 DEFAULT_FEATURES += misc.features
 DEFAULT_FEATURES += availability.features
+DEFAULT_FEATURES += hardening.features # this depends on availability, misc and libc++ macro features
diff --git a/libcxx/utils/libcxx/test/features/hardening.py b/libcxx/utils/libcxx/test/features/hardening.py
new file mode 100644
index 0000000000000..dadf1a2c547f9
--- /dev/null
+++ b/libcxx/utils/libcxx/test/features/hardening.py
@@ -0,0 +1,113 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+from libcxx.test.dsl import Feature, programSucceeds
+from lit.BooleanExpression import BooleanExpression
+
+features = []
+
+# Detect the hardening mode that the tests are being compiled with as `libcpp-hardening-mode=<mode>`.
+#
+# Note that this is the mode in effect when compiling the tests, which is not necessarily the mode
+# the library was configured with, since it can be overridden with compiler flags.
+for mode in ("none", "fast", "extensive", "debug"):
+    check_program = f"""
+        #include <stddef.h> // any header to get the definitions
+        int main(int, char**) {{
+        #if defined(_LIBCPP_VERSION) && \\
+                defined(_LIBCPP_HARDENING_MODE) && _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_{mode.upper()}
+            return 0;
+        #else
+            return 1;
+        #endif
+        }}
+    """
+    features.append(
+        Feature(
+            name=f"libcpp-hardening-mode={mode}",
+            when=lambda cfg, prog=check_program: programSucceeds(cfg, prog)
+        )
+    )
+
+# Detect the assertion semantic used by hardening as `libcpp-assertion-semantic=<semantic>`.
+#
+# Like the hardening mode above, this is the semantic in effect when compiling the tests, whether it
+# comes from how the library was configured or from elsewhere.
+for semantic in ("ignore", "observe", "quick_enforce", "enforce"):
+    check_program = f"""
+        #include <stddef.h> // any header to get the definitions
+        int main(int, char**) {{
+        #if defined(_LIBCPP_VERSION) && \\
+                defined(_LIBCPP_ASSERTION_SEMANTIC) && \\
+                _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_{semantic.upper()}
+            return 0;
+        #else
+            return 1;
+        #endif
+        }}
+    """
+    features.append(
+        Feature(
+            name=f"libcpp-assertion-semantic={semantic}",
+            when=lambda cfg, prog=check_program: programSucceeds(cfg, prog)
+        )
+    )
+
+# Whether the test suite is able to check that hardening assertions fire.
+#
+# Such tests are written using the `TEST_LIBCPP_ASSERT_FAILURE` macro from `check_assertion.h`,
+# which runs the code that should trigger the assertion in a child process and inspects how that
+# process died. That machinery requires Unix headers (it uses `fork` and pipes), C++11 or later,
+# and localization support (it uses `<regex>` and `<sstream>`).
+#
+# On top of that, a failing assertion must be observable at all, which depends on the assertion
+# semantic in effect:
+# - with `ignore`, the assertion isn't even evaluated, so there is nothing to observe;
+# - with `enforce`, the failure is reported through `std::__libcpp_verbose_abort`. When that function
+#   isn't available in the library we're running against, `_LIBCPP_VERBOSE_ABORT` degrades to a bare
+#   `abort()`, so neither the assertion message nor the way the process died match what the test suite
+#   expects.
+_can_test_hardening_assertions = " && ".join(
+    [
+        "stdlib=libc++",
+        "has-unix-headers",
+        "!c++03",
+        "!no-localization",
+        "!libcpp-assertion-semantic=ignore",
+        "!(libcpp-assertion-semantic=enforce && availability-verbose_abort-missing)",
+    ]
+)
+
+features.append(
+    Feature(
+        name="can-test-hardening-assertions",
+        when=lambda cfg: BooleanExpression.evaluate(
+            _can_test_hardening_assertions, cfg.available_features
+        ),
+    )
+)
+
+# Whether the test suite is able to check that hardening assertions of a given category fire.
+#
+# Assertion categories are enabled by different hardening modes, so the suffix is the weakest mode in
+# which the assertion under test is enabled. For example `_LIBCPP_ASSERT_NON_NULL` is enabled in the
+# `extensive` and `debug` modes, so a test for such an assertion should use
+# `can-test-hardening-assertions-extensive`.
+enabling_modes = {
+    "fast": ("fast", "extensive", "debug"),
+    "extensive": ("extensive", "debug"),
+    "debug": ("debug",),
+}
+for category, modes in enabling_modes.items():
+    expression = "can-test-hardening-assertions && ({})".format(" || ".join(f"libcpp-hardening-mode={mode}" for mode in modes))
+    features.append(
+        Feature(
+            name=f"can-test-hardening-assertions-{category}",
+            when=lambda cfg, expr=expression: BooleanExpression.evaluate(expr, cfg.available_features)
+        )
+    )
diff --git a/libcxx/utils/libcxx/test/features/libcxx_macros.py b/libcxx/utils/libcxx/test/features/libcxx_macros.py
index bcc081e2dc2de..ec149bef1a40e 100644
--- a/libcxx/utils/libcxx/test/features/libcxx_macros.py
+++ b/libcxx/utils/libcxx/test/features/libcxx_macros.py
@@ -6,7 +6,7 @@
 #
 # ===----------------------------------------------------------------------===##
 
-from libcxx.test.dsl import Feature, compilerMacros, programSucceeds
+from libcxx.test.dsl import Feature, compilerMacros
 
 features = []
 
@@ -78,22 +78,3 @@
             and compilerMacros(cfg)[m] == "0",
         )
     )
-
-for mode in ("none", "fast", "extensive", "debug"):
-    check_program = f"""
-        #include <stddef.h> // any header to get the definitions
-        int main(int, char**) {{
-        #if defined(_LIBCPP_VERSION) && \\
-                defined(_LIBCPP_HARDENING_MODE) && _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_{mode.upper()}
-            return 0;
-        #else
-            return 1;
-        #endif
-        }}
-    """
-    features.append(
-        Feature(
-            name=f"libcpp-hardening-mode={mode}",
-            when=lambda cfg, prog=check_program: programSucceeds(cfg, prog)
-        )
-    )
diff --git a/libcxx/utils/libcxx/test/params.py b/libcxx/utils/libcxx/test/params.py
index d0bb38ab84df4..909815694c2cc 100644
--- a/libcxx/utils/libcxx/test/params.py
+++ b/libcxx/utils/libcxx/test/params.py
@@ -484,7 +484,6 @@ def getSuitableClangTidy(cfg):
                 AddCompileFlag("-D_LIBCPP_ASSERTION_SEMANTIC=_LIBCPP_ASSERTION_SEMANTIC_OBSERVE")       if assertion_semantic == "observe" else None,
                 AddCompileFlag("-D_LIBCPP_ASSERTION_SEMANTIC=_LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE") if assertion_semantic == "quick_enforce" else None,
                 AddCompileFlag("-D_LIBCPP_ASSERTION_SEMANTIC=_LIBCPP_ASSERTION_SEMANTIC_ENFORCE")       if assertion_semantic == "enforce" else None,
-                AddFeature("libcpp-assertion-semantic={}".format(assertion_semantic))                   if assertion_semantic != "undefined" else None,
             ],
         ),
     ),
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 05c0f3f1c445b..23649d6200719 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -537,16 +537,19 @@ template <class ELFT>
 static void
 handleAArch64BAAndGnuProperties(ObjFile<ELFT> *file, Ctx &ctx,
                                 const AArch64BuildAttrSubsections &baInfo) {
+  // Missing subsections have zero-initialized data fields, so we must check
+  // presence before comparing against GNU properties.
+  bool baPauthInfoPresent = baInfo.Pauth.TagPlatform || baInfo.Pauth.TagSchema;
+
   if (file->aarch64PauthAbiCoreInfo) {
     // Check for data mismatch.
-    if (file->aarch64PauthAbiCoreInfo) {
-      if (baInfo.Pauth.TagPlatform != file->aarch64PauthAbiCoreInfo->platform ||
-          baInfo.Pauth.TagSchema != file->aarch64PauthAbiCoreInfo->version)
-        Err(ctx) << file
-                 << " GNU properties and build attributes have conflicting "
-                    "AArch64 PAuth data";
-    }
-    if (baInfo.AndFeatures != file->andFeatures)
+    if (baPauthInfoPresent &&
+        (baInfo.Pauth.TagPlatform != file->aarch64PauthAbiCoreInfo->platform ||
+         baInfo.Pauth.TagSchema != file->aarch64PauthAbiCoreInfo->version))
+      Err(ctx) << file
+               << " GNU properties and build attributes have conflicting "
+                  "AArch64 PAuth data";
+    if (baInfo.AndFeatures && baInfo.AndFeatures != file->andFeatures)
       Err(ctx) << file
                << " GNU properties and build attributes have conflicting "
                   "AArch64 PAuth data";
@@ -557,14 +560,14 @@ handleAArch64BAAndGnuProperties(ObjFile<ELFT> *file, Ctx &ctx,
     // PAuthAbiCoreInfo when there is at least one non-zero value. The
     // specification reserves TagPlatform = 0, TagSchema = 1 values to match the
     // 'Invalid' GNU property section with platform = 0, version = 0.
-    if (baInfo.Pauth.TagPlatform || baInfo.Pauth.TagSchema) {
+    if (baPauthInfoPresent) {
       if (baInfo.Pauth.TagPlatform == 0 && baInfo.Pauth.TagSchema == 1)
         file->aarch64PauthAbiCoreInfo = {0, 0};
       else
         file->aarch64PauthAbiCoreInfo = {baInfo.Pauth.TagPlatform,
                                          baInfo.Pauth.TagSchema};
     }
-    file->andFeatures = baInfo.AndFeatures;
+    file->andFeatures |= baInfo.AndFeatures;
   }
 }
 
diff --git a/lld/test/ELF/aarch64-build-attributes-private-subsection.s b/lld/test/ELF/aarch64-build-attributes-private-subsection.s
new file mode 100644
index 0000000000000..40adb97dd5c3c
--- /dev/null
+++ b/lld/test/ELF/aarch64-build-attributes-private-subsection.s
@@ -0,0 +1,55 @@
+// REQUIRES: aarch64
+
+/// Test that a build attributes section without 'aeabi_feature_and_bits' and
+/// 'aeabi_pauthabi' subsections does not conflict with GNU Program Properties.
+
+// RUN: llvm-mc -triple=aarch64 -mattr=+bti -aarch64-mark-bti-property -filetype=obj %s -o %t.o
+// RUN: ld.lld -shared %t.o -z force-bti -o %t.out 2>&1 | count 0
+// RUN: llvm-readobj --notes %t.out | FileCheck %s
+
+// RUN: llvm-mc -triple=aarch64 --defsym EMIT_GNU_PROPERTY=1 -filetype=obj %s -o %t.o
+// RUN: ld.lld -shared %t.o -z force-bti -o %t.out 2>&1 | count 0
+// RUN: llvm-readobj --notes %t.out | FileCheck %s --check-prefixes=CHECK,WITH_PAUTH
+
+// CHECK:      NoteSections [
+// CHECK-NEXT:   NoteSection {
+// CHECK-NEXT:     Name: .note.gnu.property
+// CHECK-NEXT:     Offset:
+// CHECK-NEXT:     Size:
+// CHECK-NEXT:     Notes [
+// CHECK-NEXT:       {
+// CHECK-NEXT:         Owner: GNU
+// CHECK-NEXT:         Data size:
+// CHECK-NEXT:         Type: NT_GNU_PROPERTY_TYPE_0 (property note)
+// CHECK-NEXT:         Property [
+// CHECK-NEXT:           aarch64 feature: BTI
+// WITH_PAUTH-NEXT:      AArch64 PAuth ABI core info: platform 0x31 (unknown), version 0x13
+// CHECK-NEXT:         ]
+// CHECK-NEXT:       }
+// CHECK-NEXT:     ]
+// CHECK-NEXT:   }
+// CHECK-NEXT: ]
+
+.aeabi_subsection anon_dummy, optional, uleb128
+.aeabi_attribute 1, 1
+
+.ifdef EMIT_GNU_PROPERTY
+.section ".note.gnu.property", "a"
+  .long 0x4           // Name length 4 ("GNU")
+  .long end - begin   // Data length
+  .long 0x5           // Type: NT_GNU_PROPERTY_TYPE_0
+  .asciz "GNU"        // Name
+  .p2align 3
+begin:
+  .long 0xc0000000    // GNU_PROPERTY_AARCH64_FEATURE_1_AND
+  .long 0x4
+  .long 0x1           // GNU_PROPERTY_AARCH64_FEATURE_1_BTI
+  .long 0x0
+  // PAuth ABI property note
+  .long 0xc0000001    // GNU_PROPERTY_AARCH64_FEATURE_PAUTH
+  .long 0x10          // Data length
+  .quad 0x31          // PAuth ABI platform
+  .quad 0x13          // PAuth ABI version
+  .p2align 3          // Align to 8 byte for 64 bit
+end:
+.endif
diff --git a/llvm/docs/HowToSubmitABug.rst b/llvm/docs/HowToSubmitABug.rst
index b6f5685b30a68..10ded5c0bdd9f 100644
--- a/llvm/docs/HowToSubmitABug.rst
+++ b/llvm/docs/HowToSubmitABug.rst
@@ -232,18 +232,8 @@ program is clean under various `sanitizers
 "LLVM bugs" that we have chased down ended up being bugs in the program being
 compiled, not LLVM.
 
-Once you determine that the program itself is not buggy, you should choose
-which code generator you wish to compile the program with (e.g., LLC or the JIT)
-and optionally a series of LLVM passes to run.  For example:
-
-.. code-block:: bash
-
-   bugpoint -run-llc [... optzn passes ...] file-to-test.bc --args -- [program arguments]
-
-bugpoint will try to narrow down your list of passes to the one pass that
-causes an error, and simplify the bitcode file as much as it can to assist
-you. It will print a message letting you know how to reproduce the
-resulting error.
-
-The :doc:`OptBisect <OptBisect>` page shows an alternative method for finding
-incorrect optimization passes.
+Once you determine that the program itself is not buggy, you should work on
+reducing the inputs required to reproduce the miscompilation. The
+:doc:`OptBisect <OptBisect>` page shows how to find the optimization pass
+causing the miscompile. You can use :doc:`llvm-reduce <CommandGuide/llvm-reduce>`
+to minimize the bitcode necessary to reproduce the miscompilation.
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index 810938303c591..f0a1439b30334 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -2062,6 +2062,75 @@ described in the `s2g.tile` mode intrinsics above.
 
 For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor).
 
+#### '`llvm.nvvm.cp.async.bulk.tensor.reduce.tile.[1-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tensor_map, i32 %d0, i64 %ch, i32 %red_op, i1 %flag_ch)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(..., i32 %d0, i32 %d1, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(..., i32 %d0, i32 %d1, i32 %d2, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+```
+
+##### Overview:
+
+The '`@llvm.nvvm.cp.async.bulk.tensor.reduce.tile.[1-5]d`' intrinsics
+correspond to the `cp.reduce.async.bulk.tensor.[1-5]d.global.shared::cta.*`
+set of PTX instructions. These instructions initiate an asynchronous reduction
+operation of tensor data in global memory with the tensor data in shared::cta
+memory, using `tile` mode. The dimension of the tensor data ranges from 1d to
+5d with the coordinates specified by the `i32 %d0 ... i32 %d4` arguments. The
+`i32 %red_op` argument selects the reduction operation to perform. It must be
+a compile-time constant in the half-open range `[0, 8)`, with the following
+encoding:
+
+| `red_op` | Reduction Operation |
+|:--------:|:--------------------|
+|    0     |   `add`             |
+|    1     |   `min`             |
+|    2     |   `max`             |
+|    3     |   `inc`             |
+|    4     |   `dec`             |
+|    5     |   `and`             |
+|    6     |   `or`              |
+|    7     |   `xor`             |
+
+The symbolic LLVM IR annotation for `red_op` and the PTX reduction suffix use
+the same canonical operator spelling.
+
+- The last argument to these intrinsics is a boolean flag indicating support
+  for cache_hint. This flag argument must be a compile-time constant. When
+  set, it indicates a valid cache_hint (`i64 %ch`) and generates the
+  `.L2::cache_hint` variant of the PTX instruction.
+
+For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor).
+
+#### '`llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.[3-5]d`'
+
+##### Syntax:
+
+```llvm
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 %red_op, i1 %flag_ch)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
+```
+
+##### Overview:
+
+The '`@llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.[3-5]d`' intrinsics
+correspond to the `cp.reduce.async.bulk.tensor.[3-5]d.global.shared::cta.*`
+set of PTX instructions. These instructions initiate an asynchronous reduction
+operation of tensor data in global memory with the tensor data in shared::cta
+memory, using `im2col` mode. In this mode, the tensor has to be at least
+three-dimensional. The supported reduction operations are the same as the ones
+in the `tile` mode. The `i32 %red_op` argument and the last boolean flag
+argument have the same functionality as described in the `tile` mode
+intrinsics above.
+
+For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor).
+
 #### '`llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.[1-5]d`'
 
 ##### Syntax:
@@ -2137,67 +2206,6 @@ functionality as described in the `tile` mode intrinsics above.
 
 For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor).
 
-#### '`llvm.nvvm.cp.async.bulk.tensor.reduce.[red_op].tile.[1-5]d`'
-
-##### Syntax:
-
-```llvm
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.1d(ptr addrspace(3) %src, ptr %tensor_map, i32 %d0, i64 %ch, i1 %flag_ch)
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.1d(ptr addrspace(3) %src, ptr %tensor_map, i32 %d0, i64 %ch, i1 %flag_ch)
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.1d(ptr addrspace(3) %src, ptr %tensor_map, i32 %d0, i64 %ch, i1 %flag_ch)
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.1d(ptr addrspace(3) %src, ptr %tensor_map, i32 %d0, i64 %ch, i1 %flag_ch)
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.1d(ptr addrspace(3) %src, ptr %tensor_map, i32 %d0, i64 %ch, i1 %flag_ch)
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.1d(ptr addrspace(3) %src, ptr %tensor_map, i32 %d0, i64 %ch, i1 %flag_ch)
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.1d(ptr addrspace(3) %src, ptr %tensor_map, i32 %d0, i64 %ch, i1 %flag_ch)
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.1d(ptr addrspace(3) %src, ptr %tensor_map, i32 %d0, i64 %ch, i1 %flag_ch)
-
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>.tile.2d(..., i32 %d0, i32 %d1, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>.tile.3d(..., i32 %d0, i32 %d1, i32 %d2, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>.tile.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>.tile.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
-```
-
-##### Overview:
-
-The '`@llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>.tile.[1-5]d`' intrinsics
-correspond to the `cp.reduce.async.bulk.tensor.[1-5]d.*` set of PTX
-instructions. These instructions initiate an asynchronous reduction operation of
-tensor data in global memory with the tensor data in shared\{::cta} memory, using
-`tile` mode. The dimension of the tensor data ranges from 1d to 5d with the
-coordinates specified by the `i32 %d0 ... i32 %d4` arguments. The supported
-reduction operations are {add, min, max, inc, dec, and, or, xor} as described in
-the `tile.1d` intrinsics.
-
-- The last argument to these intrinsics is a boolean flag indicating support for
-  cache_hint. This flag argument must be a compile-time constant. When set, it
-  indicates a valid cache_hint (`i64 %ch`) and generates the
-  `.L2::cache_hint` variant of the PTX instruction.
-
-For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor).
-
-#### '`llvm.nvvm.cp.async.bulk.tensor.reduce.[red_op].im2col.[3-5]d`'
-
-##### Syntax:
-
-```llvm
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>.im2col.3d(ptr addrspace(3) %src, ptr %tensor_map, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 %flag_ch)
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>.im2col.4d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, ...)
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>.im2col.5d(..., i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, ...)
-```
-
-##### Overview:
-
-The '`@llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>.im2col.[3-5]d`'
-intrinsics correspond to the `cp.reduce.async.bulk.tensor.[3-5]d.*` set of PTX
-instructions. These instructions initiate an asynchronous reduction operation of
-tensor data in global memory with the tensor data in shared\{::cta} memory, using
-`im2col` mode. In this mode, the tensor has to be at least three-dimensional.
-The supported reduction operations supported are the same as the ones in the
-tile mode. The last argument to these intrinsics is a boolean flag, with the
-same functionality as described in the `tile` mode intrinsics above.
-
-For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor).
-
 ### Warp Group Intrinsics
 
 #### '`llvm.nvvm.wgmma.fence.sync.aligned`'
diff --git a/llvm/docs/OptBisect.rst b/llvm/docs/OptBisect.rst
index e8a09e64e1eeb..7eee52ff1c0d6 100644
--- a/llvm/docs/OptBisect.rst
+++ b/llvm/docs/OptBisect.rst
@@ -21,13 +21,10 @@ allocation.
 
 The ``-opt-bisect-limit`` option can be used with any tool, including front ends
 such as clang, that uses the core LLVM library for optimization and code
-generation.  The exact syntax for invoking the option is discussed below.
-
-This feature is not intended to replace other debugging tools such as bugpoint.
-Rather it provides an alternate course of action when reproducing the problem
-requires a complex build infrastructure that would make using bugpoint
-impractical or when reproducing the failure requires a sequence of
-transformations that is difficult to replicate with tools like opt and llc.
+generation.  The exact syntax for invoking the option is discussed below. This
+makes ``-opt-bisect-limit`` easy to use in situations that require complex
+build infrastructure or when a full pass pipeline is needed that is difficult
+to replace in opt or llc.
 
 
 Getting Started
diff --git a/llvm/docs/Passes.md b/llvm/docs/Passes.md
index 72a5c550da614..60366dddbd300 100644
--- a/llvm/docs/Passes.md
+++ b/llvm/docs/Passes.md
@@ -589,15 +589,6 @@ eliminating loops with non-infinite computable trip counts that have no side
 effects or volatile instructions, and do not contribute to the computation of
 the function's return value.
 
-(passes-loop-extract)=
-
-### `loop-extract`: Extract loops into new functions
-
-A pass wrapper around the `ExtractLoop()` scalar transformation to extract
-each top-level loop into its own new function.  If the loop is the *only* loop
-in a given function, it is not touched.  This is a pass most useful for
-debugging via bugpoint.
-
 ### `loop-fusion`: Loop Fusion
 
 Merges adjacent loops when it can prove the transformation preserves the
@@ -896,10 +887,9 @@ algorithm:
 
 This section describes the LLVM Utility Passes.
 
-### `extract-blocks`: Extract Basic Blocks From Module (for bugpoint use)
+### `extract-blocks`: Extract Basic Blocks From Module
 
-This pass is used by bugpoint to extract all blocks from the module into their
-own functions.
+This pass extracts all blocks from the module into their own functions.
 
 ### `instnamer`: Assign names to anonymous instructions
 
diff --git a/llvm/docs/WritingAnLLVMPass.md b/llvm/docs/WritingAnLLVMPass.md
index 1627dfcb8dcd8..c1cf43d0198d2 100644
--- a/llvm/docs/WritingAnLLVMPass.md
+++ b/llvm/docs/WritingAnLLVMPass.md
@@ -346,7 +346,7 @@ the machine-dependent representation of each LLVM function in the program.
 
 Code generator passes are registered and initialized specially by
 `TargetMachine::addPassesToEmitFile` and similar routines, so they cannot
-generally be run from the {program}`opt` or {program}`bugpoint` commands.
+generally be run from the {program}`opt`.
 
 A `MachineFunctionPass` is also a `FunctionPass`, so all the restrictions
 that apply to a `FunctionPass` also apply to it.  `MachineFunctionPass`es
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index 8c429a8e1428f..823a08d7dd81b 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -1972,6 +1972,7 @@ enum {
   NT_OPENBSD_FPREGS = 21,
   NT_OPENBSD_XFPREGS = 22,
   NT_OPENBSD_WCOOKIE = 23,
+  NT_OPENBSD_PACMASK = 24,
 };
 
 // AMDGPU-specific section indices.
diff --git a/llvm/include/llvm/CodeGen/ExpandReductions.h b/llvm/include/llvm/CodeGen/ExpandReductions.h
index 25b88aef93ad4..a7c1b3692d229 100644
--- a/llvm/include/llvm/CodeGen/ExpandReductions.h
+++ b/llvm/include/llvm/CodeGen/ExpandReductions.h
@@ -14,7 +14,7 @@
 namespace llvm {
 
 class ExpandReductionsPass
-    : public OptionalPassInfoMixin<ExpandReductionsPass> {
+    : public RequiredPassInfoMixin<ExpandReductionsPass> {
 public:
   LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
diff --git a/llvm/include/llvm/CodeGen/LiveDebugValuesPass.h b/llvm/include/llvm/CodeGen/LiveDebugValuesPass.h
index 437569c08c04a..b098cf27f8057 100644
--- a/llvm/include/llvm/CodeGen/LiveDebugValuesPass.h
+++ b/llvm/include/llvm/CodeGen/LiveDebugValuesPass.h
@@ -13,7 +13,7 @@
 
 namespace llvm {
 
-class LiveDebugValuesPass : public OptionalPassInfoMixin<LiveDebugValuesPass> {
+class LiveDebugValuesPass : public RequiredPassInfoMixin<LiveDebugValuesPass> {
   const bool ShouldEmitDebugEntryValues;
 
 public:
diff --git a/llvm/include/llvm/CodeGen/RemoveRedundantDebugValues.h b/llvm/include/llvm/CodeGen/RemoveRedundantDebugValues.h
index 1c2a8321bae10..0ad6c2612a324 100644
--- a/llvm/include/llvm/CodeGen/RemoveRedundantDebugValues.h
+++ b/llvm/include/llvm/CodeGen/RemoveRedundantDebugValues.h
@@ -14,7 +14,7 @@
 namespace llvm {
 
 class RemoveRedundantDebugValuesPass
-    : public OptionalPassInfoMixin<RemoveRedundantDebugValuesPass> {
+    : public RequiredPassInfoMixin<RemoveRedundantDebugValuesPass> {
 public:
   LLVM_ABI PreservedAnalyses run(MachineFunction &MF,
                                  MachineFunctionAnalysisManager &MFAM);
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index 8ab1bb30a42b5..ea2f0f027839d 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -1138,6 +1138,11 @@ class ExecutionSession {
   /// object.
   LLVM_ABI ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC);
 
+  ExecutionSession(const ExecutionSession &) = delete;
+  ExecutionSession &operator=(const ExecutionSession &) = delete;
+  ExecutionSession(ExecutionSession &&) = delete;
+  ExecutionSession &operator=(ExecutionSession &&) = delete;
+
   /// Destroy an ExecutionSession. Verifies that endSession was called prior to
   /// destruction.
   LLVM_ABI ~ExecutionSession();
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 40f3c40a76bc6..fc04a5dc64e78 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -2833,16 +2833,22 @@ foreach dim = 1...5 in {
           [llvm_i1_ty],                     // Flag for cache_hint
           [IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>]>;
 
-    // Intrinsics for TMA Copy with reduction
-    foreach red_op = ["add", "min", "max", "inc", "dec", "and", "or", "xor"] in
-      def int_nvvm_cp_async_bulk_tensor_reduce_ # red_op # _ # mode # _ # dim # d :
-        DefaultAttrsIntrinsicFlags<[],
-            !listconcat([llvm_shared_ptr_ty,  // src_smem_ptr
-                         llvm_ptr_ty],        // tensormap_ptr
-                         tensor_dim_args,     // actual tensor dims
-                        [llvm_i64_ty]),       // cache_hint
-          [llvm_i1_ty],                       // Flag for cache_hint
-          [IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>]>;
+    defvar reduce_params =
+        !listconcat([llvm_shared_ptr_ty, // src_smem_ptr
+                     llvm_ptr_ty],       // tensormap_ptr
+                    tensor_dim_args,     // actual tensor dims
+                    [llvm_i64_ty]);      // cache_hint
+    defvar red_op_idx = !size(reduce_params);
+    def int_nvvm_cp_async_bulk_tensor_reduce_ # mode # _ # dim # d :
+      DefaultAttrsIntrinsicFlags<[],
+        reduce_params,
+        [llvm_i32_ty,    // reduction operation
+         llvm_i1_ty],    // Flag for cache_hint
+        [IntrConvergent, ReadOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>,
+        // Allowed values for red_op are {0..7} i.e. [0, 8).
+        Range<ArgIndex<red_op_idx>, 0, 8>,
+        ArgInfo<ArgIndex<red_op_idx>, [ArgName<"red_op">,
+        ImmArgPrinter<"printTMAReductionOp">]>]>;
   }
 }
 
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index 083be6d4247c1..f49224ac74171 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -19,6 +19,7 @@
 
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsNVPTX.h"
@@ -41,6 +42,28 @@ enum class TMAReductionOp : uint8_t {
   XOR = 7,
 };
 
+inline StringRef getTMATensorReductionOpName(TMAReductionOp Op) {
+  switch (Op) {
+  case TMAReductionOp::ADD:
+    return "add";
+  case TMAReductionOp::MIN:
+    return "min";
+  case TMAReductionOp::MAX:
+    return "max";
+  case TMAReductionOp::INC:
+    return "inc";
+  case TMAReductionOp::DEC:
+    return "dec";
+  case TMAReductionOp::AND:
+    return "and";
+  case TMAReductionOp::OR:
+    return "or";
+  case TMAReductionOp::XOR:
+    return "xor";
+  }
+  llvm_unreachable("invalid TMA tensorreduction operation");
+}
+
 // Enum to represent the cta_group::1 and
 // cta_group::2 variants in TMA/TCGEN05 family of
 // PTX instructions.
@@ -106,6 +129,8 @@ enum class TensormapFillMode : uint8_t {
 
 LLVM_ABI void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal);
 
+LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal);
+
 LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS,
                                            const Constant *ImmArgVal);
 
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index d80e02dcb356f..74fe01a46b5b9 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -174,7 +174,6 @@ LLVM_ABI void initializeLocalStackSlotPassPass(PassRegistry &);
 LLVM_ABI void initializeLocalizerPass(PassRegistry &);
 LLVM_ABI void initializeLogicalSROALegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeLoopDataPrefetchLegacyPassPass(PassRegistry &);
-LLVM_ABI void initializeLoopExtractorLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeLoopInfoWrapperPassPass(PassRegistry &);
 LLVM_ABI void initializeLoopPassPass(PassRegistry &);
 LLVM_ABI void initializeLoopSimplifyPass(PassRegistry &);
@@ -306,7 +305,6 @@ LLVM_ABI void
 initializeSeparateConstOffsetFromGEPLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeShadowStackGCLoweringPass(PassRegistry &);
 LLVM_ABI void initializeShrinkWrapLegacyPass(PassRegistry &);
-LLVM_ABI void initializeSingleLoopExtractorPass(PassRegistry &);
 LLVM_ABI void initializeSinkingLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeSjLjEHPreparePass(PassRegistry &);
 LLVM_ABI void initializeSlotIndexesWrapperPassPass(PassRegistry &);
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 5182341fa7a89..d1d714f499532 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 // This header file pulls in all transformation and analysis passes for tools
-// like opt and bugpoint that need this functionality.
+// like opt that need this functionality.
 //
 //===----------------------------------------------------------------------===//
 
@@ -95,7 +95,6 @@ struct ForcePassLinking {
     (void)llvm::createLCSSAPass();
     (void)llvm::createLICMPass();
     (void)llvm::createLazyValueInfoPass();
-    (void)llvm::createLoopExtractorPass();
     (void)llvm::createLoopSimplifyPass();
     (void)llvm::createLoopStrengthReducePass();
     (void)llvm::createLoopTermFoldPass();
@@ -119,7 +118,6 @@ struct ForcePassLinking {
     (void)llvm::createRegionViewerPass();
     (void)llvm::createSafeStackPass();
     (void)llvm::createSROAPass();
-    (void)llvm::createSingleLoopExtractorPass();
     (void)llvm::createTailCallEliminationPass();
     (void)llvm::createConstantHoistingPass();
     (void)llvm::createCodeGenPrepareLegacyPass();
diff --git a/llvm/include/llvm/Support/DebugCounter.h b/llvm/include/llvm/Support/DebugCounter.h
index 017c16e774160..8df853063e994 100644
--- a/llvm/include/llvm/Support/DebugCounter.h
+++ b/llvm/include/llvm/Support/DebugCounter.h
@@ -11,15 +11,15 @@
 /// thing happening.
 ///
 /// To give a use case: Imagine you have a file, very large, and you
-/// are trying to understand the minimal transformation that breaks it. Bugpoint
-/// and bisection is often helpful here in narrowing it down to a specific pass,
-/// but it's still a very large file, and a very complicated pass to try to
-/// debug.  That is where debug counting steps in.  You can instrument the pass
-/// with a debug counter before it does a certain thing, and depending on the
-/// counts, it will either execute that thing or not.  The debug counter itself
-/// consists of a list of chunks (inclusive numeric intervals). `shouldExecute`
-/// returns true iff the list is empty or the current count is in one of the
-/// chunks.
+/// are trying to understand the minimal transformation that breaks it.
+/// llvm-reduce and bisection is often helpful here in narrowing it down to a
+/// specific pass, but it's still a very large file, and a very complicated pass
+/// to try to debug.  That is where debug counting steps in.  You can instrument
+/// the pass with a debug counter before it does a certain thing, and depending
+/// on the counts, it will either execute that thing or not.  The debug counter
+/// itself consists of a list of chunks (inclusive numeric intervals).
+/// `shouldExecute` returns true iff the list is empty or the current count is
+/// in one of the chunks.
 ///
 /// Note that a counter set to a negative number will always execute. For a
 /// concrete example, during predicateinfo creation, the renaming pass replaces
diff --git a/llvm/include/llvm/Transforms/IPO.h b/llvm/include/llvm/Transforms/IPO.h
index 7523ae66429ac..6f5693110c8a4 100644
--- a/llvm/include/llvm/Transforms/IPO.h
+++ b/llvm/include/llvm/Transforms/IPO.h
@@ -28,23 +28,6 @@ class raw_ostream;
 ///
 LLVM_ABI ModulePass *createDeadArgEliminationPass();
 
-/// DeadArgHacking pass - Same as DAE, but delete arguments of external
-/// functions as well.  This is definitely not safe, and should only be used by
-/// bugpoint.
-LLVM_ABI ModulePass *createDeadArgHackingPass();
-
-//===----------------------------------------------------------------------===//
-//
-/// createLoopExtractorPass - This pass extracts all natural loops from the
-/// program into a function if it can.
-///
-LLVM_ABI Pass *createLoopExtractorPass();
-
-/// createSingleLoopExtractorPass - This pass extracts one natural loop from the
-/// program into a function if it can.  This is used by bugpoint.
-///
-LLVM_ABI Pass *createSingleLoopExtractorPass();
-
 //===----------------------------------------------------------------------===//
 /// createBarrierNoopPass - This pass is purely a module pass barrier in a pass
 /// manager.
diff --git a/llvm/include/llvm/Transforms/IPO/LoopExtractor.h b/llvm/include/llvm/Transforms/IPO/LoopExtractor.h
deleted file mode 100644
index 23328232d376d..0000000000000
--- a/llvm/include/llvm/Transforms/IPO/LoopExtractor.h
+++ /dev/null
@@ -1,35 +0,0 @@
-//===- LoopExtractor.h - Extract each loop into a new function ------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// A pass wrapper around the ExtractLoop() scalar transformation to extract each
-// top-level loop into its own new function. If the loop is the ONLY loop in a
-// given function, it is not touched. This is a pass most useful for debugging
-// via bugpoint.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TRANSFORMS_IPO_LOOPEXTRACTOR_H
-#define LLVM_TRANSFORMS_IPO_LOOPEXTRACTOR_H
-
-#include "llvm/IR/PassManager.h"
-
-namespace llvm {
-
-struct LoopExtractorPass : public OptionalPassInfoMixin<LoopExtractorPass> {
-  LoopExtractorPass(unsigned NumLoops = ~0) : NumLoops(NumLoops) {}
-  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
-  LLVM_ABI void
-  printPipeline(raw_ostream &OS,
-                function_ref<StringRef(StringRef)> MapClassName2PassName);
-
-private:
-  unsigned NumLoops;
-};
-} // namespace llvm
-
-#endif // LLVM_TRANSFORMS_IPO_LOOPEXTRACTOR_H
diff --git a/llvm/include/llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h b/llvm/include/llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h
index 11959e1c67a92..372628b26d5f6 100644
--- a/llvm/include/llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h
+++ b/llvm/include/llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h
@@ -21,7 +21,7 @@
 namespace llvm {
 
 struct ScalarizeMaskedMemIntrinPass
-    : public OptionalPassInfoMixin<ScalarizeMaskedMemIntrinPass> {
+    : public RequiredPassInfoMixin<ScalarizeMaskedMemIntrinPass> {
   LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 } // end namespace llvm
diff --git a/llvm/include/llvm/Transforms/Utils/FixIrreducible.h b/llvm/include/llvm/Transforms/Utils/FixIrreducible.h
index b10ca1a590a37..332c15970f0f6 100644
--- a/llvm/include/llvm/Transforms/Utils/FixIrreducible.h
+++ b/llvm/include/llvm/Transforms/Utils/FixIrreducible.h
@@ -12,7 +12,7 @@
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
-struct FixIrreduciblePass : OptionalPassInfoMixin<FixIrreduciblePass> {
+struct FixIrreduciblePass : RequiredPassInfoMixin<FixIrreduciblePass> {
   LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 } // namespace llvm
diff --git a/llvm/include/llvm/Transforms/Utils/LowerInvoke.h b/llvm/include/llvm/Transforms/Utils/LowerInvoke.h
index a1f5734dafc9e..5bed1309e1d06 100644
--- a/llvm/include/llvm/Transforms/Utils/LowerInvoke.h
+++ b/llvm/include/llvm/Transforms/Utils/LowerInvoke.h
@@ -19,7 +19,7 @@
 
 namespace llvm {
 
-class LowerInvokePass : public OptionalPassInfoMixin<LowerInvokePass> {
+class LowerInvokePass : public RequiredPassInfoMixin<LowerInvokePass> {
 public:
   LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
diff --git a/llvm/include/llvm/Transforms/Utils/LowerSwitch.h b/llvm/include/llvm/Transforms/Utils/LowerSwitch.h
index 652fc91c56f71..7566c340b9009 100644
--- a/llvm/include/llvm/Transforms/Utils/LowerSwitch.h
+++ b/llvm/include/llvm/Transforms/Utils/LowerSwitch.h
@@ -18,7 +18,7 @@
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
-struct LowerSwitchPass : public OptionalPassInfoMixin<LowerSwitchPass> {
+struct LowerSwitchPass : public RequiredPassInfoMixin<LowerSwitchPass> {
   LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 } // namespace llvm
diff --git a/llvm/include/llvm/Transforms/Utils/MetaRenamer.h b/llvm/include/llvm/Transforms/Utils/MetaRenamer.h
index ba189087ee3c4..2c61afa6a4b89 100644
--- a/llvm/include/llvm/Transforms/Utils/MetaRenamer.h
+++ b/llvm/include/llvm/Transforms/Utils/MetaRenamer.h
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 // This pass renames everything with metasyntatic names. The intent is to use
-// this pass after bugpoint reduction to conceal the nature of the original
+// this pass after llvm-reduce reduction to conceal the nature of the original
 // program.
 //
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 6b38991831c45..7e713da25e343 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -3458,12 +3458,19 @@ void AsmPrinter::emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
       OutStreamer->emitLabel(GetJTISymbol(JumpTableIndex, true));
 
     MCSymbol *JTISymbol = GetJTISymbol(JumpTableIndex);
+    if (JTInDiffSection && MAI.hasDotTypeDotSizeDirective())
+      OutStreamer->emitSymbolAttribute(JTISymbol, MCSA_ELF_TypeObject);
     OutStreamer->emitLabel(JTISymbol);
 
     // Defer MCAssembler based constant folding due to a performance issue. The
     // label differences will be evaluated at write time.
     for (const MachineBasicBlock *MBB : JTBBs)
       emitJumpTableEntry(MJTI, MBB, JumpTableIndex);
+
+    if (JTInDiffSection && MAI.hasDotTypeDotSizeDirective())
+      OutStreamer->emitELFSize(
+          JTISymbol, MCConstantExpr::create(
+                         JTBBs.size() * MJTI.getEntrySize(DL), OutContext));
   }
 
   if (EmitJumpTableSizesSection)
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index 0c840a7845865..73285711d0aa9 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -871,8 +871,7 @@ class DwarfDebug : public DebugHandlerBase {
     return HasAppleExtensionAttributes;
   }
 
-  /// Returns whether or not to change the current debug info for the
-  /// split dwarf proposal support.
+  /// Returns whether or not to change the current debug info for split DWARF.
   bool useSplitDwarf() const { return HasSplitDwarf; }
 
   /// Returns whether to generate a string offsets table with (possibly shared)
diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp
index 2870bf404644c..f4fe9116c3a15 100644
--- a/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -1462,10 +1462,10 @@ bool MachineBasicBlock::canSplitCriticalEdge(const MachineBasicBlock *Succ,
                          /*AllowModify*/ false))
     return false;
 
-  // Avoid bugpoint weirdness: A block may end with a conditional branch but
-  // jumps to the same MBB is either case. We have duplicate CFG edges in that
-  // case that we can't handle. Since this never happens in properly optimized
-  // code, just skip those edges.
+  // Handle weird inputs (e.g., generated by a test case reducer/fuzzer): A
+  // block may end with a conditional branch but jumps to the same MBB is either
+  // case. We have duplicate CFG edges in that case that we can't handle. Since
+  // this never happens in properly optimized code, just skip those edges.
   if (TBB && TBB == FBB) {
     LLVM_DEBUG(dbgs() << "Won't split critical edge after degenerate "
                       << printMBBReference(*this) << '\n');
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index f2a2ddbfc2e31..d2216e4072337 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -43,6 +43,7 @@
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/NVVMIntrinsicUtils.h"
 #include "llvm/IR/Value.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/Support/AMDGPUAddrSpace.h"
@@ -1192,6 +1193,42 @@ static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F,
   return Intrinsic::not_intrinsic;
 }
 
+// The legacy TMA reduction intrinsics encode the reduction operator in their
+// name, while the current ones take it as an immediate argument. Map the
+// operator part of a legacy name to the corresponding immediate value.
+static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
+  return StringSwitch<std::optional<unsigned>>(Name)
+      .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
+      .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
+      .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
+      .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
+      .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
+      .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
+      .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
+      .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
+      .Default(std::nullopt);
+}
+
+static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
+  if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
+    return Intrinsic::not_intrinsic;
+
+  auto [RedOpName, ShapeName] = Name.split('.');
+  if (!getNVPTXTMAReductionOp(RedOpName))
+    return Intrinsic::not_intrinsic;
+
+  return StringSwitch<Intrinsic::ID>(ShapeName)
+      .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
+      .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
+      .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
+      .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
+      .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
+      .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
+      .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
+      .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
+      .Default(Intrinsic::not_intrinsic);
+}
+
 static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
                                                               StringRef Name) {
   if (Name.consume_front("mapa.shared.cluster"))
@@ -1718,6 +1755,15 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
         return true;
       }
 
+      // Upgrade TMA reduction intrinsics
+      // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
+      // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
+      IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
+      if (IID != Intrinsic::not_intrinsic) {
+        NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
+        return true;
+      }
+
       // Upgrade TMA copy G2S Intrinsics
       IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name);
       if (IID != Intrinsic::not_intrinsic) {
@@ -5617,6 +5663,23 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
     CI->eraseFromParent();
     return;
   }
+  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
+  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
+  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
+  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
+  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
+  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
+  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
+  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
+    StringRef Name = F->getName();
+    Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
+    auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
+
+    SmallVector<Value *, 16> Args(CI->args());
+    Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
+    NewCall = Builder.CreateCall(NewFn, Args);
+    break;
+  }
   case Intrinsic::riscv_sha256sig0:
   case Intrinsic::riscv_sha256sig1:
   case Intrinsic::riscv_sha256sum0:
diff --git a/llvm/lib/IR/NVVMIntrinsicUtils.cpp b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
index d745a1cfb72cd..c9fab6e37c484 100644
--- a/llvm/lib/IR/NVVMIntrinsicUtils.cpp
+++ b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
@@ -16,6 +16,16 @@
 using namespace llvm;
 using namespace nvvm;
 
+void nvvm::printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal) {
+  const auto *CI = dyn_cast<ConstantInt>(ImmArgVal);
+  if (!CI || CI->getZExtValue() > static_cast<uint64_t>(TMAReductionOp::XOR))
+    llvm_unreachable(
+        "printTMAReductionOp called with invalid value for immediate argument");
+
+  OS << getTMATensorReductionOpName(
+      static_cast<TMAReductionOp>(CI->getZExtValue()));
+}
+
 void nvvm::printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal) {
   if (const auto *CI = dyn_cast<ConstantInt>(ImmArgVal)) {
     uint64_t Val = CI->getZExtValue();
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index 4e96b74bd9c15..72b0ca52ae50a 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -175,6 +175,7 @@ void ScalarEnumerationTraits<ELFYAML::ELF_NT>::enumeration(
   ECase(NT_OPENBSD_FPREGS);
   ECase(NT_OPENBSD_XFPREGS);
   ECase(NT_OPENBSD_WCOOKIE);
+  ECase(NT_OPENBSD_PACMASK);
   // AMD specific notes. (Code Object V2)
   ECase(NT_AMD_HSA_CODE_OBJECT_VERSION);
   ECase(NT_AMD_HSA_HSAIL);
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 17d096eba7e36..db4b92811f57e 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -243,7 +243,6 @@
 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
 #include "llvm/Transforms/IPO/Instrumentor.h"
 #include "llvm/Transforms/IPO/Internalize.h"
-#include "llvm/Transforms/IPO/LoopExtractor.h"
 #include "llvm/Transforms/IPO/LowerTypeTests.h"
 #include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
 #include "llvm/Transforms/IPO/MergeFunctions.h"
@@ -969,10 +968,6 @@ Expected<bool> parseDropUnnecessaryAssumesPassOptions(StringRef Params) {
                                             "DropUnnecessaryAssumes");
 }
 
-Expected<bool> parseLoopExtractorPassOptions(StringRef Params) {
-  return PassBuilder::parseSinglePassOption(Params, "single", "LoopExtractor");
-}
-
 Expected<bool> parseLowerMatrixIntrinsicsPassOptions(StringRef Params) {
   return PassBuilder::parseSinglePassOption(Params, "minimal",
                                             "LowerMatrixIntrinsics");
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 90593c1effa40..5e592945d0de7 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -246,14 +246,6 @@ MODULE_PASS_WITH_PARAMS(
 MODULE_PASS_WITH_PARAMS(
     "ipsccp", "IPSCCPPass", [](IPSCCPOptions Opts) { return IPSCCPPass(Opts); },
     parseIPSCCPOptions, "no-func-spec;func-spec")
-MODULE_PASS_WITH_PARAMS(
-    "loop-extract", "LoopExtractorPass",
-    [](bool Single) {
-      if (Single)
-        return LoopExtractorPass(1);
-      return LoopExtractorPass();
-    },
-    parseLoopExtractorPassOptions, "single")
 MODULE_PASS_WITH_PARAMS(
     "memprof-use", "MemProfUsePass",
     [](std::string Opts) { return MemProfUsePass(Opts); },
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.h b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.h
index 9a692f0979c11..57d4648b202a0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.h
@@ -14,7 +14,7 @@
 namespace llvm {
 
 class AMDGPULowerVGPREncodingPass
-    : public OptionalPassInfoMixin<AMDGPULowerVGPREncodingPass> {
+    : public RequiredPassInfoMixin<AMDGPULowerVGPREncodingPass> {
 public:
   PreservedAnalyses run(MachineFunction &MF,
                         MachineFunctionAnalysisManager &MFAM);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.h b/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.h
index b8c345cc3359b..02017c65359f5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.h
@@ -14,7 +14,7 @@
 namespace llvm {
 
 class AMDGPUPreloadKernArgPrologPass
-    : public OptionalPassInfoMixin<AMDGPUPreloadKernArgPrologPass> {
+    : public RequiredPassInfoMixin<AMDGPUPreloadKernArgPrologPass> {
 public:
   PreservedAnalyses run(MachineFunction &MF,
                         MachineFunctionAnalysisManager &AM);
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index 0b988a5257253..3e0ad792d6fab 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -196,12 +196,8 @@ getRelocPairForSize(unsigned Size) {
   }
 }
 
-// Check if an R_LARCH_ALIGN relocation is needed for an alignment directive.
-// If conditions are met, compute the padding size and create a fixup encoding
-// the padding size in the addend. If MaxBytesToEmit is smaller than the padding
-// size, the fixup encodes MaxBytesToEmit in the higher bits and references a
-// per-section marker symbol.
-bool LoongArchAsmBackend::relaxAlign(MCFragment &F, unsigned &Size) {
+// Check whether an alignment fragment needs linker relaxation.
+bool LoongArchAsmBackend::shouldRelaxAlign(const MCFragment &F) {
   // Alignments before the first linker-relaxable instruction have fixed sizes
   // and do not require relocations. Alignments after a linker-relaxable
   // instruction require a relocation, even if the STI specifies norelax.
@@ -213,17 +209,27 @@ bool LoongArchAsmBackend::relaxAlign(MCFragment &F, unsigned &Size) {
   if (F.getLayoutOrder() <= Sec->firstLinkerRelaxable())
     return false;
 
-  // Use default handling unless linker relaxation is enabled and the
-  // MaxBytesToEmit >= the nop size.
   const unsigned MinNopLen = 4;
-  unsigned MaxBytesToEmit = F.getAlignMaxBytesToEmit();
-  if (MaxBytesToEmit < MinNopLen)
+  if (F.getAlignMaxBytesToEmit() < MinNopLen)
     return false;
-
-  Size = F.getAlignment().value() - MinNopLen;
   if (F.getAlignment() <= MinNopLen)
     return false;
 
+  return true;
+}
+
+// Check if an R_LARCH_ALIGN relocation is needed for an alignment directive.
+// If conditions are met, compute the padding size and create a fixup encoding
+// the padding size in the addend. If MaxBytesToEmit is smaller than the padding
+// size, the fixup encodes MaxBytesToEmit in the higher bits and references a
+// per-section marker symbol.
+bool LoongArchAsmBackend::relaxAlign(MCFragment &F, unsigned &Size) {
+  if (!shouldRelaxAlign(F))
+    return false;
+
+  Size = F.getAlignment().value() - 4;
+  unsigned MaxBytesToEmit = F.getAlignMaxBytesToEmit();
+
   MCContext &Ctx = getContext();
   const MCExpr *Expr = nullptr;
   if (MaxBytesToEmit >= Size) {
@@ -436,14 +442,11 @@ void LoongArchAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
           isPCRelFixupResolved(Target.getSubSym(), F))
         return Fallback();
 
-      if (&SecA == &SecB) {
-        // If the section is not linker-relaxable, or if the fixup is in a .dwo
-        // section (where relocations are forbidden), we must resolve the
-        // difference directly. The computed Value in evaluateFixup is correct
-        // based on the current layout.
-        if (!SecA.isLinkerRelaxable() || SecCur.getName().ends_with(".dwo"))
-          return;
-      }
+      // In SecA == SecB case. If the section is not linker-relaxable, the
+      // FixedValue has already been calculated out in evaluateFixup,
+      // return true and avoid record relocations.
+      if (&SecA == &SecB && !SecA.isLinkerRelaxable())
+        return;
     }
 
     switch (Fixup.getKind()) {
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
index 637a6b2478f14..297bac0841326 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
@@ -48,6 +48,9 @@ class LoongArchAsmBackend : public MCAsmBackend {
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
+  /// Check whether an alignment fragment needs linker relaxation.
+  static bool shouldRelaxAlign(const MCFragment &F);
+
   bool relaxAlign(MCFragment &F, unsigned &Size) override;
   bool relaxDwarfLineAddr(MCFragment &) const override;
   bool relaxDwarfCFA(MCFragment &) const override;
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.cpp
index be62601080b79..91433d8eaddc5 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.cpp
@@ -17,6 +17,7 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCELFObjectWriter.h"
+#include "llvm/MC/MCSection.h"
 
 using namespace llvm;
 
@@ -88,15 +89,17 @@ void LoongArchTargetELFStreamer::finish() {
   W.setELFHeaderEFlags(EFlags);
 }
 
-namespace {
-class LoongArchELFStreamer : public MCELFStreamer {
-public:
-  LoongArchELFStreamer(MCContext &C, std::unique_ptr<MCAsmBackend> MAB,
-                       std::unique_ptr<MCObjectWriter> MOW,
-                       std::unique_ptr<MCCodeEmitter> MCE)
-      : MCELFStreamer(C, std::move(MAB), std::move(MOW), std::move(MCE)) {}
-};
-} // end namespace
+void LoongArchELFStreamer::emitCodeAlignment(Align Alignment,
+                                             const MCSubtargetInfo &STI,
+                                             unsigned MaxBytesToEmit) {
+  // Save the Align fragment.
+  auto *AlignFrag = getCurrentFragment();
+  MCELFStreamer::emitCodeAlignment(Alignment, STI, MaxBytesToEmit);
+
+  // Pre-mark the Align fragment as linker-relaxable.
+  if (LoongArchAsmBackend::shouldRelaxAlign(*AlignFrag))
+    AlignFrag->setLinkerRelaxable();
+}
 
 namespace llvm {
 MCELFStreamer *createLoongArchELFStreamer(MCContext &C,
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.h
index 5c27f3629e81d..a9e5ec2bb527c 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.h
@@ -29,6 +29,17 @@ class LoongArchTargetELFStreamer : public LoongArchTargetStreamer {
   void finish() override;
 };
 
+class LoongArchELFStreamer : public MCELFStreamer {
+public:
+  LoongArchELFStreamer(MCContext &C, std::unique_ptr<MCAsmBackend> MAB,
+                       std::unique_ptr<MCObjectWriter> MOW,
+                       std::unique_ptr<MCCodeEmitter> MCE)
+      : MCELFStreamer(C, std::move(MAB), std::move(MOW), std::move(MCE)) {}
+
+  void emitCodeAlignment(Align Alignment, const MCSubtargetInfo &STI,
+                         unsigned MaxBytesToEmit) override;
+};
+
 MCELFStreamer *createLoongArchELFStreamer(MCContext &C,
                                           std::unique_ptr<MCAsmBackend> MAB,
                                           std::unique_ptr<MCObjectWriter> MOW,
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index 9db838523fdcd..e013fd3126aa5 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -468,36 +468,9 @@ void NVPTXInstPrinter::printTmaReductionMode(const MCInst *MI, int OpNum,
                                              const MCSubtargetInfo &,
                                              raw_ostream &O) {
   const MCOperand &MO = MI->getOperand(OpNum);
-  using RedTy = nvvm::TMAReductionOp;
-
-  switch (static_cast<RedTy>(MO.getImm())) {
-  case RedTy::ADD:
-    O << ".add";
-    return;
-  case RedTy::MIN:
-    O << ".min";
-    return;
-  case RedTy::MAX:
-    O << ".max";
-    return;
-  case RedTy::INC:
-    O << ".inc";
-    return;
-  case RedTy::DEC:
-    O << ".dec";
-    return;
-  case RedTy::AND:
-    O << ".and";
-    return;
-  case RedTy::OR:
-    O << ".or";
-    return;
-  case RedTy::XOR:
-    O << ".xor";
-    return;
-  }
-  llvm_unreachable(
-      "Invalid Reduction Op in printCpAsyncBulkTensorReductionMode");
+  O << '.'
+    << nvvm::getTMATensorReductionOpName(
+           static_cast<nvvm::TMAReductionOp>(MO.getImm()));
 }
 
 void NVPTXInstPrinter::printCTAGroup(const MCInst *MI, int OpNum,
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index af44ee053a734..0536284c75672 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -28,8 +28,6 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsNVPTX.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/NVVMIntrinsicUtils.h"
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -1914,82 +1912,6 @@ NVPTX::Scope NVPTXScopes::operator[](SyncScope::ID ID) const {
 
 bool NVPTXScopes::empty() const { return Scopes.size() == 0; }
 
-#define CP_ASYNC_BULK_TENSOR_OPCODE(dir, dim, mode, is_s32, suffix)            \
-  (is_s32                                                                      \
-       ? NVPTX::CP_ASYNC_BULK_TENSOR_##dir##_##dim##_SHARED32_##mode##suffix   \
-       : NVPTX::CP_ASYNC_BULK_TENSOR_##dir##_##dim##_##mode##suffix)
-
-#define GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED(dim, mode, is_ch, is_s32)      \
-  (is_ch ? (CP_ASYNC_BULK_TENSOR_OPCODE(RED, dim, mode, is_s32, _CH))          \
-         : (CP_ASYNC_BULK_TENSOR_OPCODE(RED, dim, mode, is_s32, )))
-
-static unsigned GetCpAsyncBulkTensorS2GReductionOpcode(size_t Dim,
-                                                       bool IsShared32,
-                                                       bool IsCacheHint,
-                                                       bool IsIm2Col) {
-  if (IsIm2Col) {
-    switch (Dim) {
-    case 3:
-      return GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED(3D, IM2COL, IsCacheHint,
-                                                     IsShared32);
-    case 4:
-      return GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED(4D, IM2COL, IsCacheHint,
-                                                     IsShared32);
-    case 5:
-      return GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED(5D, IM2COL, IsCacheHint,
-                                                     IsShared32);
-    default:
-      llvm_unreachable("Invalid Dimension in im2col mode for "
-                       "GetCpAsyncBulkTensorS2GReductionOpcode.");
-    }
-  } else {
-    switch (Dim) {
-    case 1:
-      return GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED(1D, TILE, IsCacheHint,
-                                                     IsShared32);
-    case 2:
-      return GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED(2D, TILE, IsCacheHint,
-                                                     IsShared32);
-    case 3:
-      return GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED(3D, TILE, IsCacheHint,
-                                                     IsShared32);
-    case 4:
-      return GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED(4D, TILE, IsCacheHint,
-                                                     IsShared32);
-    case 5:
-      return GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED(5D, TILE, IsCacheHint,
-                                                     IsShared32);
-    default:
-      llvm_unreachable("Invalid Dimension in tile mode for "
-                       "GetCpAsyncBulkTensorS2GReductionOpcode.");
-    }
-  }
-}
-
-void NVPTXDAGToDAGISel::SelectCpAsyncBulkTensorReduceCommon(SDNode *N,
-                                                            unsigned RedOp,
-                                                            bool IsIm2Col) {
-  // We have {Chain, Intrinsic-ID} followed by the actual intrisic args:
-  // src, dst, dims{d0...dN}, cache_hint, cache_hint_flag
-  // NumOperands = {Chain, IID} + {Actual intrinsic args}
-  //             = {2}          + {4 + dims}
-  size_t NumOps = N->getNumOperands();
-  size_t NumDims = NumOps - 6;
-  bool IsCacheHint = N->getConstantOperandVal(NumOps - 1) == 1;
-  size_t NumArgs = NumDims + (IsCacheHint ? 3 : 2); // src, dst, cache_hint
-
-  SDLoc DL(N);
-  SmallVector<SDValue, 12> Ops(N->ops().slice(2, NumArgs));
-  Ops.push_back(getI32Imm(RedOp, DL)); // Reduction Op
-  Ops.push_back(N->getOperand(0));     // Chain operand
-
-  bool IsShared32 =
-      CurDAG->getDataLayout().getPointerSizeInBits(ADDRESS_SPACE_SHARED) == 32;
-  unsigned Opcode = GetCpAsyncBulkTensorS2GReductionOpcode(
-      NumDims, IsShared32, IsCacheHint, IsIm2Col);
-  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops));
-}
-
 #define TCGEN05_ST_OPCODE(SHAPE, NUM)                                          \
   (enableUnpack ? NVPTX::TCGEN05_ST_##SHAPE##_##NUM##_UNPACK                   \
                 : NVPTX::TCGEN05_ST_##SHAPE##_##NUM)
@@ -2105,116 +2027,9 @@ void NVPTXDAGToDAGISel::SelectTcgen05St(SDNode *N, bool hasOffset) {
 
 bool NVPTXDAGToDAGISel::tryIntrinsicVoid(SDNode *N) {
   unsigned IID = N->getConstantOperandVal(1);
-  using TMARedTy = llvm::nvvm::TMAReductionOp;
-  auto CastTy = [](TMARedTy Op) { return static_cast<unsigned>(Op); };
   switch (IID) {
   default:
     return false;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_tile_1d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_tile_2d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_tile_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_tile_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_tile_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::ADD));
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_im2col_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_im2col_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_im2col_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::ADD),
-                                        /*IsIm2Col=*/true);
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_tile_1d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_tile_2d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_tile_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_tile_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_tile_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::MIN));
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_im2col_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_im2col_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_im2col_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::MIN),
-                                        /*IsIm2Col=*/true);
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_tile_1d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_tile_2d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_tile_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_tile_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_tile_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::MAX));
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_im2col_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_im2col_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_im2col_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::MAX),
-                                        /*IsIm2Col=*/true);
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_tile_1d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_tile_2d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_tile_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_tile_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_tile_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::INC));
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_im2col_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_im2col_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_im2col_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::INC),
-                                        /*IsIm2Col=*/true);
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_tile_1d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_tile_2d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_tile_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_tile_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_tile_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::DEC));
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_im2col_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_im2col_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_im2col_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::DEC),
-                                        /*IsIm2Col=*/true);
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_tile_1d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_tile_2d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_tile_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_tile_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_tile_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::AND));
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_im2col_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_im2col_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_im2col_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::AND),
-                                        /*IsIm2Col=*/true);
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_tile_1d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_tile_2d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_tile_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_tile_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_tile_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::OR));
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_im2col_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_im2col_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_im2col_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::OR),
-                                        /*IsIm2Col=*/true);
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_tile_1d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_tile_2d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_tile_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_tile_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_tile_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::XOR));
-    return true;
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_im2col_3d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_im2col_4d:
-  case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_im2col_5d:
-    SelectCpAsyncBulkTensorReduceCommon(N, CastTy(TMARedTy::XOR),
-                                        /*IsIm2Col=*/true);
-    return true;
-
   case Intrinsic::nvvm_tcgen05_st_16x64b_x1:
   case Intrinsic::nvvm_tcgen05_st_16x64b_x2:
   case Intrinsic::nvvm_tcgen05_st_16x64b_x4:
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 5230e01261fd6..ea47ddcc02ae7 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -856,38 +856,59 @@ def TMAReductionFlags : Operand<i32> {
   let PrintMethod = "printTmaReductionMode";
 }
 
+def tma_tensor_reduction_imm :
+    TImmLeaf<i32, [{ return Imm >= 0 && Imm < 8; }]>;
+
 // TMA Copy from Shared to Global memory with Reduction
-multiclass CP_ASYNC_BULK_TENSOR_REDUCE_INTR<int dim, bit shared32, string mode> {
+multiclass CP_ASYNC_BULK_TENSOR_REDUCE_INTR<int dim, string mode> {
   defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;
   defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;
   defvar asm_str = " [$tmap, {{" # dims_str # "}}], [$src]";
-  defvar rc = !if(shared32, B32, B64);
 
   // For im2col mode, the actual asm_str is "im2col_no_offs"
   defvar mode_asm_str = !if(!eq(mode, "im2col"),
                             "im2col_no_offs", mode);
-  defvar prefix = "cp.reduce.async.bulk.tensor" # "." # dim # "d" # ".global.shared::cta";
+  defvar prefix = "cp.reduce.async.bulk.tensor"
+                  # "." # dim # "d"
+                  # ".global.shared::cta";
   defvar suffix = "." # mode_asm_str # ".bulk_group";
 
+  defvar intr = !cast<Intrinsic>(
+    "int_nvvm_cp_async_bulk_tensor_reduce_" # mode # "_" # dim # "d"
+  );
+
+  defvar intr_dag = !con(
+    (intr addr:$src, B64:$tmap),
+    !setdagop(dims_dag, intr),
+    (intr (i64 srcvalue), tma_tensor_reduction_imm:$red_op, 0)
+  );
+
+  defvar intr_dag_with_ch = !con(
+    (intr addr:$src, B64:$tmap),
+    !setdagop(dims_dag, intr),
+    (intr B64:$ch, tma_tensor_reduction_imm:$red_op, -1)
+  );
+
   def "" : NVPTXInst<(outs),
-            !con((ins rc:$src, B64:$tmap), dims_dag, (ins TMAReductionFlags:$red_op)),
-            !strconcat(prefix, "${red_op}", suffix, asm_str, ";")>,
+            !con((ins ADDR:$src, B64:$tmap), dims_dag,
+                 (ins TMAReductionFlags:$red_op)),
+            !strconcat(prefix, "${red_op}", suffix, asm_str, ";"),
+            [intr_dag]>,
             Requires<[hasPTX<80>, hasSM<90>]>;
   def _CH : NVPTXInst<(outs),
-                  !con((ins rc:$src, B64:$tmap), dims_dag, (ins B64:$ch, TMAReductionFlags:$red_op)),
-                  !strconcat(prefix, "${red_op}", suffix, ".L2::cache_hint", asm_str, ", $ch;")>,
-                  Requires<[hasPTX<80>, hasSM<90>]>;
-}
-
-foreach dim = [1, 2, 3, 4, 5] in {
-  foreach shared32 = [true, false] in {
-    foreach mode = !if(!ge(dim, 3), ["tile", "im2col"], ["tile"]) in {
-      defvar suffix = dim # "D"
-                      # !if(shared32, "_SHARED32", "")
-                      # "_" # !toupper(mode);
-      defm CP_ASYNC_BULK_TENSOR_RED_ # suffix :
-        CP_ASYNC_BULK_TENSOR_REDUCE_INTR<dim, shared32, mode>;
-    }
+              !con((ins ADDR:$src, B64:$tmap), dims_dag,
+                   (ins B64:$ch, TMAReductionFlags:$red_op)),
+              !strconcat(prefix, "${red_op}", suffix,
+                         ".L2::cache_hint", asm_str, ", $ch;"),
+              [intr_dag_with_ch]>,
+              Requires<[hasPTX<80>, hasSM<90>]>;
+}
+
+foreach dim = 1...5 in {
+  foreach mode = !if(!ge(dim, 3), ["tile", "im2col"], ["tile"]) in {
+    defvar suffix = dim # "D_" # !toupper(mode);
+    defm CP_ASYNC_BULK_TENSOR_RED_ # suffix :
+      CP_ASYNC_BULK_TENSOR_REDUCE_INTR<dim, mode>;
   }
 }
 
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index 23c610f1c15e6..ca0e140264829 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -31,7 +31,6 @@ add_llvm_component_library(LLVMipo
   InstrumentorConfigFile.cpp
   InstrumentorStubPrinter.cpp
   Internalize.cpp
-  LoopExtractor.cpp
   LowerTypeTests.cpp
   MemProfContextDisambiguation.cpp
   MergeFunctions.cpp
diff --git a/llvm/lib/Transforms/IPO/IPO.cpp b/llvm/lib/Transforms/IPO/IPO.cpp
index 61a6462c53eae..e289e31c43b36 100644
--- a/llvm/lib/Transforms/IPO/IPO.cpp
+++ b/llvm/lib/Transforms/IPO/IPO.cpp
@@ -22,6 +22,4 @@ void llvm::initializeIPO(PassRegistry &Registry) {
   initializeDAEPass(Registry);
   initializeExpandVariadicsPass(Registry);
   initializeGlobalDCELegacyPassPass(Registry);
-  initializeLoopExtractorLegacyPassPass(Registry);
-  initializeSingleLoopExtractorPass(Registry);
 }
diff --git a/llvm/lib/Transforms/IPO/LoopExtractor.cpp b/llvm/lib/Transforms/IPO/LoopExtractor.cpp
deleted file mode 100644
index 8182ef6449d02..0000000000000
--- a/llvm/lib/Transforms/IPO/LoopExtractor.cpp
+++ /dev/null
@@ -1,289 +0,0 @@
-//===- LoopExtractor.cpp - Extract each loop into a new function ----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// A pass wrapper around the ExtractLoop() scalar transformation to extract each
-// top-level loop into its own new function. If the loop is the ONLY loop in a
-// given function, it is not touched. This is a pass most useful for debugging
-// via bugpoint.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Transforms/IPO/LoopExtractor.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/AssumptionCache.h"
-#include "llvm/Analysis/LoopInfo.h"
-#include "llvm/IR/Dominators.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/PassManager.h"
-#include "llvm/InitializePasses.h"
-#include "llvm/Pass.h"
-#include "llvm/Transforms/IPO.h"
-#include "llvm/Transforms/Utils.h"
-#include "llvm/Transforms/Utils/CodeExtractor.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "loop-extract"
-
-STATISTIC(NumExtracted, "Number of loops extracted");
-
-namespace {
-struct LoopExtractorLegacyPass : public ModulePass {
-  static char ID; // Pass identification, replacement for typeid
-
-  unsigned NumLoops;
-
-  explicit LoopExtractorLegacyPass(unsigned NumLoops = ~0)
-      : ModulePass(ID), NumLoops(NumLoops) {}
-
-  bool runOnModule(Module &M) override;
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequiredID(BreakCriticalEdgesID);
-    AU.addRequired<DominatorTreeWrapperPass>();
-    AU.addRequired<LoopInfoWrapperPass>();
-    AU.addPreserved<LoopInfoWrapperPass>();
-    AU.addRequiredID(LoopSimplifyID);
-    AU.addUsedIfAvailable<AssumptionCacheTracker>();
-  }
-};
-
-struct LoopExtractor {
-  explicit LoopExtractor(
-      unsigned NumLoops,
-      function_ref<DominatorTree &(Function &)> LookupDomTree,
-      function_ref<LoopInfo &(Function &)> LookupLoopInfo,
-      function_ref<AssumptionCache *(Function &)> LookupAssumptionCache)
-      : NumLoops(NumLoops), LookupDomTree(LookupDomTree),
-        LookupLoopInfo(LookupLoopInfo),
-        LookupAssumptionCache(LookupAssumptionCache) {}
-  bool runOnModule(Module &M);
-
-private:
-  // The number of natural loops to extract from the program into functions.
-  unsigned NumLoops;
-
-  function_ref<DominatorTree &(Function &)> LookupDomTree;
-  function_ref<LoopInfo &(Function &)> LookupLoopInfo;
-  function_ref<AssumptionCache *(Function &)> LookupAssumptionCache;
-
-  bool runOnFunction(Function &F);
-
-  bool extractLoops(Loop::iterator From, Loop::iterator To, LoopInfo &LI,
-                    DominatorTree &DT);
-  bool extractLoop(Loop *L, LoopInfo &LI, DominatorTree &DT);
-};
-} // namespace
-
-char LoopExtractorLegacyPass::ID = 0;
-INITIALIZE_PASS_BEGIN(LoopExtractorLegacyPass, "loop-extract",
-                      "Extract loops into new functions", false, false)
-INITIALIZE_PASS_DEPENDENCY(BreakCriticalEdges)
-INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
-INITIALIZE_PASS_END(LoopExtractorLegacyPass, "loop-extract",
-                    "Extract loops into new functions", false, false)
-
-namespace {
-  /// SingleLoopExtractor - For bugpoint.
-struct SingleLoopExtractor : public LoopExtractorLegacyPass {
-  static char ID; // Pass identification, replacement for typeid
-  SingleLoopExtractor() : LoopExtractorLegacyPass(1) {}
-};
-} // End anonymous namespace
-
-char SingleLoopExtractor::ID = 0;
-INITIALIZE_PASS(SingleLoopExtractor, "loop-extract-single",
-                "Extract at most one loop into a new function", false, false)
-
-// createLoopExtractorPass - This pass extracts all natural loops from the
-// program into a function if it can.
-//
-Pass *llvm::createLoopExtractorPass() { return new LoopExtractorLegacyPass(); }
-
-bool LoopExtractorLegacyPass::runOnModule(Module &M) {
-  if (skipModule(M))
-    return false;
-
-  bool Changed = false;
-  auto LookupDomTree = [this](Function &F) -> DominatorTree & {
-    return this->getAnalysis<DominatorTreeWrapperPass>(F).getDomTree();
-  };
-  auto LookupLoopInfo = [this, &Changed](Function &F) -> LoopInfo & {
-    return this->getAnalysis<LoopInfoWrapperPass>(F, &Changed).getLoopInfo();
-  };
-  auto LookupACT = [this](Function &F) -> AssumptionCache * {
-    if (auto *ACT = this->getAnalysisIfAvailable<AssumptionCacheTracker>())
-      return ACT->lookupAssumptionCache(F);
-    return nullptr;
-  };
-  return LoopExtractor(NumLoops, LookupDomTree, LookupLoopInfo, LookupACT)
-             .runOnModule(M) ||
-         Changed;
-}
-
-bool LoopExtractor::runOnModule(Module &M) {
-  if (M.empty())
-    return false;
-
-  if (!NumLoops)
-    return false;
-
-  bool Changed = false;
-
-  // The end of the function list may change (new functions will be added at the
-  // end), so we run from the first to the current last.
-  auto I = M.begin(), E = --M.end();
-  while (true) {
-    Function &F = *I;
-
-    Changed |= runOnFunction(F);
-    if (!NumLoops)
-      break;
-
-    // If this is the last function.
-    if (I == E)
-      break;
-
-    ++I;
-  }
-  return Changed;
-}
-
-bool LoopExtractor::runOnFunction(Function &F) {
-  // Do not modify `optnone` functions.
-  if (F.hasOptNone())
-    return false;
-
-  if (F.empty())
-    return false;
-
-  bool Changed = false;
-  LoopInfo &LI = LookupLoopInfo(F);
-
-  // If there are no loops in the function.
-  if (LI.empty())
-    return Changed;
-
-  DominatorTree &DT = LookupDomTree(F);
-
-  // If there is more than one top-level loop in this function, extract all of
-  // the loops.
-  if (std::next(LI.begin()) != LI.end())
-    return Changed | extractLoops(LI.begin(), LI.end(), LI, DT);
-
-  // Otherwise there is exactly one top-level loop.
-  Loop *TLL = *LI.begin();
-
-  // If the loop is in LoopSimplify form, then extract it only if this function
-  // is more than a minimal wrapper around the loop.
-  if (TLL->isLoopSimplifyForm()) {
-    bool ShouldExtractLoop = false;
-
-    // Extract the loop if the entry block doesn't branch to the loop header.
-    auto *EntryTI = dyn_cast<UncondBrInst>(F.getEntryBlock().getTerminator());
-    if (EntryTI && EntryTI->getSuccessor() != TLL->getHeader()) {
-      ShouldExtractLoop = true;
-    } else {
-      // Check to see if any exits from the loop are more than just return
-      // blocks.
-      SmallVector<BasicBlock *, 8> ExitBlocks;
-      TLL->getExitBlocks(ExitBlocks);
-      for (auto *ExitBlock : ExitBlocks)
-        if (!isa<ReturnInst>(ExitBlock->getTerminator())) {
-          ShouldExtractLoop = true;
-          break;
-        }
-    }
-
-    if (ShouldExtractLoop)
-      return Changed | extractLoop(TLL, LI, DT);
-  }
-
-  // Okay, this function is a minimal container around the specified loop.
-  // If we extract the loop, we will continue to just keep extracting it
-  // infinitely... so don't extract it. However, if the loop contains any
-  // sub-loops, extract them.
-  return Changed | extractLoops(TLL->begin(), TLL->end(), LI, DT);
-}
-
-bool LoopExtractor::extractLoops(Loop::iterator From, Loop::iterator To,
-                                 LoopInfo &LI, DominatorTree &DT) {
-  bool Changed = false;
-  SmallVector<Loop *, 8> Loops;
-
-  // Save the list of loops, as it may change.
-  Loops.assign(From, To);
-  for (Loop *L : Loops) {
-    // If LoopSimplify form is not available, stay out of trouble.
-    if (!L->isLoopSimplifyForm())
-      continue;
-
-    Changed |= extractLoop(L, LI, DT);
-    if (!NumLoops)
-      break;
-  }
-  return Changed;
-}
-
-bool LoopExtractor::extractLoop(Loop *L, LoopInfo &LI, DominatorTree &DT) {
-  assert(NumLoops != 0);
-  Function &Func = *L->getHeader()->getParent();
-  AssumptionCache *AC = LookupAssumptionCache(Func);
-  CodeExtractorAnalysisCache CEAC(Func);
-  CodeExtractor Extractor(L->getBlocks(), &DT, false, nullptr, nullptr, AC);
-  if (Extractor.isEligible()) {
-    // Remove loop while blocks are still in the current function
-    LI.erase(L);
-    [[maybe_unused]] Function *ExtrF = Extractor.extractCodeRegion(CEAC);
-    assert(ExtrF && "CodeExtractor didn't extact eligible loop");
-    --NumLoops;
-    ++NumExtracted;
-    return true;
-  }
-  return false;
-}
-
-// createSingleLoopExtractorPass - This pass extracts one natural loop from the
-// program into a function if it can.  This is used by bugpoint.
-//
-Pass *llvm::createSingleLoopExtractorPass() {
-  return new SingleLoopExtractor();
-}
-
-PreservedAnalyses LoopExtractorPass::run(Module &M, ModuleAnalysisManager &AM) {
-  auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
-  auto LookupDomTree = [&FAM](Function &F) -> DominatorTree & {
-    return FAM.getResult<DominatorTreeAnalysis>(F);
-  };
-  auto LookupLoopInfo = [&FAM](Function &F) -> LoopInfo & {
-    return FAM.getResult<LoopAnalysis>(F);
-  };
-  auto LookupAssumptionCache = [&FAM](Function &F) -> AssumptionCache * {
-    return FAM.getCachedResult<AssumptionAnalysis>(F);
-  };
-  if (!LoopExtractor(NumLoops, LookupDomTree, LookupLoopInfo,
-                     LookupAssumptionCache)
-           .runOnModule(M))
-    return PreservedAnalyses::all();
-
-  PreservedAnalyses PA;
-  PA.preserve<LoopAnalysis>();
-  return PA;
-}
-
-void LoopExtractorPass::printPipeline(
-    raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
-  static_cast<PassInfoMixin<LoopExtractorPass> *>(this)->printPipeline(
-      OS, MapClassName2PassName);
-  OS << '<';
-  if (NumLoops == 1)
-    OS << "single";
-  OS << '>';
-}
diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp b/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp
index 993fc85438d89..95a9bcb839ee4 100644
--- a/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp
+++ b/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp
@@ -639,7 +639,8 @@ bool ObjCARCContract::run(Function &F, AAResults *A, DominatorTree *D) {
 
     // Function for replacing uses of Arg dominated by Inst.
     auto ReplaceArgUses = [Inst, this](Value *Arg) {
-      // If we're compiling bugpointed code, don't get in trouble.
+      // If we're compiling fuzzer generated/test reducer produced IR, don't get
+      // in trouble.
       if (!isa<Instruction>(Arg) && !isa<Argument>(Arg))
         return;
 
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index c79bf04df656a..b6e8b0efd7325 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -813,6 +813,64 @@ bool LoopInterchangeLegality::containsUnsafeInstructions(BasicBlock *BB,
   });
 }
 
+static FreezeInst *findFreezeInReNestedBlocks(Loop *OuterLoop,
+                                              Loop *InnerLoop) {
+  // adjustLoopLinks swaps the preheader bodies after changing their loop
+  // roles, so the original outer-preheader body remains outside the new outer
+  // loop and retains its execution count.
+  BasicBlock *Blocks[] = {
+      OuterLoop->getHeader(),
+      OuterLoop->getLoopLatch(),
+      InnerLoop->getLoopPreheader(),
+      InnerLoop->getExitBlock(),
+  };
+  for (BasicBlock *BB : Blocks)
+    if (BB)
+      for (Instruction &I : *BB)
+        if (auto *Freeze = dyn_cast<FreezeInst>(&I))
+          return Freeze;
+  return nullptr;
+}
+
+static FreezeInst *
+findFreezeInInnerLatchCloneSet(Loop *InnerLoop,
+                               ArrayRef<PHINode *> InnerLoopInductions) {
+  // Mirror the latch-condition and induction-update operand closure cloned by
+  // MoveInstructions in LoopInterchangeTransform::transform.
+  SmallSetVector<Instruction *, 8> Worklist;
+  auto IsDirectInnerLoopBlock = [InnerLoop](BasicBlock *BB) {
+    return InnerLoop->contains(BB) &&
+           none_of(InnerLoop->getSubLoops(),
+                   [BB](Loop *SubLoop) { return SubLoop->contains(BB); });
+  };
+  auto *LatchBranch =
+      dyn_cast<CondBrInst>(InnerLoop->getLoopLatch()->getTerminator());
+  if (LatchBranch)
+    if (auto *Condition = dyn_cast<Instruction>(LatchBranch->getCondition()))
+      Worklist.insert(Condition);
+
+  for (PHINode *Induction : InnerLoopInductions) {
+    auto *Incoming = dyn_cast<Instruction>(
+        Induction->getIncomingValueForBlock(InnerLoop->getLoopLatch()));
+    if (Incoming && !is_contained(InnerLoopInductions, Incoming))
+      Worklist.insert(Incoming);
+  }
+
+  for (unsigned I = 0; I < Worklist.size(); ++I) {
+    Instruction *Current = Worklist[I];
+    if (auto *Freeze = dyn_cast<FreezeInst>(Current))
+      return Freeze;
+    for (Value *Operand : Current->operands()) {
+      auto *OperandI = dyn_cast<Instruction>(Operand);
+      if (!OperandI || !IsDirectInnerLoopBlock(OperandI->getParent()) ||
+          is_contained(InnerLoopInductions, OperandI))
+        continue;
+      Worklist.insert(OperandI);
+    }
+  }
+  return nullptr;
+}
+
 bool LoopInterchangeLegality::tightlyNested(Loop *OuterLoop, Loop *InnerLoop) {
   BasicBlock *OuterLoopHeader = OuterLoop->getHeader();
   BasicBlock *InnerLoopPreHeader = InnerLoop->getLoopPreheader();
@@ -1583,6 +1641,21 @@ bool LoopInterchangeLegality::canInterchangeLoops(unsigned InnerLoopId,
     return false;
   }
 
+  FreezeInst *Freeze = findFreezeInReNestedBlocks(OuterLoop, InnerLoop);
+  if (!Freeze)
+    Freeze = findFreezeInInnerLatchCloneSet(InnerLoop, InnerLoopInductions);
+  if (Freeze) {
+    LLVM_DEBUG(dbgs() << "Interchange would re-nest or duplicate freeze\n");
+    ORE->emit([&]() {
+      return OptimizationRemarkMissed(DEBUG_TYPE, "UnsafeInst",
+                                      Freeze->getDebugLoc(),
+                                      Freeze->getParent())
+             << "Cannot interchange loops because re-nesting or duplicating "
+                "freeze may change its sampling behavior.";
+    });
+    return false;
+  }
+
   // TODO: The loops could not be interchanged due to current limitations in the
   // transform module.
   if (currentLimitations()) {
@@ -2136,6 +2209,8 @@ void LoopInterchangeTransform::transform(
       SplitBlock(InnerLoop->getLoopLatch(),
                  InnerLoop->getLoopLatch()->getTerminator(), DT, LI);
 
+  // Keep these seeds and the operand filter aligned with
+  // findFreezeInInnerLatchCloneSet.
   SmallSetVector<Instruction *, 4> WorkList;
   unsigned i = 0;
   auto MoveInstructions = [&i, &WorkList, this, &InductionPHIs, NewLatch]() {
diff --git a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
index 44600acf5b418..91222c9a9af1b 100644
--- a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
+++ b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
@@ -664,7 +664,7 @@ InsertSafepointPoll(BasicBlock::iterator InsertBefore,
   BasicBlock::iterator Start = IsBegin ? OrigBB->begin() : std::next(Before);
 
   // If your poll function includes an unreachable at the end, that's not
-  // valid.  Bugpoint likes to create this, so check for it.
+  // valid. Fuzzers/test case reducers can create this, so check for it.
   assert(isPotentiallyReachable(&*Start, &*After) &&
          "malformed poll function");
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 033b00cf03a8b..579ba09855c04 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1074,20 +1074,44 @@ InstructionCost VPlan::cost(ElementCount VF, VPCostContext &Ctx) {
   return Cost;
 }
 
-VPRegionBlock *VPlan::getVectorLoopRegion() {
-  // TODO: Cache if possible.
-  for (VPBlockBase *B : vp_depth_first_shallow(getEntry()))
+// Find the vector loop region by following the last successor of each block,
+// starting from the plan's entry. The vector code path is always the last
+// successor of the entry (and of the min-iters bypass block, if present), and
+// every block on the path to the region has a single predecessor. Stop at the
+// first block with multiple predecessors: in a plain CFG that is the loop
+// header (no region exists yet), and in a rolled CFG it is the middle block
+// following the region.
+static VPRegionBlock *findVectorLoopRegion(VPBlockBase *Entry) {
+  for (VPBlockBase *B = Entry; B && B->getNumPredecessors() <= 1;
+       B = B->hasSuccessors() ? B->getSuccessors().back() : nullptr)
     if (auto *R = dyn_cast<VPRegionBlock>(B))
       return R->isReplicator() ? nullptr : R;
   return nullptr;
 }
 
-const VPRegionBlock *VPlan::getVectorLoopRegion() const {
-  for (const VPBlockBase *B : vp_depth_first_shallow(getEntry()))
+#ifdef EXPENSIVE_CHECKS
+// Reference lookup that scans every top-level block. Used only to validate
+// findVectorLoopRegion() when the invariants of the last-successor walk change.
+static VPRegionBlock *findVectorLoopRegionByScan(VPBlockBase *Entry) {
+  for (VPBlockBase *B : vp_depth_first_shallow(Entry))
     if (auto *R = dyn_cast<VPRegionBlock>(B))
       return R->isReplicator() ? nullptr : R;
   return nullptr;
 }
+#endif
+
+VPRegionBlock *VPlan::getVectorLoopRegion() {
+  VPRegionBlock *LoopRegion = findVectorLoopRegion(getEntry());
+#ifdef EXPENSIVE_CHECKS
+  assert(LoopRegion == findVectorLoopRegionByScan(getEntry()) &&
+         "fast vector loop region lookup disagrees with full CFG scan");
+#endif
+  return LoopRegion;
+}
+
+const VPRegionBlock *VPlan::getVectorLoopRegion() const {
+  return const_cast<VPlan *>(this)->getVectorLoopRegion();
+}
 
 bool VPlan::isOuterLoop() const {
   const VPRegionBlock *LoopRegion = getVectorLoopRegion();
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index c59c9bda203b8..cc66175fbe8b2 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -125,6 +125,22 @@ declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.3d(ptr addrspace(3) %d,
 declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.4d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i16 %im2col0, i16 %im2col1, i16 %mc, i64 %ch, i1 %f1, i1 %f2);
 declare void @llvm.nvvm.cp.async.bulk.tensor.g2s.im2col.5d(ptr addrspace(3) %d, ptr addrspace(3) %bar, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i16 %im2col0, i16 %im2col1, i16 %im2col2, i16 %mc, i64 %ch, i1 %f1, i1 %f2);
 
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.1d(ptr addrspace(3), ptr, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.2d(ptr addrspace(3), ptr, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.3d(ptr addrspace(3), ptr, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.3d(ptr addrspace(3), ptr, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.3d(ptr addrspace(3), ptr, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.3d(ptr addrspace(3), ptr, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.3d(ptr addrspace(3), ptr, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.3d(ptr addrspace(3), ptr, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.3d(ptr addrspace(3), ptr, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.3d(ptr addrspace(3), ptr, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.4d(ptr addrspace(3), ptr, i32, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.5d(ptr addrspace(3), ptr, i32, i32, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.3d(ptr addrspace(3), ptr, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.4d(ptr addrspace(3), ptr, i32, i32, i32, i32, i64, i1)
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.5d(ptr addrspace(3), ptr, i32, i32, i32, i32, i32, i64, i1)
+
 declare void @llvm.nvvm.barrier0()
 declare void @llvm.nvvm.barrier.n(i32)
 declare void @llvm.nvvm.bar.sync(i32)
@@ -482,6 +498,46 @@ define void @nvvm_cp_async_bulk_tensor_g2s_tile(ptr addrspace(3) %d, ptr addrspa
   ret void
 }
 
+; CHECK-LABEL: @nvvm_cp_async_bulk_tensor_reduce_ops
+define void @nvvm_cp_async_bulk_tensor_reduce_ops(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch) {
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, i1 true)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 true)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=min */ i32 1, i1 false)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 false)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=max */ i32 2, i1 true)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 true)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=inc */ i32 3, i1 false)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 false)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=dec */ i32 4, i1 true)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 true)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=and */ i32 5, i1 false)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 false)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=or */ i32 6, i1 true)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 true)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=xor */ i32 7, i1 false)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: @nvvm_cp_async_bulk_tensor_reduce_shapes
+define void @nvvm_cp_async_bulk_tensor_reduce_shapes(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=add */ i32 0, i1 false)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 false)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, i1 true)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 true)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, i1 false)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 false)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, i1 true)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 true)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, i1 false)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 false)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, i1 true)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 true)
+; CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, i1 false)
+  call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 false)
+  ret void
+}
+
 define void @cta_barriers(i32 %x, i32 %y, i32 %z) {
 ; CHECK: call void @llvm.nvvm.barrier.cta.sync.aligned.all(i32 0)
 ; CHECK: call void @llvm.nvvm.barrier.cta.sync.aligned.all(i32 %x)
diff --git a/llvm/test/CodeGen/ARM/execute-only.ll b/llvm/test/CodeGen/ARM/execute-only.ll
index 9159579c1b0b5..ec71a0120cdb6 100644
--- a/llvm/test/CodeGen/ARM/execute-only.ll
+++ b/llvm/test/CodeGen/ARM/execute-only.ll
@@ -58,6 +58,7 @@ define i32 @jump_table(i32 %c, i32 %a, i32 %b) #0 {
 ; CHECK-T1-NEXT: mov  pc, [[REG_ENTRY]]
 ; CHECK-T1:      .section .rodata,"a",%progbits
 ; CHECK-T1-NEXT: .p2align 2, 0x0
+; CHECK-T1-NEXT: .type	.LJTI1_0,%object
 ; CHECK-T1-NEXT: .LJTI1_0:
 ; CHECK-T1-NEXT: .long
 ; CHECK-T1-NEXT: .long
diff --git a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce.ll b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce.ll
index 2dac6c48ca86f..de9f78415db5a 100644
--- a/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce.ll
+++ b/llvm/test/CodeGen/NVPTX/cp-async-bulk-tensor-reduce.ll
@@ -1,426 +1,571 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80| FileCheck --check-prefixes=CHECK-PTX %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80| FileCheck %s
+; RUN: llvm-as < %s | llvm-dis | FileCheck --check-prefixes=CHECK-FORMAT %s
 ; RUN: %if ptxas-sm_90 && ptxas-isa-8.0 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80| %ptxas-verify -arch=sm_90 %}
 
 target triple = "nvptx64-nvidia-cuda"
 
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i64 %ch, i1 %flag_ch);
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i64 %ch, i1 %flag_ch);
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 %flag_ch);
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 %flag_ch);
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 %flag_ch);
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i64 %ch, i32 %red_op, i1 %flag_ch);
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i64 %ch, i32 %red_op, i1 %flag_ch);
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 %red_op, i1 %flag_ch);
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 %red_op, i1 %flag_ch);
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 %red_op, i1 %flag_ch);
 
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 %flag_ch);
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 %flag_ch);
-declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 %flag_ch);
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 %red_op, i1 %flag_ch);
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 %red_op, i1 %flag_ch);
+declare void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %s, ptr %tm, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 %red_op, i1 %flag_ch);
 
 ; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_1d
 define void @cp_async_bulk_tensor_reduce_tile_1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch) {
-; CHECK-PTX-LABEL: cp_async_bulk_tensor_reduce_tile_1d(
-; CHECK-PTX:       {
-; CHECK-PTX-NEXT:    .reg .b32 %r<2>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<4>;
-; CHECK-PTX-EMPTY:
-; CHECK-PTX-NEXT:  // %bb.0:
-; CHECK-PTX-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_1d_param_0];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_1d_param_1];
-; CHECK-PTX-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_1d_param_2];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_1d_param_3];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.min.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.max.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.inc.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.dec.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.and.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.or.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.xor.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group [%rd2, {%r1}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.min.tile.bulk_group [%rd2, {%r1}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.max.tile.bulk_group [%rd2, {%r1}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.inc.tile.bulk_group [%rd2, {%r1}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.dec.tile.bulk_group [%rd2, {%r1}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.and.tile.bulk_group [%rd2, {%r1}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.or.tile.bulk_group [%rd2, {%r1}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.xor.tile.bulk_group [%rd2, {%r1}], [%rd1];
-; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 1)
+; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_1d(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_1d_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_1d_param_1];
+; CHECK-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_1d_param_2];
+; CHECK-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_1d_param_3];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.min.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.max.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.inc.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.dec.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.and.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.or.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.xor.tile.bulk_group.L2::cache_hint [%rd2, {%r1}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group [%rd2, {%r1}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.min.tile.bulk_group [%rd2, {%r1}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.max.tile.bulk_group [%rd2, {%r1}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.inc.tile.bulk_group [%rd2, {%r1}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.dec.tile.bulk_group [%rd2, {%r1}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.and.tile.bulk_group [%rd2, {%r1}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.or.tile.bulk_group [%rd2, {%r1}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.1d.global.shared::cta.xor.tile.bulk_group [%rd2, {%r1}], [%rd1];
+; CHECK-NEXT:    ret;
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i1 0)
+; CHECK-FORMAT-LABEL: define void @cp_async_bulk_tensor_reduce_tile_1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=add */ i32 0, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=min */ i32 1, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=max */ i32 2, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=inc */ i32 3, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=dec */ i32 4, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=and */ i32 5, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=or */ i32 6, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=xor */ i32 7, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=add */ i32 0, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=min */ i32 1, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=max */ i32 2, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=inc */ i32 3, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=dec */ i32 4, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=and */ i32 5, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=or */ i32 6, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, /* red_op=xor */ i32 7, i1 false)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 1, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 2, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 3, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 4, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 5, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 6, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 7, i1 1)
+
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 1, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 2, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 3, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 4, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 5, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 6, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i64 %ch, i32 7, i1 0)
   ret void
 }
 
 ; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_2d
 define void @cp_async_bulk_tensor_reduce_tile_2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch) {
-; CHECK-PTX-LABEL: cp_async_bulk_tensor_reduce_tile_2d(
-; CHECK-PTX:       {
-; CHECK-PTX-NEXT:    .reg .b32 %r<3>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<4>;
-; CHECK-PTX-EMPTY:
-; CHECK-PTX-NEXT:  // %bb.0:
-; CHECK-PTX-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_2d_param_0];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_2d_param_1];
-; CHECK-PTX-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_2d_param_2];
-; CHECK-PTX-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_tile_2d_param_3];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_2d_param_4];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.min.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.max.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.inc.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.dec.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.and.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.or.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.xor.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.min.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.max.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.inc.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.dec.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.and.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.or.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.xor.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
-; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 1)
+; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_2d(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_2d_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_2d_param_1];
+; CHECK-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_2d_param_2];
+; CHECK-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_tile_2d_param_3];
+; CHECK-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_2d_param_4];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.min.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.max.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.inc.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.dec.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.and.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.or.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.xor.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.min.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.max.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.inc.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.dec.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.and.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.or.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.2d.global.shared::cta.xor.tile.bulk_group [%rd2, {%r1, %r2}], [%rd1];
+; CHECK-NEXT:    ret;
+
+; CHECK-FORMAT-LABEL: define void @cp_async_bulk_tensor_reduce_tile_2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=min */ i32 1, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=max */ i32 2, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=inc */ i32 3, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=dec */ i32 4, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=and */ i32 5, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=or */ i32 6, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=xor */ i32 7, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=add */ i32 0, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=min */ i32 1, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=max */ i32 2, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=inc */ i32 3, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=dec */ i32 4, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=and */ i32 5, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=or */ i32 6, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, /* red_op=xor */ i32 7, i1 false)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 1, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 2, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 3, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 4, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 5, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 6, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 7, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 1, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 2, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 3, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 4, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 5, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 6, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i64 %ch, i32 7, i1 0)
   ret void
 }
 
 ; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_3d
 define void @cp_async_bulk_tensor_reduce_tile_3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch) {
-; CHECK-PTX-LABEL: cp_async_bulk_tensor_reduce_tile_3d(
-; CHECK-PTX:       {
-; CHECK-PTX-NEXT:    .reg .b32 %r<4>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<4>;
-; CHECK-PTX-EMPTY:
-; CHECK-PTX-NEXT:  // %bb.0:
-; CHECK-PTX-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_3d_param_0];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_3d_param_1];
-; CHECK-PTX-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_3d_param_2];
-; CHECK-PTX-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_tile_3d_param_3];
-; CHECK-PTX-NEXT:    ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_tile_3d_param_4];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_3d_param_5];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
+; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_3d(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_3d_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_3d_param_1];
+; CHECK-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_3d_param_2];
+; CHECK-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_tile_3d_param_3];
+; CHECK-NEXT:    ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_tile_3d_param_4];
+; CHECK-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_3d_param_5];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.tile.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    ret;
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+; CHECK-FORMAT-LABEL: define void @cp_async_bulk_tensor_reduce_tile_3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=min */ i32 1, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=max */ i32 2, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=inc */ i32 3, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=dec */ i32 4, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=and */ i32 5, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=or */ i32 6, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=xor */ i32 7, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=min */ i32 1, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=max */ i32 2, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=inc */ i32 3, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=dec */ i32 4, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=and */ i32 5, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=or */ i32 6, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=xor */ i32 7, i1 false)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 1, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 2, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 3, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 4, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 5, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 6, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 7, i1 1)
+
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 1, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 2, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 3, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 4, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 5, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 6, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 7, i1 0)
   ret void
 }
 
 ; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_4d
 define void @cp_async_bulk_tensor_reduce_tile_4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch) {
-; CHECK-PTX-LABEL: cp_async_bulk_tensor_reduce_tile_4d(
-; CHECK-PTX:       {
-; CHECK-PTX-NEXT:    .reg .b32 %r<5>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<4>;
-; CHECK-PTX-EMPTY:
-; CHECK-PTX-NEXT:  // %bb.0:
-; CHECK-PTX-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_4d_param_0];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_4d_param_1];
-; CHECK-PTX-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_4d_param_2];
-; CHECK-PTX-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_tile_4d_param_3];
-; CHECK-PTX-NEXT:    ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_tile_4d_param_4];
-; CHECK-PTX-NEXT:    ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_tile_4d_param_5];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_4d_param_6];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.min.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.max.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.inc.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.dec.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.and.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.or.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.xor.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.min.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.max.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.inc.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.dec.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.and.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.or.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.xor.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
+; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_4d(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_4d_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_4d_param_1];
+; CHECK-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_4d_param_2];
+; CHECK-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_tile_4d_param_3];
+; CHECK-NEXT:    ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_tile_4d_param_4];
+; CHECK-NEXT:    ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_tile_4d_param_5];
+; CHECK-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_4d_param_6];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.min.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.max.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.inc.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.dec.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.and.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.or.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.xor.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.min.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.max.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.inc.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.dec.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.and.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.or.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.xor.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    ret;
+
+; CHECK-FORMAT-LABEL: define void @cp_async_bulk_tensor_reduce_tile_4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=min */ i32 1, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=max */ i32 2, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=inc */ i32 3, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=dec */ i32 4, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=and */ i32 5, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=or */ i32 6, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=xor */ i32 7, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=min */ i32 1, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=max */ i32 2, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=inc */ i32 3, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=dec */ i32 4, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=and */ i32 5, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=or */ i32 6, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=xor */ i32 7, i1 false)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 1, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 2, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 3, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 4, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 5, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 6, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 7, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 1, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 2, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 3, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 4, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 5, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 6, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 7, i1 0)
   ret void
 }
 
 ; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_5d
 define void @cp_async_bulk_tensor_reduce_tile_5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
-; CHECK-PTX-LABEL: cp_async_bulk_tensor_reduce_tile_5d(
-; CHECK-PTX:       {
-; CHECK-PTX-NEXT:    .reg .b32 %r<6>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<4>;
-; CHECK-PTX-EMPTY:
-; CHECK-PTX-NEXT:  // %bb.0:
-; CHECK-PTX-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_5d_param_0];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_5d_param_1];
-; CHECK-PTX-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_5d_param_2];
-; CHECK-PTX-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_tile_5d_param_3];
-; CHECK-PTX-NEXT:    ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_tile_5d_param_4];
-; CHECK-PTX-NEXT:    ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_tile_5d_param_5];
-; CHECK-PTX-NEXT:    ld.param.b32 %r5, [cp_async_bulk_tensor_reduce_tile_5d_param_6];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_5d_param_7];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.min.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.max.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.inc.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.dec.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.and.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.or.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.xor.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.min.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.max.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.inc.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.dec.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.and.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.or.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.xor.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+; CHECK-LABEL: cp_async_bulk_tensor_reduce_tile_5d(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<6>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_tile_5d_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_tile_5d_param_1];
+; CHECK-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_tile_5d_param_2];
+; CHECK-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_tile_5d_param_3];
+; CHECK-NEXT:    ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_tile_5d_param_4];
+; CHECK-NEXT:    ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_tile_5d_param_5];
+; CHECK-NEXT:    ld.param.b32 %r5, [cp_async_bulk_tensor_reduce_tile_5d_param_6];
+; CHECK-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_tile_5d_param_7];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.min.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.max.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.inc.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.dec.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.and.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.or.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.xor.tile.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.min.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.max.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.inc.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.dec.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.and.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.or.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.xor.tile.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    ret;
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+; CHECK-FORMAT-LABEL: define void @cp_async_bulk_tensor_reduce_tile_5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=min */ i32 1, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=max */ i32 2, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=inc */ i32 3, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=dec */ i32 4, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=and */ i32 5, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=or */ i32 6, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=xor */ i32 7, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=min */ i32 1, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=max */ i32 2, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=inc */ i32 3, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=dec */ i32 4, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=and */ i32 5, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=or */ i32 6, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=xor */ i32 7, i1 false)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 1, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 2, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 3, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 4, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 5, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 6, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 7, i1 1)
+
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 1, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 2, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 3, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 4, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 5, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 6, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 7, i1 0)
   ret void
 }
 
 ; CHECK-LABEL: cp_async_bulk_tensor_reduce_im2col_3d
 define void @cp_async_bulk_tensor_reduce_im2col_3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch) {
-; CHECK-PTX-LABEL: cp_async_bulk_tensor_reduce_im2col_3d(
-; CHECK-PTX:       {
-; CHECK-PTX-NEXT:    .reg .b32 %r<4>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<4>;
-; CHECK-PTX-EMPTY:
-; CHECK-PTX-NEXT:  // %bb.0:
-; CHECK-PTX-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_im2col_3d_param_0];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_im2col_3d_param_1];
-; CHECK-PTX-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_im2col_3d_param_2];
-; CHECK-PTX-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_im2col_3d_param_3];
-; CHECK-PTX-NEXT:    ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_im2col_3d_param_4];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_im2col_3d_param_5];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
-; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 1)
+; CHECK-LABEL: cp_async_bulk_tensor_reduce_im2col_3d(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_im2col_3d_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_im2col_3d_param_1];
+; CHECK-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_im2col_3d_param_2];
+; CHECK-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_im2col_3d_param_3];
+; CHECK-NEXT:    ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_im2col_3d_param_4];
+; CHECK-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_im2col_3d_param_5];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3}], [%rd1];
+; CHECK-NEXT:    ret;
+
+; CHECK-FORMAT-LABEL: define void @cp_async_bulk_tensor_reduce_im2col_3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=min */ i32 1, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=max */ i32 2, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=inc */ i32 3, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=dec */ i32 4, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=and */ i32 5, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=or */ i32 6, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=xor */ i32 7, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=add */ i32 0, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=min */ i32 1, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=max */ i32 2, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=inc */ i32 3, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=dec */ i32 4, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=and */ i32 5, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=or */ i32 6, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, /* red_op=xor */ i32 7, i1 false)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 1, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 2, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 3, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 4, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 5, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 6, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 7, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 1, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 2, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 3, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 4, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 5, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 6, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i64 %ch, i32 7, i1 0)
   ret void
 }
 
 ; CHECK-LABEL: cp_async_bulk_tensor_reduce_im2col_4d
 define void @cp_async_bulk_tensor_reduce_im2col_4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch) {
-; CHECK-PTX-LABEL: cp_async_bulk_tensor_reduce_im2col_4d(
-; CHECK-PTX:       {
-; CHECK-PTX-NEXT:    .reg .b32 %r<5>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<4>;
-; CHECK-PTX-EMPTY:
-; CHECK-PTX-NEXT:  // %bb.0:
-; CHECK-PTX-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_im2col_4d_param_0];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_im2col_4d_param_1];
-; CHECK-PTX-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_im2col_4d_param_2];
-; CHECK-PTX-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_im2col_4d_param_3];
-; CHECK-PTX-NEXT:    ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_im2col_4d_param_4];
-; CHECK-PTX-NEXT:    ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_im2col_4d_param_5];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_im2col_4d_param_6];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.min.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.max.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.inc.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.dec.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.and.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.or.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.xor.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.min.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.max.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.inc.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.dec.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.and.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.or.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.xor.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
-; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 1)
+; CHECK-LABEL: cp_async_bulk_tensor_reduce_im2col_4d(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_im2col_4d_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_im2col_4d_param_1];
+; CHECK-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_im2col_4d_param_2];
+; CHECK-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_im2col_4d_param_3];
+; CHECK-NEXT:    ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_im2col_4d_param_4];
+; CHECK-NEXT:    ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_im2col_4d_param_5];
+; CHECK-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_im2col_4d_param_6];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.min.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.max.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.inc.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.dec.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.and.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.or.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.xor.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.min.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.max.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.inc.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.dec.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.and.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.or.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.4d.global.shared::cta.xor.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4}], [%rd1];
+; CHECK-NEXT:    ret;
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i1 0)
+; CHECK-FORMAT-LABEL: define void @cp_async_bulk_tensor_reduce_im2col_4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=min */ i32 1, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=max */ i32 2, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=inc */ i32 3, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=dec */ i32 4, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=and */ i32 5, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=or */ i32 6, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=xor */ i32 7, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=add */ i32 0, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=min */ i32 1, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=max */ i32 2, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=inc */ i32 3, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=dec */ i32 4, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=and */ i32 5, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=or */ i32 6, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, /* red_op=xor */ i32 7, i1 false)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 1, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 2, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 3, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 4, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 5, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 6, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 7, i1 1)
+
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 1, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 2, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 3, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 4, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 5, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 6, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i64 %ch, i32 7, i1 0)
   ret void
 }
 
 ; CHECK-LABEL: cp_async_bulk_tensor_reduce_im2col_5d
 define void @cp_async_bulk_tensor_reduce_im2col_5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
-; CHECK-PTX-LABEL: cp_async_bulk_tensor_reduce_im2col_5d(
-; CHECK-PTX:       {
-; CHECK-PTX-NEXT:    .reg .b32 %r<6>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<4>;
-; CHECK-PTX-EMPTY:
-; CHECK-PTX-NEXT:  // %bb.0:
-; CHECK-PTX-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_im2col_5d_param_0];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_im2col_5d_param_1];
-; CHECK-PTX-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_im2col_5d_param_2];
-; CHECK-PTX-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_im2col_5d_param_3];
-; CHECK-PTX-NEXT:    ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_im2col_5d_param_4];
-; CHECK-PTX-NEXT:    ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_im2col_5d_param_5];
-; CHECK-PTX-NEXT:    ld.param.b32 %r5, [cp_async_bulk_tensor_reduce_im2col_5d_param_6];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_im2col_5d_param_7];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.min.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.max.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.inc.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.dec.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.and.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.or.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.xor.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.min.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.max.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.inc.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.dec.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.and.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.or.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.xor.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
-; CHECK-PTX-NEXT:    ret;
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 1)
+; CHECK-LABEL: cp_async_bulk_tensor_reduce_im2col_5d(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<6>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [cp_async_bulk_tensor_reduce_im2col_5d_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [cp_async_bulk_tensor_reduce_im2col_5d_param_1];
+; CHECK-NEXT:    ld.param.b32 %r1, [cp_async_bulk_tensor_reduce_im2col_5d_param_2];
+; CHECK-NEXT:    ld.param.b32 %r2, [cp_async_bulk_tensor_reduce_im2col_5d_param_3];
+; CHECK-NEXT:    ld.param.b32 %r3, [cp_async_bulk_tensor_reduce_im2col_5d_param_4];
+; CHECK-NEXT:    ld.param.b32 %r4, [cp_async_bulk_tensor_reduce_im2col_5d_param_5];
+; CHECK-NEXT:    ld.param.b32 %r5, [cp_async_bulk_tensor_reduce_im2col_5d_param_6];
+; CHECK-NEXT:    ld.param.b64 %rd3, [cp_async_bulk_tensor_reduce_im2col_5d_param_7];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.min.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.max.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.inc.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.dec.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.and.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.or.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.xor.im2col_no_offs.bulk_group.L2::cache_hint [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1], %rd3;
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.min.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.max.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.inc.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.dec.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.and.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.or.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    cp.reduce.async.bulk.tensor.5d.global.shared::cta.xor.im2col_no_offs.bulk_group [%rd2, {%r1, %r2, %r3, %r4, %r5}], [%rd1];
+; CHECK-NEXT:    ret;
+
+; CHECK-FORMAT-LABEL: define void @cp_async_bulk_tensor_reduce_im2col_5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch) {
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=min */ i32 1, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=max */ i32 2, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=inc */ i32 3, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=dec */ i32 4, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=and */ i32 5, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=or */ i32 6, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=xor */ i32 7, i1 true)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=add */ i32 0, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=min */ i32 1, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=max */ i32 2, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=inc */ i32 3, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=dec */ i32 4, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=and */ i32 5, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=or */ i32 6, i1 false)
+; CHECK-FORMAT: tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, /* red_op=xor */ i32 7, i1 false)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 1, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 2, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 3, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 4, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 5, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 6, i1 1)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 7, i1 1)
 
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
-  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 0, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 1, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 2, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 3, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 4, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 5, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 6, i1 0)
+  tail call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %src, ptr %tmap, i32 %d0, i32 %d1, i32 %d2, i32 %d3, i32 %d4, i64 %ch, i32 7, i1 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/RISCV/jumptable-sizes.ll b/llvm/test/CodeGen/RISCV/jumptable-sizes.ll
new file mode 100644
index 0000000000000..12f47baa2810f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/jumptable-sizes.ll
@@ -0,0 +1,58 @@
+; RUN: llc -mtriple=riscv32-elf -filetype=obj -code-model=small -verify-machineinstrs < %s \
+; RUN:   | llvm-readelf -s - \
+; RUN:   | FileCheck %s -check-prefixes=RV32I-SMALL
+; RUN: llc -mtriple=riscv32-elf -filetype=obj -code-model=medium -verify-machineinstrs < %s \
+; RUN:   | llvm-readelf -s - \
+; RUN:   | FileCheck %s -check-prefixes=RV32I-MEDIUM
+; RUN: llc -mtriple=riscv32-elf -filetype=obj -relocation-model=pic -verify-machineinstrs < %s \
+; RUN:   | llvm-readelf -s - \
+; RUN:   | FileCheck %s -check-prefixes=RV32I-PIC
+; RUN: llc -mtriple=riscv64-elf -filetype=obj -code-model=small -verify-machineinstrs < %s \
+; RUN:   | llvm-readelf -s - \
+; RUN:   | FileCheck %s -check-prefixes=RV64I-SMALL
+; RUN: llc -mtriple=riscv64-elf -filetype=obj -code-model=medium -verify-machineinstrs < %s \
+; RUN:   | llvm-readelf -s - \
+; RUN:   | FileCheck %s -check-prefixes=RV64I-MEDIUM
+; RUN: llc -mtriple=riscv64-elf -filetype=obj -relocation-model=pic -verify-machineinstrs < %s \
+; RUN:   | llvm-readelf -s - \
+; RUN:   | FileCheck %s -check-prefixes=RV64I-PIC
+
+
+define void @above_threshold(i32 signext %in, ptr %out) nounwind {
+; RV32I-SMALL:  24 OBJECT LOCAL DEFAULT [[#]] .LJTI0_0
+; RV32I-MEDIUM: 24 OBJECT LOCAL DEFAULT [[#]] .LJTI0_0
+; RV32I-PIC:    24 OBJECT LOCAL DEFAULT [[#]] .LJTI0_0
+; RV64I-SMALL:  24 OBJECT LOCAL DEFAULT [[#]] .LJTI0_0
+; RV64I-MEDIUM: 48 OBJECT LOCAL DEFAULT [[#]] .LJTI0_0
+; RV64I-PIC:    24 OBJECT LOCAL DEFAULT [[#]] .LJTI0_0
+
+entry:
+  switch i32 %in, label %exit [
+    i32 1, label %bb1
+    i32 2, label %bb2
+    i32 3, label %bb3
+    i32 4, label %bb4
+    i32 5, label %bb5
+    i32 6, label %bb6
+  ]
+bb1:
+  store i32 4, ptr %out
+  br label %exit
+bb2:
+  store i32 3, ptr %out
+  br label %exit
+bb3:
+  store i32 2, ptr %out
+  br label %exit
+bb4:
+  store i32 1, ptr %out
+  br label %exit
+bb5:
+  store i32 100, ptr %out
+  br label %exit
+bb6:
+  store i32 200, ptr %out
+  br label %exit
+exit:
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/pic.ll b/llvm/test/CodeGen/X86/pic.ll
index ef2849ca0cde6..d5b53c37e7e29 100644
--- a/llvm/test/CodeGen/X86/pic.ll
+++ b/llvm/test/CodeGen/X86/pic.ll
@@ -223,6 +223,7 @@ bb12:
 ; CHECK-X32:	jmpq	*%rax
 
 ; CHECK:	.p2align 2
+; CHECK-NEXT:	.type	.LJTI7_0,@object
 ; CHECK-NEXT:	.LJTI7_0:
 ; CHECK-I686:	.long	 .LBB7_2@GOTOFF
 ; CHECK-I686:	.long	 .LBB7_8@GOTOFF
diff --git a/llvm/test/Other/new-pm-print-pipeline.ll b/llvm/test/Other/new-pm-print-pipeline.ll
index 110ff23131667..2a1192b6b66bd 100644
--- a/llvm/test/Other/new-pm-print-pipeline.ll
+++ b/llvm/test/Other/new-pm-print-pipeline.ll
@@ -43,9 +43,6 @@
 ; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='module(hwasan<>,hwasan<kernel;recover>)' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-14
 ; CHECK-14: hwasan<>,hwasan<kernel;recover>
 
-; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='module(loop-extract<>,loop-extract<single>)' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-16
-; CHECK-16: loop-extract<>,loop-extract<single>
-
 ; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='function(print<stack-lifetime><may>,print<stack-lifetime><must>)' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-17
 ; CHECK-17: function(print<stack-lifetime><may>,print<stack-lifetime><must>)
 
diff --git a/llvm/test/Transforms/CodeExtractor/2004-03-13-LoopExtractorCrash.ll b/llvm/test/Transforms/CodeExtractor/2004-03-13-LoopExtractorCrash.ll
deleted file mode 100644
index bcf418e17aae4..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/2004-03-13-LoopExtractorCrash.ll
+++ /dev/null
@@ -1,75 +0,0 @@
-; RUN: opt < %s -passes='function(loop-simplify),loop-extract'  -disable-output
-
-define void @solve() {
-entry:
-	br label %loopentry.0
-
-loopentry.0:		; preds = %endif.0, %entry
-	br i1 false, label %no_exit.0, label %loopexit.0
-
-no_exit.0:		; preds = %loopentry.0
-	br i1 false, label %then.0, label %endif.0
-
-then.0:		; preds = %no_exit.0
-	br i1 false, label %shortcirc_done, label %shortcirc_next
-
-shortcirc_next:		; preds = %then.0
-	br label %shortcirc_done
-
-shortcirc_done:		; preds = %shortcirc_next, %then.0
-	br i1 false, label %then.1, label %endif.1
-
-then.1:		; preds = %shortcirc_done
-	br i1 false, label %cond_true, label %cond_false
-
-cond_true:		; preds = %then.1
-	br label %cond_continue
-
-cond_false:		; preds = %then.1
-	br label %cond_continue
-
-cond_continue:		; preds = %cond_false, %cond_true
-	br label %return
-
-after_ret.0:		; No predecessors!
-	br label %endif.1
-
-endif.1:		; preds = %after_ret.0, %shortcirc_done
-	br label %endif.0
-
-endif.0:		; preds = %endif.1, %no_exit.0
-	br label %loopentry.0
-
-loopexit.0:		; preds = %loopentry.0
-	br i1 false, label %then.2, label %endif.2
-
-then.2:		; preds = %loopexit.0
-	br i1 false, label %then.3, label %endif.3
-
-then.3:		; preds = %then.2
-	br label %return
-
-after_ret.1:		; No predecessors!
-	br label %endif.3
-
-endif.3:		; preds = %after_ret.1, %then.2
-	br label %endif.2
-
-endif.2:		; preds = %endif.3, %loopexit.0
-	br label %loopentry.1
-
-loopentry.1:		; preds = %no_exit.1, %endif.2
-	br i1 false, label %no_exit.1, label %loopexit.1
-
-no_exit.1:		; preds = %loopentry.1
-	br label %loopentry.1
-
-loopexit.1:		; preds = %loopentry.1
-	br label %return
-
-after_ret.2:		; No predecessors!
-	br label %return
-
-return:		; preds = %after_ret.2, %loopexit.1, %then.3, %cond_continue
-	ret void
-}
diff --git a/llvm/test/Transforms/CodeExtractor/2004-03-14-DominanceProblem.ll b/llvm/test/Transforms/CodeExtractor/2004-03-14-DominanceProblem.ll
deleted file mode 100644
index 480b3d7da1409..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/2004-03-14-DominanceProblem.ll
+++ /dev/null
@@ -1,33 +0,0 @@
-; RUN: opt < %s -passes='function(loop-simplify),loop-extract'  -disable-output
-; This testcase is failing the loop extractor because not all exit blocks
-; are dominated by all of the live-outs.
-
-define i32 @ab(i32 %alpha, i32 %beta) {
-entry:
-        br label %loopentry.1.preheader
-
-loopentry.1.preheader:          ; preds = %entry
-        br label %loopentry.1
-
-loopentry.1:            ; preds = %no_exit.1, %loopentry.1.preheader
-        br i1 false, label %no_exit.1, label %loopexit.0.loopexit1
-
-no_exit.1:              ; preds = %loopentry.1
-        %tmp.53 = load i32, ptr null                ; <i32> [#uses=1]
-        br i1 false, label %shortcirc_next.2, label %loopentry.1
-
-shortcirc_next.2:               ; preds = %no_exit.1
-        %tmp.563 = call i32 @wins( i32 0, i32 %tmp.53, i32 3 )          ; <i32> [#uses=0]
-        ret i32 0
-
-loopexit.0.loopexit1:           ; preds = %loopentry.1
-        br label %loopexit.0
-
-loopexit.0:             ; preds = %loopexit.0.loopexit1
-        ret i32 0
-}
-
-declare i32 @wins(i32, i32, i32)
-
-declare i16 @ab_code()
-
diff --git a/llvm/test/Transforms/CodeExtractor/2004-03-14-NoSwitchSupport.ll b/llvm/test/Transforms/CodeExtractor/2004-03-14-NoSwitchSupport.ll
deleted file mode 100644
index 67b929d77376e..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/2004-03-14-NoSwitchSupport.ll
+++ /dev/null
@@ -1,28 +0,0 @@
-; RUN: opt < %s -passes='function(loop-simplify),loop-extract<single>' -disable-output
-
-define void @ab() {
-entry:
-        br label %codeReplTail
-
-then.1:         ; preds = %codeReplTail
-        br label %loopentry.1
-
-loopentry.1:            ; preds = %no_exit.1, %then.1
-        br i1 false, label %no_exit.1, label %loopexit.0.loopexit1
-
-no_exit.1:              ; preds = %loopentry.1
-        br label %loopentry.1
-
-loopexit.0.loopexit:            ; preds = %codeReplTail
-        ret void
-
-loopexit.0.loopexit1:           ; preds = %loopentry.1
-        ret void
-
-codeReplTail:           ; preds = %codeReplTail, %entry
-        switch i16 0, label %codeReplTail [
-                 i16 0, label %loopexit.0.loopexit
-                 i16 1, label %then.1
-        ]
-}
-
diff --git a/llvm/test/Transforms/CodeExtractor/2004-03-17-MissedLiveIns.ll b/llvm/test/Transforms/CodeExtractor/2004-03-17-MissedLiveIns.ll
deleted file mode 100644
index 81c715a631fac..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/2004-03-17-MissedLiveIns.ll
+++ /dev/null
@@ -1,47 +0,0 @@
-; RUN: opt < %s -passes='function(loop-simplify),loop-extract'  -disable-output
-
-define void @sendMTFValues() {
-entry:
-	br i1 false, label %then.1, label %endif.1
-
-then.1:		; preds = %entry
-	br i1 false, label %loopentry.6.preheader, label %else.0
-
-endif.1:		; preds = %entry
-	ret void
-
-else.0:		; preds = %then.1
-	ret void
-
-loopentry.6.preheader:		; preds = %then.1
-	br i1 false, label %endif.7.preheader, label %loopexit.9
-
-endif.7.preheader:		; preds = %loopentry.6.preheader
-	%tmp.183 = add i32 0, -1		; <i32> [#uses=1]
-	br label %endif.7
-
-endif.7:		; preds = %loopexit.15, %endif.7.preheader
-	br i1 false, label %loopentry.10, label %loopentry.12
-
-loopentry.10:		; preds = %endif.7
-	br label %loopentry.12
-
-loopentry.12:		; preds = %loopentry.10, %endif.7
-	%ge.2.1 = phi i32 [ 0, %loopentry.10 ], [ %tmp.183, %endif.7 ]		; <i32> [#uses=0]
-	br i1 false, label %loopexit.14, label %no_exit.11
-
-no_exit.11:		; preds = %loopentry.12
-	ret void
-
-loopexit.14:		; preds = %loopentry.12
-	br i1 false, label %loopexit.15, label %no_exit.14
-
-no_exit.14:		; preds = %loopexit.14
-	ret void
-
-loopexit.15:		; preds = %loopexit.14
-	br i1 false, label %endif.7, label %loopexit.9
-
-loopexit.9:		; preds = %loopexit.15, %loopentry.6.preheader
-	ret void
-}
diff --git a/llvm/test/Transforms/CodeExtractor/2004-03-17-UpdatePHIsOutsideRegion.ll b/llvm/test/Transforms/CodeExtractor/2004-03-17-UpdatePHIsOutsideRegion.ll
deleted file mode 100644
index 5068cfe1a073b..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/2004-03-17-UpdatePHIsOutsideRegion.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; RUN: opt < %s -passes='function(loop-simplify),loop-extract'  -disable-output
-
-define void @maketree() {
-entry:
-        br i1 false, label %no_exit.1, label %loopexit.0
-
-no_exit.1:              ; preds = %endif, %expandbox.entry, %entry
-        br i1 false, label %endif, label %expandbox.entry
-
-expandbox.entry:                ; preds = %no_exit.1
-        br i1 false, label %loopexit.1, label %no_exit.1
-
-endif:          ; preds = %no_exit.1
-        br i1 false, label %loopexit.1, label %no_exit.1
-
-loopexit.1:             ; preds = %endif, %expandbox.entry
-        %ic.i.0.0.4 = phi i32 [ 0, %expandbox.entry ], [ 0, %endif ]            ; <i32> [#uses=0]
-        ret void
-
-loopexit.0:             ; preds = %entry
-        ret void
-}
-
diff --git a/llvm/test/Transforms/CodeExtractor/2004-03-18-InvokeHandling.ll b/llvm/test/Transforms/CodeExtractor/2004-03-18-InvokeHandling.ll
deleted file mode 100644
index c132bb058acb6..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/2004-03-18-InvokeHandling.ll
+++ /dev/null
@@ -1,198 +0,0 @@
-; RUN: opt < %s -passes='function(loop-simplify),loop-extract'  -disable-output
-
-declare i32 @_IO_getc()
-
-declare void @__errno_location()
-
-define void @yylex() personality ptr @__gcc_personality_v0 {
-entry:
-	switch i32 0, label %label.126 [
-		 i32 0, label %return
-		 i32 61, label %combine
-		 i32 33, label %combine
-		 i32 94, label %combine
-		 i32 37, label %combine
-		 i32 47, label %combine
-		 i32 42, label %combine
-		 i32 62, label %combine
-		 i32 60, label %combine
-		 i32 58, label %combine
-		 i32 124, label %combine
-		 i32 38, label %combine
-		 i32 45, label %combine
-		 i32 43, label %combine
-		 i32 34, label %string_constant
-		 i32 39, label %char_constant
-		 i32 46, label %loopexit.2
-		 i32 57, label %loopexit.2
-		 i32 56, label %loopexit.2
-		 i32 55, label %loopexit.2
-		 i32 54, label %loopexit.2
-		 i32 53, label %loopexit.2
-		 i32 52, label %loopexit.2
-		 i32 51, label %loopexit.2
-		 i32 50, label %loopexit.2
-		 i32 49, label %loopexit.2
-		 i32 48, label %loopexit.2
-		 i32 95, label %letter
-		 i32 122, label %letter
-		 i32 121, label %letter
-		 i32 120, label %letter
-		 i32 119, label %letter
-		 i32 118, label %letter
-		 i32 117, label %letter
-		 i32 116, label %letter
-		 i32 115, label %letter
-		 i32 114, label %letter
-		 i32 113, label %letter
-		 i32 112, label %letter
-		 i32 111, label %letter
-		 i32 110, label %letter
-		 i32 109, label %letter
-		 i32 108, label %letter
-		 i32 107, label %letter
-		 i32 106, label %letter
-		 i32 105, label %letter
-		 i32 104, label %letter
-		 i32 103, label %letter
-		 i32 102, label %letter
-		 i32 101, label %letter
-		 i32 100, label %letter
-		 i32 99, label %letter
-		 i32 98, label %letter
-		 i32 97, label %letter
-		 i32 90, label %letter
-		 i32 89, label %letter
-		 i32 88, label %letter
-		 i32 87, label %letter
-		 i32 86, label %letter
-		 i32 85, label %letter
-		 i32 84, label %letter
-		 i32 83, label %letter
-		 i32 82, label %letter
-		 i32 81, label %letter
-		 i32 80, label %letter
-		 i32 79, label %letter
-		 i32 78, label %letter
-		 i32 77, label %letter
-		 i32 75, label %letter
-		 i32 74, label %letter
-		 i32 73, label %letter
-		 i32 72, label %letter
-		 i32 71, label %letter
-		 i32 70, label %letter
-		 i32 69, label %letter
-		 i32 68, label %letter
-		 i32 67, label %letter
-		 i32 66, label %letter
-		 i32 65, label %letter
-		 i32 64, label %label.13
-		 i32 76, label %label.12
-		 i32 36, label %label.11
-		 i32 -1, label %label.10
-	]
-
-label.10:		; preds = %entry
-	ret void
-
-label.11:		; preds = %entry
-	ret void
-
-label.12:		; preds = %entry
-	ret void
-
-label.13:		; preds = %entry
-	ret void
-
-letter:		; preds = %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry
-	ret void
-
-loopexit.2:		; preds = %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry
-	switch i32 0, label %shortcirc_next.14 [
-		 i32 48, label %then.20
-		 i32 46, label %endif.38
-	]
-
-then.20:		; preds = %loopexit.2
-	switch i32 0, label %else.4 [
-		 i32 120, label %then.21
-		 i32 88, label %then.21
-	]
-
-then.21:		; preds = %then.20, %then.20
-	ret void
-
-else.4:		; preds = %then.20
-	ret void
-
-shortcirc_next.14:		; preds = %loopexit.2
-	ret void
-
-endif.38:		; preds = %loopexit.2
-	br i1 false, label %then.40, label %then.39
-
-then.39:		; preds = %endif.38
-	ret void
-
-then.40:		; preds = %endif.38
-	invoke void @__errno_location( )
-			to label %switchexit.2 unwind label %LongJmpBlkPre
-
-loopentry.6:		; preds = %endif.52
-	switch i32 0, label %switchexit.2 [
-		 i32 73, label %label.82
-		 i32 105, label %label.82
-		 i32 76, label %label.80
-		 i32 108, label %label.80
-		 i32 70, label %label.78
-		 i32 102, label %label.78
-	]
-
-label.78:		; preds = %loopentry.6, %loopentry.6
-	ret void
-
-label.80:		; preds = %loopentry.6, %loopentry.6
-	ret void
-
-label.82:		; preds = %loopentry.6, %loopentry.6
-	%c.0.15.5 = phi i32 [ %tmp.79417, %loopentry.6 ], [ %tmp.79417, %loopentry.6 ]		; <i32> [#uses=0]
-	ret void
-
-switchexit.2:		; preds = %loopentry.6, %then.40
-	br i1 false, label %endif.51, label %loopexit.6
-
-endif.51:		; preds = %switchexit.2
-	br i1 false, label %endif.52, label %then.52
-
-then.52:		; preds = %endif.51
-	ret void
-
-endif.52:		; preds = %endif.51
-	%tmp.79417 = invoke i32 @_IO_getc( )
-			to label %loopentry.6 unwind label %LongJmpBlkPre		; <i32> [#uses=2]
-
-loopexit.6:		; preds = %switchexit.2
-	ret void
-
-char_constant:		; preds = %entry
-	ret void
-
-string_constant:		; preds = %entry
-	ret void
-
-combine:		; preds = %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry
-	ret void
-
-label.126:		; preds = %entry
-	ret void
-
-return:		; preds = %entry
-	ret void
-
-LongJmpBlkPre:		; preds = %endif.52, %then.40
-        %exn = landingpad { ptr, i32 }
-                 catch ptr null
-	ret void
-}
-
-declare i32 @__gcc_personality_v0(...)
diff --git a/llvm/test/Transforms/CodeExtractor/BlockAddressReference.ll b/llvm/test/Transforms/CodeExtractor/BlockAddressReference.ll
deleted file mode 100644
index fc7875cda7898..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/BlockAddressReference.ll
+++ /dev/null
@@ -1,36 +0,0 @@
-; RUN: opt < %s -passes='function(loop-simplify),loop-extract'  -S | FileCheck %s
-
-@label = common local_unnamed_addr global ptr null
-
-; CHECK: define
-; no outlined function
-; CHECK-NOT: define
-define i32 @sterix(i32 %n) {
-entry:
-  %tobool = icmp ne i32 %n, 0
-  ; this blockaddress references a basic block that goes in the extracted loop
-  %cond = select i1 %tobool, ptr blockaddress(@sterix, %for.cond), ptr blockaddress(@sterix, %exit)
-  store ptr %cond, ptr @label
-  %cmp5 = icmp sgt i32 %n, 0
-  br i1 %cmp5, label %for.body, label %exit
-
-for.cond:
-  %mul = shl nsw i32 %s.06, 1
-  %exitcond = icmp eq i32 %inc, %n
-  br i1 %exitcond, label %exit.loopexit, label %for.body
-
-for.body:
-  %i.07 = phi i32 [ %inc, %for.cond ], [ 0, %entry ]
-  %s.06 = phi i32 [ %mul, %for.cond ], [ 1, %entry ]
-  %inc = add nuw nsw i32 %i.07, 1
-  br label %for.cond
-
-exit.loopexit:
-  %phitmp = icmp ne i32 %s.06, 2
-  %phitmp8 = zext i1 %phitmp to i32
-  br label %exit
-
-exit:
-  %s.1 = phi i32 [ 1, %entry ], [ %phitmp8, %exit.loopexit ]
-  ret i32 %s.1
-}
diff --git a/llvm/test/Transforms/CodeExtractor/BlockAddressSelfReference.ll b/llvm/test/Transforms/CodeExtractor/BlockAddressSelfReference.ll
deleted file mode 100644
index ce71ffa779cb4..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/BlockAddressSelfReference.ll
+++ /dev/null
@@ -1,50 +0,0 @@
-; RUN: opt < %s -passes='function(loop-simplify),loop-extract'  -S | FileCheck %s
-
-@choum.addr = internal unnamed_addr constant [3 x ptr] [ptr blockaddress(@choum, %bb10), ptr blockaddress(@choum, %bb14), ptr blockaddress(@choum, %bb18)]
-
-; CHECK: define
-; no outlined function
-; CHECK-NOT: define
-
-define void @choum(i32 %arg, ptr nocapture %arg1, i32 %arg2) {
-bb:
-  %tmp = icmp sgt i32 %arg, 0
-  br i1 %tmp, label %bb3, label %bb24
-
-bb3:                                              ; preds = %bb
-  %tmp4 = sext i32 %arg2 to i64
-  %tmp5 = getelementptr inbounds [3 x ptr], ptr @choum.addr, i64 0, i64 %tmp4
-  %tmp6 = load ptr, ptr %tmp5
-  %tmp7 = zext i32 %arg to i64
-  br label %bb8
-
-bb8:                                              ; preds = %bb18, %bb3
-  %tmp9 = phi i64 [ 0, %bb3 ], [ %tmp22, %bb18 ]
-  indirectbr ptr %tmp6, [label %bb10, label %bb14, label %bb18]
-
-bb10:                                             ; preds = %bb8
-  %tmp11 = getelementptr inbounds i32, ptr %arg1, i64 %tmp9
-  %tmp12 = load i32, ptr %tmp11
-  %tmp13 = add nsw i32 %tmp12, 1
-  store i32 %tmp13, ptr %tmp11
-  br label %bb14
-
-bb14:                                             ; preds = %bb10, %bb8
-  %tmp15 = getelementptr inbounds i32, ptr %arg1, i64 %tmp9
-  %tmp16 = load i32, ptr %tmp15
-  %tmp17 = shl nsw i32 %tmp16, 1
-  store i32 %tmp17, ptr %tmp15
-  br label %bb18
-
-bb18:                                             ; preds = %bb14, %bb8
-  %tmp19 = getelementptr inbounds i32, ptr %arg1, i64 %tmp9
-  %tmp20 = load i32, ptr %tmp19
-  %tmp21 = add nsw i32 %tmp20, -3
-  store i32 %tmp21, ptr %tmp19
-  %tmp22 = add nuw nsw i64 %tmp9, 1
-  %tmp23 = icmp eq i64 %tmp22, %tmp7
-  br i1 %tmp23, label %bb24, label %bb8
-
-bb24:                                             ; preds = %bb18, %bb
-  ret void
-}
diff --git a/llvm/test/Transforms/CodeExtractor/LoopExtractor.ll b/llvm/test/Transforms/CodeExtractor/LoopExtractor.ll
deleted file mode 100644
index f5a68fad49552..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/LoopExtractor.ll
+++ /dev/null
@@ -1,68 +0,0 @@
-; RUN: opt < %s -passes='function(break-crit-edges,loop-simplify),loop-extract' -S | FileCheck %s
-
-; This function has 2 simple loops and they should be extracted into 2 new functions.
-define void @test3() {
-; CHECK-LABEL: @test3(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label %codeRepl1
-; CHECK:       codeRepl1:
-; CHECK-NEXT:    call void @test3.loop.0()
-; CHECK-NEXT:    br label %loop.0.loop.1_crit_edge
-; CHECK:       loop.0.loop.1_crit_edge:
-; CHECK-NEXT:    br label %codeRepl
-; CHECK:       codeRepl:
-; CHECK-NEXT:    call void @test3.loop.1()
-; CHECK-NEXT:    br label %exit
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
-
-entry:
-  br label %loop.0
-
-loop.0:                                           ; preds = %loop.0, %entry
-  %index.0 = phi i32 [ 10, %entry ], [ %next.0, %loop.0 ]
-  tail call void @foo()
-  %next.0 = add nsw i32 %index.0, -1
-  %repeat.0 = icmp sgt i32 %index.0, 1
-  br i1 %repeat.0, label %loop.0, label %loop.1
-
-loop.1:                                           ; preds = %loop.0, %loop.1
-  %index.1 = phi i32 [ %next.1, %loop.1 ], [ 10, %loop.0 ]
-  tail call void @foo()
-  %next.1 = add nsw i32 %index.1, -1
-  %repeat.1 = icmp sgt i32 %index.1, 1
-  br i1 %repeat.1, label %loop.1, label %exit
-
-exit:                                             ; preds = %loop.1
-  ret void
-}
-
-declare void @foo()
-
-; CHECK-LABEL: define internal void @test3.loop.1()
-; CHECK-NEXT:  newFuncRoot:
-; CHECK-NEXT:    br label %loop.1
-; CHECK:       loop.1:
-; CHECK-NEXT:    %index.1 = phi i32 [ %next.1, %loop.1.loop.1_crit_edge ], [ 10, %newFuncRoot ]
-; CHECK-NEXT:    tail call void @foo()
-; CHECK-NEXT:    %next.1 = add nsw i32 %index.1, -1
-; CHECK-NEXT:    %repeat.1 = icmp sgt i32 %index.1, 1
-; CHECK-NEXT:    br i1 %repeat.1, label %loop.1.loop.1_crit_edge, label %exit.exitStub
-; CHECK:       loop.1.loop.1_crit_edge:
-; CHECK-NEXT:    br label %loop.1
-; CHECK:       exit.exitStub:
-; CHECK-NEXT:    ret void
-
-; CHECK-LABEL: define internal void @test3.loop.0()
-; CHECK-NEXT:  newFuncRoot:
-; CHECK-NEXT:    br label %loop.0
-; CHECK:       loop.0:
-; CHECK-NEXT:    %index.0 = phi i32 [ 10, %newFuncRoot ], [ %next.0, %loop.0.loop.0_crit_edge ]
-; CHECK-NEXT:    tail call void @foo()
-; CHECK-NEXT:    %next.0 = add nsw i32 %index.0, -1
-; CHECK-NEXT:    %repeat.0 = icmp sgt i32 %index.0, 1
-; CHECK-NEXT:    br i1 %repeat.0, label %loop.0.loop.0_crit_edge, label %loop.0.loop.1_crit_edge.exitStub
-; CHECK:       loop.0.loop.0_crit_edge:
-; CHECK-NEXT:    br label %loop.0
-; CHECK:       loop.0.loop.1_crit_edge.exitStub:
-; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/CodeExtractor/LoopExtractor_alloca.ll b/llvm/test/Transforms/CodeExtractor/LoopExtractor_alloca.ll
deleted file mode 100644
index 09abf1f3cd85b..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/LoopExtractor_alloca.ll
+++ /dev/null
@@ -1,54 +0,0 @@
-; RUN: opt -passes=debugify,loop-simplify,loop-extract -S < %s | FileCheck %s
-
-; This tests 2 cases:
-; 1. loop1 should be extracted into a function, without extracting %v1 alloca.
-; 2. loop2 should be extracted into a function, with the %v2 alloca.
-;
-; This used to produce an invalid IR, where `memcpy` will have a reference to
-; the, now, external value (local to the extracted loop function).
-
-; CHECK-LABEL: define void @test()
-; CHECK-NEXT: entry:
-; CHECK-NEXT:   %v1 = alloca i32
-; CHECK-NEXT:   #dbg_value(ptr %v1
-; CHECK-NEXT:   call void @llvm.memcpy.p0.p0.i64(ptr align 4 undef, ptr %v1, i64 4, i1 true)
-
-; CHECK-LABEL: define internal void @test.loop2()
-; CHECK-NEXT: newFuncRoot:
-; CHECK-NEXT:   %v2 = alloca i32
-
-; CHECK-LABEL: define internal void @test.loop1(ptr %v1)
-; CHECK-NEXT: newFuncRoot:
-; CHECK-NEXT: #dbg_value
-; CHECK-NEXT:   br
-
-define void @test() {
-entry:
-  %v1 = alloca i32, align 4
-  %v2 = alloca i32, align 4
-  call void @llvm.memcpy.p0.p0.i64(ptr align 4 undef, ptr %v1, i64 4, i1 true)
-  br label %loop1
-
-loop1:
-  call void @llvm.lifetime.start.p0(ptr %v1)
-  %r1 = call i32 @foo(ptr %v1)
-  call void @llvm.lifetime.end.p0(ptr %v1)
-  %cmp1 = icmp ne i32 %r1, 0
-  br i1 %cmp1, label %loop1, label %loop2
-
-loop2:
-  call void @llvm.lifetime.start.p0(ptr %v2)
-  %r2 = call i32 @foo(ptr %v2)
-  call void @llvm.lifetime.end.p0(ptr %v2)
-  %cmp2 = icmp ne i32 %r2, 0
-  br i1 %cmp2, label %loop2, label %exit
-
-exit:
-  ret void
-}
-
-declare i32 @foo(ptr)
-
-declare void @llvm.lifetime.start.p0(ptr nocapture)
-declare void @llvm.lifetime.end.p0(ptr nocapture)
-declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
diff --git a/llvm/test/Transforms/CodeExtractor/LoopExtractor_crash.ll b/llvm/test/Transforms/CodeExtractor/LoopExtractor_crash.ll
deleted file mode 100644
index 6bd2b9791fffc..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/LoopExtractor_crash.ll
+++ /dev/null
@@ -1,46 +0,0 @@
-; RUN: opt < %s -passes='cgscc(inline,loop-simplify),loop-extract' -S | FileCheck %s
-; RUN: opt < %s -passes='cgscc(argpromotion,loop-simplify),loop-extract' -S | FileCheck %s
-
-; This test used to trigger an assert (PR8929).
-
-define void @test() {
-; CHECK-LABEL: define void @test()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label %codeRepl
-; CHECK:       codeRepl:
-; CHECK-NEXT:    call void @test.loopentry()
-; CHECK-NEXT:    br label %loopexit
-; CHECK:       loopexit:
-; CHECK-NEXT:    br label %exit
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
-
-entry:
-  br label %loopentry
-
-loopentry:                                        ; preds = %loopbody, %entry
-  br i1 undef, label %loopbody, label %loopexit
-
-loopbody:                                         ; preds = %codeRepl1
-  call void @foo()
-  br label %loopentry
-
-loopexit:                                         ; preds = %codeRepl
-  br label %exit
-
-exit:                                             ; preds = %loopexit
-  ret void
-}
-
-declare void @foo()
-
-; CHECK-LABEL: define internal void @test.loopentry()
-; CHECK-NEXT:  newFuncRoot:
-; CHECK-NEXT:    br label %loopentry
-; CHECK:       loopentry:
-; CHECK-NEXT:    br i1 false, label %loopbody, label %loopexit.exitStub
-; CHECK:       loopbody:
-; CHECK-NEXT:    call void @foo()
-; CHECK-NEXT:    br label %loopentry
-; CHECK:       loopexit.exitStub:
-; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/CodeExtractor/LoopExtractor_infinite.ll b/llvm/test/Transforms/CodeExtractor/LoopExtractor_infinite.ll
deleted file mode 100644
index b70785671bbb6..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/LoopExtractor_infinite.ll
+++ /dev/null
@@ -1,53 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs
-; RUN: opt < %s -passes=loop-extract -S | FileCheck %s
-
-; This test used to enter an infinite loop, until out of memory (PR3082).
-
-define void @test(i1 %arg) {
-
-entry:
-  br label %loopentry
-
-loopentry:
-  br i1 %arg, label %exit.1, label %loopexit
-
-loopexit:
-  br i1 %arg, label %loopentry, label %exit.0
-
-exit.0:
-  br label %unified
-
-exit.1:
-  br label %unified
-
-unified:
-  ret void
-}
-; CHECK-LABEL: define {{[^@]+}}@test
-; CHECK-SAME: (i1 [[ARG:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[CODEREPL:%.*]]
-; CHECK:       codeRepl:
-; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @test.loopentry(i1 [[ARG]])
-; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[EXIT_1:%.*]], label [[EXIT_0:%.*]]
-; CHECK:       exit.0:
-; CHECK-NEXT:    br label [[UNIFIED:%.*]]
-; CHECK:       exit.1:
-; CHECK-NEXT:    br label [[UNIFIED]]
-; CHECK:       unified:
-; CHECK-NEXT:    ret void
-;
-;
-; CHECK-LABEL: define {{[^@]+}}@test.loopentry
-; CHECK-SAME: (i1 [[ARG:%.*]]) {
-; CHECK-NEXT:  newFuncRoot:
-; CHECK-NEXT:    br label [[LOOPENTRY:%.*]]
-; CHECK:       loopentry:
-; CHECK-NEXT:    br i1 [[ARG]], label [[EXIT_1_EXITSTUB:%.*]], label [[LOOPEXIT:%.*]]
-; CHECK:       loopexit:
-; CHECK-NEXT:    br i1 [[ARG]], label [[LOOPENTRY]], label [[EXIT_0_EXITSTUB:%.*]]
-; CHECK:       exit.1.exitStub:
-; CHECK-NEXT:    ret i1 true
-; CHECK:       exit.0.exitStub:
-; CHECK-NEXT:    ret i1 false
-;
diff --git a/llvm/test/Transforms/CodeExtractor/LoopExtractor_min_wrapper.ll b/llvm/test/Transforms/CodeExtractor/LoopExtractor_min_wrapper.ll
deleted file mode 100644
index 3f1bdaebda697..0000000000000
--- a/llvm/test/Transforms/CodeExtractor/LoopExtractor_min_wrapper.ll
+++ /dev/null
@@ -1,35 +0,0 @@
-; RUN: opt < %s -passes='function(break-crit-edges,loop-simplify),loop-extract' -S | FileCheck %s
-
-; This function is just a minimal wrapper around a loop and should not be extracted.
-define void @test() {
-; CHECK-LABEL: @test(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label %loop
-; CHECK:       loop:
-; CHECK-NEXT:    %index = phi i32 [ 0, %entry ], [ %next, %loop.loop_crit_edge ]
-; CHECK-NEXT:    call void @foo()
-; CHECK-NEXT:    %next = add nsw i32 %index, -1
-; CHECK-NEXT:    %repeat = icmp sgt i32 %index, 1
-; CHECK-NEXT:    br i1 %repeat, label %loop.loop_crit_edge, label %exit
-; CHECK:       loop.loop_crit_edge:
-; CHECK-NEXT:    br label %loop
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
-
-entry:
-  br label %loop
-
-loop:                                             ; preds = %loop, %entry
-  %index = phi i32 [ 0, %entry ], [ %next, %loop ]
-  call void @foo()
-  %next = add nsw i32 %index, -1
-  %repeat = icmp sgt i32 %index, 1
-  br i1 %repeat, label %loop, label %exit
-
-exit:                                             ; preds = %loop
-  ret void
-}
-
-declare void @foo()
-
-; CHECK-NOT: define
diff --git a/llvm/test/Transforms/LoopInterchange/freeze.ll b/llvm/test/Transforms/LoopInterchange/freeze.ll
new file mode 100644
index 0000000000000..17316ee451d73
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/freeze.ll
@@ -0,0 +1,868 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-interchange -loop-interchange-profitabilities=ignore -S %s | FileCheck %s
+
+; choice = freeze(poison);
+; sum = 0.0;
+; for (i = 0; i < 4; i++)
+;   for (j = 0; j < 4; j++)
+;     sum += choice ? A[j][i] : 0.0;
+; Interchange is allowed because the outer preheader keeps its loop depth and
+; execution count.
+define void @outer_preheader_freeze(ptr noalias %A, ptr noalias %R) {
+; CHECK-LABEL: define void @outer_preheader_freeze(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[R:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[INNER_HEADER_PREHEADER:.*]]
+; CHECK:       [[OUTER_PREHEADER:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[OUTER_PREHEADER]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[SUM_J:%.*]] = phi double [ [[SUM_NEXT_J:%.*]], %[[OUTER_LATCH]] ], [ [[SUM_I:%.*]], %[[OUTER_PREHEADER]] ]
+; CHECK-NEXT:    br label %[[INNER_HEADER_SPLIT1:.*]]
+; CHECK:       [[INNER_HEADER_PREHEADER]]:
+; CHECK-NEXT:    [[CHOICE:%.*]] = freeze i1 poison
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[TMP0:%.*]], %[[INNER_HEADER_SPLIT:.*]] ], [ 0, %[[INNER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    [[SUM_I]] = phi double [ 0.000000e+00, %[[INNER_HEADER_PREHEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_HEADER_SPLIT]] ]
+; CHECK-NEXT:    br label %[[OUTER_PREHEADER]]
+; CHECK:       [[INNER_HEADER_SPLIT1]]:
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [4 x double], ptr [[A]], i64 [[J]], i64 [[I]]
+; CHECK-NEXT:    [[VALUE:%.*]] = load double, ptr [[IDX]], align 8
+; CHECK-NEXT:    [[SELECTED:%.*]] = select i1 [[CHOICE]], double [[VALUE]], double 0.000000e+00
+; CHECK-NEXT:    [[SUM_NEXT_J]] = fadd reassoc double [[SUM_J]], [[SELECTED]]
+; CHECK-NEXT:    [[J_NEXT:%.*]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[J_EC:%.*]] = icmp eq i64 [[J_NEXT]], 4
+; CHECK-NEXT:    br label %[[OUTER_LATCH]]
+; CHECK:       [[INNER_HEADER_SPLIT]]:
+; CHECK-NEXT:    [[SUM_NEXT]] = phi double [ [[SUM_NEXT_J]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[TMP0]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[TMP0]], 4
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[EXIT:.*]], label %[[INNER_HEADER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[I_EC:%.*]] = icmp eq i64 [[I_NEXT]], 4
+; CHECK-NEXT:    br i1 [[I_EC]], label %[[INNER_HEADER_SPLIT]], label %[[OUTER_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SUM_RESULT:%.*]] = phi double [ [[SUM_NEXT]], %[[INNER_HEADER_SPLIT]] ]
+; CHECK-NEXT:    store double [[SUM_RESULT]], ptr [[R]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.preheader
+
+outer.preheader:
+  %choice = freeze i1 poison
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %outer.preheader ], [ %i.next, %outer.latch ]
+  %sum.i = phi double [ 0.000000e+00, %outer.preheader ], [ %sum.next, %outer.latch ]
+  br label %inner.header
+
+inner.header:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.header ]
+  %sum.j = phi double [ %sum.i, %outer.header ], [ %sum.next.j, %inner.header ]
+  %idx = getelementptr inbounds [4 x double], ptr %A, i64 %j, i64 %i
+  %value = load double, ptr %idx, align 8
+  %selected = select i1 %choice, double %value, double 0.000000e+00
+  %sum.next.j = fadd reassoc double %sum.j, %selected
+  %j.next = add i64 %j, 1
+  %j.ec = icmp eq i64 %j.next, 4
+  br i1 %j.ec, label %outer.latch, label %inner.header
+
+outer.latch:
+  %sum.next = phi double [ %sum.next.j, %inner.header ]
+  %i.next = add i64 %i, 1
+  %i.ec = icmp eq i64 %i.next, 4
+  br i1 %i.ec, label %exit, label %outer.header
+
+exit:
+  %sum.result = phi double [ %sum.next, %outer.latch ]
+  store double %sum.result, ptr %R, align 8
+  ret void
+}
+
+; sum = 0.0;
+; for (i = 0; i < 4; i++)
+;   for (j = 0; j < 4; j++) {
+;     choice = freeze(poison);
+;     sum += choice ? A[j][i] : 0.0;
+;   }
+; Interchange is allowed because the freeze in the inner-loop body keeps its
+; loop depth and is not cloned.
+define void @inner_body_freeze(ptr noalias %A, ptr noalias %R) {
+; CHECK-LABEL: define void @inner_body_freeze(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[R:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[INNER_HEADER_PREHEADER:.*]]
+; CHECK:       [[OUTER_HEADER_PREHEADER:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[OUTER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    [[SUM_J:%.*]] = phi double [ [[SUM_NEXT_J:%.*]], %[[OUTER_LATCH]] ], [ [[SUM_I:%.*]], %[[OUTER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label %[[INNER_HEADER_SPLIT1:.*]]
+; CHECK:       [[INNER_HEADER_PREHEADER]]:
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[TMP0:%.*]], %[[INNER_HEADER_SPLIT:.*]] ], [ 0, %[[INNER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    [[SUM_I]] = phi double [ [[SUM_NEXT:%.*]], %[[INNER_HEADER_SPLIT]] ], [ 0.000000e+00, %[[INNER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER_PREHEADER]]
+; CHECK:       [[INNER_HEADER_SPLIT1]]:
+; CHECK-NEXT:    [[CHOICE:%.*]] = freeze i1 poison
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [4 x double], ptr [[A]], i64 [[J]], i64 [[I]]
+; CHECK-NEXT:    [[VALUE:%.*]] = load double, ptr [[IDX]], align 8
+; CHECK-NEXT:    [[SELECTED:%.*]] = select i1 [[CHOICE]], double [[VALUE]], double 0.000000e+00
+; CHECK-NEXT:    [[SUM_NEXT_J]] = fadd reassoc double [[SUM_J]], [[SELECTED]]
+; CHECK-NEXT:    [[J_NEXT:%.*]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[J_EC:%.*]] = icmp eq i64 [[J_NEXT]], 4
+; CHECK-NEXT:    br label %[[OUTER_LATCH]]
+; CHECK:       [[INNER_HEADER_SPLIT]]:
+; CHECK-NEXT:    [[SUM_NEXT]] = phi double [ [[SUM_NEXT_J]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[TMP0]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[TMP0]], 4
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[EXIT:.*]], label %[[INNER_HEADER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[I_EC:%.*]] = icmp eq i64 [[I_NEXT]], 4
+; CHECK-NEXT:    br i1 [[I_EC]], label %[[INNER_HEADER_SPLIT]], label %[[OUTER_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SUM_RESULT:%.*]] = phi double [ [[SUM_NEXT]], %[[INNER_HEADER_SPLIT]] ]
+; CHECK-NEXT:    store double [[SUM_RESULT]], ptr [[R]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  %sum.i = phi double [ 0.000000e+00, %entry ], [ %sum.next, %outer.latch ]
+  br label %inner.header
+
+inner.header:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.header ]
+  %sum.j = phi double [ %sum.i, %outer.header ], [ %sum.next.j, %inner.header ]
+  %choice = freeze i1 poison
+  %idx = getelementptr inbounds [4 x double], ptr %A, i64 %j, i64 %i
+  %value = load double, ptr %idx, align 8
+  %selected = select i1 %choice, double %value, double 0.000000e+00
+  %sum.next.j = fadd reassoc double %sum.j, %selected
+  %j.next = add i64 %j, 1
+  %j.ec = icmp eq i64 %j.next, 4
+  br i1 %j.ec, label %outer.latch, label %inner.header
+
+outer.latch:
+  %sum.next = phi double [ %sum.next.j, %inner.header ]
+  %i.next = add i64 %i, 1
+  %i.ec = icmp eq i64 %i.next, 4
+  br i1 %i.ec, label %exit, label %outer.header
+
+exit:
+  %sum.result = phi double [ %sum.next, %outer.latch ]
+  store double %sum.result, ptr %R, align 8
+  ret void
+}
+
+; Interchange is allowed because the distinct outer-header and inner-preheader
+; blocks contain no freeze.
+define void @header_preheader_control(ptr noalias %A, i1 %choice) {
+; CHECK-LABEL: define void @header_preheader_control(
+; CHECK-SAME: ptr noalias [[A:%.*]], i1 [[CHOICE:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[INNER_PREHEADER:.*]]
+; CHECK:       [[OUTER_HEADER_PREHEADER:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[OUTER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    [[HEADER_BIT:%.*]] = xor i1 [[CHOICE]], true
+; CHECK-NEXT:    [[PREHEADER_BIT:%.*]] = xor i1 [[CHOICE]], false
+; CHECK-NEXT:    [[SELECTED_BIT:%.*]] = and i1 [[HEADER_BIT]], [[PREHEADER_BIT]]
+; CHECK-NEXT:    br label %[[INNER_HEADER_SPLIT1:.*]]
+; CHECK:       [[INNER_PREHEADER]]:
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[INNER_PREHEADER]] ], [ [[TMP0:%.*]], %[[INNER_HEADER_SPLIT:.*]] ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER_PREHEADER]]
+; CHECK:       [[INNER_HEADER_SPLIT1]]:
+; CHECK-NEXT:    [[SELECTED:%.*]] = select i1 [[SELECTED_BIT]], double 1.000000e+00, double 0.000000e+00
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [4 x double], ptr [[A]], i64 [[J]], i64 [[I]]
+; CHECK-NEXT:    store double [[SELECTED]], ptr [[IDX]], align 8
+; CHECK-NEXT:    [[J_NEXT:%.*]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[J_EC:%.*]] = icmp eq i64 [[J_NEXT]], 4
+; CHECK-NEXT:    br label %[[OUTER_LATCH]]
+; CHECK:       [[INNER_HEADER_SPLIT]]:
+; CHECK-NEXT:    [[TMP0]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[TMP0]], 4
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[EXIT:.*]], label %[[INNER_HEADER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[I_EC:%.*]] = icmp eq i64 [[I_NEXT]], 4
+; CHECK-NEXT:    br i1 [[I_EC]], label %[[INNER_HEADER_SPLIT]], label %[[OUTER_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  %header.bit = xor i1 %choice, true
+  br label %inner.preheader
+
+inner.preheader:
+  %preheader.bit = xor i1 %choice, false
+  %selected.bit = and i1 %header.bit, %preheader.bit
+  br label %inner.header
+
+inner.header:
+  %j = phi i64 [ 0, %inner.preheader ], [ %j.next, %inner.header ]
+  %selected = select i1 %selected.bit, double 1.000000e+00, double 0.000000e+00
+  %idx = getelementptr inbounds [4 x double], ptr %A, i64 %j, i64 %i
+  store double %selected, ptr %idx, align 8
+  %j.next = add i64 %j, 1
+  %j.ec = icmp eq i64 %j.next, 4
+  br i1 %j.ec, label %outer.latch, label %inner.header
+
+outer.latch:
+  %i.next = add i64 %i, 1
+  %i.ec = icmp eq i64 %i.next, 4
+  br i1 %i.ec, label %exit, label %outer.header
+
+exit:
+  ret void
+}
+
+; choice = freeze(poison) in the outer header, before a distinct inner
+; preheader.
+; Interchange is rejected because the freeze in the outer header would move to
+; a different loop depth.
+define void @outer_header_freeze(ptr noalias %A, i1 %choice) {
+; CHECK-LABEL: define void @outer_header_freeze(
+; CHECK-SAME: ptr noalias [[A:%.*]], i1 [[CHOICE:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[HEADER_BIT:%.*]] = freeze i1 poison
+; CHECK-NEXT:    br label %[[INNER_PREHEADER:.*]]
+; CHECK:       [[INNER_PREHEADER]]:
+; CHECK-NEXT:    [[PREHEADER_BIT:%.*]] = xor i1 [[CHOICE]], false
+; CHECK-NEXT:    [[SELECTED_BIT:%.*]] = and i1 [[HEADER_BIT]], [[PREHEADER_BIT]]
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[INNER_PREHEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[SELECTED:%.*]] = select i1 [[SELECTED_BIT]], double 1.000000e+00, double 0.000000e+00
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [4 x double], ptr [[A]], i64 [[J]], i64 [[I]]
+; CHECK-NEXT:    store double [[SELECTED]], ptr [[IDX]], align 8
+; CHECK-NEXT:    [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[J_EC:%.*]] = icmp eq i64 [[J_NEXT]], 4
+; CHECK-NEXT:    br i1 [[J_EC]], label %[[OUTER_LATCH]], label %[[INNER_HEADER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[I_EC:%.*]] = icmp eq i64 [[I_NEXT]], 4
+; CHECK-NEXT:    br i1 [[I_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  %header.bit = freeze i1 poison
+  br label %inner.preheader
+
+inner.preheader:
+  %preheader.bit = xor i1 %choice, false
+  %selected.bit = and i1 %header.bit, %preheader.bit
+  br label %inner.header
+
+inner.header:
+  %j = phi i64 [ 0, %inner.preheader ], [ %j.next, %inner.header ]
+  %selected = select i1 %selected.bit, double 1.000000e+00, double 0.000000e+00
+  %idx = getelementptr inbounds [4 x double], ptr %A, i64 %j, i64 %i
+  store double %selected, ptr %idx, align 8
+  %j.next = add i64 %j, 1
+  %j.ec = icmp eq i64 %j.next, 4
+  br i1 %j.ec, label %outer.latch, label %inner.header
+
+outer.latch:
+  %i.next = add i64 %i, 1
+  %i.ec = icmp eq i64 %i.next, 4
+  br i1 %i.ec, label %exit, label %outer.header
+
+exit:
+  ret void
+}
+
+; choice = freeze(poison) in the inner preheader, after a distinct outer
+; header.
+; Interchange is rejected because the freeze in the inner preheader would move
+; to a different loop depth.
+define void @inner_preheader_freeze(ptr noalias %A, i1 %choice) {
+; CHECK-LABEL: define void @inner_preheader_freeze(
+; CHECK-SAME: ptr noalias [[A:%.*]], i1 [[CHOICE:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[HEADER_BIT:%.*]] = xor i1 [[CHOICE]], true
+; CHECK-NEXT:    br label %[[INNER_PREHEADER:.*]]
+; CHECK:       [[INNER_PREHEADER]]:
+; CHECK-NEXT:    [[PREHEADER_BIT:%.*]] = freeze i1 poison
+; CHECK-NEXT:    [[SELECTED_BIT:%.*]] = and i1 [[HEADER_BIT]], [[PREHEADER_BIT]]
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[INNER_PREHEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[SELECTED:%.*]] = select i1 [[SELECTED_BIT]], double 1.000000e+00, double 0.000000e+00
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [4 x double], ptr [[A]], i64 [[J]], i64 [[I]]
+; CHECK-NEXT:    store double [[SELECTED]], ptr [[IDX]], align 8
+; CHECK-NEXT:    [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[J_EC:%.*]] = icmp eq i64 [[J_NEXT]], 4
+; CHECK-NEXT:    br i1 [[J_EC]], label %[[OUTER_LATCH]], label %[[INNER_HEADER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[I_EC:%.*]] = icmp eq i64 [[I_NEXT]], 4
+; CHECK-NEXT:    br i1 [[I_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  %header.bit = xor i1 %choice, true
+  br label %inner.preheader
+
+inner.preheader:
+  %preheader.bit = freeze i1 poison
+  %selected.bit = and i1 %header.bit, %preheader.bit
+  br label %inner.header
+
+inner.header:
+  %j = phi i64 [ 0, %inner.preheader ], [ %j.next, %inner.header ]
+  %selected = select i1 %selected.bit, double 1.000000e+00, double 0.000000e+00
+  %idx = getelementptr inbounds [4 x double], ptr %A, i64 %j, i64 %i
+  store double %selected, ptr %idx, align 8
+  %j.next = add i64 %j, 1
+  %j.ec = icmp eq i64 %j.next, 4
+  br i1 %j.ec, label %outer.latch, label %inner.header
+
+outer.latch:
+  %i.next = add i64 %i, 1
+  %i.ec = icmp eq i64 %i.next, 4
+  br i1 %i.ec, label %exit, label %outer.header
+
+exit:
+  ret void
+}
+
+; Interchange is allowed because the distinct inner-exit and outer-latch blocks
+; contain no freeze.
+define void @exit_latch_control(ptr noalias %A, ptr noalias %R, i1 %choice) {
+; CHECK-LABEL: define void @exit_latch_control(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[R:%.*]], i1 [[CHOICE:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER_PREHEADER:.*]]:
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[OUTER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label %[[INNER_HEADER_SPLIT1:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    br label %[[INNER_HEADER1:.*]]
+; CHECK:       [[INNER_HEADER1]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[TMP2:%.*]], %[[INNER_HEADER_SPLIT:.*]] ], [ 0, %[[OUTER_HEADER]] ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER_PREHEADER]]
+; CHECK:       [[INNER_HEADER_SPLIT1]]:
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [4 x double], ptr [[A]], i64 [[J]], i64 [[I]]
+; CHECK-NEXT:    store double 1.000000e+00, ptr [[IDX]], align 8
+; CHECK-NEXT:    [[J_NEXT:%.*]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[J_EC:%.*]] = icmp eq i64 [[J_NEXT]], 4
+; CHECK-NEXT:    br label %[[INNER_EXIT:.*]]
+; CHECK:       [[INNER_HEADER_SPLIT]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i1 [ [[EXIT_BIT:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[LATCH_BIT:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[TMP2]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[TMP2]], 4
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[EXIT:.*]], label %[[INNER_HEADER1]]
+; CHECK:       [[INNER_EXIT]]:
+; CHECK-NEXT:    [[EXIT_BIT]] = xor i1 [[CHOICE]], true
+; CHECK-NEXT:    br label %[[OUTER_LATCH]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[LATCH_BIT]] = xor i1 [[CHOICE]], false
+; CHECK-NEXT:    [[I_EC:%.*]] = icmp eq i64 [[I_NEXT]], 4
+; CHECK-NEXT:    br i1 [[I_EC]], label %[[INNER_HEADER_SPLIT]], label %[[INNER_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[EXIT_OUT:%.*]] = phi i1 [ [[TMP0]], %[[INNER_HEADER_SPLIT]] ]
+; CHECK-NEXT:    [[LATCH_OUT:%.*]] = phi i1 [ [[TMP1]], %[[INNER_HEADER_SPLIT]] ]
+; CHECK-NEXT:    [[SELECTED_OUT:%.*]] = and i1 [[EXIT_OUT]], [[LATCH_OUT]]
+; CHECK-NEXT:    [[VALUE:%.*]] = select i1 [[SELECTED_OUT]], double 1.000000e+00, double 0.000000e+00
+; CHECK-NEXT:    store double [[VALUE]], ptr [[R]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  br label %inner.header
+
+inner.header:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.header ]
+  %idx = getelementptr inbounds [4 x double], ptr %A, i64 %j, i64 %i
+  store double 1.000000e+00, ptr %idx, align 8
+  %j.next = add i64 %j, 1
+  %j.ec = icmp eq i64 %j.next, 4
+  br i1 %j.ec, label %inner.exit, label %inner.header
+
+inner.exit:
+  %exit.bit = xor i1 %choice, true
+  br label %outer.latch
+
+outer.latch:
+  %exit.bit.lcssa = phi i1 [ %exit.bit, %inner.exit ]
+  %i.next = add i64 %i, 1
+  %latch.bit = xor i1 %choice, false
+  %i.ec = icmp eq i64 %i.next, 4
+  br i1 %i.ec, label %exit, label %outer.header
+
+exit:
+  %exit.out = phi i1 [ %exit.bit.lcssa, %outer.latch ]
+  %latch.out = phi i1 [ %latch.bit, %outer.latch ]
+  %selected.out = and i1 %exit.out, %latch.out
+  %value = select i1 %selected.out, double 1.000000e+00, double 0.000000e+00
+  store double %value, ptr %R, align 8
+  ret void
+}
+
+; choice = freeze(poison) in the inner exit, before a distinct outer latch.
+; Interchange is rejected because the freeze in the inner exit would move to a
+; different loop depth.
+define void @inner_exit_freeze(ptr noalias %A, ptr noalias %R, i1 %choice) {
+; CHECK-LABEL: define void @inner_exit_freeze(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[R:%.*]], i1 [[CHOICE:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [4 x double], ptr [[A]], i64 [[J]], i64 [[I]]
+; CHECK-NEXT:    store double 1.000000e+00, ptr [[IDX]], align 8
+; CHECK-NEXT:    [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[J_EC:%.*]] = icmp eq i64 [[J_NEXT]], 4
+; CHECK-NEXT:    br i1 [[J_EC]], label %[[INNER_EXIT:.*]], label %[[INNER_HEADER]]
+; CHECK:       [[INNER_EXIT]]:
+; CHECK-NEXT:    [[EXIT_BIT:%.*]] = freeze i1 poison
+; CHECK-NEXT:    br label %[[OUTER_LATCH]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[EXIT_BIT_LCSSA:%.*]] = phi i1 [ [[EXIT_BIT]], %[[INNER_EXIT]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[LATCH_BIT:%.*]] = xor i1 [[CHOICE]], false
+; CHECK-NEXT:    [[I_EC:%.*]] = icmp eq i64 [[I_NEXT]], 4
+; CHECK-NEXT:    br i1 [[I_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[EXIT_OUT:%.*]] = phi i1 [ [[EXIT_BIT_LCSSA]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[LATCH_OUT:%.*]] = phi i1 [ [[LATCH_BIT]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[SELECTED_OUT:%.*]] = and i1 [[EXIT_OUT]], [[LATCH_OUT]]
+; CHECK-NEXT:    [[VALUE:%.*]] = select i1 [[SELECTED_OUT]], double 1.000000e+00, double 0.000000e+00
+; CHECK-NEXT:    store double [[VALUE]], ptr [[R]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  br label %inner.header
+
+inner.header:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.header ]
+  %idx = getelementptr inbounds [4 x double], ptr %A, i64 %j, i64 %i
+  store double 1.000000e+00, ptr %idx, align 8
+  %j.next = add i64 %j, 1
+  %j.ec = icmp eq i64 %j.next, 4
+  br i1 %j.ec, label %inner.exit, label %inner.header
+
+inner.exit:
+  %exit.bit = freeze i1 poison
+  br label %outer.latch
+
+outer.latch:
+  %exit.bit.lcssa = phi i1 [ %exit.bit, %inner.exit ]
+  %i.next = add i64 %i, 1
+  %latch.bit = xor i1 %choice, false
+  %i.ec = icmp eq i64 %i.next, 4
+  br i1 %i.ec, label %exit, label %outer.header
+
+exit:
+  %exit.out = phi i1 [ %exit.bit.lcssa, %outer.latch ]
+  %latch.out = phi i1 [ %latch.bit, %outer.latch ]
+  %selected.out = and i1 %exit.out, %latch.out
+  %value = select i1 %selected.out, double 1.000000e+00, double 0.000000e+00
+  store double %value, ptr %R, align 8
+  ret void
+}
+
+; choice = freeze(poison) in the outer latch, after a distinct inner exit.
+; Interchange is rejected because the freeze in the outer latch would move to a
+; different loop depth.
+define void @outer_latch_freeze(ptr noalias %A, ptr noalias %R, i1 %choice) {
+; CHECK-LABEL: define void @outer_latch_freeze(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[R:%.*]], i1 [[CHOICE:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [4 x double], ptr [[A]], i64 [[J]], i64 [[I]]
+; CHECK-NEXT:    store double 1.000000e+00, ptr [[IDX]], align 8
+; CHECK-NEXT:    [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[J_EC:%.*]] = icmp eq i64 [[J_NEXT]], 4
+; CHECK-NEXT:    br i1 [[J_EC]], label %[[INNER_EXIT:.*]], label %[[INNER_HEADER]]
+; CHECK:       [[INNER_EXIT]]:
+; CHECK-NEXT:    [[EXIT_BIT:%.*]] = xor i1 [[CHOICE]], true
+; CHECK-NEXT:    br label %[[OUTER_LATCH]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[EXIT_BIT_LCSSA:%.*]] = phi i1 [ [[EXIT_BIT]], %[[INNER_EXIT]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[LATCH_BIT:%.*]] = freeze i1 poison
+; CHECK-NEXT:    [[I_EC:%.*]] = icmp eq i64 [[I_NEXT]], 4
+; CHECK-NEXT:    br i1 [[I_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[EXIT_OUT:%.*]] = phi i1 [ [[EXIT_BIT_LCSSA]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[LATCH_OUT:%.*]] = phi i1 [ [[LATCH_BIT]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[SELECTED_OUT:%.*]] = and i1 [[EXIT_OUT]], [[LATCH_OUT]]
+; CHECK-NEXT:    [[VALUE:%.*]] = select i1 [[SELECTED_OUT]], double 1.000000e+00, double 0.000000e+00
+; CHECK-NEXT:    store double [[VALUE]], ptr [[R]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  br label %inner.header
+
+inner.header:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.header ]
+  %idx = getelementptr inbounds [4 x double], ptr %A, i64 %j, i64 %i
+  store double 1.000000e+00, ptr %idx, align 8
+  %j.next = add i64 %j, 1
+  %j.ec = icmp eq i64 %j.next, 4
+  br i1 %j.ec, label %inner.exit, label %inner.header
+
+inner.exit:
+  %exit.bit = xor i1 %choice, true
+  br label %outer.latch
+
+outer.latch:
+  %exit.bit.lcssa = phi i1 [ %exit.bit, %inner.exit ]
+  %i.next = add i64 %i, 1
+  %latch.bit = freeze i1 poison
+  %i.ec = icmp eq i64 %i.next, 4
+  br i1 %i.ec, label %exit, label %outer.header
+
+exit:
+  %exit.out = phi i1 [ %exit.bit.lcssa, %outer.latch ]
+  %latch.out = phi i1 [ %latch.bit, %outer.latch ]
+  %selected.out = and i1 %exit.out, %latch.out
+  %value = select i1 %selected.out, double 1.000000e+00, double 0.000000e+00
+  store double %value, ptr %R, align 8
+  ret void
+}
+
+; Interchange is allowed because neither cloned operand set contains a freeze.
+define void @inner_latch_clone_control(ptr noalias %A, ptr noalias %R,
+; CHECK-LABEL: define void @inner_latch_clone_control(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[R:%.*]], i1 [[CONDITION_ARG:%.*]], i1 [[STEP_ARG:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[INNER_HEADER_PREHEADER:.*]]
+; CHECK:       [[OUTER_HEADER_PREHEADER:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[OUTER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label %[[INNER_HEADER_SPLIT1:.*]]
+; CHECK:       [[INNER_HEADER_PREHEADER]]:
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ [[TMP9:%.*]], %[[INNER_HEADER_SPLIT:.*]] ], [ 0, %[[INNER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    [[K:%.*]] = phi i64 [ [[TMP4:%.*]], %[[INNER_HEADER_SPLIT]] ], [ 0, %[[INNER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER_PREHEADER]]
+; CHECK:       [[INNER_HEADER_SPLIT1]]:
+; CHECK-NEXT:    [[CONDITION_CHOICE:%.*]] = xor i1 [[CONDITION_ARG]], true
+; CHECK-NEXT:    [[STEP_CHOICE:%.*]] = xor i1 [[STEP_ARG]], true
+; CHECK-NEXT:    [[CONDITION_EXT:%.*]] = zext i1 [[CONDITION_CHOICE]] to i64
+; CHECK-NEXT:    [[CONDITION_ZERO:%.*]] = mul i64 [[CONDITION_EXT]], 0
+; CHECK-NEXT:    [[BOUND:%.*]] = add i64 4, [[CONDITION_ZERO]]
+; CHECK-NEXT:    [[STEP_EXT:%.*]] = zext i1 [[STEP_CHOICE]] to i64
+; CHECK-NEXT:    [[STEP_ZERO:%.*]] = mul i64 [[STEP_EXT]], 0
+; CHECK-NEXT:    [[STEP:%.*]] = add i64 1, [[STEP_ZERO]]
+; CHECK-NEXT:    [[K_NEXT:%.*]] = add i64 [[K]], [[STEP]]
+; CHECK-NEXT:    [[K_FP:%.*]] = sitofp i64 [[K]] to double
+; CHECK-NEXT:    [[SELECTED_BIT:%.*]] = and i1 [[CONDITION_CHOICE]], [[STEP_CHOICE]]
+; CHECK-NEXT:    [[SELECTED:%.*]] = select i1 [[SELECTED_BIT]], double 1.000000e+00, double 0.000000e+00
+; CHECK-NEXT:    [[VALUE:%.*]] = fadd double [[SELECTED]], [[K_FP]]
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [4 x double], ptr [[A]], i64 [[J]], i64 [[I]]
+; CHECK-NEXT:    store double [[VALUE]], ptr [[IDX]], align 8
+; CHECK-NEXT:    [[J_NEXT:%.*]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[J_EC:%.*]] = icmp eq i64 [[J_NEXT]], [[BOUND]]
+; CHECK-NEXT:    br label %[[OUTER_LATCH]]
+; CHECK:       [[INNER_HEADER_SPLIT]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[STEP_ARG]], true
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 1, [[TMP2]]
+; CHECK-NEXT:    [[TMP4]] = add i64 [[K]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i1 [[CONDITION_ARG]], true
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 4, [[TMP7]]
+; CHECK-NEXT:    [[TMP9]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[EXIT:.*]], label %[[INNER_HEADER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[I_EC:%.*]] = icmp eq i64 [[I_NEXT]], 4
+; CHECK-NEXT:    br i1 [[I_EC]], label %[[INNER_HEADER_SPLIT]], label %[[OUTER_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[CONDITION_OUT:%.*]] = phi i1 [ [[TMP5]], %[[INNER_HEADER_SPLIT]] ]
+; CHECK-NEXT:    [[STEP_OUT:%.*]] = phi i1 [ [[TMP0]], %[[INNER_HEADER_SPLIT]] ]
+; CHECK-NEXT:    [[SELECTED_OUT:%.*]] = and i1 [[CONDITION_OUT]], [[STEP_OUT]]
+; CHECK-NEXT:    [[RESULT:%.*]] = select i1 [[SELECTED_OUT]], double 2.000000e+00, double 3.000000e+00
+; CHECK-NEXT:    store double [[RESULT]], ptr [[R]], align 8
+; CHECK-NEXT:    ret void
+;
+  i1 %condition.arg, i1 %step.arg) {
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  br label %inner.header
+
+inner.header:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.header ]
+  %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner.header ]
+  %condition.choice = xor i1 %condition.arg, true
+  %step.choice = xor i1 %step.arg, true
+  %condition.ext = zext i1 %condition.choice to i64
+  %condition.zero = mul i64 %condition.ext, 0
+  %bound = add i64 4, %condition.zero
+  %step.ext = zext i1 %step.choice to i64
+  %step.zero = mul i64 %step.ext, 0
+  %step = add i64 1, %step.zero
+  %k.next = add i64 %k, %step
+  %k.fp = sitofp i64 %k to double
+  %selected.bit = and i1 %condition.choice, %step.choice
+  %selected = select i1 %selected.bit, double 1.000000e+00, double 0.000000e+00
+  %value = fadd double %selected, %k.fp
+  %idx = getelementptr inbounds [4 x double], ptr %A, i64 %j, i64 %i
+  store double %value, ptr %idx, align 8
+  %j.next = add i64 %j, 1
+  %j.ec = icmp eq i64 %j.next, %bound
+  br i1 %j.ec, label %outer.latch, label %inner.header
+
+outer.latch:
+  %condition.lcssa = phi i1 [ %condition.choice, %inner.header ]
+  %step.lcssa = phi i1 [ %step.choice, %inner.header ]
+  %i.next = add i64 %i, 1
+  %i.ec = icmp eq i64 %i.next, 4
+  br i1 %i.ec, label %exit, label %outer.header
+
+exit:
+  %condition.out = phi i1 [ %condition.lcssa, %outer.latch ]
+  %step.out = phi i1 [ %step.lcssa, %outer.latch ]
+  %selected.out = and i1 %condition.out, %step.out
+  %result = select i1 %selected.out, double 2.000000e+00, double 3.000000e+00
+  store double %result, ptr %R, align 8
+  ret void
+}
+
+; Interchange is rejected because the latch-condition computation would clone
+; the freeze.
+define void @inner_latch_condition_freeze(ptr noalias %A, ptr noalias %R,
+; CHECK-LABEL: define void @inner_latch_condition_freeze(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[R:%.*]], i1 [[STEP_ARG:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[CONDITION_CHOICE:%.*]] = freeze i1 poison
+; CHECK-NEXT:    [[STEP_CHOICE:%.*]] = xor i1 [[STEP_ARG]], true
+; CHECK-NEXT:    [[CONDITION_EXT:%.*]] = zext i1 [[CONDITION_CHOICE]] to i64
+; CHECK-NEXT:    [[CONDITION_ZERO:%.*]] = mul i64 [[CONDITION_EXT]], 0
+; CHECK-NEXT:    [[BOUND:%.*]] = add i64 4, [[CONDITION_ZERO]]
+; CHECK-NEXT:    [[STEP_EXT:%.*]] = zext i1 [[STEP_CHOICE]] to i64
+; CHECK-NEXT:    [[STEP_ZERO:%.*]] = mul i64 [[STEP_EXT]], 0
+; CHECK-NEXT:    [[STEP:%.*]] = add i64 1, [[STEP_ZERO]]
+; CHECK-NEXT:    [[K_NEXT]] = add i64 [[K]], [[STEP]]
+; CHECK-NEXT:    [[K_FP:%.*]] = sitofp i64 [[K]] to double
+; CHECK-NEXT:    [[SELECTED_BIT:%.*]] = and i1 [[CONDITION_CHOICE]], [[STEP_CHOICE]]
+; CHECK-NEXT:    [[SELECTED:%.*]] = select i1 [[SELECTED_BIT]], double 1.000000e+00, double 0.000000e+00
+; CHECK-NEXT:    [[VALUE:%.*]] = fadd double [[SELECTED]], [[K_FP]]
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [4 x double], ptr [[A]], i64 [[J]], i64 [[I]]
+; CHECK-NEXT:    store double [[VALUE]], ptr [[IDX]], align 8
+; CHECK-NEXT:    [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[J_EC:%.*]] = icmp eq i64 [[J_NEXT]], [[BOUND]]
+; CHECK-NEXT:    br i1 [[J_EC]], label %[[OUTER_LATCH]], label %[[INNER_HEADER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[CONDITION_LCSSA:%.*]] = phi i1 [ [[CONDITION_CHOICE]], %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[STEP_LCSSA:%.*]] = phi i1 [ [[STEP_CHOICE]], %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[I_EC:%.*]] = icmp eq i64 [[I_NEXT]], 4
+; CHECK-NEXT:    br i1 [[I_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[CONDITION_OUT:%.*]] = phi i1 [ [[CONDITION_LCSSA]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[STEP_OUT:%.*]] = phi i1 [ [[STEP_LCSSA]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[SELECTED_OUT:%.*]] = and i1 [[CONDITION_OUT]], [[STEP_OUT]]
+; CHECK-NEXT:    [[RESULT:%.*]] = select i1 [[SELECTED_OUT]], double 2.000000e+00, double 3.000000e+00
+; CHECK-NEXT:    store double [[RESULT]], ptr [[R]], align 8
+; CHECK-NEXT:    ret void
+;
+  i1 %step.arg) {
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  br label %inner.header
+
+inner.header:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.header ]
+  %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner.header ]
+  %condition.choice = freeze i1 poison
+  %step.choice = xor i1 %step.arg, true
+  %condition.ext = zext i1 %condition.choice to i64
+  %condition.zero = mul i64 %condition.ext, 0
+  %bound = add i64 4, %condition.zero
+  %step.ext = zext i1 %step.choice to i64
+  %step.zero = mul i64 %step.ext, 0
+  %step = add i64 1, %step.zero
+  %k.next = add i64 %k, %step
+  %k.fp = sitofp i64 %k to double
+  %selected.bit = and i1 %condition.choice, %step.choice
+  %selected = select i1 %selected.bit, double 1.000000e+00, double 0.000000e+00
+  %value = fadd double %selected, %k.fp
+  %idx = getelementptr inbounds [4 x double], ptr %A, i64 %j, i64 %i
+  store double %value, ptr %idx, align 8
+  %j.next = add i64 %j, 1
+  %j.ec = icmp eq i64 %j.next, %bound
+  br i1 %j.ec, label %outer.latch, label %inner.header
+
+outer.latch:
+  %condition.lcssa = phi i1 [ %condition.choice, %inner.header ]
+  %step.lcssa = phi i1 [ %step.choice, %inner.header ]
+  %i.next = add i64 %i, 1
+  %i.ec = icmp eq i64 %i.next, 4
+  br i1 %i.ec, label %exit, label %outer.header
+
+exit:
+  %condition.out = phi i1 [ %condition.lcssa, %outer.latch ]
+  %step.out = phi i1 [ %step.lcssa, %outer.latch ]
+  %selected.out = and i1 %condition.out, %step.out
+  %result = select i1 %selected.out, double 2.000000e+00, double 3.000000e+00
+  store double %result, ptr %R, align 8
+  ret void
+}
+
+; Interchange is rejected because the second induction update would clone the
+; freeze.
+define void @inner_latch_induction_freeze(ptr noalias %A, ptr noalias %R,
+; CHECK-LABEL: define void @inner_latch_induction_freeze(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[R:%.*]], i1 [[CONDITION_ARG:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
+; CHECK:       [[INNER_HEADER]]:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[CONDITION_CHOICE:%.*]] = xor i1 [[CONDITION_ARG]], true
+; CHECK-NEXT:    [[STEP_CHOICE:%.*]] = freeze i1 poison
+; CHECK-NEXT:    [[CONDITION_EXT:%.*]] = zext i1 [[CONDITION_CHOICE]] to i64
+; CHECK-NEXT:    [[CONDITION_ZERO:%.*]] = mul i64 [[CONDITION_EXT]], 0
+; CHECK-NEXT:    [[BOUND:%.*]] = add i64 4, [[CONDITION_ZERO]]
+; CHECK-NEXT:    [[STEP_EXT:%.*]] = zext i1 [[STEP_CHOICE]] to i64
+; CHECK-NEXT:    [[STEP_ZERO:%.*]] = mul i64 [[STEP_EXT]], 0
+; CHECK-NEXT:    [[STEP:%.*]] = add i64 1, [[STEP_ZERO]]
+; CHECK-NEXT:    [[K_NEXT]] = add i64 [[K]], [[STEP]]
+; CHECK-NEXT:    [[K_FP:%.*]] = sitofp i64 [[K]] to double
+; CHECK-NEXT:    [[SELECTED_BIT:%.*]] = and i1 [[CONDITION_CHOICE]], [[STEP_CHOICE]]
+; CHECK-NEXT:    [[SELECTED:%.*]] = select i1 [[SELECTED_BIT]], double 1.000000e+00, double 0.000000e+00
+; CHECK-NEXT:    [[VALUE:%.*]] = fadd double [[SELECTED]], [[K_FP]]
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr inbounds [4 x double], ptr [[A]], i64 [[J]], i64 [[I]]
+; CHECK-NEXT:    store double [[VALUE]], ptr [[IDX]], align 8
+; CHECK-NEXT:    [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT:    [[J_EC:%.*]] = icmp eq i64 [[J_NEXT]], [[BOUND]]
+; CHECK-NEXT:    br i1 [[J_EC]], label %[[OUTER_LATCH]], label %[[INNER_HEADER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[CONDITION_LCSSA:%.*]] = phi i1 [ [[CONDITION_CHOICE]], %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[STEP_LCSSA:%.*]] = phi i1 [ [[STEP_CHOICE]], %[[INNER_HEADER]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT:    [[I_EC:%.*]] = icmp eq i64 [[I_NEXT]], 4
+; CHECK-NEXT:    br i1 [[I_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[CONDITION_OUT:%.*]] = phi i1 [ [[CONDITION_LCSSA]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[STEP_OUT:%.*]] = phi i1 [ [[STEP_LCSSA]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT:    [[SELECTED_OUT:%.*]] = and i1 [[CONDITION_OUT]], [[STEP_OUT]]
+; CHECK-NEXT:    [[RESULT:%.*]] = select i1 [[SELECTED_OUT]], double 2.000000e+00, double 3.000000e+00
+; CHECK-NEXT:    store double [[RESULT]], ptr [[R]], align 8
+; CHECK-NEXT:    ret void
+;
+  i1 %condition.arg) {
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
+  br label %inner.header
+
+inner.header:
+  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.header ]
+  %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner.header ]
+  %condition.choice = xor i1 %condition.arg, true
+  %step.choice = freeze i1 poison
+  %condition.ext = zext i1 %condition.choice to i64
+  %condition.zero = mul i64 %condition.ext, 0
+  %bound = add i64 4, %condition.zero
+  %step.ext = zext i1 %step.choice to i64
+  %step.zero = mul i64 %step.ext, 0
+  %step = add i64 1, %step.zero
+  %k.next = add i64 %k, %step
+  %k.fp = sitofp i64 %k to double
+  %selected.bit = and i1 %condition.choice, %step.choice
+  %selected = select i1 %selected.bit, double 1.000000e+00, double 0.000000e+00
+  %value = fadd double %selected, %k.fp
+  %idx = getelementptr inbounds [4 x double], ptr %A, i64 %j, i64 %i
+  store double %value, ptr %idx, align 8
+  %j.next = add i64 %j, 1
+  %j.ec = icmp eq i64 %j.next, %bound
+  br i1 %j.ec, label %outer.latch, label %inner.header
+
+outer.latch:
+  %condition.lcssa = phi i1 [ %condition.choice, %inner.header ]
+  %step.lcssa = phi i1 [ %step.choice, %inner.header ]
+  %i.next = add i64 %i, 1
+  %i.ec = icmp eq i64 %i.next, 4
+  br i1 %i.ec, label %exit, label %outer.header
+
+exit:
+  %condition.out = phi i1 [ %condition.lcssa, %outer.latch ]
+  %step.out = phi i1 [ %step.lcssa, %outer.latch ]
+  %selected.out = and i1 %condition.out, %step.out
+  %result = select i1 %selected.out, double 2.000000e+00, double 3.000000e+00
+  store double %result, ptr %R, align 8
+  ret void
+}
diff --git a/llvm/test/tools/llvm-readobj/ELF/note-openbsd-core.test b/llvm/test/tools/llvm-readobj/ELF/note-openbsd-core.test
index 3044fded0d6fa..3868a5761c16b 100644
--- a/llvm/test/tools/llvm-readobj/ELF/note-openbsd-core.test
+++ b/llvm/test/tools/llvm-readobj/ELF/note-openbsd-core.test
@@ -18,6 +18,8 @@ Sections:
         Type: NT_OPENBSD_AUXV
       - Name: OpenBSD
         Type: NT_OPENBSD_WCOOKIE
+      - Name: OpenBSD
+        Type: NT_OPENBSD_PACMASK
       - Name: OpenBSD@31337
         Type: NT_OPENBSD_REGS
       - Name: OpenBSD@31337
@@ -27,11 +29,12 @@ ProgramHeaders:
     FirstSec: .note.foo
     LastSec:  .note.foo
 
-# GNU:      Displaying notes found at file offset 0x00000078 with length 0x00000074:
+# GNU:      Displaying notes found at file offset 0x00000078 with length 0x00000088:
 # GNU-NEXT:   Owner                Data size 	Description
 # GNU-NEXT:   OpenBSD              0x00000000	NT_OPENBSD_PROCINFO (procinfo structure)
 # GNU-NEXT:   OpenBSD              0x00000000	NT_OPENBSD_AUXV (ELF auxiliary vector data)
 # GNU-NEXT:   OpenBSD              0x00000000	NT_OPENBSD_WCOOKIE (window cookie)
+# GNU-NEXT:   OpenBSD              0x00000000	NT_OPENBSD_PACMASK (AArch64 Pointer Authentication Code mask)
 # GNU-NEXT:   OpenBSD@31337        0x00000000	NT_OPENBSD_REGS (regular registers)
 # GNU-NEXT:   OpenBSD@31337        0x00000000	NT_OPENBSD_FPREGS (floating point registers)
 
@@ -39,7 +42,7 @@ ProgramHeaders:
 # LLVM-NEXT:   NoteSection {
 # LLVM-NEXT:     Name: <?>
 # LLVM-NEXT:     Offset: 0x78
-# LLVM-NEXT:     Size: 0x74
+# LLVM-NEXT:     Size: 0x88
 # LLVM-NEXT:     Notes [
 # LLVM-NEXT:        {
 # LLVM-NEXT:            Owner: OpenBSD
@@ -57,6 +60,11 @@ ProgramHeaders:
 # LLVM-NEXT:            Type: NT_OPENBSD_WCOOKIE (window cookie)
 # LLVM-NEXT:        }
 # LLVM-NEXT:        {
+# LLVM-NEXT:            Owner: OpenBSD
+# LLVM-NEXT:            Data size: 0x0
+# LLVM-NEXT:            Type: NT_OPENBSD_PACMASK (AArch64 Pointer Authentication Code mask)
+# LLVM-NEXT:        }
+# LLVM-NEXT:        {
 # LLVM-NEXT:            Owner: OpenBSD@31337
 # LLVM-NEXT:            Data size: 0x0
 # LLVM-NEXT:            Type: NT_OPENBSD_REGS (regular registers)
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 6bd9efa0542a1..de0a97abb08ee 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -6255,6 +6255,8 @@ const NoteType OpenBSDCoreNoteTypes[] = {
     {ELF::NT_OPENBSD_REGS, "NT_OPENBSD_REGS (regular registers)"},
     {ELF::NT_OPENBSD_FPREGS, "NT_OPENBSD_FPREGS (floating point registers)"},
     {ELF::NT_OPENBSD_WCOOKIE, "NT_OPENBSD_WCOOKIE (window cookie)"},
+    {ELF::NT_OPENBSD_PACMASK,
+     "NT_OPENBSD_PACMASK (AArch64 Pointer Authentication Code mask)"},
 };
 
 const NoteType AMDNoteTypes[] = {
diff --git a/llvm/tools/llvm-reduce/llvm-reduce.cpp b/llvm/tools/llvm-reduce/llvm-reduce.cpp
index ffda00d1a5785..953e25afb9fd8 100644
--- a/llvm/tools/llvm-reduce/llvm-reduce.cpp
+++ b/llvm/tools/llvm-reduce/llvm-reduce.cpp
@@ -8,9 +8,7 @@
 //
 // This program tries to reduce an IR test case for a given interesting-ness
 // test. It runs multiple delta debugging passes in order to minimize the input
-// file. It's worth noting that this is a part of the bugpoint redesign
-// proposal, and thus a *temporary* tool that will eventually be integrated
-// into the bugpoint tool itself.
+// file.
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index b29cba96a410f..a7793ddbaf7e8 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -4411,144 +4411,42 @@ CpAsyncBulkTensorSharedCTAToGlobalOp::getIntrinsicIDAndArgs(
 NVVM::IDArgPair CpAsyncBulkTensorReduceOp::getIntrinsicIDAndArgs(
     Operation &op, LLVM::ModuleTranslation &mt, llvm::IRBuilderBase &builder) {
   auto thisOp = cast<NVVM::CpAsyncBulkTensorReduceOp>(op);
-  llvm::LLVMContext &ctx = mt.getLLVMContext();
 
   llvm::SmallVector<llvm::Value *> args;
-
-  // Arguments to the intrinsic:
-  // shared_mem_ptr, tmaDesc, tensorDims
-  // cache_hint(if applicable) and flag(boolean)
   args.push_back(mt.lookupValue(thisOp.getSrcMem()));
   args.push_back(mt.lookupValue(thisOp.getTmaDescriptor()));
-
   for (Value v : thisOp.getCoordinates())
     args.push_back(mt.lookupValue(v));
 
   mlir::Value cacheHint = thisOp.getL2CacheHint();
   const bool hasCacheHint = static_cast<bool>(cacheHint);
-  llvm::Value *i64ZeroValue =
-      llvm::ConstantInt::get(llvm::Type::getInt64Ty(ctx), 0);
-  args.push_back(hasCacheHint ? mt.lookupValue(cacheHint) : i64ZeroValue);
+  args.push_back(hasCacheHint ? mt.lookupValue(cacheHint)
+                              : builder.getInt64(0));
+  args.push_back(builder.getInt32(static_cast<uint32_t>(thisOp.getRedKind())));
   args.push_back(builder.getInt1(hasCacheHint));
 
-  const llvm::Intrinsic::ID notIntrinsic = llvm::Intrinsic::not_intrinsic;
-
-  constexpr unsigned numRedKinds = 8; // ADD, MIN, MAX, INC, DEC, AND, OR, XOR
-  constexpr unsigned numLayouts = 2;  // TILE, IM2COL
-  constexpr unsigned maxDim = 5;      // 1D to 5D
-  using row = std::array<llvm::Intrinsic::ID, maxDim + 1>;
-  using layoutTable = std::array<row, numLayouts>;
-  using fullTable = std::array<layoutTable, numRedKinds>;
-  static constexpr fullTable IDTable{
-      {// RedTy::ADD
-       {{{{notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_tile_1d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_tile_2d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_tile_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_tile_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_tile_5d}},
-         {{notIntrinsic, notIntrinsic, notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_im2col_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_im2col_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_add_im2col_5d}}}},
-       // RedTy::MIN
-       {{{{notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_tile_1d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_tile_2d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_tile_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_tile_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_tile_5d}},
-         {{notIntrinsic, notIntrinsic, notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_im2col_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_im2col_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_min_im2col_5d}}}},
-       // RedTy::MAX
-       {{{{notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_tile_1d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_tile_2d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_tile_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_tile_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_tile_5d}},
-         {{notIntrinsic, notIntrinsic, notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_im2col_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_im2col_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_max_im2col_5d}}}},
-       // RedTy::INC
-       {{{{notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_tile_1d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_tile_2d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_tile_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_tile_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_tile_5d}},
-         {{notIntrinsic, notIntrinsic, notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_im2col_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_im2col_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_inc_im2col_5d}}}},
-       // RedTy::DEC
-       {{{{notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_tile_1d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_tile_2d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_tile_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_tile_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_tile_5d}},
-         {{notIntrinsic, notIntrinsic, notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_im2col_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_im2col_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_dec_im2col_5d}}}},
-       // RedTy::AND
-       {{{{notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_tile_1d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_tile_2d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_tile_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_tile_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_tile_5d}},
-         {{notIntrinsic, notIntrinsic, notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_im2col_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_im2col_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_and_im2col_5d}}}},
-       // RedTy::OR
-       {{{{notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_tile_1d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_tile_2d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_tile_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_tile_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_tile_5d}},
-         {{notIntrinsic, notIntrinsic, notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_im2col_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_im2col_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_or_im2col_5d}}}},
-       // RedTy::XOR
-       {{{{notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_tile_1d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_tile_2d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_tile_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_tile_4d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_tile_5d}},
-         {{notIntrinsic, notIntrinsic, notIntrinsic,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_im2col_3d,
-           llvm::Intrinsic::nvvm_cp_async_bulk_tensor_reduce_xor_im2col_4d,
-           llvm::Intrinsic::
-               nvvm_cp_async_bulk_tensor_reduce_xor_im2col_5d}}}}}};
-
-  static_assert(getMaxEnumValForTMAReduxKind() == std::size(IDTable) - 1,
-                "TMAReduxKinds must match number of rows in IDTable");
-
-  size_t redKind = static_cast<size_t>(thisOp.getRedKind());
+  using namespace llvm::Intrinsic;
+  const unsigned NI = not_intrinsic;
+  static constexpr ID IDTable[][6] = {
+      {NI, nvvm_cp_async_bulk_tensor_reduce_tile_1d,
+       nvvm_cp_async_bulk_tensor_reduce_tile_2d,
+       nvvm_cp_async_bulk_tensor_reduce_tile_3d,
+       nvvm_cp_async_bulk_tensor_reduce_tile_4d,
+       nvvm_cp_async_bulk_tensor_reduce_tile_5d},
+      {NI, NI, NI, nvvm_cp_async_bulk_tensor_reduce_im2col_3d,
+       nvvm_cp_async_bulk_tensor_reduce_im2col_4d,
+       nvvm_cp_async_bulk_tensor_reduce_im2col_5d}};
+
   size_t mode = static_cast<size_t>(thisOp.getMode());
   size_t dim = thisOp.getCoordinates().size();
-
-  assert(redKind < IDTable.size() &&
-         "Invalid redKind for CpAsyncBulkTensorReduceOp");
-  assert(mode < IDTable[redKind].size() &&
+  assert(mode < std::size(IDTable) &&
          "Invalid mode for CpAsyncBulkTensorReduceOp");
-  assert(dim < IDTable[redKind][mode].size() &&
+  assert(dim < std::size(IDTable[mode]) &&
          "Invalid dim for CpAsyncBulkTensorReduceOp");
 
-  llvm::Intrinsic::ID intrinsicID = IDTable[redKind][mode][dim];
-
-  assert(intrinsicID != notIntrinsic &&
-         "Invalid intrinsic for CpAsyncBulkTensorReduceOp.");
-
+  ID intrinsicID = IDTable[mode][dim];
+  assert(intrinsicID != NI &&
+         "Invalid intrinsic for CpAsyncBulkTensorReduceOp");
   return {intrinsicID, std::move(args)};
 }
 
diff --git a/mlir/test/Target/LLVMIR/nvvm/tma_store_reduce.mlir b/mlir/test/Target/LLVMIR/nvvm/tma_store_reduce.mlir
index 2231f1dabd504..2a6f5bedbd3e7 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tma_store_reduce.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tma_store_reduce.mlir
@@ -2,14 +2,14 @@
 
 // CHECK-LABEL: define void @tma_store_reduce_1d(
 llvm.func @tma_store_reduce_1d(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr, %d0 : i32, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.1d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i64 %[[CH:.*]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i64 %[[CH:.*]], /* red_op=add */ i32 0, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], /* red_op=min */ i32 1, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], /* red_op=max */ i32 2, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], /* red_op=inc */ i32 3, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], /* red_op=dec */ i32 4, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], /* red_op=and */ i32 5, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], /* red_op=or */ i32 6, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 %[[CH]], /* red_op=xor */ i32 7, i1 true)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<add>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<min>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<max>} : !llvm.ptr, !llvm.ptr<3>
@@ -19,14 +19,14 @@ llvm.func @tma_store_reduce_1d(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr, %d0 :
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<or>}  : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<xor>} : !llvm.ptr, !llvm.ptr<3>
 
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, /* red_op=add */ i32 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, /* red_op=min */ i32 1, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, /* red_op=max */ i32 2, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, /* red_op=inc */ i32 3, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, /* red_op=dec */ i32 4, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, /* red_op=and */ i32 5, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, /* red_op=or */ i32 6, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.1d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i64 0, /* red_op=xor */ i32 7, i1 false)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0] {redKind = #nvvm.tma_redux_kind<add>, mode = #nvvm.tma_store_mode<tile>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0] {redKind = #nvvm.tma_redux_kind<min>, mode = #nvvm.tma_store_mode<tile>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0] {redKind = #nvvm.tma_redux_kind<max>, mode = #nvvm.tma_store_mode<tile>} : !llvm.ptr, !llvm.ptr<3>
@@ -42,14 +42,14 @@ llvm.func @tma_store_reduce_1d(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr, %d0 :
 
 // CHECK-LABEL: define void @tma_store_reduce_2d(
 llvm.func @tma_store_reduce_2d(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.2d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i64 %[[CH:.*]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i64 %[[CH:.*]], /* red_op=add */ i32 0, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], /* red_op=min */ i32 1, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], /* red_op=max */ i32 2, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], /* red_op=inc */ i32 3, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], /* red_op=dec */ i32 4, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], /* red_op=and */ i32 5, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], /* red_op=or */ i32 6, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 %[[CH]], /* red_op=xor */ i32 7, i1 true)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<add>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<min>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<max>} : !llvm.ptr, !llvm.ptr<3>
@@ -59,14 +59,14 @@ llvm.func @tma_store_reduce_2d(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr, %d0 :
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<or>}  : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<xor>} : !llvm.ptr, !llvm.ptr<3>
 
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, /* red_op=add */ i32 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, /* red_op=min */ i32 1, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, /* red_op=max */ i32 2, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, /* red_op=inc */ i32 3, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, /* red_op=dec */ i32 4, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, /* red_op=and */ i32 5, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, /* red_op=or */ i32 6, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.2d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i64 0, /* red_op=xor */ i32 7, i1 false)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1] {redKind = #nvvm.tma_redux_kind<add>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1] {redKind = #nvvm.tma_redux_kind<min>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1] {redKind = #nvvm.tma_redux_kind<max>} : !llvm.ptr, !llvm.ptr<3>
@@ -82,14 +82,14 @@ llvm.func @tma_store_reduce_2d(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr, %d0 :
 
 // CHECK-LABEL: define void @tma_store_reduce_3d_tile(
 llvm.func @tma_store_reduce_3d_tile(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.3d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i32 %[[D2:.*]], i64 %[[CH:.*]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i32 %[[D2:.*]], i64 %[[CH:.*]], /* red_op=add */ i32 0, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=min */ i32 1, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=max */ i32 2, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=inc */ i32 3, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=dec */ i32 4, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=and */ i32 5, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=or */ i32 6, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=xor */ i32 7, i1 true)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<add>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<min>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<max>} : !llvm.ptr, !llvm.ptr<3>
@@ -99,14 +99,14 @@ llvm.func @tma_store_reduce_3d_tile(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr,
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<or>}  : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<xor>} : !llvm.ptr, !llvm.ptr<3>
 
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=add */ i32 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=min */ i32 1, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=max */ i32 2, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=inc */ i32 3, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=dec */ i32 4, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=and */ i32 5, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=or */ i32 6, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=xor */ i32 7, i1 false)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] {redKind = #nvvm.tma_redux_kind<add>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] {redKind = #nvvm.tma_redux_kind<min>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] {redKind = #nvvm.tma_redux_kind<max>} : !llvm.ptr, !llvm.ptr<3>
@@ -120,14 +120,14 @@ llvm.func @tma_store_reduce_3d_tile(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr,
 
 // CHECK-LABEL: define void @tma_store_reduce_3d_im2col(
 llvm.func @tma_store_reduce_3d_im2col(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.3d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i32 %[[D2:.*]], i64 %[[CH:.*]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i32 %[[D2:.*]], i64 %[[CH:.*]], /* red_op=add */ i32 0, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=min */ i32 1, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=max */ i32 2, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=inc */ i32 3, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=dec */ i32 4, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=and */ i32 5, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=or */ i32 6, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 %[[CH]], /* red_op=xor */ i32 7, i1 true)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<add>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<min>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<max>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
@@ -137,14 +137,14 @@ llvm.func @tma_store_reduce_3d_im2col(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<or>, mode = #nvvm.tma_store_mode<im2col>}  : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<xor>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
 
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=add */ i32 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=min */ i32 1, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=max */ i32 2, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=inc */ i32 3, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=dec */ i32 4, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=and */ i32 5, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=or */ i32 6, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.3d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i64 0, /* red_op=xor */ i32 7, i1 false)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] {redKind = #nvvm.tma_redux_kind<add>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] {redKind = #nvvm.tma_redux_kind<min>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2] {redKind = #nvvm.tma_redux_kind<max>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
@@ -160,14 +160,14 @@ llvm.func @tma_store_reduce_3d_im2col(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr
 
 // CHECK-LABEL: define void @tma_store_reduce_4d_tile(
 llvm.func @tma_store_reduce_4d_tile(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %d3 : i32, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.4d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i32 %[[D2:.*]], i32 %[[D3:.*]], i64 %[[CH:.*]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i32 %[[D2:.*]], i32 %[[D3:.*]], i64 %[[CH:.*]], /* red_op=add */ i32 0, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=min */ i32 1, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=max */ i32 2, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=inc */ i32 3, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=dec */ i32 4, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=and */ i32 5, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=or */ i32 6, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=xor */ i32 7, i1 true)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<add>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<min>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<max>} : !llvm.ptr, !llvm.ptr<3>
@@ -177,14 +177,14 @@ llvm.func @tma_store_reduce_4d_tile(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr,
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<or>}  : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<xor>} : !llvm.ptr, !llvm.ptr<3>
 
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=add */ i32 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=min */ i32 1, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=max */ i32 2, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=inc */ i32 3, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=dec */ i32 4, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=and */ i32 5, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=or */ i32 6, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=xor */ i32 7, i1 false)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] {redKind = #nvvm.tma_redux_kind<add>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] {redKind = #nvvm.tma_redux_kind<min>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] {redKind = #nvvm.tma_redux_kind<max>} : !llvm.ptr, !llvm.ptr<3>
@@ -198,14 +198,14 @@ llvm.func @tma_store_reduce_4d_tile(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr,
 
 // CHECK-LABEL: define void @tma_store_reduce_4d_im2col(
 llvm.func @tma_store_reduce_4d_im2col(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %d3 : i32, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.4d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i32 %[[D2:.*]], i32 %[[D3:.*]], i64 %[[CH:.*]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i32 %[[D2:.*]], i32 %[[D3:.*]], i64 %[[CH:.*]], /* red_op=add */ i32 0, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=min */ i32 1, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=max */ i32 2, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=inc */ i32 3, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=dec */ i32 4, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=and */ i32 5, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=or */ i32 6, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 %[[CH]], /* red_op=xor */ i32 7, i1 true)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<add>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<min>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<max>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
@@ -215,14 +215,14 @@ llvm.func @tma_store_reduce_4d_im2col(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<or>, mode = #nvvm.tma_store_mode<im2col>}  : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<xor>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
 
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=add */ i32 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=min */ i32 1, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=max */ i32 2, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=inc */ i32 3, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=dec */ i32 4, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=and */ i32 5, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=or */ i32 6, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.4d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i64 0, /* red_op=xor */ i32 7, i1 false)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] {redKind = #nvvm.tma_redux_kind<add>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] {redKind = #nvvm.tma_redux_kind<min>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3] {redKind = #nvvm.tma_redux_kind<max>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
@@ -238,14 +238,14 @@ llvm.func @tma_store_reduce_4d_im2col(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr
 
 // CHECK-LABEL: define void @tma_store_reduce_5d_tile(
 llvm.func @tma_store_reduce_5d_tile(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %d3 : i32, %d4 : i32, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.5d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i32 %[[D2:.*]], i32 %[[D3:.*]], i32 %[[D4:.*]], i64 %[[CH:.*]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i32 %[[D2:.*]], i32 %[[D3:.*]], i32 %[[D4:.*]], i64 %[[CH:.*]], /* red_op=add */ i32 0, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=min */ i32 1, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=max */ i32 2, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=inc */ i32 3, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=dec */ i32 4, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=and */ i32 5, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=or */ i32 6, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=xor */ i32 7, i1 true)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<add>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<min>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<max>} : !llvm.ptr, !llvm.ptr<3>
@@ -255,14 +255,14 @@ llvm.func @tma_store_reduce_5d_tile(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr,
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<or>}  : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<xor>} : !llvm.ptr, !llvm.ptr<3>
 
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=add */ i32 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=min */ i32 1, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=max */ i32 2, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=inc */ i32 3, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=dec */ i32 4, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=and */ i32 5, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=or */ i32 6, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.tile.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=xor */ i32 7, i1 false)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] {redKind = #nvvm.tma_redux_kind<add>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] {redKind = #nvvm.tma_redux_kind<min>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] {redKind = #nvvm.tma_redux_kind<max>} : !llvm.ptr, !llvm.ptr<3>
@@ -276,14 +276,14 @@ llvm.func @tma_store_reduce_5d_tile(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr,
 
 // CHECK-LABEL: define void @tma_store_reduce_5d_im2col(
 llvm.func @tma_store_reduce_5d_im2col(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %d3 : i32, %d4 : i32, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.5d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i32 %[[D2:.*]], i32 %[[D3:.*]], i32 %[[D4:.*]], i64 %[[CH:.*]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC:.*]], ptr %[[DST:.*]], i32 %[[D0:.*]], i32 %[[D1:.*]], i32 %[[D2:.*]], i32 %[[D3:.*]], i32 %[[D4:.*]], i64 %[[CH:.*]], /* red_op=add */ i32 0, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=min */ i32 1, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=max */ i32 2, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=inc */ i32 3, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=dec */ i32 4, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=and */ i32 5, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=or */ i32 6, i1 true)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 %[[CH]], /* red_op=xor */ i32 7, i1 true)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<add>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<min>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<max>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
@@ -293,14 +293,14 @@ llvm.func @tma_store_reduce_5d_im2col(%src : !llvm.ptr<3>, %tma_desc : !llvm.ptr
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<or>, mode = #nvvm.tma_store_mode<im2col>}  : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] l2_cache_hint = %ch {redKind = #nvvm.tma_redux_kind<xor>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
 
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.add.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.min.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.max.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.inc.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.dec.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.and.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.or.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.xor.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=add */ i32 0, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=min */ i32 1, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=max */ i32 2, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=inc */ i32 3, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=dec */ i32 4, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=and */ i32 5, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=or */ i32 6, i1 false)
+  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.reduce.im2col.5d(ptr addrspace(3) %[[SRC]], ptr %[[DST]], i32 %[[D0]], i32 %[[D1]], i32 %[[D2]], i32 %[[D3]], i32 %[[D4]], i64 0, /* red_op=xor */ i32 7, i1 false)
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] {redKind = #nvvm.tma_redux_kind<add>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] {redKind = #nvvm.tma_redux_kind<min>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>
   nvvm.cp.async.bulk.tensor.reduce %tma_desc, %src, box[%d0, %d1, %d2, %d3, %d4] {redKind = #nvvm.tma_redux_kind<max>, mode = #nvvm.tma_store_mode<im2col>} : !llvm.ptr, !llvm.ptr<3>