diff --git a/clang/docs/OpenMPSupport.md b/clang/docs/OpenMPSupport.md index ff9dd01978040..840cc5ef0fecd 100644 --- a/clang/docs/OpenMPSupport.md +++ b/clang/docs/OpenMPSupport.md @@ -134,7 +134,7 @@ implementation. | device | support close modifier on map clause | {good}`done` | [D55719][D55719],[D55892][D55892] | | device | teams construct on the host device | {good}`done` | r371553 | | device | support non-contiguous array sections for target update | {good}`done` | [PR144635][PR144635] | -| device | pointer attachment | {part}`being repaired` | @abhinavgaba ([PR153683][PR153683]) | +| device | pointer attachment | {part}`being repaired` | @abhinavgaba ([PR153683][PR153683], [PR210213][PR210213]) | | atomic | hints for the atomic construct | {good}`done` | [D51233][D51233] | | base language | C11 support | {good}`done` | | | base language | C++11/14/17 support | {good}`done` | | @@ -386,7 +386,7 @@ implementation. | dyn_groupprivate clause | {part}`partial` | {part}`In Progress` | C/C++: Host device support missing | | loop flatten transformation | {none}`unclaimed` | {none}`unclaimed` | | | loop grid/tile modifiers for sizes clause | {none}`unclaimed` | {none}`unclaimed` | | -| attach map-type modifier | {part}`In Progress` | {none}`unclaimed` | C/C++: @abhinavgaba; RT: @abhinavgaba ([PR149036][PR149036], [PR158370][PR158370]) | +| attach map-type modifier | {part}`In Progress` | {none}`unclaimed` | C/C++: @abhinavgaba; RT: @abhinavgaba ([PR149036][PR149036], [PR158370][PR158370], [PR210213][PR210213]) | | need_device_ptr modifier for adjust_args clause | {part}`partial` | {none}`unclaimed` | Clang Parsing/Sema: [PR168905][PR168905] [PR169558][PR169558] | | fallback modifier for use_device_ptr clause | {good}`done` | {none}`unclaimed` | Clang: @abhinavgaba ([PR170578][PR170578], [PR173931][PR173931]) RT: @abhinavgaba ([PR169603][PR169603]) | | dims modifier for num_teams, thread_limit, and num_threads clauses | {part}`partial` | {part}`In Progress` | C/C++: @kevinsala ([PR206412]); Fortran: @skc7, @kparzysz, @mjklemm | @@ -562,5 +562,6 @@ considered for standardization. Please post on the [PR194168]: https://github.com/llvm/llvm-project/pull/194168 [PR195829]: https://github.com/llvm/llvm-project/pull/195829 [PR196431]: https://github.com/llvm/llvm-project/pull/196431 +[PR210213]: https://github.com/llvm/llvm-project/pull/210213 [discourse forums (runtimes - openmp category)]: https://discourse.llvm.org/c/runtimes/openmp/35 diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md index 7108392abbaa1..321bd7cd24ac0 100644 --- a/clang/docs/ReleaseNotes.md +++ b/clang/docs/ReleaseNotes.md @@ -548,6 +548,9 @@ features cannot lower the translation-unit ABI level; `thread_limit` clauses for OpenMP 6.1 or later. - Map-type-modifying modifiers applied to a list item with a user-defined mapper are now propagated onto the maps the mapper expands to. +- Mapping of expressions with base-pointers through a user-defined mapper (e.g. + `map(s.p[0:n])`) now conforms to OpenMP's conditional pointer-attachment, + matching the behavior of such maps outside a mapper. ### SYCL Support diff --git a/clang/include/clang/Basic/TargetID.h b/clang/include/clang/Basic/TargetID.h index 8871b76859fd7..902151d76556d 100644 --- a/clang/include/clang/Basic/TargetID.h +++ b/clang/include/clang/Basic/TargetID.h @@ -9,29 +9,53 @@ #ifndef LLVM_CLANG_BASIC_TARGETID_H #define LLVM_CLANG_BASIC_TARGETID_H -#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" #include "llvm/TargetParser/Triple.h" #include -#include -#include +#include namespace clang { +/// Get all feature strings that can be used in target ID for \p Processor. +/// Target ID is a processor name with optional feature strings +/// postfixed by a plus or minus sign delimited by colons, e.g. +/// gfx908:xnack+:sramecc-. Each processor have a limited +/// number of predefined features when showing up in a target ID. +llvm::SmallVector +getAllPossibleTargetIDFeatures(const llvm::Triple &T, + llvm::StringRef Processor); + /// Get processor name from target ID. /// Returns canonical processor name or empty if the processor name is invalid. llvm::StringRef getProcessorFromTargetID(const llvm::Triple &T, llvm::StringRef OffloadArch); -/// A device triple paired with a target ID (processor and feature modifiers) -/// for that triple, e.g. {amdgcn-amd-amdhsa, "gfx906:xnack+"}. -using TargetIDEntry = std::pair; +/// Parse a target ID to get processor and feature map. +/// Returns canonicalized processor name or std::nullopt if the target ID is +/// invalid. Returns target ID features in \p FeatureMap if it is not null +/// pointer. This function assumes \p OffloadArch is a valid target ID. +/// If the target ID contains feature+, map it to true. +/// If the target ID contains feature-, map it to false. +/// If the target ID does not contain a feature (default), do not map it. +std::optional parseTargetID(const llvm::Triple &T, + llvm::StringRef OffloadArch, + llvm::StringMap *FeatureMap); + +/// Returns canonical target ID, assuming \p Processor is canonical and all +/// entries in \p Features are valid. +std::string getCanonicalTargetID(llvm::StringRef Processor, + const llvm::StringMap &Features); /// Get the conflicted pair of target IDs for a compilation or a bundled code -/// object. Two entries conflict when they resolve to the same processor but -/// disagree on whether a feature (xnack/sramecc) is explicitly specified. If -/// there is no conflict, returns std::nullopt. +/// object, assuming \p TargetIDs are canonicalized. If there is no conflicts, +/// returns std::nullopt. std::optional> -getConflictTargetIDCombination(llvm::ArrayRef Entries); +getConflictTargetIDCombination(const std::set &TargetIDs); + +/// Check whether the provided target ID is compatible with the requested +/// target ID. +bool isCompatibleTargetID(llvm::StringRef Provided, llvm::StringRef Requested); /// Sanitize a target ID string for use in a file name. /// Replaces invalid characters (like ':') with safe characters (like '@'). diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp index e72e7875f6937..01cfdeb3ef0fb 100644 --- a/clang/lib/AST/ByteCode/Compiler.cpp +++ b/clang/lib/AST/ByteCode/Compiler.cpp @@ -2629,7 +2629,7 @@ bool Compiler::visitCallArgs(ArrayRef Args, return false; } else { - DeclTy Source = Arg; + DeclOrExpr Source = Arg; if (FuncDecl) { // Try to use the parameter declaration instead of the argument // expression as a source. @@ -5313,37 +5313,37 @@ bool Compiler::emitConst(const APSInt &Value, const Expr *E) { } template -unsigned Compiler::allocateLocalPrimitive(DeclTy &&Src, PrimType Ty, - bool IsConst, +unsigned Compiler::allocateLocalPrimitive(DeclOrExpr &&Src, + PrimType Ty, bool IsConst, bool IsVolatile, ScopeKind SC) { - // FIXME: There are cases where Src.is() is wrong, e.g. + // FIXME: There are cases where Src.isExpr() is wrong, e.g. // (int){12} in C. Consider using Expr::isTemporaryObject() instead // or isa(). Descriptor *D = P.createDescriptor(Src, Ty, nullptr, Descriptor::InlineDescMD, - IsConst, isa(Src), + IsConst, Src.isExpr(), /*IsMutable=*/false, IsVolatile); D->IsConstexprUnknown = this->VariablesAreConstexprUnknown; Scope::Local Local = this->createLocal(D); - if (auto *VD = dyn_cast_if_present(Src.dyn_cast())) + if (auto *VD = Src.asValueDecl()) Locals.insert({VD, Local}); VarScope->addForScopeKind(Local, SC); return Local.Offset; } template -UnsignedOrNone Compiler::allocateLocal(DeclTy &&Src, QualType Ty, +UnsignedOrNone Compiler::allocateLocal(DeclOrExpr &&Src, QualType Ty, ScopeKind SC) { const ValueDecl *Key = nullptr; const Expr *Init = nullptr; bool IsTemporary = false; - if (auto *VD = dyn_cast_if_present(Src.dyn_cast())) { + if (auto *VD = Src.asValueDecl()) { Key = VD; if (const auto *VarD = dyn_cast(VD)) Init = VarD->getInit(); } - if (auto *E = Src.dyn_cast()) { + if (const auto *E = Src.asExpr()) { IsTemporary = true; if (Ty.isNull()) Ty = E->getType(); @@ -8704,7 +8704,7 @@ bool Compiler::emitDestructionPop(const Descriptor *Desc, /// Create a dummy pointer for the given decl (or expr) and /// push a pointer to it on the stack. template -bool Compiler::emitDummyPtr(const DeclTy &D, const Expr *E, bool CU) { +bool Compiler::emitDummyPtr(DeclOrExpr D, const Expr *E, bool CU) { assert(!DiscardResult && "Should've been checked before"); unsigned DummyID = P.getOrCreateDummy(D, CU); diff --git a/clang/lib/AST/ByteCode/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h index ebeffdd804d82..46d6b1a49c001 100644 --- a/clang/lib/AST/ByteCode/Compiler.h +++ b/clang/lib/AST/ByteCode/Compiler.h @@ -14,6 +14,7 @@ #define LLVM_CLANG_AST_INTERP_BYTECODEEXPRGEN_H #include "ByteCodeEmitter.h" +#include "DeclOrExpr.h" #include "EvalEmitter.h" #include "Pointer.h" #include "PrimType.h" @@ -339,12 +340,12 @@ class Compiler : public ConstStmtVisitor, bool>, bool Activate, bool IsOperatorCall); /// Creates a local primitive value. - unsigned allocateLocalPrimitive(DeclTy &&Decl, PrimType Ty, bool IsConst, + unsigned allocateLocalPrimitive(DeclOrExpr &&Decl, PrimType Ty, bool IsConst, bool IsVolatile = false, ScopeKind SC = ScopeKind::Block); /// Allocates a space storing a local given its type. - UnsignedOrNone allocateLocal(DeclTy &&Decl, QualType Ty = QualType(), + UnsignedOrNone allocateLocal(DeclOrExpr &&Decl, QualType Ty = QualType(), ScopeKind = ScopeKind::Block); UnsignedOrNone allocateTemporary(const Expr *E); @@ -427,7 +428,7 @@ class Compiler : public ConstStmtVisitor, bool>, const BinaryOperator *E); bool emitRecordDestructionPop(const Record *R, SourceInfo Loc); bool emitDestructionPop(const Descriptor *Desc, SourceInfo Loc); - bool emitDummyPtr(const DeclTy &D, const Expr *E, bool CU = false); + bool emitDummyPtr(DeclOrExpr D, const Expr *E, bool CU = false); bool emitFloat(const APFloat &F, SourceInfo Info); unsigned collectBaseOffset(const QualType BaseType, const QualType DerivedType); diff --git a/clang/lib/AST/ByteCode/DeclOrExpr.h b/clang/lib/AST/ByteCode/DeclOrExpr.h new file mode 100644 index 0000000000000..e170b52c6e51d --- /dev/null +++ b/clang/lib/AST/ByteCode/DeclOrExpr.h @@ -0,0 +1,65 @@ +//===------------------------- DeclOrExpr.h ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_AST_INTERP_DECLOREXPR_H +#define LLVM_CLANG_AST_INTERP_DECLOREXPR_H + +#include "clang/AST/Decl.h" +#include "clang/AST/Expr.h" +#include "clang/AST/TypeBase.h" +#include "llvm/ADT/PointerUnion.h" + +namespace clang { +namespace interp { + +struct DeclOrExpr { + llvm::PointerUnion V; + + DeclOrExpr() : V(nullptr) {} + DeclOrExpr(std::nullptr_t) : V(nullptr) {} + DeclOrExpr(const Decl *VD) : V(VD) {} + DeclOrExpr(const Expr *E) : V(E) {} + + bool isExpr() const { return isa_and_nonnull(V); } + bool isDecl() const { return isa_and_nonnull(V); } + bool isValueDecl() const { return isa_and_nonnull(asDecl()); } + + const Expr *asExpr() const { return V.dyn_cast(); } + const Decl *asDecl() const { return V.dyn_cast(); } + const ValueDecl *asValueDecl() const { + return dyn_cast_if_present(asDecl()); + } + const VarDecl *asVarDecl() const { + return dyn_cast_if_present(asDecl()); + } + + const void *getOpaqueValue() const { return V.getOpaqueValue(); } + + bool operator==(DeclOrExpr O) const { return O.V == V; } + bool operator!=(DeclOrExpr O) const { return O.V != V; } + explicit operator bool() const { return !V.isNull(); } + + QualType getType() const { + if (const auto *VD = asValueDecl()) + return VD->getType(); + return asExpr()->getType(); + } +}; +static_assert(sizeof(DeclOrExpr) == sizeof(void *)); + +inline DeclOrExpr getSwappedBytes(DeclOrExpr F) { return F; } + +inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, DeclOrExpr D) { + OS << D.getOpaqueValue(); + return OS; +} + +} // namespace interp +} // namespace clang + +#endif diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp index fb41c98dd68cb..3d661455460e3 100644 --- a/clang/lib/AST/ByteCode/Descriptor.cpp +++ b/clang/lib/AST/ByteCode/Descriptor.cpp @@ -280,7 +280,7 @@ static BlockDtorFn getDtorArrayPrim(PrimType Type) { } /// Primitives. -Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, PrimType Type, +Descriptor::Descriptor(DeclOrExpr D, const Type *SourceTy, PrimType Type, MetadataSize MD, bool IsConst, bool IsTemporary, bool IsMutable, bool IsVolatile) : Source(D), SourceType(SourceTy), ElemSize(primSize(Type)), Size(ElemSize), @@ -293,7 +293,7 @@ Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, PrimType Type, } /// Primitive arrays. -Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, PrimType Type, +Descriptor::Descriptor(DeclOrExpr D, const Type *SourceTy, PrimType Type, MetadataSize MD, size_t NumElems, bool IsConst, bool IsTemporary, bool IsMutable, bool IsVolatile) : Source(D), SourceType(SourceTy), ElemSize(primSize(Type)), @@ -307,7 +307,7 @@ Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, PrimType Type, } /// Primitive unknown-size arrays. -Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, +Descriptor::Descriptor(DeclOrExpr D, PrimType Type, MetadataSize MD, bool IsTemporary, bool IsConst, UnknownSize) : Source(D), ElemSize(primSize(Type)), Size(UnknownSizeMark), MDSize(MD.value_or(0)), @@ -319,7 +319,7 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, } /// Arrays of composite elements. -Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, +Descriptor::Descriptor(DeclOrExpr D, const Type *SourceTy, const Descriptor *Elem, MetadataSize MD, unsigned NumElems, bool IsConst, bool IsTemporary, bool IsMutable) @@ -334,7 +334,7 @@ Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, } /// Unknown-size arrays of composite elements. -Descriptor::Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD, +Descriptor::Descriptor(DeclOrExpr D, const Descriptor *Elem, MetadataSize MD, bool IsTemporary, UnknownSize) : Source(D), ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)), Size(UnknownSizeMark), MDSize(MD.value_or(0)), @@ -345,7 +345,7 @@ Descriptor::Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD, } /// Composite records. -Descriptor::Descriptor(const DeclTy &D, const Record *R, MetadataSize MD, +Descriptor::Descriptor(DeclOrExpr D, const Record *R, MetadataSize MD, bool IsConst, bool IsTemporary, bool IsMutable, bool IsVolatile) : Source(D), ElemSize(std::max(alignof(void *), R->getFullSize())), @@ -357,7 +357,7 @@ Descriptor::Descriptor(const DeclTy &D, const Record *R, MetadataSize MD, } /// Dummy. -Descriptor::Descriptor(const DeclTy &D, MetadataSize MD) +Descriptor::Descriptor(DeclOrExpr D, MetadataSize MD) : Source(D), ElemSize(1), Size(1), MDSize(MD.value_or(0)), AllocSize(MDSize), ElemRecord(nullptr), IsConst(true), IsMutable(false), IsTemporary(false) { @@ -470,17 +470,17 @@ QualType Descriptor::getDataType(const ASTContext &Ctx) const { } SourceLocation Descriptor::getLocation() const { - if (auto *D = dyn_cast(Source)) + if (auto *D = Source.asDecl()) return D->getLocation(); - if (auto *E = dyn_cast(Source)) + if (auto *E = Source.asExpr()) return E->getExprLoc(); llvm_unreachable("Invalid descriptor type"); } SourceInfo Descriptor::getLoc() const { - if (const auto *D = dyn_cast(Source)) + if (const auto *D = Source.asDecl()) return SourceInfo(D); - if (const auto *E = dyn_cast(Source)) + if (const auto *E = Source.asExpr()) return SourceInfo(E); llvm_unreachable("Invalid descriptor type"); } diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h index a2df48cf1e7fb..275e2aa594669 100644 --- a/clang/lib/AST/ByteCode/Descriptor.h +++ b/clang/lib/AST/ByteCode/Descriptor.h @@ -13,6 +13,7 @@ #ifndef LLVM_CLANG_AST_INTERP_DESCRIPTOR_H #define LLVM_CLANG_AST_INTERP_DESCRIPTOR_H +#include "DeclOrExpr.h" #include "InitMap.h" #include "PrimType.h" #include "clang/AST/Decl.h" @@ -26,8 +27,6 @@ class SourceInfo; struct Descriptor; enum PrimType : uint8_t; -using DeclTy = llvm::PointerUnion; - /// Invoked whenever a block is created. The constructor method fills in the /// inline descriptors of all fields and array elements. It also initializes /// all the fields which contain non-trivial types. @@ -123,7 +122,7 @@ static_assert(sizeof(GlobalInlineDescriptor) != sizeof(InlineDescriptor), ""); struct Descriptor final { private: /// Original declaration, used to emit the error message. - const DeclTy Source; + const DeclOrExpr Source; const Type *SourceType = nullptr; /// Size of an element, in host bytes. const unsigned ElemSize; @@ -174,34 +173,33 @@ struct Descriptor final { const BlockDtorFn DtorFn = nullptr; /// Allocates a descriptor for a primitive. - Descriptor(const DeclTy &D, const Type *SourceTy, PrimType Type, - MetadataSize MD, bool IsConst, bool IsTemporary, bool IsMutable, - bool IsVolatile); + Descriptor(DeclOrExpr D, const Type *SourceTy, PrimType Type, MetadataSize MD, + bool IsConst, bool IsTemporary, bool IsMutable, bool IsVolatile); /// Allocates a descriptor for an array of primitives. - Descriptor(const DeclTy &D, const Type *SourceTy, PrimType Type, - MetadataSize MD, size_t NumElems, bool IsConst, bool IsTemporary, - bool IsMutable, bool IsVolatile); + Descriptor(DeclOrExpr D, const Type *SourceTy, PrimType Type, MetadataSize MD, + size_t NumElems, bool IsConst, bool IsTemporary, bool IsMutable, + bool IsVolatile); /// Allocates a descriptor for an array of primitives of unknown size. - Descriptor(const DeclTy &D, PrimType Type, MetadataSize MDSize, bool IsConst, + Descriptor(DeclOrExpr D, PrimType Type, MetadataSize MDSize, bool IsConst, bool IsTemporary, UnknownSize); /// Allocates a descriptor for an array of composites. - Descriptor(const DeclTy &D, const Type *SourceTy, const Descriptor *Elem, + Descriptor(DeclOrExpr D, const Type *SourceTy, const Descriptor *Elem, MetadataSize MD, unsigned NumElems, bool IsConst, bool IsTemporary, bool IsMutable); /// Allocates a descriptor for an array of composites of unknown size. - Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD, + Descriptor(DeclOrExpr D, const Descriptor *Elem, MetadataSize MD, bool IsTemporary, UnknownSize); /// Allocates a descriptor for a record. - Descriptor(const DeclTy &D, const Record *R, MetadataSize MD, bool IsConst, + Descriptor(DeclOrExpr D, const Record *R, MetadataSize MD, bool IsConst, bool IsTemporary, bool IsMutable, bool IsVolatile); /// Allocates a dummy descriptor. - Descriptor(const DeclTy &D, MetadataSize MD = std::nullopt); + Descriptor(DeclOrExpr D, MetadataSize MD = std::nullopt); QualType getType() const; QualType getElemQualType() const; @@ -209,9 +207,9 @@ struct Descriptor final { SourceLocation getLocation() const; SourceInfo getLoc() const; - const Decl *asDecl() const { return dyn_cast(Source); } - const Expr *asExpr() const { return dyn_cast(Source); } - const DeclTy &getSource() const { return Source; } + const Decl *asDecl() const { return Source.asDecl(); } + const Expr *asExpr() const { return Source.asExpr(); } + DeclOrExpr getSource() const { return Source; } const ValueDecl *asValueDecl() const { return dyn_cast_if_present(asDecl()); diff --git a/clang/lib/AST/ByteCode/EvaluationResult.cpp b/clang/lib/AST/ByteCode/EvaluationResult.cpp index 0cd4bf837e56a..19a2744f73f15 100644 --- a/clang/lib/AST/ByteCode/EvaluationResult.cpp +++ b/clang/lib/AST/ByteCode/EvaluationResult.cpp @@ -148,9 +148,9 @@ bool EvaluationResult::checkFullyInitialized(InterpState &S, return true; SourceLocation InitLoc; - if (const auto *D = dyn_cast(Source)) + if (const auto *D = Source.asDecl()) InitLoc = cast(D)->getAnyInitializer()->getExprLoc(); - else if (const auto *E = dyn_cast(Source)) + else if (const auto *E = Source.asExpr()) InitLoc = E->getExprLoc(); if (const Record *R = Ptr.getRecord()) diff --git a/clang/lib/AST/ByteCode/EvaluationResult.h b/clang/lib/AST/ByteCode/EvaluationResult.h index 381600955440d..1edd9bc55406a 100644 --- a/clang/lib/AST/ByteCode/EvaluationResult.h +++ b/clang/lib/AST/ByteCode/EvaluationResult.h @@ -9,6 +9,7 @@ #ifndef LLVM_CLANG_AST_INTERP_EVALUATION_RESULT_H #define LLVM_CLANG_AST_INTERP_EVALUATION_RESULT_H +#include "DeclOrExpr.h" #include "clang/AST/APValue.h" #include "clang/AST/Decl.h" #include "clang/AST/Expr.h" @@ -36,17 +37,15 @@ class EvaluationResult final { Valid, // Result is valid and empty. }; - using DeclTy = llvm::PointerUnion; - private: #ifndef NDEBUG const Context *Ctx = nullptr; #endif APValue Value; ResultKind Kind = Empty; - DeclTy Source = nullptr; + DeclOrExpr Source = nullptr; - void setSource(DeclTy D) { Source = D; } + void setSource(DeclOrExpr D) { Source = D; } void takeValue(APValue &&V) { assert(empty()); @@ -85,10 +84,9 @@ class EvaluationResult final { const Pointer &Ptr, SourceInfo Info); QualType getSourceType() const { - if (const auto *D = - dyn_cast_if_present(Source.dyn_cast())) + if (const auto *D = Source.asValueDecl()) return D->getType(); - if (const auto *E = Source.dyn_cast()) + if (const auto *E = Source.asExpr()) return E->getType(); return QualType(); } diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h index c06347318dafa..a11031fb7a240 100644 --- a/clang/lib/AST/ByteCode/Pointer.h +++ b/clang/lib/AST/ByteCode/Pointer.h @@ -544,15 +544,15 @@ class Pointer { SourceLocation getDeclLoc() const { return getDeclDesc()->getLocation(); } /// Returns the expression or declaration the pointer has been created for. - DeclTy getSource() const { + DeclOrExpr getSource() const { if (isBlockPointer()) return getDeclDesc()->getSource(); if (isFunctionPointer()) { const Function *F = Fn.Func; - return F ? F->getDecl() : DeclTy(); + return F ? F->getDecl() : DeclOrExpr(); } llvm_unreachable("Unsupported pointer type in getSource()"); - return DeclTy(); + return DeclOrExpr(); } /// Returns a pointer to the object of which this pointer is a field. diff --git a/clang/lib/AST/ByteCode/Program.cpp b/clang/lib/AST/ByteCode/Program.cpp index 378a190184be9..564d2d8fc422d 100644 --- a/clang/lib/AST/ByteCode/Program.cpp +++ b/clang/lib/AST/ByteCode/Program.cpp @@ -124,10 +124,10 @@ UnsignedOrNone Program::getOrCreateGlobal(const ValueDecl *VD, return std::nullopt; } -unsigned Program::getOrCreateDummy(DeclTy D, bool IsConstexprUnknown) { +unsigned Program::getOrCreateDummy(DeclOrExpr D, bool IsConstexprUnknown) { assert(D); - if (const auto *VD = dyn_cast_if_present(dyn_cast(D))) + if (const auto *VD = D.asVarDecl()) D = VD->getFirstDecl(); // Dedup blocks since they are immutable and pointers cannot be compared. @@ -137,10 +137,10 @@ unsigned Program::getOrCreateDummy(DeclTy D, bool IsConstexprUnknown) { QualType QT; bool IsWeak = false; - if (const auto *E = dyn_cast(D)) { + if (const auto *E = D.asExpr()) { QT = E->getType(); } else { - const auto *VD = cast(cast(D)); + const auto *VD = D.asValueDecl(); IsWeak = VD->isWeak(); QT = VD->getType(); if (QT->isPointerOrReferenceType()) @@ -250,8 +250,8 @@ UnsignedOrNone Program::createGlobal(const Expr *E, QualType ExprType) { return std::nullopt; } -UnsignedOrNone Program::createGlobal(const DeclTy &D, QualType Ty, - bool IsStatic, bool IsExtern, bool IsWeak, +UnsignedOrNone Program::createGlobal(DeclOrExpr D, QualType Ty, bool IsStatic, + bool IsExtern, bool IsWeak, bool IsConstexprUnknown, const Expr *Init) { // Since this global variable is constexpr-unknown and a reference, register @@ -263,7 +263,7 @@ UnsignedOrNone Program::createGlobal(const DeclTy &D, QualType Ty, // Create a descriptor for the global. Descriptor *Desc; const bool IsConst = Ty.isConstQualified(); - const bool IsTemporary = D.dyn_cast(); + const bool IsTemporary = D.isExpr(); const bool IsVolatile = Ty.isVolatileQualified(); if (OptPrimType T = Ctx.classify(Ty)) Desc = createDescriptor(D, *T, nullptr, Descriptor::GlobalMD, IsConst, @@ -412,7 +412,7 @@ Record *Program::getOrCreateRecord(const RecordDecl *RD) { return R; } -Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty, +Descriptor *Program::createDescriptor(DeclOrExpr D, const Type *Ty, Descriptor::MetadataSize MDSize, bool IsConst, bool IsTemporary, bool IsMutable, bool IsVolatile, diff --git a/clang/lib/AST/ByteCode/Program.h b/clang/lib/AST/ByteCode/Program.h index b0ef993258637..7687f437ca680 100644 --- a/clang/lib/AST/ByteCode/Program.h +++ b/clang/lib/AST/ByteCode/Program.h @@ -13,6 +13,7 @@ #ifndef LLVM_CLANG_AST_INTERP_PROGRAM_H #define LLVM_CLANG_AST_INTERP_PROGRAM_H +#include "DeclOrExpr.h" #include "Function.h" #include "Pointer.h" #include "PrimType.h" @@ -88,7 +89,7 @@ class Program final { const Expr *Init = nullptr); /// Returns or creates a dummy value for unknown declarations. - unsigned getOrCreateDummy(DeclTy D, bool IsConstexprUnknown = false); + unsigned getOrCreateDummy(DeclOrExpr D, bool IsConstexprUnknown = false); /// Creates a global and returns its index. UnsignedOrNone createGlobal(const ValueDecl *VD, const Expr *Init, @@ -119,7 +120,7 @@ class Program final { Record *getOrCreateRecord(const RecordDecl *RD); /// Creates a descriptor for a primitive type. - Descriptor *createDescriptor(const DeclTy &D, PrimType T, + Descriptor *createDescriptor(DeclOrExpr D, PrimType T, const Type *SourceTy = nullptr, Descriptor::MetadataSize MDSize = std::nullopt, bool IsConst = false, bool IsTemporary = false, @@ -130,7 +131,7 @@ class Program final { } /// Creates a descriptor for a composite type. - Descriptor *createDescriptor(const DeclTy &D, const Type *Ty, + Descriptor *createDescriptor(DeclOrExpr D, const Type *Ty, Descriptor::MetadataSize MDSize = std::nullopt, bool IsConst = false, bool IsTemporary = false, bool IsMutable = false, bool IsVolatile = false, @@ -168,7 +169,7 @@ class Program final { private: friend class DeclScope; - UnsignedOrNone createGlobal(const DeclTy &D, QualType Ty, bool IsStatic, + UnsignedOrNone createGlobal(DeclOrExpr D, QualType Ty, bool IsStatic, bool IsExtern, bool IsWeak, bool IsConstexprUnknown, const Expr *Init = nullptr); diff --git a/clang/lib/Basic/TargetID.cpp b/clang/lib/Basic/TargetID.cpp index d2e1228897ceb..29d5d4a5d2996 100644 --- a/clang/lib/Basic/TargetID.cpp +++ b/clang/lib/Basic/TargetID.cpp @@ -7,52 +7,188 @@ //===----------------------------------------------------------------------===// #include "clang/Basic/TargetID.h" -#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Support/Path.h" #include "llvm/TargetParser/AMDGPUTargetParser.h" +#include "llvm/TargetParser/Triple.h" +#include +#include +#include namespace clang { -llvm::StringRef getProcessorFromTargetID(const llvm::Triple &T, - llvm::StringRef OffloadArch) { - auto Split = OffloadArch.split(':'); +static llvm::SmallVector +getAllPossibleAMDGPUTargetIDFeatures(const llvm::Triple &T, + llvm::StringRef Proc) { + // Entries in returned vector should be in alphabetical order. + llvm::SmallVector Ret; + if (!T.isAMDGCN()) + return Ret; + llvm::AMDGPU::GPUKind ProcKind = llvm::AMDGPU::parseArchAMDGCN(Proc); + if (ProcKind == llvm::AMDGPU::GK_NONE) + return Ret; + unsigned Features = llvm::AMDGPU::getArchAttrAMDGCN(ProcKind); + if (Features & llvm::AMDGPU::FEATURE_SRAMECC) + Ret.push_back("sramecc"); + // Only allow xnack in target ID if the processor supports on/off modes. + if (Features & llvm::AMDGPU::FEATURE_XNACK_ON_OFF_MODES) + Ret.push_back("xnack"); + return Ret; +} + +llvm::SmallVector +getAllPossibleTargetIDFeatures(const llvm::Triple &T, + llvm::StringRef Processor) { + llvm::SmallVector Ret; + if (T.isAMDGPU()) + return getAllPossibleAMDGPUTargetIDFeatures(T, Processor); + return Ret; +} + +/// Returns canonical processor name or empty string if \p Processor is invalid. +static llvm::StringRef getCanonicalProcessorName(const llvm::Triple &T, + llvm::StringRef Processor) { if (T.isAMDGPU()) - return llvm::AMDGPU::getCanonicalArchName(T, Split.first); - return Split.first; + return llvm::AMDGPU::getCanonicalArchName(T, Processor); + return Processor; +} + +llvm::StringRef getProcessorFromTargetID(const llvm::Triple &T, + llvm::StringRef TargetID) { + auto Split = TargetID.split(':'); + return getCanonicalProcessorName(T, Split.first); +} + +// Parse a target ID with format checking only. Do not check whether processor +// name or features are valid for the processor. +// +// A target ID is a processor name followed by a list of target features +// delimited by colon. Each target feature is a string post-fixed by a plus +// or minus sign, e.g. gfx908:sramecc+:xnack-. +static std::optional +parseTargetIDWithFormatCheckingOnly(llvm::StringRef TargetID, + llvm::StringMap *FeatureMap) { + llvm::StringRef Processor; + + if (TargetID.empty()) + return llvm::StringRef(); + + auto Split = TargetID.split(':'); + Processor = Split.first; + if (Processor.empty()) + return std::nullopt; + + auto Features = Split.second; + if (Features.empty()) + return Processor; + + llvm::StringMap LocalFeatureMap; + if (!FeatureMap) + FeatureMap = &LocalFeatureMap; + + while (!Features.empty()) { + auto Splits = Features.split(':'); + if (Splits.first.empty()) + return std::nullopt; + auto Sign = Splits.first.back(); + auto Feature = Splits.first.drop_back(); + if (Sign != '+' && Sign != '-') + return std::nullopt; + bool IsOn = Sign == '+'; + // Each feature can only show up at most once in target ID. + if (!FeatureMap->try_emplace(Feature, IsOn).second) + return std::nullopt; + Features = Splits.second; + } + return Processor; +} + +std::optional +parseTargetID(const llvm::Triple &T, llvm::StringRef TargetID, + llvm::StringMap *FeatureMap) { + auto OptionalProcessor = + parseTargetIDWithFormatCheckingOnly(TargetID, FeatureMap); + + if (!OptionalProcessor) + return std::nullopt; + + llvm::StringRef Processor = getCanonicalProcessorName(T, *OptionalProcessor); + if (Processor.empty()) + return std::nullopt; + + llvm::SmallSet AllFeatures( + llvm::from_range, getAllPossibleTargetIDFeatures(T, Processor)); + + for (auto &&F : *FeatureMap) + if (!AllFeatures.count(F.first())) + return std::nullopt; + + return Processor; +} + +// A canonical target ID is a target ID containing a canonical processor name +// and features in alphabetical order. +std::string getCanonicalTargetID(llvm::StringRef Processor, + const llvm::StringMap &Features) { + std::string TargetID = Processor.str(); + std::map OrderedMap; + for (const auto &F : Features) + OrderedMap[F.first()] = F.second; + for (const auto &F : OrderedMap) + TargetID = TargetID + ':' + F.first.str() + (F.second ? "+" : "-"); + return TargetID; } // For a specific processor, a feature either shows up in all target IDs, or -// does not show up in any target IDs. Otherwise the target ID combination is -// invalid. +// does not show up in any target IDs. Otherwise the target ID combination +// is invalid. std::optional> -getConflictTargetIDCombination(llvm::ArrayRef Entries) { +getConflictTargetIDCombination(const std::set &TargetIDs) { struct Info { llvm::StringRef TargetID; - bool HasXnack; - bool HasSramEcc; + llvm::StringMap Features; + Info(llvm::StringRef TargetID, const llvm::StringMap &Features) + : TargetID(TargetID), Features(Features) {} }; - - llvm::SmallDenseMap Seen; - for (const auto &[T, ID] : Entries) { - std::optional Parsed = - llvm::AMDGPU::TargetID::parse(T, ID); - if (!Parsed) - continue; - - // A feature is present in a target ID only when an explicit '+'/'-' - // modifier is given, not when it is left unspecified. - Info Cur{ID, Parsed->isXnackOnOrOff(), Parsed->isSramEccOnOrOff()}; - auto [Loc, Inserted] = Seen.try_emplace(Parsed->getGPUKind(), Cur); - if (Inserted) - continue; - - const Info &Prev = Loc->second; - if (Cur.HasXnack != Prev.HasXnack || Cur.HasSramEcc != Prev.HasSramEcc) - return std::make_pair(Prev.TargetID, ID); + llvm::StringMap FeatureMap; + for (auto &&ID : TargetIDs) { + llvm::StringMap Features; + llvm::StringRef Proc = *parseTargetIDWithFormatCheckingOnly(ID, &Features); + auto [Loc, Inserted] = FeatureMap.try_emplace(Proc, ID, Features); + if (!Inserted) { + auto &ExistingFeatures = Loc->second.Features; + if (llvm::any_of(Features, [&](auto &F) { + return ExistingFeatures.count(F.first()) == 0; + })) + return std::make_pair(Loc->second.TargetID, ID); + } } return std::nullopt; } +bool isCompatibleTargetID(llvm::StringRef Provided, llvm::StringRef Requested) { + llvm::StringMap ProvidedFeatures, RequestedFeatures; + llvm::StringRef ProvidedProc = + *parseTargetIDWithFormatCheckingOnly(Provided, &ProvidedFeatures); + llvm::StringRef RequestedProc = + *parseTargetIDWithFormatCheckingOnly(Requested, &RequestedFeatures); + if (ProvidedProc != RequestedProc) + return false; + for (const auto &F : ProvidedFeatures) { + auto Loc = RequestedFeatures.find(F.first()); + // The default (unspecified) value of a feature is 'All', which can match + // either 'On' or 'Off'. + if (Loc == RequestedFeatures.end()) + return false; + // If a feature is specified, it must have exact match. + if (Loc->second != F.second) + return false; + } + return true; +} + std::string sanitizeTargetIDInFileName(llvm::StringRef TargetID) { std::string FileName = TargetID.str(); if (llvm::sys::path::is_style_windows(llvm::sys::path::Style::native)) diff --git a/clang/lib/Basic/Targets/AMDGPU.cpp b/clang/lib/Basic/Targets/AMDGPU.cpp index c6efd02d912e9..4109066ec910e 100644 --- a/clang/lib/Basic/Targets/AMDGPU.cpp +++ b/clang/lib/Basic/Targets/AMDGPU.cpp @@ -236,17 +236,13 @@ AMDGPUTargetInfo::AMDGPUTargetInfo(const llvm::Triple &Triple, HalfArgsAndReturns = true; if (Opts.AMDGPUXnackState != TargetOptions::AMDGPUFeatureState::Any) { - XnackSetting = - Opts.AMDGPUXnackState == TargetOptions::AMDGPUFeatureState::Enabled - ? llvm::AMDGPU::TargetIDSetting::On - : llvm::AMDGPU::TargetIDSetting::Off; + OffloadArchFeatures["xnack"] = + Opts.AMDGPUXnackState == TargetOptions::AMDGPUFeatureState::Enabled; } if (Opts.AMDGPUSramEccState != TargetOptions::AMDGPUFeatureState::Any) { - SramEccSetting = - Opts.AMDGPUSramEccState == TargetOptions::AMDGPUFeatureState::Enabled - ? llvm::AMDGPU::TargetIDSetting::On - : llvm::AMDGPU::TargetIDSetting::Off; + OffloadArchFeatures["sramecc"] = + Opts.AMDGPUSramEccState == TargetOptions::AMDGPUFeatureState::Enabled; } } @@ -311,25 +307,22 @@ void AMDGPUTargetInfo::getTargetDefines(const LangOptions &Opts, Twine("__")); Builder.defineMacro("__amdgcn_processor__", Twine("\"") + Twine(CanonName) + Twine("\"")); - llvm::AMDGPU::TargetID TargetID(GPUKind, getTriple(), XnackSetting, - SramEccSetting); - Builder.defineMacro("__amdgcn_target_id__", - Twine("\"") + - Twine(TargetID.getCanonicalTargetIDString()) + - Twine("\"")); - auto DefineFeatureMacro = [&](StringRef Feature, - llvm::AMDGPU::TargetIDSetting Setting) { - if (Setting == llvm::AMDGPU::TargetIDSetting::On || - Setting == llvm::AMDGPU::TargetIDSetting::Off) { - std::string NewF = Feature.str(); + Builder.defineMacro( + "__amdgcn_target_id__", + Twine("\"") + + Twine(getCanonicalTargetID(getArchNameAMDGCN(GPUKind), + OffloadArchFeatures)) + + Twine("\"")); + for (auto F : getAllPossibleTargetIDFeatures(getTriple(), CanonName)) { + auto Loc = OffloadArchFeatures.find(F); + if (Loc != OffloadArchFeatures.end()) { + std::string NewF = F.str(); llvm::replace(NewF, '-', '_'); - Builder.defineMacro( - Twine("__amdgcn_feature_") + Twine(NewF) + Twine("__"), - Setting == llvm::AMDGPU::TargetIDSetting::On ? "1" : "0"); + Builder.defineMacro(Twine("__amdgcn_feature_") + Twine(NewF) + + Twine("__"), + Loc->second ? "1" : "0"); } - }; - DefineFeatureMacro("xnack", XnackSetting); - DefineFeatureMacro("sramecc", SramEccSetting); + } } if (Opts.AtomicIgnoreDenormalMode) diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h index 61c87456f581f..f8933ebee8ffd 100644 --- a/clang/lib/Basic/Targets/AMDGPU.h +++ b/clang/lib/Basic/Targets/AMDGPU.h @@ -42,13 +42,13 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo { /// Whether having image instructions. bool HasImage = false; - /// Explicit xnack/sramecc target-id feature settings from the command line, - /// e.g. gfx908:xnack+:sramecc-. "Unsupported" means the feature was not - /// specified (or is not a valid target-id modifier for the processor). - llvm::AMDGPU::TargetIDSetting XnackSetting = - llvm::AMDGPU::TargetIDSetting::Unsupported; - llvm::AMDGPU::TargetIDSetting SramEccSetting = - llvm::AMDGPU::TargetIDSetting::Unsupported; + /// Target ID is device name followed by optional feature name postfixed + /// by plus or minus sign delimitted by colon, e.g. gfx908:xnack+:sramecc-. + /// If the target ID contains feature+, map it to true. + /// If the target ID contains feature-, map it to false. + /// If the target ID does not contain a feature (default), do not map it. + llvm::StringMap OffloadArchFeatures; + std::string TargetID; bool hasFP64() const { return getTriple().isAMDGCN(); } @@ -461,7 +461,8 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo { bool handleTargetFeatures(std::vector &Features, DiagnosticsEngine &Diags) override { HasFullBFloat16 = true; - unsigned ArchAttr = llvm::AMDGPU::getArchAttrAMDGCN(GPUKind); + auto TargetIDFeatures = + getAllPossibleTargetIDFeatures(getTriple(), getArchNameAMDGCN(GPUKind)); for (const auto &F : Features) { assert(F.front() == '+' || F.front() == '-'); if (F == "+wavefrontsize64") @@ -472,17 +473,12 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo { CUMode = false; else if (F == "+image-insts") HasImage = true; - llvm::AMDGPU::TargetIDSetting Setting = - F.front() == '+' ? llvm::AMDGPU::TargetIDSetting::On - : llvm::AMDGPU::TargetIDSetting::Off; + bool IsOn = F.front() == '+'; StringRef Name = StringRef(F).drop_front(); - // xnack is a valid target-id modifier only when the processor supports - // on/off modes; sramecc when the processor supports sramecc. - if (Name == "xnack" && - (ArchAttr & llvm::AMDGPU::FEATURE_XNACK_ON_OFF_MODES)) - XnackSetting = Setting; - else if (Name == "sramecc" && (ArchAttr & llvm::AMDGPU::FEATURE_SRAMECC)) - SramEccSetting = Setting; + if (!llvm::is_contained(TargetIDFeatures, Name)) + continue; + assert(!OffloadArchFeatures.contains(Name)); + OffloadArchFeatures[Name] = IsOn; } return true; } diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 9e71e82e5ee86..6edabd4e42d88 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -7605,6 +7605,8 @@ class MappableExprsHandler { AttachInfo.AttachPteeAddr.emitRawPointer(CGF)); CombinedInfo.Sizes.push_back(PointerSize); CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_ATTACH); + // ATTACH entries themselves don't "have" a base attach-ptr. + CombinedInfo.HasAttachPtr.push_back(false); CombinedInfo.Mappers.push_back(nullptr); CombinedInfo.NonContigInfo.Dims.push_back(1); } @@ -7706,6 +7708,7 @@ class MappableExprsHandler { CombinedInfo.Sizes.push_back( CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/false)); CombinedInfo.Types.push_back(Flags); + CombinedInfo.HasAttachPtr.push_back(false); CombinedInfo.Mappers.push_back(nullptr); CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize : 1); } @@ -8393,10 +8396,14 @@ class MappableExprsHandler { } } - if (!IsMappingWholeStruct) + if (!IsMappingWholeStruct) { CombinedInfo.Types.push_back(Flags); - else + // HasAttachPtr marks pointee entries, which have a base attach-ptr. + CombinedInfo.HasAttachPtr.push_back(HasAttachPtr); + } else { StructBaseCombinedInfo.Types.push_back(Flags); + StructBaseCombinedInfo.HasAttachPtr.push_back(HasAttachPtr); + } } // If we have encountered a member expression so far, keep track of the @@ -8997,6 +9004,7 @@ class MappableExprsHandler { if (HasUdpFbNullify) Flags |= OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY; UseDeviceDataCombinedInfo.Types.push_back(Flags); + UseDeviceDataCombinedInfo.HasAttachPtr.push_back(false); UseDeviceDataCombinedInfo.Mappers.push_back(nullptr); }; @@ -9404,7 +9412,28 @@ class MappableExprsHandler { /// Constructor for the declare mapper directive. MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF) - : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {} + : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) { + auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) { + for (auto L : C->component_lists()) { + OMPClauseMappableExprCommon::MappableExprComponentListRef Components = + std::get<1>(L); + if (!Components.empty()) + collectAttachPtrExprInfo(Components, CurDir); + } + }; + + // Populate the AttachPtrExprMap for all component lists from map-related + // clauses in the declare mapper directive, to enable attach-style mapping + // for mappers. + for (const auto *Cl : Dir.clauses()) { + if (const auto *C = dyn_cast(Cl)) + CollectAttachPtrExprsForClauseComponents(C); + else if (const auto *C = dyn_cast(Cl)) + CollectAttachPtrExprsForClauseComponents(C); + else if (const auto *C = dyn_cast(Cl)) + CollectAttachPtrExprsForClauseComponents(C); + } + } /// Generate code for the combined entry if we have a partially mapped struct /// and take care of the mapping flags of the arguments corresponding to @@ -9478,6 +9507,14 @@ class MappableExprsHandler { : !PartialStruct.PreliminaryMapData.BasePointers.empty() ? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM); + // A combined entry has a base attach-ptr if its constituents do. e.g.: + // map(s2.s1p->x, s2.s1p->y) + // combined entry: + // s2.s1p[0], s2.s1p->x, sizeof(s1p->x..y), ALLOC + // here s2.s1p is the attach-ptr for the combined entry. + // See the inline comments in emitUserDefinedMapper's definition for how + // entries with an attach-ptr are treated. + CombinedInfo.HasAttachPtr.push_back(AttachInfo.isValid()); // If any element has the present modifier, then make sure the runtime // doesn't attempt to allocate the struct. if (CurTypes.end() != @@ -9593,6 +9630,7 @@ class MappableExprsHandler { OpenMPOffloadMappingFlags::OMP_MAP_LITERAL | OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF | OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT); + CombinedInfo.HasAttachPtr.push_back(false); CombinedInfo.Mappers.push_back(nullptr); } for (const LambdaCapture &LC : RD->captures()) { @@ -9633,6 +9671,7 @@ class MappableExprsHandler { OpenMPOffloadMappingFlags::OMP_MAP_LITERAL | OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF | OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT); + CombinedInfo.HasAttachPtr.push_back(false); CombinedInfo.Mappers.push_back(nullptr); } } @@ -9925,6 +9964,7 @@ class MappableExprsHandler { CurCaptureVarInfo.Types.push_back( OpenMPOffloadMappingFlags::OMP_MAP_LITERAL | OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM); + CurCaptureVarInfo.HasAttachPtr.push_back(false); CurCaptureVarInfo.Mappers.push_back(nullptr); return; } @@ -10323,6 +10363,7 @@ class MappableExprsHandler { if (IsImplicit) CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT; + CombinedInfo.HasAttachPtr.push_back(false); // No user-defined mapper for default mapping. CombinedInfo.Mappers.push_back(nullptr); } @@ -10539,14 +10580,31 @@ getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) { /// size*sizeof(Ty), clearToFromMember(type)); /// // Map members. /// for (unsigned i = 0; i < size; i++) { +/// N = __tgt_mapper_num_components(rt_mapper_handle); /// // For each component specified by this mapper: /// for (auto c : begin[i]->all_components) { +/// // MEMBER_OF grouping: tie this component to the current array element +/// // (component N) by adding N<<48. Exceptions: +/// // - ATTACH entries are not members of any struct storage range. +/// // - Pointee entries (reached via a pointer member) occupy separate +/// // storage; their inner MEMBER_OF bits are shifted by N instead. +/// if (c.isAttach() || c.isPointee()) +/// member_type = c.arg_type + (c.hasInnerMemberOf() ? N<<48 : 0); +/// else +/// member_type = c.arg_type + N<<48; +/// // Map-type-modifying bits (ALWAYS, DELETE, CLOSE) from the outer map +/// // clause are propagated to each component, except ATTACH entries +/// // (ATTACH|ALWAYS is reserved for attach(always), and other modifier +/// // bits have no meaning for ATTACH). PRESENT is handled separately. +/// imported_modifier_bits = type & (ALWAYS | DELETE | CLOSE); +/// effective_type = c.isAttach() ? member_type +/// : member_type | imported_modifier_bits; /// if (c.hasMapper()) /// (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size, -/// c.arg_type, c.arg_name); +/// effective_type, c.arg_name); /// else /// __tgt_push_mapper_component(rt_mapper_handle, c.arg_base, -/// c.arg_begin, c.arg_size, c.arg_type, +/// c.arg_begin, c.arg_size, effective_type, /// c.arg_name); /// } /// } @@ -10762,6 +10820,7 @@ static void genMapInfoForCaptures( CurInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_LITERAL | OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM | OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT); + CurInfo.HasAttachPtr.push_back(false); CurInfo.Mappers.push_back(nullptr); } else { const ValueDecl *CapturedVD = @@ -10919,6 +10978,7 @@ static void emitTargetCallKernelLaunch( CombinedInfo.Sizes.push_back(CGF.Builder.getInt64(0)); CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM | OpenMPOffloadMappingFlags::OMP_MAP_LITERAL); + CombinedInfo.HasAttachPtr.push_back(false); if (!CombinedInfo.Names.empty()) CombinedInfo.Names.push_back(NullPtr); CombinedInfo.Exprs.push_back(nullptr); diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp index 5c5fefe32c06c..9ff0c37ca77fc 100644 --- a/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -3441,7 +3441,8 @@ LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, llvm::Value *Val = CGF.CGM.GetAddrOfGlobalVar(VD); llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Val); - llvm::CallInst *CallVal = CGF.Builder.CreateCall(Wrapper); + llvm::CallInst *CallVal = + CGF.Builder.CreateCall(Wrapper, {}, CGF.getBundlesForFunclet(Wrapper)); CallVal->setCallingConv(Wrapper->getCallingConv()); LValue LV; diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index b4f114b5b9f7a..5bd46db170d96 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -108,7 +108,6 @@ #include "llvm/Support/TarWriter.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/TargetParser/AMDGPUTargetParser.h" #include "llvm/TargetParser/Host.h" #include "llvm/TargetParser/RISCVISAInfo.h" #include // ::getenv @@ -4896,17 +4895,14 @@ static StringRef getCanonicalArchString(Compilation &C, if (IsNVIDIAOffloadArch(Arch)) return Args.MakeArgStringRef(OffloadArchToString(Arch)); - // AMDGCN target IDs carry a processor and xnack/sramecc modifiers to - // canonicalize. Other AMD offload arches (e.g. the amdgcnspirv pseudo-arch on - // a SPIR-V triple) have no target-id features and pass through unchanged. - if (IsAMDOffloadArch(Arch) && Triple.isAMDGCN()) { - std::optional ID = - llvm::AMDGPU::TargetID::parse(Triple, ArchStr); - if (!ID) { + if (IsAMDOffloadArch(Arch)) { + llvm::StringMap Features; + std::optional Arch = parseTargetID(Triple, ArchStr, &Features); + if (!Arch) { C.getDriver().Diag(clang::diag::err_drv_bad_target_id) << ArchStr; return StringRef(); } - return Args.MakeArgStringRef(ID->getCanonicalTargetIDString()); + return Args.MakeArgStringRef(getCanonicalTargetID(*Arch, Features)); } // If the input isn't CUDA or HIP just return the architecture. @@ -4917,18 +4913,13 @@ static StringRef getCanonicalArchString(Compilation &C, /// incompatible pair if a conflict occurs. static std::optional> getConflictOffloadArchCombination(const llvm::DenseSet &Archs, - const llvm::Triple &Triple) { + llvm::Triple Triple) { if (!Triple.isAMDGPU()) return std::nullopt; - // Sort for a deterministic conflicting pair in the diagnostic. - llvm::SmallVector ArchList(Archs.begin(), Archs.end()); - llvm::sort(ArchList); - - llvm::SmallVector Entries; - for (StringRef Arch : ArchList) - Entries.emplace_back(Triple, Arch); - return getConflictTargetIDCombination(Entries); + std::set ArchSet; + llvm::copy(Archs, std::inserter(ArchSet, ArchSet.begin())); + return getConflictTargetIDCombination(ArchSet); } llvm::SmallVector diff --git a/clang/lib/Driver/OffloadBundler.cpp b/clang/lib/Driver/OffloadBundler.cpp index a356b5f84c51e..b397ee4c5b075 100644 --- a/clang/lib/Driver/OffloadBundler.cpp +++ b/clang/lib/Driver/OffloadBundler.cpp @@ -48,7 +48,6 @@ #include "llvm/Support/Timer.h" #include "llvm/Support/WithColor.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/TargetParser/AMDGPUTargetParser.h" #include "llvm/TargetParser/Host.h" #include "llvm/TargetParser/Triple.h" #include @@ -1115,15 +1114,15 @@ bool isCodeObjectCompatible(const OffloadTargetInfo &CodeObjectInfo, } // Incompatible if Processors mismatch. - std::optional CodeObjectID = - llvm::AMDGPU::TargetID::parse(CodeObjectInfo.Triple, - CodeObjectInfo.TargetID); - std::optional TargetID = - llvm::AMDGPU::TargetID::parse(TargetInfo.Triple, TargetInfo.TargetID); - - // Both target IDs must be valid and name the same processor. - if (!CodeObjectID || !TargetID || - CodeObjectID->getGPUKind() != TargetID->getGPUKind()) { + llvm::StringMap CodeObjectFeatureMap, TargetFeatureMap; + std::optional CodeObjectProc = clang::parseTargetID( + CodeObjectInfo.Triple, CodeObjectInfo.TargetID, &CodeObjectFeatureMap); + std::optional TargetProc = clang::parseTargetID( + TargetInfo.Triple, TargetInfo.TargetID, &TargetFeatureMap); + + // Both TargetProc and CodeObjectProc can't be empty here. + if (!TargetProc || !CodeObjectProc || + CodeObjectProc.value() != TargetProc.value()) { DEBUG_WITH_TYPE("CodeObjectCompatibility", dbgs() << "Incompatible: Processor mismatch \t[CodeObject: " << CodeObjectInfo.str() @@ -1131,30 +1130,44 @@ bool isCodeObjectCompatible(const OffloadTargetInfo &CodeObjectInfo, return false; } - // A feature (xnack/sramecc) is compatible if the code object leaves it - // unspecified ("Any"), or specifies it with the same value the target does. - // A feature the code object specifies but the target leaves unspecified is - // incompatible, as is a differing explicit value. - auto FeatureCompatible = [&](llvm::AMDGPU::TargetIDSetting CodeObject, - llvm::AMDGPU::TargetIDSetting Target) { - bool CodeObjectExplicit = CodeObject == llvm::AMDGPU::TargetIDSetting::On || - CodeObject == llvm::AMDGPU::TargetIDSetting::Off; - if (!CodeObjectExplicit) - return true; - return CodeObject == Target; - }; - - if (!FeatureCompatible(CodeObjectID->getXnackSetting(), - TargetID->getXnackSetting()) || - !FeatureCompatible(CodeObjectID->getSramEccSetting(), - TargetID->getSramEccSetting())) { + // Incompatible if CodeObject has more features than Target, irrespective of + // type or sign of features. + if (CodeObjectFeatureMap.getNumItems() > TargetFeatureMap.getNumItems()) { DEBUG_WITH_TYPE("CodeObjectCompatibility", - dbgs() << "Incompatible: Feature mismatch \t[CodeObject: " + dbgs() << "Incompatible: CodeObject has more features " + "than target \t[CodeObject: " << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str() << "]\n"); return false; } + // Compatible if each target feature specified by target is compatible with + // target feature of code object. The target feature is compatible if the + // code object does not specify it (meaning Any), or if it specifies it + // with the same value (meaning On or Off). + for (const auto &CodeObjectFeature : CodeObjectFeatureMap) { + auto TargetFeature = TargetFeatureMap.find(CodeObjectFeature.getKey()); + if (TargetFeature == TargetFeatureMap.end()) { + DEBUG_WITH_TYPE( + "CodeObjectCompatibility", + dbgs() + << "Incompatible: Value of CodeObject's non-ANY feature is " + "not matching with Target feature's ANY value \t[CodeObject: " + << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str() + << "]\n"); + return false; + } else if (TargetFeature->getValue() != CodeObjectFeature.getValue()) { + DEBUG_WITH_TYPE( + "CodeObjectCompatibility", + dbgs() << "Incompatible: Value of CodeObject's non-ANY feature is " + "not matching with Target feature's non-ANY value " + "\t[CodeObject: " + << CodeObjectInfo.str() + << "]\t:\t[Target: " << TargetInfo.str() << "]\n"); + return false; + } + } + // CodeObject is compatible if all features of Target are: // - either, present in the Code Object's features map with the same sign, // - or, the feature is missing from CodeObjects's features map i.e. it is @@ -1523,18 +1536,8 @@ CheckHeterogeneousArchive(StringRef ArchiveName, if (CodeObjectFileError) return CodeObjectFileError; - // A single bundle may contain several triples. Pair each target ID with its - // own triple; the conflict check groups by resolved processor, which is - // spelling-independent. - llvm::SmallVector Infos; - for (StringRef BundleId : BundleIds) - Infos.emplace_back(BundleId, BundlerConfig); - llvm::SmallVector Entries; - for (const OffloadTargetInfo &Info : Infos) - Entries.emplace_back(Info.Triple, Info.TargetID); - - if (auto &&ConflictingArchs = - clang::getConflictTargetIDCombination(Entries)) { + auto &&ConflictingArchs = clang::getConflictTargetIDCombination(BundleIds); + if (ConflictingArchs) { std::string ErrMsg = Twine("conflicting TargetIDs [" + ConflictingArchs.value().first + ", " + ConflictingArchs.value().second + "] found in " + diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp index 4f2e6278a10b0..7bce060de0596 100644 --- a/clang/lib/Driver/ToolChains/AMDGPU.cpp +++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp @@ -771,24 +771,26 @@ AMDGPUToolChain::TranslateArgs(const DerivedArgList &Args, BoundArch BA, } if (!getTriple().isSPIRV()) { - std::optional PTID = checkTargetID(*DAL); - - // Synthesize feature flags for explicit target ID modifiers (xnack, - // sramecc). - if (PTID) { - using llvm::AMDGPU::TargetIDSetting; - if (PTID->isXnackOnOrOff()) - DAL->AddFlagArg(nullptr, Opts.getOption(PTID->getXnackSetting() == - TargetIDSetting::On + AMDGPUToolChain::ParsedTargetIDType PTID = checkTargetID(*DAL); + + // Synthesize feature flags for target ID modifiers (xnack, sramecc). + if (PTID.OptionalFeatureMap) { + const llvm::StringMap &FeatureMap = *PTID.OptionalFeatureMap; + + auto XnackIt = FeatureMap.find("xnack"); + if (XnackIt != FeatureMap.end()) { + DAL->AddFlagArg(nullptr, Opts.getOption(XnackIt->second ? options::OPT_mxnack : options::OPT_mno_xnack)); + } - if (PTID->isSramEccOnOrOff()) - DAL->AddFlagArg( - nullptr, - Opts.getOption(PTID->getSramEccSetting() == TargetIDSetting::On - ? options::OPT_msramecc - : options::OPT_mno_sramecc)); + auto SrameccIt = FeatureMap.find("sramecc"); + if (SrameccIt != FeatureMap.end()) { + DAL->AddFlagArg(nullptr, + Opts.getOption(SrameccIt->second + ? options::OPT_msramecc + : options::OPT_mno_sramecc)); + } } } @@ -1002,33 +1004,28 @@ AMDGPUToolChain::getGPUArch(const llvm::opt::ArgList &DriverArgs) const { getTriple(), DriverArgs.getLastArgValue(options::OPT_mcpu_EQ)); } -StringRef -AMDGPUToolChain::getTargetIDArg(const llvm::opt::ArgList &DriverArgs) const { - // Target IDs are only meaningful for AMDGCN targets. - if (!getTriple().isAMDGCN()) - return StringRef(); - return DriverArgs.getLastArgValue(options::OPT_mcpu_EQ); -} - -std::optional +AMDGPUToolChain::ParsedTargetIDType AMDGPUToolChain::getParsedTargetID(const llvm::opt::ArgList &DriverArgs) const { - StringRef TargetID = getTargetIDArg(DriverArgs); + StringRef TargetID = DriverArgs.getLastArgValue(options::OPT_mcpu_EQ); if (TargetID.empty()) - return std::nullopt; + return {}; + + llvm::StringMap FeatureMap; + auto OptionalGpuArch = parseTargetID(getTriple(), TargetID, &FeatureMap); + if (!OptionalGpuArch) + return {TargetID.str(), std::nullopt, std::nullopt}; - return llvm::AMDGPU::TargetID::parse(getTriple(), TargetID); + return {TargetID.str(), OptionalGpuArch->str(), FeatureMap}; } -std::optional +AMDGPUToolChain::ParsedTargetIDType AMDGPUToolChain::checkTargetID(const llvm::opt::ArgList &DriverArgs) const { - std::optional ID = getParsedTargetID(DriverArgs); - // Diagnose a non-empty but invalid target ID. - if (!ID) { - StringRef TargetID = getTargetIDArg(DriverArgs); - if (!TargetID.empty()) - getDriver().Diag(clang::diag::err_drv_bad_target_id) << TargetID; + auto PTID = getParsedTargetID(DriverArgs); + if (PTID.OptionalTargetID && !PTID.OptionalGPUArch) { + getDriver().Diag(clang::diag::err_drv_bad_target_id) + << *PTID.OptionalTargetID; } - return ID; + return PTID; } Expected> @@ -1308,21 +1305,26 @@ LTOKind AMDGPUToolChain::getLTOMode(const ArgList &Args, } static bool isXnackAvailable(const llvm::Triple &TT, llvm::StringRef TargetID) { - std::optional ID = - llvm::AMDGPU::TargetID::parse(TT, TargetID); - if (!ID) + // Arch-specific check - only report as supported if arch has xnack+ + if (!TT.isAMDGCN()) return false; - - unsigned Features = llvm::AMDGPU::getArchAttrAMDGCN(ID->getGPUKind()); - - // If the processor has xnack but doesn't support on/off modes, xnack is - // always on. - if ((Features & llvm::AMDGPU::FEATURE_XNACK) && - !(Features & llvm::AMDGPU::FEATURE_XNACK_ON_OFF_MODES)) + llvm::StringRef Processor = getProcessorFromTargetID(TT, TargetID); + llvm::AMDGPU::GPUKind ProcKind = llvm::AMDGPU::parseArchAMDGCN(Processor); + unsigned Features = llvm::AMDGPU::getArchAttrAMDGCN(ProcKind); + + // If processor has xnack but doesn't support on/off modes, xnack is always on + bool XnackAlwaysOn = (Features & llvm::AMDGPU::FEATURE_XNACK) && + !(Features & llvm::AMDGPU::FEATURE_XNACK_ON_OFF_MODES); + if (XnackAlwaysOn) return true; - // Otherwise, it is available only if the target ID explicitly enables it. - return ID->getXnackSetting() == llvm::AMDGPU::TargetIDSetting::On; + // Otherwise, check if xnack+ is explicitly enabled in the target ID + llvm::StringMap FeatureMap; + auto OptionalGpuArch = parseTargetID(TT, TargetID, &FeatureMap); + if (!OptionalGpuArch) + return false; + auto Loc = FeatureMap.find("xnack"); + return (Loc != FeatureMap.end() && Loc->second); } SanitizerMask AMDGPUToolChain::getSupportedSanitizers( diff --git a/clang/lib/Driver/ToolChains/AMDGPU.h b/clang/lib/Driver/ToolChains/AMDGPU.h index fd71b53064d3e..027a6e3b47dca 100644 --- a/clang/lib/Driver/ToolChains/AMDGPU.h +++ b/clang/lib/Driver/ToolChains/AMDGPU.h @@ -161,20 +161,23 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUToolChain : public Generic_ELF { Action::OffloadKind DeviceOffloadingKind) const; protected: - /// Check and diagnose an invalid target ID specified by -mcpu. Returns the - /// parsed target ID, or std::nullopt if -mcpu is absent or invalid - virtual std::optional + /// The struct type returned by getParsedTargetID. + struct ParsedTargetIDType { + std::optional OptionalTargetID; + std::optional OptionalGPUArch; + std::optional> OptionalFeatureMap; + }; + + /// Check and diagnose invalid target ID specified by -mcpu. + /// Returns the parsed target ID. + virtual ParsedTargetIDType checkTargetID(const llvm::opt::ArgList &DriverArgs) const; - /// Parse the target ID specified by -mcpu. Returns the parsed target ID, or - /// std::nullopt if -mcpu is absent or invalid. - std::optional + /// Get target ID, GPU arch, and target ID features if the target ID is + /// specified and valid. + ParsedTargetIDType getParsedTargetID(const llvm::opt::ArgList &DriverArgs) const; - /// Get the raw target ID string from -mcpu, or an empty string if -mcpu is - /// absent or the target is not AMDGCN. - StringRef getTargetIDArg(const llvm::opt::ArgList &DriverArgs) const; - /// Get GPU arch from -mcpu without checking. StringRef getGPUArch(const llvm::opt::ArgList &DriverArgs) const; diff --git a/clang/test/CodeGenCXX/wasm-eh.cpp b/clang/test/CodeGenCXX/wasm-eh.cpp index f243f37ecb435..0b87107e476c7 100644 --- a/clang/test/CodeGenCXX/wasm-eh.cpp +++ b/clang/test/CodeGenCXX/wasm-eh.cpp @@ -41,7 +41,7 @@ void multiple_catches_wo_catch_all() { // CHECK-NEXT: %[[EXN:.*]] = call ptr @llvm.wasm.get.exception(token %[[CATCHPAD]]) // CHECK-NEXT: store ptr %[[EXN]], ptr %exn.slot // CHECK-NEXT: %[[SELECTOR:.*]] = call i32 @llvm.wasm.get.ehselector(token %[[CATCHPAD]]) -// CHECK-NEXT: %[[TYPEID:.*]] = call i32 @llvm.eh.typeid.for.p0(ptr @_ZTIi) #7 +// CHECK-NEXT: %[[TYPEID:.*]] = call i32 @llvm.eh.typeid.for.p0(ptr @_ZTIi) {{.*}} // CHECK-NEXT: %[[MATCHES:.*]] = icmp eq i32 %[[SELECTOR]], %[[TYPEID]] // CHECK-NEXT: br i1 %[[MATCHES]], label %[[CATCH_INT_BB:.*]], label %[[CATCH_FALLTHROUGH_BB:.*]] @@ -58,7 +58,7 @@ void multiple_catches_wo_catch_all() { // CHECK-NEXT: br label %[[TRY_CONT_BB:.*]] // CHECK: [[CATCH_FALLTHROUGH_BB]] -// CHECK-NEXT: %[[TYPEID:.*]] = call i32 @llvm.eh.typeid.for.p0(ptr @_ZTId) #7 +// CHECK-NEXT: %[[TYPEID:.*]] = call i32 @llvm.eh.typeid.for.p0(ptr @_ZTId) {{.*}} // CHECK-NEXT: %[[MATCHES:.*]] = icmp eq i32 %[[SELECTOR]], %[[TYPEID]] // CHECK-NEXT: br i1 %[[MATCHES]], label %[[CATCH_FLOAT_BB:.*]], label %[[RETHROW_BB:.*]] @@ -388,6 +388,34 @@ void noexcept_throw() noexcept { throw 3; } +int get_val() { return 42; } +thread_local int tls = get_val(); + +// CHECK-LABEL: @_Z26tls_wrapper_within_funcletv() +// CHECK: %[[CATCHSWITCH:.*]] = catchswitch within none [label %[[CATCHSTART_BB:.*]]] unwind to caller + +// CHECK: [[CATCHSTART_BB]]: +// CHECK-NEXT: %[[CATCHPAD:.*]] = catchpad within %[[CATCHSWITCH]] [ptr null] +// CHECK-NEXT: %[[EXN:.*]] = call ptr @llvm.wasm.get.exception(token %[[CATCHPAD]]) +// CHECK-NEXT: store ptr %[[EXN]], ptr %exn.slot +// CHECK-NEXT: {{.*}} call i32 @llvm.wasm.get.ehselector(token %[[CATCHPAD]]) +// CHECK-NEXT: br label %[[CATCH_ALL_BB:.*]] + +// CHECK: [[CATCH_ALL_BB]]: +// CHECK-NEXT: %[[EXN_LOAD:.*]] = load ptr, ptr %exn.slot +// CHECK-NEXT: call ptr @__cxa_begin_catch(ptr %[[EXN_LOAD]]) {{.*}} [ "funclet"(token %[[CATCHPAD]]) ] +// CHECK-NEXT: call {{.*}} @_ZTW3tls() [ "funclet"(token %[[CATCHPAD]]) ] + +// Thread-local wrapper calls within a funclet should have a catchpad/cleanuppad +// funclet bundle argument. +int tls_wrapper_within_funclet() { + try { + throw 1; + } catch (...) { + return tls; + } +} + // CATCH-LABEL: define void @_Z14noexcept_throwv() // CHECK: %{{.*}} = cleanuppad within none [] // CHECK-NEXT: call void @_ZSt9terminatev() diff --git a/clang/test/OffloadTools/clang-offload-bundler/basic.c b/clang/test/OffloadTools/clang-offload-bundler/basic.c index bd2ca595c4a7e..b10c9cde08921 100644 --- a/clang/test/OffloadTools/clang-offload-bundler/basic.c +++ b/clang/test/OffloadTools/clang-offload-bundler/basic.c @@ -535,17 +535,6 @@ // RUN: not clang-offload-bundler -type=o -targets=host-x86_64-unknown-linux-gnu,openmp-amdgpu9.06-amd-amdhsa--gfx906,openmp-amdgpu9.06-amd-amdhsa--gfx906:sramecc+ -input=%t.o -input=%t.tgt1 -input=%t.tgt2 -output=%t.bad.bundle 2>&1 | FileCheck %s -check-prefix=BADTARGETS // BADTARGETS: error: Cannot bundle inputs with conflicting targets: 'openmp-amdgpu9.06-amd-amdhsa--gfx906' and 'openmp-amdgpu9.06-amd-amdhsa--gfx906:sramecc+' -// Check the per-member TargetID conflict detection performed by -// -check-input-archive. The bundle-time conflict check groups by offload kind -// and triple, so "gfx906" and "gfx906:xnack+" placed under different offload -// kinds (hip vs hipv4) bundle successfully. The archive check instead groups by -// resolved processor and must flag them as conflicting for the same gfx906. - -// RUN: clang-offload-bundler -type=o -targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa--gfx906,hipv4-amdgcn-amd-amdhsa--gfx906:xnack+ -input=%t.o -input=%t.tgt1 -input=%t.tgt2 -output=%t.conflict.bundle -// RUN: llvm-ar cr %t.conflict-archive.a %t.conflict.bundle -// RUN: not clang-offload-bundler -unbundle -type=a -check-input-archive -targets=hip-amdgcn-amd-amdhsa--gfx906 -input=%t.conflict-archive.a -output=%t.conflict-out.a 2>&1 | FileCheck %s -check-prefix=CONFLICTARCHIVE -// CONFLICTARCHIVE: error: conflicting TargetIDs [gfx906, gfx906:xnack+] found in {{.*}}conflict.bundle of {{.*}}conflict-archive.a - // Check for error if no compatible code object is found in the heterogeneous archive library // RUN: not clang-offload-bundler -unbundle -type=a -targets=openmp-amdgpu8.03-amd-amdhsa--gfx803 -input=%t.input-archive.a -output=%t-archive-gfx803-incompatible.a 2>&1 | FileCheck %s -check-prefix=INCOMPATIBLEARCHIVE // INCOMPATIBLEARCHIVE: error: no compatible code object found for the target 'openmp-amdgpu8.03-amd-amdhsa--gfx803' in heterogeneous archive library diff --git a/clang/test/OpenMP/declare_mapper_codegen.cpp b/clang/test/OpenMP/declare_mapper_codegen.cpp index eb90f218f5ca8..3eb4c2f28efb2 100644 --- a/clang/test/OpenMP/declare_mapper_codegen.cpp +++ b/clang/test/OpenMP/declare_mapper_codegen.cpp @@ -85,6 +85,11 @@ class C { }; #pragma omp declare mapper(id: C s) map(s.a, s.b[0:2]) +// +// Per-element entries (N = __tgt_mapper_num_components()): +// &s, &s.a, sizeof(int), MEMBER_OF(N) | TO | FROM | modifiers +// &s.b[0], &s.b[0], sizeof(double)*2, TO | FROM | modifiers +// &s.b, &s.b[0], sizeof(double*), ATTACH // CK0: define {{.*}}void [[MPRFUNC:@[.]omp_mapper[.].*C[.]id]](ptr noundef [[HANDLE:%.+]], ptr noundef [[BPTR:%.+]], ptr noundef [[BEGIN:%.+]], i64 noundef [[BYTESIZE:%.+]], i64 noundef [[TYPE:%.+]], ptr{{.*}}) // CK0-64-DAG: [[SIZE:%.+]] = udiv exact i64 [[BYTESIZE]], 16 @@ -114,20 +119,14 @@ class C { // CK0: [[PTR:%.+]] = phi ptr [ [[BEGIN]], %{{.+}} ], [ [[PTRNEXT:%.+]], %[[LCORRECT:[^,]+]] ] // CK0-DAG: [[ABEGIN:%.+]] = getelementptr inbounds nuw %class.C, ptr [[PTR]], i32 0, i32 0 // CK0-DAG: [[BBEGIN:%.+]] = getelementptr inbounds nuw %class.C, ptr [[PTR]], i32 0, i32 1 +// CK0-DAG: [[BARRBASE:%.+]] = load ptr, ptr [[BBEGIN]] // CK0-DAG: [[BBEGIN2:%.+]] = getelementptr inbounds nuw %class.C, ptr [[PTR]], i32 0, i32 1 // CK0-DAG: [[BARRBEGIN:%.+]] = load ptr, ptr [[BBEGIN2]] // CK0-DAG: [[BARRBEGINGEP:%.+]] = getelementptr inbounds nuw double, ptr [[BARRBEGIN]], i[[sz:64|32]] 0 -// CK0-DAG: [[BEND:%.+]] = getelementptr ptr, ptr [[BBEGIN]], i32 1 -// CK0-64-DAG: [[ABEGINI:%.+]] = ptrtoaddr ptr [[ABEGIN]] to i64 -// CK0-64-DAG: [[BENDI:%.+]] = ptrtoaddr ptr [[BEND]] to i64 -// CK0-64-DAG: [[CUSIZE:%.+]] = sub i64 [[BENDI]], [[ABEGINI]] -// CK0-32-DAG: [[ABEGINI:%.+]] = ptrtoaddr ptr [[ABEGIN]] to i32 -// CK0-32-DAG: [[BENDI:%.+]] = ptrtoaddr ptr [[BEND]] to i32 -// CK0-32-DAG: [[CSIZE:%.+]] = sub i32 [[BENDI]], [[ABEGINI]] -// CK0-32-DAG: [[CUSIZE:%.+]] = zext i32 [[CSIZE]] to i64 // CK0-DAG: [[PRESIZE:%.+]] = call i64 @__tgt_mapper_num_components(ptr [[HANDLE]]) // CK0-DAG: [[SHIPRESIZE:%.+]] = shl i64 [[PRESIZE]], 48 -// CK0-DAG: [[MEMBERTYPE:%.+]] = add nuw i64 0, [[SHIPRESIZE]] +// &s, &s.a, sizeof(int), MEMBER_OF(N) | TO | FROM | modifiers +// CK0-DAG: [[MEMBERTYPE:%.+]] = add nuw i64 3, [[SHIPRESIZE]] // CK0-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3 // CK0-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0 // CK0-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]] @@ -150,57 +149,48 @@ class C { // CK0-DAG: [[PHITYPE0:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ] // CK0-DAG: [[MODMASK0:%.+]] = and i64 [[TYPE]], 1036 // CK0-DAG: [[PHITYPE0_MOD:%.+]] = or i64 [[PHITYPE0]], [[MODMASK0]] -// CK0: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[PTR]], ptr [[ABEGIN]], i64 [[CUSIZE]], i64 [[PHITYPE0_MOD]], {{.*}}) -// 281474976710659 == 0x1,000,000,003 -// CK0-DAG: [[MEMBERTYPE:%.+]] = add nuw i64 281474976710659, [[SHIPRESIZE]] +// CK0: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[PTR]], ptr [[ABEGIN]], i64 4, i64 [[PHITYPE0_MOD]], {{.*}}) +// &s.b[0], &s.b[0], sizeof(double)*2, TO | FROM | modifiers // CK0-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3 // CK0-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0 // CK0-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]] // CK0-DAG: [[ALLOC]] -// CK0-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4 // CK0-DAG: br label %[[TYEND:[^,]+]] // CK0-DAG: [[ALLOCELSE]] // CK0-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1 // CK0-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]] // CK0-DAG: [[TO]] -// CK0-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3 // CK0-DAG: br label %[[TYEND]] // CK0-DAG: [[TOELSE]] // CK0-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2 // CK0-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]] // CK0-DAG: [[FROM]] -// CK0-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2 // CK0-DAG: br label %[[TYEND]] // CK0-DAG: [[TYEND]] -// CK0-DAG: [[TYPE1:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ] +// CK0-DAG: [[TYPE1:%.+]] = phi i64 [ 0, %[[ALLOC]] ], [ 1, %[[TO]] ], [ 2, %[[FROM]] ], [ 3, %[[TOELSE]] ] // CK0-DAG: [[MODMASK1:%.+]] = and i64 [[TYPE]], 1036 // CK0-DAG: [[TYPE1_MOD:%.+]] = or i64 [[TYPE1]], [[MODMASK1]] -// CK0: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[PTR]], ptr [[ABEGIN]], i64 4, i64 [[TYPE1_MOD]], {{.*}}) -// 281474976710675 == 0x1,000,000,013 -// CK0-DAG: [[MEMBERTYPE:%.+]] = add nuw i64 281474976710675, [[SHIPRESIZE]] +// CK0: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[BARRBASE]], ptr [[BARRBEGINGEP]], i64 16, i64 [[TYPE1_MOD]], {{.*}}) +// &s.b, &s.b[0], sizeof(double*), ATTACH // CK0-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3 // CK0-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0 // CK0-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]] // CK0-DAG: [[ALLOC]] -// CK0-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4 // CK0-DAG: br label %[[TYEND:[^,]+]] // CK0-DAG: [[ALLOCELSE]] // CK0-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1 // CK0-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]] // CK0-DAG: [[TO]] -// CK0-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3 // CK0-DAG: br label %[[TYEND]] // CK0-DAG: [[TOELSE]] // CK0-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2 // CK0-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]] // CK0-DAG: [[FROM]] -// CK0-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2 // CK0-DAG: br label %[[TYEND]] // CK0-DAG: [[TYEND]] -// CK0-DAG: [[TYPE2:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ] -// CK0-DAG: [[MODMASK2:%.+]] = and i64 [[TYPE]], 1036 -// CK0-DAG: [[TYPE2_MOD:%.+]] = or i64 [[TYPE2]], [[MODMASK2]] -// CK0: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[BBEGIN]], ptr [[BARRBEGINGEP]], i64 16, i64 [[TYPE2_MOD]], {{.*}}) +// CK0-DAG: [[TYPE2:%.+]] = phi i64 [ 16384, %[[ALLOC]] ], [ 16384, %[[TO]] ], [ 16384, %[[FROM]] ], [ 16384, %[[TOELSE]] ] +// CK0-64: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[BBEGIN]], ptr [[BARRBEGINGEP]], i64 8, i64 [[TYPE2]], {{.*}}) +// CK0-32: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[BBEGIN]], ptr [[BARRBEGINGEP]], i64 4, i64 [[TYPE2]], {{.*}}) // CK0: [[PTRNEXT]] = getelementptr %class.C, ptr [[PTR]], i32 1 // CK0: [[ISDONE:%.+]] = icmp eq ptr [[PTRNEXT]], [[PTREND]] // CK0: br i1 [[ISDONE]], label %[[LEXIT:[^,]+]], label %[[LBODY]] @@ -592,6 +582,9 @@ class C { }; #pragma omp declare mapper(id: C s) map(s.a) +// +// Per-element entries (N = __tgt_mapper_num_components()): +// &s, &s.a, sizeof(int), MEMBER_OF(N) | TO | FROM | modifiers // CK1: define {{.*}}void @.omp_mapper.{{.*}}C{{.*}}.id{{.*}}(ptr noundef [[HANDLE:%.+]], ptr noundef [[BPTR:%.+]], ptr noundef [[BEGIN:%.+]], i64 noundef [[BYTESIZE:%.+]], i64 noundef [[TYPE:%.+]], ptr{{.*}}) // CK1-DAG: [[SIZE:%.+]] = udiv exact i64 [[BYTESIZE]], 4 @@ -699,6 +692,9 @@ class C { #pragma omp declare mapper(B s) map(s.a) #pragma omp declare mapper(id: C s) map(s.b) +// +// Per-element entries emitted (N = __tgt_mapper_num_components()): +// &s, &s.b, sizeof(B), MEMBER_OF(N) | TO | FROM | modifiers (dispatches to B mapper) // CK2: define {{.*}}void [[BMPRFUNC:@[.]omp_mapper[.].*B[.]default]](ptr{{.*}}, ptr{{.*}}, ptr{{.*}}, i64{{.*}}, i64{{.*}}, ptr{{.*}}) @@ -894,6 +890,11 @@ class C { }; #pragma omp declare mapper(id: C s) map(s.a, s.b[0:2]) +// +// Per-element entries (N = __tgt_mapper_num_components()): +// &s, &s.a, sizeof(int), MEMBER_OF(N) | TO | FROM | modifiers +// &s.b[0], &s.b[0], sizeof(double)*2, TO | FROM | modifiers +// &s.b, &s.b[0], sizeof(double*), ATTACH // CK4: define {{.*}}void [[MPRFUNC:@[.]omp_mapper[.].*C[.]id]](ptr noundef [[HANDLE:%.+]], ptr noundef [[BPTR:%.+]], ptr noundef [[BEGIN:%.+]], i64 noundef [[BYTESIZE:%.+]], i64 noundef [[TYPE:%.+]], ptr{{.*}}) // CK4-64-DAG: [[SIZE:%.+]] = udiv exact i64 [[BYTESIZE]], 16 @@ -924,20 +925,14 @@ class C { // CK4: [[PTR:%.+]] = phi ptr [ [[BEGIN]], %{{.+}} ], [ [[PTRNEXT:%.+]], %[[LCORRECT:[^,]+]] ] // CK4-DAG: [[ABEGIN:%.+]] = getelementptr inbounds nuw %class.C, ptr [[PTR]], i32 0, i32 0 // CK4-DAG: [[BBEGIN:%.+]] = getelementptr inbounds nuw %class.C, ptr [[PTR]], i32 0, i32 1 +// CK4-DAG: [[BARRBASE:%.+]] = load ptr, ptr [[BBEGIN]] // CK4-DAG: [[BBEGIN2:%.+]] = getelementptr inbounds nuw %class.C, ptr [[PTR]], i32 0, i32 1 // CK4-DAG: [[BARRBEGIN:%.+]] = load ptr, ptr [[BBEGIN2]] // CK4-DAG: [[BARRBEGINGEP:%.+]] = getelementptr inbounds nuw double, ptr [[BARRBEGIN]], i[[sz:64|32]] 0 -// CK4-DAG: [[BEND:%.+]] = getelementptr ptr, ptr [[BBEGIN]], i32 1 -// CK4-64-DAG: [[ABEGINI:%.+]] = ptrtoaddr ptr [[ABEGIN]] to i64 -// CK4-64-DAG: [[BENDI:%.+]] = ptrtoaddr ptr [[BEND]] to i64 -// CK4-64-DAG: [[CUSIZE:%.+]] = sub i64 [[BENDI]], [[ABEGINI]] -// CK4-32-DAG: [[ABEGINI:%.+]] = ptrtoaddr ptr [[ABEGIN]] to i32 -// CK4-32-DAG: [[BENDI:%.+]] = ptrtoaddr ptr [[BEND]] to i32 -// CK4-32-DAG: [[CSIZE:%.+]] = sub i32 [[BENDI]], [[ABEGINI]] -// CK4-32-DAG: [[CUSIZE:%.+]] = zext i32 [[CSIZE]] to i64 // CK4-DAG: [[PRESIZE:%.+]] = call i64 @__tgt_mapper_num_components(ptr [[HANDLE]]) // CK4-DAG: [[SHIPRESIZE:%.+]] = shl i64 [[PRESIZE]], 48 -// CK4-DAG: [[MEMBERTYPE:%.+]] = add nuw i64 0, [[SHIPRESIZE]] +// &s, &s.a, sizeof(int), MEMBER_OF(N) | TO | FROM | modifiers +// CK4-DAG: [[MEMBERTYPE:%.+]] = add nuw i64 3, [[SHIPRESIZE]] // CK4-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3 // CK4-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0 // CK4-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]] @@ -960,57 +955,48 @@ class C { // CK4-DAG: [[PHITYPE0:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ] // CK4-DAG: [[MODMASK0:%.+]] = and i64 [[TYPE]], 1036 // CK4-DAG: [[PHITYPE0_MOD:%.+]] = or i64 [[PHITYPE0]], [[MODMASK0]] -// CK4: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[PTR]], ptr [[ABEGIN]], i64 [[CUSIZE]], i64 [[PHITYPE0_MOD]], {{.*}}) -// 281474976710659 == 0x1,000,000,003 -// CK4-DAG: [[MEMBERTYPE:%.+]] = add nuw i64 281474976710659, [[SHIPRESIZE]] +// CK4: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[PTR]], ptr [[ABEGIN]], i64 4, i64 [[PHITYPE0_MOD]], {{.*}}) +// &s.b[0], &s.b[0], sizeof(double)*2, TO | FROM | modifiers // CK4-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3 // CK4-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0 // CK4-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]] // CK4-DAG: [[ALLOC]] -// CK4-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4 // CK4-DAG: br label %[[TYEND:[^,]+]] // CK4-DAG: [[ALLOCELSE]] // CK4-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1 // CK4-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]] // CK4-DAG: [[TO]] -// CK4-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3 // CK4-DAG: br label %[[TYEND]] // CK4-DAG: [[TOELSE]] // CK4-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2 // CK4-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]] // CK4-DAG: [[FROM]] -// CK4-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2 // CK4-DAG: br label %[[TYEND]] // CK4-DAG: [[TYEND]] -// CK4-DAG: [[TYPE1:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ] +// CK4-DAG: [[TYPE1:%.+]] = phi i64 [ 0, %[[ALLOC]] ], [ 1, %[[TO]] ], [ 2, %[[FROM]] ], [ 3, %[[TOELSE]] ] // CK4-DAG: [[MODMASK1:%.+]] = and i64 [[TYPE]], 1036 // CK4-DAG: [[TYPE1_MOD:%.+]] = or i64 [[TYPE1]], [[MODMASK1]] -// CK4: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[PTR]], ptr [[ABEGIN]], i64 4, i64 [[TYPE1_MOD]], {{.*}}) -// 281474976710675 == 0x1,000,000,013 -// CK4-DAG: [[MEMBERTYPE:%.+]] = add nuw i64 281474976710675, [[SHIPRESIZE]] +// CK4: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[BARRBASE]], ptr [[BARRBEGINGEP]], i64 16, i64 [[TYPE1_MOD]], {{.*}}) +// &s.b, &s.b[0], sizeof(double*), ATTACH // CK4-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3 // CK4-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0 // CK4-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]] // CK4-DAG: [[ALLOC]] -// CK4-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4 // CK4-DAG: br label %[[TYEND:[^,]+]] // CK4-DAG: [[ALLOCELSE]] // CK4-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1 // CK4-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]] // CK4-DAG: [[TO]] -// CK4-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3 // CK4-DAG: br label %[[TYEND]] // CK4-DAG: [[TOELSE]] // CK4-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2 // CK4-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]] // CK4-DAG: [[FROM]] -// CK4-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2 // CK4-DAG: br label %[[TYEND]] // CK4-DAG: [[TYEND]] -// CK4-DAG: [[TYPE2:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ] -// CK4-DAG: [[MODMASK2:%.+]] = and i64 [[TYPE]], 1036 -// CK4-DAG: [[TYPE2_MOD:%.+]] = or i64 [[TYPE2]], [[MODMASK2]] -// CK4: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[BBEGIN]], ptr [[BARRBEGINGEP]], i64 16, i64 [[TYPE2_MOD]], {{.*}}) +// CK4-DAG: [[TYPE2:%.+]] = phi i64 [ 16384, %[[ALLOC]] ], [ 16384, %[[TO]] ], [ 16384, %[[FROM]] ], [ 16384, %[[TOELSE]] ] +// CK4-64: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[BBEGIN]], ptr [[BARRBEGINGEP]], i64 8, i64 [[TYPE2]], {{.*}}) +// CK4-32: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[BBEGIN]], ptr [[BARRBEGINGEP]], i64 4, i64 [[TYPE2]], {{.*}}) // CK4: [[PTRNEXT]] = getelementptr %class.C, ptr [[PTR]], i32 1 // CK4: [[ISDONE:%.+]] = icmp eq ptr [[PTRNEXT]], [[PTREND]] // CK4: br i1 [[ISDONE]], label %[[LEXIT:[^,]+]], label %[[LBODY]] @@ -1083,9 +1069,14 @@ typedef struct myvec { } myvec_t; #pragma omp declare mapper(id: myvec_t v) map(iterator(it=0:v.a), tofrom: v.b[it]) +// +// Per-element entries emitted for struct v (N = __tgt_mapper_num_components()): +// &v.b[it], &v.b[it], sizeof(double), TO | FROM | modifiers +// &v.b, &v.b[it], sizeof(double*), ATTACH + // CK5: @[[ITER:[a-zA-Z0-9_]+]] = global i32 0, align 4 -void foo(){ +void foo(){ myvec_t s; #pragma omp target map(mapper(id), to:s) { @@ -1117,34 +1108,51 @@ void foo(){ // CK5: br i1 [[ISEMPTY]], label %[[DONE:[^,]+]], label %[[LBODY:[^,]+]] // CK5: [[LBODY]] // CK5: [[PTR:%.+]] = phi ptr [ [[BEGIN]], %{{.+}} ], [ [[PTRNEXT:%.+]], %[[LCORRECT:[^,]+]] ] -// CK5-DAG: [[ABEGIN:%.+]] = getelementptr inbounds nuw %struct.myvec, ptr [[PTR]], i32 0, i32 1 +// CK5-DAG: [[BBEGIN:%.+]] = getelementptr inbounds nuw %struct.myvec, ptr [[PTR]], i32 0, i32 1 +// CK5-DAG: [[BBASE:%.+]] = load ptr, ptr [[BBEGIN]], align {{.*}} // CK5-DAG: load i32, ptr @[[ITER]], align 4 // CK5-DAG: [[PRESIZE:%.+]] = call i64 @__tgt_mapper_num_components(ptr [[HANDLE]]) // CK5-DAG: [[SHIPRESIZE:%.+]] = shl i64 [[PRESIZE]], 48 -// CK5-DAG: [[MEMBERTYPE:%.+]] = add nuw i64 0, [[SHIPRESIZE]] +// &v.b[it], &v.b[it], sizeof(double), TO | FROM | modifiers // CK5-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3 // CK5-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0 // CK5-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]] // CK5-DAG: [[ALLOC]] -// CK5-DAG: [[ALLOCTYPE:%.+]] = and i64 [[MEMBERTYPE]], -4 // CK5-DAG: br label %[[TYEND:[^,]+]] // CK5-DAG: [[ALLOCELSE]] // CK5-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1 // CK5-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]] // CK5-DAG: [[TO]] -// CK5-DAG: [[TOTYPE:%.+]] = and i64 [[MEMBERTYPE]], -3 // CK5-DAG: br label %[[TYEND]] // CK5-DAG: [[TOELSE]] // CK5-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2 // CK5-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]] // CK5-DAG: [[FROM]] -// CK5-DAG: [[FROMTYPE:%.+]] = and i64 [[MEMBERTYPE]], -2 // CK5-DAG: br label %[[TYEND]] // CK5-DAG: [[TYEND]] -// CK5-DAG: [[TYPE1:%.+]] = phi i64 [ [[ALLOCTYPE]], %[[ALLOC]] ], [ [[TOTYPE]], %[[TO]] ], [ [[FROMTYPE]], %[[FROM]] ], [ [[MEMBERTYPE]], %[[TOELSE]] ] +// CK5-DAG: [[TYPE1:%.+]] = phi i64 [ 0, %[[ALLOC]] ], [ 1, %[[TO]] ], [ 2, %[[FROM]] ], [ 3, %[[TOELSE]] ] // CK5-DAG: [[MODMASK:%.+]] = and i64 [[TYPE]], 1036 // CK5-DAG: [[TYPE1_MOD:%.+]] = or i64 [[TYPE1]], [[MODMASK]] -// CK5: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[PTR]], ptr [[ABEGIN]], i64 {{.*}}, i64 [[TYPE1_MOD]], {{.*}}) +// CK5: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[BBASE]], ptr {{.*}}, i64 {{.*}}, i64 [[TYPE1_MOD]], {{.*}}) +// &v.b, &v.b[it], sizeof(double*), ATTACH +// CK5-DAG: [[TYPETF:%.+]] = and i64 [[TYPE]], 3 +// CK5-DAG: [[ISALLOC:%.+]] = icmp eq i64 [[TYPETF]], 0 +// CK5-DAG: br i1 [[ISALLOC]], label %[[ALLOC:[^,]+]], label %[[ALLOCELSE:[^,]+]] +// CK5-DAG: [[ALLOC]] +// CK5-DAG: br label %[[TYEND:[^,]+]] +// CK5-DAG: [[ALLOCELSE]] +// CK5-DAG: [[ISTO:%.+]] = icmp eq i64 [[TYPETF]], 1 +// CK5-DAG: br i1 [[ISTO]], label %[[TO:[^,]+]], label %[[TOELSE:[^,]+]] +// CK5-DAG: [[TO]] +// CK5-DAG: br label %[[TYEND]] +// CK5-DAG: [[TOELSE]] +// CK5-DAG: [[ISFROM:%.+]] = icmp eq i64 [[TYPETF]], 2 +// CK5-DAG: br i1 [[ISFROM]], label %[[FROM:[^,]+]], label %[[TYEND]] +// CK5-DAG: [[FROM]] +// CK5-DAG: br label %[[TYEND]] +// CK5-DAG: [[TYEND]] +// CK5-DAG: [[TYPE2:%.+]] = phi i64 [ 16384, %[[ALLOC]] ], [ 16384, %[[TO]] ], [ 16384, %[[FROM]] ], [ 16384, %[[TOELSE]] ] +// CK5: call void @__tgt_push_mapper_component(ptr [[HANDLE]], ptr [[BBEGIN]], ptr {{.*}}, i64 {{.*}}, i64 [[TYPE2]], {{.*}}) // CK5: [[PTRNEXT]] = getelementptr %struct.myvec, ptr [[PTR]], i32 1 // CK5: [[ISDONE:%.+]] = icmp eq ptr [[PTRNEXT]], [[PTREND]] // CK5: br i1 [[ISDONE]], label %[[LEXIT:[^,]+]], label %[[LBODY]] diff --git a/clang/test/OpenMP/target_map_nested_ptr_member_mapper_codegen.cpp b/clang/test/OpenMP/target_map_nested_ptr_member_mapper_codegen.cpp index bd9189dce7c9f..821b5fd1652c4 100644 --- a/clang/test/OpenMP/target_map_nested_ptr_member_mapper_codegen.cpp +++ b/clang/test/OpenMP/target_map_nested_ptr_member_mapper_codegen.cpp @@ -98,132 +98,150 @@ void foo(S2 *arr) { // CHECK: [[OMP_ARRAYMAP_ISEMPTY:%.*]] = icmp eq ptr [[TMP2]], [[TMP7]] // CHECK: br i1 [[OMP_ARRAYMAP_ISEMPTY]], label [[OMP_DONE:%.*]], label [[OMP_ARRAYMAP_BODY:%.*]] // CHECK: omp.arraymap.body: -// CHECK: [[OMP_ARRAYMAP_PTRCURRENT:%.*]] = phi ptr [ [[TMP2]], [[OMP_ARRAYMAP_HEAD]] ], [ [[OMP_ARRAYMAP_NEXT:%.*]], [[OMP_TYPE_END25:%.*]] ] +// CHECK: [[OMP_ARRAYMAP_PTRCURRENT:%.*]] = phi ptr [ [[TMP2]], [[OMP_ARRAYMAP_HEAD]] ], [ [[OMP_ARRAYMAP_NEXT:%.*]], [[OMP_TYPE_END33:%.*]] ] // CHECK: [[Z:%.*]] = getelementptr inbounds nuw [[STRUCT_S2]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 1 // CHECK: [[S1P:%.*]] = getelementptr inbounds nuw [[STRUCT_S2]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 0 +// CHECK: [[TMP15:%.*]] = load ptr, ptr [[S1P]], align 8 // CHECK: [[S1P1:%.*]] = getelementptr inbounds nuw [[STRUCT_S2]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 0 -// CHECK: [[TMP15:%.*]] = load ptr, ptr [[S1P1]], align 8 -// CHECK: [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_S1:%.*]], ptr [[TMP15]], i32 0, i32 0 +// CHECK: [[TMP16:%.*]] = load ptr, ptr [[S1P1]], align 8 +// CHECK: [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_S1:%.*]], ptr [[TMP16]], i32 0, i32 0 // CHECK: [[S1P2:%.*]] = getelementptr inbounds nuw [[STRUCT_S2]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 0 +// CHECK: [[TMP17:%.*]] = load ptr, ptr [[S1P2]], align 8 // CHECK: [[S1P3:%.*]] = getelementptr inbounds nuw [[STRUCT_S2]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 0 -// CHECK: [[TMP16:%.*]] = load ptr, ptr [[S1P3]], align 8 -// CHECK: [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S1]], ptr [[TMP16]], i32 0, i32 1 -// CHECK: [[TMP17:%.*]] = getelementptr i32, ptr [[Z]], i32 1 -// CHECK: [[TMP18:%.*]] = ptrtoaddr ptr [[TMP17]] to i64 -// CHECK: [[TMP19:%.*]] = ptrtoaddr ptr [[S1P]] to i64 -// CHECK: [[TMP20:%.*]] = sub i64 [[TMP18]], [[TMP19]] -// CHECK: [[TMP21:%.*]] = call i64 @__tgt_mapper_num_components(ptr [[TMP0]]) -// CHECK: [[TMP22:%.*]] = shl i64 [[TMP21]], 48 -// CHECK: [[TMP23:%.*]] = add nuw i64 0, [[TMP22]] -// CHECK: [[TMP24:%.*]] = and i64 [[TMP4]], 3 -// CHECK: [[TMP25:%.*]] = icmp eq i64 [[TMP24]], 0 -// CHECK: br i1 [[TMP25]], label [[OMP_TYPE_ALLOC:%.*]], label [[OMP_TYPE_ALLOC_ELSE:%.*]] +// CHECK: [[TMP18:%.*]] = load ptr, ptr [[S1P3]], align 8 +// CHECK: [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S1]], ptr [[TMP18]], i32 0, i32 1 +// CHECK: [[TMP19:%.*]] = getelementptr i32, ptr [[Y]], i32 1 +// CHECK: [[TMP20:%.*]] = ptrtoaddr ptr [[TMP19]] to i64 +// CHECK: [[TMP21:%.*]] = ptrtoaddr ptr [[X]] to i64 +// CHECK: [[TMP22:%.*]] = sub i64 [[TMP20]], [[TMP21]] +// CHECK: [[TMP23:%.*]] = call i64 @__tgt_mapper_num_components(ptr [[TMP0]]) +// CHECK: [[TMP24:%.*]] = shl i64 [[TMP23]], 48 +// CHECK: [[TMP25:%.*]] = add nuw i64 3, [[TMP24]] +// CHECK: [[TMP26:%.*]] = and i64 [[TMP4]], 3 +// CHECK: [[TMP27:%.*]] = icmp eq i64 [[TMP26]], 0 +// CHECK: br i1 [[TMP27]], label [[OMP_TYPE_ALLOC:%.*]], label [[OMP_TYPE_ALLOC_ELSE:%.*]] // CHECK: omp.type.alloc: -// CHECK: [[TMP26:%.*]] = and i64 [[TMP23]], -4 +// CHECK: [[TMP28:%.*]] = and i64 [[TMP25]], -4 // CHECK: br label [[OMP_TYPE_END:%.*]] // CHECK: omp.type.alloc.else: -// CHECK: [[TMP27:%.*]] = icmp eq i64 [[TMP24]], 1 -// CHECK: br i1 [[TMP27]], label [[OMP_TYPE_TO:%.*]], label [[OMP_TYPE_TO_ELSE:%.*]] +// CHECK: [[TMP29:%.*]] = icmp eq i64 [[TMP26]], 1 +// CHECK: br i1 [[TMP29]], label [[OMP_TYPE_TO:%.*]], label [[OMP_TYPE_TO_ELSE:%.*]] // CHECK: omp.type.to: -// CHECK: [[TMP28:%.*]] = and i64 [[TMP23]], -3 +// CHECK: [[TMP30:%.*]] = and i64 [[TMP25]], -3 // CHECK: br label [[OMP_TYPE_END]] // CHECK: omp.type.to.else: -// CHECK: [[TMP29:%.*]] = icmp eq i64 [[TMP24]], 2 -// CHECK: br i1 [[TMP29]], label [[OMP_TYPE_FROM:%.*]], label [[OMP_TYPE_END]] +// CHECK: [[TMP31:%.*]] = icmp eq i64 [[TMP26]], 2 +// CHECK: br i1 [[TMP31]], label [[OMP_TYPE_FROM:%.*]], label [[OMP_TYPE_END]] // CHECK: omp.type.from: -// CHECK: [[TMP30:%.*]] = and i64 [[TMP23]], -2 +// CHECK: [[TMP32:%.*]] = and i64 [[TMP25]], -2 // CHECK: br label [[OMP_TYPE_END]] // CHECK: omp.type.end: -// CHECK: [[OMP_MAPTYPE:%.*]] = phi i64 [ [[TMP26]], [[OMP_TYPE_ALLOC]] ], [ [[TMP28]], [[OMP_TYPE_TO]] ], [ [[TMP30]], [[OMP_TYPE_FROM]] ], [ [[TMP23]], [[OMP_TYPE_TO_ELSE]] ] -// CHECK: [[TMP31:%.*]] = and i64 [[TMP4]], 1036 -// CHECK: [[OMP_MAPTYPE_WITH_MODIFIERS:%.*]] = or i64 [[OMP_MAPTYPE]], [[TMP31]] -// CHECK: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], ptr [[S1P]], i64 [[TMP20]], i64 [[OMP_MAPTYPE_WITH_MODIFIERS]], ptr null) -// CHECK: [[TMP32:%.*]] = add nuw i64 281474976710659, [[TMP22]] -// CHECK: [[TMP33:%.*]] = and i64 [[TMP4]], 3 -// CHECK: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 0 -// CHECK: br i1 [[TMP34]], label [[OMP_TYPE_ALLOC4:%.*]], label [[OMP_TYPE_ALLOC_ELSE5:%.*]] +// CHECK: [[OMP_MAPTYPE:%.*]] = phi i64 [ [[TMP28]], [[OMP_TYPE_ALLOC]] ], [ [[TMP30]], [[OMP_TYPE_TO]] ], [ [[TMP32]], [[OMP_TYPE_FROM]] ], [ [[TMP25]], [[OMP_TYPE_TO_ELSE]] ] +// CHECK: [[TMP33:%.*]] = and i64 [[TMP4]], 1036 +// CHECK: [[OMP_MAPTYPE_WITH_MODIFIERS:%.*]] = or i64 [[OMP_MAPTYPE]], [[TMP33]] +// CHECK: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], ptr [[Z]], i64 4, i64 [[OMP_MAPTYPE_WITH_MODIFIERS]], ptr null) +// CHECK: [[TMP34:%.*]] = and i64 [[TMP4]], 3 +// CHECK: [[TMP35:%.*]] = icmp eq i64 [[TMP34]], 0 +// CHECK: br i1 [[TMP35]], label [[OMP_TYPE_ALLOC4:%.*]], label [[OMP_TYPE_ALLOC_ELSE5:%.*]] // CHECK: omp.type.alloc4: -// CHECK: [[TMP35:%.*]] = and i64 [[TMP32]], -4 // CHECK: br label [[OMP_TYPE_END9:%.*]] // CHECK: omp.type.alloc.else5: -// CHECK: [[TMP36:%.*]] = icmp eq i64 [[TMP33]], 1 +// CHECK: [[TMP36:%.*]] = icmp eq i64 [[TMP34]], 1 // CHECK: br i1 [[TMP36]], label [[OMP_TYPE_TO6:%.*]], label [[OMP_TYPE_TO_ELSE7:%.*]] // CHECK: omp.type.to6: -// CHECK: [[TMP37:%.*]] = and i64 [[TMP32]], -3 // CHECK: br label [[OMP_TYPE_END9]] // CHECK: omp.type.to.else7: -// CHECK: [[TMP38:%.*]] = icmp eq i64 [[TMP33]], 2 -// CHECK: br i1 [[TMP38]], label [[OMP_TYPE_FROM8:%.*]], label [[OMP_TYPE_END9]] +// CHECK: [[TMP37:%.*]] = icmp eq i64 [[TMP34]], 2 +// CHECK: br i1 [[TMP37]], label [[OMP_TYPE_FROM8:%.*]], label [[OMP_TYPE_END9]] // CHECK: omp.type.from8: -// CHECK: [[TMP39:%.*]] = and i64 [[TMP32]], -2 // CHECK: br label [[OMP_TYPE_END9]] // CHECK: omp.type.end9: -// CHECK: [[OMP_MAPTYPE10:%.*]] = phi i64 [ [[TMP35]], [[OMP_TYPE_ALLOC4]] ], [ [[TMP37]], [[OMP_TYPE_TO6]] ], [ [[TMP39]], [[OMP_TYPE_FROM8]] ], [ [[TMP32]], [[OMP_TYPE_TO_ELSE7]] ] -// CHECK: [[TMP40:%.*]] = and i64 [[TMP4]], 1036 -// CHECK: [[OMP_MAPTYPE_WITH_MODIFIERS11:%.*]] = or i64 [[OMP_MAPTYPE10]], [[TMP40]] -// CHECK: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], ptr [[Z]], i64 4, i64 [[OMP_MAPTYPE_WITH_MODIFIERS11]], ptr null) -// CHECK: [[TMP41:%.*]] = add nuw i64 281474976710675, [[TMP22]] -// CHECK: [[TMP42:%.*]] = and i64 [[TMP4]], 3 -// CHECK: [[TMP43:%.*]] = icmp eq i64 [[TMP42]], 0 -// CHECK: br i1 [[TMP43]], label [[OMP_TYPE_ALLOC12:%.*]], label [[OMP_TYPE_ALLOC_ELSE13:%.*]] +// CHECK: [[OMP_MAPTYPE10:%.*]] = phi i64 [ 0, [[OMP_TYPE_ALLOC4]] ], [ 0, [[OMP_TYPE_TO6]] ], [ 0, [[OMP_TYPE_FROM8]] ], [ 0, [[OMP_TYPE_TO_ELSE7]] ] +// CHECK: [[TMP38:%.*]] = and i64 [[TMP4]], 1036 +// CHECK: [[OMP_MAPTYPE_WITH_MODIFIERS11:%.*]] = or i64 [[OMP_MAPTYPE10]], [[TMP38]] +// CHECK: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP15]], ptr [[X]], i64 [[TMP22]], i64 [[OMP_MAPTYPE_WITH_MODIFIERS11]], ptr null) +// CHECK: [[TMP39:%.*]] = add nuw i64 562949953421315, [[TMP24]] +// CHECK: [[TMP40:%.*]] = and i64 [[TMP4]], 3 +// CHECK: [[TMP41:%.*]] = icmp eq i64 [[TMP40]], 0 +// CHECK: br i1 [[TMP41]], label [[OMP_TYPE_ALLOC12:%.*]], label [[OMP_TYPE_ALLOC_ELSE13:%.*]] // CHECK: omp.type.alloc12: -// CHECK: [[TMP44:%.*]] = and i64 [[TMP41]], -4 +// CHECK: [[TMP42:%.*]] = and i64 [[TMP39]], -4 // CHECK: br label [[OMP_TYPE_END17:%.*]] // CHECK: omp.type.alloc.else13: -// CHECK: [[TMP45:%.*]] = icmp eq i64 [[TMP42]], 1 -// CHECK: br i1 [[TMP45]], label [[OMP_TYPE_TO14:%.*]], label [[OMP_TYPE_TO_ELSE15:%.*]] +// CHECK: [[TMP43:%.*]] = icmp eq i64 [[TMP40]], 1 +// CHECK: br i1 [[TMP43]], label [[OMP_TYPE_TO14:%.*]], label [[OMP_TYPE_TO_ELSE15:%.*]] // CHECK: omp.type.to14: -// CHECK: [[TMP46:%.*]] = and i64 [[TMP41]], -3 +// CHECK: [[TMP44:%.*]] = and i64 [[TMP39]], -3 // CHECK: br label [[OMP_TYPE_END17]] // CHECK: omp.type.to.else15: -// CHECK: [[TMP47:%.*]] = icmp eq i64 [[TMP42]], 2 -// CHECK: br i1 [[TMP47]], label [[OMP_TYPE_FROM16:%.*]], label [[OMP_TYPE_END17]] +// CHECK: [[TMP45:%.*]] = icmp eq i64 [[TMP40]], 2 +// CHECK: br i1 [[TMP45]], label [[OMP_TYPE_FROM16:%.*]], label [[OMP_TYPE_END17]] // CHECK: omp.type.from16: -// CHECK: [[TMP48:%.*]] = and i64 [[TMP41]], -2 +// CHECK: [[TMP46:%.*]] = and i64 [[TMP39]], -2 // CHECK: br label [[OMP_TYPE_END17]] // CHECK: omp.type.end17: -// CHECK: [[OMP_MAPTYPE18:%.*]] = phi i64 [ [[TMP44]], [[OMP_TYPE_ALLOC12]] ], [ [[TMP46]], [[OMP_TYPE_TO14]] ], [ [[TMP48]], [[OMP_TYPE_FROM16]] ], [ [[TMP41]], [[OMP_TYPE_TO_ELSE15]] ] -// CHECK: [[TMP49:%.*]] = and i64 [[TMP4]], 1036 -// CHECK: [[OMP_MAPTYPE_WITH_MODIFIERS19:%.*]] = or i64 [[OMP_MAPTYPE18]], [[TMP49]] -// CHECK: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[S1P]], ptr [[X]], i64 4, i64 [[OMP_MAPTYPE_WITH_MODIFIERS19]], ptr null) -// CHECK: [[TMP50:%.*]] = add nuw i64 281474976710675, [[TMP22]] -// CHECK: [[TMP51:%.*]] = and i64 [[TMP4]], 3 -// CHECK: [[TMP52:%.*]] = icmp eq i64 [[TMP51]], 0 -// CHECK: br i1 [[TMP52]], label [[OMP_TYPE_ALLOC20:%.*]], label [[OMP_TYPE_ALLOC_ELSE21:%.*]] +// CHECK: [[OMP_MAPTYPE18:%.*]] = phi i64 [ [[TMP42]], [[OMP_TYPE_ALLOC12]] ], [ [[TMP44]], [[OMP_TYPE_TO14]] ], [ [[TMP46]], [[OMP_TYPE_FROM16]] ], [ [[TMP39]], [[OMP_TYPE_TO_ELSE15]] ] +// CHECK: [[TMP47:%.*]] = and i64 [[TMP4]], 1036 +// CHECK: [[OMP_MAPTYPE_WITH_MODIFIERS19:%.*]] = or i64 [[OMP_MAPTYPE18]], [[TMP47]] +// CHECK: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP15]], ptr [[X]], i64 4, i64 [[OMP_MAPTYPE_WITH_MODIFIERS19]], ptr null) +// CHECK: [[TMP48:%.*]] = add nuw i64 562949953421315, [[TMP24]] +// CHECK: [[TMP49:%.*]] = and i64 [[TMP4]], 3 +// CHECK: [[TMP50:%.*]] = icmp eq i64 [[TMP49]], 0 +// CHECK: br i1 [[TMP50]], label [[OMP_TYPE_ALLOC20:%.*]], label [[OMP_TYPE_ALLOC_ELSE21:%.*]] // CHECK: omp.type.alloc20: -// CHECK: [[TMP53:%.*]] = and i64 [[TMP50]], -4 -// CHECK: br label [[OMP_TYPE_END25]] +// CHECK: [[TMP51:%.*]] = and i64 [[TMP48]], -4 +// CHECK: br label [[OMP_TYPE_END25:%.*]] // CHECK: omp.type.alloc.else21: -// CHECK: [[TMP54:%.*]] = icmp eq i64 [[TMP51]], 1 -// CHECK: br i1 [[TMP54]], label [[OMP_TYPE_TO22:%.*]], label [[OMP_TYPE_TO_ELSE23:%.*]] +// CHECK: [[TMP52:%.*]] = icmp eq i64 [[TMP49]], 1 +// CHECK: br i1 [[TMP52]], label [[OMP_TYPE_TO22:%.*]], label [[OMP_TYPE_TO_ELSE23:%.*]] // CHECK: omp.type.to22: -// CHECK: [[TMP55:%.*]] = and i64 [[TMP50]], -3 +// CHECK: [[TMP53:%.*]] = and i64 [[TMP48]], -3 // CHECK: br label [[OMP_TYPE_END25]] // CHECK: omp.type.to.else23: -// CHECK: [[TMP56:%.*]] = icmp eq i64 [[TMP51]], 2 -// CHECK: br i1 [[TMP56]], label [[OMP_TYPE_FROM24:%.*]], label [[OMP_TYPE_END25]] +// CHECK: [[TMP54:%.*]] = icmp eq i64 [[TMP49]], 2 +// CHECK: br i1 [[TMP54]], label [[OMP_TYPE_FROM24:%.*]], label [[OMP_TYPE_END25]] // CHECK: omp.type.from24: -// CHECK: [[TMP57:%.*]] = and i64 [[TMP50]], -2 +// CHECK: [[TMP55:%.*]] = and i64 [[TMP48]], -2 // CHECK: br label [[OMP_TYPE_END25]] // CHECK: omp.type.end25: -// CHECK: [[OMP_MAPTYPE26:%.*]] = phi i64 [ [[TMP53]], [[OMP_TYPE_ALLOC20]] ], [ [[TMP55]], [[OMP_TYPE_TO22]] ], [ [[TMP57]], [[OMP_TYPE_FROM24]] ], [ [[TMP50]], [[OMP_TYPE_TO_ELSE23]] ] -// CHECK: [[TMP58:%.*]] = and i64 [[TMP4]], 1036 -// CHECK: [[OMP_MAPTYPE_WITH_MODIFIERS27:%.*]] = or i64 [[OMP_MAPTYPE26]], [[TMP58]] -// CHECK: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[S1P2]], ptr [[Y]], i64 4, i64 [[OMP_MAPTYPE_WITH_MODIFIERS27]], ptr null) +// CHECK: [[OMP_MAPTYPE26:%.*]] = phi i64 [ [[TMP51]], [[OMP_TYPE_ALLOC20]] ], [ [[TMP53]], [[OMP_TYPE_TO22]] ], [ [[TMP55]], [[OMP_TYPE_FROM24]] ], [ [[TMP48]], [[OMP_TYPE_TO_ELSE23]] ] +// CHECK: [[TMP56:%.*]] = and i64 [[TMP4]], 1036 +// CHECK: [[OMP_MAPTYPE_WITH_MODIFIERS27:%.*]] = or i64 [[OMP_MAPTYPE26]], [[TMP56]] +// CHECK: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP17]], ptr [[Y]], i64 4, i64 [[OMP_MAPTYPE_WITH_MODIFIERS27]], ptr null) +// CHECK: [[TMP57:%.*]] = and i64 [[TMP4]], 3 +// CHECK: [[TMP58:%.*]] = icmp eq i64 [[TMP57]], 0 +// CHECK: br i1 [[TMP58]], label [[OMP_TYPE_ALLOC28:%.*]], label [[OMP_TYPE_ALLOC_ELSE29:%.*]] +// CHECK: omp.type.alloc28: +// CHECK: br label [[OMP_TYPE_END33]] +// CHECK: omp.type.alloc.else29: +// CHECK: [[TMP59:%.*]] = icmp eq i64 [[TMP57]], 1 +// CHECK: br i1 [[TMP59]], label [[OMP_TYPE_TO30:%.*]], label [[OMP_TYPE_TO_ELSE31:%.*]] +// CHECK: omp.type.to30: +// CHECK: br label [[OMP_TYPE_END33]] +// CHECK: omp.type.to.else31: +// CHECK: [[TMP60:%.*]] = icmp eq i64 [[TMP57]], 2 +// CHECK: br i1 [[TMP60]], label [[OMP_TYPE_FROM32:%.*]], label [[OMP_TYPE_END33]] +// CHECK: omp.type.from32: +// CHECK: br label [[OMP_TYPE_END33]] +// CHECK: omp.type.end33: +// CHECK: [[OMP_MAPTYPE34:%.*]] = phi i64 [ 16384, [[OMP_TYPE_ALLOC28]] ], [ 16384, [[OMP_TYPE_TO30]] ], [ 16384, [[OMP_TYPE_FROM32]] ], [ 16384, [[OMP_TYPE_TO_ELSE31]] ] +// CHECK: [[TMP61:%.*]] = and i64 [[TMP4]], 1036 +// CHECK: [[OMP_MAPTYPE_WITH_MODIFIERS35:%.*]] = or i64 [[OMP_MAPTYPE34]], [[TMP61]] +// CHECK: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[S1P2]], ptr [[X]], i64 8, i64 [[OMP_MAPTYPE34]], ptr null) // CHECK: [[OMP_ARRAYMAP_NEXT]] = getelementptr [[STRUCT_S2]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 1 // CHECK: [[OMP_ARRAYMAP_ISDONE:%.*]] = icmp eq ptr [[OMP_ARRAYMAP_NEXT]], [[TMP7]] // CHECK: br i1 [[OMP_ARRAYMAP_ISDONE]], label [[OMP_ARRAYMAP_EXIT:%.*]], label [[OMP_ARRAYMAP_BODY]] // CHECK: omp.arraymap.exit: -// CHECK: [[OMP_ARRAYINIT_ISARRAY28:%.*]] = icmp sgt i64 [[TMP6]], 1 -// CHECK: [[TMP59:%.*]] = and i64 [[TMP4]], 8 -// CHECK: [[DOTOMP_ARRAY__DEL__DELETE:%.*]] = icmp ne i64 [[TMP59]], 0 -// CHECK: [[TMP60:%.*]] = and i1 [[OMP_ARRAYINIT_ISARRAY28]], [[DOTOMP_ARRAY__DEL__DELETE]] -// CHECK: br i1 [[TMP60]], label [[DOTOMP_ARRAY__DEL:%.*]], label [[OMP_DONE]] +// CHECK: [[OMP_ARRAYINIT_ISARRAY36:%.*]] = icmp sgt i64 [[TMP6]], 1 +// CHECK: [[TMP62:%.*]] = and i64 [[TMP4]], 8 +// CHECK: [[DOTOMP_ARRAY__DEL__DELETE:%.*]] = icmp ne i64 [[TMP62]], 0 +// CHECK: [[TMP63:%.*]] = and i1 [[OMP_ARRAYINIT_ISARRAY36]], [[DOTOMP_ARRAY__DEL__DELETE]] +// CHECK: br i1 [[TMP63]], label [[DOTOMP_ARRAY__DEL:%.*]], label [[OMP_DONE]] // CHECK: .omp.array..del: -// CHECK: [[TMP61:%.*]] = mul nuw i64 [[TMP6]], 16 -// CHECK: [[TMP62:%.*]] = and i64 [[TMP4]], -4 -// CHECK: [[TMP63:%.*]] = or i64 [[TMP62]], 512 -// CHECK: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP1]], ptr [[TMP2]], i64 [[TMP61]], i64 [[TMP63]], ptr [[TMP5]]) +// CHECK: [[TMP64:%.*]] = mul nuw i64 [[TMP6]], 16 +// CHECK: [[TMP65:%.*]] = and i64 [[TMP4]], -4 +// CHECK: [[TMP66:%.*]] = or i64 [[TMP65]], 512 +// CHECK: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP1]], ptr [[TMP2]], i64 [[TMP64]], i64 [[TMP66]], ptr [[TMP5]]) // CHECK: br label [[OMP_DONE]] // CHECK: omp.done: // CHECK: ret void @@ -276,132 +294,150 @@ void foo(S2 *arr) { // CHECK-60: [[OMP_ARRAYMAP_ISEMPTY:%.*]] = icmp eq ptr [[TMP2]], [[TMP7]] // CHECK-60: br i1 [[OMP_ARRAYMAP_ISEMPTY]], label [[OMP_DONE:%.*]], label [[OMP_ARRAYMAP_BODY:%.*]] // CHECK-60: omp.arraymap.body: -// CHECK-60: [[OMP_ARRAYMAP_PTRCURRENT:%.*]] = phi ptr [ [[TMP2]], [[OMP_ARRAYMAP_HEAD]] ], [ [[OMP_ARRAYMAP_NEXT:%.*]], [[OMP_TYPE_END25:%.*]] ] +// CHECK-60: [[OMP_ARRAYMAP_PTRCURRENT:%.*]] = phi ptr [ [[TMP2]], [[OMP_ARRAYMAP_HEAD]] ], [ [[OMP_ARRAYMAP_NEXT:%.*]], [[OMP_TYPE_END33:%.*]] ] // CHECK-60: [[Z:%.*]] = getelementptr inbounds nuw [[STRUCT_S2]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 1 // CHECK-60: [[S1P:%.*]] = getelementptr inbounds nuw [[STRUCT_S2]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 0 +// CHECK-60: [[TMP15:%.*]] = load ptr, ptr [[S1P]], align 8 // CHECK-60: [[S1P1:%.*]] = getelementptr inbounds nuw [[STRUCT_S2]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 0 -// CHECK-60: [[TMP15:%.*]] = load ptr, ptr [[S1P1]], align 8 -// CHECK-60: [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_S1:%.*]], ptr [[TMP15]], i32 0, i32 0 +// CHECK-60: [[TMP16:%.*]] = load ptr, ptr [[S1P1]], align 8 +// CHECK-60: [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_S1:%.*]], ptr [[TMP16]], i32 0, i32 0 // CHECK-60: [[S1P2:%.*]] = getelementptr inbounds nuw [[STRUCT_S2]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 0 +// CHECK-60: [[TMP17:%.*]] = load ptr, ptr [[S1P2]], align 8 // CHECK-60: [[S1P3:%.*]] = getelementptr inbounds nuw [[STRUCT_S2]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 0, i32 0 -// CHECK-60: [[TMP16:%.*]] = load ptr, ptr [[S1P3]], align 8 -// CHECK-60: [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S1]], ptr [[TMP16]], i32 0, i32 1 -// CHECK-60: [[TMP17:%.*]] = getelementptr i32, ptr [[Z]], i32 1 -// CHECK-60: [[TMP18:%.*]] = ptrtoaddr ptr [[TMP17]] to i64 -// CHECK-60: [[TMP19:%.*]] = ptrtoaddr ptr [[S1P]] to i64 -// CHECK-60: [[TMP20:%.*]] = sub i64 [[TMP18]], [[TMP19]] -// CHECK-60: [[TMP21:%.*]] = call i64 @__tgt_mapper_num_components(ptr [[TMP0]]) -// CHECK-60: [[TMP22:%.*]] = shl i64 [[TMP21]], 48 -// CHECK-60: [[TMP23:%.*]] = add nuw i64 0, [[TMP22]] -// CHECK-60: [[TMP24:%.*]] = and i64 [[TMP4]], 3 -// CHECK-60: [[TMP25:%.*]] = icmp eq i64 [[TMP24]], 0 -// CHECK-60: br i1 [[TMP25]], label [[OMP_TYPE_ALLOC:%.*]], label [[OMP_TYPE_ALLOC_ELSE:%.*]] +// CHECK-60: [[TMP18:%.*]] = load ptr, ptr [[S1P3]], align 8 +// CHECK-60: [[Y:%.*]] = getelementptr inbounds nuw [[STRUCT_S1]], ptr [[TMP18]], i32 0, i32 1 +// CHECK-60: [[TMP19:%.*]] = getelementptr i32, ptr [[Y]], i32 1 +// CHECK-60: [[TMP20:%.*]] = ptrtoaddr ptr [[TMP19]] to i64 +// CHECK-60: [[TMP21:%.*]] = ptrtoaddr ptr [[X]] to i64 +// CHECK-60: [[TMP22:%.*]] = sub i64 [[TMP20]], [[TMP21]] +// CHECK-60: [[TMP23:%.*]] = call i64 @__tgt_mapper_num_components(ptr [[TMP0]]) +// CHECK-60: [[TMP24:%.*]] = shl i64 [[TMP23]], 48 +// CHECK-60: [[TMP25:%.*]] = add nuw i64 3, [[TMP24]] +// CHECK-60: [[TMP26:%.*]] = and i64 [[TMP4]], 3 +// CHECK-60: [[TMP27:%.*]] = icmp eq i64 [[TMP26]], 0 +// CHECK-60: br i1 [[TMP27]], label [[OMP_TYPE_ALLOC:%.*]], label [[OMP_TYPE_ALLOC_ELSE:%.*]] // CHECK-60: omp.type.alloc: -// CHECK-60: [[TMP26:%.*]] = and i64 [[TMP23]], -4 +// CHECK-60: [[TMP28:%.*]] = and i64 [[TMP25]], -4 // CHECK-60: br label [[OMP_TYPE_END:%.*]] // CHECK-60: omp.type.alloc.else: -// CHECK-60: [[TMP27:%.*]] = icmp eq i64 [[TMP24]], 1 -// CHECK-60: br i1 [[TMP27]], label [[OMP_TYPE_TO:%.*]], label [[OMP_TYPE_TO_ELSE:%.*]] +// CHECK-60: [[TMP29:%.*]] = icmp eq i64 [[TMP26]], 1 +// CHECK-60: br i1 [[TMP29]], label [[OMP_TYPE_TO:%.*]], label [[OMP_TYPE_TO_ELSE:%.*]] // CHECK-60: omp.type.to: -// CHECK-60: [[TMP28:%.*]] = and i64 [[TMP23]], -3 +// CHECK-60: [[TMP30:%.*]] = and i64 [[TMP25]], -3 // CHECK-60: br label [[OMP_TYPE_END]] // CHECK-60: omp.type.to.else: -// CHECK-60: [[TMP29:%.*]] = icmp eq i64 [[TMP24]], 2 -// CHECK-60: br i1 [[TMP29]], label [[OMP_TYPE_FROM:%.*]], label [[OMP_TYPE_END]] +// CHECK-60: [[TMP31:%.*]] = icmp eq i64 [[TMP26]], 2 +// CHECK-60: br i1 [[TMP31]], label [[OMP_TYPE_FROM:%.*]], label [[OMP_TYPE_END]] // CHECK-60: omp.type.from: -// CHECK-60: [[TMP30:%.*]] = and i64 [[TMP23]], -2 +// CHECK-60: [[TMP32:%.*]] = and i64 [[TMP25]], -2 // CHECK-60: br label [[OMP_TYPE_END]] // CHECK-60: omp.type.end: -// CHECK-60: [[OMP_MAPTYPE:%.*]] = phi i64 [ [[TMP26]], [[OMP_TYPE_ALLOC]] ], [ [[TMP28]], [[OMP_TYPE_TO]] ], [ [[TMP30]], [[OMP_TYPE_FROM]] ], [ [[TMP23]], [[OMP_TYPE_TO_ELSE]] ] -// CHECK-60: [[TMP31:%.*]] = and i64 [[TMP4]], 1036 -// CHECK-60: [[OMP_MAPTYPE_WITH_MODIFIERS:%.*]] = or i64 [[OMP_MAPTYPE]], [[TMP31]] -// CHECK-60: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], ptr [[S1P]], i64 [[TMP20]], i64 [[OMP_MAPTYPE_WITH_MODIFIERS]], ptr null) -// CHECK-60: [[TMP32:%.*]] = add nuw i64 281474976710659, [[TMP22]] -// CHECK-60: [[TMP33:%.*]] = and i64 [[TMP4]], 3 -// CHECK-60: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 0 -// CHECK-60: br i1 [[TMP34]], label [[OMP_TYPE_ALLOC4:%.*]], label [[OMP_TYPE_ALLOC_ELSE5:%.*]] +// CHECK-60: [[OMP_MAPTYPE:%.*]] = phi i64 [ [[TMP28]], [[OMP_TYPE_ALLOC]] ], [ [[TMP30]], [[OMP_TYPE_TO]] ], [ [[TMP32]], [[OMP_TYPE_FROM]] ], [ [[TMP25]], [[OMP_TYPE_TO_ELSE]] ] +// CHECK-60: [[TMP33:%.*]] = and i64 [[TMP4]], 1036 +// CHECK-60: [[OMP_MAPTYPE_WITH_MODIFIERS:%.*]] = or i64 [[OMP_MAPTYPE]], [[TMP33]] +// CHECK-60: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], ptr [[Z]], i64 4, i64 [[OMP_MAPTYPE_WITH_MODIFIERS]], ptr null) +// CHECK-60: [[TMP34:%.*]] = and i64 [[TMP4]], 3 +// CHECK-60: [[TMP35:%.*]] = icmp eq i64 [[TMP34]], 0 +// CHECK-60: br i1 [[TMP35]], label [[OMP_TYPE_ALLOC4:%.*]], label [[OMP_TYPE_ALLOC_ELSE5:%.*]] // CHECK-60: omp.type.alloc4: -// CHECK-60: [[TMP35:%.*]] = and i64 [[TMP32]], -4 // CHECK-60: br label [[OMP_TYPE_END9:%.*]] // CHECK-60: omp.type.alloc.else5: -// CHECK-60: [[TMP36:%.*]] = icmp eq i64 [[TMP33]], 1 +// CHECK-60: [[TMP36:%.*]] = icmp eq i64 [[TMP34]], 1 // CHECK-60: br i1 [[TMP36]], label [[OMP_TYPE_TO6:%.*]], label [[OMP_TYPE_TO_ELSE7:%.*]] // CHECK-60: omp.type.to6: -// CHECK-60: [[TMP37:%.*]] = and i64 [[TMP32]], -3 // CHECK-60: br label [[OMP_TYPE_END9]] // CHECK-60: omp.type.to.else7: -// CHECK-60: [[TMP38:%.*]] = icmp eq i64 [[TMP33]], 2 -// CHECK-60: br i1 [[TMP38]], label [[OMP_TYPE_FROM8:%.*]], label [[OMP_TYPE_END9]] +// CHECK-60: [[TMP37:%.*]] = icmp eq i64 [[TMP34]], 2 +// CHECK-60: br i1 [[TMP37]], label [[OMP_TYPE_FROM8:%.*]], label [[OMP_TYPE_END9]] // CHECK-60: omp.type.from8: -// CHECK-60: [[TMP39:%.*]] = and i64 [[TMP32]], -2 // CHECK-60: br label [[OMP_TYPE_END9]] // CHECK-60: omp.type.end9: -// CHECK-60: [[OMP_MAPTYPE10:%.*]] = phi i64 [ [[TMP35]], [[OMP_TYPE_ALLOC4]] ], [ [[TMP37]], [[OMP_TYPE_TO6]] ], [ [[TMP39]], [[OMP_TYPE_FROM8]] ], [ [[TMP32]], [[OMP_TYPE_TO_ELSE7]] ] -// CHECK-60: [[TMP40:%.*]] = and i64 [[TMP4]], 1036 -// CHECK-60: [[OMP_MAPTYPE_WITH_MODIFIERS11:%.*]] = or i64 [[OMP_MAPTYPE10]], [[TMP40]] -// CHECK-60: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], ptr [[Z]], i64 4, i64 [[OMP_MAPTYPE_WITH_MODIFIERS11]], ptr null) -// CHECK-60: [[TMP41:%.*]] = add nuw i64 281474976710675, [[TMP22]] -// CHECK-60: [[TMP42:%.*]] = and i64 [[TMP4]], 3 -// CHECK-60: [[TMP43:%.*]] = icmp eq i64 [[TMP42]], 0 -// CHECK-60: br i1 [[TMP43]], label [[OMP_TYPE_ALLOC12:%.*]], label [[OMP_TYPE_ALLOC_ELSE13:%.*]] +// CHECK-60: [[OMP_MAPTYPE10:%.*]] = phi i64 [ 0, [[OMP_TYPE_ALLOC4]] ], [ 0, [[OMP_TYPE_TO6]] ], [ 0, [[OMP_TYPE_FROM8]] ], [ 0, [[OMP_TYPE_TO_ELSE7]] ] +// CHECK-60: [[TMP38:%.*]] = and i64 [[TMP4]], 1036 +// CHECK-60: [[OMP_MAPTYPE_WITH_MODIFIERS11:%.*]] = or i64 [[OMP_MAPTYPE10]], [[TMP38]] +// CHECK-60: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP15]], ptr [[X]], i64 [[TMP22]], i64 [[OMP_MAPTYPE_WITH_MODIFIERS11]], ptr null) +// CHECK-60: [[TMP39:%.*]] = add nuw i64 562949953421315, [[TMP24]] +// CHECK-60: [[TMP40:%.*]] = and i64 [[TMP4]], 3 +// CHECK-60: [[TMP41:%.*]] = icmp eq i64 [[TMP40]], 0 +// CHECK-60: br i1 [[TMP41]], label [[OMP_TYPE_ALLOC12:%.*]], label [[OMP_TYPE_ALLOC_ELSE13:%.*]] // CHECK-60: omp.type.alloc12: -// CHECK-60: [[TMP44:%.*]] = and i64 [[TMP41]], -4 +// CHECK-60: [[TMP42:%.*]] = and i64 [[TMP39]], -4 // CHECK-60: br label [[OMP_TYPE_END17:%.*]] // CHECK-60: omp.type.alloc.else13: -// CHECK-60: [[TMP45:%.*]] = icmp eq i64 [[TMP42]], 1 -// CHECK-60: br i1 [[TMP45]], label [[OMP_TYPE_TO14:%.*]], label [[OMP_TYPE_TO_ELSE15:%.*]] +// CHECK-60: [[TMP43:%.*]] = icmp eq i64 [[TMP40]], 1 +// CHECK-60: br i1 [[TMP43]], label [[OMP_TYPE_TO14:%.*]], label [[OMP_TYPE_TO_ELSE15:%.*]] // CHECK-60: omp.type.to14: -// CHECK-60: [[TMP46:%.*]] = and i64 [[TMP41]], -3 +// CHECK-60: [[TMP44:%.*]] = and i64 [[TMP39]], -3 // CHECK-60: br label [[OMP_TYPE_END17]] // CHECK-60: omp.type.to.else15: -// CHECK-60: [[TMP47:%.*]] = icmp eq i64 [[TMP42]], 2 -// CHECK-60: br i1 [[TMP47]], label [[OMP_TYPE_FROM16:%.*]], label [[OMP_TYPE_END17]] +// CHECK-60: [[TMP45:%.*]] = icmp eq i64 [[TMP40]], 2 +// CHECK-60: br i1 [[TMP45]], label [[OMP_TYPE_FROM16:%.*]], label [[OMP_TYPE_END17]] // CHECK-60: omp.type.from16: -// CHECK-60: [[TMP48:%.*]] = and i64 [[TMP41]], -2 +// CHECK-60: [[TMP46:%.*]] = and i64 [[TMP39]], -2 // CHECK-60: br label [[OMP_TYPE_END17]] // CHECK-60: omp.type.end17: -// CHECK-60: [[OMP_MAPTYPE18:%.*]] = phi i64 [ [[TMP44]], [[OMP_TYPE_ALLOC12]] ], [ [[TMP46]], [[OMP_TYPE_TO14]] ], [ [[TMP48]], [[OMP_TYPE_FROM16]] ], [ [[TMP41]], [[OMP_TYPE_TO_ELSE15]] ] -// CHECK-60: [[TMP49:%.*]] = and i64 [[TMP4]], 1036 -// CHECK-60: [[OMP_MAPTYPE_WITH_MODIFIERS19:%.*]] = or i64 [[OMP_MAPTYPE18]], [[TMP49]] -// CHECK-60: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[S1P]], ptr [[X]], i64 4, i64 [[OMP_MAPTYPE_WITH_MODIFIERS19]], ptr null) -// CHECK-60: [[TMP50:%.*]] = add nuw i64 281474976710675, [[TMP22]] -// CHECK-60: [[TMP51:%.*]] = and i64 [[TMP4]], 3 -// CHECK-60: [[TMP52:%.*]] = icmp eq i64 [[TMP51]], 0 -// CHECK-60: br i1 [[TMP52]], label [[OMP_TYPE_ALLOC20:%.*]], label [[OMP_TYPE_ALLOC_ELSE21:%.*]] +// CHECK-60: [[OMP_MAPTYPE18:%.*]] = phi i64 [ [[TMP42]], [[OMP_TYPE_ALLOC12]] ], [ [[TMP44]], [[OMP_TYPE_TO14]] ], [ [[TMP46]], [[OMP_TYPE_FROM16]] ], [ [[TMP39]], [[OMP_TYPE_TO_ELSE15]] ] +// CHECK-60: [[TMP47:%.*]] = and i64 [[TMP4]], 1036 +// CHECK-60: [[OMP_MAPTYPE_WITH_MODIFIERS19:%.*]] = or i64 [[OMP_MAPTYPE18]], [[TMP47]] +// CHECK-60: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP15]], ptr [[X]], i64 4, i64 [[OMP_MAPTYPE_WITH_MODIFIERS19]], ptr null) +// CHECK-60: [[TMP48:%.*]] = add nuw i64 562949953421315, [[TMP24]] +// CHECK-60: [[TMP49:%.*]] = and i64 [[TMP4]], 3 +// CHECK-60: [[TMP50:%.*]] = icmp eq i64 [[TMP49]], 0 +// CHECK-60: br i1 [[TMP50]], label [[OMP_TYPE_ALLOC20:%.*]], label [[OMP_TYPE_ALLOC_ELSE21:%.*]] // CHECK-60: omp.type.alloc20: -// CHECK-60: [[TMP53:%.*]] = and i64 [[TMP50]], -4 -// CHECK-60: br label [[OMP_TYPE_END25]] +// CHECK-60: [[TMP51:%.*]] = and i64 [[TMP48]], -4 +// CHECK-60: br label [[OMP_TYPE_END25:%.*]] // CHECK-60: omp.type.alloc.else21: -// CHECK-60: [[TMP54:%.*]] = icmp eq i64 [[TMP51]], 1 -// CHECK-60: br i1 [[TMP54]], label [[OMP_TYPE_TO22:%.*]], label [[OMP_TYPE_TO_ELSE23:%.*]] +// CHECK-60: [[TMP52:%.*]] = icmp eq i64 [[TMP49]], 1 +// CHECK-60: br i1 [[TMP52]], label [[OMP_TYPE_TO22:%.*]], label [[OMP_TYPE_TO_ELSE23:%.*]] // CHECK-60: omp.type.to22: -// CHECK-60: [[TMP55:%.*]] = and i64 [[TMP50]], -3 +// CHECK-60: [[TMP53:%.*]] = and i64 [[TMP48]], -3 // CHECK-60: br label [[OMP_TYPE_END25]] // CHECK-60: omp.type.to.else23: -// CHECK-60: [[TMP56:%.*]] = icmp eq i64 [[TMP51]], 2 -// CHECK-60: br i1 [[TMP56]], label [[OMP_TYPE_FROM24:%.*]], label [[OMP_TYPE_END25]] +// CHECK-60: [[TMP54:%.*]] = icmp eq i64 [[TMP49]], 2 +// CHECK-60: br i1 [[TMP54]], label [[OMP_TYPE_FROM24:%.*]], label [[OMP_TYPE_END25]] // CHECK-60: omp.type.from24: -// CHECK-60: [[TMP57:%.*]] = and i64 [[TMP50]], -2 +// CHECK-60: [[TMP55:%.*]] = and i64 [[TMP48]], -2 // CHECK-60: br label [[OMP_TYPE_END25]] // CHECK-60: omp.type.end25: -// CHECK-60: [[OMP_MAPTYPE26:%.*]] = phi i64 [ [[TMP53]], [[OMP_TYPE_ALLOC20]] ], [ [[TMP55]], [[OMP_TYPE_TO22]] ], [ [[TMP57]], [[OMP_TYPE_FROM24]] ], [ [[TMP50]], [[OMP_TYPE_TO_ELSE23]] ] -// CHECK-60: [[TMP58:%.*]] = and i64 [[TMP4]], 1036 -// CHECK-60: [[OMP_MAPTYPE_WITH_MODIFIERS27:%.*]] = or i64 [[OMP_MAPTYPE26]], [[TMP58]] -// CHECK-60: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[S1P2]], ptr [[Y]], i64 4, i64 [[OMP_MAPTYPE_WITH_MODIFIERS27]], ptr null) +// CHECK-60: [[OMP_MAPTYPE26:%.*]] = phi i64 [ [[TMP51]], [[OMP_TYPE_ALLOC20]] ], [ [[TMP53]], [[OMP_TYPE_TO22]] ], [ [[TMP55]], [[OMP_TYPE_FROM24]] ], [ [[TMP48]], [[OMP_TYPE_TO_ELSE23]] ] +// CHECK-60: [[TMP56:%.*]] = and i64 [[TMP4]], 1036 +// CHECK-60: [[OMP_MAPTYPE_WITH_MODIFIERS27:%.*]] = or i64 [[OMP_MAPTYPE26]], [[TMP56]] +// CHECK-60: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP17]], ptr [[Y]], i64 4, i64 [[OMP_MAPTYPE_WITH_MODIFIERS27]], ptr null) +// CHECK-60: [[TMP57:%.*]] = and i64 [[TMP4]], 3 +// CHECK-60: [[TMP58:%.*]] = icmp eq i64 [[TMP57]], 0 +// CHECK-60: br i1 [[TMP58]], label [[OMP_TYPE_ALLOC28:%.*]], label [[OMP_TYPE_ALLOC_ELSE29:%.*]] +// CHECK-60: omp.type.alloc28: +// CHECK-60: br label [[OMP_TYPE_END33]] +// CHECK-60: omp.type.alloc.else29: +// CHECK-60: [[TMP59:%.*]] = icmp eq i64 [[TMP57]], 1 +// CHECK-60: br i1 [[TMP59]], label [[OMP_TYPE_TO30:%.*]], label [[OMP_TYPE_TO_ELSE31:%.*]] +// CHECK-60: omp.type.to30: +// CHECK-60: br label [[OMP_TYPE_END33]] +// CHECK-60: omp.type.to.else31: +// CHECK-60: [[TMP60:%.*]] = icmp eq i64 [[TMP57]], 2 +// CHECK-60: br i1 [[TMP60]], label [[OMP_TYPE_FROM32:%.*]], label [[OMP_TYPE_END33]] +// CHECK-60: omp.type.from32: +// CHECK-60: br label [[OMP_TYPE_END33]] +// CHECK-60: omp.type.end33: +// CHECK-60: [[OMP_MAPTYPE34:%.*]] = phi i64 [ 16384, [[OMP_TYPE_ALLOC28]] ], [ 16384, [[OMP_TYPE_TO30]] ], [ 16384, [[OMP_TYPE_FROM32]] ], [ 16384, [[OMP_TYPE_TO_ELSE31]] ] +// CHECK-60: [[TMP61:%.*]] = and i64 [[TMP4]], 1036 +// CHECK-60: [[OMP_MAPTYPE_WITH_MODIFIERS35:%.*]] = or i64 [[OMP_MAPTYPE34]], [[TMP61]] +// CHECK-60: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[S1P2]], ptr [[X]], i64 8, i64 [[OMP_MAPTYPE34]], ptr null) // CHECK-60: [[OMP_ARRAYMAP_NEXT]] = getelementptr [[STRUCT_S2]], ptr [[OMP_ARRAYMAP_PTRCURRENT]], i32 1 // CHECK-60: [[OMP_ARRAYMAP_ISDONE:%.*]] = icmp eq ptr [[OMP_ARRAYMAP_NEXT]], [[TMP7]] // CHECK-60: br i1 [[OMP_ARRAYMAP_ISDONE]], label [[OMP_ARRAYMAP_EXIT:%.*]], label [[OMP_ARRAYMAP_BODY]] // CHECK-60: omp.arraymap.exit: -// CHECK-60: [[OMP_ARRAYINIT_ISARRAY28:%.*]] = icmp sgt i64 [[TMP6]], 1 -// CHECK-60: [[TMP59:%.*]] = and i64 [[TMP4]], 8 -// CHECK-60: [[DOTOMP_ARRAY__DEL__DELETE:%.*]] = icmp ne i64 [[TMP59]], 0 -// CHECK-60: [[TMP60:%.*]] = and i1 [[OMP_ARRAYINIT_ISARRAY28]], [[DOTOMP_ARRAY__DEL__DELETE]] -// CHECK-60: br i1 [[TMP60]], label [[DOTOMP_ARRAY__DEL:%.*]], label [[OMP_DONE]] +// CHECK-60: [[OMP_ARRAYINIT_ISARRAY36:%.*]] = icmp sgt i64 [[TMP6]], 1 +// CHECK-60: [[TMP62:%.*]] = and i64 [[TMP4]], 8 +// CHECK-60: [[DOTOMP_ARRAY__DEL__DELETE:%.*]] = icmp ne i64 [[TMP62]], 0 +// CHECK-60: [[TMP63:%.*]] = and i1 [[OMP_ARRAYINIT_ISARRAY36]], [[DOTOMP_ARRAY__DEL__DELETE]] +// CHECK-60: br i1 [[TMP63]], label [[DOTOMP_ARRAY__DEL:%.*]], label [[OMP_DONE]] // CHECK-60: .omp.array..del: -// CHECK-60: [[TMP61:%.*]] = mul nuw i64 [[TMP6]], 16 -// CHECK-60: [[TMP62:%.*]] = and i64 [[TMP4]], -4 -// CHECK-60: [[TMP63:%.*]] = or i64 [[TMP62]], 512 -// CHECK-60: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP1]], ptr [[TMP2]], i64 [[TMP61]], i64 [[TMP63]], ptr [[TMP5]]) +// CHECK-60: [[TMP64:%.*]] = mul nuw i64 [[TMP6]], 16 +// CHECK-60: [[TMP65:%.*]] = and i64 [[TMP4]], -4 +// CHECK-60: [[TMP66:%.*]] = or i64 [[TMP65]], 512 +// CHECK-60: call void @__tgt_push_mapper_component(ptr [[TMP0]], ptr [[TMP1]], ptr [[TMP2]], i64 [[TMP64]], i64 [[TMP66]], ptr [[TMP5]]) // CHECK-60: br label [[OMP_DONE]] // CHECK-60: omp.done: // CHECK-60: ret void diff --git a/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp b/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp index 72ead7c0b34db..40d77abe2ef7c 100644 --- a/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp +++ b/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -349,8 +349,8 @@ int main(int argc, const char **argv) { unsigned HostTargetNum = 0u; bool HIPOnly = true; llvm::DenseSet ParsedTargets; - // Map {offload-kind}-{triple} to its device triple and target IDs. - std::map>> TargetIDs; + // Map {offload-kind}-{triple} to target IDs. + std::map> TargetIDs; // Standardize target names to include env field std::vector StandardizedTargetNames; for (StringRef Target : TargetNames) { @@ -385,10 +385,8 @@ int main(int argc, const char **argv) { return reportError(createStringError(errc::invalid_argument, Msg.str())); } - auto &Entry = TargetIDs[OffloadInfo.OffloadKind.str() + "-" + - OffloadInfo.Triple.str()]; - Entry.first = OffloadInfo.Triple; - Entry.second.insert(OffloadInfo.TargetID); + TargetIDs[OffloadInfo.OffloadKind.str() + "-" + OffloadInfo.Triple.str()] + .insert(OffloadInfo.TargetID); if (KindIsValid && OffloadInfo.hasHostKind()) { ++HostTargetNum; // Save the index of the input that refers to the host. @@ -404,17 +402,14 @@ int main(int argc, const char **argv) { BundlerConfig.TargetNames.assign(StandardizedTargetNames.begin(), StandardizedTargetNames.end()); - for (const auto &[Key, TripleAndIDs] : TargetIDs) { - const auto &[Triple, IDs] = TripleAndIDs; - llvm::SmallVector Entries; - for (StringRef ID : IDs) - Entries.emplace_back(Triple, ID); - if (auto ConflictingTID = clang::getConflictTargetIDCombination(Entries)) { + for (const auto &TargetID : TargetIDs) { + if (auto ConflictingTID = + clang::getConflictTargetIDCombination(TargetID.second)) { SmallVector Buf; raw_svector_ostream Msg(Buf); Msg << "Cannot bundle inputs with conflicting targets: '" - << Key + "-" + ConflictingTID->first << "' and '" - << Key + "-" + ConflictingTID->second << "'"; + << TargetID.first + "-" + ConflictingTID->first << "' and '" + << TargetID.first + "-" + ConflictingTID->second << "'"; return reportError(createStringError(errc::invalid_argument, Msg.str())); } } diff --git a/flang/lib/Optimizer/Transforms/CUDA/CUFAddConstructor.cpp b/flang/lib/Optimizer/Transforms/CUDA/CUFAddConstructor.cpp index 04e4012c283a3..5c70342d1e0fa 100644 --- a/flang/lib/Optimizer/Transforms/CUDA/CUFAddConstructor.cpp +++ b/flang/lib/Optimizer/Transforms/CUDA/CUFAddConstructor.cpp @@ -90,6 +90,14 @@ static bool isDeviceExternReference(fir::GlobalOp hostGlobal, return !gpuGlobal.isInitialized(); } +/// Return true if \p globalOp defines the variable rather than just declaring +/// it. A variable USEd from another translation unit has no body here; its +/// device symbol belongs to the defining unit's device module, so registering +/// the declaration would bind the host address to the wrong module. +static bool definesGlobal(fir::GlobalOp globalOp) { + return globalOp.isInitialized(); +} + /// Return true if \p hostGlobal is a host module-scope global that has been /// mirrored in the GPU module as an external (no-body) declaration by the /// CUFDeviceGlobal pass under -gpu=mem:unified. @@ -214,9 +222,13 @@ static bool hasRegisteredGlobals(mlir::ModuleOp mod, } if (!gpuSymTable.lookup(globalOp.getSymName())) continue; + // Non-allocatable managed globals register a companion pointer local to + // this translation unit, so they register even when defined elsewhere. if (attr.getValue() == cuf::DataAttribute::Managed && !mlir::isa(globalOp.getType())) return true; + if (!definesGlobal(globalOp)) + continue; switch (attr.getValue()) { case cuf::DataAttribute::Device: case cuf::DataAttribute::Constant: @@ -365,6 +377,12 @@ struct CUFAddConstructor attr.getValue() == cuf::DataAttribute::Managed && !mlir::isa(globalOp.getType()); + // Non-allocatable managed globals register a companion pointer local + // to this translation unit, so they register even when defined + // elsewhere. + if (!definesGlobal(globalOp) && !isNonAllocManagedGlobal) + continue; + switch (attr.getValue()) { case cuf::DataAttribute::Device: case cuf::DataAttribute::Constant: diff --git a/flang/test/Fir/CUDA/cuda-constructor-2.f90 b/flang/test/Fir/CUDA/cuda-constructor-2.f90 index 42125a6460495..e93a721a192f0 100644 --- a/flang/test/Fir/CUDA/cuda-constructor-2.f90 +++ b/flang/test/Fir/CUDA/cuda-constructor-2.f90 @@ -311,3 +311,43 @@ module attributes {dlti.dl_spec = #dlti.dl_spec = dense<32> : vec // NOUNIFIED: fir.call @_FortranACUFRegisterVariable // UNIFIED: cuf.register_variable_static @_QMallocmodEac("_QMallocmodEac", 40) {deviceResident} // UNIFIED: cuf.register_variable_static @_QMallocmodEad("_QMallocmodEad", 48) {deviceResident} + +// ----- + +// A translation unit that only USEs the module sees its variables as +// declarations (no body). Registering them here would bind the host address +// to a device module that does not contain the symbol. + +module attributes {dlti.dl_spec = #dlti.dl_spec : vector<2xi64>, i16 = dense<16> : vector<2xi64>, i1 = dense<8> : vector<2xi64>, !llvm.ptr = dense<64> : vector<4xi64>, f80 = dense<128> : vector<2xi64>, i128 = dense<128> : vector<2xi64>, i64 = dense<64> : vector<2xi64>, !llvm.ptr<271> = dense<32> : vector<4xi64>, !llvm.ptr<272> = dense<64> : vector<4xi64>, f128 = dense<128> : vector<2xi64>, !llvm.ptr<270> = dense<32> : vector<4xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, i32 = dense<32> : vector<2xi64>, "dlti.stack_alignment" = 128 : i64, "dlti.endianness" = "little">, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} { + fir.global @_QMotherEdev_var {data_attr = #cuf.cuda} : !fir.array<5xi32> + fir.global @_QMotherEman_var {data_attr = #cuf.cuda} : !fir.box>> + gpu.module @cuda_device_mod { + gpu.func @_QMotherPkernel() kernel { + gpu.return + } + fir.global @_QMotherEdev_var {data_attr = #cuf.cuda} : !fir.array<5xi32> + fir.global @_QMotherEman_var {data_attr = #cuf.cuda} : !fir.box>> + } +} + +// CHECK: llvm.func internal @__cudaFortranConstructor() +// CHECK: cuf.register_module @cuda_device_mod +// CHECK-NOT: fir.call @_FortranACUFRegisterVariable +// CHECK-NOT: fir.call @_FortranACUFRegisterManagedVariable +// CHECK: llvm.mlir.global_ctors ctors = [@__cudaFortranConstructor] + +// ----- + +// Same without a kernel: module registration is skipped as well. + +module attributes {dlti.dl_spec = #dlti.dl_spec : vector<2xi64>, i16 = dense<16> : vector<2xi64>, i1 = dense<8> : vector<2xi64>, !llvm.ptr = dense<64> : vector<4xi64>, f80 = dense<128> : vector<2xi64>, i128 = dense<128> : vector<2xi64>, i64 = dense<64> : vector<2xi64>, !llvm.ptr<271> = dense<32> : vector<4xi64>, !llvm.ptr<272> = dense<64> : vector<4xi64>, f128 = dense<128> : vector<2xi64>, !llvm.ptr<270> = dense<32> : vector<4xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, i32 = dense<32> : vector<2xi64>, "dlti.stack_alignment" = 128 : i64, "dlti.endianness" = "little">, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} { + fir.global @_QMotherEdev_var {data_attr = #cuf.cuda} : !fir.array<5xi32> + gpu.module @cuda_device_mod { + fir.global @_QMotherEdev_var {data_attr = #cuf.cuda} : !fir.array<5xi32> + } +} + +// CHECK: llvm.func internal @__cudaFortranConstructor() +// CHECK-NOT: cuf.register_module +// CHECK-NOT: fir.call @_FortranACUFRegisterVariable +// CHECK: llvm.mlir.global_ctors ctors = [@__cudaFortranConstructor] diff --git a/lldb/include/lldb/API/SBAttachInfo.h b/lldb/include/lldb/API/SBAttachInfo.h index c18655fee77e0..94f687cd56649 100644 --- a/lldb/include/lldb/API/SBAttachInfo.h +++ b/lldb/include/lldb/API/SBAttachInfo.h @@ -199,7 +199,7 @@ class LLDB_API SBAttachInfo { friend class SBTarget; friend class SBPlatform; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; lldb_private::ProcessAttachInfo &ref(); diff --git a/lldb/include/lldb/API/SBBreakpoint.h b/lldb/include/lldb/API/SBBreakpoint.h index fe19ba998ea67..95c32fbb583bc 100644 --- a/lldb/include/lldb/API/SBBreakpoint.h +++ b/lldb/include/lldb/API/SBBreakpoint.h @@ -171,7 +171,7 @@ class LLDB_API SBBreakpoint { friend class SBBreakpointName; friend class SBTarget; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; friend class lldb_private::python::SWIGBridge; SBBreakpoint(const lldb::BreakpointSP &bp_sp); diff --git a/lldb/include/lldb/API/SBBreakpointLocation.h b/lldb/include/lldb/API/SBBreakpointLocation.h index 9b0d4839aca82..3255a51d9269f 100644 --- a/lldb/include/lldb/API/SBBreakpointLocation.h +++ b/lldb/include/lldb/API/SBBreakpointLocation.h @@ -24,7 +24,7 @@ class SWIGBridge; namespace lldb { class LLDB_API SBBreakpointLocation { - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; public: SBBreakpointLocation(); diff --git a/lldb/include/lldb/API/SBCommandReturnObject.h b/lldb/include/lldb/API/SBCommandReturnObject.h index b80a11b52c77f..4a7ea3f955305 100644 --- a/lldb/include/lldb/API/SBCommandReturnObject.h +++ b/lldb/include/lldb/API/SBCommandReturnObject.h @@ -145,7 +145,7 @@ class LLDB_API SBCommandReturnObject { friend class lldb_private::CommandPluginInterfaceImplementation; friend class lldb_private::python::SWIGBridge; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; SBCommandReturnObject(lldb_private::CommandReturnObject &ref); diff --git a/lldb/include/lldb/API/SBData.h b/lldb/include/lldb/API/SBData.h index 89a699f2f713a..d2fe33a3e0b83 100644 --- a/lldb/include/lldb/API/SBData.h +++ b/lldb/include/lldb/API/SBData.h @@ -154,7 +154,7 @@ class LLDB_API SBData { friend class SBTarget; friend class SBValue; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; lldb::DataExtractorSP m_opaque_sp; }; diff --git a/lldb/include/lldb/API/SBDebugger.h b/lldb/include/lldb/API/SBDebugger.h index 3e302f121f5ec..bb13413e7a556 100644 --- a/lldb/include/lldb/API/SBDebugger.h +++ b/lldb/include/lldb/API/SBDebugger.h @@ -678,7 +678,7 @@ class LLDB_API SBDebugger { protected: friend class lldb_private::CommandPluginInterfaceImplementation; friend class lldb_private::python::SWIGBridge; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; friend class lldb_private::SystemInitializerFull; SBDebugger(const lldb::DebuggerSP &debugger_sp); diff --git a/lldb/include/lldb/API/SBError.h b/lldb/include/lldb/API/SBError.h index dd8c0f939775f..5f2717120006a 100644 --- a/lldb/include/lldb/API/SBError.h +++ b/lldb/include/lldb/API/SBError.h @@ -109,7 +109,7 @@ class LLDB_API SBError { friend class SBValueList; friend class SBWatchpoint; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; friend class lldb_private::python::SWIGBridge; SBError(lldb_private::Status &&error); diff --git a/lldb/include/lldb/API/SBEvent.h b/lldb/include/lldb/API/SBEvent.h index 85b401ca8cc10..99f13fc90124d 100644 --- a/lldb/include/lldb/API/SBEvent.h +++ b/lldb/include/lldb/API/SBEvent.h @@ -74,7 +74,7 @@ class LLDB_API SBEvent { friend class SBThread; friend class SBWatchpoint; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; friend class lldb_private::python::SWIGBridge; SBEvent(lldb::EventSP &event_sp); diff --git a/lldb/include/lldb/API/SBExecutionContext.h b/lldb/include/lldb/API/SBExecutionContext.h index 20584271ff36c..3b2d0e0aa139d 100644 --- a/lldb/include/lldb/API/SBExecutionContext.h +++ b/lldb/include/lldb/API/SBExecutionContext.h @@ -57,7 +57,7 @@ class LLDB_API SBExecutionContext { protected: friend class SBInstructionList; friend class lldb_private::python::SWIGBridge; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; lldb_private::ExecutionContextRef *get() const; diff --git a/lldb/include/lldb/API/SBFrame.h b/lldb/include/lldb/API/SBFrame.h index eaf9a4bfece96..7094ce3ad8fb7 100644 --- a/lldb/include/lldb/API/SBFrame.h +++ b/lldb/include/lldb/API/SBFrame.h @@ -231,7 +231,7 @@ class LLDB_API SBFrame { friend class SBThread; friend class SBValue; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; friend class lldb_private::python::SWIGBridge; friend class lldb_private::lua::SWIGBridge; diff --git a/lldb/include/lldb/API/SBFrameList.h b/lldb/include/lldb/API/SBFrameList.h index 0039ffb1f863f..ef38510d9fa66 100644 --- a/lldb/include/lldb/API/SBFrameList.h +++ b/lldb/include/lldb/API/SBFrameList.h @@ -78,7 +78,7 @@ class LLDB_API SBFrameList { friend class lldb_private::python::SWIGBridge; friend class lldb_private::lua::SWIGBridge; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; private: SBFrameList(const lldb::StackFrameListSP &frame_list_sp); diff --git a/lldb/include/lldb/API/SBLaunchInfo.h b/lldb/include/lldb/API/SBLaunchInfo.h index 06e72efc30f9f..043a9a54734a1 100644 --- a/lldb/include/lldb/API/SBLaunchInfo.h +++ b/lldb/include/lldb/API/SBLaunchInfo.h @@ -210,7 +210,7 @@ class LLDB_API SBLaunchInfo { friend class SBPlatform; friend class SBTarget; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; const lldb_private::ProcessLaunchInfo &ref() const; void set_ref(const lldb_private::ProcessLaunchInfo &info); diff --git a/lldb/include/lldb/API/SBMemoryRegionInfo.h b/lldb/include/lldb/API/SBMemoryRegionInfo.h index dc5aa0858e1e3..034279f6e5593 100644 --- a/lldb/include/lldb/API/SBMemoryRegionInfo.h +++ b/lldb/include/lldb/API/SBMemoryRegionInfo.h @@ -132,7 +132,7 @@ class LLDB_API SBMemoryRegionInfo { friend class SBProcess; friend class SBMemoryRegionInfoList; friend class SBSaveCoreOptions; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; lldb_private::MemoryRegionInfo &ref(); diff --git a/lldb/include/lldb/API/SBStream.h b/lldb/include/lldb/API/SBStream.h index 21f9d21e0e717..1ba375b600df6 100644 --- a/lldb/include/lldb/API/SBStream.h +++ b/lldb/include/lldb/API/SBStream.h @@ -108,7 +108,7 @@ class LLDB_API SBStream { friend class SBValue; friend class SBWatchpoint; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; lldb_private::Stream *operator->(); diff --git a/lldb/include/lldb/API/SBSymbolContext.h b/lldb/include/lldb/API/SBSymbolContext.h index 19f29c629d094..c67f5ba0e0658 100644 --- a/lldb/include/lldb/API/SBSymbolContext.h +++ b/lldb/include/lldb/API/SBSymbolContext.h @@ -66,7 +66,7 @@ class LLDB_API SBSymbolContext { friend class SBTarget; friend class SBSymbolContextList; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; friend class lldb_private::python::SWIGBridge; SBSymbolContext(const lldb_private::SymbolContext &sc_ptr); @@ -81,8 +81,6 @@ class LLDB_API SBSymbolContext { lldb_private::SymbolContext *get() const; - friend class lldb_private::ScriptInterpreter; - private: std::unique_ptr m_opaque_up; }; diff --git a/lldb/include/lldb/API/SBTarget.h b/lldb/include/lldb/API/SBTarget.h index fd795c843330e..84cbcfb4e69d2 100644 --- a/lldb/include/lldb/API/SBTarget.h +++ b/lldb/include/lldb/API/SBTarget.h @@ -1067,7 +1067,7 @@ class LLDB_API SBTarget { friend class lldb_private::python::SWIGBridge; friend class lldb_private::lua::SWIGBridge; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; // Constructors are private, use static Target::Create function to create an // instance of this class. diff --git a/lldb/include/lldb/API/SBThread.h b/lldb/include/lldb/API/SBThread.h index 97d3b838492fb..a5edb529c2c6a 100644 --- a/lldb/include/lldb/API/SBThread.h +++ b/lldb/include/lldb/API/SBThread.h @@ -256,7 +256,7 @@ class LLDB_API SBThread { friend class SBThreadPlan; friend class SBTrace; - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; friend class lldb_private::python::SWIGBridge; SBThread(const lldb::ThreadSP &lldb_object_sp); diff --git a/lldb/include/lldb/API/SBValue.h b/lldb/include/lldb/API/SBValue.h index 9d746f74c9b09..68ae063295418 100644 --- a/lldb/include/lldb/API/SBValue.h +++ b/lldb/include/lldb/API/SBValue.h @@ -532,7 +532,7 @@ class LLDB_API SBValue { bool use_synthetic, const char *name); protected: - friend class lldb_private::ScriptInterpreter; + friend class lldb_private::ScriptInterpreterBridge; private: typedef std::shared_ptr ValueImplSP; diff --git a/lldb/include/lldb/Interpreter/Interfaces/ScriptedFrameInterface.h b/lldb/include/lldb/Interpreter/Interfaces/ScriptedFrameInterface.h index 43914ef705dbf..b2a37bf497504 100644 --- a/lldb/include/lldb/Interpreter/Interfaces/ScriptedFrameInterface.h +++ b/lldb/include/lldb/Interpreter/Interfaces/ScriptedFrameInterface.h @@ -10,7 +10,6 @@ #define LLDB_INTERPRETER_INTERFACES_SCRIPTEDFRAMEINTERFACE_H #include "ScriptedInterface.h" -#include "lldb/API/SBValueList.h" #include "lldb/Core/StructuredDataImpl.h" #include "lldb/Symbol/SymbolContext.h" #include "lldb/lldb-private.h" diff --git a/lldb/include/lldb/Interpreter/ScriptInterpreter.h b/lldb/include/lldb/Interpreter/ScriptInterpreter.h index 925b7b08e3291..7ad530b3233f2 100644 --- a/lldb/include/lldb/Interpreter/ScriptInterpreter.h +++ b/lldb/include/lldb/Interpreter/ScriptInterpreter.h @@ -9,21 +9,6 @@ #ifndef LLDB_INTERPRETER_SCRIPTINTERPRETER_H #define LLDB_INTERPRETER_SCRIPTINTERPRETER_H -#include "lldb/API/SBAttachInfo.h" -#include "lldb/API/SBBreakpoint.h" -#include "lldb/API/SBBreakpointLocation.h" -#include "lldb/API/SBCommandReturnObject.h" -#include "lldb/API/SBData.h" -#include "lldb/API/SBDebugger.h" -#include "lldb/API/SBError.h" -#include "lldb/API/SBEvent.h" -#include "lldb/API/SBExecutionContext.h" -#include "lldb/API/SBFrameList.h" -#include "lldb/API/SBLaunchInfo.h" -#include "lldb/API/SBMemoryRegionInfo.h" -#include "lldb/API/SBStream.h" -#include "lldb/API/SBSymbolContext.h" -#include "lldb/API/SBThread.h" #include "lldb/Breakpoint/BreakpointOptions.h" #include "lldb/Core/PluginInterface.h" #include "lldb/Core/SearchFilter.h" @@ -37,7 +22,6 @@ #include "lldb/Interpreter/Interfaces/ScriptedProcessInterface.h" #include "lldb/Interpreter/Interfaces/ScriptedThreadInterface.h" #include "lldb/Interpreter/ScriptObject.h" -#include "lldb/Symbol/SymbolContext.h" #include "lldb/Utility/Broadcaster.h" #include "lldb/Utility/Status.h" #include "lldb/Utility/StructuredData.h" @@ -531,54 +515,6 @@ class ScriptInterpreter : public PluginInterface { virtual SanitizedScriptingModuleName GetSanitizedScriptingModuleName(llvm::StringRef name); - lldb::DataExtractorSP - GetDataExtractorFromSBData(const lldb::SBData &data) const; - - Status GetStatusFromSBError(const lldb::SBError &error) const; - - Event *GetOpaqueTypeFromSBEvent(const lldb::SBEvent &event) const; - - lldb::StreamSP GetOpaqueTypeFromSBStream(const lldb::SBStream &stream) const; - - lldb::ThreadSP GetOpaqueTypeFromSBThread(const lldb::SBThread &exe_ctx) const; - - lldb::StackFrameSP GetOpaqueTypeFromSBFrame(const lldb::SBFrame &frame) const; - - SymbolContext - GetOpaqueTypeFromSBSymbolContext(const lldb::SBSymbolContext &sym_ctx) const; - - lldb::BreakpointSP - GetOpaqueTypeFromSBBreakpoint(const lldb::SBBreakpoint &breakpoint) const; - - lldb::BreakpointLocationSP GetOpaqueTypeFromSBBreakpointLocation( - const lldb::SBBreakpointLocation &break_loc) const; - - CommandReturnObject *GetOpaqueTypeFromSBCommandReturnObject( - const lldb::SBCommandReturnObject &cmd_retobj) const; - - lldb::DebuggerSP - GetOpaqueTypeFromSBDebugger(const lldb::SBDebugger &debugger) const; - - lldb::ProcessAttachInfoSP - GetOpaqueTypeFromSBAttachInfo(const lldb::SBAttachInfo &attach_info) const; - - lldb::ProcessLaunchInfoSP - GetOpaqueTypeFromSBLaunchInfo(const lldb::SBLaunchInfo &launch_info) const; - - std::optional GetOpaqueTypeFromSBMemoryRegionInfo( - const lldb::SBMemoryRegionInfo &mem_region) const; - - lldb::ExecutionContextRefSP GetOpaqueTypeFromSBExecutionContext( - const lldb::SBExecutionContext &exe_ctx) const; - - lldb::StackFrameListSP - GetOpaqueTypeFromSBFrameList(const lldb::SBFrameList &exe_ctx) const; - - lldb::ValueObjectSP - GetOpaqueTypeFromSBValue(const lldb::SBValue &value) const; - - lldb::TargetSP GetOpaqueTypeFromSBTarget(const lldb::SBTarget &target) const; - /// Get the debugger associated with this script interpreter. Debugger &GetDebugger() { return m_debugger; } const Debugger &GetDebugger() const { return m_debugger; } diff --git a/lldb/include/lldb/Utility/StreamString.h b/lldb/include/lldb/Utility/StreamString.h index 1a6444fc29c24..5fcda832d4cf8 100644 --- a/lldb/include/lldb/Utility/StreamString.h +++ b/lldb/include/lldb/Utility/StreamString.h @@ -47,7 +47,7 @@ class StreamString : public Stream { void FillLastLineToColumn(uint32_t column, char fill_char); protected: - friend class ScriptInterpreter; + friend class ScriptInterpreterBridge; std::string m_packet; size_t WriteImpl(const void *s, size_t length) override; diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h index 2a4044e9a9b88..47362915d6a56 100644 --- a/lldb/include/lldb/lldb-forward.h +++ b/lldb/include/lldb/lldb-forward.h @@ -188,6 +188,7 @@ class RichManglingContext; class SaveCoreOptions; class Scalar; class ScriptInterpreter; +class ScriptInterpreterBridge; class ScriptInterpreterLocker; class ScriptedFrameInterface; class ScriptedFrameProviderInterface; diff --git a/lldb/source/API/CMakeLists.txt b/lldb/source/API/CMakeLists.txt index 83ecb428d8ea4..d3a417a260270 100644 --- a/lldb/source/API/CMakeLists.txt +++ b/lldb/source/API/CMakeLists.txt @@ -116,6 +116,7 @@ add_lldb_library(liblldb SHARED ${option_framework} SBVariablesOptions.cpp SBWatchpoint.cpp SBWatchpointOptions.cpp + ScriptInterpreterBridge.cpp SystemInitializerFull.cpp ADDITIONAL_HEADER_DIRS diff --git a/lldb/source/API/ScriptInterpreterBridge.cpp b/lldb/source/API/ScriptInterpreterBridge.cpp new file mode 100644 index 0000000000000..e818a0c424d16 --- /dev/null +++ b/lldb/source/API/ScriptInterpreterBridge.cpp @@ -0,0 +1,147 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ScriptInterpreterBridge.h" +#include "API/SBCommandReturnObjectImpl.h" +#include "lldb/API/SBAttachInfo.h" +#include "lldb/API/SBBreakpoint.h" +#include "lldb/API/SBBreakpointLocation.h" +#include "lldb/API/SBCommandReturnObject.h" +#include "lldb/API/SBData.h" +#include "lldb/API/SBDebugger.h" +#include "lldb/API/SBError.h" +#include "lldb/API/SBEvent.h" +#include "lldb/API/SBExecutionContext.h" +#include "lldb/API/SBFrame.h" +#include "lldb/API/SBFrameList.h" +#include "lldb/API/SBLaunchInfo.h" +#include "lldb/API/SBMemoryRegionInfo.h" +#include "lldb/API/SBStream.h" +#include "lldb/API/SBSymbolContext.h" +#include "lldb/API/SBTarget.h" +#include "lldb/API/SBThread.h" +#include "lldb/API/SBValue.h" +#include "lldb/Host/ProcessLaunchInfo.h" +#include "lldb/Interpreter/CommandReturnObject.h" +#include "lldb/Target/ExecutionContext.h" +#include "lldb/Utility/StreamString.h" +#include "lldb/ValueObject/ValueObject.h" + +using namespace lldb; +using namespace lldb_private; + +lldb::DataExtractorSP +ScriptInterpreterBridge::GetDataExtractor(const lldb::SBData &data) { + return data.m_opaque_sp; +} + +lldb::BreakpointSP +ScriptInterpreterBridge::GetBreakpoint(const lldb::SBBreakpoint &breakpoint) { + return breakpoint.m_opaque_wp.lock(); +} + +lldb::BreakpointLocationSP ScriptInterpreterBridge::GetBreakpointLocation( + const lldb::SBBreakpointLocation &break_loc) { + return break_loc.m_opaque_wp.lock(); +} + +CommandReturnObject *ScriptInterpreterBridge::GetCommandReturnObject( + const lldb::SBCommandReturnObject &cmd_retobj) { + return cmd_retobj.m_opaque_up->get(); +} + +lldb::DebuggerSP +ScriptInterpreterBridge::GetDebugger(const lldb::SBDebugger &debugger) { + return debugger.m_opaque_sp; +} + +lldb::ProcessAttachInfoSP ScriptInterpreterBridge::GetProcessAttachInfo( + const lldb::SBAttachInfo &attach_info) { + return attach_info.m_opaque_sp; +} + +lldb::ProcessLaunchInfoSP ScriptInterpreterBridge::GetProcessLaunchInfo( + const lldb::SBLaunchInfo &launch_info) { + return std::make_shared( + *reinterpret_cast(launch_info.m_opaque_sp.get())); +} + +Status ScriptInterpreterBridge::GetStatus(const lldb::SBError &error) { + if (error.m_opaque_up) + return error.m_opaque_up->Clone(); + + return Status(); +} + +lldb::ThreadSP +ScriptInterpreterBridge::GetThread(const lldb::SBThread &thread) { + if (thread.m_opaque_sp) + return thread.m_opaque_sp->GetThreadSP(); + return nullptr; +} + +lldb::StackFrameSP +ScriptInterpreterBridge::GetStackFrame(const lldb::SBFrame &frame) { + if (frame.m_opaque_sp) + return frame.m_opaque_sp->GetFrameSP(); + return nullptr; +} + +Event *ScriptInterpreterBridge::GetEvent(const lldb::SBEvent &event) { + return event.m_opaque_ptr; +} + +lldb::StreamSP +ScriptInterpreterBridge::GetStream(const lldb::SBStream &stream) { + if (stream.m_opaque_up) { + lldb::StreamSP s = std::make_shared(); + *s << reinterpret_cast(stream.m_opaque_up.get())->m_packet; + return s; + } + + return nullptr; +} + +SymbolContext ScriptInterpreterBridge::GetSymbolContext( + const lldb::SBSymbolContext &sb_sym_ctx) { + if (sb_sym_ctx.m_opaque_up) + return *sb_sym_ctx.m_opaque_up; + return {}; +} + +std::optional +ScriptInterpreterBridge::GetMemoryRegionInfo( + const lldb::SBMemoryRegionInfo &mem_region) { + if (!mem_region.m_opaque_up) + return std::nullopt; + return *mem_region.m_opaque_up.get(); +} + +lldb::ExecutionContextRefSP ScriptInterpreterBridge::GetExecutionContextRef( + const lldb::SBExecutionContext &exe_ctx) { + return exe_ctx.m_exe_ctx_sp; +} + +lldb::StackFrameListSP ScriptInterpreterBridge::GetStackFrameList( + const lldb::SBFrameList &frame_list) { + return frame_list.m_opaque_sp; +} + +lldb::TargetSP +ScriptInterpreterBridge::GetTarget(const lldb::SBTarget &target) { + return target.m_opaque_sp; +} + +lldb::ValueObjectSP +ScriptInterpreterBridge::GetValueObject(const lldb::SBValue &value) { + if (!value.m_opaque_sp) + return lldb::ValueObjectSP(); + + lldb_private::ValueLocker locker; + return locker.GetLockedSP(*value.m_opaque_sp); +} diff --git a/lldb/source/API/ScriptInterpreterBridge.h b/lldb/source/API/ScriptInterpreterBridge.h new file mode 100644 index 0000000000000..d0dbb049a6aff --- /dev/null +++ b/lldb/source/API/ScriptInterpreterBridge.h @@ -0,0 +1,78 @@ +//===-- ScriptInterpreterBridge.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLDB_SOURCE_API_SCRIPTINTERPRETERBRIDGE_H +#define LLDB_SOURCE_API_SCRIPTINTERPRETERBRIDGE_H + +#include "lldb/API/SBDefines.h" +#include "lldb/Symbol/SymbolContext.h" +#include "lldb/Target/MemoryRegionInfo.h" +#include "lldb/Utility/Status.h" +#include "lldb/lldb-forward.h" +#include + +namespace lldb_private { + +class CommandReturnObject; +class Event; + +/// Unwraps the opaque internal object held by an SB (public API) instance, +/// for scripting-language plugins that need to convert values passed across +/// the script callback boundary back to their lldb_private form. Every SB +/// class this needs to reach into grants friendship to this class alone, so +/// that access to API internals stays confined to this single bridge rather +/// than spreading across the Interpreter layer. +class ScriptInterpreterBridge { +public: + static lldb::DataExtractorSP GetDataExtractor(const lldb::SBData &data); + + static Status GetStatus(const lldb::SBError &error); + + static Event *GetEvent(const lldb::SBEvent &event); + + static lldb::StreamSP GetStream(const lldb::SBStream &stream); + + static lldb::ThreadSP GetThread(const lldb::SBThread &thread); + + static lldb::StackFrameSP GetStackFrame(const lldb::SBFrame &frame); + + static SymbolContext GetSymbolContext(const lldb::SBSymbolContext &sym_ctx); + + static lldb::BreakpointSP GetBreakpoint(const lldb::SBBreakpoint &breakpoint); + + static lldb::BreakpointLocationSP + GetBreakpointLocation(const lldb::SBBreakpointLocation &break_loc); + + static CommandReturnObject * + GetCommandReturnObject(const lldb::SBCommandReturnObject &cmd_retobj); + + static lldb::DebuggerSP GetDebugger(const lldb::SBDebugger &debugger); + + static lldb::ProcessAttachInfoSP + GetProcessAttachInfo(const lldb::SBAttachInfo &attach_info); + + static lldb::ProcessLaunchInfoSP + GetProcessLaunchInfo(const lldb::SBLaunchInfo &launch_info); + + static std::optional + GetMemoryRegionInfo(const lldb::SBMemoryRegionInfo &mem_region); + + static lldb::ExecutionContextRefSP + GetExecutionContextRef(const lldb::SBExecutionContext &exe_ctx); + + static lldb::StackFrameListSP + GetStackFrameList(const lldb::SBFrameList &frame_list); + + static lldb::ValueObjectSP GetValueObject(const lldb::SBValue &value); + + static lldb::TargetSP GetTarget(const lldb::SBTarget &target); +}; + +} // namespace lldb_private + +#endif // LLDB_SOURCE_API_SCRIPTINTERPRETERBRIDGE_H diff --git a/lldb/source/Interpreter/CMakeLists.txt b/lldb/source/Interpreter/CMakeLists.txt index 4bb59d7550b0d..8a9605b568865 100644 --- a/lldb/source/Interpreter/CMakeLists.txt +++ b/lldb/source/Interpreter/CMakeLists.txt @@ -65,12 +65,26 @@ add_lldb_library(lldbInterpreter NO_PLUGIN_DEPENDENCIES Support LINK_LIBS lldbInterpreterInterfaces + lldbBreakpoint lldbCommands lldbCore lldbDataFormatters lldbHost + lldbSymbol lldbTarget lldbUtility + lldbValueObject + ALLOWED_INTERNAL_DEPENDENCIES + lldbInterpreterInterfaces + lldbBreakpoint + lldbCommands + lldbCore + lldbDataFormatters + lldbHost + lldbSymbol + lldbTarget + lldbUtility + lldbValueObject ) add_dependencies(lldbInterpreter diff --git a/lldb/source/Interpreter/ScriptInterpreter.cpp b/lldb/source/Interpreter/ScriptInterpreter.cpp index c16716bbc00fb..a53f3b744f02e 100644 --- a/lldb/source/Interpreter/ScriptInterpreter.cpp +++ b/lldb/source/Interpreter/ScriptInterpreter.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "lldb/Interpreter/ScriptInterpreter.h" -#include "API/SBCommandReturnObjectImpl.h" #include "lldb/Core/Debugger.h" #include "lldb/Host/ConnectionFileDescriptor.h" #include "lldb/Host/Pipe.h" @@ -17,7 +16,6 @@ #include "lldb/Utility/Stream.h" #include "lldb/Utility/StringList.h" #include "lldb/Utility/UnimplementedError.h" -#include "lldb/ValueObject/ValueObject.h" #include "llvm/ADT/StringSwitch.h" #if defined(_WIN32) #include "lldb/Host/windows/ConnectionGenericFileWindows.h" @@ -81,121 +79,6 @@ std::string ScriptInterpreter::LanguageToString(lldb::ScriptLanguage language) { llvm_unreachable("Unhandled ScriptInterpreter!"); } -lldb::DataExtractorSP -ScriptInterpreter::GetDataExtractorFromSBData(const lldb::SBData &data) const { - return data.m_opaque_sp; -} - -lldb::BreakpointSP ScriptInterpreter::GetOpaqueTypeFromSBBreakpoint( - const lldb::SBBreakpoint &breakpoint) const { - return breakpoint.m_opaque_wp.lock(); -} - -lldb::BreakpointLocationSP -ScriptInterpreter::GetOpaqueTypeFromSBBreakpointLocation( - const lldb::SBBreakpointLocation &break_loc) const { - return break_loc.m_opaque_wp.lock(); -} - -CommandReturnObject *ScriptInterpreter::GetOpaqueTypeFromSBCommandReturnObject( - const lldb::SBCommandReturnObject &cmd_retobj) const { - return cmd_retobj.m_opaque_up->get(); -} - -lldb::DebuggerSP ScriptInterpreter::GetOpaqueTypeFromSBDebugger( - const lldb::SBDebugger &debugger) const { - return debugger.m_opaque_sp; -} - -lldb::ProcessAttachInfoSP ScriptInterpreter::GetOpaqueTypeFromSBAttachInfo( - const lldb::SBAttachInfo &attach_info) const { - return attach_info.m_opaque_sp; -} - -lldb::ProcessLaunchInfoSP ScriptInterpreter::GetOpaqueTypeFromSBLaunchInfo( - const lldb::SBLaunchInfo &launch_info) const { - return std::make_shared( - *reinterpret_cast(launch_info.m_opaque_sp.get())); -} - -Status -ScriptInterpreter::GetStatusFromSBError(const lldb::SBError &error) const { - if (error.m_opaque_up) - return error.m_opaque_up->Clone(); - - return Status(); -} - -lldb::ThreadSP ScriptInterpreter::GetOpaqueTypeFromSBThread( - const lldb::SBThread &thread) const { - if (thread.m_opaque_sp) - return thread.m_opaque_sp->GetThreadSP(); - return nullptr; -} - -lldb::StackFrameSP -ScriptInterpreter::GetOpaqueTypeFromSBFrame(const lldb::SBFrame &frame) const { - if (frame.m_opaque_sp) - return frame.m_opaque_sp->GetFrameSP(); - return nullptr; -} - -Event * -ScriptInterpreter::GetOpaqueTypeFromSBEvent(const lldb::SBEvent &event) const { - return event.m_opaque_ptr; -} - -lldb::StreamSP ScriptInterpreter::GetOpaqueTypeFromSBStream( - const lldb::SBStream &stream) const { - if (stream.m_opaque_up) { - lldb::StreamSP s = std::make_shared(); - *s << reinterpret_cast(stream.m_opaque_up.get())->m_packet; - return s; - } - - return nullptr; -} - -SymbolContext ScriptInterpreter::GetOpaqueTypeFromSBSymbolContext( - const lldb::SBSymbolContext &sb_sym_ctx) const { - if (sb_sym_ctx.m_opaque_up) - return *sb_sym_ctx.m_opaque_up; - return {}; -} - -std::optional -ScriptInterpreter::GetOpaqueTypeFromSBMemoryRegionInfo( - const lldb::SBMemoryRegionInfo &mem_region) const { - if (!mem_region.m_opaque_up) - return std::nullopt; - return *mem_region.m_opaque_up.get(); -} - -lldb::ExecutionContextRefSP -ScriptInterpreter::GetOpaqueTypeFromSBExecutionContext( - const lldb::SBExecutionContext &exe_ctx) const { - return exe_ctx.m_exe_ctx_sp; -} - -lldb::StackFrameListSP ScriptInterpreter::GetOpaqueTypeFromSBFrameList( - const lldb::SBFrameList &frame_list) const { - return frame_list.m_opaque_sp; -} - -lldb::TargetSP ScriptInterpreter::GetOpaqueTypeFromSBTarget( - const lldb::SBTarget &target) const { - return target.m_opaque_sp; -} - -lldb::ValueObjectSP -ScriptInterpreter::GetOpaqueTypeFromSBValue(const lldb::SBValue &value) const { - if (!value.m_opaque_sp) - return lldb::ValueObjectSP(); - - lldb_private::ValueLocker locker; - return locker.GetLockedSP(*value.m_opaque_sp); -} - lldb::ScriptLanguage ScriptInterpreter::StringToLanguage(const llvm::StringRef &language) { if (language.equals_insensitive(LanguageToString(eScriptLanguageNone))) diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedPythonInterface.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedPythonInterface.cpp index 96391f7be8da0..e08b4795b9297 100644 --- a/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedPythonInterface.cpp +++ b/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedPythonInterface.cpp @@ -8,7 +8,9 @@ #include "../lldb-python.h" -#include "lldb/API/SBDebugger.h" +#include "API/ScriptInterpreterBridge.h" +#include "lldb/API/SBValue.h" +#include "lldb/API/SBValueList.h" #include "lldb/Host/Config.h" #include "lldb/Utility/Log.h" #include "lldb/lldb-enumerations.h" @@ -47,7 +49,7 @@ Status ScriptedPythonInterface::ExtractValueFromPythonObject( python::PythonObject &p, Status &error) { if (lldb::SBError *sb_error = reinterpret_cast( python::LLDBSWIGPython_CastPyObjectToSBError(p.get()))) - return m_interpreter.GetStatusFromSBError(*sb_error); + return ScriptInterpreterBridge::GetStatus(*sb_error); error = Status::FromErrorString("Couldn't cast lldb::SBError to lldb::Status."); @@ -59,7 +61,7 @@ Event *ScriptedPythonInterface::ExtractValueFromPythonObject( python::PythonObject &p, Status &error) { if (lldb::SBEvent *sb_event = reinterpret_cast( python::LLDBSWIGPython_CastPyObjectToSBEvent(p.get()))) - return m_interpreter.GetOpaqueTypeFromSBEvent(*sb_event); + return ScriptInterpreterBridge::GetEvent(*sb_event); error = Status::FromErrorString( "Couldn't cast lldb::SBEvent to lldb_private::Event."); @@ -74,7 +76,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( reinterpret_cast( python::LLDBSWIGPython_CastPyObjectToSBCommandReturnObject( p.get()))) - return m_interpreter.GetOpaqueTypeFromSBCommandReturnObject(*sb_cmd_retobj); + return ScriptInterpreterBridge::GetCommandReturnObject(*sb_cmd_retobj); error = Status::FromErrorString("couldn't cast lldb::SBCommandReturnObject to " "lldb_private::CommandReturnObject."); @@ -87,7 +89,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( python::PythonObject &p, Status &error) { if (lldb::SBStream *sb_stream = reinterpret_cast( python::LLDBSWIGPython_CastPyObjectToSBStream(p.get()))) - return m_interpreter.GetOpaqueTypeFromSBStream(*sb_stream); + return ScriptInterpreterBridge::GetStream(*sb_stream); error = Status::FromErrorString( "Couldn't cast lldb::SBStream to lldb_private::Stream."); @@ -100,7 +102,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( python::PythonObject &p, Status &error) { if (lldb::SBFrame *sb_frame = reinterpret_cast( python::LLDBSWIGPython_CastPyObjectToSBFrame(p.get()))) - return m_interpreter.GetOpaqueTypeFromSBFrame(*sb_frame); + return ScriptInterpreterBridge::GetStackFrame(*sb_frame); error = Status::FromErrorString( "Couldn't cast lldb::SBFrame to lldb_private::StackFrame."); @@ -113,7 +115,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( python::PythonObject &p, Status &error) { if (lldb::SBThread *sb_thread = reinterpret_cast( python::LLDBSWIGPython_CastPyObjectToSBThread(p.get()))) - return m_interpreter.GetOpaqueTypeFromSBThread(*sb_thread); + return ScriptInterpreterBridge::GetThread(*sb_thread); error = Status::FromErrorString( "Couldn't cast lldb::SBThread to lldb_private::Thread."); @@ -127,7 +129,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( if (lldb::SBSymbolContext *sb_symbol_context = reinterpret_cast( python::LLDBSWIGPython_CastPyObjectToSBSymbolContext(p.get()))) - return m_interpreter.GetOpaqueTypeFromSBSymbolContext(*sb_symbol_context); + return ScriptInterpreterBridge::GetSymbolContext(*sb_symbol_context); error = Status::FromErrorString( "Couldn't cast lldb::SBSymbolContext to lldb_private::SymbolContext."); @@ -147,7 +149,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( return nullptr; } - return m_interpreter.GetDataExtractorFromSBData(*sb_data); + return ScriptInterpreterBridge::GetDataExtractor(*sb_data); } template <> @@ -163,7 +165,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( return nullptr; } - return m_interpreter.GetOpaqueTypeFromSBBreakpoint(*sb_breakpoint); + return ScriptInterpreterBridge::GetBreakpoint(*sb_breakpoint); } template <> @@ -181,7 +183,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject< return nullptr; } - return m_interpreter.GetOpaqueTypeFromSBBreakpointLocation(*sb_break_loc); + return ScriptInterpreterBridge::GetBreakpointLocation(*sb_break_loc); } template <> @@ -196,7 +198,7 @@ lldb::ProcessAttachInfoSP ScriptedPythonInterface::ExtractValueFromPythonObject< return nullptr; } - return m_interpreter.GetOpaqueTypeFromSBAttachInfo(*sb_attach_info); + return ScriptInterpreterBridge::GetProcessAttachInfo(*sb_attach_info); } template <> @@ -211,7 +213,7 @@ lldb::ProcessLaunchInfoSP ScriptedPythonInterface::ExtractValueFromPythonObject< return nullptr; } - return m_interpreter.GetOpaqueTypeFromSBLaunchInfo(*sb_launch_info); + return ScriptInterpreterBridge::GetProcessLaunchInfo(*sb_launch_info); } template <> @@ -230,7 +232,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject< return {}; } - return m_interpreter.GetOpaqueTypeFromSBMemoryRegionInfo(*sb_mem_reg_info); + return ScriptInterpreterBridge::GetMemoryRegionInfo(*sb_mem_reg_info); } template <> @@ -249,7 +251,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject< return {}; } - return m_interpreter.GetOpaqueTypeFromSBExecutionContext(*sb_exe_ctx); + return ScriptInterpreterBridge::GetExecutionContextRef(*sb_exe_ctx); } template <> @@ -284,7 +286,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( return {}; } - return m_interpreter.GetOpaqueTypeFromSBFrameList(*sb_frame_list); + return ScriptInterpreterBridge::GetStackFrameList(*sb_frame_list); } template <> @@ -299,7 +301,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( return {}; } - return m_interpreter.GetOpaqueTypeFromSBValue(*sb_value); + return ScriptInterpreterBridge::GetValueObject(*sb_value); } template <> @@ -314,7 +316,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( return {}; } - return m_interpreter.GetOpaqueTypeFromSBTarget(*sb_target); + return ScriptInterpreterBridge::GetTarget(*sb_target); } template <> @@ -330,7 +332,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( python::LLDBSWIGPython_CastPyObjectToSBValueList(p.get()))) { for (uint32_t i = 0, e = sb_value_list->GetSize(); i < e; ++i) { SBValue value = sb_value_list->GetValueAtIndex(i); - out->Append(m_interpreter.GetOpaqueTypeFromSBValue(value)); + out->Append(ScriptInterpreterBridge::GetValueObject(value)); } return out; } @@ -359,7 +361,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( python::LLDBSWIGPython_CastPyObjectToSBValue( static_cast(generic->GetValue()))); if (sb_value) - if (auto valobj_sp = m_interpreter.GetOpaqueTypeFromSBValue(*sb_value)) + if (auto valobj_sp = ScriptInterpreterBridge::GetValueObject(*sb_value)) out->Append(valobj_sp); ++index; return true; @@ -403,7 +405,7 @@ ScriptedPythonInterface::ExtractValueFromPythonObject( python::PythonObject &p, Status &error) { if (lldb::SBDebugger *sb_dbg = reinterpret_cast( python::LLDBSWIGPython_CastPyObjectToSBDebugger(p.get()))) - return m_interpreter.GetOpaqueTypeFromSBDebugger(*sb_dbg); + return ScriptInterpreterBridge::GetDebugger(*sb_dbg); error = Status::FromErrorString( "couldn't cast lldb::SBDebugger to lldb::DebuggerSP."); return {}; diff --git a/llvm/include/llvm/ADT/Hashing.h b/llvm/include/llvm/ADT/Hashing.h index bb965bea86d0c..b05598486abc7 100644 --- a/llvm/include/llvm/ADT/Hashing.h +++ b/llvm/include/llvm/ADT/Hashing.h @@ -306,7 +306,7 @@ template hash_code hash_combine(const Ts &...args) { constexpr size_t Total = hashing::detail::total_hashable_size(); // Round up so `data()` is non-null when Total == 0; combine_bytes won't // read the buffer in that case (len=0 short-circuits in xxh3_64bits). - std::array(1, Total)> buf; + std::array(1, Total)> buf{}; [[maybe_unused]] size_t off = 0; (hashing::detail::store_hashable_data(buf.data(), off, args), ...); return hashing::detail::combine_bytes(buf.data(), Total); diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index d5b5a014d9ef6..fc0aaeafc4fcf 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -2960,6 +2960,7 @@ class OpenMPIRBuilder { using MapNamesArrayTy = SmallVector; using MapDimArrayTy = SmallVector; using MapNonContiguousArrayTy = SmallVector; + using MapHasAttachPtrArrayTy = SmallVector; /// This structure contains combined information generated for mappable /// clauses, including base pointers, pointers, sizes, map types, user-defined @@ -2978,6 +2979,29 @@ class OpenMPIRBuilder { MapValuesArrayTy Sizes; MapFlagsArrayTy Types; MapNamesArrayTy Names; + /// True for entries that have an attach ptr, and thus an accompanying + /// ATTACH entry linking that ptr to its ptee. + /// + /// This is a property of the storage block an entry describes, not of the + /// map clause list item: it is true iff the entry's storage is the pointee + /// storage reached through some attach ptr. So, for `int *p; map(p[1:10])`, + /// which produces + /// &p[1], &p[1], 10*sizeof(int), TO|FROM <- HasAttachPtr = true + /// &p, &p[1], sizeof(void*), ATTACH <- HasAttachPtr = false + /// it is set on the pointee entry only. It is never set on the ATTACH entry + /// itself, nor on an entry that maps the pointer as an object in its own + /// right (e.g. the `map(p)` entry for the pointer's own storage). + /// + /// It is set on every entry whose storage lies in a pointee block, + /// including a combined struct entry for such a block and the individual + /// member entries that are MEMBER_OF it. e.g. for + /// `map(s2.s1p->x, s2.s1p->y)`: + /// &s2.s1p[0], &s2.s1p->x, sizeof(x..y), ALLOC <- true + /// &s2.s1p[0], &s2.s1p->x, 4, MEMBER_OF(1)|TO|FROM <- true + /// &s2.s1p[0], &s2.s1p->y, 4, MEMBER_OF(1)|TO|FROM <- true + /// &s2.s1p, &s2.s1p->x, sizeof(void*), ATTACH <- false + /// with s2.s1p as the attach ptr for all three. + MapHasAttachPtrArrayTy HasAttachPtr; StructNonContiguousInfo NonContigInfo; /// Append arrays in \a CurInfo. @@ -2990,6 +3014,8 @@ class OpenMPIRBuilder { Sizes.append(CurInfo.Sizes.begin(), CurInfo.Sizes.end()); Types.append(CurInfo.Types.begin(), CurInfo.Types.end()); Names.append(CurInfo.Names.begin(), CurInfo.Names.end()); + HasAttachPtr.append(CurInfo.HasAttachPtr.begin(), + CurInfo.HasAttachPtr.end()); NonContigInfo.Dims.append(CurInfo.NonContigInfo.Dims.begin(), CurInfo.NonContigInfo.Dims.end()); NonContigInfo.Offsets.append(CurInfo.NonContigInfo.Offsets.begin(), diff --git a/llvm/include/llvm/TargetParser/AMDGPUTargetParser.h b/llvm/include/llvm/TargetParser/AMDGPUTargetParser.h index f5e5137b027b4..c2e394d82292f 100644 --- a/llvm/include/llvm/TargetParser/AMDGPUTargetParser.h +++ b/llvm/include/llvm/TargetParser/AMDGPUTargetParser.h @@ -299,7 +299,7 @@ class LLVM_ABI TargetID { /// \returns the canonical processor name followed by any explicit xnack and /// sramecc feature modifiers order (e.g. "gfx908:sramecc-:xnack+"), without /// the triple prefix. - std::string getCanonicalTargetIDString() const; + std::string getCanonicalFeatureString() const; bool operator==(const TargetID &Other) const; bool operator!=(const TargetID &Other) const { return !(*this == Other); } diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 6fe6f268f1ef5..db55fc3ab0fa0 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -10635,19 +10635,78 @@ Expected OpenMPIRBuilder::emitUserDefinedMapper( ? Info->Names[I] : Constant::getNullValue(Builder.getPtrTy()); - // Extract the MEMBER_OF field from the map type. Value *OriMapType = Builder.getInt64( static_cast>( Info->Types[I])); + auto RawType = + static_cast>( + Info->Types[I]); + constexpr uint64_t MemberOfMask = + static_cast(OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF); + constexpr uint64_t AttachBit = + static_cast>( + OpenMPOffloadMappingFlags::OMP_MAP_ATTACH); + + // Add MEMBER_OF (ShiftedPreviousSize) to group this sub-map with the + // current array element (N = __tgt_mapper_num_components() at loop body + // start). + // + // Example 1: + // struct S { int x; int *p; }; + // + // mapper: #pragma omp declare mapper(id: S s) map(s.x, s.p[0:10]) + // use: S arr[2]; ... map(arr) + // entries per element: + // + // &arr[i], &arr[i].x, sizeof(int), MEMBER_OF(N)|TO|FROM + // &arr[i].p[0], &arr[i].p[0], 10*sizeof(int), TO|FROM (*) + // &arr[i].p, &arr[i].p[0], sizeof(int*), ATTACH (**) + // + // Example 2: + // struct S1 { int x; int y; }; + // struct S2 { int z; S1 *s1p; }; + // + // mapper: #pragma omp declare mapper(S2 s2) map(s2.z, s2.s1p->x, + // s2.s1p->y) + // use: S2 arr[2]; ... map(arr) + // entries per element: + // + // &arr[i], &arr[i].z, sizeof(int), MEMBER_OF(N)|TO|FROM + // &arr[i].s1p[0], &arr[i].s1p->x, sizeof(s1p->x..y), ALLOC (*) + // &arr[i].s1p[0], &arr[i].s1p->x, 4, MEMBER_OF(N+2)|TO|FROM (*)(***) + // &arr[i].s1p[0], &arr[i].s1p->y, 4, MEMBER_OF(N+2)|TO|FROM (*)(***) + // &arr[i].s1p, &arr[i].s1p->x, sizeof(ptr), ATTACH (**) + // + // x/y carry inner MEMBER_OF(2) + // which is shifted by N to become MEMBER_OF(N+2). + // + // HasAttachPtr is set on all of the s1p entries except the ATTACH one: + // the combined ALLOC entry for the s1p->x..y block, and the individual + // x/y entries that are MEMBER_OF that block, all describe storage + // reached through the attach ptr arr[i].s1p. + // + // Entries of the following kinds do NOT receive a new outer MEMBER_OF + // linking them to the parent struct: + // + // * (*) Entries with HasAttachPtr: they represent pointee data that + // occupies a different storage block than the struct being mapped, so + // they are not a member of it. They may still be MEMBER_OF an entry + // within that pointee block, in which case those pre-existing bits are + // shifted -- see (***). + // * (**) ATTACH entries: they are not a member of anything — they just + // link a ptr to its ptee. + // * All entries when PreserveMemberOfFlags is set (the Flang/MLIR path): + // its pre-shaped entries already carry their final MEMBER_OF bits. + // TODO: set HasAttachPtr from Flang for entries whose storage is the + // pointee's (e.g. s%p(0:10)) and drop PreserveMemberOfFlags in favor of + // it. + // + // (***) If such an entry already has its own MEMBER_OF bits (e.g. the + // s1p->x/y entries above), those bits are still shifted by N. Value *MemberMapType; - if (PreserveMemberOfFlags) { - constexpr uint64_t MemberOfMask = - static_cast(OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF); - uint64_t OrigFlags = - static_cast>( - Info->Types[I]); - bool HasMemberOf = (OrigFlags & MemberOfMask) != 0; - if (HasMemberOf) + if (PreserveMemberOfFlags || (RawType & AttachBit) || + Info->HasAttachPtr[I]) { + if (RawType & MemberOfMask) MemberMapType = Builder.CreateNUWAdd(OriMapType, ShiftedPreviousSize); else MemberMapType = OriMapType; @@ -10758,12 +10817,6 @@ Expected OpenMPIRBuilder::emitUserDefinedMapper( // ATTACH entries must not receive map-type-modifying bits: ATTACH|ALWAYS is // reserved for the attach(always) map-type modifier, and other modifier // bits (DELETE, CLOSE) have no meaning for an ATTACH entry. - auto RawType = - static_cast>( - Info->Types[I]); - constexpr uint64_t AttachBit = - static_cast>( - OpenMPOffloadMappingFlags::OMP_MAP_ATTACH); Value *FinalMapType = (RawType & AttachBit) ? CurMapType : CurMapTypeWithModifiers; diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp index bcd8d0bef062d..c1fa828f38bbf 100644 --- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp @@ -1793,10 +1793,16 @@ void SIFrameLowering::processFunctionBeforeFrameFinalized( // Add an emergency spill slot RS->addScavengingFrameIndex(FuncInfo->getScavengeFI(MFI, *TRI)); - // If we are spilling SGPRs to memory with a large frame, we may need a - // second VGPR emergency frame index. - if (HaveSGPRToVMemSpill && - allocateScavengingFrameIndexesNearIncomingSP(MF)) { + if (HaveSGPRToVMemSpill && FuncInfo->hasNoWWMPoolSGPRSpillFallback()) { + // The no-WWM-pool fallback can reach SGPR-to-memory lowering while an + // ordinary frame-index scavenge is live. It may then need one slot for + // its temporary VGPR and another for recursive address materialization. + RS->addScavengingFrameIndex(MFI.CreateSpillStackObject(4, Align(4))); + RS->addScavengingFrameIndex(MFI.CreateSpillStackObject(4, Align(4))); + } else if (HaveSGPRToVMemSpill && + allocateScavengingFrameIndexesNearIncomingSP(MF)) { + // Existing large-frame SGPR-to-memory spills need one additional VGPR + // emergency frame index. RS->addScavengingFrameIndex(MFI.CreateSpillStackObject(4, Align(4))); } } diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp index 8cc7714c6fa2e..5bf34ddcb2816 100644 --- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp +++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp @@ -20,6 +20,7 @@ #include "GCNSubtarget.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIMachineFunctionInfo.h" +#include "SIPreAllocateWWMRegs.h" #include "SISpillUtils.h" #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/MachineCycleAnalysis.h" @@ -80,7 +81,9 @@ class SILowerSGPRSpills { void updateLaneVGPRDomInstr( int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt, DenseMap &LaneVGPRDomInstr); - void determineRegsForWWMAllocation(MachineFunction &MF, BitVector &RegMask); + SmallVector determineRegsForWWMAllocation(MachineFunction &MF); + void assignWWMRegs(MachineFunction &MF, ArrayRef WWMRegCandidates, + bool RequiresFullWWMPool); }; class SILowerSGPRSpillsLegacy : public MachineFunctionPass { @@ -372,42 +375,64 @@ void SILowerSGPRSpills::updateLaneVGPRDomInstr( } } -void SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF, - BitVector &RegMask) { - // Determine an optimal number of VGPRs for WWM allocation. The complement - // list will be available for allocating other VGPR virtual registers. - SIMachineFunctionInfo *MFI = MF.getInfo(); +SmallVector +SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF) { + SmallVector WWMRegCandidates; + if (!MaxNumVGPRsForWwmAllocation) + return WWMRegCandidates; + MachineRegisterInfo &MRI = MF.getRegInfo(); BitVector ReservedRegs = TRI->getReservedRegs(MF); - BitVector NonWwmAllocMask(TRI->getNumRegs()); const GCNSubtarget &ST = MF.getSubtarget(); + unsigned MaxNumVGPRs = ST.getMaxNumVectorRegs(MF.getFunction()).first; - // FIXME: MaxNumVGPRsForWwmAllocation might need to be adjusted in the future - // to have a balanced allocation between WWM values and per-thread vector - // register operands. - unsigned NumRegs = MaxNumVGPRsForWwmAllocation; - NumRegs = - std::min(static_cast(MFI->getSGPRSpillVGPRs().size()), NumRegs); - - auto [MaxNumVGPRs, MaxNumAGPRs] = ST.getMaxNumVectorRegs(MF.getFunction()); // Try to use the highest available registers for now. Later after // vgpr-regalloc, they can be shifted to the lowest range. - unsigned I = 0; for (unsigned Reg = AMDGPU::VGPR0 + MaxNumVGPRs - 1; - (I < NumRegs) && (Reg >= AMDGPU::VGPR0); --Reg) { + WWMRegCandidates.size() < MaxNumVGPRsForWwmAllocation && + Reg >= AMDGPU::VGPR0; + --Reg) { if (!ReservedRegs.test(Reg) && - !MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/true)) { - TRI->markSuperRegs(RegMask, Reg); - ++I; - } + !MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/true)) + WWMRegCandidates.push_back(Reg); } - if (I != NumRegs) { + return WWMRegCandidates; +} + +void SILowerSGPRSpills::assignWWMRegs(MachineFunction &MF, + ArrayRef WWMRegCandidates, + bool RequiresFullWWMPool) { + SIMachineFunctionInfo *FuncInfo = MF.getInfo(); + if (FuncInfo->getSGPRSpillVGPRs().empty()) + return; + + BitVector WwmRegMask(TRI->getNumRegs()); + + unsigned DesiredPoolSize = + std::min(static_cast(FuncInfo->getSGPRSpillVGPRs().size()), + static_cast(MaxNumVGPRsForWwmAllocation)); + unsigned SelectedPoolSize = + std::min(DesiredPoolSize, WWMRegCandidates.size()); + // WWM register candidates are ordered high-to-low, so take the highest + // available registers when the desired pool is smaller than the candidate + // list. + for (MCRegister Reg : WWMRegCandidates.take_front(SelectedPoolSize)) + TRI->markSuperRegs(WwmRegMask, Reg); + + if (RequiresFullWWMPool && SelectedPoolSize != DesiredPoolSize) { // Reserve an arbitrary register and report the error. - TRI->markSuperRegs(RegMask, AMDGPU::VGPR0); + TRI->markSuperRegs(WwmRegMask, AMDGPU::VGPR0); MF.getFunction().getContext().emitError( "cannot find enough VGPRs for wwm-regalloc"); } + + BitVector NonWwmRegMask(WwmRegMask); + NonWwmRegMask.flip().clearBitsNotInMask(TRI->getAllVGPRRegMask()); + + // The complement set will be the registers for non-wwm (per-thread) vgpr + // allocation. + FuncInfo->updateNonWWMRegMask(NonWwmRegMask); } bool SILowerSGPRSpillsLegacy::runOnMachineFunction(MachineFunction &MF) { @@ -465,8 +490,19 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) { // To track the IMPLICIT_DEF insertion point for the lane vgprs. DenseMap LaneVGPRDomInstr; + // Defer ordinary spills until physical CSR spills have reserved their + // lane VGPRs and the WWM allocation pool can be selected. + SmallVector OrdinarySGPRSpills; + bool HasStrictWWMRegion = false; + for (MachineBasicBlock &MBB : MF) { for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) { + if (MI.getOpcode() == AMDGPU::ENTER_STRICT_WWM || + MI.getOpcode() == AMDGPU::ENTER_STRICT_WQM) { + HasStrictWWMRegion = true; + continue; + } + if (!TII->isSGPRSpill(MI)) continue; @@ -500,18 +536,39 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) { llvm_unreachable( "failed to spill SGPR to physical VGPR lane when allocated"); } - } else { - MachineInstrSpan MIS(&MI, &MBB); - if (FuncInfo->allocateSGPRSpillToVGPRLane(MF, FI)) { - bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex( - MI, FI, nullptr, Indexes, LIS); - if (!Spilled) - llvm_unreachable( - "failed to spill SGPR to virtual VGPR lane when allocated"); - SpillFIs.set(FI); - updateLaneVGPRDomInstr(FI, &MBB, MIS.begin(), LaneVGPRDomInstr); - SpilledToVirtVGPRLanes = true; - } + } else + OrdinarySGPRSpills.push_back(&MI); + } + } + + // Select candidates once, before ordinary lane lowering creates virtual + // VGPRs and changes the number of registers desired for the WWM pool. + SmallVector WWMRegCandidates; + // These non-spillable WWM users retain the old all-or-nothing pool policy. + const bool RequiresFullWWMPool = + HasStrictWWMRegion || isPreallocateSGPRSpillVGPRsEnabled(MF); + if (!OrdinarySGPRSpills.empty()) + WWMRegCandidates = determineRegsForWWMAllocation(MF); + + const bool ShouldLowerOrdinarySpillsToVGPRLanes = + RequiresFullWWMPool || !WWMRegCandidates.empty(); + if (!ShouldLowerOrdinarySpillsToVGPRLanes && !OrdinarySGPRSpills.empty()) + FuncInfo->setNoWWMPoolSGPRSpillFallback(); + + if (ShouldLowerOrdinarySpillsToVGPRLanes) { + for (MachineInstr *MI : OrdinarySGPRSpills) { + int FI = TII->getNamedOperand(*MI, AMDGPU::OpName::addr)->getIndex(); + if (FuncInfo->allocateSGPRSpillToVGPRLane(MF, FI)) { + MachineBasicBlock *MBB = MI->getParent(); + MachineInstrSpan MIS(MI, MBB); + bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex( + *MI, FI, nullptr, Indexes, LIS); + if (!Spilled) + llvm_unreachable( + "failed to spill SGPR to virtual VGPR lane when allocated"); + SpillFIs.set(FI); + updateLaneVGPRDomInstr(FI, MBB, MIS.begin(), LaneVGPRDomInstr); + SpilledToVirtVGPRLanes = true; } } } @@ -538,20 +595,9 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) { } } - // Determine the registers for WWM allocation and also compute the register - // mask for non-wwm VGPR allocation. - if (FuncInfo->getSGPRSpillVGPRs().size()) { - BitVector WwmRegMask(TRI->getNumRegs()); - - determineRegsForWWMAllocation(MF, WwmRegMask); - - BitVector NonWwmRegMask(WwmRegMask); - NonWwmRegMask.flip().clearBitsNotInMask(TRI->getAllVGPRRegMask()); - - // The complement set will be the registers for non-wwm (per-thread) vgpr - // allocation. - FuncInfo->updateNonWWMRegMask(NonWwmRegMask); - } + // Assign the WWM pool from the pre-selected candidates and compute the + // complement mask for per-thread VGPR allocation. + assignWWMRegs(MF, WWMRegCandidates, RequiresFullWWMPool); for (MachineBasicBlock &MBB : MF) clearDebugInfoForSpillFIs(MFI, MBB, SpillFIs); diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp index 59971923e4a5d..950019b37a462 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp @@ -739,6 +739,7 @@ yaml::SIMachineFunctionInfo::SIMachineFunctionInfo( WaveLimiter(MFI.needsWaveLimiter()), HasSpilledSGPRs(MFI.hasSpilledSGPRs()), HasSpilledVGPRs(MFI.hasSpilledVGPRs()), + HasNoWWMPoolSGPRSpillFallback(MFI.hasNoWWMPoolSGPRSpillFallback()), NumWaveDispatchSGPRs(MFI.getNumWaveDispatchSGPRs()), NumWaveDispatchVGPRs(MFI.getNumWaveDispatchVGPRs()), HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()), @@ -798,6 +799,7 @@ bool SIMachineFunctionInfo::initializeBaseYamlFields( WaveLimiter = YamlMFI.WaveLimiter; HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs; HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs; + HasNoWWMPoolSGPRSpillFallback = YamlMFI.HasNoWWMPoolSGPRSpillFallback; NumWaveDispatchSGPRs = YamlMFI.NumWaveDispatchSGPRs; NumWaveDispatchVGPRs = YamlMFI.NumWaveDispatchVGPRs; BytesInStackArgArea = YamlMFI.BytesInStackArgArea; diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h index 7374e996837a5..2d1b77fc4fb8f 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h @@ -272,6 +272,7 @@ struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo { bool WaveLimiter = false; bool HasSpilledSGPRs = false; bool HasSpilledVGPRs = false; + bool HasNoWWMPoolSGPRSpillFallback = false; uint16_t NumWaveDispatchSGPRs = 0; uint16_t NumWaveDispatchVGPRs = 0; uint32_t HighBitsOf32BitAddress = 0; @@ -334,6 +335,8 @@ template <> struct MappingTraits { YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false); YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false); YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false); + YamlIO.mapOptional("hasNoWWMPoolSGPRSpillFallback", + MFI.HasNoWWMPoolSGPRSpillFallback, false); YamlIO.mapOptional("numWaveDispatchSGPRs", MFI.NumWaveDispatchSGPRs, false); YamlIO.mapOptional("numWaveDispatchVGPRs", MFI.NumWaveDispatchVGPRs, false); YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg, @@ -613,6 +616,11 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunctionInfo, // frame, so save it here and add it to the RegScavenger later. std::optional ScavengeFI; + // Ordinary SGPR spills fell back to memory because no WWM VGPR pool was + // available. This path may require additional nested VGPR scavenging slots + // during frame-index elimination. + bool HasNoWWMPoolSGPRSpillFallback = false; + // Map each VGPR CSR to the mask needed to save and restore it using block // load/store instructions. Only used if the subtarget feature for VGPR block // load/store is enabled. @@ -853,6 +861,11 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunctionInfo, int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI); std::optional getOptionalScavengeFI() const { return ScavengeFI; } + void setNoWWMPoolSGPRSpillFallback() { HasNoWWMPoolSGPRSpillFallback = true; } + bool hasNoWWMPoolSGPRSpillFallback() const { + return HasNoWWMPoolSGPRSpillFallback; + } + unsigned getBytesInStackArgArea() const { return BytesInStackArgArea; } diff --git a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp index bf484cef98da4..0d9ef77b70562 100644 --- a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp +++ b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp @@ -35,6 +35,11 @@ static cl::opt EnablePreallocateSGPRSpillVGPRs("amdgpu-prealloc-sgpr-spill-vgprs", cl::init(false), cl::Hidden); +bool llvm::isPreallocateSGPRSpillVGPRsEnabled(const MachineFunction &MF) { + return EnablePreallocateSGPRSpillVGPRs || + MF.getFunction().hasFnAttribute("amdgpu-prealloc-sgpr-spill-vgprs"); +} + namespace { class SIPreAllocateWWMRegs { @@ -221,9 +226,7 @@ bool SIPreAllocateWWMRegs::run(MachineFunction &MF) { TRI = &TII->getRegisterInfo(); MRI = &MF.getRegInfo(); - bool PreallocateSGPRSpillVGPRs = - EnablePreallocateSGPRSpillVGPRs || - MF.getFunction().hasFnAttribute("amdgpu-prealloc-sgpr-spill-vgprs"); + bool PreallocateSGPRSpillVGPRs = isPreallocateSGPRSpillVGPRsEnabled(MF); bool RegsAssigned = false; diff --git a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h index 8d3fa21035966..2fcd9f9cba351 100644 --- a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h +++ b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h @@ -13,6 +13,8 @@ namespace llvm { +bool isPreallocateSGPRSpillVGPRsEnabled(const MachineFunction &MF); + class SIPreAllocateWWMRegsPass : public RequiredPassInfoMixin { public: diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index f36f27769a353..c4eee0173bdcb 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -3597,10 +3597,6 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base, /// compressible) standard load/store instructions. bool RISCVDAGToDAGISel::SelectAddrRegImm26(SDValue Addr, SDValue &Base, SDValue &Offset) { - - if (SelectAddrFrameIndex(Addr, Base, Offset)) - return true; - SDLoc DL(Addr); MVT VT = Addr.getSimpleValueType(); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index e23427482c1e4..cd03dbed23ee7 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -15730,8 +15730,8 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, // If the FP type needs to be softened, emit a library call to lround. We'll // need to truncate the result. We assume any value that doesn't fit in i32 // is allowed to return an unspecified value. - RTLIB::Libcall LC = - Op0.getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32; + RTLIB::Libcall LC = RTLIB::getLROUND(Op0.getValueType()); + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected FP type for LROUND!"); MakeLibCallOptions CallOptions; EVT OpVT = Op0.getValueType(); CallOptions.setTypeListBeforeSoften(OpVT, MVT::i64); diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp index 95562e9fc45fd..f51f56cb10a46 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp @@ -573,6 +573,29 @@ void RISCVRegisterInfo::lowerSegmentSpillReload(MachineBasicBlock::iterator II, II->eraseFromParent(); } +static unsigned getXqciloWideOpcode(unsigned Opc) { + switch (Opc) { + case RISCV::LW: + return RISCV::QC_E_LW; + case RISCV::SW: + return RISCV::QC_E_SW; + case RISCV::LB: + return RISCV::QC_E_LB; + case RISCV::LBU: + return RISCV::QC_E_LBU; + case RISCV::LH: + return RISCV::QC_E_LH; + case RISCV::LHU: + return RISCV::QC_E_LHU; + case RISCV::SB: + return RISCV::QC_E_SB; + case RISCV::SH: + return RISCV::QC_E_SH; + default: + return 0; + } +} + bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, int SPAdj, unsigned FIOperandNum, RegScavenger *RS) const { @@ -581,7 +604,9 @@ bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, MachineInstr &MI = *II; MachineFunction &MF = *MI.getParent()->getParent(); MachineRegisterInfo &MRI = MF.getRegInfo(); - bool Is64Bit = MF.getSubtarget().is64Bit(); + const RISCVSubtarget &ST = MF.getSubtarget(); + const RISCVInstrInfo *TII = ST.getInstrInfo(); + bool Is64Bit = ST.is64Bit(); DebugLoc DL = MI.getDebugLoc(); int FrameIndex = MI.getOperand(FIOperandNum).getIndex(); @@ -627,6 +652,17 @@ bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, // instruction will add 4 to the immediate. If that would overflow 12 // bits, we can't fold the offset. MI.getOperand(FIOperandNum + 1).ChangeToImmediate(0); + } else if (unsigned WideOpc = getXqciloWideOpcode(Opc); + !isInt<12>(Val) && ST.hasVendorXqcilo() && WideOpc) { + // The resolved frame offset exceeds simm12 but the instruction is a + // standard load/store (LW/SW/etc). Promote to the wide Xqcilo equivalent + // so the full 26-bit offset folds directly, avoiding a separate + // base-adjust instruction. This runs post-RA and does not affect + // register allocation decisions. + MI.setDesc(TII->get(WideOpc)); + MI.getOperand(FIOperandNum + 1).ChangeToImmediate(Lo26); + Offset = StackOffset::get((uint64_t)Val - (uint64_t)Lo26, + Offset.getScalable()); } else if (Opc == RISCV::QC_E_ADDI || RISCVInstrInfo::isBaseQCLoad(MI) || RISCVInstrInfo::isBaseQCStore(MI)) { MI.getOperand(FIOperandNum + 1).ChangeToImmediate(Lo26); diff --git a/llvm/lib/TargetParser/AMDGPUTargetParser.cpp b/llvm/lib/TargetParser/AMDGPUTargetParser.cpp index bd87e635a00ee..61addf60b284c 100644 --- a/llvm/lib/TargetParser/AMDGPUTargetParser.cpp +++ b/llvm/lib/TargetParser/AMDGPUTargetParser.cpp @@ -716,7 +716,7 @@ void TargetID::printCanonicalTargetIDString(raw_ostream &OS) const { printFeatureModifiers(OS, getSramEccSetting(), getXnackSetting()); } -std::string TargetID::getCanonicalTargetIDString() const { +std::string TargetID::getCanonicalFeatureString() const { std::string Str; raw_string_ostream OS(Str); printCanonicalTargetIDString(OS); diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 33b08a648d45f..089892dc573f3 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -3545,13 +3545,77 @@ static ConstantInt *getKnownValueOnEdge(Value *V, BasicBlock *From, return nullptr; } +static bool isUncontrolledConvergentCall(CallBase *CB) { + return CB->isConvergent() && !isa(CB) && + !CB->getConvergenceControlToken(); +} + +static bool reachesUncontrolledConvergentCallBeforeBlock(BasicBlock *From, + BasicBlock *StopBB) { + static constexpr unsigned MaxInstructionsToScan = 512; + + // Walk predecessors of StopBB to find blocks that can reach it. Only + // convergent calls on a cycle with StopBB matter - a convergent call on a + // path to function exit cannot have its dynamic instance changed by + // threading. + SmallPtrSet CanReachStop; + SmallPtrSet BlocksWithUncontrolledConvergentCalls; + SmallVector Worklist; + for (BasicBlock *Pred : predecessors(StopBB)) + Worklist.push_back(Pred); + + // Cache blocks with relevant calls while building CanReachStop. This keeps + // the instruction scan bounded without a separate block limit. + unsigned NumScannedInstructions = 0; + while (!Worklist.empty()) { + BasicBlock *BB = Worklist.pop_back_val(); + if (BB == StopBB) + continue; + if (!CanReachStop.insert(BB).second) + continue; + + for (Instruction &I : *BB) { + if (++NumScannedInstructions > MaxInstructionsToScan) + return true; + auto *CB = dyn_cast(&I); + if (CB && isUncontrolledConvergentCall(CB)) { + BlocksWithUncontrolledConvergentCalls.insert(BB); + break; + } + } + + append_range(Worklist, predecessors(BB)); + } + + if (!CanReachStop.contains(From)) + return false; + + SmallPtrSet Visited; + Worklist.push_back(From); + + while (!Worklist.empty()) { + BasicBlock *BB = Worklist.pop_back_val(); + if (BB == StopBB || !CanReachStop.contains(BB)) + continue; + + if (!Visited.insert(BB).second) + continue; + + if (BlocksWithUncontrolledConvergentCalls.contains(BB)) + return true; + + append_range(Worklist, successors(BB)); + } + + return false; +} + /// If we have a conditional branch on something for which we know the constant /// value in predecessors (e.g. a phi node in the current block), thread edges /// from the predecessor to their ultimate destination. -static std::optional -foldCondBranchOnValueKnownInPredecessorImpl(CondBrInst *BI, DomTreeUpdater *DTU, - const DataLayout &DL, - AssumptionCache *AC) { +static std::optional foldCondBranchOnValueKnownInPredecessorImpl( + CondBrInst *BI, const TargetTransformInfo &TTI, DomTreeUpdater *DTU, + AssumptionCache *AC, const DataLayout &DL) { SmallMapVector, 2> KnownValues; BasicBlock *BB = BI->getParent(); Value *Cond = BI->getCondition(); @@ -3621,6 +3685,14 @@ foldCondBranchOnValueKnownInPredecessorImpl(CondBrInst *BI, DomTreeUpdater *DTU, if (ReachesNonLocalUseBlocks.contains(RealDest)) continue; + // Threading through a branch can bypass a reconvergence point. If the + // destination can execute an uncontrolled convergent operation before + // returning to this block, this may change the dynamic instance of that + // operation. + if (TTI.hasBranchDivergence(BB->getParent()) && + reachesUncontrolledConvergentCallBeforeBlock(RealDest, BB)) + continue; + LLVM_DEBUG({ dbgs() << "Condition " << *Cond << " in " << BB->getName() << " has value " << *Pair.first << " in predecessors:\n"; @@ -3742,8 +3814,8 @@ bool SimplifyCFGOpt::foldCondBranchOnValueKnownInPredecessor(CondBrInst *BI) { bool EverChanged = false; do { // Note that None means "we changed things, but recurse further." - Result = - foldCondBranchOnValueKnownInPredecessorImpl(BI, DTU, DL, Options.AC); + Result = foldCondBranchOnValueKnownInPredecessorImpl(BI, TTI, DTU, + Options.AC, DL); EverChanged |= Result == std::nullopt || *Result; } while (Result == std::nullopt); return EverChanged; diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll index e9bb77f3db5dd..30202943afa84 100644 --- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll +++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll @@ -16,7 +16,10 @@ i64 ; vcc } -; ERR-GCNTRACKERS: ran out of registers during register allocation +; With the tracker enabled, no separate WWM VGPR is available and the SGPR +; spill falls back to memory. This case cannot preserve EXEC because SCC is +; live and no SGPR can be scavenged. +; ERR-GCNTRACKERS: unhandled SGPR spill to memory ; GCN-NOT: ran out of registers during register allocation ; FIXME: GCN Trackers do not track pressure from PhysRegs, so scheduling is actually worse @@ -62,4 +65,3 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0 attributes #0 = { nounwind alignstack=64 "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="10,10" "no-realign-stack" } attributes #1 = { nounwind alignstack=16 "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="10,10" "no-realign-stack" } - diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir index b23aa0f45a05c..8b64e4fb09308 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir @@ -4,6 +4,10 @@ # Check that we allocate 2 emergency stack slots if we're spilling # SGPRs to memory and potentially have an offset larger than fits in # the addressing mode of the memory instructions. +# +# The no-WWM-pool fallback needs 3 emergency slots even for a small frame: +# one for an ordinary frame-index scavenge, one for the SGPR spill temporary, +# and one for recursive address materialization. --- name: test @@ -49,3 +53,47 @@ body: | renamable $sgpr10 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 S_SETPC_B64 $sgpr30_sgpr31, implicit $scc ... +--- +name: fallback_scavenging +tracksRegLiveness: true +frameInfo: + maxAlignment: 4 +stack: + - { id: 0, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill } +machineFunctionInfo: + isEntryFunction: false + scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' + stackPtrOffsetReg: '$sgpr32' + frameOffsetReg: '$sgpr33' + hasSpilledSGPRs: true + hasNoWWMPoolSGPRSpillFallback: true +body: | + bb.0: + liveins: $sgpr30_sgpr31, $sgpr10, $sgpr11 + ; CHECK-LABEL: name: fallback_scavenging + ; CHECK: liveins: $sgpr10, $sgpr11, $sgpr30_sgpr31 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: frame-setup CFI_INSTRUCTION llvm_def_aspace_cfa $sgpr32, 0, 6 + ; CHECK-NEXT: frame-setup CFI_INSTRUCTION llvm_register_pair $pc_reg, $sgpr30, 32, $sgpr31, 32 + ; CHECK-NEXT: frame-setup CFI_INSTRUCTION undefined $sgpr10 + ; CHECK-NEXT: S_CMP_EQ_U32 0, 0, implicit-def $scc + ; CHECK-NEXT: $sgpr4_sgpr5 = S_MOV_B64 $exec + ; CHECK-NEXT: $exec = S_MOV_B64 1, implicit-def $vgpr1 + ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.1, addrspace 5) + ; CHECK-NEXT: $vgpr1 = SI_SPILL_S32_TO_VGPR $sgpr10, 0, undef $vgpr1 + ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5) + ; CHECK-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.1, addrspace 5) + ; CHECK-NEXT: $exec = S_MOV_B64 killed $sgpr4_sgpr5, implicit killed $vgpr1 + ; CHECK-NEXT: $sgpr4_sgpr5 = S_MOV_B64 $exec + ; CHECK-NEXT: $exec = S_MOV_B64 1, implicit-def $vgpr1 + ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.1, addrspace 5) + ; CHECK-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.0, addrspace 5) + ; CHECK-NEXT: $sgpr10 = SI_RESTORE_S32_FROM_VGPR killed $vgpr1, 0 + ; CHECK-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.1, addrspace 5) + ; CHECK-NEXT: $exec = S_MOV_B64 killed $sgpr4_sgpr5, implicit killed $vgpr1 + ; CHECK-NEXT: S_SETPC_B64 $sgpr30_sgpr31, implicit $scc + S_CMP_EQ_U32 0, 0, implicit-def $scc + SI_SPILL_S32_SAVE killed $sgpr10, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + renamable $sgpr10 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + S_SETPC_B64 $sgpr30_sgpr31, implicit $scc +... diff --git a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll deleted file mode 100644 index 2367af90d9555..0000000000000 --- a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll +++ /dev/null @@ -1,29 +0,0 @@ -; RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -stress-regalloc=2 -filetype=null %s 2>&1 | FileCheck %s - -; A negative test to capture the expected error when the VGPRs are insufficient for wwm-regalloc. - -; CHECK: error: cannot find enough VGPRs for wwm-regalloc - -define amdgpu_kernel void @test(i32 %in) { -entry: - call void asm sideeffect "", "~{v[0:7]}" () - call void asm sideeffect "", "~{v[8:15]}" () - call void asm sideeffect "", "~{v[16:23]}" () - call void asm sideeffect "", "~{v[24:31]}" () - call void asm sideeffect "", "~{v[32:39]}" () - call void asm sideeffect "", "~{v[40:47]}" () - call void asm sideeffect "", "~{v[48:55]}" () - call void asm sideeffect "", "~{v[56:63]}" () - %val0 = call i32 asm sideeffect "; def $0", "=s" () - %val1 = call i32 asm sideeffect "; def $0", "=s" () - %val2 = call i32 asm sideeffect "; def $0", "=s" () - %cmp = icmp eq i32 %in, 0 - br i1 %cmp, label %bb0, label %ret -bb0: - call void asm sideeffect "; use $0", "s"(i32 %val0) - call void asm sideeffect "; use $0", "s"(i32 %val1) - call void asm sideeffect "; use $0", "s"(i32 %val2) - br label %ret -ret: - ret void -} diff --git a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-memory-fallback.ll b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-memory-fallback.ll new file mode 100644 index 0000000000000..b6e8341494153 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-memory-fallback.ll @@ -0,0 +1,219 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck %s + +; All allocatable VGPRs are clobbered while several large SGPR values remain +; live. Verify end-to-end that SGPR spills use scratch when no WWM VGPR pool is +; available, without relying on register-allocation stress options. + +define amdgpu_kernel void @repro(i32 %in) { +; CHECK-LABEL: repro: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_mov_b64 s[70:71], s[2:3] +; CHECK-NEXT: s_mov_b64 s[68:69], s[0:1] +; CHECK-NEXT: s_load_dword s0, s[8:9], 0x0 +; CHECK-NEXT: s_add_u32 s68, s68, s17 +; CHECK-NEXT: s_addc_u32 s69, s69, 0 +; CHECK-NEXT: s_mov_b64 s[2:3], exec +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; VGPR clobbers +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_mov_b64 exec, 0xffffffff +; CHECK-NEXT: buffer_store_dword v0, off, s[68:71], 0 offset:256 +; CHECK-NEXT: v_writelane_b32 v0, s36, 0 +; CHECK-NEXT: v_writelane_b32 v0, s37, 1 +; CHECK-NEXT: v_writelane_b32 v0, s38, 2 +; CHECK-NEXT: v_writelane_b32 v0, s39, 3 +; CHECK-NEXT: v_writelane_b32 v0, s40, 4 +; CHECK-NEXT: v_writelane_b32 v0, s41, 5 +; CHECK-NEXT: v_writelane_b32 v0, s42, 6 +; CHECK-NEXT: v_writelane_b32 v0, s43, 7 +; CHECK-NEXT: v_writelane_b32 v0, s44, 8 +; CHECK-NEXT: v_writelane_b32 v0, s45, 9 +; CHECK-NEXT: v_writelane_b32 v0, s46, 10 +; CHECK-NEXT: v_writelane_b32 v0, s47, 11 +; CHECK-NEXT: v_writelane_b32 v0, s48, 12 +; CHECK-NEXT: v_writelane_b32 v0, s49, 13 +; CHECK-NEXT: v_writelane_b32 v0, s50, 14 +; CHECK-NEXT: v_writelane_b32 v0, s51, 15 +; CHECK-NEXT: v_writelane_b32 v0, s52, 16 +; CHECK-NEXT: v_writelane_b32 v0, s53, 17 +; CHECK-NEXT: v_writelane_b32 v0, s54, 18 +; CHECK-NEXT: v_writelane_b32 v0, s55, 19 +; CHECK-NEXT: v_writelane_b32 v0, s56, 20 +; CHECK-NEXT: v_writelane_b32 v0, s57, 21 +; CHECK-NEXT: v_writelane_b32 v0, s58, 22 +; CHECK-NEXT: v_writelane_b32 v0, s59, 23 +; CHECK-NEXT: v_writelane_b32 v0, s60, 24 +; CHECK-NEXT: v_writelane_b32 v0, s61, 25 +; CHECK-NEXT: v_writelane_b32 v0, s62, 26 +; CHECK-NEXT: v_writelane_b32 v0, s63, 27 +; CHECK-NEXT: v_writelane_b32 v0, s64, 28 +; CHECK-NEXT: v_writelane_b32 v0, s65, 29 +; CHECK-NEXT: v_writelane_b32 v0, s66, 30 +; CHECK-NEXT: v_writelane_b32 v0, s67, 31 +; CHECK-NEXT: buffer_store_dword v0, off, s[68:71], 0 ; 4-byte Folded Spill +; CHECK-NEXT: buffer_load_dword v0, off, s[68:71], 0 offset:256 +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_mov_b64 exec, s[2:3] +; CHECK-NEXT: s_waitcnt lgkmcnt(0) +; CHECK-NEXT: s_cmp_lg_u32 s0, 0 +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; VGPR clobbers +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; VGPR clobbers +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_cbranch_scc0 .LBB0_2 +; CHECK-NEXT: ; %bb.1: ; %exit +; CHECK-NEXT: s_endpgm +; CHECK-NEXT: .LBB0_2: ; %use +; CHECK-NEXT: s_mov_b64 s[34:35], exec +; CHECK-NEXT: s_mov_b64 exec, 0xffffffff +; CHECK-NEXT: buffer_store_dword v0, off, s[68:71], 0 offset:256 +; CHECK-NEXT: v_writelane_b32 v0, s36, 0 +; CHECK-NEXT: v_writelane_b32 v0, s37, 1 +; CHECK-NEXT: v_writelane_b32 v0, s38, 2 +; CHECK-NEXT: v_writelane_b32 v0, s39, 3 +; CHECK-NEXT: v_writelane_b32 v0, s40, 4 +; CHECK-NEXT: v_writelane_b32 v0, s41, 5 +; CHECK-NEXT: v_writelane_b32 v0, s42, 6 +; CHECK-NEXT: v_writelane_b32 v0, s43, 7 +; CHECK-NEXT: v_writelane_b32 v0, s44, 8 +; CHECK-NEXT: v_writelane_b32 v0, s45, 9 +; CHECK-NEXT: v_writelane_b32 v0, s46, 10 +; CHECK-NEXT: v_writelane_b32 v0, s47, 11 +; CHECK-NEXT: v_writelane_b32 v0, s48, 12 +; CHECK-NEXT: v_writelane_b32 v0, s49, 13 +; CHECK-NEXT: v_writelane_b32 v0, s50, 14 +; CHECK-NEXT: v_writelane_b32 v0, s51, 15 +; CHECK-NEXT: v_writelane_b32 v0, s52, 16 +; CHECK-NEXT: v_writelane_b32 v0, s53, 17 +; CHECK-NEXT: v_writelane_b32 v0, s54, 18 +; CHECK-NEXT: v_writelane_b32 v0, s55, 19 +; CHECK-NEXT: v_writelane_b32 v0, s56, 20 +; CHECK-NEXT: v_writelane_b32 v0, s57, 21 +; CHECK-NEXT: v_writelane_b32 v0, s58, 22 +; CHECK-NEXT: v_writelane_b32 v0, s59, 23 +; CHECK-NEXT: v_writelane_b32 v0, s60, 24 +; CHECK-NEXT: v_writelane_b32 v0, s61, 25 +; CHECK-NEXT: v_writelane_b32 v0, s62, 26 +; CHECK-NEXT: v_writelane_b32 v0, s63, 27 +; CHECK-NEXT: v_writelane_b32 v0, s64, 28 +; CHECK-NEXT: v_writelane_b32 v0, s65, 29 +; CHECK-NEXT: v_writelane_b32 v0, s66, 30 +; CHECK-NEXT: v_writelane_b32 v0, s67, 31 +; CHECK-NEXT: buffer_store_dword v0, off, s[68:71], 0 offset:128 ; 4-byte Folded Spill +; CHECK-NEXT: buffer_load_dword v0, off, s[68:71], 0 offset:256 +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_mov_b64 exec, s[34:35] +; CHECK-NEXT: s_mov_b64 s[34:35], exec +; CHECK-NEXT: s_mov_b64 exec, 0xffffffff +; CHECK-NEXT: buffer_store_dword v0, off, s[68:71], 0 offset:256 +; CHECK-NEXT: buffer_load_dword v0, off, s[68:71], 0 ; 4-byte Folded Reload +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_readlane_b32 s36, v0, 0 +; CHECK-NEXT: v_readlane_b32 s37, v0, 1 +; CHECK-NEXT: v_readlane_b32 s38, v0, 2 +; CHECK-NEXT: v_readlane_b32 s39, v0, 3 +; CHECK-NEXT: v_readlane_b32 s40, v0, 4 +; CHECK-NEXT: v_readlane_b32 s41, v0, 5 +; CHECK-NEXT: v_readlane_b32 s42, v0, 6 +; CHECK-NEXT: v_readlane_b32 s43, v0, 7 +; CHECK-NEXT: v_readlane_b32 s44, v0, 8 +; CHECK-NEXT: v_readlane_b32 s45, v0, 9 +; CHECK-NEXT: v_readlane_b32 s46, v0, 10 +; CHECK-NEXT: v_readlane_b32 s47, v0, 11 +; CHECK-NEXT: v_readlane_b32 s48, v0, 12 +; CHECK-NEXT: v_readlane_b32 s49, v0, 13 +; CHECK-NEXT: v_readlane_b32 s50, v0, 14 +; CHECK-NEXT: v_readlane_b32 s51, v0, 15 +; CHECK-NEXT: v_readlane_b32 s52, v0, 16 +; CHECK-NEXT: v_readlane_b32 s53, v0, 17 +; CHECK-NEXT: v_readlane_b32 s54, v0, 18 +; CHECK-NEXT: v_readlane_b32 s55, v0, 19 +; CHECK-NEXT: v_readlane_b32 s56, v0, 20 +; CHECK-NEXT: v_readlane_b32 s57, v0, 21 +; CHECK-NEXT: v_readlane_b32 s58, v0, 22 +; CHECK-NEXT: v_readlane_b32 s59, v0, 23 +; CHECK-NEXT: v_readlane_b32 s60, v0, 24 +; CHECK-NEXT: v_readlane_b32 s61, v0, 25 +; CHECK-NEXT: v_readlane_b32 s62, v0, 26 +; CHECK-NEXT: v_readlane_b32 s63, v0, 27 +; CHECK-NEXT: v_readlane_b32 s64, v0, 28 +; CHECK-NEXT: v_readlane_b32 s65, v0, 29 +; CHECK-NEXT: v_readlane_b32 s66, v0, 30 +; CHECK-NEXT: v_readlane_b32 s67, v0, 31 +; CHECK-NEXT: buffer_load_dword v0, off, s[68:71], 0 offset:256 +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_mov_b64 exec, s[34:35] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; VGPR clobbers +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; VGPR clobbers +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_mov_b64 s[34:35], exec +; CHECK-NEXT: s_mov_b64 exec, 0xffffffff +; CHECK-NEXT: buffer_store_dword v0, off, s[68:71], 0 offset:256 +; CHECK-NEXT: buffer_load_dword v0, off, s[68:71], 0 offset:128 ; 4-byte Folded Reload +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_readlane_b32 s0, v0, 0 +; CHECK-NEXT: v_readlane_b32 s1, v0, 1 +; CHECK-NEXT: v_readlane_b32 s2, v0, 2 +; CHECK-NEXT: v_readlane_b32 s3, v0, 3 +; CHECK-NEXT: v_readlane_b32 s4, v0, 4 +; CHECK-NEXT: v_readlane_b32 s5, v0, 5 +; CHECK-NEXT: v_readlane_b32 s6, v0, 6 +; CHECK-NEXT: v_readlane_b32 s7, v0, 7 +; CHECK-NEXT: v_readlane_b32 s8, v0, 8 +; CHECK-NEXT: v_readlane_b32 s9, v0, 9 +; CHECK-NEXT: v_readlane_b32 s10, v0, 10 +; CHECK-NEXT: v_readlane_b32 s11, v0, 11 +; CHECK-NEXT: v_readlane_b32 s12, v0, 12 +; CHECK-NEXT: v_readlane_b32 s13, v0, 13 +; CHECK-NEXT: v_readlane_b32 s14, v0, 14 +; CHECK-NEXT: v_readlane_b32 s15, v0, 15 +; CHECK-NEXT: v_readlane_b32 s16, v0, 16 +; CHECK-NEXT: v_readlane_b32 s17, v0, 17 +; CHECK-NEXT: v_readlane_b32 s18, v0, 18 +; CHECK-NEXT: v_readlane_b32 s19, v0, 19 +; CHECK-NEXT: v_readlane_b32 s20, v0, 20 +; CHECK-NEXT: v_readlane_b32 s21, v0, 21 +; CHECK-NEXT: v_readlane_b32 s22, v0, 22 +; CHECK-NEXT: v_readlane_b32 s23, v0, 23 +; CHECK-NEXT: v_readlane_b32 s24, v0, 24 +; CHECK-NEXT: v_readlane_b32 s25, v0, 25 +; CHECK-NEXT: v_readlane_b32 s26, v0, 26 +; CHECK-NEXT: v_readlane_b32 s27, v0, 27 +; CHECK-NEXT: v_readlane_b32 s28, v0, 28 +; CHECK-NEXT: v_readlane_b32 s29, v0, 29 +; CHECK-NEXT: v_readlane_b32 s30, v0, 30 +; CHECK-NEXT: v_readlane_b32 s31, v0, 31 +; CHECK-NEXT: buffer_load_dword v0, off, s[68:71], 0 offset:256 +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_mov_b64 exec, s[34:35] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; VGPR clobbers +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_endpgm +entry: + call void asm sideeffect "", "~{v[0:7]},~{v[8:15]},~{v[16:23]},~{v[24:31]},~{v[32:39]},~{v[40:47]},~{v[48:55]},~{v[56:63]}" () + + %s0 = call <32 x i32> asm sideeffect "; VGPR clobbers", "=s" () + %s1 = call <32 x i32> asm sideeffect "; VGPR clobbers", "=s" () + %s2 = call <32 x i32> asm sideeffect "; VGPR clobbers", "=s" () + + %cmp = icmp eq i32 %in, 0 + br i1 %cmp, label %use, label %exit + +use: + call void asm sideeffect "; VGPR clobbers", "s"(<32 x i32> %s0) + call void asm sideeffect "; VGPR clobbers", "s"(<32 x i32> %s1) + call void asm sideeffect "; VGPR clobbers", "s"(<32 x i32> %s2) + br label %exit + +exit: + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-partial-pool.mir b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-partial-pool.mir new file mode 100644 index 0000000000000..6c79c594771b9 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-partial-pool.mir @@ -0,0 +1,50 @@ +# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -O0 -amdgpu-stress-vgpr=1 -amdgpu-num-vgprs-for-wwm-alloc=2 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,1 -verify-machineinstrs -o - %s | FileCheck %s +# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -O0 -amdgpu-num-vgprs-for-wwm-alloc=0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,1 -verify-machineinstrs -o - %s | FileCheck --check-prefix=ZERO %s + +# CHECK-LABEL: name: partial_pool +# CHECK-COUNT-2: flags: [ WWM_REG ] +# CHECK: renamable $vgpr0 = IMPLICIT_DEF +# CHECK: SI_SPILL_WWM_V32_SAVE $vgpr0 +# CHECK: $vgpr0 = SI_SPILL_WWM_V32_RESTORE + +# With an explicitly empty pool, otherwise available VGPRs must remain for the +# per-thread allocator and the SGPR spill pseudos must remain on the memory path. +# ZERO-LABEL: name: partial_pool +# ZERO-NOT: WWM_REG +# ZERO: SI_SPILL_S1024_SAVE +# ZERO: SI_SPILL_S32_SAVE + +--- | + define amdgpu_kernel void @partial_pool() #0 { + ret void + } + + attributes #0 = { "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" } +... +--- +name: partial_pool +tracksRegLiveness: true +frameInfo: + maxAlignment: 4 +stack: + - { id: 0, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill } + - { id: 1, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill } + - { id: 2, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill } +machineFunctionInfo: + isEntryFunction: true + scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' + stackPtrOffsetReg: '$sgpr32' + frameOffsetReg: '$sgpr33' + hasSpilledSGPRs: true +body: | + bb.0: + liveins: $sgpr34, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 + + SI_SPILL_S1024_SAVE killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + SI_SPILL_S1024_SAVE killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, %stack.1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + SI_SPILL_S32_SAVE killed $sgpr34, %stack.2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + renamable $sgpr34 = SI_SPILL_S32_RESTORE %stack.2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67 = SI_SPILL_S1024_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + S_ENDPGM 0 +... diff --git a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-preallocation-guard.mir b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-preallocation-guard.mir new file mode 100644 index 0000000000000..8647c6314c88e --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-preallocation-guard.mir @@ -0,0 +1,101 @@ +# RUN: split-file %s %t +# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -amdgpu-stress-vgpr=1 -amdgpu-num-vgprs-for-wwm-alloc=2 -run-pass=si-lower-sgpr-spills -filetype=null %t/strict.mir 2>&1 | FileCheck %s +# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -amdgpu-stress-vgpr=1 -amdgpu-num-vgprs-for-wwm-alloc=2 -amdgpu-prealloc-sgpr-spill-vgprs -run-pass=si-lower-sgpr-spills -filetype=null %t/option.mir 2>&1 | FileCheck %s +# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -amdgpu-stress-vgpr=1 -amdgpu-num-vgprs-for-wwm-alloc=2 -run-pass=si-lower-sgpr-spills -filetype=null %t/attribute.mir 2>&1 | FileCheck %s + +# CHECK: error: cannot find enough VGPRs for wwm-regalloc + +#--- strict.mir +--- | + define amdgpu_kernel void @strict() #0 { + ret void + } + attributes #0 = { "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" } +... +--- +name: strict +tracksRegLiveness: true +frameInfo: + maxAlignment: 4 +stack: + - { id: 0, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill } + - { id: 1, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill } + - { id: 2, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill } +machineFunctionInfo: + isEntryFunction: true + scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' + stackPtrOffsetReg: '$sgpr32' + frameOffsetReg: '$sgpr33' + hasSpilledSGPRs: true +body: | + bb.0: + liveins: $sgpr34, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 + %0:sreg_64 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec + $exec = EXIT_STRICT_WWM killed %0 + SI_SPILL_S1024_SAVE killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + SI_SPILL_S1024_SAVE killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, %stack.1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + SI_SPILL_S32_SAVE killed $sgpr34, %stack.2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + S_ENDPGM 0 +... + +#--- option.mir +--- | + define amdgpu_kernel void @option() #0 { + ret void + } + attributes #0 = { "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" } +... +--- +name: option +tracksRegLiveness: true +frameInfo: + maxAlignment: 4 +stack: + - { id: 0, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill } + - { id: 1, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill } + - { id: 2, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill } +machineFunctionInfo: + isEntryFunction: true + scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' + stackPtrOffsetReg: '$sgpr32' + frameOffsetReg: '$sgpr33' + hasSpilledSGPRs: true +body: | + bb.0: + liveins: $sgpr34, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 + SI_SPILL_S1024_SAVE killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + SI_SPILL_S1024_SAVE killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, %stack.1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + SI_SPILL_S32_SAVE killed $sgpr34, %stack.2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + S_ENDPGM 0 +... + +#--- attribute.mir +--- | + define amdgpu_kernel void @attribute() #0 { + ret void + } + attributes #0 = { "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-prealloc-sgpr-spill-vgprs" } +... +--- +name: attribute +tracksRegLiveness: true +frameInfo: + maxAlignment: 4 +stack: + - { id: 0, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill } + - { id: 1, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill } + - { id: 2, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill } +machineFunctionInfo: + isEntryFunction: true + scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' + stackPtrOffsetReg: '$sgpr32' + frameOffsetReg: '$sgpr33' + hasSpilledSGPRs: true +body: | + bb.0: + liveins: $sgpr34, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 + SI_SPILL_S1024_SAVE killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + SI_SPILL_S1024_SAVE killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, %stack.1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + SI_SPILL_S32_SAVE killed $sgpr34, %stack.2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32 + S_ENDPGM 0 +... diff --git a/llvm/test/CodeGen/MIR/AMDGPU/long-branch-reg-all-sgpr-used.ll b/llvm/test/CodeGen/MIR/AMDGPU/long-branch-reg-all-sgpr-used.ll index d8c8ea1a87895..025008fac5144 100644 --- a/llvm/test/CodeGen/MIR/AMDGPU/long-branch-reg-all-sgpr-used.ll +++ b/llvm/test/CodeGen/MIR/AMDGPU/long-branch-reg-all-sgpr-used.ll @@ -16,6 +16,7 @@ ; CHECK-NEXT: waveLimiter: false ; CHECK-NEXT: hasSpilledSGPRs: false ; CHECK-NEXT: hasSpilledVGPRs: false +; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false ; CHECK-NEXT: numWaveDispatchSGPRs: 0 ; CHECK-NEXT: numWaveDispatchVGPRs: 0 ; CHECK-NEXT: scratchRSrcReg: '$sgpr96_sgpr97_sgpr98_sgpr99' @@ -289,6 +290,7 @@ ; CHECK-NEXT: waveLimiter: false ; CHECK-NEXT: hasSpilledSGPRs: false ; CHECK-NEXT: hasSpilledVGPRs: false +; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false ; CHECK-NEXT: numWaveDispatchSGPRs: 0 ; CHECK-NEXT: numWaveDispatchVGPRs: 0 ; CHECK-NEXT: scratchRSrcReg: '$sgpr96_sgpr97_sgpr98_sgpr99' diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-after-pei.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-after-pei.ll index d8bb718866274..2c31f5c9c3477 100644 --- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-after-pei.ll +++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-after-pei.ll @@ -15,6 +15,7 @@ ; AFTER-PEI-NEXT: waveLimiter: false ; AFTER-PEI-NEXT: hasSpilledSGPRs: true ; AFTER-PEI-NEXT: hasSpilledVGPRs: false +; AFTER-PEI-NEXT: hasNoWWMPoolSGPRSpillFallback: false ; AFTER-PEI-NEXT: numWaveDispatchSGPRs: 0 ; AFTER-PEI-NEXT: numWaveDispatchVGPRs: 0 ; AFTER-PEI-NEXT: scratchRSrcReg: '$sgpr68_sgpr69_sgpr70_sgpr71' diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll index 796814baa195d..5497316fc1936 100644 --- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll +++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll @@ -16,6 +16,7 @@ ; CHECK-NEXT: waveLimiter: false ; CHECK-NEXT: hasSpilledSGPRs: false ; CHECK-NEXT: hasSpilledVGPRs: false +; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false ; CHECK-NEXT: numWaveDispatchSGPRs: 0 ; CHECK-NEXT: numWaveDispatchVGPRs: 0 ; CHECK-NEXT: scratchRSrcReg: '$sgpr96_sgpr97_sgpr98_sgpr99' diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll index 13a958c4b1ed5..8aab1c0fa55c8 100644 --- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll +++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll @@ -16,6 +16,7 @@ ; CHECK-NEXT: waveLimiter: false ; CHECK-NEXT: hasSpilledSGPRs: false ; CHECK-NEXT: hasSpilledVGPRs: false +; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false ; CHECK-NEXT: numWaveDispatchSGPRs: 0 ; CHECK-NEXT: numWaveDispatchVGPRs: 0 ; CHECK-NEXT: scratchRSrcReg: '$sgpr96_sgpr97_sgpr98_sgpr99' diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir index 22e74925f8139..9d1236b0f3739 100644 --- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir +++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir @@ -16,6 +16,7 @@ # FULL-NEXT: waveLimiter: true # FULL-NEXT: hasSpilledSGPRs: false # FULL-NEXT: hasSpilledVGPRs: false +# FULL-NEXT: hasNoWWMPoolSGPRSpillFallback: false # FULL-NEXT: numWaveDispatchSGPRs: 0 # FULL-NEXT: numWaveDispatchVGPRs: 0 # FULL-NEXT: scratchRSrcReg: '$sgpr8_sgpr9_sgpr10_sgpr11' @@ -129,6 +130,7 @@ body: | # FULL-NEXT: waveLimiter: false # FULL-NEXT: hasSpilledSGPRs: false # FULL-NEXT: hasSpilledVGPRs: false +# FULL-NEXT: hasNoWWMPoolSGPRSpillFallback: false # FULL-NEXT: numWaveDispatchSGPRs: 0 # FULL-NEXT: numWaveDispatchVGPRs: 0 # FULL-NEXT: scratchRSrcReg: '$private_rsrc_reg' @@ -212,6 +214,7 @@ body: | # FULL-NEXT: waveLimiter: false # FULL-NEXT: hasSpilledSGPRs: false # FULL-NEXT: hasSpilledVGPRs: false +# FULL-NEXT: hasNoWWMPoolSGPRSpillFallback: false # FULL-NEXT: numWaveDispatchSGPRs: 0 # FULL-NEXT: numWaveDispatchVGPRs: 0 # FULL-NEXT: scratchRSrcReg: '$private_rsrc_reg' @@ -296,6 +299,7 @@ body: | # FULL-NEXT: waveLimiter: false # FULL-NEXT: hasSpilledSGPRs: false # FULL-NEXT: hasSpilledVGPRs: false +# FULL-NEXT: hasNoWWMPoolSGPRSpillFallback: false # FULL-NEXT: numWaveDispatchSGPRs: 0 # FULL-NEXT: numWaveDispatchVGPRs: 0 # FULL-NEXT: scratchRSrcReg: '$private_rsrc_reg' diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll index 108a80950b59a..770718c166d4c 100644 --- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll +++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll @@ -19,6 +19,7 @@ ; CHECK-NEXT: waveLimiter: false ; CHECK-NEXT: hasSpilledSGPRs: false ; CHECK-NEXT: hasSpilledVGPRs: false +; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false ; CHECK-NEXT: numWaveDispatchSGPRs: 0 ; CHECK-NEXT: numWaveDispatchVGPRs: 0 ; CHECK-NEXT: scratchRSrcReg: '$sgpr96_sgpr97_sgpr98_sgpr99' @@ -82,6 +83,7 @@ define amdgpu_kernel void @kernel(i32 %arg0, i64 %arg1, <16 x i32> %arg2) { ; CHECK-NEXT: waveLimiter: false ; CHECK-NEXT: hasSpilledSGPRs: false ; CHECK-NEXT: hasSpilledVGPRs: false +; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false ; CHECK-NEXT: numWaveDispatchSGPRs: 3 ; CHECK-NEXT: numWaveDispatchVGPRs: 1 ; CHECK-NEXT: scratchRSrcReg: '$sgpr96_sgpr97_sgpr98_sgpr99' @@ -149,6 +151,7 @@ define amdgpu_ps void @gds_size_shader(i32 %arg0, i32 inreg %arg1) #5 { ; CHECK-NEXT: waveLimiter: false ; CHECK-NEXT: hasSpilledSGPRs: false ; CHECK-NEXT: hasSpilledVGPRs: false +; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false ; CHECK-NEXT: numWaveDispatchSGPRs: 16 ; CHECK-NEXT: numWaveDispatchVGPRs: 0 ; CHECK-NEXT: scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' @@ -208,6 +211,7 @@ define void @function() { ; CHECK-NEXT: waveLimiter: false ; CHECK-NEXT: hasSpilledSGPRs: false ; CHECK-NEXT: hasSpilledVGPRs: false +; CHECK-NEXT: hasNoWWMPoolSGPRSpillFallback: false ; CHECK-NEXT: numWaveDispatchSGPRs: 16 ; CHECK-NEXT: numWaveDispatchVGPRs: 0 ; CHECK-NEXT: scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' diff --git a/llvm/test/CodeGen/RISCV/fp128.ll b/llvm/test/CodeGen/RISCV/fp128.ll index 10f5fe9946bb7..9beecf0c025c8 100644 --- a/llvm/test/CodeGen/RISCV/fp128.ll +++ b/llvm/test/CodeGen/RISCV/fp128.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \ ; RUN: | FileCheck -check-prefix=RV32I %s +; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefix=RV64I %s @x = local_unnamed_addr global fp128 0xL00000000000000007FFF000000000000, align 16 @y = local_unnamed_addr global fp128 0xL00000000000000007FFF000000000000, align 16 @@ -38,6 +40,22 @@ define i32 @test_load_and_cmp() nounwind { ; RV32I-NEXT: lw ra, 44(sp) # 4-byte Folded Reload ; RV32I-NEXT: addi sp, sp, 48 ; RV32I-NEXT: ret +; +; RV64I-LABEL: test_load_and_cmp: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -16 +; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: lui a1, %hi(x) +; RV64I-NEXT: lui a3, %hi(y) +; RV64I-NEXT: ld a0, %lo(x)(a1) +; RV64I-NEXT: ld a1, %lo(x+8)(a1) +; RV64I-NEXT: ld a2, %lo(y)(a3) +; RV64I-NEXT: ld a3, %lo(y+8)(a3) +; RV64I-NEXT: call __netf2 +; RV64I-NEXT: snez a0, a0 +; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: addi sp, sp, 16 +; RV64I-NEXT: ret %1 = load fp128, ptr @x, align 16 %2 = load fp128, ptr @y, align 16 %cmp = fcmp une fp128 %1, %2 @@ -85,6 +103,22 @@ define i32 @test_add_and_fptosi() nounwind { ; RV32I-NEXT: lw ra, 76(sp) # 4-byte Folded Reload ; RV32I-NEXT: addi sp, sp, 80 ; RV32I-NEXT: ret +; +; RV64I-LABEL: test_add_and_fptosi: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -16 +; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: lui a1, %hi(x) +; RV64I-NEXT: lui a3, %hi(y) +; RV64I-NEXT: ld a0, %lo(x)(a1) +; RV64I-NEXT: ld a1, %lo(x+8)(a1) +; RV64I-NEXT: ld a2, %lo(y)(a3) +; RV64I-NEXT: ld a3, %lo(y+8)(a3) +; RV64I-NEXT: call __addtf3 +; RV64I-NEXT: call __fixtfsi +; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: addi sp, sp, 16 +; RV64I-NEXT: ret %1 = load fp128, ptr @x, align 16 %2 = load fp128, ptr @y, align 16 %3 = fadd fp128 %1, %2 @@ -137,6 +171,19 @@ define fp128 @fmaximum(fp128 %x, fp128 %y) { ; RV32I-NEXT: addi sp, sp, 64 ; RV32I-NEXT: .cfi_def_cfa_offset 0 ; RV32I-NEXT: ret +; +; RV64I-LABEL: fmaximum: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -16 +; RV64I-NEXT: .cfi_def_cfa_offset 16 +; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: .cfi_offset ra, -8 +; RV64I-NEXT: call fmaximuml +; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: .cfi_restore ra +; RV64I-NEXT: addi sp, sp, 16 +; RV64I-NEXT: .cfi_def_cfa_offset 0 +; RV64I-NEXT: ret %a = call fp128 @llvm.maximum.fp128(fp128 %x, fp128 %y) ret fp128 %a } @@ -186,6 +233,19 @@ define fp128 @fminimum(fp128 %x, fp128 %y) { ; RV32I-NEXT: addi sp, sp, 64 ; RV32I-NEXT: .cfi_def_cfa_offset 0 ; RV32I-NEXT: ret +; +; RV64I-LABEL: fminimum: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -16 +; RV64I-NEXT: .cfi_def_cfa_offset 16 +; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: .cfi_offset ra, -8 +; RV64I-NEXT: call fminimuml +; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: .cfi_restore ra +; RV64I-NEXT: addi sp, sp, 16 +; RV64I-NEXT: .cfi_def_cfa_offset 0 +; RV64I-NEXT: ret %a = call fp128 @llvm.minimum.fp128(fp128 %x, fp128 %y) ret fp128 %a } @@ -229,6 +289,28 @@ define { fp128, fp128 } @modf(fp128 %a) nounwind { ; RV32I-NEXT: lw s0, 56(sp) # 4-byte Folded Reload ; RV32I-NEXT: addi sp, sp, 64 ; RV32I-NEXT: ret +; +; RV64I-LABEL: modf: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -32 +; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill +; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill +; RV64I-NEXT: mv a3, a2 +; RV64I-NEXT: mv s0, a0 +; RV64I-NEXT: mv a2, sp +; RV64I-NEXT: mv a0, a1 +; RV64I-NEXT: mv a1, a3 +; RV64I-NEXT: call modfl +; RV64I-NEXT: ld a2, 0(sp) +; RV64I-NEXT: ld a3, 8(sp) +; RV64I-NEXT: sd a0, 0(s0) +; RV64I-NEXT: sd a1, 8(s0) +; RV64I-NEXT: sd a2, 16(s0) +; RV64I-NEXT: sd a3, 24(s0) +; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; RV64I-NEXT: addi sp, sp, 32 +; RV64I-NEXT: ret %result = call { fp128, fp128 } @llvm.modf.f128(fp128 %a) ret { fp128, fp128 } %result } @@ -443,6 +525,51 @@ define i96 @fptosi_fp128_to_i96(fp128 %a) nounwind { ; RV32I-NEXT: lw s10, 64(sp) # 4-byte Folded Reload ; RV32I-NEXT: addi sp, sp, 112 ; RV32I-NEXT: ret +; +; RV64I-LABEL: fptosi_fp128_to_i96: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -16 +; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: call __fixtfti +; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: addi sp, sp, 16 +; RV64I-NEXT: ret %result = fptosi fp128 %a to i96 ret i96 %result } + +define i32 @lround_f128() nounwind { +; RV32I-LABEL: lround_f128: +; RV32I: # %bb.0: +; RV32I-NEXT: addi sp, sp, -32 +; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32I-NEXT: lui a0, %hi(x) +; RV32I-NEXT: lw a1, %lo(x)(a0) +; RV32I-NEXT: lw a2, %lo(x+4)(a0) +; RV32I-NEXT: lw a3, %lo(x+8)(a0) +; RV32I-NEXT: lw a4, %lo(x+12)(a0) +; RV32I-NEXT: addi a0, sp, 8 +; RV32I-NEXT: sw a1, 8(sp) +; RV32I-NEXT: sw a2, 12(sp) +; RV32I-NEXT: sw a3, 16(sp) +; RV32I-NEXT: sw a4, 20(sp) +; RV32I-NEXT: call lroundl +; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32I-NEXT: addi sp, sp, 32 +; RV32I-NEXT: ret +; +; RV64I-LABEL: lround_f128: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -16 +; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: lui a1, %hi(x) +; RV64I-NEXT: ld a0, %lo(x)(a1) +; RV64I-NEXT: ld a1, %lo(x+8)(a1) +; RV64I-NEXT: call lroundl +; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: addi sp, sp, 16 +; RV64I-NEXT: ret + %1 = load fp128, ptr @x, align 16 + %r = call i32 @llvm.lround.f128(fp128 %1) + ret i32 %r +} diff --git a/llvm/test/CodeGen/RISCV/xqcilo-xqcilia-frame-index.ll b/llvm/test/CodeGen/RISCV/xqcilo-xqcilia-frame-index.ll index eaf5b78f30ff2..681418c16816c 100644 --- a/llvm/test/CodeGen/RISCV/xqcilo-xqcilia-frame-index.ll +++ b/llvm/test/CodeGen/RISCV/xqcilo-xqcilia-frame-index.ll @@ -111,3 +111,105 @@ define void @bare_fi_high_frame_store(i32 %x) nounwind { store i32 %x, ptr %small ret void } + +; Register-pressure regression test: bare frame-index loads from fixed stack +; slots (incoming args on stack) must select standard LW at ISel — not QC_E_LW — +; so that the register allocator can rematerialize them. With QC_E_LW selected, +; isLoadFromStackSlot does not recognize it, RA cannot rematerialize, and under +; high pressure it spills excessively. +declare dso_local i32 @sink(i32 noundef) local_unnamed_addr + +define dso_local i32 @regpressure(i32 noundef %a0, i32 noundef %a1, i32 noundef %a2, i32 noundef %a3, i32 noundef %a4, i32 noundef %a5, i32 noundef %a6, i32 noundef %a7, i32 noundef %s0, i32 noundef %s1, i32 noundef %s2, i32 noundef %s3, i32 noundef %s4, i32 noundef %s5, i32 noundef %s6, i32 noundef %s7, i32 noundef %s8, i32 noundef %s9, i32 noundef %s10, i32 noundef %s11) local_unnamed_addr nounwind { +; CHECK-LABEL: regpressure: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: c.addi16sp sp, -64 +; CHECK-NEXT: c.swsp ra, 60(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.swsp s0, 56(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.swsp s1, 52(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.swsp s2, 48(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.swsp s3, 44(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.swsp s4, 40(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.swsp s5, 36(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.swsp s6, 32(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.swsp s7, 28(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.swsp s8, 24(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.swsp s9, 20(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.swsp s10, 16(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.swsp s11, 12(sp) # 4-byte Folded Spill +; CHECK-NEXT: c.mv s0, a7 +; CHECK-NEXT: c.mv s1, a6 +; CHECK-NEXT: c.mv s2, a5 +; CHECK-NEXT: c.mv s3, a4 +; CHECK-NEXT: c.mv s4, a3 +; CHECK-NEXT: c.mv s5, a2 +; CHECK-NEXT: c.mv s6, a1 +; CHECK-NEXT: c.lwsp s8, 80(sp) +; CHECK-NEXT: c.lwsp s10, 76(sp) +; CHECK-NEXT: c.lwsp s11, 72(sp) +; CHECK-NEXT: c.lwsp s9, 68(sp) +; CHECK-NEXT: c.lwsp s7, 64(sp) +; CHECK-NEXT: call sink +; CHECK-NEXT: c.add s2, s3 +; CHECK-NEXT: c.add s0, s1 +; CHECK-NEXT: c.add s0, s2 +; CHECK-NEXT: c.add s9, s11 +; CHECK-NEXT: c.add s0, s7 +; CHECK-NEXT: c.add s8, s10 +; CHECK-NEXT: c.add s0, s9 +; CHECK-NEXT: c.lwsp a1, 84(sp) +; CHECK-NEXT: c.add s8, a1 +; CHECK-NEXT: c.add s0, s8 +; CHECK-NEXT: c.lwsp a1, 92(sp) +; CHECK-NEXT: c.lwsp a2, 88(sp) +; CHECK-NEXT: c.add a1, a2 +; CHECK-NEXT: c.lwsp a2, 108(sp) +; CHECK-NEXT: c.lwsp a3, 104(sp) +; CHECK-NEXT: c.add a2, a3 +; CHECK-NEXT: c.lwsp a3, 96(sp) +; CHECK-NEXT: c.add a1, a3 +; CHECK-NEXT: c.add a0, a2 +; CHECK-NEXT: c.lwsp a2, 100(sp) +; CHECK-NEXT: c.add a1, a2 +; CHECK-NEXT: c.add a0, s6 +; CHECK-NEXT: c.add a1, s0 +; CHECK-NEXT: c.add a0, s5 +; CHECK-NEXT: c.add a0, a1 +; CHECK-NEXT: c.add a0, s4 +; CHECK-NEXT: c.lwsp ra, 60(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.lwsp s0, 56(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.lwsp s1, 52(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.lwsp s2, 48(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.lwsp s3, 44(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.lwsp s4, 40(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.lwsp s5, 36(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.lwsp s6, 32(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.lwsp s7, 28(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.lwsp s8, 24(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.lwsp s9, 20(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.lwsp s10, 16(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.lwsp s11, 12(sp) # 4-byte Folded Reload +; CHECK-NEXT: c.addi16sp sp, 64 +; CHECK-NEXT: c.jr ra +entry: + %call = tail call i32 @sink(i32 noundef %a0) + %add2 = add i32 %a5, %a4 + %add4 = add i32 %add2, %a6 + %add6 = add i32 %add4, %a7 + %add7 = add i32 %add6, %s0 + %add8 = add i32 %add7, %s1 + %add9 = add i32 %add8, %s2 + %add10 = add i32 %add9, %s3 + %add11 = add i32 %add10, %s4 + %add12 = add i32 %add11, %s5 + %add13 = add i32 %add12, %s6 + %add14 = add i32 %add13, %s7 + %add15 = add i32 %add14, %s8 + %add16 = add i32 %add15, %s9 + %add17 = add i32 %add16, %s10 + %add18 = add i32 %add17, %s11 + %add19 = add i32 %add18, %call + %add20 = add i32 %add19, %a1 + %add21 = add i32 %add20, %a2 + %add22 = add i32 %add21, %a3 + ret i32 %add22 +} diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/convergent-jump-threading.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/convergent-jump-threading.ll new file mode 100644 index 0000000000000..88cf0d1d7d86e --- /dev/null +++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/convergent-jump-threading.ll @@ -0,0 +1,143 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -S -mtriple=amdgpu9.00-amd-amdhsa -passes='simplifycfg' < %s | FileCheck %s + +declare void @barrier() convergent +declare void @plain() + +define void @preserve_loop_header_branch(i1 %cond, ptr %ptr) convergent { +; CHECK-LABEL: define void @preserve_loop_header_branch( +; CHECK-SAME: i1 [[COND:%.*]], ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: call void @barrier() #[[ATTR0]] +; CHECK-NEXT: br i1 [[COND]], label %[[PRE_THEN:.*]], label %[[LOOP_HEADER:.*]] +; CHECK: [[PRE_THEN]]: +; CHECK-NEXT: store i32 1, ptr [[PTR]], align 4 +; CHECK-NEXT: br label %[[LOOP_HEADER]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_BODY:.*]], label %[[LOOP_LATCH:.*]] +; CHECK: [[LOOP_BODY]]: +; CHECK-NEXT: store i32 2, ptr [[PTR]], align 4 +; CHECK-NEXT: br label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: call void @barrier() #[[ATTR0]] +; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_HEADER]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %pre + +pre: + call void @barrier() convergent + br i1 %cond, label %pre.then, label %loop.header + +pre.then: + store i32 1, ptr %ptr, align 4 + br label %loop.header + +loop.header: + br i1 %cond, label %loop.body, label %loop.latch + +loop.body: + store i32 2, ptr %ptr, align 4 + br label %loop.latch + +loop.latch: + call void @barrier() convergent + br i1 %cond, label %loop.header, label %exit + +exit: + ret void +} + +define void @thread_non_convergent_loop_header_branch(i1 %cond, ptr %ptr) { +; CHECK-LABEL: define void @thread_non_convergent_loop_header_branch( +; CHECK-SAME: i1 [[COND:%.*]], ptr [[PTR:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: call void @plain() +; CHECK-NEXT: br i1 [[COND]], label %[[PRE_THEN:.*]], label %[[EXIT_CRITEDGE:.*]] +; CHECK: [[PRE_THEN]]: +; CHECK-NEXT: store i32 1, ptr [[PTR]], align 4 +; CHECK-NEXT: br label %[[LOOP_BODY:.*]] +; CHECK: [[LOOP_BODY]]: +; CHECK-NEXT: store i32 2, ptr [[PTR]], align 4 +; CHECK-NEXT: call void @plain() +; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_BODY]], label %[[EXIT:.*]] +; CHECK: [[EXIT_CRITEDGE]]: +; CHECK-NEXT: call void @plain() +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %pre + +pre: + call void @plain() + br i1 %cond, label %pre.then, label %loop.header + +pre.then: + store i32 1, ptr %ptr, align 4 + br label %loop.header + +loop.header: + br i1 %cond, label %loop.body, label %loop.latch + +loop.body: + store i32 2, ptr %ptr, align 4 + br label %loop.latch + +loop.latch: + call void @plain() + br i1 %cond, label %loop.header, label %exit + +exit: + ret void +} + +define void @thread_convergent_only_on_exit_path(i1 %cond, ptr %ptr) convergent { +; CHECK-LABEL: define void @thread_convergent_only_on_exit_path( +; CHECK-SAME: i1 [[COND:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: call void @plain() +; CHECK-NEXT: br i1 [[COND]], label %[[PRE_THEN:.*]], label %[[EXIT_CRITEDGE:.*]] +; CHECK: [[PRE_THEN]]: +; CHECK-NEXT: store i32 1, ptr [[PTR]], align 4 +; CHECK-NEXT: br label %[[LOOP_BODY:.*]] +; CHECK: [[LOOP_BODY]]: +; CHECK-NEXT: store i32 2, ptr [[PTR]], align 4 +; CHECK-NEXT: call void @plain() +; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_BODY]], label %[[EXIT:.*]] +; CHECK: [[EXIT_CRITEDGE]]: +; CHECK-NEXT: call void @plain() +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: call void @barrier() #[[ATTR0]] +; CHECK-NEXT: ret void +; +entry: + br label %pre + +pre: + call void @plain() + br i1 %cond, label %pre.then, label %loop.header + +pre.then: + store i32 1, ptr %ptr, align 4 + br label %loop.header + +loop.header: + br i1 %cond, label %loop.body, label %loop.latch + +loop.body: + store i32 2, ptr %ptr, align 4 + br label %loop.latch + +loop.latch: + call void @plain() + br i1 %cond, label %loop.header, label %exit + +exit: + call void @barrier() convergent + ret void +} diff --git a/llvm/tools/llvm-gpu-loader/llvm-gpu-loader.cpp b/llvm/tools/llvm-gpu-loader/llvm-gpu-loader.cpp index 2747757e1c200..9505355e751ed 100644 --- a/llvm/tools/llvm-gpu-loader/llvm-gpu-loader.cpp +++ b/llvm/tools/llvm-gpu-loader/llvm-gpu-loader.cpp @@ -65,6 +65,11 @@ static cl::alias Blocks("blocks", cl::aliasopt(BlocksX), cl::desc("Alias for --blocks-x"), cl::cat(LoaderCategory)); +static cl::list Kernels( + "kernel", cl::value_desc("name"), + cl::desc("Launch '(void)' instead of the 'main' entry point."), + cl::cat(LoaderCategory)); + static cl::opt File(cl::Positional, cl::Required, cl::desc(""), cl::cat(LoaderCategory)); @@ -263,21 +268,29 @@ int main(int argc, const char **argv, const char **envp) { OFFLOAD_ERR(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, sizeof(int), &DevRet)); OFFLOAD_ERR(olMemcpy(Queue, DevRet, Device, &Zero, Host, sizeof(int))); - ol_kernel_launch_size_args_t BeginLaunch{1, {1, 1, 1}, {1, 1, 1}, 0}; - launchKernel(Queue, Device, Program, "_begin", BeginLaunch, DevArgc, DevArgv, - DevEnvp); - OFFLOAD_ERR(olSyncQueue(Queue)); - uint32_t Dims = (BlocksZ > 1) ? 3 : (BlocksY > 1) ? 2 : 1; ol_kernel_launch_size_args_t StartLaunch{Dims, {BlocksX, BlocksY, BlocksZ}, {ThreadsX, ThreadsY, ThreadsZ}, /*SharedMemBytes=*/0}; - launchKernel(Queue, Device, Program, "_start", StartLaunch, DevArgc, DevArgv, - DevEnvp, DevRet); - - ol_kernel_launch_size_args_t EndLaunch{1, {1, 1, 1}, {1, 1, 1}, 0}; - launchKernel(Queue, Device, Program, "_end", EndLaunch); + if (!Kernels.empty()) { + // Launch the user-specified kernels in order. These must take no arguments. + for (const std::string &Kernel : Kernels) + launchKernel(Queue, Device, Program, Kernel.c_str(), StartLaunch); + } else { + // The '_begin' and '_end' kernels perform libc startup and teardown. Global + // constructors and destructors are handled automatically by the runtime. + ol_kernel_launch_size_args_t BeginLaunch{1, {1, 1, 1}, {1, 1, 1}, 0}; + launchKernel(Queue, Device, Program, "_begin", BeginLaunch, DevArgc, + DevArgv, DevEnvp); + OFFLOAD_ERR(olSyncQueue(Queue)); + + launchKernel(Queue, Device, Program, "_start", StartLaunch, DevArgc, + DevArgv, DevEnvp, DevRet); + + ol_kernel_launch_size_args_t EndLaunch{1, {1, 1, 1}, {1, 1, 1}, 0}; + launchKernel(Queue, Device, Program, "_end", EndLaunch); + } int Ret; OFFLOAD_ERR(olMemcpy(Queue, &Ret, Host, DevRet, Device, sizeof(int))); diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp index 90061026e6225..084dcb0a5847f 100644 --- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp +++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp @@ -8345,6 +8345,7 @@ TEST_F(OpenMPIRBuilderTest, EmitOffloadingArraysNonContigCountExpression) { CombinedInfo.Types.push_back(static_cast( omp::OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG | omp::OpenMPOffloadMappingFlags::OMP_MAP_TO)); + CombinedInfo.HasAttachPtr.push_back(false); CombinedInfo.Names.push_back( Builder.CreateGlobalString("data", "data_name", 0, M.get())); diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp index 377a3304231c8..f13704b138d88 100644 --- a/llvm/unittests/TargetParser/TargetParserTest.cpp +++ b/llvm/unittests/TargetParser/TargetParserTest.cpp @@ -3120,12 +3120,12 @@ TEST(TargetParserTest, testAMDGPUParseTargetIDString) { } EXPECT_EQ(TargetID::parse(AMDHSA, "gfx908:xnack+:sramecc-") - ->getCanonicalTargetIDString(), + ->getCanonicalFeatureString(), "gfx908:sramecc-:xnack+"); - EXPECT_EQ(TargetID::parse(AMDHSA, "gfx908")->getCanonicalTargetIDString(), + EXPECT_EQ(TargetID::parse(AMDHSA, "gfx908")->getCanonicalFeatureString(), "gfx908"); EXPECT_EQ(TargetID::parse(Triple("amdgcn-amd-amdpal"), "gfx908:xnack-") - ->getCanonicalTargetIDString(), + ->getCanonicalFeatureString(), "gfx908:xnack-"); EXPECT_TRUE(TargetID::parse(AMDHSA, "").has_value()); EXPECT_FALSE(TargetID::parse(AMDHSA, "gfxbogus").has_value()); diff --git a/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn index c637c1df59208..e2bae2a1d1f07 100644 --- a/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn +++ b/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn @@ -38,6 +38,7 @@ static_library("Utility") { "ProcessInfo.cpp", "RealpathPrefixes.cpp", "RegisterFlags.cpp", + "RegisterType.cpp", "RegisterValue.cpp", "RegularExpression.cpp", "Scalar.cpp", diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn index 2a2e0f4ff8430..202e6c7d6e955 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn @@ -57,6 +57,8 @@ static_library("Utils") { "LoopPeel.cpp", "LoopRotationUtils.cpp", "LoopSimplify.cpp", + "LoopSplitUtils.cpp", + "LoopSplitUtilsPass.cpp", "LoopUnroll.cpp", "LoopUnrollAndJam.cpp", "LoopUnrollRuntime.cpp", diff --git a/llvm/utils/lit/lit/ShellEnvironment.py b/llvm/utils/lit/lit/ShellEnvironment.py index 1945865b19199..e2cf602de96ff 100644 --- a/llvm/utils/lit/lit/ShellEnvironment.py +++ b/llvm/utils/lit/lit/ShellEnvironment.py @@ -1,7 +1,11 @@ +from __future__ import annotations + +import io import os import platform import subprocess import tempfile +from typing import BinaryIO, Protocol, TextIO import lit.util from lit.ShCommands import GlobItem @@ -206,6 +210,67 @@ def processRedirects(cmd, stdin_source, cmd_shenv, opened_files): return std_fds +def as_binary_reader(stream: None | int | io.TextIOBase | BinaryIO) -> BinaryIO: + """Adapts a builtin's stdin source into a binary, read()-able stream. + + Args: + stream: Standard input source from pipeline dispatch. Supported types: + - None or subprocess sentinel (PIPE, DEVNULL, STDOUT) for no input + - Binary stream (BytesIO, temporary/spooled file, or 'rb' mode) + - Text stream (text-mode '<' redirect or upstream universal_newlines pipe) + + Returns: + A binary, read()-able stream. Text streams are unwrapped to their + underlying buffer so an in-process builtin sees the same raw bytes a + child process would. + """ + if stream is None or isinstance(stream, int): + # No real input to read. + return io.BytesIO(b"") + if isinstance(stream, io.TextIOBase): + buffer = getattr(stream, "buffer", None) + if buffer is not None: + return buffer + data = stream.read() + return io.BytesIO(data.encode() if isinstance(data, str) else data) + # Already a binary reader. + assert hasattr(stream, "read"), f"expected a binary reader, got {type(stream)!r}" + return stream + + +class BinaryFileWriter: + """Writes bytes straight to a file descriptor, matching Popen's behavior. + + Writes directly to the underlying file descriptor with os.write, bypassing + the wrapped file object's text-mode buffering and newline translation, so + the exact bytes passed in reach the fd unchanged. + """ + + # TODO: Replace __slots__ with @dataclass(slots=True) + # once the minimum Python version is bumped to 3.10 + # https://github.com/llvm/llvm-project/issues/200531 + __slots__ = ("fd",) + + def __init__(self, fileobj: TextIO) -> None: + # Owned by opened_files, never closed here + self.fd = fileobj.fileno() + + def write(self, data: bytes) -> int: + return os.write(self.fd, data) + + +def binary_fd(fileobj: TextIO) -> BinaryFileWriter: + """Wraps a redirect file object as a byte-exact writer for in-process builtins.""" + return BinaryFileWriter(fileobj) + + +class ByteWriter(Protocol): + """Structural type for objects that support writing a bytes buffer.""" + + def write(self, data: bytes) -> object: + ... + + def expand_glob(arg, cwd): if isinstance(arg, GlobItem): return sorted(arg.resolve(cwd)) diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py index 97cb54c63be64..681a56d9b45db 100644 --- a/llvm/utils/lit/lit/TestRunner.py +++ b/llvm/utils/lit/lit/TestRunner.py @@ -1,6 +1,7 @@ from __future__ import annotations import enum +import io import os import pathlib import re @@ -11,17 +12,23 @@ import tempfile import threading import traceback +from typing import IO, BinaryIO, Callable, List, TextIO, Union import lit.InprocBuiltins as InprocBuiltins import lit.ShUtil as ShUtil import lit.Test as Test import lit.util +import lit.builtin_commands.cat as builtin_cat +import lit.builtin_commands.diff as builtin_diff from lit.BooleanExpression import BooleanExpression from lit.ShCommands import Command from lit.ShellEnvironment import ( + ByteWriter, InternalShellError, ShellCommandResult, ShellEnvironment, + as_binary_reader, + binary_fd, expand_glob, expand_glob_expressions, kAvoidDevNull, @@ -212,6 +219,299 @@ def _replaceReadFile(match): return arguments +class PipeIOConfig: + """Configuration for how one pipeline stage's output is routed, whether + to the next stage, a redirect file, or captured for the caller, and the + working directory the stage runs in. + + Attributes: + out_sink: Where a non-last stage's output goes if the next stage + reads it. None for the last stage or a real redirect. + redirect_out: Binary writer for a '>'/'>>' target, else None. + capture_out: True if this stage's output should be captured and + returned to the caller instead of piped or redirected. + err_sink: Binary writer for a '2>file' target, else None. + cwd: The shell environment's current working directory. + merge_err: True for '2>&1', stderr writes to the out target instead. + """ + + # TODO: Replace __slots__ with @dataclass(slots=True) + # once the minimum Python version is bumped to 3.10 + # https://github.com/llvm/llvm-project/issues/200531 + __slots__ = ( + "out_sink", + "redirect_out", + "capture_out", + "err_sink", + "cwd", + "merge_err", + ) + + def __init__( + self, + out_sink: IO[bytes] | None, + redirect_out: ByteWriter | None, + capture_out: bool, + err_sink: ByteWriter | None, + cwd: str, + merge_err: bool = False, + ) -> None: + self.out_sink = out_sink + self.redirect_out = redirect_out + self.capture_out = capture_out + self.err_sink = err_sink + self.cwd = cwd + self.merge_err = merge_err + + +# A builtin's run(argv, stdin, stdout, stderr, cwd) entry point, e.g. +# builtin_cat.run or builtin_diff.run. stdout/stderr accept anything with a +# byte-oriented write() method: a real BytesIO or the duck-typed ByteWriter. +RunFn = Callable[ + [ + List[str], + BinaryIO, + Union[IO[bytes], ByteWriter, None], + Union[IO[bytes], ByteWriter, None], + str, + ], + int, +] + + +class InProcessPipe: + """Popen-compatible shim for an I/O-heavy builtin command. + + Runs the builtin (e.g., cat or diff) in-process instead of spawning + it. Anything that isn't a lit builtin still spawns a real subprocess. + + Exposes the same interface as subprocess.Popen, so it's a drop-in + replacement wherever a real subprocess is expected. + + Attributes: + returncode: The builtin's exit code. + stdout: The captured stdout if this stage's output was captured, + else None. + stderr: The captured stderr. + """ + + # TODO: Replace __slots__ with @dataclass(slots=True) + # once the minimum Python version is bumped to 3.10 + # https://github.com/llvm/llvm-project/issues/200531 + __slots__ = ("returncode", "stdout", "stderr", "_out", "_err") + + def __init__( + self, + run_fn: RunFn, + args: List[str], + stdin: None | int | io.TextIOBase | BinaryIO, + stage_io: PipeIOConfig, + ) -> None: + """Runs run_fn synchronously and captures the result. + + Args: + run_fn: The builtin's run function, e.g. cat.run or diff.run. + args: The argv to pass to run_fn, args[0] is the command name. + stdin: This stage's input stream. + stage_io: Where this stage's output and error streams go, and + its cwd. + """ + in_stream = as_binary_reader(stdin) + out_buf = io.BytesIO() if stage_io.capture_out else None + out_target = ( + out_buf + if stage_io.capture_out + else (stage_io.redirect_out or stage_io.out_sink) + ) + if stage_io.merge_err: + err_target, err_buf = out_target, None + elif stage_io.err_sink is not None: + err_target, err_buf = stage_io.err_sink, None + else: + err_buf = io.BytesIO() + err_target = err_buf + + self.returncode = run_fn(args, in_stream, out_target, err_target, stage_io.cwd) + + self._out = out_buf.getvalue() if out_buf is not None else b"" + self._err = err_buf.getvalue() if err_buf is not None else b"" + self.stdout = io.BytesIO(self._out) if stage_io.capture_out else None + self.stderr = io.BytesIO(self._err) + + def communicate(self) -> tuple[bytes, bytes]: + return (self._out, self._err) + + def wait(self) -> int: + return self.returncode + + def poll(self) -> int: + return self.returncode + + def kill(self) -> None: + pass + + def terminate(self) -> None: + pass + + +def _resolve_redirect_out(stdout: int | TextIO) -> ByteWriter | None: + """Wraps a '>' or '>>' redirect target as a binary writer. + + Args: + stdout: This stage's stdout target, a subprocess sentinel + (PIPE/STDOUT) or an open file object for a real redirect. + + Returns: + A binary writer wrapping the redirect file. None if stdout is a + PIPE or STDOUT sentinel instead of a real redirect file, since + that case doesn't need this sink. + """ + if isinstance(stdout, int): + return None + return binary_fd(stdout) + + +def _should_capture(is_last: bool, stdout: int | TextIO) -> bool: + """Whether this stage's output should be captured for the caller. + + Args: + is_last: True for the pipeline's final stage, the only stage allowed + to capture instead of feeding the next stage or a redirect. + stdout: The stage's stdout target. Capture only applies when + this is subprocess.PIPE, ruling out a real redirect file. + + Returns: + True only for the last stage of a piped command reading from a + PIPE, the one stage that reports output back to the caller + instead of writing to a temp file or redirect. + """ + return is_last and stdout == subprocess.PIPE + + +def _make_out_sink(stdout: int | TextIO, is_last: bool) -> IO[bytes] | None: + """Builds the output sink for a non-last stage feeding a pipe. + + Args: + stdout: The stage's stdout target. Only subprocess.PIPE gets a + sink here, a real redirect target writes there directly. + is_last: True for the pipeline's final stage, which streams to + the caller instead of spooling for a downstream stage. + + Returns: + A SpooledTemporaryFile, which stays in memory unless its size or + a downstream call to .fileno() forces it to disk. None for the + last stage or a non-pipe target, which don't need this sink. + """ + if stdout == subprocess.PIPE and not is_last: + return tempfile.SpooledTemporaryFile(max_size=1 << 20) + return None + + +def _resolve_err_sink(stderr: int | TextIO, merge_err: bool) -> ByteWriter | None: + """Wraps a '2>file' redirect target as a binary writer. + + Args: + stderr: This stage's stderr target. Only a real redirect file + needs wrapping here, PIPE/STDOUT are handled elsewhere. + merge_err: Whether stderr is merged into stdout (2>&1). + + Returns: + A binary writer wrapping the redirect file. None when stderr is + merged into stdout or is a PIPE/STDOUT sentinel, since both cases + are captured into the result instead of written through a sink + here. + """ + if merge_err or isinstance(stderr, int): + return None + return binary_fd(stderr) + + +def _should_run_inproc( + builtin_fn: RunFn | None, + not_crash: bool, + cmd_shenv: ShellEnvironment, + shenv: ShellEnvironment, +) -> bool: + """Whether this pipeline stage can run in-process instead of spawning. + + Args: + builtin_fn: The builtin's run function for this command (e.g. + builtin_cat.run), or None if the command has no in-process + implementation. + not_crash: True if the command is wrapped in 'not --crash'. + cmd_shenv: The environment this specific command runs under. + shenv: The pipeline's shared environment. + + Returns: + False if builtin_fn is None, if not_crash is set, or if cmd_shenv + is not shenv (a per-command 'env' only takes effect on a spawned + child, not an in-process call sharing our environment). True + otherwise. + """ + return builtin_fn is not None and not not_crash and cmd_shenv is shenv + + +def _run_inproc_stage( + args: List[str], + builtin_fn: RunFn, + stdin: None | int | io.TextIOBase | BinaryIO, + stdout: int | TextIO, + stderr: int | TextIO, + cmd_shenv: ShellEnvironment, + is_last: bool, + named_temp_files: List[str], +) -> tuple[InProcessPipe, IO[bytes] | int]: + """Runs one in-process pipeline stage and returns its default_stdin. + + Args: + args: The command's argument vector. + builtin_fn: The builtin's run function to execute (e.g., + cat.run or diff.run). + stdin: Current stage's input. None or a placeholder int means no + real input, otherwise an open binary or text stream to read + from. + stdout: Target for the stage's stdout, either subprocess.PIPE or + an open redirect file. Whether PIPE feeds the next stage or + is captured depends on is_last. + stderr: Target for the stage's stderr. Set to subprocess.STDOUT + when merged into stdout. + cmd_shenv: Shell environment the stage runs under. + is_last: True for the final stage in the pipeline. Together with + stdout being subprocess.PIPE, this determines whether output + is captured instead of feeding a downstream stage. + named_temp_files: Caller-owned cleanup list. Any temp file created + to stand in for /dev/null is appended here. + + Returns: + The InProcessPipe wrapping the stage, and the stdin the next stage + should read from (the spooled output, or subprocess.PIPE). + """ + if kAvoidDevNull: + for arg_idx, arg in enumerate(args): + if isinstance(arg, str) and kDevNull in arg: + devnull = tempfile.NamedTemporaryFile(delete=False) + devnull.close() + named_temp_files.append(devnull.name) + args[arg_idx] = arg.replace(kDevNull, devnull.name) + args = expand_glob_expressions(args, cmd_shenv.cwd) + merge_err = stderr == subprocess.STDOUT + stage_io = PipeIOConfig( + out_sink=_make_out_sink(stdout, is_last), + redirect_out=_resolve_redirect_out(stdout), + capture_out=_should_capture(is_last, stdout), + err_sink=_resolve_err_sink(stderr, merge_err), + cwd=cmd_shenv.cwd, + merge_err=merge_err, + ) + proc = InProcessPipe(builtin_fn, args, stdin, stage_io) + if stage_io.out_sink is not None: + stage_io.out_sink.seek(0) + default_stdin = stage_io.out_sink + else: + default_stdin = subprocess.PIPE + return proc, default_stdin + + def _executeShCmd(cmd, shenv, results, timeoutHelper): if timeoutHelper.timeoutReached(): # Prevent further recursion if the timeout has been hit @@ -268,6 +568,10 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper): "umask": InprocBuiltins.executeBuiltinUmask, ":": InprocBuiltins.executeBuiltinColon, } + pipeline_builtins = { + "cat": builtin_cat.run, + "diff": builtin_diff.run, + } # To avoid deadlock, we use a single stderr stream for piped # output. This is null until we have seen some output using # stderr. @@ -362,13 +666,14 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper): results.append(result) return result.exitCode - # Resolve any out-of-process builtin command before adding back 'not' - # commands. - if args[0] in builtin_commands: + builtin_fn = pipeline_builtins.get(args[0]) + use_inproc = _should_run_inproc(builtin_fn, not_crash, cmd_shenv, shenv) + if not use_inproc and args[0] in builtin_commands: args.insert(0, sys.executable) cmd_shenv.env["PYTHONPATH"] = os.path.dirname(os.path.abspath(__file__)) args[1] = os.path.join(builtin_commands_dir, args[1] + ".py") + # We had to search through the 'not' commands to find all the 'env' # commands and any other in-process builtin command. We don't want to # reimplement 'not' and its '--crash' here, so just push all 'not' @@ -395,6 +700,21 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper): j, default_stdin, cmd_shenv, opened_files ) + if use_inproc: + proc, default_stdin = _run_inproc_stage( + args, + builtin_fn, + stdin, + stdout, + stderr, + cmd_shenv, + is_last=j is cmd.commands[-1], + named_temp_files=named_temp_files, + ) + procs.append(proc) + proc_not_counts.append(not_count) + proc_not_fail_if_crash.append(False) + continue # If stderr wants to come from stdout, but stdout isn't a pipe, then put # stderr on a pipe and treat it as stdout. if stderr == subprocess.STDOUT and stdout != subprocess.PIPE: diff --git a/llvm/utils/lit/lit/builtin_commands/diff.py b/llvm/utils/lit/lit/builtin_commands/diff.py index 9d1a398106664..3a85d2210705f 100644 --- a/llvm/utils/lit/lit/builtin_commands/diff.py +++ b/llvm/utils/lit/lit/builtin_commands/diff.py @@ -92,11 +92,7 @@ def compareTwoBinaryFiles(flags, filepaths, filelines, stdout): ) for diff in diffs: - stdout.write( - diff.decode(errors="backslashreplace").encode( - locale.getpreferredencoding(False) - ) - ) + stdout.write(diff.decode(errors="backslashreplace").encode("utf-8")) exitCode = 1 return exitCode @@ -153,9 +149,7 @@ def printDirVsFile(dir_path, file_path, stdout): msg = "File %s is a directory while file %s is a regular file" else: msg = "File %s is a directory while file %s is a regular empty file" - stdout.write( - (msg % (dir_path, file_path) + "\n").encode(locale.getpreferredencoding(False)) - ) + stdout.write((msg % (dir_path, file_path) + "\n").encode("utf-8")) def printFileVsDir(file_path, dir_path, stdout): @@ -163,16 +157,12 @@ def printFileVsDir(file_path, dir_path, stdout): msg = "File %s is a regular file while file %s is a directory" else: msg = "File %s is a regular empty file while file %s is a directory" - stdout.write( - (msg % (file_path, dir_path) + "\n").encode(locale.getpreferredencoding(False)) - ) + stdout.write((msg % (file_path, dir_path) + "\n").encode("utf-8")) def printOnlyIn(basedir, path, name, stdout): stdout.write( - ("Only in %s: %s\n" % (os.path.join(basedir, path), name)).encode( - locale.getpreferredencoding(False) - ) + ("Only in %s: %s\n" % (os.path.join(basedir, path), name)).encode("utf-8") ) diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp index 727c98aef6619..8a45836426931 100644 --- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp +++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp @@ -615,12 +615,13 @@ struct TransferReadLowering : public OpRewritePattern { AffineMap readMap = readOp.getPermutationMap(); bool isTransposeLoad = isInnermostTwoDimsTransposed(readMap); - // Prefer an nd block load. It requires HW block-load support, a non-0D - // vector backed by a scalar-element memref, and a map the block load can - // realize. Out-of-bounds reads are allowed as long as the padding matches - // load_nd's implicit zero padding. + // Prefer an nd block load. It requires HW block-load support, a vector of + // rank >= 2 backed by a scalar-element memref, and a map the block load can + // realize. 1D vectors use the scattered xegpu.load path instead, which has + // a richer interface (e.g. layout capabilities). Out-of-bounds reads are + // allowed as long as the padding matches load_nd's implicit zero padding. bool canLowerToLoadNd = - hasBlockLoadSupport && loadedVecTy.getRank() > 0 && + hasBlockLoadSupport && loadedVecTy.getRank() > 1 && (readMap.isMinorIdentity() || isTransposeLoad) && readMemTy.getElementType().isIntOrFloat() && (!isOutOfBounds || isZeroOrPoisonPadding(readOp.getPadding())); @@ -726,12 +727,13 @@ struct TransferWriteLowering bool hasBlockStoreSupport = (chip == "pvc" || chip == "bmg" || chip == "cri"); - // Prefer an nd block store. It requires HW block-store support, a non-0D - // vector backed by a scalar-element memref, and a minor-identity map (block - // stores have no transpose support). Out-of-bounds writes are handled by - // the descriptor's boundary check. + // Prefer an nd block store. It requires HW block-store support, a vector of + // rank >= 2 backed by a scalar-element memref, and a minor-identity map + // (block stores have no transpose support). 1D vectors use the scattered + // xegpu.store path instead, which has a richer interface. Out-of-bounds + // writes are handled by the descriptor's boundary check. AffineMap map = writeOp.getPermutationMap(); - bool canLowerToStoreNd = hasBlockStoreSupport && vecTy.getRank() > 0 && + bool canLowerToStoreNd = hasBlockStoreSupport && vecTy.getRank() > 1 && map.isMinorIdentity() && writeMemTy.getElementType().isIntOrFloat(); @@ -924,6 +926,63 @@ struct StoreLowering : public OpRewritePattern { } }; +// If `indexingMaps` describe a (batched) row-major matmul +// lhs[b..., m, k], rhs[b..., k, n], acc[b..., m, n] +// return the number of leading batch dims (0 for a plain 2D matmul); +// otherwise return std::nullopt. +static std::optional +getRowMajorMatmulBatchRank(ArrayAttr indexingMaps) { + if (indexingMaps.size() != 3) + return std::nullopt; + + AffineMap mapA = cast(indexingMaps[0]).getValue(); + AffineMap mapB = cast(indexingMaps[1]).getValue(); + AffineMap mapC = cast(indexingMaps[2]).getValue(); + + // The result map exposes the batch dims followed by the core (m, n) dims. + if (mapC.getNumResults() < 2) + return std::nullopt; + int64_t batchRank = mapC.getNumResults() - 2; + + // A single `k` reduction gives batchRank + 3 iteration dims; each operand + // map exposes batchRank + 2 dims (batch dims + 2 core dims). + unsigned numDims = static_cast(batchRank) + 3; + unsigned numOperandResults = static_cast(batchRank) + 2; + if (mapA.getNumInputs() != numDims || mapB.getNumInputs() != numDims || + mapC.getNumInputs() != numDims) + return std::nullopt; + if (mapA.getNumResults() != numOperandResults || + mapB.getNumResults() != numOperandResults) + return std::nullopt; + + // Reconstruct the canonical maps from the batch/m/n dims of the result and + // the k dim of lhs, then compare against the actual maps. + MLIRContext *context = indexingMaps.getContext(); + ArrayRef batchDims = mapC.getResults().take_front(batchRank); + AffineExpr m = mapC.getResult(batchRank); + AffineExpr n = mapC.getResult(batchRank + 1); + AffineExpr k = mapA.getResult(batchRank + 1); + + SmallVector aDims = llvm::to_vector(batchDims); + aDims.push_back(m); + aDims.push_back(k); + SmallVector bDims = llvm::to_vector(batchDims); + bDims.push_back(k); + bDims.push_back(n); + SmallVector cDims = llvm::to_vector(batchDims); + cDims.push_back(m); + cDims.push_back(n); + + auto expected = ArrayAttr::get( + context, + {AffineMapAttr::get(AffineMap::get(numDims, 0, aDims, context)), + AffineMapAttr::get(AffineMap::get(numDims, 0, bDims, context)), + AffineMapAttr::get(AffineMap::get(numDims, 0, cDims, context))}); + if (indexingMaps != expected) + return std::nullopt; + return batchRank; +} + struct ContractionLowering : public OpRewritePattern { using Base::Base; @@ -935,21 +994,26 @@ struct ContractionLowering : public OpRewritePattern { return rewriter.notifyMatchFailure(contractOp, "Expects add combining kind"); + TypedValue lhs = contractOp.getLhs(); + TypedValue rhs = contractOp.getRhs(); TypedValue acc = contractOp.getAcc(); VectorType accType = dyn_cast(acc.getType()); - if (!accType || accType.getRank() != 2) - return rewriter.notifyMatchFailure(contractOp, "Expects acc 2D vector"); + if (!accType) + return rewriter.notifyMatchFailure(contractOp, "Expects vector acc"); - // Accept only plain 2D data layout. - // VNNI packing is applied to DPAS as a separate lowering step. - TypedValue lhs = contractOp.getLhs(); - TypedValue rhs = contractOp.getRhs(); - if (lhs.getType().getRank() != 2 || rhs.getType().getRank() != 2) - return rewriter.notifyMatchFailure(contractOp, - "Expects lhs and rhs 2D vectors"); + std::optional batchRank = + getRowMajorMatmulBatchRank(contractOp.getIndexingMapsAttr()); + if (!batchRank) + return rewriter.notifyMatchFailure( + contractOp, + "Expects a (batched) row-major matmul: leading dims must " + "be batch dims shared by lhs, rhs, and acc; innermost two " + "dims must be (M, K), (K, N), and (M, N)"); - if (!isRowMajorMatmul(contractOp.getIndexingMapsAttr())) - return rewriter.notifyMatchFailure(contractOp, "Invalid indexing maps"); + // xegpu.dpas operands are limited to 2 batch + 2 core dims. + if (*batchRank > 2) + return rewriter.notifyMatchFailure(contractOp, + "Expects operands of rank 4 or less"); auto dpasOp = xegpu::DpasOp::create(rewriter, loc, TypeRange{contractOp.getResultType()}, diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp index dcaeec51c5b7a..07f46d53ad48c 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -6697,6 +6697,8 @@ static void collectMapDataFromMapOperands( builder, moduleTranslation)); mapData.MapClause.push_back(mapOp.getOperation()); mapData.Types.push_back(convertClauseMapFlags(mapOp.getMapType())); + // TODO: set HasAttachPtr from Flang for pointee-storage entries. + mapData.HasAttachPtr.push_back(false); mapData.Names.push_back(LLVM::createMappingInformation( mapOp.getLoc(), *moduleTranslation.getOpenMPBuilder())); mapData.DevicePointers.push_back(llvm::OpenMPIRBuilder::DeviceInfoTy::None); @@ -6765,6 +6767,8 @@ static void collectMapDataFromMapOperands( mapData.MapClause.push_back(mapOp.getOperation()); mapData.Types.push_back( llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM); + // TODO: set HasAttachPtr from Flang for pointee-storage entries. + mapData.HasAttachPtr.push_back(false); mapData.Names.push_back(LLVM::createMappingInformation( mapOp.getLoc(), *moduleTranslation.getOpenMPBuilder())); mapData.DevicePointers.push_back(devInfoTy); @@ -6805,6 +6809,8 @@ static void collectMapDataFromMapOperands( // rematerialized, so the address of the decriptor for a given object // may change from one place to another. mapData.Types.push_back(mapType); + // TODO: set HasAttachPtr from Flang for pointee-storage entries. + mapData.HasAttachPtr.push_back(false); // Technically it's possible for a non-descriptor mapping to have // both has-device-addr and ALWAYS, so lookup the mapper in case it // exists. @@ -6821,6 +6827,8 @@ static void collectMapDataFromMapOperands( mapData.Types.push_back( isDevicePtr ? mapType : llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL); + // TODO: set HasAttachPtr from Flang for pointee-storage entries. + mapData.HasAttachPtr.push_back(false); mapData.Mappers.push_back(nullptr); } mapData.Names.push_back(LLVM::createMappingInformation( @@ -7126,6 +7134,8 @@ processIndividualMap(llvm::IRBuilderBase &builder, combinedInfo.Mappers.emplace_back(mapData.Mappers[mapDataIdx]); combinedInfo.Names.emplace_back(mapData.Names[mapDataIdx]); combinedInfo.Types.emplace_back(mapFlag); + // TODO: set HasAttachPtr from Flang for pointee-storage entries. + combinedInfo.HasAttachPtr.emplace_back(false); combinedInfo.Sizes.emplace_back( isPtrTy ? builder.CreateSelect( builder.CreateIsNull(mapData.Pointers[mapDataIdx]), @@ -7189,6 +7199,8 @@ static void mapParentWithMembers( } combinedInfo.Types.emplace_back(baseFlag); + // TODO: set HasAttachPtr from Flang for pointee-storage entries. + combinedInfo.HasAttachPtr.emplace_back(false); combinedInfo.DevicePointers.emplace_back( mapData.DevicePointers[mapDataIndex]); // Only attach the mapper to the base entry when we are mapping the whole @@ -7289,6 +7301,8 @@ static void mapParentWithMembers( if (targetDirective == TargetDirectiveEnumTy::TargetUpdate || hasMapClose || overlapIdxs.size() == 1) { combinedInfo.Types.emplace_back(mapFlag); + // TODO: set HasAttachPtr from Flang for pointee-storage entries. + combinedInfo.HasAttachPtr.emplace_back(false); combinedInfo.DevicePointers.emplace_back( mapData.DevicePointers[mapDataIndex]); combinedInfo.Names.emplace_back(LLVM::createMappingInformation( @@ -7329,6 +7343,8 @@ static void mapParentWithMembers( auto isPtrMap = checkIfPointerMap( llvm::cast(mapData.MapClause[mapDataOverlapIdx])); combinedInfo.Types.emplace_back(mapFlag); + // TODO: set HasAttachPtr from Flang for pointee-storage entries. + combinedInfo.HasAttachPtr.emplace_back(false); combinedInfo.DevicePointers.emplace_back( llvm::OpenMPIRBuilder::DeviceInfoTy::None); combinedInfo.Names.emplace_back(LLVM::createMappingInformation( @@ -7357,6 +7373,8 @@ static void mapParentWithMembers( } combinedInfo.Types.emplace_back(mapFlag); + // TODO: set HasAttachPtr from Flang for pointee-storage entries. + combinedInfo.HasAttachPtr.emplace_back(false); combinedInfo.DevicePointers.emplace_back( llvm::OpenMPIRBuilder::DeviceInfoTy::None); combinedInfo.Names.emplace_back(LLVM::createMappingInformation( @@ -8937,6 +8955,8 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, combinedInfos.Types.push_back( llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL); + // TODO: set HasAttachPtr from Flang for pointee-storage entries. + combinedInfos.HasAttachPtr.push_back(false); if (!combinedInfos.Names.empty()) combinedInfos.Names.push_back(nullPtr); combinedInfos.Mappers.push_back(nullptr); diff --git a/mlir/test/Conversion/VectorToXeGPU/contract-to-xegpu.mlir b/mlir/test/Conversion/VectorToXeGPU/contract-to-xegpu.mlir index 292e4ff882000..98b3133465b3e 100644 --- a/mlir/test/Conversion/VectorToXeGPU/contract-to-xegpu.mlir +++ b/mlir/test/Conversion/VectorToXeGPU/contract-to-xegpu.mlir @@ -180,3 +180,73 @@ func.func @negative_accumulator_shape(%lhs: vector<8x16xf16>, %rhs: vector<16x16 // CHECK-LABEL: @negative_accumulator_shape( // CHECK: vector.contract + +// ----- + +// A batched matmul whose leading dimensions are batch dimensions shared across +// lhs, rhs, and acc lowers to a batched xegpu.dpas. Two batch dims exercise the +// upper bound (rank 4 = 2 batch + 2 core dims). + +#map = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d4)> +#map1 = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d4, d3)> +#map2 = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3)> +func.func @dpas_batched_gemm(%lhs: vector<2x4x8x16xf16>, %rhs: vector<2x4x16x16xf16>, + %acc: vector<2x4x8x16xf32>) -> vector<2x4x8x16xf32> { + %3 = vector.contract + {indexing_maps = [#map, #map1, #map2], + iterator_types = ["parallel", "parallel", "parallel", "parallel", "reduction"], + kind = #vector.kind} %lhs, %rhs, %acc + : vector<2x4x8x16xf16>, vector<2x4x16x16xf16> into vector<2x4x8x16xf32> + return %3 : vector<2x4x8x16xf32> +} + +// CHECK-LABEL: @dpas_batched_gemm( +// CHECK-SAME: %[[LHS:.+]]: vector<2x4x8x16xf16>, +// CHECK-SAME: %[[RHS:.+]]: vector<2x4x16x16xf16>, +// CHECK-SAME: %[[ACC:.+]]: vector<2x4x8x16xf32> +// CHECK: %[[DPAS:.+]] = xegpu.dpas +// CHECK-SAME: %[[LHS]], %[[RHS]], %[[ACC]] +// CHECK-SAME: {{.*}}-> vector<2x4x8x16xf32> +// CHECK: return %[[DPAS]] + +// ----- + +// An N-D contraction whose leading dimension is not a batch dimension shared +// across lhs, rhs, and acc does not map to a (batched) row-major matmul. + +#map = affine_map<(d0, d1, d2, d3) -> (d1, d0, d3)> +#map1 = affine_map<(d0, d1, d2, d3) -> (d3, d2)> +#map2 = affine_map<(d0, d1, d2, d3) -> (d1, d2)> +func.func @negative_non_batched_nd(%lhs: vector<128x64x64xf16>, %rhs: vector<64x64xf16>, + %acc: vector<128x64xf16>) -> vector<128x64xf16> { + %3 = vector.contract + {indexing_maps = [#map, #map1, #map2], + iterator_types = ["parallel", "parallel", "parallel", "reduction"], + kind = #vector.kind} %lhs, %rhs, %acc + : vector<128x64x64xf16>, vector<64x64xf16> into vector<128x64xf16> + return %3 : vector<128x64xf16> +} + +// CHECK-LABEL: @negative_non_batched_nd( +// CHECK: vector.contract + +// ----- + +// A row-major batched matmul with more than 2 batch dims (rank 5) exceeds the +// xegpu.dpas limit of 2 batch + 2 core dims and is not lowered. + +#map = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d2, d3, d5)> +#map1 = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d2, d5, d4)> +#map2 = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d2, d3, d4)> +func.func @negative_too_many_batch_dims(%lhs: vector<2x2x4x8x16xf16>, %rhs: vector<2x2x4x16x16xf16>, + %acc: vector<2x2x4x8x16xf32>) -> vector<2x2x4x8x16xf32> { + %3 = vector.contract + {indexing_maps = [#map, #map1, #map2], + iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel", "reduction"], + kind = #vector.kind} %lhs, %rhs, %acc + : vector<2x2x4x8x16xf16>, vector<2x2x4x16x16xf16> into vector<2x2x4x8x16xf32> + return %3 : vector<2x2x4x8x16xf32> +} + +// CHECK-LABEL: @negative_too_many_batch_dims( +// CHECK: vector.contract diff --git a/mlir/test/Conversion/VectorToXeGPU/gather-to-xegpu.mlir b/mlir/test/Conversion/VectorToXeGPU/gather-to-xegpu.mlir index 2a319869a7b06..9617d347af2b6 100644 --- a/mlir/test/Conversion/VectorToXeGPU/gather-to-xegpu.mlir +++ b/mlir/test/Conversion/VectorToXeGPU/gather-to-xegpu.mlir @@ -15,8 +15,8 @@ gpu.func @load_1D_vector(%source: memref<8x16x32xf32>, // CHECK-SAME: %[[INDICES:.+]]: vector<8xindex> // CHECK-SAME: %[[MASK:.+]]: vector<8xi1> // CHECK-SAME: %[[PASS_THRU:.+]]: vector<8xf32>) -> vector<8xf32> { -// CHECK-COUNT2: arith.muli {{.*}} : index -// CHECK-COUNT2: arith.addi {{.*}} : index +// CHECK : arith.muli {{.*}} : index +// CHECK : arith.addi {{.*}} : index // CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8xindex> // CHECK: %[[LIN_IDX:.+]] = arith.addi %[[SPLAT]], %[[INDICES]] : vector<8xindex> // CHECK: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x16x32xf32> -> index @@ -42,8 +42,8 @@ gpu.func @load_2D_memref(%source: memref<8x32xf32>, // CHECK-SAME: %[[INDICES:.+]]: vector<8xindex> // CHECK-SAME: %[[MASK:.+]]: vector<8xi1> // CHECK-SAME: %[[PASS_THRU:.+]]: vector<8xf32>) -> vector<8xf32> { -// CHECK-COUNT1: arith.muli {{.*}} : index -// CHECK-COUNT1: arith.addi {{.*}} : index +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi {{.*}} : index // CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8xindex> // CHECK: %[[LIN_IDX:.+]] = arith.addi %[[SPLAT]], %[[INDICES]] : vector<8xindex> // CHECK: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x32xf32> -> index @@ -69,8 +69,8 @@ gpu.func @load_2D_vector(%source: memref<8x16x32xf32>, // CHECK-SAME: %[[INDICES:.+]]: vector<8x16xindex> // CHECK-SAME: %[[MASK:.+]]: vector<8x16xi1> // CHECK-SAME: %[[PASS_THRU:.+]]: vector<8x16xf32>) -> vector<8x16xf32> { -// CHECK-COUNT2: arith.muli {{.*}} : index -// CHECK-COUNT2: arith.addi {{.*}} : index +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi {{.*}} : index // CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8x16xindex> // CHECK: %[[LIN_IDX:.+]] = arith.addi %[[SPLAT]], %[[INDICES]] : vector<8x16xindex> // CHECK: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x16x32xf32> -> index @@ -97,8 +97,8 @@ gpu.func @load_dynamic_source(%source: memref, // CHECK-SAME: %[[MASK:.+]]: vector<8x16xi1> // CHECK-SAME: %[[PASS_THRU:.+]]: vector<8x16xf32>) -> vector<8x16xf32> { // CHECK: memref.extract_strided_metadata %[[SRC]] -// CHECK-COUNT2: arith.muli {{.*}} : index -// CHECK-COUNT2: arith.addi {{.*}} : index +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi {{.*}} : index // CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8x16xindex> // CHECK: %[[LIN_IDX:.+]] = arith.addi %[[SPLAT]], %[[INDICES]] : vector<8x16xindex> // CHECK: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref -> index @@ -125,8 +125,8 @@ gpu.func @load_dynamic_source2(%source: memref, // CHECK-SAME: %[[MASK:.+]]: vector<8x16xi1> // CHECK-SAME: %[[PASS_THRU:.+]]: vector<8x16xf32>) -> vector<8x16xf32> { // CHECK-NOT: memref.extract_strided_metadata %[[SRC]] -// CHECK-COUNT2: arith.muli {{.*}} : index -// CHECK-COUNT2: arith.addi {{.*}} : index +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi {{.*}} : index // CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8x16xindex> // CHECK: %[[LIN_IDX:.+]] = arith.addi %[[SPLAT]], %[[INDICES]] : vector<8x16xindex> // CHECK: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref -> index @@ -238,8 +238,8 @@ gpu.func @non_unit_inner_stride_3D( // CHECK: %[[BB:.+]], %[[M_OFF:.+]], %[[SIZES:.+]]:3, %[[STRIDES:.+]]:3 = memref.extract_strided_metadata %[[SRC]] // CHECK: arith.muli %[[OFF0]], %[[STRIDES]]#0 : index // CHECK: arith.addi {{.*}} : index -// CHECK-COUNT2: arith.muli {{.*}} : index -// CHECK-COUNT2: arith.addi {{.*}} : index +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi {{.*}} : index // CHECK: %[[STRD_INDICES:.+]] = arith.muli {{.*}}%[[INDICES]]{{.*}} : vector<8xindex> // CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}} : index to vector<8xindex> // CHECK: %[[LIN_IDX:.+]] = arith.addi %[[SPLAT]], %[[STRD_INDICES]] : vector<8xindex> diff --git a/mlir/test/Conversion/VectorToXeGPU/scatter-to-xegpu.mlir b/mlir/test/Conversion/VectorToXeGPU/scatter-to-xegpu.mlir index ffd3f170c0fad..b9dd241990222 100644 --- a/mlir/test/Conversion/VectorToXeGPU/scatter-to-xegpu.mlir +++ b/mlir/test/Conversion/VectorToXeGPU/scatter-to-xegpu.mlir @@ -12,8 +12,8 @@ gpu.func @store_1D_vector(%vec: vector<8xf32>, %source: memref<8x16x32xf32>, // CHECK-SAME: %[[VAL:.+]]: vector<8xf32>, %[[SRC:.+]]: memref<8x16x32xf32>, // CHECK-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index, %[[OFF3:.+]]: index, // CHECK-SAME: %[[INDICES:.+]]: vector<8xindex>, %[[MASK:.+]]: vector<8xi1>) { -// CHECK-COUNT2: arith.muli {{.*}} : index -// CHECK-COUNT2: arith.addi {{.*}} : index +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi {{.*}} : index // CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8xindex> // CHECK: %[[LIN_IDX:.+]] = arith.addi %[[SPLAT]], %[[INDICES]] : vector<8xindex> // CHECK: %[[BASE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x16x32xf32> -> index @@ -35,8 +35,8 @@ gpu.func @store_2D_memref(%vec: vector<8xf32>, %source: memref<8x32xf32>, // CHECK-SAME: %[[VAL:.+]]: vector<8xf32>, %[[SRC:.+]]: memref<8x32xf32>, // CHECK-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index // CHECK-SAME: %[[INDICES:.+]]: vector<8xindex>, %[[MASK:.+]]: vector<8xi1>) { -// CHECK-COUNT1: arith.muli {{.*}} : index -// CHECK-COUNT1: arith.addi {{.*}} : index +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi {{.*}} : index // CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8xindex> // CHECK: %[[LIN_IDX:.+]] = arith.addi %[[SPLAT]], %[[INDICES]] : vector<8xindex> // CHECK: %[[BASE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x32xf32> -> index @@ -58,8 +58,8 @@ gpu.func @store_2D_vector(%vec: vector<8x16xf32>, %source: memref<8x16x32xf32>, // CHECK-SAME: %[[VAL:.+]]: vector<8x16xf32>, %[[SRC:.+]]: memref<8x16x32xf32>, // CHECK-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index, %[[OFF3:.+]]: index, // CHECK-SAME: %[[INDICES:.+]]: vector<8x16xindex>, %[[MASK:.+]]: vector<8x16xi1>) { -// CHECK-COUNT2: arith.muli {{.*}} : index -// CHECK-COUNT2: arith.addi {{.*}} : index +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi {{.*}} : index // CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8x16xindex> // CHECK: %[[LIN_IDX:.+]] = arith.addi %[[SPLAT]], %[[INDICES]] : vector<8x16xindex> // CHECK: %[[BASE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x16x32xf32> -> index @@ -82,8 +82,8 @@ gpu.func @store_dynamic_source(%vec: vector<8x16xf32>, %source: memref, %[[MASK:.+]]: vector<8x16xi1>) { // CHECK: memref.extract_strided_metadata %[[SRC]] -// CHECK-COUNT2: arith.muli {{.*}} : index -// CHECK-COUNT2: arith.addi {{.*}} : index +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi {{.*}} : index // CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8x16xindex> // CHECK: %[[LIN_IDX:.+]] = arith.addi %[[SPLAT]], %[[INDICES]] : vector<8x16xindex> // CHECK: %[[BASE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref -> index @@ -106,8 +106,8 @@ gpu.func @store_dynamic_source2(%vec: vector<8x16xf32>, %source: memref, %[[MASK:.+]]: vector<8x16xi1>) { // CHECK-NOT: memref.extract_strided_metadata %[[SRC]] -// CHECK-COUNT2: arith.muli {{.*}} : index -// CHECK-COUNT2: arith.addi {{.*}} : index +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi {{.*}} : index // CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8x16xindex> // CHECK: %[[LIN_IDX:.+]] = arith.addi %[[SPLAT]], %[[INDICES]] : vector<8x16xindex> // CHECK: %[[BASE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref -> index @@ -161,8 +161,8 @@ gpu.func @non_unit_inner_stride_3D( // CHECK: %[[BB:.+]], %[[M_OFF:.+]], %[[SIZES:.+]]:3, %[[STRIDES:.+]]:3 = memref.extract_strided_metadata %[[SRC]] // CHECK: arith.muli %[[OFF0]], %[[STRIDES]]#0 : index // CHECK: arith.addi {{.*}} : index -// CHECK-COUNT2: arith.muli {{.*}} : index -// CHECK-COUNT2: arith.addi {{.*}} : index +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi {{.*}} : index // CHECK: %[[STRD_INDICES:.+]] = arith.muli {{.*}}%[[INDICES]]{{.*}} : vector<8xindex> // CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}} : index to vector<8xindex> // CHECK: %[[LIN_IDX:.+]] = arith.addi %[[SPLAT]], %[[STRD_INDICES]] : vector<8xindex> diff --git a/mlir/test/Conversion/VectorToXeGPU/transfer-read-to-xegpu.mlir b/mlir/test/Conversion/VectorToXeGPU/transfer-read-to-xegpu.mlir index e22e682053a2a..4cc4a0db5b63c 100644 --- a/mlir/test/Conversion/VectorToXeGPU/transfer-read-to-xegpu.mlir +++ b/mlir/test/Conversion/VectorToXeGPU/transfer-read-to-xegpu.mlir @@ -9,36 +9,17 @@ gpu.func @load_1D_vector(%source: memref<8x16x32xf32>, %offset: index) -> vector gpu.return %0 : vector<8xf32> } -// LOAD-ND-LABEL: @load_1D_vector( -// LOAD-ND-SAME: %[[SRC:.+]]: memref<8x16x32xf32>, -// LOAD-ND-SAME: %[[OFFSET:.+]]: index -// LOAD-ND: %[[ELEM_BYTES:.+]] = arith.constant 4 : index -// LOAD-ND: %[[COLLAPSED:.+]] = memref.subview %[[SRC]][%[[OFFSET]], %[[OFFSET]], 0] -// LOAD-ND: %[[BASE_BUFFER:.*]], %[[OFF1:.*]], %[[SIZES:.*]], %[[STRIDES:.*]] = memref.extract_strided_metadata %[[COLLAPSED]] -// LOAD-ND-SAME: : memref<32xf32, strided<[1], offset: ?>> -> memref, index, index, index -// LOAD-ND: %[[INTPTR:.*]] = memref.extract_aligned_pointer_as_index %[[BASE_BUFFER]] -// LOAD-ND-SAME: : memref -> index -// LOAD-ND: %[[MUL:.*]] = arith.muli %[[OFF1]], %[[ELEM_BYTES]] : index -// LOAD-ND: %[[ADD:.*]] = arith.addi %[[INTPTR]], %[[MUL]] : index -// LOAD-ND: %[[I64PTR:.*]] = arith.index_cast %[[ADD]] : index to i64 -// LOAD-ND: %[[DESC:.+]] = xegpu.create_nd_tdesc %[[I64PTR]], shape : [32], -// LOAD-ND-SAME: strides : [1] : i64 -> !xegpu.tensor_desc<8xf32, -// LOAD-ND-SAME: #xegpu.block_tdesc_attr> -// LOAD-ND: %[[VEC:.+]] = xegpu.load_nd %[[DESC]][%[[OFFSET]]] -// LOAD-ND-SAME: : !xegpu.tensor_desc<8xf32, #xegpu.block_tdesc_attr> -> vector<8xf32> - -// LOAD-GATHER-LABEL: @load_1D_vector( -// LOAD-GATHER-SAME: %[[SRC:.+]]: memref<8x16x32xf32>, -// LOAD-GATHER: %[[CST:.+]] = arith.constant dense : vector<8xi1> -// LOAD-GATHER: %[[STEP:.+]] = vector.step : vector<8xindex> -// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index -// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index -// LOAD-GATHER: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8xindex> -// LOAD-GATHER: %[[IDX:.+]] = arith.addi %[[SPLAT]], %[[STEP]] : vector<8xindex> -// LOAD-GATHER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x16x32xf32> -> index -// LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 -// LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : i64, vector<8xindex>, vector<8xi1> -> vector<8xf32> - +// CHECK-LABEL: @load_1D_vector( +// CHECK-SAME: %[[SRC:.+]]: memref<8x16x32xf32>, +// CHECK: %[[CST:.+]] = arith.constant dense : vector<8xi1> +// CHECK: %[[STEP:.+]] = vector.step : vector<8xindex> +// CHECK-COUNT-2: arith.muli {{.*}} : index +// CHECK-COUNT-2: arith.addi {{.*}} : index +// CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8xindex> +// CHECK: %[[IDX:.+]] = arith.addi %[[SPLAT]], %[[STEP]] : vector<8xindex> +// CHECK: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x16x32xf32> -> index +// CHECK: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 +// CHECK: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : i64, vector<8xindex>, vector<8xi1> -> vector<8xf32> } // ----- @@ -81,10 +62,8 @@ gpu.func @load_2D_vector(%source: memref<8x16x32xf32>, // LOAD-GATHER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x16x32xf32> -> index // LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 // LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : i64, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf32> - } - // ----- gpu.module @xevm_module { gpu.func @load_zero_pad_out_of_bounds(%source: memref<32x64xf32>, @@ -105,10 +84,8 @@ gpu.func @load_zero_pad_out_of_bounds(%source: memref<32x64xf32>, // LOAD-GATHER-LABEL: @load_zero_pad_out_of_bounds( // LOAD-GATHER: vector.transfer_read - } - // ----- gpu.module @xevm_module { gpu.func @load_transposed(%source: memref<32x64xf32>, @@ -131,7 +108,6 @@ gpu.func @load_transposed(%source: memref<32x64xf32>, // LOAD-ND: %[[VEC_TRANSPOSED:.+]] = vector.transpose %[[VEC]], [1, 0] : vector<16x8xf32> to vector<8x16xf32> // LOAD-ND: return %[[VEC_TRANSPOSED]] - // LOAD-GATHER-LABEL: @load_transposed( // LOAD-GATHER-SAME: %[[SRC:.+]]: memref<32x64xf32>, // LOAD-GATHER: %[[CST:.+]] = arith.constant dense : vector<8x16xi1> @@ -145,7 +121,6 @@ gpu.func @load_transposed(%source: memref<32x64xf32>, // LOAD-GATHER: %[[COLLAPSE:.*]] = memref.extract_aligned_pointer_as_index %arg0 : memref<32x64xf32> -> index // LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 // LOAD-GATHER: %[[LOAD:.*]] = xegpu.load %[[COLLAPSE_I]][%[[IDX]]], %[[CST]] : i64, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf32> - } // ----- @@ -185,7 +160,6 @@ gpu.func @load_transpose_3d_memref(%source: memref<32x64x128xf32>, // LOAD-GATHER: %[[INTPTR:.*]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<32x64x128xf32> -> index // LOAD-GATHER-NEXT: %[[I64PTR:.+]] = arith.index_cast %[[INTPTR]] : index to i64 // LOAD-GATHER-NEXT: %[[LOAD:.*]] = xegpu.load %[[I64PTR]][%[[IDX]]], %{{.*}} : i64, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf32> - } // ----- @@ -240,7 +214,6 @@ gpu.func @load_dynamic_source(%source: memref, // LOAD-ND: %[[VEC:.+]] = xegpu.load_nd %[[DESC]][%[[OFF1]], %[[OFF2]]]{{.*}}-> vector<8x16xf32> // LOAD-ND: return %[[VEC]] - // LOAD-GATHER-LABEL: @load_dynamic_source( // LOAD-GATHER-SAME: %[[ARG0:.+]]: memref, // LOAD-GATHER: %[[CST:.+]] = arith.constant dense : vector<8x16xi1> @@ -295,7 +268,6 @@ gpu.func @load_dynamic_source2(%source: memref, // LOAD-GATHER-DAG: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %arg0 : memref -> index // LOAD-GATHER-DAG: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 // LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[OFFSETS]]{{\]}}, %[[CST_0]] : i64, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf32> - } // ----- @@ -366,7 +338,6 @@ gpu.func @load_high_dim_vector(%source: memref<16x32x64xf32>, // LOAD-GATHER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %arg0 : memref<16x32x64xf32> -> index // LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 // LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]][%[[IDX]]], %[[CST]] : i64, vector<8x16x32xindex>, vector<8x16x32xi1> -> vector<8x16x32xf32> - } // ----- @@ -398,7 +369,6 @@ gpu.func @load_8D_vector(%source: memref<2x2x2x2x2x2x2x2xf32>, // LOAD-GATHER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<2x2x2x2x2x2x2x2xf32> -> index // LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 // LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]][%[[IDX]]], %[[CST]] : i64, vector<2x2x2x2x2x2x2x2xindex>, vector<2x2x2x2x2x2x2x2xi1> -> vector<2x2x2x2x2x2x2x2xf32> - } // ----- @@ -443,11 +413,8 @@ gpu.func @no_load_out_of_bounds_non_zero_pad(%source: memref<32x64xf32>, gpu.return %0, %1 : vector<8x16xf32>, vector<8x16xf32> } -// LOAD-ND-LABEL: @no_load_out_of_bounds_non_zero_pad( -// LOAD-ND-COUNT-2: vector.transfer_read - -// LOAD-GATHER-LABEL: @no_load_out_of_bounds_non_zero_pad( -// LOAD-GATHER-COUNT-2: vector.transfer_read +// CHECK-LABEL: @no_load_out_of_bounds_non_zero_pad( +// CHECK-COUNT-2: vector.transfer_read } // ----- @@ -460,11 +427,8 @@ gpu.func @no_load_out_of_bounds_1D_vector(%source: memref<8x16x32xf32>, gpu.return %0 : vector<8xf32> } -// LOAD-ND-LABEL: @no_load_out_of_bounds_1D_vector( -// LOAD-ND: vector.transfer_read - -// LOAD-GATHER-LABEL: @no_load_out_of_bounds_1D_vector( -// LOAD-GATHER: vector.transfer_read +// CHECK-LABEL: @no_load_out_of_bounds_1D_vector( +// CHECK: vector.transfer_read } // ----- @@ -478,11 +442,8 @@ gpu.func @no_load_masked(%source : memref<4xf32>, gpu.return %0 : vector<4xf32> } -// LOAD-ND-LABEL: @no_load_masked( -// LOAD-ND: vector.transfer_read - -// LOAD-GATHER-LABEL: @no_load_masked( -// LOAD-GATHER: vector.transfer_read +// CHECK-LABEL: @no_load_masked( +// CHECK: vector.transfer_read } // ----- @@ -495,14 +456,10 @@ gpu.func @no_load_tensor(%source: tensor<32x64xf32>, gpu.return %0 : vector<8x16xf32> } -// LOAD-ND-LABEL: @no_load_tensor( -// LOAD-ND: vector.transfer_read - -// LOAD-GATHER-LABEL: @no_load_tensor( -// LOAD-GATHER: vector.transfer_read +// CHECK-LABEL: @no_load_tensor( +// CHECK: vector.transfer_read } - // ----- gpu.module @xevm_module { gpu.func @no_load_non_unit_inner_stride( @@ -514,14 +471,10 @@ gpu.func @no_load_non_unit_inner_stride( gpu.return %0 : vector<8xf32> } -// LOAD-ND-LABEL: @no_load_non_unit_inner_stride( -// LOAD-ND: vector.transfer_read - -// LOAD-GATHER-LABEL: @no_load_non_unit_inner_stride( -// LOAD-GATHER: vector.transfer_read +// CHECK-LABEL: @no_load_non_unit_inner_stride( +// CHECK: vector.transfer_read } - // ----- gpu.module @xevm_module { gpu.func @no_load_unsupported_map(%source: memref<16x32x64xf32>, @@ -533,11 +486,8 @@ gpu.func @no_load_unsupported_map(%source: memref<16x32x64xf32>, gpu.return %0 : vector<8x16xf32> } -// LOAD-ND-LABEL: @no_load_unsupported_map( -// LOAD-ND: vector.transfer_read - -// LOAD-GATHER-LABEL: @no_load_unsupported_map( -// LOAD-GATHER: vector.transfer_read +// CHECK-LABEL: @no_load_unsupported_map( +// CHECK: vector.transfer_read } // ----- @@ -550,36 +500,21 @@ gpu.func @load_from_subview_1D(%source: memref<4096x4096xf16>, %off1: index, %of gpu.return %0 : vector<8xf16> } -// LOAD-ND-LABEL: @load_from_subview_1D( -// LOAD-ND-SAME: %[[SRC:.+]]: memref<4096x4096xf16>, -// LOAD-ND-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index -// LOAD-ND: %[[ELEM_BYTES:.+]] = arith.constant 2 : index -// LOAD-ND: %[[SUBVIEW:.+]] = memref.subview %[[SRC]][%[[OFF1]], %[[OFF2]]] [256, 256] [1, 1] : memref<4096x4096xf16> to memref<256x256xf16, strided<[4096, 1], offset: ?>> -// LOAD-ND: %[[COLLAPSED:.+]] = memref.subview %[[SUBVIEW]][%[[OFF2]], 0] -// LOAD-ND: %[[BASE_BUFFER:.*]], %[[OFFSET:.*]], %[[SIZES:.*]], %[[STRIDES:.*]] = memref.extract_strided_metadata %[[COLLAPSED]] -// LOAD-ND: %[[INTPTR:.*]] = memref.extract_aligned_pointer_as_index %[[BASE_BUFFER]] -// LOAD-ND: %[[MUL:.+]] = arith.muli %[[OFFSET]], %[[ELEM_BYTES]] : index -// LOAD-ND: %[[ADD:.+]] = arith.addi %[[INTPTR]], %[[MUL]] : index -// LOAD-ND: %[[I64PTR:.*]] = arith.index_cast %[[ADD]] : index to i64 -// LOAD-ND: %[[DESC:.*]] = xegpu.create_nd_tdesc %[[I64PTR]], shape : [256], strides : [1] : i64 -> -// LOAD-ND-SAME: !xegpu.tensor_desc<8xf16, #xegpu.block_tdesc_attr> -// LOAD-ND: %[[VEC:.+]] = xegpu.load_nd %[[DESC]][%[[OFF2]]] : !xegpu.tensor_desc<8xf16, #xegpu.block_tdesc_attr> -> vector<8xf16> - -// LOAD-GATHER-LABEL: @load_from_subview_1D( -// LOAD-GATHER-SAME: %[[SRC:.+]]: memref<4096x4096xf16>, -// LOAD-GATHER-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index -// LOAD-GATHER: %[[CST:.+]] = arith.constant dense : vector<8xi1> -// LOAD-GATHER: %[[SUBVIEW:.+]] = memref.subview %[[SRC]][%[[OFF1]], %[[OFF2]]] [256, 256] [1, 1] : memref<4096x4096xf16> to memref<256x256xf16, strided<[4096, 1], offset: ?>> -// LOAD-GATHER: %[[BB:.+]], %[[OFFSET:.+]],{{.*}},{{.*}} = memref.extract_strided_metadata %[[SUBVIEW]] : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> memref, index, index, index, index, index -// LOAD-GATHER: %[[STEP:.+]] = vector.step : vector<8xindex> -// LOAD-GATHER: arith.muli {{.*}} : index -// LOAD-GATHER: arith.addi %[[OFFSET]]{{.*}} : index -// LOAD-GATHER: arith.addi {{.*}} : index -// LOAD-GATHER: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8xindex> -// LOAD-GATHER: %[[IDX:.+]] = arith.addi %[[SPLAT]], %[[STEP]] : vector<8xindex> -// LOAD-GATHER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SUBVIEW]] : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> index -// LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 -// LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : i64, vector<8xindex>, vector<8xi1> -> vector<8xf16> +// CHECK-LABEL: @load_from_subview_1D( +// CHECK-SAME: %[[SRC:.+]]: memref<4096x4096xf16>, +// CHECK-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index +// CHECK: %[[CST:.+]] = arith.constant dense : vector<8xi1> +// CHECK: %[[SUBVIEW:.+]] = memref.subview %[[SRC]][%[[OFF1]], %[[OFF2]]] [256, 256] [1, 1] : memref<4096x4096xf16> to memref<256x256xf16, strided<[4096, 1], offset: ?>> +// CHECK: %[[BB:.+]], %[[OFFSET:.+]],{{.*}},{{.*}} = memref.extract_strided_metadata %[[SUBVIEW]] : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> memref, index, index, index, index, index +// CHECK: %[[STEP:.+]] = vector.step : vector<8xindex> +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi %[[OFFSET]]{{.*}} : index +// CHECK: arith.addi {{.*}} : index +// CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8xindex> +// CHECK: %[[IDX:.+]] = arith.addi %[[SPLAT]], %[[STEP]] : vector<8xindex> +// CHECK: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SUBVIEW]] : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> index +// CHECK: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 +// CHECK: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : i64, vector<8xindex>, vector<8xi1> -> vector<8xf16> } // ----- @@ -635,20 +570,12 @@ gpu.func @load_2D_vector_addrspace3(%source: memref<16x32xf32, 3>, gpu.return %0 : vector<8x16xf32> } -// LOAD-ND-LABEL: @load_2D_vector_addrspace3 -// LOAD-ND-SAME: %[[SOURCE:.+]]: memref<16x32xf32, 3> -// LOAD-ND-SAME: %[[OFFSET:.+]]: index -// LOAD-ND: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[SOURCE]] : memref<16x32xf32, 3> -> !xegpu.mem_desc<16x32xf32> -// LOAD-ND: %[[DATA:.+]] = xegpu.load_matrix %[[MEM_DESC]][%[[OFFSET]], %[[OFFSET]]] : !xegpu.mem_desc<16x32xf32>, index, index -> vector<8x16xf32> -// LOAD-ND: gpu.return %[[DATA]] : vector<8x16xf32> - -// LOAD-GATHER-LABEL: @load_2D_vector_addrspace3 -// LOAD-GATHER-SAME: %[[SOURCE:.+]]: memref<16x32xf32, 3> -// LOAD-GATHER-SAME: %[[OFFSET:.+]]: index -// LOAD-GATHER: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[SOURCE]] : memref<16x32xf32, 3> -> !xegpu.mem_desc<16x32xf32> -// LOAD-GATHER: %[[DATA:.+]] = xegpu.load_matrix %[[MEM_DESC]][%[[OFFSET]], %[[OFFSET]]] : !xegpu.mem_desc<16x32xf32>, index, index -> vector<8x16xf32> -// LOAD-GATHER: gpu.return %[[DATA]] : vector<8x16xf32> - +// CHECK-LABEL: @load_2D_vector_addrspace3 +// CHECK-SAME: %[[SOURCE:.+]]: memref<16x32xf32, 3> +// CHECK-SAME: %[[OFFSET:.+]]: index +// CHECK: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[SOURCE]] : memref<16x32xf32, 3> -> !xegpu.mem_desc<16x32xf32> +// CHECK: %[[DATA:.+]] = xegpu.load_matrix %[[MEM_DESC]][%[[OFFSET]], %[[OFFSET]]] : !xegpu.mem_desc<16x32xf32>, index, index -> vector<8x16xf32> +// CHECK: gpu.return %[[DATA]] : vector<8x16xf32> } // ----- @@ -661,20 +588,12 @@ gpu.func @load_1D_vector_addrspace3(%source: memref<32xf32, 3>, gpu.return %0 : vector<8xf32> } -// LOAD-ND-LABEL: @load_1D_vector_addrspace3 -// LOAD-ND-SAME: %[[SOURCE:.+]]: memref<32xf32, 3> -// LOAD-ND-SAME: %[[OFFSET:.+]]: index -// LOAD-ND: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[SOURCE]] : memref<32xf32, 3> -> !xegpu.mem_desc<32xf32> -// LOAD-ND: %[[DATA:.+]] = xegpu.load_matrix %[[MEM_DESC]][%[[OFFSET]]] : !xegpu.mem_desc<32xf32>, index -> vector<8xf32> -// LOAD-ND: gpu.return %[[DATA]] : vector<8xf32> - -// LOAD-GATHER-LABEL: @load_1D_vector_addrspace3 -// LOAD-GATHER-SAME: %[[SOURCE:.+]]: memref<32xf32, 3> -// LOAD-GATHER-SAME: %[[OFFSET:.+]]: index -// LOAD-GATHER: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[SOURCE]] : memref<32xf32, 3> -> !xegpu.mem_desc<32xf32> -// LOAD-GATHER: %[[DATA:.+]] = xegpu.load_matrix %[[MEM_DESC]][%[[OFFSET]]] : !xegpu.mem_desc<32xf32>, index -> vector<8xf32> -// LOAD-GATHER: gpu.return %[[DATA]] : vector<8xf32> - +// CHECK-LABEL: @load_1D_vector_addrspace3 +// CHECK-SAME: %[[SOURCE:.+]]: memref<32xf32, 3> +// CHECK-SAME: %[[OFFSET:.+]]: index +// CHECK: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[SOURCE]] : memref<32xf32, 3> -> !xegpu.mem_desc<32xf32> +// CHECK: %[[DATA:.+]] = xegpu.load_matrix %[[MEM_DESC]][%[[OFFSET]]] : !xegpu.mem_desc<32xf32>, index -> vector<8xf32> +// CHECK: gpu.return %[[DATA]] : vector<8xf32> } // ----- @@ -691,16 +610,10 @@ gpu.func @load_2D_vector_alloca_promoted_to_slm(%offset: index) gpu.return %0 : vector<8x16xf32> } -// LOAD-ND-LABEL: @load_2D_vector_alloca_promoted_to_slm -// LOAD-ND: %[[BUF:.+]] = memref.alloca() : memref<16x32xf32, 3> -// LOAD-ND: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[BUF]] : memref<16x32xf32, 3> -> !xegpu.mem_desc<16x32xf32> -// LOAD-ND: xegpu.load_matrix %[[MEM_DESC]] - -// LOAD-GATHER-LABEL: @load_2D_vector_alloca_promoted_to_slm -// LOAD-GATHER: %[[BUF:.+]] = memref.alloca() : memref<16x32xf32, 3> -// LOAD-GATHER: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[BUF]] : memref<16x32xf32, 3> -> !xegpu.mem_desc<16x32xf32> -// LOAD-GATHER: xegpu.load_matrix %[[MEM_DESC]] - +// CHECK-LABEL: @load_2D_vector_alloca_promoted_to_slm +// CHECK: %[[BUF:.+]] = memref.alloca() : memref<16x32xf32, 3> +// CHECK: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[BUF]] : memref<16x32xf32, 3> -> !xegpu.mem_desc<16x32xf32> +// CHECK: xegpu.load_matrix %[[MEM_DESC]] } // ----- @@ -716,16 +629,10 @@ gpu.func @load_1D_vector_alloca_promoted_to_slm(%offset: index) gpu.return %0 : vector<8xf32> } -// LOAD-ND-LABEL: @load_1D_vector_alloca_promoted_to_slm -// LOAD-ND: %[[BUF:.+]] = memref.alloca() : memref<16xf32, 3> -// LOAD-ND: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[BUF]] : memref<16xf32, 3> -> !xegpu.mem_desc<16xf32> -// LOAD-ND: xegpu.load_matrix %[[MEM_DESC]] - -// LOAD-GATHER-LABEL: @load_1D_vector_alloca_promoted_to_slm -// LOAD-GATHER: %[[BUF:.+]] = memref.alloca() : memref<16xf32, 3> -// LOAD-GATHER: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[BUF]] : memref<16xf32, 3> -> !xegpu.mem_desc<16xf32> -// LOAD-GATHER: xegpu.load_matrix %[[MEM_DESC]] - +// CHECK-LABEL: @load_1D_vector_alloca_promoted_to_slm +// CHECK: %[[BUF:.+]] = memref.alloca() : memref<16xf32, 3> +// CHECK: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[BUF]] : memref<16xf32, 3> -> !xegpu.mem_desc<16xf32> +// CHECK: xegpu.load_matrix %[[MEM_DESC]] } // ----- @@ -738,7 +645,6 @@ gpu.func @load_0D_memref_unsupported(%source: memref) -> vector { // CHECK-LABEL: @load_0D_memref_unsupported // CHECK: vector.transfer_read - } // ----- @@ -751,12 +657,8 @@ gpu.func @load_0D_vector_unsupported(%source: memref<3xf32>, gpu.return %0 : vector } -// LOAD-ND-LABEL: @load_0D_vector_unsupported -// LOAD-ND: vector.transfer_read - -// LOAD-GATHER-LABEL: @load_0D_vector_unsupported -// LOAD-GATHER: vector.transfer_read - +// CHECK-LABEL: @load_0D_vector_unsupported +// CHECK: vector.transfer_read } // ----- @@ -816,5 +718,4 @@ gpu.func @transpose_1x1024x24x64( // LOAD-GATHER: arith.muli %block_id_x, %[[C65536]] : index // LOAD-GATHER: arith.muli %{{.+}}, %[[C64]] : index // LOAD-GATHER: xegpu.store {{.*}} - } diff --git a/mlir/test/Conversion/VectorToXeGPU/transfer-write-to-xegpu.mlir b/mlir/test/Conversion/VectorToXeGPU/transfer-write-to-xegpu.mlir index 2d55a785595f7..dad2740f5c0ea 100644 --- a/mlir/test/Conversion/VectorToXeGPU/transfer-write-to-xegpu.mlir +++ b/mlir/test/Conversion/VectorToXeGPU/transfer-write-to-xegpu.mlir @@ -1,7 +1,6 @@ // RUN: mlir-opt %s --xevm-attach-target='module=xevm_* O=3 chip=pvc' -convert-vector-to-xegpu -split-input-file | FileCheck %s --check-prefixes=STORE-ND,CHECK // RUN: mlir-opt %s -convert-vector-to-xegpu -split-input-file | FileCheck %s --check-prefixes=STORE-SCATTER,CHECK - gpu.module @xevm_module { gpu.func @store_1D_vector(%vec: vector<8xf32>, %source: memref<8x16x32xf32>, %offset: index) { @@ -11,36 +10,18 @@ gpu.func @store_1D_vector(%vec: vector<8xf32>, gpu.return } -// STORE-ND-LABEL: @store_1D_vector( -// STORE-ND-SAME: %[[VEC:.+]]: vector<8xf32>, -// STORE-ND-SAME: %[[SRC:.+]]: memref<8x16x32xf32>, -// STORE-ND-SAME: %[[OFFSET:.+]]: index -// STORE-ND: %[[ELEM_BYTES:.+]] = arith.constant 4 : index -// STORE-ND: %[[COLLAPSED:.+]] = memref.subview %[[SRC]][%[[OFFSET]], %[[OFFSET]], 0] -// STORE-ND: %[[BASE_BUFFER:.+]], %[[OFFSET1:.+]], %[[SIZES:.+]], %[[STRIDES:.+]] = memref.extract_strided_metadata %[[COLLAPSED]] -// STORE-ND-SAME: : memref<32xf32, strided<[1], offset: ?>> -> memref, index, index, index -// STORE-ND: %[[INTPTR:.+]] = memref.extract_aligned_pointer_as_index %[[BASE_BUFFER]] -// STORE-ND-SAME: : memref -> index -// STORE-ND: %[[MUL:.+]] = arith.muli %[[OFFSET1]], %[[ELEM_BYTES]] : index -// STORE-ND: %[[ADD:.+]] = arith.addi %[[INTPTR]], %[[MUL]] : index -// STORE-ND: %[[I64PTR:.+]] = arith.index_cast %[[ADD]] : index to i64 -// STORE-ND: %[[DESC:.+]] = xegpu.create_nd_tdesc %[[I64PTR]], shape : [32], -// STORE-ND-SAME: strides : [1] : i64 -> !xegpu.tensor_desc<8xf32, -// STORE-ND-SAME: boundary_check = false -// STORE-ND: xegpu.store_nd %[[VEC]], %[[DESC]][%[[OFFSET]]] : vector<8xf32> - -// STORE-SCATTER-LABEL: @store_1D_vector( -// STORE-SCATTER-SAME: %[[VEC:.+]]: vector<8xf32>, -// STORE-SCATTER-SAME: %[[SRC:.+]]: memref<8x16x32xf32>, -// STORE-SCATTER-DAG: %[[CST:.+]] = arith.constant dense : vector<8xi1> -// STORE-SCATTER-DAG: %[[STEP:.+]] = vector.step -// STORE-SCATTER-COUNT2: arith.muli {{.*}} : index -// STORE-SCATTER-COUNT2: arith.addi {{.*}} : index -// STORE-SCATTER-DAG: %[[BCAST:.+]] = vector.broadcast {{.*}} : index to vector<8xindex> -// STORE-SCATTER-DAG: %[[IDX:.+]] = arith.addi %[[BCAST]], %{{.*}} : vector<8xindex> -// STORE-SCATTER-DAG: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x16x32xf32> -> index -// STORE-SCATTER-DAG: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 -// STORE-SCATTER: xegpu.store %[[VEC]], %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : vector<8xf32>, i64, vector<8xindex>, vector<8xi1> +// CHECK-LABEL: @store_1D_vector( +// CHECK-SAME: %[[VEC:.+]]: vector<8xf32>, +// CHECK-SAME: %[[SRC:.+]]: memref<8x16x32xf32>, +// CHECK-DAG: %[[CST:.+]] = arith.constant dense : vector<8xi1> +// CHECK-DAG: %[[STEP:.+]] = vector.step +// CHECK : arith.muli {{.*}} : index +// CHECK : arith.addi {{.*}} : index +// CHECK-DAG: %[[BCAST:.+]] = vector.broadcast {{.*}} : index to vector<8xindex> +// CHECK-DAG: %[[IDX:.+]] = arith.addi %[[BCAST]], %{{.*}} : vector<8xindex> +// CHECK-DAG: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x16x32xf32> -> index +// CHECK-DAG: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 +// CHECK: xegpu.store %[[VEC]], %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : vector<8xf32>, i64, vector<8xindex>, vector<8xi1> } // ----- @@ -166,9 +147,9 @@ gpu.func @store_transposed(%vec: vector<8x16xf32>, // CHECK-SAME: %[[SRC:.+]]: memref<32x64xf32>, // CHECK-SAME: %[[OFFSET:.+]]: index // CHECK: %[[CST:.+]] = arith.constant dense : vector<8x16xi1> -// CHECK-COUNT2: %[[STEP:.+]] = vector.step -// CHECK-COUNT2: vector.shape_cast {{.*}} -// CHECK-COUNT2: vector.broadcast {{.*}} : vector<8x16xindex> +// CHECK : %[[STEP:.+]] = vector.step +// CHECK : vector.shape_cast {{.*}} +// CHECK : vector.broadcast {{.*}} : vector<8x16xindex> // CHECK-DAG: %[[BCAST2:.+]] = vector.broadcast {{.*}} : index to vector<8x16xindex> // CHECK-DAG: %[[IDX:.+]] = arith.addi %[[BCAST2]], {{.*}} : vector<8x16xindex> // CHECK-DAG: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<32x64xf32> -> index @@ -274,11 +255,8 @@ gpu.func @no_store_masked(%vec: vector<4xf32>, gpu.return } -// STORE-ND-LABEL: @no_store_masked( -// STORE-ND: vector.transfer_write - -// STORE-SCATTER-LABEL: @no_store_masked( -// STORE-SCATTER: vector.transfer_write +// CHECK-LABEL: @no_store_masked( +// CHECK: vector.transfer_write } // ----- @@ -291,11 +269,8 @@ gpu.func @no_store_tensor(%vec: vector<8x16xf32>, gpu.return %0 : tensor<32x64xf32> } -// STORE-ND-LABEL: @no_store_tensor( -// STORE-ND: vector.transfer_write - -// STORE-SCATTER-LABEL: @no_store_tensor( -// STORE-SCATTER: vector.transfer_write +// CHECK-LABEL: @no_store_tensor( +// CHECK: vector.transfer_write } // ----- @@ -308,11 +283,8 @@ gpu.func @no_store_non_unit_inner_stride(%vec: vector<8xf32>, gpu.return } -// STORE-ND-LABEL: @no_store_non_unit_inner_stride( -// STORE-ND: vector.transfer_write - -// STORE-SCATTER-LABEL: @no_store_non_unit_inner_stride( -// STORE-SCATTER: vector.transfer_write +// CHECK-LABEL: @no_store_non_unit_inner_stride( +// CHECK: vector.transfer_write } // ----- @@ -326,11 +298,8 @@ gpu.func @no_store_unsupported_map(%vec: vector<8x16xf32>, gpu.return } -// STORE-ND-LABEL: @no_store_unsupported_map( -// STORE-ND: vector.transfer_write - -// STORE-SCATTER-LABEL: @no_store_unsupported_map( -// STORE-SCATTER: vector.transfer_write +// CHECK-LABEL: @no_store_unsupported_map( +// CHECK: vector.transfer_write } // ----- @@ -343,11 +312,8 @@ gpu.func @no_store_out_of_bounds_1D_vector(%vec: vector<8xf32>, gpu.return } -// STORE-ND-LABEL: @no_store_out_of_bounds_1D_vector( -// STORE-ND: vector.transfer_write - -// STORE-SCATTER-LABEL: @no_store_out_of_bounds_1D_vector( -// STORE-SCATTER: vector.transfer_write +// CHECK-LABEL: @no_store_out_of_bounds_1D_vector( +// CHECK: vector.transfer_write } // ----- @@ -362,41 +328,26 @@ gpu.func @store_to_subview(%vec: vector<8xf16>, : vector<8xf16>, memref<256x256xf16, strided<[4096, 1], offset: ?>> gpu.return } -// STORE-ND-LABEL: @store_to_subview( -// STORE-ND-SAME: %[[VEC:.+]]: vector<8xf16>, -// STORE-ND-SAME: %[[SRC:.+]]: memref<4096x4096xf16>, -// STORE-ND-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index -// STORE-ND: %[[ELEM_BYTES:.+]] = arith.constant 2 : index -// STORE-ND: %[[SUBVIEW:.+]] = memref.subview %[[SRC]][%[[OFF1]], %[[OFF2]]] [256, 256] [1, 1] : memref<4096x4096xf16> to memref<256x256xf16, strided<[4096, 1], offset: ?>> -// STORE-ND: %[[COLLAPSED:.+]] = memref.subview %[[SUBVIEW]][%[[OFF2]], 0] -// STORE-ND: %[[BASE_BUFFER:.*]], %[[OFFSET:.*]], %[[SIZES:.*]], %[[STRIDES:.*]] = memref.extract_strided_metadata %[[COLLAPSED]] -// STORE-ND: %[[INTPTR:.*]] = memref.extract_aligned_pointer_as_index %[[BASE_BUFFER]] -// STORE-ND: %[[MUL:.+]] = arith.muli %[[OFFSET]], %[[ELEM_BYTES]] : index -// STORE-ND: %[[ADD:.+]] = arith.addi %[[INTPTR]], %[[MUL]] : index -// STORE-ND: %[[I64PTR:.*]] = arith.index_cast %[[ADD]] : index to i64 -// STORE-ND: %[[DESC:.*]] = xegpu.create_nd_tdesc %[[I64PTR]], shape : [256], strides : [1] : i64 -> -// STORE-ND-SAME: !xegpu.tensor_desc<8xf16, #xegpu.block_tdesc_attr> -// STORE-ND: xegpu.store_nd %[[VEC]], %[[DESC]][%[[OFF2]]] : vector<8xf16> - -// STORE-SCATTER-LABEL: @store_to_subview( -// STORE-SCATTER-SAME: %[[VEC:.+]]: vector<8xf16>, -// STORE-SCATTER-SAME: %[[SRC:.+]]: memref<4096x4096xf16>, -// STORE-SCATTER-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index -// STORE-SCATTER: %[[CST:.+]] = arith.constant dense : vector<8xi1> -// STORE-SCATTER: %[[SUBVIEW:.+]] = memref.subview %[[SRC]][%[[OFF1]], %[[OFF2]]] [256, 256] [1, 1] -// STORE-SCATTER-SAME: : memref<4096x4096xf16> to memref<256x256xf16, strided<[4096, 1], offset: ?>> -// STORE-SCATTER: %[[BB:.+]], %[[OFFSET:.+]], {{.*}}, {{.*}} = memref.extract_strided_metadata %[[SUBVIEW]] -// STORE-SCATTER-SAME: : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> memref, index, index, index, index, index -// STORE-SCATTER: %[[STEP:.+]] = vector.step : vector<8xindex> -// STORE-SCATTER: arith.muli {{.*}} : index -// STORE-SCATTER: arith.addi %[[OFFSET]]{{.*}} : index -// STORE-SCATTER: arith.addi {{.*}} : index -// STORE-SCATTER: %[[SPLAT:.+]] = vector.broadcast {{.*}} : index to vector<8xindex> -// STORE-SCATTER: %[[IDX:.+]] = arith.addi %[[SPLAT]], %[[STEP]] : vector<8xindex> -// STORE-SCATTER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SUBVIEW]] -// STORE-SCATTER-SAME: : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> index -// STORE-SCATTER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 -// STORE-SCATTER: xegpu.store %[[VEC]], %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : vector<8xf16>, i64, vector<8xindex>, vector<8xi1> + +// CHECK-LABEL: @store_to_subview( +// CHECK-SAME: %[[VEC:.+]]: vector<8xf16>, +// CHECK-SAME: %[[SRC:.+]]: memref<4096x4096xf16>, +// CHECK-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index +// CHECK: %[[CST:.+]] = arith.constant dense : vector<8xi1> +// CHECK: %[[SUBVIEW:.+]] = memref.subview %[[SRC]][%[[OFF1]], %[[OFF2]]] [256, 256] [1, 1] +// CHECK-SAME: : memref<4096x4096xf16> to memref<256x256xf16, strided<[4096, 1], offset: ?>> +// CHECK: %[[BB:.+]], %[[OFFSET:.+]], {{.*}}, {{.*}} = memref.extract_strided_metadata %[[SUBVIEW]] +// CHECK-SAME: : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> memref, index, index, index, index, index +// CHECK: %[[STEP:.+]] = vector.step : vector<8xindex> +// CHECK: arith.muli {{.*}} : index +// CHECK: arith.addi %[[OFFSET]]{{.*}} : index +// CHECK: arith.addi {{.*}} : index +// CHECK: %[[SPLAT:.+]] = vector.broadcast {{.*}} : index to vector<8xindex> +// CHECK: %[[IDX:.+]] = arith.addi %[[SPLAT]], %[[STEP]] : vector<8xindex> +// CHECK: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SUBVIEW]] +// CHECK-SAME: : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> index +// CHECK: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64 +// CHECK: xegpu.store %[[VEC]], %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : vector<8xf16>, i64, vector<8xindex>, vector<8xi1> } // ----- @@ -409,22 +360,13 @@ gpu.func @store_2D_vector_addrspace3(%vec: vector<8x16xf32>, gpu.return } -// STORE-ND-LABEL: @store_2D_vector_addrspace3 -// STORE-ND-SAME: %[[VEC:.+]]: vector<8x16xf32> -// STORE-ND-SAME: %[[SOURCE:.+]]: memref<16x32xf32, 3> -// STORE-ND-SAME: %[[OFFSET:.+]]: index -// STORE-ND: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[SOURCE]] : memref<16x32xf32, 3> -> !xegpu.mem_desc<16x32xf32> -// STORE-ND: xegpu.store_matrix %[[VEC]], %[[MEM_DESC]][%[[OFFSET]], %[[OFFSET]]] : vector<8x16xf32>, !xegpu.mem_desc<16x32xf32>, index, index -// STORE-ND: gpu.return - -// STORE-SCATTER-LABEL: @store_2D_vector_addrspace3 -// STORE-SCATTER-SAME: %[[VEC:.+]]: vector<8x16xf32> -// STORE-SCATTER-SAME: %[[SOURCE:.+]]: memref<16x32xf32, 3> -// STORE-SCATTER-SAME: %[[OFFSET:.+]]: index -// STORE-SCATTER: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[SOURCE]] : memref<16x32xf32, 3> -> !xegpu.mem_desc<16x32xf32> -// STORE-SCATTER: xegpu.store_matrix %[[VEC]], %[[MEM_DESC]][%[[OFFSET]], %[[OFFSET]]] : vector<8x16xf32>, !xegpu.mem_desc<16x32xf32>, index, index -// STORE-SCATTER: gpu.return - +// CHECK-LABEL: @store_2D_vector_addrspace3 +// CHECK-SAME: %[[VEC:.+]]: vector<8x16xf32> +// CHECK-SAME: %[[SOURCE:.+]]: memref<16x32xf32, 3> +// CHECK-SAME: %[[OFFSET:.+]]: index +// CHECK: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[SOURCE]] : memref<16x32xf32, 3> -> !xegpu.mem_desc<16x32xf32> +// CHECK: xegpu.store_matrix %[[VEC]], %[[MEM_DESC]][%[[OFFSET]], %[[OFFSET]]] : vector<8x16xf32>, !xegpu.mem_desc<16x32xf32>, index, index +// CHECK: gpu.return } // ----- @@ -437,22 +379,13 @@ gpu.func @store_1D_vector_addrspace3(%vec: vector<8xf32>, gpu.return } -// STORE-ND-LABEL: @store_1D_vector_addrspace3 -// STORE-ND-SAME: %[[VEC:.+]]: vector<8xf32> -// STORE-ND-SAME: %[[SOURCE:.+]]: memref<32xf32, 3> -// STORE-ND-SAME: %[[OFFSET:.+]]: index -// STORE-ND: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[SOURCE]] : memref<32xf32, 3> -> !xegpu.mem_desc<32xf32> -// STORE-ND: xegpu.store_matrix %[[VEC]], %[[MEM_DESC]][%[[OFFSET]]] : vector<8xf32>, !xegpu.mem_desc<32xf32>, index -// STORE-ND: gpu.return - -// STORE-SCATTER-LABEL: @store_1D_vector_addrspace3 -// STORE-SCATTER-SAME: %[[VEC:.+]]: vector<8xf32> -// STORE-SCATTER-SAME: %[[SOURCE:.+]]: memref<32xf32, 3> -// STORE-SCATTER-SAME: %[[OFFSET:.+]]: index -// STORE-SCATTER: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[SOURCE]] : memref<32xf32, 3> -> !xegpu.mem_desc<32xf32> -// STORE-SCATTER: xegpu.store_matrix %[[VEC]], %[[MEM_DESC]][%[[OFFSET]]] : vector<8xf32>, !xegpu.mem_desc<32xf32>, index -// STORE-SCATTER: gpu.return - +// CHECK-LABEL: @store_1D_vector_addrspace3 +// CHECK-SAME: %[[VEC:.+]]: vector<8xf32> +// CHECK-SAME: %[[SOURCE:.+]]: memref<32xf32, 3> +// CHECK-SAME: %[[OFFSET:.+]]: index +// CHECK: %[[MEM_DESC:.+]] = xegpu.create_mem_desc %[[SOURCE]] : memref<32xf32, 3> -> !xegpu.mem_desc<32xf32> +// CHECK: xegpu.store_matrix %[[VEC]], %[[MEM_DESC]][%[[OFFSET]]] : vector<8xf32>, !xegpu.mem_desc<32xf32>, index +// CHECK: gpu.return } // ----- @@ -464,10 +397,6 @@ gpu.func @store_0D_vector_unsupported(%vec: vector, gpu.return } -// STORE-ND-LABEL: @store_0D_vector_unsupported -// STORE-ND: vector.transfer_write - -// STORE-SCATTER-LABEL: @store_0D_vector_unsupported -// STORE-SCATTER: vector.transfer_write - +// CHECK-LABEL: @store_0D_vector_unsupported +// CHECK: vector.transfer_write } diff --git a/offload/plugins-nextgen/common/include/JIT.h b/offload/plugins-nextgen/common/include/JIT.h index b4e3712d9c980..96dea06335730 100644 --- a/offload/plugins-nextgen/common/include/JIT.h +++ b/offload/plugins-nextgen/common/include/JIT.h @@ -103,6 +103,8 @@ struct JITEngine { StringEnvar("LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE"); StringEnvar PostOptIRModuleFileName = StringEnvar("LIBOMPTARGET_JIT_POST_OPT_IR_MODULE"); + StringEnvar SaveImageFileName = + StringEnvar("LIBOMPTARGET_JIT_SAVE_IMAGE_FILENAME"); UInt32Envar JITOptLevel = UInt32Envar("LIBOMPTARGET_JIT_OPT_LEVEL", 3); BoolEnvar JITSkipOpt = BoolEnvar("LIBOMPTARGET_JIT_SKIP_OPT", false); }; diff --git a/offload/plugins-nextgen/common/src/JIT.cpp b/offload/plugins-nextgen/common/src/JIT.cpp index 9153059887757..b78c9b1768139 100644 --- a/offload/plugins-nextgen/common/src/JIT.cpp +++ b/offload/plugins-nextgen/common/src/JIT.cpp @@ -300,5 +300,16 @@ JITEngine::process(StringRef Image, target::plugin::GenericDeviceTy &Device) { return Device.doJITPostProcessing(std::move(MB)); }; - return compile(Image, ComputeUnitKind, PostProcessing); + auto ImageOrError = compile(Image, ComputeUnitKind, PostProcessing); + + if (SaveImageFileName.isPresent() && ImageOrError) { + std::error_code EC; + raw_fd_ostream OS(SaveImageFileName.get(), EC); + if (EC) + return createStringError(error::ErrorCode::HOST_IO, + "saving JIT image file\n"); + OS << ImageOrError.get()->getBuffer(); + OS.close(); + } + return ImageOrError; } diff --git a/offload/test/jit/save_image.c b/offload/test/jit/save_image.c new file mode 100644 index 0000000000000..66936e196c528 --- /dev/null +++ b/offload/test/jit/save_image.c @@ -0,0 +1,20 @@ +// clang-format off +// RUN: %libomptarget-compileopt-generic -fopenmp-target-jit +// RUN: rm -f %t.image +// RUN: env LIBOMPTARGET_JIT_SAVE_IMAGE_FILENAME=%t.image %libomptarget-run-generic +// RUN: test -s %t.image +// clang-format on + +// REQUIRES: gpu +// XFAIL: intelgpu + +int main() { + int X = 0; + +#pragma omp target map(tofrom : X) + { + X = 1; + } + + return X != 1; +} diff --git a/offload/test/mapping/mapper_enter_data_always_present_ptee.c b/offload/test/mapping/mapper_enter_data_always_present_ptee.c index 9fd3a6bf0d794..810ec89eb8eb9 100644 --- a/offload/test/mapping/mapper_enter_data_always_present_ptee.c +++ b/offload/test/mapping/mapper_enter_data_always_present_ptee.c @@ -2,31 +2,27 @@ // "target enter data map(always, present, to : s)" clause instead of a // "target update to(present : s)" motion clause. Both invoke the mapper; this // checks present propagation to the pointee is consistent across the two paths. -// -// FIXME: this test currently run-fails at every version/bounds combination. -// The mapper maps the struct member (s.y) with a combined entry whose size does -// not match the member's own storage, so the map clause aborts with an -// "explicit extension not allowed" error before the present modifier is ever -// considered. This is fixed once the mapper emits attach-style maps for pointer -// members (so the member and pointee occupy separate, correctly-sized entries). -// -// EXPECTED final state: -// inbounds, 5.2 and 6.0: run succeeds, prints "333 333". -// out-of-bounds (s.p[0:20] over the mapped x[0:10]): -// 5.2: succeeds (present is not applied to the pointee before 6.0). -// 6.0: run-fails; the failure should be the 'present' map-type-modifier -// check on s.p[0:20] (an accompanying "explicit extension" message is -// incidental -- for a map clause it is user error to map 20 elements -// when only 10 are present). +// Inbounds: the pointee region is fully present; the run succeeds at every +// OpenMP version. // RUN: %libomptarget-compile-generic -fopenmp-version=52 -// RUN: %libomptarget-run-fail-generic 2>&1 | %fcheck-generic +// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic // RUN: %libomptarget-compile-generic -fopenmp-version=60 -// RUN: %libomptarget-run-fail-generic 2>&1 | %fcheck-generic +// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic + +// Out-of-bounds: s.p[0:20] extends beyond the mapped region x[0:10]. Because a +// map clause performs an actual mapping, requesting 20 elements when only 10 +// are present is an error, so the run fails with an "explicit extension" +// diagnostic at every version. +// FIXME: at OpenMP 6.0 the present modifier is also propagated to the pointee, +// so the failure should additionally report the 'present' map-type-modifier +// check on s.p[0:20]. The extension diagnostic currently fires first. // RUN: %libomptarget-compile-generic -fopenmp-version=52 -DOUT_OF_BOUNDS -// RUN: %libomptarget-run-fail-generic 2>&1 | %fcheck-generic +// RUN: %libomptarget-run-fail-generic 2>&1 \ +// RUN: | %fcheck-generic --check-prefix=CHECK-OOB // RUN: %libomptarget-compile-generic -fopenmp-version=60 -DOUT_OF_BOUNDS -// RUN: %libomptarget-run-fail-generic 2>&1 | %fcheck-generic +// RUN: %libomptarget-run-fail-generic 2>&1 \ +// RUN: | %fcheck-generic --check-prefix=CHECK-OOB #include @@ -63,14 +59,12 @@ int main() { fprintf(stderr, "addr=%p, size=%zu\n", &s.p[0], 20 * sizeof(s.p[0])); - // FIXME: the map clause aborts here with an "explicit extension not allowed" - // error on the mapper's combined member entry, at every version. Fixed once - // the mapper uses attach-style maps for pointer members. - // CHECK: explicit extension not allowed + // CHECK-OOB: explicit extension not allowed #pragma omp target data map(from : s.y, x) { f1(); } + // CHECK: 333 333 printf("%d %d\n", x[0], s.y); } diff --git a/offload/test/mapping/mapper_map_mbr_ptee_then_present_mbr_ptee.c b/offload/test/mapping/mapper_map_mbr_ptee_then_present_mbr_ptee.c index 9da3d0afd326b..fb3abc68159ce 100644 --- a/offload/test/mapping/mapper_map_mbr_ptee_then_present_mbr_ptee.c +++ b/offload/test/mapping/mapper_map_mbr_ptee_then_present_mbr_ptee.c @@ -1,15 +1,7 @@ // Check that it's ok to first map a member of a struct and its pointee, and // then do a map(present) on a mapper that maps them internally. -// -// FIXME: This currently run-fails, because the mapper does not yet emit -// attach-style maps for the pointee: the combined entry over the whole struct -// s1 (40016 bytes) triggers an "explicit extension not allowed" error against -// the 4-byte device allocation of s1.x. Once attach-style maps are emitted for -// the pointee, the present check should pass and the run should complete the -// present/delete sequence below. -// RUN: %libomptarget-compile-generic -// RUN: %libomptarget-run-fail-generic 2>&1 \ -// RUN: | %fcheck-generic --check-prefix=CHECK + +// RUN: %libomptarget-compile-run-and-check-generic #include #include @@ -35,24 +27,20 @@ int main() { s1.p = (int *)&x; #pragma omp target enter data map(alloc : s1.x, s1.p[0 : 10]) - // EXPECTED: After mapping - print_status(&s1.x, "x"); // EXPECTED: x is present - print_status(&s1.dummy, "dummy"); // EXPECTED: dummy is not present - print_status(&s1.p, "p"); // EXPECTED: p is not present - print_status(&s1.p[0], "p[0]"); // EXPECTED: p[0] is present - - // This present check currently fails (explicit extension); once attach-style - // maps are emitted for the pointee, it should pass. - // clang-format off - // CHECK: message: explicit extension not allowed - // CHECK: fatal error 1: failure of target construct while offloading is mandatory - // clang-format on + printf("After mapping\n"); + print_status(&s1.x, "x"); // CHECK: x is present + print_status(&s1.dummy, "dummy"); // CHECK: dummy is not present + print_status(&s1.p, "p"); // CHECK: p is not present + print_status(&s1.p[0], "p[0]"); // CHECK: p[0] is present + printf("\n"); + + // This present check should pass. #pragma omp target enter data map(present, alloc : s1) #pragma omp target exit data map(delete : s1) - // EXPECTED: After deleting - print_status(&s1.x, "x"); // EXPECTED: x is not present - print_status(&s1.dummy, "dummy"); // EXPECTED: dummy is not present - print_status(&s1.p, "p"); // EXPECTED: p is not present - print_status(&s1.p[0], "p[0]"); // EXPECTED: p[0] is not present + printf("After deleting\n"); + print_status(&s1.x, "x"); // CHECK: x is not present + print_status(&s1.dummy, "dummy"); // CHECK: dummy is not present + print_status(&s1.p, "p"); // CHECK: p is not present + print_status(&s1.p[0], "p[0]"); // CHECK: p[0] is not present } diff --git a/offload/test/mapping/mapper_map_mbr_then_present_mbr_ptee.c b/offload/test/mapping/mapper_map_mbr_then_present_mbr_ptee.c index 0ee75ada5d00e..bb8b6aa3c3e76 100644 --- a/offload/test/mapping/mapper_map_mbr_then_present_mbr_ptee.c +++ b/offload/test/mapping/mapper_map_mbr_then_present_mbr_ptee.c @@ -1,22 +1,19 @@ // The mapper maps a struct member (s.x) and a pointee (s.p[0:10]). We pre-map // only s.x, then do map(present) on the mapper. The pointee s.p[0:10] is not // present, so once PRESENT is propagated to the pointee (a follow-on, at OpenMP -// >= 6.0) the check must fail; at <= 5.2 present is not propagated, so it would -// pass. +// >= 6.0) the check must fail; at <= 5.2 present is not propagated, so it +// passes. // -// FIXME: This currently run-fails at BOTH versions, because the mapper does not -// yet emit attach-style maps for the pointee: the combined entry over the whole -// struct s1 (40016 bytes) triggers an "explicit extension not allowed" error -// against the 4-byte device allocation of s1.x. Once attach-style maps are -// emitted for the pointee: +// FIXME: PRESENT is not propagated to the pointee yet, so the run currently +// completes ("done") at BOTH versions. Once it is propagated: // EXPECTED (5.2): the run completes ("done"). // EXPECTED (6.0): the present check fails for the absent pointee s1.p[0:10]. // RUN: %libomptarget-compile-generic -fopenmp-version=52 -// RUN: %libomptarget-run-fail-generic 2>&1 \ -// RUN: | %fcheck-generic --check-prefixes=CHECK +// RUN: %libomptarget-run-generic 2>&1 \ +// RUN: | %fcheck-generic --check-prefixes=CHECK,CHECK-52 // RUN: %libomptarget-compile-generic -fopenmp-version=60 -// RUN: %libomptarget-run-fail-generic 2>&1 \ -// RUN: | %fcheck-generic --check-prefixes=CHECK +// RUN: %libomptarget-run-generic 2>&1 \ +// RUN: | %fcheck-generic --check-prefixes=CHECK,CHECK-60 #include #include @@ -41,22 +38,21 @@ void print_status(void *p, const char *name) { int main() { s1.p = (int *)&x; + // CHECK: addr=0x[[#%x,HOST_ADDR:]], size=[[#%u,SIZE:]] fprintf(stderr, "addr=%p, size=%ld\n", &s1.p[0], 10 * sizeof(s1.p[0])); #pragma omp target enter data map(alloc : s1.x) - print_status(&s1.x, "x"); // EXPECTED: x is present - print_status(&s1.dummy, "dummy"); // EXPECTED: dummy is not present - print_status(&s1.p, "p"); // EXPECTED: p is not present - print_status(&s1.p[0], "p[0]"); // EXPECTED: p[0] is not present + print_status(&s1.x, "x"); // CHECK: x is present + print_status(&s1.dummy, "dummy"); // CHECK: dummy is not present + print_status(&s1.p, "p"); // CHECK: p is not present + print_status(&s1.p[0], "p[0]"); // CHECK: p[0] is not present #pragma omp target enter data map(present, alloc : s1) - // Once attach-style maps are emitted for the pointee, at 5.2 the run - // completes past this point; at 6.0 the present check on the absent pointee - // s1.p[0:10] fails here. - // clang-format off - // CHECK: message: explicit extension not allowed - // CHECK: fatal error 1: failure of target construct while offloading is mandatory - // clang-format on + // Once PRESENT is propagated to the pointee, at 5.2 the run completes past + // this point; at 6.0 the present check on the absent pointee s1.p[0:10] + // fails here. + // CHECK-52: done + // CHECK-60: done fprintf(stderr, "done\n"); } diff --git a/offload/test/mapping/mapper_map_present_ptee.c b/offload/test/mapping/mapper_map_present_ptee.c index ce6887a12f8de..7a0ed0b126c5a 100644 --- a/offload/test/mapping/mapper_map_present_ptee.c +++ b/offload/test/mapping/mapper_map_present_ptee.c @@ -1,21 +1,12 @@ // A user-defined mapper maps s.y and, with present written directly in the // mapper's own clause, the pointee s.p. present in the mapper clause applies at // every OpenMP version -- this is not the outer-clause propagation that a -// follow-on gates on the version. +// follow-on gates on the version, so both 5.2 and 6.0 behave the same here. // -// FIXME: This currently run-fails at BOTH the inbounds and out-of-bounds cases, -// because the mapper does not yet emit attach-style maps for the pointee: the -// present check runs against the whole struct s (16 bytes), which is not -// present. Once attach-style maps are emitted for the pointee: -// EXPECTED (inbounds): the present check passes and the run prints "333 333". -// EXPECTED (out-of-bounds): the present check fails only for the pointee -// region s.p[0:20] that is not present. -// RUN: %libomptarget-compile-generic -// RUN: %libomptarget-run-fail-generic 2>&1 \ -// RUN: | %fcheck-generic --check-prefix=CHECK +// RUN: %libomptarget-compile-run-and-check-generic // RUN: %libomptarget-compile-generic -DOUT_OF_BOUNDS // RUN: %libomptarget-run-fail-generic 2>&1 \ -// RUN: | %fcheck-generic --check-prefix=CHECK +// RUN: | %fcheck-generic --check-prefix=CHECK-OOB #include @@ -26,6 +17,7 @@ typedef struct { } S; #ifdef OUT_OF_BOUNDS +// s.p[0:20] extends beyond the mapped region x[0:10]; the present check fails. #pragma omp declare mapper(S s) map(s.y) map(present, tofrom : s.p[0 : 20]) #else #pragma omp declare mapper(S s) map(s.y) map(present, tofrom : s.p[0 : 2]) @@ -33,7 +25,7 @@ typedef struct { S s; void f1() { - // The mapper runs here; the present check currently fails here at both cases. + // The mapper runs here; for OUT_OF_BOUNDS the present check fails here. #pragma omp target update to(s) #pragma omp target data use_device_addr(s, x) @@ -49,6 +41,7 @@ int main() { s.y = 111; s.p = &x[0]; + // CHECK-OOB: addr=0x[[#%x,HOST_ADDR:]], size=[[#%u,SIZE:]] fprintf(stderr, "addr=%p, size=%zu\n", &s.p[0], 20 * sizeof(s.p[0])); #pragma omp target data map(from : s.y, x) @@ -56,10 +49,9 @@ int main() { f1(); } - // EXPECTED (inbounds): 333 333 - fprintf(stderr, "%d %d\n", x[0], s.y); + printf("%d %d\n", x[0], s.y); // CHECK: 333 333 // clang-format off - // CHECK: message: device mapping required by 'present' motion modifier does not exist for host address - // CHECK: fatal error 1: failure of target construct while offloading is mandatory + // CHECK-OOB: message: device mapping required by 'present' motion modifier does not exist for host address 0x{{0*}}[[#HOST_ADDR]] ([[#SIZE]] bytes) + // CHECK-OOB: fatal error 1: failure of target construct while offloading is mandatory // clang-format on } diff --git a/offload/test/mapping/mapper_map_ptee_only.c b/offload/test/mapping/mapper_map_ptee_only.c index 6e9904bb1917e..3944545b6e23c 100644 --- a/offload/test/mapping/mapper_map_ptee_only.c +++ b/offload/test/mapping/mapper_map_ptee_only.c @@ -28,16 +28,10 @@ int main() { #pragma omp target enter data map(alloc : s1) printf("After mapping\n"); - print_status(&s1.x, "x"); // CHECK: x is present - // FIXME: mapper should not map s.dummy or s.p; will be fixed when mapper - // emits attach-style maps for pointer members. - print_status(&s1.dummy, "dummy"); // CHECK: dummy is present - // EXPECTED: dummy is not present - // FIXME: mapper should not map s.dummy or s.p; will be fixed when mapper - // emits attach-style maps for pointer members. - print_status(&s1.p, "p"); // CHECK: p is present - // EXPECTED: p is not present - print_status(&s1.p[0], "p[0]"); // CHECK: p[0] is present + print_status(&s1.x, "x"); // CHECK: x is present + print_status(&s1.dummy, "dummy"); // CHECK: dummy is not present + print_status(&s1.p, "p"); // CHECK: p is not present + print_status(&s1.p[0], "p[0]"); // CHECK: p[0] is present printf("\n"); #pragma omp target exit data map(delete : s1) diff --git a/offload/test/mapping/mapper_map_ptee_only_2_ptr_indirections.c b/offload/test/mapping/mapper_map_ptee_only_2_ptr_indirections.c index 1b91d87071cbd..c9c1a2c8ae7a9 100644 --- a/offload/test/mapping/mapper_map_ptee_only_2_ptr_indirections.c +++ b/offload/test/mapping/mapper_map_ptee_only_2_ptr_indirections.c @@ -42,10 +42,8 @@ int main() { print_status(&s2.s1p->y, "y"); // CHECK: y is present print_status(&s2.z, "z"); // CHECK: z is present print_status(&s2.s1p->dummy, "dummy"); // CHECK: dummy is not present - print_status(&s2.s1p->p, "p"); // CHECK: p is present - // FIXME: mapper should emit attach-style maps for pointer members. - // EXPECTED: p is not present - print_status(&s2.s1p->p[0], "p[0]"); // CHECK: p[0] is present + print_status(&s2.s1p->p, "p"); // CHECK: p is not present + print_status(&s2.s1p->p[0], "p[0]"); // CHECK: p[0] is present printf("\n"); #pragma omp target exit data map(delete : s2) diff --git a/offload/test/mapping/mapper_map_ptee_only_2_ptr_indirections_array.c b/offload/test/mapping/mapper_map_ptee_only_2_ptr_indirections_array.c index 09eed525d4340..2f1c5a9a7e615 100644 --- a/offload/test/mapping/mapper_map_ptee_only_2_ptr_indirections_array.c +++ b/offload/test/mapping/mapper_map_ptee_only_2_ptr_indirections_array.c @@ -6,6 +6,10 @@ // Array variant of mapper_map_ptee_only_2_ptr_indirections.c. // The mapper maps s2.z, s2.s1p->x, s2.s1p->y, and s2.s1p->p[0:10]. // s2.s1p->dummy and s2.s1p->p itself are not mapped. +// This exercises the nested-pointer-chain case: the inner MEMBER_OF bits for +// s2.s1p->x/y/p[0:10] must be shifted correctly, and outer MEMBER_OF must not +// be applied to the pointee entry (s2.s1p->p[0:10]) or the ATTACH entry +// (s2.s1p). int x[2][10]; @@ -45,11 +49,8 @@ int main() { print_status(&s2arr[0].z, "s2arr[0].z"); // CHECK: s2arr[0].z is present print_status(&s2arr[0].s1p->dummy, "s2arr[0].dummy"); // CHECK: s2arr[0].dummy is not present - // FIXME: mapper should not map s1p->p; will be fixed when mapper emits - // attach-style maps for pointer members. print_status(&s2arr[0].s1p->p, - "s2arr[0].p"); // CHECK: s2arr[0].p is present - // EXPECTED: s2arr[0].p is not present + "s2arr[0].p"); // CHECK: s2arr[0].p is not present print_status(&s2arr[0].s1p->p[0], "s2arr[0].p[0]"); // CHECK: s2arr[0].p[0] is present print_status(&s2arr[1].s1p->x, "s2arr[1].x"); // CHECK: s2arr[1].x is present @@ -57,11 +58,8 @@ int main() { print_status(&s2arr[1].z, "s2arr[1].z"); // CHECK: s2arr[1].z is present print_status(&s2arr[1].s1p->dummy, "s2arr[1].dummy"); // CHECK: s2arr[1].dummy is not present - // FIXME: mapper should not map s1p->p; will be fixed when mapper emits - // attach-style maps for pointer members. print_status(&s2arr[1].s1p->p, - "s2arr[1].p"); // CHECK: s2arr[1].p is present - // EXPECTED: s2arr[1].p is not present + "s2arr[1].p"); // CHECK: s2arr[1].p is not present print_status(&s2arr[1].s1p->p[0], "s2arr[1].p[0]"); // CHECK: s2arr[1].p[0] is present printf("\n"); diff --git a/offload/test/mapping/mapper_map_ptee_only_2ndlevel.c b/offload/test/mapping/mapper_map_ptee_only_2ndlevel.c index d7803b4e24d9a..2d5404a6d0cea 100644 --- a/offload/test/mapping/mapper_map_ptee_only_2ndlevel.c +++ b/offload/test/mapping/mapper_map_ptee_only_2ndlevel.c @@ -35,18 +35,14 @@ int main() { printf("After mapping\n"); print_status(&s2.s1.x, "x"); // CHECK: x is present print_status(&s2.s1.dummy, "dummy"); // CHECK: dummy is not present - print_status(&s2.s1.p, "p"); // CHECK: p is present - // FIXME: mapper should emit attach-style maps for pointer members. - // EXPECTED: p is not present - print_status(&s2.s1.p[0], "p[0]"); // CHECK: p[0] is present + print_status(&s2.s1.p, "p"); // CHECK: p is not present + print_status(&s2.s1.p[0], "p[0]"); // CHECK: p[0] is present printf("\n"); #pragma omp target exit data map(delete : s2) printf("After deleting\n"); print_status(&s2.s1.x, "x"); // CHECK: x is not present print_status(&s2.s1.dummy, "dummy"); // CHECK: dummy is not present - print_status(&s2.s1.p, "p"); // CHECK: p is present - // FIXME: mapper should emit attach-style maps for pointer members. - // EXPECTED: p is not present - print_status(&s2.s1.p[0], "p[0]"); // CHECK: p[0] is not present + print_status(&s2.s1.p, "p"); // CHECK: p is not present + print_status(&s2.s1.p[0], "p[0]"); // CHECK: p[0] is not present } diff --git a/openmp/docs/design/Runtimes.rst b/openmp/docs/design/Runtimes.rst index 4e3137abd6fb7..14e300a0f531f 100644 --- a/openmp/docs/design/Runtimes.rst +++ b/openmp/docs/design/Runtimes.rst @@ -741,6 +741,7 @@ variables is defined below. * ``LIBOMPTARGET_JIT_REPLACEMENT_MODULE= (LLVM-IR file)`` * ``LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE= (LLVM-IR file)`` * ``LIBOMPTARGET_JIT_POST_OPT_IR_MODULE= (LLVM-IR file)`` + * ``LIBOMPTARGET_JIT_SAVE_IMAGE_FILENAME= (device image file)`` * ``LIBOMPTARGET_MIN_THREADS_FOR_LOW_TRIP_COUNT= (default: 32)`` * ``LIBOMPTARGET_REUSE_BLOCKS_FOR_HIGH_TRIP_COUNT=[TRUE/FALSE] (default TRUE)`` * ``OFFLOAD_TRACK_ALLOCATION_TRACES=[TRUE/FALSE] (default FALSE)`` @@ -1162,6 +1163,14 @@ which the LLVM-IR module is written. The module can be the analyzed, and transformed and loaded back into the JIT pipeline via :ref:`LIBOMPTARGET_JIT_REPLACEMENT_MODULE`. +.. _libomptarget_jit_save_image_filename: + +LIBOMPTARGET_JIT_SAVE_IMAGE_FILENAME +"""""""""""""""""""""""""""""""""""" + +This environment variable can be used to save the device image produced by the +device JIT after target-specific post-processing. The value is expected to be a +filename into which the binary device image is written. LIBOMPTARGET_MIN_THREADS_FOR_LOW_TRIP_COUNT """""""""""""""""""""""""""""""""""""""""""