diff --git a/CMakeLists.txt b/CMakeLists.txt index 92c399e..ace4a40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,10 +39,12 @@ endif() option(USE_AMDGPU "Build the AMD GPU Execution Provider" OFF) option(USE_DML "Build the DirectML Execution Provider" OFF) option(USE_MIGRAPHX "Build the MIGraphX Execution Provider" OFF) +option(USE_HIP "Build the HIP (morphizen) backend shim" OFF) if(USE_AMDGPU) - set(CACHE{USE_DML} TYPE BOOL FORCE VALUE ON) - set(CACHE{USE_MIGRAPHX} TYPE BOOL FORCE VALUE ON) + set(CACHE{USE_DML} TYPE BOOL FORCE VALUE OFF) + set(CACHE{USE_MIGRAPHX} TYPE BOOL FORCE VALUE OFF) + set(CACHE{USE_HIP} TYPE BOOL FORCE VALUE ON) endif() add_subdirectory(src) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7648cd2..7526e72 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -167,3 +167,6 @@ endif() if(USE_MIGRAPHX) add_subdirectory(migraphx) endif() +if(USE_HIP) + add_subdirectory(hip) +endif() diff --git a/src/amdgpu/CMakeLists.txt b/src/amdgpu/CMakeLists.txt index f1b1e83..e3e85be 100644 --- a/src/amdgpu/CMakeLists.txt +++ b/src/amdgpu/CMakeLists.txt @@ -21,11 +21,19 @@ add_library(amdgpu-ep MODULE target_link_libraries(amdgpu-ep PRIVATE plugin-ep-utils flatbuffers) -target_include_directories(amdgpu-ep PRIVATE - $) +if(USE_MIGRAPHX) + target_include_directories(amdgpu-ep PRIVATE + $) +endif() + +if(USE_DML) + target_include_directories(amdgpu-ep PRIVATE + $) +endif() -target_include_directories(amdgpu-ep PRIVATE - $) +target_compile_definitions(amdgpu-ep PRIVATE + $<$:USE_DML> + $<$:USE_MIGRAPHX>) if(WIN32) set_property(TARGET amdgpu-ep APPEND_STRING diff --git a/src/amdgpu/gpu_ep.cc b/src/amdgpu/gpu_ep.cc index 8979077..9a99abb 100644 --- a/src/amdgpu/gpu_ep.cc +++ b/src/amdgpu/gpu_ep.cc @@ -5,7 +5,9 @@ #include "gpu_ep.h" #include "gpu_options.h" +#ifdef USE_MIGRAPHX #include "mgx_options.h" +#endif #define EP_CALL_T(backend, fn, defval, ...) \ do { \ @@ -116,6 +118,14 @@ ExecutionProvider::ExecutionProvider(ProviderFactory& factory, std::string_view }; }; + const auto create_hip_backend = [&] { + // hip backend manages allocator/data-transfer at the backend factory level, + // reached through the amdgpu Allocator/DataTransfer wrappers — leave + // OrtEp::CreateAllocator null so ORT falls back to ep_factory_.CreateAllocator. + THROW_IF_ERROR(factory.CreateHipBackend(local_session_options, logger, backend_ep_)); + }; + +#ifdef USE_MIGRAPHX const auto create_migraphx_backend = [&] { const auto get_name = [](const std::string_view sv) { return std::string{"ep."}.append(kMIGraphXBackend).append(".").append(sv); @@ -158,15 +168,34 @@ ExecutionProvider::ExecutionProvider(ProviderFactory& factory, std::string_view } THROW_IF_ERROR(factory.CreateMIGraphXBackend(local_session_options, logger, backend_ep_)); }; +#endif - if (info.profile == Profile::Eager) { + bool backend_created = false; + if (info.profile == Profile::Eager || info.profile == Profile::DirectML) { create_directml_backend(); - } else if (info.profile == Profile::DirectML) { - create_directml_backend(); - } else if (info.profile == Profile::MIGraphX) { + backend_created = true; + } +#ifdef USE_MIGRAPHX + if (!backend_created && info.profile == Profile::MIGraphX) { create_migraphx_backend(); - } else { + backend_created = true; + } +#endif + if (!backend_created && info.profile == Profile::Llm) { + create_hip_backend(); + backend_created = true; + } +#ifdef USE_MIGRAPHX + if (!backend_created) { create_migraphx_backend(); + backend_created = true; + } +#endif + if (!backend_created) { + ort_api.ReleaseSessionOptions(local_session_options); + THROW_IF_ERROR(MAKE_STATUS(ORT_EP_FAIL, + "amdgpu EP: no backend available for the requested profile " + "(check build flags USE_DML/USE_MIGRAPHX)")); } ort_api.ReleaseSessionOptions(local_session_options); } diff --git a/src/amdgpu/gpu_factory.cc b/src/amdgpu/gpu_factory.cc index d2b2d93..8a16c0d 100644 --- a/src/amdgpu/gpu_factory.cc +++ b/src/amdgpu/gpu_factory.cc @@ -47,6 +47,7 @@ namespace gpu_ep { namespace { constexpr auto directmlBackend{LIBRARY_PREFIX ORT_TSTR("directml-backend") LIBRARY_SUFFIX}; constexpr auto migraphxBackend{LIBRARY_PREFIX ORT_TSTR("migraphx-backend") LIBRARY_SUFFIX}; +constexpr auto hipBackend{LIBRARY_PREFIX ORT_TSTR("hip-backend") LIBRARY_SUFFIX}; } ProviderFactory::ProviderFactory(const ApiPtrs& api_ptrs, const OrtApiBase* ort_api_base, const char* ep_name, const OrtLogger* default_logger) @@ -118,32 +119,54 @@ ProviderFactory::ProviderFactory(const ApiPtrs& api_ptrs, const OrtApiBase* ort_ API_CALL_S(ProviderFactory, this_, GetCustomOpDomains, domains, num_domains); }; - THROW_IF_ERROR(LoadDynamicLibrary(directmlBackend, &dml_backend_)); - THROW_IF_ERROR(GetSymbolFromLibrary(dml_backend_, - "ReleaseEpFactory", reinterpret_cast(&dml_release_ep_factory_))); +#ifdef USE_DML + { + THROW_IF_ERROR(LoadDynamicLibrary(directmlBackend, &dml_backend_)); + THROW_IF_ERROR(GetSymbolFromLibrary(dml_backend_, + "ReleaseEpFactory", reinterpret_cast(&dml_release_ep_factory_))); + + CreateEpFactories_t dml_create_ep_factories{}; + THROW_IF_ERROR(GetSymbolFromLibrary(dml_backend_, + "CreateEpFactories", reinterpret_cast(&dml_create_ep_factories))); + + size_t factories_created{}; + // Pass ep_name_ (e.g. "amdgpu") so the directml backend registers its kernels, + // allocators, and node assignments under the same name ORT sees for this EP. + // Using kDirectMLBackend ("directml") would cause a provider name mismatch: + // ORT would look up kernels under "amdgpu" but find them stamped as "directml". + THROW_IF_ERROR(dml_create_ep_factories(ep_name_.c_str(), ort_api_base, default_logger, + &dml_ep_factory_, 1, &factories_created)); + } +#endif - CreateEpFactories_t dml_create_ep_factories{}; - THROW_IF_ERROR(GetSymbolFromLibrary(dml_backend_, - "CreateEpFactories", reinterpret_cast(&dml_create_ep_factories))); +#ifdef USE_MIGRAPHX + { + THROW_IF_ERROR(LoadDynamicLibrary(migraphxBackend, &mgx_backend_)); + THROW_IF_ERROR(GetSymbolFromLibrary(mgx_backend_, + "ReleaseEpFactory", reinterpret_cast(&mgx_release_ep_factory_))); - size_t factories_created{}; - // Pass ep_name_ (e.g. "amdgpu") so the directml backend registers its kernels, - // allocators, and node assignments under the same name ORT sees for this EP. - // Using kDirectMLBackend ("directml") would cause a provider name mismatch: - // ORT would look up kernels under "amdgpu" but find them stamped as "directml". - THROW_IF_ERROR(dml_create_ep_factories(ep_name_.c_str(), ort_api_base, default_logger, - &dml_ep_factory_, 1, &factories_created)); + CreateEpFactories_t mgx_create_ep_factories{}; + THROW_IF_ERROR(GetSymbolFromLibrary(mgx_backend_, + "CreateEpFactories", reinterpret_cast(&mgx_create_ep_factories))); + + size_t factories_created{}; + THROW_IF_ERROR(mgx_create_ep_factories(kMIGraphXBackend, ort_api_base, default_logger, + &mgx_ep_factory_, 1, &factories_created)); + } +#endif - THROW_IF_ERROR(LoadDynamicLibrary(migraphxBackend, &mgx_backend_)); - THROW_IF_ERROR(GetSymbolFromLibrary(mgx_backend_, - "ReleaseEpFactory", reinterpret_cast(&mgx_release_ep_factory_))); + THROW_IF_ERROR(LoadDynamicLibrary(hipBackend, &hip_backend_)); + THROW_IF_ERROR(GetSymbolFromLibrary(hip_backend_, + "ReleaseEpFactory", reinterpret_cast(&hip_release_ep_factory_))); - CreateEpFactories_t mgx_create_ep_factories{}; - THROW_IF_ERROR(GetSymbolFromLibrary(mgx_backend_, - "CreateEpFactories", reinterpret_cast(&mgx_create_ep_factories))); + CreateEpFactories_t hip_create_ep_factories{}; + THROW_IF_ERROR(GetSymbolFromLibrary(hip_backend_, + "CreateEpFactories", reinterpret_cast(&hip_create_ep_factories))); - THROW_IF_ERROR(mgx_create_ep_factories(kMIGraphXBackend, ort_api_base, default_logger, - &mgx_ep_factory_, 1, &factories_created)); + size_t factories_created{}; + // Pass ep_name_ so the hip backend's EP reports the same name ORT sees. + THROW_IF_ERROR(hip_create_ep_factories(ep_name_.c_str(), ort_api_base, default_logger, + &hip_ep_factory_, 1, &factories_created)); data_transfer_ = std::make_unique(*this); } @@ -163,12 +186,19 @@ ProviderFactory::~ProviderFactory() { if (pinned_memory_info_) { ort_api.ReleaseMemoryInfo(pinned_memory_info_); } +#ifdef USE_DML if (!UnloadDynamicLibrary(dml_backend_).IsOK()) { /* TODO: log failure while unloading DirectML EP library */ } +#endif +#ifdef USE_MIGRAPHX if (!UnloadDynamicLibrary(mgx_backend_).IsOK()) { /* TODO: log failure while unloading MIGraphX EP library */ } +#endif + if (!UnloadDynamicLibrary(hip_backend_).IsOK()) { + /* TODO: log failure while unloading hip EP library */ + } } const char* ProviderFactory::GetName() const { diff --git a/src/amdgpu/gpu_factory.h b/src/amdgpu/gpu_factory.h index 8f9d399..b3775e7 100644 --- a/src/amdgpu/gpu_factory.h +++ b/src/amdgpu/gpu_factory.h @@ -24,6 +24,12 @@ struct ProviderFactory : OrtEpFactory, ApiPtrs { return STATUS_OK; } + Ort::Status CreateHipBackend(const OrtSessionOptions* session_options, const OrtLogger* logger, OrtEp*& ep) { + RETURN_IF_ERROR(hip_ep_factory_->CreateEp(hip_ep_factory_, nullptr, nullptr, 0, session_options, logger, &ep)); + backend_ep_factory_ = hip_ep_factory_; + return STATUS_OK; + } + OrtEpFactory* GetBackendFactory() const noexcept { return backend_ep_factory_; } @@ -87,6 +93,11 @@ struct ProviderFactory : OrtEpFactory, ApiPtrs { OrtEpFactory* mgx_ep_factory_{}; + void* hip_backend_{}; + ReleaseEpFactory_t hip_release_ep_factory_{}; + + OrtEpFactory* hip_ep_factory_{}; + OrtHardwareDevice* virtual_device_{}; // Owned memory infos for the GPU and pinned device slots, registered with OrtEpDevice. diff --git a/src/amdgpu/gpu_info.cc b/src/amdgpu/gpu_info.cc index 0c0d1d8..b7cc036 100644 --- a/src/amdgpu/gpu_info.cc +++ b/src/amdgpu/gpu_info.cc @@ -27,6 +27,8 @@ ProviderInfo::ProviderInfo(const ProviderOptions& provider_options) { profile = Profile::MIGraphX; } else if (lower == "directml" || value == "4") { profile = Profile::DirectML; + } else if (lower == "llm" || value == "5") { + profile = Profile::Llm; } else { return MAKE_STATUS(ORT_FAIL, "unknown profile: '", value, "'"); } diff --git a/src/amdgpu/gpu_info.h b/src/amdgpu/gpu_info.h index 18ded75..5e2a8ba 100644 --- a/src/amdgpu/gpu_info.h +++ b/src/amdgpu/gpu_info.h @@ -15,7 +15,8 @@ enum class Profile { Eager, Optimized, MIGraphX, - DirectML + DirectML, + Llm }; struct ProviderInfo { diff --git a/src/amdgpu/gpu_options.h b/src/amdgpu/gpu_options.h index 63d5181..7f91338 100644 --- a/src/amdgpu/gpu_options.h +++ b/src/amdgpu/gpu_options.h @@ -10,6 +10,7 @@ namespace gpu_ep { constexpr auto kDirectMLBackend = "directml"; constexpr auto kMIGraphXBackend = "migraphx"; +constexpr auto kHipBackend = "hip"; namespace provider_option { constexpr auto kDeviceId = "device_id"sv; diff --git a/src/hip/CMakeLists.txt b/src/hip/CMakeLists.txt new file mode 100644 index 0000000..70f1b3a --- /dev/null +++ b/src/hip/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright (c) Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +add_library(hip-ep MODULE hip_factory.cc) + +target_link_libraries(hip-ep PRIVATE plugin-ep-utils) + +if(WIN32) + set_property(TARGET hip-ep APPEND_STRING + PROPERTY LINK_FLAGS /DEF:${PROJECT_SOURCE_DIR}/src/shared/symbols.def) + + install(TARGETS hip-ep + DESTINATION ".") + + install(FILES $ + DESTINATION "." OPTIONAL) +else() + set_property(TARGET hip-ep APPEND_STRING + PROPERTY LINK_FLAGS "-Xlinker --version-script=${PROJECT_SOURCE_DIR}/src/shared/version_script.lds -Xlinker --gc-sections") + + install(TARGETS hip-ep + DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() + +set_target_properties(hip-ep PROPERTIES OUTPUT_NAME "hip-backend") + +add_version_info( + TARGET hip-ep + DESCRIPTION "HIP Execution Provider - ONNX Runtime plugin" + PRODUCT "HIP Plugin EP") diff --git a/src/hip/hip_factory.cc b/src/hip/hip_factory.cc new file mode 100644 index 0000000..8d4dc01 --- /dev/null +++ b/src/hip/hip_factory.cc @@ -0,0 +1,71 @@ +// Copyright (c) Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include + +#include "common/dynamic_library.h" +#include "common/plugin_ep_utils.h" + +namespace { + +// The HIP EP ships as a separate prebuilt DLL. hip-backend is a thin +// pass-through: it loads that DLL, forwards CreateEpFactories, and hands back +// its OrtEpFactory unchanged so the parent EP forwards to it directly. This +// presents a repo-built backend parallel to directml/migraphx while keeping the +// HIP EP external. The DLL search directory is set by whoever loads this shim +// (the AMD GPU umbrella, or the executable that registers it). +constexpr auto hipLib{LIBRARY_PREFIX ORT_TSTR("hipgpu") LIBRARY_SUFFIX}; + +using CreateEpFactories_t = OrtStatus* (*)(const char*, const OrtApiBase*, const OrtLogger*, + OrtEpFactory**, size_t, size_t*); +using ReleaseEpFactory_t = OrtStatus* (*)(OrtEpFactory*); + +// CreateEpFactories may be called more than once (e.g. a real session plus a +// device-init session in OGA). Reference-count the underlying HIP EP DLL so it +// is loaded on the first factory and unloaded only after the last is released. +void* g_hip_module{}; +CreateEpFactories_t g_hip_create{}; +ReleaseEpFactory_t g_hip_release{}; +std::atomic g_refcount{0}; + +} // namespace + +extern "C" { + +OrtStatus* CreateEpFactories(const char* registration_name, const OrtApiBase* ort_api_base, + const OrtLogger* default_logger, OrtEpFactory** factories, size_t max_factories, size_t* num_factories) +{ + try { + const OrtApi* ort_api{ort_api_base->GetApi(ORT_API_VERSION)}; + Ort::InitApi(ort_api); + + if (g_refcount++ == 0) { + THROW_IF_ERROR(LoadDynamicLibrary(hipLib, &g_hip_module)); + THROW_IF_ERROR(GetSymbolFromLibrary(g_hip_module, "CreateEpFactories", + reinterpret_cast(&g_hip_create))); + THROW_IF_ERROR(GetSymbolFromLibrary(g_hip_module, "ReleaseEpFactory", + reinterpret_cast(&g_hip_release))); + } + + return g_hip_create(registration_name, ort_api_base, default_logger, + factories, max_factories, num_factories); + } + catch (const std::exception& e) { + // Roll back the refcount bump so a later retry re-attempts the load. + --g_refcount; + RETURN_STATUS(ORT_EP_FAIL, e.what()); + } +} + +OrtStatus* ReleaseEpFactory(OrtEpFactory* factory) { + OrtStatus* status = (g_hip_release != nullptr) ? g_hip_release(factory) : nullptr; + if (--g_refcount == 0 && g_hip_module != nullptr) { + UnloadDynamicLibrary(g_hip_module); + g_hip_module = nullptr; + g_hip_create = nullptr; + g_hip_release = nullptr; + } + return status; +} + +} // extern "C"