diff --git a/.flake/pkgs/legion.nix b/.flake/pkgs/legion.nix index 814ef85e00..361a66c4ff 100644 --- a/.flake/pkgs/legion.nix +++ b/.flake/pkgs/legion.nix @@ -2,7 +2,6 @@ , stdenv , fetchFromGitLab , cmake -, python3 , cudaPackages ? { } , cudaCapabilities ? [ "60" "70" "80" "86" ] , maxDim ? 5 @@ -17,15 +16,14 @@ let in stdenv.mkDerivation rec { - pname = "legion_flexflow"; - version = "2024-03-13"; + pname = "legion"; + version = "2025-01-06"; src = fetchFromGitLab { owner = "StanfordLegion"; repo = "legion"; - rev = "24e8c452341dea41427e0ce61e154d61715e6835"; - sha256 = "sha256-NjCSjphOIew/V24i74I6DModSGcWKLeiSIjts3cFtx4="; - fetchSubmodules = true; + rev = "7be1abd0207eb1126c7629b16d1123fa6f58ce9d"; + sha256 = "sha256-gTjnGYYTQwTsrV1WcY0qqpTrlwbzAPcndurRy6XnG8A="; }; nativeBuildInputs = [ @@ -33,21 +31,18 @@ stdenv.mkDerivation rec { ]; cmakeFlags = [ - "-DLegion_USE_Python=1" - "-DLegion_BUILD_BINDINGS=1" "-DLegion_USE_CUDA=1" "-DLegion_CUDA_ARCH=${lib.concatStringsSep "," cudaCapabilities}" "-DLegion_MAX_DIM=${toString maxDim}" ]; buildInputs = [ - python3 cudatoolkit ]; meta = with lib; { description = "Legion is a parallel programming model for distributed, heterogeneous machines"; - homepage = "https://github.com/StanfordLegion/legion"; + homepage = "https://legion.stanford.edu/"; license = licenses.asl20; }; } diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b313f5d4f..c2239cdcb0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,22 +33,15 @@ set(FF_MAX_NUM_TASK_REGIONS "20" CACHE STRING set(FF_MAX_NUM_TASK_ARGUMENTS "5" CACHE STRING "Maximum number of arguments that can be declared in a TaskSignature") option(FF_USE_NCCL "Run FlexFlow with NCCL" OFF) -option(FF_USE_PREBUILT_LEGION "Enable use of Legion pre-compiled library, if available" ON) option(FF_USE_ALL_PREBUILT_LIBRARIES "Enable use of all pre-compiled libraries, if available" OFF) option(FF_USE_PYTHON "Enable Python" ON) option(FF_BUILD_FROM_PYPI "Build from pypi" OFF) option(FF_USE_CODE_COVERAGE "Enable code coverage" OFF) -set(FF_GASNET_CONDUITS aries udp mpi ibv ucx) -set(FF_GASNET_CONDUIT "mpi" CACHE STRING "Select GASNet conduit ${FF_GASNET_CONDUITS}") -set_property(CACHE FF_GASNET_CONDUIT PROPERTY STRINGS ${FF_GASNET_CONDUITS}) -set(FF_LEGION_NETWORKS "" CACHE STRING "Network backend(s) to use") - set(FF_GPU_BACKENDS cuda hip_cuda hip_rocm intel) set(FF_GPU_BACKEND "cuda" CACHE STRING "Select GPU Backend ${FF_GPU_BACKENDS}") set_property(CACHE FF_GPU_BACKEND PROPERTY STRINGS ${FF_GPU_BACKENDS}) -option(FF_USE_EXTERNAL_LEGION "Use pre-installed Legion" OFF) option(FF_USE_EXTERNAL_NCCL "Use pre-installed NCCL" OFF) option(FF_USE_EXTERNAL_JSON "Use pre-installed nlohmann::json" OFF) option(FF_USE_EXTERNAL_FMT "Use pre-installed fmt" OFF) @@ -104,7 +97,7 @@ include(gbenchmark) include(libassert) include(CTest) include(fmt) -include(legion) +include(realm) include(rapidcheck) #include(gtest) diff --git a/cmake/legion.cmake b/cmake/legion.cmake deleted file mode 100644 index b904b66efc..0000000000 --- a/cmake/legion.cmake +++ /dev/null @@ -1,161 +0,0 @@ -if(FF_USE_EXTERNAL_LEGION) - if(NOT "${LEGION_ROOT}" STREQUAL "") - set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${LEGION_ROOT}/share/Legion/cmake) - endif() - find_package(Legion REQUIRED) - get_target_property(LEGION_INCLUDE_DIRS Legion::RealmRuntime INTERFACE_INCLUDE_DIRECTORIES) - string(REGEX REPLACE "/include" "" LEGION_ROOT_TMP ${LEGION_INCLUDE_DIRS}) - if("${LEGION_ROOT}" STREQUAL "") - set(LEGION_ROOT ${LEGION_ROOT_TMP}) - else() - if(NOT "${LEGION_ROOT}" STREQUAL ${LEGION_ROOT_TMP}) - message( FATAL_ERROR "LEGION_ROOT is not set correctly ${LEGION_ROOT} ${LEGION_ROOT_TMP}") - endif() - endif() - message(STATUS "Use external Legion cmake found: ${LEGION_ROOT_TMP}") - message(STATUS "Use external Legion: ${LEGION_ROOT}") - set(LEGION_LIBRARY Legion::Legion) -else() - # Check availability of precompiled Legion library - set(LEGION_URL "") - if((FF_USE_PREBUILT_LEGION OR FF_USE_ALL_PREBUILT_LIBRARIES) AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "x86_64" AND - FF_USE_PYTHON AND NOT "${FF_LEGION_NETWORKS}" STREQUAL "gasnet" AND FF_MAX_DIM EQUAL 5) - # For now, reusing pre-compiled Legion library only works when the Python library on the target machine - # is stored at the path `/opt/conda/lib/libpython3.10.so`. Here, we check if this is the case. - find_package(PythonInterp) - find_package(PythonLibs) - if(PYTHON_LIBRARIES STREQUAL "/opt/conda/lib/libpython3.10.so") - if(LINUX_VERSION MATCHES "20.04") - if (FF_GPU_BACKEND STREQUAL "cuda") - if (CUDA_VERSION VERSION_EQUAL "11.0") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-20.04_11.0.3.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.1") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-20.04_11.1.1.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.2") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-20.04_11.2.2.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.3") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-20.04_11.3.1.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.4") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-20.04_11.4.3.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.5") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-20.04_11.5.2.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.6") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-20.04_11.6.2.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.7") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-20.04_11.7.0.tar.gz") - endif() - elseif(FF_GPU_BACKEND STREQUAL "hip_rocm") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-20.04_hip_rocm.tar.gz") - endif() - elseif(LINUX_VERSION MATCHES "18.04") - if (FF_GPU_BACKEND STREQUAL "cuda") - if (CUDA_VERSION VERSION_EQUAL "10.1") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-18.04_10.1.243.tar.gz") - elseif (CUDA_VERSION VERSION_EQUAL "10.2") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-18.04_10.2.89.tar.gz") - elseif (CUDA_VERSION VERSION_EQUAL "11.0") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-18.04_11.0.3.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.1") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-18.04_11.1.1.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.2") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-18.04_11.2.2.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.3") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-18.04_11.3.1.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.4") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-18.04_11.4.3.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.5") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-18.04_11.5.2.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.6") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-18.04_11.6.2.tar.gz") - elseif(CUDA_VERSION VERSION_EQUAL "11.7") - set(LEGION_URL "https://github.com/flexflow/flexflow-third-party/releases/latest/download/legion_ubuntu-18.04_11.7.0.tar.gz") - endif() - endif() - endif() - endif() - endif() - - if(LEGION_URL) - # Download and import pre-compiled Legion library - message(STATUS "Using pre-compiled Legion library") - message(STATUS "LEGION_URL: ${LEGION_URL}") - set(LEGION_NAME legion) - set(LEGION_LIBRARY legion) - set(REALM_LIBRARY realm) - - include(FetchContent) - FetchContent_Declare(${LEGION_NAME} - URL ${LEGION_URL} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - ) - FetchContent_GetProperties(${LEGION_NAME}) - if(NOT ${LEGION_NAME}_POPULATED) - FetchContent_Populate(${LEGION_NAME}) - endif() - - set(LEGION_FOLDER_PATH ${${LEGION_NAME}_SOURCE_DIR}/export/${LEGION_NAME}) - SET(LEGION_INCLUDE_DIR ${LEGION_FOLDER_PATH}/include) - SET(LEGION_DEF_DIR ${LEGION_INCLUDE_DIR}) - SET(LEGION_BIN_DIR ${LEGION_FOLDER_PATH}/bin/) - SET(LEGION_LIB_DIR ${LEGION_FOLDER_PATH}/lib) - SET(LEGION_SHARE_DIR ${LEGION_FOLDER_PATH}/share/) - message(STATUS "Legion library path: ${LEGION_FOLDER_PATH}") - - add_library(${LEGION_LIBRARY} SHARED IMPORTED) - add_library(${REALM_LIBRARY} SHARED IMPORTED) - set_target_properties(${LEGION_LIBRARY} PROPERTIES IMPORTED_LOCATION ${LEGION_LIB_DIR}/liblegion${LIBEXT}) - set_target_properties(${REALM_LIBRARY} PROPERTIES IMPORTED_LOCATION ${LEGION_LIB_DIR}/librealm${LIBEXT}) - - list(APPEND FLEXFLOW_INCLUDE_DIRS - ${LEGION_INCLUDE_DIR} - ${LEGION_INCLUDE_DIR}/hip_cuda_compat - ${LEGION_INCLUDE_DIR}/legion - ${LEGION_INCLUDE_DIR}/mappers - ${LEGION_INCLUDE_DIR}/mathtypes - ${LEGION_INCLUDE_DIR}/realm - ) - - install(DIRECTORY ${LEGION_SHARE_DIR} DESTINATION share) - install(DIRECTORY ${LEGION_BIN_DIR} DESTINATION bin) - install(DIRECTORY ${LEGION_LIB_DIR}/ DESTINATION lib) - - else() - # Build Legion from source - message(STATUS "Building Legion from source") - if(FF_USE_PYTHON) - set(Legion_USE_Python ON CACHE BOOL "enable Legion_USE_Python") - endif() - if("${FF_LEGION_NETWORKS}" STREQUAL "gasnet") - set(Legion_EMBED_GASNet ON CACHE BOOL "Use embed GASNet") - set(Legion_EMBED_GASNet_VERSION "GASNet-2022.3.0" CACHE STRING "GASNet version") - set(Legion_NETWORKS "gasnetex" CACHE STRING "GASNet conduit") - set(GASNet_CONDUIT ${FF_GASNET_CONDUIT}) - endif() - message(STATUS "GASNET ROOT: $ENV{GASNet_ROOT_DIR}") - set(Legion_MAX_DIM ${FF_MAX_DIM} CACHE STRING "Maximum number of dimensions") - if ((FF_LEGION_NETWORKS STREQUAL "gasnet" AND FF_GASNET_CONDUIT STREQUAL "ucx") OR FF_LEGION_NETWORKS STREQUAL "ucx") - include(ucx) - else() - message(STATUS "FF_GASNET_CONDUIT: ${FF_GASNET_CONDUIT}") - endif() - if (FF_GPU_BACKEND STREQUAL "cuda") - set(Legion_USE_CUDA ON CACHE BOOL "enable Legion_USE_CUDA" FORCE) - set(Legion_CUDA_ARCH ${FF_CUDA_ARCH} CACHE STRING "Legion CUDA ARCH" FORCE) - elseif (FF_GPU_BACKEND STREQUAL "hip_cuda" OR FF_GPU_BACKEND STREQUAL "hip_rocm") - set(Legion_USE_HIP ON CACHE BOOL "enable Legion_USE_HIP" FORCE) - if (FF_GPU_BACKEND STREQUAL "hip_cuda") - set(Legion_HIP_TARGET "CUDA" CACHE STRING "Legion_HIP_TARGET CUDA" FORCE) - elseif(FF_GPU_BACKEND STREQUAL "hip_rocm") - set(Legion_HIP_TARGET "ROCM" CACHE STRING "Legion HIP_TARGET ROCM" FORCE) - endif() - endif() - add_subdirectory(deps/legion) - set(LEGION_LIBRARY Legion) - - set(LEGION_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/deps/legion/runtime) - set(LEGION_DEF_DIR ${CMAKE_BINARY_DIR}/deps/legion/runtime) - endif() -endif() - -alias_library(legion "${LEGION_LIBRARY}") diff --git a/cmake/realm.cmake b/cmake/realm.cmake new file mode 100644 index 0000000000..2d915f1211 --- /dev/null +++ b/cmake/realm.cmake @@ -0,0 +1 @@ +find_package(Realm REQUIRED) diff --git a/cmake/ucx.cmake b/cmake/ucx.cmake deleted file mode 100644 index 813ab0e595..0000000000 --- a/cmake/ucx.cmake +++ /dev/null @@ -1,87 +0,0 @@ -if("${FF_UCX_URL}" STREQUAL "") - set(UCX_URL "https://github.com/openucx/ucx/releases/download/v1.14.0-rc1/ucx-1.14.0.tar.gz") -else() - set(UCX_URL "${FF_UCX_URL}") -endif() - -set(UCX_DIR ${CMAKE_CURRENT_BINARY_DIR}/ucx) -get_filename_component(UCX_COMPRESSED_FILE_NAME "${UCX_URL}" NAME) -# message(STATUS "UCX_URL: ${UCX_URL}") -# message(STATUS "UCX_COMPRESSED_FILE_NAME: ${UCX_COMPRESSED_FILE_NAME}") -set(UCX_COMPRESSED_FILE_PATH "${CMAKE_CURRENT_BINARY_DIR}/${UCX_COMPRESSED_FILE_NAME}") -set(UCX_BUILD_NEEDED OFF) -set(UCX_CONFIG_FILE ${UCX_DIR}/config.txt) -set(UCX_BUILD_OUTPUT ${UCX_DIR}/build.log) - -if(EXISTS ${UCX_CONFIG_FILE}) - file(READ ${UCX_CONFIG_FILE} PREV_UCX_CONFIG) - # message(STATUS "PREV_UCX_CONFIG: ${PREV_UCX_CONFIG}") - if("${UCX_URL}" STREQUAL "${PREV_UCX_CONFIG}") - # configs match - no build needed - set(UCX_BUILD_NEEDED OFF) - else() - message(STATUS "UCX configuration has changed - rebuilding...") - set(UCX_BUILD_NEEDED ON) - endif() -else() - message(STATUS "Configuring and building UCX...") - set(UCX_BUILD_NEEDED ON) -endif() - -if(UCX_BUILD_NEEDED) - if(NOT EXISTS "${UCX_COMPRESSED_FILE_PATH}") - message(STATUS "Downloading openucx/ucx from: ${UCX_URL}") - file( - DOWNLOAD - "${UCX_URL}" "${UCX_COMPRESSED_FILE_PATH}" - SHOW_PROGRESS - STATUS status - LOG log - ) - - list(GET status 0 status_code) - list(GET status 1 status_string) - - if(status_code EQUAL 0) - message(STATUS "Downloading... done") - else() - message(FATAL_ERROR "error: downloading '${UCX_URL}' failed - status_code: ${status_code} - status_string: ${status_string} - log: - --- LOG BEGIN --- - ${log} - --- LOG END ---" - ) - endif() - else() - message(STATUS "${UCX_COMPRESSED_FILE_NAME} already exists") - endif() - - execute_process(COMMAND mkdir -p ${UCX_DIR}) - execute_process(COMMAND tar xzf ${UCX_COMPRESSED_FILE_PATH} -C ${UCX_DIR} --strip-components 1) - message(STATUS "Building UCX...") - execute_process( - COMMAND sh -c "cd ${UCX_DIR} && ${UCX_DIR}/contrib/configure-release --prefix=${UCX_DIR}/install --enable-mt && make -j8 && make install" - RESULT_VARIABLE UCX_BUILD_STATUS - OUTPUT_FILE ${UCX_BUILD_OUTPUT} - ERROR_FILE ${UCX_BUILD_OUTPUT} - ) - - if(UCX_BUILD_STATUS) - message(FATAL_ERROR "UCX build result = ${UCX_BUILD_STATUS} - see ${UCX_BUILD_OUTPUT} for more details") - endif() - - # Currently, we use default build configurations for UCX and therefore only save URL as configuration settings - file(WRITE ${UCX_CONFIG_FILE} "${UCX_URL}") -endif() - -if (FF_LEGION_NETWORKS STREQUAL "gasnet" AND FF_GASNET_CONDUIT STREQUAL "ucx") - set(ENV{UCX_HOME} "${UCX_DIR}/install") -endif() - -if (FF_LEGION_NETWORKS STREQUAL "ucx") - set(ucx_DIR ${UCX_DIR}/cmake) - set(ENV{Legion_NETWORKS} "ucx") - message(STATUS "Legion_NETWORKS: $ENV{Legion_NETWORKS}") -endif()