MPI optional

CMakeLists.txt

@@ -30,11 +30,6 @@ add_subdirectory(externals)
 add_subdirectory(lib)
 add_subdirectory(scripts)
 
-if(PROJECT_IS_TOP_LEVEL)
-  enable_testing()
-  add_subdirectory(test)
-endif()
-
 write_basic_package_version_file(
   ${CMAKE_CURRENT_BINARY_DIR}/cusanConfigVersion.cmake
   VERSION ${PROJECT_VERSION}
@@ -68,3 +63,8 @@ if(PROJECT_IS_TOP_LEVEL)
       QUIET_ON_EMPTY
     )
 endif()
+
+if(PROJECT_IS_TOP_LEVEL)
+  enable_testing()
+  add_subdirectory(test)
+endif()

cmake/cusanToolchain.cmake

@@ -26,7 +26,7 @@ string(COMPARE EQUAL "${CMAKE_SOURCE_DIR}" "${PROJECT_SOURCE_DIR}"
 )
 
 find_package(CUDAToolkit REQUIRED)
-find_package(MPI REQUIRED)
+find_package(MPI)
 
 option(CUSAN_TEST_CONFIGURE_IDE "Add targets for tests to help the IDE with completion etc." ON)
 mark_as_advanced(CUSAN_TEST_CONFIGURE_IDE)

lib/runtime/CMakeLists.txt

@@ -1,67 +1,69 @@
-find_package(MPI)
-add_library(cusan_mpi_interceptor SHARED MPIInterception.cpp)
-target_link_libraries(cusan_mpi_interceptor
-  PRIVATE MPI::MPI_CXX
-)
-set_target_properties(
-  cusan_mpi_interceptor
-  PROPERTIES
-  OUTPUT_NAME "CusanMPIInterceptor"
-  EXPORT_NAME "MPIInterceptor"
-)
-add_library(cusan::MPI_Interceptor ALIAS cusan_mpi_interceptor)
-target_compile_features(cusan_mpi_interceptor PUBLIC cxx_std_17)
-target_compile_definitions(
-  cusan_mpi_interceptor
-  PRIVATE CUSAN_LOG_LEVEL=${CUSAN_LOG_LEVEL_RT}
-  LLVM_VERSION_MAJOR=${LLVM_VERSION_MAJOR}
-   $<$<BOOL:${CUSAN_FIBERPOOL}>:CUSAN_FIBERPOOL=1>
-   $<$<BOOL:${CUSAN_TYPEART}>:CUSAN_TYPEART=1>
-   $<$<BOOL:${CUSAN_SOFTCOUNTER}>:CUSAN_SOFTCOUNTER>
-)
-
-target_include_directories(cusan_mpi_interceptor ${warning_guard}
-  PUBLIC $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  PRIVATE $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/lib>
-)
-
-target_include_directories(cusan_mpi_interceptor
-  SYSTEM
-  PRIVATE
-  ${LLVM_INCLUDE_DIRS}
-)
-
-target_link_libraries(cusan_mpi_interceptor PRIVATE LLVMSupport)
-
-if(CUSAN_FIBERPOOL)
-  target_link_libraries(cusan_mpi_interceptor PUBLIC cusan::fiberpool)
-  set_target_properties(cusan_mpi_interceptor PROPERTIES
-        BUILD_WITH_INSTALL_RPATH FALSE
-        LINK_FLAGS "-Wl,-rpath,${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+if (MPI_FOUND)
+  add_library(cusan_mpi_interceptor SHARED MPIInterception.cpp)
+  target_link_libraries(cusan_mpi_interceptor
+    PRIVATE MPI::MPI_CXX
+  )
+  set_target_properties(
+    cusan_mpi_interceptor
+    PROPERTIES
+    OUTPUT_NAME "CusanMPIInterceptor"
+    EXPORT_NAME "MPIInterceptor"
+  )
+  add_library(cusan::MPI_Interceptor ALIAS cusan_mpi_interceptor)
+
+  target_compile_features(cusan_mpi_interceptor PUBLIC cxx_std_17)
+  target_compile_definitions(
+    cusan_mpi_interceptor
+    PRIVATE CUSAN_LOG_LEVEL=${CUSAN_LOG_LEVEL_RT}
+    LLVM_VERSION_MAJOR=${LLVM_VERSION_MAJOR}
+    $<$<BOOL:${CUSAN_FIBERPOOL}>:CUSAN_FIBERPOOL=1>
+    $<$<BOOL:${CUSAN_TYPEART}>:CUSAN_TYPEART=1>
+    $<$<BOOL:${CUSAN_SOFTCOUNTER}>:CUSAN_SOFTCOUNTER>
+  )
+
+  target_include_directories(cusan_mpi_interceptor ${warning_guard}
+    PUBLIC $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+    PRIVATE $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/lib>
+  )
+
+  target_include_directories(cusan_mpi_interceptor
+    SYSTEM
+    PRIVATE
+    ${LLVM_INCLUDE_DIRS}
+  )
+
+  target_link_libraries(cusan_mpi_interceptor PRIVATE LLVMSupport)
+
+  if(CUSAN_FIBERPOOL)
+    target_link_libraries(cusan_mpi_interceptor PUBLIC cusan::fiberpool)
+    set_target_properties(cusan_mpi_interceptor PROPERTIES
+          BUILD_WITH_INSTALL_RPATH FALSE
+          LINK_FLAGS "-Wl,-rpath,${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+  endif()
+
+  set(CONFIG_NAME cusanMPIInterceptor)
+  set(TARGETS_EXPORT_NAME ${CONFIG_NAME}Targets)
+
+  install(
+    TARGETS cusan_mpi_interceptor
+    EXPORT ${TARGETS_EXPORT_NAME}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  )
+
+  install(
+    EXPORT ${TARGETS_EXPORT_NAME}
+    NAMESPACE cusan::
+    DESTINATION ${CUSAN_INSTALL_CONFIGDIR}
+  )
+
+  export(
+    EXPORT ${TARGETS_EXPORT_NAME}
+    FILE ${CMAKE_BINARY_DIR}/${TARGETS_EXPORT_NAME}.cmake
+    NAMESPACE cusan::
+  )
 endif()
 
-set(CONFIG_NAME cusanMPIInterceptor)
-set(TARGETS_EXPORT_NAME ${CONFIG_NAME}Targets)
-
-install(
-  TARGETS cusan_mpi_interceptor
-  EXPORT ${TARGETS_EXPORT_NAME}
-  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-)
-
-install(
-  EXPORT ${TARGETS_EXPORT_NAME}
-  NAMESPACE cusan::
-  DESTINATION ${CUSAN_INSTALL_CONFIGDIR}
-)
-
-export(
-  EXPORT ${TARGETS_EXPORT_NAME}
-  FILE ${CMAKE_BINARY_DIR}/${TARGETS_EXPORT_NAME}.cmake
-  NAMESPACE cusan::
-)
-
 add_library(cusan_Runtime SHARED CusanRuntime.cpp CusanRuntime_cudaSpecific.cpp)
 set_target_properties(
   cusan_Runtime

lib/runtime/TSanInterface.h

@@ -1,9 +1,11 @@
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wformat"
-#pragma GCC diagnostic ignored "-Wmacro-redefined"
 #pragma GCC diagnostic ignored "-Wunused-parameter"
-#pragma GCC diagnostic ignored "-Wformat-pedantic"
 #pragma GCC diagnostic ignored "-Wcast-qual"
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wmacro-redefined"
+#pragma clang diagnostic ignored "-Wformat-pedantic"
+#endif
 
 #include "TSan_External.h"
 

scripts/CMakeLists.txt

@@ -175,7 +175,6 @@ if(NOT CUSAN_TYPEART)
   )
 endif()
 
-find_package(MPI)
 if(MPI_C_FOUND)
   configure_cusan_script(
     ${CUSAN_WRAPPER} cusan-mpicc${CMAKE_DEBUG_POSTFIX}

test/CMakeLists.txt

@@ -36,6 +36,7 @@ function(configure_cusan_lit_site_cfg input output)
 
   pythonize_bool(${CUSAN_TEST_WORKAROUNDS} CUSAN_TEST_WORKAROUND)
   pythonize_bool(${CUSAN_TYPEART} CUSAN_HAS_TYPEART)
+  pythonize_bool(${MPI_FOUND} CUSAN_HAS_MPI)
 
   cusan_target_generate_file(${input} ${output})
 endfunction()
@@ -90,16 +91,32 @@ function(cusan_add_lit_target)
   endforeach()
 endfunction()
 
+
+if(NOT MPI_FOUND)
+  # mock target to make test generation work
+  add_library(cusan_mpi_interceptor_mock SHARED IMPORTED)
+  set_target_properties(
+    cusan_mpi_interceptor_mock
+    PROPERTIES
+    OUTPUT_NAME "CusanMPIInterceptorPlaceholder"
+    IMPORTED_LOCATION "${PROJECT_SOURCE_DIR}/lib/runtime"
+  )
+  set(_cusan_interceptor_target cusan_mpi_interceptor_mock)
+  add_library(cusan::MPI_Interceptor ALIAS cusan_mpi_interceptor_mock)
+endif()
+
 set(CUSAN_TEST_DEPENDS
   cusan::TransformPass
-  cusan::MPI_Interceptor
   cusan::Runtime
   cusan::Analysis
-
 )
 
+if(FOUND_MPI)
+  list(APPEND CUSAN_TEST_DEPENDS cusan::MPI_Interceptor)
+endif()
+
 if(CUSAN_TYPEART)
-list(APPEND CUSAN_TEST_DEPENDS typeart::Runtime typeart::TransformPass)
+  list(APPEND CUSAN_TEST_DEPENDS typeart::Runtime typeart::TransformPass)
 endif()
 
 set(CUSAN_SUITES

test/kernel_analysis/03_struct_write.c

@@ -22,6 +22,8 @@
 // CHECK-NEXT: subarg: {{.*}}ptr: 0, rw: ReadWrite
 // CHECK-NOT: Handling Arg:
 
+// REQUIRES: mpi
+
 // clang-format on
 
 #include "../support/gpu_mpi.h"

test/kernel_analysis/08_big_struct_write.c

@@ -22,6 +22,8 @@
 // CHECK-NEXT: subarg: {{.*}}ptr: 0, rw: ReadWrite
 // CHECK-NOT: Handling Arg:
 
+// REQUIRES: mpi
+
 // clang-format on
 
 #include "../support/gpu_mpi.h"

test/lit.cfg

@@ -45,6 +45,8 @@ if config.cusan_typeart:
 
 if config.mpiexec:
   config.available_features.add('mpiexec')
+if config.cusan_mpi:
+  config.available_features.add('mpi')
 
 config.substitutions.append(("%clang-cpp", clang_cpp))
 config.substitutions.append(("%clang-cc", clang_cc))

test/lit.site.cfg.in

@@ -31,6 +31,7 @@ config.cusan_pass = "$<TARGET_FILE_NAME:cusan::TransformPass>"
 config.cusan_mpi_interceptor = "$<TARGET_FILE:cusan::MPI_Interceptor>"
 config.cusan_use_workarounds = @CUSAN_TEST_WORKAROUND@
 config.cusan_typeart = @CUSAN_HAS_TYPEART@
+config.cusan_mpi = @CUSAN_HAS_MPI@
 
 # Let the main config do the real work.
 config.loaded_site_config = True

test/pass/03_cuda_to_mpi.c

@@ -2,11 +2,11 @@
 // RUN: %rm-file %t.yaml 
 // RUN: %wrapper-mpicc %clang-pass-only-args --cusan-kernel-data=%t.yaml -x cuda --cuda-gpu-arch=sm_72 %s 2>&1 | %filecheck %s  -DFILENAME=%s --allow-empty --check-prefix CHECK-LLVM-IR
 
-
-
 // CHECK-LLVM-IR: {{call|invoke}} i32 @cudaMemcpy({{i8\*|ptr}} {{.*}}[[target:%[0-9a-z]+]], {{i8\*|ptr}} {{.*}}[[from:%[0-9a-z]+]], 
 // CHECK-LLVM-IR: {{call|invoke}} void @_cusan_memcpy({{i8\*|ptr}} {{.*}}[[target]], {{i8\*|ptr}} {{.*}}[[from]],
 
+// REQUIRES: mpi
+
 // clang-format on
 
 #include "../support/gpu_mpi.h"

test/pass/04_mpi_to_cuda.c

@@ -8,6 +8,8 @@
 // CHECK-LLVM-IR: {{(call|invoke)}} i32 @cudaMemcpy({{i8\*|ptr}} {{.*}}[[target:%[0-9a-z]+]], {{i8\*|ptr}} {{.*}}[[from:%[0-9a-z]+]],
 // CHECK-LLVM-IR: {{call|invoke}} void @_cusan_memcpy({{i8\*|ptr}} {{.*}}[[target]], {{i8\*|ptr}} {{.*}}[[from]],
 
+// REQUIRES: mpi
+
 // clang-format on
 
 #include "../support/gpu_mpi.h"

test/pass/05_cuda_to_mpi_stream.c

@@ -10,6 +10,8 @@
 // CHECK-LLVM-IR: {{(call|invoke)}} i32 @cudaStreamSynchronize
 // CHECK-LLVM-IR: {{call|invoke}} void @_cusan_sync_stream
 
+// REQUIRES: mpi
+
 // clang-format on
 
 #include "../support/gpu_mpi.h"

test/pass/06_cuda_to_mpi_event.c

@@ -10,6 +10,8 @@
 // CHECK-LLVM-IR: {{(call|invoke)}} i32 @cudaMemcpy({{i8\*|ptr}} {{.*}}[[target:%[0-9a-z]+]], {{i8\*|ptr}} {{.*}}[[from:%[0-9a-z]+]], 
 // CHECK-LLVM-IR: {{call|invoke}} void @_cusan_memcpy({{i8\*|ptr}} {{.*}}[[target]], {{i8\*|ptr}} {{.*}}[[from]],
 
+// REQUIRES: mpi
+
 // clang-format on
 
 #include "../support/gpu_mpi.h"

test/pass/07_cuda_to_mpi_read.c

@@ -11,6 +11,8 @@
 // CHECK-LLVM-IR: {{(call|invoke)}} i32 @cudaMemcpy({{i8\*|ptr}} {{.*}}[[target:%[0-9a-z]+]], {{i8\*|ptr}} {{.*}}[[from:%[0-9a-z]+]],
 // CHECK-LLVM-IR: {{(call|invoke)}} void @_cusan_memcpy({{i8\*|ptr}} {{.*}}[[target]], {{i8\*|ptr}} {{.*}}[[from]],
 
+// REQUIRES: mpi
+
 // clang-format on
 
 #include "../support/gpu_mpi.h"

test/pass/08_cudamemcpy_to_mpi.c

@@ -15,6 +15,8 @@
 // CHECK-LLVM-IR: {{(call|invoke)}} i32 @cudaStreamSynchronize
 // CHECK-LLVM-IR: {{call|invoke}} void @_cusan_sync_stream
 
+// REQUIRES: mpi
+
 // clang-format on
 
 // Tsan sometimes crashes with this test it seems

test/pass/11_cuda_to_mpi_struct_of_buff.c

@@ -3,6 +3,8 @@
 
 // RUN: %wrapper-mpicc %clang-pass-only-args --cusan-kernel-data=%t.yaml -x cuda --cuda-gpu-arch=sm_72 %s 2>&1 | %filecheck %s  -DFILENAME=%s --allow-empty --check-prefix CHECK-LLVM-IR
 
+// REQUIRES: mpi
+
 // CHECK-LLVM-IR: {{call|invoke}} i32 @cudaStreamCreate
 // CHECK-LLVM-IR: {{call|invoke}} void @_cusan_create_stream
 // CHECK-LLVM-IR: {{call|invoke}} i32 @cudaStreamCreate

test/pass/18_cuda_to_mpi_event_query_busy_loop.c

@@ -3,6 +3,7 @@
 
 // RUN: %wrapper-mpicc %clang-pass-only-args --cusan-kernel-data=%t.yaml -x cuda --cuda-gpu-arch=sm_72 %s 2>&1 | %filecheck %s  -DFILENAME=%s --allow-empty --check-prefix CHECK-LLVM-IR
 
+// REQUIRES: mpi
 
 // CHECK-LLVM-IR: {{(call|invoke)}} i32 @cudaStreamCreate
 // CHECK-LLVM-IR: {{(call|invoke)}} void @_cusan_create_stream

test/pass/19_cuda_to_mpi_send_cudaMemcpyAsyncH2H_implicit_sync.c

@@ -3,6 +3,7 @@
 
 // RUN: %wrapper-mpicc %clang-pass-only-args --cusan-kernel-data=%t.yaml -x cuda --cuda-gpu-arch=sm_72 %s 2>&1 | %filecheck %s  -DFILENAME=%s --allow-empty --check-prefix CHECK-LLVM-IR
 
+// REQUIRES: mpi
 
 // CHECK-LLVM-IR: {{(call|invoke)}} i32 @cudaStreamCreate
 // CHECK-LLVM-IR: {{(call|invoke)}} void @_cusan_create_stream

test/pass/20_cuda_to_mpi_send_ds_sync_w_r.c

@@ -3,6 +3,8 @@
 
 // RUN: %wrapper-mpicc %clang-pass-only-args --cusan-kernel-data=%t.yaml -x cuda --cuda-gpu-arch=sm_72 %s 2>&1 | %filecheck %s  -DFILENAME=%s --allow-empty --check-prefix CHECK-LLVM-IR
 
+// REQUIRES: mpi
+
 // CHECK-LLVM-IR: {{call|invoke}} i32 @cudaStreamCreate
 // CHECK-LLVM-IR: {{call|invoke}} void @_cusan_create_stream
 // CHECK-LLVM-IR: {{call|invoke}} i32 @cudaMemset

test/pass/21_chunked_streams_example.c

@@ -3,6 +3,8 @@
 
 // RUN: %wrapper-mpicc %clang-pass-only-args --cusan-kernel-data=%t.yaml -x cuda --cuda-gpu-arch=sm_72 %s 2>&1 | %filecheck %s  -DFILENAME=%s --allow-empty --check-prefix CHECK-LLVM-IR
 
+// REQUIRES: mpi
+
 // CHECK-LLVM-IR: {{call|invoke}} i32 @cudaStreamCreate
 // CHECK-LLVM-IR: {{call|invoke}} void @_cusan_create_stream
 // CHECK-LLVM-IR: {{call|invoke}} i32 @cudaStreamCreate

test/pass/22_cuda_to_mpi_partial_buff_write.c

@@ -3,6 +3,8 @@
 
 // RUN: %wrapper-mpicc %clang-pass-only-args --cusan-kernel-data=%t.yaml -x cuda --cuda-gpu-arch=sm_72 %s 2>&1 | %filecheck %s  -DFILENAME=%s --allow-empty --check-prefix CHECK-LLVM-IR
 
+// REQUIRES: mpi
+
 // CHECK-LLVM-IR: {{call|invoke}} i32 @cudaStreamCreate
 // CHECK-LLVM-IR: {{call|invoke}} void @_cusan_create_stream
 // CHECK-LLVM-IR: {{call|invoke}} i32 @cudaStreamSynchronize

test/pass/26_malloc_pitch.c

@@ -6,6 +6,8 @@
 // {{.*}}[[from:%[0-9a-z]+]], CHECK-LLVM-IR: {{call|invoke}} void @_cusan_memcpy({{i8\*|ptr}} {{.*}}[[target]],
 // {{i8\*|ptr}} {{.*}}[[from]],
 
+// REQUIRES: mpi
+
 // clang-format on
 
 #include "../support/gpu_mpi.h"

test/pass/29_tsan_cuda_to_mpi.c

@@ -3,6 +3,7 @@
 
 // RUN: %wrapper-mpicc %clang-pass-only-args --cusan-kernel-data=%t.yaml -x cuda --cuda-gpu-arch=sm_72 %s 2>&1 | %filecheck %s  -DFILENAME=%s --allow-empty --check-prefix CHECK-LLVM-IR
 
+// REQUIRES: mpi
 
 // CHECK-LLVM-IR: {{(call|invoke)}} i32 @cudaMemcpy
 // CHECK-LLVM-IR: {{(call|invoke)}} void @_cusan_memcpy

test/pass/30_tsan_annotate_cuda_to_mpi.c

@@ -8,6 +8,8 @@
 // CHECK-LLVM-IR: {{(call|invoke)}} i32 @cudaFree
 // CHECK-LLVM-IR: {{(call|invoke)}} void @_cusan_device_free
 
+// REQUIRES: mpi
+
 // clang-format on
 
 #include "../support/gpu_mpi.h"