Use SST_MPI_Comm_spawn_multiple in Airel PIN backend

configure.ac

@@ -48,8 +48,6 @@ LTDL_INIT([recursive])
 
 AC_CACHE_SAVE
 
-SST_CHECK_ARIEL_MPI([], [AC_MSG_ERROR([Can't enable Ariel MPI])])
-
 SST_CORE_CHECK_INSTALL()
 
 SST_CHECK_OSX()

src/sst/elements/ariel/Makefile.am

@@ -1,7 +1,4 @@
 SUBDIRS = api
-if SST_USE_ARIEL_MPI
-SUBDIRS += mpi
-endif
 
 AM_CPPFLAGS += \
 	-I$(top_srcdir)/src
@@ -57,8 +54,6 @@ libariel_la_SOURCES = \
 EXTRA_DIST = \
 	api/arielapi.c \
 	api/arielapi.h \
-	mpi/mpilauncher.cc \
-	mpi/fakepin.cc \
 	frontend/pin3/fesimple.cc \
 	frontend/simple/fesimple.cc \
 	frontend/simple/examples/multicore.py \
@@ -123,7 +118,6 @@ libariel_la_CPPFLAGS = \
 	-DARIEL_TRACE_LIB=$(libdir)/sst \
 	-DARIEL_TOOL_DIR="$(libexecdir)" \
 	-DPINTOOL_EXECUTABLE="$(PINTOOL_RUNTIME)" \
-	-DMPILAUNCHER_EXECUTABLE="$(CURDIR)/mpi/mpilauncher" \
 	$(AM_CPPFLAGS) \
 	$(CPPFLAGS)
 

src/sst/elements/ariel/api/Makefile.am

@@ -1,15 +1,3 @@
-if SST_USE_ARIEL_MPI
-CC_LOCAL = $(ARIEL_MPICC)
-CFLAGS_LOCAL = -fopenmp $(ARIEL_MPI_CFLAGS) -DENABLE_ARIEL_MPI=1
-else
-CC_LOCAL = $(CC)
-CFLAGS_LOCAL =
-endif
-
-AM_CPPFLAGS += \
-	$(MPI_CPPFLAGS) \
-	-I$(top_srcdir)/src
-
 lib_LTLIBRARIES = libarielapi.la
 
 libarielapi_la_SOURCES = \

src/sst/elements/ariel/api/arielapi.c

@@ -17,9 +17,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <inttypes.h>
-#ifdef ENABLE_ARIEL_MPI
-#include <mpi.h>
-#endif
 
 /* These definitions are replaced during simulation */
 
@@ -40,6 +37,7 @@ void ariel_fence() {
 }
 
 uint64_t ariel_cycles() {
+    printf("ARIEL: ariel_cycles called in Ariel API.\n");
     return 0;
 }
 
@@ -51,59 +49,10 @@ void ariel_malloc_flag(int64_t id, int count, int level) {
     printf("ARIEL: flagging next %d mallocs at id %" PRId64 "\n", count, id);
 }
 
-// To ensure that the Pintool (fesimple.cc) numbers our application's OpenMP threads
-// from 0..N-1, we need to run an OpenMP parallel region before calling MPI Init.
-// Otherwise, some MPI threads which aren't used for our application will be
-// numbered 1 and 2.
-void omp_parallel_region() {
-    volatile int x = 0;
-#if defined(_OPENMP)
-#pragma omp parallel
-    {
-#pragma omp critical
-        {
-            x += 1;
-        }
-    }
-#else
-    printf("ERROR: arielapi.c: libarielapi was compiled without OpenMP enabled\n");
-    exit(1);
-#endif
+void ariel_output_stats_begin_region(const char *name) {
+    printf("ARIEL: Request to print statistics and begin region:%s\n", name);
 }
 
-// This function only exists to get mapped by the frontend. It should only be called
-// from MPI_Init or MPI_Init_thread to allow the frontend to distinguish between our
-// custom versions of of those functions and the normal MPI library's versions.
-void _api_mpi_init() {
-    printf("notifying fesimple\n");
-}
-
-// Custom version of MPI_Init. We override the normal version in order to call an
-// OpenMP parallel region to ensure threads are numbered properly by the frontend.
-int MPI_Init(int *argc, char ***argv) {
-#ifdef ENABLE_ARIEL_MPI
-    // Communicate to the frontend that we have replaced the nomal MPI_Init with
-    // the one in the Ariel API
-    _api_mpi_init();
-    omp_parallel_region();
-    return PMPI_Init(argc, argv);
-#else
-    printf("Error: arielapi.c: MPI_Init called in arielapi.c but this file was compiled without MPI.\n");
-    exit(1);
-#endif
-}
-
-// Custom version of MPI_Init_thread. We override the normal verison in order to call an
-// OpenMP parallel region to ensure threads are numbered properly by the frontend.
-int MPI_Init_thread(int *argc, char ***argv, int required, int *provided) {
-#ifdef ENABLE_ARIEL_MPI
-    // Communicate to the frontend that we have replaced the nomal MPI_Init_thread with
-    // the one in the Ariel API
-    _api_mpi_init();
-    omp_parallel_region();
-    return PMPI_Init_thread(argc, argv, required, provided);
-#else
-    printf("Error: arielapi.c: MPI_Init_thread called in arielapi.c but this file was compiled without MPI. Please recompile the API with `CC=mpicc make`123123.\n");
-    exit(1);
-#endif
+void ariel_output_stats_end_region(const char *name) {
+    printf("ARIEL: Request to print statistics and end region:%s\n", name);
 }

src/sst/elements/ariel/api/arielapi.h

@@ -48,6 +48,13 @@ uint64_t ariel_cycles();
 /* Trigger the simulation to output statistics */
 void ariel_output_stats();
 
+/* Trigger the simulation to output statistics along with a region name.
+ * This interface may be changed or removed in the future if named
+ * regions are added to sst-core.
+ */
+void ariel_output_stats_begin_region(const char *name);
+void ariel_output_stats_end_region(const char *name);
+
 /* Control which memory pool (level) the next 'count' allocations encountered should map to
  *
  */

src/sst/elements/ariel/ariel_shmem.h

@@ -55,6 +55,8 @@ enum ArielShmemCmd_t {
     ARIEL_SWITCH_POOL = 110,
     ARIEL_NOOP = 128,
     ARIEL_OUTPUT_STATS = 140,
+    ARIEL_OUTPUT_STATS_BEGIN_REGION = 141,
+    ARIEL_OUTPUT_STATS_END_REGION = 142,
     ARIEL_ISSUE_RTL = 150,
     ARIEL_FLUSHLINE_INSTRUCTION = 154,
     ARIEL_FENCE_INSTRUCTION = 155,

src/sst/elements/ariel/arielcore.cc

@@ -412,6 +412,7 @@ bool ArielCore::refillQueue() {
                             coreID, (uint32_t) coreQ->size(), (uint32_t) maxQLength));
 
         ArielCommand ac;
+        char *region_name;
         const bool avail = tunnel->readMessageNB(coreID, &ac);
 
         if ( !avail ) {
@@ -428,6 +429,18 @@ bool ArielCore::refillQueue() {
                 performGlobalStatisticOutput();
                 break;
 
+            case ARIEL_OUTPUT_STATS_BEGIN_REGION:
+                region_name = (char*)ac.inst.payload;
+                fprintf(stdout, "ARIEL_REGION_BEGIN %s %" PRIu64 "\n", region_name, getCurrentSimTime(getCoreTimeBase().toString()));
+                performGlobalStatisticOutput();
+                break;
+
+            case ARIEL_OUTPUT_STATS_END_REGION:
+                region_name = (char*)ac.inst.payload;
+                fprintf(stdout, "ARIEL_REGION_END %s %" PRIu64 "\n", region_name, getCurrentSimTime(getCoreTimeBase().toString()));
+                performGlobalStatisticOutput();
+                break;
+
             case ARIEL_START_INSTRUCTION:
                 if(ARIEL_INST_SP_FP == ac.inst.instClass) {
                         statFPSPIns->addData(1);

src/sst/elements/ariel/arielcpu.h

@@ -64,10 +64,9 @@ class ArielCPU : public SST::Component {
         {"appstderrappend", "If appstderr is set, set this to 1 to append the file intead of overwriting", "0"},
         {"launchparamcount", "Number of parameters supplied for the launch tool", "0" },
         {"launchparam%(launchparamcount)d", "Set the parameter to the launcher", "" },
-        {"mpimode", "Whether to use <mpilauncher> to to launch <launcher> in order to trace MPI-enabled applications.", "0"},
-        {"mpilauncher", "Specify a launcher to be used for MPI executables in conjuction with <launcher>", STRINGIZE(MPILAUNCHER_EXECUTABLE)},
-        {"mpiranks", "Number of ranks to be launched by <mpilauncher>. Only <mpitracerank> will be traced by <launcher>.", "1" },
-        {"mpitracerank", "Rank to be traced by <launcher>.", "0" },
+        {"mpimode", "Whether to use MPI_Comm_spawn_multiple to to launch app. Use for MPI aplications. SST-Core must be compiled with MPI enabled.", "0"},
+        {"mpiranks", "Number of ranks to be launched. Only <mpitracerank> will be traced by <launcher>.", "1" },
+        {"mpitracerank", "Rank to be traced", "0" },
         {"envparamcount", "Number of environment parameters to supply to the Ariel executable, default=-1 (use SST environment)", "-1"},
         {"envparamname%(envparamcount)d", "Sets the environment parameter name", ""},
         {"envparamval%(envparamcount)d", "Sets the environment parameter value", ""},

src/sst/elements/ariel/arielfrontendcommon.h

@@ -57,10 +57,9 @@ class ArielFrontendCommon : public ArielFrontend {
         {"appstdoutappend", "If appstdout is set, set this to 1 to append the file intead of overwriting", "0"},
         {"appstderr", "Specify a file to use for the program's stderr", ""},
         {"appstderrappend", "If appstderr is set, set this to 1 to append the file intead of overwriting", "0"},
-        {"mpimode", "Whether to use <mpilauncher> to to launch <launcher> in order to trace MPI-enabled applications.", "0"},
-        {"mpilauncher", "Specify a launcher to be used for MPI executables in conjuction with <launcher>", STRINGIZE(MPILAUNCHER_EXECUTABLE)},
-        {"mpiranks", "Number of ranks to be launched by <mpilauncher>. Only <mpitracerank> will be traced by <launcher>.", "1" },
-        {"mpitracerank", "Rank to be traced by <launcher>.", "0" },
+        {"mpimode", "Whether to use MPI_Comm_spawn_multiple to to launch app. Use for MPI aplications. SST-Core must be compiled with MPI enabled.", "0"},
+        {"mpiranks", "Number of ranks to be launched. Only <mpitracerank> will be traced by <launcher>.", "1" },
+        {"mpitracerank", "Rank to be traced", "0" },
         {"appargcount", "Number of arguments to the traced executable", "0"},
         {"apparg%(appargcount)d", "Arguments for the traced executable", ""},
         {"envparamcount", "Number of environment parameters to supply to the Ariel executable, default=-1 (use SST environment)", "-1"},

src/sst/elements/ariel/configure.m4

@@ -11,13 +11,7 @@ AC_DEFUN([SST_ariel_CONFIG], [
   # Use LIBZ
   SST_CHECK_LIBZ()
 
-  AC_SUBST([ARIEL_MPICC])
-  AC_SUBST([ARIEL_MPICXX])
-  AC_SUBST([ARIEL_MPI_CFLAGS])
-  AC_SUBST([ARIEL_MPI_LIBS])
-
   AC_CONFIG_FILES([src/sst/elements/ariel/api/Makefile])
-  AC_CONFIG_FILES([src/sst/elements/ariel/mpi/Makefile])
 
   AS_IF([test "$sst_check_ariel" = "yes"], [$1], [$2])
 ])

src/sst/elements/ariel/frontend/epa/epafrontend.cc

@@ -52,9 +52,14 @@ EPAFrontend::EPAFrontend(ComponentId_t id, Params& params, uint32_t cores,
     // Put together execute_args for fork
     setForkArguments();
     // If mpi, use mpi launcher. Otherwise launch instrumented app
-    app_name = (mpimode == 1) ? mpilauncher : executable;
+    //app_name = (mpimode == 1) ? mpilauncher : executable;
+    app_name = executable;
 
-//    output->verbose(CALL_INFO, 1, 0, "Processing application arguments...\n");
+    if (mpimode == 1) {
+        output->fatal(CALL_INFO, -1, "Error: Running the EPA frontend with MPI is currently not supported. The code needs to be adapted to use SST_MPI_Comm_spawn_multiple() to launch the app.\n");
+    }
+
+    //output->verbose(CALL_INFO, 1, 0, "Processing application arguments...\n");
 
 
     // Remember that the list of arguments must be NULL terminated for execution
@@ -75,7 +80,7 @@ int EPAFrontend::forkChildProcess(const char* app, char** args, std::map<std::st
     char* full_execute_line = (char*) malloc(sizeof(char) * 16384);
 
     memset(full_execute_line, 0, sizeof(char) * 16384);
-// TODO Why is no command appearing?
+    // TODO Why is no command appearing?
     while (NULL != args[next_arg_index]) {
         int copy_char_index = 0;
 
@@ -95,7 +100,6 @@ int EPAFrontend::forkChildProcess(const char* app, char** args, std::map<std::st
     output->verbose(CALL_INFO, 1, 0, "Executing command: %s\n", full_execute_line);
     free(full_execute_line);
 
-
     // Fork this binary, then exec to get around waiting for
     // child to exit.
     pid_t the_child;