Fixes for glibc bug around locale and TLS crashes

config.h.in

@@ -85,6 +85,9 @@
    */
 #undef LT_OBJDIR
 
+/* Defined if GLIBC has locale TLS bug */
+#undef NEWTHREAD_LOCALE_BUG
+
 /* Exclusions for numa optimization */
 #undef NUMA_EXCLUDES
 

configure

@@ -16747,7 +16747,7 @@ fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for glibc dtv allocation bug" >&5
 $as_echo_n "checking for glibc dtv allocation bug... " >&6; }
-if test "x`${SPINDLE_SOURCE_ROOT}/scripts/dtvtest/run_dtvtest.sh ${SPINDLE_SOURCE_ROOT}/scripts/dtvtest ${SPINDLE_BUILD_ROOT}/dtvtest '$CC' '$CFLAGS'`" == "x0"; then
+if ${SPINDLE_SOURCE_ROOT}/scripts/dtvtest/run_dtvtest.sh ${SPINDLE_SOURCE_ROOT}/scripts/dtvtest ${SPINDLE_BUILD_ROOT}/dtvtest '$CC' '$CFLAGS'; then
    DTV_ALLOC_BUG=0
    { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
 $as_echo "no" >&6; }
@@ -16762,6 +16762,24 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for locale tls fault glibc bug" >&5
+$as_echo_n "checking for locale tls fault glibc bug... " >&6; }
+if ${SPINDLE_SOURCE_ROOT}/scripts/locale_tls_fault/tls_locale_fault.sh ${SPINDLE_SOURCE_ROOT}/scripts/locale_tls_fault ${SPINDLE_BUILD_ROOT}/locale_tls_fault $CC; then
+   LOCALE_FAULT_BUG=0
+   { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+else
+   LOCALE_FAULT_BUG=1
+   { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+fi
+
+cat >>confdefs.h <<_ACEOF
+#define NEWTHREAD_LOCALE_BUG $LOCALE_FAULT_BUG
+_ACEOF
+
+
+
 #OS Detection
 # Check whether --enable-bluegene was given.
 if test "${enable_bluegene+set}" = set; then :

configure.common.ac

@@ -87,7 +87,7 @@ fi
 AM_CONDITIONAL([BLD_FLUXPLUGIN], [test "x$ENABLE_FLUX_PLUGIN" = "xtrue"])
 
 AC_MSG_CHECKING([for glibc dtv allocation bug])
-if test "x`${SPINDLE_SOURCE_ROOT}/scripts/dtvtest/run_dtvtest.sh ${SPINDLE_SOURCE_ROOT}/scripts/dtvtest ${SPINDLE_BUILD_ROOT}/dtvtest '$CC' '$CFLAGS'`" == "x0"; then
+if ${SPINDLE_SOURCE_ROOT}/scripts/dtvtest/run_dtvtest.sh ${SPINDLE_SOURCE_ROOT}/scripts/dtvtest ${SPINDLE_BUILD_ROOT}/dtvtest '$CC' '$CFLAGS'; then
    DTV_ALLOC_BUG=0
    AC_MSG_RESULT([no])
 else
@@ -96,6 +96,17 @@ else
 fi
 AC_DEFINE_UNQUOTED([DTV_ALLOCATION_BUG],[$DTV_ALLOC_BUG],[Defined if GLIBC has an allocation bug])
 
+AC_MSG_CHECKING([for locale tls fault glibc bug])
+if ${SPINDLE_SOURCE_ROOT}/scripts/locale_tls_fault/tls_locale_fault.sh ${SPINDLE_SOURCE_ROOT}/scripts/locale_tls_fault ${SPINDLE_BUILD_ROOT}/locale_tls_fault $CC; then
+   LOCALE_FAULT_BUG=0
+   AC_MSG_RESULT([no])
+else
+   LOCALE_FAULT_BUG=1
+   AC_MSG_RESULT([yes])
+fi
+AC_DEFINE_UNQUOTED([NEWTHREAD_LOCALE_BUG],[$LOCALE_FAULT_BUG],[Defined if GLIBC has locale TLS bug])
+
+
 #OS Detection
 AC_ARG_ENABLE(bluegene,
               [AS_HELP_STRING([--enable-bluegene],[Build Spindle for BlueGene/Q])],

scripts/dtvtest/Makefile

@@ -1,5 +1,5 @@
-DEP_NUMS = 01 02 03 04 05 06 07 08 09 10 11 12 13
-DLOPEN_NUMS = 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15
+DEP_NUMS = 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18
+DLOPEN_NUMS = 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18
 
 DEP_LIBS = $(DEP_NUMS:%=libdeptls%.so)
 DEP_LINKS = $(DEP_NUMS:%=-ldeptls%)
@@ -23,9 +23,8 @@ libminaudit.so: $(SRCDIR)/minaudit.c
 	gcc -shared -fPIC -o $@ $<
 
 dtvtest: $(SRCDIR)/dtvtest.c $(DEP_LIBS)
-	gcc -o $@ $(SRCDIR)/dtvtest.c -ldl -lpthread -g -Wl,-rpath,`pwd` -L`pwd` $(DEP_LINKS)
+	gcc -o $@ $(SRCDIR)/dtvtest.c -ldl -lpthread -g -Wl,-rpath,`pwd` -L`pwd` -Wl,--no-as-needed -Wl,--disable-new-dtags $(DEP_LINKS)
 
 run: dtvtest libminaudit.so $(DLOPEN_LIBS)
-	echo LD_AUDIT=./libminaudit.so ./dtvtest $(DLOPEN_NUMS)
 	LD_AUDIT=./libminaudit.so ./dtvtest $(DLOPEN_NUMS)
-	echo $?
+	@echo $$?

scripts/locale_tls_fault/app.c

@@ -0,0 +1,39 @@
+#include <pthread.h>
+#include <stdio.h>
+#include <dlfcn.h>
+#include <stdlib.h>
+#include <signal.h>
+
+void onsig(int sig)
+{
+   exit(-1);
+}
+
+void *thrdrun(void *arg)
+{
+   void *result;
+
+   //Any audit-triggering operation will do
+   result = dlopen("libm.so.6", RTLD_NOW);
+   if (!result) {
+      return NULL;
+   }
+
+   return NULL;
+}
+
+int main(int argc, char *argv[])
+{
+   pthread_t thrd;
+   int result;
+
+   signal(SIGSEGV, onsig);
+
+   result = pthread_create(&thrd, NULL, thrdrun, NULL);
+   if (result != 0) {
+      return -1;
+   }
+
+   pthread_join(thrd, NULL);
+   return 0;
+}

scripts/locale_tls_fault/auditor.c

@@ -0,0 +1,22 @@
+#define _GNU_SOURCE
+#include <link.h>
+#include <stdio.h>
+#include <string.h>
+
+extern int isspace(int c);
+
+unsigned int la_version(unsigned int v)
+{
+   return LAV_CURRENT;
+}
+
+unsigned int la_objopen(struct link_map *map, Lmid_t lmid, uintptr_t *cookie)
+{
+   //Need something that accesses TLS through R_X86_64_TPOFF64, which
+   //isspace does by getting the locale
+   int result;
+   if (strstr(map->l_name, "libm.so")) {
+      result = isspace(' ');
+   }
+   return 0;
+}

scripts/locale_tls_fault/tls_locale_fault.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+TEST_SRCDIR=$1
+TEST_BUILDDIR=$2
+COMPILE=$3
+
+mkdir -p $TEST_BUILDDIR
+$COMPILE -o $TEST_BUILDDIR/localeapp $TEST_SRCDIR/app.c -ldl -lpthread
+$COMPILE -shared -fPIC -o $TEST_BUILDDIR/liblocaleaudit.so $TEST_SRCDIR/auditor.c
+LD_AUDIT=$TEST_BUILDDIR/liblocaleaudit.so $TEST_BUILDDIR/localeapp
+

src/client/auditclient/Makefile.am

@@ -14,7 +14,7 @@ AM_CFLAGS = -fvisibility=hidden
 
 AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib
 
-BASE_SRCS = auditclient.c auditclient_common.c patch_linkmap.c redirect.c bindgot.c writablegot.c patch_bad_dtv.c
+BASE_SRCS = auditclient.c auditclient_common.c patch_linkmap.c redirect.c bindgot.c writablegot.c patch_bad_dtv.c malloc_wrapper.c fixlocale.c
 if X86_64_BLD
 ARCH_SRCS = auditclient_x86_64.c 
 endif

src/client/auditclient/Makefile.in

@@ -143,10 +143,11 @@ libspindle_audit_biter_la_DEPENDENCIES =  \
 	$(top_builddir)/client/libspindlec_biter.la $(AUDITLIB)
 am__libspindle_audit_biter_la_SOURCES_DIST = auditclient.c \
 	auditclient_common.c patch_linkmap.c redirect.c bindgot.c \
-	writablegot.c patch_bad_dtv.c auditclient_aarch64.c \
-	auditclient_ppc64.c auditclient_x86_64.c
+	writablegot.c patch_bad_dtv.c malloc_wrapper.c fixlocale.c \
+	auditclient_aarch64.c auditclient_ppc64.c auditclient_x86_64.c
 am__objects_1 = auditclient.lo auditclient_common.lo patch_linkmap.lo \
-	redirect.lo bindgot.lo writablegot.lo patch_bad_dtv.lo
+	redirect.lo bindgot.lo writablegot.lo patch_bad_dtv.lo \
+	malloc_wrapper.lo fixlocale.lo
 @AARCH64_BLD_FALSE@@PPC64LE_BLD_FALSE@@PPC64_BLD_FALSE@@X86_64_BLD_TRUE@am__objects_2 = auditclient_x86_64.lo
 @AARCH64_BLD_FALSE@@PPC64LE_BLD_FALSE@@PPC64_BLD_TRUE@am__objects_2 = auditclient_ppc64.lo
 @AARCH64_BLD_FALSE@@PPC64LE_BLD_TRUE@am__objects_2 =  \
@@ -169,8 +170,8 @@ libspindle_audit_pipe_la_DEPENDENCIES =  \
 	$(top_builddir)/client/libspindlec_pipe.la $(AUDITLIB)
 am__libspindle_audit_pipe_la_SOURCES_DIST = auditclient.c \
 	auditclient_common.c patch_linkmap.c redirect.c bindgot.c \
-	writablegot.c patch_bad_dtv.c auditclient_aarch64.c \
-	auditclient_ppc64.c auditclient_x86_64.c
+	writablegot.c patch_bad_dtv.c malloc_wrapper.c fixlocale.c \
+	auditclient_aarch64.c auditclient_ppc64.c auditclient_x86_64.c
 am_libspindle_audit_pipe_la_OBJECTS = $(am__objects_1) \
 	$(am__objects_2)
 libspindle_audit_pipe_la_OBJECTS =  \
@@ -184,8 +185,8 @@ libspindle_audit_socket_la_DEPENDENCIES =  \
 	$(top_builddir)/client/libspindlec_socket.la $(AUDITLIB)
 am__libspindle_audit_socket_la_SOURCES_DIST = auditclient.c \
 	auditclient_common.c patch_linkmap.c redirect.c bindgot.c \
-	writablegot.c patch_bad_dtv.c auditclient_aarch64.c \
-	auditclient_ppc64.c auditclient_x86_64.c
+	writablegot.c patch_bad_dtv.c malloc_wrapper.c fixlocale.c \
+	auditclient_aarch64.c auditclient_ppc64.c auditclient_x86_64.c
 am_libspindle_audit_socket_la_OBJECTS = $(am__objects_1) \
 	$(am__objects_2)
 libspindle_audit_socket_la_OBJECTS =  \
@@ -216,6 +217,7 @@ am__depfiles_remade = ./$(DEPDIR)/auditclient.Plo \
 	./$(DEPDIR)/auditclient_common.Plo \
 	./$(DEPDIR)/auditclient_ppc64.Plo \
 	./$(DEPDIR)/auditclient_x86_64.Plo ./$(DEPDIR)/bindgot.Plo \
+	./$(DEPDIR)/fixlocale.Plo ./$(DEPDIR)/malloc_wrapper.Plo \
 	./$(DEPDIR)/patch_bad_dtv.Plo ./$(DEPDIR)/patch_linkmap.Plo \
 	./$(DEPDIR)/redirect.Plo ./$(DEPDIR)/writablegot.Plo
 am__mv = mv -f
@@ -407,7 +409,7 @@ top_srcdir = @top_srcdir@
 pkglib_LTLIBRARIES = $(am__append_1) $(am__append_2) $(am__append_3)
 AM_CFLAGS = -fvisibility=hidden
 AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib
-BASE_SRCS = auditclient.c auditclient_common.c patch_linkmap.c redirect.c bindgot.c writablegot.c patch_bad_dtv.c
+BASE_SRCS = auditclient.c auditclient_common.c patch_linkmap.c redirect.c bindgot.c writablegot.c patch_bad_dtv.c malloc_wrapper.c fixlocale.c
 @AARCH64_BLD_TRUE@ARCH_SRCS = auditclient_aarch64.c
 @PPC64LE_BLD_TRUE@ARCH_SRCS = auditclient_ppc64.c
 @PPC64_BLD_TRUE@ARCH_SRCS = auditclient_ppc64.c
@@ -512,6 +514,8 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/auditclient_ppc64.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/auditclient_x86_64.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bindgot.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fixlocale.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/malloc_wrapper.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/patch_bad_dtv.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/patch_linkmap.Plo@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/redirect.Plo@am__quote@ # am--include-marker
@@ -687,6 +691,8 @@ distclean: distclean-am
 	-rm -f ./$(DEPDIR)/auditclient_ppc64.Plo
 	-rm -f ./$(DEPDIR)/auditclient_x86_64.Plo
 	-rm -f ./$(DEPDIR)/bindgot.Plo
+	-rm -f ./$(DEPDIR)/fixlocale.Plo
+	-rm -f ./$(DEPDIR)/malloc_wrapper.Plo
 	-rm -f ./$(DEPDIR)/patch_bad_dtv.Plo
 	-rm -f ./$(DEPDIR)/patch_linkmap.Plo
 	-rm -f ./$(DEPDIR)/redirect.Plo
@@ -742,6 +748,8 @@ maintainer-clean: maintainer-clean-am
 	-rm -f ./$(DEPDIR)/auditclient_ppc64.Plo
 	-rm -f ./$(DEPDIR)/auditclient_x86_64.Plo
 	-rm -f ./$(DEPDIR)/bindgot.Plo
+	-rm -f ./$(DEPDIR)/fixlocale.Plo
+	-rm -f ./$(DEPDIR)/malloc_wrapper.Plo
 	-rm -f ./$(DEPDIR)/patch_bad_dtv.Plo
 	-rm -f ./$(DEPDIR)/patch_linkmap.Plo
 	-rm -f ./$(DEPDIR)/redirect.Plo

src/client/auditclient/fixlocale.c

@@ -0,0 +1,59 @@
+/*
+This file is part of Spindle.  For copyright information see the COPYRIGHT 
+file in the top level directory, or at 
+https://github.com/hpc/Spindle/blob/master/COPYRIGHT
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU Lesser General Public License (as published by the Free Software
+Foundation) version 2.1 dated February 1999.  This program is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED
+WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms 
+and conditions of the GNU Lesser General Public License for more details.  You should 
+have received a copy of the GNU Lesser General Public License along with this 
+program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+#include <locale.h>
+#include <unistd.h>
+
+#include "spindle_debug.h"
+#include "config.h"
+
+#if defined(NEWTHREAD_LOCALE_BUG)
+#include <sys/syscall.h>
+
+static __thread int fixed_locale = 0;
+
+static pid_t gettid()
+{
+   return syscall(SYS_gettid);   
+}
+
+#endif
+
+/**
+ * This works around the LD_AUDIT bug documented at:
+ *  https://sourceware.org/bugzilla/show_bug.cgi?id=32483
+ *
+ * We need to set the locale for app-space created threads in the audit space.
+ * Otherwise we see seg faults under malloc in spindle when malloc calls
+ * get_nprocs.
+ **/
+void check_for_new_thread()
+{
+#if defined(NEWTHREAD_LOCALE_BUG)
+   locale_t l;
+   if (fixed_locale)
+      return;
+   fixed_locale = 1;
+   if (gettid() == getpid()) {
+      debug_printf3("Primary thread. Not updating locale.\n");      
+      return;
+   }
+
+   debug_printf2("Identified new thread. Fixing locale\n");
+   l = uselocale((locale_t) 0);
+   uselocale(l);
+#endif
+}

src/client/auditclient/fixlocale.h

@@ -0,0 +1,22 @@
+/*
+This file is part of Spindle.  For copyright information see the COPYRIGHT 
+file in the top level directory, or at 
+https://github.com/hpc/Spindle/blob/master/COPYRIGHT
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU Lesser General Public License (as published by the Free Software
+Foundation) version 2.1 dated February 1999.  This program is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED
+WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms 
+and conditions of the GNU Lesser General Public License for more details.  You should 
+have received a copy of the GNU Lesser General Public License along with this 
+program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+#if !defined(FIXLOCALE_H_)
+#define FIXLOCALE_H_
+
+void check_for_new_thread();
+
+#endif