From 51dadc3cd723b50d9fad50ecb8d129ee1f224fcc Mon Sep 17 00:00:00 2001 From: Alexander A Oganezov Date: Sat, 11 Jul 2026 00:06:48 +0000 Subject: [PATCH] DAOS-19298 cart: fix test fini sequence - In number of cart clients, the group was destroyed before all the client threads joined. If during testing any RPC timed out, then the client could try to access already-destroyed group in the delayed completion callback handler, leading to a segfault. Signed-off-by: Alexander A Oganezov --- src/tests/ftest/cart/iv_client.c | 4 ++-- src/tests/ftest/cart/no_pmix_launcher_client.c | 8 ++++---- src/tests/ftest/cart/rpc_test_srv.c | 11 ++++++----- src/tests/ftest/cart/test_ep_cred_client.c | 12 ++++++------ src/tests/ftest/cart/test_multisend_client.c | 15 +++++++-------- src/tests/ftest/cart/test_no_timeout.c | 14 +++++++------- src/tests/ftest/cart/test_proto_client.c | 12 ++++++------ 7 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/tests/ftest/cart/iv_client.c b/src/tests/ftest/cart/iv_client.c index eb263b58c3f..5802bf5a8c8 100644 --- a/src/tests/ftest/cart/iv_client.c +++ b/src/tests/ftest/cart/iv_client.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2021 Intel Corporation. + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -670,12 +671,11 @@ int main(int argc, char **argv) return -1; } - crt_group_detach(srv_grp); - g_do_shutdown = true; pthread_join(progress_thread, NULL); DBG_PRINT("Exiting client\n"); + crt_group_detach(srv_grp); if (log_file != stdout) fclose(log_file); diff --git a/src/tests/ftest/cart/no_pmix_launcher_client.c b/src/tests/ftest/cart/no_pmix_launcher_client.c index 52652c5e416..ec14855eda5 100644 --- a/src/tests/ftest/cart/no_pmix_launcher_client.c +++ b/src/tests/ftest/cart/no_pmix_launcher_client.c @@ -1,6 +1,6 @@ /* * (C) Copyright 2018-2022 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -206,6 +206,9 @@ int main(int argc, char **argv) DBG_PRINT("RPC response received from rank=%d\n", rank); } + g_do_shutdown = true; + pthread_join(progress_thread, NULL); + D_FREE(rank_list->rl_ranks); D_FREE(rank_list); @@ -215,9 +218,6 @@ int main(int argc, char **argv) assert(0); } - g_do_shutdown = true; - pthread_join(progress_thread, NULL); - sem_destroy(&sem); rc = crt_finalize(); diff --git a/src/tests/ftest/cart/rpc_test_srv.c b/src/tests/ftest/cart/rpc_test_srv.c index 5b7604d41e4..5186d7d6478 100644 --- a/src/tests/ftest/cart/rpc_test_srv.c +++ b/src/tests/ftest/cart/rpc_test_srv.c @@ -1,5 +1,6 @@ /* * (C) Copyright 2017-2021 Intel Corporation. + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -222,16 +223,16 @@ srv_rpc_finalize(void) rc = crt_group_rank(NULL, &myrank); D_ASSERTF(rc == 0, "crt_group_rank failed %d\n", rc); - if (rpc_srv.target_multitier_grp != NULL) { - rc = crt_group_detach(rpc_srv.target_multitier_grp); - D_ASSERTF(rc == 0, "crt_group_detach failed %d\n", rc); - } - dbg("main thread wait progress thread ...\n"); if (rpc_srv.progress_thid) pthread_join(rpc_srv.progress_thid, NULL); + if (rpc_srv.target_multitier_grp != NULL) { + rc = crt_group_detach(rpc_srv.target_multitier_grp); + D_ASSERTF(rc == 0, "crt_group_detach failed %d\n", rc); + } + rc = crt_context_destroy(rpc_srv.crt_ctx, 1); D_ASSERTF(rc == 0, "crt_context_destroy failed %d\n", rc); diff --git a/src/tests/ftest/cart/test_ep_cred_client.c b/src/tests/ftest/cart/test_ep_cred_client.c index a6b84a38d68..958f01cdede 100644 --- a/src/tests/ftest/cart/test_ep_cred_client.c +++ b/src/tests/ftest/cart/test_ep_cred_client.c @@ -146,6 +146,12 @@ test_run() crtu_sem_timedwait(&test.tg_token_to_proceed, 61, __LINE__); } + crtu_progress_stop(); + + rc = pthread_join(test.tg_tid, NULL); + D_ASSERTF(rc == 0, "pthread_join failed. rc: %d\n", rc); + DBG_PRINT("joined progress thread.\n"); + d_rank_list_free(rank_list); rank_list = NULL; @@ -158,12 +164,6 @@ test_run() "crt_group_view_destroy() failed; rc=%d\n", rc); } - crtu_progress_stop(); - - rc = pthread_join(test.tg_tid, NULL); - D_ASSERTF(rc == 0, "pthread_join failed. rc: %d\n", rc); - DBG_PRINT("joined progress thread.\n"); - rc = sem_destroy(&test.tg_token_to_proceed); D_ASSERTF(rc == 0, "sem_destroy() failed.\n"); diff --git a/src/tests/ftest/cart/test_multisend_client.c b/src/tests/ftest/cart/test_multisend_client.c index f1cafdddc65..a434399b388 100644 --- a/src/tests/ftest/cart/test_multisend_client.c +++ b/src/tests/ftest/cart/test_multisend_client.c @@ -225,6 +225,13 @@ test_run() } } + crtu_progress_stop(); + + for (i = 0; i < test.tg_num_ctx; i++) { + rc = pthread_join(test.tg_tid[i], NULL); + D_ASSERTF(rc == 0, "pthread_join failed. rc: %d\n", rc); + } + D_DEBUG(DB_TRACE, "joined progress threads.\n"); d_rank_list_free(rank_list); rank_list = NULL; @@ -236,14 +243,6 @@ test_run() D_ASSERTF(rc == 0, "crt_group_view_destroy() failed; rc=%d\n", rc); } - crtu_progress_stop(); - - for (i = 0; i < test.tg_num_ctx; i++) { - rc = pthread_join(test.tg_tid[i], NULL); - D_ASSERTF(rc == 0, "pthread_join failed. rc: %d\n", rc); - } - D_DEBUG(DB_TRACE, "joined progress threads.\n"); - rc = sem_destroy(&test.tg_token_to_proceed); D_ASSERTF(rc == 0, "sem_destroy() failed.\n"); diff --git a/src/tests/ftest/cart/test_no_timeout.c b/src/tests/ftest/cart/test_no_timeout.c index 28c5585ce48..768b023f2fc 100644 --- a/src/tests/ftest/cart/test_no_timeout.c +++ b/src/tests/ftest/cart/test_no_timeout.c @@ -133,6 +133,13 @@ test_run(void) } } + crtu_progress_stop(); + + rc = pthread_join(test_g.t_tid[0], NULL); + if (rc != 0) + fprintf(stderr, "pthread_join failed. rc: %d\n", rc); + D_DEBUG(DB_TEST, "joined progress thread.\n"); + d_rank_list_free(rank_list); rank_list = NULL; @@ -145,13 +152,6 @@ test_run(void) "crt_group_view_destroy() failed; rc=%d\n", rc); } - crtu_progress_stop(); - - rc = pthread_join(test_g.t_tid[0], NULL); - if (rc != 0) - fprintf(stderr, "pthread_join failed. rc: %d\n", rc); - D_DEBUG(DB_TEST, "joined progress thread.\n"); - rc = sem_destroy(&test_g.t_token_to_proceed); D_ASSERTF(rc == 0, "sem_destroy() failed.\n"); diff --git a/src/tests/ftest/cart/test_proto_client.c b/src/tests/ftest/cart/test_proto_client.c index eb91fa652cd..c38489ba7d1 100644 --- a/src/tests/ftest/cart/test_proto_client.c +++ b/src/tests/ftest/cart/test_proto_client.c @@ -164,6 +164,12 @@ test_run() crtu_sem_timedwait(&test.tg_token_to_proceed, 61, __LINE__); } + crtu_progress_stop(); + + rc = pthread_join(test.tg_tid, NULL); + D_ASSERTF(rc == 0, "pthread_join failed. rc: %d\n", rc); + D_DEBUG(DB_TRACE, "joined progress thread.\n"); + d_rank_list_free(rank_list); rank_list = NULL; @@ -176,12 +182,6 @@ test_run() "crt_group_view_destroy() failed; rc=%d\n", rc); } - crtu_progress_stop(); - - rc = pthread_join(test.tg_tid, NULL); - D_ASSERTF(rc == 0, "pthread_join failed. rc: %d\n", rc); - D_DEBUG(DB_TRACE, "joined progress thread.\n"); - rc = sem_destroy(&test.tg_token_to_proceed); D_ASSERTF(rc == 0, "sem_destroy() failed.\n");