Enhance unit tests with timeout handling for blob creation and sync and jenkins.

.jenkins/Jenkinsfile

@@ -65,6 +65,9 @@ pipeline {
                     conan create ${BUILD_MISSING} -s:h build_type=RelWithDebInfo -o sisl/*:malloc_impl=tcmalloc ${CONAN_FLAGS} . ; \
                     "
             }
+             post {
+                       failure { script { sleep 3600000  } }
+                   }
         }
 
         stage("Deploy") {
@@ -82,7 +85,7 @@ pipeline {
                expression { (env.BRANCH_NAME == "${TARGET_BRANCH}") }
                expression { (!"${upstream_triggered}") || ("${upstream_triggered}" == "") }
             } }
-        
+
             stages {
                 stage('StorageManager') {
                     steps {

src/lib/homestore_backend/CMakeLists.txt

@@ -138,6 +138,7 @@ add_test(NAME HomestoreResyncTestWithLeaderRestart
         --override_config homestore_config.consensus.max_grpc_message_size=138412032
         --override_config homestore_config.consensus.replace_member_sync_check_interval_ms=1000
         --override_config homestore_config.consensus.laggy_threshold=2000
+        --log_mods homeobject:trace,replication:trace
         --gtest_filter=HomeObjectFixture.RestartLeader*)
 #add_test(NAME HomestoreReplaceMemberRollbackTest
 #        COMMAND homestore_test_dynamic -csv error --executor immediate --config_path ./

src/lib/homestore_backend/tests/homeobj_fixture.hpp

@@ -49,7 +49,7 @@ class HomeObjectFixture : public ::testing::Test {
 
         HSHomeObject::_hs_chunk_size = SISL_OPTIONS["chunk_size"].as< uint64_t >() * Mi;
         _obj_inst = std::dynamic_pointer_cast< HSHomeObject >(g_helper->build_new_homeobject());
-        
+
         // Used to export metrics, it should be called after init_homeobject
         if (SISL_OPTIONS["enable_http"].as< bool >()) { g_helper->app->start_http_server(); }
         if (!g_helper->is_current_testcase_restarted()) {
@@ -785,12 +785,16 @@ class HomeObjectFixture : public ::testing::Test {
 
     // wait for the last blob to be created locally, which means all the blob before this blob are created
     void wait_for_blob(shard_id_t shard_id, blob_id_t blob_id) {
+        static constexpr auto k_blob_timeout = std::chrono::minutes(10);
+        auto deadline = std::chrono::steady_clock::now() + k_blob_timeout;
         while (true) {
             if (blob_exist(shard_id, blob_id)) {
                 LOGINFO("shard {} blob {} is created locally, which means all the blob before {} are created", shard_id,
                         blob_id, blob_id);
                 return;
             }
+            RELEASE_ASSERT(std::chrono::steady_clock::now() < deadline,
+                           "wait_for_blob timed out after 5min: shard={} blob={}", shard_id, blob_id);
             std::this_thread::sleep_for(1s);
         }
     }

src/lib/homestore_backend/tests/hs_repl_test_helper.hpp

@@ -52,6 +52,7 @@ class HSReplTestHelper {
 protected:
     struct IPCData {
         void sync(uint64_t sync_point, uint32_t max_count) {
+            static constexpr auto k_sync_timeout = std::chrono::minutes(10);
             std::unique_lock lg(mtx_);
             LOGINFO("=== Syncing: replica={}(total {}), sync_point_num={} ===", homeobject_replica_count_, max_count,
                     sync_point);
@@ -63,7 +64,9 @@ class HSReplTestHelper {
                 auxiliary_uint64_id_ = UINT64_MAX;
                 cv_.notify_all();
             } else {
-                cv_.wait(lg, [this, sync_point]() { return sync_point_num_ == sync_point; });
+                bool ok =
+                    cv_.wait_for(lg, k_sync_timeout, [this, sync_point]() { return sync_point_num_ == sync_point; });
+                RELEASE_ASSERT(ok, "sync timed out after 5min at sync_point={}, a replica likely crashed", sync_point);
             }
         }