HBASE-28804: Implement asynchronous retrieval of bucket-cache data from persistence (#6182) (#6219)

The cherry-pick of HBASE-29727 into branch-2.6 accidently brought most of HBASE-28804 into branch-2, this is to backport the remaining original change in HBASE-28804.

Signed-off-by: Wellington Chevreuil <wchevreuil@apache.org>
Change-Id: I816141ccebf91191aaa4cb6afa04343b64663eb0

Author: jhungund

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java

@@ -37,6 +37,9 @@ public class HFilePreadReader extends HFileReaderImpl {
   public HFilePreadReader(ReaderContext context, HFileInfo fileInfo, CacheConfig cacheConf,
     Configuration conf) throws IOException {
     super(context, fileInfo, cacheConf, conf);
+    cacheConf.getBlockCache()
+      .ifPresent(cache -> cache.waitForCacheInitialization(WAIT_TIME_FOR_CACHE_INITIALIZATION));
+
     // Initialize HFileInfo object with metadata for caching decisions
     fileInfo.initMetaAndIndex(this);
     // master hosted regions, like the master procedures store wouldn't have a block cache

hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestVerifyBucketCacheFile.java

@@ -108,7 +108,8 @@ public void testRetrieveFromFile() throws Exception {
       bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
         constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen,
         testDir + "/bucket.persistence", DEFAULT_ERROR_TOLERATION_DURATION, conf);
-      assertTrue(bucketCache.waitForCacheInitialization(10000));long usedSize = bucketCache.getAllocator().getUsedSize();
+      assertTrue(bucketCache.waitForCacheInitialization(10000));
+      long usedSize = bucketCache.getAllocator().getUsedSize();
       assertEquals(0, usedSize);
       CacheTestUtils.HFileBlockPair[] blocks =
         CacheTestUtils.generateHFileBlocks(constructedBlockSize, 1);
@@ -125,30 +126,30 @@ public void testRetrieveFromFile() throws Exception {
         constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen,
         testDir + "/bucket.persistence", DEFAULT_ERROR_TOLERATION_DURATION, conf);
       waitPersistentCacheValidation(conf, bucketCache);
-    assertTrue(bucketCache.waitForCacheInitialization(10000));
+      assertTrue(bucketCache.waitForCacheInitialization(10000));
       assertEquals(usedSize, bucketCache.getAllocator().getUsedSize());
       // persist cache to file
       bucketCache.shutdown();
 
-    // 2.delete bucket cache file
-    final java.nio.file.Path cacheFile =
-      FileSystems.getDefault().getPath(testDir.toString(), "bucket.cache");
-    assertTrue(Files.deleteIfExists(cacheFile));
-    // can't restore cache from file
-    bucketCache =
-      new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize,
-        constructedBlockSizes, writeThreads, writerQLen, testDir + "/bucket.persistence");
-    assertTrue(bucketCache.waitForCacheInitialization(10000));
-    assertEquals(0, bucketCache.getAllocator().getUsedSize());
-    assertEquals(0, bucketCache.backingMap.size());
-    // Add blocks
-    for (CacheTestUtils.HFileBlockPair block : blocks) {
-      cacheAndWaitUntilFlushedToBucket(bucketCache, block.getBlockName(), block.getBlock());
-    }
-    usedSize = bucketCache.getAllocator().getUsedSize();
-    assertNotEquals(0, usedSize);
-    // persist cache to file
-    bucketCache.shutdown();
+      // 2.delete bucket cache file
+      final java.nio.file.Path cacheFile =
+        FileSystems.getDefault().getPath(testDir.toString(), "bucket.cache");
+      assertTrue(Files.deleteIfExists(cacheFile));
+      // can't restore cache from file
+      bucketCache =
+        new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize,
+          constructedBlockSizes, writeThreads, writerQLen, testDir + "/bucket.persistence");
+      assertTrue(bucketCache.waitForCacheInitialization(10000));
+      assertEquals(0, bucketCache.getAllocator().getUsedSize());
+      assertEquals(0, bucketCache.backingMap.size());
+      // Add blocks
+      for (CacheTestUtils.HFileBlockPair block : blocks) {
+        cacheAndWaitUntilFlushedToBucket(bucketCache, block.getBlockName(), block.getBlock());
+      }
+      usedSize = bucketCache.getAllocator().getUsedSize();
+      assertNotEquals(0, usedSize);
+      // persist cache to file
+      bucketCache.shutdown();
 
       // 3.delete backingMap persistence file
       final java.nio.file.Path mapFile =
@@ -186,7 +187,8 @@ public void testRetrieveFromFileAfterDelete() throws Exception {
       bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
         constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, mapFileName,
         DEFAULT_ERROR_TOLERATION_DURATION, conf);
-      assertTrue(bucketCache.waitForCacheInitialization(10000));    long usedSize = bucketCache.getAllocator().getUsedSize();
+      assertTrue(bucketCache.waitForCacheInitialization(10000));
+      long usedSize = bucketCache.getAllocator().getUsedSize();
       assertEquals(0, usedSize);
       CacheTestUtils.HFileBlockPair[] blocks =
         CacheTestUtils.generateHFileBlocks(constructedBlockSize, 1);
@@ -238,7 +240,8 @@ public void testModifiedBucketCacheFileData() throws Exception {
       bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
         constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen,
         testDir + "/bucket.persistence", DEFAULT_ERROR_TOLERATION_DURATION, conf);
-      assertTrue(bucketCache.waitForCacheInitialization(10000));long usedSize = bucketCache.getAllocator().getUsedSize();
+      assertTrue(bucketCache.waitForCacheInitialization(10000));
+      long usedSize = bucketCache.getAllocator().getUsedSize();
       assertEquals(0, usedSize);
 
       CacheTestUtils.HFileBlockPair[] blocks =
@@ -300,7 +303,8 @@ public void testModifiedBucketCacheFileTime() throws Exception {
       bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
         constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen,
         testDir + "/bucket.persistence", DEFAULT_ERROR_TOLERATION_DURATION, conf);
-      assertTrue(bucketCache.waitForCacheInitialization(10000));long usedSize = bucketCache.getAllocator().getUsedSize();
+      assertTrue(bucketCache.waitForCacheInitialization(10000));
+      long usedSize = bucketCache.getAllocator().getUsedSize();
       assertEquals(0, usedSize);
 
       Pair<String, Long> myPair = new Pair<>();
@@ -327,7 +331,8 @@ public void testModifiedBucketCacheFileTime() throws Exception {
         constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen,
         testDir + "/bucket.persistence", DEFAULT_ERROR_TOLERATION_DURATION, conf);
       waitPersistentCacheValidation(conf, bucketCache);
-      assertTrue(bucketCache.waitForCacheInitialization(10000));assertEquals(usedSize, bucketCache.getAllocator().getUsedSize());
+      assertTrue(bucketCache.waitForCacheInitialization(10000));
+      assertEquals(usedSize, bucketCache.getAllocator().getUsedSize());
       assertEquals(blockCount, bucketCache.backingMap.size());
     } finally {
       if (bucketCache != null) {
@@ -360,7 +365,7 @@ public void testBucketCacheRecovery() throws Exception {
       bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
         constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, mapFileName,
         DEFAULT_ERROR_TOLERATION_DURATION, conf);
-    assertTrue(bucketCache.waitForCacheInitialization(10000));
+      assertTrue(bucketCache.waitForCacheInitialization(10000));
 
       CacheTestUtils.HFileBlockPair[] blocks =
         CacheTestUtils.generateHFileBlocks(constructedBlockSize, 4);
@@ -428,7 +433,7 @@ private void testChunkedBackingMapRecovery(int chunkSize, int numBlocks) throws
       bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
         constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, mapFileName,
         DEFAULT_ERROR_TOLERATION_DURATION, conf);
-    assertTrue(bucketCache.waitForCacheInitialization(10000));
+      assertTrue(bucketCache.waitForCacheInitialization(10000));
 
       CacheTestUtils.HFileBlockPair[] blocks =
         CacheTestUtils.generateHFileBlocks(constructedBlockSize, numBlocks);