HDFS-17918. WebHDFS client-side failover fails during NameNode fsimage loading in HA

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java

@@ -521,7 +521,7 @@ public HttpServer2 getHttpServer() {
   public void queueExternalCall(ExternalCall<?> extCall)
       throws IOException, InterruptedException {
     if (rpcServer == null) {
-      throw new RetriableException("Namenode is in startup mode");
+      throw NameNodeUtils.startupModeException(this);
     }
     rpcServer.getClientRpcServer().queueCall(extCall);
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java

@@ -24,9 +24,12 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.ipc.RetriableException;
+import org.apache.hadoop.ipc.StandbyException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.net.URI;
 import java.util.Collection;
 
@@ -41,6 +44,8 @@
 public final class NameNodeUtils {
   public static final Logger LOG = LoggerFactory.getLogger(NameNodeUtils.class);
 
+  public static final String STARTUP_MODE = "Namenode is in startup mode";
+
   /**
    * Return the namenode address that will be used by clients to access this
    * namenode or name service. This needs to be called before the config
@@ -119,6 +124,21 @@ static String getClientNamenodeAddress(
     }
   }
 
+  /**
+   * Build the exception to throw when the NameNode receives a request while
+   * its RPC server is not yet available (e.g. fsimage loading). Returns a
+   * {@link StandbyException} when this NameNode is not in active state so the
+   * client fails over to the peer NameNode; otherwise a
+   * {@link RetriableException} so the client retries against the same node.
+   */
+  public static IOException startupModeException(NameNode namenode) {
+    if (!namenode.isActiveState()) {
+      return new StandbyException(STARTUP_MODE);
+    } else {
+      return new RetriableException(STARTUP_MODE);
+    }
+  }
+
   private NameNodeUtils() {
     // Disallow construction
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java

@@ -65,6 +65,7 @@
 import org.apache.hadoop.fs.QuotaUsage;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -115,7 +116,6 @@
 import org.apache.hadoop.http.JettyUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.ExternalCall;
-import org.apache.hadoop.ipc.RetriableException;
 import org.apache.hadoop.net.Node;
 import org.apache.hadoop.net.NodeBase;
 import org.apache.hadoop.security.Credentials;
@@ -184,18 +184,18 @@ protected void init(final UserGroupInformation ugi,
 
   private static NamenodeProtocols getRPCServer(NameNode namenode)
       throws IOException {
-     final NamenodeProtocols np = namenode.getRpcServer();
-     if (np == null) {
-       throw new RetriableException("Namenode is in startup mode");
-     }
-     return np;
+    final NamenodeProtocols np = namenode.getRpcServer();
+    if (np == null) {
+      throw NameNodeUtils.startupModeException(namenode);
+    }
+    return np;
   }
 
   protected ClientProtocol getRpcClientProtocol() throws IOException {
     final NameNode namenode = (NameNode)context.getAttribute("name.node");
     final ClientProtocol cp = namenode.getRpcServer();
     if (cp == null) {
-      throw new RetriableException("Namenode is in startup mode");
+      throw NameNodeUtils.startupModeException(namenode);
     }
     return cp;
   }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java

@@ -22,9 +22,11 @@
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
@@ -48,14 +50,17 @@
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeUtils;
 import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.hdfs.web.resources.ExceptionHandler;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.RetriableException;
 import org.apache.hadoop.ipc.StandbyException;
 import org.apache.hadoop.security.token.SecretManager;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.Whitebox;
 import org.apache.hadoop.util.concurrent.SubjectInheritingThread;
 import org.junit.jupiter.api.Test;
@@ -332,4 +337,65 @@ public void work() {
       }
     }
   }
+
+  /**
+   * Make sure a StandbyException is thrown when rpcServer is null in
+   * NamenodeWebHdfsMethods in HA Setup.
+   */
+  @Test
+  @Timeout(120)
+  public void testThrowStandbyExceptionWhileNNStartup() throws Exception {
+    final Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo)
+          .numDataNodes(0).build();
+      HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME);
+      cluster.waitActive();
+      cluster.transitionToActive(1);
+
+      final NameNode namenode = cluster.getNameNode(0);
+      final NamenodeProtocols rpcServer = namenode.getRpcServer();
+      Whitebox.setInternalState(namenode, "rpcServer", null);
+
+      String standbyHttpAddress = namenode.getHttpAddress().getHostName()
+          + ":" + namenode.getHttpAddress().getPort();
+      URI webhdfsUri = URI.create(WebHdfsConstants.WEBHDFS_SCHEME + "://" + standbyHttpAddress);
+      FileSystem fs = FileSystem.get(webhdfsUri, conf);
+
+      Path foo = new Path("/foo");
+      try {
+        fs.mkdirs(foo);
+        fail("Expected StandbyException");
+      } catch (Exception e) {
+        if (e instanceof StandbyException) {
+          GenericTestUtils.assertExceptionContains(NameNodeUtils.STARTUP_MODE, e);
+        } else {
+          fail("Expected StandbyException");
+        }
+      } finally {
+        Whitebox.setInternalState(namenode, "rpcServer", rpcServer);
+      }
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
+
+  /**
+   * Active NameNode in startup mode must yield a RetriableException so the
+   * client keeps retrying the same node instead of failing over.
+   */
+  @Test
+  public void testStartupModeExceptionWhenActive() {
+    NameNode active = mock(NameNode.class);
+    when(active.isActiveState()).thenReturn(true);
+
+    IOException ex = NameNodeUtils.startupModeException(active);
+    assertTrue(ex instanceof RetriableException,
+        "Active NameNode in startup mode should yield RetriableException, got "
+            + ex.getClass().getName());
+    GenericTestUtils.assertExceptionContains(NameNodeUtils.STARTUP_MODE, ex);
+  }
 }