diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 5b02699adb7a74..06edc10e54ca4f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -521,7 +521,7 @@ public HttpServer2 getHttpServer() { public void queueExternalCall(ExternalCall extCall) throws IOException, InterruptedException { if (rpcServer == null) { - throw new RetriableException("Namenode is in startup mode"); + throw NameNodeUtils.startupModeException(this); } rpcServer.getClientRpcServer().queueCall(extCall); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java index af3ed61cfe35b3..6db0ceb2b854cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java @@ -24,9 +24,12 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.ipc.RetriableException; +import org.apache.hadoop.ipc.StandbyException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.net.URI; import java.util.Collection; @@ -41,6 +44,8 @@ public final class NameNodeUtils { public static final Logger LOG = LoggerFactory.getLogger(NameNodeUtils.class); + public static final String STARTUP_MODE = "Namenode is in startup mode"; + /** * Return the namenode address that will be used by clients to access this * namenode or name service. This needs to be called before the config @@ -119,6 +124,21 @@ static String getClientNamenodeAddress( } } + /** + * Build the exception to throw when the NameNode receives a request while + * its RPC server is not yet available (e.g. fsimage loading). Returns a + * {@link StandbyException} when this NameNode is not in active state so the + * client fails over to the peer NameNode; otherwise a + * {@link RetriableException} so the client retries against the same node. + */ + public static IOException startupModeException(NameNode namenode) { + if (!namenode.isActiveState()) { + return new StandbyException(STARTUP_MODE); + } else { + return new RetriableException(STARTUP_MODE); + } + } + private NameNodeUtils() { // Disallow construction } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 443c1836351ed3..2086e83dea5d50 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -65,6 +65,7 @@ import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.namenode.NameNodeUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -115,7 +116,6 @@ import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.ExternalCall; -import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.security.Credentials; @@ -184,18 +184,18 @@ protected void init(final UserGroupInformation ugi, private static NamenodeProtocols getRPCServer(NameNode namenode) throws IOException { - final NamenodeProtocols np = namenode.getRpcServer(); - if (np == null) { - throw new RetriableException("Namenode is in startup mode"); - } - return np; + final NamenodeProtocols np = namenode.getRpcServer(); + if (np == null) { + throw NameNodeUtils.startupModeException(namenode); + } + return np; } protected ClientProtocol getRpcClientProtocol() throws IOException { final NameNode namenode = (NameNode)context.getAttribute("name.node"); final ClientProtocol cp = namenode.getRpcServer(); if (cp == null) { - throw new RetriableException("Namenode is in startup mode"); + throw NameNodeUtils.startupModeException(namenode); } return cp; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java index 48751695e48c9f..1fbbaa21c5d8a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java @@ -22,9 +22,11 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; import java.io.DataInputStream; @@ -48,14 +50,17 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.hdfs.server.namenode.NameNodeUtils; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.web.resources.ExceptionHandler; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.Whitebox; import org.apache.hadoop.util.concurrent.SubjectInheritingThread; import org.junit.jupiter.api.Test; @@ -332,4 +337,65 @@ public void work() { } } } + + /** + * Make sure a StandbyException is thrown when rpcServer is null in + * NamenodeWebHdfsMethods in HA Setup. + */ + @Test + @Timeout(120) + public void testThrowStandbyExceptionWhileNNStartup() throws Exception { + final Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo) + .numDataNodes(0).build(); + HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME); + cluster.waitActive(); + cluster.transitionToActive(1); + + final NameNode namenode = cluster.getNameNode(0); + final NamenodeProtocols rpcServer = namenode.getRpcServer(); + Whitebox.setInternalState(namenode, "rpcServer", null); + + String standbyHttpAddress = namenode.getHttpAddress().getHostName() + + ":" + namenode.getHttpAddress().getPort(); + URI webhdfsUri = URI.create(WebHdfsConstants.WEBHDFS_SCHEME + "://" + standbyHttpAddress); + FileSystem fs = FileSystem.get(webhdfsUri, conf); + + Path foo = new Path("/foo"); + try { + fs.mkdirs(foo); + fail("Expected StandbyException"); + } catch (Exception e) { + if (e instanceof StandbyException) { + GenericTestUtils.assertExceptionContains(NameNodeUtils.STARTUP_MODE, e); + } else { + fail("Expected StandbyException"); + } + } finally { + Whitebox.setInternalState(namenode, "rpcServer", rpcServer); + } + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + /** + * Active NameNode in startup mode must yield a RetriableException so the + * client keeps retrying the same node instead of failing over. + */ + @Test + public void testStartupModeExceptionWhenActive() { + NameNode active = mock(NameNode.class); + when(active.isActiveState()).thenReturn(true); + + IOException ex = NameNodeUtils.startupModeException(active); + assertTrue(ex instanceof RetriableException, + "Active NameNode in startup mode should yield RetriableException, got " + + ex.getClass().getName()); + GenericTestUtils.assertExceptionContains(NameNodeUtils.STARTUP_MODE, ex); + } }