YARN-11960. Fix NodeManager OOM caused by large client.json

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -2726,6 +2726,16 @@ public static boolean isAclEnabled(Configuration conf) {
   public static final String NM_DOCKER_DEFAULT_TMPFS_MOUNTS =
       DOCKER_CONTAINER_RUNTIME_PREFIX + "default-tmpfs-mounts";
 
+  /**
+   * Maximum allowed file size in bytes for the Docker client configuration
+   * file (config.json). If the file size exceeds this value, the container
+   * launch will be rejected.
+   */
+  public static final String NM_DOCKER_CLIENT_CONFIG_MAX_SIZE_BYTES =
+      DOCKER_CONTAINER_RUNTIME_PREFIX + "client-config-file-max-size-bytes";
+  public static final int DEFAULT_NM_DOCKER_CLIENT_CONFIG_MAX_SIZE_BYTES =
+      16 * 1024;
+
   /** The mode in which the Java Container Sandbox should run detailed by
    *  the JavaSandboxLinuxContainerRuntime. */
   public static final String YARN_CONTAINER_SANDBOX =

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java

@@ -19,6 +19,7 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.commons.io.IOUtils;
@@ -27,6 +28,7 @@
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.security.DockerCredentialTokenIdentifier;
 
 import com.fasterxml.jackson.core.JsonFactory;
@@ -80,19 +82,37 @@ private DockerClientConfigHandler() { }
    */
   public static Credentials readCredentialsFromConfigFile(Path configFile,
       Configuration conf, String applicationId) throws IOException {
-    // Read the config file
-    String contents = null;
+
+    int configFileMaxSizeByte = conf.getInt(
+        YarnConfiguration.NM_DOCKER_CLIENT_CONFIG_MAX_SIZE_BYTES,
+        YarnConfiguration.DEFAULT_NM_DOCKER_CLIENT_CONFIG_MAX_SIZE_BYTES);
+    if (configFileMaxSizeByte <= 0) {
+      LOG.warn("Invalid value {} for {}, falling back to default {} bytes.",
+          configFileMaxSizeByte,
+          YarnConfiguration.NM_DOCKER_CLIENT_CONFIG_MAX_SIZE_BYTES,
+          YarnConfiguration.DEFAULT_NM_DOCKER_CLIENT_CONFIG_MAX_SIZE_BYTES);
+      configFileMaxSizeByte =
+          YarnConfiguration.DEFAULT_NM_DOCKER_CLIENT_CONFIG_MAX_SIZE_BYTES;
+    }
+
+    // Read the config file. Check the file size up front so
+    // a large config.json cannot OOM the NodeManager via IOUtils.toString.
     configFile = new Path(configFile.toUri());
     FileSystem fs = configFile.getFileSystem(conf);
-    if (fs != null) {
-      FSDataInputStream fileHandle = fs.open(configFile);
-      if (fileHandle != null) {
-        contents = IOUtils.toString(fileHandle, StandardCharsets.UTF_8);
-      }
+    FileStatus status = fs.getFileStatus(configFile);
+    if (status.getLen() > configFileMaxSizeByte) {
+      throw new IOException("Docker client config " + configFile + " ("
+          + status.getLen() + " bytes) is larger than maximum allowed size ("
+          + configFileMaxSizeByte + " bytes)");
+    }
+
+    String contents;
+    try (FSDataInputStream fileHandle = fs.open(configFile)) {
+      contents = IOUtils.toString(fileHandle, StandardCharsets.UTF_8);
     }
-    if (contents == null) {
-      throw new IOException("Failed to read Docker client configuration: "
-          + configFile);
+    if (contents == null || contents.trim().isEmpty()) {
+      throw new IOException(
+          "Failed to read Docker client configuration: " + configFile);
     }
 
     // Parse the JSON and create the Tokens/Credentials.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -2336,6 +2336,16 @@
     <value></value>
   </property>
 
+  <property>
+    <description>
+      Maximum allowed file size in bytes for the Docker client configuration file
+      (config.json). If the file size exceeds this value, the container launch
+      will be rejected. Default is 16 KiB.
+    </description>
+    <name>yarn.nodemanager.runtime.linux.docker.client-config-file-max-size-bytes</name>
+    <value>16384</value>
+  </property>
+
   <property>
     <description>The runC image tag to manifest plugin
       class to be used.</description>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java

@@ -910,7 +910,7 @@ private Credentials getAdditionalDockerClientCredentials(String clientConfig,
                 containerIdStr);
       } catch (IOException e) {
         throw new RuntimeException(
-            "Fail to read additional docker client config file from " + clientConfig);
+            "Failed to read additional docker client config file: " + clientConfig, e);
       }
     }
     return additionalDockerCredentials;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java

@@ -58,6 +58,7 @@
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
 import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -118,7 +119,9 @@
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime.CONTAINER_PID_NAMESPACE_SUFFIX;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime.RUN_PRIVILEGED_CONTAINER_SUFFIX;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime.formatOciEnvKey;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
@@ -2512,6 +2515,72 @@ public void testLaunchContainerWithAdditionalDockerClientConfig(boolean pHttps)
     testLaunchContainer(null, getDockerClientConfigFile());
   }
 
+  @Test
+  public void testLaunchContainerWithBigDockerClientConfig() throws Exception {
+    initHttps(false);
+    int maxSize = YarnConfiguration.DEFAULT_NM_DOCKER_CLIENT_CONFIG_MAX_SIZE_BYTES;
+    int invalidSize = maxSize + 1;
+
+    File file = new File(tmpPath, "docker-client-config-big");
+    try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))) {
+      bw.write("x".repeat(invalidSize));
+    }
+    RuntimeException e = assertThrows(RuntimeException.class,
+        () -> testLaunchContainer(null, file));
+    assertNotNull(e.getCause(), "Wrapped IOException cause expected");
+    assertThat(e.getCause().getMessage())
+        .contains("(" + invalidSize + " bytes) is larger than maximum allowed size ("
+            + maxSize + " bytes)");
+  }
+
+  @Test
+  public void testLaunchContainerWithDockerClientConfigAtMaxSize() throws Exception {
+    initHttps(false);
+    int maxSize = YarnConfiguration.DEFAULT_NM_DOCKER_CLIENT_CONFIG_MAX_SIZE_BYTES;
+
+    // Build a valid Docker client config JSON of exactly maxSize bytes by
+    // padding whitespace after the opening brace (JSON ignores whitespace
+    // between tokens). This verifies the boundary is accepted by the size
+    // gate and that the resulting content still parses as JSON downstream.
+    String base = TestDockerClientConfigHandler.JSON;
+    int padLen = maxSize - base.length();
+    assertTrue(padLen >= 0,
+        "Base JSON (" + base.length() + " bytes) must fit within maxSize ("
+            + maxSize + " bytes) for this test to be meaningful");
+    String content = "{" + " ".repeat(padLen) + base.substring(1);
+    assertEquals(maxSize, content.getBytes(StandardCharsets.UTF_8).length,
+        "Padded JSON must be exactly maxSize bytes");
+
+    File file = new File(tmpPath, "docker-client-config-boundary");
+    try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))) {
+      bw.write(content);
+    }
+    // Must NOT throw the size-gate error. Container launch should proceed
+    // through the normal path with this boundary-sized valid config.
+    testLaunchContainer(null, file);
+  }
+
+  @Test
+  public void testLaunchContainerWithInvalidMaxSizeFallsBackToDefault()
+      throws Exception {
+    initHttps(false);
+
+    conf.setInt(YarnConfiguration.NM_DOCKER_CLIENT_CONFIG_MAX_SIZE_BYTES, -1);
+    int defaultMax = YarnConfiguration.DEFAULT_NM_DOCKER_CLIENT_CONFIG_MAX_SIZE_BYTES;
+    int invalidSize = defaultMax + 1;
+
+    File file = new File(tmpPath, "docker-client-neg-max");
+    try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))) {
+      bw.write("x".repeat(invalidSize));
+    }
+    RuntimeException e = assertThrows(RuntimeException.class,
+        () -> testLaunchContainer(null, file));
+    assertNotNull(e.getCause(), "Wrapped IOException cause expected");
+    assertThat(e.getCause().getMessage())
+        .contains("(" + invalidSize + " bytes) is larger than maximum allowed size ("
+            + defaultMax + " bytes)");
+  }
+
   public void testLaunchContainer(ByteBuffer tokens, File dockerConfigFile)
       throws ContainerExecutionException, PrivilegedOperationException,
       IOException {