split cmd for better caching

Author: kevinjqliu

dev/spark/Dockerfile

@@ -41,25 +41,27 @@ COPY --chown=spark:spark spark-defaults.conf ${SPARK_HOME}/conf/
 RUN mkdir -p /home/iceberg/spark-events && \
     chown -R spark:spark /home/iceberg
 
-# Required JAR dependencies
-ENV JARS_TO_DOWNLOAD="\
-    org/apache/spark/spark-connect_${SCALA_VERSION}/${SPARK_VERSION}/spark-connect_${SCALA_VERSION}-${SPARK_VERSION}.jar \
-    org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
-    org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \
-    org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar \
-    software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar"
+# Download version-independent JARs first (cached across Iceberg version changes)
+RUN cd "${SPARK_HOME}/jars" && \
+    curl -fsSL --retry 3 --retry-delay 5 \
+         -o "spark-connect_${SCALA_VERSION}-${SPARK_VERSION}.jar" \
+         "${MAVEN_MIRROR}/org/apache/spark/spark-connect_${SCALA_VERSION}/${SPARK_VERSION}/spark-connect_${SCALA_VERSION}-${SPARK_VERSION}.jar" && \
+    curl -fsSL --retry 3 --retry-delay 5 \
+         -o "hadoop-aws-${HADOOP_VERSION}.jar" \
+         "${MAVEN_MIRROR}/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar" && \
+    curl -fsSL --retry 3 --retry-delay 5 \
+         -o "bundle-${AWS_SDK_VERSION}.jar" \
+         "${MAVEN_MIRROR}/software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar" && \
+    chown -R spark:spark "${SPARK_HOME}/jars"
 
-# Download JARs with retry logic
-RUN set -e && \
-    cd "${SPARK_HOME}/jars" && \
-    for jar_path in ${JARS_TO_DOWNLOAD}; do \
-        jar_name=$(basename "${jar_path}") && \
-        echo "Downloading ${jar_name}..." && \
-        curl -fsSL --retry 3 --retry-delay 5 \
-             -o "${jar_name}" \
-             "${MAVEN_MIRROR}/${jar_path}" && \
-        echo "✓ Downloaded ${jar_name}"; \
-    done && \
+# Download Iceberg-specific JARs (only invalidated when ICEBERG_VERSION changes)
+RUN cd "${SPARK_HOME}/jars" && \
+    curl -fsSL --retry 3 --retry-delay 5 \
+         -o "iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar" \
+         "${MAVEN_MIRROR}/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar" && \
+    curl -fsSL --retry 3 --retry-delay 5 \
+         -o "iceberg-aws-bundle-${ICEBERG_VERSION}.jar" \
+         "${MAVEN_MIRROR}/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar" && \
     chown -R spark:spark "${SPARK_HOME}/jars"
 
 USER spark