From d8a001916bb9691b59da95152acf334c6cac3b16 Mon Sep 17 00:00:00 2001 From: Sergei Nikolaev Date: Sat, 21 Mar 2026 16:29:46 +0100 Subject: [PATCH] Spark: preload delete files to avoid deadlocks Signed-off-by: Sergei Nikolaev --- .../java/org/apache/iceberg/spark/source/BaseReader.java | 5 +++++ .../java/org/apache/iceberg/spark/source/BaseReader.java | 5 +++++ .../java/org/apache/iceberg/spark/source/BaseReader.java | 5 +++++ .../java/org/apache/iceberg/spark/source/BaseReader.java | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index 150d4bf28174..2fd23e9491bc 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -219,6 +219,11 @@ protected class SparkDeleteFilter extends DeleteFilter { this.asStructLike = new InternalRowWrapper( SparkSchemaUtil.convert(requiredSchema()), requiredSchema().asStruct()); + // Preload delete files to avoid deadlocking the connection pool: + // data file loading holds a connection while + // lazy delete file loading tries to acquire another. + deletedRowPositions(); + eqDeletedRowFilter(); } @Override diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index 0333f1e45dd4..8f304e96bf4d 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -209,6 +209,11 @@ protected class SparkDeleteFilter extends DeleteFilter { this.asStructLike = new InternalRowWrapper( SparkSchemaUtil.convert(requiredSchema()), requiredSchema().asStruct()); + // Preload delete files to avoid deadlocking the connection pool: + // data file loading holds a connection while + // lazy delete file loading tries to acquire another. + deletedRowPositions(); + eqDeletedRowFilter(); } @Override diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index 0333f1e45dd4..8f304e96bf4d 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -209,6 +209,11 @@ protected class SparkDeleteFilter extends DeleteFilter { this.asStructLike = new InternalRowWrapper( SparkSchemaUtil.convert(requiredSchema()), requiredSchema().asStruct()); + // Preload delete files to avoid deadlocking the connection pool: + // data file loading holds a connection while + // lazy delete file loading tries to acquire another. + deletedRowPositions(); + eqDeletedRowFilter(); } @Override diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index a4d9766ae713..9c4ba314c5c6 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -208,6 +208,11 @@ protected class SparkDeleteFilter extends DeleteFilter { this.asStructLike = new InternalRowWrapper( SparkSchemaUtil.convert(requiredSchema()), requiredSchema().asStruct()); + // Preload delete files to avoid deadlocking the connection pool: + // data file loading holds a connection while + // lazy delete file loading tries to acquire another. + deletedRowPositions(); + eqDeletedRowFilter(); } @Override