From 66dfb39ee39a0a1658dcfa1632f0ddd0a92eac2a Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Wed, 22 Oct 2025 14:32:13 +0200 Subject: [PATCH 01/33] create project skeleton --- .../iceberg-1.6/library/build.gradle.kts | 7 +++++++ .../iceberg/v1_6/IcebergTelemetry.java | 21 +++++++++++++++++++ .../iceberg/v1_6/IcebergTelemetryBuilder.java | 14 +++++++++++++ settings.gradle.kts | 1 + 4 files changed, 43 insertions(+) create mode 100644 instrumentation/iceberg-1.6/library/build.gradle.kts create mode 100644 instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java create mode 100644 instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetryBuilder.java diff --git a/instrumentation/iceberg-1.6/library/build.gradle.kts b/instrumentation/iceberg-1.6/library/build.gradle.kts new file mode 100644 index 000000000000..71ebfd13a7c4 --- /dev/null +++ b/instrumentation/iceberg-1.6/library/build.gradle.kts @@ -0,0 +1,7 @@ +plugins { + id("otel.library-instrumentation") +} + +dependencies { + implementation("org.apache.iceberg:iceberg-api:1.6.1") +} \ No newline at end of file diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java new file mode 100644 index 000000000000..d35ea54f4be5 --- /dev/null +++ b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java @@ -0,0 +1,21 @@ +package opentlelemetry.instrumentation.iceberg.v1_6; + +import org.omg.PortableInterceptor.Interceptor; + +import io.opentelemetry.api.OpenTelemetry; + +public class IcebergTelemetry { + public static IcebergTelemetry create(OpenTelemetry openTelemetry) { + return builder(openTelemetry).build(); + } + + public static IcebergTelemetryBuilder builder(OpenTelemetry openTelemetry) { + return new IcebergTelemetryBuilder(openTelemetry); + } + + IcebergTelemetry() {} + + public Interceptor newTracingInterceptor() { + return null; + } +} diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetryBuilder.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetryBuilder.java new file mode 100644 index 000000000000..4fdab6101fff --- /dev/null +++ b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetryBuilder.java @@ -0,0 +1,14 @@ +package opentlelemetry.instrumentation.iceberg.v1_6; + +import io.opentelemetry.api.OpenTelemetry; + +public class IcebergTelemetryBuilder { + + IcebergTelemetryBuilder(OpenTelemetry openTelemetry) { + + } + + public IcebergTelemetry build() { + return null; + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index a961f5e4a209..ad67626a6954 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -264,6 +264,7 @@ include(":instrumentation:hikaricp-3.0:library") include(":instrumentation:hikaricp-3.0:testing") include(":instrumentation:http-url-connection:javaagent") include(":instrumentation:hystrix-1.4:javaagent") +include(":instrumentation:iceberg-1.6:library") include(":instrumentation:influxdb-2.4:javaagent") include(":instrumentation:internal:internal-application-logger:bootstrap") include(":instrumentation:internal:internal-application-logger:javaagent") From cfa569502f867cb9a1e02d09086f4911f22c1ef0 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Wed, 22 Oct 2025 17:52:38 +0200 Subject: [PATCH 02/33] import correct iceberg library --- .../iceberg-1.6/library/build.gradle.kts | 2 +- .../iceberg/v1_6/IcebergMetrics.java | 25 +++++++++++++++++++ .../iceberg/v1_6/IcebergTelemetry.java | 14 ++++------- .../iceberg/v1_6/IcebergTelemetryBuilder.java | 14 ----------- 4 files changed, 31 insertions(+), 24 deletions(-) create mode 100644 instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetrics.java delete mode 100644 instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetryBuilder.java diff --git a/instrumentation/iceberg-1.6/library/build.gradle.kts b/instrumentation/iceberg-1.6/library/build.gradle.kts index 71ebfd13a7c4..74bf4de61588 100644 --- a/instrumentation/iceberg-1.6/library/build.gradle.kts +++ b/instrumentation/iceberg-1.6/library/build.gradle.kts @@ -3,5 +3,5 @@ plugins { } dependencies { - implementation("org.apache.iceberg:iceberg-api:1.6.1") + implementation("org.apache.iceberg:iceberg-core:1.6.1") } \ No newline at end of file diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetrics.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetrics.java new file mode 100644 index 000000000000..46ba6a4fc3f0 --- /dev/null +++ b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetrics.java @@ -0,0 +1,25 @@ +package opentlelemetry.instrumentation.iceberg.v1_6; + +import org.apache.iceberg.metrics.MetricsReport; +import org.apache.iceberg.metrics.MetricsReporter; +import org.apache.iceberg.metrics.ScanReport; + +import io.opentelemetry.api.OpenTelemetry; + +public class IcebergMetrics implements MetricsReporter { + private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg-1.6"; + + private final OpenTelemetry openTelemetry; + + IcebergMetrics(OpenTelemetry openTelemetry) { + this.openTelemetry = openTelemetry; + } + + @Override + public void report(MetricsReport report) { + if (report instanceof ScanReport) { + + } + } + +} diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java index d35ea54f4be5..535191dbb59b 100644 --- a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java +++ b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java @@ -1,21 +1,17 @@ package opentlelemetry.instrumentation.iceberg.v1_6; -import org.omg.PortableInterceptor.Interceptor; import io.opentelemetry.api.OpenTelemetry; public class IcebergTelemetry { + private final OpenTelemetry openTelemetry; + public static IcebergTelemetry create(OpenTelemetry openTelemetry) { - return builder(openTelemetry).build(); + return new IcebergTelemetry(openTelemetry); } - public static IcebergTelemetryBuilder builder(OpenTelemetry openTelemetry) { - return new IcebergTelemetryBuilder(openTelemetry); + IcebergTelemetry(OpenTelemetry openTelemetry) { + this.openTelemetry = openTelemetry; } - IcebergTelemetry() {} - - public Interceptor newTracingInterceptor() { - return null; - } } diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetryBuilder.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetryBuilder.java deleted file mode 100644 index 4fdab6101fff..000000000000 --- a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetryBuilder.java +++ /dev/null @@ -1,14 +0,0 @@ -package opentlelemetry.instrumentation.iceberg.v1_6; - -import io.opentelemetry.api.OpenTelemetry; - -public class IcebergTelemetryBuilder { - - IcebergTelemetryBuilder(OpenTelemetry openTelemetry) { - - } - - public IcebergTelemetry build() { - return null; - } -} From caca4af3286ebe8bdd5f4fb8afe83ccb21eb53a8 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Thu, 23 Oct 2025 18:23:22 +0200 Subject: [PATCH 03/33] initial ScanMetrics builders implementation --- .../iceberg/v1_6/IcebergMetrics.java | 25 ----- .../v1_6/internal/IcebergMetricsReporter.java | 43 +++++++++ .../v1_6/internal/ScanMetricsBuilder.java | 96 +++++++++++++++++++ 3 files changed, 139 insertions(+), 25 deletions(-) delete mode 100644 instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetrics.java create mode 100644 instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/IcebergMetricsReporter.java create mode 100644 instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/ScanMetricsBuilder.java diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetrics.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetrics.java deleted file mode 100644 index 46ba6a4fc3f0..000000000000 --- a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetrics.java +++ /dev/null @@ -1,25 +0,0 @@ -package opentlelemetry.instrumentation.iceberg.v1_6; - -import org.apache.iceberg.metrics.MetricsReport; -import org.apache.iceberg.metrics.MetricsReporter; -import org.apache.iceberg.metrics.ScanReport; - -import io.opentelemetry.api.OpenTelemetry; - -public class IcebergMetrics implements MetricsReporter { - private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg-1.6"; - - private final OpenTelemetry openTelemetry; - - IcebergMetrics(OpenTelemetry openTelemetry) { - this.openTelemetry = openTelemetry; - } - - @Override - public void report(MetricsReport report) { - if (report instanceof ScanReport) { - - } - } - -} diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/IcebergMetricsReporter.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/IcebergMetricsReporter.java new file mode 100644 index 000000000000..7f1331256b7e --- /dev/null +++ b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/IcebergMetricsReporter.java @@ -0,0 +1,43 @@ +package opentlelemetry.instrumentation.iceberg.v1_6.internal; + +import org.apache.iceberg.metrics.CommitReport; +import org.apache.iceberg.metrics.MetricsReport; +import org.apache.iceberg.metrics.MetricsReporter; +import org.apache.iceberg.metrics.ScanMetricsResult; +import org.apache.iceberg.metrics.ScanReport; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.DoubleGauge; + +public class IcebergMetricsReporter implements MetricsReporter { + private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg_1.6"; + + private final OpenTelemetry openTelemetry; + + IcebergMetricsReporter(OpenTelemetry openTelemetry) { + this.openTelemetry = openTelemetry; + } + + @Override + public void report(MetricsReport report) { + if (report instanceof ScanReport) { + reportScanMetrics((ScanReport) report); + } else if (report instanceof CommitReport) { + reportCommitMetrics((CommitReport) report); + } + } + + void reportScanMetrics(ScanReport scanReport) { + final ScanMetricsResult metrics = scanReport.scanMetrics(); + + if (metrics.indexedDeleteFiles() != null) { + DoubleGauge metric = ScanMetricsBuilder.indexedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(metrics.indexedDeleteFiles().value()); + } + } + + void reportCommitMetrics(CommitReport commitReport) { + + } + +} diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/ScanMetricsBuilder.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/ScanMetricsBuilder.java new file mode 100644 index 000000000000..8e805e545b14 --- /dev/null +++ b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/ScanMetricsBuilder.java @@ -0,0 +1,96 @@ +package opentlelemetry.instrumentation.iceberg.v1_6.internal; + +import io.opentelemetry.api.metrics.DoubleGauge; +import io.opentelemetry.api.metrics.Meter; + +public class ScanMetricsBuilder { + private static final String ROOT = "iceberg.scan"; + private static final String TOTAL_PLANNING_DURATION = ROOT + ".planning.duration"; + private static final String RESULT_DATA_FILES = ROOT + ".scanned.data_files.count"; + private static final String RESULT_DELETE_FILES = ROOT + ".scanned.delete_files.count"; + private static final String SCANNED_DATA_MANIFESTS = ROOT + ".scanned.data_manifests.count"; + private static final String SCANNED_DELETE_MANIFESTS = ROOT + ".scanned.delete_manifests.count"; + private static final String TOTAL_DATA_MANIFESTS = ROOT + ".total.data_manifests.count"; + private static final String TOTAL_DELETE_MANIFESTS = ROOT + ".total.delete_manifests.count"; + private static final String TOTAL_FILE_SIZE_IN_BYTES = ROOT + ".scanned.data_files.size"; + private static final String TOTAL_DELETE_FILE_SIZE_IN_BYTES = ROOT + "scanned.delete_files.size"; + private static final String SKIPPED_DATA_MANIFESTS = ROOT + ".skipped.data_manifests.count"; + private static final String SKIPPED_DELETE_MANIFESTS = ROOT + ".skipped.delete_manifests.count"; + private static final String SKIPPED_DATA_FILES = ROOT + ".skipped.data_files.count"; + private static final String SKIPPED_DELETE_FILES = ROOT + ".skipped.delete_files.count"; + private static final String INDEXED_DELETE_FILES = ROOT + ".scanned.indexed_delete_files.count"; + private static final String EQUALITY_DELETE_FILES = ROOT + ".scanned.equality_delete_files.count"; + private static final String POSITIONAL_DELETE_FILES = ROOT + ".scanned.positional_delete_files.count"; + private static final String DVS = ROOT + ".scanned.dvs.count"; + + + static DoubleGauge totalPlanningDuration(Meter meter) { + return meter.gaugeBuilder(TOTAL_PLANNING_DURATION).setDescription("The total duration needed to plan the scan.").setUnit("ms").build(); + } + + static DoubleGauge scannedDataFilesCount(Meter meter) { + return meter.gaugeBuilder(RESULT_DATA_FILES).setDescription("The number of scanned data files.").setUnit("{file}").build(); + } + + static DoubleGauge scannedDeleteFilesCount(Meter meter) { + return meter.gaugeBuilder(RESULT_DELETE_FILES).setDescription("The number of scanned delete files.").setUnit("{file}").build(); + } + + static DoubleGauge scannedDataManifestsCount(Meter meter) { + return meter.gaugeBuilder(SCANNED_DATA_MANIFESTS).setDescription("The number of scanned data manifests.").setUnit("{file}").build(); + } + + static DoubleGauge scannedDeleteManifestsCount(Meter meter) { + return meter.gaugeBuilder(SCANNED_DELETE_MANIFESTS).setDescription("The number of scanned delete manifests.").setUnit("{file}").build(); + } + + static DoubleGauge totalDataManifestsCount(Meter meter) { + return meter.gaugeBuilder(TOTAL_DATA_MANIFESTS).setDescription("The number of all data manifests.").setUnit("{file}").build(); + } + + static DoubleGauge totalDeleteManifestsCount(Meter meter) { + return meter.gaugeBuilder(TOTAL_DELETE_MANIFESTS).setDescription("The number of all delete manifests.").setUnit("{file}").build(); + } + + static DoubleGauge scannedDataFilesSize(Meter meter) { + return meter.gaugeBuilder(TOTAL_FILE_SIZE_IN_BYTES).setDescription("The total size of all scanned data files.").setUnit("byte").build(); + } + + static DoubleGauge scannedDeleteFilesSize(Meter meter) { + return meter.gaugeBuilder(TOTAL_DELETE_FILE_SIZE_IN_BYTES).setDescription("The total size of all scanned delete files.").setUnit("byte").build(); + } + + static DoubleGauge skippedDataManifests(Meter meter) { + return meter.gaugeBuilder(SKIPPED_DATA_MANIFESTS).setDescription("The number of data manifests that were skipped during the scan.").setUnit("{file}").build(); + } + + static DoubleGauge skippedDeleteManifests(Meter meter) { + return meter.gaugeBuilder(SKIPPED_DELETE_MANIFESTS).setDescription("The number of delete manifests that were skipped during the scan.").setUnit("{file}").build(); + } + + static DoubleGauge skippedDataFiles(Meter meter) { + return meter.gaugeBuilder(SKIPPED_DATA_FILES).setDescription("The number of data files that were skipped during the scan.").setUnit("{file}").build(); + } + + static DoubleGauge skippedDeleteFiles(Meter meter) { + return meter.gaugeBuilder(SKIPPED_DELETE_FILES).setDescription("The number of delete files that were skipped during the scan.").setUnit("{file}").build(); + } + + static DoubleGauge indexedDeleteFiles(Meter meter) { + return meter.gaugeBuilder(INDEXED_DELETE_FILES).setDescription("The number of delete files constituting the delete file index for this scan.").setUnit("{file}").build(); + } + + static DoubleGauge equalityDeleteFiles(Meter meter) { + return meter.gaugeBuilder(EQUALITY_DELETE_FILES).setDescription("The number of equality delete files relevant for the current scan.").setUnit("{file}").build(); + } + + static DoubleGauge positionDeleteFiles(Meter meter) { + return meter.gaugeBuilder(POSITIONAL_DELETE_FILES).setDescription("The number of position delete files relevant for the current scan.").setUnit("{file}").build(); + } + + static DoubleGauge deletionVectorFiles(Meter meter) { + return meter.gaugeBuilder(DVS).setDescription("The number of deletion vector (DV) files relevant for the current scan.").setUnit("{file}").build(); + } + + +} From e56fb2e8a2bb0e25d20bab837eac31c4288c22d7 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Thu, 23 Oct 2025 19:23:13 +0200 Subject: [PATCH 04/33] apply spotless --- .../iceberg-1.6/library/build.gradle.kts | 2 +- .../IcebergMetricsReporter.java | 24 +-- .../iceberg/v1_6/IcebergTelemetry.java | 10 +- .../iceberg/v1_6/ScanMetricsBuilder.java | 168 ++++++++++++++++++ .../v1_6/internal/ScanMetricsBuilder.java | 96 ---------- 5 files changed, 191 insertions(+), 109 deletions(-) rename instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/{internal => }/IcebergMetricsReporter.java (77%) create mode 100644 instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/ScanMetricsBuilder.java delete mode 100644 instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/ScanMetricsBuilder.java diff --git a/instrumentation/iceberg-1.6/library/build.gradle.kts b/instrumentation/iceberg-1.6/library/build.gradle.kts index 74bf4de61588..845767fe3faf 100644 --- a/instrumentation/iceberg-1.6/library/build.gradle.kts +++ b/instrumentation/iceberg-1.6/library/build.gradle.kts @@ -4,4 +4,4 @@ plugins { dependencies { implementation("org.apache.iceberg:iceberg-core:1.6.1") -} \ No newline at end of file +} diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/IcebergMetricsReporter.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java similarity index 77% rename from instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/IcebergMetricsReporter.java rename to instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java index 7f1331256b7e..69de2019b361 100644 --- a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java @@ -1,17 +1,21 @@ -package opentlelemetry.instrumentation.iceberg.v1_6.internal; +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ +package opentlelemetry.instrumentation.iceberg.v1_6; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.DoubleGauge; import org.apache.iceberg.metrics.CommitReport; import org.apache.iceberg.metrics.MetricsReport; import org.apache.iceberg.metrics.MetricsReporter; import org.apache.iceberg.metrics.ScanMetricsResult; import org.apache.iceberg.metrics.ScanReport; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.metrics.DoubleGauge; - public class IcebergMetricsReporter implements MetricsReporter { private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg_1.6"; - + private final OpenTelemetry openTelemetry; IcebergMetricsReporter(OpenTelemetry openTelemetry) { @@ -24,20 +28,18 @@ public void report(MetricsReport report) { reportScanMetrics((ScanReport) report); } else if (report instanceof CommitReport) { reportCommitMetrics((CommitReport) report); - } + } } void reportScanMetrics(ScanReport scanReport) { final ScanMetricsResult metrics = scanReport.scanMetrics(); if (metrics.indexedDeleteFiles() != null) { - DoubleGauge metric = ScanMetricsBuilder.indexedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + DoubleGauge metric = + ScanMetricsBuilder.indexedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(metrics.indexedDeleteFiles().value()); } } - void reportCommitMetrics(CommitReport commitReport) { - - } - + void reportCommitMetrics(CommitReport commitReport) {} } diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java index 535191dbb59b..3859548493a0 100644 --- a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java +++ b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java @@ -1,7 +1,12 @@ -package opentlelemetry.instrumentation.iceberg.v1_6; +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ +package opentlelemetry.instrumentation.iceberg.v1_6; import io.opentelemetry.api.OpenTelemetry; +import org.apache.iceberg.TableScan; public class IcebergTelemetry { private final OpenTelemetry openTelemetry; @@ -14,4 +19,7 @@ public static IcebergTelemetry create(OpenTelemetry openTelemetry) { this.openTelemetry = openTelemetry; } + public TableScan wrapTableScan(TableScan tableScan) { + return tableScan.metricsReporter(new IcebergMetricsReporter(openTelemetry)); + } } diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/ScanMetricsBuilder.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/ScanMetricsBuilder.java new file mode 100644 index 000000000000..733069f3c1c4 --- /dev/null +++ b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/ScanMetricsBuilder.java @@ -0,0 +1,168 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package opentlelemetry.instrumentation.iceberg.v1_6; + +import io.opentelemetry.api.metrics.DoubleGauge; +import io.opentelemetry.api.metrics.Meter; + +public class ScanMetricsBuilder { + private static final String ROOT = "iceberg.scan"; + private static final String TOTAL_PLANNING_DURATION = ROOT + ".planning.duration"; + private static final String RESULT_DATA_FILES = ROOT + ".scanned.data_files.count"; + private static final String RESULT_DELETE_FILES = ROOT + ".scanned.delete_files.count"; + private static final String SCANNED_DATA_MANIFESTS = ROOT + ".scanned.data_manifests.count"; + private static final String SCANNED_DELETE_MANIFESTS = ROOT + ".scanned.delete_manifests.count"; + private static final String TOTAL_DATA_MANIFESTS = ROOT + ".total.data_manifests.count"; + private static final String TOTAL_DELETE_MANIFESTS = ROOT + ".total.delete_manifests.count"; + private static final String TOTAL_FILE_SIZE_IN_BYTES = ROOT + ".scanned.data_files.size"; + private static final String TOTAL_DELETE_FILE_SIZE_IN_BYTES = ROOT + "scanned.delete_files.size"; + private static final String SKIPPED_DATA_MANIFESTS = ROOT + ".skipped.data_manifests.count"; + private static final String SKIPPED_DELETE_MANIFESTS = ROOT + ".skipped.delete_manifests.count"; + private static final String SKIPPED_DATA_FILES = ROOT + ".skipped.data_files.count"; + private static final String SKIPPED_DELETE_FILES = ROOT + ".skipped.delete_files.count"; + private static final String INDEXED_DELETE_FILES = ROOT + ".scanned.indexed_delete_files.count"; + private static final String EQUALITY_DELETE_FILES = ROOT + ".scanned.equality_delete_files.count"; + private static final String POSITIONAL_DELETE_FILES = + ROOT + ".scanned.positional_delete_files.count"; + private static final String DVS = ROOT + ".scanned.dvs.count"; + + static DoubleGauge totalPlanningDuration(Meter meter) { + return meter + .gaugeBuilder(TOTAL_PLANNING_DURATION) + .setDescription("The total duration needed to plan the scan.") + .setUnit("ms") + .build(); + } + + static DoubleGauge scannedDataFilesCount(Meter meter) { + return meter + .gaugeBuilder(RESULT_DATA_FILES) + .setDescription("The number of scanned data files.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge scannedDeleteFilesCount(Meter meter) { + return meter + .gaugeBuilder(RESULT_DELETE_FILES) + .setDescription("The number of scanned delete files.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge scannedDataManifestsCount(Meter meter) { + return meter + .gaugeBuilder(SCANNED_DATA_MANIFESTS) + .setDescription("The number of scanned data manifests.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge scannedDeleteManifestsCount(Meter meter) { + return meter + .gaugeBuilder(SCANNED_DELETE_MANIFESTS) + .setDescription("The number of scanned delete manifests.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge totalDataManifestsCount(Meter meter) { + return meter + .gaugeBuilder(TOTAL_DATA_MANIFESTS) + .setDescription("The number of all data manifests.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge totalDeleteManifestsCount(Meter meter) { + return meter + .gaugeBuilder(TOTAL_DELETE_MANIFESTS) + .setDescription("The number of all delete manifests.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge scannedDataFilesSize(Meter meter) { + return meter + .gaugeBuilder(TOTAL_FILE_SIZE_IN_BYTES) + .setDescription("The total size of all scanned data files.") + .setUnit("byte") + .build(); + } + + static DoubleGauge scannedDeleteFilesSize(Meter meter) { + return meter + .gaugeBuilder(TOTAL_DELETE_FILE_SIZE_IN_BYTES) + .setDescription("The total size of all scanned delete files.") + .setUnit("byte") + .build(); + } + + static DoubleGauge skippedDataManifests(Meter meter) { + return meter + .gaugeBuilder(SKIPPED_DATA_MANIFESTS) + .setDescription("The number of data manifests that were skipped during the scan.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge skippedDeleteManifests(Meter meter) { + return meter + .gaugeBuilder(SKIPPED_DELETE_MANIFESTS) + .setDescription("The number of delete manifests that were skipped during the scan.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge skippedDataFiles(Meter meter) { + return meter + .gaugeBuilder(SKIPPED_DATA_FILES) + .setDescription("The number of data files that were skipped during the scan.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge skippedDeleteFiles(Meter meter) { + return meter + .gaugeBuilder(SKIPPED_DELETE_FILES) + .setDescription("The number of delete files that were skipped during the scan.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge indexedDeleteFiles(Meter meter) { + return meter + .gaugeBuilder(INDEXED_DELETE_FILES) + .setDescription( + "The number of delete files constituting the delete file index for this scan.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge equalityDeleteFiles(Meter meter) { + return meter + .gaugeBuilder(EQUALITY_DELETE_FILES) + .setDescription("The number of equality delete files relevant for the current scan.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge positionDeleteFiles(Meter meter) { + return meter + .gaugeBuilder(POSITIONAL_DELETE_FILES) + .setDescription("The number of position delete files relevant for the current scan.") + .setUnit("{file}") + .build(); + } + + static DoubleGauge deletionVectorFiles(Meter meter) { + return meter + .gaugeBuilder(DVS) + .setDescription("The number of deletion vector (DV) files relevant for the current scan.") + .setUnit("{file}") + .build(); + } +} diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/ScanMetricsBuilder.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/ScanMetricsBuilder.java deleted file mode 100644 index 8e805e545b14..000000000000 --- a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/internal/ScanMetricsBuilder.java +++ /dev/null @@ -1,96 +0,0 @@ -package opentlelemetry.instrumentation.iceberg.v1_6.internal; - -import io.opentelemetry.api.metrics.DoubleGauge; -import io.opentelemetry.api.metrics.Meter; - -public class ScanMetricsBuilder { - private static final String ROOT = "iceberg.scan"; - private static final String TOTAL_PLANNING_DURATION = ROOT + ".planning.duration"; - private static final String RESULT_DATA_FILES = ROOT + ".scanned.data_files.count"; - private static final String RESULT_DELETE_FILES = ROOT + ".scanned.delete_files.count"; - private static final String SCANNED_DATA_MANIFESTS = ROOT + ".scanned.data_manifests.count"; - private static final String SCANNED_DELETE_MANIFESTS = ROOT + ".scanned.delete_manifests.count"; - private static final String TOTAL_DATA_MANIFESTS = ROOT + ".total.data_manifests.count"; - private static final String TOTAL_DELETE_MANIFESTS = ROOT + ".total.delete_manifests.count"; - private static final String TOTAL_FILE_SIZE_IN_BYTES = ROOT + ".scanned.data_files.size"; - private static final String TOTAL_DELETE_FILE_SIZE_IN_BYTES = ROOT + "scanned.delete_files.size"; - private static final String SKIPPED_DATA_MANIFESTS = ROOT + ".skipped.data_manifests.count"; - private static final String SKIPPED_DELETE_MANIFESTS = ROOT + ".skipped.delete_manifests.count"; - private static final String SKIPPED_DATA_FILES = ROOT + ".skipped.data_files.count"; - private static final String SKIPPED_DELETE_FILES = ROOT + ".skipped.delete_files.count"; - private static final String INDEXED_DELETE_FILES = ROOT + ".scanned.indexed_delete_files.count"; - private static final String EQUALITY_DELETE_FILES = ROOT + ".scanned.equality_delete_files.count"; - private static final String POSITIONAL_DELETE_FILES = ROOT + ".scanned.positional_delete_files.count"; - private static final String DVS = ROOT + ".scanned.dvs.count"; - - - static DoubleGauge totalPlanningDuration(Meter meter) { - return meter.gaugeBuilder(TOTAL_PLANNING_DURATION).setDescription("The total duration needed to plan the scan.").setUnit("ms").build(); - } - - static DoubleGauge scannedDataFilesCount(Meter meter) { - return meter.gaugeBuilder(RESULT_DATA_FILES).setDescription("The number of scanned data files.").setUnit("{file}").build(); - } - - static DoubleGauge scannedDeleteFilesCount(Meter meter) { - return meter.gaugeBuilder(RESULT_DELETE_FILES).setDescription("The number of scanned delete files.").setUnit("{file}").build(); - } - - static DoubleGauge scannedDataManifestsCount(Meter meter) { - return meter.gaugeBuilder(SCANNED_DATA_MANIFESTS).setDescription("The number of scanned data manifests.").setUnit("{file}").build(); - } - - static DoubleGauge scannedDeleteManifestsCount(Meter meter) { - return meter.gaugeBuilder(SCANNED_DELETE_MANIFESTS).setDescription("The number of scanned delete manifests.").setUnit("{file}").build(); - } - - static DoubleGauge totalDataManifestsCount(Meter meter) { - return meter.gaugeBuilder(TOTAL_DATA_MANIFESTS).setDescription("The number of all data manifests.").setUnit("{file}").build(); - } - - static DoubleGauge totalDeleteManifestsCount(Meter meter) { - return meter.gaugeBuilder(TOTAL_DELETE_MANIFESTS).setDescription("The number of all delete manifests.").setUnit("{file}").build(); - } - - static DoubleGauge scannedDataFilesSize(Meter meter) { - return meter.gaugeBuilder(TOTAL_FILE_SIZE_IN_BYTES).setDescription("The total size of all scanned data files.").setUnit("byte").build(); - } - - static DoubleGauge scannedDeleteFilesSize(Meter meter) { - return meter.gaugeBuilder(TOTAL_DELETE_FILE_SIZE_IN_BYTES).setDescription("The total size of all scanned delete files.").setUnit("byte").build(); - } - - static DoubleGauge skippedDataManifests(Meter meter) { - return meter.gaugeBuilder(SKIPPED_DATA_MANIFESTS).setDescription("The number of data manifests that were skipped during the scan.").setUnit("{file}").build(); - } - - static DoubleGauge skippedDeleteManifests(Meter meter) { - return meter.gaugeBuilder(SKIPPED_DELETE_MANIFESTS).setDescription("The number of delete manifests that were skipped during the scan.").setUnit("{file}").build(); - } - - static DoubleGauge skippedDataFiles(Meter meter) { - return meter.gaugeBuilder(SKIPPED_DATA_FILES).setDescription("The number of data files that were skipped during the scan.").setUnit("{file}").build(); - } - - static DoubleGauge skippedDeleteFiles(Meter meter) { - return meter.gaugeBuilder(SKIPPED_DELETE_FILES).setDescription("The number of delete files that were skipped during the scan.").setUnit("{file}").build(); - } - - static DoubleGauge indexedDeleteFiles(Meter meter) { - return meter.gaugeBuilder(INDEXED_DELETE_FILES).setDescription("The number of delete files constituting the delete file index for this scan.").setUnit("{file}").build(); - } - - static DoubleGauge equalityDeleteFiles(Meter meter) { - return meter.gaugeBuilder(EQUALITY_DELETE_FILES).setDescription("The number of equality delete files relevant for the current scan.").setUnit("{file}").build(); - } - - static DoubleGauge positionDeleteFiles(Meter meter) { - return meter.gaugeBuilder(POSITIONAL_DELETE_FILES).setDescription("The number of position delete files relevant for the current scan.").setUnit("{file}").build(); - } - - static DoubleGauge deletionVectorFiles(Meter meter) { - return meter.gaugeBuilder(DVS).setDescription("The number of deletion vector (DV) files relevant for the current scan.").setUnit("{file}").build(); - } - - -} From d900bd4f0e6aa71e5cdb4878df8d6505c3a94c79 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Fri, 24 Oct 2025 17:28:30 +0200 Subject: [PATCH 05/33] update iceberg version --- .fossa.yml | 3 + .../iceberg/v1_6/IcebergMetricsReporter.java | 45 ----- .../library/build.gradle.kts | 2 +- .../iceberg/v1_6/IcebergMetricsReporter.java | 178 ++++++++++++++++++ .../iceberg/v1_6/IcebergTelemetry.java | 0 .../iceberg/v1_6/ScanMetricsBuilder.java | 0 settings.gradle.kts | 2 +- 7 files changed, 183 insertions(+), 47 deletions(-) delete mode 100644 instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java rename instrumentation/{iceberg-1.6 => iceberg-1.8}/library/build.gradle.kts (53%) create mode 100644 instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java rename instrumentation/{iceberg-1.6 => iceberg-1.8}/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java (100%) rename instrumentation/{iceberg-1.6 => iceberg-1.8}/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/ScanMetricsBuilder.java (100%) diff --git a/.fossa.yml b/.fossa.yml index 161ceb50792f..6f5dd0dfe0a6 100644 --- a/.fossa.yml +++ b/.fossa.yml @@ -154,6 +154,9 @@ targets: - type: gradle path: ./ target: ':instrumentation:hystrix-1.4:javaagent' + - type: gradle + path: ./ + target: ':instrumentation:iceberg-1.8:library' - type: gradle path: ./ target: ':instrumentation:influxdb-2.4:javaagent' diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java b/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java deleted file mode 100644 index 69de2019b361..000000000000 --- a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package opentlelemetry.instrumentation.iceberg.v1_6; - -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.metrics.DoubleGauge; -import org.apache.iceberg.metrics.CommitReport; -import org.apache.iceberg.metrics.MetricsReport; -import org.apache.iceberg.metrics.MetricsReporter; -import org.apache.iceberg.metrics.ScanMetricsResult; -import org.apache.iceberg.metrics.ScanReport; - -public class IcebergMetricsReporter implements MetricsReporter { - private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg_1.6"; - - private final OpenTelemetry openTelemetry; - - IcebergMetricsReporter(OpenTelemetry openTelemetry) { - this.openTelemetry = openTelemetry; - } - - @Override - public void report(MetricsReport report) { - if (report instanceof ScanReport) { - reportScanMetrics((ScanReport) report); - } else if (report instanceof CommitReport) { - reportCommitMetrics((CommitReport) report); - } - } - - void reportScanMetrics(ScanReport scanReport) { - final ScanMetricsResult metrics = scanReport.scanMetrics(); - - if (metrics.indexedDeleteFiles() != null) { - DoubleGauge metric = - ScanMetricsBuilder.indexedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(metrics.indexedDeleteFiles().value()); - } - } - - void reportCommitMetrics(CommitReport commitReport) {} -} diff --git a/instrumentation/iceberg-1.6/library/build.gradle.kts b/instrumentation/iceberg-1.8/library/build.gradle.kts similarity index 53% rename from instrumentation/iceberg-1.6/library/build.gradle.kts rename to instrumentation/iceberg-1.8/library/build.gradle.kts index 845767fe3faf..6b534d00a2f9 100644 --- a/instrumentation/iceberg-1.6/library/build.gradle.kts +++ b/instrumentation/iceberg-1.8/library/build.gradle.kts @@ -3,5 +3,5 @@ plugins { } dependencies { - implementation("org.apache.iceberg:iceberg-core:1.6.1") + implementation("org.apache.iceberg:iceberg-core:1.8.1") } diff --git a/instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java new file mode 100644 index 000000000000..ec3999233500 --- /dev/null +++ b/instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java @@ -0,0 +1,178 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package opentlelemetry.instrumentation.iceberg.v1_6; + +import org.apache.iceberg.metrics.CommitReport; +import org.apache.iceberg.metrics.CounterResult; +import org.apache.iceberg.metrics.MetricsReport; +import org.apache.iceberg.metrics.MetricsReporter; +import org.apache.iceberg.metrics.ScanMetricsResult; +import org.apache.iceberg.metrics.ScanReport; +import org.apache.iceberg.metrics.TimerResult; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.DoubleGauge; + +public class IcebergMetricsReporter implements MetricsReporter { + private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg_1.6"; + + private final OpenTelemetry openTelemetry; + + IcebergMetricsReporter(OpenTelemetry openTelemetry) { + this.openTelemetry = openTelemetry; + } + + @Override + public void report(MetricsReport report) { + if (report instanceof ScanReport) { + reportScanMetrics((ScanReport) report); + } else if (report instanceof CommitReport) { + reportCommitMetrics((CommitReport) report); + } + } + + void reportScanMetrics(ScanReport scanReport) { + final ScanMetricsResult metrics = scanReport.scanMetrics(); + TimerResult duration = metrics.totalPlanningDuration(); + + if (duration != null) { + DoubleGauge metric = + ScanMetricsBuilder.totalPlanningDuration(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(duration.totalDuration().toMillis()); + } + + CounterResult current = metrics.resultDataFiles(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.scannedDataFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.resultDeleteFiles(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.scannedDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.scannedDataManifests(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.scannedDataManifestsCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.scannedDeleteManifests(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.scannedDeleteManifestsCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.totalDataManifests(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.totalDataManifestsCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.totalDeleteManifests(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.totalDeleteManifestsCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.totalFileSizeInBytes(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.scannedDataFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.totalDeleteFileSizeInBytes(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.scannedDeleteFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.skippedDataManifests(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.skippedDataManifests(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.skippedDeleteManifests(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.skippedDeleteManifests(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.skippedDataFiles(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.skippedDataFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.skippedDeleteFiles(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.skippedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.indexedDeleteFiles(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.indexedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.equalityDeleteFiles(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.equalityDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.positionalDeleteFiles(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.positionDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + current = metrics.dvs(); + + if (current != null) { + DoubleGauge metric = + ScanMetricsBuilder.deletionVectorFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(current.value()); + } + + } + + void reportCommitMetrics(CommitReport commitReport) {} +} diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java b/instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java similarity index 100% rename from instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java rename to instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java diff --git a/instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/ScanMetricsBuilder.java b/instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/ScanMetricsBuilder.java similarity index 100% rename from instrumentation/iceberg-1.6/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/ScanMetricsBuilder.java rename to instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/ScanMetricsBuilder.java diff --git a/settings.gradle.kts b/settings.gradle.kts index 50ca2bd4de87..54de8be89101 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -303,7 +303,7 @@ include(":instrumentation:hikaricp-3.0:library") include(":instrumentation:hikaricp-3.0:testing") include(":instrumentation:http-url-connection:javaagent") include(":instrumentation:hystrix-1.4:javaagent") -include(":instrumentation:iceberg-1.6:library") +include(":instrumentation:iceberg-1.8:library") include(":instrumentation:influxdb-2.4:javaagent") include(":instrumentation:internal:internal-application-logger:bootstrap") include(":instrumentation:internal:internal-application-logger:javaagent") From ad6c673fbf68b6acaebe22afc1a9b3a30c123fc5 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Thu, 30 Oct 2025 10:09:18 +0100 Subject: [PATCH 06/33] fix some build issues --- instrumentation/iceberg-1.8/library/build.gradle.kts | 2 +- .../iceberg/v1_8}/IcebergMetricsReporter.java | 12 +++++++----- .../iceberg/v1_8}/IcebergTelemetry.java | 5 +++-- .../iceberg/v1_8}/ScanMetricsBuilder.java | 8 ++++++-- 4 files changed, 17 insertions(+), 10 deletions(-) rename instrumentation/iceberg-1.8/library/src/main/java/{opentlelemetry/instrumentation/iceberg/v1_6 => io/opentelemetry/instrumentation/iceberg/v1_8}/IcebergMetricsReporter.java (92%) rename instrumentation/iceberg-1.8/library/src/main/java/{opentlelemetry/instrumentation/iceberg/v1_6 => io/opentelemetry/instrumentation/iceberg/v1_8}/IcebergTelemetry.java (91%) rename instrumentation/iceberg-1.8/library/src/main/java/{opentlelemetry/instrumentation/iceberg/v1_6 => io/opentelemetry/instrumentation/iceberg/v1_8}/ScanMetricsBuilder.java (97%) diff --git a/instrumentation/iceberg-1.8/library/build.gradle.kts b/instrumentation/iceberg-1.8/library/build.gradle.kts index 6b534d00a2f9..fab8da96b885 100644 --- a/instrumentation/iceberg-1.8/library/build.gradle.kts +++ b/instrumentation/iceberg-1.8/library/build.gradle.kts @@ -3,5 +3,5 @@ plugins { } dependencies { - implementation("org.apache.iceberg:iceberg-core:1.8.1") + library("org.apache.iceberg:iceberg-core:1.8.1") } diff --git a/instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java similarity index 92% rename from instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java rename to instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index ec3999233500..de3fdb2943dd 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package opentlelemetry.instrumentation.iceberg.v1_6; +package io.opentelemetry.instrumentation.iceberg.v1_8; import org.apache.iceberg.metrics.CommitReport; import org.apache.iceberg.metrics.CounterResult; @@ -64,7 +64,8 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { DoubleGauge metric = - ScanMetricsBuilder.scannedDataManifestsCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + ScanMetricsBuilder.scannedDataManifestsCount( + openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -72,7 +73,8 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { DoubleGauge metric = - ScanMetricsBuilder.scannedDeleteManifestsCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + ScanMetricsBuilder.scannedDeleteManifestsCount( + openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -88,7 +90,8 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { DoubleGauge metric = - ScanMetricsBuilder.totalDeleteManifestsCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + ScanMetricsBuilder.totalDeleteManifestsCount( + openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -171,7 +174,6 @@ void reportScanMetrics(ScanReport scanReport) { ScanMetricsBuilder.deletionVectorFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } - } void reportCommitMetrics(CommitReport commitReport) {} diff --git a/instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java similarity index 91% rename from instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java rename to instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java index 3859548493a0..8642cc866bf6 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/IcebergTelemetry.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java @@ -3,11 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -package opentlelemetry.instrumentation.iceberg.v1_6; +package io.opentelemetry.instrumentation.iceberg.v1_8; -import io.opentelemetry.api.OpenTelemetry; import org.apache.iceberg.TableScan; +import io.opentelemetry.api.OpenTelemetry; + public class IcebergTelemetry { private final OpenTelemetry openTelemetry; diff --git a/instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/ScanMetricsBuilder.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java similarity index 97% rename from instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/ScanMetricsBuilder.java rename to instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java index 733069f3c1c4..91cf7d97f9db 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/opentlelemetry/instrumentation/iceberg/v1_6/ScanMetricsBuilder.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java @@ -3,12 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -package opentlelemetry.instrumentation.iceberg.v1_6; +package io.opentelemetry.instrumentation.iceberg.v1_8; import io.opentelemetry.api.metrics.DoubleGauge; import io.opentelemetry.api.metrics.Meter; -public class ScanMetricsBuilder { +final class ScanMetricsBuilder { private static final String ROOT = "iceberg.scan"; private static final String TOTAL_PLANNING_DURATION = ROOT + ".planning.duration"; private static final String RESULT_DATA_FILES = ROOT + ".scanned.data_files.count"; @@ -29,6 +29,10 @@ public class ScanMetricsBuilder { ROOT + ".scanned.positional_delete_files.count"; private static final String DVS = ROOT + ".scanned.dvs.count"; + private ScanMetricsBuilder() { + // prevents instantiation + } + static DoubleGauge totalPlanningDuration(Meter meter) { return meter .gaugeBuilder(TOTAL_PLANNING_DURATION) From 5d94635314215fb8151385ce6b95a6d59335a310 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Thu, 30 Oct 2025 11:32:59 +0100 Subject: [PATCH 07/33] Convert DoubleGauge to LongGauge --- .../iceberg/v1_8/IcebergMetricsReporter.java | 43 ++++++++------- .../iceberg/v1_8/IcebergTelemetry.java | 3 +- .../iceberg/v1_8/ScanMetricsBuilder.java | 53 ++++++++++++------- 3 files changed, 57 insertions(+), 42 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index de3fdb2943dd..02a372bb0256 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -5,6 +5,8 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.LongGauge; import org.apache.iceberg.metrics.CommitReport; import org.apache.iceberg.metrics.CounterResult; import org.apache.iceberg.metrics.MetricsReport; @@ -13,11 +15,8 @@ import org.apache.iceberg.metrics.ScanReport; import org.apache.iceberg.metrics.TimerResult; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.metrics.DoubleGauge; - public class IcebergMetricsReporter implements MetricsReporter { - private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg_1.6"; + private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg_1.8"; private final OpenTelemetry openTelemetry; @@ -35,11 +34,11 @@ public void report(MetricsReport report) { } void reportScanMetrics(ScanReport scanReport) { - final ScanMetricsResult metrics = scanReport.scanMetrics(); + ScanMetricsResult metrics = scanReport.scanMetrics(); TimerResult duration = metrics.totalPlanningDuration(); if (duration != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.totalPlanningDuration(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(duration.totalDuration().toMillis()); } @@ -47,7 +46,7 @@ void reportScanMetrics(ScanReport scanReport) { CounterResult current = metrics.resultDataFiles(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.scannedDataFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -55,7 +54,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.resultDeleteFiles(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.scannedDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -63,7 +62,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.scannedDataManifests(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.scannedDataManifestsCount( openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); @@ -72,7 +71,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.scannedDeleteManifests(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.scannedDeleteManifestsCount( openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); @@ -81,7 +80,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.totalDataManifests(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.totalDataManifestsCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -89,7 +88,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.totalDeleteManifests(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.totalDeleteManifestsCount( openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); @@ -98,7 +97,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.totalFileSizeInBytes(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.scannedDataFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -106,7 +105,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.totalDeleteFileSizeInBytes(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.scannedDeleteFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -114,7 +113,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.skippedDataManifests(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.skippedDataManifests(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -122,7 +121,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.skippedDeleteManifests(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.skippedDeleteManifests(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -130,7 +129,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.skippedDataFiles(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.skippedDataFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -138,7 +137,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.skippedDeleteFiles(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.skippedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -146,7 +145,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.indexedDeleteFiles(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.indexedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -154,7 +153,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.equalityDeleteFiles(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.equalityDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -162,7 +161,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.positionalDeleteFiles(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.positionDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } @@ -170,7 +169,7 @@ void reportScanMetrics(ScanReport scanReport) { current = metrics.dvs(); if (current != null) { - DoubleGauge metric = + LongGauge metric = ScanMetricsBuilder.deletionVectorFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(current.value()); } diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java index 8642cc866bf6..e19f8be251cd 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java @@ -5,9 +5,8 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; -import org.apache.iceberg.TableScan; - import io.opentelemetry.api.OpenTelemetry; +import org.apache.iceberg.TableScan; public class IcebergTelemetry { private final OpenTelemetry openTelemetry; diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java index 91cf7d97f9db..a0c74099f4c6 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java @@ -5,7 +5,7 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; -import io.opentelemetry.api.metrics.DoubleGauge; +import io.opentelemetry.api.metrics.LongGauge; import io.opentelemetry.api.metrics.Meter; final class ScanMetricsBuilder { @@ -33,140 +33,157 @@ private ScanMetricsBuilder() { // prevents instantiation } - static DoubleGauge totalPlanningDuration(Meter meter) { + static LongGauge totalPlanningDuration(Meter meter) { return meter .gaugeBuilder(TOTAL_PLANNING_DURATION) .setDescription("The total duration needed to plan the scan.") .setUnit("ms") + .ofLongs() .build(); } - static DoubleGauge scannedDataFilesCount(Meter meter) { + static LongGauge scannedDataFilesCount(Meter meter) { return meter .gaugeBuilder(RESULT_DATA_FILES) .setDescription("The number of scanned data files.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge scannedDeleteFilesCount(Meter meter) { + static LongGauge scannedDeleteFilesCount(Meter meter) { return meter .gaugeBuilder(RESULT_DELETE_FILES) .setDescription("The number of scanned delete files.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge scannedDataManifestsCount(Meter meter) { + static LongGauge scannedDataManifestsCount(Meter meter) { return meter .gaugeBuilder(SCANNED_DATA_MANIFESTS) .setDescription("The number of scanned data manifests.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge scannedDeleteManifestsCount(Meter meter) { + static LongGauge scannedDeleteManifestsCount(Meter meter) { return meter .gaugeBuilder(SCANNED_DELETE_MANIFESTS) .setDescription("The number of scanned delete manifests.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge totalDataManifestsCount(Meter meter) { + static LongGauge totalDataManifestsCount(Meter meter) { return meter .gaugeBuilder(TOTAL_DATA_MANIFESTS) .setDescription("The number of all data manifests.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge totalDeleteManifestsCount(Meter meter) { + static LongGauge totalDeleteManifestsCount(Meter meter) { return meter .gaugeBuilder(TOTAL_DELETE_MANIFESTS) .setDescription("The number of all delete manifests.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge scannedDataFilesSize(Meter meter) { + static LongGauge scannedDataFilesSize(Meter meter) { return meter .gaugeBuilder(TOTAL_FILE_SIZE_IN_BYTES) .setDescription("The total size of all scanned data files.") .setUnit("byte") + .ofLongs() .build(); } - static DoubleGauge scannedDeleteFilesSize(Meter meter) { + static LongGauge scannedDeleteFilesSize(Meter meter) { return meter .gaugeBuilder(TOTAL_DELETE_FILE_SIZE_IN_BYTES) .setDescription("The total size of all scanned delete files.") .setUnit("byte") + .ofLongs() .build(); } - static DoubleGauge skippedDataManifests(Meter meter) { + static LongGauge skippedDataManifests(Meter meter) { return meter .gaugeBuilder(SKIPPED_DATA_MANIFESTS) .setDescription("The number of data manifests that were skipped during the scan.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge skippedDeleteManifests(Meter meter) { + static LongGauge skippedDeleteManifests(Meter meter) { return meter .gaugeBuilder(SKIPPED_DELETE_MANIFESTS) .setDescription("The number of delete manifests that were skipped during the scan.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge skippedDataFiles(Meter meter) { + static LongGauge skippedDataFiles(Meter meter) { return meter .gaugeBuilder(SKIPPED_DATA_FILES) .setDescription("The number of data files that were skipped during the scan.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge skippedDeleteFiles(Meter meter) { + static LongGauge skippedDeleteFiles(Meter meter) { return meter .gaugeBuilder(SKIPPED_DELETE_FILES) .setDescription("The number of delete files that were skipped during the scan.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge indexedDeleteFiles(Meter meter) { + static LongGauge indexedDeleteFiles(Meter meter) { return meter .gaugeBuilder(INDEXED_DELETE_FILES) .setDescription( "The number of delete files constituting the delete file index for this scan.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge equalityDeleteFiles(Meter meter) { + static LongGauge equalityDeleteFiles(Meter meter) { return meter .gaugeBuilder(EQUALITY_DELETE_FILES) .setDescription("The number of equality delete files relevant for the current scan.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge positionDeleteFiles(Meter meter) { + static LongGauge positionDeleteFiles(Meter meter) { return meter .gaugeBuilder(POSITIONAL_DELETE_FILES) .setDescription("The number of position delete files relevant for the current scan.") .setUnit("{file}") + .ofLongs() .build(); } - static DoubleGauge deletionVectorFiles(Meter meter) { + static LongGauge deletionVectorFiles(Meter meter) { return meter .gaugeBuilder(DVS) .setDescription("The number of deletion vector (DV) files relevant for the current scan.") .setUnit("{file}") + .ofLongs() .build(); } } From b8acfe8880f1399f5b03f3667b3631f87e85f50b Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Thu, 30 Oct 2025 12:10:15 +0100 Subject: [PATCH 08/33] Convert LongGauge to LongCounter --- .../iceberg/v1_8/IcebergMetricsReporter.java | 83 ++++++++++--------- .../iceberg/v1_8/ScanMetricsBuilder.java | 81 ++++++++---------- 2 files changed, 76 insertions(+), 88 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index 02a372bb0256..8ee709a24ade 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -6,6 +6,7 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.metrics.LongGauge; import org.apache.iceberg.metrics.CommitReport; import org.apache.iceberg.metrics.CounterResult; @@ -46,132 +47,134 @@ void reportScanMetrics(ScanReport scanReport) { CounterResult current = metrics.resultDataFiles(); if (current != null) { - LongGauge metric = + LongCounter metric = ScanMetricsBuilder.scannedDataFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + metric.add(current.value()); } current = metrics.resultDeleteFiles(); if (current != null) { - LongGauge metric = + LongCounter metric = ScanMetricsBuilder.scannedDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + metric.add(current.value()); } current = metrics.scannedDataManifests(); if (current != null) { - LongGauge metric = + LongCounter metric = ScanMetricsBuilder.scannedDataManifestsCount( openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + metric.add(current.value()); } current = metrics.scannedDeleteManifests(); if (current != null) { - LongGauge metric = + LongCounter metric = ScanMetricsBuilder.scannedDeleteManifestsCount( openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + metric.add(current.value()); } current = metrics.totalDataManifests(); if (current != null) { - LongGauge metric = + LongCounter metric = ScanMetricsBuilder.totalDataManifestsCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + metric.add(current.value()); } current = metrics.totalDeleteManifests(); if (current != null) { - LongGauge metric = + LongCounter metric = ScanMetricsBuilder.totalDeleteManifestsCount( openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + metric.add(current.value()); } current = metrics.totalFileSizeInBytes(); if (current != null) { - LongGauge metric = + LongCounter metric = ScanMetricsBuilder.scannedDataFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + metric.add(current.value()); } current = metrics.totalDeleteFileSizeInBytes(); if (current != null) { - LongGauge metric = + LongCounter metric = ScanMetricsBuilder.scannedDeleteFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + metric.add(current.value()); } current = metrics.skippedDataManifests(); if (current != null) { - LongGauge metric = - ScanMetricsBuilder.skippedDataManifests(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + LongCounter metric = + ScanMetricsBuilder.skippedDataManifestsCount( + openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value()); } current = metrics.skippedDeleteManifests(); if (current != null) { - LongGauge metric = - ScanMetricsBuilder.skippedDeleteManifests(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + LongCounter metric = + ScanMetricsBuilder.skippedDeleteManifestsCount( + openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value()); } current = metrics.skippedDataFiles(); if (current != null) { - LongGauge metric = - ScanMetricsBuilder.skippedDataFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + LongCounter metric = + ScanMetricsBuilder.skippedDataFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value()); } current = metrics.skippedDeleteFiles(); if (current != null) { - LongGauge metric = - ScanMetricsBuilder.skippedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + LongCounter metric = + ScanMetricsBuilder.skippedDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value()); } current = metrics.indexedDeleteFiles(); if (current != null) { - LongGauge metric = - ScanMetricsBuilder.indexedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + LongCounter metric = + ScanMetricsBuilder.indexedDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value()); } current = metrics.equalityDeleteFiles(); if (current != null) { - LongGauge metric = - ScanMetricsBuilder.equalityDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + LongCounter metric = + ScanMetricsBuilder.equalityDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value()); } current = metrics.positionalDeleteFiles(); if (current != null) { - LongGauge metric = - ScanMetricsBuilder.positionDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + LongCounter metric = + ScanMetricsBuilder.positionDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value()); } current = metrics.dvs(); if (current != null) { - LongGauge metric = - ScanMetricsBuilder.deletionVectorFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(current.value()); + LongCounter metric = + ScanMetricsBuilder.deletionVectorFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value()); } } diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java index a0c74099f4c6..5d31059189b3 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java @@ -5,6 +5,7 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; +import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.metrics.LongGauge; import io.opentelemetry.api.metrics.Meter; @@ -42,148 +43,132 @@ static LongGauge totalPlanningDuration(Meter meter) { .build(); } - static LongGauge scannedDataFilesCount(Meter meter) { + static LongCounter scannedDataFilesCount(Meter meter) { return meter - .gaugeBuilder(RESULT_DATA_FILES) + .counterBuilder(RESULT_DATA_FILES) .setDescription("The number of scanned data files.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge scannedDeleteFilesCount(Meter meter) { + static LongCounter scannedDeleteFilesCount(Meter meter) { return meter - .gaugeBuilder(RESULT_DELETE_FILES) + .counterBuilder(RESULT_DELETE_FILES) .setDescription("The number of scanned delete files.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge scannedDataManifestsCount(Meter meter) { + static LongCounter scannedDataManifestsCount(Meter meter) { return meter - .gaugeBuilder(SCANNED_DATA_MANIFESTS) + .counterBuilder(SCANNED_DATA_MANIFESTS) .setDescription("The number of scanned data manifests.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge scannedDeleteManifestsCount(Meter meter) { + static LongCounter scannedDeleteManifestsCount(Meter meter) { return meter - .gaugeBuilder(SCANNED_DELETE_MANIFESTS) + .counterBuilder(SCANNED_DELETE_MANIFESTS) .setDescription("The number of scanned delete manifests.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge totalDataManifestsCount(Meter meter) { + static LongCounter totalDataManifestsCount(Meter meter) { return meter - .gaugeBuilder(TOTAL_DATA_MANIFESTS) + .counterBuilder(TOTAL_DATA_MANIFESTS) .setDescription("The number of all data manifests.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge totalDeleteManifestsCount(Meter meter) { + static LongCounter totalDeleteManifestsCount(Meter meter) { return meter - .gaugeBuilder(TOTAL_DELETE_MANIFESTS) + .counterBuilder(TOTAL_DELETE_MANIFESTS) .setDescription("The number of all delete manifests.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge scannedDataFilesSize(Meter meter) { + static LongCounter scannedDataFilesSize(Meter meter) { return meter - .gaugeBuilder(TOTAL_FILE_SIZE_IN_BYTES) + .counterBuilder(TOTAL_FILE_SIZE_IN_BYTES) .setDescription("The total size of all scanned data files.") .setUnit("byte") - .ofLongs() .build(); } - static LongGauge scannedDeleteFilesSize(Meter meter) { + static LongCounter scannedDeleteFilesSize(Meter meter) { return meter - .gaugeBuilder(TOTAL_DELETE_FILE_SIZE_IN_BYTES) + .counterBuilder(TOTAL_DELETE_FILE_SIZE_IN_BYTES) .setDescription("The total size of all scanned delete files.") .setUnit("byte") - .ofLongs() .build(); } - static LongGauge skippedDataManifests(Meter meter) { + static LongCounter skippedDataManifestsCount(Meter meter) { return meter - .gaugeBuilder(SKIPPED_DATA_MANIFESTS) + .counterBuilder(SKIPPED_DATA_MANIFESTS) .setDescription("The number of data manifests that were skipped during the scan.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge skippedDeleteManifests(Meter meter) { + static LongCounter skippedDeleteManifestsCount(Meter meter) { return meter - .gaugeBuilder(SKIPPED_DELETE_MANIFESTS) + .counterBuilder(SKIPPED_DELETE_MANIFESTS) .setDescription("The number of delete manifests that were skipped during the scan.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge skippedDataFiles(Meter meter) { + static LongCounter skippedDataFilesCount(Meter meter) { return meter - .gaugeBuilder(SKIPPED_DATA_FILES) + .counterBuilder(SKIPPED_DATA_FILES) .setDescription("The number of data files that were skipped during the scan.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge skippedDeleteFiles(Meter meter) { + static LongCounter skippedDeleteFilesCount(Meter meter) { return meter - .gaugeBuilder(SKIPPED_DELETE_FILES) + .counterBuilder(SKIPPED_DELETE_FILES) .setDescription("The number of delete files that were skipped during the scan.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge indexedDeleteFiles(Meter meter) { + static LongCounter indexedDeleteFilesCount(Meter meter) { return meter - .gaugeBuilder(INDEXED_DELETE_FILES) + .counterBuilder(INDEXED_DELETE_FILES) .setDescription( "The number of delete files constituting the delete file index for this scan.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge equalityDeleteFiles(Meter meter) { + static LongCounter equalityDeleteFilesCount(Meter meter) { return meter - .gaugeBuilder(EQUALITY_DELETE_FILES) + .counterBuilder(EQUALITY_DELETE_FILES) .setDescription("The number of equality delete files relevant for the current scan.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge positionDeleteFiles(Meter meter) { + static LongCounter positionDeleteFilesCount(Meter meter) { return meter - .gaugeBuilder(POSITIONAL_DELETE_FILES) + .counterBuilder(POSITIONAL_DELETE_FILES) .setDescription("The number of position delete files relevant for the current scan.") .setUnit("{file}") - .ofLongs() .build(); } - static LongGauge deletionVectorFiles(Meter meter) { + static LongCounter deletionVectorFilesCount(Meter meter) { return meter - .gaugeBuilder(DVS) + .counterBuilder(DVS) .setDescription("The number of deletion vector (DV) files relevant for the current scan.") .setUnit("{file}") - .ofLongs() .build(); } } From 77dd60655d3cf0048e265c581f1582dcb22ffbd7 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Thu, 30 Oct 2025 13:48:15 +0100 Subject: [PATCH 09/33] add attributes to scan metrics --- .../iceberg/v1_8/IcebergMetricsReporter.java | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index 8ee709a24ade..f1a8dfcd2369 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -6,6 +6,8 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.metrics.LongGauge; import org.apache.iceberg.metrics.CommitReport; @@ -18,6 +20,10 @@ public class IcebergMetricsReporter implements MetricsReporter { private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg_1.8"; + private static final AttributeKey SCHEMA_ID = AttributeKey.longKey("iceberg.schema.id"); + private static final AttributeKey TABLE_NAME = + AttributeKey.stringKey("iceberg.table.name"); + private static final AttributeKey SNAPHSOT_ID = AttributeKey.longKey("iceberg.snapshot.id"); private final OpenTelemetry openTelemetry; @@ -35,13 +41,21 @@ public void report(MetricsReport report) { } void reportScanMetrics(ScanReport scanReport) { + Attributes scanAttributes = + Attributes.of( + SCHEMA_ID, + Long.valueOf(scanReport.schemaId()), + TABLE_NAME, + scanReport.tableName(), + SNAPHSOT_ID, + scanReport.snapshotId()); ScanMetricsResult metrics = scanReport.scanMetrics(); TimerResult duration = metrics.totalPlanningDuration(); if (duration != null) { LongGauge metric = ScanMetricsBuilder.totalPlanningDuration(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(duration.totalDuration().toMillis()); + metric.set(duration.totalDuration().toMillis(), scanAttributes); } CounterResult current = metrics.resultDataFiles(); @@ -49,7 +63,7 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { LongCounter metric = ScanMetricsBuilder.scannedDataFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.resultDeleteFiles(); @@ -57,7 +71,7 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { LongCounter metric = ScanMetricsBuilder.scannedDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.scannedDataManifests(); @@ -66,7 +80,7 @@ void reportScanMetrics(ScanReport scanReport) { LongCounter metric = ScanMetricsBuilder.scannedDataManifestsCount( openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.scannedDeleteManifests(); @@ -75,7 +89,7 @@ void reportScanMetrics(ScanReport scanReport) { LongCounter metric = ScanMetricsBuilder.scannedDeleteManifestsCount( openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.totalDataManifests(); @@ -83,7 +97,7 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { LongCounter metric = ScanMetricsBuilder.totalDataManifestsCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.totalDeleteManifests(); @@ -92,7 +106,7 @@ void reportScanMetrics(ScanReport scanReport) { LongCounter metric = ScanMetricsBuilder.totalDeleteManifestsCount( openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.totalFileSizeInBytes(); @@ -100,7 +114,7 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { LongCounter metric = ScanMetricsBuilder.scannedDataFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.totalDeleteFileSizeInBytes(); @@ -108,7 +122,7 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { LongCounter metric = ScanMetricsBuilder.scannedDeleteFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.skippedDataManifests(); @@ -117,7 +131,7 @@ void reportScanMetrics(ScanReport scanReport) { LongCounter metric = ScanMetricsBuilder.skippedDataManifestsCount( openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.skippedDeleteManifests(); @@ -126,7 +140,7 @@ void reportScanMetrics(ScanReport scanReport) { LongCounter metric = ScanMetricsBuilder.skippedDeleteManifestsCount( openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.skippedDataFiles(); @@ -134,7 +148,7 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { LongCounter metric = ScanMetricsBuilder.skippedDataFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.skippedDeleteFiles(); @@ -142,7 +156,7 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { LongCounter metric = ScanMetricsBuilder.skippedDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.indexedDeleteFiles(); @@ -150,7 +164,7 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { LongCounter metric = ScanMetricsBuilder.indexedDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.equalityDeleteFiles(); @@ -158,7 +172,7 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { LongCounter metric = ScanMetricsBuilder.equalityDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.positionalDeleteFiles(); @@ -166,7 +180,7 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { LongCounter metric = ScanMetricsBuilder.positionDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } current = metrics.dvs(); @@ -174,7 +188,7 @@ void reportScanMetrics(ScanReport scanReport) { if (current != null) { LongCounter metric = ScanMetricsBuilder.deletionVectorFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value()); + metric.add(current.value(), scanAttributes); } } From b7a4b5da5f5c39591feb082e849a379440816228 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Thu, 30 Oct 2025 18:33:27 +0100 Subject: [PATCH 10/33] create commit metrics builder --- .../iceberg/v1_8/CommitMetricsBuilder.java | 285 ++++++++++++++++++ .../iceberg/v1_8/IcebergMetricsReporter.java | 127 +++++++- 2 files changed, 406 insertions(+), 6 deletions(-) create mode 100644 instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/CommitMetricsBuilder.java diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/CommitMetricsBuilder.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/CommitMetricsBuilder.java new file mode 100644 index 000000000000..9edfed3d56e9 --- /dev/null +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/CommitMetricsBuilder.java @@ -0,0 +1,285 @@ +package io.opentelemetry.instrumentation.iceberg.v1_8; + +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongGauge; +import io.opentelemetry.api.metrics.Meter; + +final class CommitMetricsBuilder { + + private static final String ROOT = "iceberg.commit"; + private static final String DURATION = ROOT + ".duration"; + private static final String ATTEMPTS = ROOT + ".attempts.count"; + private static final String ADDED_DATA_FILES = ROOT + ".added.data_files.count"; + private static final String REMOVED_DATA_FILES = ROOT + ".removed.data_files.count"; + private static final String TOTAL_DATA_FILES = ROOT + ".total.data_files.count"; + private static final String ADDED_DELETE_FILES = ROOT + ".added.delete_files.count"; + private static final String ADDED_EQ_DELETE_FILES = ROOT + ".added.equality_delete_files.count"; + private static final String ADDED_POS_DELETE_FILES = ROOT + ".added.position_delete_files.count"; + private static final String ADDED_DVS = ROOT + ".added.dvs.count"; + private static final String REMOVED_POS_DELETE_FILES = ROOT + ".removed.positional_delete_files.count"; + private static final String REMOVED_DVS = ROOT + ".removed.dvs.count"; + private static final String REMOVED_EQ_DELETE_FILES = ROOT + ".removed.equality_delete_files.count"; + private static final String REMOVED_DELETE_FILES = ROOT + ".removed.delete_files.count"; + private static final String TOTAL_DELETE_FILES = ROOT + ".total.delete_files.count"; + private static final String ADDED_RECORDS = ROOT + ".added.records.count"; + private static final String REMOVED_RECORDS = ROOT + ".removed.records.count"; + private static final String TOTAL_RECORDS = ROOT + ".total.records.count"; + private static final String ADDED_FILE_SIZE_BYTES = ROOT + ".added.files.size"; + private static final String REMOVED_FILE_SIZE_BYTES = ROOT + ".removed.files.size"; + private static final String TOTAL_FILE_SIZE_BYTES = ROOT + ".total.files.size"; + private static final String ADDED_POS_DELETES = ROOT + ".added.position_deletes.count"; + private static final String REMOVED_POS_DELETES = ROOT + ".removed.position_deletes.count"; + private static final String TOTAL_POS_DELETES = ROOT + ".total.position_deletes.count"; + private static final String ADDED_EQ_DELETES = ROOT + ".added.equality_deletes.count"; + private static final String REMOVED_EQ_DELETES = ROOT + ".removed.equality_deletes.count"; + private static final String TOTAL_EQ_DELETES = ROOT + ".total.equality_deletes.count"; + private static final String KEPT_MANIFESTS_COUNT = ROOT + ".manifests_kept.count"; + private static final String CREATED_MANIFESTS_COUNT = ROOT + ".manifests_created.count"; + private static final String REPLACED_MANIFESTS_COUNT = ROOT + ".manifests_replaced.count"; + private static final String PROCESSED_MANIFEST_ENTRY_COUNT = ROOT + ".manifest_entries_processed.count"; + + private CommitMetricsBuilder() { + // prevents instantiation + } + + static LongGauge duration(Meter meter) { + return meter + .gaugeBuilder(DURATION) + .setDescription("The duration taken to process the commit.") + .setUnit("ms") + .ofLongs() + .build(); + } + + static LongCounter attempts(Meter meter) { + return meter + .counterBuilder(ATTEMPTS) + .setDescription("The number of attempts made to complete this commit.") + .setUnit("{attempt}") + .build(); + } + + static LongCounter addedDataFiles(Meter meter) { + return meter + .counterBuilder(ADDED_DATA_FILES) + .setDescription("The number of data files added as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter removedDataFiles(Meter meter) { + return meter + .counterBuilder(REMOVED_DATA_FILES) + .setDescription("The number of data files removed as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter totalDataFiles(Meter meter) { + return meter + .counterBuilder(TOTAL_DATA_FILES) + .setDescription("The number of data files added or removed as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter addedDeleteFiles(Meter meter) { + return meter + .counterBuilder(ADDED_DELETE_FILES) + .setDescription("The overall number of delete files added as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter addedEqualityDeleteFiles(Meter meter) { + return meter + .counterBuilder(ADDED_EQ_DELETE_FILES) + .setDescription("The number of equality delete files added as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter addedPositionDeleteFiles(Meter meter) { + return meter + .counterBuilder(ADDED_POS_DELETE_FILES) + .setDescription("The number of position delete files added as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter addedDeletionVectors(Meter meter) { + return meter + .counterBuilder(ADDED_DVS) + .setDescription("The number of deletion vector files added as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter removedPositionDeleteFiles(Meter meter) { + return meter + .counterBuilder(REMOVED_POS_DELETE_FILES) + .setDescription("The number of position delete files removed as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter removedDeletionVectors(Meter meter) { + return meter + .counterBuilder(REMOVED_DVS) + .setDescription("The number of deletion vector files removed as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter removedEqualityDeleteFiles(Meter meter) { + return meter + .counterBuilder(REMOVED_EQ_DELETE_FILES) + .setDescription("The number of equality delete files removed as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter removedDeleteFiles(Meter meter) { + return meter + .counterBuilder(REMOVED_DELETE_FILES) + .setDescription("The overall number of delete files removed as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter totalDeleteFiles(Meter meter) { + return meter + .counterBuilder(TOTAL_DELETE_FILES) + .setDescription("The overall number of delete files added or removed as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter addedRecords(Meter meter) { + return meter + .counterBuilder(ADDED_RECORDS) + .setDescription("The number of records added as part of the commit.") + .setUnit("{record}") + .build(); + } + + static LongCounter removedRecords(Meter meter) { + return meter + .counterBuilder(REMOVED_RECORDS) + .setDescription("The number of records removed as part of the commit.") + .setUnit("{record}") + .build(); + } + + static LongCounter totalRecords(Meter meter) { + return meter + .counterBuilder(TOTAL_RECORDS) + .setDescription("The overall number of records added or removed as part of the commit.") + .setUnit("{record}") + .build(); + } + + static LongCounter addedFilesSize(Meter meter) { + return meter + .counterBuilder(ADDED_FILE_SIZE_BYTES) + .setDescription("The overall size of the data and delete files added as part of the commit.") + .setUnit("byte") + .build(); + } + + static LongCounter removedFilesSize(Meter meter) { + return meter + .counterBuilder(REMOVED_FILE_SIZE_BYTES) + .setDescription("The overall size of the data or delete files removed as part of the commit.") + .setUnit("byte") + .build(); + } + + static LongCounter totalFilesSize(Meter meter) { + return meter + .counterBuilder(TOTAL_FILE_SIZE_BYTES) + .setDescription("The overall size of the data or delete files added or removed as part of the commit.") + .setUnit("byte") + .build(); + } + + static LongCounter addedPositionDeletes(Meter meter) { + return meter + .counterBuilder(ADDED_POS_DELETES) + .setDescription("The overall number of position delete entries added as part of the commit.") + .setUnit("{record}") + .build(); + } + + static LongCounter removedPositionDeletes(Meter meter) { + return meter + .counterBuilder(REMOVED_POS_DELETES) + .setDescription("The overall number of position delete entries removed as part of the commit.") + .setUnit("{record}") + .build(); + } + + static LongCounter totalPositionDeletes(Meter meter) { + return meter + .counterBuilder(TOTAL_POS_DELETES) + .setDescription("The overall number of position delete entries added or removed as part of the commit.") + .setUnit("{record}") + .build(); + } + + static LongCounter addedEqualityDeletes(Meter meter) { + return meter + .counterBuilder(ADDED_EQ_DELETES) + .setDescription("The overall number of equality delete entries added as part of the commit.") + .setUnit("{record}") + .build(); + } + + static LongCounter removedEqualityDeletes(Meter meter) { + return meter + .counterBuilder(REMOVED_EQ_DELETES) + .setDescription("The overall number of equality delete entries removed as part of the commit.") + .setUnit("{record}") + .build(); + } + + static LongCounter totalEqualityDeletes(Meter meter) { + return meter + .counterBuilder(TOTAL_EQ_DELETES) + .setDescription("The overall number of equality delete entries added or removed as part of the commit.") + .setUnit("{record}") + .build(); + } + + static LongCounter keptManifests(Meter meter) { + return meter + .counterBuilder(KEPT_MANIFESTS_COUNT) + .setDescription("The number of manifests that are kept as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter createdManfiests(Meter meter) { + return meter + .counterBuilder(CREATED_MANIFESTS_COUNT) + .setDescription("The number of manifests that are created as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter replacedManifests(Meter meter) { + return meter + .counterBuilder(REPLACED_MANIFESTS_COUNT) + .setDescription("The overall number of manifests that are deleted or overwritten as part of the commit.") + .setUnit("{file}") + .build(); + } + + static LongCounter processedManfiestEntries(Meter meter) { + return meter + .counterBuilder(PROCESSED_MANIFEST_ENTRY_COUNT) + .setDescription("The overall number of manifest entries (referenced files) that are processed as part of the commit.") + .setUnit("{file}") + .build(); + } +} diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index f1a8dfcd2369..8272ec41e3e8 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -5,11 +5,7 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.common.AttributeKey; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.LongCounter; -import io.opentelemetry.api.metrics.LongGauge; +import org.apache.iceberg.metrics.CommitMetricsResult; import org.apache.iceberg.metrics.CommitReport; import org.apache.iceberg.metrics.CounterResult; import org.apache.iceberg.metrics.MetricsReport; @@ -18,12 +14,19 @@ import org.apache.iceberg.metrics.ScanReport; import org.apache.iceberg.metrics.TimerResult; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongGauge; + public class IcebergMetricsReporter implements MetricsReporter { private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg_1.8"; private static final AttributeKey SCHEMA_ID = AttributeKey.longKey("iceberg.schema.id"); private static final AttributeKey TABLE_NAME = AttributeKey.stringKey("iceberg.table.name"); private static final AttributeKey SNAPHSOT_ID = AttributeKey.longKey("iceberg.snapshot.id"); + private static final AttributeKey SEQUENCE_NUMBER = AttributeKey.longKey("iceberg.commit.sequence_number"); private final OpenTelemetry openTelemetry; @@ -192,5 +195,117 @@ void reportScanMetrics(ScanReport scanReport) { } } - void reportCommitMetrics(CommitReport commitReport) {} + void reportCommitMetrics(CommitReport commitReport) { + Attributes commitAttributes = + Attributes.of(SEQUENCE_NUMBER, + Long.valueOf(commitReport.sequenceNumber()), + TABLE_NAME, + commitReport.tableName(), + SNAPHSOT_ID, + commitReport.snapshotId()); + CommitMetricsResult metrics = commitReport.commitMetrics(); + TimerResult duration = metrics.totalDuration(); + + if (duration != null) { + LongGauge metric = CommitMetricsBuilder.duration(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.set(duration.totalDuration().toMillis(), commitAttributes); + } + + CounterResult current = metrics.attempts(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.attempts(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.addedDataFiles(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.addedDataFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.removedDataFiles(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.removedDataFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.totalDataFiles(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.totalDataFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.addedDeleteFiles(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.addedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.addedEqualityDeleteFiles(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.addedEqualityDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.addedPositionalDeleteFiles(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.addedPositionDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.addedDVs(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.addedDeletionVectors(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.removedPositionalDeleteFiles(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.removedPositionDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.removedDVs(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.removedDeletionVectors(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.removedEqualityDeleteFiles(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.removedEqualityDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.removedDeleteFiles(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.removedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + } } From 98ee4127b9b573e340e70bdc2dba5422b1168762 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Fri, 31 Oct 2025 09:46:55 +0100 Subject: [PATCH 11/33] Finalize IcebergMetricsReporter implementation --- .../iceberg/v1_8/CommitMetricsBuilder.java | 267 ++++++++---------- .../iceberg/v1_8/IcebergMetricsReporter.java | 137 ++++++++- 2 files changed, 247 insertions(+), 157 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/CommitMetricsBuilder.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/CommitMetricsBuilder.java index 9edfed3d56e9..afab06ce391d 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/CommitMetricsBuilder.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/CommitMetricsBuilder.java @@ -1,3 +1,8 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + package io.opentelemetry.instrumentation.iceberg.v1_8; import io.opentelemetry.api.metrics.LongCounter; @@ -16,9 +21,11 @@ final class CommitMetricsBuilder { private static final String ADDED_EQ_DELETE_FILES = ROOT + ".added.equality_delete_files.count"; private static final String ADDED_POS_DELETE_FILES = ROOT + ".added.position_delete_files.count"; private static final String ADDED_DVS = ROOT + ".added.dvs.count"; - private static final String REMOVED_POS_DELETE_FILES = ROOT + ".removed.positional_delete_files.count"; + private static final String REMOVED_POS_DELETE_FILES = + ROOT + ".removed.positional_delete_files.count"; private static final String REMOVED_DVS = ROOT + ".removed.dvs.count"; - private static final String REMOVED_EQ_DELETE_FILES = ROOT + ".removed.equality_delete_files.count"; + private static final String REMOVED_EQ_DELETE_FILES = + ROOT + ".removed.equality_delete_files.count"; private static final String REMOVED_DELETE_FILES = ROOT + ".removed.delete_files.count"; private static final String TOTAL_DELETE_FILES = ROOT + ".total.delete_files.count"; private static final String ADDED_RECORDS = ROOT + ".added.records.count"; @@ -33,10 +40,6 @@ final class CommitMetricsBuilder { private static final String ADDED_EQ_DELETES = ROOT + ".added.equality_deletes.count"; private static final String REMOVED_EQ_DELETES = ROOT + ".removed.equality_deletes.count"; private static final String TOTAL_EQ_DELETES = ROOT + ".total.equality_deletes.count"; - private static final String KEPT_MANIFESTS_COUNT = ROOT + ".manifests_kept.count"; - private static final String CREATED_MANIFESTS_COUNT = ROOT + ".manifests_created.count"; - private static final String REPLACED_MANIFESTS_COUNT = ROOT + ".manifests_replaced.count"; - private static final String PROCESSED_MANIFEST_ENTRY_COUNT = ROOT + ".manifest_entries_processed.count"; private CommitMetricsBuilder() { // prevents instantiation @@ -44,242 +47,220 @@ private CommitMetricsBuilder() { static LongGauge duration(Meter meter) { return meter - .gaugeBuilder(DURATION) - .setDescription("The duration taken to process the commit.") - .setUnit("ms") - .ofLongs() - .build(); + .gaugeBuilder(DURATION) + .setDescription("The duration taken to process the commit.") + .setUnit("ms") + .ofLongs() + .build(); } static LongCounter attempts(Meter meter) { return meter - .counterBuilder(ATTEMPTS) - .setDescription("The number of attempts made to complete this commit.") - .setUnit("{attempt}") - .build(); + .counterBuilder(ATTEMPTS) + .setDescription("The number of attempts made to complete this commit.") + .setUnit("{attempt}") + .build(); } static LongCounter addedDataFiles(Meter meter) { return meter - .counterBuilder(ADDED_DATA_FILES) - .setDescription("The number of data files added as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(ADDED_DATA_FILES) + .setDescription("The number of data files added as part of the commit.") + .setUnit("{file}") + .build(); } static LongCounter removedDataFiles(Meter meter) { return meter - .counterBuilder(REMOVED_DATA_FILES) - .setDescription("The number of data files removed as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(REMOVED_DATA_FILES) + .setDescription("The number of data files removed as part of the commit.") + .setUnit("{file}") + .build(); } static LongCounter totalDataFiles(Meter meter) { return meter - .counterBuilder(TOTAL_DATA_FILES) - .setDescription("The number of data files added or removed as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(TOTAL_DATA_FILES) + .setDescription("The number of data files added or removed as part of the commit.") + .setUnit("{file}") + .build(); } static LongCounter addedDeleteFiles(Meter meter) { return meter - .counterBuilder(ADDED_DELETE_FILES) - .setDescription("The overall number of delete files added as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(ADDED_DELETE_FILES) + .setDescription("The overall number of delete files added as part of the commit.") + .setUnit("{file}") + .build(); } static LongCounter addedEqualityDeleteFiles(Meter meter) { return meter - .counterBuilder(ADDED_EQ_DELETE_FILES) - .setDescription("The number of equality delete files added as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(ADDED_EQ_DELETE_FILES) + .setDescription("The number of equality delete files added as part of the commit.") + .setUnit("{file}") + .build(); } static LongCounter addedPositionDeleteFiles(Meter meter) { return meter - .counterBuilder(ADDED_POS_DELETE_FILES) - .setDescription("The number of position delete files added as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(ADDED_POS_DELETE_FILES) + .setDescription("The number of position delete files added as part of the commit.") + .setUnit("{file}") + .build(); } static LongCounter addedDeletionVectors(Meter meter) { return meter - .counterBuilder(ADDED_DVS) - .setDescription("The number of deletion vector files added as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(ADDED_DVS) + .setDescription("The number of deletion vector files added as part of the commit.") + .setUnit("{file}") + .build(); } static LongCounter removedPositionDeleteFiles(Meter meter) { return meter - .counterBuilder(REMOVED_POS_DELETE_FILES) - .setDescription("The number of position delete files removed as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(REMOVED_POS_DELETE_FILES) + .setDescription("The number of position delete files removed as part of the commit.") + .setUnit("{file}") + .build(); } static LongCounter removedDeletionVectors(Meter meter) { return meter - .counterBuilder(REMOVED_DVS) - .setDescription("The number of deletion vector files removed as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(REMOVED_DVS) + .setDescription("The number of deletion vector files removed as part of the commit.") + .setUnit("{file}") + .build(); } static LongCounter removedEqualityDeleteFiles(Meter meter) { return meter - .counterBuilder(REMOVED_EQ_DELETE_FILES) - .setDescription("The number of equality delete files removed as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(REMOVED_EQ_DELETE_FILES) + .setDescription("The number of equality delete files removed as part of the commit.") + .setUnit("{file}") + .build(); } static LongCounter removedDeleteFiles(Meter meter) { return meter - .counterBuilder(REMOVED_DELETE_FILES) - .setDescription("The overall number of delete files removed as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(REMOVED_DELETE_FILES) + .setDescription("The overall number of delete files removed as part of the commit.") + .setUnit("{file}") + .build(); } static LongCounter totalDeleteFiles(Meter meter) { return meter - .counterBuilder(TOTAL_DELETE_FILES) - .setDescription("The overall number of delete files added or removed as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(TOTAL_DELETE_FILES) + .setDescription( + "The overall number of delete files added or removed as part of the commit.") + .setUnit("{file}") + .build(); } static LongCounter addedRecords(Meter meter) { return meter - .counterBuilder(ADDED_RECORDS) - .setDescription("The number of records added as part of the commit.") - .setUnit("{record}") - .build(); + .counterBuilder(ADDED_RECORDS) + .setDescription("The number of records added as part of the commit.") + .setUnit("{record}") + .build(); } static LongCounter removedRecords(Meter meter) { return meter - .counterBuilder(REMOVED_RECORDS) - .setDescription("The number of records removed as part of the commit.") - .setUnit("{record}") - .build(); + .counterBuilder(REMOVED_RECORDS) + .setDescription("The number of records removed as part of the commit.") + .setUnit("{record}") + .build(); } static LongCounter totalRecords(Meter meter) { return meter - .counterBuilder(TOTAL_RECORDS) - .setDescription("The overall number of records added or removed as part of the commit.") - .setUnit("{record}") - .build(); + .counterBuilder(TOTAL_RECORDS) + .setDescription("The overall number of records added or removed as part of the commit.") + .setUnit("{record}") + .build(); } static LongCounter addedFilesSize(Meter meter) { return meter - .counterBuilder(ADDED_FILE_SIZE_BYTES) - .setDescription("The overall size of the data and delete files added as part of the commit.") - .setUnit("byte") - .build(); + .counterBuilder(ADDED_FILE_SIZE_BYTES) + .setDescription( + "The overall size of the data and delete files added as part of the commit.") + .setUnit("byte") + .build(); } static LongCounter removedFilesSize(Meter meter) { return meter - .counterBuilder(REMOVED_FILE_SIZE_BYTES) - .setDescription("The overall size of the data or delete files removed as part of the commit.") - .setUnit("byte") - .build(); + .counterBuilder(REMOVED_FILE_SIZE_BYTES) + .setDescription( + "The overall size of the data or delete files removed as part of the commit.") + .setUnit("byte") + .build(); } static LongCounter totalFilesSize(Meter meter) { return meter - .counterBuilder(TOTAL_FILE_SIZE_BYTES) - .setDescription("The overall size of the data or delete files added or removed as part of the commit.") - .setUnit("byte") - .build(); + .counterBuilder(TOTAL_FILE_SIZE_BYTES) + .setDescription( + "The overall size of the data or delete files added or removed as part of the commit.") + .setUnit("byte") + .build(); } static LongCounter addedPositionDeletes(Meter meter) { return meter - .counterBuilder(ADDED_POS_DELETES) - .setDescription("The overall number of position delete entries added as part of the commit.") - .setUnit("{record}") - .build(); + .counterBuilder(ADDED_POS_DELETES) + .setDescription( + "The overall number of position delete entries added as part of the commit.") + .setUnit("{record}") + .build(); } static LongCounter removedPositionDeletes(Meter meter) { return meter - .counterBuilder(REMOVED_POS_DELETES) - .setDescription("The overall number of position delete entries removed as part of the commit.") - .setUnit("{record}") - .build(); + .counterBuilder(REMOVED_POS_DELETES) + .setDescription( + "The overall number of position delete entries removed as part of the commit.") + .setUnit("{record}") + .build(); } static LongCounter totalPositionDeletes(Meter meter) { return meter - .counterBuilder(TOTAL_POS_DELETES) - .setDescription("The overall number of position delete entries added or removed as part of the commit.") - .setUnit("{record}") - .build(); + .counterBuilder(TOTAL_POS_DELETES) + .setDescription( + "The overall number of position delete entries added or removed as part of the commit.") + .setUnit("{record}") + .build(); } static LongCounter addedEqualityDeletes(Meter meter) { return meter - .counterBuilder(ADDED_EQ_DELETES) - .setDescription("The overall number of equality delete entries added as part of the commit.") - .setUnit("{record}") - .build(); + .counterBuilder(ADDED_EQ_DELETES) + .setDescription( + "The overall number of equality delete entries added as part of the commit.") + .setUnit("{record}") + .build(); } static LongCounter removedEqualityDeletes(Meter meter) { return meter - .counterBuilder(REMOVED_EQ_DELETES) - .setDescription("The overall number of equality delete entries removed as part of the commit.") - .setUnit("{record}") - .build(); + .counterBuilder(REMOVED_EQ_DELETES) + .setDescription( + "The overall number of equality delete entries removed as part of the commit.") + .setUnit("{record}") + .build(); } static LongCounter totalEqualityDeletes(Meter meter) { return meter - .counterBuilder(TOTAL_EQ_DELETES) - .setDescription("The overall number of equality delete entries added or removed as part of the commit.") - .setUnit("{record}") - .build(); - } - - static LongCounter keptManifests(Meter meter) { - return meter - .counterBuilder(KEPT_MANIFESTS_COUNT) - .setDescription("The number of manifests that are kept as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter createdManfiests(Meter meter) { - return meter - .counterBuilder(CREATED_MANIFESTS_COUNT) - .setDescription("The number of manifests that are created as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter replacedManifests(Meter meter) { - return meter - .counterBuilder(REPLACED_MANIFESTS_COUNT) - .setDescription("The overall number of manifests that are deleted or overwritten as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter processedManfiestEntries(Meter meter) { - return meter - .counterBuilder(PROCESSED_MANIFEST_ENTRY_COUNT) - .setDescription("The overall number of manifest entries (referenced files) that are processed as part of the commit.") - .setUnit("{file}") - .build(); + .counterBuilder(TOTAL_EQ_DELETES) + .setDescription( + "The overall number of equality delete entries added or removed as part of the commit.") + .setUnit("{record}") + .build(); } } diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index 8272ec41e3e8..b34b8253ba7a 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -5,6 +5,11 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongGauge; import org.apache.iceberg.metrics.CommitMetricsResult; import org.apache.iceberg.metrics.CommitReport; import org.apache.iceberg.metrics.CounterResult; @@ -14,19 +19,14 @@ import org.apache.iceberg.metrics.ScanReport; import org.apache.iceberg.metrics.TimerResult; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.common.AttributeKey; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.LongCounter; -import io.opentelemetry.api.metrics.LongGauge; - public class IcebergMetricsReporter implements MetricsReporter { private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg_1.8"; private static final AttributeKey SCHEMA_ID = AttributeKey.longKey("iceberg.schema.id"); private static final AttributeKey TABLE_NAME = AttributeKey.stringKey("iceberg.table.name"); private static final AttributeKey SNAPHSOT_ID = AttributeKey.longKey("iceberg.snapshot.id"); - private static final AttributeKey SEQUENCE_NUMBER = AttributeKey.longKey("iceberg.commit.sequence_number"); + private static final AttributeKey SEQUENCE_NUMBER = + AttributeKey.longKey("iceberg.commit.sequence_number"); private final OpenTelemetry openTelemetry; @@ -197,7 +197,8 @@ void reportScanMetrics(ScanReport scanReport) { void reportCommitMetrics(CommitReport commitReport) { Attributes commitAttributes = - Attributes.of(SEQUENCE_NUMBER, + Attributes.of( + SEQUENCE_NUMBER, Long.valueOf(commitReport.sequenceNumber()), TABLE_NAME, commitReport.tableName(), @@ -207,7 +208,8 @@ void reportCommitMetrics(CommitReport commitReport) { TimerResult duration = metrics.totalDuration(); if (duration != null) { - LongGauge metric = CommitMetricsBuilder.duration(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + LongGauge metric = + CommitMetricsBuilder.duration(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.set(duration.totalDuration().toMillis(), commitAttributes); } @@ -218,7 +220,7 @@ void reportCommitMetrics(CommitReport commitReport) { CommitMetricsBuilder.attempts(openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.add(current.value(), commitAttributes); } - + current = metrics.addedDataFiles(); if (current != null) { @@ -255,7 +257,8 @@ void reportCommitMetrics(CommitReport commitReport) { if (current != null) { LongCounter metric = - CommitMetricsBuilder.addedEqualityDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + CommitMetricsBuilder.addedEqualityDeleteFiles( + openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.add(current.value(), commitAttributes); } @@ -263,7 +266,8 @@ void reportCommitMetrics(CommitReport commitReport) { if (current != null) { LongCounter metric = - CommitMetricsBuilder.addedPositionDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + CommitMetricsBuilder.addedPositionDeleteFiles( + openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.add(current.value(), commitAttributes); } @@ -279,7 +283,8 @@ void reportCommitMetrics(CommitReport commitReport) { if (current != null) { LongCounter metric = - CommitMetricsBuilder.removedPositionDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + CommitMetricsBuilder.removedPositionDeleteFiles( + openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.add(current.value(), commitAttributes); } @@ -295,7 +300,8 @@ void reportCommitMetrics(CommitReport commitReport) { if (current != null) { LongCounter metric = - CommitMetricsBuilder.removedEqualityDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + CommitMetricsBuilder.removedEqualityDeleteFiles( + openTelemetry.getMeter(INSTRUMENTATION_NAME)); metric.add(current.value(), commitAttributes); } @@ -307,5 +313,108 @@ void reportCommitMetrics(CommitReport commitReport) { metric.add(current.value(), commitAttributes); } + current = metrics.totalDataFiles(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.totalDataFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.addedRecords(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.addedRecords(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.removedRecords(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.removedRecords(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.totalRecords(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.totalRecords(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.addedFilesSizeInBytes(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.addedFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.removedFilesSizeInBytes(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.removedFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.totalFilesSizeInBytes(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.totalFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.addedPositionalDeletes(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.addedPositionDeletes(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.removedPositionalDeletes(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.removedPositionDeletes(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.totalPositionalDeletes(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.totalPositionDeletes(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.addedEqualityDeletes(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.addedEqualityDeletes(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.removedEqualityDeletes(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.removedEqualityDeletes(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } + + current = metrics.totalEqualityDeletes(); + + if (current != null) { + LongCounter metric = + CommitMetricsBuilder.totalEqualityDeletes(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + metric.add(current.value(), commitAttributes); + } } } From b912f8f0795a270d6d9535498a5f92949869a642 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Mon, 3 Nov 2025 18:14:10 +0100 Subject: [PATCH 12/33] initial testing implementation --- instrumentation/iceberg-1.8/README.md | 42 +++++++++ .../iceberg-1.8/library/build.gradle.kts | 2 + .../iceberg/v1_8/IcebergTelemetry.java | 9 +- .../iceberg/v1_8/IcebergLibraryTest.java | 25 +++++ instrumentation/iceberg-1.8/metadata.yaml | 2 + .../iceberg-1.8/testing/build.gradle.kts | 14 +++ .../iceberg/v1_8/AbstractIcebergTest.java | 93 +++++++++++++++++++ settings.gradle.kts | 1 + 8 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 instrumentation/iceberg-1.8/README.md create mode 100644 instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java create mode 100644 instrumentation/iceberg-1.8/metadata.yaml create mode 100644 instrumentation/iceberg-1.8/testing/build.gradle.kts create mode 100644 instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java diff --git a/instrumentation/iceberg-1.8/README.md b/instrumentation/iceberg-1.8/README.md new file mode 100644 index 000000000000..958f3bcf3a8c --- /dev/null +++ b/instrumentation/iceberg-1.8/README.md @@ -0,0 +1,42 @@ +# Library Instrumentation for Apache Iceberg Version 1.8 and Higher + +Provides OpenTelemetry instrumentation for [Apache Iceberg](https://iceberg.apache.org/). + +## Quickstart + +### Add These Dependencies to Your Project + +Replace `OPENTELEMETRY_VERSION` with the [latest release](https://central.sonatype.com/artifact/io.opentelemetry.instrumentation/opentelemetry-iceberg-1.8). + +For Maven, add to your `pom.xml` dependencies: + +```xml + + + io.opentelemetry.instrumentation + opentelemetry-iceberg-1.8 + OPENTELEMETRY_VERSION + + +``` + +For Gradle, add to your dependencies: + +```groovy +implementation("io.opentelemetry.instrumentation:opentelemetry-iceberg-1.8:OPENTELEMETRY_VERSION") +``` + +### Usage + +The instrumentation library allows creating instrumented `Scan` (e.g., `TableScan`) instances for collecting +OpenTelemetry-based metrics for scans and commits. For example: + +```java +OpenTelemetry openTelemetry = ... +IcebergTelemetry icebergTelemetry = IcebergTelemetry.create(openTelemetry); +TableScan tableScan = icebergTelemetry.wrapScan(table.newScan()); + +try (CloseableIterable fileScanTasks = tableScan.planFiles()) { + // ... +} +``` diff --git a/instrumentation/iceberg-1.8/library/build.gradle.kts b/instrumentation/iceberg-1.8/library/build.gradle.kts index fab8da96b885..5d0d1f3004b9 100644 --- a/instrumentation/iceberg-1.8/library/build.gradle.kts +++ b/instrumentation/iceberg-1.8/library/build.gradle.kts @@ -1,7 +1,9 @@ plugins { id("otel.library-instrumentation") + id("otel.nullaway-conventions") } dependencies { library("org.apache.iceberg:iceberg-core:1.8.1") + testImplementation(project(":instrumentation:iceberg-1.8:testing")) } diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java index e19f8be251cd..dad86ef55146 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java @@ -5,8 +5,11 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; +import org.apache.iceberg.Scan; +import org.apache.iceberg.ScanTask; +import org.apache.iceberg.ScanTaskGroup; + import io.opentelemetry.api.OpenTelemetry; -import org.apache.iceberg.TableScan; public class IcebergTelemetry { private final OpenTelemetry openTelemetry; @@ -19,7 +22,7 @@ public static IcebergTelemetry create(OpenTelemetry openTelemetry) { this.openTelemetry = openTelemetry; } - public TableScan wrapTableScan(TableScan tableScan) { - return tableScan.metricsReporter(new IcebergMetricsReporter(openTelemetry)); + public > ThisT wrapScan(Scan scan) { + return scan.metricsReporter(new IcebergMetricsReporter(openTelemetry)); } } diff --git a/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java b/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java new file mode 100644 index 000000000000..b0ed265bc85c --- /dev/null +++ b/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java @@ -0,0 +1,25 @@ +package io.opentelemetry.instrumentation.iceberg.v1_8; + +import org.apache.iceberg.TableScan; +import org.junit.jupiter.api.extension.RegisterExtension; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; +import io.opentelemetry.instrumentation.testing.junit.LibraryInstrumentationExtension; + +public class IcebergLibraryTest extends AbstractIcebergTest { + @RegisterExtension + final InstrumentationExtension testing = LibraryInstrumentationExtension.create(); + + @Override + protected InstrumentationExtension testing() { + return this.testing; + } + + @Override + protected TableScan configure(TableScan tableScan) { + OpenTelemetry openTelemetry = testing.getOpenTelemetry(); + IcebergTelemetry icebergTelemetry = IcebergTelemetry.create(openTelemetry); + return icebergTelemetry.wrapScan(tableScan); + } +} diff --git a/instrumentation/iceberg-1.8/metadata.yaml b/instrumentation/iceberg-1.8/metadata.yaml new file mode 100644 index 000000000000..be8ff24b3949 --- /dev/null +++ b/instrumentation/iceberg-1.8/metadata.yaml @@ -0,0 +1,2 @@ +description: This standalone instrumentation enables metrics for Apache Iceberg scans and commits. +library_link: https://iceberg.apache.org/ diff --git a/instrumentation/iceberg-1.8/testing/build.gradle.kts b/instrumentation/iceberg-1.8/testing/build.gradle.kts new file mode 100644 index 000000000000..d6491fcb2899 --- /dev/null +++ b/instrumentation/iceberg-1.8/testing/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + id("otel.java-conventions") +} + +dependencies { + implementation("org.apache.iceberg:iceberg-core:1.8.1") + implementation("org.apache.iceberg:iceberg-core:1.8.1") { + artifact { + classifier = "tests" + } + } + implementation("org.apache.commons:commons-compress:1.26.2") + api(project(":testing-common")) +} \ No newline at end of file diff --git a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java new file mode 100644 index 000000000000..03f08bea3dac --- /dev/null +++ b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java @@ -0,0 +1,93 @@ +package io.opentelemetry.instrumentation.iceberg.v1_8; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.TableScan; +import org.apache.iceberg.TestTables; +import org.apache.iceberg.TestTables.TestTable; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.metrics.MetricsReport; +import org.apache.iceberg.metrics.MetricsReporter; +import org.apache.iceberg.types.Types.IntegerType; +import org.apache.iceberg.types.Types.NestedField; +import org.apache.iceberg.types.Types.StringType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; + +abstract class AbstractIcebergTest { + protected abstract InstrumentationExtension testing(); + + protected abstract TableScan configure(TableScan tableScan); + + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractIcebergTest.class); + protected static final int FORMAT_VERSION = 2; + protected static final Schema SCHEMA = new Schema(NestedField.required(3, "id", IntegerType.get()), NestedField.required(4, "data", StringType.get())); + protected static final int BUCKETS_NUMBER = 16; + protected static final PartitionSpec SPEC = PartitionSpec.builderFor(SCHEMA).bucket("data", 16).build(); + protected static final DataFile FILE_1 = DataFiles.builder(SPEC).withPath("/path/to/data-a.parquet").withFileSizeInBytes(10L).withPartitionPath("data_bucket=0").withRecordCount(1L).build(); + protected static final DataFile FILE_2 = DataFiles.builder(SPEC).withPath("/path/to/data-b.parquet").withFileSizeInBytes(10L).withPartitionPath("data_bucket=1").withRecordCount(1L).withSplitOffsets(Arrays.asList(1L)).build(); + + @TempDir + protected File tableDir = null; + protected TestTable table; + + @BeforeEach + void init() { + this.table = TestTables.create(this.tableDir, "test", SCHEMA, SPEC, FORMAT_VERSION); + this.table.newFastAppend().appendFile(FILE_1).appendFile(FILE_2).commit(); + } + + @Test + void testCreateTelemetry() throws IOException { + SimpleReporter reporter = new SimpleReporter(); + + TableScan scan = table.newScan() + .select("id", "data"); + assertNotNull(scan); + assertNull(reporter.report); + + Schema projection = scan.schema(); + assertNotNull(projection); + + try (CloseableIterable tasks = scan.planFiles()) { + assertNotNull(tasks); + int counter = 0; + + for (FileScanTask fileTask : tasks) { + LOGGER.info(fileTask.file().location()); + counter++; + } + + assertEquals(2, counter); + assertNotNull(reporter.report); + } + } + + static class SimpleReporter implements MetricsReporter { + MetricsReport report; + + @Override + public void report(MetricsReport report) { + LOGGER.error("I am invoked!"); + this.report = report; + } + + } + +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 80bd790591e4..6ad081f0249a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -304,6 +304,7 @@ include(":instrumentation:hikaricp-3.0:testing") include(":instrumentation:http-url-connection:javaagent") include(":instrumentation:hystrix-1.4:javaagent") include(":instrumentation:iceberg-1.8:library") +include(":instrumentation:iceberg-1.8:testing") include(":instrumentation:influxdb-2.4:javaagent") include(":instrumentation:internal:internal-application-logger:bootstrap") include(":instrumentation:internal:internal-application-logger:javaagent") From 9498d36b386a59b1bf2a54fada557cf7c6727a48 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Tue, 4 Nov 2025 11:29:52 +0100 Subject: [PATCH 13/33] finish implementing unit test --- .../iceberg/v1_8/IcebergMetricsReporter.java | 5 +- .../iceberg/v1_8/IcebergTelemetry.java | 6 +- .../iceberg/v1_8/ScanMetricsBuilder.java | 6 +- .../iceberg/v1_8/IcebergLibraryTest.java | 10 +- .../iceberg-1.8/testing/build.gradle.kts | 2 +- .../iceberg/v1_8/AbstractIcebergTest.java | 360 ++++++++++++++++-- 6 files changed, 342 insertions(+), 47 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index b34b8253ba7a..a73920368b86 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -10,6 +10,7 @@ import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.metrics.LongGauge; +import java.util.Locale; import org.apache.iceberg.metrics.CommitMetricsResult; import org.apache.iceberg.metrics.CommitReport; import org.apache.iceberg.metrics.CounterResult; @@ -57,7 +58,9 @@ void reportScanMetrics(ScanReport scanReport) { if (duration != null) { LongGauge metric = - ScanMetricsBuilder.totalPlanningDuration(openTelemetry.getMeter(INSTRUMENTATION_NAME)); + ScanMetricsBuilder.totalPlanningDuration( + openTelemetry.getMeter(INSTRUMENTATION_NAME), + duration.timeUnit().name().toLowerCase(Locale.getDefault())); metric.set(duration.totalDuration().toMillis(), scanAttributes); } diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java index dad86ef55146..90af18e987d5 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java @@ -5,12 +5,11 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; +import io.opentelemetry.api.OpenTelemetry; import org.apache.iceberg.Scan; import org.apache.iceberg.ScanTask; import org.apache.iceberg.ScanTaskGroup; -import io.opentelemetry.api.OpenTelemetry; - public class IcebergTelemetry { private final OpenTelemetry openTelemetry; @@ -22,7 +21,8 @@ public static IcebergTelemetry create(OpenTelemetry openTelemetry) { this.openTelemetry = openTelemetry; } - public > ThisT wrapScan(Scan scan) { + public > ThisT wrapScan( + Scan scan) { return scan.metricsReporter(new IcebergMetricsReporter(openTelemetry)); } } diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java index 5d31059189b3..760fd5933921 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java @@ -19,7 +19,7 @@ final class ScanMetricsBuilder { private static final String TOTAL_DATA_MANIFESTS = ROOT + ".total.data_manifests.count"; private static final String TOTAL_DELETE_MANIFESTS = ROOT + ".total.delete_manifests.count"; private static final String TOTAL_FILE_SIZE_IN_BYTES = ROOT + ".scanned.data_files.size"; - private static final String TOTAL_DELETE_FILE_SIZE_IN_BYTES = ROOT + "scanned.delete_files.size"; + private static final String TOTAL_DELETE_FILE_SIZE_IN_BYTES = ROOT + ".scanned.delete_files.size"; private static final String SKIPPED_DATA_MANIFESTS = ROOT + ".skipped.data_manifests.count"; private static final String SKIPPED_DELETE_MANIFESTS = ROOT + ".skipped.delete_manifests.count"; private static final String SKIPPED_DATA_FILES = ROOT + ".skipped.data_files.count"; @@ -34,11 +34,11 @@ private ScanMetricsBuilder() { // prevents instantiation } - static LongGauge totalPlanningDuration(Meter meter) { + static LongGauge totalPlanningDuration(Meter meter, String unit) { return meter .gaugeBuilder(TOTAL_PLANNING_DURATION) .setDescription("The total duration needed to plan the scan.") - .setUnit("ms") + .setUnit(unit) .ofLongs() .build(); } diff --git a/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java b/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java index b0ed265bc85c..c2bc2ca798ea 100644 --- a/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java +++ b/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java @@ -1,11 +1,15 @@ -package io.opentelemetry.instrumentation.iceberg.v1_8; +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ -import org.apache.iceberg.TableScan; -import org.junit.jupiter.api.extension.RegisterExtension; +package io.opentelemetry.instrumentation.iceberg.v1_8; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; import io.opentelemetry.instrumentation.testing.junit.LibraryInstrumentationExtension; +import org.apache.iceberg.TableScan; +import org.junit.jupiter.api.extension.RegisterExtension; public class IcebergLibraryTest extends AbstractIcebergTest { @RegisterExtension diff --git a/instrumentation/iceberg-1.8/testing/build.gradle.kts b/instrumentation/iceberg-1.8/testing/build.gradle.kts index d6491fcb2899..1bb747a6840d 100644 --- a/instrumentation/iceberg-1.8/testing/build.gradle.kts +++ b/instrumentation/iceberg-1.8/testing/build.gradle.kts @@ -11,4 +11,4 @@ dependencies { } implementation("org.apache.commons:commons-compress:1.26.2") api(project(":testing-common")) -} \ No newline at end of file +} diff --git a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java index 03f08bea3dac..a369fc3f3ea8 100644 --- a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java +++ b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java @@ -1,13 +1,20 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + package io.opentelemetry.instrumentation.iceberg.v1_8; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; import java.io.File; import java.io.IOException; import java.util.Arrays; - +import java.util.Locale; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileScanTask; @@ -16,37 +23,53 @@ import org.apache.iceberg.TableScan; import org.apache.iceberg.TestTables; import org.apache.iceberg.TestTables.TestTable; +import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.metrics.CounterResult; import org.apache.iceberg.metrics.MetricsReport; import org.apache.iceberg.metrics.MetricsReporter; +import org.apache.iceberg.metrics.ScanReport; +import org.apache.iceberg.metrics.TimerResult; import org.apache.iceberg.types.Types.IntegerType; import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.types.Types.StringType; +import org.assertj.core.api.Condition; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; abstract class AbstractIcebergTest { - protected abstract InstrumentationExtension testing(); - - protected abstract TableScan configure(TableScan tableScan); - - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractIcebergTest.class); protected static final int FORMAT_VERSION = 2; - protected static final Schema SCHEMA = new Schema(NestedField.required(3, "id", IntegerType.get()), NestedField.required(4, "data", StringType.get())); + protected static final Schema SCHEMA = + new Schema( + NestedField.required(3, "id", IntegerType.get()), + NestedField.required(4, "data", StringType.get())); protected static final int BUCKETS_NUMBER = 16; - protected static final PartitionSpec SPEC = PartitionSpec.builderFor(SCHEMA).bucket("data", 16).build(); - protected static final DataFile FILE_1 = DataFiles.builder(SPEC).withPath("/path/to/data-a.parquet").withFileSizeInBytes(10L).withPartitionPath("data_bucket=0").withRecordCount(1L).build(); - protected static final DataFile FILE_2 = DataFiles.builder(SPEC).withPath("/path/to/data-b.parquet").withFileSizeInBytes(10L).withPartitionPath("data_bucket=1").withRecordCount(1L).withSplitOffsets(Arrays.asList(1L)).build(); + protected static final PartitionSpec SPEC = + PartitionSpec.builderFor(SCHEMA).bucket("data", 16).build(); + protected static final DataFile FILE_1 = + DataFiles.builder(SPEC) + .withPath("/path/to/data-a.parquet") + .withFileSizeInBytes(10L) + .withPartitionPath("data_bucket=0") + .withRecordCount(1L) + .build(); + protected static final DataFile FILE_2 = + DataFiles.builder(SPEC) + .withPath("/path/to/data-b.parquet") + .withFileSizeInBytes(10L) + .withPartitionPath("data_bucket=1") + .withRecordCount(1L) + .withSplitOffsets(Arrays.asList(1L)) + .build(); - @TempDir - protected File tableDir = null; + @TempDir protected File tableDir = null; protected TestTable table; + protected abstract InstrumentationExtension testing(); + + protected abstract TableScan configure(TableScan tableScan); + @BeforeEach void init() { this.table = TestTables.create(this.tableDir, "test", SCHEMA, SPEC, FORMAT_VERSION); @@ -55,39 +78,304 @@ void init() { @Test void testCreateTelemetry() throws IOException { - SimpleReporter reporter = new SimpleReporter(); - TableScan scan = table.newScan() - .select("id", "data"); - assertNotNull(scan); - assertNull(reporter.report); - - Schema projection = scan.schema(); - assertNotNull(projection); + SimpleReporter reporter = new SimpleReporter(); + TableScan scan = + table + .newScan() + .filter(Expressions.lessThan("id", 5)) + .select("id", "data") + .metricsReporter(reporter); + scan = configure(scan); try (CloseableIterable tasks = scan.planFiles()) { assertNotNull(tasks); - int counter = 0; + assertNotNull(tasks.iterator()); + } + + assertNotNull(reporter.report); + assertTrue(reporter.report instanceof ScanReport); + ScanReport expected = (ScanReport) reporter.report; + CounterResult currentExpectedMetric = expected.scanMetrics().resultDataFiles(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.scanned.data_files.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.scanned.data_files.count"); + } + + currentExpectedMetric = expected.scanMetrics().resultDeleteFiles(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.scanned.delete_files.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.scanned.delete_files.count"); + } + + currentExpectedMetric = expected.scanMetrics().scannedDataManifests(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.scanned.data_manifests.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.scanned.data_manifests.count"); + } + + currentExpectedMetric = expected.scanMetrics().scannedDeleteManifests(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.scanned.delete_manifests.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.scanned.delete_manifests.count"); + } + + currentExpectedMetric = expected.scanMetrics().totalDataManifests(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.total.data_manifests.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.total.data_manifests.count"); + } + + currentExpectedMetric = expected.scanMetrics().totalDeleteManifests(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.total.delete_manifests.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.total.delete_manifests.count"); + } + + currentExpectedMetric = expected.scanMetrics().totalFileSizeInBytes(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.scanned.data_files.size", "byte", expected, currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.scanned.data_files.size"); + } + + currentExpectedMetric = expected.scanMetrics().totalDeleteFileSizeInBytes(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.scanned.delete_files.size", + "byte", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.scanned.delete_files.size"); + } + + currentExpectedMetric = expected.scanMetrics().skippedDataManifests(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.skipped.data_manifests.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.skipped.data_manifests.count"); + } + + currentExpectedMetric = expected.scanMetrics().skippedDeleteManifests(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.skipped.delete_manifests.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.skipped.delete_manifests.count"); + } + + currentExpectedMetric = expected.scanMetrics().skippedDataFiles(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.skipped.data_files.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.skipped.data_files.count"); + } + + currentExpectedMetric = expected.scanMetrics().skippedDeleteFiles(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.skipped.delete_files.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.skipped.delete_files.count"); + } + + currentExpectedMetric = expected.scanMetrics().indexedDeleteFiles(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.scanned.indexed_delete_files.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.scanned.indexed_delete_files.count"); + } + + currentExpectedMetric = expected.scanMetrics().equalityDeleteFiles(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.scanned.equality_delete_files.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.scanned.equality_delete_files.count"); + } + + currentExpectedMetric = expected.scanMetrics().positionalDeleteFiles(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.scanned.positional_delete_files.count", + "{file}", + expected, + currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.scanned.positional_delete_files.count"); + } + + currentExpectedMetric = expected.scanMetrics().dvs(); + + if (currentExpectedMetric != null) { + assertIcebergCounterMetric( + "iceberg.scan.scanned.dvs.count", "{file}", expected, currentExpectedMetric.value()); + } else { + assertIcebergMetricNotReported("iceberg.scan.scanned.dvs.count"); + } - for (FileScanTask fileTask : tasks) { - LOGGER.info(fileTask.file().location()); - counter++; - } + TimerResult timer = expected.scanMetrics().totalPlanningDuration(); - assertEquals(2, counter); - assertNotNull(reporter.report); + if (timer != null) { + assertIcebergGaugeMetric( + "iceberg.scan.planning.duration", + timer.timeUnit().name().toLowerCase(Locale.getDefault()), + expected, + timer.totalDuration().toMillis()); + } else { + assertIcebergMetricNotReported("iceberg.scan.planning.duration"); } } - static class SimpleReporter implements MetricsReporter { + private void assertIcebergMetricNotReported(String otelMetricName) { + testing() + .waitAndAssertMetrics( + otelMetricName, + metricAssert -> + metricAssert.doesNotHave( + new Condition<>( + spanData -> otelMetricName.equals(spanData.getName()), + "metric is not reported"))); + } + + private void assertIcebergGaugeMetric( + String otelMetricName, String expectedUnit, ScanReport expectedReport, long expectedValue) { + testing() + .waitAndAssertMetrics( + "io.opentelemetry.iceberg_1.8", + metricAssert -> + metricAssert + .hasName(otelMetricName) + .hasUnit(expectedUnit) + .hasLongGaugeSatisfying( + sum -> + sum.hasPointsSatisfying( + longAssert -> + longAssert + .hasValue(expectedValue) + .hasAttributesSatisfying( + attributes -> + assertEquals( + Attributes.builder() + .put( + "iceberg.schema.id", + expectedReport.schemaId()) + .put( + "iceberg.table.name", + expectedReport.tableName()) + .put( + "iceberg.snapshot.id", + expectedReport.snapshotId()) + .build(), + attributes))))); + } + + private void assertIcebergCounterMetric( + String otelMetricName, String expectedUnit, ScanReport expectedReport, long expectedValue) { + testing() + .waitAndAssertMetrics( + "io.opentelemetry.iceberg_1.8", + metricAssert -> + metricAssert + .hasName(otelMetricName) + .hasUnit(expectedUnit) + .hasLongSumSatisfying( + sum -> + sum.hasPointsSatisfying( + longSumAssert -> + longSumAssert + .hasValue(expectedValue) + .hasAttributesSatisfying( + attributes -> + assertEquals( + Attributes.builder() + .put( + "iceberg.schema.id", + expectedReport.schemaId()) + .put( + "iceberg.table.name", + expectedReport.tableName()) + .put( + "iceberg.snapshot.id", + expectedReport.snapshotId()) + .build(), + attributes))))); + } + + static final class SimpleReporter implements MetricsReporter { MetricsReport report; @Override public void report(MetricsReport report) { - LOGGER.error("I am invoked!"); this.report = report; } - } - } From a35cf9e75ce594494f088b7e8530ff123339e14a Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Tue, 4 Nov 2025 11:36:15 +0100 Subject: [PATCH 14/33] remove commit metrics --- instrumentation/iceberg-1.8/README.md | 9 +- .../iceberg/v1_8/CommitMetricsBuilder.java | 266 ------------------ .../iceberg/v1_8/IcebergMetricsReporter.java | 229 --------------- instrumentation/iceberg-1.8/metadata.yaml | 2 +- .../iceberg-1.8/testing/build.gradle.kts | 1 - 5 files changed, 6 insertions(+), 501 deletions(-) delete mode 100644 instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/CommitMetricsBuilder.java diff --git a/instrumentation/iceberg-1.8/README.md b/instrumentation/iceberg-1.8/README.md index 958f3bcf3a8c..1ec0afb57af7 100644 --- a/instrumentation/iceberg-1.8/README.md +++ b/instrumentation/iceberg-1.8/README.md @@ -28,15 +28,16 @@ implementation("io.opentelemetry.instrumentation:opentelemetry-iceberg-1.8:OPENT ### Usage -The instrumentation library allows creating instrumented `Scan` (e.g., `TableScan`) instances for collecting -OpenTelemetry-based metrics for scans and commits. For example: +The instrumentation library allows creating instrumented `Scan` (e.g., `TableScan`) instances for collecting and reporting OpenTelemetry-based sacn metrics. For example: ```java -OpenTelemetry openTelemetry = ... +OpenTelemetry openTelemetry = // ... IcebergTelemetry icebergTelemetry = IcebergTelemetry.create(openTelemetry); TableScan tableScan = icebergTelemetry.wrapScan(table.newScan()); try (CloseableIterable fileScanTasks = tableScan.planFiles()) { - // ... + // Process the scan tasks } + +// The metrics will be reported after the scan tasks iterable is closed ``` diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/CommitMetricsBuilder.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/CommitMetricsBuilder.java deleted file mode 100644 index afab06ce391d..000000000000 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/CommitMetricsBuilder.java +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package io.opentelemetry.instrumentation.iceberg.v1_8; - -import io.opentelemetry.api.metrics.LongCounter; -import io.opentelemetry.api.metrics.LongGauge; -import io.opentelemetry.api.metrics.Meter; - -final class CommitMetricsBuilder { - - private static final String ROOT = "iceberg.commit"; - private static final String DURATION = ROOT + ".duration"; - private static final String ATTEMPTS = ROOT + ".attempts.count"; - private static final String ADDED_DATA_FILES = ROOT + ".added.data_files.count"; - private static final String REMOVED_DATA_FILES = ROOT + ".removed.data_files.count"; - private static final String TOTAL_DATA_FILES = ROOT + ".total.data_files.count"; - private static final String ADDED_DELETE_FILES = ROOT + ".added.delete_files.count"; - private static final String ADDED_EQ_DELETE_FILES = ROOT + ".added.equality_delete_files.count"; - private static final String ADDED_POS_DELETE_FILES = ROOT + ".added.position_delete_files.count"; - private static final String ADDED_DVS = ROOT + ".added.dvs.count"; - private static final String REMOVED_POS_DELETE_FILES = - ROOT + ".removed.positional_delete_files.count"; - private static final String REMOVED_DVS = ROOT + ".removed.dvs.count"; - private static final String REMOVED_EQ_DELETE_FILES = - ROOT + ".removed.equality_delete_files.count"; - private static final String REMOVED_DELETE_FILES = ROOT + ".removed.delete_files.count"; - private static final String TOTAL_DELETE_FILES = ROOT + ".total.delete_files.count"; - private static final String ADDED_RECORDS = ROOT + ".added.records.count"; - private static final String REMOVED_RECORDS = ROOT + ".removed.records.count"; - private static final String TOTAL_RECORDS = ROOT + ".total.records.count"; - private static final String ADDED_FILE_SIZE_BYTES = ROOT + ".added.files.size"; - private static final String REMOVED_FILE_SIZE_BYTES = ROOT + ".removed.files.size"; - private static final String TOTAL_FILE_SIZE_BYTES = ROOT + ".total.files.size"; - private static final String ADDED_POS_DELETES = ROOT + ".added.position_deletes.count"; - private static final String REMOVED_POS_DELETES = ROOT + ".removed.position_deletes.count"; - private static final String TOTAL_POS_DELETES = ROOT + ".total.position_deletes.count"; - private static final String ADDED_EQ_DELETES = ROOT + ".added.equality_deletes.count"; - private static final String REMOVED_EQ_DELETES = ROOT + ".removed.equality_deletes.count"; - private static final String TOTAL_EQ_DELETES = ROOT + ".total.equality_deletes.count"; - - private CommitMetricsBuilder() { - // prevents instantiation - } - - static LongGauge duration(Meter meter) { - return meter - .gaugeBuilder(DURATION) - .setDescription("The duration taken to process the commit.") - .setUnit("ms") - .ofLongs() - .build(); - } - - static LongCounter attempts(Meter meter) { - return meter - .counterBuilder(ATTEMPTS) - .setDescription("The number of attempts made to complete this commit.") - .setUnit("{attempt}") - .build(); - } - - static LongCounter addedDataFiles(Meter meter) { - return meter - .counterBuilder(ADDED_DATA_FILES) - .setDescription("The number of data files added as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter removedDataFiles(Meter meter) { - return meter - .counterBuilder(REMOVED_DATA_FILES) - .setDescription("The number of data files removed as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter totalDataFiles(Meter meter) { - return meter - .counterBuilder(TOTAL_DATA_FILES) - .setDescription("The number of data files added or removed as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter addedDeleteFiles(Meter meter) { - return meter - .counterBuilder(ADDED_DELETE_FILES) - .setDescription("The overall number of delete files added as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter addedEqualityDeleteFiles(Meter meter) { - return meter - .counterBuilder(ADDED_EQ_DELETE_FILES) - .setDescription("The number of equality delete files added as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter addedPositionDeleteFiles(Meter meter) { - return meter - .counterBuilder(ADDED_POS_DELETE_FILES) - .setDescription("The number of position delete files added as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter addedDeletionVectors(Meter meter) { - return meter - .counterBuilder(ADDED_DVS) - .setDescription("The number of deletion vector files added as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter removedPositionDeleteFiles(Meter meter) { - return meter - .counterBuilder(REMOVED_POS_DELETE_FILES) - .setDescription("The number of position delete files removed as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter removedDeletionVectors(Meter meter) { - return meter - .counterBuilder(REMOVED_DVS) - .setDescription("The number of deletion vector files removed as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter removedEqualityDeleteFiles(Meter meter) { - return meter - .counterBuilder(REMOVED_EQ_DELETE_FILES) - .setDescription("The number of equality delete files removed as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter removedDeleteFiles(Meter meter) { - return meter - .counterBuilder(REMOVED_DELETE_FILES) - .setDescription("The overall number of delete files removed as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter totalDeleteFiles(Meter meter) { - return meter - .counterBuilder(TOTAL_DELETE_FILES) - .setDescription( - "The overall number of delete files added or removed as part of the commit.") - .setUnit("{file}") - .build(); - } - - static LongCounter addedRecords(Meter meter) { - return meter - .counterBuilder(ADDED_RECORDS) - .setDescription("The number of records added as part of the commit.") - .setUnit("{record}") - .build(); - } - - static LongCounter removedRecords(Meter meter) { - return meter - .counterBuilder(REMOVED_RECORDS) - .setDescription("The number of records removed as part of the commit.") - .setUnit("{record}") - .build(); - } - - static LongCounter totalRecords(Meter meter) { - return meter - .counterBuilder(TOTAL_RECORDS) - .setDescription("The overall number of records added or removed as part of the commit.") - .setUnit("{record}") - .build(); - } - - static LongCounter addedFilesSize(Meter meter) { - return meter - .counterBuilder(ADDED_FILE_SIZE_BYTES) - .setDescription( - "The overall size of the data and delete files added as part of the commit.") - .setUnit("byte") - .build(); - } - - static LongCounter removedFilesSize(Meter meter) { - return meter - .counterBuilder(REMOVED_FILE_SIZE_BYTES) - .setDescription( - "The overall size of the data or delete files removed as part of the commit.") - .setUnit("byte") - .build(); - } - - static LongCounter totalFilesSize(Meter meter) { - return meter - .counterBuilder(TOTAL_FILE_SIZE_BYTES) - .setDescription( - "The overall size of the data or delete files added or removed as part of the commit.") - .setUnit("byte") - .build(); - } - - static LongCounter addedPositionDeletes(Meter meter) { - return meter - .counterBuilder(ADDED_POS_DELETES) - .setDescription( - "The overall number of position delete entries added as part of the commit.") - .setUnit("{record}") - .build(); - } - - static LongCounter removedPositionDeletes(Meter meter) { - return meter - .counterBuilder(REMOVED_POS_DELETES) - .setDescription( - "The overall number of position delete entries removed as part of the commit.") - .setUnit("{record}") - .build(); - } - - static LongCounter totalPositionDeletes(Meter meter) { - return meter - .counterBuilder(TOTAL_POS_DELETES) - .setDescription( - "The overall number of position delete entries added or removed as part of the commit.") - .setUnit("{record}") - .build(); - } - - static LongCounter addedEqualityDeletes(Meter meter) { - return meter - .counterBuilder(ADDED_EQ_DELETES) - .setDescription( - "The overall number of equality delete entries added as part of the commit.") - .setUnit("{record}") - .build(); - } - - static LongCounter removedEqualityDeletes(Meter meter) { - return meter - .counterBuilder(REMOVED_EQ_DELETES) - .setDescription( - "The overall number of equality delete entries removed as part of the commit.") - .setUnit("{record}") - .build(); - } - - static LongCounter totalEqualityDeletes(Meter meter) { - return meter - .counterBuilder(TOTAL_EQ_DELETES) - .setDescription( - "The overall number of equality delete entries added or removed as part of the commit.") - .setUnit("{record}") - .build(); - } -} diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index a73920368b86..89bd56e37841 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -11,8 +11,6 @@ import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.metrics.LongGauge; import java.util.Locale; -import org.apache.iceberg.metrics.CommitMetricsResult; -import org.apache.iceberg.metrics.CommitReport; import org.apache.iceberg.metrics.CounterResult; import org.apache.iceberg.metrics.MetricsReport; import org.apache.iceberg.metrics.MetricsReporter; @@ -26,8 +24,6 @@ public class IcebergMetricsReporter implements MetricsReporter { private static final AttributeKey TABLE_NAME = AttributeKey.stringKey("iceberg.table.name"); private static final AttributeKey SNAPHSOT_ID = AttributeKey.longKey("iceberg.snapshot.id"); - private static final AttributeKey SEQUENCE_NUMBER = - AttributeKey.longKey("iceberg.commit.sequence_number"); private final OpenTelemetry openTelemetry; @@ -39,8 +35,6 @@ public class IcebergMetricsReporter implements MetricsReporter { public void report(MetricsReport report) { if (report instanceof ScanReport) { reportScanMetrics((ScanReport) report); - } else if (report instanceof CommitReport) { - reportCommitMetrics((CommitReport) report); } } @@ -197,227 +191,4 @@ void reportScanMetrics(ScanReport scanReport) { metric.add(current.value(), scanAttributes); } } - - void reportCommitMetrics(CommitReport commitReport) { - Attributes commitAttributes = - Attributes.of( - SEQUENCE_NUMBER, - Long.valueOf(commitReport.sequenceNumber()), - TABLE_NAME, - commitReport.tableName(), - SNAPHSOT_ID, - commitReport.snapshotId()); - CommitMetricsResult metrics = commitReport.commitMetrics(); - TimerResult duration = metrics.totalDuration(); - - if (duration != null) { - LongGauge metric = - CommitMetricsBuilder.duration(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.set(duration.totalDuration().toMillis(), commitAttributes); - } - - CounterResult current = metrics.attempts(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.attempts(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.addedDataFiles(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.addedDataFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.removedDataFiles(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.removedDataFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.totalDataFiles(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.totalDataFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.addedDeleteFiles(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.addedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.addedEqualityDeleteFiles(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.addedEqualityDeleteFiles( - openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.addedPositionalDeleteFiles(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.addedPositionDeleteFiles( - openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.addedDVs(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.addedDeletionVectors(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.removedPositionalDeleteFiles(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.removedPositionDeleteFiles( - openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.removedDVs(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.removedDeletionVectors(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.removedEqualityDeleteFiles(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.removedEqualityDeleteFiles( - openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.removedDeleteFiles(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.removedDeleteFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.totalDataFiles(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.totalDataFiles(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.addedRecords(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.addedRecords(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.removedRecords(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.removedRecords(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.totalRecords(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.totalRecords(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.addedFilesSizeInBytes(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.addedFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.removedFilesSizeInBytes(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.removedFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.totalFilesSizeInBytes(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.totalFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.addedPositionalDeletes(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.addedPositionDeletes(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.removedPositionalDeletes(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.removedPositionDeletes(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.totalPositionalDeletes(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.totalPositionDeletes(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.addedEqualityDeletes(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.addedEqualityDeletes(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.removedEqualityDeletes(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.removedEqualityDeletes(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - - current = metrics.totalEqualityDeletes(); - - if (current != null) { - LongCounter metric = - CommitMetricsBuilder.totalEqualityDeletes(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), commitAttributes); - } - } } diff --git a/instrumentation/iceberg-1.8/metadata.yaml b/instrumentation/iceberg-1.8/metadata.yaml index be8ff24b3949..3000e45310d8 100644 --- a/instrumentation/iceberg-1.8/metadata.yaml +++ b/instrumentation/iceberg-1.8/metadata.yaml @@ -1,2 +1,2 @@ -description: This standalone instrumentation enables metrics for Apache Iceberg scans and commits. +description: This standalone instrumentation enables metrics for Apache Iceberg scans. library_link: https://iceberg.apache.org/ diff --git a/instrumentation/iceberg-1.8/testing/build.gradle.kts b/instrumentation/iceberg-1.8/testing/build.gradle.kts index 1bb747a6840d..88527621ebb4 100644 --- a/instrumentation/iceberg-1.8/testing/build.gradle.kts +++ b/instrumentation/iceberg-1.8/testing/build.gradle.kts @@ -9,6 +9,5 @@ dependencies { classifier = "tests" } } - implementation("org.apache.commons:commons-compress:1.26.2") api(project(":testing-common")) } From 920dd26165774c146e279bcc149e6b81d990919a Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Tue, 4 Nov 2025 12:48:39 +0100 Subject: [PATCH 15/33] Fix style issue --- .../instrumentation/iceberg/v1_8/IcebergTelemetry.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java index 90af18e987d5..5178628894fd 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java @@ -5,11 +5,12 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; -import io.opentelemetry.api.OpenTelemetry; import org.apache.iceberg.Scan; import org.apache.iceberg.ScanTask; import org.apache.iceberg.ScanTaskGroup; +import io.opentelemetry.api.OpenTelemetry; + public class IcebergTelemetry { private final OpenTelemetry openTelemetry; @@ -21,8 +22,8 @@ public static IcebergTelemetry create(OpenTelemetry openTelemetry) { this.openTelemetry = openTelemetry; } - public > ThisT wrapScan( - Scan scan) { + public > T1 wrapScan( + Scan scan) { return scan.metricsReporter(new IcebergMetricsReporter(openTelemetry)); } } From 6a758b718d44ea0e63a83d4d634483b76dd04e3f Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Tue, 4 Nov 2025 13:00:14 +0100 Subject: [PATCH 16/33] reorder imports --- .../instrumentation/iceberg/v1_8/IcebergTelemetry.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java index 5178628894fd..41f43c3b9a79 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java @@ -5,12 +5,11 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; +import io.opentelemetry.api.OpenTelemetry; import org.apache.iceberg.Scan; import org.apache.iceberg.ScanTask; import org.apache.iceberg.ScanTaskGroup; -import io.opentelemetry.api.OpenTelemetry; - public class IcebergTelemetry { private final OpenTelemetry openTelemetry; From a42c03e987a3269c684a95d10e02b7b36db1e733 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Tue, 4 Nov 2025 13:20:17 +0100 Subject: [PATCH 17/33] make only class in testing project public to pass javadoc task --- instrumentation/iceberg-1.8/library/build.gradle.kts | 6 ++++++ .../instrumentation/iceberg/v1_8/AbstractIcebergTest.java | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/instrumentation/iceberg-1.8/library/build.gradle.kts b/instrumentation/iceberg-1.8/library/build.gradle.kts index 5d0d1f3004b9..4bda55bc587b 100644 --- a/instrumentation/iceberg-1.8/library/build.gradle.kts +++ b/instrumentation/iceberg-1.8/library/build.gradle.kts @@ -7,3 +7,9 @@ dependencies { library("org.apache.iceberg:iceberg-core:1.8.1") testImplementation(project(":instrumentation:iceberg-1.8:testing")) } + +tasks { + test { + systemProperty("collectMetadata", findProperty("collectMetadata")?.toString() ?: "false") + } +} diff --git a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java index a369fc3f3ea8..b595fd885ccc 100644 --- a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java +++ b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java @@ -38,7 +38,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -abstract class AbstractIcebergTest { +public abstract class AbstractIcebergTest { protected static final int FORMAT_VERSION = 2; protected static final Schema SCHEMA = new Schema( From 061f58f849816cc4ba7c9ef7908f3d0e08858632 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Tue, 4 Nov 2025 14:03:33 +0100 Subject: [PATCH 18/33] set minJavaVersionSupported to 11 since this is required by the Iceberg API --- instrumentation/iceberg-1.8/library/build.gradle.kts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/build.gradle.kts b/instrumentation/iceberg-1.8/library/build.gradle.kts index 4bda55bc587b..739bb00d9f1a 100644 --- a/instrumentation/iceberg-1.8/library/build.gradle.kts +++ b/instrumentation/iceberg-1.8/library/build.gradle.kts @@ -8,8 +8,6 @@ dependencies { testImplementation(project(":instrumentation:iceberg-1.8:testing")) } -tasks { - test { - systemProperty("collectMetadata", findProperty("collectMetadata")?.toString() ?: "false") - } +otelJava { + minJavaVersionSupported.set(JavaVersion.VERSION_11) } From e672e3b563fe8c2cdd3cd5e1d7aa9f5812e0ad38 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Tue, 4 Nov 2025 16:18:37 +0100 Subject: [PATCH 19/33] Apply suggestions from code review Co-authored-by: Jay DeLuca --- instrumentation/iceberg-1.8/README.md | 2 +- .../instrumentation/iceberg/v1_8/IcebergMetricsReporter.java | 2 +- .../instrumentation/iceberg/v1_8/IcebergLibraryTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/instrumentation/iceberg-1.8/README.md b/instrumentation/iceberg-1.8/README.md index 1ec0afb57af7..e218632b3b83 100644 --- a/instrumentation/iceberg-1.8/README.md +++ b/instrumentation/iceberg-1.8/README.md @@ -28,7 +28,7 @@ implementation("io.opentelemetry.instrumentation:opentelemetry-iceberg-1.8:OPENT ### Usage -The instrumentation library allows creating instrumented `Scan` (e.g., `TableScan`) instances for collecting and reporting OpenTelemetry-based sacn metrics. For example: +The instrumentation library allows creating instrumented `Scan` (e.g., `TableScan`) instances for collecting and reporting OpenTelemetry-based scan metrics. For example: ```java OpenTelemetry openTelemetry = // ... diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index 89bd56e37841..5075c5a491eb 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -19,7 +19,7 @@ import org.apache.iceberg.metrics.TimerResult; public class IcebergMetricsReporter implements MetricsReporter { - private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg_1.8"; + private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg-1.8"; private static final AttributeKey SCHEMA_ID = AttributeKey.longKey("iceberg.schema.id"); private static final AttributeKey TABLE_NAME = AttributeKey.stringKey("iceberg.table.name"); diff --git a/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java b/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java index c2bc2ca798ea..3cd02e78f357 100644 --- a/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java +++ b/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java @@ -17,7 +17,7 @@ public class IcebergLibraryTest extends AbstractIcebergTest { @Override protected InstrumentationExtension testing() { - return this.testing; + return testing; } @Override From af1dd704ac51162cc0b5e8cf4603b0fce29e5bd0 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Tue, 4 Nov 2025 16:21:04 +0100 Subject: [PATCH 20/33] apply suggestions from code review --- .../iceberg/v1_8/ScanMetricsBuilder.java | 4 ++-- .../{IcebergLibraryTest.java => IcebergTest.java} | 7 ++++--- .../iceberg/v1_8/AbstractIcebergTest.java | 14 ++++++++------ 3 files changed, 14 insertions(+), 11 deletions(-) rename instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/{IcebergLibraryTest.java => IcebergTest.java} (93%) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java index 760fd5933921..c6ffb9544fd0 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java @@ -95,7 +95,7 @@ static LongCounter scannedDataFilesSize(Meter meter) { return meter .counterBuilder(TOTAL_FILE_SIZE_IN_BYTES) .setDescription("The total size of all scanned data files.") - .setUnit("byte") + .setUnit("By") .build(); } @@ -103,7 +103,7 @@ static LongCounter scannedDeleteFilesSize(Meter meter) { return meter .counterBuilder(TOTAL_DELETE_FILE_SIZE_IN_BYTES) .setDescription("The total size of all scanned delete files.") - .setUnit("byte") + .setUnit("By") .build(); } diff --git a/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java b/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTest.java similarity index 93% rename from instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java rename to instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTest.java index c2bc2ca798ea..c7dd0de8cbc8 100644 --- a/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergLibraryTest.java +++ b/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTest.java @@ -5,13 +5,14 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; +import org.apache.iceberg.TableScan; +import org.junit.jupiter.api.extension.RegisterExtension; + import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; import io.opentelemetry.instrumentation.testing.junit.LibraryInstrumentationExtension; -import org.apache.iceberg.TableScan; -import org.junit.jupiter.api.extension.RegisterExtension; -public class IcebergLibraryTest extends AbstractIcebergTest { +class IcebergTest extends AbstractIcebergTest { @RegisterExtension final InstrumentationExtension testing = LibraryInstrumentationExtension.create(); diff --git a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java index b595fd885ccc..e54a33ca8e4a 100644 --- a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java +++ b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java @@ -9,12 +9,11 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Locale; + import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileScanTask; @@ -38,6 +37,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; + public abstract class AbstractIcebergTest { protected static final int FORMAT_VERSION = 2; protected static final Schema SCHEMA = @@ -172,7 +174,7 @@ void testCreateTelemetry() throws IOException { if (currentExpectedMetric != null) { assertIcebergCounterMetric( - "iceberg.scan.scanned.data_files.size", "byte", expected, currentExpectedMetric.value()); + "iceberg.scan.scanned.data_files.size", "By", expected, currentExpectedMetric.value()); } else { assertIcebergMetricNotReported("iceberg.scan.scanned.data_files.size"); } @@ -182,7 +184,7 @@ void testCreateTelemetry() throws IOException { if (currentExpectedMetric != null) { assertIcebergCounterMetric( "iceberg.scan.scanned.delete_files.size", - "byte", + "By", expected, currentExpectedMetric.value()); } else { @@ -310,7 +312,7 @@ private void assertIcebergGaugeMetric( String otelMetricName, String expectedUnit, ScanReport expectedReport, long expectedValue) { testing() .waitAndAssertMetrics( - "io.opentelemetry.iceberg_1.8", + "io.opentelemetry.iceberg-1.8", metricAssert -> metricAssert .hasName(otelMetricName) @@ -342,7 +344,7 @@ private void assertIcebergCounterMetric( String otelMetricName, String expectedUnit, ScanReport expectedReport, long expectedValue) { testing() .waitAndAssertMetrics( - "io.opentelemetry.iceberg_1.8", + "io.opentelemetry.iceberg-1.8", metricAssert -> metricAssert .hasName(otelMetricName) From 8d975b9d6540bfdcf2d66382871b594187447938 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Tue, 4 Nov 2025 16:22:02 +0100 Subject: [PATCH 21/33] reorder imports --- .../instrumentation/iceberg/v1_8/IcebergTest.java | 5 ++--- .../iceberg/v1_8/AbstractIcebergTest.java | 11 +++-------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTest.java b/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTest.java index a3b802a0b8bd..177cb7f8a8a0 100644 --- a/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTest.java +++ b/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTest.java @@ -5,12 +5,11 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; -import org.apache.iceberg.TableScan; -import org.junit.jupiter.api.extension.RegisterExtension; - import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; import io.opentelemetry.instrumentation.testing.junit.LibraryInstrumentationExtension; +import org.apache.iceberg.TableScan; +import org.junit.jupiter.api.extension.RegisterExtension; class IcebergTest extends AbstractIcebergTest { @RegisterExtension diff --git a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java index e54a33ca8e4a..9b506bd3daf6 100644 --- a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java +++ b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java @@ -9,11 +9,12 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Locale; - import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileScanTask; @@ -37,9 +38,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; - public abstract class AbstractIcebergTest { protected static final int FORMAT_VERSION = 2; protected static final Schema SCHEMA = @@ -183,10 +181,7 @@ void testCreateTelemetry() throws IOException { if (currentExpectedMetric != null) { assertIcebergCounterMetric( - "iceberg.scan.scanned.delete_files.size", - "By", - expected, - currentExpectedMetric.value()); + "iceberg.scan.scanned.delete_files.size", "By", expected, currentExpectedMetric.value()); } else { assertIcebergMetricNotReported("iceberg.scan.scanned.delete_files.size"); } From e9aeabef0b12cd654b523e297c0901aa7a5489fa Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Thu, 13 Nov 2025 10:57:20 +0100 Subject: [PATCH 22/33] apply review comments --- docs/supported-libraries.md | 1 + .../iceberg/v1_8/IcebergMetricsReporter.java | 12 ++++------ .../iceberg/v1_8/IcebergTelemetry.java | 2 +- .../iceberg/v1_8/ScanMetricsBuilder.java | 7 +++--- .../iceberg/v1_8/AbstractIcebergTest.java | 24 ++++++++----------- 5 files changed, 20 insertions(+), 26 deletions(-) diff --git a/docs/supported-libraries.md b/docs/supported-libraries.md index 2eb0694bcc62..4c44925dbba7 100644 --- a/docs/supported-libraries.md +++ b/docs/supported-libraries.md @@ -31,6 +31,7 @@ These are the supported libraries and frameworks: | [Apache Dubbo](https://github.com/apache/dubbo/) | 2.7+ | [opentelemetry-apache-dubbo-2.7](../instrumentation/apache-dubbo-2.7/library-autoconfigure) | [RPC Client Spans], [RPC Server Spans] | | [Apache HttpAsyncClient](https://hc.apache.org/index.html) | 4.1+ | N/A | [HTTP Client Spans], [HTTP Client Metrics] | | [Apache HttpClient](https://hc.apache.org/index.html) | 2.0+ | [opentelemetry-apache-httpclient-4.3](../instrumentation/apache-httpclient/apache-httpclient-4.3/library),
[opentelemetry-apache-httpclient-5.2](../instrumentation/apache-httpclient/apache-httpclient-5.2/library) | [HTTP Client Spans], [HTTP Client Metrics] | +| [Apache Iceberg](https://iceberg.apache.org/) | N/A | [opentelemetry-iceberg-1.8](../instrumentation/iceberg-1.8/library/) | none | | [Apache ShenYu](https://shenyu.apache.org/) | 2.4+ | N/A | Provides `http.route` [2] | | [Apache Kafka Connect API](https://kafka.apache.org/documentation/#connect) | 2.6+ | N/A | [Messaging Spans] | | [Apache Kafka Producer/Consumer API](https://kafka.apache.org/documentation/#producerapi) | 0.11+ | [opentelemetry-kafka-clients-2.6](../instrumentation/kafka/kafka-clients/kafka-clients-2.6/library) | [Messaging Spans] | diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index 5075c5a491eb..848a08d6a88e 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -8,9 +8,8 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleHistogram; import io.opentelemetry.api.metrics.LongCounter; -import io.opentelemetry.api.metrics.LongGauge; -import java.util.Locale; import org.apache.iceberg.metrics.CounterResult; import org.apache.iceberg.metrics.MetricsReport; import org.apache.iceberg.metrics.MetricsReporter; @@ -18,7 +17,7 @@ import org.apache.iceberg.metrics.ScanReport; import org.apache.iceberg.metrics.TimerResult; -public class IcebergMetricsReporter implements MetricsReporter { +final class IcebergMetricsReporter implements MetricsReporter { private static final String INSTRUMENTATION_NAME = "io.opentelemetry.iceberg-1.8"; private static final AttributeKey SCHEMA_ID = AttributeKey.longKey("iceberg.schema.id"); private static final AttributeKey TABLE_NAME = @@ -51,11 +50,10 @@ void reportScanMetrics(ScanReport scanReport) { TimerResult duration = metrics.totalPlanningDuration(); if (duration != null) { - LongGauge metric = + DoubleHistogram metric = ScanMetricsBuilder.totalPlanningDuration( - openTelemetry.getMeter(INSTRUMENTATION_NAME), - duration.timeUnit().name().toLowerCase(Locale.getDefault())); - metric.set(duration.totalDuration().toMillis(), scanAttributes); + openTelemetry.getMeter(INSTRUMENTATION_NAME), "s"); + metric.record(duration.totalDuration().toMillis() / 1000.0, scanAttributes); } CounterResult current = metrics.resultDataFiles(); diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java index 41f43c3b9a79..23f30b9fd704 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java @@ -10,7 +10,7 @@ import org.apache.iceberg.ScanTask; import org.apache.iceberg.ScanTaskGroup; -public class IcebergTelemetry { +public final class IcebergTelemetry { private final OpenTelemetry openTelemetry; public static IcebergTelemetry create(OpenTelemetry openTelemetry) { diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java index c6ffb9544fd0..4b7853c6a61d 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java @@ -5,8 +5,8 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; +import io.opentelemetry.api.metrics.DoubleHistogram; import io.opentelemetry.api.metrics.LongCounter; -import io.opentelemetry.api.metrics.LongGauge; import io.opentelemetry.api.metrics.Meter; final class ScanMetricsBuilder { @@ -34,12 +34,11 @@ private ScanMetricsBuilder() { // prevents instantiation } - static LongGauge totalPlanningDuration(Meter meter, String unit) { + static DoubleHistogram totalPlanningDuration(Meter meter, String unit) { return meter - .gaugeBuilder(TOTAL_PLANNING_DURATION) + .histogramBuilder(TOTAL_PLANNING_DURATION) .setDescription("The total duration needed to plan the scan.") .setUnit(unit) - .ofLongs() .build(); } diff --git a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java index 9b506bd3daf6..272962cdb838 100644 --- a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java +++ b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java @@ -14,7 +14,6 @@ import java.io.File; import java.io.IOException; import java.util.Arrays; -import java.util.Locale; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileScanTask; @@ -282,11 +281,7 @@ void testCreateTelemetry() throws IOException { TimerResult timer = expected.scanMetrics().totalPlanningDuration(); if (timer != null) { - assertIcebergGaugeMetric( - "iceberg.scan.planning.duration", - timer.timeUnit().name().toLowerCase(Locale.getDefault()), - expected, - timer.totalDuration().toMillis()); + assertIcebergHistogramMetric("iceberg.scan.planning.duration", "s", expected); } else { assertIcebergMetricNotReported("iceberg.scan.planning.duration"); } @@ -303,8 +298,8 @@ private void assertIcebergMetricNotReported(String otelMetricName) { "metric is not reported"))); } - private void assertIcebergGaugeMetric( - String otelMetricName, String expectedUnit, ScanReport expectedReport, long expectedValue) { + private void assertIcebergHistogramMetric( + String otelMetricName, String expectedUnit, ScanReport expectedReport) { testing() .waitAndAssertMetrics( "io.opentelemetry.iceberg-1.8", @@ -312,12 +307,13 @@ private void assertIcebergGaugeMetric( metricAssert .hasName(otelMetricName) .hasUnit(expectedUnit) - .hasLongGaugeSatisfying( - sum -> - sum.hasPointsSatisfying( - longAssert -> - longAssert - .hasValue(expectedValue) + .hasHistogramSatisfying( + histogram -> + histogram.hasPointsSatisfying( + point -> + point + .hasSumGreaterThan(0.0) + .hasCount(1) .hasAttributesSatisfying( attributes -> assertEquals( From 6f8e403042a93f9ef49f692d7bcf98119d919b27 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Fri, 21 Nov 2025 16:47:22 +0100 Subject: [PATCH 23/33] merge metrics --- .../iceberg/v1_8/IcebergMetricsReporter.java | 200 +++++++++++------- .../iceberg/v1_8/ScanMetricsBuilder.java | 137 ++---------- .../iceberg/v1_8/IcebergTest.java | 2 +- .../iceberg-1.8/testing/build.gradle.kts | 6 +- .../iceberg/v1_8/AbstractIcebergTest.java | 21 +- 5 files changed, 160 insertions(+), 206 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index 848a08d6a88e..8955f3d6e4c6 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -10,6 +10,7 @@ import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.DoubleHistogram; import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.Meter; import org.apache.iceberg.metrics.CounterResult; import org.apache.iceberg.metrics.MetricsReport; import org.apache.iceberg.metrics.MetricsReporter; @@ -23,11 +24,27 @@ final class IcebergMetricsReporter implements MetricsReporter { private static final AttributeKey TABLE_NAME = AttributeKey.stringKey("iceberg.table.name"); private static final AttributeKey SNAPHSOT_ID = AttributeKey.longKey("iceberg.snapshot.id"); - - private final OpenTelemetry openTelemetry; + private static final AttributeKey SCAN_STATE = + AttributeKey.stringKey("iceberg.scan.state"); + private static final AttributeKey DELETE_TYPE = + AttributeKey.stringKey("iceberg.delete_file.type"); + private final DoubleHistogram planningDuration; + private final LongCounter dataFilesCount; + private final LongCounter dataFilesSize; + private final LongCounter deleteFilesCount; + private final LongCounter deleteFilesSize; + private final LongCounter dataManifestsCount; + private final LongCounter deleteManifestsCount; IcebergMetricsReporter(OpenTelemetry openTelemetry) { - this.openTelemetry = openTelemetry; + Meter meter = openTelemetry.getMeter(INSTRUMENTATION_NAME); + planningDuration = ScanMetricsBuilder.totalPlanningDuration(meter, "s"); + dataFilesCount = ScanMetricsBuilder.dataFilesCount(meter); + dataFilesSize = ScanMetricsBuilder.dataFilesSize(meter); + deleteFilesCount = ScanMetricsBuilder.deleteFilesCount(meter); + deleteFilesSize = ScanMetricsBuilder.deleteFilesSize(meter); + dataManifestsCount = ScanMetricsBuilder.dataManifestsCount(meter); + deleteManifestsCount = ScanMetricsBuilder.deleteManifestsCount(meter); } @Override @@ -50,143 +67,164 @@ void reportScanMetrics(ScanReport scanReport) { TimerResult duration = metrics.totalPlanningDuration(); if (duration != null) { - DoubleHistogram metric = - ScanMetricsBuilder.totalPlanningDuration( - openTelemetry.getMeter(INSTRUMENTATION_NAME), "s"); - metric.record(duration.totalDuration().toMillis() / 1000.0, scanAttributes); + planningDuration.record(duration.totalDuration().toMillis() / 1000.0, scanAttributes); } + // Data files metrics CounterResult current = metrics.resultDataFiles(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.scannedDataFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + addValueToLongCounter(dataFilesCount, current.value(), scanAttributes, SCAN_STATE, "scanned"); } - current = metrics.resultDeleteFiles(); - - if (current != null) { - LongCounter metric = - ScanMetricsBuilder.scannedDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); - } - - current = metrics.scannedDataManifests(); + current = metrics.skippedDataFiles(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.scannedDataManifestsCount( - openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + addValueToLongCounter(dataFilesCount, current.value(), scanAttributes, SCAN_STATE, "skipped"); } - current = metrics.scannedDeleteManifests(); + current = metrics.totalFileSizeInBytes(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.scannedDeleteManifestsCount( - openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + dataFilesSize.add(current.value(), scanAttributes); } - current = metrics.totalDataManifests(); + // Delete files metrics + current = metrics.totalDeleteFileSizeInBytes(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.totalDataManifestsCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + deleteFilesSize.add(current.value(), scanAttributes); } - current = metrics.totalDeleteManifests(); + current = metrics.resultDeleteFiles(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.totalDeleteManifestsCount( - openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + addValueToLongCounter( + deleteFilesCount, + current.value(), + scanAttributes, + SCAN_STATE, + "scanned", + DELETE_TYPE, + "all"); } - current = metrics.totalFileSizeInBytes(); + current = metrics.skippedDeleteFiles(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.scannedDataFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + addValueToLongCounter( + deleteFilesCount, + current.value(), + scanAttributes, + SCAN_STATE, + "skipped", + DELETE_TYPE, + "all"); } - current = metrics.totalDeleteFileSizeInBytes(); + current = metrics.indexedDeleteFiles(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.scannedDeleteFilesSize(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + addValueToLongCounter( + deleteFilesCount, + current.value(), + scanAttributes, + SCAN_STATE, + "scanned", + DELETE_TYPE, + "indexed"); } - current = metrics.skippedDataManifests(); + current = metrics.equalityDeleteFiles(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.skippedDataManifestsCount( - openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + addValueToLongCounter( + deleteFilesCount, + current.value(), + scanAttributes, + SCAN_STATE, + "scanned", + DELETE_TYPE, + "equality"); } - current = metrics.skippedDeleteManifests(); + current = metrics.positionalDeleteFiles(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.skippedDeleteManifestsCount( - openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + addValueToLongCounter( + deleteFilesCount, + current.value(), + scanAttributes, + SCAN_STATE, + "scanned", + DELETE_TYPE, + "position"); } - current = metrics.skippedDataFiles(); + current = metrics.dvs(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.skippedDataFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + addValueToLongCounter( + deleteFilesCount, + current.value(), + scanAttributes, + SCAN_STATE, + "scanned", + DELETE_TYPE, + "dvs"); } - current = metrics.skippedDeleteFiles(); + // Data manifests metrics + current = metrics.scannedDataManifests(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.skippedDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + addValueToLongCounter( + dataManifestsCount, current.value(), scanAttributes, SCAN_STATE, "scanned"); } - current = metrics.indexedDeleteFiles(); + current = metrics.skippedDataManifests(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.indexedDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + addValueToLongCounter( + dataManifestsCount, current.value(), scanAttributes, SCAN_STATE, "skipped"); } - current = metrics.equalityDeleteFiles(); + // Delete manifests metrics + current = metrics.scannedDeleteManifests(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.equalityDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + addValueToLongCounter( + deleteManifestsCount, current.value(), scanAttributes, SCAN_STATE, "scanned"); } - current = metrics.positionalDeleteFiles(); + current = metrics.skippedDeleteManifests(); if (current != null) { - LongCounter metric = - ScanMetricsBuilder.positionDeleteFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); + addValueToLongCounter( + deleteManifestsCount, current.value(), scanAttributes, SCAN_STATE, "skipped"); } + } - current = metrics.dvs(); + private void addValueToLongCounter( + LongCounter metric, + long measurement, + Attributes attributes, + AttributeKey att1Key, + String att1Value) { + Attributes newAttributes = attributes.toBuilder().put(att1Key, att1Value).build(); + metric.add(measurement, newAttributes); + } - if (current != null) { - LongCounter metric = - ScanMetricsBuilder.deletionVectorFilesCount(openTelemetry.getMeter(INSTRUMENTATION_NAME)); - metric.add(current.value(), scanAttributes); - } + private void addValueToLongCounter( + LongCounter metric, + long measurement, + Attributes attributes, + AttributeKey att1Key, + String att1Value, + AttributeKey att2Key, + String att2Value) { + Attributes newAttributes = + attributes.toBuilder().put(att1Key, att1Value).put(att2Key, att2Value).build(); + metric.add(measurement, newAttributes); } } diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java index 4b7853c6a61d..9d5db92d870d 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java @@ -12,23 +12,13 @@ final class ScanMetricsBuilder { private static final String ROOT = "iceberg.scan"; private static final String TOTAL_PLANNING_DURATION = ROOT + ".planning.duration"; - private static final String RESULT_DATA_FILES = ROOT + ".scanned.data_files.count"; - private static final String RESULT_DELETE_FILES = ROOT + ".scanned.delete_files.count"; - private static final String SCANNED_DATA_MANIFESTS = ROOT + ".scanned.data_manifests.count"; - private static final String SCANNED_DELETE_MANIFESTS = ROOT + ".scanned.delete_manifests.count"; - private static final String TOTAL_DATA_MANIFESTS = ROOT + ".total.data_manifests.count"; - private static final String TOTAL_DELETE_MANIFESTS = ROOT + ".total.delete_manifests.count"; - private static final String TOTAL_FILE_SIZE_IN_BYTES = ROOT + ".scanned.data_files.size"; - private static final String TOTAL_DELETE_FILE_SIZE_IN_BYTES = ROOT + ".scanned.delete_files.size"; - private static final String SKIPPED_DATA_MANIFESTS = ROOT + ".skipped.data_manifests.count"; - private static final String SKIPPED_DELETE_MANIFESTS = ROOT + ".skipped.delete_manifests.count"; - private static final String SKIPPED_DATA_FILES = ROOT + ".skipped.data_files.count"; - private static final String SKIPPED_DELETE_FILES = ROOT + ".skipped.delete_files.count"; - private static final String INDEXED_DELETE_FILES = ROOT + ".scanned.indexed_delete_files.count"; - private static final String EQUALITY_DELETE_FILES = ROOT + ".scanned.equality_delete_files.count"; - private static final String POSITIONAL_DELETE_FILES = - ROOT + ".scanned.positional_delete_files.count"; - private static final String DVS = ROOT + ".scanned.dvs.count"; + + private static final String DATA_FILES_COUNT = ROOT + ".data_files.count"; + private static final String DATA_FILES_SIZE = ROOT + ".data_files.size"; + private static final String DELETE_FILES_SIZE = ROOT + ".delete_files.size"; + private static final String DELETE_FILES_COUNT = ROOT + ".delete_files.count"; + private static final String DATA_MANIFESTS_COUNT = ROOT + ".data_manifests.count"; + private static final String DELETE_MANIFESTS_COUNT = ROOT + ".delete_manifests.count"; private ScanMetricsBuilder() { // prevents instantiation @@ -42,132 +32,51 @@ static DoubleHistogram totalPlanningDuration(Meter meter, String unit) { .build(); } - static LongCounter scannedDataFilesCount(Meter meter) { + static LongCounter dataFilesCount(Meter meter) { return meter - .counterBuilder(RESULT_DATA_FILES) - .setDescription("The number of scanned data files.") + .counterBuilder(DATA_FILES_COUNT) + .setDescription("The number of data files.") .setUnit("{file}") .build(); } - static LongCounter scannedDeleteFilesCount(Meter meter) { + static LongCounter deleteFilesCount(Meter meter) { return meter - .counterBuilder(RESULT_DELETE_FILES) - .setDescription("The number of scanned delete files.") + .counterBuilder(DELETE_FILES_COUNT) + .setDescription("The number of delete files.") .setUnit("{file}") .build(); } - static LongCounter scannedDataManifestsCount(Meter meter) { + static LongCounter dataManifestsCount(Meter meter) { return meter - .counterBuilder(SCANNED_DATA_MANIFESTS) - .setDescription("The number of scanned data manifests.") + .counterBuilder(DATA_MANIFESTS_COUNT) + .setDescription("The number of data manifests.") .setUnit("{file}") .build(); } - static LongCounter scannedDeleteManifestsCount(Meter meter) { + static LongCounter deleteManifestsCount(Meter meter) { return meter - .counterBuilder(SCANNED_DELETE_MANIFESTS) - .setDescription("The number of scanned delete manifests.") + .counterBuilder(DELETE_MANIFESTS_COUNT) + .setDescription("The number of delete manifests.") .setUnit("{file}") .build(); } - static LongCounter totalDataManifestsCount(Meter meter) { + static LongCounter dataFilesSize(Meter meter) { return meter - .counterBuilder(TOTAL_DATA_MANIFESTS) - .setDescription("The number of all data manifests.") - .setUnit("{file}") - .build(); - } - - static LongCounter totalDeleteManifestsCount(Meter meter) { - return meter - .counterBuilder(TOTAL_DELETE_MANIFESTS) - .setDescription("The number of all delete manifests.") - .setUnit("{file}") - .build(); - } - - static LongCounter scannedDataFilesSize(Meter meter) { - return meter - .counterBuilder(TOTAL_FILE_SIZE_IN_BYTES) + .counterBuilder(DATA_FILES_SIZE) .setDescription("The total size of all scanned data files.") .setUnit("By") .build(); } - static LongCounter scannedDeleteFilesSize(Meter meter) { + static LongCounter deleteFilesSize(Meter meter) { return meter - .counterBuilder(TOTAL_DELETE_FILE_SIZE_IN_BYTES) + .counterBuilder(DELETE_FILES_SIZE) .setDescription("The total size of all scanned delete files.") .setUnit("By") .build(); } - - static LongCounter skippedDataManifestsCount(Meter meter) { - return meter - .counterBuilder(SKIPPED_DATA_MANIFESTS) - .setDescription("The number of data manifests that were skipped during the scan.") - .setUnit("{file}") - .build(); - } - - static LongCounter skippedDeleteManifestsCount(Meter meter) { - return meter - .counterBuilder(SKIPPED_DELETE_MANIFESTS) - .setDescription("The number of delete manifests that were skipped during the scan.") - .setUnit("{file}") - .build(); - } - - static LongCounter skippedDataFilesCount(Meter meter) { - return meter - .counterBuilder(SKIPPED_DATA_FILES) - .setDescription("The number of data files that were skipped during the scan.") - .setUnit("{file}") - .build(); - } - - static LongCounter skippedDeleteFilesCount(Meter meter) { - return meter - .counterBuilder(SKIPPED_DELETE_FILES) - .setDescription("The number of delete files that were skipped during the scan.") - .setUnit("{file}") - .build(); - } - - static LongCounter indexedDeleteFilesCount(Meter meter) { - return meter - .counterBuilder(INDEXED_DELETE_FILES) - .setDescription( - "The number of delete files constituting the delete file index for this scan.") - .setUnit("{file}") - .build(); - } - - static LongCounter equalityDeleteFilesCount(Meter meter) { - return meter - .counterBuilder(EQUALITY_DELETE_FILES) - .setDescription("The number of equality delete files relevant for the current scan.") - .setUnit("{file}") - .build(); - } - - static LongCounter positionDeleteFilesCount(Meter meter) { - return meter - .counterBuilder(POSITIONAL_DELETE_FILES) - .setDescription("The number of position delete files relevant for the current scan.") - .setUnit("{file}") - .build(); - } - - static LongCounter deletionVectorFilesCount(Meter meter) { - return meter - .counterBuilder(DVS) - .setDescription("The number of deletion vector (DV) files relevant for the current scan.") - .setUnit("{file}") - .build(); - } } diff --git a/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTest.java b/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTest.java index 177cb7f8a8a0..5e6abde53978 100644 --- a/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTest.java +++ b/instrumentation/iceberg-1.8/library/src/test/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTest.java @@ -13,7 +13,7 @@ class IcebergTest extends AbstractIcebergTest { @RegisterExtension - final InstrumentationExtension testing = LibraryInstrumentationExtension.create(); + static final InstrumentationExtension testing = LibraryInstrumentationExtension.create(); @Override protected InstrumentationExtension testing() { diff --git a/instrumentation/iceberg-1.8/testing/build.gradle.kts b/instrumentation/iceberg-1.8/testing/build.gradle.kts index 88527621ebb4..5dcd9d3a148b 100644 --- a/instrumentation/iceberg-1.8/testing/build.gradle.kts +++ b/instrumentation/iceberg-1.8/testing/build.gradle.kts @@ -4,10 +4,6 @@ plugins { dependencies { implementation("org.apache.iceberg:iceberg-core:1.8.1") - implementation("org.apache.iceberg:iceberg-core:1.8.1") { - artifact { - classifier = "tests" - } - } + implementation("org.apache.hadoop:hadoop-common:3.4.2") api(project(":testing-common")) } diff --git a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java index 272962cdb838..f569647f1094 100644 --- a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java +++ b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java @@ -14,15 +14,18 @@ import java.io.File; import java.io.IOException; import java.util.Arrays; +import java.util.Collections; +import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; import org.apache.iceberg.TableScan; -import org.apache.iceberg.TestTables; -import org.apache.iceberg.TestTables.TestTable; +import org.apache.iceberg.Tables; import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.metrics.CounterResult; import org.apache.iceberg.metrics.MetricsReport; @@ -43,7 +46,6 @@ public abstract class AbstractIcebergTest { new Schema( NestedField.required(3, "id", IntegerType.get()), NestedField.required(4, "data", StringType.get())); - protected static final int BUCKETS_NUMBER = 16; protected static final PartitionSpec SPEC = PartitionSpec.builderFor(SCHEMA).bucket("data", 16).build(); protected static final DataFile FILE_1 = @@ -63,7 +65,8 @@ public abstract class AbstractIcebergTest { .build(); @TempDir protected File tableDir = null; - protected TestTable table; + protected Tables tables; + protected Table table; protected abstract InstrumentationExtension testing(); @@ -71,7 +74,15 @@ public abstract class AbstractIcebergTest { @BeforeEach void init() { - this.table = TestTables.create(this.tableDir, "test", SCHEMA, SPEC, FORMAT_VERSION); + Configuration conf = new Configuration(); + this.tables = new HadoopTables(conf); + String tableLocation = new File(tableDir, "test").toString(); + this.table = + tables.create( + SCHEMA, + SPEC, + Collections.singletonMap("format-version", String.valueOf(FORMAT_VERSION)), + tableLocation); this.table.newFastAppend().appendFile(FILE_1).appendFile(FILE_2).commit(); } From 088e2dd453bbe6a6fcf78ba50b432e3ab1135500 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Fri, 21 Nov 2025 18:21:51 +0100 Subject: [PATCH 24/33] adapt unit tests to new metrics --- .../iceberg/v1_8/IcebergMetricsReporter.java | 8 +- .../iceberg/v1_8/AbstractIcebergTest.java | 560 ++++++++++-------- 2 files changed, 317 insertions(+), 251 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index 8955f3d6e4c6..7cd9d897f2ef 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -23,7 +23,7 @@ final class IcebergMetricsReporter implements MetricsReporter { private static final AttributeKey SCHEMA_ID = AttributeKey.longKey("iceberg.schema.id"); private static final AttributeKey TABLE_NAME = AttributeKey.stringKey("iceberg.table.name"); - private static final AttributeKey SNAPHSOT_ID = AttributeKey.longKey("iceberg.snapshot.id"); + private static final AttributeKey SNAPSHOT_ID = AttributeKey.longKey("iceberg.snapshot.id"); private static final AttributeKey SCAN_STATE = AttributeKey.stringKey("iceberg.scan.state"); private static final AttributeKey DELETE_TYPE = @@ -61,7 +61,7 @@ void reportScanMetrics(ScanReport scanReport) { Long.valueOf(scanReport.schemaId()), TABLE_NAME, scanReport.tableName(), - SNAPHSOT_ID, + SNAPSHOT_ID, scanReport.snapshotId()); ScanMetricsResult metrics = scanReport.scanMetrics(); TimerResult duration = metrics.totalPlanningDuration(); @@ -205,7 +205,7 @@ void reportScanMetrics(ScanReport scanReport) { } } - private void addValueToLongCounter( + private static void addValueToLongCounter( LongCounter metric, long measurement, Attributes attributes, @@ -215,7 +215,7 @@ private void addValueToLongCounter( metric.add(measurement, newAttributes); } - private void addValueToLongCounter( + private static void addValueToLongCounter( LongCounter metric, long measurement, Attributes attributes, diff --git a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java index f569647f1094..93fc9fa191e4 100644 --- a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java +++ b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java @@ -5,11 +5,11 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static io.opentelemetry.api.common.AttributeKey.longKey; +import static io.opentelemetry.api.common.AttributeKey.stringKey; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo; +import static org.assertj.core.api.Assertions.assertThat; -import io.opentelemetry.api.common.Attributes; import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; import java.io.File; import java.io.IOException; @@ -27,15 +27,12 @@ import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.metrics.CounterResult; import org.apache.iceberg.metrics.MetricsReport; import org.apache.iceberg.metrics.MetricsReporter; import org.apache.iceberg.metrics.ScanReport; -import org.apache.iceberg.metrics.TimerResult; import org.apache.iceberg.types.Types.IntegerType; import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.types.Types.StringType; -import org.assertj.core.api.Condition; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -99,279 +96,348 @@ void testCreateTelemetry() throws IOException { scan = configure(scan); try (CloseableIterable tasks = scan.planFiles()) { - assertNotNull(tasks); - assertNotNull(tasks.iterator()); + assertThat(tasks).isNotNull(); + assertThat(tasks.iterator()).isNotNull(); } - assertNotNull(reporter.report); - assertTrue(reporter.report instanceof ScanReport); + assertThat(reporter.report).isNotNull(); + assertThat(reporter.report).isInstanceOf(ScanReport.class); ScanReport expected = (ScanReport) reporter.report; - CounterResult currentExpectedMetric = expected.scanMetrics().resultDataFiles(); - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.scanned.data_files.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.scanned.data_files.count"); - } - - currentExpectedMetric = expected.scanMetrics().resultDeleteFiles(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.scanned.delete_files.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.scanned.delete_files.count"); - } - - currentExpectedMetric = expected.scanMetrics().scannedDataManifests(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.scanned.data_manifests.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.scanned.data_manifests.count"); - } - - currentExpectedMetric = expected.scanMetrics().scannedDeleteManifests(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.scanned.delete_manifests.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.scanned.delete_manifests.count"); - } - - currentExpectedMetric = expected.scanMetrics().totalDataManifests(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.total.data_manifests.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.total.data_manifests.count"); - } - - currentExpectedMetric = expected.scanMetrics().totalDeleteManifests(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.total.delete_manifests.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.total.delete_manifests.count"); - } - - currentExpectedMetric = expected.scanMetrics().totalFileSizeInBytes(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.scanned.data_files.size", "By", expected, currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.scanned.data_files.size"); - } - - currentExpectedMetric = expected.scanMetrics().totalDeleteFileSizeInBytes(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.scanned.delete_files.size", "By", expected, currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.scanned.delete_files.size"); - } - - currentExpectedMetric = expected.scanMetrics().skippedDataManifests(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.skipped.data_manifests.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.skipped.data_manifests.count"); - } - - currentExpectedMetric = expected.scanMetrics().skippedDeleteManifests(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.skipped.delete_manifests.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.skipped.delete_manifests.count"); - } - - currentExpectedMetric = expected.scanMetrics().skippedDataFiles(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.skipped.data_files.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.skipped.data_files.count"); - } - - currentExpectedMetric = expected.scanMetrics().skippedDeleteFiles(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.skipped.delete_files.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.skipped.delete_files.count"); - } - - currentExpectedMetric = expected.scanMetrics().indexedDeleteFiles(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.scanned.indexed_delete_files.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.scanned.indexed_delete_files.count"); - } - - currentExpectedMetric = expected.scanMetrics().equalityDeleteFiles(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.scanned.equality_delete_files.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.scanned.equality_delete_files.count"); - } - - currentExpectedMetric = expected.scanMetrics().positionalDeleteFiles(); - - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.scanned.positional_delete_files.count", - "{file}", - expected, - currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.scanned.positional_delete_files.count"); - } - - currentExpectedMetric = expected.scanMetrics().dvs(); + assertScanDurationMetric(expected); + assertDataFilesCountMetrics(expected); + assertDeleteFilesCountMetrics(expected); + assertDataManifestCountMetrics(expected); + assertDeleteManifestCountMetrics(expected); + assertSizeMetric( + "iceberg.scan.data_files.size", + expected, + expected.scanMetrics().totalFileSizeInBytes().value()); + assertSizeMetric( + "iceberg.scan.delete_files.size", + expected, + expected.scanMetrics().totalDeleteFileSizeInBytes().value()); + } - if (currentExpectedMetric != null) { - assertIcebergCounterMetric( - "iceberg.scan.scanned.dvs.count", "{file}", expected, currentExpectedMetric.value()); - } else { - assertIcebergMetricNotReported("iceberg.scan.scanned.dvs.count"); - } + private void assertScanDurationMetric(ScanReport expectedReport) { + testing() + .waitAndAssertMetrics( + "io.opentelemetry.iceberg-1.8", + metricAssert -> + metricAssert + .hasName("iceberg.scan.planning.duration") + .hasUnit("s") + .hasHistogramSatisfying( + histogram -> + histogram.hasPointsSatisfying( + point -> + point + .hasSumGreaterThan(0.0) + .hasCount(1) + .hasAttributesSatisfyingExactly( + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId()))))); + } - TimerResult timer = expected.scanMetrics().totalPlanningDuration(); + private void assertDataFilesCountMetrics(ScanReport expectedReport) { + testing() + .waitAndAssertMetrics( + "io.opentelemetry.iceberg-1.8", + metricAssert -> + metricAssert + .hasName("iceberg.scan.data_files.count") + .hasUnit("{file}") + .hasLongSumSatisfying( + sum -> + sum.hasPointsSatisfying( + longSumAssert -> + longSumAssert + .hasValue( + expectedReport.scanMetrics().resultDataFiles().value()) + .hasAttributesSatisfyingExactly( + equalTo(stringKey("iceberg.scan.state"), "scanned"), + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId())), + longSumAssert -> + longSumAssert + .hasValue( + expectedReport.scanMetrics().skippedDataFiles().value()) + .hasAttributesSatisfyingExactly( + equalTo(stringKey("iceberg.scan.state"), "skipped"), + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId()))))); + } - if (timer != null) { - assertIcebergHistogramMetric("iceberg.scan.planning.duration", "s", expected); - } else { - assertIcebergMetricNotReported("iceberg.scan.planning.duration"); - } + private void assertDataManifestCountMetrics(ScanReport expectedReport) { + testing() + .waitAndAssertMetrics( + "io.opentelemetry.iceberg-1.8", + metricAssert -> + metricAssert + .hasName("iceberg.scan.data_manifests.count") + .hasUnit("{file}") + .hasLongSumSatisfying( + sum -> + sum.hasPointsSatisfying( + longSumAssert -> + longSumAssert + .hasValue( + expectedReport + .scanMetrics() + .scannedDataManifests() + .value()) + .hasAttributesSatisfyingExactly( + equalTo(stringKey("iceberg.scan.state"), "scanned"), + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId())), + longSumAssert -> + longSumAssert + .hasValue( + expectedReport + .scanMetrics() + .skippedDataManifests() + .value()) + .hasAttributesSatisfyingExactly( + equalTo(stringKey("iceberg.scan.state"), "skipped"), + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId()))))); } - private void assertIcebergMetricNotReported(String otelMetricName) { + private void assertDeleteManifestCountMetrics(ScanReport expectedReport) { testing() .waitAndAssertMetrics( - otelMetricName, + "io.opentelemetry.iceberg-1.8", metricAssert -> - metricAssert.doesNotHave( - new Condition<>( - spanData -> otelMetricName.equals(spanData.getName()), - "metric is not reported"))); + metricAssert + .hasName("iceberg.scan.delete_manifests.count") + .hasUnit("{file}") + .hasLongSumSatisfying( + sum -> + sum.hasPointsSatisfying( + longSumAssert -> + longSumAssert + .hasValue( + expectedReport + .scanMetrics() + .scannedDeleteManifests() + .value()) + .hasAttributesSatisfyingExactly( + equalTo(stringKey("iceberg.scan.state"), "scanned"), + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId())), + longSumAssert -> + longSumAssert + .hasValue( + expectedReport + .scanMetrics() + .skippedDeleteManifests() + .value()) + .hasAttributesSatisfyingExactly( + equalTo(stringKey("iceberg.scan.state"), "skipped"), + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId()))))); } - private void assertIcebergHistogramMetric( - String otelMetricName, String expectedUnit, ScanReport expectedReport) { + private void assertDeleteFilesCountMetrics(ScanReport expectedReport) { testing() .waitAndAssertMetrics( "io.opentelemetry.iceberg-1.8", metricAssert -> metricAssert - .hasName(otelMetricName) - .hasUnit(expectedUnit) - .hasHistogramSatisfying( - histogram -> - histogram.hasPointsSatisfying( - point -> - point - .hasSumGreaterThan(0.0) - .hasCount(1) - .hasAttributesSatisfying( - attributes -> - assertEquals( - Attributes.builder() - .put( - "iceberg.schema.id", - expectedReport.schemaId()) - .put( - "iceberg.table.name", - expectedReport.tableName()) - .put( - "iceberg.snapshot.id", - expectedReport.snapshotId()) - .build(), - attributes))))); + .hasName("iceberg.scan.delete_files.count") + .hasUnit("{file}") + .hasLongSumSatisfying( + sum -> + sum.hasPointsSatisfying( + longSumAssert -> + longSumAssert + .hasValue( + expectedReport + .scanMetrics() + .resultDeleteFiles() + .value()) + .hasAttributesSatisfyingExactly( + equalTo(stringKey("iceberg.scan.state"), "scanned"), + equalTo(stringKey("iceberg.delete_file.type"), "all"), + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId())), + longSumAssert -> + longSumAssert + .hasValue( + expectedReport + .scanMetrics() + .skippedDeleteFiles() + .value()) + .hasAttributesSatisfyingExactly( + equalTo(stringKey("iceberg.scan.state"), "skipped"), + equalTo(stringKey("iceberg.delete_file.type"), "all"), + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId())), + longSumAssert -> + longSumAssert + .hasValue( + expectedReport + .scanMetrics() + .indexedDeleteFiles() + .value()) + .hasAttributesSatisfyingExactly( + equalTo(stringKey("iceberg.scan.state"), "scanned"), + equalTo( + stringKey("iceberg.delete_file.type"), "indexed"), + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId())), + longSumAssert -> + longSumAssert + .hasValue( + expectedReport + .scanMetrics() + .equalityDeleteFiles() + .value()) + .hasAttributesSatisfyingExactly( + equalTo(stringKey("iceberg.scan.state"), "scanned"), + equalTo( + stringKey("iceberg.delete_file.type"), "equality"), + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId())), + longSumAssert -> + longSumAssert + .hasValue( + expectedReport + .scanMetrics() + .positionalDeleteFiles() + .value()) + .hasAttributesSatisfyingExactly( + equalTo(stringKey("iceberg.scan.state"), "scanned"), + equalTo( + stringKey("iceberg.delete_file.type"), "position"), + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId())), + longSumAssert -> + longSumAssert + .hasValue(expectedReport.scanMetrics().dvs().value()) + .hasAttributesSatisfyingExactly( + equalTo(stringKey("iceberg.scan.state"), "scanned"), + equalTo(stringKey("iceberg.delete_file.type"), "dvs"), + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId()))))); } - private void assertIcebergCounterMetric( - String otelMetricName, String expectedUnit, ScanReport expectedReport, long expectedValue) { + private void assertSizeMetric( + String otelMetricName, ScanReport expectedReport, long expectedValue) { testing() .waitAndAssertMetrics( "io.opentelemetry.iceberg-1.8", metricAssert -> metricAssert .hasName(otelMetricName) - .hasUnit(expectedUnit) + .hasUnit("By") .hasLongSumSatisfying( sum -> sum.hasPointsSatisfying( longSumAssert -> longSumAssert .hasValue(expectedValue) - .hasAttributesSatisfying( - attributes -> - assertEquals( - Attributes.builder() - .put( - "iceberg.schema.id", - expectedReport.schemaId()) - .put( - "iceberg.table.name", - expectedReport.tableName()) - .put( - "iceberg.snapshot.id", - expectedReport.snapshotId()) - .build(), - attributes))))); + .hasAttributesSatisfyingExactly( + equalTo( + longKey("iceberg.schema.id"), + expectedReport.schemaId()), + equalTo( + stringKey("iceberg.table.name"), + expectedReport.tableName()), + equalTo( + longKey("iceberg.snapshot.id"), + expectedReport.snapshotId()))))); } static final class SimpleReporter implements MetricsReporter { From ea200d3054049baf3754a7fcdb9a82022f987eab Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Sat, 22 Nov 2025 11:55:22 +0100 Subject: [PATCH 25/33] limit java version for tests to 17 to allow hadoop to run --- instrumentation/iceberg-1.8/library/build.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/instrumentation/iceberg-1.8/library/build.gradle.kts b/instrumentation/iceberg-1.8/library/build.gradle.kts index 739bb00d9f1a..56b3c53c7739 100644 --- a/instrumentation/iceberg-1.8/library/build.gradle.kts +++ b/instrumentation/iceberg-1.8/library/build.gradle.kts @@ -6,8 +6,10 @@ plugins { dependencies { library("org.apache.iceberg:iceberg-core:1.8.1") testImplementation(project(":instrumentation:iceberg-1.8:testing")) + implementation("io.opentelemetry:opentelemetry-api-incubator") } otelJava { minJavaVersionSupported.set(JavaVersion.VERSION_11) + maxJavaVersionForTests.set(JavaVersion.VERSION_17) } From d7f72ef65ee1c6bea411bdd2d78a4ae481ddb2cc Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Sat, 22 Nov 2025 12:21:26 +0100 Subject: [PATCH 26/33] add javadocs to the public class --- .../iceberg/v1_8/IcebergTelemetry.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java index 23f30b9fd704..11886c4915ca 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java @@ -5,14 +5,17 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; -import io.opentelemetry.api.OpenTelemetry; import org.apache.iceberg.Scan; import org.apache.iceberg.ScanTask; import org.apache.iceberg.ScanTaskGroup; +import io.opentelemetry.api.OpenTelemetry; + +/** Entrypoint for instrumenting Apache Iceberg scan metrics */ public final class IcebergTelemetry { private final OpenTelemetry openTelemetry; + /** Returns a new {@link IcebergTelemetry} configured with the given {@link OpenTelemetry}. */ public static IcebergTelemetry create(OpenTelemetry openTelemetry) { return new IcebergTelemetry(openTelemetry); } @@ -21,6 +24,15 @@ public static IcebergTelemetry create(OpenTelemetry openTelemetry) { this.openTelemetry = openTelemetry; } + /** + * Creates a new {@link Scan} instance based on an existing {@link Scan} instance. The new instance is associated + * with a custom {@link org.apache.iceberg.metrics.MetricsReporter} that reports scan metrics using the configured {@link OpenTelemetry} instance. + * @param the child class, returned by method chaining, e.g., {@link Scan#project(org.apache.iceberg.Schema)} + * @param the type of tasks produces by this scan + * @param the type of task groups produces by this scan + * @param scan the original scan instance that will be instrumented + * @return an instrumented {@link Scan} instance based on the provided instance + */ public > T1 wrapScan( Scan scan) { return scan.metricsReporter(new IcebergMetricsReporter(openTelemetry)); From f5d96e348c14e69a27400528db3a5f4c1a077a0a Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Sat, 22 Nov 2025 13:34:42 +0100 Subject: [PATCH 27/33] apply advice to exclude snapshot_id attribute --- .../iceberg/v1_8/IcebergMetricsReporter.java | 46 ++++++++++-- .../iceberg/v1_8/IcebergTelemetry.java | 12 ++-- ...er.java => ScanMetricsBuilderFactory.java} | 43 +++++------- .../iceberg/v1_8/AbstractIcebergTest.java | 70 ++++--------------- 4 files changed, 78 insertions(+), 93 deletions(-) rename instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/{ScanMetricsBuilder.java => ScanMetricsBuilderFactory.java} (66%) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index 7cd9d897f2ef..b224338d4fda 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -8,9 +8,14 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.incubator.metrics.ExtendedDoubleHistogramBuilder; +import io.opentelemetry.api.incubator.metrics.ExtendedLongCounterBuilder; import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleHistogramBuilder; import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongCounterBuilder; import io.opentelemetry.api.metrics.Meter; +import java.util.List; import org.apache.iceberg.metrics.CounterResult; import org.apache.iceberg.metrics.MetricsReport; import org.apache.iceberg.metrics.MetricsReporter; @@ -28,6 +33,10 @@ final class IcebergMetricsReporter implements MetricsReporter { AttributeKey.stringKey("iceberg.scan.state"); private static final AttributeKey DELETE_TYPE = AttributeKey.stringKey("iceberg.delete_file.type"); + private static final List> advice1 = List.of(SCHEMA_ID, TABLE_NAME); + private static final List> advice2 = List.of(SCHEMA_ID, TABLE_NAME, SCAN_STATE); + private static final List> advice3 = + List.of(SCHEMA_ID, TABLE_NAME, SCAN_STATE, DELETE_TYPE); private final DoubleHistogram planningDuration; private final LongCounter dataFilesCount; private final LongCounter dataFilesSize; @@ -38,13 +47,18 @@ final class IcebergMetricsReporter implements MetricsReporter { IcebergMetricsReporter(OpenTelemetry openTelemetry) { Meter meter = openTelemetry.getMeter(INSTRUMENTATION_NAME); - planningDuration = ScanMetricsBuilder.totalPlanningDuration(meter, "s"); - dataFilesCount = ScanMetricsBuilder.dataFilesCount(meter); - dataFilesSize = ScanMetricsBuilder.dataFilesSize(meter); - deleteFilesCount = ScanMetricsBuilder.deleteFilesCount(meter); - deleteFilesSize = ScanMetricsBuilder.deleteFilesSize(meter); - dataManifestsCount = ScanMetricsBuilder.dataManifestsCount(meter); - deleteManifestsCount = ScanMetricsBuilder.deleteManifestsCount(meter); + planningDuration = + applyAdvice(advice1, ScanMetricsBuilderFactory.totalPlanningDuration(meter, "s")).build(); + dataFilesCount = applyAdvice(advice2, ScanMetricsBuilderFactory.dataFilesCount(meter)).build(); + dataFilesSize = applyAdvice(advice1, ScanMetricsBuilderFactory.dataFilesSize(meter)).build(); + deleteFilesCount = + applyAdvice(advice3, ScanMetricsBuilderFactory.deleteFilesCount(meter)).build(); + deleteFilesSize = + applyAdvice(advice1, ScanMetricsBuilderFactory.deleteFilesSize(meter)).build(); + dataManifestsCount = + applyAdvice(advice2, ScanMetricsBuilderFactory.dataManifestsCount(meter)).build(); + deleteManifestsCount = + applyAdvice(advice2, ScanMetricsBuilderFactory.deleteManifestsCount(meter)).build(); } @Override @@ -227,4 +241,22 @@ private static void addValueToLongCounter( attributes.toBuilder().put(att1Key, att1Value).put(att2Key, att2Value).build(); metric.add(measurement, newAttributes); } + + private static LongCounterBuilder applyAdvice( + List> attributeKeys, LongCounterBuilder builder) { + if (builder instanceof ExtendedLongCounterBuilder) { + return ((ExtendedLongCounterBuilder) builder).setAttributesAdvice(attributeKeys); + } + + return builder; + } + + private static DoubleHistogramBuilder applyAdvice( + List> attributeKeys, DoubleHistogramBuilder builder) { + if (builder instanceof ExtendedDoubleHistogramBuilder) { + return ((ExtendedDoubleHistogramBuilder) builder).setAttributesAdvice(attributeKeys); + } + + return builder; + } } diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java index 11886c4915ca..97f2e124129c 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java @@ -5,12 +5,11 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; +import io.opentelemetry.api.OpenTelemetry; import org.apache.iceberg.Scan; import org.apache.iceberg.ScanTask; import org.apache.iceberg.ScanTaskGroup; -import io.opentelemetry.api.OpenTelemetry; - /** Entrypoint for instrumenting Apache Iceberg scan metrics */ public final class IcebergTelemetry { private final OpenTelemetry openTelemetry; @@ -25,9 +24,12 @@ public static IcebergTelemetry create(OpenTelemetry openTelemetry) { } /** - * Creates a new {@link Scan} instance based on an existing {@link Scan} instance. The new instance is associated - * with a custom {@link org.apache.iceberg.metrics.MetricsReporter} that reports scan metrics using the configured {@link OpenTelemetry} instance. - * @param the child class, returned by method chaining, e.g., {@link Scan#project(org.apache.iceberg.Schema)} + * Creates a new {@link Scan} instance based on an existing {@link Scan} instance. The new + * instance is associated with a custom {@link org.apache.iceberg.metrics.MetricsReporter} that + * reports scan metrics using the configured {@link OpenTelemetry} instance. + * + * @param the child class, returned by method chaining, e.g., {@link + * Scan#project(org.apache.iceberg.Schema)} * @param the type of tasks produces by this scan * @param the type of task groups produces by this scan * @param scan the original scan instance that will be instrumented diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilderFactory.java similarity index 66% rename from instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java rename to instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilderFactory.java index 9d5db92d870d..8870f2ea018b 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilder.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/ScanMetricsBuilderFactory.java @@ -5,11 +5,11 @@ package io.opentelemetry.instrumentation.iceberg.v1_8; -import io.opentelemetry.api.metrics.DoubleHistogram; -import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.DoubleHistogramBuilder; +import io.opentelemetry.api.metrics.LongCounterBuilder; import io.opentelemetry.api.metrics.Meter; -final class ScanMetricsBuilder { +final class ScanMetricsBuilderFactory { private static final String ROOT = "iceberg.scan"; private static final String TOTAL_PLANNING_DURATION = ROOT + ".planning.duration"; @@ -20,63 +20,56 @@ final class ScanMetricsBuilder { private static final String DATA_MANIFESTS_COUNT = ROOT + ".data_manifests.count"; private static final String DELETE_MANIFESTS_COUNT = ROOT + ".delete_manifests.count"; - private ScanMetricsBuilder() { + private ScanMetricsBuilderFactory() { // prevents instantiation } - static DoubleHistogram totalPlanningDuration(Meter meter, String unit) { + static DoubleHistogramBuilder totalPlanningDuration(Meter meter, String unit) { return meter .histogramBuilder(TOTAL_PLANNING_DURATION) .setDescription("The total duration needed to plan the scan.") - .setUnit(unit) - .build(); + .setUnit(unit); } - static LongCounter dataFilesCount(Meter meter) { + static LongCounterBuilder dataFilesCount(Meter meter) { return meter .counterBuilder(DATA_FILES_COUNT) .setDescription("The number of data files.") - .setUnit("{file}") - .build(); + .setUnit("{file}"); } - static LongCounter deleteFilesCount(Meter meter) { + static LongCounterBuilder deleteFilesCount(Meter meter) { return meter .counterBuilder(DELETE_FILES_COUNT) .setDescription("The number of delete files.") - .setUnit("{file}") - .build(); + .setUnit("{file}"); } - static LongCounter dataManifestsCount(Meter meter) { + static LongCounterBuilder dataManifestsCount(Meter meter) { return meter .counterBuilder(DATA_MANIFESTS_COUNT) .setDescription("The number of data manifests.") - .setUnit("{file}") - .build(); + .setUnit("{file}"); } - static LongCounter deleteManifestsCount(Meter meter) { + static LongCounterBuilder deleteManifestsCount(Meter meter) { return meter .counterBuilder(DELETE_MANIFESTS_COUNT) .setDescription("The number of delete manifests.") - .setUnit("{file}") - .build(); + .setUnit("{file}"); } - static LongCounter dataFilesSize(Meter meter) { + static LongCounterBuilder dataFilesSize(Meter meter) { return meter .counterBuilder(DATA_FILES_SIZE) .setDescription("The total size of all scanned data files.") - .setUnit("By") - .build(); + .setUnit("By"); } - static LongCounter deleteFilesSize(Meter meter) { + static LongCounterBuilder deleteFilesSize(Meter meter) { return meter .counterBuilder(DELETE_FILES_SIZE) .setDescription("The total size of all scanned delete files.") - .setUnit("By") - .build(); + .setUnit("By"); } } diff --git a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java index 93fc9fa191e4..d680cb39ff6a 100644 --- a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java +++ b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java @@ -140,10 +140,7 @@ private void assertScanDurationMetric(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId()))))); + expectedReport.tableName()))))); } private void assertDataFilesCountMetrics(ScanReport expectedReport) { @@ -168,10 +165,7 @@ private void assertDataFilesCountMetrics(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId())), + expectedReport.tableName())), longSumAssert -> longSumAssert .hasValue( @@ -183,10 +177,7 @@ private void assertDataFilesCountMetrics(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId()))))); + expectedReport.tableName()))))); } private void assertDataManifestCountMetrics(ScanReport expectedReport) { @@ -214,10 +205,7 @@ private void assertDataManifestCountMetrics(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId())), + expectedReport.tableName())), longSumAssert -> longSumAssert .hasValue( @@ -232,10 +220,7 @@ private void assertDataManifestCountMetrics(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId()))))); + expectedReport.tableName()))))); } private void assertDeleteManifestCountMetrics(ScanReport expectedReport) { @@ -263,10 +248,7 @@ private void assertDeleteManifestCountMetrics(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId())), + expectedReport.tableName())), longSumAssert -> longSumAssert .hasValue( @@ -281,10 +263,7 @@ private void assertDeleteManifestCountMetrics(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId()))))); + expectedReport.tableName()))))); } private void assertDeleteFilesCountMetrics(ScanReport expectedReport) { @@ -313,10 +292,7 @@ private void assertDeleteFilesCountMetrics(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId())), + expectedReport.tableName())), longSumAssert -> longSumAssert .hasValue( @@ -332,10 +308,7 @@ private void assertDeleteFilesCountMetrics(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId())), + expectedReport.tableName())), longSumAssert -> longSumAssert .hasValue( @@ -352,10 +325,7 @@ private void assertDeleteFilesCountMetrics(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId())), + expectedReport.tableName())), longSumAssert -> longSumAssert .hasValue( @@ -372,10 +342,7 @@ private void assertDeleteFilesCountMetrics(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId())), + expectedReport.tableName())), longSumAssert -> longSumAssert .hasValue( @@ -392,10 +359,7 @@ private void assertDeleteFilesCountMetrics(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId())), + expectedReport.tableName())), longSumAssert -> longSumAssert .hasValue(expectedReport.scanMetrics().dvs().value()) @@ -407,10 +371,7 @@ private void assertDeleteFilesCountMetrics(ScanReport expectedReport) { expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId()))))); + expectedReport.tableName()))))); } private void assertSizeMetric( @@ -434,10 +395,7 @@ private void assertSizeMetric( expectedReport.schemaId()), equalTo( stringKey("iceberg.table.name"), - expectedReport.tableName()), - equalTo( - longKey("iceberg.snapshot.id"), - expectedReport.snapshotId()))))); + expectedReport.tableName()))))); } static final class SimpleReporter implements MetricsReporter { From 90f50478de2561ad215f07544f07f60d6ea4126d Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Sat, 22 Nov 2025 13:35:10 +0100 Subject: [PATCH 28/33] try unit tests with java 21 --- instrumentation/iceberg-1.8/library/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/iceberg-1.8/library/build.gradle.kts b/instrumentation/iceberg-1.8/library/build.gradle.kts index 56b3c53c7739..ab52c05f1ef2 100644 --- a/instrumentation/iceberg-1.8/library/build.gradle.kts +++ b/instrumentation/iceberg-1.8/library/build.gradle.kts @@ -11,5 +11,5 @@ dependencies { otelJava { minJavaVersionSupported.set(JavaVersion.VERSION_11) - maxJavaVersionForTests.set(JavaVersion.VERSION_17) + maxJavaVersionForTests.set(JavaVersion.VERSION_21) } From 0b0a16125ad5759e8d361af7a07631fdef02fa53 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Wed, 26 Nov 2025 13:28:00 +0100 Subject: [PATCH 29/33] revert to using iceberg's TestTables in unit tests to ensure we can test against latest JDK version --- .../iceberg-1.8/library/build.gradle.kts | 1 - .../iceberg-1.8/testing/build.gradle.kts | 8 +++++++- .../iceberg/v1_8/AbstractIcebergTest.java | 16 ++-------------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/build.gradle.kts b/instrumentation/iceberg-1.8/library/build.gradle.kts index ab52c05f1ef2..c1d3d3cb9c41 100644 --- a/instrumentation/iceberg-1.8/library/build.gradle.kts +++ b/instrumentation/iceberg-1.8/library/build.gradle.kts @@ -11,5 +11,4 @@ dependencies { otelJava { minJavaVersionSupported.set(JavaVersion.VERSION_11) - maxJavaVersionForTests.set(JavaVersion.VERSION_21) } diff --git a/instrumentation/iceberg-1.8/testing/build.gradle.kts b/instrumentation/iceberg-1.8/testing/build.gradle.kts index 5dcd9d3a148b..104ea3633ec0 100644 --- a/instrumentation/iceberg-1.8/testing/build.gradle.kts +++ b/instrumentation/iceberg-1.8/testing/build.gradle.kts @@ -4,6 +4,12 @@ plugins { dependencies { implementation("org.apache.iceberg:iceberg-core:1.8.1") - implementation("org.apache.hadoop:hadoop-common:3.4.2") + // The follwoing dependency allows us to use the following Iceberg test classes TestTables and TestTable + // which are not published by default + implementation("org.apache.iceberg:iceberg-core:1.8.1") { + artifact { + classifier = "tests" + } + } api(project(":testing-common")) } diff --git a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java index d680cb39ff6a..e8945ea19e0e 100644 --- a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java +++ b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java @@ -14,8 +14,6 @@ import java.io.File; import java.io.IOException; import java.util.Arrays; -import java.util.Collections; -import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileScanTask; @@ -23,9 +21,8 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableScan; -import org.apache.iceberg.Tables; +import org.apache.iceberg.TestTables; import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.metrics.MetricsReport; import org.apache.iceberg.metrics.MetricsReporter; @@ -62,7 +59,6 @@ public abstract class AbstractIcebergTest { .build(); @TempDir protected File tableDir = null; - protected Tables tables; protected Table table; protected abstract InstrumentationExtension testing(); @@ -71,15 +67,7 @@ public abstract class AbstractIcebergTest { @BeforeEach void init() { - Configuration conf = new Configuration(); - this.tables = new HadoopTables(conf); - String tableLocation = new File(tableDir, "test").toString(); - this.table = - tables.create( - SCHEMA, - SPEC, - Collections.singletonMap("format-version", String.valueOf(FORMAT_VERSION)), - tableLocation); + this.table = TestTables.create(this.tableDir, "test", SCHEMA, SPEC, FORMAT_VERSION); this.table.newFastAppend().appendFile(FILE_1).appendFile(FILE_2).commit(); } From aee6f7be6141dbfb3cd7bcb5bcc40ea0104f2500 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Thu, 27 Nov 2025 10:47:06 +0100 Subject: [PATCH 30/33] Update comment in instrumentation/iceberg-1.8/testing/build.gradle.kts Co-authored-by: Jay DeLuca --- instrumentation/iceberg-1.8/testing/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/iceberg-1.8/testing/build.gradle.kts b/instrumentation/iceberg-1.8/testing/build.gradle.kts index 104ea3633ec0..e83b377b1492 100644 --- a/instrumentation/iceberg-1.8/testing/build.gradle.kts +++ b/instrumentation/iceberg-1.8/testing/build.gradle.kts @@ -4,7 +4,7 @@ plugins { dependencies { implementation("org.apache.iceberg:iceberg-core:1.8.1") - // The follwoing dependency allows us to use the following Iceberg test classes TestTables and TestTable + // The following dependency allows us to use the following Iceberg test classes TestTables and TestTable // which are not published by default implementation("org.apache.iceberg:iceberg-core:1.8.1") { artifact { From c5b4b9763c9fadc3f54f00d164ad305977ab8252 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Thu, 4 Dec 2025 17:24:07 +0100 Subject: [PATCH 31/33] Apply enahncements suggested by the Copilot PR review --- .../iceberg/v1_8/IcebergMetricsReporter.java | 40 +++++++++++++------ .../iceberg/v1_8/IcebergTelemetry.java | 4 +- .../iceberg/v1_8/AbstractIcebergTest.java | 2 +- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index b224338d4fda..88de648011f4 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -33,10 +33,7 @@ final class IcebergMetricsReporter implements MetricsReporter { AttributeKey.stringKey("iceberg.scan.state"); private static final AttributeKey DELETE_TYPE = AttributeKey.stringKey("iceberg.delete_file.type"); - private static final List> advice1 = List.of(SCHEMA_ID, TABLE_NAME); - private static final List> advice2 = List.of(SCHEMA_ID, TABLE_NAME, SCAN_STATE); - private static final List> advice3 = - List.of(SCHEMA_ID, TABLE_NAME, SCAN_STATE, DELETE_TYPE); + private final DoubleHistogram planningDuration; private final LongCounter dataFilesCount; private final LongCounter dataFilesSize; @@ -47,18 +44,37 @@ final class IcebergMetricsReporter implements MetricsReporter { IcebergMetricsReporter(OpenTelemetry openTelemetry) { Meter meter = openTelemetry.getMeter(INSTRUMENTATION_NAME); + List> BASE_ADVICE = List.of(SCHEMA_ID, TABLE_NAME); + List> ADVICE_FOR_DATA_AND_MANIFEST_FILE_COUNTS = + List.of(SCHEMA_ID, TABLE_NAME, SCAN_STATE); + List> ADVICE_FOR_DELETE_FILE_COUNTS = + List.of(SCHEMA_ID, TABLE_NAME, SCAN_STATE, DELETE_TYPE); planningDuration = - applyAdvice(advice1, ScanMetricsBuilderFactory.totalPlanningDuration(meter, "s")).build(); - dataFilesCount = applyAdvice(advice2, ScanMetricsBuilderFactory.dataFilesCount(meter)).build(); - dataFilesSize = applyAdvice(advice1, ScanMetricsBuilderFactory.dataFilesSize(meter)).build(); + applyAdvice(BASE_ADVICE, ScanMetricsBuilderFactory.totalPlanningDuration(meter, "s")) + .build(); + dataFilesCount = + applyAdvice( + ADVICE_FOR_DATA_AND_MANIFEST_FILE_COUNTS, + ScanMetricsBuilderFactory.dataFilesCount(meter)) + .build(); + dataFilesSize = + applyAdvice(BASE_ADVICE, ScanMetricsBuilderFactory.dataFilesSize(meter)).build(); deleteFilesCount = - applyAdvice(advice3, ScanMetricsBuilderFactory.deleteFilesCount(meter)).build(); + applyAdvice( + ADVICE_FOR_DELETE_FILE_COUNTS, ScanMetricsBuilderFactory.deleteFilesCount(meter)) + .build(); deleteFilesSize = - applyAdvice(advice1, ScanMetricsBuilderFactory.deleteFilesSize(meter)).build(); + applyAdvice(BASE_ADVICE, ScanMetricsBuilderFactory.deleteFilesSize(meter)).build(); dataManifestsCount = - applyAdvice(advice2, ScanMetricsBuilderFactory.dataManifestsCount(meter)).build(); + applyAdvice( + ADVICE_FOR_DATA_AND_MANIFEST_FILE_COUNTS, + ScanMetricsBuilderFactory.dataManifestsCount(meter)) + .build(); deleteManifestsCount = - applyAdvice(advice2, ScanMetricsBuilderFactory.deleteManifestsCount(meter)).build(); + applyAdvice( + ADVICE_FOR_DATA_AND_MANIFEST_FILE_COUNTS, + ScanMetricsBuilderFactory.deleteManifestsCount(meter)) + .build(); } @Override @@ -72,7 +88,7 @@ void reportScanMetrics(ScanReport scanReport) { Attributes scanAttributes = Attributes.of( SCHEMA_ID, - Long.valueOf(scanReport.schemaId()), + (long) scanReport.schemaId(), TABLE_NAME, scanReport.tableName(), SNAPSHOT_ID, diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java index 97f2e124129c..81d0f20d0832 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergTelemetry.java @@ -30,8 +30,8 @@ public static IcebergTelemetry create(OpenTelemetry openTelemetry) { * * @param the child class, returned by method chaining, e.g., {@link * Scan#project(org.apache.iceberg.Schema)} - * @param the type of tasks produces by this scan - * @param the type of task groups produces by this scan + * @param the type of tasks produced by this scan + * @param the type of task groups produced by this scan * @param scan the original scan instance that will be instrumented * @return an instrumented {@link Scan} instance based on the provided instance */ diff --git a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java index e8945ea19e0e..359d7996d354 100644 --- a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java +++ b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java @@ -34,7 +34,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -public abstract class AbstractIcebergTest { +abstract class AbstractIcebergTest { protected static final int FORMAT_VERSION = 2; protected static final Schema SCHEMA = new Schema( From 86610f7ba0a7b4d4f8a01f26d410b38e42979b99 Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Thu, 4 Dec 2025 19:26:44 +0100 Subject: [PATCH 32/33] fix checkstyle issue --- .../iceberg/v1_8/IcebergMetricsReporter.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java index 88de648011f4..7ac909346d00 100644 --- a/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java +++ b/instrumentation/iceberg-1.8/library/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/IcebergMetricsReporter.java @@ -33,6 +33,11 @@ final class IcebergMetricsReporter implements MetricsReporter { AttributeKey.stringKey("iceberg.scan.state"); private static final AttributeKey DELETE_TYPE = AttributeKey.stringKey("iceberg.delete_file.type"); + private static final List> BASE_ADVICE = List.of(SCHEMA_ID, TABLE_NAME); + private static final List> ADVICE_FOR_DATA_AND_MANIFEST_FILE_COUNTS = + List.of(SCHEMA_ID, TABLE_NAME, SCAN_STATE); + private static final List> ADVICE_FOR_DELETE_FILE_COUNTS = + List.of(SCHEMA_ID, TABLE_NAME, SCAN_STATE, DELETE_TYPE); private final DoubleHistogram planningDuration; private final LongCounter dataFilesCount; @@ -44,11 +49,7 @@ final class IcebergMetricsReporter implements MetricsReporter { IcebergMetricsReporter(OpenTelemetry openTelemetry) { Meter meter = openTelemetry.getMeter(INSTRUMENTATION_NAME); - List> BASE_ADVICE = List.of(SCHEMA_ID, TABLE_NAME); - List> ADVICE_FOR_DATA_AND_MANIFEST_FILE_COUNTS = - List.of(SCHEMA_ID, TABLE_NAME, SCAN_STATE); - List> ADVICE_FOR_DELETE_FILE_COUNTS = - List.of(SCHEMA_ID, TABLE_NAME, SCAN_STATE, DELETE_TYPE); + planningDuration = applyAdvice(BASE_ADVICE, ScanMetricsBuilderFactory.totalPlanningDuration(meter, "s")) .build(); From 77068a2ddbe66f2097fe6f04e829e9ef8feeb36a Mon Sep 17 00:00:00 2001 From: Ghareeb Falazi Date: Thu, 4 Dec 2025 20:47:19 +0100 Subject: [PATCH 33/33] make AbstractIcebergTest class public to avoid javadoc error --- .../instrumentation/iceberg/v1_8/AbstractIcebergTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java index 359d7996d354..e8945ea19e0e 100644 --- a/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java +++ b/instrumentation/iceberg-1.8/testing/src/main/java/io/opentelemetry/instrumentation/iceberg/v1_8/AbstractIcebergTest.java @@ -34,7 +34,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -abstract class AbstractIcebergTest { +public abstract class AbstractIcebergTest { protected static final int FORMAT_VERSION = 2; protected static final Schema SCHEMA = new Schema(