From 2680cbc9fb4356652c68d9599b0a00e0bac6ee2e Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Fri, 7 Nov 2025 15:04:48 +0800 Subject: [PATCH 1/2] [Improvement]: Support get statistics from Paimon --- .../server/dashboard/DashboardServer.java | 2 + .../MixedAndIcebergTableDescriptor.java | 6 ++ .../dashboard/ServerTableDescriptor.java | 7 ++ .../dashboard/controller/TableController.java | 16 +++++ .../descriptor/FormatTableDescriptor.java | 3 + .../table/descriptor/StatisticsBaseInfo.java | 71 +++++++++++++++++++ .../formats/hudi/HudiTableDescriptor.java | 7 ++ .../formats/paimon/PaimonTableDescriptor.java | 19 +++++ .../paimon/info/PaimonStatisticInfo.java | 18 +++++ 9 files changed, 149 insertions(+) create mode 100644 amoro-common/src/main/java/org/apache/amoro/table/descriptor/StatisticsBaseInfo.java create mode 100644 amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/info/PaimonStatisticInfo.java diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java index 7e342159f4..2aabea5fc6 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java @@ -272,6 +272,8 @@ private EndpointGroup apiGroup() { get( "/catalogs/{catalog}/dbs/{db}/tables/{table}/consumers", tableController::getTableConsumerInfos); + get("/catalogs/{catalog}/dbs/{db}/tables/{table}/statistics", + tableController::getTableStatistics); post( "/catalogs/{catalog}/dbs/{db}/tables/{table}/optimizing-processes/{processId}/cancel", tableController::cancelOptimizingProcess); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java index df035ee0b4..e1c2b0efcf 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java @@ -67,6 +67,7 @@ import org.apache.amoro.table.descriptor.PartitionBaseInfo; import org.apache.amoro.table.descriptor.PartitionFileBaseInfo; import org.apache.amoro.table.descriptor.ServerTableMeta; +import org.apache.amoro.table.descriptor.StatisticsBaseInfo; import org.apache.amoro.table.descriptor.TableSummary; import org.apache.amoro.table.descriptor.TagOrBranchInfo; import org.apache.amoro.utils.MixedDataFiles; @@ -509,6 +510,11 @@ public List getTableConsumerInfos(AmoroTable amoroTable) { return Collections.emptyList(); } + @Override + public StatisticsBaseInfo getTableStatistics(AmoroTable amoroTable) { + return new StatisticsBaseInfo(); + } + @Override public Pair, Integer> getOptimizingProcessesInfo( AmoroTable amoroTable, String type, ProcessStatus status, int limit, int offset) { diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java index 4e2b1aef8c..e4c993ae6d 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java @@ -37,6 +37,7 @@ import org.apache.amoro.table.descriptor.PartitionBaseInfo; import org.apache.amoro.table.descriptor.PartitionFileBaseInfo; import org.apache.amoro.table.descriptor.ServerTableMeta; +import org.apache.amoro.table.descriptor.StatisticsBaseInfo; import org.apache.amoro.table.descriptor.TagOrBranchInfo; import org.apache.commons.lang3.tuple.Pair; import org.apache.iceberg.util.ThreadPools; @@ -128,6 +129,12 @@ public List getTableConsumersInfos(TableIdentifier tableIdentifier return formatTableDescriptor.getTableConsumerInfos(amoroTable); } + public StatisticsBaseInfo getTableStatistic(TableIdentifier tableIdentifier) { + AmoroTable amoroTable = loadTable(tableIdentifier); + FormatTableDescriptor formatTableDescriptor = formatDescriptorMap.get(amoroTable.format()); + return formatTableDescriptor.getTableStatistics(amoroTable); + } + public Pair, Integer> getOptimizingProcessesInfo( TableIdentifier tableIdentifier, String type, ProcessStatus status, int limit, int offset) { AmoroTable amoroTable = loadTable(tableIdentifier); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java index 8b4acba44a..83c29a6fad 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java @@ -72,6 +72,7 @@ import org.apache.amoro.table.descriptor.PartitionBaseInfo; import org.apache.amoro.table.descriptor.PartitionFileBaseInfo; import org.apache.amoro.table.descriptor.ServerTableMeta; +import org.apache.amoro.table.descriptor.StatisticsBaseInfo; import org.apache.amoro.table.descriptor.TableSummary; import org.apache.amoro.table.descriptor.TagOrBranchInfo; import org.apache.amoro.utils.CatalogUtil; @@ -82,6 +83,7 @@ import org.apache.iceberg.SnapshotRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.reflect.internal.util.StatisticsStatics; import java.util.Collections; import java.util.Comparator; @@ -670,6 +672,20 @@ public void getTableConsumerInfos(Context ctx) { ctx.json(OkResponse.of(amsPageResult)); } + public void getTableStatistics(Context ctx) { + String catalog = ctx.pathParam("catalog"); + String database = ctx.pathParam("db"); + String table = ctx.pathParam("table"); + Integer page = ctx.queryParamAsClass("page", Integer.class).getOrDefault(1); + Integer pageSize = ctx.queryParamAsClass("pageSize", Integer.class).getOrDefault(20); + int offset = (page - 1) * pageSize; + StatisticsBaseInfo statisticsStatics = + tableDescriptor.getTableStatistic( + TableIdentifier.of(catalog, database, table).buildTableIdentifier()); + PageResult amsPageResult = PageResult.of(Collections.singletonList(statisticsStatics), offset, pageSize); + ctx.json(OkResponse.of(amsPageResult)); + } + /** * cancel the running optimizing process of one certain table. * diff --git a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java index 1e8bc62c2c..89b6411950 100644 --- a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java +++ b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java @@ -79,4 +79,7 @@ Pair, Integer> getOptimizingProcessesInfo( /** Get the consumer information of the {@link AmoroTable}. */ List getTableConsumerInfos(AmoroTable amoroTable); + + /** Get the partition information of the {@link AmoroTable}. */ + StatisticsBaseInfo getTableStatistics(AmoroTable amoroTable); } diff --git a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/StatisticsBaseInfo.java b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/StatisticsBaseInfo.java new file mode 100644 index 0000000000..ae7e96200a --- /dev/null +++ b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/StatisticsBaseInfo.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.table.descriptor; + +public class StatisticsBaseInfo { + + private Long snapshotId; + + private Long schemaId; + + private Long mergedRecordCount; + + private Long mergedRecordSize; + + public StatisticsBaseInfo() {} + + public StatisticsBaseInfo(Long snapshotId, Long schemaId, Long mergedRecordCount, Long mergedRecordSize) { + this.snapshotId = snapshotId; + this.schemaId = schemaId; + this.mergedRecordCount = mergedRecordCount; + this.mergedRecordSize = mergedRecordSize; + } + + public long getSnapshotId() { + return snapshotId; + } + + public void setSnapshotId(Long snapshotId) { + this.snapshotId = snapshotId; + } + + public long getSchemaId() { + return schemaId; + } + + public void setSchemaId(Long schemaId) { + this.schemaId = schemaId; + } + + public Long getMergedRecordCount() { + return mergedRecordCount; + } + + public void setMergedRecordCount(Long mergedRecordCount) { + this.mergedRecordCount = mergedRecordCount; + } + + public Long getMergedRecordSize() { + return mergedRecordSize; + } + + public void setMergedRecordSize(Long mergedRecordSize) { + this.mergedRecordSize = mergedRecordSize; + } +} diff --git a/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java b/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java index 04d089233a..97292e30ec 100644 --- a/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java +++ b/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java @@ -38,6 +38,7 @@ import org.apache.amoro.table.descriptor.PartitionBaseInfo; import org.apache.amoro.table.descriptor.PartitionFileBaseInfo; import org.apache.amoro.table.descriptor.ServerTableMeta; +import org.apache.amoro.table.descriptor.StatisticsBaseInfo; import org.apache.amoro.table.descriptor.TableSummary; import org.apache.amoro.table.descriptor.TagOrBranchInfo; import org.apache.amoro.utils.CommonUtil; @@ -725,6 +726,12 @@ public List getTableConsumerInfos(AmoroTable amoroTable) { return Collections.emptyList(); } + @Override + public StatisticsBaseInfo getTableStatistics(AmoroTable amoroTable) { + // hudi doesn't support statistics + return new StatisticsBaseInfo(); + } + private long parseHoodieCommitTime(String commitTime) { try { Date date = HoodieInstantTimeGenerator.parseDateFromInstantTime(commitTime); diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java index f938276586..48a6135b19 100644 --- a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java @@ -23,6 +23,7 @@ import org.apache.amoro.AmoroTable; import org.apache.amoro.TableFormat; import org.apache.amoro.api.CommitMetaProducer; +import org.apache.amoro.formats.paimon.info.PaimonStatisticInfo; import org.apache.amoro.process.ProcessStatus; import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; @@ -42,6 +43,7 @@ import org.apache.amoro.table.descriptor.PartitionBaseInfo; import org.apache.amoro.table.descriptor.PartitionFileBaseInfo; import org.apache.amoro.table.descriptor.ServerTableMeta; +import org.apache.amoro.table.descriptor.StatisticsBaseInfo; import org.apache.amoro.table.descriptor.TableSummary; import org.apache.amoro.table.descriptor.TagOrBranchInfo; import org.apache.amoro.utils.CommonUtil; @@ -58,6 +60,7 @@ import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.ManifestList; +import org.apache.paimon.stats.Statistics; import org.apache.paimon.table.DataTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.utils.BranchManager; @@ -579,6 +582,22 @@ public List getTableConsumerInfos(AmoroTable amoroTable) { return consumerInfos; } + @Override + public StatisticsBaseInfo getTableStatistics(AmoroTable amoroTable) { + FileStoreTable table = getTable(amoroTable); + if (!table.statistics().isPresent()) { + return new PaimonStatisticInfo(); + } + Statistics statistics = table.statistics().get(); + return new PaimonStatisticInfo( + statistics.snapshotId(), + statistics.schemaId(), + statistics.mergedRecordCount().getAsLong(), + statistics.mergedRecordSize().getAsLong(), + Collections.emptyMap() + ); + } + private AmoroSnapshotsOfTable manifestListInfo( FileStore store, Snapshot snapshot, diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/info/PaimonStatisticInfo.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/info/PaimonStatisticInfo.java new file mode 100644 index 0000000000..aa4b38d596 --- /dev/null +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/info/PaimonStatisticInfo.java @@ -0,0 +1,18 @@ +package org.apache.amoro.formats.paimon.info; + +import org.apache.amoro.table.descriptor.StatisticsBaseInfo; +import org.apache.paimon.stats.ColStats; + +import java.util.Map; + +public class PaimonStatisticInfo extends StatisticsBaseInfo { + private Map> colStas; + + public PaimonStatisticInfo(){} + + public PaimonStatisticInfo(Long snapshotId, Long schemaId, Long mergedRecordCount, Long mergedRecordSize, Map> colStas) { + super(snapshotId, schemaId, mergedRecordCount, mergedRecordSize); + this.colStas = colStas; + } + +} From 128207dbced1494c5e5a6dc0febf83d06259a71c Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Fri, 7 Nov 2025 15:37:36 +0800 Subject: [PATCH 2/2] style changed --- .../server/dashboard/DashboardServer.java | 5 +- .../dashboard/controller/TableController.java | 8 +-- .../table/descriptor/StatisticsBaseInfo.java | 71 ++++++++++--------- .../formats/paimon/PaimonTableDescriptor.java | 11 ++- .../paimon/info/PaimonStatisticInfo.java | 36 ++++++++-- 5 files changed, 77 insertions(+), 54 deletions(-) diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java index 2aabea5fc6..0215c03b0e 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java @@ -272,8 +272,9 @@ private EndpointGroup apiGroup() { get( "/catalogs/{catalog}/dbs/{db}/tables/{table}/consumers", tableController::getTableConsumerInfos); - get("/catalogs/{catalog}/dbs/{db}/tables/{table}/statistics", - tableController::getTableStatistics); + get( + "/catalogs/{catalog}/dbs/{db}/tables/{table}/statistics", + tableController::getTableStatistics); post( "/catalogs/{catalog}/dbs/{db}/tables/{table}/optimizing-processes/{processId}/cancel", tableController::cancelOptimizingProcess); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java index 83c29a6fad..ab51116fc0 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java @@ -83,7 +83,6 @@ import org.apache.iceberg.SnapshotRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.reflect.internal.util.StatisticsStatics; import java.util.Collections; import java.util.Comparator; @@ -680,9 +679,10 @@ public void getTableStatistics(Context ctx) { Integer pageSize = ctx.queryParamAsClass("pageSize", Integer.class).getOrDefault(20); int offset = (page - 1) * pageSize; StatisticsBaseInfo statisticsStatics = - tableDescriptor.getTableStatistic( - TableIdentifier.of(catalog, database, table).buildTableIdentifier()); - PageResult amsPageResult = PageResult.of(Collections.singletonList(statisticsStatics), offset, pageSize); + tableDescriptor.getTableStatistic( + TableIdentifier.of(catalog, database, table).buildTableIdentifier()); + PageResult amsPageResult = + PageResult.of(Collections.singletonList(statisticsStatics), offset, pageSize); ctx.json(OkResponse.of(amsPageResult)); } diff --git a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/StatisticsBaseInfo.java b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/StatisticsBaseInfo.java index ae7e96200a..302bb70d61 100644 --- a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/StatisticsBaseInfo.java +++ b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/StatisticsBaseInfo.java @@ -20,52 +20,53 @@ public class StatisticsBaseInfo { - private Long snapshotId; + private Long snapshotId; - private Long schemaId; + private Long schemaId; - private Long mergedRecordCount; + private Long mergedRecordCount; - private Long mergedRecordSize; + private Long mergedRecordSize; - public StatisticsBaseInfo() {} + public StatisticsBaseInfo() {} - public StatisticsBaseInfo(Long snapshotId, Long schemaId, Long mergedRecordCount, Long mergedRecordSize) { - this.snapshotId = snapshotId; - this.schemaId = schemaId; - this.mergedRecordCount = mergedRecordCount; - this.mergedRecordSize = mergedRecordSize; - } + public StatisticsBaseInfo( + Long snapshotId, Long schemaId, Long mergedRecordCount, Long mergedRecordSize) { + this.snapshotId = snapshotId; + this.schemaId = schemaId; + this.mergedRecordCount = mergedRecordCount; + this.mergedRecordSize = mergedRecordSize; + } - public long getSnapshotId() { - return snapshotId; - } + public long getSnapshotId() { + return snapshotId; + } - public void setSnapshotId(Long snapshotId) { - this.snapshotId = snapshotId; - } + public void setSnapshotId(Long snapshotId) { + this.snapshotId = snapshotId; + } - public long getSchemaId() { - return schemaId; - } + public long getSchemaId() { + return schemaId; + } - public void setSchemaId(Long schemaId) { - this.schemaId = schemaId; - } + public void setSchemaId(Long schemaId) { + this.schemaId = schemaId; + } - public Long getMergedRecordCount() { - return mergedRecordCount; - } + public Long getMergedRecordCount() { + return mergedRecordCount; + } - public void setMergedRecordCount(Long mergedRecordCount) { - this.mergedRecordCount = mergedRecordCount; - } + public void setMergedRecordCount(Long mergedRecordCount) { + this.mergedRecordCount = mergedRecordCount; + } - public Long getMergedRecordSize() { - return mergedRecordSize; - } + public Long getMergedRecordSize() { + return mergedRecordSize; + } - public void setMergedRecordSize(Long mergedRecordSize) { - this.mergedRecordSize = mergedRecordSize; - } + public void setMergedRecordSize(Long mergedRecordSize) { + this.mergedRecordSize = mergedRecordSize; + } } diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java index 48a6135b19..1d7c8e4ade 100644 --- a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java @@ -590,12 +590,11 @@ public StatisticsBaseInfo getTableStatistics(AmoroTable amoroTable) { } Statistics statistics = table.statistics().get(); return new PaimonStatisticInfo( - statistics.snapshotId(), - statistics.schemaId(), - statistics.mergedRecordCount().getAsLong(), - statistics.mergedRecordSize().getAsLong(), - Collections.emptyMap() - ); + statistics.snapshotId(), + statistics.schemaId(), + statistics.mergedRecordCount().getAsLong(), + statistics.mergedRecordSize().getAsLong(), + Collections.emptyMap()); } private AmoroSnapshotsOfTable manifestListInfo( diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/info/PaimonStatisticInfo.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/info/PaimonStatisticInfo.java index aa4b38d596..a881fae2f3 100644 --- a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/info/PaimonStatisticInfo.java +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/info/PaimonStatisticInfo.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.amoro.formats.paimon.info; import org.apache.amoro.table.descriptor.StatisticsBaseInfo; @@ -6,13 +24,17 @@ import java.util.Map; public class PaimonStatisticInfo extends StatisticsBaseInfo { - private Map> colStas; - - public PaimonStatisticInfo(){} + private Map> colStas; - public PaimonStatisticInfo(Long snapshotId, Long schemaId, Long mergedRecordCount, Long mergedRecordSize, Map> colStas) { - super(snapshotId, schemaId, mergedRecordCount, mergedRecordSize); - this.colStas = colStas; - } + public PaimonStatisticInfo() {} + public PaimonStatisticInfo( + Long snapshotId, + Long schemaId, + Long mergedRecordCount, + Long mergedRecordSize, + Map> colStas) { + super(snapshotId, schemaId, mergedRecordCount, mergedRecordSize); + this.colStas = colStas; + } }