diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java index f24f9c3330..571baaeb04 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java @@ -282,6 +282,9 @@ private EndpointGroup apiGroup() { get( "/catalogs/{catalog}/dbs/{db}/tables/{table}/consumers", tableController::getTableConsumerInfos); + get( + "/catalogs/{catalog}/dbs/{db}/tables/{table}/statistics", + tableController::getTableStatistics); post( "/catalogs/{catalog}/dbs/{db}/tables/{table}/optimizing-processes/{processId}/cancel", tableController::cancelOptimizingProcess); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java index df035ee0b4..e1c2b0efcf 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java @@ -67,6 +67,7 @@ import org.apache.amoro.table.descriptor.PartitionBaseInfo; import org.apache.amoro.table.descriptor.PartitionFileBaseInfo; import org.apache.amoro.table.descriptor.ServerTableMeta; +import org.apache.amoro.table.descriptor.StatisticsBaseInfo; import org.apache.amoro.table.descriptor.TableSummary; import org.apache.amoro.table.descriptor.TagOrBranchInfo; import org.apache.amoro.utils.MixedDataFiles; @@ -509,6 +510,11 @@ public List getTableConsumerInfos(AmoroTable amoroTable) { return Collections.emptyList(); } + @Override + public StatisticsBaseInfo getTableStatistics(AmoroTable amoroTable) { + return new StatisticsBaseInfo(); + } + @Override public Pair, Integer> getOptimizingProcessesInfo( AmoroTable amoroTable, String type, ProcessStatus status, int limit, int offset) { diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java index 4e2b1aef8c..e4c993ae6d 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java @@ -37,6 +37,7 @@ import org.apache.amoro.table.descriptor.PartitionBaseInfo; import org.apache.amoro.table.descriptor.PartitionFileBaseInfo; import org.apache.amoro.table.descriptor.ServerTableMeta; +import org.apache.amoro.table.descriptor.StatisticsBaseInfo; import org.apache.amoro.table.descriptor.TagOrBranchInfo; import org.apache.commons.lang3.tuple.Pair; import org.apache.iceberg.util.ThreadPools; @@ -128,6 +129,12 @@ public List getTableConsumersInfos(TableIdentifier tableIdentifier return formatTableDescriptor.getTableConsumerInfos(amoroTable); } + public StatisticsBaseInfo getTableStatistic(TableIdentifier tableIdentifier) { + AmoroTable amoroTable = loadTable(tableIdentifier); + FormatTableDescriptor formatTableDescriptor = formatDescriptorMap.get(amoroTable.format()); + return formatTableDescriptor.getTableStatistics(amoroTable); + } + public Pair, Integer> getOptimizingProcessesInfo( TableIdentifier tableIdentifier, String type, ProcessStatus status, int limit, int offset) { AmoroTable amoroTable = loadTable(tableIdentifier); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java index 8b4acba44a..ab51116fc0 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java @@ -72,6 +72,7 @@ import org.apache.amoro.table.descriptor.PartitionBaseInfo; import org.apache.amoro.table.descriptor.PartitionFileBaseInfo; import org.apache.amoro.table.descriptor.ServerTableMeta; +import org.apache.amoro.table.descriptor.StatisticsBaseInfo; import org.apache.amoro.table.descriptor.TableSummary; import org.apache.amoro.table.descriptor.TagOrBranchInfo; import org.apache.amoro.utils.CatalogUtil; @@ -670,6 +671,21 @@ public void getTableConsumerInfos(Context ctx) { ctx.json(OkResponse.of(amsPageResult)); } + public void getTableStatistics(Context ctx) { + String catalog = ctx.pathParam("catalog"); + String database = ctx.pathParam("db"); + String table = ctx.pathParam("table"); + Integer page = ctx.queryParamAsClass("page", Integer.class).getOrDefault(1); + Integer pageSize = ctx.queryParamAsClass("pageSize", Integer.class).getOrDefault(20); + int offset = (page - 1) * pageSize; + StatisticsBaseInfo statisticsStatics = + tableDescriptor.getTableStatistic( + TableIdentifier.of(catalog, database, table).buildTableIdentifier()); + PageResult amsPageResult = + PageResult.of(Collections.singletonList(statisticsStatics), offset, pageSize); + ctx.json(OkResponse.of(amsPageResult)); + } + /** * cancel the running optimizing process of one certain table. * diff --git a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java index 1e8bc62c2c..89b6411950 100644 --- a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java +++ b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java @@ -79,4 +79,7 @@ Pair, Integer> getOptimizingProcessesInfo( /** Get the consumer information of the {@link AmoroTable}. */ List getTableConsumerInfos(AmoroTable amoroTable); + + /** Get the partition information of the {@link AmoroTable}. */ + StatisticsBaseInfo getTableStatistics(AmoroTable amoroTable); } diff --git a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/StatisticsBaseInfo.java b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/StatisticsBaseInfo.java new file mode 100644 index 0000000000..302bb70d61 --- /dev/null +++ b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/StatisticsBaseInfo.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.table.descriptor; + +public class StatisticsBaseInfo { + + private Long snapshotId; + + private Long schemaId; + + private Long mergedRecordCount; + + private Long mergedRecordSize; + + public StatisticsBaseInfo() {} + + public StatisticsBaseInfo( + Long snapshotId, Long schemaId, Long mergedRecordCount, Long mergedRecordSize) { + this.snapshotId = snapshotId; + this.schemaId = schemaId; + this.mergedRecordCount = mergedRecordCount; + this.mergedRecordSize = mergedRecordSize; + } + + public long getSnapshotId() { + return snapshotId; + } + + public void setSnapshotId(Long snapshotId) { + this.snapshotId = snapshotId; + } + + public long getSchemaId() { + return schemaId; + } + + public void setSchemaId(Long schemaId) { + this.schemaId = schemaId; + } + + public Long getMergedRecordCount() { + return mergedRecordCount; + } + + public void setMergedRecordCount(Long mergedRecordCount) { + this.mergedRecordCount = mergedRecordCount; + } + + public Long getMergedRecordSize() { + return mergedRecordSize; + } + + public void setMergedRecordSize(Long mergedRecordSize) { + this.mergedRecordSize = mergedRecordSize; + } +} diff --git a/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java b/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java index 04d089233a..97292e30ec 100644 --- a/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java +++ b/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java @@ -38,6 +38,7 @@ import org.apache.amoro.table.descriptor.PartitionBaseInfo; import org.apache.amoro.table.descriptor.PartitionFileBaseInfo; import org.apache.amoro.table.descriptor.ServerTableMeta; +import org.apache.amoro.table.descriptor.StatisticsBaseInfo; import org.apache.amoro.table.descriptor.TableSummary; import org.apache.amoro.table.descriptor.TagOrBranchInfo; import org.apache.amoro.utils.CommonUtil; @@ -725,6 +726,12 @@ public List getTableConsumerInfos(AmoroTable amoroTable) { return Collections.emptyList(); } + @Override + public StatisticsBaseInfo getTableStatistics(AmoroTable amoroTable) { + // hudi doesn't support statistics + return new StatisticsBaseInfo(); + } + private long parseHoodieCommitTime(String commitTime) { try { Date date = HoodieInstantTimeGenerator.parseDateFromInstantTime(commitTime); diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java index f938276586..1d7c8e4ade 100644 --- a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java @@ -23,6 +23,7 @@ import org.apache.amoro.AmoroTable; import org.apache.amoro.TableFormat; import org.apache.amoro.api.CommitMetaProducer; +import org.apache.amoro.formats.paimon.info.PaimonStatisticInfo; import org.apache.amoro.process.ProcessStatus; import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; @@ -42,6 +43,7 @@ import org.apache.amoro.table.descriptor.PartitionBaseInfo; import org.apache.amoro.table.descriptor.PartitionFileBaseInfo; import org.apache.amoro.table.descriptor.ServerTableMeta; +import org.apache.amoro.table.descriptor.StatisticsBaseInfo; import org.apache.amoro.table.descriptor.TableSummary; import org.apache.amoro.table.descriptor.TagOrBranchInfo; import org.apache.amoro.utils.CommonUtil; @@ -58,6 +60,7 @@ import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.ManifestList; +import org.apache.paimon.stats.Statistics; import org.apache.paimon.table.DataTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.utils.BranchManager; @@ -579,6 +582,21 @@ public List getTableConsumerInfos(AmoroTable amoroTable) { return consumerInfos; } + @Override + public StatisticsBaseInfo getTableStatistics(AmoroTable amoroTable) { + FileStoreTable table = getTable(amoroTable); + if (!table.statistics().isPresent()) { + return new PaimonStatisticInfo(); + } + Statistics statistics = table.statistics().get(); + return new PaimonStatisticInfo( + statistics.snapshotId(), + statistics.schemaId(), + statistics.mergedRecordCount().getAsLong(), + statistics.mergedRecordSize().getAsLong(), + Collections.emptyMap()); + } + private AmoroSnapshotsOfTable manifestListInfo( FileStore store, Snapshot snapshot, diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/info/PaimonStatisticInfo.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/info/PaimonStatisticInfo.java new file mode 100644 index 0000000000..a881fae2f3 --- /dev/null +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/info/PaimonStatisticInfo.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.formats.paimon.info; + +import org.apache.amoro.table.descriptor.StatisticsBaseInfo; +import org.apache.paimon.stats.ColStats; + +import java.util.Map; + +public class PaimonStatisticInfo extends StatisticsBaseInfo { + private Map> colStas; + + public PaimonStatisticInfo() {} + + public PaimonStatisticInfo( + Long snapshotId, + Long schemaId, + Long mergedRecordCount, + Long mergedRecordSize, + Map> colStas) { + super(snapshotId, schemaId, mergedRecordCount, mergedRecordSize); + this.colStas = colStas; + } +}