Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,9 @@ private EndpointGroup apiGroup() {
get(
"/catalogs/{catalog}/dbs/{db}/tables/{table}/consumers",
tableController::getTableConsumerInfos);
get(
"/catalogs/{catalog}/dbs/{db}/tables/{table}/statistics",
tableController::getTableStatistics);
post(
"/catalogs/{catalog}/dbs/{db}/tables/{table}/optimizing-processes/{processId}/cancel",
tableController::cancelOptimizingProcess);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
import org.apache.amoro.table.descriptor.PartitionBaseInfo;
import org.apache.amoro.table.descriptor.PartitionFileBaseInfo;
import org.apache.amoro.table.descriptor.ServerTableMeta;
import org.apache.amoro.table.descriptor.StatisticsBaseInfo;
import org.apache.amoro.table.descriptor.TableSummary;
import org.apache.amoro.table.descriptor.TagOrBranchInfo;
import org.apache.amoro.utils.MixedDataFiles;
Expand Down Expand Up @@ -509,6 +510,11 @@ public List<ConsumerInfo> getTableConsumerInfos(AmoroTable<?> amoroTable) {
return Collections.emptyList();
}

@Override
public StatisticsBaseInfo getTableStatistics(AmoroTable<?> amoroTable) {
return new StatisticsBaseInfo();
}

@Override
public Pair<List<OptimizingProcessInfo>, Integer> getOptimizingProcessesInfo(
AmoroTable<?> amoroTable, String type, ProcessStatus status, int limit, int offset) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.apache.amoro.table.descriptor.PartitionBaseInfo;
import org.apache.amoro.table.descriptor.PartitionFileBaseInfo;
import org.apache.amoro.table.descriptor.ServerTableMeta;
import org.apache.amoro.table.descriptor.StatisticsBaseInfo;
import org.apache.amoro.table.descriptor.TagOrBranchInfo;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.iceberg.util.ThreadPools;
Expand Down Expand Up @@ -128,6 +129,12 @@ public List<ConsumerInfo> getTableConsumersInfos(TableIdentifier tableIdentifier
return formatTableDescriptor.getTableConsumerInfos(amoroTable);
}

public StatisticsBaseInfo getTableStatistic(TableIdentifier tableIdentifier) {
AmoroTable<?> amoroTable = loadTable(tableIdentifier);
FormatTableDescriptor formatTableDescriptor = formatDescriptorMap.get(amoroTable.format());
return formatTableDescriptor.getTableStatistics(amoroTable);
}

public Pair<List<OptimizingProcessInfo>, Integer> getOptimizingProcessesInfo(
TableIdentifier tableIdentifier, String type, ProcessStatus status, int limit, int offset) {
AmoroTable<?> amoroTable = loadTable(tableIdentifier);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
import org.apache.amoro.table.descriptor.PartitionBaseInfo;
import org.apache.amoro.table.descriptor.PartitionFileBaseInfo;
import org.apache.amoro.table.descriptor.ServerTableMeta;
import org.apache.amoro.table.descriptor.StatisticsBaseInfo;
import org.apache.amoro.table.descriptor.TableSummary;
import org.apache.amoro.table.descriptor.TagOrBranchInfo;
import org.apache.amoro.utils.CatalogUtil;
Expand Down Expand Up @@ -670,6 +671,21 @@ public void getTableConsumerInfos(Context ctx) {
ctx.json(OkResponse.of(amsPageResult));
}

public void getTableStatistics(Context ctx) {
String catalog = ctx.pathParam("catalog");
String database = ctx.pathParam("db");
String table = ctx.pathParam("table");
Integer page = ctx.queryParamAsClass("page", Integer.class).getOrDefault(1);
Integer pageSize = ctx.queryParamAsClass("pageSize", Integer.class).getOrDefault(20);
int offset = (page - 1) * pageSize;
StatisticsBaseInfo statisticsStatics =
tableDescriptor.getTableStatistic(
TableIdentifier.of(catalog, database, table).buildTableIdentifier());
PageResult<StatisticsBaseInfo> amsPageResult =
PageResult.of(Collections.singletonList(statisticsStatics), offset, pageSize);
ctx.json(OkResponse.of(amsPageResult));
}

/**
* cancel the running optimizing process of one certain table.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,7 @@ Pair<List<OptimizingProcessInfo>, Integer> getOptimizingProcessesInfo(

/** Get the consumer information of the {@link AmoroTable}. */
List<ConsumerInfo> getTableConsumerInfos(AmoroTable<?> amoroTable);

/** Get the partition information of the {@link AmoroTable}. */
StatisticsBaseInfo getTableStatistics(AmoroTable<?> amoroTable);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.amoro.table.descriptor;

public class StatisticsBaseInfo {

private Long snapshotId;

private Long schemaId;

private Long mergedRecordCount;

private Long mergedRecordSize;

public StatisticsBaseInfo() {}

public StatisticsBaseInfo(
Long snapshotId, Long schemaId, Long mergedRecordCount, Long mergedRecordSize) {
this.snapshotId = snapshotId;
this.schemaId = schemaId;
this.mergedRecordCount = mergedRecordCount;
this.mergedRecordSize = mergedRecordSize;
}

public long getSnapshotId() {
return snapshotId;
}

public void setSnapshotId(Long snapshotId) {
this.snapshotId = snapshotId;
}

public long getSchemaId() {
return schemaId;
}

public void setSchemaId(Long schemaId) {
this.schemaId = schemaId;
}

public Long getMergedRecordCount() {
return mergedRecordCount;
}

public void setMergedRecordCount(Long mergedRecordCount) {
this.mergedRecordCount = mergedRecordCount;
}

public Long getMergedRecordSize() {
return mergedRecordSize;
}

public void setMergedRecordSize(Long mergedRecordSize) {
this.mergedRecordSize = mergedRecordSize;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.apache.amoro.table.descriptor.PartitionBaseInfo;
import org.apache.amoro.table.descriptor.PartitionFileBaseInfo;
import org.apache.amoro.table.descriptor.ServerTableMeta;
import org.apache.amoro.table.descriptor.StatisticsBaseInfo;
import org.apache.amoro.table.descriptor.TableSummary;
import org.apache.amoro.table.descriptor.TagOrBranchInfo;
import org.apache.amoro.utils.CommonUtil;
Expand Down Expand Up @@ -725,6 +726,12 @@ public List<ConsumerInfo> getTableConsumerInfos(AmoroTable<?> amoroTable) {
return Collections.emptyList();
}

@Override
public StatisticsBaseInfo getTableStatistics(AmoroTable<?> amoroTable) {
// hudi doesn't support statistics
return new StatisticsBaseInfo();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The default implementation we can make in the interface is:throw new UnsupportedOperationException

}

private long parseHoodieCommitTime(String commitTime) {
try {
Date date = HoodieInstantTimeGenerator.parseDateFromInstantTime(commitTime);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.amoro.AmoroTable;
import org.apache.amoro.TableFormat;
import org.apache.amoro.api.CommitMetaProducer;
import org.apache.amoro.formats.paimon.info.PaimonStatisticInfo;
import org.apache.amoro.process.ProcessStatus;
import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
Expand All @@ -42,6 +43,7 @@
import org.apache.amoro.table.descriptor.PartitionBaseInfo;
import org.apache.amoro.table.descriptor.PartitionFileBaseInfo;
import org.apache.amoro.table.descriptor.ServerTableMeta;
import org.apache.amoro.table.descriptor.StatisticsBaseInfo;
import org.apache.amoro.table.descriptor.TableSummary;
import org.apache.amoro.table.descriptor.TagOrBranchInfo;
import org.apache.amoro.utils.CommonUtil;
Expand All @@ -58,6 +60,7 @@
import org.apache.paimon.manifest.ManifestFile;
import org.apache.paimon.manifest.ManifestFileMeta;
import org.apache.paimon.manifest.ManifestList;
import org.apache.paimon.stats.Statistics;
import org.apache.paimon.table.DataTable;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.utils.BranchManager;
Expand Down Expand Up @@ -579,6 +582,21 @@ public List<ConsumerInfo> getTableConsumerInfos(AmoroTable<?> amoroTable) {
return consumerInfos;
}

@Override
public StatisticsBaseInfo getTableStatistics(AmoroTable<?> amoroTable) {
FileStoreTable table = getTable(amoroTable);
if (!table.statistics().isPresent()) {
return new PaimonStatisticInfo();
}
Statistics statistics = table.statistics().get();
return new PaimonStatisticInfo(
statistics.snapshotId(),
statistics.schemaId(),
statistics.mergedRecordCount().getAsLong(),
statistics.mergedRecordSize().getAsLong(),
Collections.emptyMap());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's about colStats info ?

}

private AmoroSnapshotsOfTable manifestListInfo(
FileStore<?> store,
Snapshot snapshot,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.amoro.formats.paimon.info;

import org.apache.amoro.table.descriptor.StatisticsBaseInfo;
import org.apache.paimon.stats.ColStats;

import java.util.Map;

public class PaimonStatisticInfo extends StatisticsBaseInfo {
private Map<String, ColStats<?>> colStas;

public PaimonStatisticInfo() {}

public PaimonStatisticInfo(
Long snapshotId,
Long schemaId,
Long mergedRecordCount,
Long mergedRecordSize,
Map<String, ColStats<?>> colStas) {
super(snapshotId, schemaId, mergedRecordCount, mergedRecordSize);
this.colStas = colStas;
}
}