diff --git a/core/src/main/java/org/apache/iceberg/MetadataTableType.java b/core/src/main/java/org/apache/iceberg/MetadataTableType.java index 733a11d900f1..6cfab39ffe6a 100644 --- a/core/src/main/java/org/apache/iceberg/MetadataTableType.java +++ b/core/src/main/java/org/apache/iceberg/MetadataTableType.java @@ -18,7 +18,10 @@ */ package org.apache.iceberg; +import java.util.Arrays; +import java.util.List; import java.util.Locale; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; public enum MetadataTableType { ENTRIES, @@ -36,7 +39,14 @@ public enum MetadataTableType { ALL_FILES, ALL_MANIFESTS, ALL_ENTRIES, - POSITION_DELETES; + POSITION_DELETES, + METADATA_TABLES; + + private final String lower; + + MetadataTableType() { + this.lower = name().toLowerCase(Locale.ROOT); + } public static MetadataTableType from(String name) { try { @@ -45,4 +55,14 @@ public static MetadataTableType from(String name) { return null; } } + + public String tableName() { + return this.lower; + } + + public static List lowerNames() { + return Arrays.stream(values()) + .map(MetadataTableType::tableName) + .collect(ImmutableList.toImmutableList()); + } } diff --git a/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java b/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java index adb0f18ba1ad..05d69d345833 100644 --- a/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java +++ b/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java @@ -88,6 +88,8 @@ private static Table createMetadataTableInstance( return new AllEntriesTable(baseTable, metadataTableName); case POSITION_DELETES: return new PositionDeletesTable(baseTable, metadataTableName); + case METADATA_TABLES: + return new MetadataTables(baseTable, metadataTableName); default: throw new NoSuchTableException( "Unknown metadata table type: %s for %s", type, metadataTableName); diff --git a/core/src/main/java/org/apache/iceberg/MetadataTables.java b/core/src/main/java/org/apache/iceberg/MetadataTables.java new file mode 100644 index 000000000000..e894077abca0 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/MetadataTables.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.util.Arrays; +import org.apache.iceberg.types.Types; + +/** A {@link Table} implementation that exposes a table's metadata tables as rows. */ +public class MetadataTables extends BaseMetadataTable { + MetadataTables(Table table) { + this(table, table.name() + "." + MetadataTableType.METADATA_TABLES.tableName()); + } + + MetadataTables(Table table, String name) { + super(table, name); + } + + private static final Schema METADATA_TABLES_SCHEMA = + new Schema(Types.NestedField.required(1, "metadata_table_name", Types.StringType.get())); + + @Override + public TableScan newScan() { + return new MetadataTablesScan(table()); + } + + @Override + public Schema schema() { + return METADATA_TABLES_SCHEMA; + } + + @Override + MetadataTableType metadataTableType() { + return MetadataTableType.METADATA_TABLES; + } + + private DataTask task(BaseTableScan scan) { + StaticDataTask.Row[] rows = + Arrays.stream(MetadataTableType.values()) + .map(metadataType -> StaticDataTask.Row.of(metadataType.tableName())) + .toArray(StaticDataTask.Row[]::new); + + DataFile metadataFile = + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath(table().location() + "#" + MetadataTableType.METADATA_TABLES.tableName()) + .withFormat(FileFormat.METADATA) + .withRecordCount(MetadataTableType.values().length) + .withFileSizeInBytes(0) + .build(); + + return new StaticDataTask(metadataFile, schema(), scan.schema(), rows); + } + + private class MetadataTablesScan extends StaticTableScan { + MetadataTablesScan(Table table) { + super( + table, + METADATA_TABLES_SCHEMA, + MetadataTableType.METADATA_TABLES, + MetadataTables.this::task); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java b/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java index d5a0402a7aa4..d6b5d7903dc1 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java +++ b/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java @@ -1833,6 +1833,28 @@ public void testDeleteFilesTableScanOnBranch() throws IOException { .isEqualTo(2); } + @TestTemplate + public void testMetadataTablesScan() throws IOException { + table.newFastAppend().appendFile(FILE_A).commit(); + Table metadataTable = new MetadataTables(table); + + TableScan scan = metadataTable.newScan(); + List tasks = Lists.newArrayList(scan.planFiles()); + assertThat(tasks).hasSize(1); + + List expected = MetadataTableType.lowerNames(); + + List actual = Lists.newArrayList(); + try (CloseableIterable rs = tasks.get(0).asDataTask().rows()) { + for (StructLike r : rs) { + actual.add(r.get(0, String.class)); + } + } + assertThat(actual) + .as("MetadataTables should return all metadata table names") + .containsExactlyInAnyOrderElementsOf(expected); + } + private int rowCount(TableScan scan) throws IOException { int count = 0; try (CloseableIterable tasks = scan.planFiles()) { diff --git a/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java b/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java index ece9b24af3d1..9e91e751b6d8 100644 --- a/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java +++ b/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java @@ -194,7 +194,11 @@ public void testSerializableMetadataTablesPlanning(boolean fromSerialized) throw Set newFiles = getFiles(getMetaDataTable(table, type)); // Expect that the new data is changed in the meantime - assertThat(deserializedFiles).isNotEqualTo(newFiles); + if (type == MetadataTableType.METADATA_TABLES) { + assertThat(deserializedFiles).isEqualTo(newFiles); + } else { + assertThat(deserializedFiles).isNotEqualTo(newFiles); + } } } diff --git a/docs/docs/spark-queries.md b/docs/docs/spark-queries.md index 91f1759f568c..61858aacd23f 100644 --- a/docs/docs/spark-queries.md +++ b/docs/docs/spark-queries.md @@ -340,6 +340,32 @@ To inspect a table's history, snapshots, and other metadata, Iceberg supports me Metadata tables are identified by adding the metadata table name after the original table name. For example, history for `db.table` is read using `db.table.history`. +### Metadata Tables +To list the metadata tables supported by a table: + +```sql +SELECT * FROM prod.db.table.metadata_tables; +``` + +| metadata_table_name | +|----------------------| +| entries | +| files | +| data_files | +| history | +| metadata_log_entries | +| snapshots | +| refs | +| manifests | +| partitions | +| all_data_files | +| all_delete_files | +| all_files | +| all_manifests | +| all_entries | +| position_deletes | +| metadata_tables | + ### History To show table history: