diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index ab1bf1be36926d..91ba0a7bb75192 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -388,6 +388,7 @@ private void setExternalTableAutoAnalyzePolicy(ExternalTable table, List alterOps) throws UserException { + long updateTime = System.currentTimeMillis(); for (AlterOp alterOp : alterOps) { if (alterOp instanceof ModifyTablePropertiesOp) { setExternalTableAutoAnalyzePolicy(table, alterOps); @@ -430,7 +431,7 @@ private void processAlterTableForExternalTable( AddPartitionFieldOp addPartitionField = (AddPartitionFieldOp) alterOp; if (table instanceof IcebergExternalTable) { ((IcebergExternalCatalog) table.getCatalog()).addPartitionField( - (IcebergExternalTable) table, addPartitionField); + (IcebergExternalTable) table, addPartitionField, updateTime); } else { throw new UserException("ADD PARTITION KEY is only supported for Iceberg tables"); } @@ -438,7 +439,7 @@ private void processAlterTableForExternalTable( DropPartitionFieldOp dropPartitionField = (DropPartitionFieldOp) alterOp; if (table instanceof IcebergExternalTable) { ((IcebergExternalCatalog) table.getCatalog()).dropPartitionField( - (IcebergExternalTable) table, dropPartitionField); + (IcebergExternalTable) table, dropPartitionField, updateTime); } else { throw new UserException("DROP PARTITION KEY is only supported for Iceberg tables"); } @@ -446,7 +447,7 @@ private void processAlterTableForExternalTable( ReplacePartitionFieldOp replacePartitionField = (ReplacePartitionFieldOp) alterOp; if (table instanceof IcebergExternalTable) { ((IcebergExternalCatalog) table.getCatalog()).replacePartitionField( - (IcebergExternalTable) table, replacePartitionField); + (IcebergExternalTable) table, replacePartitionField, updateTime); } else { throw new UserException("REPLACE PARTITION KEY is only supported for Iceberg tables"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index 2802055a8873c3..54237f2d38edb0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -146,10 +146,10 @@ public void handleRefreshTable(String catalogName, String dbName, String tableNa } return; } - refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, 0); - + long updateTime = System.currentTimeMillis(); + refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, updateTime); ExternalObjectLog log = ExternalObjectLog.createForRefreshTable(catalog.getId(), db.getFullName(), - table.getName()); + table.getName(), updateTime); Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log); } @@ -194,6 +194,9 @@ public void replayRefreshTable(ExternalObjectLog log) { HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() .getMetaStoreCache((HMSExternalCatalog) catalog); cache.refreshAffectedPartitionsCache((HMSExternalTable) table.get(), modifiedPartNames, newPartNames); + if (table.get() instanceof HMSExternalTable && log.getLastUpdateTime() > 0) { + ((HMSExternalTable) table.get()).setUpdateTime(log.getLastUpdateTime()); + } LOG.info("replay refresh partitions for table {}, " + "modified partitions count: {}, " + "new partitions count: {}", @@ -229,12 +232,12 @@ public void refreshExternalTableFromEvent(String catalogName, String dbName, Str public void refreshTableInternal(ExternalDatabase db, ExternalTable table, long updateTime) { table.unsetObjectCreated(); - if (table instanceof HMSExternalTable && updateTime > 0) { - ((HMSExternalTable) table).setEventUpdateTime(updateTime); + if (updateTime > 0) { + table.setUpdateTime(updateTime); } Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(table); - LOG.info("refresh table {}, id {} from db {} in catalog {}", - table.getName(), table.getId(), db.getFullName(), db.getCatalog().getName()); + LOG.info("refresh table {}, id {} from db {} in catalog {}, update time: {}", + table.getName(), table.getId(), db.getFullName(), db.getCatalog().getName(), updateTime); } // Refresh partition @@ -268,7 +271,7 @@ public void refreshPartitions(String catalogName, String dbName, String tableNam } Env.getCurrentEnv().getExtMetaCacheMgr().invalidatePartitionsCache((ExternalTable) table, partitionNames); - ((HMSExternalTable) table).setEventUpdateTime(updateTime); + ((HMSExternalTable) table).setUpdateTime(updateTime); } public void addToRefreshMap(long catalogId, Integer[] sec) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 692d32c5931afc..16eeebd7dffab8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -726,7 +726,7 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl HMSExternalTable hmsTable = (HMSExternalTable) table; Env.getCurrentEnv().getExtMetaCacheMgr().addPartitionsCache(catalog.getId(), hmsTable, partitionNames); - hmsTable.setEventUpdateTime(updateTime); + hmsTable.setUpdateTime(updateTime); } public void dropExternalPartitions(String catalogName, String dbName, String tableName, @@ -757,7 +757,7 @@ public void dropExternalPartitions(String catalogName, String dbName, String tab HMSExternalTable hmsTable = (HMSExternalTable) table; Env.getCurrentEnv().getExtMetaCacheMgr().dropPartitionsCache(catalog.getId(), hmsTable, partitionNames); - hmsTable.setEventUpdateTime(updateTime); + hmsTable.setUpdateTime(updateTime); } public void registerCatalogRefreshListener(Env env) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 3e347c9d353687..c0e89a3978ae93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -1137,8 +1137,9 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa partitions = partitionNamesInfo.getPartitionNames(); } ExternalTable dorisTable = getDbOrDdlException(dbName).getTableOrDdlException(tableName); - metadataOps.truncateTable(dorisTable, partitions); - TruncateTableInfo info = new TruncateTableInfo(getName(), dbName, tableName, partitions); + long updateTime = System.currentTimeMillis(); + metadataOps.truncateTable(dorisTable, partitions, updateTime); + TruncateTableInfo info = new TruncateTableInfo(getName(), dbName, tableName, partitions, updateTime); Env.getCurrentEnv().getEditLog().logTruncateTable(info); } catch (Exception e) { LOG.warn("Failed to truncate table {}.{} in catalog {}", dbName, tableName, getName(), e); @@ -1148,7 +1149,7 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa public void replayTruncateTable(TruncateTableInfo info) { if (metadataOps != null) { - metadataOps.afterTruncateTable(info.getDb(), info.getTable()); + metadataOps.afterTruncateTable(info.getDb(), info.getTable(), info.getUpdateTime()); } } @@ -1320,11 +1321,11 @@ public void resetMetaCacheNames() { } // log the refresh external table operation - private void logRefreshExternalTable(ExternalTable dorisTable) { + private void logRefreshExternalTable(ExternalTable dorisTable, long updateTime) { Env.getCurrentEnv().getEditLog() .logRefreshExternalTable( ExternalObjectLog.createForRefreshTable(dorisTable.getCatalog().getId(), - dorisTable.getDbName(), dorisTable.getName())); + dorisTable.getDbName(), dorisTable.getName(), updateTime)); } @Override @@ -1336,8 +1337,9 @@ public void addColumn(TableIf dorisTable, Column column, ColumnPosition position throw new DdlException("Add column operation is not supported for catalog: " + getName()); } try { - metadataOps.addColumn(externalTable, column, position); - logRefreshExternalTable(externalTable); + long updateTime = System.currentTimeMillis(); + metadataOps.addColumn(externalTable, column, position, updateTime); + logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to add column {} to table {}.{} in catalog {}", column.getName(), externalTable.getDbName(), externalTable.getName(), getName(), e); @@ -1354,8 +1356,9 @@ public void addColumns(TableIf dorisTable, List columns) throws UserExce throw new DdlException("Add columns operation is not supported for catalog: " + getName()); } try { - metadataOps.addColumns(externalTable, columns); - logRefreshExternalTable(externalTable); + long updateTime = System.currentTimeMillis(); + metadataOps.addColumns(externalTable, columns, updateTime); + logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to add columns to table {}.{} in catalog {}", externalTable.getDbName(), externalTable.getName(), getName(), e); @@ -1372,8 +1375,9 @@ public void dropColumn(TableIf dorisTable, String columnName) throws UserExcepti throw new DdlException("Drop column operation is not supported for catalog: " + getName()); } try { - metadataOps.dropColumn(externalTable, columnName); - logRefreshExternalTable(externalTable); + long updateTime = System.currentTimeMillis(); + metadataOps.dropColumn(externalTable, columnName, updateTime); + logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to drop column {} from table {}.{} in catalog {}", columnName, externalTable.getDbName(), externalTable.getName(), getName(), e); @@ -1390,8 +1394,9 @@ public void renameColumn(TableIf dorisTable, String oldName, String newName) thr throw new DdlException("Rename column operation is not supported for catalog: " + getName()); } try { - metadataOps.renameColumn(externalTable, oldName, newName); - logRefreshExternalTable(externalTable); + long updateTime = System.currentTimeMillis(); + metadataOps.renameColumn(externalTable, oldName, newName, updateTime); + logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to rename column {} to {} in table {}.{} in catalog {}", oldName, newName, externalTable.getDbName(), externalTable.getName(), getName(), e); @@ -1408,8 +1413,9 @@ public void modifyColumn(TableIf dorisTable, Column column, ColumnPosition colum throw new DdlException("Modify column operation is not supported for catalog: " + getName()); } try { - metadataOps.modifyColumn(externalTable, column, columnPosition); - logRefreshExternalTable(externalTable); + long updateTime = System.currentTimeMillis(); + metadataOps.modifyColumn(externalTable, column, columnPosition, updateTime); + logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to modify column {} in table {}.{} in catalog {}", column.getName(), externalTable.getDbName(), externalTable.getName(), getName(), e); @@ -1426,8 +1432,9 @@ public void reorderColumns(TableIf dorisTable, List newOrder) throws Use throw new DdlException("Reorder columns operation is not supported for catalog: " + getName()); } try { - metadataOps.reorderColumns(externalTable, newOrder); - logRefreshExternalTable(externalTable); + long updateTime = System.currentTimeMillis(); + metadataOps.reorderColumns(externalTable, newOrder, updateTime); + logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to reorder columns in table {}.{} in catalog {}", externalTable.getDbName(), externalTable.getName(), getName(), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java index 43a0675d783c41..84f0605b8d8557 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java @@ -75,22 +75,25 @@ public static ExternalObjectLog createForRefreshDb(long catalogId, String dbName return externalObjectLog; } - public static ExternalObjectLog createForRefreshTable(long catalogId, String dbName, String tblName) { + public static ExternalObjectLog createForRefreshTable(long catalogId, String dbName, String tblName, + long updateTime) { ExternalObjectLog externalObjectLog = new ExternalObjectLog(); externalObjectLog.setCatalogId(catalogId); externalObjectLog.setDbName(dbName); externalObjectLog.setTableName(tblName); + externalObjectLog.setLastUpdateTime(updateTime); return externalObjectLog; } public static ExternalObjectLog createForRefreshPartitions(long catalogId, String dbName, String tblName, - List modifiedPartNames, List newPartNames) { + List modifiedPartNames, List newPartNames, long updateTime) { ExternalObjectLog externalObjectLog = new ExternalObjectLog(); externalObjectLog.setCatalogId(catalogId); externalObjectLog.setDbName(dbName); externalObjectLog.setTableName(tblName); externalObjectLog.setPartitionNames(modifiedPartNames); externalObjectLog.setNewPartitionNames(newPartNames); + externalObjectLog.setLastUpdateTime(updateTime); return externalObjectLog; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index cf74ef2fb54898..4a81bb3c7f7193 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -85,8 +85,8 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { @SerializedName(value = "ta") private final TableAttributes tableAttributes = new TableAttributes(); - // this field will be refreshed after reloading schema - protected volatile long schemaUpdateTime; + // record the table update time, like insert/alter/delete + protected volatile long updateTime = 0; protected long dbId; protected boolean objectCreated; @@ -276,16 +276,15 @@ public long getCreateTime() { return 0; } - // return schema update time as default - // override this method if there is some other kinds of update time - // use getSchemaUpdateTime if just need the schema update time @Override + // Returns the table update time, tracking when the table was last modified + // (for example, by insert, alter, or refresh operations). public long getUpdateTime() { - return this.schemaUpdateTime; + return updateTime; } - public void setUpdateTime(long schemaUpdateTime) { - this.schemaUpdateTime = schemaUpdateTime; + public void setUpdateTime(long updateTime) { + this.updateTime = updateTime; } @Override @@ -345,7 +344,7 @@ public Optional getColumnStatistic(String colName) { * @return */ public Optional initSchemaAndUpdateTime(SchemaCacheKey key) { - schemaUpdateTime = System.currentTimeMillis(); + setUpdateTime(System.currentTimeMillis()); return initSchema(key); } @@ -484,10 +483,6 @@ public int hashCode() { return Objects.hashCode(name, db); } - public long getSchemaUpdateTime() { - return schemaUpdateTime; - } - public long getDbId() { return dbId; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java index 467887fd476de3..ab0884d0ce1637 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java @@ -51,7 +51,7 @@ public boolean registerTable(TableIf tableIf) { super.registerTable(tableIf); HMSExternalTable table = getTableNullable(tableIf.getName()); if (table != null) { - table.setEventUpdateTime(tableIf.getUpdateTime()); + table.setUpdateTime(tableIf.getUpdateTime()); } return true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 8e8556c8716976..47ed5ff6a6a8d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -190,9 +190,6 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI private HMSDlaTable dlaTable; - // record the event update time when enable hms event listener - protected volatile long eventUpdateTime; - public enum DLAType { UNKNOWN, HIVE, HUDI, ICEBERG } @@ -648,11 +645,11 @@ public Set getPartitionNames() { public Optional initSchemaAndUpdateTime(SchemaCacheKey key) { Table table = loadHiveTable(); // try to use transient_lastDdlTime from hms client - schemaUpdateTime = MapUtils.isNotEmpty(table.getParameters()) + setUpdateTime(MapUtils.isNotEmpty(table.getParameters()) && table.getParameters().containsKey(TBL_PROP_TRANSIENT_LAST_DDL_TIME) ? Long.parseLong(table.getParameters().get(TBL_PROP_TRANSIENT_LAST_DDL_TIME)) * 1000 // use current timestamp if lastDdlTime does not exist (hive views don't have this prop) - : System.currentTimeMillis(); + : System.currentTimeMillis()); return initSchema(key); } @@ -902,17 +899,6 @@ private void setStatData(Column col, ColumnStatisticsData data, ColumnStatisticB builder.setMaxValue(Double.POSITIVE_INFINITY); } - public void setEventUpdateTime(long updateTime) { - this.eventUpdateTime = updateTime; - } - - @Override - // get the max value of `schemaUpdateTime` and `eventUpdateTime` - // eventUpdateTime will be refreshed after processing events with hms event listener enabled - public long getUpdateTime() { - return Math.max(this.schemaUpdateTime, this.eventUpdateTime); - } - @Override public void gsonPostProcess() throws IOException { super.gsonPostProcess(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index b1ede3926be5c4..7d8501f449fb50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -337,7 +337,7 @@ public void truncateTableImpl(ExternalTable dorisTable, List partitions) } @Override - public void afterTruncateTable(String dbName, String tblName) { + public void afterTruncateTable(String dbName, String tblName, long updateTime) { try { // Invalidate cache. Optional> db = catalog.getDbForReplay(dbName); @@ -345,7 +345,7 @@ public void afterTruncateTable(String dbName, String tblName) { Optional tbl = db.get().getTableForReplay(tblName); if (tbl.isPresent()) { Env.getCurrentEnv().getRefreshManager() - .refreshTableInternal(db.get(), (ExternalTable) tbl.get(), 0); + .refreshTableInternal(db.get(), (ExternalTable) tbl.get(), updateTime); } } } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 8d08e3e8eae934..fb15e9afc9b014 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -221,46 +221,48 @@ public boolean viewExists(String dbName, String viewName) { /** * Add partition field to Iceberg table for partition evolution */ - public void addPartitionField(IcebergExternalTable table, AddPartitionFieldOp op) throws UserException { + public void addPartitionField(IcebergExternalTable table, AddPartitionFieldOp op, long updateTime) + throws UserException { makeSureInitialized(); if (metadataOps == null) { throw new UserException("Add partition field operation is not supported for catalog: " + getName()); } - ((IcebergMetadataOps) metadataOps).addPartitionField(table, op); + ((IcebergMetadataOps) metadataOps).addPartitionField(table, op, updateTime); Env.getCurrentEnv().getEditLog() .logRefreshExternalTable( ExternalObjectLog.createForRefreshTable(table.getCatalog().getId(), - table.getDbName(), table.getName())); + table.getDbName(), table.getName(), updateTime)); } /** * Drop partition field from Iceberg table for partition evolution */ - public void dropPartitionField(IcebergExternalTable table, DropPartitionFieldOp op) throws UserException { + public void dropPartitionField(IcebergExternalTable table, DropPartitionFieldOp op, long updateTime) + throws UserException { makeSureInitialized(); if (metadataOps == null) { throw new UserException("Drop partition field operation is not supported for catalog: " + getName()); } - ((IcebergMetadataOps) metadataOps).dropPartitionField(table, op); + ((IcebergMetadataOps) metadataOps).dropPartitionField(table, op, updateTime); Env.getCurrentEnv().getEditLog() .logRefreshExternalTable( ExternalObjectLog.createForRefreshTable(table.getCatalog().getId(), - table.getDbName(), table.getName())); + table.getDbName(), table.getName(), updateTime)); } /** * Replace partition field in Iceberg table for partition evolution */ public void replacePartitionField(IcebergExternalTable table, - ReplacePartitionFieldOp op) throws UserException { + ReplacePartitionFieldOp op, long updateTime) throws UserException { makeSureInitialized(); if (metadataOps == null) { throw new UserException("Replace partition field operation is not supported for catalog: " + getName()); } - ((IcebergMetadataOps) metadataOps).replacePartitionField(table, op); + ((IcebergMetadataOps) metadataOps).replacePartitionField(table, op, updateTime); Env.getCurrentEnv().getEditLog() .logRefreshExternalTable( ExternalObjectLog.createForRefreshTable(table.getCatalog().getId(), - table.getDbName(), table.getName())); + table.getDbName(), table.getName(), updateTime)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 55378d41589088..bdec6802004c05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -623,19 +623,19 @@ private void applyPosition(UpdateSchema updateSchema, ColumnPosition position, S } } - private void refreshTable(ExternalTable dorisTable) { + private void refreshTable(ExternalTable dorisTable, long updateTime) { Optional> db = dorisCatalog.getDbForReplay(dorisTable.getRemoteDbName()); if (db.isPresent()) { Optional tbl = db.get().getTableForReplay(dorisTable.getRemoteName()); if (tbl.isPresent()) { Env.getCurrentEnv().getRefreshManager() - .refreshTableInternal(db.get(), (ExternalTable) tbl.get(), System.currentTimeMillis()); + .refreshTableInternal(db.get(), (ExternalTable) tbl.get(), updateTime); } } } @Override - public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position) + public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { validateCommonColumnInfo(column); Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); @@ -650,11 +650,11 @@ public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition po throw new UserException("Failed to add column: " + column.getName() + " to table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); } @Override - public void addColumns(ExternalTable dorisTable, List columns) throws UserException { + public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); UpdateSchema updateSchema = icebergTable.updateSchema(); for (Column column : columns) { @@ -667,11 +667,11 @@ public void addColumns(ExternalTable dorisTable, List columns) throws Us throw new UserException("Failed to add columns to table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); } @Override - public void dropColumn(ExternalTable dorisTable, String columnName) throws UserException { + public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); UpdateSchema updateSchema = icebergTable.updateSchema(); updateSchema.deleteColumn(columnName); @@ -681,11 +681,12 @@ public void dropColumn(ExternalTable dorisTable, String columnName) throws UserE throw new UserException("Failed to drop column: " + columnName + " from table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); } @Override - public void renameColumn(ExternalTable dorisTable, String oldName, String newName) throws UserException { + public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) + throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); UpdateSchema updateSchema = icebergTable.updateSchema(); updateSchema.renameColumn(oldName, newName); @@ -695,11 +696,11 @@ public void renameColumn(ExternalTable dorisTable, String oldName, String newNam throw new UserException("Failed to rename column: " + oldName + " to " + newName + " in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); } @Override - public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position) + public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); validateForModifyColumn(column, icebergTable); @@ -720,7 +721,7 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition throw new UserException("Failed to modify column: " + column.getName() + " in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); } private void validateForModifyColumn(Column column, Table icebergTable) throws UserException { @@ -752,7 +753,7 @@ private void validateCommonColumnInfo(Column column) throws UserException { } @Override - public void reorderColumns(ExternalTable dorisTable, List newOrder) throws UserException { + public void reorderColumns(ExternalTable dorisTable, List newOrder, long updateTime) throws UserException { if (newOrder == null || newOrder.isEmpty()) { throw new UserException("Reorder column failed, new order is empty."); } @@ -768,7 +769,7 @@ public void reorderColumns(ExternalTable dorisTable, List newOrder) thro throw new UserException("Failed to reorder columns in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); } public ExecutionAuthenticator getExecutionAuthenticator() { @@ -810,7 +811,8 @@ private Term getTransform(String transformName, String columnName, Integer trans /** * Add partition field to Iceberg table for partition evolution */ - public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldOp op) throws UserException { + public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldOp op, long updateTime) + throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); @@ -832,7 +834,7 @@ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldOp op) throw new UserException("Failed to add partition field to table: " + icebergTable.name() + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); // Reset cached isValidRelatedTable flag after partition evolution ((IcebergExternalTable) dorisTable).setIsValidRelatedTableCached(false); } @@ -840,7 +842,8 @@ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldOp op) /** * Drop partition field from Iceberg table for partition evolution */ - public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldOp op) throws UserException { + public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldOp op, long updateTime) + throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); @@ -860,7 +863,7 @@ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldOp op throw new UserException("Failed to drop partition field from table: " + icebergTable.name() + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); // Reset cached isValidRelatedTable flag after partition evolution ((IcebergExternalTable) dorisTable).setIsValidRelatedTableCached(false); } @@ -868,7 +871,7 @@ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldOp op /** * Replace partition field in Iceberg table for partition evolution */ - public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFieldOp op) + public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFieldOp op, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); @@ -903,7 +906,7 @@ public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFiel throw new UserException("Failed to replace partition field in table: " + icebergTable.name() + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); // Reset cached isValidRelatedTable flag after partition evolution ((IcebergExternalTable) dorisTable).setIsValidRelatedTableCached(false); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java index 35fcea590486e6..3407e8e7cec09b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java @@ -140,14 +140,14 @@ default void afterRenameTable(String dbName, String oldName, String newName) { * @param dorisTable * @param partitions */ - default void truncateTable(ExternalTable dorisTable, List partitions) throws DdlException { + default void truncateTable(ExternalTable dorisTable, List partitions, long updateTime) throws DdlException { truncateTableImpl(dorisTable, partitions); - afterTruncateTable(dorisTable.getDbName(), dorisTable.getName()); + afterTruncateTable(dorisTable.getDbName(), dorisTable.getName(), updateTime); } void truncateTableImpl(ExternalTable dorisTable, List partitions) throws DdlException; - default void afterTruncateTable(String dbName, String tblName) { + default void afterTruncateTable(String dbName, String tblName, long updateTime) { } /** @@ -223,7 +223,7 @@ default void dropBranch(ExternalTable dorisTable, DropBranchInfo branchInfo) * @param position * @throws UserException */ - default void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position) + default void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { throw new UnsupportedOperationException("Add column operation is not supported for this table type."); } @@ -235,7 +235,7 @@ default void addColumn(ExternalTable dorisTable, Column column, ColumnPosition p * @param columns * @throws UserException */ - default void addColumns(ExternalTable dorisTable, List columns) + default void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { throw new UnsupportedOperationException("Add columns operation is not supported for this table type."); } @@ -247,7 +247,7 @@ default void addColumns(ExternalTable dorisTable, List columns) * @param columnName * @throws UserException */ - default void dropColumn(ExternalTable dorisTable, String columnName) + default void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { throw new UnsupportedOperationException("Drop column operation is not supported for this table type."); } @@ -260,7 +260,7 @@ default void dropColumn(ExternalTable dorisTable, String columnName) * @param newName * @throws UserException */ - default void renameColumn(ExternalTable dorisTable, String oldName, String newName) + default void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { throw new UnsupportedOperationException("Rename column operation is not supported for this table type."); } @@ -273,7 +273,7 @@ default void renameColumn(ExternalTable dorisTable, String oldName, String newNa * @param position * @throws UserException */ - default void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position) + default void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { throw new UnsupportedOperationException("Modify column operation is not supported for this table type."); } @@ -285,7 +285,7 @@ default void modifyColumn(ExternalTable dorisTable, Column column, ColumnPositio * @param newOrder * @throws UserException */ - default void reorderColumns(ExternalTable dorisTable, List newOrder) + default void reorderColumns(ExternalTable dorisTable, List newOrder, long updateTime) throws UserException { throw new UnsupportedOperationException("Reorder columns operation is not supported for this table type."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java index 6d08b10403bb76..d3277fe0756137 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java @@ -41,6 +41,7 @@ public TestExternalTable(long id, String name, String remoteName, TestExternalCa public synchronized void makeSureInitialized() { super.makeSureInitialized(); this.objectCreated = true; + setUpdateTime(System.currentTimeMillis()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteActionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteActionCommand.java index 511b5010c044ff..edb3a059078732 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteActionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteActionCommand.java @@ -105,7 +105,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { action.validate(tableNameInfo, ctx.getCurrentUserIdentity()); ResultSet resultSet = action.execute(table); - logRefreshTable(table); + logRefreshTable(table, System.currentTimeMillis()); if (resultSet != null) { executor.sendResultSet(resultSet); } @@ -150,7 +150,7 @@ public Optional getWhereCondition() { * @param table the table to log * @throws UserException if the table type is not supported */ - private void logRefreshTable(TableIf table) throws UserException { + private void logRefreshTable(TableIf table, long updateTime) throws UserException { if (table instanceof ExternalTable) { ExternalTable externalTable = (ExternalTable) table; Env.getCurrentEnv().getEditLog() @@ -158,7 +158,7 @@ private void logRefreshTable(TableIf table) throws UserException { ExternalObjectLog.createForRefreshTable( externalTable.getCatalog().getId(), externalTable.getDbName(), - externalTable.getName())); + externalTable.getName(), updateTime)); } else { // support more table in future throw new UserException("Unsupported table type: " + table.getClass().getName() + " for refresh table"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java index 9f8be52afec30d..fef4d283165cbe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java @@ -98,6 +98,8 @@ protected void doAfterCommit() throws DdlException { } // Write edit log to notify other FEs + long updateTime = System.currentTimeMillis(); + hmsTable.setUpdateTime(updateTime); ExternalObjectLog log; if (!modifiedPartNames.isEmpty() || !newPartNames.isEmpty()) { // Partition-level refresh for other FEs @@ -106,13 +108,14 @@ protected void doAfterCommit() throws DdlException { table.getDatabase().getFullName(), table.getName(), modifiedPartNames, - newPartNames); + newPartNames, + updateTime); } else { // Full table refresh for other FEs log = ExternalObjectLog.createForRefreshTable( hmsTable.getCatalog().getId(), table.getDatabase().getFullName(), - table.getName()); + table.getName(), updateTime); } Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java index b9d77ca6818444..b846d1acbdc1a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java @@ -58,6 +58,8 @@ public class TruncateTableInfo implements Writable { private boolean force = true; // older version it was forced always. @SerializedName(value = "ur") private Map updateRecords; + @SerializedName(value = "ut") + private long updateTime; public TruncateTableInfo() { @@ -82,11 +84,12 @@ public TruncateTableInfo(long dbId, String db, long tblId, String table, List partNames) { + public TruncateTableInfo(String ctl, String db, String table, List partNames, long updateTime) { this.ctl = ctl; this.db = db; this.table = table; this.extPartNames = partNames; + this.updateTime = updateTime; } public String getCtl() { @@ -137,6 +140,10 @@ public Map getUpdateRecords() { return updateRecords; } + public long getUpdateTime() { + return updateTime; + } + public static TruncateTableInfo read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, TruncateTableInfo.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java index 32703c682a304a..c9dd169bc78a12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java @@ -136,7 +136,7 @@ private void checkCacheConfig() { enableSqlCache = true; } } - // alread remove the entrance of partition cache, so we force set to false + // already remove the entrance of partition cache, so we force set to false enablePartitionCache = false; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java index 31f019d5145592..797635307e27f5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java @@ -88,12 +88,12 @@ public void testRefreshTable() throws Exception { CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); TestExternalTable table = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); Assertions.assertFalse(table.isObjectCreated()); - long l1 = table.getSchemaUpdateTime(); - Assertions.assertTrue(l1 == 0); + long l1 = table.getUpdateTime(); + // Assertions.assertEquals(0, l1); table.makeSureInitialized(); Assertions.assertTrue(table.isObjectCreated()); - long l2 = table.getSchemaUpdateTime(); - Assertions.assertTrue(l2 == l1); + long l2 = table.getUpdateTime(); + Assertions.assertTrue(l2 >= l1); TableNameInfo tableNameInfo = new TableNameInfo("test1", "db1", "tbl11"); try { Env.getCurrentEnv().getRefreshManager() @@ -102,15 +102,15 @@ public void testRefreshTable() throws Exception { // Do nothing } Assertions.assertFalse(table.isObjectCreated()); - long l3 = table.getSchemaUpdateTime(); - Assertions.assertTrue(l3 == l2); + long l3 = table.getUpdateTime(); + Assertions.assertTrue(l3 >= l2); table.getFullSchema(); // only table.getFullSchema() can change table.lastUpdateTime - long l4 = table.getSchemaUpdateTime(); + long l4 = table.getUpdateTime(); Assertions.assertTrue(l4 > l3); // updateTime is equal to schema update time as default long l5 = table.getUpdateTime(); - Assertions.assertTrue(l5 == l4); + Assertions.assertTrue(l5 >= l4); // external info schema db ExternalInfoSchemaDatabase infoDb = (ExternalInfoSchemaDatabase) test1.getDbNullable(InfoSchemaDb.DATABASE_NAME); diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java index 7d358073ab89fa..43d5669e6ad74a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java @@ -106,8 +106,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { Deencapsulation.setField(db, "initialized", true); Deencapsulation.setField(tbl, "objectCreated", true); - Deencapsulation.setField(tbl, "schemaUpdateTime", NOW); - Deencapsulation.setField(tbl, "eventUpdateTime", 0); + Deencapsulation.setField(tbl, "updateTime", NOW); Deencapsulation.setField(tbl, "catalog", hmsCatalog); Deencapsulation.setField(tbl, "dbName", "hms_db"); Deencapsulation.setField(tbl, "name", "hms_tbl"); @@ -163,8 +162,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view1, "objectCreated", true); - Deencapsulation.setField(view1, "schemaUpdateTime", NOW); - Deencapsulation.setField(view1, "eventUpdateTime", 0); + Deencapsulation.setField(view1, "updateTime", NOW); Deencapsulation.setField(view1, "catalog", hmsCatalog); Deencapsulation.setField(view1, "dbName", "hms_db"); Deencapsulation.setField(view1, "name", "hms_view1"); @@ -215,8 +213,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view2, "objectCreated", true); - Deencapsulation.setField(view2, "schemaUpdateTime", NOW); - Deencapsulation.setField(view2, "eventUpdateTime", 0); + Deencapsulation.setField(view2, "updateTime", NOW); Deencapsulation.setField(view2, "catalog", hmsCatalog); Deencapsulation.setField(view2, "dbName", "hms_db"); Deencapsulation.setField(view2, "name", "hms_view2"); @@ -268,8 +265,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view3, "objectCreated", true); - Deencapsulation.setField(view3, "schemaUpdateTime", NOW); - Deencapsulation.setField(view3, "eventUpdateTime", 0); + Deencapsulation.setField(view3, "updateTime", NOW); Deencapsulation.setField(view3, "catalog", hmsCatalog); Deencapsulation.setField(view3, "dbName", "hms_db"); Deencapsulation.setField(view3, "name", "hms_view3"); @@ -321,8 +317,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view4, "objectCreated", true); - Deencapsulation.setField(view4, "schemaUpdateTime", NOW); - Deencapsulation.setField(view4, "eventUpdateTime", 0); + Deencapsulation.setField(view4, "updateTime", NOW); Deencapsulation.setField(view4, "catalog", hmsCatalog); Deencapsulation.setField(view4, "dbName", "hms_db"); Deencapsulation.setField(view4, "name", "hms_view4"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index bbca8c183f1539..2e69f7e31f9905 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -136,8 +136,7 @@ private void init(HMSExternalCatalog hmsCatalog) { setField(db, "initialized", true); setField(tbl, "objectCreated", true); - setField(tbl, "schemaUpdateTime", NOW); - setField(tbl, "eventUpdateTime", 0); + setField(tbl, "updateTime", NOW); setField(tbl, "catalog", hmsCatalog); setField(tbl, "dbName", "hms_db"); setField(tbl, "name", "hms_tbl"); @@ -159,8 +158,7 @@ private void init(HMSExternalCatalog hmsCatalog) { .thenReturn(Optional.empty()); setField(tbl2, "objectCreated", true); - setField(tbl2, "schemaUpdateTime", NOW); - setField(tbl2, "eventUpdateTime", 0); + setField(tbl2, "updateTime", NOW); setField(tbl2, "catalog", hmsCatalog); setField(tbl2, "dbName", "hms_db"); setField(tbl2, "name", "hms_tbl2"); @@ -178,11 +176,11 @@ private void init(HMSExternalCatalog hmsCatalog) { Mockito.when(tbl2.getDatabase()).thenReturn(db); Mockito.when(tbl2.getSupportedSysTables()).thenReturn(SupportedSysTables.HIVE_SUPPORTED_SYS_TABLES); Mockito.when(tbl2.getUpdateTime()).thenReturn(NOW); - Mockito.when(tbl2.getSchemaUpdateTime()).thenReturn(NOW); + Mockito.when(tbl2.getUpdateTime()).thenReturn(NOW); // mock initSchemaAndUpdateTime and do nothing Mockito.when(tbl2.initSchemaAndUpdateTime(Mockito.any(ExternalSchemaCache.SchemaCacheKey.class))) .thenReturn(Optional.empty()); - Mockito.doNothing().when(tbl2).setEventUpdateTime(Mockito.anyLong()); + Mockito.doNothing().when(tbl2).setUpdateTime(Mockito.anyLong()); setField(view1, "objectCreated", true); @@ -259,7 +257,7 @@ public void testHitSqlCacheByNereidsAfterPartitionUpdateTimeChanged() { SqlCache sqlCache1 = (SqlCache) ca.getCache(); // latestTime is equals to the schema update time if not set partition update time - Assert.assertEquals(tbl2.getSchemaUpdateTime(), sqlCache1.getLatestTime()); + Assert.assertEquals(tbl2.getUpdateTime(), sqlCache1.getLatestTime()); // wait a second and set partition update time try { diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy index f5d38350e7a1fb..b8838eadbaa269 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy @@ -53,6 +53,9 @@ suite("iceberg_branch_tag_operate", "p0,external,doris,external_docker,external_ sql """ alter table test_branch_tag_operate create branch b1 """ sql """ alter table test_branch_tag_operate create branch if not exists b1 """ + def result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="test_db" and TABLE_NAME='test_branch_tag_operate'""" + def update_time1 = result[0][0]; + sleep(1000) test { sql """ alter table test_branch_tag_operate create or replace branch b1 """ @@ -72,7 +75,11 @@ suite("iceberg_branch_tag_operate", "p0,external,doris,external_docker,external_ sql """ insert into test_branch_tag_operate values (3) """ sql """ insert into test_branch_tag_operate values (4) """ sql """ insert into test_branch_tag_operate values (5) """ - + result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="test_db" and TABLE_NAME='test_branch_tag_operate'""" + def update_time2 = result[0][0]; + logger.info("get update times " + update_time1 + " vs. " + update_time2) + assertTrue(update_time2 > update_time1); + sleep(1000) List> snapshots = sql """ select snapshot_id from iceberg_meta("table" = "${catalog_name}.test_db.test_branch_tag_operate", "query_type" = "snapshots") order by committed_at; """ String s0 = snapshots.get(0)[0] @@ -137,6 +144,10 @@ suite("iceberg_branch_tag_operate", "p0,external,doris,external_docker,external_ exception "Cannot set b8 to unknown snapshot: 11223344" } + result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="test_db" and TABLE_NAME='test_branch_tag_operate'""" + def update_time3 = result[0][0]; + logger.info("get update times " + update_time2 + " vs. " + update_time3) + assertTrue(update_time3 > update_time2); // tag sql """ alter table test_branch_tag_operate create tag t2 as of version ${s0} """ diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy index bc6002500ea756..33ce064811021e 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy @@ -73,6 +73,9 @@ suite("iceberg_schema_change_ddl", "p0,external,doris,external_docker,external_d (2, 'Bob', 30, 87.2), (3, 'Charlie', 22, 92.8) """ + def result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="iceberg_schema_change_ddl_db" and TABLE_NAME='${table_name}'""" + def update_time1 = result[0][0]; + sleep(1000) // Verify initial state qt_init_1 """ DESC ${table_name} """ @@ -104,6 +107,10 @@ suite("iceberg_schema_change_ddl", "p0,external,doris,external_docker,external_d // Test 3: ADD complex type column sql """ ALTER TABLE ${table_name} ADD COLUMN address STRUCT """ + result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="iceberg_schema_change_ddl_db" and TABLE_NAME='${table_name}'""" + def update_time2 = result[0][0]; + logger.info("get update times " + update_time1 + " vs. " + update_time2) + assertTrue(update_time2 > update_time1); qt_add_multi_1 """ DESC ${table_name} """ diff --git a/regression-test/suites/external_table_p0/test_external_table_update_time.groovy b/regression-test/suites/external_table_p0/test_external_table_update_time.groovy new file mode 100644 index 00000000000000..9d18db8aa03727 --- /dev/null +++ b/regression-test/suites/external_table_p0/test_external_table_update_time.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_external_table_update_time", "p0,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Hive test.") + return + } + + for (String hivePrefix : ["hive3"]) { + String extHiveHmsHost = context.config.otherConfigs.get("externalEnvIp") + String extHiveHmsPort = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalog_name = hivePrefix + "_test_update_time_ctl" + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + logger.info("catalog " + catalog_name + " created") + sql """switch ${catalog_name};""" + + sql "drop database if exists test_update_time_db force"; + sql "create database test_update_time_db"; + sql "use test_update_time_db"; + sql "create table test_update_time_tbl(k1 int)" + sql "insert into test_update_time_tbl values(1)" + def result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="test_update_time_db" and TABLE_NAME="test_update_time_tbl"""" + def update_time1 = result[0][0]; + sleep(2000); + sql "insert into test_update_time_tbl values(2)" + result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="test_update_time_db" and TABLE_NAME="test_update_time_tbl"""" + def update_time2 = result[0][0]; + logger.info("get update times " + update_time1 + " vs. " + update_time2) + assertTrue(update_time2 > update_time1); + sleep(2000); + sql "truncate table test_update_time_tbl"; + result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="test_update_time_db" and TABLE_NAME="test_update_time_tbl"""" + def update_time3 = result[0][0]; + logger.info("get update times " + update_time2 + " vs. " + update_time3) + assertTrue(update_time3 > update_time2); + } +} +