Materialize indexes

Author: CyberROFL

ydb/core/backup/common/metadata.cpp

@@ -34,6 +34,17 @@ const std::optional<std::vector<TChangefeedMetadata>>& TMetadata::GetChangefeeds
     return Changefeeds;
 }
 
+void TMetadata::AddIndex(const TIndexMetadata& index) {
+    if (!Indexes) {
+        Indexes.emplace();
+    }
+    Indexes->push_back(index);
+}
+
+const std::optional<std::vector<TIndexMetadata>>& TMetadata::GetIndexes() const {
+    return Indexes;
+}
+
 void TMetadata::SetEnablePermissions(bool enablePermissions) {
     EnablePermissions = enablePermissions;
 }
@@ -75,9 +86,19 @@ TString TMetadata::Serialize() const {
             changefeeds.AppendValue(std::move(changefeedMap));
         }
     }
-    // We always serialize changefeeds in order to list them explicitly during import
     m["changefeeds"] = changefeeds;
 
+    NJson::TJsonArray indexes;
+    if (Indexes) {
+        for (const auto& index : *Indexes) {
+            NJson::TJsonMap indexMap;
+            indexMap["export_prefix"] = index.ExportPrefix;
+            indexMap["impl_table_prefix"] = index.ImplTablePrefix;
+            indexes.AppendValue(std::move(indexMap));
+        }
+    }
+    m["indexes"] = indexes;
+
     return NJson::WriteJson(&m, false);
 }
 
@@ -108,6 +129,17 @@ TMetadata TMetadata::Deserialize(const TString& metadata) {
         }
     }
 
+    if (json.Has("indexes")) {
+        result.Indexes.emplace();
+        const NJson::TJsonValue& indexes = json["indexes"];
+        for (const NJson::TJsonValue& index : indexes.GetArray()) {
+            result.AddIndex({
+                .ExportPrefix = index["export_prefix"].GetString(),
+                .ImplTablePrefix = index["impl_table_prefix"].GetString(),
+            });
+        }
+    }
+
     return result;
 }
 

ydb/core/backup/common/metadata.h

@@ -41,6 +41,11 @@ struct TChangefeedMetadata {
     TString Name;
 };
 
+struct TIndexMetadata {
+    TString ExportPrefix;
+    TString ImplTablePrefix;
+};
+
 class TMetadata {
 public:
     TMetadata() = default;
@@ -52,9 +57,13 @@ class TMetadata {
     void SetVersion(ui64 version);
     bool HasVersion() const;
     ui64 GetVersion() const;
+
     void AddChangefeed(const TChangefeedMetadata& changefeed);
     const std::optional<std::vector<TChangefeedMetadata>>& GetChangefeeds() const;
 
+    void AddIndex(const TIndexMetadata& index);
+    const std::optional<std::vector<TIndexMetadata>>& GetIndexes() const;
+
     void SetEnablePermissions(bool enablePermissions = true);
     bool HasEnablePermissions() const;
     bool GetEnablePermissions() const;
@@ -74,6 +83,12 @@ class TMetadata {
     // [...]: The export must have all changefeeds listed here
     std::optional<std::vector<TChangefeedMetadata>> Changefeeds;
 
+    // Indexes:
+    // Undefined (previous versions): we don't know if we see the export with _materialized_ indexes or without them, so list suitable S3 files to find out all materialized indexes
+    // []: The export has no materialized indexes
+    // [...]: The export must have all materialized indexes listed here
+    std::optional<std::vector<TIndexMetadata>> Indexes;
+
     // EnablePermissions:
     // Undefined (previous versions): we don't know if we see the export with permissions or without them, so check S3 for the permissions file existence
     // 0: The export has no permissions

ydb/core/base/table_index.cpp

@@ -133,6 +133,30 @@ TString InvalidIndexType(NKikimrSchemeOp::EIndexType indexType) {
     return TStringBuilder() << "Invalid index type " << static_cast<int>(indexType);
 }
 
+std::optional<NKikimrSchemeOp::EIndexType> TryConvertIndexType(Ydb::Table::TableIndex::TypeCase type) {
+    switch (type) {
+        case Ydb::Table::TableIndex::TypeCase::TYPE_NOT_SET:
+        case Ydb::Table::TableIndex::TypeCase::kGlobalIndex:
+            return NKikimrSchemeOp::EIndexTypeGlobal;
+        case Ydb::Table::TableIndex::TypeCase::kGlobalAsyncIndex:
+            return NKikimrSchemeOp::EIndexTypeGlobalAsync;
+        case Ydb::Table::TableIndex::TypeCase::kGlobalUniqueIndex:
+            return NKikimrSchemeOp::EIndexTypeGlobalUnique;
+        case Ydb::Table::TableIndex::TypeCase::kGlobalVectorKmeansTreeIndex:
+            return NKikimrSchemeOp::EIndexTypeGlobalVectorKmeansTree;
+        case Ydb::Table::TableIndex::TypeCase::kGlobalFulltextIndex:
+            return NKikimrSchemeOp::EIndexTypeGlobalFulltext;
+        default:
+            return std::nullopt;
+    }
+}
+
+NKikimrSchemeOp::EIndexType ConvertIndexType(Ydb::Table::TableIndex::TypeCase type) {
+    const auto result = TryConvertIndexType(type);
+    Y_ENSURE(result);
+    return *result;
+}
+
 bool IsCompatibleIndex(NKikimrSchemeOp::EIndexType indexType, const TTableColumns& table, const TIndexColumns& index, TString& explain) {
     if (const auto* broken = IsContains(table.Keys, table.Columns)) {
         explain = TStringBuilder()
@@ -232,7 +256,11 @@ bool DoesIndexSupportTTL(NKikimrSchemeOp::EIndexType indexType) {
     }
 }
 
-std::span<const std::string_view> GetImplTables(NKikimrSchemeOp::EIndexType indexType, std::span<const TString> indexKeys) {
+std::span<const std::string_view> GetImplTables(
+        NKikimrSchemeOp::EIndexType indexType,
+        std::span<const TString> indexKeys,
+        std::optional<Ydb::Table::FulltextIndexSettings::Layout> layout)
+{
     switch (indexType) {
         case NKikimrSchemeOp::EIndexTypeGlobal:
         case NKikimrSchemeOp::EIndexTypeGlobalAsync:
@@ -244,6 +272,9 @@ std::span<const std::string_view> GetImplTables(NKikimrSchemeOp::EIndexType inde
             } else {
                 return PrefixedGlobalKMeansTreeImplTables;
             }
+        case NKikimrSchemeOp::EIndexTypeGlobalFulltext:
+            Y_ENSURE(layout);
+            return GetFulltextImplTables(*layout);
         default:
             Y_ENSURE(false, InvalidIndexType(indexType));
     }

ydb/core/base/table_index.h

@@ -10,6 +10,7 @@
 #include <util/generic/string.h>
 #include <util/string/builder.h>
 
+#include <optional>
 #include <span>
 #include <string_view>
 
@@ -47,8 +48,13 @@ bool DoesIndexSupportTTL(NKikimrSchemeOp::EIndexType indexType);
 
 NKikimrSchemeOp::EIndexType GetIndexType(const NKikimrSchemeOp::TIndexCreationConfig& indexCreation);
 TString InvalidIndexType(NKikimrSchemeOp::EIndexType indexType);
+std::optional<NKikimrSchemeOp::EIndexType> TryConvertIndexType(Ydb::Table::TableIndex::TypeCase type);
+NKikimrSchemeOp::EIndexType ConvertIndexType(Ydb::Table::TableIndex::TypeCase type);
 
-std::span<const std::string_view> GetImplTables(NKikimrSchemeOp::EIndexType indexType, std::span<const TString> indexKeys);
+std::span<const std::string_view> GetImplTables(
+    NKikimrSchemeOp::EIndexType indexType,
+    std::span<const TString> indexKeys,
+    std::optional<Ydb::Table::FulltextIndexSettings::Layout> layout = std::nullopt);
 std::span<const std::string_view> GetFulltextImplTables(Ydb::Table::FulltextIndexSettings::Layout layout);
 bool IsImplTable(std::string_view tableName);
 bool IsBuildImplTable(std::string_view tableName);

ydb/core/protos/feature_flags.proto

@@ -244,4 +244,5 @@ message TFeatureFlags {
     optional bool EnableShuttingDownNodeState = 218 [default = false];
     optional bool EnableStreamingQueriesCounters = 219 [default = false];
     optional bool UseYdsTopicStorageMetering = 220 [default = false];
-    }
+    optional bool EnableIndexMaterialization = 221 [default = false];
+}

ydb/core/testlib/basics/feature_flags.h

@@ -88,6 +88,7 @@ class TTestFeatureFlagsHolder {
     FEATURE_FLAG_SETTER(EnableTopicMessageLevelParallelism)
     FEATURE_FLAG_SETTER(EnableTopicAutopartitioningForReplication)
     FEATURE_FLAG_SETTER(EnableAccessToIndexImplTables)
+    FEATURE_FLAG_SETTER(EnableIndexMaterialization)
 
     #undef FEATURE_FLAG_SETTER
 };

ydb/core/tx/datashard/export_s3_uploader.cpp

@@ -6,6 +6,7 @@
 #include "extstorage_usage_config.h"
 
 #include <ydb/core/base/appdata.h>
+#include <ydb/core/base/table_index.h>
 #include <ydb/core/protos/flat_scheme_op.pb.h>
 #include <ydb/library/services/services.pb.h>
 #include <ydb/core/backup/common/checksum.h>
@@ -978,6 +979,36 @@ IActor* TS3Export::CreateUploader(const TActorId& dataShard, ui64 txId) const {
         }
     }
 
+    if (scheme) {
+        int idx = changefeeds.size() + 1;
+        for (const auto& index : scheme->indexes()) {
+            const auto indexType = NTableIndex::ConvertIndexType(index.type_case());
+            const TVector<TString> indexColumns(index.index_columns().begin(), index.index_columns().end());
+            std::optional<Ydb::Table::FulltextIndexSettings::Layout> layout;
+            if (indexType == NKikimrSchemeOp::EIndexTypeGlobalFulltext) {
+                const auto& settings = index.global_fulltext_index().fulltext_settings();
+                layout = settings.has_layout() ? settings.layout() : Ydb::Table::FulltextIndexSettings::LAYOUT_UNSPECIFIED;
+            }
+
+            for (const auto& implTable : NTableIndex::GetImplTables(indexType, indexColumns, layout)) {
+                const TString implTablePrefix = TStringBuilder() << index.name() << "/" << implTable;
+                TString exportPrefix;
+                if (encrypted) {
+                    std::stringstream prefix;
+                    prefix << std::setfill('0') << std::setw(3) << std::right << idx++;
+                    exportPrefix = prefix.str();
+                } else {
+                    exportPrefix = implTablePrefix;
+                }
+
+                metadata.AddIndex(TIndexMetadata{
+                    .ExportPrefix = exportPrefix,
+                    .ImplTablePrefix = implTablePrefix,
+                });
+            }
+        }
+    }
+
     auto permissions = (Task.GetEnablePermissions() && Task.GetShardNum() == 0)
         ? GenYdbPermissions(Task.GetTable())
         : Nothing();

ydb/core/tx/schemeshard/schemeshard__init.cpp

@@ -4457,6 +4457,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
                     exportInfo->EndTime = TInstant::Seconds(rowset.GetValueOrDefault<Schema::Exports::EndTime>());
                     exportInfo->EnableChecksums = rowset.GetValueOrDefault<Schema::Exports::EnableChecksums>(false);
                     exportInfo->EnablePermissions = rowset.GetValueOrDefault<Schema::Exports::EnablePermissions>(false);
+                    exportInfo->MaterializeIndexes = rowset.GetValueOrDefault<Schema::Exports::MaterializeIndexes>(false);
 
                     if (rowset.HaveValue<Schema::Exports::ExportMetadata>()) {
                         exportInfo->ExportMetadata = rowset.GetValue<Schema::Exports::ExportMetadata>();
@@ -4506,6 +4507,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
                     item.SourcePathId.OwnerId = rowset.GetValueOrDefault<Schema::ExportItems::SourceOwnerPathId>(selfId);
                     item.SourcePathId.LocalPathId = rowset.GetValue<Schema::ExportItems::SourcePathId>();
                     item.SourcePathType = rowset.GetValue<Schema::ExportItems::SourcePathType>();
+                    item.ParentIdx = rowset.GetValueOrDefault<Schema::ExportItems::ParentIndex>(Max<ui32>());
 
                     item.State = static_cast<TExportInfo::EState>(rowset.GetValue<Schema::ExportItems::State>());
                     item.WaitTxId = rowset.GetValueOrDefault<Schema::ExportItems::BackupTxId>(InvalidTxId);
@@ -4651,6 +4653,17 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
                     item.Issue = rowset.GetValueOrDefault<Schema::ImportItems::Issue>(TString());
                     item.SrcPrefix = rowset.GetValueOrDefault<Schema::ImportItems::SrcPrefix>(TString());
                     item.SrcPath = rowset.GetValueOrDefault<Schema::ImportItems::SrcPath>(TString());
+
+                    item.ParentIdx = rowset.GetValueOrDefault<Schema::ImportItems::ParentIndex>(Max<ui32>());
+                    if (item.ParentIdx != Max<ui32>()) {
+                        Y_VERIFY_S(item.ParentIdx < importInfo->Items.size(), "Invalid item's index"
+                                   << ": importId# " << importId
+                                   << ", itemIdx# " << item.ParentIdx);
+
+                        TImportInfo::TItem& parentItem = importInfo->Items[item.ParentIdx];
+                        parentItem.ChildItems.push_back(itemIdx);
+                    }
+
                     if (rowset.HaveValue<Schema::ImportItems::EncryptionIV>()) {
                         item.ExportItemIV = NBackup::TEncryptionIV::FromBinaryString(rowset.GetValue<Schema::ImportItems::EncryptionIV>());
                     }

ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h

@@ -578,6 +578,7 @@ class TBackupRestoreOperationBase: public TSubOperation {
 
         const TString& parentPath = Transaction.GetWorkingDir();
         const TString name = TKind::GetTableName(Transaction);
+        const bool internal = Transaction.HasInternal() && Transaction.GetInternal();
 
         LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
                      TKind::Name() << " Propose"
@@ -618,9 +619,13 @@ class TBackupRestoreOperationBase: public TSubOperation {
                 .NotDeleted()
                 .IsTable()
                 .NotAsyncReplicaTable()
-                .NotUnderOperation()
-                .IsCommonSensePath() //forbid alter impl index tables
-                .CanBackupTable(); //forbid backup table with indexes
+                .NotUnderOperation();
+
+            if (!internal) {
+                checks
+                    .IsCommonSensePath() // forbid alter impl index tables
+                    .CanBackupTable(); // forbid backup table with indexes
+            }
 
             if (!checks) {
                 result->SetError(checks.GetStatus(), checks.GetError());

ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp

@@ -52,8 +52,8 @@ static std::optional<NKikimrSchemeOp::TModifyScheme> CreateIndexTask(NKikimr::NS
 
     auto operation = scheme.MutableCreateTableIndex();
     operation->SetName(dst.LeafName());
-
     operation->SetType(indexInfo->Type);
+    operation->SetState(indexInfo->State);
 
     for (const auto& keyName: indexInfo->IndexKeys) {
         *operation->MutableKeyColumnNames()->Add() = keyName;
@@ -269,6 +269,7 @@ bool CreateConsistentCopyTables(
                                        TStringBuilder{} << "Consistent copy table doesn't support table with index type " << indexInfo->Type)};
                 return false;
             }
+            scheme->SetInternal(tx.GetInternal());
             result.push_back(CreateNewTableIndex(NextPartId(nextId, result), *scheme));
 
             for (const auto& [srcImplTableName, srcImplTablePathId] : srcIndexPath.Base()->GetChildren()) {