Skip to content

Commit 3b22f46

Browse files
pixccpixcc
authored
Add restore backup from filesystem in tablet recovery mode (#29602)
1 parent a30e5b9 commit 3b22f46

File tree

10 files changed

+1299
-74
lines changed

10 files changed

+1299
-74
lines changed

‎ydb/core/base/appdata.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include <ydb/core/protos/memory_controller_config.pb.h>
2929
#include <ydb/core/protos/netclassifier.pb.h>
3030
#include <ydb/core/protos/pqconfig.pb.h>
31+
#include <ydb/core/protos/recoveryshard_config.pb.h>
3132
#include <ydb/core/protos/replication.pb.h>
3233
#include <ydb/core/protos/shared_cache.pb.h>
3334
#include <ydb/core/protos/stream.pb.h>
@@ -82,6 +83,7 @@ struct TAppData::TImpl {
8283
NKikimrConfig::TStatisticsConfig StatisticsConfig;
8384
TMetricsConfig MetricsConfig;
8485
NKikimrConfig::TSystemTabletBackupConfig SystemTabletBackupConfig;
86+
NKikimrConfig::TRecoveryShardConfig RecoveryShardConfig;
8587
};
8688

8789
TAppData::TAppData(
@@ -149,6 +151,7 @@ TAppData::TAppData(
149151
, StatisticsConfig(Impl->StatisticsConfig)
150152
, MetricsConfig(Impl->MetricsConfig)
151153
, SystemTabletBackupConfig(Impl->SystemTabletBackupConfig)
154+
, RecoveryShardConfig(Impl->RecoveryShardConfig)
152155
, KikimrShouldContinue(kikimrShouldContinue)
153156
, TracingConfigurator(MakeIntrusive<NJaegerTracing::TSamplingThrottlingConfigurator>(TimeProvider, RandomProvider))
154157
{}

‎ydb/core/base/appdata_fwd.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ namespace NKikimrConfig {
8282
class TBridgeConfig;
8383
class TStatisticsConfig;
8484
class TSystemTabletBackupConfig;
85+
class TRecoveryShardConfig;
8586
}
8687

8788
namespace NKikimrReplication {
@@ -272,6 +273,7 @@ struct TAppData {
272273
NKikimrConfig::TStatisticsConfig& StatisticsConfig;
273274
TMetricsConfig& MetricsConfig;
274275
NKikimrConfig::TSystemTabletBackupConfig& SystemTabletBackupConfig;
276+
NKikimrConfig::TRecoveryShardConfig& RecoveryShardConfig;
275277
bool EnforceUserTokenRequirement = false;
276278
bool EnforceUserTokenCheckRequirement = false; // check token if it was specified
277279
bool AllowHugeKeyValueDeletes = true; // delete when all clients limit deletes per request

‎ydb/core/driver_lib/run/run.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
#include <ydb/core/protos/datashard_config.pb.h>
6565
#include <ydb/core/protos/http_config.pb.h>
6666
#include <ydb/core/protos/node_broker.pb.h>
67+
#include <ydb/core/protos/recoveryshard_config.pb.h>
6768
#include <ydb/core/protos/replication.pb.h>
6869
#include <ydb/core/protos/stream.pb.h>
6970
#include <ydb/core/protos/workload_manager_config.pb.h>
@@ -1561,6 +1562,10 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig)
15611562
AppData->DataIntegrityTrailsConfig = runConfig.AppConfig.GetDataIntegrityTrailsConfig();
15621563
}
15631564

1565+
if (runConfig.AppConfig.HasRecoveryShardConfig()) {
1566+
AppData->RecoveryShardConfig = runConfig.AppConfig.GetRecoveryShardConfig();
1567+
}
1568+
15641569
// setup resource profiles
15651570
AppData->ResourceProfiles = new TResourceProfiles;
15661571
if (runConfig.AppConfig.GetBootstrapConfig().ResourceProfilesSize())

‎ydb/core/protos/config.proto‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import "ydb/core/protos/memory_controller_config.proto";
2828
import "ydb/core/protos/netclassifier.proto";
2929
import "ydb/core/protos/node_broker.proto";
3030
import "ydb/core/protos/pqconfig.proto";
31+
import "ydb/core/protos/recoveryshard_config.proto";
3132
import "ydb/core/protos/replication.proto";
3233
import "ydb/core/protos/resource_broker.proto";
3334
import "ydb/core/protos/shared_cache.proto";
@@ -2965,6 +2966,8 @@ message TAppConfig {
29652966

29662967
// Built-in AWS client options (used, for example, to interact with S3).
29672968
optional TAwsClientConfig AwsClientConfig = 116;
2969+
2970+
optional TRecoveryShardConfig RecoveryShardConfig = 117;
29682971
}
29692972

29702973
message TYdbVersion {

‎ydb/core/protos/recoveryshard_config.proto‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package NKikimrConfig;
2+
option java_package = "ru.yandex.kikimr.proto";
3+
4+
message TRecoveryShardConfig {
5+
optional uint64 RestoreInFlightBytes = 1 [default = 52428800]; // 50 MB
6+
optional uint64 RestoreTxMaxLines = 2 [default = 10000];
7+
optional uint64 RestoreTxMaxRedoBytes = 3 [default = 134217728]; // 128 MB
8+
}

‎ydb/core/protos/ya.make‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ SRCS(
119119
pqconfig.proto
120120
profiler.proto
121121
query_stats.proto
122+
recoveryshard_config.proto
122123
replication.proto
123124
resource_broker.proto
124125
s3_settings.proto

‎ydb/core/tablet_flat/flat_executor_backup.cpp‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include <ydb/library/services/services.pb.h>
1717
#include <yql/essentials/types/binary_json/read.h>
1818

19+
#include <library/cpp/json/json_reader.h>
1920
#include <library/cpp/json/json_writer.h>
2021
#include <library/cpp/protobuf/json/proto2json.h>
2122
#include <library/cpp/protobuf/json/util.h>
@@ -48,6 +49,12 @@ EScanStatus ToScanStatus(EStatus status) {
4849
return EScanStatus::InProgress;
4950
}
5051

52+
void WriteJson(TStringBuf in, NJsonWriter::TBuf& out) {
53+
NJson::TJsonValue value;
54+
Y_ENSURE(NJson::ReadJsonTree(in, &value));
55+
out.WriteJsonValue(&value);
56+
}
57+
5158
void WriteColumnToJson(const TString& columnName, NScheme::TTypeId columnType,
5259
const TCell& columnData, NJsonWriter::TBuf& writer)
5360
{
@@ -91,13 +98,13 @@ void WriteColumnToJson(const TString& columnName, NScheme::TTypeId columnType,
9198
writer.WriteKey(columnName).WriteFloat(columnData.AsValue<float>());
9299
break;
93100
case NScheme::NTypeIds::Date:
94-
writer.WriteKey(columnName).WriteULongLong(columnData.AsValue<ui16>());
101+
writer.WriteKey(columnName).WriteString(TInstant::Days(columnData.AsValue<ui16>()).ToString());
95102
break;
96103
case NScheme::NTypeIds::Datetime:
97-
writer.WriteKey(columnName).WriteULongLong(columnData.AsValue<ui32>());
104+
writer.WriteKey(columnName).WriteString(TInstant::Seconds(columnData.AsValue<ui32>()).ToString());
98105
break;
99106
case NScheme::NTypeIds::Timestamp:
100-
writer.WriteKey(columnName).WriteULongLong(columnData.AsValue<ui64>());
107+
writer.WriteKey(columnName).WriteString(TInstant::MicroSeconds(columnData.AsValue<ui64>()).ToString());
101108
break;
102109
case NScheme::NTypeIds::Interval:
103110
writer.WriteKey(columnName).WriteLongLong(columnData.AsValue<i64>());
@@ -111,11 +118,15 @@ void WriteColumnToJson(const TString& columnName, NScheme::TTypeId columnType,
111118
writer.WriteKey(columnName).WriteLongLong(columnData.AsValue<i64>());
112119
break;
113120
case NScheme::NTypeIds::Utf8:
114-
case NScheme::NTypeIds::Json:
115121
writer.WriteKey(columnName).WriteString(columnData.AsBuf());
116122
break;
123+
case NScheme::NTypeIds::Json:
124+
writer.WriteKey(columnName);
125+
WriteJson(columnData.AsBuf(), writer);
126+
break;
117127
case NScheme::NTypeIds::JsonDocument:
118-
writer.WriteKey(columnName).WriteString(Base64Encode(NBinaryJson::SerializeToJson(columnData.AsBuf())));
128+
writer.WriteKey(columnName);
129+
WriteJson(NBinaryJson::SerializeToJson(columnData.AsBuf()), writer);
119130
break;
120131
case NScheme::NTypeIds::PairUi64Ui64: {
121132
auto pair = columnData.AsValue<std::pair<ui64, ui64>>();
@@ -131,6 +142,7 @@ void WriteColumnToJson(const TString& columnName, NScheme::TTypeId columnType,
131142
writer.WriteKey(columnName).WriteString(actorId.ToString());
132143
break;
133144
}
145+
case NScheme::NTypeIds::String:
134146
default:
135147
writer.WriteKey(columnName).WriteString(Base64Encode(columnData.AsBuf()));
136148
break;

0 commit comments

Comments
 (0)