From ce29c91f9a0054edec8ae9cbade4ce43ad053cbd Mon Sep 17 00:00:00 2001 From: flown4qqqq Date: Tue, 28 Jul 2026 08:43:46 +0000 Subject: [PATCH 01/56] Set not null: KQP (#44067) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 - codecov.yml | 25 ----------- .../client/iam/common/generic_provider.h | 3 +- src/api/protos/ydb_table.proto | 7 +++ src/client/topic/impl/producer.cpp | 44 +------------------ .../client/retry_range/retry_range_ut.cpp | 20 --------- 7 files changed, 12 insertions(+), 91 deletions(-) delete mode 100644 codecov.yml diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 3a09c931bb..b94d11a222 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -fb25fb6453264260a02c6ec49492382fcaf801ce +d91eb90fbe745d4bbc555898d7c52dc4494c17ca diff --git a/CHANGELOG.md b/CHANGELOG.md index f9d4fe04a0..d91bbbb942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,3 @@ -## v3.20.0 - * Added automatic retries for unary methods of table and query clients(ExecuteQuery, ExecuteScript, BulkUpsert, ReadRows). * Implemented native ranges(TRowRange) and iterators over both streaming query results and TResultSet. diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index e4cddb4661..0000000000 --- a/codecov.yml +++ /dev/null @@ -1,25 +0,0 @@ -coverage: - status: - project: - default: - target: auto # do not drop below the current level... - threshold: 1% # ...by more than 1% - patch: - default: - target: 80% # new/changed code in the PR diff must be >= 80% covered - -# Cross-repo uniform metric: every YDB SDK repo defines this same component_id, -# so native-SDK coverage is queryable identically via the Codecov API -# (?component_id=native-sdk), regardless of how each repo tags its uploads. -# -# The SDK proper lives in src/ (implementation) and include/ (public headers). -# The vendored Arcadia libraries (util/, library/, contrib/, third_party/) and -# examples/tests/tools are not part of the native SDK and are pinned out by listing -# only the SDK paths. -component_management: - individual_components: - - component_id: native-sdk - name: Native SDK - paths: - - "src/**" - - "include/**" diff --git a/include/ydb-cpp-sdk/client/iam/common/generic_provider.h b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h index 37d3184e4b..5505ca6814 100644 --- a/include/ydb-cpp-sdk/client/iam/common/generic_provider.h +++ b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h @@ -168,6 +168,7 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } }; auto facility = weakFacility.lock(); + auto self = weakSelf.lock(); try { if (facility) { @@ -177,7 +178,7 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } catch (...) { } - if (auto self = weakSelf.lock()) { + if (self) { std::lock_guard guard(self->Lock_); self->ResetContextImpl(); } diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index b7ced0a0bd..3f870b7768 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -461,6 +461,10 @@ message SetColumnConstraintState { } } +message SetNotNullItem { + optional string column_name = 1; +} + // Description of index building operation message IndexBuildDescription { string path = 1; @@ -1187,6 +1191,9 @@ message AlterTableRequest { MetricsSettings set_metrics_settings = 25; google.protobuf.Empty drop_metrics_settings = 26; } + + // Start set not null for table + repeated SetNotNullItem set_not_null = 27; } message AlterTableResponse { diff --git a/src/client/topic/impl/producer.cpp b/src/client/topic/impl/producer.cpp index d5ffe71f64..34880fe59a 100644 --- a/src/client/topic/impl/producer.cpp +++ b/src/client/topic/impl/producer.cpp @@ -13,39 +13,6 @@ namespace NYdb::inline V3::NTopic { namespace { static constexpr auto PARTITION_KEY_META_KEY = "__partition_key"; -static constexpr size_t DESCRIBE_TOPIC_ATTEMPTS = 3; -static constexpr TDuration DESCRIBE_TOPIC_RETRY_DELAY = TDuration::MilliSeconds(100); - -TDescribeTopicResult DescribeTopicWithRetries( - TTopicClient::TImpl* client, - const std::string& path, - const TDescribeTopicSettings& settings, - const TDbDriverStatePtr& dbDriverState, - const std::string& logPrefix) { - for (size_t attempt = 1; attempt <= DESCRIBE_TOPIC_ATTEMPTS; ++attempt) { - auto result = client->DescribeTopic(path, settings).GetValueSync(); - if (result.IsSuccess() && !result.GetTopicDescription().GetPartitions().empty()) { - return result; - } - - if (attempt == DESCRIBE_TOPIC_ATTEMPTS) { - return result; - } - - TStringBuilder message; - message << logPrefix << "DescribeTopic returned "; - if (result.IsSuccess()) { - message << "no partitions"; - } else { - message << "status " << result.GetStatus(); - } - message << ", retry attempt " << attempt; - LOG_LAZY(dbDriverState->Log, TLOG_DEBUG, message); - Sleep(DESCRIBE_TOPIC_RETRY_DELAY); - } - - Y_UNREACHABLE(); -} } // namespace @@ -1642,16 +1609,12 @@ TProducer::TProducer( } TDescribeTopicSettings describeTopicSettings; - auto topicConfig = DescribeTopicWithRetries(client.get(), settings.Path_, describeTopicSettings, DbDriverState, LogPrefix()); + auto topicConfig = client->DescribeTopic(settings.Path_, describeTopicSettings).GetValueSync(); auto partitions = topicConfig.GetTopicDescription().GetPartitions(); std::sort(partitions.begin(), partitions.end(), [](const auto& a, const auto& b) -> bool { return a.GetPartitionId() < b.GetPartitionId(); }); - if (partitions.empty()) { - ythrow TContractViolation("Topic has no partitions"); - } - auto partitionChooserStrategy = settings.PartitionChooserStrategy_; auto strategy = topicConfig.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(); auto autoPartitioningEnabled = (strategy != EAutoPartitioningStrategy::Disabled && @@ -1695,7 +1658,6 @@ TProducer::TProducer( case TProducerSettings::EPartitionChooserStrategy::Bound: PartitioningKeyHasher = settings.PartitioningKeyHasher_; PartitionChooser = std::make_unique(this); - for (size_t i = 0; i < partitions.size(); ++i) { const auto& partition = partitions[i]; if (i > 0 && !partition.GetFromBound().has_value() && !partition.GetToBound().has_value()) { @@ -2297,9 +2259,7 @@ std::pair TProducer::TBoundPartitionChooser::ChooseP TProducer::THashPartitionChooser::THashPartitionChooser(std::vector&& partitions) : Partitions(std::move(partitions)) { - if (Partitions.empty()) { - ythrow TContractViolation("THashPartitionChooser requires at least one partition"); - } + Y_ABORT_UNLESS(!Partitions.empty(), "THashPartitionChooser requires at least one partition"); } std::pair TProducer::THashPartitionChooser::ChoosePartition(const std::string_view key) { diff --git a/tests/unit/client/retry_range/retry_range_ut.cpp b/tests/unit/client/retry_range/retry_range_ut.cpp index 5062c14e9e..60dd6b8f1c 100644 --- a/tests/unit/client/retry_range/retry_range_ut.cpp +++ b/tests/unit/client/retry_range/retry_range_ut.cpp @@ -158,16 +158,6 @@ struct TTableClientFixture { .SetDatabase("/Root/My/DB")); Client = std::make_unique(*Driver); } - - ~TTableClientFixture() { - Client.reset(); - if (Driver) { - Driver->Stop(true); - } - if (GrpcServer) { - GrpcServer->Shutdown(); - } - } }; struct TQueryClientFixture { @@ -190,16 +180,6 @@ struct TQueryClientFixture { .SetDatabase("/Root/My/DB")); Client = std::make_unique(*Driver); } - - ~TQueryClientFixture() { - Client.reset(); - if (Driver) { - Driver->Stop(true); - } - if (GrpcServer) { - GrpcServer->Shutdown(); - } - } }; } // namespace From ccad027643d11f349b548e073c96840415fb8f0c Mon Sep 17 00:00:00 2001 From: Dmitry Kardymon Date: Tue, 28 Jul 2026 08:43:58 +0000 Subject: [PATCH 02/56] YDB-3355 Set 1M max query text size in yandex query (#42596) --- .github/last_commit.txt | 2 +- src/api/protos/draft/fq.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index b94d11a222..a9e8d7eaff 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -d91eb90fbe745d4bbc555898d7c52dc4494c17ca +cba72777bcd0959867f4dde845b2fd0850f3989f diff --git a/src/api/protos/draft/fq.proto b/src/api/protos/draft/fq.proto index 1864a04d81..20f4878397 100644 --- a/src/api/protos/draft/fq.proto +++ b/src/api/protos/draft/fq.proto @@ -118,7 +118,7 @@ message QueryContent { string name = 2 [(Ydb.length).le = 1024]; Acl acl = 3; Limits limits = 4; - string text = 5 [(Ydb.length).range = {min: 1, max: 102400}]; // The text of the query itself + string text = 5 [(Ydb.length).range = {min: 1, max: 1024000}]; // The text of the query itself bool automatic = 6; // Is used for queries that are created by automatic systems (robots, jdbc driver, ...) string description = 7 [(Ydb.length).le = 10240]; // Description of the query, there can be any text // Specified settings for query's executor From c489fd186255c085aa1c94998791ca9b32a89848 Mon Sep 17 00:00:00 2001 From: Yuriy Kaminskiy Date: Tue, 28 Jul 2026 08:44:07 +0000 Subject: [PATCH 03/56] iam credentials provider: error propagation (#44723) --- .github/last_commit.txt | 2 +- src/client/iam/iam.cpp | 26 ++++++++++- tests/unit/client/iam/http_iam_ut.cpp | 65 ++++++++++++++++++++++++++- 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index a9e8d7eaff..606d735553 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -cba72777bcd0959867f4dde845b2fd0850f3989f +f1440b15b1d7e70b93059b27770545dbf70d7799 diff --git a/src/client/iam/iam.cpp b/src/client/iam/iam.cpp index 9a152b8556..b4a21d2742 100644 --- a/src/client/iam/iam.cpp +++ b/src/client/iam/iam.cpp @@ -28,18 +28,31 @@ class TIAMCredentialsProvider : public ICredentialsProvider { std::string GetAuthInfo() const override { std::string ticket; TInstant nextTicketUpdate; + auto now = TInstant::Now(); + std::optional lastErrorMessage; { std::lock_guard lock(Lock_); + if (LastErrorMessage_.has_value() && now > ExpiresAt_) { + Ticket_.clear(); + } ticket = Ticket_; nextTicketUpdate = NextTicketUpdate_; + lastErrorMessage = LastErrorMessage_; } - if (TInstant::Now() >= nextTicketUpdate) { + if (now >= nextTicketUpdate) { GetTicket(); { std::lock_guard lock(Lock_); + if (LastErrorMessage_.has_value() && now > ExpiresAt_) { + Ticket_.clear(); + } ticket = Ticket_; + lastErrorMessage = LastErrorMessage_; } } + if (ticket.empty() && lastErrorMessage.has_value()) { + throw yexception() << *lastErrorMessage; + } return ticket; } @@ -53,6 +66,8 @@ class TIAMCredentialsProvider : public ICredentialsProvider { mutable std::mutex Lock_; mutable std::string Ticket_; mutable TInstant NextTicketUpdate_; + mutable TInstant ExpiresAt_ = TInstant::Zero(); + mutable std::optional LastErrorMessage_; TDuration RefreshPeriod_; void GetTicket() const { @@ -75,18 +90,22 @@ class TIAMCredentialsProvider : public ICredentialsProvider { const auto now = TInstant::Now(); TInstant nextUpdate; TDuration expiresIn; + TInstant expiresAt = TInstant::Max(); if (auto it = respMap.find("expires_in"); it != respMap.end()) { auto seconds = it->second.GetUInteger(); if (seconds > 0) { expiresIn = TDuration::Seconds(seconds); + expiresAt = now + expiresIn; } } else if (auto it = respMap.find("expiry"); it != respMap.end()) { try { TInstant expiry; if (TInstant::TryParseIso8601(it->second.GetStringSafe(), expiry) && expiry > now) { expiresIn = expiry - now; + expiresAt = expiry; } } catch (...) { + expiresAt = now; } } if (expiresIn > TDuration::Zero()) { @@ -101,8 +120,13 @@ class TIAMCredentialsProvider : public ICredentialsProvider { std::lock_guard lock(Lock_); Ticket_ = std::move(ticket); NextTicketUpdate_ = nextUpdate; + ExpiresAt_ = expiresAt; + LastErrorMessage_.reset(); } } catch (...) { + std::lock_guard lock(Lock_); + NextTicketUpdate_ = TInstant::Now() + std::min(RefreshPeriod_, TDuration::Seconds(10)); + LastErrorMessage_ = CurrentExceptionMessage(); } } }; diff --git a/tests/unit/client/iam/http_iam_ut.cpp b/tests/unit/client/iam/http_iam_ut.cpp index f2ed793fe8..fd2fe0383c 100644 --- a/tests/unit/client/iam/http_iam_ut.cpp +++ b/tests/unit/client/iam/http_iam_ut.cpp @@ -4,6 +4,8 @@ #include +#include + #include #include #include @@ -56,7 +58,68 @@ TEST(IamCredentialsProvider, ServerError) { auto factory = CreateIamCredentialsProviderFactory(params); auto provider = factory->CreateProvider(); - EXPECT_EQ(provider->GetAuthInfo(), ""); + EXPECT_THROW(provider->GetAuthInfo(), yexception); +} + +TEST(IamCredentialsProvider, GracePeriodOnRefreshError) { + TMetadataServer server; + server.SetStrictMode(false); + server.SetResponse(HTTP_OK, MakeTokenResponse("old-token", 3600)); + + TIamHost params = MakeMetadataParams(server.Port); + params.RefreshPeriod = TDuration::MilliSeconds(100); + + auto provider = CreateIamCredentialsProviderFactory(params)->CreateProvider(); + EXPECT_EQ(provider->GetAuthInfo(), "old-token"); + + int countBeforeRefresh = server.GetRequestCount(); + Sleep(TDuration::MilliSeconds(150)); + + server.SetResponse(HTTP_INTERNAL_SERVER_ERROR, ""); + EXPECT_EQ(provider->GetAuthInfo(), "old-token"); + EXPECT_GT(server.GetRequestCount(), countBeforeRefresh); +} + +TEST(IamCredentialsProvider, ThrowAfterTokenExpiredOnRefreshError) { + TMetadataServer server; + server.SetStrictMode(false); + server.SetResponse(HTTP_OK, MakeTokenResponse("old-token", 1)); + + TIamHost params = MakeMetadataParams(server.Port); + params.RefreshPeriod = TDuration::MilliSeconds(100); + + auto provider = CreateIamCredentialsProviderFactory(params)->CreateProvider(); + EXPECT_EQ(provider->GetAuthInfo(), "old-token"); + + Sleep(TDuration::MilliSeconds(150)); + server.SetResponse(HTTP_INTERNAL_SERVER_ERROR, ""); + EXPECT_EQ(provider->GetAuthInfo(), "old-token"); + + Sleep(TDuration::Seconds(1)); + EXPECT_THROW(provider->GetAuthInfo(), yexception); +} + +TEST(IamCredentialsProvider, RecoveryAfterRefreshError) { + TMetadataServer server; + server.SetStrictMode(false); + server.SetResponse(HTTP_OK, MakeTokenResponse("token-1", 3600)); + + TIamHost params = MakeMetadataParams(server.Port); + params.RefreshPeriod = TDuration::MilliSeconds(100); + + auto provider = CreateIamCredentialsProviderFactory(params)->CreateProvider(); + EXPECT_EQ(provider->GetAuthInfo(), "token-1"); + + Sleep(TDuration::MilliSeconds(150)); + server.SetResponse(HTTP_INTERNAL_SERVER_ERROR, ""); + EXPECT_EQ(provider->GetAuthInfo(), "token-1"); + + server.SetResponse(HTTP_OK, MakeTokenResponse("token-2", 3600)); + int countBeforeRecovery = server.GetRequestCount(); + Sleep(TDuration::MilliSeconds(150)); + + EXPECT_EQ(provider->GetAuthInfo(), "token-2"); + EXPECT_GT(server.GetRequestCount(), countBeforeRecovery); } TEST(IamCredentialsProvider, ConcurrentAccess) { From 324e5241d504eba74eca63c86d993acc649ffe86 Mon Sep 17 00:00:00 2001 From: Kuzin Roman Date: Tue, 28 Jul 2026 08:44:18 +0000 Subject: [PATCH 04/56] LOGBROKER-10406 Fixed memory leak in sdk & fixed mirrorer test (#45159) --- .github/last_commit.txt | 2 +- src/client/topic/impl/read_session_impl.h | 14 +- src/client/topic/impl/read_session_impl.ipp | 160 +++++++++++++++----- 3 files changed, 132 insertions(+), 44 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 606d735553..20c5907660 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -f1440b15b1d7e70b93059b27770545dbf70d7799 +e731d2ce12cb8730f160b4c25481804f9f1e2475 diff --git a/src/client/topic/impl/read_session_impl.h b/src/client/topic/impl/read_session_impl.h index 483450d0e3..5ea791a07a 100644 --- a/src/client/topic/impl/read_session_impl.h +++ b/src/client/topic/impl/read_session_impl.h @@ -257,8 +257,9 @@ class TDataDecompressionInfo : public std::enable_shared_from_this& deferred); - void PlanDecompressionTasks(double averageCompressionRatio, - TIntrusivePtr> partitionStream); + bool PlanDecompressionTasks(double averageCompressionRatio, + TIntrusivePtr> partitionStream, + TDeferredActions& deferred); void OnDestroyReadSession(); @@ -628,6 +629,7 @@ class TRawPartitionStreamEventQueue { TReadSessionEventsQueue& queue, TDeferredActions& deferred); void DeleteNotReadyTail(TDeferredActions& deferred); + void Cleanup(TDeferredActions& deferred); void GetDataEventImpl(TIntrusivePtr> partitionStream, size_t& maxEventsCount, @@ -962,6 +964,9 @@ class TReadSessionEventsQueue: public TBaseSessionEventsQueue info(event); @@ -1268,7 +1273,7 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext* deferred = nullptr); void Close(std::function callback); void AbortSession(TASessionClosedEvent&& closeEvent); @@ -1337,6 +1342,7 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext& deferred); // Destroy all streams before setting new connection // Assumes that we're under lock. + void CleanupDecompressionQueueImpl(TDeferredActions& deferred); // Assumes that we're under lock. // Initing. inline void InitImpl(TDeferredActions& deferred); // Assumes that we're under lock. @@ -1378,7 +1384,7 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext* deferred = nullptr); void UpdateMemoryUsageStatisticsImpl(); void UpdateReadSizeBudgetCounter(i64 value); diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index c2a6c38e3b..5160353776 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -267,6 +267,38 @@ void TRawPartitionStreamEventQueue::DeleteNotReadyTail(TDe swap(ready, NotReady); } +template +void TRawPartitionStreamEventQueue::Cleanup(TDeferredActions& deferred) +{ + std::vector> infos; + TUserRetrievedEventsInfoAccumulator accumulator; + + auto cleanupEvent = [&](TRawPartitionStreamEvent& event) { + if (!event.IsDataEvent()) { + return; + } + + auto& dataEvent = event.GetDataEvent(); + if (event.IsReady() || !dataEvent.SetAbandoned()) { + accumulator.Add(dataEvent.GetParent(), dataEvent.GetDataSize(), dataEvent.GetMessageCount()); + } else { + infos.push_back(dataEvent.GetParent()); + } + }; + + for (auto& event : Ready) { + cleanupEvent(event); + } + for (auto& event : NotReady) { + cleanupEvent(event); + } + + deferred.DeferDestroyDecompressionInfos(std::move(infos)); + deferred.DeferOnUserRetrievedEvent(std::move(accumulator)); + Ready.clear(); + NotReady.clear(); +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // TDecompressionQueueItem @@ -462,6 +494,7 @@ void TSingleClusterReadSessionImpl::BreakConnectionAndReco template void TSingleClusterReadSessionImpl::OnConnectTimeout(const NYdbGrpc::IQueueClientContextPtr& connectTimeoutContext) { + TDeferredActions deferred; { std::lock_guard guard(Lock); if (ConnectTimeoutContext == connectTimeoutContext) { @@ -471,7 +504,7 @@ void TSingleClusterReadSessionImpl::OnConnectTimeout(const ConnectDelayContext = nullptr; if (Closing || Aborting) { - CallCloseCallbackImpl(); + CallCloseCallbackImpl(&deferred); return; } } else { @@ -500,7 +533,7 @@ void TSingleClusterReadSessionImpl::OnConnect( ConnectDelayContext = nullptr; if (Closing || Aborting) { - CallCloseCallbackImpl(); + CallCloseCallbackImpl(&deferred); return; } @@ -788,7 +821,7 @@ void TSingleClusterReadSessionImpl::ConfirmPartitionStream deferred); } if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } TClientMessage req; @@ -938,7 +971,7 @@ void TSingleClusterReadSessionImpl::ReadFromProcessorImpl( } if (Closing && !HasCommitsInflightImpl()) { Processor->Cancel(); - CallCloseCallbackImpl(); + CallCloseCallbackImpl(&deferred); return; } @@ -1158,8 +1191,11 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( Settings.Decompress_); Y_ABORT_UNLESS(decompressionInfo); - decompressionInfo->PlanDecompressionTasks(AverageCompressionRatio, - partitionStream); + if (!decompressionInfo->PlanDecompressionTasks(AverageCompressionRatio, + partitionStream, + deferred)) { + return; + } DecompressionQueue.emplace_back(decompressionInfo, partitionStream); StartDecompressionTasksImpl(deferred); @@ -1196,7 +1232,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( currentPartitionStream, NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent::EReason::Lost), deferred); if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } currentPartitionStream = partitionStream; @@ -1208,7 +1244,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent(partitionStream, msg.read_offset(), msg.end_offset()), deferred); if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } } @@ -1242,7 +1278,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( } if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } @@ -1271,7 +1307,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( partitionStream, partitionStream->GetMaxCommittedOffset()), deferred); if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } } @@ -1286,7 +1322,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent(partitionStream, rangeProto.end_offset()), deferred); if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } } @@ -1311,7 +1347,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( msg.end_offset(), TInstant::MilliSeconds(msg.write_watermark_ms())), deferred); if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } } @@ -1408,8 +1444,11 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( serverBytesSize = 0; Y_ABORT_UNLESS(decompressionInfo); - decompressionInfo->PlanDecompressionTasks(AverageCompressionRatio, - partitionStream); + if (!decompressionInfo->PlanDecompressionTasks(AverageCompressionRatio, + partitionStream, + deferred)) { + return; + } DecompressionQueue.emplace_back(decompressionInfo, partitionStream); StartDecompressionTasksImpl(deferred); } @@ -1453,7 +1492,7 @@ inline void TSingleClusterReadSessionImpl::StopPartitionSessionImpl( } if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); } } @@ -1589,7 +1628,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( deferred); if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } } @@ -1615,7 +1654,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( deferred); if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } } @@ -1728,7 +1767,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( TReadSessionEvent::TEndPartitionSessionEvent(std::move(partitionStream), std::move(adjacentPartitionIds), std::move(childPartitionIds)), deferred); if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } } @@ -1752,7 +1791,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( partitionStream, rangeProto.committed_offset()), deferred); if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } } @@ -1780,7 +1819,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( msg.write_time_high_watermark()))), deferred); if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } } @@ -1843,7 +1882,7 @@ void TSingleClusterReadSessionImpl::DestroyAllPartitionStr TClosedEvent(std::move(partitionStream), TClosedEvent::EReason::ConnectionLost), deferred); if (!pushRes) { - AbortImpl(); + AbortImpl(&deferred); return; } } @@ -1851,6 +1890,24 @@ void TSingleClusterReadSessionImpl::DestroyAllPartitionStr CookieMapping.ClearMapping(); } +template +void TSingleClusterReadSessionImpl::CleanupDecompressionQueueImpl(TDeferredActions& deferred) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (DecompressionQueue.empty()) { + return; + } + + std::vector> infos; + infos.reserve(DecompressionQueue.size()); + for (auto& item : DecompressionQueue) { + infos.push_back(item.BatchInfo); + } + DecompressionQueue.clear(); + + deferred.DeferDestroyDecompressionInfos(std::move(infos)); +} + template void TSingleClusterReadSessionImpl::OnCreateNewDecompressionTask() { ++DecompressionTasksInflight; @@ -1948,8 +2005,9 @@ template void TSingleClusterReadSessionImpl::Abort() { LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Abort session to cluster"); + TDeferredActions deferred; std::lock_guard guard(Lock); - AbortImpl(); + AbortImpl(&deferred); } template @@ -1962,12 +2020,12 @@ void TSingleClusterReadSessionImpl::AbortSession(TASession template -void TSingleClusterReadSessionImpl::AbortImpl() { +void TSingleClusterReadSessionImpl::AbortImpl(TDeferredActions* deferred) { Y_ABORT_UNLESS(Lock.IsLocked()); if (!Aborting) { Aborting = true; - CallCloseCallbackImpl(); + CallCloseCallbackImpl(deferred); // Cancel(ClientContext); // Don't cancel, because this is used only as factory for other contexts. Cancel(ConnectContext); @@ -1985,13 +2043,19 @@ void TSingleClusterReadSessionImpl::AbortImpl() { } } } + + if (deferred) { + CleanupDecompressionQueueImpl(*deferred); + } } template void TSingleClusterReadSessionImpl::Close(std::function callback) { + TDeferredActions deferred; std::lock_guard guard(Lock); if (Aborting) { callback(); + return; } if (!Closing) { @@ -2004,27 +2068,31 @@ void TSingleClusterReadSessionImpl::Close(std::functionCancel(); - CallCloseCallbackImpl(); + CallCloseCallbackImpl(&deferred); } } } - AbortImpl(); + AbortImpl(&deferred); } template -void TSingleClusterReadSessionImpl::CallCloseCallbackImpl() { +void TSingleClusterReadSessionImpl::CallCloseCallbackImpl(TDeferredActions* deferred) { Y_ABORT_UNLESS(Lock.IsLocked()); if (CloseCallback) { CloseCallback(); CloseCallback = {}; } - AbortImpl(); + if (!Aborting) { + AbortImpl(deferred); + } else if (deferred) { + CleanupDecompressionQueueImpl(*deferred); + } } template @@ -2875,14 +2943,24 @@ void TReadSessionEventsQueue::GetDataEventCallbackSettings template void TReadSessionEventsQueue::ClearAllEvents() { - std::lock_guard guard(TParent::Mutex); - while (!TParent::Events.empty()) { - auto& event = TParent::Events.front(); - if (event.PartitionStream && event.PartitionStream->HasEvents()) { - event.PartitionStream->PopEvent(); + TDeferredActions deferred; + std::vector> deferredDelete; + { + std::lock_guard guard(TParent::Mutex); + deferredDelete.reserve(TParent::Events.size()); + while (!TParent::Events.empty()) { + auto& event = TParent::Events.front(); + if (!event.IsEmpty()) { + deferredDelete.push_back(event.PartitionStream->ExtractQueue()); + } + TParent::Events.pop(); } - TParent::Events.pop(); } + + for (auto& queue : deferredDelete) { + queue.Cleanup(deferred); + } + deferredDelete.clear(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -3042,8 +3120,9 @@ i64 TDataDecompressionInfo::StartDecompressionTasks( } template -void TDataDecompressionInfo::PlanDecompressionTasks(double averageCompressionRatio, - TIntrusivePtr> partitionStream) { +bool TDataDecompressionInfo::PlanDecompressionTasks(double averageCompressionRatio, + TIntrusivePtr> partitionStream, + TDeferredActions& deferred) { constexpr size_t TASK_LIMIT = 512_KB; Y_ABORT_UNLESS(partitionStream); @@ -3074,8 +3153,9 @@ void TDataDecompressionInfo::PlanDecompressionTasks(double ReadyThresholds.back().Ready, ReadyThresholds.back().Abandoned); if (!pushRes) { - session->AbortImpl(); - return; + deferred.DeferDestroyDecompressionInfos({TDataDecompressionInfo::shared_from_this()}); + session->AbortImpl(&deferred); + return false; } } @@ -3099,6 +3179,8 @@ void TDataDecompressionInfo::PlanDecompressionTasks(double } else { ReadyThresholds.pop_back(); // Revert. } + + return true; } template From 523823f517e5acb80c92b528cb6fc178125b7072 Mon Sep 17 00:00:00 2001 From: Kuzin Roman Date: Tue, 28 Jul 2026 08:44:28 +0000 Subject: [PATCH 05/56] LOGBROKER-10206 Change abort to exception & add describe retries (#45267) --- .github/last_commit.txt | 2 +- src/client/topic/impl/producer.cpp | 44 ++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 20c5907660..d81cb54ce7 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -e731d2ce12cb8730f160b4c25481804f9f1e2475 +7db195bd6c4d429a649c66a018eaedbabc7bd471 diff --git a/src/client/topic/impl/producer.cpp b/src/client/topic/impl/producer.cpp index 34880fe59a..d5ffe71f64 100644 --- a/src/client/topic/impl/producer.cpp +++ b/src/client/topic/impl/producer.cpp @@ -13,6 +13,39 @@ namespace NYdb::inline V3::NTopic { namespace { static constexpr auto PARTITION_KEY_META_KEY = "__partition_key"; +static constexpr size_t DESCRIBE_TOPIC_ATTEMPTS = 3; +static constexpr TDuration DESCRIBE_TOPIC_RETRY_DELAY = TDuration::MilliSeconds(100); + +TDescribeTopicResult DescribeTopicWithRetries( + TTopicClient::TImpl* client, + const std::string& path, + const TDescribeTopicSettings& settings, + const TDbDriverStatePtr& dbDriverState, + const std::string& logPrefix) { + for (size_t attempt = 1; attempt <= DESCRIBE_TOPIC_ATTEMPTS; ++attempt) { + auto result = client->DescribeTopic(path, settings).GetValueSync(); + if (result.IsSuccess() && !result.GetTopicDescription().GetPartitions().empty()) { + return result; + } + + if (attempt == DESCRIBE_TOPIC_ATTEMPTS) { + return result; + } + + TStringBuilder message; + message << logPrefix << "DescribeTopic returned "; + if (result.IsSuccess()) { + message << "no partitions"; + } else { + message << "status " << result.GetStatus(); + } + message << ", retry attempt " << attempt; + LOG_LAZY(dbDriverState->Log, TLOG_DEBUG, message); + Sleep(DESCRIBE_TOPIC_RETRY_DELAY); + } + + Y_UNREACHABLE(); +} } // namespace @@ -1609,12 +1642,16 @@ TProducer::TProducer( } TDescribeTopicSettings describeTopicSettings; - auto topicConfig = client->DescribeTopic(settings.Path_, describeTopicSettings).GetValueSync(); + auto topicConfig = DescribeTopicWithRetries(client.get(), settings.Path_, describeTopicSettings, DbDriverState, LogPrefix()); auto partitions = topicConfig.GetTopicDescription().GetPartitions(); std::sort(partitions.begin(), partitions.end(), [](const auto& a, const auto& b) -> bool { return a.GetPartitionId() < b.GetPartitionId(); }); + if (partitions.empty()) { + ythrow TContractViolation("Topic has no partitions"); + } + auto partitionChooserStrategy = settings.PartitionChooserStrategy_; auto strategy = topicConfig.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(); auto autoPartitioningEnabled = (strategy != EAutoPartitioningStrategy::Disabled && @@ -1658,6 +1695,7 @@ TProducer::TProducer( case TProducerSettings::EPartitionChooserStrategy::Bound: PartitioningKeyHasher = settings.PartitioningKeyHasher_; PartitionChooser = std::make_unique(this); + for (size_t i = 0; i < partitions.size(); ++i) { const auto& partition = partitions[i]; if (i > 0 && !partition.GetFromBound().has_value() && !partition.GetToBound().has_value()) { @@ -2259,7 +2297,9 @@ std::pair TProducer::TBoundPartitionChooser::ChooseP TProducer::THashPartitionChooser::THashPartitionChooser(std::vector&& partitions) : Partitions(std::move(partitions)) { - Y_ABORT_UNLESS(!Partitions.empty(), "THashPartitionChooser requires at least one partition"); + if (Partitions.empty()) { + ythrow TContractViolation("THashPartitionChooser requires at least one partition"); + } } std::pair TProducer::THashPartitionChooser::ChoosePartition(const std::string_view key) { From 7e2be4b2968390d163836133ea2578c4f6d8afa9 Mon Sep 17 00:00:00 2001 From: Kuzin Roman Date: Tue, 28 Jul 2026 08:44:38 +0000 Subject: [PATCH 06/56] fix topic sdk setup (#45282) --- .github/last_commit.txt | 2 +- src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index d81cb54ce7..085b226b25 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -7db195bd6c4d429a649c66a018eaedbabc7bd471 +262ec8a26c3590ca1c71eb7021fcb43570908af2 diff --git a/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp b/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp index 890356e19f..b6d9da190c 100644 --- a/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp +++ b/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp @@ -102,7 +102,8 @@ TTopicSdkTestSetup::TReadResult::TReadResult(TDriver& driver) TTopicSdkTestSetup::TReadResult::~TReadResult() { if (Reader) { - Reader->Close(); + StartPartitionSessionEvents.clear(); + Reader->Close(TDuration::Seconds(5)); Reader.reset(); } } From 4281433da0be71d94edafa818c95337fc01b5abb Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:44:48 +0000 Subject: [PATCH 07/56] create a dependency leak check helper (#45139) --- .github/last_commit.txt | 2 +- allowed_peerdirs.txt | 6 +++ scripts/check_peerdirs.py | 85 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 allowed_peerdirs.txt create mode 100644 scripts/check_peerdirs.py diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 085b226b25..2afc0fef86 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -262ec8a26c3590ca1c71eb7021fcb43570908af2 +e844854ba6d72bbdc3af36e45ede13f2e0de0edd diff --git a/allowed_peerdirs.txt b/allowed_peerdirs.txt new file mode 100644 index 0000000000..49259c99a7 --- /dev/null +++ b/allowed_peerdirs.txt @@ -0,0 +1,6 @@ +# Mirrors ydb-cpp-sdk/.github/scripts/copy_sources.sh. +ydb/public/sdk/cpp/ +ydb/public/api/ +library/cpp/ +contrib/ +util/ diff --git a/scripts/check_peerdirs.py b/scripts/check_peerdirs.py new file mode 100644 index 0000000000..0ec91d0b4c --- /dev/null +++ b/scripts/check_peerdirs.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +SDK = Path(__file__).resolve().parent.parent +PREFIX = Path("ydb/public/sdk/cpp") +ALLOWLIST = SDK / "allowed_peerdirs.txt" +ALLOWED = [ + line.split("#", 1)[0].strip() + for line in ALLOWLIST.read_text(encoding="utf-8").splitlines() + if line.split("#", 1)[0].strip() +] +IN_SCOPE = {"src", "include", "plugins", "examples"} + + +def ok(rel): + parts = Path(rel).parts + return ( + parts + and parts[0] in IN_SCOPE + and "ut" not in parts + and "ut_utils" not in parts + ) + + +def peerdirs(text): + in_block = False + for i, line in enumerate(text.splitlines(), 1): + s = line.split("#", 1)[0].strip() + if s.startswith("PEERDIR("): + if s.endswith(")"): + dep = s[len("PEERDIR("):-1].strip() + if dep: + yield i, dep + else: + in_block = True + continue + if in_block: + if s == ")": + in_block = False + elif s: + yield i, s + + +def targets(changed_path): + all_targets = [ + p for p in sorted(SDK.rglob("ya.make")) if ok(p.relative_to(SDK)) + ] + if not changed_path: + return all_targets + changed = [ + Path(line.strip()) + for line in Path(changed_path).read_text(encoding="utf-8").splitlines() + if line.strip() + ] + if PREFIX / "allowed_peerdirs.txt" in changed or PREFIX / "scripts/check_peerdirs.py" in changed: + return all_targets + out = [] + for path in changed: + try: + rel = path.relative_to(PREFIX) + except ValueError: + continue + if path.name != "ya.make" or not ok(rel): + continue + out.append(SDK / rel) + return out + + +def main(): + files = targets(sys.argv[1] if len(sys.argv) > 1 else None) + bad = [] + for ya_make in files: + for line_no, dep in peerdirs(ya_make.read_text(encoding="utf-8")): + if not any(dep.startswith(p) for p in ALLOWED): + rel = PREFIX / ya_make.relative_to(SDK) + bad.append(f"{rel}:{line_no}: forbidden PEERDIR {dep!r}") + if bad: + print("\n".join(bad), file=sys.stderr) + raise SystemExit(1) + print(f"ok ({len(files)} ya.make)") + + +if __name__ == "__main__": + main() From 9a110eaea9805bb26ff7f03c5e4b90e0a0b9fd24 Mon Sep 17 00:00:00 2001 From: flown4qqqq Date: Tue, 28 Jul 2026 08:44:58 +0000 Subject: [PATCH 08/56] Rename SetColumnConstraint in public api (#45409) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_table.proto | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 2afc0fef86..03c0019565 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -e844854ba6d72bbdc3af36e45ede13f2e0de0edd +1bd70ffd044b6d2c8854668b083e0145a393e530 diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index 3f870b7768..b63b573db2 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -449,8 +449,8 @@ message IndexBuildState { } } -// State of set column constraint operation -message SetColumnConstraintState { +// State of set not null operation +message SetNotNullState { enum State { STATE_UNSPECIFIED = 0; STATE_PREPARING = 1; From 7f75422a34de0e8b40834536f19b7a6774220150 Mon Sep 17 00:00:00 2001 From: stanislav_shchetinin Date: Tue, 28 Jul 2026 08:45:07 +0000 Subject: [PATCH 09/56] KIKIMR-25851: Support roles in Maintenance ListNodes Response (#44342) --- .github/last_commit.txt | 2 +- src/api/protos/draft/ydb_maintenance.proto | 29 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 03c0019565..bda4a6e2cf 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -1bd70ffd044b6d2c8854668b083e0145a393e530 +e295d9202c47b69c26da4f1ec423fc00b11fc1d6 diff --git a/src/api/protos/draft/ydb_maintenance.proto b/src/api/protos/draft/ydb_maintenance.proto index 3b629b7705..be2f73550f 100644 --- a/src/api/protos/draft/ydb_maintenance.proto +++ b/src/api/protos/draft/ydb_maintenance.proto @@ -24,6 +24,28 @@ enum ItemState { ITEM_STATE_DOWN = 3; } +// NodeRole is an additional cluster responsibility a node carries on top of its +// type, hosting a fault-tolerance-sensitive subsystem that maintenance must respect. +message NodeRole { + // Node hosts a state storage replica. + message StateStorage { + } + + // Node hosts VDisks of a static blobstorage group. + message StaticGroup { + } + + // Node is configured to host system tablets. + message SystemTablet { + } + + oneof role { + StateStorage state_storage = 1; + StaticGroup static_group = 2; + SystemTablet system_tablet = 3; + } +} + message Node { message StorageNode { } @@ -37,6 +59,10 @@ message Node { uint32 port = 3; Ydb.Discovery.NodeLocation location = 4; ItemState state = 5; + // Type is the node's base kind: + // - a static storage node; + // - or a dynamic (compute) node serving a tenant. + // Exactly one is set. oneof type { StorageNode storage = 6; DynamicNode dynamic = 7; @@ -46,6 +72,9 @@ message Node { // version defines YDB version for current Node. // For example, 'ydb-stable-24-1'. string version = 9; + // roles lists extra cluster responsibilities the node carries beyond its type; + // empty when the node hosts none of them. + repeated NodeRole roles = 10; } message ListClusterNodesRequest { From 2b5805a7ec346dfe03bdf064669277948463acd8 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:45:17 +0000 Subject: [PATCH 10/56] fix: too early provider lock can keep it alive on grpc thread (#45164) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/iam/common/generic_provider.h | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index bda4a6e2cf..c6857135f7 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -e295d9202c47b69c26da4f1ec423fc00b11fc1d6 +d309a896f261f6d493a15b878efd455d5852e360 diff --git a/include/ydb-cpp-sdk/client/iam/common/generic_provider.h b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h index 5505ca6814..37d3184e4b 100644 --- a/include/ydb-cpp-sdk/client/iam/common/generic_provider.h +++ b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h @@ -168,7 +168,6 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } }; auto facility = weakFacility.lock(); - auto self = weakSelf.lock(); try { if (facility) { @@ -178,7 +177,7 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } catch (...) { } - if (self) { + if (auto self = weakSelf.lock()) { std::lock_guard guard(self->Lock_); self->ResetContextImpl(); } From 96c5c88966f5bd29e09ac821cc723d38a0b9d7c0 Mon Sep 17 00:00:00 2001 From: Kuzin Roman Date: Tue, 28 Jul 2026 08:45:27 +0000 Subject: [PATCH 11/56] LOGBROKER-10406 Add batch logic to compactification topics (#45458) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/topic/codecs.h | 2 ++ src/client/topic/codecs/codecs.cpp | 4 ++++ src/client/topic/impl/write_session_impl.cpp | 14 ++++++++++++++ src/client/topic/impl/write_session_impl.h | 3 +++ 5 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index c6857135f7..f1eab915ad 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -d309a896f261f6d493a15b878efd455d5852e360 +fb4f64231fd3a56f2b97522d9311a75387469de2 diff --git a/include/ydb-cpp-sdk/client/topic/codecs.h b/include/ydb-cpp-sdk/client/topic/codecs.h index 00c4e8a77f..2fc66a8f2b 100644 --- a/include/ydb-cpp-sdk/client/topic/codecs.h +++ b/include/ydb-cpp-sdk/client/topic/codecs.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -43,6 +44,7 @@ struct TWriteBlockCompression { ECodec Codec = ECodec::RAW; std::vector& Payloads; const std::vector& CreatedAt; + const std::vector>& MessageKeys; TBuffer& Data; ui32& CodecID; bool& Compressed; diff --git a/src/client/topic/codecs/codecs.cpp b/src/client/topic/codecs/codecs.cpp index b8a1c21e56..f14746b746 100644 --- a/src/client/topic/codecs/codecs.cpp +++ b/src/client/topic/codecs/codecs.cpp @@ -172,6 +172,7 @@ void TKafkaBatchCodec::CompressWriteBlock(TWriteBlockCompression& ctx) const { using namespace NKafka; Y_ABORT_UNLESS(ctx.Payloads.size() == ctx.CreatedAt.size()); + Y_ABORT_UNLESS(ctx.Payloads.size() == ctx.MessageKeys.size()); Y_ABORT_UNLESS(!ctx.Payloads.empty()); TKafkaRecordBatch kafkaBatch; @@ -192,6 +193,9 @@ void TKafkaBatchCodec::CompressWriteBlock(TWriteBlockCompression& ctx) const { record.OffsetDelta = static_cast(i); record.TimestampDelta = ctx.CreatedAt[i].MilliSeconds() - baseTimestamp; kafkaBatch.MaxTimestamp = Max(kafkaBatch.MaxTimestamp, static_cast(ctx.CreatedAt[i].MilliSeconds())); + if (ctx.MessageKeys[i]) { + record.SetKey(TString{ctx.MessageKeys[i]->data(), ctx.MessageKeys[i]->size()}); + } record.Value = TKafkaRawBytes(ctx.Payloads[i].data(), ctx.Payloads[i].size()); record.Length = record.Size(2) - NKafka::NPrivate::SizeOfVarint(0); kafkaBatch.Records.push_back(std::move(record)); diff --git a/src/client/topic/impl/write_session_impl.cpp b/src/client/topic/impl/write_session_impl.cpp index 0e737dea2a..8a7ff36fce 100644 --- a/src/client/topic/impl/write_session_impl.cpp +++ b/src/client/topic/impl/write_session_impl.cpp @@ -30,6 +30,18 @@ namespace { using TTxId = std::pair; +constexpr std::string_view MESSAGE_ATTRIBUTE_KEY = "__key"; + +std::optional GetMessageKey(const std::vector>& messageMeta) { + for (const auto& [key, value] : messageMeta) { + if (key == MESSAGE_ATTRIBUTE_KEY) { + return value; + } + } + + return std::nullopt; +} + bool ValidateWriteSessionSettings(const TWriteSessionSettings& settings, NYdb::NIssue::TIssues& issues) { if (!settings.BatchInnerCodec_.has_value()) { return true; @@ -1346,6 +1358,7 @@ void TWriteSessionImpl::CompressImpl(TBlock&& block_) { .Codec = codec, .Payloads = blockPtr->OriginalDataRefs, .CreatedAt = blockPtr->CreatedAt, + .MessageKeys = blockPtr->MessageKeys, .Data = blockPtr->Data, .CodecID = blockPtr->CodecID, .Compressed = blockPtr->Compressed, @@ -1508,6 +1521,7 @@ size_t TWriteSessionImpl::WriteBatchImpl() { block.OriginalMemoryUsage += datum.size(); block.OriginalDataRefs.emplace_back(datum); block.CreatedAt.emplace_back(createTs); + block.MessageKeys.emplace_back(GetMessageKey(currMessage.MessageMeta)); if (CurrentBatch.Messages[i].Codec.has_value()) { Y_ABORT_UNLESS(CurrentBatch.Messages.size() == 1); block.CodecID = static_cast(*currMessage.Codec); diff --git a/src/client/topic/impl/write_session_impl.h b/src/client/topic/impl/write_session_impl.h index 313352ee44..ff32dcf188 100644 --- a/src/client/topic/impl/write_session_impl.h +++ b/src/client/topic/impl/write_session_impl.h @@ -234,6 +234,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, ui32 CodecID = static_cast(ECodec::RAW); mutable std::vector OriginalDataRefs; mutable std::vector CreatedAt; + mutable std::vector> MessageKeys; mutable TBuffer Data; bool Compressed = false; mutable bool Valid = true; @@ -252,12 +253,14 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, CodecID = rhs.CodecID; OriginalDataRefs.swap(rhs.OriginalDataRefs); CreatedAt.swap(rhs.CreatedAt); + MessageKeys.swap(rhs.MessageKeys); Data.Swap(rhs.Data); Compressed = rhs.Compressed; rhs.Data.Clear(); rhs.OriginalDataRefs.clear(); rhs.CreatedAt.clear(); + rhs.MessageKeys.clear(); } }; From 0a7eaafa6f97555b7e2ee65dea9df42836048447 Mon Sep 17 00:00:00 2001 From: Yuriy Kaminskiy Date: Tue, 28 Jul 2026 08:45:37 +0000 Subject: [PATCH 12/56] fix locking of TDbDriverState (#45069) --- .github/last_commit.txt | 2 +- .../impl/internal/db_driver_state/state.cpp | 82 +++++++++++-------- .../impl/internal/db_driver_state/state.h | 1 + 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index f1eab915ad..fc84889697 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -fb4f64231fd3a56f2b97522d9311a75387469de2 +02aad105a6d192c1932b48c0ca3e112eaca19d9e diff --git a/src/client/impl/internal/db_driver_state/state.cpp b/src/client/impl/internal/db_driver_state/state.cpp index 2c87fd7195..2f1f51137f 100644 --- a/src/client/impl/internal/db_driver_state/state.cpp +++ b/src/client/impl/internal/db_driver_state/state.cpp @@ -189,53 +189,67 @@ TDbDriverStatePtr TDbDriverStateTracker::GetDriverState( } } TDbDriverStatePtr strongState; - for (;;) { + { std::unique_lock lock(Lock_); - { + Notify_.wait(lock, [&]() { auto state = States_.find(key); - if (state != States_.end()) { - auto strong = state->second.lock(); - if (strong) { - return strong; - } else { - // We could find state record, but couldn't promote weak to shared - // this means weak ptr already expired but dtor hasn't been - // called yet. Likely other thread now is waiting on mutex to - // remove expired record from hashmap. So give him chance - // to do it after that we will be able to create new state - lock.unlock(); - std::this_thread::yield(); - continue; - } + if (state == States_.end()) { + return true; + } + strongState = state->second.lock(); + if (strongState) { + return true; } + return false; + }); + if (strongState) { + return strongState; } { auto deleter = [this, key](TDbDriverState* p) { { std::unique_lock lock(Lock_); States_.erase(key); + Notify_.notify_all(); } delete p; }; - strongState = std::shared_ptr( - new TDbDriverState( - quotedDatabase, - discoveryEndpoint, - discoveryMode, - sslCredentials, - DiscoveryClient_), - deleter); - - strongState->SetCredentialsProvider( - credentialsProviderFactory - ? credentialsProviderFactory->CreateProvider(strongState) - : CreateInsecureCredentialsProviderFactory()->CreateProvider(strongState)); - - if (discoveryMode != EDiscoveryMode::Off) { - DiscoveryClient_->AddPeriodicTask(CreatePeriodicDiscoveryTask(strongState), DISCOVERY_RECHECK_PERIOD); + + auto [it, inserted] = States_.try_emplace(key); // creates empty weak_ptr + auto& weakState = it->second; + lock.unlock(); // temporarily release lock + + try { + Y_ABORT_UNLESS(inserted); + strongState = std::shared_ptr( + new TDbDriverState( + quotedDatabase, + discoveryEndpoint, + discoveryMode, + sslCredentials, + DiscoveryClient_), + deleter); + + strongState->SetCredentialsProvider( + credentialsProviderFactory + ? credentialsProviderFactory->CreateProvider(strongState) + : CreateInsecureCredentialsProviderFactory()->CreateProvider(strongState)); + + if (discoveryMode != EDiscoveryMode::Off) { + DiscoveryClient_->AddPeriodicTask(CreatePeriodicDiscoveryTask(strongState), DISCOVERY_RECHECK_PERIOD); + } + } catch (...) { + lock.lock(); + Y_ABORT_UNLESS(weakState.expired()); + Y_ABORT_UNLESS(States_.erase(key)); + Notify_.notify_all(); + throw; } - Y_ABORT_UNLESS(States_.emplace(key, strongState).second); - break; + + lock.lock(); // re-acquire lock + Y_ABORT_UNLESS(weakState.expired()); + weakState = strongState; // reference remains valid + Notify_.notify_all(); } } diff --git a/src/client/impl/internal/db_driver_state/state.h b/src/client/impl/internal/db_driver_state/state.h index c975aaaf31..3dbf6eaaab 100644 --- a/src/client/impl/internal/db_driver_state/state.h +++ b/src/client/impl/internal/db_driver_state/state.h @@ -112,6 +112,7 @@ class TDbDriverStateTracker { IInternalClient* DiscoveryClient_; std::unordered_map, TStateKeyHash> States_; std::shared_mutex Lock_; + std::condition_variable_any Notify_; }; using TDbDriverStatePtr = TDbDriverState::TPtr; From 20be3ba0c43b5baf24ae244d676c447c42edb85a Mon Sep 17 00:00:00 2001 From: Yuriy Kaminskiy Date: Tue, 28 Jul 2026 08:45:47 +0000 Subject: [PATCH 13/56] implement GetClientIdentity for several CredentialProviders (#45173) --- .github/last_commit.txt | 2 +- src/client/iam/iam.cpp | 6 ++++++ src/client/iam_private/common/iam.h | 12 ++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index fc84889697..04ccdae31a 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -02aad105a6d192c1932b48c0ca3e112eaca19d9e +308bd8315eb1eb26523401ef5924e54743f8d3f4 diff --git a/src/client/iam/iam.cpp b/src/client/iam/iam.cpp index b4a21d2742..dd7c55fa9f 100644 --- a/src/client/iam/iam.cpp +++ b/src/client/iam/iam.cpp @@ -139,6 +139,12 @@ class TIamCredentialsProviderFactory : public ICredentialsProviderFactory { return std::make_shared(Params_); } + std::string GetClientIdentity() const final { + return TStringBuilder() << + "TIamCredentialsProviderFactory" << '\t' << + Params_.Host << ':' << Params_.Port << '@' << Params_.RefreshPeriod; + } + private: TIamHost Params_; }; diff --git a/src/client/iam_private/common/iam.h b/src/client/iam_private/common/iam.h index 21e8354abc..4f03c9df32 100644 --- a/src/client/iam_private/common/iam.h +++ b/src/client/iam_private/common/iam.h @@ -71,6 +71,18 @@ class TIamServiceCredentialsProviderFactory : public ICredentialsProviderFactory return std::make_shared(Params_, std::move(facility)); } + std::string GetClientIdentity() const override final { + return TStringBuilder() + << "TIamServiceCredentialsProviderFactory" + << '\t' << Params_.ServiceId + << '\t' << Params_.MicroserviceId + << '\t' << Params_.ResourceId + << '\t' << Params_.ResourceType + << '\t' << Params_.TargetServiceAccountId + << '\t' << Params_.SystemServiceAccountCredentials->GetClientIdentity() + ; + } + private: TIamServiceParams Params_; }; From d51a9cbfef97084d602c1238ec18f50771a7afd4 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:45:57 +0000 Subject: [PATCH 14/56] added deferred session creation support to the query client (#45019) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 4 + include/ydb-cpp-sdk/client/query/client.h | 4 + src/client/query/client.cpp | 23 ++- .../query/deferred_session_creation_ut.cpp | 143 ++++++++++++++++++ 5 files changed, 172 insertions(+), 4 deletions(-) create mode 100644 tests/unit/client/query/deferred_session_creation_ut.cpp diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 04ccdae31a..6219011f49 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -308bd8315eb1eb26523401ef5924e54743f8d3f4 +190c9f8e21a566743e27cdea63d323a6853cc170 diff --git a/CHANGELOG.md b/CHANGELOG.md index d91bbbb942..8f9b29d8f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +* Added a flag to support deferred session creation(when client timeout exceeded, the session is created in the backgroud) + +# v3.20.0 + * Added automatic retries for unary methods of table and query clients(ExecuteQuery, ExecuteScript, BulkUpsert, ReadRows). * Implemented native ranges(TRowRange) and iterators over both streaming query results and TResultSet. diff --git a/include/ydb-cpp-sdk/client/query/client.h b/include/ydb-cpp-sdk/client/query/client.h index 3b684dfb53..5bb3224a65 100644 --- a/include/ydb-cpp-sdk/client/query/client.h +++ b/include/ydb-cpp-sdk/client/query/client.h @@ -51,6 +51,10 @@ struct TSessionPoolSettings { // Min number of session in session pool. // Sessions will not be closed by CloseIdleThreshold if the number of sessions less then this limit. FLUENT_SETTING_DEFAULT(uint32_t, MinPoolSize, 10); + + // Create session in the background even after client timeout. + // This is useful for applications with short session timeouts. + FLUENT_SETTING_DEFAULT(bool, UseDeferredSessionCreation, false); }; struct TClientSettings : public TCommonClientSettingsBase { diff --git a/src/client/query/client.cpp b/src/client/query/client.cpp index 4ff31745aa..19723562ee 100644 --- a/src/client/query/client.cpp +++ b/src/client/query/client.cpp @@ -523,15 +523,32 @@ class TQueryClient::TImpl: public TClientImplCommon, public } void ReplyNewSession() override { - Client->CreateAttachedSession(RpcSettings).Subscribe( - [promise{std::move(Promise)}, obs = Observation](TAsyncCreateSessionResult future) mutable + TRpcRequestSettings deferredRpcSettings = RpcSettings; + deferredRpcSettings.Deadline = TDeadline::Max(); + Client->CreateAttachedSession( + this->Client->Settings_.SessionPoolSettings_.UseDeferredSessionCreation_ ? + deferredRpcSettings : + RpcSettings).Subscribe( + [promise = Promise, obs = Observation](TAsyncCreateSessionResult future) mutable { auto val = future.ExtractValue(); if (obs) { obs->End(val.GetStatus(), val.GetEndpoint()); } - promise.SetValue(std::move(val)); + promise.TrySetValue(std::move(val)); }); + if (Client->Settings_.SessionPoolSettings_.UseDeferredSessionCreation_) { + Client->Connections_->ScheduleDelayedTask( + [promise = Promise, obs = Observation, client = Client]() mutable { + TSession session; + promise.TrySetValue(TCreateSessionResult(TStatus(TPlainStatus(EStatus::CLIENT_DEADLINE_EXCEEDED, "GetSession deadline exceeded")), std::move(session))); + if (obs) { + obs->End(EStatus::CLIENT_DEADLINE_EXCEEDED, "GetSession deadline exceeded"); + } + }, + GetDeadline() + ); + } } void ScheduleOnDeadlineWaiterCleanup() override { diff --git a/tests/unit/client/query/deferred_session_creation_ut.cpp b/tests/unit/client/query/deferred_session_creation_ut.cpp new file mode 100644 index 0000000000..42c2cb8ed7 --- /dev/null +++ b/tests/unit/client/query/deferred_session_creation_ut.cpp @@ -0,0 +1,143 @@ +#include +#include + +#include +#include + +#include + +#include + +#include +#include +#include + +#include + +using namespace NYdb; +using namespace NYdb::NQuery; + +namespace { + +constexpr TDuration kShortDeadline = TDuration::MilliSeconds(50); +constexpr TDuration kSlowAttach = TDuration::MilliSeconds(300); + +class TDelayedMockQueryService : public Ydb::Query::V1::QueryService::Service { +public: + TDuration AttachDelay = TDuration::Zero(); + + grpc::Status CreateSession( + grpc::ServerContext*, + const Ydb::Query::CreateSessionRequest*, + Ydb::Query::CreateSessionResponse* response) override + { + response->set_status(Ydb::StatusIds::SUCCESS); + response->set_session_id("fake-query-session-id"); + response->set_node_id(1); + return grpc::Status::OK; + } + + grpc::Status AttachSession( + grpc::ServerContext* context, + const Ydb::Query::AttachSessionRequest*, + grpc::ServerWriter* writer) override + { + if (AttachDelay != TDuration::Zero()) { + std::this_thread::sleep_for(std::chrono::milliseconds(AttachDelay.MilliSeconds())); + } + Ydb::Query::SessionState state; + state.set_status(Ydb::StatusIds::SUCCESS); + writer->Write(state); + while (!context->IsCancelled()) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + return grpc::Status::OK; + } +}; + +template +std::unique_ptr StartGrpcServer(const std::string& address, TService& service) { + return grpc::ServerBuilder() + .AddListeningPort(TString{address}, grpc::InsecureServerCredentials()) + .RegisterService(&service) + .BuildAndStart(); +} + +TCreateSessionSettings ShortDeadlineSettings() { + return TCreateSessionSettings() + .ClientTimeout(kShortDeadline) + .Deadline(TDeadline::AfterDuration(kShortDeadline)); +} + +std::unique_ptr MakeClient(TDriver& driver, bool deferred) { + return std::make_unique( + driver, + TClientSettings().SessionPoolSettings( + TSessionPoolSettings().UseDeferredSessionCreation(deferred))); +} + +} // namespace + +Y_UNIT_TEST_SUITE(DeferredGetSession) { + +Y_UNIT_TEST(TimeoutThenPoolWarmup) { + NTesting::InitPortManagerFromEnv(); + const auto port = NTesting::GetFreePort(); + const auto endpoint = TStringBuilder() << "127.0.0.1:" << port; + + TDelayedMockQueryService service; + service.AttachDelay = kSlowAttach; + auto server = StartGrpcServer(endpoint, service); + + TDriver driver( + TDriverConfig() + .SetEndpoint(endpoint) + .SetDiscoveryMode(EDiscoveryMode::Off) + .SetDatabase("/Root/My/DB")); + auto client = MakeClient(driver, /*deferred=*/true); + + const auto result = client->GetSession(ShortDeadlineSettings()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::CLIENT_DEADLINE_EXCEEDED); + + for (int i = 0; i < 40; ++i) { + if (client->GetCurrentPoolSize() == 1 && client->GetActiveSessionCount() == 0) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + UNIT_ASSERT_EQUAL(client->GetCurrentPoolSize(), 1); + UNIT_ASSERT_EQUAL(client->GetActiveSessionCount(), 0); + + client.reset(); + driver.Stop(true); +} + +Y_UNIT_TEST(DisabledWaitsForAttach) { + NTesting::InitPortManagerFromEnv(); + const auto port = NTesting::GetFreePort(); + const auto endpoint = TStringBuilder() << "127.0.0.1:" << port; + + TDelayedMockQueryService service; + service.AttachDelay = kSlowAttach; + auto server = StartGrpcServer(endpoint, service); + + TDriver driver( + TDriverConfig() + .SetEndpoint(endpoint) + .SetDiscoveryMode(EDiscoveryMode::Off) + .SetDatabase("/Root/My/DB")); + auto client = MakeClient(driver, /*deferred=*/false); + + const auto started = TInstant::Now(); + const auto result = client->GetSession(ShortDeadlineSettings()).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT(!result.GetSession().GetId().empty()); + UNIT_ASSERT_GE(TInstant::Now() - started, kSlowAttach); + UNIT_ASSERT_EQUAL(client->GetActiveSessionCount(), 1); + + client.reset(); + driver.Stop(true); +} + +} From 3b8b99f87cea3fc19a376bd5fc218a2cdcc03235 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:46:07 +0000 Subject: [PATCH 15/56] destroy the driver on a detached thread (#45273) --- .github/last_commit.txt | 2 +- src/client/driver/driver.cpp | 16 +++++++++++-- src/library/grpc/client/grpc_client_low.cpp | 25 +++++++++++++++++++++ src/library/grpc/client/grpc_client_low.h | 2 ++ 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 6219011f49..9a69c30e54 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -190c9f8e21a566743e27cdea63d323a6853cc170 +3a30c980ed2168ca0b04f5264eeb992fecd6c73f diff --git a/src/client/driver/driver.cpp b/src/client/driver/driver.cpp index 4dfdb88477..a999266eef 100644 --- a/src/client/driver/driver.cpp +++ b/src/client/driver/driver.cpp @@ -13,7 +13,7 @@ #include #include #include - +#include namespace NYdb::inline V3 { using NYdbGrpc::TGRpcClientLow; @@ -23,6 +23,7 @@ using NYdbGrpc::TGRpcClientConfig; using NYdbGrpc::TResponseCallback; using NYdbGrpc::TGrpcStatus; using NYdbGrpc::TTcpKeepAliveSettings; +using NYdbGrpc::IsGRpcCompletionThread; using Ydb::StatusIds; @@ -361,7 +362,18 @@ TDriver::TDriver(const TDriverConfig& config) { ythrow yexception() << "Invalid config object"; } - Impl_.reset(new TGRpcConnectionsImpl(config.Impl_)); + Impl_ = std::shared_ptr(new TGRpcConnectionsImpl(config.Impl_), [] (TGRpcConnectionsImpl* impl) { + auto destroyImpl = [impl] { + impl->Stop(true); + delete impl; + }; + + if (IsGRpcCompletionThread()) { + std::thread(std::move(destroyImpl)).detach(); + } else { + destroyImpl(); + } + }); } void TDriver::Stop(bool wait) { diff --git a/src/library/grpc/client/grpc_client_low.cpp b/src/library/grpc/client/grpc_client_low.cpp index a54553e211..30d7f29af4 100644 --- a/src/library/grpc/client/grpc_client_low.cpp +++ b/src/library/grpc/client/grpc_client_low.cpp @@ -216,7 +216,32 @@ void TChannelPool::EraseFromQueueByTime(const TInstant& lastUseTime, const std:: LastUsedQueue_.erase(pos); } +namespace { + +thread_local bool IsGrpcWorkerThread = false; + +class TGrpcWorkerThreadGuard { +public: + TGrpcWorkerThreadGuard() { + IsGrpcWorkerThread = true; + } + + ~TGrpcWorkerThreadGuard() { + IsGrpcWorkerThread = PreviousValue_; + } + +private: + const bool PreviousValue_ = IsGrpcWorkerThread; +}; + +} // namespace + +bool IsGRpcCompletionThread() { + return IsGrpcWorkerThread; +} + static void PullEvents(grpc::CompletionQueue* cq) { + TGrpcWorkerThreadGuard guard; TThread::SetCurrentThreadName("grpc_client"); while (true) { void* tag; diff --git a/src/library/grpc/client/grpc_client_low.h b/src/library/grpc/client/grpc_client_low.h index fb0ae5b55a..07fb200c5f 100644 --- a/src/library/grpc/client/grpc_client_low.h +++ b/src/library/grpc/client/grpc_client_low.h @@ -34,6 +34,8 @@ const size_t DEFAULT_NUM_THREADS = 2; void EnableGRpcTracing(); +bool IsGRpcCompletionThread(); + //////////////////////////////////////////////////////////////////////////////// struct TTcpKeepAliveSettings { From 3f635ec975849d76ed147f58cc1b412a296e031c Mon Sep 17 00:00:00 2001 From: Maksim Zinal Date: Tue, 28 Jul 2026 08:46:16 +0000 Subject: [PATCH 16/56] sdk: Move first query result chunk into buffer (#39472) --- .github/last_commit.txt | 2 +- src/client/query/impl/exec_query.cpp | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 9a69c30e54..881f61a03b 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -3a30c980ed2168ca0b04f5264eeb992fecd6c73f +af2085ab3be83b4c086194707d61590eca29efdf diff --git a/src/client/query/impl/exec_query.cpp b/src/client/query/impl/exec_query.cpp index 0a7814b73d..7b9ba781dc 100644 --- a/src/client/query/impl/exec_query.cpp +++ b/src/client/query/impl/exec_query.cpp @@ -162,6 +162,7 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { std::optional Tx_; std::vector ArrowSchemas_; std::vector> BytesData_; + std::vector ResultSetSeen_; void Next() { TPtr self(this); @@ -216,29 +217,36 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { if (part.HasResultSet()) { auto inRs = part.ExtractResultSet(); - auto& inRsProto = TProtoAccessor::GetProto(inRs); + auto& inRsProto = inRs.MutableProto(); // TODO: Use result sets metadata if (self->ResultSets_.size() <= part.GetResultSetIndex()) { self->ResultSets_.resize(part.GetResultSetIndex() + 1); + self->ResultSetSeen_.resize(part.GetResultSetIndex() + 1); } auto& resultSet = self->ResultSets_[part.GetResultSetIndex()]; - resultSet.set_format(inRsProto.format()); + const auto resultSetIndex = part.GetResultSetIndex(); - switch (resultSet.format()) { + switch (inRsProto.format()) { case Ydb::ResultSet::FORMAT_UNSPECIFIED: case Ydb::ResultSet::FORMAT_VALUE: { - self->CollectYdbValues(resultSet, inRsProto); + if (!self->ResultSetSeen_[resultSetIndex]) { + resultSet = std::move(inRsProto); + } else { + self->CollectYdbValues(resultSet, inRsProto); + } break; } case Ydb::ResultSet::FORMAT_ARROW: { - self->CollectArrowBytes(resultSet, inRs.MutableProto(), part.GetResultSetIndex()); + resultSet.set_format(inRsProto.format()); + self->CollectArrowBytes(resultSet, inRsProto, part.GetResultSetIndex()); break; } default: break; } + self->ResultSetSeen_[resultSetIndex] = true; } if (const auto& tx = part.GetTransaction()) { From 810367ef64ca42524218492a64bcb9b9ffe5babe Mon Sep 17 00:00:00 2001 From: Alek5andr-Kotov Date: Tue, 28 Jul 2026 08:46:26 +0000 Subject: [PATCH 17/56] Add canonical Topic/Kafka transaction wire with legacy dual-write. (#45858) --- .github/last_commit.txt | 2 +- src/client/topic/ut/ut_utils/txusage_fixture.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 881f61a03b..5cac4c738c 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -af2085ab3be83b4c086194707d61590eca29efdf +d3235585a7c046b39aa17e41651f93a2d28204f3 diff --git a/src/client/topic/ut/ut_utils/txusage_fixture.cpp b/src/client/topic/ut/ut_utils/txusage_fixture.cpp index d38b1d5c6e..b0c820a601 100644 --- a/src/client/topic/ut/ut_utils/txusage_fixture.cpp +++ b/src/client/topic/ut/ut_utils/txusage_fixture.cpp @@ -1174,7 +1174,7 @@ void TFixture::SendLongTxLockStatus(const NActors::TActorId& actorId, NKikimrLongTxService::TEvLockStatus::EStatus status) { auto event = - std::make_unique(writeId.KeyId, writeId.NodeId, + std::make_unique(writeId.GetKeyId(), writeId.GetNodeId(), status); auto& runtime = Setup->GetRuntime(); runtime.SendToPipe(tabletId, actorId, event.release()); From bf17aad627d2b2e13238c8f6a1ba28da70319d62 Mon Sep 17 00:00:00 2001 From: Aleksandr Usenko Date: Tue, 28 Jul 2026 08:46:36 +0000 Subject: [PATCH 18/56] [Console, ConfigsDispatcher, YDB CLI] Implement database yaml config selectors (#44420) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_cms.proto | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 5cac4c738c..84deb4ab51 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -d3235585a7c046b39aa17e41651f93a2d28204f3 +8c4027d82ae1d105bf58374b6beeef5fc592f5d2 diff --git a/src/api/protos/ydb_cms.proto b/src/api/protos/ydb_cms.proto index fe90f5b1de..f72bb967a9 100644 --- a/src/api/protos/ydb_cms.proto +++ b/src/api/protos/ydb_cms.proto @@ -101,7 +101,7 @@ message DatabaseQuotas { // A minimum value of `TtlSettings.run_interval_seconds` that can be specified. // Default is 1800 (15 minutes). uint32 ttl_min_run_internal_seconds = 4; - + message StorageQuotas { // in theory an arbitrary string, but in practice "hdd" or "ssd" string unit_kind = 1; @@ -120,7 +120,7 @@ message ScaleRecommenderPolicies { message TargetTrackingPolicy { oneof target { // A percentage of compute resources' average CPU utilization. - uint32 average_cpu_utilization_percent = 1 [(Ydb.value) = "[10; 90]"]; + uint32 average_cpu_utilization_percent = 1 [(Ydb.value) = "[10; 90]"]; } } @@ -216,6 +216,8 @@ message GetDatabaseStatusResult { // Outstanding problems related to database resources // (e.g. storage pool allocation failures). repeated Ydb.Issue.IssueMessage issues = 12; + // Current database attributes + map attributes = 13; } // Change resources allocated for database. From 6551e407cbcce0831d11cf9a012d0b839eacaad0 Mon Sep 17 00:00:00 2001 From: mregrock Date: Tue, 28 Jul 2026 08:46:46 +0000 Subject: [PATCH 19/56] Refactor grpc, sdk and cli for test shard scheme object (#45361) --- .github/last_commit.txt | 2 +- .../client/test_shard/test_shard.h | 56 ++++++++++++ src/api/grpc/draft/ydb_test_shard_v1.proto | 15 ---- src/api/grpc/ydb_test_shard_v1.proto | 15 ++++ .../protos/{draft => }/ydb_test_shard.proto | 43 ++++----- src/client/test_shard/test_shard.cpp | 88 +++++++++++++++++++ 6 files changed, 176 insertions(+), 43 deletions(-) create mode 100644 include/ydb-cpp-sdk/client/test_shard/test_shard.h delete mode 100644 src/api/grpc/draft/ydb_test_shard_v1.proto create mode 100644 src/api/grpc/ydb_test_shard_v1.proto rename src/api/protos/{draft => }/ydb_test_shard.proto (75%) create mode 100644 src/client/test_shard/test_shard.cpp diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 84deb4ab51..70cb61c329 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -8c4027d82ae1d105bf58374b6beeef5fc592f5d2 +174bc811b6d306a4445b6bf7e5dcfd5b3af93064 diff --git a/include/ydb-cpp-sdk/client/test_shard/test_shard.h b/include/ydb-cpp-sdk/client/test_shard/test_shard.h new file mode 100644 index 0000000000..7855f1d5bb --- /dev/null +++ b/include/ydb-cpp-sdk/client/test_shard/test_shard.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace NYdb::NTestShardSet { + +struct TCreateTestShardSetSettings : public TOperationRequestSettings {}; + +struct TDeleteTestShardSetSettings : public TOperationRequestSettings {}; + +class TCreateTestShardSetResult : public TStatus { +public: + TCreateTestShardSetResult(TStatus&& status, std::vector tabletIds) + : TStatus(std::move(status)) + , TabletIds_(std::move(tabletIds)) + {} + + const std::vector& GetTabletIds() const { + return TabletIds_; + } + +private: + std::vector TabletIds_; +}; + +using TAsyncCreateTestShardSetResult = NThreading::TFuture; + +class TTestShardSetClient { +public: + explicit TTestShardSetClient(const TDriver& driver, const TCommonClientSettings& settings = {}); + ~TTestShardSetClient(); + + TAsyncCreateTestShardSetResult CreateTestShardSet( + const std::string& path, + const std::vector& channels, + uint32_t count, + const std::string& config, + const TCreateTestShardSetSettings& settings = {}); + + TAsyncStatus DeleteTestShardSet( + const std::string& path, + const TDeleteTestShardSetSettings& settings = {}); + +private: + class TImpl; + std::unique_ptr Impl_; +}; + +} // namespace NYdb::NTestShardSet diff --git a/src/api/grpc/draft/ydb_test_shard_v1.proto b/src/api/grpc/draft/ydb_test_shard_v1.proto deleted file mode 100644 index 02d61d01af..0000000000 --- a/src/api/grpc/draft/ydb_test_shard_v1.proto +++ /dev/null @@ -1,15 +0,0 @@ -syntax = "proto3"; - -package Ydb.TestShard.V1; - -import "src/api/protos/draft/ydb_test_shard.proto"; - -service TestShardService { - // Creates TestShard tablet(s) - rpc CreateTestShard(Ydb.TestShard.CreateTestShardRequest) - returns (Ydb.TestShard.CreateTestShardResponse); - - // Deletes TestShard tablet(s) - rpc DeleteTestShard(Ydb.TestShard.DeleteTestShardRequest) - returns (Ydb.TestShard.DeleteTestShardResponse); -} diff --git a/src/api/grpc/ydb_test_shard_v1.proto b/src/api/grpc/ydb_test_shard_v1.proto new file mode 100644 index 0000000000..8cda7263dc --- /dev/null +++ b/src/api/grpc/ydb_test_shard_v1.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package Ydb.TestShardSet.V1; + +import "src/api/protos/ydb_test_shard.proto"; + +service TestShardSetService { + // Creates TestShardSet tablet(s) + rpc CreateTestShardSet(Ydb.TestShardSet.CreateTestShardSetRequest) + returns (Ydb.TestShardSet.CreateTestShardSetResponse); + + // Deletes TestShardSet tablet(s) + rpc DeleteTestShardSet(Ydb.TestShardSet.DeleteTestShardSetRequest) + returns (Ydb.TestShardSet.DeleteTestShardSetResponse); +} diff --git a/src/api/protos/draft/ydb_test_shard.proto b/src/api/protos/ydb_test_shard.proto similarity index 75% rename from src/api/protos/draft/ydb_test_shard.proto rename to src/api/protos/ydb_test_shard.proto index 1ef8a662c4..f42dd882f5 100644 --- a/src/api/protos/draft/ydb_test_shard.proto +++ b/src/api/protos/ydb_test_shard.proto @@ -1,27 +1,23 @@ syntax = "proto3"; -package Ydb.TestShard; +package Ydb.TestShardSet; -option java_package = "com.yandex.ydb.test_shard"; +option java_package = "com.yandex.ydb.test_shard_set"; import "src/api/protos/ydb_operation.proto"; -message CreateTestShardRequest { +message CreateTestShardSetRequest { Ydb.Operations.OperationParams operation_params = 1; - // Database path - string database = 2; - - // Base index for tablet ID generation - // Same owner_idx = same tablets (idempotent duplicate detection) - uint64 owner_idx = 3; + // Path to the TestShardSet object to create + string path = 2; // Storage pool names for tablet channels (optional) // If not provided, uses first 3 storage pools from the database domain - repeated string channels = 4; + repeated string channels = 3; // Number of tablets to create - uint32 count = 5; + uint32 count = 4; // TestShard initialization config in YAML format // @@ -59,41 +55,34 @@ message CreateTestShardRequest { // stall_counter: 1000 # Barrier sync every N requests // // validation: # Optional external validation - // server: "host:9999" # Validation server address + // server: "host:port" # Validation server address // after_bytes: 1000000 # Start validation after N bytes // // tracing: # Optional request tracing // put_fraction_ppm: 1000 # Trace fraction (parts per million) // verbosity: 15 # Trace detail level (0-15) // ``` - string config = 6; + string config = 5; } -message CreateTestShardResponse { +message CreateTestShardSetResponse { Ydb.Operations.Operation operation = 1; } -message CreateTestShardResult { +message CreateTestShardSetResult { // Created tablet IDs repeated uint64 tablet_ids = 1; } -message DeleteTestShardRequest { +message DeleteTestShardSetRequest { Ydb.Operations.OperationParams operation_params = 1; - // Database path - string database = 2; - - // Owner index identifying tablets to delete - uint64 owner_idx = 3; - - // Number of consecutive tablets to delete starting from owner_idx - // If not specified, deletes all tablets with this owner_idx - uint32 count = 4; + // Path to the TestShardSet object to delete + string path = 2; } -message DeleteTestShardResponse { +message DeleteTestShardSetResponse { Ydb.Operations.Operation operation = 1; } -message DeleteTestShardResult {} +message DeleteTestShardSetResult {} diff --git a/src/client/test_shard/test_shard.cpp b/src/client/test_shard/test_shard.cpp new file mode 100644 index 0000000000..4cff4e9607 --- /dev/null +++ b/src/client/test_shard/test_shard.cpp @@ -0,0 +1,88 @@ +#include + +#include +#include + +#include +#include + +namespace NYdb::NTestShardSet { + +class TTestShardSetClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + {} + + TAsyncCreateTestShardSetResult CreateTestShardSet(const std::string& path, + const std::vector& channels, uint32_t count, + const std::string& config, + const TCreateTestShardSetSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(path); + for (const auto& channel : channels) { + request.add_channels(channel); + } + request.set_count(count); + if (!config.empty()) { + request.set_config(config); + } + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] (google::protobuf::Any* any, TPlainStatus status) mutable { + std::vector tabletIds; + if (any) { + Ydb::TestShardSet::CreateTestShardSetResult result; + if (any->UnpackTo(&result)) { + tabletIds.reserve(result.tablet_ids_size()); + for (int i = 0; i < result.tablet_ids_size(); ++i) { + tabletIds.push_back(result.tablet_ids(i)); + } + } + } + promise.SetValue(TCreateTestShardSetResult(TStatus(std::move(status)), std::move(tabletIds))); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &Ydb::TestShardSet::V1::TestShardSetService::Stub::AsyncCreateTestShardSet, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + + TAsyncStatus DeleteTestShardSet(const std::string& path, + const TDeleteTestShardSetSettings& settings) { + auto request = MakeOperationRequest(settings); + request.set_path(path); + + return RunSimple( + std::move(request), + &Ydb::TestShardSet::V1::TestShardSetService::Stub::AsyncDeleteTestShardSet, + TRpcRequestSettings::Make(settings)); + } +}; + +TTestShardSetClient::TTestShardSetClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(new TImpl(CreateInternalInterface(driver), settings)) +{} + +TTestShardSetClient::~TTestShardSetClient() = default; + +TAsyncCreateTestShardSetResult TTestShardSetClient::CreateTestShardSet(const std::string& path, + const std::vector& channels, uint32_t count, + const std::string& config, + const TCreateTestShardSetSettings& settings) { + return Impl_->CreateTestShardSet(path, channels, count, config, settings); +} + +TAsyncStatus TTestShardSetClient::DeleteTestShardSet(const std::string& path, + const TDeleteTestShardSetSettings& settings) { + return Impl_->DeleteTestShardSet(path, settings); +} + +} // namespace NYdb::NTestShardSet From e651f3ec7ae18fadb1aabcd0f259beb0e56e9b37 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:46:57 +0000 Subject: [PATCH 20/56] Fix driver uaf (#45962) --- .github/last_commit.txt | 2 +- src/client/coordination/coordination.cpp | 73 ++- src/client/driver/driver.cpp | 18 +- .../impl/internal/db_driver_state/state.cpp | 9 +- .../impl/internal/db_driver_state/state.h | 5 +- .../impl/internal/grpc_connections/actions.h | 44 +- .../grpc_connections/grpc_connections.cpp | 247 +++++++++- .../grpc_connections/grpc_connections.h | 431 ++++++++++++------ src/client/table/impl/table_client.cpp | 33 +- src/client/table/impl/table_client.h | 3 +- src/client/topic/impl/direct_reader.cpp | 3 +- src/client/topic/impl/read_session_impl.h | 11 +- src/client/topic/impl/read_session_impl.ipp | 14 +- src/library/grpc/client/grpc_client_low.h | 174 +++++-- .../client/coordination/coordination_ut.cpp | 50 +- tests/unit/client/table/table_ut.cpp | 343 ++++++++++++++ 16 files changed, 1203 insertions(+), 257 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 70cb61c329..7af8cda066 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -174bc811b6d306a4445b6bf7e5dcfd5b3af93064 +1a85306db085572a96d030acac1482ce9f5523c1 diff --git a/src/client/coordination/coordination.cpp b/src/client/coordination/coordination.cpp index 4c3cacfb68..af352fd995 100644 --- a/src/client/coordination/coordination.cpp +++ b/src/client/coordination/coordination.cpp @@ -177,11 +177,11 @@ class TSessionContext : public TThrRefBase { public: TSessionContext( - TGRpcConnectionsImpl* connections, + std::shared_ptr connections, TDbDriverStatePtr dbState, const std::string& path, const TSessionSettings& settings) - : Connections_(connections) + : Connections_(std::move(connections)) , DbDriverState_(dbState) , Path_(path) , Settings_(settings) @@ -197,8 +197,8 @@ class TSessionContext : public TThrRefBase { return result; } - void Start(IQueueClientContextProvider* provider) { - auto context = provider->CreateContext(); + void Start() { + auto context = Connections_->CreateContext(); if (!context) { auto status = MakeStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"); auto promise = std::move(SessionPromise); @@ -333,11 +333,16 @@ class TSessionContext : public TThrRefBase { struct TDescribeSemaphoreOp : public TSimpleOp { const std::string Name; TDescribeSemaphoreSettings Settings; + NYdbGrpc::TQueueClientCallbackGuardFactory CallbackGuardFactory; TPromise Promise = NewPromise(); - TDescribeSemaphoreOp(const std::string& name, const TDescribeSemaphoreSettings& settings) + TDescribeSemaphoreOp( + const std::string& name, + const TDescribeSemaphoreSettings& settings, + NYdbGrpc::TQueueClientCallbackGuardFactory callbackGuardFactory) : Name(name) , Settings(settings) + , CallbackGuardFactory(std::move(callbackGuardFactory)) {} void FillRequest(TRequest& req, uint64_t reqId) const override { @@ -356,7 +361,9 @@ class TSessionContext : public TThrRefBase { } else if (Settings.OnChanged_) { std::function callback; callback.swap(Settings.OnChanged_); - callback(false); + NYdbGrpc::RunQueueClientCallback(CallbackGuardFactory, [&] { + callback(false); + }); } } }; @@ -527,7 +534,10 @@ class TSessionContext : public TThrRefBase { if (IsClosed()) { return MakeClosedResult(); } - auto op = std::make_unique(name, settings); + auto op = std::make_unique( + name, + settings, + Connections_->GetCallbackGuardFactory()); auto future = op->Promise.GetFuture(); if (IsWriteAllowed()) { DoSendSimpleOp(std::move(op)); @@ -856,6 +866,13 @@ class TSessionContext : public TThrRefBase { } private: + template + void RunUserCallback(TCallback&& callback) { + NYdbGrpc::RunQueueClientCallback( + Connections_->GetCallbackGuardFactory(), + std::forward(callback)); + } + void OnProcessorStatus(TStatus status) { std::shared_ptr context; TDbDriverStatePtr dbDriverState; @@ -1023,10 +1040,14 @@ class TSessionContext : public TThrRefBase { if (stopped) { if (notifyExpired && Settings_.OnStateChanged_) { - Settings_.OnStateChanged_(ESessionState::EXPIRED); + RunUserCallback([this] { + Settings_.OnStateChanged_(ESessionState::EXPIRED); + }); } if (Settings_.OnStopped_) { - Settings_.OnStopped_(); + RunUserCallback([this] { + Settings_.OnStopped_(); + }); } return; } @@ -1228,7 +1249,9 @@ class TSessionContext : public TThrRefBase { } if (detached && Settings_.OnStateChanged_) { - Settings_.OnStateChanged_(ESessionState::DETACHED); + RunUserCallback([this] { + Settings_.OnStateChanged_(ESessionState::DETACHED); + }); } if (sessionStartTimeoutContext) { @@ -1442,9 +1465,13 @@ class TSessionContext : public TThrRefBase { } } if (expired && Settings_.OnStateChanged_) { - Settings_.OnStateChanged_(ESessionState::EXPIRED); + RunUserCallback([this] { + Settings_.OnStateChanged_(ESessionState::EXPIRED); + }); } else if (detached && Settings_.OnStateChanged_) { - Settings_.OnStateChanged_(ESessionState::DETACHED); + RunUserCallback([this] { + Settings_.OnStateChanged_(ESessionState::DETACHED); + }); } return false; } @@ -1475,7 +1502,9 @@ class TSessionContext : public TThrRefBase { } } if (Settings_.OnStateChanged_) { - Settings_.OnStateChanged_(ESessionState::ATTACHED); + RunUserCallback([this] { + Settings_.OnStateChanged_(ESessionState::ATTACHED); + }); } if (replyPromise.Initialized()) { // If there are no listeners session destructor will be immediately called outside of the lock @@ -1518,7 +1547,9 @@ class TSessionContext : public TThrRefBase { Y_ABORT_UNLESS(SessionState != ESessionState::ATTACHED); } if (expired && Settings_.OnStateChanged_) { - Settings_.OnStateChanged_(ESessionState::EXPIRED); + RunUserCallback([this] { + Settings_.OnStateChanged_(ESessionState::EXPIRED); + }); } return false; } @@ -1555,7 +1586,9 @@ class TSessionContext : public TThrRefBase { supersededPromise.SetValue(TResult(std::move(status), false)); } if (acceptedCallback) { - acceptedCallback(); + RunUserCallback([callback = std::move(acceptedCallback)] { + callback(); + }); } return true; } @@ -1636,7 +1669,9 @@ class TSessionContext : public TThrRefBase { } } if (callback) { - callback(triggered); + RunUserCallback([callback = std::move(callback), triggered] { + callback(triggered); + }); } return true; } @@ -1730,7 +1765,7 @@ class TSessionContext : public TThrRefBase { } private: - TGRpcConnectionsImpl* const Connections_; + std::shared_ptr Connections_; TDbDriverStatePtr DbDriverState_; const std::string Path_; const TSessionSettings Settings_; @@ -1850,14 +1885,14 @@ class TClient::TImpl : public TClientImplCommon { const TSessionSettings& settings) { auto session = MakeIntrusive( - Connections_.get(), + Connections_, DbDriverState_, path, settings); auto result = session->TakeStartResult(); - session->Start(Connections_.get()); + session->Start(); return result; } diff --git a/src/client/driver/driver.cpp b/src/client/driver/driver.cpp index a999266eef..2cd004faa5 100644 --- a/src/client/driver/driver.cpp +++ b/src/client/driver/driver.cpp @@ -362,22 +362,14 @@ TDriver::TDriver(const TDriverConfig& config) { ythrow yexception() << "Invalid config object"; } - Impl_ = std::shared_ptr(new TGRpcConnectionsImpl(config.Impl_), [] (TGRpcConnectionsImpl* impl) { - auto destroyImpl = [impl] { - impl->Stop(true); - delete impl; - }; - - if (IsGRpcCompletionThread()) { - std::thread(std::move(destroyImpl)).detach(); - } else { - destroyImpl(); - } - }); + Impl_.reset(new TGRpcConnectionsImpl(config.Impl_), TGRpcConnectionsDeleter()); } void TDriver::Stop(bool wait) { - Impl_->Stop(wait); + auto impl = Impl_; + TGRpcConnectionsImpl::DeferOrRunNow(impl->StopState_, [impl, wait]() mutable { + impl->Stop(wait); + }); } TDriverConfig TDriver::GetConfig() const { diff --git a/src/client/impl/internal/db_driver_state/state.cpp b/src/client/impl/internal/db_driver_state/state.cpp index 2f1f51137f..8a072fb330 100644 --- a/src/client/impl/internal/db_driver_state/state.cpp +++ b/src/client/impl/internal/db_driver_state/state.cpp @@ -17,6 +17,7 @@ namespace { url.assign(to, Quote(to, TStringBuf(url), safe)); } + } namespace NYdb::inline V3 { @@ -286,7 +287,8 @@ void TDbDriverState::PostToResponseQueue(TPostTaskCb&& f) { } NThreading::TFuture TDbDriverStateTracker::SendNotification( - TDbDriverState::ENotifyType type + TDbDriverState::ENotifyType type, + TNotificationCbRunner cbRunner ) { std::vector> states; { @@ -303,7 +305,7 @@ NThreading::TFuture TDbDriverStateTracker::SendNotification( std::lock_guard lock(strong->NotifyCbsLock); for (auto& cb : strong->NotifyCbs[static_cast(type)]) { if (cb) { - auto future = cb(); + auto future = cbRunner ? cbRunner(cb) : cb(); if (!future.HasException()) { results.push_back(future); } @@ -312,6 +314,9 @@ NThreading::TFuture TDbDriverStateTracker::SendNotification( } } } + if (results.empty()) { + return NThreading::MakeFuture(); + } return NThreading::WaitExceptionOrAll(results); } diff --git a/src/client/impl/internal/db_driver_state/state.h b/src/client/impl/internal/db_driver_state/state.h index 3dbf6eaaab..1db84201b2 100644 --- a/src/client/impl/internal/db_driver_state/state.h +++ b/src/client/impl/internal/db_driver_state/state.h @@ -98,6 +98,8 @@ class TDbDriverStateTracker { }; public: TDbDriverStateTracker(IInternalClient* client); + using TNotificationCbRunner = std::function(TDbDriverState::TCb& cb)>; + TDbDriverState::TPtr GetDriverState( const std::string& database, const std::string& discoveryEndpoint, @@ -106,7 +108,8 @@ class TDbDriverStateTracker { std::shared_ptr credentialsProviderFactory ); NThreading::TFuture SendNotification( - TDbDriverState::ENotifyType type); + TDbDriverState::ENotifyType type, + TNotificationCbRunner cbRunner = {}); void SetMetricRegistry(::NMonitoring::TMetricRegistry *sensorsRegistry); private: IInternalClient* DiscoveryClient_; diff --git a/src/client/impl/internal/grpc_connections/actions.h b/src/client/impl/internal/grpc_connections/actions.h index 4c64a32d7b..0f67b7cbd2 100644 --- a/src/client/impl/internal/grpc_connections/actions.h +++ b/src/client/impl/internal/grpc_connections/actions.h @@ -14,6 +14,8 @@ #include +#include + namespace NYdb::inline V3 { using NYdbGrpc::IQueueClientContext; @@ -28,6 +30,15 @@ using TResponseCb = std::function; using TDeferredOperationCb = std::function; using TDelayedCb = std::function; +inline TPlainStatus MakeClientStoppedStatus() { + return TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"); +} + +class TQueueResponse : public IObjectInQueue { +public: + virtual void Cancel() = 0; +}; + template class TGenericCbHolder { protected: @@ -87,11 +98,16 @@ class TAlarmActionBase LocalContext_.reset(); } - if (ok) { - OnAlarm(); - } else { - OnError(); - } + auto guardFactory = this->Context_ + ? this->Context_->GetCallbackGuardFactory() + : NYdbGrpc::TQueueClientCallbackGuardFactory(); + NYdbGrpc::RunQueueClientCallback(guardFactory, [&] { + if (ok) { + OnAlarm(); + } else { + OnError(); + } + }); return false; } @@ -112,7 +128,7 @@ class TAlarmActionBase template class TGRpcErrorResponse : public TGenericCbHolder> - , public IObjectInQueue + , public TQueueResponse { public: TGRpcErrorResponse( @@ -135,10 +151,17 @@ class TGRpcErrorResponse status.Issues.AddIssue(NYdb::NIssue::TIssue(msg)); } + this->Context_.reset(); this->UserResponseCb_(nullptr, status); delete this; } + void Cancel() override { + this->Context_.reset(); + this->UserResponseCb_(nullptr, MakeClientStoppedStatus()); + delete this; + } + private: NYdbGrpc::TGrpcStatus GRpcStatus_; std::string Endpoint_; @@ -147,7 +170,7 @@ class TGRpcErrorResponse template class TResult : public TGenericCbHolder> - , public IObjectInQueue + , public TQueueResponse { public: TResult( @@ -165,10 +188,17 @@ class TResult , Metadata_(std::move(metadata)) {} void Process(void*) override { + this->Context_.reset(); this->UserResponseCb_(&Response_, TPlainStatus{GRpcStatus_, Endpoint_, std::move(Metadata_)}); delete this; } + void Cancel() override { + this->Context_.reset(); + this->UserResponseCb_(nullptr, MakeClientStoppedStatus()); + delete this; + } + private: TResponse Response_; NYdbGrpc::TGrpcStatus GRpcStatus_; diff --git a/src/client/impl/internal/grpc_connections/grpc_connections.cpp b/src/client/impl/internal/grpc_connections/grpc_connections.cpp index f739bf20aa..915f9c4b35 100644 --- a/src/client/impl/internal/grpc_connections/grpc_connections.cpp +++ b/src/client/impl/internal/grpc_connections/grpc_connections.cpp @@ -5,10 +5,142 @@ #include #include +#include +#include namespace NYdb::inline V3 { +namespace { + thread_local ui32 SdkResponseCallbackDepth = 0; + + template + void RunAfterCurrentSdkCallback(std::shared_ptr stopState, TCallback&& callback) { + // A fresh detached thread that runs to completion and exits: deferring onto a single + // long-lived worker instead was tried and regressed (a permanently-blocked worker + // faults at process teardown). Teardown-from-callback is rare, so on-demand is fine. + try { + std::thread([stopState = std::move(stopState), callback = std::forward(callback)]() mutable { + // Wait until every in-flight SDK callback (including the one that scheduled + // us) has returned before touching the driver, so we never free it or join + // its threads from under a live callback frame. + if (stopState) { + stopState->WaitCallbacksDrained(); + } + callback(); + }).detach(); + } catch (...) { + Y_ABORT("Failed to defer YDB driver action from SDK callback thread"); + } + } + + TQueueClientCallbackGuardFactory MakeSdkCallbackGuardFactory(std::shared_ptr stopState) { + return [stopState = std::move(stopState)] { + return std::make_unique(stopState); + }; + } + + class TSdkQueueClientContext final : public NYdbGrpc::IQueueClientContext { + public: + TSdkQueueClientContext(IQueueClientContextPtr underlying, std::shared_ptr stopState) + : Underlying_(std::move(underlying)) + , StopState_(std::move(stopState)) + { + Y_ABORT_UNLESS(Underlying_); + Y_ABORT_UNLESS(StopState_); + } + + IQueueClientContextPtr CreateContext() override { + auto child = Underlying_->CreateContext(); + if (!child) { + return nullptr; + } + return std::make_shared(std::move(child), StopState_); + } + + TQueueClientCallbackGuardFactory GetCallbackGuardFactory() override { + return MakeSdkCallbackGuardFactory(StopState_); + } + + grpc::CompletionQueue* CompletionQueue() override { + return Underlying_->CompletionQueue(); + } + + bool IsCancelled() const override { + return Underlying_->IsCancelled(); + } + + bool Cancel() override { + return Underlying_->Cancel(); + } + + void SubscribeCancel(std::function callback) override { + Underlying_->SubscribeCancel(std::move(callback)); + } + + private: + IQueueClientContextPtr Underlying_; + std::shared_ptr StopState_; + }; +} + +bool TDriverStopState::TryEnterCallback() noexcept { + std::unique_lock lock(Mutex_); + if (Stopped_) { + return false; + } + ++InFlightCallbacks_; + return true; +} + +void TDriverStopState::LeaveCallback() noexcept { + std::unique_lock lock(Mutex_); + Y_ABORT_UNLESS(InFlightCallbacks_ > 0); + if (--InFlightCallbacks_ == 0) { + Drained_.notify_all(); + } +} + +void TDriverStopState::WaitCallbacksDrained() { + std::unique_lock lock(Mutex_); + Drained_.wait(lock, [this] { + return InFlightCallbacks_ == 0; + }); +} + +void TDriverStopState::MarkStopped() noexcept { + std::unique_lock lock(Mutex_); + Stopped_ = true; + if (InFlightCallbacks_ == 0) { + Drained_.notify_all(); + } +} + +TSdkCallbackGuard::TSdkCallbackGuard(std::shared_ptr stopState) + : StopState_(std::move(stopState)) +{ + Entered_ = !StopState_ || StopState_->TryEnterCallback(); + if (Entered_) { + ++SdkResponseCallbackDepth; + } +} + +TSdkCallbackGuard::~TSdkCallbackGuard() { + if (!Entered_) { + return; + } + + --SdkResponseCallbackDepth; + + if (StopState_) { + StopState_->LeaveCallback(); + } +} + +bool TSdkCallbackGuard::IsEntered() const noexcept { + return Entered_; +} + bool IsTokenCorrect(const std::string& in) { for (char c : in) { if (!(IsAsciiAlnum(c) || IsAsciiPunct(c) || c == ' ')) { @@ -62,13 +194,24 @@ class TScheduledObject : public TThrRefBase { return static_cast(this); } + void Complete(bool ok) { + bool entered = true; + std::unique_ptr guard; + if (CallbackGuardFactory) { + guard = CallbackGuardFactory(); + entered = !guard || guard->IsEntered(); + } + Derived()->OnComplete(entered ? ok : false); + } + protected: TScheduledObject() { } void Start(TDuration timeout, IQueueClientContextProvider* provider) { + CallbackGuardFactory = provider->GetCallbackGuardFactory(); auto context = provider->CreateContext(); if (!context) { - Derived()->OnComplete(false); + Complete(false); return; } @@ -95,13 +238,14 @@ class TScheduledObject : public TThrRefBase { Context.reset(); } - Derived()->OnComplete(ok); + Complete(ok); } private: std::mutex Mutex; IQueueClientContextPtr Context; grpc::Alarm Alarm; + TQueueClientCallbackGuardFactory CallbackGuardFactory; private: using TFixedEvent = NYdbGrpc::TQueueClientFixedEvent; @@ -124,8 +268,8 @@ class TScheduledCallback : public TScheduledObject { } void OnComplete(bool ok) { - Callback(ok); - Callback = { }; + auto callback = std::move(Callback); + callback(ok); } private: @@ -160,6 +304,7 @@ class TScheduledFuture : public TScheduledObject { TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr params) : MetricRegistryPtr_(nullptr) , ClientThreadsNum_(params->GetClientThreadsNum()) + , StopState_(std::make_shared()) , DefaultDiscoveryEndpoint_(params->GetEndpoint()) , SslCredentials_(params->GetSslCredentials()) , DefaultDatabase_(params->GetDatabase()) @@ -228,15 +373,40 @@ TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr p } TGRpcConnectionsImpl::~TGRpcConnectionsImpl() { - GRpcClientLow_.Stop(true); - ResponseQueue_->Stop(); + Stop(true); + StopState_->MarkStopped(); +} + +bool TGRpcConnectionsImpl::IsCurrentThreadInSdkCallback() noexcept { + return SdkResponseCallbackDepth != 0; +} + +void TGRpcConnectionsImpl::DeferOrRunNow(std::shared_ptr stopState, std::function action) { + if (!IsCurrentThreadInSdkCallback() && !NYdbGrpc::IsGRpcCompletionThread()) { + action(); + return; + } + + RunAfterCurrentSdkCallback(std::move(stopState), std::move(action)); +} + +void TGRpcConnectionsDeleter::operator()(TGRpcConnectionsImpl* connections) const noexcept { + if (!connections) { + return; + } + + TGRpcConnectionsImpl::DeferOrRunNow(connections->StopState_, [connections] { + delete connections; + }); } void TGRpcConnectionsImpl::AddPeriodicTask(TPeriodicCb&& cb, TDeadline::Duration period) { std::shared_ptr context; if (!TryCreateContext(context)) { NYdb::NIssue::TIssues issues; - cb(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR); + RunGuarded(StopState_, + [&] { cb(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR); }, + [&] { cb(std::move(issues), EStatus::CLIENT_CANCELLED); }); } else { auto action = MakeIntrusive( std::move(cb), @@ -248,7 +418,12 @@ void TGRpcConnectionsImpl::AddPeriodicTask(TPeriodicCb&& cb, TDeadline::Duration } void TGRpcConnectionsImpl::PostToResponseQueue(std::function&& f) { - ResponseQueue_->Post(std::move(f)); + auto stopState = StopState_; + ResponseQueue_->Post([f = std::move(f), stopState = std::move(stopState)]() mutable { + RunGuarded(stopState, + [&] { auto callback = std::move(f); callback(); }, + [] {}); + }); } void TGRpcConnectionsImpl::ScheduleDelayedTask(TSimpleCb&& fn, TDeadline deadline) { @@ -291,7 +466,7 @@ NThreading::TFuture TGRpcConnectionsImpl::ScheduleFuture( { IQueueClientContextProvider* provider = context.get(); if (!provider) { - provider = &GRpcClientLow_; + provider = this; } return MakeIntrusive() @@ -305,7 +480,7 @@ void TGRpcConnectionsImpl::ScheduleCallback( { IQueueClientContextProvider* provider = context.get(); if (!provider) { - provider = &GRpcClientLow_; + provider = this; } return MakeIntrusive(std::move(callback)) @@ -328,7 +503,15 @@ TDbDriverStatePtr TGRpcConnectionsImpl::GetDriverState( } IQueueClientContextPtr TGRpcConnectionsImpl::CreateContext() { - return GRpcClientLow_.CreateContext(); + auto context = GRpcClientLow_.CreateContext(); + if (!context) { + return nullptr; + } + return std::make_shared(std::move(context), StopState_); +} + +TQueueClientCallbackGuardFactory TGRpcConnectionsImpl::GetCallbackGuardFactory() { + return MakeSdkCallbackGuardFactory(StopState_); } bool TGRpcConnectionsImpl::TryCreateContext(IQueueClientContextPtr& context) { @@ -347,8 +530,22 @@ void TGRpcConnectionsImpl::WaitIdle() { } void TGRpcConnectionsImpl::Stop(bool wait) { - StateTracker_.SendNotification(TDbDriverState::ENotifyType::STOP).Wait(); + auto stopState = StopState_; + StateTracker_.SendNotification( + TDbDriverState::ENotifyType::STOP, + [stopState = std::move(stopState)](TDbDriverState::TCb& cb) { + TSdkCallbackGuard guard(stopState); + if (guard.IsEntered()) { + return cb(); + } + + return NThreading::MakeFuture(); + }).Wait(); GRpcClientLow_.Stop(wait); + if (wait) { + StopResponseQueue(); + StopState_->WaitCallbacksDrained(); + } } void TGRpcConnectionsImpl::SetGrpcKeepAlive(NYdbGrpc::TGRpcClientConfig& config, const TDeadline::Duration& timeout, bool permitWithoutCalls) { @@ -402,9 +599,14 @@ TAsyncListEndpointsResult TGRpcConnectionsImpl::GetEndpoints(TDbDriverStatePtr d std::weak_ptr weakState = dbState; - return promise.GetFuture().Apply([this, weakState](NThreading::TFuture future){ + auto stopState = StopState_; + return promise.GetFuture().Apply([this, weakState, stopState = std::move(stopState)](NThreading::TFuture future){ auto strong = weakState.lock(); auto result = future.ExtractValue(); + TSdkCallbackGuard guard(stopState); + if (!guard.IsEntered()) { + return NThreading::MakeFuture(std::move(result)); + } if (strong && result.DiscoveryStatus.IsTransportError()) { strong->StatCollector.IncDiscoveryFailDueTransportError(); } @@ -492,8 +694,23 @@ const TLog& TGRpcConnectionsImpl::GetLog() const { } void TGRpcConnectionsImpl::EnqueueResponse(IObjectInQueue* action) { - ResponseQueue_->Post([action]() { - action->Process(nullptr); + auto stopState = StopState_; + ResponseQueue_->Post([action, stopState = std::move(stopState)]() { + RunGuarded(stopState, + [&] { action->Process(nullptr); }, + [&] { + if (auto* response = dynamic_cast(action)) { + response->Cancel(); + } else { + delete action; + } + }); + }); +} + +void TGRpcConnectionsImpl::StopResponseQueue() { + std::call_once(ResponseQueueStopOnce_, [this] { + ResponseQueue_->Stop(); }); } diff --git a/src/client/impl/internal/grpc_connections/grpc_connections.h b/src/client/impl/internal/grpc_connections/grpc_connections.h index 329cec719e..241e920254 100644 --- a/src/client/impl/internal/grpc_connections/grpc_connections.h +++ b/src/client/impl/internal/grpc_connections/grpc_connections.h @@ -15,6 +15,9 @@ #include +#include +#include +#include #include namespace NYdb::inline V3 { @@ -34,12 +37,54 @@ constexpr TDeadline::Duration GET_ENDPOINTS_TIMEOUT = std::chrono::seconds(10); using NYdbGrpc::TCallMeta; using NYdbGrpc::IQueueClientContextPtr; using NYdbGrpc::IQueueClientContextProvider; +using NYdbGrpc::IQueueClientCallbackGuard; +using NYdbGrpc::TQueueClientCallbackGuardFactory; class ICredentialsProvider; // Deferred callbacks using TDeferredResultCb = std::function; +class TDriverStopState { +public: + bool TryEnterCallback() noexcept; + void LeaveCallback() noexcept; + + void WaitCallbacksDrained(); + void MarkStopped() noexcept; + +private: + std::mutex Mutex_; + std::condition_variable Drained_; + ui64 InFlightCallbacks_ = 0; + bool Stopped_ = false; +}; + +class TSdkCallbackGuard final : public IQueueClientCallbackGuard { +public: + explicit TSdkCallbackGuard(std::shared_ptr stopState = {}); + ~TSdkCallbackGuard(); + + bool IsEntered() const noexcept override; + +private: + std::shared_ptr StopState_; + bool Entered_ = false; +}; + +// Runs onEntered() while the driver is not stopping, otherwise onStopped(). +// The single choke point behind every SDK-level guarded callback: it decides +// run-vs-substitute and keeps the in-flight-callback drain counter correct. +template +void RunGuarded(const std::shared_ptr& stopState, TOnEntered&& onEntered, TOnStopped&& onStopped) { + TSdkCallbackGuard guard(stopState); + if (guard.IsEntered()) { + std::forward(onEntered)(); + } else { + std::forward(onStopped)(); + } +} + std::string GetAuthInfo(TDbDriverStatePtr p); std::string CreateSDKBuildInfo(); @@ -49,10 +94,20 @@ class TGRpcConnectionsImpl { friend class TDeferredAction; friend class TDriver; + friend struct TGRpcConnectionsDeleter; public: TGRpcConnectionsImpl(std::shared_ptr params); ~TGRpcConnectionsImpl(); + static bool IsCurrentThreadInSdkCallback() noexcept; + + // Runs action() now if the caller is on a normal thread, otherwise defers it to a + // fresh thread that first waits for all in-flight callbacks to drain. Used for + // Stop()/delete triggered from within a callback, where running inline would + // deadlock (self-join) or free the driver under a live callback frame. + static void DeferOrRunNow(std::shared_ptr stopState, std::function action); + +public: void AddPeriodicTask(TPeriodicCb&& cb, TDeadline::Duration period) override; void PostToResponseQueue(std::function&& f) override; @@ -79,6 +134,7 @@ class TGRpcConnectionsImpl const std::optional>& credentialsProviderFactory ); IQueueClientContextPtr CreateContext() override; + TQueueClientCallbackGuardFactory GetCallbackGuardFactory() override; bool TryCreateContext(IQueueClientContextPtr& context); void WaitIdle(); void Stop(bool wait = false); @@ -158,6 +214,43 @@ class TGRpcConnectionsImpl TRequest, TResponse>::TAsyncRequest; + template + void RunResponseCallback( + TResponseCb& callback, + TResponse* response, + TPlainStatus status, + const std::shared_ptr& stopState) + { + RunGuarded(stopState, + [&] { callback(response, std::move(status)); }, + [&] { callback(nullptr, MakeClientStoppedStatus()); }); + } + + template + void RunStreamCallback( + TCallback& callback, + TPlainStatus status, + TProcessor processor, + const std::shared_ptr& stopState) + { + RunGuarded(stopState, + [&] { callback(std::move(status), std::move(processor)); }, + [&] { callback(MakeClientStoppedStatus(), nullptr); }); + } + + template + void RunServiceConnectionCallback( + TCallback& callback, + TPlainStatus status, + std::unique_ptr> serviceConnection, + TEndpointKey endpoint, + const std::shared_ptr& stopState) + { + RunGuarded(stopState, + [&] { callback(std::move(status), std::move(serviceConnection), std::move(endpoint)); }, + [&] { callback(MakeClientStoppedStatus(), std::unique_ptr>{nullptr}, TEndpointKey{}); }); + } + template class TRequestWrapper { public: @@ -218,13 +311,12 @@ class TGRpcConnectionsImpl Y_ABORT_UNLESS(dbState); if (auto tlsValidationStatus = ValidateClientTlsCredentials(dbState)) { - userResponseCb(nullptr, std::move(*tlsValidationStatus)); + RunResponseCallback(userResponseCb, nullptr, std::move(*tlsValidationStatus), StopState_); return; } if (!TryCreateContext(context)) { - TPlainStatus status(EStatus::CLIENT_CANCELLED, "Client is stopped"); - userResponseCb(nullptr, TPlainStatus{status.Status, std::move(status.Issues)}); + RunResponseCallback(userResponseCb, nullptr, MakeClientStoppedStatus(), StopState_); return; } @@ -246,79 +338,80 @@ class TGRpcConnectionsImpl WithServiceConnection( [this, requestWrapper = std::move(requestWrapper), userResponseCb = std::move(userResponseCb), rpc, requestSettings, context = std::move(context), dbState] - (TPlainStatus status, TConnection serviceConnection, TEndpointKey endpoint) mutable -> void { - if (!status.Ok()) { - userResponseCb( - nullptr, - std::move(status)); - return; - } - - Y_ABORT_UNLESS(serviceConnection != nullptr); + (TPlainStatus status, TConnection serviceConnection, TEndpointKey endpoint) mutable -> void { + if (!status.Ok()) { + context.reset(); + RunResponseCallback(userResponseCb, nullptr, std::move(status), StopState_); + return; + } - TCallMeta meta; + Y_ABORT_UNLESS(serviceConnection != nullptr); - try { - meta = MakeCallMeta(requestSettings, dbState); - } catch (const TYdbException& e) { - userResponseCb( - nullptr, - TPlainStatus(dynamic_cast(&e) ? EStatus::CLIENT_UNAUTHENTICATED : EStatus::UNAVAILABLE, e.what()) - ); - return; - } + TCallMeta meta; - dbState->StatCollector.IncGRpcInFlight(); - dbState->StatCollector.IncGRpcInFlightByHost(endpoint.GetEndpoint()); + try { + meta = MakeCallMeta(requestSettings, dbState); + } catch (const TYdbException& e) { + context.reset(); + RunResponseCallback( + userResponseCb, + nullptr, + TPlainStatus(dynamic_cast(&e) ? EStatus::CLIENT_UNAUTHENTICATED : EStatus::UNAVAILABLE, e.what()), + StopState_); + return; + } - NYdbGrpc::TAdvancedResponseCallback responseCbLow = - [this, context, userResponseCb = std::move(userResponseCb), endpoint, dbState] - (const grpc::ClientContext& ctx, TGrpcStatus&& grpcStatus, TResponse&& response) mutable -> void { - dbState->StatCollector.DecGRpcInFlight(); - dbState->StatCollector.DecGRpcInFlightByHost(endpoint.GetEndpoint()); + dbState->StatCollector.IncGRpcInFlight(); + dbState->StatCollector.IncGRpcInFlightByHost(endpoint.GetEndpoint()); - if (NYdbGrpc::IsGRpcStatusGood(grpcStatus)) { - std::multimap metadata; + NYdbGrpc::TAdvancedResponseCallback responseCbLow = + [this, context, userResponseCb = std::move(userResponseCb), endpoint, dbState] + (const grpc::ClientContext& ctx, TGrpcStatus&& grpcStatus, TResponse&& response) mutable -> void { + dbState->StatCollector.DecGRpcInFlight(); + dbState->StatCollector.DecGRpcInFlightByHost(endpoint.GetEndpoint()); - for (const auto& [name, value] : ctx.GetServerInitialMetadata()) { - metadata.emplace( - std::string(name.begin(), name.end()), - std::string(value.begin(), value.end())); - } - for (const auto& [name, value] : ctx.GetServerTrailingMetadata()) { - metadata.emplace( - std::string(name.begin(), name.end()), - std::string(value.begin(), value.end())); - } + if (NYdbGrpc::IsGRpcStatusGood(grpcStatus)) { + std::multimap metadata; - auto resp = new TResult( - std::move(response), - std::move(grpcStatus), - std::move(userResponseCb), - this, - std::move(context), - endpoint.GetEndpoint(), - std::move(metadata)); - - EnqueueResponse(resp); - } else { - dbState->StatCollector.IncReqFailDueTransportError(); - dbState->StatCollector.IncTransportErrorsByHost(endpoint.GetEndpoint()); + for (const auto& [name, value] : ctx.GetServerInitialMetadata()) { + metadata.emplace( + std::string(name.begin(), name.end()), + std::string(value.begin(), value.end())); + } + for (const auto& [name, value] : ctx.GetServerTrailingMetadata()) { + metadata.emplace( + std::string(name.begin(), name.end()), + std::string(value.begin(), value.end())); + } - auto resp = new TGRpcErrorResponse( - std::move(grpcStatus), - std::move(userResponseCb), - this, - std::move(context), - endpoint.GetEndpoint()); + auto resp = new TResult( + std::move(response), + std::move(grpcStatus), + std::move(userResponseCb), + this, + std::move(context), + endpoint.GetEndpoint(), + std::move(metadata)); + + EnqueueResponse(resp); + } else { + dbState->StatCollector.IncReqFailDueTransportError(); + dbState->StatCollector.IncTransportErrorsByHost(endpoint.GetEndpoint()); + + auto resp = new TGRpcErrorResponse( + std::move(grpcStatus), + std::move(userResponseCb), + this, + std::move(context), + endpoint.GetEndpoint()); - dbState->EndpointPool.BanEndpoint(endpoint.GetEndpoint()); + dbState->EndpointPool.BanEndpoint(endpoint.GetEndpoint()); - EnqueueResponse(resp); - } - }; + EnqueueResponse(resp); + } + }; - requestWrapper.DoRequest(serviceConnection, std::move(responseCbLow), rpc, meta, context.get()); + requestWrapper.DoRequest(serviceConnection, std::move(responseCbLow), rpc, meta, context.get()); }, dbState, requestSettings.PreferredEndpoint, requestSettings.EndpointPolicy); } @@ -334,8 +427,7 @@ class TGRpcConnectionsImpl std::shared_ptr context = nullptr) { if (!TryCreateContext(context)) { - TPlainStatus status(EStatus::CLIENT_CANCELLED, "Client is stopped"); - userResponseCb(nullptr, status); + userResponseCb(nullptr, MakeClientStoppedStatus()); return; } @@ -360,10 +452,12 @@ class TGRpcConnectionsImpl } else { NYdb::NIssue::TIssues opIssues; NYdb::NIssue::IssuesFromMessage(operation->issues(), opIssues); + context.reset(); userResponseCb(operation, TPlainStatus{static_cast(operation->status()), std::move(opIssues), status.Endpoint, std::move(status.Metadata)}); } } else { + context.reset(); userResponseCb(nullptr, status); } }; @@ -454,19 +548,20 @@ class TGRpcConnectionsImpl using TProcessor = typename NYdbGrpc::IStreamRequestReadProcessor::TPtr; if (auto tlsValidationStatus = ValidateClientTlsCredentials(dbState)) { - responseCb(std::move(*tlsValidationStatus), nullptr); + RunStreamCallback(responseCb, std::move(*tlsValidationStatus), nullptr, StopState_); return; } if (!TryCreateContext(context)) { - responseCb(TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"), nullptr); + RunStreamCallback(responseCb, MakeClientStoppedStatus(), nullptr, StopState_); return; } WithServiceConnection( [this, request, responseCb = std::move(responseCb), rpc, requestSettings, context = std::move(context), dbState](TPlainStatus status, TConnection serviceConnection, TEndpointKey endpoint) mutable { if (!status.Ok()) { - responseCb(std::move(status), nullptr); + context.reset(); + RunStreamCallback(responseCb, std::move(status), nullptr, StopState_); return; } @@ -476,17 +571,19 @@ class TGRpcConnectionsImpl try { meta = MakeCallMeta(requestSettings, dbState); } catch (const TYdbException& e) { - responseCb( + context.reset(); + RunStreamCallback( + responseCb, TPlainStatus(dynamic_cast(&e) ? EStatus::CLIENT_UNAUTHENTICATED : EStatus::UNAVAILABLE, e.what()), - nullptr - ); + nullptr, + StopState_); return; } dbState->StatCollector.IncGRpcInFlight(); dbState->StatCollector.IncGRpcInFlightByHost(endpoint.GetEndpoint()); - auto lowCallback = [responseCb = std::move(responseCb), dbState, endpoint] + auto lowCallback = [responseCb = std::move(responseCb), dbState, endpoint, stopState = StopState_] (TGrpcStatus grpcStatus, TProcessor processor) mutable { dbState->StatCollector.DecGRpcInFlight(); dbState->StatCollector.DecGRpcInFlightByHost(endpoint.GetEndpoint()); @@ -500,7 +597,9 @@ class TGRpcConnectionsImpl }; processor->AddFinishedCallback(std::move(finishedCallback)); TPlainStatus status(std::move(grpcStatus), endpoint.GetEndpoint(), {}); - responseCb(std::move(status), std::move(processor)); + RunGuarded(stopState, + [&] { responseCb(std::move(status), std::move(processor)); }, + [&] { responseCb(MakeClientStoppedStatus(), nullptr); }); } else { dbState->StatCollector.IncReqFailDueTransportError(); dbState->StatCollector.IncTransportErrorsByHost(endpoint.GetEndpoint()); @@ -508,7 +607,9 @@ class TGRpcConnectionsImpl dbState->EndpointPool.BanEndpoint(endpoint.GetEndpoint()); } TPlainStatus status(std::move(grpcStatus), endpoint.GetEndpoint(), {}); - responseCb(std::move(status), nullptr); + RunGuarded(stopState, + [&] { responseCb(std::move(status), nullptr); }, + [&] { responseCb(MakeClientStoppedStatus(), nullptr); }); } }; @@ -534,70 +635,77 @@ class TGRpcConnectionsImpl using TProcessor = typename NYdbGrpc::IStreamRequestReadWriteProcessor::TPtr; if (auto tlsValidationStatus = ValidateClientTlsCredentials(dbState)) { - connectedCallback(std::move(*tlsValidationStatus), nullptr); + RunStreamCallback(connectedCallback, std::move(*tlsValidationStatus), nullptr, StopState_); return; } if (!TryCreateContext(context)) { - connectedCallback(TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"), nullptr); + RunStreamCallback(connectedCallback, MakeClientStoppedStatus(), nullptr, StopState_); return; } WithServiceConnection( [this, connectedCallback = std::move(connectedCallback), rpc, requestSettings, context = std::move(context), dbState] - (TPlainStatus status, TConnection serviceConnection, TEndpointKey endpoint) mutable { - if (!status.Ok()) { - connectedCallback(std::move(status), nullptr); - return; - } - - Y_ABORT_UNLESS(serviceConnection != nullptr); - - TCallMeta meta; - try { - meta = MakeCallMeta(requestSettings, dbState); - } catch (const TYdbException& e) { - connectedCallback( - TPlainStatus(dynamic_cast(&e) ? EStatus::CLIENT_UNAUTHENTICATED : EStatus::UNAVAILABLE, e.what()), - nullptr - ); - return; - } - - dbState->StatCollector.IncGRpcInFlight(); - dbState->StatCollector.IncGRpcInFlightByHost(endpoint.GetEndpoint()); + (TPlainStatus status, TConnection serviceConnection, TEndpointKey endpoint) mutable { + if (!status.Ok()) { + context.reset(); + RunStreamCallback(connectedCallback, std::move(status), nullptr, StopState_); + return; + } - auto lowCallback = [connectedCallback = std::move(connectedCallback), dbState, endpoint] - (TGrpcStatus grpcStatus, TProcessor processor) { - dbState->StatCollector.DecGRpcInFlight(); - dbState->StatCollector.DecGRpcInFlightByHost(endpoint.GetEndpoint()); + Y_ABORT_UNLESS(serviceConnection != nullptr); + + TCallMeta meta; + try { + meta = MakeCallMeta(requestSettings, dbState); + } catch (const TYdbException& e) { + context.reset(); + RunStreamCallback( + connectedCallback, + TPlainStatus(dynamic_cast(&e) ? EStatus::CLIENT_UNAUTHENTICATED : EStatus::UNAVAILABLE, e.what()), + nullptr, + StopState_); + return; + } - if (grpcStatus.Ok()) { - Y_ABORT_UNLESS(processor); - auto finishedCallback = [dbState, endpoint] (TGrpcStatus grpcStatus) { - if (!grpcStatus.Ok() && grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { + dbState->StatCollector.IncGRpcInFlight(); + dbState->StatCollector.IncGRpcInFlightByHost(endpoint.GetEndpoint()); + + auto lowCallback = [connectedCallback = std::move(connectedCallback), dbState, endpoint, stopState = StopState_] + (TGrpcStatus grpcStatus, TProcessor processor) { + dbState->StatCollector.DecGRpcInFlight(); + dbState->StatCollector.DecGRpcInFlightByHost(endpoint.GetEndpoint()); + + if (grpcStatus.Ok()) { + Y_ABORT_UNLESS(processor); + auto finishedCallback = [dbState, endpoint] (TGrpcStatus grpcStatus) { + if (!grpcStatus.Ok() && grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { + dbState->EndpointPool.BanEndpoint(endpoint.GetEndpoint()); + } + }; + processor->AddFinishedCallback(std::move(finishedCallback)); + TPlainStatus status(std::move(grpcStatus), endpoint.GetEndpoint(), {}); + RunGuarded(stopState, + [&] { connectedCallback(std::move(status), std::move(processor)); }, + [&] { connectedCallback(MakeClientStoppedStatus(), nullptr); }); + } else { + dbState->StatCollector.IncReqFailDueTransportError(); + dbState->StatCollector.IncTransportErrorsByHost(endpoint.GetEndpoint()); + if (grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { dbState->EndpointPool.BanEndpoint(endpoint.GetEndpoint()); } - }; - processor->AddFinishedCallback(std::move(finishedCallback)); - TPlainStatus status(std::move(grpcStatus), endpoint.GetEndpoint(), {}); - connectedCallback(std::move(status), std::move(processor)); - } else { - dbState->StatCollector.IncReqFailDueTransportError(); - dbState->StatCollector.IncTransportErrorsByHost(endpoint.GetEndpoint()); - if (grpcStatus.GRpcStatusCode != grpc::StatusCode::CANCELLED) { - dbState->EndpointPool.BanEndpoint(endpoint.GetEndpoint()); + TPlainStatus status(std::move(grpcStatus), endpoint.GetEndpoint(), {}); + RunGuarded(stopState, + [&] { connectedCallback(std::move(status), nullptr); }, + [&] { connectedCallback(MakeClientStoppedStatus(), nullptr); }); } - TPlainStatus status(std::move(grpcStatus), endpoint.GetEndpoint(), {}); - connectedCallback(std::move(status), nullptr); - } - }; + }; - serviceConnection->template DoStreamRequest( - std::move(lowCallback), - std::move(rpc), - std::move(meta), - context.get()); + serviceConnection->template DoStreamRequest( + std::move(lowCallback), + std::move(rpc), + std::move(meta), + context.get()); }, dbState, requestSettings.PreferredEndpoint, requestSettings.EndpointPolicy); } @@ -653,11 +761,12 @@ class TGRpcConnectionsImpl errString << "No endpoint for database " << dbState->Database; errString << ", cluster endpoint " << dbState->DiscoveryEndpoint; dbState->StatCollector.IncReqFailNoEndpoint(); - callback( + RunServiceConnectionCallback( + callback, TPlainStatus(EStatus::UNAVAILABLE, errString.Str()), TConnection{nullptr}, - TEndpointKey{ }); - + TEndpointKey{}, + StopState_); } else if (dbState->DiscoveryMode == EDiscoveryMode::Sync) { TStringStream errString; errString << "Endpoint list is empty for database " << dbState->Database; @@ -673,14 +782,16 @@ class TGRpcConnectionsImpl errString << " while last discovery returned success status. Unable to continue processing."; discoveryStatus = TPlainStatus(EStatus::UNAVAILABLE, errString.Str()); } else { - errString <<"."; + errString << "."; discoveryStatus.Issues.AddIssues({NYdb::NIssue::TIssue(errString.Str())}); } dbState->StatCollector.IncReqFailNoEndpoint(); - callback( - discoveryStatus, + RunServiceConnectionCallback( + callback, + std::move(discoveryStatus), TConnection{nullptr}, - TEndpointKey{ }); + TEndpointKey{}, + StopState_); } else { int64_t newVal; int64_t val; @@ -688,48 +799,58 @@ class TGRpcConnectionsImpl val = QueuedRequests_.load(); if (val >= MaxQueuedRequests_) { dbState->StatCollector.IncReqFailQueueOverflow(); - callback( + RunServiceConnectionCallback( + callback, TPlainStatus(EStatus::CLIENT_LIMITS_REACHED, "Requests queue limit reached"), TConnection{nullptr}, - TEndpointKey{ }); + TEndpointKey{}, + StopState_); return; } newVal = val + 1; } while (!QueuedRequests_.compare_exchange_weak(val, newVal)); - // UpdateAsync guarantee one update in progress for state + // UpdateAsync guarantees one update in progress for state. auto asyncResult = dbState->EndpointPool.UpdateAsync(); const bool needUpdateChannels = asyncResult.second; - asyncResult.first.Subscribe([this, callback = std::move(callback), needUpdateChannels, dbState, preferredEndpoint, endpointPolicy] + auto stopState = StopState_; + asyncResult.first.Subscribe([this, callback = std::move(callback), needUpdateChannels, dbState, preferredEndpoint, endpointPolicy, stopState = std::move(stopState)] (const NThreading::TFuture& future) mutable { --QueuedRequests_; - const auto& updateResult = future.GetValue(); - if (needUpdateChannels) { + RunGuarded(stopState, + [&] { + const auto& updateResult = future.GetValue(); + if (needUpdateChannels) { #ifndef YDB_GRPC_BYPASS_CHANNEL_POOL - DeleteChannels(updateResult.Removed); + DeleteChannels(updateResult.Removed); #endif - } - auto discoveryStatus = updateResult.DiscoveryStatus; - if (discoveryStatus.Status == EStatus::SUCCESS) { - WithServiceConnection(std::move(callback), dbState, preferredEndpoint, endpointPolicy); - } else { - callback( - TPlainStatus(discoveryStatus.Status, std::move(discoveryStatus.Issues)), - TConnection{nullptr}, - TEndpointKey{ }); - } + } + auto discoveryStatus = updateResult.DiscoveryStatus; + if (discoveryStatus.Status == EStatus::SUCCESS) { + WithServiceConnection(std::move(callback), dbState, preferredEndpoint, endpointPolicy); + } else { + callback( + TPlainStatus(discoveryStatus.Status, std::move(discoveryStatus.Issues)), + TConnection{nullptr}, + TEndpointKey{}); + } + }, + [&] { callback(MakeClientStoppedStatus(), TConnection{nullptr}, TEndpointKey{}); }); }); } return; } - callback( - TPlainStatus{ }, + RunServiceConnectionCallback( + callback, + TPlainStatus{}, std::move(serviceConnection), - std::move(endpoint)); + std::move(endpoint), + StopState_); } void EnqueueResponse(IObjectInQueue* action); + void StopResponseQueue(); private: TCallMeta MakeCallMeta(const TRpcRequestSettings& requestSettings, const TDbDriverStatePtr& dbState) const; @@ -739,6 +860,8 @@ class TGRpcConnectionsImpl const std::size_t ClientThreadsNum_; std::shared_ptr ResponseQueue_; + std::once_flag ResponseQueueStopOnce_; + std::shared_ptr StopState_; const std::string DefaultDiscoveryEndpoint_; const TSslCredentials SslCredentials_; @@ -785,4 +908,8 @@ class TGRpcConnectionsImpl TLog Log; }; +struct TGRpcConnectionsDeleter { + void operator()(TGRpcConnectionsImpl* connections) const noexcept; +}; + } // namespace NYdb diff --git a/src/client/table/impl/table_client.cpp b/src/client/table/impl/table_client.cpp index 9a6970699f..d73ea40c0e 100644 --- a/src/client/table/impl/table_client.cpp +++ b/src/client/table/impl/table_client.cpp @@ -11,6 +11,14 @@ using namespace NThreading; const TKeepAliveSettings TTableClient::TImpl::KeepAliveSettings = TKeepAliveSettings().ClientTimeout(KEEP_ALIVE_CLIENT_TIMEOUT); +namespace { + NThreading::TFuture MakeReadyFuture() { + auto promise = NThreading::NewPromise(); + auto future = promise.GetFuture(); + promise.SetValue(); + return future; + } +} TDuration GetMinTimeToTouch(const TSessionPoolSettings& settings) { return Min(settings.CloseIdleThreshold_, settings.KeepAliveIdleThreshold_); @@ -73,7 +81,11 @@ std::shared_ptr TTableClient::TImpl::CreateRetryAt TTableClient::TImpl::~TImpl() { if (Connections_->GetDrainOnDtors()) { - Drain().Wait(DRAIN_TIMEOUT); + const bool closeRemote = !TGRpcConnectionsImpl::IsCurrentThreadInSdkCallback(); + auto drainFuture = Drain(closeRemote); + if (closeRemote) { + drainFuture.Wait(DRAIN_TIMEOUT); + } } } @@ -86,9 +98,7 @@ void TTableClient::TImpl::InitStopper() { auto cb = [weak]() mutable { auto strong = weak.lock(); if (!strong) { - auto promise = NThreading::NewPromise(); - promise.SetException("no more client"); - return promise.GetFuture(); + return MakeReadyFuture(); } return strong->Drain(); }; @@ -96,7 +106,7 @@ void TTableClient::TImpl::InitStopper() { DbDriverState_->AddCb(std::move(cb), TDbDriverState::ENotifyType::STOP); } -NThreading::TFuture TTableClient::TImpl::Drain() { +NThreading::TFuture TTableClient::TImpl::Drain(bool closeRemote) { std::vector> sessions; // No realocations under lock sessions.reserve(Settings_.SessionPoolSettings_.MaxActiveSessions_); @@ -108,10 +118,16 @@ NThreading::TFuture TTableClient::TImpl::Drain() { std::vector closeResults; for (auto& s : sessions) { if (!s->GetId().empty()) { - closeResults.push_back(CloseInternal(s.get())); + if (closeRemote) { + closeResults.push_back(CloseInternal(s.get())); + } + DbDriverState_->StatCollector.DecSessionsOnHost(s->GetEndpoint()); } } sessions.clear(); + if (closeResults.empty()) { + return MakeReadyFuture(); + } return NThreading::WaitExceptionOrAll(closeResults); } @@ -1150,8 +1166,11 @@ void TTableClient::TImpl::DeleteSession(TKqpSessionCommon* sessionImpl) { SessionPool_.DecrementActiveCounter(); } + const bool closeRemote = !TGRpcConnectionsImpl::IsCurrentThreadInSdkCallback(); if (!sessionImpl->GetId().empty()) { - CloseInternal(sessionImpl); + if (closeRemote) { + CloseInternal(sessionImpl); + } DbDriverState_->StatCollector.DecSessionsOnHost(sessionImpl->GetEndpoint()); } diff --git a/src/client/table/impl/table_client.h b/src/client/table/impl/table_client.h index c9d77b9afc..a0960be7ba 100644 --- a/src/client/table/impl/table_client.h +++ b/src/client/table/impl/table_client.h @@ -21,7 +21,6 @@ #include - namespace NYdb::inline V3 { namespace NTable { @@ -45,7 +44,7 @@ class TTableClient::TImpl: public TClientImplCommon, public bool LinkObjToEndpoint(const TEndpointKey& endpoint, TEndpointObj* obj, const void* tag); void InitStopper(); - NThreading::TFuture Drain(); + NThreading::TFuture Drain(bool closeRemote = true); NThreading::TFuture Stop(); void ScheduleTaskUnsafe(std::function&& fn, TDeadline::Duration timeout); void StartPeriodicSessionPoolTask(); diff --git a/src/client/topic/impl/direct_reader.cpp b/src/client/topic/impl/direct_reader.cpp index 4cd2388ea5..332e0659de 100644 --- a/src/client/topic/impl/direct_reader.cpp +++ b/src/client/topic/impl/direct_reader.cpp @@ -588,7 +588,8 @@ void TDirectReadSession::OnReadDone(NYdbGrpc::TGrpcStatus&& grpcStatus, size_t c cbContext = SelfContext, partitionSessionId = partitionSessionId.value() ]() { callbacks->OnDirectReadDone(messages); - } + }, + ClientContext->GetCallbackGuardFactory() ); } diff --git a/src/client/topic/impl/read_session_impl.h b/src/client/topic/impl/read_session_impl.h index 5ea791a07a..de0282b953 100644 --- a/src/client/topic/impl/read_session_impl.h +++ b/src/client/topic/impl/read_session_impl.h @@ -150,7 +150,9 @@ class TDeferredActions { // TODO(qyryq) Extract a separate TDeferredDirectReadActions class? void DeferReadFromProcessor(const typename IDirectReadProcessor::TPtr& processor, TDirectReadServerMessage* dst, typename IDirectReadProcessor::TReadCallback callback); void DeferScheduleCallback(TDuration delay, std::function callback, TSingleClusterReadSessionContextPtr); - void DeferCallback(std::function callback); + void DeferCallback( + std::function callback, + NYdbGrpc::TQueueClientCallbackGuardFactory callbackGuardFactory = {}); void DeferReadFromProcessor(const typename IProcessor::TPtr& processor, TServerMessage* dst, typename IProcessor::TReadCallback callback); void DeferStartExecutorTask(const typename IExecutor::TPtr& executor, typename IExecutor::TFunction&& task); @@ -202,7 +204,12 @@ class TDeferredActions { }; std::optional ScheduledCallback; - std::optional> Callback; + struct TCallback { + std::function Callback; + NYdbGrpc::TQueueClientCallbackGuardFactory CallbackGuardFactory; + }; + + std::optional Callback; } DirectReadActions; // Executor tasks. diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index 5160353776..8ed3d76e19 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -3657,9 +3657,15 @@ void TDeferredActions::DeferScheduleCallback(TDuration del } template -void TDeferredActions::DeferCallback(std::function callback) { +void TDeferredActions::DeferCallback( + std::function callback, + NYdbGrpc::TQueueClientCallbackGuardFactory callbackGuardFactory) +{ Y_ASSERT(!DirectReadActions.Callback); - DirectReadActions.Callback = std::move(callback); + DirectReadActions.Callback = typename TDirectReadDeferredActions::TCallback{ + std::move(callback), + std::move(callbackGuardFactory) + }; } template @@ -3789,7 +3795,9 @@ template void TDeferredActions::DirectReadCallback() { auto& callback = DirectReadActions.Callback; if (callback) { - (*callback)(); + NYdbGrpc::RunQueueClientCallback(callback->CallbackGuardFactory, [&] { + callback->Callback(); + }); } } diff --git a/src/library/grpc/client/grpc_client_low.h b/src/library/grpc/client/grpc_client_low.h index 07fb200c5f..42852001e8 100644 --- a/src/library/grpc/client/grpc_client_low.h +++ b/src/library/grpc/client/grpc_client_low.h @@ -14,6 +14,8 @@ #include #include +#include +#include #include #include #include @@ -93,14 +95,48 @@ class TQueueClientFixedEvent : private IQueueClientEvent { class IQueueClientContext; using IQueueClientContextPtr = std::shared_ptr; +class IQueueClientCallbackGuard { +public: + virtual ~IQueueClientCallbackGuard() = default; + virtual bool IsEntered() const noexcept = 0; +}; + +class TNoopQueueClientCallbackGuard final : public IQueueClientCallbackGuard { +public: + bool IsEntered() const noexcept override { + return true; + } +}; + +using TQueueClientCallbackGuardFactory = std::function()>; + // Provider of IQueueClientContext instances class IQueueClientContextProvider { public: virtual ~IQueueClientContextProvider() = default; virtual IQueueClientContextPtr CreateContext() = 0; + + virtual TQueueClientCallbackGuardFactory GetCallbackGuardFactory() { + return [] { + return std::make_unique(); + }; + } }; +template +void RunQueueClientCallback(const TQueueClientCallbackGuardFactory& guardFactory, F&& f) { + std::unique_ptr guard; + if (guardFactory) { + guard = guardFactory(); + } else { + guard = std::make_unique(); + } + if (guard->IsEntered()) { + f(); + } +} + // Activity context for a low-level client class IQueueClientContext : public IQueueClientContextProvider { public: @@ -210,9 +246,19 @@ class TGRpcRequestProcessorCommon { void GetInitialMetadata(std::unordered_multimap* metadata); + void InitCallbackGuard(IQueueClientContextProvider* provider) { + CallbackGuardFactory_ = provider->GetCallbackGuardFactory(); + } + + template + void RunGuarded(F&& f) { + RunQueueClientCallback(CallbackGuardFactory_, std::forward(f)); + } + grpc::Status Status; grpc::ClientContext Context; std::shared_ptr LocalContext; + TQueueClientCallbackGuardFactory CallbackGuardFactory_; }; template @@ -232,7 +278,9 @@ class TSimpleRequestProcessor ~TSimpleRequestProcessor() { if (!Replied_ && Callback_) { - Callback_(TGrpcStatus::Internal("request left unhandled"), std::move(Reply_)); + RunGuarded([&] { + Callback_(TGrpcStatus::Internal("request left unhandled"), std::move(Reply_)); + }); Callback_ = nullptr; // free resources as early as possible } } @@ -249,7 +297,9 @@ class TSimpleRequestProcessor status = TGrpcStatus::Internal("Unexpected error"); } Replied_ = true; - Callback_(std::move(status), std::move(Reply_)); + RunGuarded([&] { + Callback_(std::move(status), std::move(Reply_)); + }); Callback_ = nullptr; // free resources as early as possible return false; } @@ -265,10 +315,13 @@ class TSimpleRequestProcessor } void Start(TStub& stub, TAsyncRequest asyncRequest, const TRequest& request, IQueueClientContextProvider* provider) { + InitCallbackGuard(provider); auto context = provider->CreateContext(); if (!context) { Replied_ = true; - Callback_(TGrpcStatus(grpc::StatusCode::CANCELLED, "Client is shutting down"), std::move(Reply_)); + RunGuarded([&] { + Callback_(TGrpcStatus(grpc::StatusCode::CANCELLED, "Client is shutting down"), std::move(Reply_)); + }); Callback_ = nullptr; return; } @@ -312,7 +365,9 @@ class TAdvancedRequestProcessor ~TAdvancedRequestProcessor() { if (!Replied_ && Callback_) { - Callback_(Context, TGrpcStatus::Internal("request left unhandled"), std::move(Reply_)); + RunGuarded([&] { + Callback_(Context, TGrpcStatus::Internal("request left unhandled"), std::move(Reply_)); + }); Callback_ = nullptr; // free resources as early as possible } } @@ -329,7 +384,9 @@ class TAdvancedRequestProcessor status = TGrpcStatus::Internal("Unexpected error"); } Replied_ = true; - Callback_(Context, std::move(status), std::move(Reply_)); + RunGuarded([&] { + Callback_(Context, std::move(status), std::move(Reply_)); + }); Callback_ = nullptr; // free resources as early as possible return false; } @@ -345,10 +402,13 @@ class TAdvancedRequestProcessor } void Start(TStub& stub, TAsyncRequest asyncRequest, const TRequest& request, IQueueClientContextProvider* provider) { + InitCallbackGuard(provider); auto context = provider->CreateContext(); if (!context) { Replied_ = true; - Callback_(Context, TGrpcStatus(grpc::StatusCode::CANCELLED, "Client is shutting down"), std::move(Reply_)); + RunGuarded([&] { + Callback_(Context, TGrpcStatus(grpc::StatusCode::CANCELLED, "Client is shutting down"), std::move(Reply_)); + }); Callback_ = nullptr; return; } @@ -585,7 +645,9 @@ class TStreamRequestReadProcessor } } - callback(std::move(status)); + RunGuarded([&] { + callback(std::move(status)); + }); } void Read(TResponse* message, TReadCallback callback) override { @@ -613,7 +675,9 @@ class TStreamRequestReadProcessor status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); } - callback(std::move(status)); + RunGuarded([&] { + callback(std::move(status)); + }); } void Finish(TReadCallback callback) override { @@ -638,7 +702,9 @@ class TStreamRequestReadProcessor } } - callback(std::move(status)); + RunGuarded([&] { + callback(std::move(status)); + }); } void AddFinishedCallback(TReadCallback callback) override { @@ -662,16 +728,21 @@ class TStreamRequestReadProcessor } } - callback(std::move(status)); + RunGuarded([&] { + callback(std::move(status)); + }); } private: void Start(TStub& stub, const TRequest& request, TAsyncRequest asyncRequest, IQueueClientContextProvider* provider) { + InitCallbackGuard(provider); auto context = provider->CreateContext(); if (!context) { auto callback = std::move(Callback); TGrpcStatus status(grpc::StatusCode::CANCELLED, "Client is shutting down"); - callback(std::move(status), nullptr); + RunGuarded([&] { + callback(std::move(status), nullptr); + }); return; } @@ -719,7 +790,9 @@ class TStreamRequestReadProcessor GetInitialMetadata(initialMetadata); } - callback(std::move(status)); + RunGuarded([&] { + callback(std::move(status)); + }); } void OnStartDone(bool ok) { @@ -737,7 +810,9 @@ class TStreamRequestReadProcessor Callback = nullptr; } - callback({ }, typename TBase::TPtr(this)); + RunGuarded([&] { + callback({ }, typename TBase::TPtr(this)); + }); } void OnFinished(bool ok) { @@ -782,14 +857,18 @@ class TStreamRequestReadProcessor for (auto& finishedCallback : finishedCallbacks) { auto statusCopy = status; - finishedCallback(std::move(statusCopy)); + RunGuarded([&] { + finishedCallback(std::move(statusCopy)); + }); } if (startCallback) { if (status.Ok()) { status = TGrpcStatus(grpc::StatusCode::UNKNOWN, "Unknown stream failure"); } - startCallback(std::move(status), nullptr); + RunGuarded([&] { + startCallback(std::move(status), nullptr); + }); } else if (readCallback) { if (status.Ok()) { status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); @@ -799,9 +878,13 @@ class TStreamRequestReadProcessor std::string(value.begin(), value.end())); } } - readCallback(std::move(status)); + RunGuarded([&] { + readCallback(std::move(status)); + }); } else if (finishCallback) { - finishCallback(std::move(status)); + RunGuarded([&] { + finishCallback(std::move(status)); + }); } } @@ -888,7 +971,9 @@ class TStreamRequestReadWriteProcessor } if (!status.Ok() && callback) { - callback(std::move(status)); + RunGuarded([&] { + callback(std::move(status)); + }); } } @@ -918,7 +1003,9 @@ class TStreamRequestReadWriteProcessor } } - callback(std::move(status)); + RunGuarded([&] { + callback(std::move(status)); + }); } void Read(TResponse* message, TReadCallback callback) override { @@ -946,7 +1033,9 @@ class TStreamRequestReadWriteProcessor status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); } - callback(std::move(status)); + RunGuarded([&] { + callback(std::move(status)); + }); } void Finish(TReadCallback callback) override { @@ -976,7 +1065,9 @@ class TStreamRequestReadWriteProcessor } } - callback(std::move(status)); + RunGuarded([&] { + callback(std::move(status)); + }); } void AddFinishedCallback(TReadCallback callback) override { @@ -1000,18 +1091,23 @@ class TStreamRequestReadWriteProcessor } } - callback(std::move(status)); + RunGuarded([&] { + callback(std::move(status)); + }); } private: template friend class TServiceConnection; void Start(TStub& stub, TAsyncRequest asyncRequest, IQueueClientContextProvider* provider) { + InitCallbackGuard(provider); auto context = provider->CreateContext(); if (!context) { auto callback = std::move(ConnectedCallback); TGrpcStatus status(grpc::StatusCode::CANCELLED, "Client is shutting down"); - callback(std::move(status), nullptr); + RunGuarded([&] { + callback(std::move(status), nullptr); + }); return; } @@ -1044,7 +1140,9 @@ class TStreamRequestReadWriteProcessor ConnectedCallback = nullptr; } - callback({ }, typename TBase::TPtr(this)); + RunGuarded([&] { + callback({ }, typename TBase::TPtr(this)); + }); } void OnReadDone(bool ok) { @@ -1084,7 +1182,9 @@ class TStreamRequestReadWriteProcessor GetInitialMetadata(initialMetadata); } - callback(std::move(status)); + RunGuarded([&] { + callback(std::move(status)); + }); } void OnWriteDone(bool ok) { @@ -1123,7 +1223,9 @@ class TStreamRequestReadWriteProcessor } if (okCallback) { - okCallback(TGrpcStatus()); + RunGuarded([&] { + okCallback(TGrpcStatus()); + }); } } @@ -1174,20 +1276,26 @@ class TStreamRequestReadWriteProcessor if (writeStatus.Ok()) { writeStatus = TGrpcStatus(grpc::StatusCode::CANCELLED, "Write request dropped"); } - item.Callback(std::move(writeStatus)); + RunGuarded([&] { + item.Callback(std::move(writeStatus)); + }); } } for (auto& finishedCallback : finishedCallbacks) { TGrpcStatus statusCopy = status; - finishedCallback(std::move(statusCopy)); + RunGuarded([&] { + finishedCallback(std::move(statusCopy)); + }); } if (connectedCallback) { if (status.Ok()) { status = TGrpcStatus(grpc::StatusCode::UNKNOWN, "Unknown stream failure"); } - connectedCallback(std::move(status), nullptr); + RunGuarded([&] { + connectedCallback(std::move(status), nullptr); + }); } else if (readCallback) { if (status.Ok()) { status = TGrpcStatus(grpc::StatusCode::OUT_OF_RANGE, "Read EOF"); @@ -1197,9 +1305,13 @@ class TStreamRequestReadWriteProcessor std::string(value.begin(), value.end())); } } - readCallback(std::move(status)); + RunGuarded([&] { + readCallback(std::move(status)); + }); } else if (finishCallback) { - finishCallback(std::move(status)); + RunGuarded([&] { + finishCallback(std::move(status)); + }); } } diff --git a/tests/unit/client/coordination/coordination_ut.cpp b/tests/unit/client/coordination/coordination_ut.cpp index 75a8dcb1b1..ad510f658a 100644 --- a/tests/unit/client/coordination/coordination_ut.cpp +++ b/tests/unit/client/coordination/coordination_ut.cpp @@ -81,7 +81,10 @@ namespace { size_t pings_received = 0; while (stream->Read(&request)) { std::cerr << "Session request: " << request.ShortDebugString() << std::endl; - Y_ABORT_UNLESS(request.has_ping(), "Only ping requests are supported"); + if (request.has_session_stop()) { + return grpc::Status::OK; + } + Y_ABORT_UNLESS(request.has_ping(), "Only ping and stop requests are supported"); if (++pings_received <= 2) { // Only reply to the first 2 ping requests Ydb::Coordination::SessionResponse response; @@ -295,4 +298,49 @@ Y_UNIT_TEST_SUITE(Coordination) { UNIT_ASSERT_VALUES_EQUAL_C(res2.GetStatus(), EStatus::CLIENT_CANCELLED, res2.GetIssues().ToString()); } + Y_UNIT_TEST(SessionDropsDriverFromStateCallback) { + TPortManager pm; + + TMockCoordinationService coordinationService; + ui16 coordinationPort = pm.GetPort(); + auto coordinationServer = StartGrpcServer( + TStringBuilder() << "0.0.0.0:" << coordinationPort, + coordinationService); + + TMockDiscoveryService discoveryService; + { + auto& dbResult = discoveryService.MockResults["/Root/My/DB"]; + auto* endpoint = dbResult.add_endpoints(); + endpoint->set_address("localhost"); + endpoint->set_port(coordinationPort); + } + + ui16 discoveryPort = pm.GetPort(); + auto discoveryServer = StartGrpcServer( + TStringBuilder() << "0.0.0.0:" << discoveryPort, + discoveryService); + + auto config = TDriverConfig() + .SetEndpoint(TStringBuilder() << "localhost:" << discoveryPort) + .SetDatabase("/Root/My/DB"); + std::optional driver(std::in_place, config); + std::optional client(std::in_place, *driver); + + auto droppedPromise = NThreading::NewPromise(); + auto droppedFuture = droppedPromise.GetFuture(); + auto settings = TSessionSettings() + .OnStateChanged([&](auto state) mutable { + if (state == ESessionState::ATTACHED) { + client.reset(); + driver.reset(); + droppedPromise.SetValue(); + } + }) + .Timeout(TDuration::MilliSeconds(1000)); + + auto res = client->StartSession("/Some/Path", settings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::SUCCESS, res.GetIssues().ToString()); + UNIT_ASSERT(droppedFuture.Wait(TDuration::Seconds(10))); + } + } diff --git a/tests/unit/client/table/table_ut.cpp b/tests/unit/client/table/table_ut.cpp index 1f2bd3198b..b013f16860 100644 --- a/tests/unit/client/table/table_ut.cpp +++ b/tests/unit/client/table/table_ut.cpp @@ -13,6 +13,11 @@ #include +#include +#include +#include +#include + using namespace NYdb; namespace { @@ -59,6 +64,11 @@ namespace { // + if (CreateTableStarted) { + CreateTableStarted->set_value(); + ContinueCreateTable.wait(); + } + auto op = response->mutable_operation(); op->set_ready(true); @@ -69,6 +79,24 @@ namespace { return grpc::Status::OK; } + virtual grpc::Status DeleteSession( + grpc::ServerContext* /* context */, + const Ydb::Table::DeleteSessionRequest* request, + Ydb::Table::DeleteSessionResponse* response + ) override { + std::cerr << "DeleteSession():" << std::endl + << request->DebugString() + << std::endl; + + ++DeleteSessionRequests; + + auto op = response->mutable_operation(); + op->set_ready(true); + op->set_status(Ydb::StatusIds::SUCCESS); + + return grpc::Status::OK; + } + virtual grpc::Status AlterTable( grpc::ServerContext* /* context */, const Ydb::Table::AlterTableRequest* request, @@ -92,6 +120,9 @@ namespace { std::optional LastCreateTableRequest; std::optional LastAlterTableRequest; + std::atomic_uint DeleteSessionRequests = 0; + std::promise* CreateTableStarted = nullptr; + std::shared_future ContinueCreateTable; }; /** @@ -112,6 +143,18 @@ namespace { .BuildAndStart(); } + template + bool WaitUntil(TPredicate&& predicate, std::chrono::milliseconds timeout = std::chrono::seconds(10)) { + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) { + if (predicate()) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + return predicate(); + } + /** * Configure and start a local GRPC server with the mocked table API service. * @@ -160,6 +203,306 @@ namespace { } // namespace +TEST(TableTest, SessionHandleDestructionSendsDeleteSession) { + TMockTableService tableService; + std::unique_ptr grpcServer; + std::unique_ptr driver; + std::unique_ptr tableClient; + std::unique_ptr tableSession; + + StartServerWithTableService( + tableService, + grpcServer, + driver, + tableClient, + tableSession + ); + + tableSession.reset(); + ASSERT_TRUE(WaitUntil([&] { + return tableService.DeleteSessionRequests.load() == 1u; + })); + + tableClient.reset(); + driver.reset(); +} + +TEST(TableTest, ClientDestructorSendsDeleteSessionForPooledSessions) { + TMockTableService tableService; + std::unique_ptr grpcServer; + std::unique_ptr driver; + std::unique_ptr tableClient; + std::unique_ptr tableSession; + + StartServerWithTableService( + tableService, + grpcServer, + driver, + tableClient, + tableSession + ); + + tableSession.reset(); + ASSERT_TRUE(WaitUntil([&] { + return tableService.DeleteSessionRequests.load() == 1u; + })); + tableService.DeleteSessionRequests.store(0); + + { + auto pooledSessionResult = tableClient->GetSession().ExtractValueSync(); + ASSERT_TRUE(pooledSessionResult.IsSuccess()); + auto pooledSession = pooledSessionResult.GetSession(); + } + + tableClient.reset(); + ASSERT_TRUE(WaitUntil([&] { + return tableService.DeleteSessionRequests.load() == 1u; + })); + + driver.reset(); +} + +TEST(TableTest, ExplicitStopClosesPooledSessions) { + TMockTableService tableService; + std::unique_ptr grpcServer; + std::unique_ptr driver; + std::unique_ptr tableClient; + std::unique_ptr tableSession; + + StartServerWithTableService( + tableService, + grpcServer, + driver, + tableClient, + tableSession + ); + + tableSession.reset(); + ASSERT_TRUE(WaitUntil([&] { + return tableService.DeleteSessionRequests.load() == 1u; + })); + tableService.DeleteSessionRequests.store(0); + + { + auto pooledSessionResult = tableClient->GetSession().ExtractValueSync(); + ASSERT_TRUE(pooledSessionResult.IsSuccess()); + auto pooledSession = pooledSessionResult.GetSession(); + } + + ASSERT_EQ(tableService.DeleteSessionRequests.load(), 0u); + + ASSERT_TRUE(tableClient->Stop().Wait(TDuration::Seconds(10))); + ASSERT_EQ(tableService.DeleteSessionRequests.load(), 1u); +} + +TEST(TableTest, CheckedOutPooledSessionClosesRemotelyAfterExplicitStop) { + TMockTableService tableService; + std::unique_ptr grpcServer; + std::unique_ptr driver; + std::unique_ptr tableClient; + std::unique_ptr tableSession; + + StartServerWithTableService( + tableService, + grpcServer, + driver, + tableClient, + tableSession + ); + + tableSession.reset(); + ASSERT_TRUE(WaitUntil([&] { + return tableService.DeleteSessionRequests.load() == 1u; + })); + tableService.DeleteSessionRequests.store(0); + + { + auto pooledSessionResult = tableClient->GetSession().ExtractValueSync(); + ASSERT_TRUE(pooledSessionResult.IsSuccess()); + auto pooledSession = pooledSessionResult.GetSession(); + + ASSERT_TRUE(tableClient->Stop().Wait(TDuration::Seconds(10))); + ASSERT_EQ(tableService.DeleteSessionRequests.load(), 0u); + } + + ASSERT_TRUE(WaitUntil([&] { + return tableService.DeleteSessionRequests.load() == 1u; + })); +} + +TEST(TableTest, DriverStopFromResponseCallbackRunsStopNotifications) { + TMockTableService tableService; + std::unique_ptr grpcServer; + std::unique_ptr driver; + std::unique_ptr tableClient; + std::unique_ptr tableSession; + + StartServerWithTableService( + tableService, + grpcServer, + driver, + tableClient, + tableSession + ); + + { + auto pooledSessionResult = tableClient->GetSession().ExtractValueSync(); + ASSERT_TRUE(pooledSessionResult.IsSuccess()); + auto pooledSession = pooledSessionResult.GetSession(); + } + + std::promise createTableStarted; + auto createTableStartedFuture = createTableStarted.get_future(); + std::promise continueCreateTable; + tableService.CreateTableStarted = &createTableStarted; + tableService.ContinueCreateTable = continueCreateTable.get_future().share(); + + std::promise callbackDone; + auto callbackDoneFuture = callbackDone.get_future(); + std::atomic_bool success = false; + + auto requestFuture = tableSession->CreateTable( + "/Root/My/DB/driver_stop_from_callback", + NTable::TTableBuilder().Build() + ); + + ASSERT_EQ(createTableStartedFuture.wait_for(std::chrono::seconds(10)), std::future_status::ready); + + requestFuture.Subscribe([&](const NThreading::TFuture& future) mutable { + success.store(future.GetValue().IsSuccess()); + driver->Stop(true); + callbackDone.set_value(); + }); + + continueCreateTable.set_value(); + + ASSERT_EQ(callbackDoneFuture.wait_for(std::chrono::seconds(10)), std::future_status::ready); + ASSERT_TRUE(success.load()); + + ASSERT_TRUE(WaitUntil([&] { + return tableService.DeleteSessionRequests.load() >= 1u; + })); + ASSERT_TRUE(WaitUntil([&] { + auto stoppedSessionResult = tableClient->CreateSession().ExtractValueSync(); + return stoppedSessionResult.GetStatus() == EStatus::CLIENT_CANCELLED; + })); +} + +TEST(TableTest, DropLastOwnersFromResponseCallbackDoesNotDeadlock) { + TMockTableService tableService; + std::unique_ptr grpcServer; + std::unique_ptr driver; + std::unique_ptr tableClient; + std::unique_ptr tableSession; + + StartServerWithTableService( + tableService, + grpcServer, + driver, + tableClient, + tableSession + ); + + std::weak_ptr connections = CreateInternalInterface(*driver); + + { + auto pooledSessionResult = tableClient->GetSession().ExtractValueSync(); + ASSERT_TRUE(pooledSessionResult.IsSuccess()); + auto pooledSession = pooledSessionResult.GetSession(); + } + + std::promise createTableStarted; + auto createTableStartedFuture = createTableStarted.get_future(); + std::promise continueCreateTable; + tableService.CreateTableStarted = &createTableStarted; + tableService.ContinueCreateTable = continueCreateTable.get_future().share(); + + std::promise callbackDone; + auto callbackDoneFuture = callbackDone.get_future(); + std::atomic_bool success = false; + + auto requestFuture = tableSession->CreateTable( + "/Root/My/DB/drop_owners", + NTable::TTableBuilder().Build() + ); + + ASSERT_EQ(createTableStartedFuture.wait_for(std::chrono::seconds(10)), std::future_status::ready); + + requestFuture.Subscribe([&](const NThreading::TFuture& future) mutable { + success.store(future.GetValue().IsSuccess()); + tableSession.reset(); + tableClient.reset(); + driver.reset(); + callbackDone.set_value(); + }); + + continueCreateTable.set_value(); + + ASSERT_EQ(callbackDoneFuture.wait_for(std::chrono::seconds(10)), std::future_status::ready); + ASSERT_TRUE(success.load()); + ASSERT_TRUE(WaitUntil([&] { + return connections.expired(); + })); +} + +TEST(TableTest, DriverStopFromResponseCallbackThenDropOwnersDoesNotDeadlock) { + TMockTableService tableService; + std::unique_ptr grpcServer; + std::unique_ptr driver; + std::unique_ptr tableClient; + std::unique_ptr tableSession; + + StartServerWithTableService( + tableService, + grpcServer, + driver, + tableClient, + tableSession + ); + + std::weak_ptr connections = CreateInternalInterface(*driver); + + { + auto pooledSessionResult = tableClient->GetSession().ExtractValueSync(); + ASSERT_TRUE(pooledSessionResult.IsSuccess()); + auto pooledSession = pooledSessionResult.GetSession(); + } + + std::promise createTableStarted; + auto createTableStartedFuture = createTableStarted.get_future(); + std::promise continueCreateTable; + tableService.CreateTableStarted = &createTableStarted; + tableService.ContinueCreateTable = continueCreateTable.get_future().share(); + + std::promise callbackDone; + auto callbackDoneFuture = callbackDone.get_future(); + std::atomic_bool success = false; + + auto requestFuture = tableSession->CreateTable( + "/Root/My/DB/driver_stop_drop_owners", + NTable::TTableBuilder().Build() + ); + + ASSERT_EQ(createTableStartedFuture.wait_for(std::chrono::seconds(10)), std::future_status::ready); + + requestFuture.Subscribe([&](const NThreading::TFuture& future) mutable { + success.store(future.GetValue().IsSuccess()); + driver->Stop(true); + tableSession.reset(); + tableClient.reset(); + driver.reset(); + callbackDone.set_value(); + }); + + continueCreateTable.set_value(); + + ASSERT_EQ(callbackDoneFuture.wait_for(std::chrono::seconds(10)), std::future_status::ready); + ASSERT_TRUE(success.load()); + ASSERT_TRUE(WaitUntil([&] { + return connections.expired(); + })); +} + /** * Verify that the SDK creates the CREATE TABLE request correctly, * when no metrics configuration is provided. From dc67f8461519df364ab19bc44acb8670b3cdab56 Mon Sep 17 00:00:00 2001 From: azevaykin <145343289+azevaykin@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:47:07 +0000 Subject: [PATCH 21/56] Multi-column statistics: KQP & SDK (#46118) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/proto/accessor.h | 3 + include/ydb-cpp-sdk/client/table/fwd.h | 1 + include/ydb-cpp-sdk/client/table/table.h | 47 +++++++++ src/api/protos/ydb_table.proto | 32 ++++++ src/client/table/proto_accessor.cpp | 8 ++ src/client/table/table.cpp | 108 ++++++++++++++++++++ 7 files changed, 200 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 7af8cda066..a37bae21f1 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -1a85306db085572a96d030acac1482ce9f5523c1 +3e5dc8335f1cf18a17d4839537eddcdf0c2671c3 diff --git a/include/ydb-cpp-sdk/client/proto/accessor.h b/include/ydb-cpp-sdk/client/proto/accessor.h index b3b9c2cfef..7b6c650f49 100644 --- a/include/ydb-cpp-sdk/client/proto/accessor.h +++ b/include/ydb-cpp-sdk/client/proto/accessor.h @@ -58,6 +58,9 @@ class TProtoAccessor { static NTable::TIndexDescription FromProto(const Ydb::Table::TableIndex& tableIndex); static NTable::TIndexDescription FromProto(const Ydb::Table::TableIndexDescription& tableIndexDesc); + static NTable::TMultiColumnStatisticsDescription FromProto(const Ydb::Table::TableMultiColumnStatistics& tableMultiColumnStatistics); + static NTable::TMultiColumnStatisticsDescription FromProto(const Ydb::Table::TableMultiColumnStatisticsDescription& tableMultiColumnStatisticsDesc); + static NTable::TChangefeedDescription FromProto(const Ydb::Table::Changefeed& changefeed); static NTable::TChangefeedDescription FromProto(const Ydb::Table::ChangefeedDescription& changefeed); diff --git a/include/ydb-cpp-sdk/client/table/fwd.h b/include/ydb-cpp-sdk/client/table/fwd.h index 8315bce41e..11b0cbfcb9 100644 --- a/include/ydb-cpp-sdk/client/table/fwd.h +++ b/include/ydb-cpp-sdk/client/table/fwd.h @@ -13,6 +13,7 @@ struct TPartitionStats; struct TSequenceDescription; class TChangefeedDescription; class TIndexDescription; +class TMultiColumnStatisticsDescription; class TColumnFamilyDescription; class TTableDescription; diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index fa892c3169..4bbdd81fc7 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -41,6 +41,8 @@ class TtlSettings; class TtlTier; class TableIndex; class TableIndexDescription; +class TableMultiColumnStatistics; +class TableMultiColumnStatisticsDescription; class ValueSinceUnixEpochModeSettings; class EvictionToExternalStorageSettings; class CompactItem; @@ -1016,6 +1018,44 @@ enum class EStoreType { Column = 1 }; +enum class EMultiColumnStatisticsType { + Unknown = 0, + CountMinSketch = 1, +}; + +//! Represents multi-column table statistics description +class TMultiColumnStatisticsDescription { + friend class NYdb::TProtoAccessor; + +public: + TMultiColumnStatisticsDescription( + const std::string& name, + const std::vector& columns, + const std::vector& types + ); + + const std::string& GetName() const; + const std::vector& GetColumns() const; + const std::vector& GetTypes() const; + + void SerializeTo(Ydb::Table::TableMultiColumnStatistics& proto) const; + +private: + explicit TMultiColumnStatisticsDescription(const Ydb::Table::TableMultiColumnStatistics& proto); + explicit TMultiColumnStatisticsDescription(const Ydb::Table::TableMultiColumnStatisticsDescription& proto); + + template + static TMultiColumnStatisticsDescription FromProto(const TProto& proto); + +private: + std::string Name_; + std::vector Columns_; + std::vector Types_; +}; + +bool operator==(const TMultiColumnStatisticsDescription& lhs, const TMultiColumnStatisticsDescription& rhs); +bool operator!=(const TMultiColumnStatisticsDescription& lhs, const TMultiColumnStatisticsDescription& rhs); + //! Represents table description class TTableDescription { friend class TTableBuilder; @@ -1031,6 +1071,7 @@ class TTableDescription { std::vector GetColumns() const; std::vector GetTableColumns() const; std::vector GetIndexDescriptions() const; + std::vector GetMultiColumnStatisticsDescriptions() const; std::vector GetChangefeedDescriptions() const; std::optional GetTtlSettings() const; // Deprecated. Use GetTtlSettings() instead @@ -1124,6 +1165,9 @@ class TTableDescription { void AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); void AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); + // multi-column statistics + void AddMultiColumnStatistics(const TMultiColumnStatisticsDescription& statisticsDescription); + void SetTtlSettings(TTtlSettings&& settings); void SetTtlSettings(const TTtlSettings& settings); @@ -1379,6 +1423,9 @@ class TTableBuilder { TTableBuilder& AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); TTableBuilder& AddSecondaryIndex(const std::string& indexName, const std::string& indexColumn); + // multi-column statistics + TTableBuilder& AddMultiColumnStatistics(const TMultiColumnStatisticsDescription& statisticsDescription); + TTableBuilder& SetTtlSettings(TTtlSettings&& settings); TTableBuilder& SetTtlSettings(const TTtlSettings& settings); TTableBuilder& SetTtlSettings(const std::string& columnName, const TDuration& expireAfter = TDuration::Zero()); diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index b63b573db2..85ec4ef143 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -434,6 +434,30 @@ message TableIndexDescription { uint64 size_bytes = 7; } +// Represent multi-column table statistics +message TableMultiColumnStatistics { + enum MultiColumnStatisticsType { + STATISTIC_TYPE_UNSPECIFIED = 0; + COUNT_MIN_SKETCH = 1; + } + // Name of statistics + string name = 1; + // List of columns the statistics is built on + repeated string columns = 2; + // Types of statistics to build + repeated MultiColumnStatisticsType types = 3; +} + +// Represent multi-column table statistics description +message TableMultiColumnStatisticsDescription { + // Name of statistics + string name = 1; + // List of columns the statistics is built on + repeated string columns = 2; + // Types of statistics to build + repeated TableMultiColumnStatistics.MultiColumnStatisticsType types = 3; +} + // State of index building operation message IndexBuildState { enum State { @@ -1058,6 +1082,8 @@ message CreateTableRequest { * The metrics configuration for the given table. */ optional MetricsSettings metrics_settings = 21; + // List of multi-column table statistics + repeated TableMultiColumnStatistics statistics = 22; } message CreateTableResponse { @@ -1194,6 +1220,10 @@ message AlterTableRequest { // Start set not null for table repeated SetNotNullItem set_not_null = 27; + // Add multi-column table statistics + repeated TableMultiColumnStatistics add_statistics = 28; + // Remove multi-column table statistics (by its names) + repeated string drop_statistics = 29; } message AlterTableResponse { @@ -1325,6 +1355,8 @@ message DescribeTableResult { * The metrics configuration for the given table. */ optional MetricsSettings metrics_settings = 19; + // List of multi-column table statistics + repeated TableMultiColumnStatisticsDescription statistics = 20; } message Query { diff --git a/src/client/table/proto_accessor.cpp b/src/client/table/proto_accessor.cpp index d4e69ccfd1..f10a628503 100644 --- a/src/client/table/proto_accessor.cpp +++ b/src/client/table/proto_accessor.cpp @@ -40,6 +40,14 @@ NTable::TIndexDescription TProtoAccessor::FromProto(const Ydb::Table::TableIndex return NTable::TIndexDescription(tableIndexDesc); } +NTable::TMultiColumnStatisticsDescription TProtoAccessor::FromProto(const Ydb::Table::TableMultiColumnStatistics& tableMultiColumnStatistics) { + return NTable::TMultiColumnStatisticsDescription(tableMultiColumnStatistics); +} + +NTable::TMultiColumnStatisticsDescription TProtoAccessor::FromProto(const Ydb::Table::TableMultiColumnStatisticsDescription& tableMultiColumnStatisticsDesc) { + return NTable::TMultiColumnStatisticsDescription(tableMultiColumnStatisticsDesc); +} + NTable::TChangefeedDescription TProtoAccessor::FromProto(const Ydb::Table::Changefeed& changefeed) { return NTable::TChangefeedDescription(changefeed); } diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index ff46e22add..b1f452bccd 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -390,6 +390,12 @@ class TTableDescription::TImpl { Indexes_.emplace_back(TProtoAccessor::FromProto(index)); } + // statistics + MultiColumnStatistics_.reserve(proto.statistics_size()); + for (const auto& statistics : proto.statistics()) { + MultiColumnStatistics_.emplace_back(TProtoAccessor::FromProto(statistics)); + } + if constexpr (std::is_same_v) { // changefeeds Changefeeds_.reserve(proto.changefeeds_size()); @@ -529,6 +535,10 @@ class TTableDescription::TImpl { Indexes_.emplace_back(indexDescription); } + void AddMultiColumnStatistics(const TMultiColumnStatisticsDescription& statisticsDescription) { + MultiColumnStatistics_.emplace_back(statisticsDescription); + } + void AddVectorKMeansTreeIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const TKMeansTreeSettings& indexSettings) { Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, {}, {}, indexSettings)); } @@ -628,6 +638,10 @@ class TTableDescription::TImpl { return Indexes_; } + const std::vector& GetMultiColumnStatisticsDescriptions() const { + return MultiColumnStatistics_; + } + const std::vector& GetChangefeedDescriptions() const { return Changefeeds_; } @@ -723,6 +737,7 @@ class TTableDescription::TImpl { std::vector PrimaryKey_; std::vector Columns_; std::vector Indexes_; + std::vector MultiColumnStatistics_; std::vector Changefeeds_; std::optional TtlSettings_; std::string Owner_; @@ -784,6 +799,10 @@ std::vector TTableDescription::GetIndexDescriptions() const { return Impl_->GetIndexDescriptions(); } +std::vector TTableDescription::GetMultiColumnStatisticsDescriptions() const { + return Impl_->GetMultiColumnStatisticsDescriptions(); +} + std::vector TTableDescription::GetChangefeedDescriptions() const { return Impl_->GetChangefeedDescriptions(); } @@ -836,6 +855,10 @@ void TTableDescription::AddSecondaryIndex(const TIndexDescription& indexDescript Impl_->AddSecondaryIndex(indexDescription); } +void TTableDescription::AddMultiColumnStatistics(const TMultiColumnStatisticsDescription& statisticsDescription) { + Impl_->AddMultiColumnStatistics(statisticsDescription); +} + void TTableDescription::AddSyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns) { AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns); } @@ -1036,6 +1059,10 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con index.SerializeTo(*request.add_indexes()); } + for (const auto& statistics : Impl_->GetMultiColumnStatisticsDescriptions()) { + statistics.SerializeTo(*request.add_statistics()); + } + if (const auto& ttl = Impl_->GetTtlSettings()) { ttl->SerializeTo(*request.mutable_ttl_settings()); } @@ -1320,6 +1347,11 @@ TTableBuilder& TTableBuilder::AddSecondaryIndex(const TIndexDescription& indexDe return *this; } +TTableBuilder& TTableBuilder::AddMultiColumnStatistics(const TMultiColumnStatisticsDescription& statisticsDescription) { + TableDescription_.AddMultiColumnStatistics(statisticsDescription); + return *this; +} + TTableBuilder& TTableBuilder::AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const std::vector& dataColumns) { TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns); return *this; @@ -3388,6 +3420,82 @@ bool operator!=(const TIndexDescription& lhs, const TIndexDescription& rhs) { //////////////////////////////////////////////////////////////////////////////// +TMultiColumnStatisticsDescription::TMultiColumnStatisticsDescription( + const std::string& name, + const std::vector& columns, + const std::vector& types) + : Name_(name) + , Columns_(columns) + , Types_(types) +{} + +TMultiColumnStatisticsDescription::TMultiColumnStatisticsDescription(const Ydb::Table::TableMultiColumnStatistics& proto) + : TMultiColumnStatisticsDescription(FromProto(proto)) +{} + +TMultiColumnStatisticsDescription::TMultiColumnStatisticsDescription(const Ydb::Table::TableMultiColumnStatisticsDescription& proto) + : TMultiColumnStatisticsDescription(FromProto(proto)) +{} + +template +TMultiColumnStatisticsDescription TMultiColumnStatisticsDescription::FromProto(const TProto& proto) { + std::vector columns(proto.columns().begin(), proto.columns().end()); + std::vector types; + types.reserve(proto.types_size()); + for (const auto type : proto.types()) { + switch (type) { + case Ydb::Table::TableMultiColumnStatistics::COUNT_MIN_SKETCH: + types.push_back(EMultiColumnStatisticsType::CountMinSketch); + break; + default: + types.push_back(EMultiColumnStatisticsType::Unknown); + break; + } + } + return TMultiColumnStatisticsDescription(proto.name(), columns, types); +} + +const std::string& TMultiColumnStatisticsDescription::GetName() const { + return Name_; +} + +const std::vector& TMultiColumnStatisticsDescription::GetColumns() const { + return Columns_; +} + +const std::vector& TMultiColumnStatisticsDescription::GetTypes() const { + return Types_; +} + +void TMultiColumnStatisticsDescription::SerializeTo(Ydb::Table::TableMultiColumnStatistics& proto) const { + proto.set_name(TStringType{Name_}); + for (const auto& column : Columns_) { + proto.add_columns(TStringType{column}); + } + for (const auto type : Types_) { + switch (type) { + case EMultiColumnStatisticsType::CountMinSketch: + proto.add_types(Ydb::Table::TableMultiColumnStatistics::COUNT_MIN_SKETCH); + break; + case EMultiColumnStatisticsType::Unknown: + proto.add_types(Ydb::Table::TableMultiColumnStatistics::STATISTIC_TYPE_UNSPECIFIED); + break; + } + } +} + +bool operator==(const TMultiColumnStatisticsDescription& lhs, const TMultiColumnStatisticsDescription& rhs) { + return lhs.GetName() == rhs.GetName() + && lhs.GetColumns() == rhs.GetColumns() + && lhs.GetTypes() == rhs.GetTypes(); +} + +bool operator!=(const TMultiColumnStatisticsDescription& lhs, const TMultiColumnStatisticsDescription& rhs) { + return !(lhs == rhs); +} + +//////////////////////////////////////////////////////////////////////////////// + TChangefeedDescription::TChangefeedDescription(const std::string& name, EChangefeedMode mode, EChangefeedFormat format) : Name_(name) , Mode_(mode) From 079dda8d6a67da30f75bee7617ea709991efcffa Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:47:16 +0000 Subject: [PATCH 22/56] fix the mute by gracefully closing in table client (#46158) --- .github/last_commit.txt | 2 +- src/client/table/impl/table_client.cpp | 17 ++++++----------- src/client/table/impl/table_client.h | 2 +- tests/unit/client/table/table_ut.cpp | 1 + 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index a37bae21f1..f88bafdbb0 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -3e5dc8335f1cf18a17d4839537eddcdf0c2671c3 +c2e2dee52ffa1513c18f0abb396516c377d2f43f diff --git a/src/client/table/impl/table_client.cpp b/src/client/table/impl/table_client.cpp index d73ea40c0e..93e5ac4753 100644 --- a/src/client/table/impl/table_client.cpp +++ b/src/client/table/impl/table_client.cpp @@ -81,9 +81,9 @@ std::shared_ptr TTableClient::TImpl::CreateRetryAt TTableClient::TImpl::~TImpl() { if (Connections_->GetDrainOnDtors()) { - const bool closeRemote = !TGRpcConnectionsImpl::IsCurrentThreadInSdkCallback(); - auto drainFuture = Drain(closeRemote); - if (closeRemote) { + const bool waitForDrain = !TGRpcConnectionsImpl::IsCurrentThreadInSdkCallback(); + auto drainFuture = Drain(); + if (waitForDrain) { drainFuture.Wait(DRAIN_TIMEOUT); } } @@ -106,7 +106,7 @@ void TTableClient::TImpl::InitStopper() { DbDriverState_->AddCb(std::move(cb), TDbDriverState::ENotifyType::STOP); } -NThreading::TFuture TTableClient::TImpl::Drain(bool closeRemote) { +NThreading::TFuture TTableClient::TImpl::Drain() { std::vector> sessions; // No realocations under lock sessions.reserve(Settings_.SessionPoolSettings_.MaxActiveSessions_); @@ -118,9 +118,7 @@ NThreading::TFuture TTableClient::TImpl::Drain(bool closeRemote) { std::vector closeResults; for (auto& s : sessions) { if (!s->GetId().empty()) { - if (closeRemote) { - closeResults.push_back(CloseInternal(s.get())); - } + closeResults.push_back(CloseInternal(s.get())); DbDriverState_->StatCollector.DecSessionsOnHost(s->GetEndpoint()); } } @@ -1166,11 +1164,8 @@ void TTableClient::TImpl::DeleteSession(TKqpSessionCommon* sessionImpl) { SessionPool_.DecrementActiveCounter(); } - const bool closeRemote = !TGRpcConnectionsImpl::IsCurrentThreadInSdkCallback(); if (!sessionImpl->GetId().empty()) { - if (closeRemote) { - CloseInternal(sessionImpl); - } + CloseInternal(sessionImpl); DbDriverState_->StatCollector.DecSessionsOnHost(sessionImpl->GetEndpoint()); } diff --git a/src/client/table/impl/table_client.h b/src/client/table/impl/table_client.h index a0960be7ba..38e7237ae1 100644 --- a/src/client/table/impl/table_client.h +++ b/src/client/table/impl/table_client.h @@ -44,7 +44,7 @@ class TTableClient::TImpl: public TClientImplCommon, public bool LinkObjToEndpoint(const TEndpointKey& endpoint, TEndpointObj* obj, const void* tag); void InitStopper(); - NThreading::TFuture Drain(bool closeRemote = true); + NThreading::TFuture Drain(); NThreading::TFuture Stop(); void ScheduleTaskUnsafe(std::function&& fn, TDeadline::Duration timeout); void StartPeriodicSessionPoolTask(); diff --git a/tests/unit/client/table/table_ut.cpp b/tests/unit/client/table/table_ut.cpp index b013f16860..5788a703b6 100644 --- a/tests/unit/client/table/table_ut.cpp +++ b/tests/unit/client/table/table_ut.cpp @@ -443,6 +443,7 @@ TEST(TableTest, DropLastOwnersFromResponseCallbackDoesNotDeadlock) { ASSERT_TRUE(WaitUntil([&] { return connections.expired(); })); + ASSERT_EQ(tableService.DeleteSessionRequests.load(), 2u); } TEST(TableTest, DriverStopFromResponseCallbackThenDropOwnersDoesNotDeadlock) { From a8fd990ff114e4233b9ee26c638e67004ba93a23 Mon Sep 17 00:00:00 2001 From: Aleksey Myasnikov Date: Tue, 28 Jul 2026 08:47:27 +0000 Subject: [PATCH 23/56] Remove legacy pgwire support from ydbd (#45922) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_query.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index f88bafdbb0..4c39f42d03 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -c2e2dee52ffa1513c18f0abb396516c377d2f43f +43b689c3a99b980d8e4be3d2b7b4a292c02debdc diff --git a/src/api/protos/ydb_query.proto b/src/api/protos/ydb_query.proto index ecce71efb1..2ec4ea6824 100644 --- a/src/api/protos/ydb_query.proto +++ b/src/api/protos/ydb_query.proto @@ -159,7 +159,7 @@ message RollbackTransactionResponse { enum Syntax { SYNTAX_UNSPECIFIED = 0; SYNTAX_YQL_V1 = 1; // YQL - SYNTAX_PG = 2; // PostgresQL + SYNTAX_PG = 2 [deprecated = true]; // Removed: PostgreSQL syntax is no longer supported } message QueryContent { From 7e703533540ce38fb256dee8c4fe4e2d545ae853 Mon Sep 17 00:00:00 2001 From: Dmitry Azhichakov Date: Tue, 28 Jul 2026 08:47:37 +0000 Subject: [PATCH 24/56] Export to S3 in Parquet (#40202) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/export/export.h | 14 ++++++++++++ include/ydb-cpp-sdk/client/proto/accessor.h | 2 ++ src/api/protos/ydb_export.proto | 2 +- src/client/export/export.cpp | 25 +++++++++++++++++++++ src/client/proto/accessor.cpp | 12 ++++++++++ 6 files changed, 55 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 4c39f42d03..73597cdcfc 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -43b689c3a99b980d8e4be3d2b7b4a292c02debdc +41ebdaf011be47d89113a112f5f037ebe1d9221a diff --git a/include/ydb-cpp-sdk/client/export/export.h b/include/ydb-cpp-sdk/client/export/export.h index 35cec0b943..02f73af609 100644 --- a/include/ydb-cpp-sdk/client/export/export.h +++ b/include/ydb-cpp-sdk/client/export/export.h @@ -4,6 +4,8 @@ #include #include +#include + namespace NYdb::inline V3 { namespace NExport { @@ -32,6 +34,15 @@ struct TEncryptionAlgorithm { static const std::string CHACHA_20_POLY_1305; }; +struct TYdbDumpFormat { +}; + +struct TParquetFormat { + using TSelf = TParquetFormat; + + FLUENT_SETTING_DEFAULT(uint32_t, RowGroupSize, 10000); +}; + /// YT struct TExportToYtSettings : public TOperationRequestSettings { struct TItem { @@ -104,6 +115,9 @@ struct TExportToS3Settings : public TOperationRequestSettings; + FLUENT_SETTING(FormatVariant, Format); + TSelf& SymmetricEncryption(const std::string& algorithm, const std::string& key) { EncryptionAlgorithm_ = algorithm; SymmetricKey_ = key; diff --git a/include/ydb-cpp-sdk/client/proto/accessor.h b/include/ydb-cpp-sdk/client/proto/accessor.h index 7b6c650f49..da3da5707c 100644 --- a/include/ydb-cpp-sdk/client/proto/accessor.h +++ b/include/ydb-cpp-sdk/client/proto/accessor.h @@ -76,6 +76,8 @@ class TProtoAccessor { static Ydb::Export::ExportToS3Settings::StorageClass GetProto(NExport::TExportToS3Settings::EStorageClass value); static NExport::TExportToS3Settings::EStorageClass FromProto(Ydb::Export::ExportToS3Settings::StorageClass value); static NExport::EExportProgress FromProto(Ydb::Export::ExportProgress::Progress value); + static NExport::TYdbDumpFormat FromProto(const Ydb::Export::YdbDumpFormat& value); + static NExport::TParquetFormat FromProto(const Ydb::Export::ParquetFormat& value); static NImport::EImportProgress FromProto(Ydb::Import::ImportProgress::Progress value); static Ydb::Import::ImportFromS3Settings::IndexPopulationMode GetProto(NImport::EIndexPopulationMode value); static NImport::EIndexPopulationMode FromProto(Ydb::Import::ImportFromS3Settings::IndexPopulationMode value); diff --git a/src/api/protos/ydb_export.proto b/src/api/protos/ydb_export.proto index 95822a81da..1514e3822b 100644 --- a/src/api/protos/ydb_export.proto +++ b/src/api/protos/ydb_export.proto @@ -75,7 +75,7 @@ message YdbDumpFormat { } message ParquetFormat { - uint32 row_group_size = 1; + uint32 row_group_size = 1 [(value) = "[1; 10000000]"]; } /// S3 diff --git a/src/client/export/export.cpp b/src/client/export/export.cpp index 6ac2846989..0cdcf9e88c 100644 --- a/src/client/export/export.cpp +++ b/src/client/export/export.cpp @@ -28,6 +28,10 @@ const std::string TEncryptionAlgorithm::CHACHA_20_POLY_1305 = "ChaCha20-Poly1305 /// Common namespace { +// helper type for the visitor +template +struct overloads : Ts... { using Ts::operator()...; }; + std::vector ItemsProgressFromProto(const google::protobuf::RepeatedPtrField& proto) { std::vector result; result.reserve(proto.size()); @@ -101,6 +105,16 @@ TExportToS3Response::TExportToS3Response(TStatus&& status, Ydb::Operations::Oper Metadata_.Settings.Compression(metadata.settings().compression()); } + switch (metadata.settings().format_case()) { + case Ydb::Export::ExportToS3Settings::FORMAT_NOT_SET: + case Ydb::Export::ExportToS3Settings::kYdbDump: + Metadata_.Settings.Format(TProtoAccessor::FromProto(metadata.settings().ydb_dump())); + break; + case Ydb::Export::ExportToS3Settings::kParquet: + Metadata_.Settings.Format(TProtoAccessor::FromProto(metadata.settings().parquet())); + break; + } + // progress Metadata_.Progress = TProtoAccessor::FromProto(metadata.progress()); Metadata_.ItemsProgress = ItemsProgressFromProto(metadata.items_progress()); @@ -226,6 +240,17 @@ TFuture TExportClient::ExportToS3(const TExportToS3Settings request.mutable_settings()->set_access_key(TStringType{settings.AccessKey_}); request.mutable_settings()->set_secret_key(TStringType{settings.SecretKey_}); + // Set format based on the Format_ field + std::visit(overloads{ + [&request](const TYdbDumpFormat&) { + request.mutable_settings()->mutable_ydb_dump(); + }, + [&request](const TParquetFormat& format) { + auto parquet = request.mutable_settings()->mutable_parquet(); + parquet->set_row_group_size(format.RowGroupSize_); + } + }, settings.Format_); + for (const auto& item : settings.Item_) { auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); protoItem.set_source_path(TStringType{item.Src}); diff --git a/src/client/proto/accessor.cpp b/src/client/proto/accessor.cpp index 8a5cbd79c7..5fbdd858ba 100644 --- a/src/client/proto/accessor.cpp +++ b/src/client/proto/accessor.cpp @@ -164,4 +164,16 @@ NImport::EIndexPopulationMode TProtoAccessor::FromProto(Ydb::Import::ImportFromS } } +NExport::TYdbDumpFormat TProtoAccessor::FromProto(const Ydb::Export::YdbDumpFormat&) { + return NExport::TYdbDumpFormat(); +} + +NExport::TParquetFormat TProtoAccessor::FromProto(const Ydb::Export::ParquetFormat& value) { + NExport::TParquetFormat result; + if (value.row_group_size()) { + result.RowGroupSize(value.row_group_size()); + } + return result; +} + } // namespace NYdb From 8014266326e31a16ca190480791f71a16abfa293 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:47:47 +0000 Subject: [PATCH 25/56] Distributed mutex based off of coordination service (#43658) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 + .../client/coordination/coordination.h | 8 + .../client/coordination/distributed_lock.h | 42 ++++ src/client/coordination/coordination.cpp | 64 +++++ src/client/coordination/distributed_lock.cpp | 206 ++++++++++++++++ .../coordination/coordination_grpc_mock.h | 206 ++++++++++++++++ .../client/coordination/coordination_ut.cpp | 132 +--------- .../coordination/distributed_lock_ut.cpp | 227 ++++++++++++++++++ 9 files changed, 759 insertions(+), 130 deletions(-) create mode 100644 include/ydb-cpp-sdk/client/coordination/distributed_lock.h create mode 100644 src/client/coordination/distributed_lock.cpp create mode 100644 tests/unit/client/coordination/coordination_grpc_mock.h create mode 100644 tests/unit/client/coordination/distributed_lock_ut.cpp diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 73597cdcfc..b6d89ca6ab 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -41ebdaf011be47d89113a112f5f037ebe1d9221a +f23903e3c2953fb93c984354f164a2fa4bb66c79 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f9b29d8f9..80bb49f2d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ * Added a flag to support deferred session creation(when client timeout exceeded, the session is created in the backgroud) +* Added a distributed lock primitive based on the coordination service, which implements basic_lockable concept. + # v3.20.0 * Added automatic retries for unary methods of table and query clients(ExecuteQuery, ExecuteScript, BulkUpsert, ReadRows). diff --git a/include/ydb-cpp-sdk/client/coordination/coordination.h b/include/ydb-cpp-sdk/client/coordination/coordination.h index 4b46428ca9..45bda4be92 100644 --- a/include/ydb-cpp-sdk/client/coordination/coordination.h +++ b/include/ydb-cpp-sdk/client/coordination/coordination.h @@ -291,6 +291,9 @@ struct TDescribeSemaphoreSettings { //////////////////////////////////////////////////////////////////////////////// +class TDistributedLock; +struct TDistributedLockSettings; + class TClient { public: TClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); @@ -323,6 +326,7 @@ class TSessionContext; class TSession { friend class TSessionContext; + friend class TDistributedLock; public: TSession() = default; @@ -360,9 +364,13 @@ class TSession { TAsyncResult DeleteSemaphore(const std::string& name, bool force = false); + TDistributedLock CreateDistributedLock(const TDistributedLockSettings& settings); + private: explicit TSession(TSessionContext* context); + std::shared_ptr SubscribeSessionLost(std::function callback); + private: class TImpl; std::shared_ptr Impl_; diff --git a/include/ydb-cpp-sdk/client/coordination/distributed_lock.h b/include/ydb-cpp-sdk/client/coordination/distributed_lock.h new file mode 100644 index 0000000000..56edbdac1f --- /dev/null +++ b/include/ydb-cpp-sdk/client/coordination/distributed_lock.h @@ -0,0 +1,42 @@ +#pragma once +#include +#include +namespace NYdb { +namespace NCoordination { + struct TYdbLockException : public TYdbException { + TYdbLockException(const std::string& message) : TYdbException(message) {} + }; + struct TDistributedLockSettings { + using TSelf = TDistributedLockSettings; + FLUENT_SETTING(std::string, Name); + FLUENT_SETTING_DEFAULT(TDuration, Timeout, TDuration::Seconds(5)); + }; + // Distributed exclusive lock backed by a YDB coordination semaphore. + // Satisfies BasicLockable (lock/unlock) for std::lock_guard; not a blocking Lockable. + class TDistributedLock { + public: + TDistributedLock(TSession session, const TDistributedLockSettings& settings); + ~TDistributedLock(); + TDistributedLock(const TDistributedLock&) = delete; + TDistributedLock& operator=(const TDistributedLock&) = delete; + TDistributedLock(TDistributedLock&&) = delete; + TDistributedLock& operator=(TDistributedLock&&) = delete; + // Throws TYdbLockException on session start failure, acquire timeout, transport + // error, or contention timeout (timeout bounds the acquire wait). + void lock(); + // Same as lock() + void Acquire(); + // noexcept. Undefined behavior if called when the lock is not held (same as std::mutex). + void unlock() noexcept; + // Same as unlock() + void Release() noexcept; + // noexcept. Returns false on any failure without throwing. + bool try_lock() noexcept; + // Signals lock loss for the current hold; refreshed on successful acquire — call again after re-lock. + std::stop_token getStopToken() const; + private: + struct TImpl; + std::unique_ptr impl_; + }; +} +} diff --git a/src/client/coordination/coordination.cpp b/src/client/coordination/coordination.cpp index af352fd995..02a967f965 100644 --- a/src/client/coordination/coordination.cpp +++ b/src/client/coordination/coordination.cpp @@ -172,6 +172,7 @@ class TSessionContext : public TThrRefBase { using TResponse = Ydb::Coordination::SessionResponse; using TGrpcStatus = NYdbGrpc::TGrpcStatus; using IProcessor = NYdbGrpc::IStreamRequestReadWriteProcessor; + using TSessionLostCallback = std::function; friend class TSession::TImpl; @@ -592,6 +593,32 @@ class TSessionContext : public TThrRefBase { return future; } + std::shared_ptr DoSubscribeSessionLost(TSessionLostCallback callback) { + TSessionLostCallback callbackToCall; + uint64_t callbackId = 0; + { + std::lock_guard guard(Lock); + if (SessionLossNotified || IsClosed() || SessionState == ESessionState::EXPIRED || ConnectionState == EConnectionState::STOPPED) { + callbackToCall = std::move(callback); + } else { + callbackId = NextSessionLostCallbackId++; + SessionLostCallbacks.emplace(callbackId, std::move(callback)); + } + } + + if (callbackToCall) { + callbackToCall(); + return {}; + } + + return std::shared_ptr( + new uint64_t(callbackId), + [self = TPtr(this), callbackId](void* ptr) { + delete static_cast(ptr); + self->DoUnsubscribeSessionLost(callbackId); + }); + } + TDuration GetConnectTimeout() { // Use a separate connect timeout if available if (Settings_.ConnectTimeout_) { @@ -865,6 +892,26 @@ class TSessionContext : public TThrRefBase { return dynamic_cast(it->second.get()); } + void DoUnsubscribeSessionLost(uint64_t callbackId) { + std::lock_guard guard(Lock); + SessionLostCallbacks.erase(callbackId); + } + + std::vector TakeSessionLostCallbacksLocked() { + std::vector callbacks; + if (SessionLossNotified) { + return callbacks; + } + + SessionLossNotified = true; + callbacks.reserve(SessionLostCallbacks.size()); + for (auto& [_, callback] : SessionLostCallbacks) { + callbacks.emplace_back(std::move(callback)); + } + SessionLostCallbacks.clear(); + return callbacks; + } + private: template void RunUserCallback(TCallback&& callback) { @@ -886,6 +933,7 @@ class TSessionContext : public TThrRefBase { std::deque> failedSemaphoreOps; std::deque> failedSimpleOps; TResultPromise closePromise; + std::vector sessionLostCallbacks; { std::lock_guard guard(Lock); @@ -942,6 +990,7 @@ class TSessionContext : public TThrRefBase { SessionState = ESessionState::EXPIRED; notifyExpired = true; } + sessionLostCallbacks = TakeSessionLostCallbacksLocked(); } else { context = LocalContext; ConnectionState = EConnectionState::DISCONNECTED; @@ -1034,6 +1083,10 @@ class TSessionContext : public TThrRefBase { op->SetFailure(status); } + for (auto& callback : sessionLostCallbacks) { + callback(); + } + if (closePromise.Initialized()) { closePromise.SetValue(TResult(expired ? MakeStatus() : status)); } @@ -1795,6 +1848,9 @@ class TSessionContext : public TThrRefBase { std::deque> PendingRequests; std::unordered_map> SentRequests; TResultPromise ReconnectPromise; + std::unordered_map SessionLostCallbacks; + uint64_t NextSessionLostCallbackId = 1; + bool SessionLossNotified = false; // These are used to manage session timeout IQueueClientContextPtr SessionStartTimeoutContext; @@ -2084,6 +2140,10 @@ class TSession::TImpl { return Context->DoDeleteSemaphore(name, force); } + std::shared_ptr SubscribeSessionLost(std::function callback) { + return Context->DoSubscribeSessionLost(std::move(callback)); + } + private: const TIntrusivePtr Context; }; @@ -2158,5 +2218,9 @@ TAsyncResult TSession::DeleteSemaphore( return Impl_->DeleteSemaphore(name, force); } +std::shared_ptr TSession::SubscribeSessionLost(std::function callback) { + return Impl_->SubscribeSessionLost(std::move(callback)); +} + } } diff --git a/src/client/coordination/distributed_lock.cpp b/src/client/coordination/distributed_lock.cpp new file mode 100644 index 0000000000..a0f5149df8 --- /dev/null +++ b/src/client/coordination/distributed_lock.cpp @@ -0,0 +1,206 @@ +#include + +#include + +#include + +namespace NYdb { +namespace NCoordination { + struct TDistributedLock::TImpl { + struct TLockState { + std::stop_token GetStopToken() const { + std::lock_guard guard(Mutex); + return StopSource.get_token(); + } + + bool TryBeginHold() { + std::lock_guard guard(Mutex); + if (SessionLost) { + return false; + } + StopSource = std::stop_source{}; + Holding = true; + return true; + } + + void EndHold() { + std::lock_guard guard(Mutex); + Holding = false; + } + + void RequestStopIfHolding() { + std::lock_guard guard(Mutex); + if (Holding) { + StopSource.request_stop(); + } + } + + void NotifySessionLost() { + std::lock_guard guard(Mutex); + SessionLost = true; + if (Holding) { + StopSource.request_stop(); + } + } + + mutable std::mutex Mutex; + std::stop_source StopSource; + bool Holding = false; + bool SessionLost = false; + }; + + TSession Session; + TAcquireSemaphoreSettings Settings; + std::string Name; + TDuration Timeout; + std::shared_ptr LockState; + std::shared_ptr SessionLostSubscription; + + bool CancelAcquire() noexcept try { + auto releaseFuture = Session.ReleaseSemaphore(Name); + if (!releaseFuture.Wait(Timeout)) { + return false; + } + const auto result = releaseFuture.GetValue(); + return result.IsSuccess(); + } catch (...) { + return false; + } + + bool IsSessionLost() { + return Session.GetSessionState() == ESessionState::EXPIRED || + Session.GetConnectionState() == EConnectionState::STOPPED; + } + + bool TryBeginHold() { + if (IsSessionLost()) { + LockState->NotifySessionLost(); + return false; + } + if (!LockState->TryBeginHold()) { + return false; + } + if (IsSessionLost()) { + LockState->NotifySessionLost(); + LockState->EndHold(); + return false; + } + return true; + } + + TImpl(TSession session, const TDistributedLockSettings& lockSettings) + : Session(std::move(session)) + , Name(lockSettings.Name_) + , Timeout(lockSettings.Timeout_) + , LockState(std::make_shared()) + { + if (!Session) { + throw TYdbLockException("Session is not initialized"); + } + + Settings = TAcquireSemaphoreSettings() + .Exclusive() + .Data(FQDNHostName()) + .Ephemeral() + .Timeout(Timeout); + + std::weak_ptr weakLockState = LockState; + SessionLostSubscription = Session.SubscribeSessionLost([weakLockState] { + if (auto lockState = weakLockState.lock()) { + lockState->NotifySessionLost(); + } + }); + } + + bool try_lock() noexcept try { + auto acquireFuture = Session.AcquireSemaphore(Name, Settings); + if (!acquireFuture.Wait(Timeout)) { + CancelAcquire(); + return false; + } + const auto result = acquireFuture.GetValue(); + if (!result.IsSuccess()) { + return false; + } + if (!result.GetResult()) { + return false; + } + if (!TryBeginHold()) { + return false; + } + return true; + } catch (...) { + return false; + } + + void lock() { + auto acquireFuture = Session.AcquireSemaphore(Name, Settings); + if (!acquireFuture.Wait(Timeout)) { + CancelAcquire(); + throw TYdbLockException("Failed to acquire semaphore"); + } + const auto result = acquireFuture.GetValue(); + if (!result.IsSuccess()) { + throw TYdbLockException("Failed to acquire semaphore"); + } + if (!result.GetResult()) { + throw TYdbLockException("Failed to acquire semaphore"); + } + if (!TryBeginHold()) { + throw TYdbLockException("Failed to acquire semaphore"); + } + } + + void unlock() noexcept try { + auto releaseFuture = Session.ReleaseSemaphore(Name); + if (releaseFuture.Wait(Timeout)) { + const auto result = releaseFuture.GetValue(); + if (result.IsSuccess() && result.GetResult()) { + LockState->EndHold(); + return; + } + } + + LockState->RequestStopIfHolding(); + LockState->EndHold(); + } catch (...) { + LockState->RequestStopIfHolding(); + LockState->EndHold(); + } + }; + + TDistributedLock::TDistributedLock(TSession session, const TDistributedLockSettings& settings) { + impl_ = std::make_unique(std::move(session), settings); + } + + TDistributedLock TSession::CreateDistributedLock(const TDistributedLockSettings& settings) { + return TDistributedLock(*this, settings); + } + + TDistributedLock::~TDistributedLock() = default; + + void TDistributedLock::lock() { + impl_->lock(); + } + + void TDistributedLock::unlock() noexcept { + impl_->unlock(); + } + + void TDistributedLock::Acquire() { + impl_->lock(); + } + + void TDistributedLock::Release() noexcept { + impl_->unlock(); + } + + bool TDistributedLock::try_lock() noexcept { + return impl_->try_lock(); + } + + std::stop_token TDistributedLock::getStopToken() const { + return impl_->LockState->GetStopToken(); + } +} +} diff --git a/tests/unit/client/coordination/coordination_grpc_mock.h b/tests/unit/client/coordination/coordination_grpc_mock.h new file mode 100644 index 0000000000..98922d8d9e --- /dev/null +++ b/tests/unit/client/coordination/coordination_grpc_mock.h @@ -0,0 +1,206 @@ +#pragma once + +#include + +#include +#include + +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include + +namespace NCoordinationTest { + +class TMockDiscoveryService : public Ydb::Discovery::V1::DiscoveryService::Service { +public: + grpc::Status ListEndpoints( + grpc::ServerContext* context, + const Ydb::Discovery::ListEndpointsRequest* request, + Ydb::Discovery::ListEndpointsResponse* response) override + { + Y_UNUSED(context); + + const auto* result = MapFindPtr(MockResults, request->database()); + Y_ABORT_UNLESS(result, "Mock service doesn't have a result for database '%s'", request->database().c_str()); + + auto* op = response->mutable_operation(); + op->set_ready(true); + op->set_status(Ydb::StatusIds::SUCCESS); + op->mutable_result()->PackFrom(*result); + return grpc::Status::OK; + } + + std::unordered_map MockResults; +}; + +class TMockCoordinationService : public Ydb::Coordination::V1::CoordinationService::Service { +public: + std::atomic FailNextAcquire{false}; + std::atomic FailNextRelease{false}; + std::atomic BreakNextPingWithoutSessionLoss{false}; + // Zero means unlimited pong responses; set to a positive value to stop responding + // after N pings (used by SessionPingTimeout). + std::atomic MaxPingResponses{0}; + + grpc::Status Session( + grpc::ServerContext* context, + grpc::ServerReaderWriter< + Ydb::Coordination::SessionResponse, + Ydb::Coordination::SessionRequest>* stream) override + { + Y_UNUSED(context); + + Ydb::Coordination::SessionRequest request; + + if (!stream->Read(&request)) { + return grpc::Status::OK; + } + Y_ABORT_UNLESS(request.has_session_start(), "Expected session start"); + auto& start = request.session_start(); + uint64_t sessionId = start.session_id(); + if (!sessionId) { + sessionId = ++LastSessionId_; + } + { + Ydb::Coordination::SessionResponse response; + auto* started = response.mutable_session_started(); + started->set_session_id(sessionId); + started->set_timeout_millis(start.timeout_millis()); + stream->Write(response); + } + request.Clear(); + + size_t pingsReceived = 0; + while (stream->Read(&request)) { + if (request.has_ping()) { + if (BreakNextPingWithoutSessionLoss.exchange(false)) { + return grpc::Status(grpc::StatusCode::UNAVAILABLE, "Injected recoverable transport failure"); + } + const size_t maxResponses = MaxPingResponses.load(); + if (!maxResponses || ++pingsReceived <= maxResponses) { + Ydb::Coordination::SessionResponse response; + auto* pong = response.mutable_pong(); + pong->set_opaque(request.ping().opaque()); + stream->Write(response); + } + } else if (request.has_acquire_semaphore()) { + const auto& acquire = request.acquire_semaphore(); + Ydb::Coordination::SessionResponse response; + auto* result = response.mutable_acquire_semaphore_result(); + result->set_req_id(acquire.req_id()); + if (FailNextAcquire.exchange(false)) { + result->set_status(Ydb::StatusIds::BAD_REQUEST); + result->set_acquired(false); + } else { + std::lock_guard guard(SemaphoreLock_); + auto& sem = Semaphores_[acquire.name()]; + if (!sem.Held) { + sem.Held = true; + sem.OwnerSessionId = sessionId; + sem.Data = acquire.data(); + result->set_status(Ydb::StatusIds::SUCCESS); + result->set_acquired(true); + } else { + result->set_status(Ydb::StatusIds::SUCCESS); + result->set_acquired(false); + } + } + stream->Write(response); + } else if (request.has_release_semaphore()) { + const auto& release = request.release_semaphore(); + Ydb::Coordination::SessionResponse response; + auto* result = response.mutable_release_semaphore_result(); + result->set_req_id(release.req_id()); + if (FailNextRelease.exchange(false)) { + result->set_status(Ydb::StatusIds::BAD_REQUEST); + result->set_released(false); + } else { + std::lock_guard guard(SemaphoreLock_); + auto it = Semaphores_.find(release.name()); + if (it != Semaphores_.end() && it->second.Held && it->second.OwnerSessionId == sessionId) { + it->second.Held = false; + it->second.OwnerSessionId = 0; + it->second.Data.clear(); + result->set_status(Ydb::StatusIds::SUCCESS); + result->set_released(true); + } else { + result->set_status(Ydb::StatusIds::SUCCESS); + result->set_released(false); + } + } + stream->Write(response); + } else if (request.has_describe_semaphore()) { + const auto& describe = request.describe_semaphore(); + Ydb::Coordination::SessionResponse response; + auto* result = response.mutable_describe_semaphore_result(); + result->set_req_id(describe.req_id()); + result->set_status(Ydb::StatusIds::SUCCESS); + auto* desc = result->mutable_semaphore_description(); + desc->set_name(describe.name()); + desc->set_ephemeral(true); + std::lock_guard guard(SemaphoreLock_); + if (const auto* sem = MapFindPtr(Semaphores_, describe.name()); sem && sem->Held) { + auto* owner = desc->add_owners(); + owner->set_session_id(sem->OwnerSessionId); + owner->set_count(static_cast(-1)); + owner->set_data(sem->Data); + } + stream->Write(response); + } else if (request.has_session_stop()) { + Ydb::Coordination::SessionResponse response; + response.mutable_session_stopped(); + stream->Write(response); + return grpc::Status::OK; + } else { + Y_ABORT_UNLESS(false, "Unexpected session request: %s", request.ShortDebugString().c_str()); + } + request.Clear(); + } + + { + std::lock_guard guard(SemaphoreLock_); + for (auto& [name, sem] : Semaphores_) { + if (sem.OwnerSessionId == sessionId) { + sem = {}; + } + } + } + + return grpc::Status::OK; + } + +private: + struct TSemaphoreState { + bool Held = false; + uint64_t OwnerSessionId = 0; + std::string Data; + }; + + std::atomic LastSessionId_{0}; + std::mutex SemaphoreLock_; + std::unordered_map Semaphores_; +}; + +template +std::unique_ptr StartGrpcServer(const std::string& address, TService& service) { + grpc::ServerBuilder builder; + builder.AddListeningPort(NYdb::TStringType{address}, grpc::InsecureServerCredentials()); + builder.RegisterService(&service); + // Match YDB server defaults: allow client keepalive pings used by the SDK driver. + builder.AddChannelArgument(GRPC_ARG_HTTP2_MAX_PING_STRIKES, 0); + builder.AddChannelArgument(GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, 1000); + builder.AddChannelArgument(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, 0); + return builder.BuildAndStart(); +} + +} // namespace NCoordinationTest diff --git a/tests/unit/client/coordination/coordination_ut.cpp b/tests/unit/client/coordination/coordination_ut.cpp index ad510f658a..db6c8c5d27 100644 --- a/tests/unit/client/coordination/coordination_ut.cpp +++ b/tests/unit/client/coordination/coordination_ut.cpp @@ -2,122 +2,20 @@ #include -#include -#include - -#include -#include -#include +#include "coordination_grpc_mock.h" #include #include -#include using namespace NYdb; using namespace NYdb::NCoordination; - -namespace { - - class TMockDiscoveryService : public Ydb::Discovery::V1::DiscoveryService::Service { - public: - grpc::Status ListEndpoints( - grpc::ServerContext* context, - const Ydb::Discovery::ListEndpointsRequest* request, - Ydb::Discovery::ListEndpointsResponse* response) override - { - Y_UNUSED(context); - - std::cerr << "ListEndpoints: " << request->ShortDebugString() << std::endl; - - const auto* result = MapFindPtr(MockResults, request->database()); - Y_ABORT_UNLESS(result, "Mock service doesn't have a result for database '%s'", request->database().c_str()); - - auto* op = response->mutable_operation(); - op->set_ready(true); - op->set_status(Ydb::StatusIds::SUCCESS); - op->mutable_result()->PackFrom(*result); - return grpc::Status::OK; - } - - // From database name to result - std::unordered_map MockResults; - }; - - class TMockCoordinationService : public Ydb::Coordination::V1::CoordinationService::Service { - public: - grpc::Status Session( - grpc::ServerContext* context, - grpc::ServerReaderWriter< - Ydb::Coordination::SessionResponse, - Ydb::Coordination::SessionRequest>* stream) override - { - Y_UNUSED(context); - - std::cerr << "Session stream started" << std::endl; - - Ydb::Coordination::SessionRequest request; - - // Process session start - { - if (!stream->Read(&request)) { - // Disconnected before the request was sent - return grpc::Status::OK; - } - std::cerr << "Session request: " << request.ShortDebugString() << std::endl; - Y_ABORT_UNLESS(request.has_session_start(), "Expected session start"); - auto& start = request.session_start(); - uint64_t sessionId = start.session_id(); - if (!sessionId) { - sessionId = ++LastSessionId; - } - Ydb::Coordination::SessionResponse response; - auto* started = response.mutable_session_started(); - started->set_session_id(sessionId); - started->set_timeout_millis(start.timeout_millis()); - stream->Write(response); - request.Clear(); - } - - size_t pings_received = 0; - while (stream->Read(&request)) { - std::cerr << "Session request: " << request.ShortDebugString() << std::endl; - if (request.has_session_stop()) { - return grpc::Status::OK; - } - Y_ABORT_UNLESS(request.has_ping(), "Only ping and stop requests are supported"); - if (++pings_received <= 2) { - // Only reply to the first 2 ping requests - Ydb::Coordination::SessionResponse response; - auto* pong = response.mutable_pong(); - pong->set_opaque(request.ping().opaque()); - stream->Write(response); - } - request.Clear(); - } - - return grpc::Status::OK; - } - - private: - std::atomic LastSessionId{ 0 }; - }; - - template - std::unique_ptr StartGrpcServer(const std::string& address, TService& service) { - grpc::ServerBuilder builder; - builder.AddListeningPort(TStringType{address}, grpc::InsecureServerCredentials()); - builder.RegisterService(&service); - return builder.BuildAndStart(); - } - -} // namespace +using namespace NCoordinationTest; Y_UNIT_TEST_SUITE(Coordination) { Y_UNIT_TEST(SessionStartTimeout) { TPortManager pm; - // Start a fake endpoint on a random port ui16 fakeEndpointPort = pm.GetPort(); TInet6StreamSocket fakeEndpointSocket; { @@ -129,7 +27,6 @@ Y_UNIT_TEST_SUITE(Coordination) { "Failed to listen on port %" PRIu16, fakeEndpointPort); } - // Fill mock discovery service with our fake database TMockDiscoveryService discoveryService; { auto& dbResult = discoveryService.MockResults["/Root/My/DB"]; @@ -138,7 +35,6 @@ Y_UNIT_TEST_SUITE(Coordination) { endpoint->set_port(fakeEndpointPort); } - // Start our mock discovery service ui16 discoveryPort = pm.GetPort(); std::string discoveryAddr = TStringBuilder() << "0.0.0.0:" << discoveryPort; auto discoveryServer = StartGrpcServer(discoveryAddr, discoveryService); @@ -152,32 +48,25 @@ Y_UNIT_TEST_SUITE(Coordination) { auto settings = TSessionSettings() .Timeout(TDuration::MilliSeconds(500)); - // We expect either connection or session start timeout auto startTimestamp = TInstant::Now(); auto res = client.StartSession("/Some/Path", settings).ExtractValueSync(); auto endTimestamp = TInstant::Now(); auto elapsed = endTimestamp - startTimestamp; - std::cerr << "Got: " << ToString(res.GetStatus()) << ": " << res.GetIssues().ToString() << std::endl; - - // Both connection and session timeout return EStatus::TIMEOUT UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::TIMEOUT, res.GetIssues().ToString()); - - // Our timeout is 500ms, but allow up to 5 seconds of slack on very busy servers UNIT_ASSERT_C(elapsed < TDuration::Seconds(5), "Timeout after too much time: " << elapsed); } Y_UNIT_TEST(SessionPingTimeout) { TPortManager pm; - // Start a fake coordination service TMockCoordinationService coordinationService; + coordinationService.MaxPingResponses.store(2); ui16 coordinationPort = pm.GetPort(); auto coordinationServer = StartGrpcServer( TStringBuilder() << "0.0.0.0:" << coordinationPort, coordinationService); - // Fill a fake discovery service TMockDiscoveryService discoveryService; { auto& dbResult = discoveryService.MockResults["/Root/My/DB"]; @@ -186,13 +75,11 @@ Y_UNIT_TEST_SUITE(Coordination) { endpoint->set_port(coordinationPort); } - // Start a fake discovery service ui16 discoveryPort = pm.GetPort(); auto discoveryServer = StartGrpcServer( TStringBuilder() << "0.0.0.0:" << discoveryPort, discoveryService); - // Create a driver and a client auto config = TDriverConfig() .SetEndpoint(TStringBuilder() << "localhost:" << discoveryPort) .SetDatabase("/Root/My/DB"); @@ -219,29 +106,22 @@ Y_UNIT_TEST_SUITE(Coordination) { auto endTimestamp = TInstant::Now(); auto elapsed = endTimestamp - startTimestamp; - // We expect first few pings to succeed UNIT_ASSERT_C(elapsed > TDuration::Seconds(1), "Elapsed time too short: " << elapsed); - - // We expect timeout to hit pretty soon afterwards UNIT_ASSERT_C(elapsed < TDuration::Seconds(4), "Elapsed time too large: " << elapsed); - // Check the last failure stored in a session auto res2 = session.Close().ExtractValueSync(); - std::cerr << "Close: " << ToString(res2.GetStatus()) << ": " << res2.GetIssues().ToString() << std::endl; UNIT_ASSERT_VALUES_EQUAL_C(res2.GetStatus(), EStatus::TIMEOUT, res2.GetIssues().ToString()); } Y_UNIT_TEST(SessionCancelByDriver) { TPortManager pm; - // Start a fake coordination service TMockCoordinationService coordinationService; ui16 coordinationPort = pm.GetPort(); auto coordinationServer = StartGrpcServer( TStringBuilder() << "0.0.0.0:" << coordinationPort, coordinationService); - // Fill a fake discovery service TMockDiscoveryService discoveryService; { auto& dbResult = discoveryService.MockResults["/Root/My/DB"]; @@ -250,13 +130,11 @@ Y_UNIT_TEST_SUITE(Coordination) { endpoint->set_port(coordinationPort); } - // Start a fake discovery service ui16 discoveryPort = pm.GetPort(); auto discoveryServer = StartGrpcServer( TStringBuilder() << "0.0.0.0:" << discoveryPort, discoveryService); - // Create a driver and a client auto config = TDriverConfig() .SetEndpoint(TStringBuilder() << "localhost:" << discoveryPort) .SetDatabase("/Root/My/DB"); @@ -280,7 +158,6 @@ Y_UNIT_TEST_SUITE(Coordination) { auto session = res.ExtractResult(); - // Stop and destroy driver, forcing session to be cancelled client.reset(); driver->Stop(); driver.reset(); @@ -289,12 +166,9 @@ Y_UNIT_TEST_SUITE(Coordination) { auto endTimestamp = TInstant::Now(); auto elapsed = endTimestamp - startTimestamp; - // We expect session to be cancelled promptly UNIT_ASSERT_C(elapsed < TDuration::Seconds(1), "Elapsed time too large: " << elapsed); - // Check the last failure stored in a session auto res2 = session.Close().ExtractValueSync(); - std::cerr << "Close: " << ToString(res2.GetStatus()) << ": " << res2.GetIssues().ToString() << std::endl; UNIT_ASSERT_VALUES_EQUAL_C(res2.GetStatus(), EStatus::CLIENT_CANCELLED, res2.GetIssues().ToString()); } diff --git a/tests/unit/client/coordination/distributed_lock_ut.cpp b/tests/unit/client/coordination/distributed_lock_ut.cpp new file mode 100644 index 0000000000..1500836f51 --- /dev/null +++ b/tests/unit/client/coordination/distributed_lock_ut.cpp @@ -0,0 +1,227 @@ +#include + +#include + +#include "coordination_grpc_mock.h" + +#include +#include + +#include +#include + +#include +#include + +using namespace NYdb; +using namespace NYdb::NCoordination; +using namespace NCoordinationTest; + +namespace { + +constexpr TDuration TEST_TIMEOUT = TDuration::MilliSeconds(500); +constexpr const char* COORD_PATH = "/Some/CoordPath"; +constexpr const char* SEMAPHORE_NAME = "test-lock"; +constexpr const char* ANOTHER_SEMAPHORE_NAME = "another-test-lock"; + +struct TTestEnv { + TPortManager PortManager; + TMockDiscoveryService DiscoveryService; + TMockCoordinationService CoordinationService; + std::unique_ptr CoordinationServer; + std::unique_ptr DiscoveryServer; + std::optional Driver; + std::optional Client; + + TTestEnv() { + ui16 coordinationPort = PortManager.GetPort(); + CoordinationServer = StartGrpcServer( + TStringBuilder() << "0.0.0.0:" << coordinationPort, + CoordinationService); + + auto& dbResult = DiscoveryService.MockResults["/Root/My/DB"]; + auto* endpoint = dbResult.add_endpoints(); + endpoint->set_address("localhost"); + endpoint->set_port(coordinationPort); + + ui16 discoveryPort = PortManager.GetPort(); + DiscoveryServer = StartGrpcServer( + TStringBuilder() << "0.0.0.0:" << discoveryPort, + DiscoveryService); + + Driver.emplace(TDriverConfig() + .SetEndpoint(TStringBuilder() << "localhost:" << discoveryPort) + .SetDatabase("/Root/My/DB")); + Client.emplace(*Driver); + } + + ~TTestEnv() { + Client.reset(); + Driver.reset(); + if (CoordinationServer) { + CoordinationServer->Shutdown(); + CoordinationServer->Wait(); + } + if (DiscoveryServer) { + DiscoveryServer->Shutdown(); + DiscoveryServer->Wait(); + } + } +}; + +TSession MakeSession(TTestEnv& env) { + auto sessionResult = env.Client->StartSession( + COORD_PATH, + TSessionSettings().Timeout(TEST_TIMEOUT) + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(sessionResult.GetStatus(), EStatus::SUCCESS, sessionResult.GetIssues().ToString()); + return sessionResult.ExtractResult(); +} + +TDistributedLock MakeLock(TSession session, const char* name = SEMAPHORE_NAME) { + return session.CreateDistributedLock( + TDistributedLockSettings().Name(name).Timeout(TEST_TIMEOUT)); +} + +} // namespace + +Y_UNIT_TEST_SUITE(DistributedLock) { + + Y_UNIT_TEST(LockUnlock) { + TTestEnv env; + auto session = MakeSession(env); + auto lock = MakeLock(session); + lock.lock(); + lock.unlock(); + } + + Y_UNIT_TEST(LockGuard) { + TTestEnv env; + auto session = MakeSession(env); + auto lock = MakeLock(session); + std::lock_guard guard(lock); + } + + Y_UNIT_TEST(TryLockSuccess) { + TTestEnv env; + auto session = MakeSession(env); + auto lock = MakeLock(session); + UNIT_ASSERT(lock.try_lock()); + lock.unlock(); + } + + Y_UNIT_TEST(TryLockFailsWhenHeld) { + TTestEnv env; + auto sessionA = MakeSession(env); + auto sessionB = MakeSession(env); + auto lockA = MakeLock(sessionA); + auto lockB = MakeLock(sessionB); + lockA.lock(); + UNIT_ASSERT(!lockB.try_lock()); + lockA.unlock(); + } + + Y_UNIT_TEST(MultipleLockNamesInOneSession) { + TTestEnv env; + auto session = MakeSession(env); + auto lockA = MakeLock(session); + auto lockB = MakeLock(session, ANOTHER_SEMAPHORE_NAME); + lockA.lock(); + lockB.lock(); + lockB.unlock(); + lockA.unlock(); + } + + Y_UNIT_TEST(LockThrowsOnAcquireFailure) { + TTestEnv env; + env.CoordinationService.FailNextAcquire.store(true); + auto session = MakeSession(env); + auto lock = MakeLock(session); + UNIT_ASSERT_EXCEPTION(lock.lock(), TYdbLockException); + UNIT_ASSERT(!lock.getStopToken().stop_requested()); + lock.lock(); + lock.unlock(); + } + + Y_UNIT_TEST(UnlockFailureNotifiesStopToken) { + TTestEnv env; + auto session = MakeSession(env); + auto lock = MakeLock(session); + lock.lock(); + auto token = lock.getStopToken(); + env.CoordinationService.FailNextRelease.store(true); + lock.unlock(); + UNIT_ASSERT(token.stop_requested()); + } + + Y_UNIT_TEST(OwnerDataIsHostName) { + TTestEnv env; + auto session = MakeSession(env); + auto lock = MakeLock(session); + lock.lock(); + + auto describeResult = session.DescribeSemaphore( + SEMAPHORE_NAME, + TDescribeSemaphoreSettings().IncludeOwners(true) + ).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(describeResult.GetStatus(), EStatus::SUCCESS, describeResult.GetIssues().ToString()); + + const auto& description = describeResult.GetResult(); + UNIT_ASSERT_VALUES_EQUAL(description.GetOwners().size(), 1u); + UNIT_ASSERT_VALUES_EQUAL(description.GetOwners()[0].GetData(), FQDNHostName()); + + lock.unlock(); + } + + Y_UNIT_TEST(GetStopTokenInitiallyValid) { + TTestEnv env; + auto session = MakeSession(env); + auto lock = MakeLock(session); + UNIT_ASSERT(!lock.getStopToken().stop_requested()); + } + + Y_UNIT_TEST(SessionExpiryWhileHoldingLock) { + TTestEnv env; + env.CoordinationService.MaxPingResponses.store(2); + auto session = MakeSession(env); + auto lock = MakeLock(session); + lock.lock(); + const TInstant deadline = TInstant::Now() + TDuration::Seconds(5); + while (!lock.getStopToken().stop_requested() && TInstant::Now() < deadline) { + Sleep(TDuration::MilliSeconds(50)); + } + UNIT_ASSERT(lock.getStopToken().stop_requested()); + lock.unlock(); + + UNIT_ASSERT_EXCEPTION(lock.lock(), TYdbLockException); + + env.CoordinationService.MaxPingResponses.store(0); + auto newSession = MakeSession(env); + auto newLock = MakeLock(newSession); + newLock.lock(); + UNIT_ASSERT(!newLock.getStopToken().stop_requested()); + newLock.unlock(); + } + + Y_UNIT_TEST(RecoverableTransportFailureDoesNotNotifyStopToken) { + TTestEnv env; + auto session = MakeSession(env); + auto lock = MakeLock(session); + lock.lock(); + auto token = lock.getStopToken(); + + env.CoordinationService.BreakNextPingWithoutSessionLoss.store(true); + auto pingResult = session.Ping().ExtractValueSync(); + UNIT_ASSERT_VALUES_UNEQUAL(pingResult.GetStatus(), EStatus::SUCCESS); + + const TInstant deadline = TInstant::Now() + TDuration::Seconds(5); + while (session.GetConnectionState() != EConnectionState::CONNECTED && TInstant::Now() < deadline) { + Sleep(TDuration::MilliSeconds(50)); + } + UNIT_ASSERT_VALUES_EQUAL(session.GetConnectionState(), EConnectionState::CONNECTED); + UNIT_ASSERT(!token.stop_requested()); + + lock.unlock(); + } + +} From 6781fdca4afc09d9fc015923b41802aa31b462bb Mon Sep 17 00:00:00 2001 From: Alek5andr-Kotov Date: Tue, 28 Jul 2026 08:47:57 +0000 Subject: [PATCH 26/56] Implement deferred topic publish (staging, finalize, E2E tests) (#46290) --- .github/last_commit.txt | 2 +- src/api/grpc/{ => draft}/ydb_topic_deferred_publish_v1.proto | 2 +- src/api/protos/{ => draft}/ydb_topic_deferred_publish.proto | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/api/grpc/{ => draft}/ydb_topic_deferred_publish_v1.proto (94%) rename src/api/protos/{ => draft}/ydb_topic_deferred_publish.proto (100%) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index b6d89ca6ab..e4bd34de45 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -f23903e3c2953fb93c984354f164a2fa4bb66c79 +655fe2558c818b97dd7ae06861e8a4e57b992043 diff --git a/src/api/grpc/ydb_topic_deferred_publish_v1.proto b/src/api/grpc/draft/ydb_topic_deferred_publish_v1.proto similarity index 94% rename from src/api/grpc/ydb_topic_deferred_publish_v1.proto rename to src/api/grpc/draft/ydb_topic_deferred_publish_v1.proto index 45bc980598..52e44db651 100644 --- a/src/api/grpc/ydb_topic_deferred_publish_v1.proto +++ b/src/api/grpc/draft/ydb_topic_deferred_publish_v1.proto @@ -5,7 +5,7 @@ package Ydb.Topic.DeferredPublish.V1; option java_package = "com.yandex.ydb.topic.deferredpublish.v1"; -import "src/api/protos/ydb_topic_deferred_publish.proto"; +import "src/api/protos/draft/ydb_topic_deferred_publish.proto"; service TopicDeferredPublishService { // Starts a deferred topic publication. diff --git a/src/api/protos/ydb_topic_deferred_publish.proto b/src/api/protos/draft/ydb_topic_deferred_publish.proto similarity index 100% rename from src/api/protos/ydb_topic_deferred_publish.proto rename to src/api/protos/draft/ydb_topic_deferred_publish.proto From eca15e8e8b3679f4147cdbabff109afc0d64140b Mon Sep 17 00:00:00 2001 From: flown4qqqq Date: Tue, 28 Jul 2026 08:48:07 +0000 Subject: [PATCH 27/56] Set not null: public api (SDK, CLI) (#46394) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/table/table.h | 17 +++++++++++++++++ include/ydb-cpp-sdk/client/table/table_enum.h | 9 +++++++++ .../library/operation_id/operation_id.h | 1 + src/api/protos/ydb_table.proto | 7 +++++++ src/client/operation/operation.cpp | 6 ++++++ src/client/table/table.cpp | 15 +++++++++++++++ src/library/operation_id/operation_id.cpp | 7 +++++++ .../operation_id/protos/operation_id.proto | 1 + 9 files changed, 64 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index e4bd34de45..645fbf67bb 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -655fe2558c818b97dd7ae06861e8a4e57b992043 +5e78bc775d99d7a531145f12512af09a8e4c0b92 diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index 4bbdd81fc7..8af94679d7 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -625,6 +625,23 @@ class TAnalyzeOperation : public TOperation { TMetadata Metadata_; }; +class TSetNotNullOperation : public TOperation { +public: + using TOperation::TOperation; + TSetNotNullOperation(TStatus&& status, Ydb::Operations::Operation&& operation); + + struct TMetadata { + ESetNotNullState State = ESetNotNullState::Unspecified; + float Progress = 0; + std::string Path; + std::vector Columns; + }; + + const TMetadata& Metadata() const; +private: + TMetadata Metadata_; +}; + //////////////////////////////////////////////////////////////////////////////// //! Represents changefeed description diff --git a/include/ydb-cpp-sdk/client/table/table_enum.h b/include/ydb-cpp-sdk/client/table/table_enum.h index 6ae6d50e41..bc195a4802 100644 --- a/include/ydb-cpp-sdk/client/table/table_enum.h +++ b/include/ydb-cpp-sdk/client/table/table_enum.h @@ -52,6 +52,15 @@ enum class ECompactState { Cancelled = 3, }; +enum class ESetNotNullState { + Unspecified = 0, + Preparing = 1, + Validating = 2, + Applying = 3, + Done = 4, + Cancelled = 5, +}; + enum class EAnalyzeState { Unspecified = 0, Enqueued = 1, diff --git a/include/ydb-cpp-sdk/library/operation_id/operation_id.h b/include/ydb-cpp-sdk/library/operation_id/operation_id.h index 57b69eb267..ec30726269 100644 --- a/include/ydb-cpp-sdk/library/operation_id/operation_id.h +++ b/include/ydb-cpp-sdk/library/operation_id/operation_id.h @@ -31,6 +31,7 @@ class TOperationId { COMPACTION = 13, FULL_BACKUP = 14, ANALYZE = 15, + SET_NOT_NULL = 16, }; struct TData { diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index 85ec4ef143..1797ca9d6c 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -489,6 +489,13 @@ message SetNotNullItem { optional string column_name = 1; } +message SetNotNullMetadata { + string path = 1; + repeated string columns = 2; + SetNotNullState.State state = 3; + float progress = 4; +} + // Description of index building operation message IndexBuildDescription { string path = 1; diff --git a/src/client/operation/operation.cpp b/src/client/operation/operation.cpp index e135486424..eb7c486fbf 100644 --- a/src/client/operation/operation.cpp +++ b/src/client/operation/operation.cpp @@ -97,4 +97,10 @@ NThreading::TFuture> TOperationClient return List("analyze", pageSize, pageToken); } +template NThreading::TFuture TOperationClient::Get(const TOperation::TOperationId& id); +template <> +NThreading::TFuture> TOperationClient::List(std::uint64_t pageSize, const std::string& pageToken) { + return List("setnotnull", pageSize, pageToken); +} + } // namespace NYdb::NOperation diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index b1f452bccd..2d32819202 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -244,6 +244,21 @@ const TAnalyzeOperation::TMetadata& TAnalyzeOperation::Metadata() const { return Metadata_; } +TSetNotNullOperation::TSetNotNullOperation(TStatus &&status, Ydb::Operations::Operation &&operation) + : TOperation(std::move(status), std::move(operation)) +{ + Ydb::Table::SetNotNullMetadata metadata; + GetProto().metadata().UnpackTo(&metadata); + Metadata_.State = static_cast(metadata.state()); + Metadata_.Progress = metadata.progress(); + Metadata_.Path = metadata.path(); + Metadata_.Columns.assign(metadata.columns().begin(), metadata.columns().end()); +} + +const TSetNotNullOperation::TMetadata& TSetNotNullOperation::Metadata() const { + return Metadata_; +} + //////////////////////////////////////////////////////////////////////////////// class TPartitioningSettings::TImpl { diff --git a/src/library/operation_id/operation_id.cpp b/src/library/operation_id/operation_id.cpp index d4c78ff18b..fef2073329 100644 --- a/src/library/operation_id/operation_id.cpp +++ b/src/library/operation_id/operation_id.cpp @@ -82,6 +82,9 @@ std::string ProtoToString(const Ydb::TOperationId& proto) { case Ydb::TOperationId::ANALYZE: res << "ydb://analyze"; break; + case Ydb::TOperationId::SET_NOT_NULL: + res << "ydb://setnotnull"; + break; default: Y_ABORT_UNLESS(false, "unexpected kind"); } @@ -345,6 +348,10 @@ TOperationId::EKind ParseKind(const std::string_view value) { return TOperationId::ANALYZE; } + if (value.starts_with("setnotnull")) { + return TOperationId::SET_NOT_NULL; + } + return TOperationId::UNUSED; } diff --git a/src/library/operation_id/protos/operation_id.proto b/src/library/operation_id/protos/operation_id.proto index 3791f0d3f0..296c37761b 100644 --- a/src/library/operation_id/protos/operation_id.proto +++ b/src/library/operation_id/protos/operation_id.proto @@ -20,6 +20,7 @@ message TOperationId { COMPACTION = 13; FULL_BACKUP = 14; ANALYZE = 15; + SET_NOT_NULL = 16; } message TData { From fbee3fc0f2908b4ee0ee23803f376ca708de951a Mon Sep 17 00:00:00 2001 From: Nikolay Perfilov Date: Tue, 28 Jul 2026 08:48:17 +0000 Subject: [PATCH 28/56] Add delete session to SDK and experimental YDB CLI (#46598) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 + include/ydb-cpp-sdk/client/query/client.h | 2 + include/ydb-cpp-sdk/client/query/fwd.h | 1 + include/ydb-cpp-sdk/client/query/query.h | 3 ++ src/client/query/client.cpp | 61 +++++++++++++++++++++++ 6 files changed, 70 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 645fbf67bb..e6cd3910f5 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -5e78bc775d99d7a531145f12512af09a8e4c0b92 +2cb8291ec8707daeb126ce715c95cb185a1d2517 diff --git a/CHANGELOG.md b/CHANGELOG.md index 80bb49f2d2..c6d597d9d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Added `TQueryClient::DeleteSession` to explicitly delete a query session by session id. + * Added a flag to support deferred session creation(when client timeout exceeded, the session is created in the backgroud) * Added a distributed lock primitive based on the coordination service, which implements basic_lockable concept. diff --git a/include/ydb-cpp-sdk/client/query/client.h b/include/ydb-cpp-sdk/client/query/client.h index 5bb3224a65..3567071d5a 100644 --- a/include/ydb-cpp-sdk/client/query/client.h +++ b/include/ydb-cpp-sdk/client/query/client.h @@ -131,6 +131,8 @@ class TQueryClient { TAsyncCreateSessionResult GetSession(const TCreateSessionSettings& settings = TCreateSessionSettings()); + TAsyncStatus DeleteSession(const std::string& sessionId, const TDeleteSessionSettings& settings = TDeleteSessionSettings()); + //! Returns number of active sessions given via session pool int64_t GetActiveSessionCount() const; diff --git a/include/ydb-cpp-sdk/client/query/fwd.h b/include/ydb-cpp-sdk/client/query/fwd.h index 9f88fdc3c0..455cca65ed 100644 --- a/include/ydb-cpp-sdk/client/query/fwd.h +++ b/include/ydb-cpp-sdk/client/query/fwd.h @@ -9,6 +9,7 @@ struct TExecuteQuerySettings; struct TBeginTxSettings; struct TCommitTxSettings; struct TRollbackTxSettings; +struct TDeleteSessionSettings; struct TExecuteScriptSettings; struct TFetchScriptResultsSettings; struct TTxOnlineSettings; diff --git a/include/ydb-cpp-sdk/client/query/query.h b/include/ydb-cpp-sdk/client/query/query.h index 23fc18c555..9229d65247 100644 --- a/include/ydb-cpp-sdk/client/query/query.h +++ b/include/ydb-cpp-sdk/client/query/query.h @@ -122,6 +122,9 @@ struct TExecuteQuerySettings : public TRequestSettings { struct TBeginTxSettings : public TRequestSettings {}; struct TCommitTxSettings : public TRequestSettings {}; struct TRollbackTxSettings : public TRequestSettings {}; +struct TDeleteSessionSettings : public TRequestSettings { + FLUENT_SETTING_OPTIONAL(TRetryOperationSettings, RetrySettings); +}; diff --git a/src/client/query/client.cpp b/src/client/query/client.cpp index 19723562ee..78fb31eac2 100644 --- a/src/client/query/client.cpp +++ b/src/client/query/client.cpp @@ -377,6 +377,49 @@ class TQueryClient::TImpl: public TClientImplCommon, public return promise.GetFuture(); } + TAsyncStatus DeleteSession(const std::string& sessionId, const NYdb::NQuery::TDeleteSessionSettings& settings) { + auto request = MakeRequest(); + request.set_session_id(TStringType{sessionId}); + + auto promise = NThreading::NewPromise(); + + auto obs = MakeObservation("DeleteSession"); + + auto responseCb = [promise, obs] + (Ydb::Query::DeleteSessionResponse* response, TPlainStatus status) mutable { + try { + if (response) { + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); + TStatus deleteSessionStatus(TPlainStatus{static_cast(response->status()), std::move(opIssues), + status.Endpoint, std::move(status.Metadata)}); + + obs->End(deleteSessionStatus.GetStatus(), deleteSessionStatus.GetEndpoint()); + + promise.SetValue(std::move(deleteSessionStatus)); + } else { + obs->End(status.Status, status.Endpoint); + promise.SetValue(TStatus(std::move(status))); + } + } catch (...) { + obs->EndWithClientInternalError(); + promise.SetException(std::current_exception()); + } + }; + + auto rpcSettings = TRpcRequestSettings::Make(settings); + rpcSettings.PreferredEndpoint = TEndpointKey(GetNodeIdFromSession(sessionId)); + + Connections_->Run( + std::move(request), + responseCb, + &Ydb::Query::V1::QueryService::Stub::AsyncDeleteSession, + DbDriverState_, + rpcSettings); + + return promise.GetFuture(); + } + void DeleteSession(TKqpSessionCommon* sessionImpl) override { if (sessionImpl->IsOwnedBySessionPool()) { if (SessionPool_.CheckAndFeedWaiterNewSession(sessionImpl->NeedUpdateActiveCounter())) { @@ -813,6 +856,24 @@ TAsyncCreateSessionResult TQueryClient::GetSession(const TCreateSessionSettings& return Impl_->GetSession(settings); } +TAsyncStatus TQueryClient::DeleteSession(const std::string& sessionId, const TDeleteSessionSettings& settings) +{ + const auto resolvedRetrySettings = NRetry::ResolveRetrySettings( + Impl_->Settings_.RetrySettings_, + settings.RetrySettings_, + settings.ClientTimeout_, + NRetry::ERetryIdempotentDefault::True); + + return NRetry::RunUnaryWithRetry(*this, resolvedRetrySettings, + [this, sessionId, settings](TDuration timeout) { + auto opSettings = settings; + if (timeout != TDuration::Max()) { + opSettings.ClientTimeout(timeout); + } + return Impl_->DeleteSession(sessionId, opSettings); + }); +} + int64_t TQueryClient::GetActiveSessionCount() const { return Impl_->GetActiveSessionCount(); } From b04ec2fc99e0785f906a7a3c90473d0502c14c92 Mon Sep 17 00:00:00 2001 From: Mikhail Aksenov Date: Tue, 28 Jul 2026 08:48:27 +0000 Subject: [PATCH 29/56] Leak CloseInternal session settings to fix use-after-dtor (#46591) --- .github/last_commit.txt | 2 +- src/client/table/impl/table_client.cpp | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index e6cd3910f5..205c85f2d5 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -2cb8291ec8707daeb126ce715c95cb185a1d2517 +2a2759c00ae915faa5890a608ba191e199cd085b diff --git a/src/client/table/impl/table_client.cpp b/src/client/table/impl/table_client.cpp index 93e5ac4753..dd0d9af184 100644 --- a/src/client/table/impl/table_client.cpp +++ b/src/client/table/impl/table_client.cpp @@ -1118,8 +1118,7 @@ TAsyncStatus TTableClient::TImpl::Close(const TKqpSessionCommon* sessionImpl, co } TAsyncStatus TTableClient::TImpl::CloseInternal(const TKqpSessionCommon* sessionImpl) { - static const auto internalCloseSessionSettings = TCloseSessionSettings() - .ClientTimeout(TDuration::Seconds(2)); + const auto internalCloseSessionSettings = TCloseSessionSettings().ClientTimeout(TDuration::Seconds(2)); auto driver = Connections_; return Close(sessionImpl, internalCloseSessionSettings) From 6e961ba004d18dcc5b9acfc30daf9acf3c35b3ff Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:48:36 +0000 Subject: [PATCH 30/56] reuse equivalent IAM credentials providers (#46593) --- .github/last_commit.txt | 2 +- .../client/iam/common/generic_provider.h | 74 +++++++++++-- .../client/types/credentials/credentials.h | 15 +++ src/client/iam/iam.cpp | 9 ++ src/client/iam_private/common/iam.h | 35 +++--- src/client/types/credentials/credentials.cpp | 104 +++++++++++++++++- tests/unit/client/driver/driver_ut.cpp | 48 ++++++++ tests/unit/client/iam/grpc_iam_ut.cpp | 43 ++++++++ tests/unit/client/iam/http_iam_ut.cpp | 14 +++ .../iam_private/grpc_iam_service_ut.cpp | 96 ++++++++++++++++ 10 files changed, 415 insertions(+), 25 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 205c85f2d5..84c411aa1b 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -2a2759c00ae915faa5890a608ba191e199cd085b +1a38ab96700e21746605e59475c203dc108c4644 diff --git a/include/ydb-cpp-sdk/client/iam/common/generic_provider.h b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h index 37d3184e4b..11e1b44abb 100644 --- a/include/ydb-cpp-sdk/client/iam/common/generic_provider.h +++ b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h @@ -25,6 +25,35 @@ constexpr std::chrono::milliseconds BACKOFF_MAX{10000}; constexpr std::chrono::milliseconds PERIODIC_TICK{100}; constexpr std::chrono::milliseconds MINIMUM_REFRESH_INTERVAL{100}; +// Implementation detail for the IAM factory templates below. Symbols in NDetail are not part of +// the public YDB C++ SDK API and may change or be removed without notice. +namespace NIam::NDetail { + +template +std::string MakeClientIdentity( + const char* factoryType, + const TIamEndpoint& params, + const TExtraValues&... extraValues) +{ + TStringBuilder identity; + const auto append = [&identity](const auto& value) { + const std::string serialized = TStringBuilder() << value; + identity << serialized.size() << ':' << serialized; + }; + + append(factoryType); + append(params.Endpoint); + append(params.RefreshPeriod); + append(params.RequestTimeout); + append(params.EnableSsl); + append(params.CaCerts); + (append(extraValues), ...); + + return identity; +} + +} // namespace NIam::NDetail + // This file contains internal generic implementation of IAM credentials providers. // DO NOT USE THIS CLASS DIRECTLY. Use specialized factory methods for specific cases. template @@ -424,16 +453,32 @@ class TIamJwtCredentialsProviderFactory : public ICredentialsProviderFactory { // that don't have access to an ICoreFacility. Spins up a private TSimpleCoreFacility and ties // its lifetime to the returned provider via TOwningFacilityCredentialsProvider. TCredentialsProviderPtr CreateProvider() const final { - auto facility = CreateSimpleCoreFacility(); - auto inner = std::make_shared>( - Params_, std::weak_ptr(facility)); - return std::make_shared(std::move(facility), std::move(inner)); + return NCredentials::NDetail::GetOrCreateCachedProvider( + GetClientIdentity(), + [this] { + auto facility = CreateSimpleCoreFacility(); + auto inner = std::make_shared>( + Params_, std::weak_ptr(facility)); + return std::make_shared( + std::move(facility), std::move(inner)); + }); } TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const override { return std::make_shared>(Params_, std::move(facility)); } + std::string GetClientIdentity() const override final { + return NIam::NDetail::MakeClientIdentity( + "TIamJwtCredentialsProviderFactory", + Params_, + TService::service_full_name(), + Params_.JwtParams.AccountId, + Params_.JwtParams.KeyId, + Params_.JwtParams.PubKey, + Params_.JwtParams.PrivKey); + } + private: TIamJwtParams Params_; }; @@ -445,16 +490,29 @@ class TIamOAuthCredentialsProviderFactory : public ICredentialsProviderFactory { // Deprecated. Kept for backward compatibility — see comment on TIamJwtCredentialsProviderFactory. TCredentialsProviderPtr CreateProvider() const final { - auto facility = CreateSimpleCoreFacility(); - auto inner = std::make_shared>( - Params_, std::weak_ptr(facility)); - return std::make_shared(std::move(facility), std::move(inner)); + return NCredentials::NDetail::GetOrCreateCachedProvider( + GetClientIdentity(), + [this] { + auto facility = CreateSimpleCoreFacility(); + auto inner = std::make_shared>( + Params_, std::weak_ptr(facility)); + return std::make_shared( + std::move(facility), std::move(inner)); + }); } TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const override { return std::make_shared>(Params_, std::move(facility)); } + std::string GetClientIdentity() const override final { + return NIam::NDetail::MakeClientIdentity( + "TIamOAuthCredentialsProviderFactory", + Params_, + TService::service_full_name(), + Params_.OAuthToken); + } + private: TIamOAuth Params_; }; diff --git a/include/ydb-cpp-sdk/client/types/credentials/credentials.h b/include/ydb-cpp-sdk/client/types/credentials/credentials.h index 054db38a46..7eaa6994d6 100644 --- a/include/ydb-cpp-sdk/client/types/credentials/credentials.h +++ b/include/ydb-cpp-sdk/client/types/credentials/credentials.h @@ -2,6 +2,7 @@ #include +#include #include #include @@ -16,6 +17,20 @@ class ICredentialsProvider { using TCredentialsProviderPtr = std::shared_ptr; +// Implementation detail for SDK credentials factories. Symbols in NCredentials::NDetail are not +// part of the public YDB C++ SDK API and may change or be removed without notice. +namespace NCredentials::NDetail { + +using TCredentialsProviderCreator = std::function; + +// Process-wide weak cache for no-argument factory paths whose providers own their facilities. +// Facility-bound providers must not use it: their callbacks belong to the supplied facility. +TCredentialsProviderPtr GetOrCreateCachedProvider( + const std::string& identity, + TCredentialsProviderCreator createProvider); + +} // namespace NCredentials::NDetail + class ICoreFacility; class ICredentialsProviderFactory { public: diff --git a/src/client/iam/iam.cpp b/src/client/iam/iam.cpp index dd7c55fa9f..6cf531ab2c 100644 --- a/src/client/iam/iam.cpp +++ b/src/client/iam/iam.cpp @@ -136,6 +136,15 @@ class TIamCredentialsProviderFactory : public ICredentialsProviderFactory { TIamCredentialsProviderFactory(const TIamHost& params): Params_(params) {} TCredentialsProviderPtr CreateProvider() const final { + return NCredentials::NDetail::GetOrCreateCachedProvider( + GetClientIdentity(), + [this] { + return std::make_shared(Params_); + }); + } + + // Keep the facility-taking path driver-scoped; only the no-arg path is process-wide cached. + TCredentialsProviderPtr CreateProvider(std::weak_ptr) const final { return std::make_shared(Params_); } diff --git a/src/client/iam_private/common/iam.h b/src/client/iam_private/common/iam.h index 4f03c9df32..f892d6a046 100644 --- a/src/client/iam_private/common/iam.h +++ b/src/client/iam_private/common/iam.h @@ -59,12 +59,16 @@ class TIamServiceCredentialsProviderFactory : public ICredentialsProviderFactory // returns a TOwningFacilityCredentialsProvider). Sharing a TSimpleCoreFacility between two gRPC // IAM providers would abort: each one registers a periodic task and the facility allows only one. TCredentialsProviderPtr CreateProvider() const override final { - auto authProvider = Params_.SystemServiceAccountCredentials->CreateProvider(); - auto outerFacility = CreateSimpleCoreFacility(); - auto serviceProvider = std::make_shared( - Params_, std::weak_ptr(outerFacility), std::move(authProvider)); - return std::make_shared( - std::move(outerFacility), std::move(serviceProvider)); + return NCredentials::NDetail::GetOrCreateCachedProvider( + GetClientIdentity(), + [this] { + auto authProvider = Params_.SystemServiceAccountCredentials->CreateProvider(); + auto outerFacility = CreateSimpleCoreFacility(); + auto serviceProvider = std::make_shared( + Params_, std::weak_ptr(outerFacility), std::move(authProvider)); + return std::make_shared( + std::move(outerFacility), std::move(serviceProvider)); + }); } TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const override { @@ -72,15 +76,16 @@ class TIamServiceCredentialsProviderFactory : public ICredentialsProviderFactory } std::string GetClientIdentity() const override final { - return TStringBuilder() - << "TIamServiceCredentialsProviderFactory" - << '\t' << Params_.ServiceId - << '\t' << Params_.MicroserviceId - << '\t' << Params_.ResourceId - << '\t' << Params_.ResourceType - << '\t' << Params_.TargetServiceAccountId - << '\t' << Params_.SystemServiceAccountCredentials->GetClientIdentity() - ; + return NIam::NDetail::MakeClientIdentity( + "TIamServiceCredentialsProviderFactory", + Params_, + TService::service_full_name(), + Params_.ServiceId, + Params_.MicroserviceId, + Params_.ResourceId, + Params_.ResourceType, + Params_.TargetServiceAccountId, + Params_.SystemServiceAccountCredentials->GetClientIdentity()); } private: diff --git a/src/client/types/credentials/credentials.cpp b/src/client/types/credentials/credentials.cpp index 2a3e0a83a9..bbce466334 100644 --- a/src/client/types/credentials/credentials.cpp +++ b/src/client/types/credentials/credentials.cpp @@ -1,8 +1,111 @@ #include #include +#include +#include +#include + namespace NYdb::inline V3 { +namespace NCredentials::NDetail { + +namespace { + +struct TCredentialsProviderCacheEntry { + std::mutex Mutex; + std::condition_variable Initialized; + bool Initializing = false; + std::weak_ptr Provider; +}; + +class TCredentialsProviderCache { +public: + TCredentialsProviderPtr Get( + const std::string& identity, + TCredentialsProviderCreator createProvider) + { + std::shared_ptr entry; + { + std::lock_guard guard(Mutex_); + RemoveExpiredEntries(); + auto [it, inserted] = Entries_.try_emplace(identity); + if (inserted) { + it->second = std::make_shared(); + } + entry = it->second; + } + + std::unique_lock entryLock(entry->Mutex); + while (true) { + if (auto provider = entry->Provider.lock()) { + return provider; + } + + if (!entry->Initializing) { + entry->Initializing = true; + break; + } + + entry->Initialized.wait(entryLock, [&entry] { + return !entry->Initializing; + }); + } + + entryLock.unlock(); + + TCredentialsProviderPtr provider; + try { + provider = createProvider(); + } catch (...) { + entryLock.lock(); + entry->Initializing = false; + entryLock.unlock(); + entry->Initialized.notify_all(); + throw; + } + + entryLock.lock(); + entry->Provider = provider; + entry->Initializing = false; + entryLock.unlock(); + entry->Initialized.notify_all(); + return provider; + } + +private: + void RemoveExpiredEntries() { + for (auto it = Entries_.begin(); it != Entries_.end();) { + auto entry = it->second; + if (entry.use_count() == 2) { // The map and this local variable. + std::unique_lock entryLock(entry->Mutex, std::try_to_lock); + if (entryLock.owns_lock() && entry.use_count() == 2 && entry->Provider.expired()) { + entryLock.unlock(); + entry.reset(); + it = Entries_.erase(it); + continue; + } + } + ++it; + } + } + +private: + std::mutex Mutex_; + std::unordered_map> Entries_; +}; + +} // namespace + +TCredentialsProviderPtr GetOrCreateCachedProvider( + const std::string& identity, + TCredentialsProviderCreator createProvider) +{ + static TCredentialsProviderCache cache; + return cache.Get(identity, std::move(createProvider)); +} + +} // namespace NCredentials::NDetail + class TInsecureCredentialsProvider : public ICredentialsProvider { public: TInsecureCredentialsProvider() @@ -79,4 +182,3 @@ std::shared_ptr CreateOAuthCredentialsProviderFacto } } // namespace NYdb - diff --git a/tests/unit/client/driver/driver_ut.cpp b/tests/unit/client/driver/driver_ut.cpp index 03406f23f4..1ebc148081 100644 --- a/tests/unit/client/driver/driver_ut.cpp +++ b/tests/unit/client/driver/driver_ut.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -14,6 +15,7 @@ #include #include +#include #include @@ -77,9 +79,55 @@ namespace { return builder.BuildAndStart(); } + class TCountingCredentialsProvider final : public ICredentialsProvider { + public: + std::string GetAuthInfo() const override { + return "token"; + } + + bool IsValid() const override { + return true; + } + }; + + class TCountingCredentialsProviderFactory final : public ICredentialsProviderFactory { + public: + explicit TCountingCredentialsProviderFactory(std::atomic_int& providerCount) + : ProviderCount_(providerCount) + {} + + TCredentialsProviderPtr CreateProvider() const override { + ++ProviderCount_; + return std::make_shared(); + } + + std::string GetClientIdentity() const override { + return "same-credentials"; + } + + private: + std::atomic_int& ProviderCount_; + }; + } // namespace Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) { + Y_UNIT_TEST(ReusesCredentialsProviderForSameIdentity) { + std::atomic_int providerCount = 0; + auto driver = TDriver( + TDriverConfig() + .SetEndpoint("localhost:1") + .SetDatabase("/Root") + .SetDiscoveryMode(EDiscoveryMode::Off)); + + auto firstClient = TTableClient(driver, TClientSettings().CredentialsProviderFactory( + std::make_shared(providerCount))); + auto secondClient = TTableClient(driver, TClientSettings().CredentialsProviderFactory( + std::make_shared(providerCount))); + + UNIT_ASSERT_VALUES_EQUAL(providerCount.load(), 1); + } + Y_UNIT_TEST(InvalidRootCertificatePemFailsFast) { auto driver = TDriver( TDriverConfig() diff --git a/tests/unit/client/iam/grpc_iam_ut.cpp b/tests/unit/client/iam/grpc_iam_ut.cpp index e5eaa9ed1a..b75c44d2a1 100644 --- a/tests/unit/client/iam/grpc_iam_ut.cpp +++ b/tests/unit/client/iam/grpc_iam_ut.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -84,6 +85,48 @@ TEST(GrpcIamCredentialsProvider, TeardownWhileIamCreatePendingCompletesViaFactor server.Stop(); } +TEST(GrpcIamCredentialsProviderFactory, NoArgProvidersAreCachedAcrossFactoryInstances) { + TIamTokenServiceStub iamStub; + iamStub.SetResponseToken("unit-test-iam-token"); + TIamGrpcServer server(&iamStub); + ASSERT_TRUE(server.Start()); + + using TOAuthFactory = TIamOAuthCredentialsProviderFactory< + CreateIamTokenRequest, CreateIamTokenResponse, IamTokenService>; + const auto oauthParams = MakeOAuthParams(server.Endpoint()); + + std::vector> oauthProviders; + for (size_t i = 0; i < 8; ++i) { + auto factory = std::make_shared(oauthParams); + oauthProviders.emplace_back(std::async(std::launch::async, [factory] { + return factory->CreateProvider(); + })); + } + + auto firstOAuthProvider = oauthProviders.front().get(); + for (size_t i = 1; i < oauthProviders.size(); ++i) { + EXPECT_EQ(firstOAuthProvider, oauthProviders[i].get()); + } + EXPECT_EQ(iamStub.GetRequestCount(), 1); + + using TJwtFactory = TIamJwtCredentialsProviderFactory< + CreateIamTokenRequest, CreateIamTokenResponse, IamTokenService>; + const auto jwtParams = MakeJwtParams(server.Endpoint()); + auto firstJwtProvider = std::make_shared(jwtParams)->CreateProvider(); + auto secondJwtProvider = std::make_shared(jwtParams)->CreateProvider(); + + EXPECT_EQ(firstJwtProvider, secondJwtProvider); + EXPECT_EQ(iamStub.GetRequestCount(), 2); + + auto differentOAuthParams = oauthParams; + differentOAuthParams.OAuthToken = "different-oauth-token"; + auto differentOAuthProvider = std::make_shared(differentOAuthParams)->CreateProvider(); + EXPECT_NE(firstOAuthProvider, differentOAuthProvider); + EXPECT_EQ(iamStub.GetRequestCount(), 3); + + server.Stop(); +} + namespace { class TSlowBlockingAuthProvider final : public ICredentialsProvider { diff --git a/tests/unit/client/iam/http_iam_ut.cpp b/tests/unit/client/iam/http_iam_ut.cpp index fd2fe0383c..f0774b2098 100644 --- a/tests/unit/client/iam/http_iam_ut.cpp +++ b/tests/unit/client/iam/http_iam_ut.cpp @@ -13,6 +13,20 @@ using namespace NYdb; using namespace NYdb::NTest; +TEST(IamCredentialsProviderFactory, NoArgProviderIsCachedAcrossFactoryInstances) { + TMetadataServer server; + server.SetStrictMode(false); + server.SetResponse(HTTP_OK, MakeTokenResponse("cached-token", 3600)); + + const TIamHost params = MakeMetadataParams(server.Port); + auto firstProvider = CreateIamCredentialsProviderFactory(params)->CreateProvider(); + auto secondProvider = CreateIamCredentialsProviderFactory(params)->CreateProvider(); + + EXPECT_EQ(firstProvider, secondProvider); + EXPECT_EQ(firstProvider->GetAuthInfo(), "cached-token"); + EXPECT_EQ(server.GetRequestCount(), 1); +} + TEST(IamCredentialsProvider, ExpiryFieldSupport) { TMetadataServer server; server.SetStrictMode(false); diff --git a/tests/unit/client/iam_private/grpc_iam_service_ut.cpp b/tests/unit/client/iam_private/grpc_iam_service_ut.cpp index f557bd804c..935f985394 100644 --- a/tests/unit/client/iam_private/grpc_iam_service_ut.cpp +++ b/tests/unit/client/iam_private/grpc_iam_service_ut.cpp @@ -7,6 +7,7 @@ #include +#include #include #include #include @@ -28,6 +29,7 @@ class TIamServiceStub final : public IamTokenService::Service { const CreateIamTokenRequest*, CreateIamTokenResponse* response) override { + ++CreateRequestCount_; response->set_iam_token("inner-jwt-token"); response->mutable_expires_at()->set_seconds(4102444800); return grpc::Status::OK; @@ -38,14 +40,108 @@ class TIamServiceStub final : public IamTokenService::Service { const CreateIamTokenForServiceRequest*, CreateIamTokenResponse* response) override { + ++CreateForServiceRequestCount_; response->set_iam_token("outer-service-token"); response->mutable_expires_at()->set_seconds(4102444800); return grpc::Status::OK; } + + int GetCreateRequestCount() const { + return CreateRequestCount_.load(); + } + + int GetCreateForServiceRequestCount() const { + return CreateForServiceRequestCount_.load(); + } + +private: + std::atomic CreateRequestCount_{0}; + std::atomic CreateForServiceRequestCount_{0}; }; +using TJwtFactory = TIamJwtCredentialsProviderFactory< + CreateIamTokenRequest, CreateIamTokenResponse, IamTokenService>; + +using TOAuthFactory = TIamOAuthCredentialsProviderFactory< + CreateIamTokenRequest, CreateIamTokenResponse, IamTokenService>; + +TIamServiceParams MakeServiceParams(TCredentialsProviderFactoryPtr nestedFactory) { + TIamServiceParams params; + params.ServiceId = "unit-test-service"; + params.MicroserviceId = "unit-test-microservice"; + params.ResourceId = "unit-test-resource"; + params.ResourceType = "unit-test-resource-type"; + params.TargetServiceAccountId = "unit-test-target"; + params.SystemServiceAccountCredentials = std::move(nestedFactory); + return params; +} + } // namespace +TEST(IamCredentialsProviderIdentity, IdentityIsValueBased) { + const auto jwtParams = MakeJwtParams("iam.example:443"); + const auto firstJwtFactory = std::make_shared(jwtParams); + const auto secondJwtFactory = std::make_shared(jwtParams); + const auto jwtIdentity = firstJwtFactory->GetClientIdentity(); + EXPECT_EQ(jwtIdentity, secondJwtFactory->GetClientIdentity()); + auto differentJwtParams = jwtParams; + differentJwtParams.JwtParams.KeyId = "different-key"; + EXPECT_NE(jwtIdentity, TJwtFactory(differentJwtParams).GetClientIdentity()); + + const auto oauthParams = MakeOAuthParams("iam.example:443"); + const auto firstOAuthFactory = std::make_shared(oauthParams); + const auto secondOAuthFactory = std::make_shared(oauthParams); + const auto oauthIdentity = firstOAuthFactory->GetClientIdentity(); + EXPECT_EQ(oauthIdentity, secondOAuthFactory->GetClientIdentity()); + auto differentOAuthParams = oauthParams; + differentOAuthParams.OAuthToken = "different-token"; + EXPECT_NE(oauthIdentity, TOAuthFactory(differentOAuthParams).GetClientIdentity()); + + auto firstServiceParams = MakeServiceParams(firstJwtFactory); + const auto firstServiceFactory = CreateIamServiceCredentialsProviderFactory(firstServiceParams); + const auto secondServiceFactory = CreateIamServiceCredentialsProviderFactory( + MakeServiceParams(secondJwtFactory)); + const auto serviceIdentity = firstServiceFactory->GetClientIdentity(); + EXPECT_EQ(serviceIdentity, secondServiceFactory->GetClientIdentity()); + firstServiceParams.Endpoint = "different-iam.example:443"; + EXPECT_NE( + serviceIdentity, + CreateIamServiceCredentialsProviderFactory(firstServiceParams)->GetClientIdentity()); +} + +TEST(IamServiceCredentialsProvider, NoArgProviderIsCachedAcrossFactoryInstances) { + TIamServiceStub stub; + TIamGrpcServer server(&stub); + ASSERT_TRUE(server.Start()); + + const auto jwtParams = MakeJwtParams(server.Endpoint()); + auto firstServiceParams = MakeServiceParams(std::make_shared(jwtParams)); + firstServiceParams.Endpoint = server.Endpoint(); + firstServiceParams.EnableSsl = false; + firstServiceParams.RequestTimeout = TDuration::Seconds(5); + + auto secondServiceParams = MakeServiceParams(std::make_shared(jwtParams)); + secondServiceParams.Endpoint = server.Endpoint(); + secondServiceParams.EnableSsl = false; + secondServiceParams.RequestTimeout = TDuration::Seconds(5); + + auto firstProvider = CreateIamServiceCredentialsProviderFactory(firstServiceParams)->CreateProvider(); + auto secondProvider = CreateIamServiceCredentialsProviderFactory(secondServiceParams)->CreateProvider(); + + EXPECT_EQ(firstProvider, secondProvider); + EXPECT_EQ(firstProvider->GetAuthInfo(), "outer-service-token"); + EXPECT_EQ(stub.GetCreateRequestCount(), 1); + EXPECT_EQ(stub.GetCreateForServiceRequestCount(), 1); + + secondServiceParams.TargetServiceAccountId = "another-target"; + auto differentProvider = CreateIamServiceCredentialsProviderFactory(secondServiceParams)->CreateProvider(); + EXPECT_NE(firstProvider, differentProvider); + EXPECT_EQ(stub.GetCreateRequestCount(), 1); + EXPECT_EQ(stub.GetCreateForServiceRequestCount(), 2); + + server.Stop(); +} + // Regression test for the deprecated no-arg CreateProvider() on the IAM service-account // factory with a nested gRPC JWT auth provider. Before the fix, both providers shared a single // TSimpleCoreFacility, each registered a periodic refresh task, and TSimpleCoreFacility's From 22bf6af1439b0da02e8362e04e91f3c092b984f7 Mon Sep 17 00:00:00 2001 From: Kuzin Roman Date: Tue, 28 Jul 2026 08:48:46 +0000 Subject: [PATCH 31/56] LOGBROKER-10550 Fix verify on shutdown in sdk (#46840) --- .github/last_commit.txt | 2 +- src/client/topic/impl/read_session_impl.ipp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 84c411aa1b..eec6dbee1d 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -1a38ab96700e21746605e59475c203dc108c4644 +dac02c2aaa5c9a052f467ff0349863eec0c0e66b diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index 8ed3d76e19..cb95b69af2 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -1273,7 +1273,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( } else { pushRes = EventsQueue->PushEvent( partitionStream, - NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent(std::move(partitionStream), msg.commit_offset()), + NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent(partitionStream, msg.commit_offset()), deferred); } @@ -1476,7 +1476,7 @@ inline void TSingleClusterReadSessionImpl::StopPartitionSessionImpl( pushRes = EventsQueue->PushEvent( partitionStream, // TODO(qyryq) Is it safe to use GetMaxCommittedOffset here instead of StopPartitionSessionRequest.commmitted_offset? - TReadSessionEvent::TStopPartitionSessionEvent(std::move(partitionStream), committedOffset), + TReadSessionEvent::TStopPartitionSessionEvent(partitionStream, committedOffset), deferred); } else { // partitionStream->ConfirmDestroy(); @@ -1764,7 +1764,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( bool pushRes = EventsQueue->PushEvent( partitionStream, - TReadSessionEvent::TEndPartitionSessionEvent(std::move(partitionStream), std::move(adjacentPartitionIds), std::move(childPartitionIds)), + TReadSessionEvent::TEndPartitionSessionEvent(partitionStream, std::move(adjacentPartitionIds), std::move(childPartitionIds)), deferred); if (!pushRes) { AbortImpl(&deferred); @@ -1879,7 +1879,7 @@ void TSingleClusterReadSessionImpl::DestroyAllPartitionStr for (auto&& [key, partitionStream] : PartitionStreams) { bool pushRes = EventsQueue->PushEvent(partitionStream, - TClosedEvent(std::move(partitionStream), TClosedEvent::EReason::ConnectionLost), + TClosedEvent(partitionStream, TClosedEvent::EReason::ConnectionLost), deferred); if (!pushRes) { AbortImpl(&deferred); From 27fca2d2fe1135382b4036dd05f7132c853893bf Mon Sep 17 00:00:00 2001 From: Kuzin Roman Date: Tue, 28 Jul 2026 08:48:56 +0000 Subject: [PATCH 32/56] LOGBROKER-10550 Fix memory leak (#46748) --- .github/last_commit.txt | 2 +- src/client/topic/impl/read_session.cpp | 125 +++++++++++++------- src/client/topic/impl/read_session_impl.h | 7 ++ src/client/topic/impl/read_session_impl.ipp | 58 ++++++++- 4 files changed, 142 insertions(+), 50 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index eec6dbee1d..5b2e0f9633 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -dac02c2aaa5c9a052f467ff0349863eec0c0e66b +a32e2f886b40070e2580214f0f8adce49992330c diff --git a/src/client/topic/impl/read_session.cpp b/src/client/topic/impl/read_session.cpp index 7cc416eb3a..3c8c536ed1 100644 --- a/src/client/topic/impl/read_session.cpp +++ b/src/client/topic/impl/read_session.cpp @@ -40,7 +40,20 @@ TReadSession::~TReadSession() { Close(TDuration::Zero()); Abort(EStatus::ABORTED, "Aborted"); - ClearAllEvents(); + if (CbContext) { + if (auto session = CbContext->LockShared()) { + const TInstant closeDeadline = TInstant::Now() + TDuration::Seconds(5); + if (!session->WaitAllDecompressionTasks(closeDeadline)) { + LOG_LAZY(Log, TLOG_WARNING, GetLogPrefix() << "Some decompression tasks are still running after read session destroy timeout"); + } + ClearAllEvents(); + session->ClearAllPartitionStreamEvents(); + } else { + ClearAllEvents(); + } + } else { + ClearAllEvents(); + } if (CbContext) { CbContext->Cancel(); @@ -191,57 +204,77 @@ bool TReadSession::Close(TDuration timeout) { promise.TrySetValue(true); }; - TDeferredActions deferred; - with_lock(Lock) { - if (Closing || Aborting) { - return false; - } - - if (!timeout) { - AbortImpl(EStatus::ABORTED, "Close with zero timeout", deferred); - return false; + std::shared_ptr>> cbContextToCancel; + std::shared_ptr>> dumpCountersContextToCancel; + TInstant closeDeadline; + bool result = false; + { + TDeferredActions deferred; + with_lock(Lock) { + if (Closing || Aborting) { + return false; + } + + if (!timeout) { + AbortImpl(EStatus::ABORTED, "Close with zero timeout", deferred); + return false; + } + + Closing = true; + session = CbContext->TryGet(); } + session->Close(callback); - Closing = true; - session = CbContext->TryGet(); - } - session->Close(callback); - - callback(); // For the case when there are no subsessions yet. + callback(); // For the case when there are no subsessions yet. - auto timeoutCallback = [=](bool) mutable { - promise.TrySetValue(false); - }; + auto timeoutCallback = [=](bool) mutable { + promise.TrySetValue(false); + }; - auto timeoutContext = Connections->CreateContext(); - if (!timeoutContext) { - AbortImpl(EStatus::ABORTED, DRIVER_IS_STOPPING_DESCRIPTION, deferred); - return false; + auto timeoutContext = Connections->CreateContext(); + if (!timeoutContext) { + AbortImpl(EStatus::ABORTED, DRIVER_IS_STOPPING_DESCRIPTION, deferred); + return false; + } + closeDeadline = TInstant::Now() + timeout; + Connections->ScheduleCallback(timeout, + std::move(timeoutCallback), + timeoutContext); + + // Wait. + NThreading::TFuture resultFuture = promise.GetFuture(); + result = resultFuture.GetValueSync(); + if (result) { + Cancel(timeoutContext); + + NYdb::NIssue::TIssues issues; + issues.AddIssue("Session was gracefully closed"); + EventsQueue->Close(TSessionClosedEvent(EStatus::SUCCESS, std::move(issues)), deferred); + } else { + ++*Settings.Counters_->Errors; + session->Abort(); + + NYdb::NIssue::TIssues issues; + issues.AddIssue(TStringBuilder() << "Session was closed after waiting " << timeout); + EventsQueue->Close(TSessionClosedEvent(EStatus::TIMEOUT, std::move(issues)), deferred); + } + { + std::lock_guard guard(Lock); + Aborting = true; // Set abort flag for doing nothing on destructor. + cbContextToCancel = CbContext; + dumpCountersContextToCancel = DumpCountersContext; + } + if (!session->WaitAllDecompressionTasks(closeDeadline)) { + LOG_LAZY(Log, TLOG_WARNING, GetLogPrefix() << "Some decompression tasks are still running after read session close timeout"); + } + ClearAllEvents(); + session->ClearAllPartitionStreamEvents(); } - Connections->ScheduleCallback(timeout, - std::move(timeoutCallback), - timeoutContext); - - // Wait. - NThreading::TFuture resultFuture = promise.GetFuture(); - const bool result = resultFuture.GetValueSync(); - if (result) { - Cancel(timeoutContext); - - NYdb::NIssue::TIssues issues; - issues.AddIssue("Session was gracefully closed"); - EventsQueue->Close(TSessionClosedEvent(EStatus::SUCCESS, std::move(issues)), deferred); - } else { - ++*Settings.Counters_->Errors; - session->Abort(); - - NYdb::NIssue::TIssues issues; - issues.AddIssue(TStringBuilder() << "Session was closed after waiting " << timeout); - EventsQueue->Close(TSessionClosedEvent(EStatus::TIMEOUT, std::move(issues)), deferred); + if (cbContextToCancel) { + cbContextToCancel->Cancel(); } - { - std::lock_guard guard(Lock); - Aborting = true; // Set abort flag for doing nothing on destructor. + if (dumpCountersContextToCancel) { + dumpCountersContextToCancel->Cancel(); } return result; } diff --git a/src/client/topic/impl/read_session_impl.h b/src/client/topic/impl/read_session_impl.h index de0282b953..ee994937b4 100644 --- a/src/client/topic/impl/read_session_impl.h +++ b/src/client/topic/impl/read_session_impl.h @@ -27,7 +27,9 @@ #include #include +#include #include +#include #include @@ -1263,6 +1265,9 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext* partitionStream, ui64 startOffset, ui64 endOffset); void OnCreateNewDecompressionTask(); + void OnDecompressionTaskFinished(); + bool WaitAllDecompressionTasks(TInstant deadline) const; + void ClearAllPartitionStreamEvents(); void OnDecompressionInfoDestroy(i64 compressedSize, i64 decompressedSize, i64 messagesCount, i64 serverBytesSize); void OnDecompressionInfoDestroyImpl(i64 compressedSize, i64 decompressedSize, @@ -1547,6 +1552,8 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext CloseCallback; std::atomic DecompressionTasksInflight = 0; + mutable std::mutex DecompressionTasksInflightMutex; + mutable std::condition_variable DecompressionTasksInflightCondVar; i64 ReadSizeBudget; i64 ReadSizeServerDelta = 0; diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index cb95b69af2..0badf3eafc 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -1913,6 +1914,56 @@ void TSingleClusterReadSessionImpl::OnCreateNewDecompressi ++DecompressionTasksInflight; } +template +void TSingleClusterReadSessionImpl::OnDecompressionTaskFinished() { + Y_ABORT_UNLESS(DecompressionTasksInflight > 0); + if (--DecompressionTasksInflight == 0) { + DecompressionTasksInflightCondVar.notify_all(); + } +} + +template +bool TSingleClusterReadSessionImpl::WaitAllDecompressionTasks(TInstant deadline) const { + const TDuration timeout = deadline - TInstant::Now(); + if (timeout <= TDuration::Zero()) { + return DecompressionTasksInflight.load() == 0; + } + + std::unique_lock guard(DecompressionTasksInflightMutex); + return DecompressionTasksInflightCondVar.wait_for( + guard, + std::chrono::microseconds(timeout.MicroSeconds()), + [&] { + return DecompressionTasksInflight.load() == 0; + }); +} + +template +void TSingleClusterReadSessionImpl::ClearAllPartitionStreamEvents() { + TDeferredActions deferred; + std::vector>> streams; + std::vector> deferredDelete; + { + std::lock_guard guard(Lock); + streams.reserve(PartitionStreams.size()); + for (auto& [_, partitionStream] : PartitionStreams) { + streams.push_back(partitionStream); + } + } + + deferredDelete.reserve(streams.size()); + for (auto& stream : streams) { + std::lock_guard guard(stream->GetLock()); + if (stream->HasEvents()) { + deferredDelete.push_back(stream->ExtractQueue()); + } + } + + for (auto& queue : deferredDelete) { + queue.Cleanup(deferred); + } +} + template void TSingleClusterReadSessionImpl::OnDecompressionInfoDestroy(i64 compressedSize, i64 decompressedSize, i64 messagesCount, i64 serverBytesSize) { @@ -1943,9 +1994,6 @@ void TSingleClusterReadSessionImpl::OnDataDecompressed(i64 TDeferredActions deferred; - Y_ABORT_UNLESS(DecompressionTasksInflight > 0); - --DecompressionTasksInflight; - *Settings.Counters_->BytesRead += decompressedSize; *Settings.Counters_->BytesReadCompressed += sourceSize; *Settings.Counters_->MessagesRead += messagesCount; @@ -3598,6 +3646,10 @@ void TDataDecompressionInfo::TDecompressionTask::operator( // Message is dropped due to partition stream cancellation, we should release decompressed memory parent->OnUserRetrievedEvent(DecompressedSize, messagesProcessed); } + + if (auto session = parent->CbContext->LockShared()) { + session->OnDecompressionTaskFinished(); + } } template From 660c1cc1210c6191f7e9e5e4c33791c0a55e937e Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:49:06 +0000 Subject: [PATCH 33/56] async provider initialisation (#46135) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 + .../client/iam/common/generic_provider.h | 214 ++++++++++++++---- .../client/types/credentials/credentials.h | 9 + src/client/iam/iam.cpp | 33 +++ src/client/iam_private/common/iam.h | 40 +++- .../impl/internal/db_driver_state/state.cpp | 35 ++- .../impl/internal/db_driver_state/state.h | 21 +- .../grpc_connections/grpc_connections.cpp | 148 +++++++++++- .../grpc_connections/grpc_connections.h | 84 +++++++ .../impl/write_session_impl.cpp | 7 +- src/client/topic/impl/write_session_impl.cpp | 7 +- src/client/types/credentials/login/login.cpp | 87 +++++-- tests/unit/client/driver/driver_ut.cpp | 61 +++++ .../iam_private/grpc_iam_service_ut.cpp | 83 +++++++ 15 files changed, 743 insertions(+), 90 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 5b2e0f9633..11cfcd65d6 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -a32e2f886b40070e2580214f0f8adce49992330c +f7303ada674f0f072da2f4271a82af28e1029245 diff --git a/CHANGELOG.md b/CHANGELOG.md index c6d597d9d0..7829779d05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ * Added `TQueryClient::DeleteSession` to explicitly delete a query session by session id. +* Driver now supports async credentials initialisation: the first request is delayed until they are ready. + * Added a flag to support deferred session creation(when client timeout exceeded, the session is created in the backgroud) * Added a distributed lock primitive based on the coordination service, which implements basic_lockable concept. diff --git a/include/ydb-cpp-sdk/client/iam/common/generic_provider.h b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h index 11e1b44abb..04a07375c6 100644 --- a/include/ydb-cpp-sdk/client/iam/common/generic_provider.h +++ b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h @@ -14,9 +14,11 @@ #include #include +#include #include #include #include +#include namespace NYdb::inline V3 { @@ -86,6 +88,8 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { , Lock_() , ResponseFacility_(std::move(responseFacility)) , AuthTokenProvider_(authTokenProvider) + , FirstTokenReady_(NThreading::NewPromise()) + , FirstTokenReadySet_(false) { std::shared_ptr creds = nullptr; if (IamEndpoint_.EnableSsl) { @@ -108,20 +112,33 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { void StartPeriodicTask() { auto facility = ResponseFacility_.lock(); if (!facility) { + FailFirstToken("IAM-token provider response facility is not available"); return; } std::weak_ptr weakSelf = TGrpcIamCredentialsProvider::TImpl::weak_from_this(); - facility->AddPeriodicTask( - [weakSelf](NYdb::NIssue::TIssues&&, EStatus status) { - auto self = weakSelf.lock(); - if (!self || status != EStatus::SUCCESS) { - return false; - } - return self->OnPeriodicTick(); - }, - PERIODIC_TICK - ); + try { + facility->AddPeriodicTask( + [weakSelf](NYdb::NIssue::TIssues&&, EStatus status) { + auto self = weakSelf.lock(); + if (!self) { + return false; + } + if (status != EStatus::SUCCESS) { + self->FailFirstToken(TStringBuilder() + << "IAM-token provider periodic task failed with status " + << static_cast(status)); + return false; + } + return self->OnPeriodicTick(); + }, + PERIODIC_TICK + ); + } catch (...) { + FailFirstToken(TStringBuilder() + << "Failed to start IAM-token provider periodic task: " + << CurrentExceptionMessage()); + } } std::string GetTicket() { @@ -142,19 +159,29 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { ); } + NThreading::TFuture GetReadyFuture() const { + return FirstTokenReady_.GetFuture(); + } + void Stop() { + bool setStoppedException = false; { std::unique_lock guard(Lock_); if (NeedStop_) { return; } NeedStop_ = true; + setStoppedException = MarkFirstTokenReadyLocked(); TokenReady_.notify_all(); if (Context_.has_value()) { Context_->TryCancel(); } ContextReady_.wait(guard, [this]() { return !Context_.has_value(); }); } + if (setStoppedException) { + FirstTokenReady_.SetException( + std::make_exception_ptr(yexception() << "IAM-token provider stopped before token was ready")); + } Stub_.reset(); Channel_.reset(); } @@ -162,6 +189,23 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { private: using SysDuration = SysClock::duration; + bool MarkFirstTokenReadyLocked() { + return !std::exchange(FirstTokenReadySet_, true); + } + + void FailFirstToken(std::string error) { + bool setException = false; + { + std::lock_guard guard(Lock_); + if ((setException = MarkFirstTokenReadyLocked())) { + LastRequestError_ = error; + } + } + if (setException) { + FirstTokenReady_.SetException(std::make_exception_ptr(yexception() << error)); + } + } + static SysDuration ToBoundedSysDuration(const TDuration& d) { return std::chrono::duration_cast(TDeadline::SafeDurationCast(d)); } @@ -207,8 +251,16 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } if (auto self = weakSelf.lock()) { - std::lock_guard guard(self->Lock_); - self->ResetContextImpl(); + bool failFirstToken; + { + std::lock_guard guard(self->Lock_); + failFirstToken = self->MarkFirstTokenReadyLocked(); + self->ResetContextImpl(); + } + if (failFirstToken) { + self->FirstTokenReady_.SetException(std::make_exception_ptr( + yexception() << "IAM-token provider response facility is not available")); + } } }; @@ -217,13 +269,22 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { try { RequestFiller_(req); } catch (...) { + std::optional firstTokenError; const auto now = SysClock::now(); - std::lock_guard guard(Lock_); - LastRequestError_ = TStringBuilder() - << "Last request error was at " << FormatSysTimeUtcIsoMicros(now) - << ". Failed to prepare IAM request: " << CurrentExceptionMessage(); - ResetContextImpl(); - RescheduleOnFailure(); + { + std::lock_guard guard(Lock_); + LastRequestError_ = TStringBuilder() + << "Last request error was at " << FormatSysTimeUtcIsoMicros(now) + << ". Failed to prepare IAM request: " << CurrentExceptionMessage(); + if (MarkFirstTokenReadyLocked()) { + firstTokenError = LastRequestError_; + } + ResetContextImpl(); + RescheduleOnFailure(); + } + if (firstTokenError) { + FirstTokenReady_.SetException(std::make_exception_ptr(yexception() << *firstTokenError)); + } return; } @@ -271,6 +332,8 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } bool OnPeriodicTick() { + std::optional firstTokenError; + bool updateTicket = false; { std::unique_lock guard(Lock_); if (NeedStop_) { @@ -286,6 +349,9 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { LastRequestError_ = TStringBuilder() << "Last request error was at " << FormatSysTimeUtcIsoMicros(now) << ". Failed to prepare IAM request context: " << CurrentExceptionMessage(); + if (MarkFirstTokenReadyLocked()) { + firstTokenError = LastRequestError_; + } ResetContextImpl(); } if (NeedStop_) { @@ -294,35 +360,50 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } if (!Context_.has_value()) { RescheduleOnFailure(); - return true; + } else { + updateTicket = true; } } - UpdateTicket(); + if (firstTokenError) { + FirstTokenReady_.SetException(std::make_exception_ptr(yexception() << *firstTokenError)); + } + if (updateTicket) { + UpdateTicket(); + } return true; } void ProcessIamResponse(grpc::Status&& status, TResponse&& result) { - std::lock_guard guard(Lock_); + bool setFirstTokenReady = false; - if (!status.ok()) { - LastRequestError_ = TStringBuilder() - << "Last request error was at " << FormatSysTimeUtcIsoMicros(SysClock::now()) - << ". GrpcStatusCode: " << static_cast(status.error_code()) - << " Message: \"" << status.error_message() - << "\" iam-endpoint: \"" << IamEndpoint_.Endpoint << "\""; + { + std::lock_guard guard(Lock_); - RescheduleOnFailure(); - } else { - LastRequestError_ = ""; - Ticket_ = result.iam_token(); + if (!status.ok()) { + LastRequestError_ = TStringBuilder() + << "Last request error was at " << FormatSysTimeUtcIsoMicros(SysClock::now()) + << ". GrpcStatusCode: " << static_cast(status.error_code()) + << " Message: \"" << status.error_message() + << "\" iam-endpoint: \"" << IamEndpoint_.Endpoint << "\""; - const SysTimePoint expiresAt = SysClock::from_time_t(result.expires_at().seconds()); - RescheduleOnSuccess(expiresAt); + RescheduleOnFailure(); + } else { + LastRequestError_ = ""; + Ticket_ = result.iam_token(); - TokenReady_.notify_all(); + const SysTimePoint expiresAt = SysClock::from_time_t(result.expires_at().seconds()); + RescheduleOnSuccess(expiresAt); + + setFirstTokenReady = MarkFirstTokenReadyLocked(); + TokenReady_.notify_all(); + } + + ResetContextImpl(); } - ResetContextImpl(); + if (setFirstTokenReady) { + FirstTokenReady_.SetValue(); + } } void RescheduleOnFailure() { // call with Lock_ @@ -363,6 +444,8 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { std::mutex Lock_; std::weak_ptr ResponseFacility_; TCredentialsProviderPtr AuthTokenProvider_; + NThreading::TPromise FirstTokenReady_; + bool FirstTokenReadySet_; }; public: @@ -370,11 +453,14 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { const TRequestFiller& requestFiller, TAsyncRpc rpc, std::weak_ptr responseFacility, - TCredentialsProviderPtr authTokenProvider = nullptr) + TCredentialsProviderPtr authTokenProvider = nullptr, + bool waitForToken = true) : Impl_(std::make_shared(endpoint, requestFiller, rpc, std::move(responseFacility), authTokenProvider)) { Impl_->StartPeriodicTask(); - Impl_->WaitForToken(); + if (waitForToken) { + Impl_->WaitForToken(); + } } ~TGrpcIamCredentialsProvider() { @@ -389,6 +475,10 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { return true; } + NThreading::TFuture GetReadyFuture() const { + return Impl_->GetReadyFuture(); + } + private: std::shared_ptr Impl_; }; @@ -420,28 +510,48 @@ class TOwningFacilityCredentialsProvider : public ICredentialsProvider { TCredentialsProviderPtr Inner_; }; +namespace NPrivate { + +template +NThreading::TFuture CreateGrpcIamCredentialsProviderAsync( + const TParams& params, + std::weak_ptr facility, + std::shared_ptr ownedFacility = {}) +{ + auto inner = std::make_shared(params, std::move(facility), false); + auto ready = inner->GetReadyFuture(); + TCredentialsProviderPtr provider = std::move(inner); + if (ownedFacility) { + provider = std::make_shared( + std::move(ownedFacility), std::move(provider)); + } + return ready.Return(std::move(provider)); +} + +} // namespace NPrivate + template class TIamJwtCredentialsProvider : public TGrpcIamCredentialsProvider { public: - TIamJwtCredentialsProvider(const TIamJwtParams& params, std::weak_ptr responseFacility) + TIamJwtCredentialsProvider(const TIamJwtParams& params, std::weak_ptr responseFacility, bool waitForToken = true) : TGrpcIamCredentialsProvider(params, [jwtParams = params.JwtParams](TRequest& req) { req.set_jwt(MakeSignedJwt(jwtParams)); }, [](typename TService::Stub* stub, grpc::ClientContext* context, const TRequest* request, TResponse* response, std::function cb) { stub->async()->Create(context, request, response, std::move(cb)); - }, std::move(responseFacility), nullptr) {} + }, std::move(responseFacility), nullptr, waitForToken) {} }; template class TIamOAuthCredentialsProvider : public TGrpcIamCredentialsProvider { public: - TIamOAuthCredentialsProvider(const TIamOAuth& params, std::weak_ptr responseFacility) + TIamOAuthCredentialsProvider(const TIamOAuth& params, std::weak_ptr responseFacility, bool waitForToken = true) : TGrpcIamCredentialsProvider(params, [token = params.OAuthToken](TRequest& req) { req.set_yandex_passport_oauth_token(TStringType{token}); }, [](typename TService::Stub* stub, grpc::ClientContext* context, const TRequest* request, TResponse* response, std::function cb) { stub->async()->Create(context, request, response, std::move(cb)); - }, std::move(responseFacility), nullptr) {} + }, std::move(responseFacility), nullptr, waitForToken) {} }; template @@ -464,6 +574,12 @@ class TIamJwtCredentialsProviderFactory : public ICredentialsProviderFactory { }); } + NThreading::TFuture CreateProviderAsync() const override { + auto facility = CreateSimpleCoreFacility(); + return NPrivate::CreateGrpcIamCredentialsProviderAsync< + TIamJwtCredentialsProvider>(Params_, facility, facility); + } + TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const override { return std::make_shared>(Params_, std::move(facility)); } @@ -479,6 +595,11 @@ class TIamJwtCredentialsProviderFactory : public ICredentialsProviderFactory { Params_.JwtParams.PrivKey); } + NThreading::TFuture CreateProviderAsync(std::weak_ptr facility) const override { + return NPrivate::CreateGrpcIamCredentialsProviderAsync< + TIamJwtCredentialsProvider>(Params_, std::move(facility)); + } + private: TIamJwtParams Params_; }; @@ -501,6 +622,12 @@ class TIamOAuthCredentialsProviderFactory : public ICredentialsProviderFactory { }); } + NThreading::TFuture CreateProviderAsync() const override { + auto facility = CreateSimpleCoreFacility(); + return NPrivate::CreateGrpcIamCredentialsProviderAsync< + TIamOAuthCredentialsProvider>(Params_, facility, facility); + } + TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const override { return std::make_shared>(Params_, std::move(facility)); } @@ -513,6 +640,11 @@ class TIamOAuthCredentialsProviderFactory : public ICredentialsProviderFactory { Params_.OAuthToken); } + NThreading::TFuture CreateProviderAsync(std::weak_ptr facility) const override { + return NPrivate::CreateGrpcIamCredentialsProviderAsync< + TIamOAuthCredentialsProvider>(Params_, std::move(facility)); + } + private: TIamOAuth Params_; }; diff --git a/include/ydb-cpp-sdk/client/types/credentials/credentials.h b/include/ydb-cpp-sdk/client/types/credentials/credentials.h index 7eaa6994d6..8082c7a6a2 100644 --- a/include/ydb-cpp-sdk/client/types/credentials/credentials.h +++ b/include/ydb-cpp-sdk/client/types/credentials/credentials.h @@ -3,8 +3,11 @@ #include #include +#include + #include #include +#include namespace NYdb::inline V3 { @@ -37,9 +40,15 @@ class ICredentialsProviderFactory { virtual ~ICredentialsProviderFactory() = default; // deprecated, use CreateProvider(std::weak_ptr facility) instead virtual TCredentialsProviderPtr CreateProvider() const = 0; + virtual NThreading::TFuture CreateProviderAsync() const { + return NThreading::MakeFuture(CreateProvider()); + } virtual TCredentialsProviderPtr CreateProvider([[maybe_unused]] std::weak_ptr facility) const { return CreateProvider(); } + virtual NThreading::TFuture CreateProviderAsync(std::weak_ptr facility) const { + return NThreading::MakeFuture(CreateProvider(std::move(facility))); + } virtual std::string GetClientIdentity() const; }; diff --git a/src/client/iam/iam.cpp b/src/client/iam/iam.cpp index 6cf531ab2c..5884118d1a 100644 --- a/src/client/iam/iam.cpp +++ b/src/client/iam/iam.cpp @@ -8,6 +8,7 @@ #include #include +#include #include using namespace yandex::cloud::iam::v1; @@ -148,6 +149,14 @@ class TIamCredentialsProviderFactory : public ICredentialsProviderFactory { return std::make_shared(Params_); } + NThreading::TFuture CreateProviderAsync() const final { + return CreateProviderAsync(std::weak_ptr{}); + } + + NThreading::TFuture CreateProviderAsync(std::weak_ptr facility) const final { + return CreateProviderInBackground(Params_, std::move(facility)); + } + std::string GetClientIdentity() const final { return TStringBuilder() << "TIamCredentialsProviderFactory" << '\t' << @@ -155,6 +164,30 @@ class TIamCredentialsProviderFactory : public ICredentialsProviderFactory { } private: + static NThreading::TFuture CreateProviderInBackground( + TIamHost params, + std::weak_ptr facility) + { + auto promise = NThreading::NewPromise(); + auto createProvider = [params = std::move(params), promise]() mutable { + try { + promise.TrySetValue(std::make_shared(params)); + } catch (...) { + promise.TrySetException(std::current_exception()); + } + }; + try { + if (auto core = facility.lock()) { + core->PostToResponseQueue(std::move(createProvider)); + } else { + createProvider(); + } + } catch (...) { + promise.TrySetException(std::current_exception()); + } + return promise.GetFuture(); + } + TIamHost Params_; }; diff --git a/src/client/iam_private/common/iam.h b/src/client/iam_private/common/iam.h index f892d6a046..a3952e4e4a 100644 --- a/src/client/iam_private/common/iam.h +++ b/src/client/iam_private/common/iam.h @@ -2,6 +2,8 @@ #include +#include + namespace NYdb::inline V3 { template @@ -40,15 +42,34 @@ class TIamServiceCredentialsProviderFactory : public ICredentialsProviderFactory // because TSimpleCoreFacility allows only one periodic task. TCredentialsProvider(const TIamServiceParams& params, std::weak_ptr outerFacility, - TCredentialsProviderPtr authProvider) + TCredentialsProviderPtr authProvider, + bool waitForToken = true) : TGrpcIamCredentialsProvider(params, MakeRequestFiller(params), MakeRpc(), std::move(outerFacility), - std::move(authProvider)) + std::move(authProvider), + waitForToken) {} }; + static NThreading::TFuture CreateProviderAsyncImpl( + TIamServiceParams params, + NThreading::TFuture authProvider, + std::weak_ptr facility, + std::shared_ptr ownedFacility = {}) + { + auto serviceProvider = std::make_shared( + params, std::move(facility), co_await authProvider, false); + auto ready = serviceProvider->GetReadyFuture(); + TCredentialsProviderPtr provider = std::move(serviceProvider); + if (ownedFacility) { + provider = std::make_shared( + std::move(ownedFacility), std::move(provider)); + } + co_return co_await ready.Return(std::move(provider)); + } + public: TIamServiceCredentialsProviderFactory(const TIamServiceParams& params) : Params_(params) @@ -56,7 +77,7 @@ class TIamServiceCredentialsProviderFactory : public ICredentialsProviderFactory // Deprecated. Kept for backward compatibility — see comment on TIamJwtCredentialsProviderFactory. // The nested auth provider gets its own facility (via a recursive no-arg CreateProvider() that - // returns a TOwningFacilityCredentialsProvider). Sharing a TSimpleCoreFacility between two gRPC + // owns its private facility). Sharing a TSimpleCoreFacility between two gRPC // IAM providers would abort: each one registers a periodic task and the facility allows only one. TCredentialsProviderPtr CreateProvider() const override final { return NCredentials::NDetail::GetOrCreateCachedProvider( @@ -71,10 +92,23 @@ class TIamServiceCredentialsProviderFactory : public ICredentialsProviderFactory }); } + NThreading::TFuture CreateProviderAsync() const override { + auto outerFacility = CreateSimpleCoreFacility(); + return CreateProviderAsyncImpl( + Params_, Params_.SystemServiceAccountCredentials->CreateProviderAsync(), + outerFacility, outerFacility); + } + TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const override { return std::make_shared(Params_, std::move(facility)); } + NThreading::TFuture CreateProviderAsync(std::weak_ptr facility) const override { + return CreateProviderAsyncImpl( + Params_, Params_.SystemServiceAccountCredentials->CreateProviderAsync(facility), + facility); + } + std::string GetClientIdentity() const override final { return NIam::NDetail::MakeClientIdentity( "TIamServiceCredentialsProviderFactory", diff --git a/src/client/impl/internal/db_driver_state/state.cpp b/src/client/impl/internal/db_driver_state/state.cpp index 8a072fb330..bc392d9502 100644 --- a/src/client/impl/internal/db_driver_state/state.cpp +++ b/src/client/impl/internal/db_driver_state/state.cpp @@ -59,14 +59,35 @@ TDbDriverState::TDbDriverState( Log.SetFormatter(GetPrefixLogFormatter(GetDatabaseLogPrefix(Database))); } -void TDbDriverState::SetCredentialsProvider(std::shared_ptr credentialsProvider) { - CredentialsProvider = std::move(credentialsProvider); +void TDbDriverState::InitCredentials( + std::shared_ptr credentialsProviderFactory +) { + Credentials = credentialsProviderFactory->CreateProviderAsync(weak_from_this()).Apply( + [](const NThreading::TFuture& future) { + TCredentials result{future.GetValue()}; #ifndef YDB_GRPC_UNSECURE_AUTH - CallCredentials = grpc::MetadataCredentialsFromPlugin( - std::unique_ptr(new TYdbAuthenticator(CredentialsProvider))); + result.CallCredentials = grpc::MetadataCredentialsFromPlugin( + std::unique_ptr(new TYdbAuthenticator(result.Provider))); #endif + return result; + }); + CredentialsReady = Credentials.IgnoreResult(); } +NThreading::TFuture TDbDriverState::GetCredentialsReady() const { + return CredentialsReady; +} + +std::shared_ptr TDbDriverState::GetCredentialsProvider() const { + return Credentials.HasValue() ? Credentials.GetValue().Provider : nullptr; +} + +#ifndef YDB_GRPC_UNSECURE_AUTH +std::shared_ptr TDbDriverState::GetCallCredentials() const { + return Credentials.HasValue() ? Credentials.GetValue().CallCredentials : nullptr; +} +#endif + bool TDbDriverState::AreClientTlsCredentialsValid() const { std::call_once(ClientTlsValidationOnceFlag_, [this]() { ClientTlsValidationDetail_.clear(); @@ -231,10 +252,10 @@ TDbDriverStatePtr TDbDriverStateTracker::GetDriverState( DiscoveryClient_), deleter); - strongState->SetCredentialsProvider( + strongState->InitCredentials( credentialsProviderFactory - ? credentialsProviderFactory->CreateProvider(strongState) - : CreateInsecureCredentialsProviderFactory()->CreateProvider(strongState)); + ? std::move(credentialsProviderFactory) + : CreateInsecureCredentialsProviderFactory()); if (discoveryMode != EDiscoveryMode::Off) { DiscoveryClient_->AddPeriodicTask(CreatePeriodicDiscoveryTask(strongState), DISCOVERY_RECHECK_PERIOD); diff --git a/src/client/impl/internal/db_driver_state/state.h b/src/client/impl/internal/db_driver_state/state.h index 1db84201b2..0f31e96ac3 100644 --- a/src/client/impl/internal/db_driver_state/state.h +++ b/src/client/impl/internal/db_driver_state/state.h @@ -8,6 +8,7 @@ #include #include +#include #include namespace NYdb::inline V3 { @@ -40,6 +41,12 @@ class TDbDriverState NThreading::TFuture DiscoveryCompleted() const; void SignalDiscoveryCompleted(); + void InitCredentials(std::shared_ptr credentialsProviderFactory); + NThreading::TFuture GetCredentialsReady() const; + std::shared_ptr GetCredentialsProvider() const; +#ifndef YDB_GRPC_UNSECURE_AUTH + std::shared_ptr GetCallCredentials() const; +#endif void AddPeriodicTask(TPeriodicCb&& cb, TDeadline::Duration period) override; void PostToResponseQueue(TPostTaskCb&& f) override; @@ -50,7 +57,6 @@ class TDbDriverState void ForEachForeignEndpoint(const TEndpointElectorSafe::THandleCb& cb, const void* tag) const; TBalancingPolicy::TImpl::EPolicyType GetBalancingPolicyType() const; std::string GetEndpoint() const; - void SetCredentialsProvider(std::shared_ptr credentialsProvider); bool AreClientTlsCredentialsValid() const; const std::string& GetClientTlsValidationDetail() const; @@ -58,15 +64,11 @@ class TDbDriverState const std::string DiscoveryEndpoint; const EDiscoveryMode DiscoveryMode; const TSslCredentials SslCredentials; - std::shared_ptr CredentialsProvider; IInternalClient* Client; TEndpointPool EndpointPool; // StopCb allow client to subscribe for notifications from lower layer std::mutex NotifyCbsLock; std::array, static_cast(ENotifyType::COUNT)> NotifyCbs; -#ifndef YDB_GRPC_UNSECURE_AUTH - std::shared_ptr CallCredentials; -#endif // Status of last discovery call, used in sync mode, coresponding mutex std::shared_mutex LastDiscoveryStatusRWLock; TPlainStatus LastDiscoveryStatus; @@ -75,6 +77,15 @@ class TDbDriverState NThreading::TPromise DiscoveryCompletedPromise; private: + struct TCredentials { + std::shared_ptr Provider; +#ifndef YDB_GRPC_UNSECURE_AUTH + std::shared_ptr CallCredentials; +#endif + }; + + NThreading::TFuture CredentialsReady; + NThreading::TFuture Credentials; mutable std::once_flag ClientTlsValidationOnceFlag_; mutable bool ClientTlsCredentialsValid_ = true; mutable std::string ClientTlsValidationDetail_; diff --git a/src/client/impl/internal/grpc_connections/grpc_connections.cpp b/src/client/impl/internal/grpc_connections/grpc_connections.cpp index 915f9c4b35..83fc5930e0 100644 --- a/src/client/impl/internal/grpc_connections/grpc_connections.cpp +++ b/src/client/impl/internal/grpc_connections/grpc_connections.cpp @@ -84,6 +84,44 @@ namespace { }; } +namespace { + +using TCredentialsWaitResult = TGRpcConnectionsImpl::TCredentialsWaitResult; + +TPlainStatus InitFailedStatus(const std::exception* e = nullptr) { + TStringBuilder message; + message << "Credentials provider initialization failed"; + if (e) { + message << ". " << e->what(); + } + return TPlainStatus(EStatus::CLIENT_UNAUTHENTICATED, message); +} + +TPlainStatus InitCancelledStatus() { + return TPlainStatus(EStatus::CLIENT_CANCELLED, "Client is stopped"); +} + +TCredentialsWaitResult ReadyResult(const NThreading::TFuture& future) { + try { + future.GetValue(); + return {}; + } catch (const std::exception& e) { + return InitFailedStatus(&e); + } catch (...) { + return InitFailedStatus(); + } +} + +bool ScheduledSuccessfully(const NThreading::TFuture& future) { + try { + return future.GetValue(); + } catch (...) { + return false; + } +} + +} // anonymous namespace + bool TDriverStopState::TryEnterCallback() noexcept { std::unique_lock lock(Mutex_); if (Stopped_) { @@ -141,6 +179,102 @@ bool TSdkCallbackGuard::IsEntered() const noexcept { return Entered_; } +NThreading::TFuture TGRpcConnectionsImpl::CredentialsReadyToWaitFor( + const TDbDriverStatePtr& dbState, + const TRpcRequestSettings& requestSettings, + const IQueueClientContextPtr& context) const +{ + if (!requestSettings.UseAuth) { + return {}; + } + auto ready = dbState->GetCredentialsReady(); + return ready.HasValue() && !(context && context->IsCancelled()) + ? NThreading::TFuture{} + : ready; +} + +void TGRpcConnectionsImpl::DeferUntilCredentialsReady( + const TRpcRequestSettings& requestSettings, + IQueueClientContextPtr& context, + NThreading::TFuture credentialsReady, + TCredentialsCallback callback) +{ + auto cancelled = NThreading::NewPromise(); + if (!credentialsReady.IsReady()) { + TryCreateContext(context); + if (context) { + context->SubscribeCancel([cancelled]() mutable { + cancelled.TrySetValue(); + }); + } else { + cancelled.SetValue(); + } + } else if (context && context->IsCancelled()) { + cancelled.SetValue(); + } + + auto scheduleContext = context; + auto schedule = [this, scheduleContext, stopState = StopState_](TDeadline deadline) { + TSdkCallbackGuard guard(stopState); + if (!guard.IsEntered()) { + return NThreading::MakeFuture(false); + } + const auto now = TDeadline::Clock::now(); + const auto timeout = deadline.GetTimePoint() <= now + ? TDuration::Zero() + : TDuration::MicroSeconds(std::chrono::duration_cast( + deadline.GetTimePoint() - now).count()); + return ScheduleFuture(timeout, scheduleContext); + }; + + NThreading::TFuture wait; + if (credentialsReady.IsReady()) { + auto status = ReadyResult(credentialsReady); + wait = NThreading::MakeFuture(status || !cancelled.HasValue() + ? std::move(status) + : TCredentialsWaitResult(InitCancelledStatus())); + } else { + auto result = NThreading::NewPromise(); + wait = result.GetFuture(); + credentialsReady.Subscribe([result](const NThreading::TFuture& future) mutable { + result.TrySetValue(ReadyResult(future)); + }); + cancelled.GetFuture().Subscribe([result](const NThreading::TFuture&) mutable { + result.TrySetValue(InitCancelledStatus()); + }); + if (requestSettings.Deadline != TDeadline::Max()) { + try { + schedule(requestSettings.Deadline).Subscribe( + [result](const NThreading::TFuture& future) mutable { + result.TrySetValue(ScheduledSuccessfully(future) + ? TPlainStatus(EStatus::CLIENT_DEADLINE_EXCEEDED, + "Request deadline exceeded while waiting for credentials") + : InitCancelledStatus()); + }); + } catch (...) { + result.TrySetValue(InitCancelledStatus()); + } + } + } + + wait.Subscribe([callback = std::move(callback), schedule = std::move(schedule)] + (const NThreading::TFuture& future) mutable { + NThreading::TFuture scheduled; + try { + scheduled = schedule(TDeadline::Now()); + } catch (...) { + callback(InitCancelledStatus()); + return; + } + scheduled.Subscribe([callback = std::move(callback), status = future.GetValue()] + (const NThreading::TFuture& future) mutable { + callback(ScheduledSuccessfully(future) + ? std::move(status) + : TCredentialsWaitResult(InitCancelledStatus())); + }); + }); +} + bool IsTokenCorrect(const std::string& in) { for (char c : in) { if (!(IsAsciiAlnum(c) || IsAsciiPunct(c) || c == ' ')) { @@ -152,7 +286,11 @@ bool IsTokenCorrect(const std::string& in) { std::string GetAuthInfo(TDbDriverStatePtr p) { try { - auto token = p->CredentialsProvider->GetAuthInfo(); + auto credentialsProvider = p->GetCredentialsProvider(); + if (!credentialsProvider) { + throw TAuthenticationError("Credentials provider is not initialized"); + } + auto token = credentialsProvider->GetAuthInfo(); if (!IsTokenCorrect(token)) { throw TAuthenticationError("token is incorrect, illegal characters found"); } @@ -375,6 +513,7 @@ TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr p TGRpcConnectionsImpl::~TGRpcConnectionsImpl() { Stop(true); StopState_->MarkStopped(); + StopState_->WaitCallbacksDrained(); } bool TGRpcConnectionsImpl::IsCurrentThreadInSdkCallback() noexcept { @@ -718,9 +857,12 @@ TCallMeta TGRpcConnectionsImpl::MakeCallMeta(const TRpcRequestSettings& requestS TCallMeta meta; meta.Timeout = requestSettings.Deadline; #ifndef YDB_GRPC_UNSECURE_AUTH - meta.CallCredentials = dbState->CallCredentials; + if (requestSettings.UseAuth) { + meta.CallCredentials = dbState->GetCallCredentials(); + } #else - if (requestSettings.UseAuth && dbState->CredentialsProvider && dbState->CredentialsProvider->IsValid()) { + auto credentialsProvider = dbState->GetCredentialsProvider(); + if (requestSettings.UseAuth && credentialsProvider && credentialsProvider->IsValid()) { meta.Aux.push_back({YDB_AUTH_TICKET_HEADER, GetAuthInfo(dbState)}); } #endif diff --git a/src/client/impl/internal/grpc_connections/grpc_connections.h b/src/client/impl/internal/grpc_connections/grpc_connections.h index 241e920254..65829f5c68 100644 --- a/src/client/impl/internal/grpc_connections/grpc_connections.h +++ b/src/client/impl/internal/grpc_connections/grpc_connections.h @@ -20,6 +20,12 @@ #include #include +#if defined(_asan_enabled_) +#define YDB_ASAN_SIZE_ATTRIBUTES __attribute__((nodebug)) +#else +#define YDB_ASAN_SIZE_ATTRIBUTES +#endif + namespace NYdb::inline V3 { namespace NMetrics { @@ -146,6 +152,20 @@ class TGRpcConnectionsImpl static void SetGrpcCompressionAlgorithm(NYdbGrpc::TGRpcClientConfig& config, EGrpcCompressionAlgorithm algorithm); + using TCredentialsWaitResult = std::optional; + using TCredentialsCallback = std::function; + + NThreading::TFuture CredentialsReadyToWaitFor( + const TDbDriverStatePtr& dbState, + const TRpcRequestSettings& requestSettings, + const IQueueClientContextPtr& context) const; + + void DeferUntilCredentialsReady( + const TRpcRequestSettings& requestSettings, + IQueueClientContextPtr& context, + NThreading::TFuture credentialsReady, + TCredentialsCallback callback); + template std::pair>, TEndpointKey> GetServiceConnection( TDbDriverStatePtr dbState, const TEndpointKey& preferredEndpoint, @@ -277,6 +297,7 @@ class TGRpcConnectionsImpl TRequestWrapper& operator=(TRequestWrapper&& other) = default; template + YDB_ASAN_SIZE_ATTRIBUTES void DoRequest( std::unique_ptr>& serviceConnection, NYdbGrpc::TAdvancedResponseCallback&& responseCbLow, @@ -298,6 +319,7 @@ class TGRpcConnectionsImpl }; template + YDB_ASAN_SIZE_ATTRIBUTES void Run( TRequestWrapper&& requestWrapper, TResponseCb&& userResponseCb, @@ -310,6 +332,26 @@ class TGRpcConnectionsImpl using TConnection = std::unique_ptr>; Y_ABORT_UNLESS(dbState); + if (auto ready = CredentialsReadyToWaitFor(dbState, requestSettings, context); ready.Initialized()) { + DeferUntilCredentialsReady(requestSettings, context, std::move(ready), + [this, requestWrapper = std::move(requestWrapper), userResponseCb = std::move(userResponseCb), + rpc, dbState, requestSettings, context = std::move(context)] + (std::optional status) YDB_ASAN_SIZE_ATTRIBUTES mutable { + if (status) { + userResponseCb(nullptr, std::move(*status)); + return; + } + Run( + std::move(requestWrapper), + std::move(userResponseCb), + rpc, + std::move(dbState), + requestSettings, + std::move(context)); + }); + return; + } + if (auto tlsValidationStatus = ValidateClientTlsCredentials(dbState)) { RunResponseCallback(userResponseCb, nullptr, std::move(*tlsValidationStatus), StopState_); return; @@ -535,6 +577,7 @@ class TGRpcConnectionsImpl TResponse>::TAsyncRequest; template + YDB_ASAN_SIZE_ATTRIBUTES void StartReadStream( const TRequest& request, TCallback responseCb, @@ -547,6 +590,25 @@ class TGRpcConnectionsImpl using TConnection = std::unique_ptr>; using TProcessor = typename NYdbGrpc::IStreamRequestReadProcessor::TPtr; + if (auto ready = CredentialsReadyToWaitFor(dbState, requestSettings, context); ready.Initialized()) { + DeferUntilCredentialsReady(requestSettings, context, std::move(ready), + [this, request, responseCb = std::move(responseCb), rpc, dbState, requestSettings, context = std::move(context)] + (std::optional status) YDB_ASAN_SIZE_ATTRIBUTES mutable { + if (status) { + responseCb(std::move(*status), nullptr); + return; + } + StartReadStream( + request, + std::move(responseCb), + rpc, + std::move(dbState), + requestSettings, + std::move(context)); + }); + return; + } + if (auto tlsValidationStatus = ValidateClientTlsCredentials(dbState)) { RunStreamCallback(responseCb, std::move(*tlsValidationStatus), nullptr, StopState_); return; @@ -623,6 +685,7 @@ class TGRpcConnectionsImpl } template + YDB_ASAN_SIZE_ATTRIBUTES void StartBidirectionalStream( TCallback connectedCallback, TStreamRpc rpc, @@ -634,6 +697,24 @@ class TGRpcConnectionsImpl using TConnection = std::unique_ptr>; using TProcessor = typename NYdbGrpc::IStreamRequestReadWriteProcessor::TPtr; + if (auto ready = CredentialsReadyToWaitFor(dbState, requestSettings, context); ready.Initialized()) { + DeferUntilCredentialsReady(requestSettings, context, std::move(ready), + [this, connectedCallback = std::move(connectedCallback), rpc, dbState, requestSettings, context = std::move(context)] + (std::optional status) YDB_ASAN_SIZE_ATTRIBUTES mutable { + if (status) { + connectedCallback(std::move(*status), nullptr); + return; + } + StartBidirectionalStream( + std::move(connectedCallback), + rpc, + std::move(dbState), + requestSettings, + std::move(context)); + }); + return; + } + if (auto tlsValidationStatus = ValidateClientTlsCredentials(dbState)) { RunStreamCallback(connectedCallback, std::move(*tlsValidationStatus), nullptr, StopState_); return; @@ -748,6 +829,7 @@ class TGRpcConnectionsImpl } template + YDB_ASAN_SIZE_ATTRIBUTES void WithServiceConnection(TCallback callback, TDbDriverStatePtr dbState, const TEndpointKey& preferredEndpoint, TRpcRequestSettings::TEndpointPolicy endpointPolicy) { @@ -913,3 +995,5 @@ struct TGRpcConnectionsDeleter { }; } // namespace NYdb + +#undef YDB_ASAN_SIZE_ATTRIBUTES diff --git a/src/client/persqueue_public/impl/write_session_impl.cpp b/src/client/persqueue_public/impl/write_session_impl.cpp index 2029893d34..26bd88fa2e 100644 --- a/src/client/persqueue_public/impl/write_session_impl.cpp +++ b/src/client/persqueue_public/impl/write_session_impl.cpp @@ -29,7 +29,7 @@ TWriteSessionImpl::TWriteSessionImpl( , Client(std::move(client)) , Connections(std::move(connections)) , DbDriverState(std::move(dbDriverState)) - , PrevToken(DbDriverState->CredentialsProvider ? DbDriverState->CredentialsProvider->GetAuthInfo() : "") + , PrevToken(DbDriverState->GetCredentialsProvider() ? DbDriverState->GetCredentialsProvider()->GetAuthInfo() : "") , InitSeqNoPromise(NThreading::NewPromise()) , WakeupInterval( Settings.BatchFlushInterval_ != TDuration::Zero() ? @@ -1183,11 +1183,12 @@ void TWriteSessionImpl::UpdateTokenIfNeededImpl() { LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: try to update token"); - if (!DbDriverState->CredentialsProvider || UpdateTokenInProgress || !SessionEstablished) + auto credentialsProvider = DbDriverState->GetCredentialsProvider(); + if (!credentialsProvider || UpdateTokenInProgress || !SessionEstablished) return; TClientMessage clientMessage; auto* updateRequest = clientMessage.mutable_update_token_request(); - auto token = DbDriverState->CredentialsProvider->GetAuthInfo(); + auto token = credentialsProvider->GetAuthInfo(); if (token == PrevToken) return; UpdateTokenInProgress = true; diff --git a/src/client/topic/impl/write_session_impl.cpp b/src/client/topic/impl/write_session_impl.cpp index 8a7ff36fce..d357d987c6 100644 --- a/src/client/topic/impl/write_session_impl.cpp +++ b/src/client/topic/impl/write_session_impl.cpp @@ -114,7 +114,7 @@ TWriteSessionImpl::TWriteSessionImpl( , Client(std::move(client)) , Connections(std::move(connections)) , DbDriverState(std::move(dbDriverState)) - , PrevToken(DbDriverState->CredentialsProvider ? DbDriverState->CredentialsProvider->GetAuthInfo() : "") + , PrevToken(DbDriverState->GetCredentialsProvider() ? DbDriverState->GetCredentialsProvider()->GetAuthInfo() : "") , MaxBlockMessageCount(Settings.BatchFlushMessageCount_) , InitSeqNoPromise(NThreading::NewPromise()) , WakeupInterval( @@ -1586,11 +1586,12 @@ void TWriteSessionImpl::UpdateTokenIfNeededImpl() { LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefixImpl() << "Write session: try to update token"); - if (!DbDriverState->CredentialsProvider || UpdateTokenInProgress || !SessionEstablished) { + auto credentialsProvider = DbDriverState->GetCredentialsProvider(); + if (!credentialsProvider || UpdateTokenInProgress || !SessionEstablished) { return; } - auto token = DbDriverState->CredentialsProvider->GetAuthInfo(); + auto token = credentialsProvider->GetAuthInfo(); if (token == PrevToken) { return; } diff --git a/src/client/types/credentials/login/login.cpp b/src/client/types/credentials/login/login.cpp index 724039cb2e..af26b9b9d1 100644 --- a/src/client/types/credentials/login/login.cpp +++ b/src/client/types/credentials/login/login.cpp @@ -9,6 +9,8 @@ #include +#include + using namespace std::chrono_literals; namespace NYdb::inline V3 { @@ -42,10 +44,12 @@ class TLoginCredentialsProvider : public ICredentialsProvider { TLoginCredentialsProvider(std::weak_ptr facility, TLoginCredentialsParams params); virtual std::string GetAuthInfo() const override; virtual bool IsValid() const override; + NThreading::TFuture PrepareTokenAsync(); private: void PrepareToken(); void RequestToken(); + void FinishRequest(Ydb::Auth::LoginResponse* response, TPlainStatus status, bool facilityAvailable); bool IsOk() const; void ParseToken(); std::string GetToken() const; @@ -62,7 +66,6 @@ class TLoginCredentialsProvider : public ICredentialsProvider { TLoginCredentialsParams Params_; EState State_ = EState::Empty; std::mutex Mutex_; - std::condition_variable Notify_; std::atomic TokenReceived_ = 1; std::atomic TokenParsed_ = 0; std::optional Token_; @@ -71,11 +74,13 @@ class TLoginCredentialsProvider : public ICredentialsProvider { TInstant TokenRequestAt_; TPlainStatus Status_; Ydb::Auth::LoginResponse Response_; + NThreading::TPromise TokenReadyPromise_; }; TLoginCredentialsProvider::TLoginCredentialsProvider(std::weak_ptr facility, TLoginCredentialsParams params) : Facility_(facility) , Params_(std::move(params)) + , TokenReadyPromise_(NThreading::NewPromise()) { auto strongFacility = facility.lock(); if (strongFacility) { @@ -123,16 +128,7 @@ void TLoginCredentialsProvider::RequestToken() { auto responseCb = [facility = Facility_, this](Ydb::Auth::LoginResponse* resp, TPlainStatus status) { auto strongFacility = facility.lock(); - if (strongFacility) { - std::lock_guard lock(Mutex_); - Status_ = std::move(status); - if (resp != nullptr) { - Response_ = std::move(*resp); - } - State_ = EState::Done; - TokenReceived_++; - } - Notify_.notify_all(); + FinishRequest(resp, std::move(status), static_cast(strongFacility)); }; Ydb::Auth::LoginRequest request; @@ -144,25 +140,61 @@ void TLoginCredentialsProvider::RequestToken() { TGRpcConnectionsImpl::RunOnDiscoveryEndpoint( strongFacility, std::move(request), std::move(responseCb), &Ydb::Auth::V1::AuthService::Stub::AsyncLogin, rpcSettings); + } else { + FinishRequest(nullptr, {}, false); + } +} + +void TLoginCredentialsProvider::FinishRequest( + Ydb::Auth::LoginResponse* response, + TPlainStatus status, + bool facilityAvailable) +{ + std::optional error; + { + std::lock_guard lock(Mutex_); + State_ = EState::Done; + ++TokenReceived_; + if (facilityAvailable) { + Status_ = std::move(status); + if (response) { + Response_ = std::move(*response); + } + ParseToken(); + } else { + Token_.reset(); + Error_ = "Login credentials provider response facility is not available"; + TokenParsed_ = TokenReceived_.load(); + } + error = Error_; + } + if (error) { + TokenReadyPromise_.TrySetException(std::make_exception_ptr(yexception() << *error)); + } else { + TokenReadyPromise_.TrySetValue(); } } void TLoginCredentialsProvider::PrepareToken() { - std::unique_lock lock(Mutex_); - switch (State_) { - case EState::Empty: + PrepareTokenAsync().Wait(); + std::lock_guard lock(Mutex_); + ParseToken(); +} + +NThreading::TFuture TLoginCredentialsProvider::PrepareTokenAsync() { + bool requestToken = false; + auto future = TokenReadyPromise_.GetFuture(); + { + std::unique_lock lock(Mutex_); + if (State_ == EState::Empty) { State_ = EState::Requesting; - RequestToken(); - [[fallthrough]]; - case EState::Requesting: - Notify_.wait(lock, [&]{ - return State_ == EState::Done; - }); - [[fallthrough]]; - case EState::Done: - ParseToken(); - break; + requestToken = true; + } + } + if (requestToken) { + RequestToken(); } + return future; } bool TLoginCredentialsProvider::IsOk() const { @@ -226,6 +258,7 @@ class TLoginCredentialsProviderFactory : public ICredentialsProviderFactory { TLoginCredentialsProviderFactory(TLoginCredentialsParams params); virtual std::shared_ptr CreateProvider() const override; virtual std::shared_ptr CreateProvider(std::weak_ptr facility) const override; + virtual NThreading::TFuture CreateProviderAsync(std::weak_ptr facility) const override; private: TLoginCredentialsParams Params_; @@ -244,6 +277,12 @@ std::shared_ptr TLoginCredentialsProviderFactory::CreatePr return std::make_shared(std::move(facility), Params_); } +NThreading::TFuture TLoginCredentialsProviderFactory::CreateProviderAsync(std::weak_ptr facility) const { + auto provider = std::make_shared(std::move(facility), Params_); + TCredentialsProviderPtr result = provider; + return provider->PrepareTokenAsync().Return(std::move(result)); +} + std::shared_ptr CreateLoginCredentialsProviderFactory(TLoginCredentialsParams params) { return std::make_shared(std::move(params)); } diff --git a/tests/unit/client/driver/driver_ut.cpp b/tests/unit/client/driver/driver_ut.cpp index 1ebc148081..ef2c43eb36 100644 --- a/tests/unit/client/driver/driver_ut.cpp +++ b/tests/unit/client/driver/driver_ut.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -109,8 +110,68 @@ namespace { std::atomic_int& ProviderCount_; }; + class TDeferredCredentialsFactory final : public ICredentialsProviderFactory { + public: + TDeferredCredentialsFactory() + : Provider_(NThreading::NewPromise()) + {} + + TCredentialsProviderPtr CreateProvider() const override { + return CreateInsecureCredentialsProviderFactory()->CreateProvider(); + } + + NThreading::TFuture CreateProviderAsync(std::weak_ptr) const override { + return Provider_.GetFuture(); + } + + void SetReady() { + Provider_.SetValue(CreateProvider()); + } + + private: + NThreading::TPromise Provider_; + }; + } // namespace +Y_UNIT_TEST_SUITE(DeferredCredentialsTest) { + Y_UNIT_TEST(RequestWaitsForCredentials) { + auto factory = std::make_shared(); + auto driver = TDriver(TDriverConfig() + .SetEndpoint("localhost:100") + .SetCredentialsProviderFactory(factory)); + auto result = TTableClient(driver).CreateSession(); + + UNIT_ASSERT(!result.Wait(TDuration::MilliSeconds(100))); + factory->SetReady(); + UNIT_ASSERT(result.Wait(TDuration::Seconds(10))); + UNIT_ASSERT_VALUES_EQUAL(result.GetValue().GetStatus(), EStatus::TRANSPORT_UNAVAILABLE); + } + + Y_UNIT_TEST(RequestDeadlineWhileWaitingForCredentials) { + auto factory = std::make_shared(); + auto driver = TDriver(TDriverConfig() + .SetEndpoint("localhost:100") + .SetCredentialsProviderFactory(factory)); + auto result = TTableClient(driver).CreateSession( + TCreateSessionSettings().ClientTimeout(TDuration::MilliSeconds(100))).GetValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::CLIENT_DEADLINE_EXCEEDED); + } + + Y_UNIT_TEST(DriverStopCancelsCredentialsWait) { + auto factory = std::make_shared(); + auto driver = TDriver(TDriverConfig() + .SetEndpoint("localhost:100") + .SetCredentialsProviderFactory(factory)); + auto result = TTableClient(driver).CreateSession(); + + driver.Stop(true); + UNIT_ASSERT(result.Wait(TDuration::Seconds(10))); + UNIT_ASSERT_VALUES_EQUAL(result.GetValue().GetStatus(), EStatus::CLIENT_CANCELLED); + } +} + Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) { Y_UNIT_TEST(ReusesCredentialsProviderForSameIdentity) { std::atomic_int providerCount = 0; diff --git a/tests/unit/client/iam_private/grpc_iam_service_ut.cpp b/tests/unit/client/iam_private/grpc_iam_service_ut.cpp index 935f985394..3fdf2f1de0 100644 --- a/tests/unit/client/iam_private/grpc_iam_service_ut.cpp +++ b/tests/unit/client/iam_private/grpc_iam_service_ut.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -76,6 +77,52 @@ TIamServiceParams MakeServiceParams(TCredentialsProviderFactoryPtr nestedFactory return params; } +class TFlakyIamServiceStub final : public IamTokenService::Service { +public: + grpc::Status Create( + grpc::ServerContext*, + const CreateIamTokenRequest*, + CreateIamTokenResponse* response) override + { + if (Attempts_.fetch_add(1) == 0) { + return grpc::Status(grpc::StatusCode::UNAVAILABLE, "transient failure"); + } + response->set_iam_token("oauth-token"); + response->mutable_expires_at()->set_seconds(4102444800); + return grpc::Status::OK; + } + + size_t Attempts() const { + return Attempts_.load(); + } + +private: + std::atomic_size_t Attempts_ = 0; +}; + +class TFailingCoreFacility final : public ICoreFacility { +public: + void AddPeriodicTask(TPeriodicCb&& callback, TDeadline::Duration) override { + NYdb::NIssue::TIssues issues; + callback(std::move(issues), EStatus::CLIENT_CANCELLED); + } + + void PostToResponseQueue(TPostTaskCb&& callback) override { + callback(); + } +}; + +using TTestOAuthFactory = TIamOAuthCredentialsProviderFactory< + CreateIamTokenRequest, CreateIamTokenResponse, IamTokenService>; + +TTestOAuthFactory MakeOAuthFactory() { + TIamOAuth params; + params.Endpoint = "localhost:1"; + params.EnableSsl = false; + params.OAuthToken = "token"; + return TTestOAuthFactory(params); +} + } // namespace TEST(IamCredentialsProviderIdentity, IdentityIsValueBased) { @@ -142,6 +189,42 @@ TEST(IamServiceCredentialsProvider, NoArgProviderIsCachedAcrossFactoryInstances) server.Stop(); } +TEST(IamCredentialsProvider, AsyncCreationFailsWithExpiredFacility) { + auto future = MakeOAuthFactory().CreateProviderAsync(std::weak_ptr{}); + + ASSERT_TRUE(future.IsReady()); + EXPECT_THROW(future.GetValue(), std::exception); +} + +TEST(IamCredentialsProvider, AsyncCreationFailsWhenPeriodicTaskIsRejected) { + auto facility = std::make_shared(); + auto future = MakeOAuthFactory().CreateProviderAsync(std::weak_ptr(facility)); + + ASSERT_TRUE(future.IsReady()); + EXPECT_THROW(future.GetValue(), std::exception); +} + +TEST(IamCredentialsProvider, AsyncCreationRetriesTransientIamFailure) { + TFlakyIamServiceStub stub; + TIamGrpcServer server(&stub); + ASSERT_TRUE(server.Start()); + + TIamOAuth params; + params.Endpoint = server.Endpoint(); + params.EnableSsl = false; + params.OAuthToken = "token"; + params.RequestTimeout = TDuration::Seconds(2); + + auto future = TTestOAuthFactory(params).CreateProviderAsync(); + ASSERT_TRUE(future.Wait(TDuration::Seconds(10))); + auto provider = future.GetValue(); + + EXPECT_EQ(provider->GetAuthInfo(), "oauth-token"); + EXPECT_GE(stub.Attempts(), 2u); + + server.Stop(); +} + // Regression test for the deprecated no-arg CreateProvider() on the IAM service-account // factory with a nested gRPC JWT auth provider. Before the fix, both providers shared a single // TSimpleCoreFacility, each registered a periodic refresh task, and TSimpleCoreFacility's From ea0d80517e364158782b1184d10cdd56a18502e4 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:49:16 +0000 Subject: [PATCH 34/56] Fix createsession metric (#47132) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 ++ src/client/query/client.cpp | 19 +++++-------------- tests/integration/metrics/main.cpp | 18 +++++++++++++++--- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 11cfcd65d6..e2e87f0964 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -f7303ada674f0f072da2f4271a82af28e1029245 +28cf090af7d7beb5691b463c768d3dcb3f37c5a0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7829779d05..774b189382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Fixed Query SDK `CreateSession` metrics being recorded when reusing a session from the pool. + * Added `TQueryClient::DeleteSession` to explicitly delete a query session by session id. * Driver now supports async credentials initialisation: the first request is delayed until they are ready. diff --git a/src/client/query/client.cpp b/src/client/query/client.cpp index 78fb31eac2..83d08750ed 100644 --- a/src/client/query/client.cpp +++ b/src/client/query/client.cpp @@ -530,12 +530,10 @@ class TQueryClient::TImpl: public TClientImplCommon, public TAsyncCreateSessionResult GetSession(const TCreateSessionSettings& settings) { class TQueryClientGetSessionCtx : public NSessionPool::IGetSessionCtx { public: - TQueryClientGetSessionCtx(std::shared_ptr client, const TCreateSessionSettings& settings, - std::shared_ptr observation) + TQueryClientGetSessionCtx(std::shared_ptr client, const TCreateSessionSettings& settings) : Promise(NThreading::NewPromise()) , Client(client) , RpcSettings(TRpcRequestSettings::Make(settings)) - , Observation(std::move(observation)) {} TAsyncCreateSessionResult GetFuture() { @@ -544,9 +542,6 @@ class TQueryClient::TImpl: public TClientImplCommon, public void ReplyError(TStatus status) override { TSession session; - if (Observation) { - Observation->End(status.GetStatus(), status.GetEndpoint()); - } ScheduleReply(TCreateSessionResult(std::move(status), std::move(session))); } @@ -559,20 +554,18 @@ class TQueryClient::TImpl: public TClientImplCommon, public ) ); - if (Observation) { - Observation->End(EStatus::SUCCESS, session->GetEndpoint()); - } ScheduleReply(std::move(val)); } void ReplyNewSession() override { + auto obs = Client->MakeObservation("CreateSession"); TRpcRequestSettings deferredRpcSettings = RpcSettings; deferredRpcSettings.Deadline = TDeadline::Max(); Client->CreateAttachedSession( this->Client->Settings_.SessionPoolSettings_.UseDeferredSessionCreation_ ? deferredRpcSettings : RpcSettings).Subscribe( - [promise = Promise, obs = Observation](TAsyncCreateSessionResult future) mutable + [promise = Promise, obs](TAsyncCreateSessionResult future) mutable { auto val = future.ExtractValue(); if (obs) { @@ -582,7 +575,7 @@ class TQueryClient::TImpl: public TClientImplCommon, public }); if (Client->Settings_.SessionPoolSettings_.UseDeferredSessionCreation_) { Client->Connections_->ScheduleDelayedTask( - [promise = Promise, obs = Observation, client = Client]() mutable { + [promise = Promise, obs, client = Client]() mutable { TSession session; promise.TrySetValue(TCreateSessionResult(TStatus(TPlainStatus(EStatus::CLIENT_DEADLINE_EXCEEDED, "GetSession deadline exceeded")), std::move(session))); if (obs) { @@ -615,11 +608,9 @@ class TQueryClient::TImpl: public TClientImplCommon, public NThreading::TPromise Promise; std::shared_ptr Client; const TRpcRequestSettings RpcSettings; - std::shared_ptr Observation; }; - auto obs = MakeObservation("CreateSession"); - auto ctx = std::make_unique(shared_from_this(), settings, obs); + auto ctx = std::make_unique(shared_from_this(), settings); auto future = ctx->GetFuture(); SessionPool_.GetSession(std::move(ctx)); diff --git a/tests/integration/metrics/main.cpp b/tests/integration/metrics/main.cpp index 4ae308fd89..5616f670d5 100644 --- a/tests/integration/metrics/main.cpp +++ b/tests/integration/metrics/main.cpp @@ -188,14 +188,26 @@ TEST(QueryMetricsIntegration, CreateSessionRecordsDuration) { auto args = MakeRunArgs(); TQueryClient client(args.Driver, TClientSettings().Database(args.Database)); - auto session = client.GetSession().ExtractValueSync(); - ASSERT_TRUE(session.IsSuccess()) << session.GetIssues().ToString(); + std::string sessionId; + { + auto sessionResult = client.GetSession().ExtractValueSync(); + ASSERT_TRUE(sessionResult.IsSuccess()) << sessionResult.GetIssues().ToString(); + sessionId = sessionResult.GetSession().GetId(); + } + + ASSERT_EQ(client.GetCurrentPoolSize(), 1); + + { + auto sessionResult = client.GetSession().ExtractValueSync(); + ASSERT_TRUE(sessionResult.IsSuccess()) << sessionResult.GetIssues().ToString(); + EXPECT_EQ(sessionResult.GetSession().GetId(), sessionId); + } auto duration = args.Registry->GetHistogram( "ydb.client.operation.duration", DurationLabels(args.Database, "CreateSession", args.ServerAddress, args.ServerPort)); ASSERT_NE(duration, nullptr) << "CreateSession duration histogram not created"; - EXPECT_GE(duration->Count(), 1u); + EXPECT_EQ(duration->Count(), 1u); args.Driver.Stop(true); } From ef770c9e0b65d14fff1bfe0607924a8032b46397 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Tue, 28 Jul 2026 08:49:26 +0000 Subject: [PATCH 35/56] Implement simplified UAX#29 Standard tokenizer with Elasticsearch-compatible Unicode tests (except Emoji) (#46842) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_table.proto | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index e2e87f0964..637a4643da 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -28cf090af7d7beb5691b463c768d3dcb3f37c5a0 +d68f70459cac3cb95da03d0ab79701d59da527f6 diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index 1797ca9d6c..071eace64f 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -137,12 +137,11 @@ message FulltextIndexSettings { // Tokens: ["foo-bar", "baz_lorem", "ipsum"] WHITESPACE = 1; - // Applies general language-aware tokenization - // Splits text on whitespace and punctuation + // Splits text into sequences of alphabetic and numeric characters // Example: // Text: "foo-bar baz_lorem ipsum" // Tokens: ["foo", "bar", "baz", "lorem", "ipsum"] - STANDARD = 2; + ALPHANUMERIC = 2; // Treats the entire input as a single token // No splitting is performed @@ -150,6 +149,13 @@ message FulltextIndexSettings { // Text: "Hello World!" // Tokens: ["Hello World!"] KEYWORD = 3; + + // Applies general language-aware tokenization + // According to a simplified Unicode UAX#29 standard + // Example: + // Text: "The 24.18 Brown-Foxes jumped over the lazy dog's bone!" + // Tokens: ["The", "24.18", "Brown", "Foxes", "jumped", "over", "the", "lazy", "dog's", "bone"] + STANDARD = 4; } // Represents text analyzers settings From 3a4fea0127bdc9686457152cc84efe99f7214e2a Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:49:35 +0000 Subject: [PATCH 36/56] implemented tracing and metrics observability headers (#46668) --- .github/last_commit.txt | 2 +- .../grpc_connections/grpc_connections.cpp | 18 +++++++++-- .../grpc_connections/grpc_connections.h | 1 + .../internal/rpc_request_settings/settings.h | 1 + src/client/impl/observability/constants.h | 5 +++ tests/unit/client/driver/driver_ut.cpp | 32 +++++++++++++++++-- 6 files changed, 52 insertions(+), 7 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 637a4643da..9dad98c2b0 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -d68f70459cac3cb95da03d0ab79701d59da527f6 +5790bd6cf50cc074ab342c5e537fd0bcdce685d4 diff --git a/src/client/impl/internal/grpc_connections/grpc_connections.cpp b/src/client/impl/internal/grpc_connections/grpc_connections.cpp index 83fc5930e0..7e56f5e31e 100644 --- a/src/client/impl/internal/grpc_connections/grpc_connections.cpp +++ b/src/client/impl/internal/grpc_connections/grpc_connections.cpp @@ -313,8 +313,16 @@ std::string CreateSDKBuildInfo() { return std::string("ydb-cpp-sdk/") + GetSdkSemver(); } -std::string BuildFullBuildInfo(const IConnectionsParams& params) { +std::string BuildFullBuildInfo(const IConnectionsParams& params, bool includeObservability) { auto result = CreateSDKBuildInfo(); + if (includeObservability && params.GetTraceProvider()) { + result += " ydb-sdk-tracing/"; + result += NObservability::kTracingChainVersion; + } + if (includeObservability && params.GetExternalMetricRegistry()) { + result += " ydb-sdk-metrics/"; + result += NObservability::kMetricsChainVersion; + } auto extra = params.GetBuildInfoExtra(); if (!extra.empty()) { result += ';'; @@ -470,7 +478,8 @@ TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr p #endif , MetricRegistry_(params->GetExternalMetricRegistry()) , TraceProvider_(params->GetTraceProvider()) - , BuildInfo_(BuildFullBuildInfo(*params)) + , BuildInfoWithoutObservability_(BuildFullBuildInfo(*params, false)) + , BuildInfo_(BuildFullBuildInfo(*params, true)) , NetworkThreadsNum_(params->GetNetworkThreadsNum()) , UsePerChannelTcpConnection_(params->GetUsePerChannelTcpConnection()) , GRpcClientLow_(NetworkThreadsNum_) @@ -727,6 +736,7 @@ TAsyncListEndpointsResult TGRpcConnectionsImpl::GetEndpoints(TDbDriverStatePtr d TRpcRequestSettings rpcSettings; rpcSettings.Deadline = TDeadline::AfterDuration(GET_ENDPOINTS_TIMEOUT); + rpcSettings.IncludeObservabilityInBuildInfo = true; RunDeferred( std::move(request), @@ -892,7 +902,9 @@ TCallMeta TGRpcConnectionsImpl::MakeCallMeta(const TRpcRequestSettings& requestS static const std::string clientPid = GetClientPIDHeaderValue(); - meta.Aux.push_back({YDB_SDK_BUILD_INFO_HEADER, BuildInfo_}); + meta.Aux.push_back({ + YDB_SDK_BUILD_INFO_HEADER, + requestSettings.IncludeObservabilityInBuildInfo ? BuildInfo_ : BuildInfoWithoutObservability_}); meta.Aux.push_back({YDB_CLIENT_PID, clientPid}); meta.Aux.insert(meta.Aux.end(), requestSettings.Header.begin(), requestSettings.Header.end()); diff --git a/src/client/impl/internal/grpc_connections/grpc_connections.h b/src/client/impl/internal/grpc_connections/grpc_connections.h index 65829f5c68..203187b3c0 100644 --- a/src/client/impl/internal/grpc_connections/grpc_connections.h +++ b/src/client/impl/internal/grpc_connections/grpc_connections.h @@ -981,6 +981,7 @@ class TGRpcConnectionsImpl IDiscoveryMutatorApi::TMutatorCb DiscoveryMutatorCb; + const std::string BuildInfoWithoutObservability_; const std::string BuildInfo_; const std::size_t NetworkThreadsNum_; diff --git a/src/client/impl/internal/rpc_request_settings/settings.h b/src/client/impl/internal/rpc_request_settings/settings.h index a63ddf6532..58ee117bf3 100644 --- a/src/client/impl/internal/rpc_request_settings/settings.h +++ b/src/client/impl/internal/rpc_request_settings/settings.h @@ -18,6 +18,7 @@ struct TRpcRequestSettings { UseDiscoveryEndpoint // Use single discovery endpoint } EndpointPolicy = TEndpointPolicy::UsePreferredEndpointOptionally; bool UseAuth = true; + bool IncludeObservabilityInBuildInfo = false; NYdb::TDeadline Deadline = NYdb::TDeadline::Max(); std::string TraceParent; diff --git a/src/client/impl/observability/constants.h b/src/client/impl/observability/constants.h index 0a16fb3bc8..26a4a99c93 100644 --- a/src/client/impl/observability/constants.h +++ b/src/client/impl/observability/constants.h @@ -11,6 +11,11 @@ namespace NYdb::inline V3::NObservability { +// SDK build-info chain versions. Bump these when the corresponding +// observability integration changes incompatibly. +inline constexpr std::string_view kTracingChainVersion = "0.1.0"; +inline constexpr std::string_view kMetricsChainVersion = "0.1.0"; + // --------------------------------------------------------------------------- // OTel Semconv attribute keys shared between span attributes and metric labels. // --------------------------------------------------------------------------- diff --git a/tests/unit/client/driver/driver_ut.cpp b/tests/unit/client/driver/driver_ut.cpp index ef2c43eb36..e2e3a77681 100644 --- a/tests/unit/client/driver/driver_ut.cpp +++ b/tests/unit/client/driver/driver_ut.cpp @@ -1,7 +1,11 @@ +#include #include #include #include #include +#include +#include +#include #include #include @@ -25,6 +29,13 @@ using namespace NYdb::NTable; namespace { + std::string ReadBuildInfo(grpc::ServerContext* context) { + const auto& metadata = context->client_metadata(); + const auto it = metadata.find(YDB_SDK_BUILD_INFO_HEADER); + Y_ABORT_UNLESS(it != metadata.end()); + return {it->second.data(), it->second.length()}; + } + class TMockDiscoveryService : public Ydb::Discovery::V1::DiscoveryService::Service { public: grpc::Status ListEndpoints( @@ -32,7 +43,7 @@ namespace { const Ydb::Discovery::ListEndpointsRequest* request, Ydb::Discovery::ListEndpointsResponse* response) override { - Y_UNUSED(context); + BuildInfo = ReadBuildInfo(context); std::cerr << "ListEndpoints: " << request->ShortDebugString() << std::endl; @@ -48,6 +59,7 @@ namespace { // From database name to result std::unordered_map MockResults; + std::string BuildInfo; }; class TMockTableService : public Ydb::Table::V1::TableService::Service { @@ -57,7 +69,7 @@ namespace { const Ydb::Table::CreateSessionRequest* request, Ydb::Table::CreateSessionResponse* response) override { - Y_UNUSED(context); + BuildInfo = ReadBuildInfo(context); std::cerr << "CreateSession: " << request->ShortDebugString() << std::endl; @@ -70,6 +82,8 @@ namespace { op->mutable_result()->PackFrom(result); return grpc::Status::OK; } + + std::string BuildInfo; }; template @@ -326,7 +340,10 @@ Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) { auto driver = TDriver( TDriverConfig() .SetEndpoint(TStringBuilder() << "localhost:" << discoveryPort) - .SetDatabase("/Root/My/DB")); + .SetDatabase("/Root/My/DB") + .SetTraceProvider(std::make_shared()) + .SetMetricRegistry(std::make_shared()) + .AppendBuildInfo("test-client/1.2.3")); auto client = NTable::TTableClient(driver); auto sessionFuture = client.CreateSession(); @@ -335,6 +352,15 @@ Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) { UNIT_ASSERT(sessionResult.IsSuccess()); auto session = sessionResult.GetSession(); UNIT_ASSERT_VALUES_EQUAL(session.GetId(), "my-session-id"); + + const auto baseBuildInfo = "ydb-cpp-sdk/" + GetSdkSemver(); + UNIT_ASSERT_VALUES_EQUAL( + discoveryService.BuildInfo, + baseBuildInfo + + " ydb-sdk-tracing/" + std::string(NObservability::kTracingChainVersion) + + " ydb-sdk-metrics/" + std::string(NObservability::kMetricsChainVersion) + + ";test-client/1.2.3"); + UNIT_ASSERT_VALUES_EQUAL(tableService.BuildInfo, baseBuildInfo + ";test-client/1.2.3"); } Y_UNIT_TEST(WithoutDiscoveryDriverLevel) { From 9d9db745d8decdb5e1db220dc43a25fd31408e24 Mon Sep 17 00:00:00 2001 From: kseleznyov Date: Tue, 28 Jul 2026 08:49:45 +0000 Subject: [PATCH 37/56] [YDB_LOG] Migrate ydb/core/kafka_proxy (#43285) --- .github/last_commit.txt | 2 +- src/library/kafka/kafka_messages_int.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 9dad98c2b0..d99bd6c6b1 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -5790bd6cf50cc074ab342c5e537fd0bcdce685d4 +c0bf532526b4869dafd50b80e6951b3c3d274357 diff --git a/src/library/kafka/kafka_messages_int.h b/src/library/kafka/kafka_messages_int.h index 99469c65fa..89543bf4b5 100644 --- a/src/library/kafka/kafka_messages_int.h +++ b/src/library/kafka/kafka_messages_int.h @@ -640,7 +640,7 @@ class TypeStrategy, TKafkaArrayDesc> { // template inline void Write(TWriteCollector& collector, TKafkaWritable& writable, TKafkaInt16 version, const typename Meta::Type& value) { - if (VersionCheck(version)) { + if (VersionCheck(version)) { if (VersionCheck(version)) { if (!IsDefaultValue(value)) { ++collector.NumTaggedFields; @@ -653,7 +653,7 @@ inline void Write(TWriteCollector& collector, TKafkaWritable& writable, TKafkaIn template inline void Read(TKafkaReadable& readable, TKafkaInt16 version, typename Meta::Type& value) { - if (!VersionNone() + if (!VersionNone() && VersionCheck(version)) { return; } else { From 673386cfe05c00924956eed09b62e0f90b5e8037 Mon Sep 17 00:00:00 2001 From: Alek5andr-Kotov Date: Tue, 28 Jul 2026 08:49:55 +0000 Subject: [PATCH 38/56] Make StreamWrite ext_publication_id optional (#47344) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_topic.proto | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index d99bd6c6b1..da50339887 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -c0bf532526b4869dafd50b80e6951b3c3d274357 +6302f291f4089c6f3048cdf2c768ffd694ff61b1 diff --git a/src/api/protos/ydb_topic.proto b/src/api/protos/ydb_topic.proto index 08aa58f643..bd2fdb5652 100644 --- a/src/api/protos/ydb_topic.proto +++ b/src/api/protos/ydb_topic.proto @@ -722,12 +722,14 @@ message TransactionIdentity { } // Identity of a deferred topic publication for StreamWrite. -// int_publication_id is assigned by the server in BeginPublication. -// ext_publication_id is repeated from BeginPublication on each write batch. +// int_publication_id is assigned by the server in BeginPublication and is authoritative. +// ext_publication_id is optional and informational only (diagnostics / TWriteId display). +// When set, the server does not require a non-empty value and does not check that it +// matches BeginPublication. Omitting the field and setting an empty string are both allowed. message DeferredPublishIdentity { uint64 int_publication_id = 1; - string ext_publication_id = 2 [(Ydb.length).le = 2048]; + optional string ext_publication_id = 2 [(Ydb.length).le = 2048]; } // Add offsets to transaction request sent from client to server. From 344d3c4b3c18ae8b90825b163f5c206f64ce3acc Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Tue, 28 Jul 2026 08:50:04 +0000 Subject: [PATCH 39/56] StrictSerializable: return commit timestamp (#46795) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/query/client.h | 19 +++++- include/ydb-cpp-sdk/client/query/query.h | 7 +++ include/ydb-cpp-sdk/client/scheme/scheme.h | 20 +------ .../client/types/virtual_timestamp.h | 32 ++++++++++ src/api/protos/ydb_query.proto | 8 +++ src/client/query/client.cpp | 7 ++- src/client/query/impl/exec_query.cpp | 22 +++++-- src/client/query/query.cpp | 5 ++ src/client/scheme/scheme.cpp | 46 --------------- src/client/types/virtual_timestamp.cpp | 58 +++++++++++++++++++ .../client/query/virtual_timestamp_ut.cpp | 42 ++++++++++++++ 12 files changed, 194 insertions(+), 74 deletions(-) create mode 100644 include/ydb-cpp-sdk/client/types/virtual_timestamp.h create mode 100644 src/client/types/virtual_timestamp.cpp create mode 100644 tests/unit/client/query/virtual_timestamp_ut.cpp diff --git a/.github/last_commit.txt b/.github/last_commit.txt index da50339887..088f788fd2 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -6302f291f4089c6f3048cdf2c768ffd694ff61b1 +75900fddb7cfff27d66271c4a9c419ddd852554b diff --git a/include/ydb-cpp-sdk/client/query/client.h b/include/ydb-cpp-sdk/client/query/client.h index 3567071d5a..b0a87eb71e 100644 --- a/include/ydb-cpp-sdk/client/query/client.h +++ b/include/ydb-cpp-sdk/client/query/client.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -293,19 +294,25 @@ class TExecuteQueryPart : public TStreamPartStatus { const std::optional& GetTransaction() const { return Transaction_; } - TExecuteQueryPart(TStatus&& status, std::optional&& queryStats, std::optional&& tx) + const std::optional& GetCommitTimestamp() const { return CommitTimestamp_; } + + TExecuteQueryPart(TStatus&& status, std::optional&& queryStats, std::optional&& tx, + std::optional&& commitTimestamp = {}) : TStreamPartStatus(std::move(status)) , Stats_(std::move(queryStats)) , Transaction_(std::move(tx)) + , CommitTimestamp_(std::move(commitTimestamp)) {} TExecuteQueryPart(TStatus&& status, TResultSet&& resultSet, int64_t resultSetIndex, - std::optional&& queryStats, std::optional&& tx) + std::optional&& queryStats, std::optional&& tx, + std::optional&& commitTimestamp = {}) : TStreamPartStatus(std::move(status)) , ResultSet_(std::move(resultSet)) , ResultSetIndex_(resultSetIndex) , Stats_(std::move(queryStats)) , Transaction_(std::move(tx)) + , CommitTimestamp_(std::move(commitTimestamp)) {} private: @@ -313,6 +320,7 @@ class TExecuteQueryPart : public TStreamPartStatus { int64_t ResultSetIndex_ = 0; std::optional Stats_; std::optional Transaction_; + std::optional CommitTimestamp_; }; class TExecuteQueryResult : public TStatus { @@ -325,22 +333,27 @@ class TExecuteQueryResult : public TStatus { std::optional GetTransaction() const {return Transaction_; } + const std::optional& GetCommitTimestamp() const { return CommitTimestamp_; } + TExecuteQueryResult(TStatus&& status) : TStatus(std::move(status)) {} TExecuteQueryResult(TStatus&& status, std::vector&& resultSets, - std::optional&& stats, std::optional&& tx) + std::optional&& stats, std::optional&& tx, + std::optional&& commitTimestamp = {}) : TStatus(std::move(status)) , ResultSets_(std::move(resultSets)) , Stats_(std::move(stats)) , Transaction_(std::move(tx)) + , CommitTimestamp_(std::move(commitTimestamp)) {} private: std::vector ResultSets_; std::optional Stats_; std::optional Transaction_; + std::optional CommitTimestamp_; }; } // namespace NYdb::NQuery diff --git a/include/ydb-cpp-sdk/client/query/query.h b/include/ydb-cpp-sdk/client/query/query.h index 9229d65247..fa0c557f03 100644 --- a/include/ydb-cpp-sdk/client/query/query.h +++ b/include/ydb-cpp-sdk/client/query/query.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -131,6 +132,12 @@ struct TDeleteSessionSettings : public TRequestSettings class TCommitTransactionResult : public TStatus { public: TCommitTransactionResult(TStatus&& status); + TCommitTransactionResult(TStatus&& status, std::optional&& commitTimestamp); + + const std::optional& GetCommitTimestamp() const { return CommitTimestamp_; } + +private: + std::optional CommitTimestamp_; }; using TAsyncBeginTransactionResult = NThreading::TFuture; diff --git a/include/ydb-cpp-sdk/client/scheme/scheme.h b/include/ydb-cpp-sdk/client/scheme/scheme.h index 6ef6941b89..517bf7d2a5 100644 --- a/include/ydb-cpp-sdk/client/scheme/scheme.h +++ b/include/ydb-cpp-sdk/client/scheme/scheme.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace Ydb { class VirtualTimestamp; @@ -57,25 +58,6 @@ enum class ESchemeEntryType : i32 { Secret = 26, }; -struct TVirtualTimestamp { - uint64_t PlanStep = 0; - uint64_t TxId = 0; - - TVirtualTimestamp() = default; - TVirtualTimestamp(uint64_t planStep, uint64_t txId); - TVirtualTimestamp(const ::Ydb::VirtualTimestamp& proto); - - std::string ToString() const; - void Out(IOutputStream& out) const; - - bool operator<(const TVirtualTimestamp& rhs) const; - bool operator<=(const TVirtualTimestamp& rhs) const; - bool operator>(const TVirtualTimestamp& rhs) const; - bool operator>=(const TVirtualTimestamp& rhs) const; - bool operator==(const TVirtualTimestamp& rhs) const; - bool operator!=(const TVirtualTimestamp& rhs) const; -}; - struct TSchemeEntry { std::string Name; std::string Owner; diff --git a/include/ydb-cpp-sdk/client/types/virtual_timestamp.h b/include/ydb-cpp-sdk/client/types/virtual_timestamp.h new file mode 100644 index 0000000000..e68de28f7c --- /dev/null +++ b/include/ydb-cpp-sdk/client/types/virtual_timestamp.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +#include + +#include + +namespace NYdb::inline V3 { +namespace NScheme { + +struct TVirtualTimestamp { + uint64_t PlanStep = 0; + uint64_t TxId = 0; + + TVirtualTimestamp() = default; + TVirtualTimestamp(uint64_t planStep, uint64_t txId); + TVirtualTimestamp(const ::Ydb::VirtualTimestamp& proto); + + std::string ToString() const; + void Out(IOutputStream& out) const; + + bool operator<(const TVirtualTimestamp& rhs) const; + bool operator<=(const TVirtualTimestamp& rhs) const; + bool operator>(const TVirtualTimestamp& rhs) const; + bool operator>=(const TVirtualTimestamp& rhs) const; + bool operator==(const TVirtualTimestamp& rhs) const; + bool operator!=(const TVirtualTimestamp& rhs) const; +}; + +} // namespace NScheme +} // namespace NYdb diff --git a/src/api/protos/ydb_query.proto b/src/api/protos/ydb_query.proto index 2ec4ea6824..6263cbfe61 100644 --- a/src/api/protos/ydb_query.proto +++ b/src/api/protos/ydb_query.proto @@ -142,6 +142,10 @@ message CommitTransactionRequest { message CommitTransactionResponse { StatusIds.StatusCode status = 1; repeated Ydb.Issue.IssueMessage issues = 2; + + // Commit timestamp (PlanStep, TxId) for StrictSerializableRW write transactions. + // Present only on SUCCESS and when the transaction had write effects. + Ydb.VirtualTimestamp commit_timestamp = 3; } message RollbackTransactionRequest { @@ -255,6 +259,10 @@ message ExecuteQueryResponsePart { TransactionMeta tx_meta = 6; VirtualTimestamp snapshot_timestamp = 7; + + // Commit timestamp (PlanStep, TxId) for StrictSerializableRW write transactions. + // Present only in the final (trailing) part on SUCCESS and when the transaction had write effects. + VirtualTimestamp commit_timestamp = 8; } message ExecuteScriptRequest { diff --git a/src/client/query/client.cpp b/src/client/query/client.cpp index 83d08750ed..4a8b378f1d 100644 --- a/src/client/query/client.cpp +++ b/src/client/query/client.cpp @@ -260,7 +260,12 @@ class TQueryClient::TImpl: public TClientImplCommon, public obs->End(commitTxStatus.GetStatus(), commitTxStatus.GetEndpoint()); - TCommitTransactionResult commitTxResult(std::move(commitTxStatus)); + std::optional commitTimestamp; + if (response->has_commit_timestamp()) { + commitTimestamp = NScheme::TVirtualTimestamp(response->commit_timestamp()); + } + + TCommitTransactionResult commitTxResult(std::move(commitTxStatus), std::move(commitTimestamp)); promise.SetValue(std::move(commitTxResult)); } else { obs->End(status.Status, status.Endpoint); diff --git a/src/client/query/impl/exec_query.cpp b/src/client/query/impl/exec_query.cpp index 7b9ba781dc..3654d396dc 100644 --- a/src/client/query/impl/exec_query.cpp +++ b/src/client/query/impl/exec_query.cpp @@ -88,6 +88,7 @@ class TExecuteQueryIterator::TReaderImpl { std::optional stats; std::optional tx; + std::optional commitTimestamp; if (self->Response_.has_exec_stats()) { stats = TExecStats(std::move(*self->Response_.mutable_exec_stats())); } @@ -96,16 +97,21 @@ class TExecuteQueryIterator::TReaderImpl { tx = TTransaction(self->Session_.value(), self->Response_.tx_meta().id()); } + if (self->Response_.has_commit_timestamp()) { + commitTimestamp = NScheme::TVirtualTimestamp(self->Response_.commit_timestamp()); + } + if (self->Response_.has_result_set()) { promise.SetValue({ std::move(status), TResultSet(std::move(*self->Response_.mutable_result_set())), self->Response_.result_set_index(), std::move(stats), - std::move(tx) + std::move(tx), + std::move(commitTimestamp) }); } else { - promise.SetValue({std::move(status), std::move(stats), std::move(tx)}); + promise.SetValue({std::move(status), std::move(stats), std::move(tx), std::move(commitTimestamp)}); } } }; @@ -160,6 +166,7 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { std::vector ResultSets_; std::optional Stats_; std::optional Tx_; + std::optional CommitTimestamp_; std::vector ArrowSchemas_; std::vector> BytesData_; std::vector ResultSetSeen_; @@ -174,6 +181,10 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { self->Stats_ = st; } + if (const auto& ct = part.GetCommitTimestamp()) { + self->CommitTimestamp_ = ct; + } + if (!part.IsSuccess()) { std::optional stats; std::swap(self->Stats_, stats); @@ -182,12 +193,14 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { std::vector issues; std::vector resultProtos; std::optional tx; + std::optional commitTimestamp; std::vector arrowSchemas; std::vector> bytesData; std::swap(self->Issues_, issues); std::swap(self->ResultSets_, resultProtos); std::swap(self->Tx_, tx); + std::swap(self->CommitTimestamp_, commitTimestamp); std::swap(self->ArrowSchemas_, arrowSchemas); std::swap(self->BytesData_, bytesData); @@ -204,10 +217,11 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues(std::move(issues))), std::move(resultSets), std::move(stats), - std::move(tx) + std::move(tx), + std::move(commitTimestamp) )); } else { - self->Promise_.SetValue(TExecuteQueryResult(std::move(part), {}, std::move(stats), {})); + self->Promise_.SetValue(TExecuteQueryResult(std::move(part), {}, std::move(stats), {})); // No commit timestamp on error } return; diff --git a/src/client/query/query.cpp b/src/client/query/query.cpp index 72c036b5f0..fef7e7877c 100644 --- a/src/client/query/query.cpp +++ b/src/client/query/query.cpp @@ -66,4 +66,9 @@ TCommitTransactionResult::TCommitTransactionResult(TStatus&& status) : TStatus(std::move(status)) {} +TCommitTransactionResult::TCommitTransactionResult(TStatus&& status, std::optional&& commitTimestamp) + : TStatus(std::move(status)) + , CommitTimestamp_(std::move(commitTimestamp)) +{} + } // namespace NYdb::NQuery diff --git a/src/client/scheme/scheme.cpp b/src/client/scheme/scheme.cpp index 32d3025157..5666717f4b 100644 --- a/src/client/scheme/scheme.cpp +++ b/src/client/scheme/scheme.cpp @@ -29,52 +29,6 @@ void TPermissions::SerializeTo(::Ydb::Scheme::Permissions& proto) const { } } -TVirtualTimestamp::TVirtualTimestamp(uint64_t planStep, uint64_t txId) - : PlanStep(planStep) - , TxId(txId) -{} - -TVirtualTimestamp::TVirtualTimestamp(const ::Ydb::VirtualTimestamp& proto) - : TVirtualTimestamp(proto.plan_step(), proto.tx_id()) -{} - -std::string TVirtualTimestamp::ToString() const { - TString result; - TStringOutput out(result); - Out(out); - return result; -} - -void TVirtualTimestamp::Out(IOutputStream& out) const { - out << "{ plan_step: " << PlanStep - << ", tx_id: " << TxId - << " }"; -} - -bool TVirtualTimestamp::operator<(const TVirtualTimestamp& rhs) const { - return PlanStep < rhs.PlanStep && TxId < rhs.TxId; -} - -bool TVirtualTimestamp::operator<=(const TVirtualTimestamp& rhs) const { - return PlanStep <= rhs.PlanStep && TxId <= rhs.TxId; -} - -bool TVirtualTimestamp::operator>(const TVirtualTimestamp& rhs) const { - return PlanStep > rhs.PlanStep && TxId > rhs.TxId; -} - -bool TVirtualTimestamp::operator>=(const TVirtualTimestamp& rhs) const { - return PlanStep >= rhs.PlanStep && TxId >= rhs.TxId; -} - -bool TVirtualTimestamp::operator==(const TVirtualTimestamp& rhs) const { - return PlanStep == rhs.PlanStep && TxId == rhs.TxId; -} - -bool TVirtualTimestamp::operator!=(const TVirtualTimestamp& rhs) const { - return !(*this == rhs); -} - static ESchemeEntryType ConvertProtoEntryType(::Ydb::Scheme::Entry::Type entry) { switch (entry) { case ::Ydb::Scheme::Entry::DIRECTORY: diff --git a/src/client/types/virtual_timestamp.cpp b/src/client/types/virtual_timestamp.cpp new file mode 100644 index 0000000000..ae83730817 --- /dev/null +++ b/src/client/types/virtual_timestamp.cpp @@ -0,0 +1,58 @@ +#include + +#include +#include + +#include + +namespace NYdb::inline V3 { +namespace NScheme { + +TVirtualTimestamp::TVirtualTimestamp(uint64_t planStep, uint64_t txId) + : PlanStep(planStep) + , TxId(txId) +{} + +TVirtualTimestamp::TVirtualTimestamp(const ::Ydb::VirtualTimestamp& proto) + : TVirtualTimestamp(proto.plan_step(), proto.tx_id()) +{} + +std::string TVirtualTimestamp::ToString() const { + TString result; + TStringOutput out(result); + Out(out); + return result; +} + +void TVirtualTimestamp::Out(IOutputStream& out) const { + out << "{ plan_step: " << PlanStep + << ", tx_id: " << TxId + << " }"; +} + +bool TVirtualTimestamp::operator<(const TVirtualTimestamp& rhs) const { + return std::tie(PlanStep, TxId) < std::tie(rhs.PlanStep, rhs.TxId); +} + +bool TVirtualTimestamp::operator<=(const TVirtualTimestamp& rhs) const { + return std::tie(PlanStep, TxId) <= std::tie(rhs.PlanStep, rhs.TxId); +} + +bool TVirtualTimestamp::operator>(const TVirtualTimestamp& rhs) const { + return std::tie(PlanStep, TxId) > std::tie(rhs.PlanStep, rhs.TxId); +} + +bool TVirtualTimestamp::operator>=(const TVirtualTimestamp& rhs) const { + return std::tie(PlanStep, TxId) >= std::tie(rhs.PlanStep, rhs.TxId); +} + +bool TVirtualTimestamp::operator==(const TVirtualTimestamp& rhs) const { + return PlanStep == rhs.PlanStep && TxId == rhs.TxId; +} + +bool TVirtualTimestamp::operator!=(const TVirtualTimestamp& rhs) const { + return !(*this == rhs); +} + +} // namespace NScheme +} // namespace NYdb diff --git a/tests/unit/client/query/virtual_timestamp_ut.cpp b/tests/unit/client/query/virtual_timestamp_ut.cpp new file mode 100644 index 0000000000..11e18117f4 --- /dev/null +++ b/tests/unit/client/query/virtual_timestamp_ut.cpp @@ -0,0 +1,42 @@ +#include + +#include + +using NYdb::NScheme::TVirtualTimestamp; + +Y_UNIT_TEST_SUITE(VirtualTimestamp) { + Y_UNIT_TEST(LexicographicLess) { + UNIT_ASSERT(TVirtualTimestamp(1, 100) < TVirtualTimestamp(2, 1)); + UNIT_ASSERT(TVirtualTimestamp(1, 50) < TVirtualTimestamp(1, 100)); + UNIT_ASSERT(!(TVirtualTimestamp(2, 1) < TVirtualTimestamp(1, 100))); + UNIT_ASSERT(!(TVirtualTimestamp(1, 100) < TVirtualTimestamp(1, 100))); + } + + Y_UNIT_TEST(LexicographicGreater) { + UNIT_ASSERT(TVirtualTimestamp(2, 1) > TVirtualTimestamp(1, 100)); + UNIT_ASSERT(TVirtualTimestamp(1, 100) > TVirtualTimestamp(1, 50)); + UNIT_ASSERT(!(TVirtualTimestamp(1, 100) > TVirtualTimestamp(2, 1))); + UNIT_ASSERT(!(TVirtualTimestamp(1, 100) > TVirtualTimestamp(1, 100))); + } + + Y_UNIT_TEST(LessOrEqual) { + UNIT_ASSERT(TVirtualTimestamp(1, 100) <= TVirtualTimestamp(1, 100)); + UNIT_ASSERT(TVirtualTimestamp(1, 50) <= TVirtualTimestamp(1, 100)); + UNIT_ASSERT(TVirtualTimestamp(1, 100) <= TVirtualTimestamp(2, 1)); + UNIT_ASSERT(!(TVirtualTimestamp(2, 1) <= TVirtualTimestamp(1, 100))); + } + + Y_UNIT_TEST(GreaterOrEqual) { + UNIT_ASSERT(TVirtualTimestamp(1, 100) >= TVirtualTimestamp(1, 100)); + UNIT_ASSERT(TVirtualTimestamp(1, 100) >= TVirtualTimestamp(1, 50)); + UNIT_ASSERT(TVirtualTimestamp(2, 1) >= TVirtualTimestamp(1, 100)); + UNIT_ASSERT(!(TVirtualTimestamp(1, 50) >= TVirtualTimestamp(1, 100))); + } + + Y_UNIT_TEST(Equality) { + UNIT_ASSERT(TVirtualTimestamp(1, 100) == TVirtualTimestamp(1, 100)); + UNIT_ASSERT(!(TVirtualTimestamp(1, 100) == TVirtualTimestamp(1, 101))); + UNIT_ASSERT(!(TVirtualTimestamp(1, 100) == TVirtualTimestamp(2, 100))); + UNIT_ASSERT(TVirtualTimestamp(1, 100) != TVirtualTimestamp(2, 100)); + } +} From 3536eaba18bb59eb3509ef03821bdfb3604a8058 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:50:14 +0000 Subject: [PATCH 40/56] implement async provider methods with retries and refactor (#47249) --- .github/last_commit.txt | 2 +- .../client/iam/common/generic_provider.h | 271 +++++----------- .../types/core_facility/core_facility.h | 2 +- .../client/types/credentials/credentials.h | 46 ++- src/client/iam/iam.cpp | 244 +++++++------- src/client/iam_private/common/iam.h | 73 +---- .../impl/internal/db_driver_state/state.cpp | 17 +- .../impl/internal/db_driver_state/state.h | 3 +- .../grpc_connections/grpc_connections.h | 6 +- .../impl/write_session_impl.cpp | 56 +++- .../impl/write_session_impl.h | 3 +- src/client/topic/impl/write_session_impl.cpp | 47 ++- src/client/topic/impl/write_session_impl.h | 3 +- .../core_facility/simple_core_facility.cpp | 16 +- .../core_facility/simple_core_facility.h | 1 - src/client/types/credentials/login/login.cpp | 305 +++++++++--------- .../oauth2_token_exchange/credentials.cpp | 269 +++++++-------- .../common/iam_mocks/iam_grpc_mock_server.cpp | 8 + tests/common/iam_mocks/iam_grpc_mock_server.h | 2 + tests/unit/client/driver/driver_ut.cpp | 40 ++- tests/unit/client/iam/grpc_iam_ut.cpp | 154 ++++++--- tests/unit/client/iam/http_iam_ut.cpp | 64 +--- .../iam_private/grpc_iam_service_ut.cpp | 30 +- .../oauth2_token_exchange/credentials_ut.cpp | 203 +++--------- .../helpers/test_token_exchange_server.cpp | 8 +- 25 files changed, 885 insertions(+), 988 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 088f788fd2..13db073620 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -75900fddb7cfff27d66271c4a9c419ddd852554b +3fd87611fbcea7809d690ee9062009fb33924e4b diff --git a/include/ydb-cpp-sdk/client/iam/common/generic_provider.h b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h index 04a07375c6..e7deca3f55 100644 --- a/include/ydb-cpp-sdk/client/iam/common/generic_provider.h +++ b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h @@ -22,6 +22,8 @@ namespace NYdb::inline V3 { +using NCredentials::NDetail::TOwningFacilityCredentialsProvider; + constexpr std::chrono::milliseconds BACKOFF_START{50}; constexpr std::chrono::milliseconds BACKOFF_MAX{10000}; constexpr std::chrono::milliseconds PERIODIC_TICK{100}; @@ -77,19 +79,16 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { std::weak_ptr responseFacility, TCredentialsProviderPtr authTokenProvider) : Rpc_(rpc) - , Ticket_("") , NextTicketUpdate_(SysTimePoint{}) , IamEndpoint_(iamEndpoint) , RequestFiller_(requestFiller) , Context_(std::nullopt) - , LastRequestError_("") , NeedStop_(false) , BackoffTimeout_(BACKOFF_START) , Lock_() , ResponseFacility_(std::move(responseFacility)) , AuthTokenProvider_(authTokenProvider) - , FirstTokenReady_(NThreading::NewPromise()) - , FirstTokenReadySet_(false) + , AuthInfo_(NThreading::NewPromise()) { std::shared_ptr creds = nullptr; if (IamEndpoint_.EnableSsl) { @@ -112,7 +111,7 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { void StartPeriodicTask() { auto facility = ResponseFacility_.lock(); if (!facility) { - FailFirstToken("IAM-token provider response facility is not available"); + Fail("IAM-token provider response facility is not available"); return; } @@ -125,7 +124,7 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { return false; } if (status != EStatus::SUCCESS) { - self->FailFirstToken(TStringBuilder() + self->Fail(TStringBuilder() << "IAM-token provider periodic task failed with status " << static_cast(status)); return false; @@ -135,53 +134,30 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { PERIODIC_TICK ); } catch (...) { - FailFirstToken(TStringBuilder() + Fail(TStringBuilder() << "Failed to start IAM-token provider periodic task: " << CurrentExceptionMessage()); } } - std::string GetTicket() { + NThreading::TFuture GetAuthInfoAsync() { std::lock_guard guard(Lock_); - if (Ticket_.empty()) { - ythrow yexception() << "IAM-token not ready yet. " << LastRequestError_; - } - return Ticket_; - } - - void WaitForToken() { - std::unique_lock guard(Lock_); - TokenReady_.wait_for(guard, - std::chrono::microseconds(2 * IamEndpoint_.RequestTimeout.MicroSeconds()), - [this]() { - return NeedStop_ || !Ticket_.empty(); - } - ); - } - - NThreading::TFuture GetReadyFuture() const { - return FirstTokenReady_.GetFuture(); + return AuthInfo_.GetFuture(); } void Stop() { - bool setStoppedException = false; + NThreading::TPromise promise; { std::unique_lock guard(Lock_); - if (NeedStop_) { - return; - } NeedStop_ = true; - setStoppedException = MarkFirstTokenReadyLocked(); - TokenReady_.notify_all(); + promise = AuthInfo_; if (Context_.has_value()) { Context_->TryCancel(); } ContextReady_.wait(guard, [this]() { return !Context_.has_value(); }); } - if (setStoppedException) { - FirstTokenReady_.SetException( - std::make_exception_ptr(yexception() << "IAM-token provider stopped before token was ready")); - } + promise.TrySetException(std::make_exception_ptr( + yexception() << "IAM-token provider stopped before token was ready")); Stub_.reset(); Channel_.reset(); } @@ -189,21 +165,17 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { private: using SysDuration = SysClock::duration; - bool MarkFirstTokenReadyLocked() { - return !std::exchange(FirstTokenReadySet_, true); - } - - void FailFirstToken(std::string error) { - bool setException = false; + void Fail(std::string error) { + NThreading::TPromise promise; { std::lock_guard guard(Lock_); - if ((setException = MarkFirstTokenReadyLocked())) { - LastRequestError_ = error; + NeedStop_ = true; + promise = AuthInfo_; + if (Context_) { + Context_->TryCancel(); } } - if (setException) { - FirstTokenReady_.SetException(std::make_exception_ptr(yexception() << error)); - } + promise.TrySetException(std::make_exception_ptr(yexception() << error)); } static SysDuration ToBoundedSysDuration(const TDuration& d) { @@ -251,16 +223,11 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } if (auto self = weakSelf.lock()) { - bool failFirstToken; { std::lock_guard guard(self->Lock_); - failFirstToken = self->MarkFirstTokenReadyLocked(); self->ResetContextImpl(); } - if (failFirstToken) { - self->FirstTokenReady_.SetException(std::make_exception_ptr( - yexception() << "IAM-token provider response facility is not available")); - } + self->Fail("IAM-token provider response facility is not available"); } }; @@ -268,42 +235,36 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { try { RequestFiller_(req); + Rpc_(Stub_.get(), &*Context_, &req, response.get(), std::move(cb)); } catch (...) { - std::optional firstTokenError; - const auto now = SysClock::now(); - { - std::lock_guard guard(Lock_); - LastRequestError_ = TStringBuilder() - << "Last request error was at " << FormatSysTimeUtcIsoMicros(now) - << ". Failed to prepare IAM request: " << CurrentExceptionMessage(); - if (MarkFirstTokenReadyLocked()) { - firstTokenError = LastRequestError_; - } - ResetContextImpl(); - RescheduleOnFailure(); - } - if (firstTokenError) { - FirstTokenReady_.SetException(std::make_exception_ptr(yexception() << *firstTokenError)); - } - return; + std::lock_guard guard(Lock_); + ResetContextImpl(); + RescheduleOnFailure(); } - - Rpc_(Stub_.get(), &*Context_, &req, response.get(), std::move(cb)); } - void FillContext(std::unique_lock& guard) { + bool FillContext(std::unique_lock& guard) { std::optional authToken; if (AuthTokenProvider_) { guard.unlock(); try { - authToken = AuthTokenProvider_->GetAuthInfo(); + if (!AuthTokenInfo_.Initialized()) { + AuthTokenInfo_ = AuthTokenProvider_->GetAuthInfoAsync(); + } + if (!AuthTokenInfo_.IsReady()) { + guard.lock(); + return false; + } + authToken = AuthTokenInfo_.GetValue(); + AuthTokenInfo_ = {}; } catch (...) { + AuthTokenInfo_ = {}; guard.lock(); throw; } guard.lock(); if (NeedStop_) { - return; + return false; } } @@ -317,6 +278,7 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { if (authToken) { context.AddMetadata("authorization", "Bearer " + *authToken); } + return true; } void ResetContextImpl() { @@ -332,8 +294,9 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } bool OnPeriodicTick() { - std::optional firstTokenError; + std::optional terminalError; bool updateTicket = false; + bool authPending = false; { std::unique_lock guard(Lock_); if (NeedStop_) { @@ -342,16 +305,15 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { if (Context_.has_value() || SysClock::now() < NextTicketUpdate_) { return true; } + if (AuthInfo_.GetFuture().IsReady()) { + AuthInfo_ = NThreading::NewPromise(); + } try { - FillContext(guard); + authPending = !FillContext(guard); } catch (...) { - const auto now = SysClock::now(); - LastRequestError_ = TStringBuilder() - << "Last request error was at " << FormatSysTimeUtcIsoMicros(now) + terminalError = TStringBuilder() + << "Last request error was at " << FormatSysTimeUtcIsoMicros(SysClock::now()) << ". Failed to prepare IAM request context: " << CurrentExceptionMessage(); - if (MarkFirstTokenReadyLocked()) { - firstTokenError = LastRequestError_; - } ResetContextImpl(); } if (NeedStop_) { @@ -359,13 +321,16 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { return false; } if (!Context_.has_value()) { - RescheduleOnFailure(); + if (!authPending && !terminalError) { + RescheduleOnFailure(); + } } else { updateTicket = true; } } - if (firstTokenError) { - FirstTokenReady_.SetException(std::make_exception_ptr(yexception() << *firstTokenError)); + if (terminalError) { + Fail(*terminalError); + return false; } if (updateTicket) { UpdateTicket(); @@ -374,38 +339,52 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } void ProcessIamResponse(grpc::Status&& status, TResponse&& result) { - bool setFirstTokenReady = false; + std::optional token; + std::optional terminalError; + NThreading::TPromise promise; { std::lock_guard guard(Lock_); if (!status.ok()) { - LastRequestError_ = TStringBuilder() + const std::string error = TStringBuilder() << "Last request error was at " << FormatSysTimeUtcIsoMicros(SysClock::now()) << ". GrpcStatusCode: " << static_cast(status.error_code()) << " Message: \"" << status.error_message() << "\" iam-endpoint: \"" << IamEndpoint_.Endpoint << "\""; - RescheduleOnFailure(); + if (IsRetryable(status.error_code())) { + RescheduleOnFailure(); + } else { + terminalError = error; + } + } else if (result.iam_token().empty()) { + terminalError = "IAM-token service returned an empty token"; } else { - LastRequestError_ = ""; - Ticket_ = result.iam_token(); + token = result.iam_token(); + promise = AuthInfo_; const SysTimePoint expiresAt = SysClock::from_time_t(result.expires_at().seconds()); RescheduleOnSuccess(expiresAt); - - setFirstTokenReady = MarkFirstTokenReadyLocked(); - TokenReady_.notify_all(); } ResetContextImpl(); } - if (setFirstTokenReady) { - FirstTokenReady_.SetValue(); + if (token) { + promise.TrySetValue(std::move(*token)); + } else if (terminalError) { + Fail(*terminalError); } } + static bool IsRetryable(grpc::StatusCode code) { + return code == grpc::StatusCode::CANCELLED || code == grpc::StatusCode::UNKNOWN || + code == grpc::StatusCode::DEADLINE_EXCEEDED || code == grpc::StatusCode::RESOURCE_EXHAUSTED || + code == grpc::StatusCode::ABORTED || code == grpc::StatusCode::INTERNAL || + code == grpc::StatusCode::UNAVAILABLE; + } + void RescheduleOnFailure() { // call with Lock_ const auto now = SysClock::now(); const auto retryDelay = std::min(BackoffTimeout_, BACKOFF_MAX); @@ -431,21 +410,18 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { std::shared_ptr Stub_; TAsyncRpc Rpc_; - std::string Ticket_; SysTimePoint NextTicketUpdate_; const TIamEndpoint IamEndpoint_; const TRequestFiller RequestFiller_; std::optional Context_; std::condition_variable ContextReady_; - std::condition_variable TokenReady_; - std::string LastRequestError_; bool NeedStop_; std::chrono::milliseconds BackoffTimeout_; std::mutex Lock_; std::weak_ptr ResponseFacility_; TCredentialsProviderPtr AuthTokenProvider_; - NThreading::TPromise FirstTokenReady_; - bool FirstTokenReadySet_; + NThreading::TFuture AuthTokenInfo_; + NThreading::TPromise AuthInfo_; }; public: @@ -453,14 +429,10 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { const TRequestFiller& requestFiller, TAsyncRpc rpc, std::weak_ptr responseFacility, - TCredentialsProviderPtr authTokenProvider = nullptr, - bool waitForToken = true) + TCredentialsProviderPtr authTokenProvider = nullptr) : Impl_(std::make_shared(endpoint, requestFiller, rpc, std::move(responseFacility), authTokenProvider)) { Impl_->StartPeriodicTask(); - if (waitForToken) { - Impl_->WaitForToken(); - } } ~TGrpcIamCredentialsProvider() { @@ -468,90 +440,43 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } std::string GetAuthInfo() const override { - return Impl_->GetTicket(); + return GetAuthInfoAsync().GetValueSync(); } - bool IsValid() const override { - return true; - } - - NThreading::TFuture GetReadyFuture() const { - return Impl_->GetReadyFuture(); - } - -private: - std::shared_ptr Impl_; -}; - -// Adapter that keeps a self-owned ICoreFacility alive for the lifetime of an inner credentials -// provider. Used by deprecated no-arg ICredentialsProviderFactory::CreateProvider() paths where -// the caller hasn't supplied a facility. -class TOwningFacilityCredentialsProvider : public ICredentialsProvider { -public: - TOwningFacilityCredentialsProvider(std::shared_ptr facility, - TCredentialsProviderPtr inner) - : Facility_(std::move(facility)) - , Inner_(std::move(inner)) - {} - - std::string GetAuthInfo() const override { - return Inner_->GetAuthInfo(); + NThreading::TFuture GetAuthInfoAsync() const override { + return Impl_->GetAuthInfoAsync(); } bool IsValid() const override { - return Inner_->IsValid(); + return true; } private: - // Field declaration order matters: Inner_ is destroyed first so that its Stop() can still - // drive the facility's queue (cancel the in-flight gRPC context, drain the response callback), - // and only then is Facility_ destroyed. - std::shared_ptr Facility_; - TCredentialsProviderPtr Inner_; + std::shared_ptr Impl_; }; -namespace NPrivate { - -template -NThreading::TFuture CreateGrpcIamCredentialsProviderAsync( - const TParams& params, - std::weak_ptr facility, - std::shared_ptr ownedFacility = {}) -{ - auto inner = std::make_shared(params, std::move(facility), false); - auto ready = inner->GetReadyFuture(); - TCredentialsProviderPtr provider = std::move(inner); - if (ownedFacility) { - provider = std::make_shared( - std::move(ownedFacility), std::move(provider)); - } - return ready.Return(std::move(provider)); -} - -} // namespace NPrivate - template class TIamJwtCredentialsProvider : public TGrpcIamCredentialsProvider { public: - TIamJwtCredentialsProvider(const TIamJwtParams& params, std::weak_ptr responseFacility, bool waitForToken = true) + TIamJwtCredentialsProvider(const TIamJwtParams& params, std::weak_ptr responseFacility) : TGrpcIamCredentialsProvider(params, [jwtParams = params.JwtParams](TRequest& req) { req.set_jwt(MakeSignedJwt(jwtParams)); }, [](typename TService::Stub* stub, grpc::ClientContext* context, const TRequest* request, TResponse* response, std::function cb) { stub->async()->Create(context, request, response, std::move(cb)); - }, std::move(responseFacility), nullptr, waitForToken) {} + }, std::move(responseFacility)) {} }; template class TIamOAuthCredentialsProvider : public TGrpcIamCredentialsProvider { public: - TIamOAuthCredentialsProvider(const TIamOAuth& params, std::weak_ptr responseFacility, bool waitForToken = true) + TIamOAuthCredentialsProvider(const TIamOAuth& params, std::weak_ptr responseFacility) : TGrpcIamCredentialsProvider(params, [token = params.OAuthToken](TRequest& req) { req.set_yandex_passport_oauth_token(TStringType{token}); }, [](typename TService::Stub* stub, grpc::ClientContext* context, const TRequest* request, TResponse* response, std::function cb) { stub->async()->Create(context, request, response, std::move(cb)); - }, std::move(responseFacility), nullptr, waitForToken) {} + }, std::move(responseFacility)) {} }; template @@ -574,12 +499,6 @@ class TIamJwtCredentialsProviderFactory : public ICredentialsProviderFactory { }); } - NThreading::TFuture CreateProviderAsync() const override { - auto facility = CreateSimpleCoreFacility(); - return NPrivate::CreateGrpcIamCredentialsProviderAsync< - TIamJwtCredentialsProvider>(Params_, facility, facility); - } - TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const override { return std::make_shared>(Params_, std::move(facility)); } @@ -595,11 +514,6 @@ class TIamJwtCredentialsProviderFactory : public ICredentialsProviderFactory { Params_.JwtParams.PrivKey); } - NThreading::TFuture CreateProviderAsync(std::weak_ptr facility) const override { - return NPrivate::CreateGrpcIamCredentialsProviderAsync< - TIamJwtCredentialsProvider>(Params_, std::move(facility)); - } - private: TIamJwtParams Params_; }; @@ -622,12 +536,6 @@ class TIamOAuthCredentialsProviderFactory : public ICredentialsProviderFactory { }); } - NThreading::TFuture CreateProviderAsync() const override { - auto facility = CreateSimpleCoreFacility(); - return NPrivate::CreateGrpcIamCredentialsProviderAsync< - TIamOAuthCredentialsProvider>(Params_, facility, facility); - } - TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const override { return std::make_shared>(Params_, std::move(facility)); } @@ -640,11 +548,6 @@ class TIamOAuthCredentialsProviderFactory : public ICredentialsProviderFactory { Params_.OAuthToken); } - NThreading::TFuture CreateProviderAsync(std::weak_ptr facility) const override { - return NPrivate::CreateGrpcIamCredentialsProviderAsync< - TIamOAuthCredentialsProvider>(Params_, std::move(facility)); - } - private: TIamOAuth Params_; }; diff --git a/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h b/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h index 6e4d48dba7..0a6b1d5eeb 100644 --- a/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h +++ b/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h @@ -19,7 +19,7 @@ class ICoreFacility { // Add task to execute periodicaly // Task should return false to stop execution virtual void AddPeriodicTask(TPeriodicCb&& cb, TDeadline::Duration period) = 0; - // Post task on SDK response executor. + // Post task on SDK response executor, never inline. virtual void PostToResponseQueue(TPostTaskCb&& f) = 0; }; diff --git a/include/ydb-cpp-sdk/client/types/credentials/credentials.h b/include/ydb-cpp-sdk/client/types/credentials/credentials.h index 8082c7a6a2..f84ca505a1 100644 --- a/include/ydb-cpp-sdk/client/types/credentials/credentials.h +++ b/include/ydb-cpp-sdk/client/types/credentials/credentials.h @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -16,9 +17,17 @@ class ICredentialsProvider { virtual ~ICredentialsProvider() = default; virtual std::string GetAuthInfo() const = 0; virtual bool IsValid() const = 0; + virtual NThreading::TFuture GetAuthInfoAsync() const { + try { + return NThreading::MakeFuture(GetAuthInfo()); + } catch (...) { + return NThreading::MakeErrorFuture(std::current_exception()); + } + } }; using TCredentialsProviderPtr = std::shared_ptr; +class ICoreFacility; // Implementation detail for SDK credentials factories. Symbols in NCredentials::NDetail are not // part of the public YDB C++ SDK API and may change or be removed without notice. @@ -26,6 +35,35 @@ namespace NCredentials::NDetail { using TCredentialsProviderCreator = std::function; +class TOwningFacilityCredentialsProvider final : public ICredentialsProvider { +public: + TOwningFacilityCredentialsProvider(std::shared_ptr facility, + TCredentialsProviderPtr inner, + bool forwardAsync = false) + : Facility_(std::move(facility)) + , Inner_(std::move(inner)) + , ForwardAsync_(forwardAsync) + {} + + std::string GetAuthInfo() const override { + return Inner_->GetAuthInfo(); + } + + NThreading::TFuture GetAuthInfoAsync() const override { + return ForwardAsync_ ? Inner_->GetAuthInfoAsync() : ICredentialsProvider::GetAuthInfoAsync(); + } + + bool IsValid() const override { + return Inner_->IsValid(); + } + +private: + // Reverse destruction keeps Facility_ alive while Inner_ stops. + std::shared_ptr Facility_; + TCredentialsProviderPtr Inner_; + const bool ForwardAsync_; +}; + // Process-wide weak cache for no-argument factory paths whose providers own their facilities. // Facility-bound providers must not use it: their callbacks belong to the supplied facility. TCredentialsProviderPtr GetOrCreateCachedProvider( @@ -34,21 +72,15 @@ TCredentialsProviderPtr GetOrCreateCachedProvider( } // namespace NCredentials::NDetail -class ICoreFacility; class ICredentialsProviderFactory { public: virtual ~ICredentialsProviderFactory() = default; // deprecated, use CreateProvider(std::weak_ptr facility) instead virtual TCredentialsProviderPtr CreateProvider() const = 0; - virtual NThreading::TFuture CreateProviderAsync() const { - return NThreading::MakeFuture(CreateProvider()); - } + // The facility must outlive the returned provider. virtual TCredentialsProviderPtr CreateProvider([[maybe_unused]] std::weak_ptr facility) const { return CreateProvider(); } - virtual NThreading::TFuture CreateProviderAsync(std::weak_ptr facility) const { - return NThreading::MakeFuture(CreateProvider(std::move(facility))); - } virtual std::string GetClientIdentity() const; }; diff --git a/src/client/iam/iam.cpp b/src/client/iam/iam.cpp index 5884118d1a..c8a294a544 100644 --- a/src/client/iam/iam.cpp +++ b/src/client/iam/iam.cpp @@ -15,120 +15,156 @@ using namespace yandex::cloud::iam::v1; namespace NYdb::inline V3 { -class TIAMCredentialsProvider : public ICredentialsProvider { +class TIAMCredentialsProvider : public ICredentialsProvider, public std::enable_shared_from_this { public: - TIAMCredentialsProvider(const TIamHost& params) + TIAMCredentialsProvider(const TIamHost& params, std::weak_ptr facility) : HttpClient_(TSimpleHttpClient(TString(params.Host), params.Port)) , Request_("/computeMetadata/v1/instance/service-accounts/default/token") , NextTicketUpdate_(TInstant::Zero()) , RefreshPeriod_(params.RefreshPeriod) - { - GetTicket(); - } - - std::string GetAuthInfo() const override { - std::string ticket; - TInstant nextTicketUpdate; - auto now = TInstant::Now(); - std::optional lastErrorMessage; - { - std::lock_guard lock(Lock_); - if (LastErrorMessage_.has_value() && now > ExpiresAt_) { - Ticket_.clear(); - } - ticket = Ticket_; - nextTicketUpdate = NextTicketUpdate_; - lastErrorMessage = LastErrorMessage_; + , Facility_(std::move(facility)) + , AuthInfo_(NThreading::NewPromise()) + {} + + void Start() { + auto facility = Facility_.lock(); + if (!facility) { + Fail("IAM-token provider response facility is not available"); + return; } - if (now >= nextTicketUpdate) { - GetTicket(); - { - std::lock_guard lock(Lock_); - if (LastErrorMessage_.has_value() && now > ExpiresAt_) { - Ticket_.clear(); + try { + facility->AddPeriodicTask([weak = weak_from_this()](NYdb::NIssue::TIssues&&, EStatus status) { + if (auto self = weak.lock()) { + return self->OnPeriodicTick(status); } - ticket = Ticket_; - lastErrorMessage = LastErrorMessage_; - } - } - if (ticket.empty() && lastErrorMessage.has_value()) { - throw yexception() << *lastErrorMessage; + return false; + }, PERIODIC_TICK); + } catch (...) { + Fail(CurrentExceptionMessage()); } - return ticket; + } + + std::string GetAuthInfo() const override { + return GetAuthInfoAsync().GetValueSync(); + } + + NThreading::TFuture GetAuthInfoAsync() const override { + std::lock_guard lock(Lock_); + return AuthInfo_.GetFuture(); } bool IsValid() const override { return true; } + ~TIAMCredentialsProvider() { Fail("IAM-token provider stopped"); } + private: TSimpleHttpClient HttpClient_; std::string Request_; mutable std::mutex Lock_; - mutable std::string Ticket_; mutable TInstant NextTicketUpdate_; - mutable TInstant ExpiresAt_ = TInstant::Zero(); - mutable std::optional LastErrorMessage_; TDuration RefreshPeriod_; + std::weak_ptr Facility_; + mutable NThreading::TPromise AuthInfo_; + mutable bool Stopped_ = false; - void GetTicket() const { - try { - TStringStream out; - TSimpleHttpClient::THeaders headers; - headers["Metadata-Flavor"] = "Google"; - HttpClient_.DoGet(Request_, &out, headers); - NJson::TJsonValue resp; - NJson::ReadJsonTree(&out, &resp, true); - - auto respMap = resp.GetMap(); - - std::string ticket; - if (auto it = respMap.find("access_token"); it == respMap.end()) - ythrow yexception() << "Result doesn't contain access_token"; - else if (ticket = it->second.GetStringSafe(); ticket.empty()) - ythrow yexception() << "Got empty ticket"; - - const auto now = TInstant::Now(); - TInstant nextUpdate; - TDuration expiresIn; - TInstant expiresAt = TInstant::Max(); - if (auto it = respMap.find("expires_in"); it != respMap.end()) { - auto seconds = it->second.GetUInteger(); - if (seconds > 0) { - expiresIn = TDuration::Seconds(seconds); - expiresAt = now + expiresIn; - } - } else if (auto it = respMap.find("expiry"); it != respMap.end()) { - try { - TInstant expiry; - if (TInstant::TryParseIso8601(it->second.GetStringSafe(), expiry) && expiry > now) { - expiresIn = expiry - now; - expiresAt = expiry; - } - } catch (...) { - expiresAt = now; - } + void Fail(std::string error) const { + NThreading::TPromise promise; + { + std::lock_guard lock(Lock_); + Stopped_ = true; + promise = AuthInfo_; + } + promise.TrySetException(std::make_exception_ptr(yexception() << error)); + } + + bool OnPeriodicTick(EStatus status) const { + if (status != EStatus::SUCCESS) { + Fail("IAM-token provider periodic task failed"); + return false; + } + + NThreading::TPromise promise; + { + std::lock_guard lock(Lock_); + if (Stopped_) { + return false; + } + if (TInstant::Now() < NextTicketUpdate_) { + return true; } - if (expiresIn > TDuration::Zero()) { - const auto halfLife = expiresIn / 2; - const auto interval = std::max(std::min(halfLife, RefreshPeriod_), TDuration::MilliSeconds(100)); - nextUpdate = now + interval; - } else { - nextUpdate = now + std::min(RefreshPeriod_, TDuration::Minutes(30)); + if (AuthInfo_.GetFuture().IsReady()) { + AuthInfo_ = NThreading::NewPromise(); } + promise = AuthInfo_; + } + try { + auto [ticket, nextUpdate] = GetTicket(); { std::lock_guard lock(Lock_); - Ticket_ = std::move(ticket); NextTicketUpdate_ = nextUpdate; - ExpiresAt_ = expiresAt; - LastErrorMessage_.reset(); } + promise.TrySetValue(std::move(ticket)); } catch (...) { + const auto error = std::current_exception(); + if (!IsRetryable(error)) { + Fail(CurrentExceptionMessage()); + return false; + } std::lock_guard lock(Lock_); NextTicketUpdate_ = TInstant::Now() + std::min(RefreshPeriod_, TDuration::Seconds(10)); - LastErrorMessage_ = CurrentExceptionMessage(); } + return true; + } + + static bool IsRetryable(const std::exception_ptr& error) { + try { + std::rethrow_exception(error); + } catch (const THttpRequestException& e) { + const int code = e.GetStatusCode(); + return code == 0 || code == HTTP_REQUEST_TIME_OUT || code == HTTP_AUTHENTICATION_TIMEOUT || + code == HTTP_TOO_MANY_REQUESTS || (code >= 500 && code < 600); + } catch (const TSystemError&) { + return true; + } catch (...) { + return false; + } + } + + std::pair GetTicket() const { + TStringStream out; + TSimpleHttpClient::THeaders headers; + headers["Metadata-Flavor"] = "Google"; + HttpClient_.DoGet(Request_, &out, headers); + NJson::TJsonValue resp; + NJson::ReadJsonTree(&out, &resp, true); + + auto respMap = resp.GetMap(); + std::string ticket; + if (auto it = respMap.find("access_token"); it == respMap.end()) + ythrow yexception() << "Result doesn't contain access_token"; + else if (ticket = it->second.GetStringSafe(); ticket.empty()) + ythrow yexception() << "Got empty ticket"; + + const auto now = TInstant::Now(); + TDuration expiresIn; + if (auto it = respMap.find("expires_in"); it != respMap.end()) { + const auto seconds = it->second.GetUInteger(); + if (seconds > 0) { + expiresIn = TDuration::Seconds(seconds); + } + } else if (auto it = respMap.find("expiry"); it != respMap.end()) { + TInstant expiry; + if (TInstant::TryParseIso8601(it->second.GetStringSafe(), expiry) && expiry > now) { + expiresIn = expiry - now; + } + } + const auto interval = expiresIn > TDuration::Zero() + ? std::max(std::min(expiresIn / 2, RefreshPeriod_), TDuration::MilliSeconds(100)) + : std::min(RefreshPeriod_, TDuration::Minutes(30)); + return {std::move(ticket), now + interval}; } }; @@ -140,21 +176,17 @@ class TIamCredentialsProviderFactory : public ICredentialsProviderFactory { return NCredentials::NDetail::GetOrCreateCachedProvider( GetClientIdentity(), [this] { - return std::make_shared(Params_); + auto facility = CreateSimpleCoreFacility(); + auto provider = CreateProvider(facility); + return std::make_shared( + std::move(facility), std::move(provider)); }); } - // Keep the facility-taking path driver-scoped; only the no-arg path is process-wide cached. - TCredentialsProviderPtr CreateProvider(std::weak_ptr) const final { - return std::make_shared(Params_); - } - - NThreading::TFuture CreateProviderAsync() const final { - return CreateProviderAsync(std::weak_ptr{}); - } - - NThreading::TFuture CreateProviderAsync(std::weak_ptr facility) const final { - return CreateProviderInBackground(Params_, std::move(facility)); + TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const final { + auto provider = std::make_shared(Params_, std::move(facility)); + provider->Start(); + return provider; } std::string GetClientIdentity() const final { @@ -164,30 +196,6 @@ class TIamCredentialsProviderFactory : public ICredentialsProviderFactory { } private: - static NThreading::TFuture CreateProviderInBackground( - TIamHost params, - std::weak_ptr facility) - { - auto promise = NThreading::NewPromise(); - auto createProvider = [params = std::move(params), promise]() mutable { - try { - promise.TrySetValue(std::make_shared(params)); - } catch (...) { - promise.TrySetException(std::current_exception()); - } - }; - try { - if (auto core = facility.lock()) { - core->PostToResponseQueue(std::move(createProvider)); - } else { - createProvider(); - } - } catch (...) { - promise.TrySetException(std::current_exception()); - } - return promise.GetFuture(); - } - TIamHost Params_; }; diff --git a/src/client/iam_private/common/iam.h b/src/client/iam_private/common/iam.h index a3952e4e4a..9fe1a84233 100644 --- a/src/client/iam_private/common/iam.h +++ b/src/client/iam_private/common/iam.h @@ -2,8 +2,6 @@ #include -#include - namespace NYdb::inline V3 { template @@ -27,88 +25,45 @@ class TIamServiceCredentialsProviderFactory : public ICredentialsProviderFactory class TCredentialsProvider : public TGrpcIamCredentialsProvider { public: - // TDriver path: a shared facility (TGRpcConnectionsImpl) supports multiple periodic tasks, - // so we can hand the same weak_ptr to the nested auth provider here. - TCredentialsProvider(const TIamServiceParams& params, std::weak_ptr responseFacility) - : TGrpcIamCredentialsProvider(params, - MakeRequestFiller(params), - MakeRpc(), - responseFacility, - params.SystemServiceAccountCredentials->CreateProvider(responseFacility)) - {} - - // Standalone (no-arg) path: the caller has already built a self-owning auth provider - // backed by its OWN facility. We must not share `outerFacility` with the auth provider - // because TSimpleCoreFacility allows only one periodic task. TCredentialsProvider(const TIamServiceParams& params, - std::weak_ptr outerFacility, - TCredentialsProviderPtr authProvider, - bool waitForToken = true) + std::weak_ptr responseFacility, + TCredentialsProviderPtr authProvider = {}) : TGrpcIamCredentialsProvider(params, MakeRequestFiller(params), MakeRpc(), - std::move(outerFacility), - std::move(authProvider), - waitForToken) + responseFacility, + authProvider ? std::move(authProvider) : + params.SystemServiceAccountCredentials->CreateProvider(responseFacility)) {} }; - static NThreading::TFuture CreateProviderAsyncImpl( - TIamServiceParams params, - NThreading::TFuture authProvider, - std::weak_ptr facility, - std::shared_ptr ownedFacility = {}) - { - auto serviceProvider = std::make_shared( - params, std::move(facility), co_await authProvider, false); - auto ready = serviceProvider->GetReadyFuture(); - TCredentialsProviderPtr provider = std::move(serviceProvider); - if (ownedFacility) { - provider = std::make_shared( - std::move(ownedFacility), std::move(provider)); - } - co_return co_await ready.Return(std::move(provider)); - } - public: TIamServiceCredentialsProviderFactory(const TIamServiceParams& params) : Params_(params) {} - // Deprecated. Kept for backward compatibility — see comment on TIamJwtCredentialsProviderFactory. - // The nested auth provider gets its own facility (via a recursive no-arg CreateProvider() that - // owns its private facility). Sharing a TSimpleCoreFacility between two gRPC - // IAM providers would abort: each one registers a periodic task and the facility allows only one. TCredentialsProviderPtr CreateProvider() const override final { return NCredentials::NDetail::GetOrCreateCachedProvider( GetClientIdentity(), [this] { - auto authProvider = Params_.SystemServiceAccountCredentials->CreateProvider(); - auto outerFacility = CreateSimpleCoreFacility(); + auto authProvider = NCredentials::NDetail::GetOrCreateCachedProvider( + "async:" + Params_.SystemServiceAccountCredentials->GetClientIdentity(), [this] { + auto facility = CreateSimpleCoreFacility(); + return std::make_shared(facility, + Params_.SystemServiceAccountCredentials->CreateProvider(facility), true); + }); + auto facility = CreateSimpleCoreFacility(); auto serviceProvider = std::make_shared( - Params_, std::weak_ptr(outerFacility), std::move(authProvider)); + Params_, facility, std::move(authProvider)); return std::make_shared( - std::move(outerFacility), std::move(serviceProvider)); + std::move(facility), std::move(serviceProvider)); }); } - NThreading::TFuture CreateProviderAsync() const override { - auto outerFacility = CreateSimpleCoreFacility(); - return CreateProviderAsyncImpl( - Params_, Params_.SystemServiceAccountCredentials->CreateProviderAsync(), - outerFacility, outerFacility); - } - TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const override { return std::make_shared(Params_, std::move(facility)); } - NThreading::TFuture CreateProviderAsync(std::weak_ptr facility) const override { - return CreateProviderAsyncImpl( - Params_, Params_.SystemServiceAccountCredentials->CreateProviderAsync(facility), - facility); - } - std::string GetClientIdentity() const override final { return NIam::NDetail::MakeClientIdentity( "TIamServiceCredentialsProviderFactory", diff --git a/src/client/impl/internal/db_driver_state/state.cpp b/src/client/impl/internal/db_driver_state/state.cpp index bc392d9502..357c151819 100644 --- a/src/client/impl/internal/db_driver_state/state.cpp +++ b/src/client/impl/internal/db_driver_state/state.cpp @@ -62,29 +62,24 @@ TDbDriverState::TDbDriverState( void TDbDriverState::InitCredentials( std::shared_ptr credentialsProviderFactory ) { - Credentials = credentialsProviderFactory->CreateProviderAsync(weak_from_this()).Apply( - [](const NThreading::TFuture& future) { - TCredentials result{future.GetValue()}; + Credentials.Provider = credentialsProviderFactory->CreateProvider(weak_from_this()); #ifndef YDB_GRPC_UNSECURE_AUTH - result.CallCredentials = grpc::MetadataCredentialsFromPlugin( - std::unique_ptr(new TYdbAuthenticator(result.Provider))); + Credentials.CallCredentials = grpc::MetadataCredentialsFromPlugin( + std::unique_ptr(new TYdbAuthenticator(Credentials.Provider))); #endif - return result; - }); - CredentialsReady = Credentials.IgnoreResult(); } NThreading::TFuture TDbDriverState::GetCredentialsReady() const { - return CredentialsReady; + return Credentials.Provider->GetAuthInfoAsync().IgnoreResult(); } std::shared_ptr TDbDriverState::GetCredentialsProvider() const { - return Credentials.HasValue() ? Credentials.GetValue().Provider : nullptr; + return Credentials.Provider; } #ifndef YDB_GRPC_UNSECURE_AUTH std::shared_ptr TDbDriverState::GetCallCredentials() const { - return Credentials.HasValue() ? Credentials.GetValue().CallCredentials : nullptr; + return Credentials.CallCredentials; } #endif diff --git a/src/client/impl/internal/db_driver_state/state.h b/src/client/impl/internal/db_driver_state/state.h index 0f31e96ac3..e719eebafe 100644 --- a/src/client/impl/internal/db_driver_state/state.h +++ b/src/client/impl/internal/db_driver_state/state.h @@ -84,8 +84,7 @@ class TDbDriverState #endif }; - NThreading::TFuture CredentialsReady; - NThreading::TFuture Credentials; + TCredentials Credentials; mutable std::once_flag ClientTlsValidationOnceFlag_; mutable bool ClientTlsCredentialsValid_ = true; mutable std::string ClientTlsValidationDetail_; diff --git a/src/client/impl/internal/grpc_connections/grpc_connections.h b/src/client/impl/internal/grpc_connections/grpc_connections.h index 203187b3c0..f31b8d4f54 100644 --- a/src/client/impl/internal/grpc_connections/grpc_connections.h +++ b/src/client/impl/internal/grpc_connections/grpc_connections.h @@ -335,7 +335,7 @@ class TGRpcConnectionsImpl if (auto ready = CredentialsReadyToWaitFor(dbState, requestSettings, context); ready.Initialized()) { DeferUntilCredentialsReady(requestSettings, context, std::move(ready), [this, requestWrapper = std::move(requestWrapper), userResponseCb = std::move(userResponseCb), - rpc, dbState, requestSettings, context = std::move(context)] + rpc, dbState, requestSettings, context] (std::optional status) YDB_ASAN_SIZE_ATTRIBUTES mutable { if (status) { userResponseCb(nullptr, std::move(*status)); @@ -592,7 +592,7 @@ class TGRpcConnectionsImpl if (auto ready = CredentialsReadyToWaitFor(dbState, requestSettings, context); ready.Initialized()) { DeferUntilCredentialsReady(requestSettings, context, std::move(ready), - [this, request, responseCb = std::move(responseCb), rpc, dbState, requestSettings, context = std::move(context)] + [this, request, responseCb = std::move(responseCb), rpc, dbState, requestSettings, context] (std::optional status) YDB_ASAN_SIZE_ATTRIBUTES mutable { if (status) { responseCb(std::move(*status), nullptr); @@ -699,7 +699,7 @@ class TGRpcConnectionsImpl if (auto ready = CredentialsReadyToWaitFor(dbState, requestSettings, context); ready.Initialized()) { DeferUntilCredentialsReady(requestSettings, context, std::move(ready), - [this, connectedCallback = std::move(connectedCallback), rpc, dbState, requestSettings, context = std::move(context)] + [this, connectedCallback = std::move(connectedCallback), rpc, dbState, requestSettings, context] (std::optional status) YDB_ASAN_SIZE_ATTRIBUTES mutable { if (status) { connectedCallback(std::move(*status), nullptr); diff --git a/src/client/persqueue_public/impl/write_session_impl.cpp b/src/client/persqueue_public/impl/write_session_impl.cpp index 26bd88fa2e..d7e4fa7197 100644 --- a/src/client/persqueue_public/impl/write_session_impl.cpp +++ b/src/client/persqueue_public/impl/write_session_impl.cpp @@ -29,7 +29,6 @@ TWriteSessionImpl::TWriteSessionImpl( , Client(std::move(client)) , Connections(std::move(connections)) , DbDriverState(std::move(dbDriverState)) - , PrevToken(DbDriverState->GetCredentialsProvider() ? DbDriverState->GetCredentialsProvider()->GetAuthInfo() : "") , InitSeqNoPromise(NThreading::NewPromise()) , WakeupInterval( Settings.BatchFlushInterval_ != TDuration::Zero() ? @@ -1184,19 +1183,62 @@ void TWriteSessionImpl::UpdateTokenIfNeededImpl() { LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: try to update token"); auto credentialsProvider = DbDriverState->GetCredentialsProvider(); - if (!credentialsProvider || UpdateTokenInProgress || !SessionEstablished) + if (!credentialsProvider || UpdateTokenInProgress || !SessionEstablished || Aborting) return; - TClientMessage clientMessage; - auto* updateRequest = clientMessage.mutable_update_token_request(); - auto token = credentialsProvider->GetAuthInfo(); - if (token == PrevToken) + auto authInfo = credentialsProvider->GetAuthInfoAsync(); + if (authInfo.IsReady()) { + UpdateTokenImpl(authInfo); return; + } + UpdateTokenInProgress = true; + try { + authInfo.Subscribe([cbContext = SelfContext](const auto& future) { + if (auto self = cbContext->LockShared()) try { + self->Connections->ScheduleCallback(TDuration::Zero(), [cbContext, future](bool ok) { + if (auto self = cbContext->LockShared()) { + if (!ok) { + self->UpdateTokenInProgress = false; + return; + } + std::lock_guard guard(self->Lock); + self->UpdateTokenImpl(future); + } + }); + } catch (...) { + self->UpdateTokenInProgress = false; + } + }); + } catch (...) { + UpdateTokenInProgress = false; + } +} + +void TWriteSessionImpl::UpdateTokenImpl(const NThreading::TFuture& future) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + UpdateTokenInProgress = false; + if (!SessionEstablished || Aborting) { + return; + } + + std::string token; + try { + token = future.GetValue(); + } catch (...) { + CloseImpl(EStatus::CLIENT_UNAUTHENTICATED, CurrentExceptionMessage()); + return; + } + if (token == PrevToken) { + return; + } + UpdateTokenInProgress = true; - updateRequest->set_token(TStringType{token}); PrevToken = token; LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: updating token"); + TClientMessage clientMessage; + clientMessage.mutable_update_token_request()->set_token(TStringType{token}); Processor->Write(std::move(clientMessage)); } diff --git a/src/client/persqueue_public/impl/write_session_impl.h b/src/client/persqueue_public/impl/write_session_impl.h index 86be4fc8d3..aeff9e0c96 100644 --- a/src/client/persqueue_public/impl/write_session_impl.h +++ b/src/client/persqueue_public/impl/write_session_impl.h @@ -325,6 +325,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, TStringBuilder LogPrefix() const; void UpdateTokenIfNeededImpl(); + void UpdateTokenImpl(const NThreading::TFuture& future); void WriteInternal(TContinuationToken&& continuationToken, std::string_view data, std::optional codec, ui32 originalSize, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt); @@ -387,7 +388,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, std::shared_ptr ConnectionFactory; TDbDriverStatePtr DbDriverState; std::string PrevToken; - bool UpdateTokenInProgress = false; + std::atomic_bool UpdateTokenInProgress = false; TInstant LastTokenUpdate = TInstant::Zero(); std::shared_ptr EventsQueue; NYdbGrpc::IQueueClientContextPtr ClientContext; // Common client context. diff --git a/src/client/topic/impl/write_session_impl.cpp b/src/client/topic/impl/write_session_impl.cpp index d357d987c6..90c79da5f7 100644 --- a/src/client/topic/impl/write_session_impl.cpp +++ b/src/client/topic/impl/write_session_impl.cpp @@ -114,7 +114,6 @@ TWriteSessionImpl::TWriteSessionImpl( , Client(std::move(client)) , Connections(std::move(connections)) , DbDriverState(std::move(dbDriverState)) - , PrevToken(DbDriverState->GetCredentialsProvider() ? DbDriverState->GetCredentialsProvider()->GetAuthInfo() : "") , MaxBlockMessageCount(Settings.BatchFlushMessageCount_) , InitSeqNoPromise(NThreading::NewPromise()) , WakeupInterval( @@ -1587,11 +1586,53 @@ void TWriteSessionImpl::UpdateTokenIfNeededImpl() { LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefixImpl() << "Write session: try to update token"); auto credentialsProvider = DbDriverState->GetCredentialsProvider(); - if (!credentialsProvider || UpdateTokenInProgress || !SessionEstablished) { + if (!credentialsProvider || UpdateTokenInProgress || !SessionEstablished || Aborting) { return; } - auto token = credentialsProvider->GetAuthInfo(); + auto authInfo = credentialsProvider->GetAuthInfoAsync(); + if (authInfo.IsReady()) { + UpdateTokenImpl(authInfo); + return; + } + UpdateTokenInProgress = true; + try { + authInfo.Subscribe([cbContext = SelfContext](const auto& future) { + if (auto self = cbContext->LockShared()) try { + self->Connections->ScheduleCallback(TDuration::Zero(), [cbContext, future](bool ok) { + if (auto self = cbContext->LockShared()) { + if (!ok) { + self->UpdateTokenInProgress = false; + return; + } + std::lock_guard guard(self->Lock); + self->UpdateTokenImpl(future); + } + }); + } catch (...) { + self->UpdateTokenInProgress = false; + } + }); + } catch (...) { + UpdateTokenInProgress = false; + } +} + +void TWriteSessionImpl::UpdateTokenImpl(const NThreading::TFuture& future) { + Y_ABORT_UNLESS(Lock.IsLocked()); + + UpdateTokenInProgress = false; + if (!SessionEstablished || Aborting) { + return; + } + + std::string token; + try { + token = future.GetValue(); + } catch (...) { + CloseImpl(EStatus::CLIENT_UNAUTHENTICATED, CurrentExceptionMessage()); + return; + } if (token == PrevToken) { return; } diff --git a/src/client/topic/impl/write_session_impl.h b/src/client/topic/impl/write_session_impl.h index ff32dcf188..0acd8ee4a7 100644 --- a/src/client/topic/impl/write_session_impl.h +++ b/src/client/topic/impl/write_session_impl.h @@ -377,6 +377,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, TStringBuilder LogPrefixImpl() const; void UpdateTokenIfNeededImpl(); + void UpdateTokenImpl(const NThreading::TFuture& future); void WriteInternal(TContinuationToken&& continuationToken, TWriteMessage&& message); @@ -447,7 +448,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, std::shared_ptr ConnectionFactory; TDbDriverStatePtr DbDriverState; std::string PrevToken; - bool UpdateTokenInProgress = false; + std::atomic_bool UpdateTokenInProgress = false; TInstant LastTokenUpdate = TInstant::Zero(); std::shared_ptr EventsQueue; NYdbGrpc::IQueueClientContextPtr ClientContext; // Common client context. diff --git a/src/client/types/core_facility/simple_core_facility.cpp b/src/client/types/core_facility/simple_core_facility.cpp index 4356582b2a..3c6b46797d 100644 --- a/src/client/types/core_facility/simple_core_facility.cpp +++ b/src/client/types/core_facility/simple_core_facility.cpp @@ -2,7 +2,6 @@ #include #include -#include namespace NYdb::inline V3 { @@ -30,9 +29,6 @@ TSimpleCoreFacility::~TSimpleCoreFacility() { void TSimpleCoreFacility::AddPeriodicTask(TPeriodicCb&& cb, TDeadline::Duration period) { std::lock_guard lock(Mutex_); - Y_ABORT_UNLESS(!PeriodicStarted_); - PeriodicStarted_ = true; - auto periodicCb = std::make_shared(std::move(cb)); EnqueueTaskNoLock( TClock::now(), @@ -46,15 +42,11 @@ void TSimpleCoreFacility::PostToResponseQueue(TPostTaskCb&& f) { if (!f) { return; } - { - std::lock_guard lock(Mutex_); - if (!Stop_) { - EnqueueTaskNoLock(TClock::now(), std::move(f)); - Cv_.notify_one(); - return; - } + std::lock_guard lock(Mutex_); + if (!Stop_) { + EnqueueTaskNoLock(TClock::now(), std::move(f)); + Cv_.notify_one(); } - f(); } void TSimpleCoreFacility::EnqueueTaskNoLock(TTimePoint executeAt, TPostTaskCb&& task) { diff --git a/src/client/types/core_facility/simple_core_facility.h b/src/client/types/core_facility/simple_core_facility.h index 5444774dfc..b9d6ab3cec 100644 --- a/src/client/types/core_facility/simple_core_facility.h +++ b/src/client/types/core_facility/simple_core_facility.h @@ -51,7 +51,6 @@ class TSimpleCoreFacility final : public ICoreFacility { std::condition_variable Cv_; std::priority_queue, TScheduledTaskLess> Queue_; bool Stop_ = false; - bool PeriodicStarted_ = false; std::uint64_t NextSeqNo_ = 0; std::thread WorkerThread_; diff --git a/src/client/types/credentials/login/login.cpp b/src/client/types/credentials/login/login.cpp index af26b9b9d1..01afd3153e 100644 --- a/src/client/types/credentials/login/login.cpp +++ b/src/client/types/credentials/login/login.cpp @@ -37,75 +37,99 @@ std::chrono::system_clock::time_point GetTokenExpiresAt(const std::string& token } return {}; } + +bool IsRetryable(EStatus status) { + return status == EStatus::INTERNAL_ERROR || status == EStatus::ABORTED || status == EStatus::UNAVAILABLE || + status == EStatus::OVERLOADED || status == EStatus::GENERIC_ERROR || status == EStatus::TIMEOUT || + status == EStatus::CANCELLED || status == EStatus::UNDETERMINED || status == EStatus::SESSION_BUSY || + status == EStatus::TRANSPORT_UNAVAILABLE || status == EStatus::CLIENT_RESOURCE_EXHAUSTED || + status == EStatus::CLIENT_DEADLINE_EXCEEDED || status == EStatus::CLIENT_INTERNAL_ERROR || + status == EStatus::CLIENT_CANCELLED || status == EStatus::CLIENT_DISCOVERY_FAILED || + status == EStatus::CLIENT_LIMITS_REACHED; } -class TLoginCredentialsProvider : public ICredentialsProvider { +bool IsRetryableOperation(EStatus status) { + return status == EStatus::ABORTED || status == EStatus::UNAVAILABLE || status == EStatus::OVERLOADED || + status == EStatus::TIMEOUT || status == EStatus::SESSION_BUSY; +} +} + +class TLoginCredentialsProvider : public ICredentialsProvider, public std::enable_shared_from_this { public: TLoginCredentialsProvider(std::weak_ptr facility, TLoginCredentialsParams params); - virtual std::string GetAuthInfo() const override; - virtual bool IsValid() const override; - NThreading::TFuture PrepareTokenAsync(); + std::string GetAuthInfo() const override; + NThreading::TFuture GetAuthInfoAsync() const override; + bool IsValid() const override; + void Start(); + ~TLoginCredentialsProvider() { Fail("Login credentials provider stopped"); } private: - void PrepareToken(); + bool OnPeriodicTick(EStatus status); void RequestToken(); void FinishRequest(Ydb::Auth::LoginResponse* response, TPlainStatus status, bool facilityAvailable); - bool IsOk() const; - void ParseToken(); - std::string GetToken() const; - std::string GetError() const; - std::string GetTokenOrError() const; - - enum class EState { - Empty, - Requesting, - Done, - }; + void Fail(std::string error); + static std::string GetError(const TPlainStatus& status, const Ydb::Auth::LoginResponse& response); std::weak_ptr Facility_; TLoginCredentialsParams Params_; - EState State_ = EState::Empty; - std::mutex Mutex_; - std::atomic TokenReceived_ = 1; - std::atomic TokenParsed_ = 0; - std::optional Token_; - std::optional Error_; - TInstant TokenExpireAt_; + mutable std::mutex Mutex_; TInstant TokenRequestAt_; - TPlainStatus Status_; - Ydb::Auth::LoginResponse Response_; - NThreading::TPromise TokenReadyPromise_; + bool Requesting_ = false; + bool HasToken_ = false; + bool Stopped_ = false; + mutable NThreading::TPromise AuthInfo_; }; TLoginCredentialsProvider::TLoginCredentialsProvider(std::weak_ptr facility, TLoginCredentialsParams params) : Facility_(facility) , Params_(std::move(params)) - , TokenReadyPromise_(NThreading::NewPromise()) -{ - auto strongFacility = facility.lock(); - if (strongFacility) { - auto periodicTask = [facility, this](NYdb::NIssue::TIssues&&, EStatus status) -> bool { - if (status != EStatus::SUCCESS) { - return false; - } - - auto strongFacility = facility.lock(); - if (!strongFacility) { - return false; - } - - if (!TokenRequestAt_) { - return true; - } - - if (TInstant::Now() >= TokenRequestAt_) { - RequestToken(); + , AuthInfo_(NThreading::NewPromise()) +{} + +void TLoginCredentialsProvider::Start() { + auto facility = Facility_.lock(); + if (!facility) { + Fail("Login credentials provider response facility is not available"); + return; + } + { + std::lock_guard lock(Mutex_); + Requesting_ = true; + } + try { + facility->AddPeriodicTask([weak = weak_from_this()](NYdb::NIssue::TIssues&&, EStatus status) { + if (auto self = weak.lock()) { + return self->OnPeriodicTick(status); } + return false; + }, 1s); + } catch (...) { + Fail(CurrentExceptionMessage()); + return; + } + RequestToken(); +} +bool TLoginCredentialsProvider::OnPeriodicTick(EStatus status) { + if (status != EStatus::SUCCESS) { + Fail("Login credentials provider periodic task failed"); + return false; + } + { + std::lock_guard lock(Mutex_); + if (Stopped_) { + return false; + } + if (Requesting_ || TInstant::Now() < TokenRequestAt_) { return true; - }; - strongFacility->AddPeriodicTask(std::move(periodicTask), 1min); + } + if (AuthInfo_.GetFuture().IsReady()) { + AuthInfo_ = NThreading::NewPromise(); + } + Requesting_ = true; } + RequestToken(); + return true; } bool TLoginCredentialsProvider::IsValid() const { @@ -113,22 +137,21 @@ bool TLoginCredentialsProvider::IsValid() const { } std::string TLoginCredentialsProvider::GetAuthInfo() const { - if (TokenParsed_ == TokenReceived_) { - return GetTokenOrError(); - } else { - const_cast(this)->PrepareToken(); // will block here - return GetTokenOrError(); - } + return GetAuthInfoAsync().GetValueSync(); +} + +NThreading::TFuture TLoginCredentialsProvider::GetAuthInfoAsync() const { + std::lock_guard lock(Mutex_); + return AuthInfo_.GetFuture(); } void TLoginCredentialsProvider::RequestToken() { auto strongFacility = Facility_.lock(); if (strongFacility) { - TokenRequestAt_ = {}; - - auto responseCb = [facility = Facility_, this](Ydb::Auth::LoginResponse* resp, TPlainStatus status) { - auto strongFacility = facility.lock(); - FinishRequest(resp, std::move(status), static_cast(strongFacility)); + auto responseCb = [facility = Facility_, weak = weak_from_this()](Ydb::Auth::LoginResponse* resp, TPlainStatus status) { + if (auto self = weak.lock()) { + self->FinishRequest(resp, std::move(status), !facility.expired()); + } }; Ydb::Auth::LoginRequest request; @@ -137,9 +160,13 @@ void TLoginCredentialsProvider::RequestToken() { TRpcRequestSettings rpcSettings; rpcSettings.Deadline = TDeadline::AfterDuration(60s); - TGRpcConnectionsImpl::RunOnDiscoveryEndpoint( - strongFacility, std::move(request), std::move(responseCb), &Ydb::Auth::V1::AuthService::Stub::AsyncLogin, - rpcSettings); + try { + TGRpcConnectionsImpl::RunOnDiscoveryEndpoint( + strongFacility, std::move(request), std::move(responseCb), &Ydb::Auth::V1::AuthService::Stub::AsyncLogin, + rpcSettings); + } catch (...) { + Fail(CurrentExceptionMessage()); + } } else { FinishRequest(nullptr, {}, false); } @@ -150,107 +177,78 @@ void TLoginCredentialsProvider::FinishRequest( TPlainStatus status, bool facilityAvailable) { - std::optional error; - { - std::lock_guard lock(Mutex_); - State_ = EState::Done; - ++TokenReceived_; - if (facilityAvailable) { - Status_ = std::move(status); - if (response) { - Response_ = std::move(*response); - } - ParseToken(); - } else { - Token_.reset(); - Error_ = "Login credentials provider response facility is not available"; - TokenParsed_ = TokenReceived_.load(); - } - error = Error_; + if (!facilityAvailable) { + Fail("Login credentials provider response facility is not available"); + return; } - if (error) { - TokenReadyPromise_.TrySetException(std::make_exception_ptr(yexception() << *error)); - } else { - TokenReadyPromise_.TrySetValue(); - } -} -void TLoginCredentialsProvider::PrepareToken() { - PrepareTokenAsync().Wait(); - std::lock_guard lock(Mutex_); - ParseToken(); -} - -NThreading::TFuture TLoginCredentialsProvider::PrepareTokenAsync() { - bool requestToken = false; - auto future = TokenReadyPromise_.GetFuture(); - { - std::unique_lock lock(Mutex_); - if (State_ == EState::Empty) { - State_ = EState::Requesting; - requestToken = true; + Ydb::Auth::LoginResponse emptyResponse; + const auto& responseValue = response ? *response : emptyResponse; + const auto operationStatus = static_cast(responseValue.operation().status()); + if (!status.Ok() || operationStatus != EStatus::SUCCESS) { + bool retry; + { + std::lock_guard lock(Mutex_); + retry = (!status.Ok() && IsRetryable(status.Status)) || + (HasToken_ && status.Ok() && IsRetryableOperation(operationStatus)); + if (retry) { + Requesting_ = false; + TokenRequestAt_ = TInstant::Now() + TDuration::Seconds(1); + } } - } - if (requestToken) { - RequestToken(); - } - return future; -} - -bool TLoginCredentialsProvider::IsOk() const { - return State_ == EState::Done - && Status_.Ok() - && Response_.operation().status() == Ydb::StatusIds::SUCCESS; -} - -void TLoginCredentialsProvider::ParseToken() { // works under mutex - if (TokenParsed_ != TokenReceived_) { - if (IsOk()) { - Token_ = GetToken(); - Error_.reset(); - TInstant now = TInstant::Now(); - TokenExpireAt_ = ToInstant(GetTokenExpiresAt(Token_.value())); - TokenRequestAt_ = now + TDuration::Minutes((TokenExpireAt_ - now).Minutes() / 2); - } else { - Token_.reset(); - Error_ = GetError(); + if (retry) { + return; } - TokenParsed_ = TokenReceived_.load(); + Fail(GetError(status, responseValue)); + return; } -} -std::string TLoginCredentialsProvider::GetToken() const { Ydb::Auth::LoginResult result; - Response_.operation().result().UnpackTo(&result); - return result.token(); + if (!responseValue.operation().result().UnpackTo(&result) || result.token().empty()) { + Fail("Login service returned an empty token"); + return; + } + auto token = result.token(); + const auto now = TInstant::Now(); + NThreading::TPromise promise; + { + std::lock_guard lock(Mutex_); + Requesting_ = false; + HasToken_ = true; + TokenRequestAt_ = now + (ToInstant(GetTokenExpiresAt(token)) - now) / 2; + promise = AuthInfo_; + } + promise.TrySetValue(std::move(token)); } -std::string TLoginCredentialsProvider::GetError() const { - if (Status_.Ok()) { - if (Response_.operation().issues_size() > 0) { - return Response_.operation().issues(0).message(); - } else { - return Ydb::StatusIds_StatusCode_Name(Response_.operation().status()); - } - } else { - TStringBuilder str; - str << "Couldn't get token for provided credentials from " << Status_.Endpoint - << " with status " << Status_.Status << "."; - for (const auto& issue : Status_.Issues) { - str << Endl << "Issue: " << issue; - } - return str; +void TLoginCredentialsProvider::Fail(std::string error) { + NThreading::TPromise promise; + { + std::lock_guard lock(Mutex_); + Stopped_ = true; + Requesting_ = false; + promise = AuthInfo_; } + promise.TrySetException(std::make_exception_ptr(yexception() << error)); } -std::string TLoginCredentialsProvider::GetTokenOrError() const { - if (Token_) { - return Token_.value(); +std::string TLoginCredentialsProvider::GetError( + const TPlainStatus& status, + const Ydb::Auth::LoginResponse& response) +{ + if (status.Ok()) { + if (response.operation().issues_size() > 0) { + return response.operation().issues(0).message(); + } + return Ydb::StatusIds_StatusCode_Name(response.operation().status()); } - if (Error_) { - ythrow yexception() << Error_.value(); + TStringBuilder str; + str << "Couldn't get token for provided credentials from " << status.Endpoint + << " with status " << status.Status << "."; + for (const auto& issue : status.Issues) { + str << Endl << "Issue: " << issue; } - ythrow yexception() << "Wrong state of credentials provider"; + return str; } class TLoginCredentialsProviderFactory : public ICredentialsProviderFactory { @@ -258,7 +256,6 @@ class TLoginCredentialsProviderFactory : public ICredentialsProviderFactory { TLoginCredentialsProviderFactory(TLoginCredentialsParams params); virtual std::shared_ptr CreateProvider() const override; virtual std::shared_ptr CreateProvider(std::weak_ptr facility) const override; - virtual NThreading::TFuture CreateProviderAsync(std::weak_ptr facility) const override; private: TLoginCredentialsParams Params_; @@ -274,13 +271,9 @@ std::shared_ptr TLoginCredentialsProviderFactory::CreatePr } std::shared_ptr TLoginCredentialsProviderFactory::CreateProvider(std::weak_ptr facility) const { - return std::make_shared(std::move(facility), Params_); -} - -NThreading::TFuture TLoginCredentialsProviderFactory::CreateProviderAsync(std::weak_ptr facility) const { auto provider = std::make_shared(std::move(facility), Params_); - TCredentialsProviderPtr result = provider; - return provider->PrepareTokenAsync().Return(std::move(result)); + provider->Start(); + return provider; } std::shared_ptr CreateLoginCredentialsProviderFactory(TLoginCredentialsParams params) { diff --git a/src/client/types/credentials/oauth2_token_exchange/credentials.cpp b/src/client/types/credentials/oauth2_token_exchange/credentials.cpp index 57cbc42760..ac40fdc199 100644 --- a/src/client/types/credentials/oauth2_token_exchange/credentials.cpp +++ b/src/client/types/credentials/oauth2_token_exchange/credentials.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -85,18 +86,11 @@ bool IsRetryableError(TKeepAliveHttpClient::THttpCode code) { || code == HTTP_GATEWAY_TIME_OUT; } -ERetryErrorClass RetryPolicyClass(TKeepAliveHttpClient::THttpCode code) { - return IsRetryableError(code) ? ERetryErrorClass::LongRetry : ERetryErrorClass::ShortRetry; // In case when we already have token we know that all params are correct and all errors are temporary -} - -ERetryErrorClass SyncRetryPolicyClass(const std::exception* ex, bool retryAllErrors) { +ERetryErrorClass RetryPolicyClass(const std::exception* ex) { if (const TTokenExchangeError* err = dynamic_cast(ex)) { if (IsRetryableError(err->HttpCode)) { return ERetryErrorClass::LongRetry; } - if (retryAllErrors) { - return ERetryErrorClass::ShortRetry; - } } if (dynamic_cast(ex)) { return ERetryErrorClass::ShortRetry; @@ -106,8 +100,7 @@ ERetryErrorClass SyncRetryPolicyClass(const std::exception* ex, bool retryAllErr return ERetryErrorClass::NoRetry; } -using TRetryPolicy = IRetryPolicy; -using TSyncRetryPolicy = IRetryPolicy; +using TRetryPolicy = IRetryPolicy; struct TPrivateOauth2TokenExchangeParams: public TOauth2TokenExchangeParams { TPrivateOauth2TokenExchangeParams(const TOauth2TokenExchangeParams& params) @@ -172,48 +165,65 @@ struct TPrivateOauth2TokenExchangeParams: public TOauth2TokenExchangeParams { } }; -class TOauth2TokenExchangeProviderImpl: public std::enable_shared_from_this -{ +class TOauth2TokenExchangeProviderImpl final : public ICredentialsProvider { struct TTokenExchangeResult { std::string Token; - TInstant TokenDeadline; TInstant TokenRefreshTime; }; public: - explicit TOauth2TokenExchangeProviderImpl(const TPrivateOauth2TokenExchangeParams& params) + TOauth2TokenExchangeProviderImpl(const TPrivateOauth2TokenExchangeParams& params, + std::weak_ptr responseFacility) : Params(params) + , ResponseFacility(std::move(responseFacility)) + , AuthInfo(NThreading::NewPromise()) { - ExchangeTokenSync(TInstant::Now()); + ResponseFacility.expired() ? Stop() : Start(); + } + +private: + void Start() { + try { + WorkerThread = std::thread([this] { Run(); }); + } catch (...) { + Fail(std::current_exception()); + } } void Stop() { + NThreading::TPromise promise; { std::unique_lock lock(StopMutex); Stopping = true; StopVar.notify_all(); } + with_lock (Lock) { + promise = AuthInfo; + } + promise.TrySetException(std::make_exception_ptr(yexception() << PROV_ERR "stopped")); - if (RefreshTokenThread.joinable()) { - RefreshTokenThread.join(); + if (WorkerThread.joinable()) { + WorkerThread.join(); } } - std::string GetAuthInfo() const { - const TInstant now = TInstant::Now(); - std::string token; +public: + ~TOauth2TokenExchangeProviderImpl() { + Stop(); + } + + std::string GetAuthInfo() const override { + return GetAuthInfoAsync().GetValueSync(); + } + + NThreading::TFuture GetAuthInfoAsync() const override { with_lock (Lock) { - if (Token.empty() || now >= TokenDeadline) { // Update sync. This can be if we have repeating error during token refresh process. In this case we will try for the last time and throw an error - ExchangeTokenSync(now, true); - token = Token; - } else { - if (now >= TokenRefreshTime) { - TryRefreshToken(); - } - token = Token; // Still valid - } + return AuthInfo.GetFuture(); } - return token; + } + + bool IsValid() const override { + return true; } private: @@ -344,7 +354,6 @@ class TOauth2TokenExchangeProviderImpl: public std::enable_shared_from_this::max(), // max retries // default - Params.SyncUpdateTimeout_ // max time - )->CreateRetryState(); + TRetryPolicy::IRetryState::TPtr retryState; + TTokenExchangeResult result; + while (true) { + if (IsStopping()) { + return; + } + try { + result = ExchangeToken(TInstant::Now()); + break; + } catch (const std::exception& ex) { + if (!retryState) { + retryState = TRetryPolicy::GetExponentialBackoffPolicy( + RetryPolicyClass, + TDuration::MilliSeconds(10), + TDuration::MilliSeconds(200), + TDuration::Seconds(30))->CreateRetryState(); + } + auto delay = retryState->GetNextRetryDelay(&ex); + if (!delay) { + Fail(std::current_exception()); + return; + } + std::unique_lock lock(StopMutex); + if (StopVar.wait_for(lock, TChronoDuration(delay->GetValue()), [this] { return Stopping; })) { + return; + } + } catch (...) { + Fail(std::current_exception()); + return; + } + } + + NThreading::TPromise promise; + with_lock (Lock) { + promise = AuthInfo; + } + Complete([promise, token = std::move(result.Token)]() mutable { + promise.TrySetValue(std::move(token)); + }); + + { + std::unique_lock lock(StopMutex); + const auto delay = result.TokenRefreshTime - TInstant::Now(); + if (delay > TDuration::Zero() && + StopVar.wait_for(lock, TChronoDuration(delay.GetValue()), [this] { return Stopping; })) + { + return; } - if (auto interval = retryState->GetNextRetryDelay(&ex, retryAllErrors)) { - Sleep(*interval); - } else { - throw; + if (Stopping) { + return; } } + with_lock (Lock) { + AuthInfo = NThreading::NewPromise(); + } } } @@ -394,85 +435,32 @@ class TOauth2TokenExchangeProviderImpl: public std::enable_shared_from_this promise; with_lock (Lock) { - deadline = TokenDeadline; - } - while (!IsStopping()) { - const TInstant now = TInstant::Now(); - try { - TTokenExchangeResult result = ExchangeToken(now); - with_lock (Lock) { - Token = result.Token; - TokenDeadline = result.TokenDeadline; - TokenRefreshTime = result.TokenRefreshTime; - break; - } - } catch (const std::exception& ex) { // If this error will repeat, we finally will get it syncronously in GetAuthInfo() and pass to client - if (!retryState) { - retryState = TRetryPolicy::GetExponentialBackoffPolicy( - RetryPolicyClass, - TDuration::MilliSeconds(10), // min delay // default - TDuration::MilliSeconds(200), // min long delay // default - TDuration::Seconds(30), // max delay // default - std::numeric_limits::max(), // max retries // default - deadline - TInstant::Now() // max time - )->CreateRetryState(); - } else { - TKeepAliveHttpClient::THttpCode code = HTTP_CODE_MAX; - if (const auto* err = dynamic_cast(&ex)) { - code = err->HttpCode; - } - if (auto delay = retryState->GetNextRetryDelay(code)) { - std::unique_lock lock(StopMutex); - const bool stopping = StopVar.wait_for( - lock, - TChronoDuration(delay->GetValue()), - [this]() { - return Stopping; - } - ); - if (stopping) { - break; - } - } else { - break; - } - } - } + promise = AuthInfo; } - TokenIsRefreshing.store(false); + Complete([promise, error = std::move(error)]() mutable { + promise.TrySetException(std::move(error)); + }); } - void TryRefreshToken() const { // Is run under lock - if (TokenIsRefreshing.load()) { - return; - } - if (RefreshTokenThread.joinable()) { - RefreshTokenThread.join(); - } - - TokenIsRefreshing.store(true); - RefreshTokenThread = std::thread( - [w = weak_from_this()]() { - if (auto p = w.lock()) { - p->RefreshToken(); - } + void Complete(TPostTaskCb&& callback) const noexcept { + try { + if (auto facility = ResponseFacility.lock()) { + facility->PostToResponseQueue(std::move(callback)); } - ); + } catch (...) { + } } private: TPrivateOauth2TokenExchangeParams Params; + std::weak_ptr ResponseFacility; - TAdaptiveLock Lock; - mutable std::atomic TokenIsRefreshing = false; - mutable std::thread RefreshTokenThread; - mutable std::string Token; - mutable TInstant TokenDeadline; - mutable TInstant TokenRefreshTime; + mutable TAdaptiveLock Lock; + mutable NThreading::TPromise AuthInfo; + std::thread WorkerThread; // Stop bool Stopping = false; @@ -480,42 +468,31 @@ class TOauth2TokenExchangeProviderImpl: public std::enable_shared_from_this(params)) - { - } - - std::string GetAuthInfo() const override { - return Impl->GetAuthInfo(); - } - - bool IsValid() const override { - return true; - } - - ~TOauth2TokenExchangeProvider() { // The last link tp provider is gone - Impl->Stop(); - } - -private: - std::shared_ptr Impl; -}; - class TOauth2TokenExchangeFactory: public ICredentialsProviderFactory { public: explicit TOauth2TokenExchangeFactory(const TOauth2TokenExchangeParams& params) - : Provider(std::make_shared(params)) + : Params(params) { } TCredentialsProviderPtr CreateProvider() const override { + std::lock_guard lock(Lock); + if (!Provider) { + auto facility = CreateSimpleCoreFacility(); + Provider = std::make_shared( + facility, std::make_shared(Params, facility)); + } return Provider; } + TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const override { + return std::make_shared(Params, std::move(facility)); + } + private: - std::shared_ptr Provider; + TPrivateOauth2TokenExchangeParams Params; + mutable std::mutex Lock; + mutable TCredentialsProviderPtr Provider; }; } // namespace diff --git a/tests/common/iam_mocks/iam_grpc_mock_server.cpp b/tests/common/iam_mocks/iam_grpc_mock_server.cpp index a07a54fef8..e6bd55103e 100644 --- a/tests/common/iam_mocks/iam_grpc_mock_server.cpp +++ b/tests/common/iam_mocks/iam_grpc_mock_server.cpp @@ -10,6 +10,11 @@ void TIamTokenServiceStub::SetResponseToken(const std::string& token, int64_t ex ExpiresAtSeconds_ = expiresAtSeconds; } +void TIamTokenServiceStub::SetStatus(grpc::Status status) { + std::lock_guard lock(Lock_); + Status_ = std::move(status); +} + grpc::Status TIamTokenServiceStub::Create( grpc::ServerContext*, const yandex::cloud::iam::v1::CreateIamTokenRequest* request, @@ -19,6 +24,9 @@ grpc::Status TIamTokenServiceStub::Create( ++RequestCount_; LastRequest_ = *request; HasLastRequest_ = true; + if (!Status_.ok()) { + return Status_; + } response->set_iam_token(IamToken_); response->mutable_expires_at()->set_seconds(ExpiresAtSeconds_); response->mutable_expires_at()->set_nanos(0); diff --git a/tests/common/iam_mocks/iam_grpc_mock_server.h b/tests/common/iam_mocks/iam_grpc_mock_server.h index cb1666636e..08747041ac 100644 --- a/tests/common/iam_mocks/iam_grpc_mock_server.h +++ b/tests/common/iam_mocks/iam_grpc_mock_server.h @@ -21,6 +21,7 @@ namespace NYdb::NTest { class TIamTokenServiceStub final : public yandex::cloud::iam::v1::IamTokenService::Service { public: void SetResponseToken(const std::string& token, int64_t expiresAtSeconds = 4102444800); + void SetStatus(grpc::Status status); grpc::Status Create( grpc::ServerContext*, @@ -34,6 +35,7 @@ class TIamTokenServiceStub final : public yandex::cloud::iam::v1::IamTokenServic private: mutable std::mutex Lock_; std::string IamToken_; + grpc::Status Status_; int64_t ExpiresAtSeconds_ = 4102444800; int RequestCount_ = 0; yandex::cloud::iam::v1::CreateIamTokenRequest LastRequest_; diff --git a/tests/unit/client/driver/driver_ut.cpp b/tests/unit/client/driver/driver_ut.cpp index e2e3a77681..1e1154c2e3 100644 --- a/tests/unit/client/driver/driver_ut.cpp +++ b/tests/unit/client/driver/driver_ut.cpp @@ -124,32 +124,54 @@ namespace { std::atomic_int& ProviderCount_; }; + class TDeferredAuthProvider final : public ICredentialsProvider { + public: + TDeferredAuthProvider() + : AuthInfo_(NThreading::NewPromise()) + {} + + std::string GetAuthInfo() const override { + return AuthInfo_.GetFuture().GetValueSync(); + } + + NThreading::TFuture GetAuthInfoAsync() const override { + return AuthInfo_.GetFuture(); + } + + bool IsValid() const override { + return true; + } + + void SetReady() { + AuthInfo_.SetValue("token"); + } + + private: + NThreading::TPromise AuthInfo_; + }; + class TDeferredCredentialsFactory final : public ICredentialsProviderFactory { public: TDeferredCredentialsFactory() - : Provider_(NThreading::NewPromise()) + : Provider_(std::make_shared()) {} TCredentialsProviderPtr CreateProvider() const override { - return CreateInsecureCredentialsProviderFactory()->CreateProvider(); - } - - NThreading::TFuture CreateProviderAsync(std::weak_ptr) const override { - return Provider_.GetFuture(); + return Provider_; } void SetReady() { - Provider_.SetValue(CreateProvider()); + Provider_->SetReady(); } private: - NThreading::TPromise Provider_; + std::shared_ptr Provider_; }; } // namespace Y_UNIT_TEST_SUITE(DeferredCredentialsTest) { - Y_UNIT_TEST(RequestWaitsForCredentials) { + Y_UNIT_TEST(RequestWaitsForAuthInfo) { auto factory = std::make_shared(); auto driver = TDriver(TDriverConfig() .SetEndpoint("localhost:100") diff --git a/tests/unit/client/iam/grpc_iam_ut.cpp b/tests/unit/client/iam/grpc_iam_ut.cpp index b75c44d2a1..075357f8c1 100644 --- a/tests/unit/client/iam/grpc_iam_ut.cpp +++ b/tests/unit/client/iam/grpc_iam_ut.cpp @@ -33,19 +33,19 @@ TEST(GrpcIamCredentialsProvider, TeardownWhileIamCreatePendingCompletes) { TIamOAuth params = MakeOAuthParams(server.Endpoint()); params.RequestTimeout = TDuration::MilliSeconds(400); - auto work = [¶ms] { - auto facility = std::make_shared(); - TIamOAuthCredentialsProvider provider( - params, - facility); - (void)provider; - }; - - std::future done = std::async(std::launch::async, work); + auto facility = std::make_shared(); + auto provider = std::make_shared>(params, facility); + auto authInfo = provider->GetAuthInfoAsync(); + ASSERT_FALSE(authInfo.IsReady()); ASSERT_TRUE(iamService.WaitUntilRpcEntered(std::chrono::seconds(5))) << "server should have accepted the IAM Create call"; + std::future done = std::async(std::launch::async, [provider = std::move(provider)]() mutable { + provider.reset(); + }); + ASSERT_EQ(done.wait_for(std::chrono::seconds(20)), std::future_status::ready) << "provider destructor must finish while an IAM Create is still blocked on the server " "(in-flight RPC vs channel teardown)."; @@ -67,16 +67,15 @@ TEST(GrpcIamCredentialsProvider, TeardownWhileIamCreatePendingCompletesViaFactor auto factory = std::make_shared>(params); - auto work = [&factory] { - auto provider = factory->CreateProvider(); - (void)provider; - }; - - std::future done = std::async(std::launch::async, work); + auto provider = factory->CreateProvider(); ASSERT_TRUE(iamService.WaitUntilRpcEntered(std::chrono::seconds(5))) << "server should have accepted the IAM Create call"; + std::future done = std::async(std::launch::async, [provider = std::move(provider)]() mutable { + provider.reset(); + }); + ASSERT_EQ(done.wait_for(std::chrono::seconds(20)), std::future_status::ready) << "factory wrapper teardown must finish while an IAM Create is still blocked on the server"; done.get(); @@ -107,6 +106,7 @@ TEST(GrpcIamCredentialsProviderFactory, NoArgProvidersAreCachedAcrossFactoryInst for (size_t i = 1; i < oauthProviders.size(); ++i) { EXPECT_EQ(firstOAuthProvider, oauthProviders[i].get()); } + EXPECT_EQ(firstOAuthProvider->GetAuthInfo(), "unit-test-iam-token"); EXPECT_EQ(iamStub.GetRequestCount(), 1); using TJwtFactory = TIamJwtCredentialsProviderFactory< @@ -116,17 +116,41 @@ TEST(GrpcIamCredentialsProviderFactory, NoArgProvidersAreCachedAcrossFactoryInst auto secondJwtProvider = std::make_shared(jwtParams)->CreateProvider(); EXPECT_EQ(firstJwtProvider, secondJwtProvider); + EXPECT_EQ(firstJwtProvider->GetAuthInfo(), "unit-test-iam-token"); EXPECT_EQ(iamStub.GetRequestCount(), 2); auto differentOAuthParams = oauthParams; differentOAuthParams.OAuthToken = "different-oauth-token"; auto differentOAuthProvider = std::make_shared(differentOAuthParams)->CreateProvider(); EXPECT_NE(firstOAuthProvider, differentOAuthProvider); + EXPECT_EQ(differentOAuthProvider->GetAuthInfo(), "unit-test-iam-token"); EXPECT_EQ(iamStub.GetRequestCount(), 3); server.Stop(); } +TEST(GrpcIamCredentialsProviderFactory, ReadyCallbackCanReleaseNoArgProvider) { + TIamTokenServiceStub iamStub; + iamStub.SetResponseToken("unit-test-iam-token"); + TIamGrpcServer server(&iamStub); + ASSERT_TRUE(server.Start()); + + auto provider = std::make_shared>( + MakeOAuthParams(server.Endpoint()))->CreateProvider(); + auto released = std::make_shared>(); + auto authInfo = provider->GetAuthInfoAsync(); + ASSERT_TRUE(authInfo.IsReady()); + authInfo.Subscribe( + [provider = std::move(provider), released](const auto&) mutable { + provider.reset(); + released->set_value(); + }); + + ASSERT_EQ(released->get_future().wait_for(std::chrono::seconds(10)), std::future_status::ready); + server.Stop(); +} + namespace { class TSlowBlockingAuthProvider final : public ICredentialsProvider { @@ -134,6 +158,7 @@ class TSlowBlockingAuthProvider final : public ICredentialsProvider { std::string GetAuthInfo() const override { std::unique_lock lock(Mutex_); if (++CallCount_ > 1) { + BlockedCv_.notify_all(); BlockedCv_.wait(lock, [this] { return Released_; }); } return "slow-auth-token"; @@ -152,16 +177,8 @@ class TSlowBlockingAuthProvider final : public ICredentialsProvider { } bool WaitUntilBlocked(std::chrono::milliseconds timeout) const { - const auto deadline = std::chrono::steady_clock::now() + timeout; - while (std::chrono::steady_clock::now() < deadline) { - std::lock_guard lock(Mutex_); - if (CallCount_ > 1 && !Released_) { - return true; - } - std::this_thread::sleep_for(std::chrono::milliseconds(2)); - } - std::lock_guard lock(Mutex_); - return CallCount_ > 1 && !Released_; + std::unique_lock lock(Mutex_); + return BlockedCv_.wait_for(lock, timeout, [this] { return CallCount_ > 1; }); } private: @@ -171,30 +188,36 @@ class TSlowBlockingAuthProvider final : public ICredentialsProvider { bool Released_ = false; }; -class TFailThenSucceedAuthProvider final : public ICredentialsProvider { +class TDeferredAuthProvider final : public ICredentialsProvider { public: - explicit TFailThenSucceedAuthProvider(int failCount) - : FailCount_(failCount) + TDeferredAuthProvider() + : AuthInfo_(NThreading::NewPromise()) {} std::string GetAuthInfo() const override { - if (CallCount_.fetch_add(1) < FailCount_) { - ythrow yexception() << "auth failure"; - } - return "auth-token"; + return AuthInfo_.GetFuture().GetValueSync(); + } + + NThreading::TFuture GetAuthInfoAsync() const override { + ++CallCount_; + return AuthInfo_.GetFuture(); } bool IsValid() const override { return true; } - int GetCallCount() const { + int CallCount() const { return CallCount_.load(); } + void SetReady() { + AuthInfo_.SetValue("auth-token"); + } + private: - const int FailCount_; mutable std::atomic CallCount_{0}; + NThreading::TPromise AuthInfo_; }; } // namespace @@ -250,13 +273,13 @@ TEST(GrpcIamCredentialsProvider, StopDuringFillContextDoesNotHang) { server.Stop(); } -TEST(GrpcIamCredentialsProvider, FillContextAuthExceptionSurvivesAndRecovers) { +TEST(GrpcIamCredentialsProvider, WaitsForNestedAuthWithoutPollingIt) { TIamTokenServiceStub iamStub; iamStub.SetResponseToken("unit-test-iam-token"); TIamGrpcServer server(&iamStub); ASSERT_TRUE(server.Start()); - auto authProvider = std::make_shared(2); + auto authProvider = std::make_shared(); TIamOAuth params = MakeOAuthParams(server.Endpoint()); auto facility = std::make_shared(); @@ -273,9 +296,64 @@ TEST(GrpcIamCredentialsProvider, FillContextAuthExceptionSurvivesAndRecovers) { facility, authProvider); - EXPECT_EQ(provider.GetAuthInfo(), "unit-test-iam-token"); + auto future = provider.GetAuthInfoAsync(); + for (int i = 0; i < 100 && authProvider->CallCount() == 0; ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + ASSERT_EQ(authProvider->CallCount(), 1); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + EXPECT_EQ(authProvider->CallCount(), 1); + + authProvider->SetReady(); + ASSERT_TRUE(future.Wait(TDuration::Seconds(10))); + EXPECT_EQ(future.GetValue(), "unit-test-iam-token"); EXPECT_EQ(iamStub.GetRequestCount(), 1); - EXPECT_GE(authProvider->GetCallCount(), 3); + + server.Stop(); +} + +TEST(GrpcIamCredentialsProvider, RetriesTransientFailure) { + TIamTokenServiceStub iamStub; + iamStub.SetResponseToken("unit-test-iam-token"); + iamStub.SetStatus(grpc::Status(grpc::StatusCode::UNAVAILABLE, "retry")); + TIamGrpcServer server(&iamStub); + ASSERT_TRUE(server.Start()); + + auto facility = std::make_shared(); + auto provider = std::make_shared>( + MakeOAuthParams(server.Endpoint()), facility); + auto future = provider->GetAuthInfoAsync(); + for (int i = 0; i < 1000 && iamStub.GetRequestCount() == 0; ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + ASSERT_GT(iamStub.GetRequestCount(), 0); + ASSERT_FALSE(future.IsReady()); + + iamStub.SetStatus(grpc::Status::OK); + ASSERT_TRUE(future.Wait(TDuration::Seconds(10))); + EXPECT_EQ(future.GetValue(), "unit-test-iam-token"); + EXPECT_GT(iamStub.GetRequestCount(), 1); + + server.Stop(); +} + +TEST(GrpcIamCredentialsProvider, DoesNotRetryTerminalFailure) { + TIamTokenServiceStub iamStub; + iamStub.SetStatus(grpc::Status(grpc::StatusCode::PERMISSION_DENIED, "terminal")); + TIamGrpcServer server(&iamStub); + ASSERT_TRUE(server.Start()); + + auto facility = std::make_shared(); + auto provider = std::make_shared>( + MakeOAuthParams(server.Endpoint()), facility); + auto future = provider->GetAuthInfoAsync(); + ASSERT_TRUE(future.Wait(TDuration::Seconds(10))); + EXPECT_THROW(future.GetValue(), yexception); + const int requests = iamStub.GetRequestCount(); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + EXPECT_EQ(iamStub.GetRequestCount(), requests); server.Stop(); } diff --git a/tests/unit/client/iam/http_iam_ut.cpp b/tests/unit/client/iam/http_iam_ut.cpp index f0774b2098..e6cad44cbd 100644 --- a/tests/unit/client/iam/http_iam_ut.cpp +++ b/tests/unit/client/iam/http_iam_ut.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -65,7 +66,7 @@ TEST(IamCredentialsProvider, NoExpiryFieldFallback) { TEST(IamCredentialsProvider, ServerError) { TMetadataServer server; server.SetStrictMode(false); - server.SetResponse(HTTP_INTERNAL_SERVER_ERROR, ""); + server.SetResponse(HTTP_BAD_REQUEST, ""); TIamHost params = MakeMetadataParams(server.Port); @@ -75,7 +76,7 @@ TEST(IamCredentialsProvider, ServerError) { EXPECT_THROW(provider->GetAuthInfo(), yexception); } -TEST(IamCredentialsProvider, GracePeriodOnRefreshError) { +TEST(IamCredentialsProvider, RetriesTransientError) { TMetadataServer server; server.SetStrictMode(false); server.SetResponse(HTTP_OK, MakeTokenResponse("old-token", 3600)); @@ -83,57 +84,22 @@ TEST(IamCredentialsProvider, GracePeriodOnRefreshError) { TIamHost params = MakeMetadataParams(server.Port); params.RefreshPeriod = TDuration::MilliSeconds(100); - auto provider = CreateIamCredentialsProviderFactory(params)->CreateProvider(); + auto facility = CreateSimpleCoreFacility(); + auto provider = CreateIamCredentialsProviderFactory(params)->CreateProvider(facility); EXPECT_EQ(provider->GetAuthInfo(), "old-token"); - int countBeforeRefresh = server.GetRequestCount(); - Sleep(TDuration::MilliSeconds(150)); - + const int countBeforeRefresh = server.GetRequestCount(); server.SetResponse(HTTP_INTERNAL_SERVER_ERROR, ""); - EXPECT_EQ(provider->GetAuthInfo(), "old-token"); - EXPECT_GT(server.GetRequestCount(), countBeforeRefresh); -} - -TEST(IamCredentialsProvider, ThrowAfterTokenExpiredOnRefreshError) { - TMetadataServer server; - server.SetStrictMode(false); - server.SetResponse(HTTP_OK, MakeTokenResponse("old-token", 1)); - - TIamHost params = MakeMetadataParams(server.Port); - params.RefreshPeriod = TDuration::MilliSeconds(100); - - auto provider = CreateIamCredentialsProviderFactory(params)->CreateProvider(); - EXPECT_EQ(provider->GetAuthInfo(), "old-token"); - - Sleep(TDuration::MilliSeconds(150)); - server.SetResponse(HTTP_INTERNAL_SERVER_ERROR, ""); - EXPECT_EQ(provider->GetAuthInfo(), "old-token"); - - Sleep(TDuration::Seconds(1)); - EXPECT_THROW(provider->GetAuthInfo(), yexception); -} - -TEST(IamCredentialsProvider, RecoveryAfterRefreshError) { - TMetadataServer server; - server.SetStrictMode(false); - server.SetResponse(HTTP_OK, MakeTokenResponse("token-1", 3600)); - - TIamHost params = MakeMetadataParams(server.Port); - params.RefreshPeriod = TDuration::MilliSeconds(100); - - auto provider = CreateIamCredentialsProviderFactory(params)->CreateProvider(); - EXPECT_EQ(provider->GetAuthInfo(), "token-1"); - - Sleep(TDuration::MilliSeconds(150)); - server.SetResponse(HTTP_INTERNAL_SERVER_ERROR, ""); - EXPECT_EQ(provider->GetAuthInfo(), "token-1"); - - server.SetResponse(HTTP_OK, MakeTokenResponse("token-2", 3600)); - int countBeforeRecovery = server.GetRequestCount(); - Sleep(TDuration::MilliSeconds(150)); + for (int i = 0; i < 1000 && server.GetRequestCount() == countBeforeRefresh; ++i) { + Sleep(TDuration::MilliSeconds(10)); + } + ASSERT_GT(server.GetRequestCount(), countBeforeRefresh); + auto future = provider->GetAuthInfoAsync(); + EXPECT_FALSE(future.IsReady()); - EXPECT_EQ(provider->GetAuthInfo(), "token-2"); - EXPECT_GT(server.GetRequestCount(), countBeforeRecovery); + server.SetResponse(HTTP_OK, MakeTokenResponse("new-token", 3600)); + ASSERT_TRUE(future.Wait(TDuration::Seconds(10))); + EXPECT_EQ(future.GetValue(), "new-token"); } TEST(IamCredentialsProvider, ConcurrentAccess) { diff --git a/tests/unit/client/iam_private/grpc_iam_service_ut.cpp b/tests/unit/client/iam_private/grpc_iam_service_ut.cpp index 3fdf2f1de0..8914e6db4e 100644 --- a/tests/unit/client/iam_private/grpc_iam_service_ut.cpp +++ b/tests/unit/client/iam_private/grpc_iam_service_ut.cpp @@ -107,9 +107,7 @@ class TFailingCoreFacility final : public ICoreFacility { callback(std::move(issues), EStatus::CLIENT_CANCELLED); } - void PostToResponseQueue(TPostTaskCb&& callback) override { - callback(); - } + void PostToResponseQueue(TPostTaskCb&&) override {} }; using TTestOAuthFactory = TIamOAuthCredentialsProviderFactory< @@ -183,28 +181,31 @@ TEST(IamServiceCredentialsProvider, NoArgProviderIsCachedAcrossFactoryInstances) secondServiceParams.TargetServiceAccountId = "another-target"; auto differentProvider = CreateIamServiceCredentialsProviderFactory(secondServiceParams)->CreateProvider(); EXPECT_NE(firstProvider, differentProvider); + EXPECT_EQ(differentProvider->GetAuthInfo(), "outer-service-token"); EXPECT_EQ(stub.GetCreateRequestCount(), 1); EXPECT_EQ(stub.GetCreateForServiceRequestCount(), 2); server.Stop(); } -TEST(IamCredentialsProvider, AsyncCreationFailsWithExpiredFacility) { - auto future = MakeOAuthFactory().CreateProviderAsync(std::weak_ptr{}); +TEST(IamCredentialsProvider, AuthInfoFailsWithExpiredFacility) { + auto provider = MakeOAuthFactory().CreateProvider(std::weak_ptr{}); + auto future = provider->GetAuthInfoAsync(); ASSERT_TRUE(future.IsReady()); EXPECT_THROW(future.GetValue(), std::exception); } -TEST(IamCredentialsProvider, AsyncCreationFailsWhenPeriodicTaskIsRejected) { +TEST(IamCredentialsProvider, AuthInfoFailsWhenPeriodicTaskIsRejected) { auto facility = std::make_shared(); - auto future = MakeOAuthFactory().CreateProviderAsync(std::weak_ptr(facility)); + auto provider = MakeOAuthFactory().CreateProvider(std::weak_ptr(facility)); + auto future = provider->GetAuthInfoAsync(); ASSERT_TRUE(future.IsReady()); EXPECT_THROW(future.GetValue(), std::exception); } -TEST(IamCredentialsProvider, AsyncCreationRetriesTransientIamFailure) { +TEST(IamCredentialsProvider, AuthInfoRetriesTransientIamFailure) { TFlakyIamServiceStub stub; TIamGrpcServer server(&stub); ASSERT_TRUE(server.Start()); @@ -215,22 +216,17 @@ TEST(IamCredentialsProvider, AsyncCreationRetriesTransientIamFailure) { params.OAuthToken = "token"; params.RequestTimeout = TDuration::Seconds(2); - auto future = TTestOAuthFactory(params).CreateProviderAsync(); + auto provider = TTestOAuthFactory(params).CreateProvider(); + auto future = provider->GetAuthInfoAsync(); ASSERT_TRUE(future.Wait(TDuration::Seconds(10))); - auto provider = future.GetValue(); - EXPECT_EQ(provider->GetAuthInfo(), "oauth-token"); + EXPECT_EQ(future.GetValue(), "oauth-token"); EXPECT_GE(stub.Attempts(), 2u); server.Stop(); } -// Regression test for the deprecated no-arg CreateProvider() on the IAM service-account -// factory with a nested gRPC JWT auth provider. Before the fix, both providers shared a single -// TSimpleCoreFacility, each registered a periodic refresh task, and TSimpleCoreFacility's -// single-task invariant tripped Y_ABORT_UNLESS and killed the process. The fix gives the -// nested auth provider its own facility (via a recursive no-arg CreateProvider()), so the two -// periodic tasks land on separate facilities. +// The no-arg service provider keeps nested gRPC authentication on a separate facility. TEST(IamServiceCredentialsProvider, NoArgCreateProviderWithGrpcInnerCreds) { TIamServiceStub stub; TIamGrpcServer server(&stub); diff --git a/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp b/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp index 04d4761999..b6352029b7 100644 --- a/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp +++ b/tests/unit/client/oauth2_token_exchange/credentials_ut.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -12,6 +13,8 @@ #include #include +#include + using namespace NYdb; extern const std::string TestRSAPrivateKeyContent; @@ -108,6 +111,19 @@ struct TTestConfigFile : public TJsonFiller { }; Y_UNIT_TEST_SUITE(TestTokenExchange) { + bool WaitRequest(TTestTokenExchangeServer& server, TDuration timeout) { + const auto deadline = TInstant::Now() + timeout; + do { + bool received = false; + server.WithLock([&] { received = server.Check.InputParams.has_value(); }); + if (received) { + return true; + } + Sleep(TDuration::MilliSeconds(10)); + } while (TInstant::Now() < deadline); + return false; + } + void Exchanges(bool fromConfig) { TTestTokenExchangeServer server; server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); @@ -405,28 +421,6 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { server.Check.ExpectedInputParams.erase("scope"); - server.Check.ExpectedErrorPart = "can not connect to"; - server.Check.ExpectRequest = false; - if (fromConfig) { - server.RunFromConfig( - TTestConfigFile() - .Field("token-endpoint", "https://localhost:42/aaa") - .SubMap("subject-credentials") - .Field("type", "Fixed") - .Field("token", "test_token") - .Field("token-type", "test_token_type") - .Build() - .Build() - ); - } else { - server.Run( - TOauth2TokenExchangeParams() - .TokenEndpoint("https://localhost:42/aaa") - .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type")) - ); - } - server.Check.ExpectRequest = true; - // parsing response server.Check.StatusCode = HTTP_FORBIDDEN; server.Check.Response = R"(not json)"; @@ -488,16 +482,13 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { server.WithLock( [&]() { + server.Check.Reset(); server.Check.Response = R"({"access_token": "token_2", "token_type": "bearer", "expires_in": 1})"; } ); - Sleep(TDuration::Seconds(1)); - server.Run( - [&]() { - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_2"); - } - ); + UNIT_ASSERT(WaitRequest(server, TDuration::Seconds(10))); + UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_2"); } Y_UNIT_TEST(UpdatesToken) { @@ -540,138 +531,44 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer the_only_token"); } - Y_UNIT_TEST(UpdatesTokenInBackgroud) { - TCredentialsProviderFactoryPtr factory; - TInstant startTime; - - TTestTokenExchangeServer server; - server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); - server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); - server.Check.ExpectedInputParams.emplace("actor_token", "test_token"); - server.Check.ExpectedInputParams.emplace("actor_token_type", "test_token_type"); - - for (int i = 0; i < 2; ++i) { - server.WithLock( - [&]() { - server.Check.Response = R"({"access_token": "token_1", "token_type": "bearer", "expires_in": 2})"; - } - ); - if (!factory) { - server.Run( - [&]() { - factory = CreateOauth2TokenExchangeCredentialsProviderFactory( - TOauth2TokenExchangeParams() - .TokenEndpoint(server.GetEndpoint()) - .ActorTokenSource(CreateFixedTokenSource("test_token", "test_token_type"))); - startTime = TInstant::Now(); - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); - } - ); - } - - server.WithLock( - [&]() { - server.Check.Reset(); - if (i == 0) { - server.Check.Response = R"({"access_token": "token_2", "token_type": "bearer", "expires_in": 2})"; - } else { - server.Check.Response = R"({"access_token": "token_3", "token_type": "bearer", "expires_in": 2})"; - } - } - ); - - SleepUntil(startTime + TDuration::Seconds(1) + TDuration::MilliSeconds(5)); - const std::string token = factory->CreateProvider()->GetAuthInfo(); - TInstant halfTimeTokenValid = TInstant::Now(); - if (halfTimeTokenValid < startTime + TDuration::Seconds(2)) { // valid => got cached token, but async update must be run after half time token is valid - if (i == 0) { - UNIT_ASSERT_VALUES_EQUAL(token, "Bearer token_1"); - } else { - UNIT_ASSERT_VALUES_EQUAL(token, "Bearer token_2"); - } - do { - Sleep(TDuration::MilliSeconds(10)); - bool gotRequest = false; - server.WithLock( - [&]() { - if (server.Check.InputParams) { // InputParams are created => got the request - gotRequest = true; - } - } - ); - if (gotRequest) { - startTime = TInstant::Now(); // for second iteration - break; - } - } while (TInstant::Now() <= startTime + TDuration::Seconds(30)); - server.CheckExpectations(); - server.WithLock( - [&]() { - server.Check.Reset(); - server.Check.Response = R"(invalid response)"; // update must finish asyncronously - } - ); - Sleep(TDuration::MilliSeconds(500)); // After the request is got, it takes some time to get updated token - if (i == 0) { // Finally check that we got updated token - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_2"); - } else { - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_3"); - } - Cerr << "Checked backgroud update on " << i << " iteration" << Endl; - } - } - } - Y_UNIT_TEST(UpdatesTokenAndRetriesErrors) { TCredentialsProviderFactoryPtr factory; + auto facility = CreateSimpleCoreFacility(); + TCredentialsProviderPtr provider; TTestTokenExchangeServer server; server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); server.Check.ExpectedInputParams.emplace("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"); server.Check.ExpectedInputParams.emplace("subject_token", "test_token"); server.Check.ExpectedInputParams.emplace("subject_token_type", "test_token_type"); - server.Check.Response = R"({"access_token": "token_1", "token_type": "bearer", "expires_in": 6})"; + server.Check.Response = R"({"access_token": "token_1", "token_type": "bearer", "expires_in": 2})"; server.Run( [&]() { factory = CreateOauth2TokenExchangeCredentialsProviderFactory( TOauth2TokenExchangeParams() .TokenEndpoint(server.GetEndpoint()) .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type"))); - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); + provider = factory->CreateProvider(facility); + UNIT_ASSERT_VALUES_EQUAL(provider->GetAuthInfo(), "Bearer token_1"); } ); server.WithLock( [&]() { server.Check.Reset(); - server.Check.StatusCode = HTTP_BAD_REQUEST; // all errors are temporary, because the first attempt is always successful (in constructor) + server.Check.StatusCode = HTTP_INTERNAL_SERVER_ERROR; server.Check.Response = R"({"error": "tmp", "error_description": "temporary error"})"; } ); - Sleep(TDuration::Seconds(3) + TDuration::MilliSeconds(5)); - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); - - auto waitRequest = [&](TDuration howLong) { - TInstant startTime = TInstant::Now(); - bool gotRequest = false; - do { - Sleep(TDuration::MilliSeconds(10)); - server.WithLock( - [&]() { - if (server.Check.InputParams) { // InputParams are created => got the request - gotRequest = true; - } - } - ); - if (gotRequest) { - break; - } - } while (TInstant::Now() <= startTime + howLong); - return gotRequest; - }; - - UNIT_ASSERT(waitRequest(TDuration::Seconds(30))); + UNIT_ASSERT(WaitRequest(server, TDuration::Seconds(30))); + auto future = provider->GetAuthInfoAsync(); + UNIT_ASSERT(!future.IsReady()); + auto released = std::make_shared>(); + future.Subscribe([provider = std::move(provider), released](const auto&) mutable { + provider.reset(); + released->set_value(); + }); server.WithLock( [&]() { @@ -681,24 +578,15 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { } ); - UNIT_ASSERT(waitRequest(TDuration::Seconds(10))); - Sleep(TDuration::MilliSeconds(500)); // After the request is got, it takes some time to get updated token - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_2"); - - server.WithLock( - [&]() { - server.Check.Reset(); - server.Check.StatusCode = HTTP_INTERNAL_SERVER_ERROR; - server.Check.Response = R"({})"; - } - ); - - Sleep(TDuration::Seconds(2)); - UNIT_ASSERT_EXCEPTION(factory->CreateProvider()->GetAuthInfo(), std::runtime_error); + UNIT_ASSERT(future.Wait(TDuration::Seconds(10))); + UNIT_ASSERT_VALUES_EQUAL(future.GetValue(), "Bearer token_2"); + UNIT_ASSERT(released->get_future().wait_for(std::chrono::seconds(10)) == std::future_status::ready); } Y_UNIT_TEST(ShutdownWhileRefreshingToken) { TCredentialsProviderFactoryPtr factory; + auto facility = CreateSimpleCoreFacility(); + TCredentialsProviderPtr provider; TTestTokenExchangeServer server; server.Check.ExpectedInputParams.emplace("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); @@ -713,7 +601,8 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { TOauth2TokenExchangeParams() .TokenEndpoint(server.GetEndpoint()) .SubjectTokenSource(CreateFixedTokenSource("test_token", "test_token_type"))); - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); + provider = factory->CreateProvider(facility); + UNIT_ASSERT_VALUES_EQUAL(provider->GetAuthInfo(), "Bearer token_1"); } ); @@ -725,14 +614,16 @@ Y_UNIT_TEST_SUITE(TestTokenExchange) { } ); - Sleep(TDuration::Seconds(3) + TDuration::MilliSeconds(5)); - - UNIT_ASSERT_VALUES_EQUAL(factory->CreateProvider()->GetAuthInfo(), "Bearer token_1"); + UNIT_ASSERT(WaitRequest(server, TDuration::Seconds(30))); + auto future = provider->GetAuthInfoAsync(); + UNIT_ASSERT(!future.IsReady()); const TInstant shutdownStart = TInstant::Now(); - factory = nullptr; + provider = nullptr; const TInstant shutdownStop = TInstant::Now(); - Cerr << "Shutdown: " << (shutdownStop - shutdownStart) << Endl; + UNIT_ASSERT(shutdownStop - shutdownStart < TDuration::Seconds(1)); + UNIT_ASSERT(future.IsReady()); + UNIT_ASSERT_EXCEPTION(future.GetValue(), yexception); } Y_UNIT_TEST(ExchangesFromFileConfig) { diff --git a/tests/unit/client/oauth2_token_exchange/helpers/test_token_exchange_server.cpp b/tests/unit/client/oauth2_token_exchange/helpers/test_token_exchange_server.cpp index 1e745f8e61..26da1c120b 100644 --- a/tests/unit/client/oauth2_token_exchange/helpers/test_token_exchange_server.cpp +++ b/tests/unit/client/oauth2_token_exchange/helpers/test_token_exchange_server.cpp @@ -17,9 +17,7 @@ void TTestTokenExchangeServer::Run(const NYdb::TOauth2TokenExchangeParams& param std::string token; Run([&]() { auto factory = CreateOauth2TokenExchangeCredentialsProviderFactory(params); - if (!expectedToken.empty()) { - token = factory->CreateProvider()->GetAuthInfo(); - } + token = factory->CreateProvider()->GetAuthInfo(); }, checkExpectations); @@ -32,9 +30,7 @@ void TTestTokenExchangeServer::RunFromConfig(const std::string& fileName, const std::string token; Run([&]() { auto factory = NYdb::CreateOauth2TokenExchangeFileCredentialsProviderFactory(fileName, explicitTokenEndpoint); - if (!expectedToken.empty()) { - token = factory->CreateProvider()->GetAuthInfo(); - } + token = factory->CreateProvider()->GetAuthInfo(); }, checkExpectations); From 079cd6ed3f54374d4205b00933c74351446c0d37 Mon Sep 17 00:00:00 2001 From: flown4qqqq Date: Tue, 28 Jul 2026 08:50:24 +0000 Subject: [PATCH 41/56] Set not null state rejected (#46759) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_table.proto | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 13db073620..b41c4d1957 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -3fd87611fbcea7809d690ee9062009fb33924e4b +2c59aea8ec28e4aaae726c29cb7957a023e9b7ed diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index 071eace64f..5a40929e7f 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -488,6 +488,7 @@ message SetNotNullState { STATE_APPLYING = 3; STATE_DONE = 4; STATE_CANCELLED = 5; + STATE_REJECTED = 6; } } From 13d8eb91c8885b451110f84bb6809ecc9a4e4429 Mon Sep 17 00:00:00 2001 From: mregrock Date: Tue, 28 Jul 2026 08:50:34 +0000 Subject: [PATCH 42/56] Fix some problems with new test shard api (#47463) --- .github/last_commit.txt | 2 +- src/api/protos/ydb_test_shard.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index b41c4d1957..e3661ce6fa 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -2c59aea8ec28e4aaae726c29cb7957a023e9b7ed +0ef2f431b08e0eb983b8e2c576bfc3788e0799ce diff --git a/src/api/protos/ydb_test_shard.proto b/src/api/protos/ydb_test_shard.proto index f42dd882f5..314f48b8ba 100644 --- a/src/api/protos/ydb_test_shard.proto +++ b/src/api/protos/ydb_test_shard.proto @@ -12,7 +12,7 @@ message CreateTestShardSetRequest { // Path to the TestShardSet object to create string path = 2; - // Storage pool names for tablet channels (optional) + // Storage pool kinds for tablet channels (optional) // If not provided, uses first 3 storage pools from the database domain repeated string channels = 3; From 7f1a19fd7b085ba48fdd29881e66c8cad9b7298c Mon Sep 17 00:00:00 2001 From: Sergey M Date: Tue, 28 Jul 2026 08:50:43 +0000 Subject: [PATCH 43/56] Fix tokenizer autoreview issues (#47453) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/table/table.h | 1 + src/client/table/out.cpp | 3 +++ src/client/table/table.cpp | 4 ++++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index e3661ce6fa..fdd31d87cd 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -0ef2f431b08e0eb983b8e2c576bfc3788e0799ce +f0572f3a8a1ad3a8ef33e004d8b0c1248df2ed05 diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index 8af94679d7..979dd631a9 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -391,6 +391,7 @@ struct TFulltextIndexSettings { Whitespace, Standard, Keyword, + Alphanumeric, }; struct TAnalyzers { diff --git a/src/client/table/out.cpp b/src/client/table/out.cpp index 7d29674d4f..59ac9b1188 100644 --- a/src/client/table/out.cpp +++ b/src/client/table/out.cpp @@ -97,6 +97,9 @@ Y_DECLARE_OUT_SPEC(, NYdb::NTable::TFulltextIndexSettings::ETokenizer, stream, v case NYdb::NTable::TFulltextIndexSettings::ETokenizer::Keyword: stream << "keyword"; break; + case NYdb::NTable::TFulltextIndexSettings::ETokenizer::Alphanumeric: + stream << "alphanumeric"; + break; case NYdb::NTable::TFulltextIndexSettings::ETokenizer::Unspecified: stream << "unspecified"; break; diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index 2d32819202..3233f94305 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -3040,6 +3040,8 @@ TFulltextIndexSettings::TAnalyzers FromProto(const Ydb::Table::FulltextIndexSett return ETokenizer::Standard; case Ydb::Table::FulltextIndexSettings::KEYWORD: return ETokenizer::Keyword; + case Ydb::Table::FulltextIndexSettings::ALPHANUMERIC: + return ETokenizer::Alphanumeric; default: return ETokenizer::Unspecified; } @@ -3093,6 +3095,8 @@ Ydb::Table::FulltextIndexSettings::Analyzers ToProto(const TFulltextIndexSetting return Ydb::Table::FulltextIndexSettings::STANDARD; case ETokenizer::Keyword: return Ydb::Table::FulltextIndexSettings::KEYWORD; + case ETokenizer::Alphanumeric: + return Ydb::Table::FulltextIndexSettings::ALPHANUMERIC; case ETokenizer::Unspecified: return Ydb::Table::FulltextIndexSettings::TOKENIZER_UNSPECIFIED; } From 958b22838600e0b955aa69906e62df89f0d34500 Mon Sep 17 00:00:00 2001 From: Nikolay Shestakov Date: Tue, 28 Jul 2026 08:50:53 +0000 Subject: [PATCH 44/56] Fix ASAN leak in PersQueue read session teardown (#47727) --- .github/last_commit.txt | 2 +- .../persqueue_public/impl/read_session.cpp | 57 ++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index fdd31d87cd..eb2b767022 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -f0572f3a8a1ad3a8ef33e004d8b0c1248df2ed05 +a47655f05be85017a54e7f33ba134d529c05638c diff --git a/src/client/persqueue_public/impl/read_session.cpp b/src/client/persqueue_public/impl/read_session.cpp index 9323a353e1..8857265735 100644 --- a/src/client/persqueue_public/impl/read_session.cpp +++ b/src/client/persqueue_public/impl/read_session.cpp @@ -59,7 +59,28 @@ TReadSession::~TReadSession() { } Abort(); + + std::vector sessions; + { + std::lock_guard guard(Lock); + sessions.reserve(ClusterSessions.size()); + for (auto& [_, sessionInfo] : ClusterSessions) { + if (sessionInfo.Session) { + sessions.push_back(sessionInfo.Session); + } + } + } + + const TInstant closeDeadline = TInstant::Now() + TDuration::Seconds(5); + for (const auto& session : sessions) { + if (!session->WaitAllDecompressionTasks(closeDeadline)) { + LOG_LAZY(Log, TLOG_WARNING, GetLogPrefix() << "Some decompression tasks are still running after read session destroy timeout"); + } + } ClearAllEvents(); + for (const auto& session : sessions) { + session->ClearAllPartitionStreamEvents(); + } for (const auto& ctx : CbContexts) { ctx->Cancel(); @@ -352,6 +373,12 @@ bool TReadSession::Close(TDuration timeout) { // Log final counters. CountersLogger->Stop(); } + { + std::lock_guard guard(Lock); + if (DumpCountersContext) { + DumpCountersContext->Cancel(); + } + } std::vector sessions; NThreading::TPromise promise = NThreading::NewPromise(); @@ -362,6 +389,8 @@ bool TReadSession::Close(TDuration timeout) { } }; + std::vector cbContextsToCancel; + std::shared_ptr> dumpCountersContextToCancel; TDeferredActions deferred; { std::lock_guard guard(Lock); @@ -395,9 +424,11 @@ bool TReadSession::Close(TDuration timeout) { auto timeoutContext = Connections->CreateContext(); if (!timeoutContext) { + std::lock_guard guard(Lock); AbortImpl(EStatus::ABORTED, DRIVER_IS_STOPPING_DESCRIPTION, deferred); return false; } + const TInstant closeDeadline = TInstant::Now() + timeout; Connections->ScheduleCallback(timeout, std::move(timeoutCallback), timeoutContext); @@ -422,8 +453,30 @@ bool TReadSession::Close(TDuration timeout) { EventsQueue->Close(TSessionClosedEvent(EStatus::TIMEOUT, std::move(issues)), deferred); } - std::lock_guard guard(Lock); - Aborting = true; // Set abort flag for doing nothing on destructor. + { + std::lock_guard guard(Lock); + Aborting = true; // Set abort flag for doing nothing on destructor. + cbContextsToCancel = CbContexts; + dumpCountersContextToCancel = DumpCountersContext; + } + + for (const auto& session : sessions) { + if (!session->WaitAllDecompressionTasks(closeDeadline)) { + LOG_LAZY(Log, TLOG_WARNING, GetLogPrefix() << "Some decompression tasks are still running after read session close timeout"); + } + } + ClearAllEvents(); + for (const auto& session : sessions) { + session->ClearAllPartitionStreamEvents(); + } + + for (const auto& ctx : cbContextsToCancel) { + ctx->Cancel(); + } + if (dumpCountersContextToCancel) { + dumpCountersContextToCancel->Cancel(); + } + return result; } From ed1393f015d92d3dbf3f7f8d03381b28c79f1314 Mon Sep 17 00:00:00 2001 From: Alek5andr-Kotov Date: Tue, 28 Jul 2026 08:51:03 +0000 Subject: [PATCH 45/56] Fix flaky BasicUsage.Producer_WriteManyMessages (UAF in read helper) (#47855) --- .github/last_commit.txt | 2 +- src/client/topic/ut/basic_usage_ut.cpp | 35 +++++++++++++++++++------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index eb2b767022..01f5aca799 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -a47655f05be85017a54e7f33ba134d529c05638c +2725dd489f683062064e77b8180ee35535b65f15 diff --git a/src/client/topic/ut/basic_usage_ut.cpp b/src/client/topic/ut/basic_usage_ut.cpp index 6d34cd7c1f..266fcba33e 100644 --- a/src/client/topic/ut/basic_usage_ut.cpp +++ b/src/client/topic/ut/basic_usage_ut.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -81,9 +82,19 @@ void ReadMessagesAndAssertOrderedBySeqNo(TTopicClient& client, ui64 SeqNo; std::string Data; }; - std::vector messages; - messages.reserve(expectedCount); - NThreading::TPromise donePromise = NThreading::NewPromise(); + // Handlers may still run after Close(); keep shared ownership so late callbacks are safe. + struct TState { + std::mutex Lock; + std::vector Messages; + NThreading::TPromise DonePromise = NThreading::NewPromise(); + + std::vector CopyMessages() { + std::lock_guard guard(Lock); + return Messages; + } + }; + auto state = std::make_shared(); + state->Messages.reserve(expectedCount); TTopicReadSettings topicSettings(topicPath); topicSettings.ReadFromTimestamp(TInstant::Zero()); @@ -93,25 +104,31 @@ void ReadMessagesAndAssertOrderedBySeqNo(TTopicClient& client, .AutoPartitioningSupport(true) .AppendTopics(topicSettings); - readSettings.EventHandlers_.SimpleDataHandlers([&](TReadSessionEvent::TDataReceivedEvent& ev) { + readSettings.EventHandlers_.SimpleDataHandlers([state, expectedCount](TReadSessionEvent::TDataReceivedEvent& ev) { + std::lock_guard guard(state->Lock); for (auto& msg : ev.GetMessages()) { - messages.push_back(TMessageInfo{ + if (state->Messages.size() >= expectedCount) { + break; + } + state->Messages.push_back(TMessageInfo{ msg.GetPartitionSession()->GetPartitionId(), TString(msg.GetProducerId()), msg.GetSeqNo(), TString(msg.GetData()), }); } - if (messages.size() >= expectedCount) { - donePromise.SetValue(); + if (state->Messages.size() >= expectedCount) { + state->DonePromise.TrySetValue(); } }, true); auto readSession = client.CreateReadSession(readSettings); - UNIT_ASSERT_C(donePromise.GetFuture().Wait(timeout), - "Expected to read " << expectedCount << " messages within " << timeout << ", got " << messages.size()); + UNIT_ASSERT_C(state->DonePromise.GetFuture().Wait(timeout), + "Expected to read " << expectedCount << " messages within " << timeout); readSession->Close(TDuration::Seconds(5)); + const auto messages = state->CopyMessages(); + UNIT_ASSERT_VALUES_EQUAL_C(messages.size(), expectedCount, "Read message count mismatch: got " << messages.size() << ", expected " << expectedCount); From 88fbd0caf48ec59f00709887d4aed99f3597d0a2 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:51:13 +0000 Subject: [PATCH 46/56] MakeFuture singletone teardown fix (#47909) --- .github/last_commit.txt | 2 +- src/client/impl/internal/db_driver_state/state.cpp | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 01f5aca799..82ef5f353d 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -2725dd489f683062064e77b8180ee35535b65f15 +d79e0dd998ae37fac57ae1361dc888e7550f0191 diff --git a/src/client/impl/internal/db_driver_state/state.cpp b/src/client/impl/internal/db_driver_state/state.cpp index 357c151819..9ad2ad2e8c 100644 --- a/src/client/impl/internal/db_driver_state/state.cpp +++ b/src/client/impl/internal/db_driver_state/state.cpp @@ -331,7 +331,12 @@ NThreading::TFuture TDbDriverStateTracker::SendNotification( } } if (results.empty()) { - return NThreading::MakeFuture(); + // MakeFuture() uses a process-wide singleton that may already be + // destroyed when driver shutdown is triggered by another singleton. + auto promise = NThreading::NewPromise(); + auto future = promise.GetFuture(); + promise.SetValue(); + return future; } return NThreading::WaitExceptionOrAll(results); } From 42c86fc72053c5e4ba9c203476b8e39b6db1f24a Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:51:23 +0000 Subject: [PATCH 47/56] Fix invalid root certificate false positive (#47930) --- .github/last_commit.txt | 2 +- src/library/grpc/client/grpc_common.cpp | 31 ++++++------------- tests/unit/client/driver/driver_ut.cpp | 41 +++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 22 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 82ef5f353d..97f86fae65 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -d79e0dd998ae37fac57ae1361dc888e7550f0191 +d67b64e86f1ba7b821f0ef67c01a3d2db4ecf21c diff --git a/src/library/grpc/client/grpc_common.cpp b/src/library/grpc/client/grpc_common.cpp index 23f4dcbc2c..3a2d7efa28 100644 --- a/src/library/grpc/client/grpc_common.cpp +++ b/src/library/grpc/client/grpc_common.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -55,36 +54,26 @@ bool ValidateRootCertificates(const std::string& pemRootCerts, std::string& erro size_t certsParsed = 0; while (true) { std::unique_ptr cert( - PEM_read_bio_X509(rootCertsBio, nullptr, nullptr, nullptr), + PEM_read_bio_X509_AUX(rootCertsBio, nullptr, nullptr, nullptr), &X509_free); if (!cert) { - const unsigned long errorCode = ERR_peek_last_error(); - if (errorCode == 0 || ERR_GET_REASON(errorCode) == PEM_R_NO_START_LINE) { - ERR_clear_error(); - break; + if (certsParsed == 0) { + errorMessage = "root CA PEM: failed to parse certificate #1: " + DrainOpenSslErrors(); + return false; } - errorMessage = "root CA PEM: " + DrainOpenSslErrors(); - return false; - } - std::unique_ptr basicConstraints( - static_cast(X509_get_ext_d2i(cert.get(), NID_basic_constraints, nullptr, nullptr)), - &BASIC_CONSTRAINTS_free); - const auto isCaCert = basicConstraints && basicConstraints->ca; - - if (!isCaCert) { + // gRPC treats a read error as the end of the bundle and uses all + // certificates parsed before it. ERR_clear_error(); - errorMessage = "root CA PEM: certificate is not a CA (BasicConstraints)"; - return false; + break; } + + // Match gRPC trust store semantics: with X509_V_FLAG_PARTIAL_CHAIN an + // explicitly trusted non-CA certificate may also be a trust anchor. ERR_clear_error(); ++certsParsed; } - if (certsParsed == 0) { - errorMessage = "root CA PEM: no certificates parsed"; - return false; - } return true; } diff --git a/tests/unit/client/driver/driver_ut.cpp b/tests/unit/client/driver/driver_ut.cpp index 1e1154c2e3..54ce45af0f 100644 --- a/tests/unit/client/driver/driver_ut.cpp +++ b/tests/unit/client/driver/driver_ut.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,17 @@ using namespace NYdb::NTable; namespace { + constexpr const char LegacyV1Certificate[] = R"(-----BEGIN CERTIFICATE----- +MIIBbTCCARMCFBthJdWIg/H6ITeelffnCYoK8fDFMAoGCCqGSM49BAMCMDkxCzAJ +BgNVBAYTAlJVMQwwCgYDVQQKDANZREIxHDAaBgNVBAMME0xlZ2FjeSBUZXN0IFJv +b3QgQ0EwHhcNMjYwNzI3MDk0NDU2WhcNMzYwNzI0MDk0NDU2WjA5MQswCQYDVQQG +EwJSVTEMMAoGA1UECgwDWURCMRwwGgYDVQQDDBNMZWdhY3kgVGVzdCBSb290IENB +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4zlS2ha5hOd20QJEh17FP/mjkzsO +PmwF7iY9zJ0HILwBjqxJSCGnNMMdT+A2d+Nry6de3WC6RkR72HTe6gffuTAKBggq +hkjOPQQDAgNIADBFAiEA/0rBKAconmtFcliTZ0i9HzIkQeG+E/zVMiUvlhwpylYC +IGfPhGBVwOMnr+uhwtpj4PAOIrlOQD/fBsaRtYuBRdg2 +-----END CERTIFICATE-----)"; + std::string ReadBuildInfo(grpc::ServerContext* context) { const auto& metadata = context->client_metadata(); const auto it = metadata.find(YDB_SDK_BUILD_INFO_HEADER); @@ -237,6 +249,35 @@ Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) { UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::TRANSPORT_UNAVAILABLE); UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Client TLS credentials validation failed"); UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "root CA PEM:"); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "failed to parse certificate #1"); + } + + Y_UNIT_TEST(LegacyV1TrustAnchorPassesValidation) { + auto driver = TDriver( + TDriverConfig() + .SetEndpoint("localhost:100") + .UseSecureConnection(LegacyV1Certificate)); + auto client = NTable::TTableClient(driver); + + auto result = client.CreateSession().GetValueSync(); + auto issues = result.GetIssues().ToString(); + + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::TRANSPORT_UNAVAILABLE); + UNIT_ASSERT(issues.find("Client TLS credentials validation failed") == std::string::npos); + } + + Y_UNIT_TEST(MalformedCertificateAfterValidRootPassesValidation) { + const std::string rootBundle = std::string(LegacyV1Certificate) + R"( +-----BEGIN CERTIFICATE----- +not-base64 +-----END CERTIFICATE-----)"; + grpc::SslCredentialsOptions sslOptions{ + .pem_root_certs = NYdb::TStringType{rootBundle}, + }; + std::string validationDetail; + + UNIT_ASSERT(NYdbGrpc::ValidateTlsCredentials(sslOptions, validationDetail)); + UNIT_ASSERT(validationDetail.empty()); } Y_UNIT_TEST(EmptyRootCertificateWithoutClientCredentialsKeepsBehavior) { From ee904c512d808861566b144ceb7f69957697b810 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:51:32 +0000 Subject: [PATCH 48/56] release sdk v3.21.0 (#48074) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 97f86fae65..747fec7e66 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -d67b64e86f1ba7b821f0ef67c01a3d2db4ecf21c +1e3f62319678deefbb65acd4a3894eb0badcbb0d diff --git a/CHANGELOG.md b/CHANGELOG.md index 774b189382..9560323d42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## v3.21.0 + * Fixed Query SDK `CreateSession` metrics being recorded when reusing a session from the pool. * Added `TQueryClient::DeleteSession` to explicitly delete a query session by session id. @@ -8,7 +10,7 @@ * Added a distributed lock primitive based on the coordination service, which implements basic_lockable concept. -# v3.20.0 +## v3.20.0 * Added automatic retries for unary methods of table and query clients(ExecuteQuery, ExecuteScript, BulkUpsert, ReadRows). From ae8fb6187153ffd6b83429519cf8891c60a82576 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:51:33 +0000 Subject: [PATCH 49/56] Update import generation: 44 --- .github/import_generation.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/import_generation.txt b/.github/import_generation.txt index c739b42c4d..ea90ee3198 100644 --- a/.github/import_generation.txt +++ b/.github/import_generation.txt @@ -1 +1 @@ -44 +45 From b5467e3756e9ed2ae2375fb682402cd3506d2cba Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Tue, 28 Jul 2026 14:13:35 +0300 Subject: [PATCH 50/56] update vendored grpc versions --- .devcontainer/Dockerfile | 7 ++++--- .github/actions/prepare_vm/action.yaml | 19 ++++++++++--------- README.md | 24 +++++++++++++++++------- cmake/external_libs.cmake | 2 +- src/client/CMakeLists.txt | 1 + src/client/coordination/CMakeLists.txt | 1 + src/client/test_shard/CMakeLists.txt | 16 ++++++++++++++++ src/client/types/CMakeLists.txt | 8 +++++++- tests/slo_workloads/Dockerfile | 7 ++++--- tests/slo_workloads/Dockerfile.userver | 7 ++++--- tests/unit/client/CMakeLists.txt | 8 +++++++- 11 files changed, 72 insertions(+), 28 deletions(-) create mode 100644 src/client/test_shard/CMakeLists.txt diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a30aaa6147..f360923063 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -44,17 +44,18 @@ RUN wget -O abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz https://github.com/abseil/ab cmake --install . --config Release --prefix ${ABSEIL_CPP_INSTALL_DIR} # Install protobuf -ENV PROTOBUF_VERSION=3.21.12 +ENV PROTOBUF_VERSION=25.0 ENV PROTOBUF_INSTALL_DIR=~/ydb_deps/protobuf RUN wget -O protobuf-${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \ tar -xvzf protobuf-${PROTOBUF_VERSION}.tar.gz && cd protobuf-${PROTOBUF_VERSION} && \ mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ + cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ cmake --build . --config Release && \ cmake --install . --config Release --prefix ${PROTOBUF_INSTALL_DIR} # Install grpc -ENV GRPC_VERSION=1.54.3 +ENV GRPC_VERSION=1.60.2 ENV GRPC_INSTALL_DIR=~/ydb_deps/grpc RUN wget -O grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v${GRPC_VERSION}.tar.gz && \ tar -xvzf grpc-${GRPC_VERSION}.tar.gz && cd grpc-${GRPC_VERSION} && \ diff --git a/.github/actions/prepare_vm/action.yaml b/.github/actions/prepare_vm/action.yaml index 35819ab630..ccf3f08fc1 100644 --- a/.github/actions/prepare_vm/action.yaml +++ b/.github/actions/prepare_vm/action.yaml @@ -54,20 +54,21 @@ runs: cd ../../ # Install protobuf - wget -O protobuf-3.21.12.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz - tar -xvzf protobuf-3.21.12.tar.gz - cd protobuf-3.21.12 + wget -O protobuf-25.0.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v25.0.tar.gz + tar -xvzf protobuf-25.0.tar.gz + cd protobuf-25.0 mkdir build && cd build - cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON .. + cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="${HOME}/ydb_deps/absl" -DCMAKE_BUILD_TYPE=Release \ + -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. cmake --build . --config Release cmake --install . --config Release --prefix ~/ydb_deps/protobuf cd ../../ # Install gRPC - wget -O grpc-1.54.3.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz - tar -xvzf grpc-1.54.3.tar.gz && cd grpc-1.54.3 + wget -O grpc-1.60.2.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.60.2.tar.gz + tar -xvzf grpc-1.60.2.tar.gz && cd grpc-1.60.2 mkdir build && cd build - cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="~/ydb_deps/absl;~/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ + cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="${HOME}/ydb_deps/absl;${HOME}/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ @@ -88,6 +89,6 @@ runs: # Clean up ccache -s - sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-3.21.12.tar.gz grpc-1.54.3.tar.gz \ + sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-25.0.tar.gz grpc-1.60.2.tar.gz \ brotli-1.1.0.tar.gz abseil-cpp-20230802.0 \ - protobuf-3.21.12 grpc-1.54.3 brotli-1.1.0 + protobuf-25.0 grpc-1.60.2 brotli-1.1.0 diff --git a/README.md b/README.md index affad1c844..611251d59a 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,15 @@ ### Install dependencies +The standalone dependency bundle uses gRPC 1.60.2 to match the imported YDB sources. +Its protobuf and Abseil pins match the dependency set published with that gRPC release: + +| Dependency | Version | +|------------|---------| +| Abseil | 20230802.0 | +| protobuf | 25.0 | +| gRPC | 1.60.2 | + ```bash sudo apt-get -y update sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ @@ -65,20 +74,21 @@ cmake --install . --config Release --prefix ~/ydb_deps/absl cd ../../ # Install protobuf -wget -O protobuf-3.21.12.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz -tar -xvzf protobuf-3.21.12.tar.gz -cd protobuf-3.21.12 +wget -O protobuf-25.0.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v25.0.tar.gz +tar -xvzf protobuf-25.0.tar.gz +cd protobuf-25.0 mkdir build && cd build -cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON .. +cmake -G Ninja -DCMAKE_PREFIX_PATH="$HOME/ydb_deps/absl" -DCMAKE_BUILD_TYPE=Release \ + -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. cmake --build . --config Release cmake --install . --config Release --prefix ~/ydb_deps/protobuf cd ../../ # Install gRPC -wget -O grpc-1.54.3.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz -tar -xvzf grpc-1.54.3.tar.gz && cd grpc-1.54.3 +wget -O grpc-1.60.2.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.60.2.tar.gz +tar -xvzf grpc-1.60.2.tar.gz && cd grpc-1.60.2 mkdir build && cd build -cmake -G Ninja -DCMAKE_PREFIX_PATH="~/ydb_deps/absl;~/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ +cmake -G Ninja -DCMAKE_PREFIX_PATH="$HOME/ydb_deps/absl;$HOME/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ diff --git a/cmake/external_libs.cmake b/cmake/external_libs.cmake index 64723763a8..636321c7d6 100644 --- a/cmake/external_libs.cmake +++ b/cmake/external_libs.cmake @@ -1,7 +1,7 @@ find_package(IDN REQUIRED) find_package(Iconv REQUIRED) find_package(OpenSSL REQUIRED) -find_package(Protobuf REQUIRED) +find_package(Protobuf CONFIG REQUIRED) find_package(gRPC 1.41.0 REQUIRED) find_package(ZLIB REQUIRED) find_package(xxHash REQUIRED) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index e9b85f044e..fcf0a1b81c 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -27,6 +27,7 @@ add_subdirectory(scheme) add_subdirectory(secret) add_subdirectory(ss_tasks) add_subdirectory(table) +add_subdirectory(test_shard) add_subdirectory(topic) add_subdirectory(trace) add_subdirectory(types) diff --git a/src/client/coordination/CMakeLists.txt b/src/client/coordination/CMakeLists.txt index e8152b6dd7..04c3279513 100644 --- a/src/client/coordination/CMakeLists.txt +++ b/src/client/coordination/CMakeLists.txt @@ -15,6 +15,7 @@ target_link_libraries(client-ydb_coordination PUBLIC target_sources(client-ydb_coordination PRIVATE coordination.cpp + distributed_lock.cpp proto_accessor.cpp ) diff --git a/src/client/test_shard/CMakeLists.txt b/src/client/test_shard/CMakeLists.txt new file mode 100644 index 0000000000..e32fc3e962 --- /dev/null +++ b/src/client/test_shard/CMakeLists.txt @@ -0,0 +1,16 @@ +_ydb_sdk_add_library(client-ydb_test_shard) + +target_link_libraries(client-ydb_test_shard PUBLIC + yutil + api-grpc + impl-internal-make_request + client-ydb_common_client-impl + client-ydb_driver + client-types-status +) + +target_sources(client-ydb_test_shard PRIVATE + test_shard.cpp +) + +_ydb_sdk_make_client_component(TestShard client-ydb_test_shard) diff --git a/src/client/types/CMakeLists.txt b/src/client/types/CMakeLists.txt index d31569ed37..580c9b3da2 100644 --- a/src/client/types/CMakeLists.txt +++ b/src/client/types/CMakeLists.txt @@ -8,8 +8,14 @@ add_subdirectory(status) _ydb_sdk_add_library(client-types) target_sources(client-types PRIVATE - ydb.cpp core_facility/simple_core_facility.cpp + virtual_timestamp.cpp + ydb.cpp +) + +set_property(SOURCE virtual_timestamp.cpp APPEND PROPERTY COMPILE_OPTIONS + "$<$:/FI${YDB_SDK_SOURCE_DIR}/util/stream/str.h>" + "$<$>:-include${YDB_SDK_SOURCE_DIR}/util/stream/str.h>" ) target_link_libraries(client-types PUBLIC diff --git a/tests/slo_workloads/Dockerfile b/tests/slo_workloads/Dockerfile index 5a5c7f2821..2261ab4823 100644 --- a/tests/slo_workloads/Dockerfile +++ b/tests/slo_workloads/Dockerfile @@ -88,18 +88,19 @@ RUN wget $WGET_OPTS -O abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz https://github.co rm -rf abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz abseil-cpp-${ABSEIL_CPP_VERSION} # Install protobuf -ENV PROTOBUF_VERSION=3.21.12 +ENV PROTOBUF_VERSION=25.0 ENV PROTOBUF_INSTALL_DIR=/root/ydb_deps/protobuf RUN wget $WGET_OPTS -O protobuf-${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \ tar -xvzf protobuf-${PROTOBUF_VERSION}.tar.gz && cd protobuf-${PROTOBUF_VERSION} && \ mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ + cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ cmake --build . --config Release && \ cmake --install . --config Release --prefix ${PROTOBUF_INSTALL_DIR} && \ rm -rf protobuf-${PROTOBUF_VERSION}.tar.gz protobuf-${PROTOBUF_VERSION} # Install grpc -ENV GRPC_VERSION=1.54.3 +ENV GRPC_VERSION=1.60.2 ENV GRPC_INSTALL_DIR=/root/ydb_deps/grpc RUN wget $WGET_OPTS -O grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v${GRPC_VERSION}.tar.gz && \ tar -xvzf grpc-${GRPC_VERSION}.tar.gz && cd grpc-${GRPC_VERSION} && \ diff --git a/tests/slo_workloads/Dockerfile.userver b/tests/slo_workloads/Dockerfile.userver index c14b20bb4e..a618debce8 100644 --- a/tests/slo_workloads/Dockerfile.userver +++ b/tests/slo_workloads/Dockerfile.userver @@ -83,18 +83,19 @@ RUN wget $WGET_OPTS -O abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz https://github.co rm -rf abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz abseil-cpp-${ABSEIL_CPP_VERSION} # Install protobuf -ENV PROTOBUF_VERSION=3.21.12 +ENV PROTOBUF_VERSION=25.0 ENV PROTOBUF_INSTALL_DIR=/root/ydb_deps/protobuf RUN wget $WGET_OPTS -O protobuf-${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \ tar -xvzf protobuf-${PROTOBUF_VERSION}.tar.gz && cd protobuf-${PROTOBUF_VERSION} && \ mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ + cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ cmake --build . --config Release && \ cmake --install . --config Release --prefix ${PROTOBUF_INSTALL_DIR} && \ rm -rf protobuf-${PROTOBUF_VERSION}.tar.gz protobuf-${PROTOBUF_VERSION} # Install grpc -ENV GRPC_VERSION=1.54.3 +ENV GRPC_VERSION=1.60.2 ENV GRPC_INSTALL_DIR=/root/ydb_deps/grpc RUN wget $WGET_OPTS -O grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v${GRPC_VERSION}.tar.gz && \ tar -xvzf grpc-${GRPC_VERSION}.tar.gz && cd grpc-${GRPC_VERSION} && \ diff --git a/tests/unit/client/CMakeLists.txt b/tests/unit/client/CMakeLists.txt index b5574c44af..95d7afa228 100644 --- a/tests/unit/client/CMakeLists.txt +++ b/tests/unit/client/CMakeLists.txt @@ -12,6 +12,7 @@ add_ydb_test(NAME client-connection_string_ut GTEST add_ydb_test(NAME client-coordination_ut SOURCES coordination/coordination_ut.cpp + coordination/distributed_lock_ut.cpp LINK_LIBRARIES YDB-CPP-SDK::Coordination api-grpc @@ -174,7 +175,11 @@ add_ydb_test(NAME client-build_info_ut GTEST add_ydb_test(NAME client-query_session_ut SOURCES query/client_session_ut.cpp + query/deferred_session_creation_ut.cpp LINK_LIBRARIES + api-grpc + cpp-testing-common + YDB-CPP-SDK::Query client-ydb_query-impl impl-session library-operation_id @@ -185,6 +190,7 @@ add_ydb_test(NAME client-query_session_ut add_ydb_test(NAME client-query_stats_ut SOURCES query/query_stats_ut.cpp + query/virtual_timestamp_ut.cpp LINK_LIBRARIES YDB-CPP-SDK::Query LABELS @@ -245,4 +251,4 @@ add_ydb_test(NAME client-metric_buffer_ut GTEST client-metrics LABELS unit -) \ No newline at end of file +) From 7dbb49be6a6c1931c50f9698bca6c6421b2d7622 Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Tue, 28 Jul 2026 15:19:12 +0300 Subject: [PATCH 51/56] fix cpp20 clang support --- .github/actions/prepare_vm/action.yaml | 6 +++--- cmake/external_libs.cmake | 5 ++++- cmake/ydb-cpp-sdk-config.cmake.in | 5 ++++- src/client/types/CMakeLists.txt | 2 +- tests/slo_workloads/Dockerfile | 4 ++-- tests/slo_workloads/Dockerfile.userver | 2 +- 6 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/actions/prepare_vm/action.yaml b/.github/actions/prepare_vm/action.yaml index ccf3f08fc1..1fff4159de 100644 --- a/.github/actions/prepare_vm/action.yaml +++ b/.github/actions/prepare_vm/action.yaml @@ -39,9 +39,9 @@ runs: wget https://apt.llvm.org/llvm.sh chmod u+x llvm.sh - sudo ./llvm.sh 16 - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-16 10000 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-16 10000 + sudo ./llvm.sh 17 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 10000 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 10000 # Install abseil-cpp wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz diff --git a/cmake/external_libs.cmake b/cmake/external_libs.cmake index 636321c7d6..469df6000c 100644 --- a/cmake/external_libs.cmake +++ b/cmake/external_libs.cmake @@ -1,7 +1,10 @@ find_package(IDN REQUIRED) find_package(Iconv REQUIRED) find_package(OpenSSL REQUIRED) -find_package(Protobuf CONFIG REQUIRED) +find_package(Protobuf CONFIG QUIET) +if (NOT Protobuf_FOUND) + find_package(Protobuf MODULE REQUIRED) +endif() find_package(gRPC 1.41.0 REQUIRED) find_package(ZLIB REQUIRED) find_package(xxHash REQUIRED) diff --git a/cmake/ydb-cpp-sdk-config.cmake.in b/cmake/ydb-cpp-sdk-config.cmake.in index 5d53b62b28..c92ab92a31 100644 --- a/cmake/ydb-cpp-sdk-config.cmake.in +++ b/cmake/ydb-cpp-sdk-config.cmake.in @@ -45,7 +45,10 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/Modules") find_package(IDN REQUIRED) find_package(Iconv REQUIRED) find_package(OpenSSL REQUIRED) -find_package(Protobuf REQUIRED) +find_package(Protobuf CONFIG QUIET) +if (NOT Protobuf_FOUND) + find_package(Protobuf MODULE REQUIRED) +endif() find_package(gRPC REQUIRED) find_package(ZLIB REQUIRED) find_package(xxHash REQUIRED) diff --git a/src/client/types/CMakeLists.txt b/src/client/types/CMakeLists.txt index 580c9b3da2..d4f14cdab8 100644 --- a/src/client/types/CMakeLists.txt +++ b/src/client/types/CMakeLists.txt @@ -13,7 +13,7 @@ target_sources(client-types PRIVATE ydb.cpp ) -set_property(SOURCE virtual_timestamp.cpp APPEND PROPERTY COMPILE_OPTIONS +target_compile_options(client-types PRIVATE "$<$:/FI${YDB_SDK_SOURCE_DIR}/util/stream/str.h>" "$<$>:-include${YDB_SDK_SOURCE_DIR}/util/stream/str.h>" ) diff --git a/tests/slo_workloads/Dockerfile b/tests/slo_workloads/Dockerfile index 2261ab4823..4a4093627e 100644 --- a/tests/slo_workloads/Dockerfile +++ b/tests/slo_workloads/Dockerfile @@ -61,13 +61,13 @@ RUN wget $WGET_OPTS https://github.com/Kitware/CMake/releases/download/v${CMAKE_ && rm cmake-install.sh # Install LLVM -ENV LLVM_VERSION=16 +ENV LLVM_VERSION=17 RUN wget $WGET_OPTS https://apt.llvm.org/llvm.sh && \ chmod u+x llvm.sh && \ ./llvm.sh ${LLVM_VERSION} && \ rm llvm.sh -# Update alternatives to use clang-16 by default +# Update alternatives to use the selected Clang version by default RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 10000 && \ update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-${LLVM_VERSION} 10000 && \ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM_VERSION} 10000 diff --git a/tests/slo_workloads/Dockerfile.userver b/tests/slo_workloads/Dockerfile.userver index a618debce8..56c7a3657e 100644 --- a/tests/slo_workloads/Dockerfile.userver +++ b/tests/slo_workloads/Dockerfile.userver @@ -58,7 +58,7 @@ RUN wget $WGET_OPTS https://github.com/Kitware/CMake/releases/download/v${CMAKE_ && rm cmake-install.sh # Install LLVM -ENV LLVM_VERSION=16 +ENV LLVM_VERSION=17 RUN wget $WGET_OPTS https://apt.llvm.org/llvm.sh && \ chmod u+x llvm.sh && \ ./llvm.sh ${LLVM_VERSION} && \ From 32f35a5b4ae5a26eaa38f9ba875a12148fad58d6 Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Tue, 28 Jul 2026 16:39:36 +0300 Subject: [PATCH 52/56] fix slo build --- CMakeLists.txt | 1 + cmake/protos_public_headers.txt | 4 +++- cmake/ydb-cpp-sdk-config.cmake.in | 10 ++++++---- tests/slo_workloads/Dockerfile | 4 ++-- tests/slo_workloads/Dockerfile.userver | 4 ++-- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bec181f06..6e9fa4883d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,6 +67,7 @@ if (YDB_CPP_SDK_SLO_USE_INSTALLED_SDK) add_subdirectory(tools) add_subdirectory(contrib/libs) + add_subdirectory(src/library/string_utils/base64) add_subdirectory(library/cpp) add_subdirectory(util) add_subdirectory(tests/slo_workloads) diff --git a/cmake/protos_public_headers.txt b/cmake/protos_public_headers.txt index 8bf8cb074e..468ca6d992 100644 --- a/cmake/protos_public_headers.txt +++ b/cmake/protos_public_headers.txt @@ -1,7 +1,9 @@ src/api/grpc/draft/ydb_datastreams_v1.pb.h src/api/grpc/ydb_topic_v1.pb.h +src/api/protos/annotations/sensitive.pb.h src/api/protos/annotations/validation.pb.h src/api/protos/draft/datastreams.pb.h +src/api/protos/draft/field_transformation.pb.h src/api/protos/ydb_common.pb.h src/api/protos/ydb_federation_discovery.pb.h src/api/protos/ydb_operation.pb.h @@ -17,4 +19,4 @@ src/api/protos/ydb_issue_message.pb.h src/api/protos/ydb_export.pb.h src/api/protos/ydb_coordination.pb.h src/api/protos/ydb_status_codes.pb.h -src/api/protos/draft/ydb_replication.pb.h \ No newline at end of file +src/api/protos/draft/ydb_replication.pb.h diff --git a/cmake/ydb-cpp-sdk-config.cmake.in b/cmake/ydb-cpp-sdk-config.cmake.in index c92ab92a31..c6817d8799 100644 --- a/cmake/ydb-cpp-sdk-config.cmake.in +++ b/cmake/ydb-cpp-sdk-config.cmake.in @@ -69,11 +69,13 @@ endif() if (@YDB_SDK_USE_RAPID_JSON@) find_package(RapidJSON REQUIRED) - add_library(RapidJSON::RapidJSON INTERFACE IMPORTED) + if (NOT TARGET RapidJSON::RapidJSON) + add_library(RapidJSON::RapidJSON INTERFACE IMPORTED) - target_include_directories(RapidJSON::RapidJSON INTERFACE - ${RAPIDJSON_INCLUDE_DIRS} - ) + target_include_directories(RapidJSON::RapidJSON INTERFACE + ${RAPIDJSON_INCLUDE_DIRS} + ) + endif() endif() if (@YDB_SDK_ENABLE_OTEL_METRICS@ OR @YDB_SDK_ENABLE_OTEL_TRACE@) diff --git a/tests/slo_workloads/Dockerfile b/tests/slo_workloads/Dockerfile index 4a4093627e..e51001c12c 100644 --- a/tests/slo_workloads/Dockerfile +++ b/tests/slo_workloads/Dockerfile @@ -133,9 +133,9 @@ ENV BROTLI_INSTALL_DIR=/root/ydb_deps/brotli RUN wget $WGET_OPTS -O brotli-${BROTLI_VERSION}.tar.gz https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz && \ tar -xvzf brotli-${BROTLI_VERSION}.tar.gz && cd brotli-${BROTLI_VERSION} && \ mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \ + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${BROTLI_INSTALL_DIR} .. && \ cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${BROTLI_INSTALL_DIR} && \ + cmake --install . --config Release && \ rm -rf brotli-${BROTLI_VERSION}.tar.gz brotli-${BROTLI_VERSION} # Install jwt-cpp diff --git a/tests/slo_workloads/Dockerfile.userver b/tests/slo_workloads/Dockerfile.userver index 56c7a3657e..42b29517e6 100644 --- a/tests/slo_workloads/Dockerfile.userver +++ b/tests/slo_workloads/Dockerfile.userver @@ -128,9 +128,9 @@ ENV BROTLI_INSTALL_DIR=/root/ydb_deps/brotli RUN wget $WGET_OPTS -O brotli-${BROTLI_VERSION}.tar.gz https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz && \ tar -xvzf brotli-${BROTLI_VERSION}.tar.gz && cd brotli-${BROTLI_VERSION} && \ mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \ + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${BROTLI_INSTALL_DIR} .. && \ cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${BROTLI_INSTALL_DIR} && \ + cmake --install . --config Release && \ rm -rf brotli-${BROTLI_VERSION}.tar.gz brotli-${BROTLI_VERSION} # Install jwt-cpp From 05d8bc276a117947f47953642f44621c45eca37e Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Wed, 29 Jul 2026 13:50:19 +0300 Subject: [PATCH 53/56] fix docs --- README.md | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 611251d59a..13a29bace5 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ### Prerequisites - cmake 3.22+ -- clang 16+ +- clang 17+ - git 2.20+ - ninja 1.10+ - ragel @@ -61,7 +61,7 @@ sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-de wget https://apt.llvm.org/llvm.sh chmod u+x llvm.sh -sudo ./llvm.sh 16 +sudo ./llvm.sh 17 # Install abseil-cpp wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz @@ -111,9 +111,10 @@ cd ../../ wget -O brotli-1.1.0.tar.gz https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz tar -xvzf brotli-1.1.0.tar.gz && cd brotli-1.1.0 mkdir build && cd build -cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. +cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$HOME/ydb_deps/brotli" .. cmake --build . --config Release -cmake --install . --config Release --prefix ~/ydb_deps/brotli +cmake --install . --config Release cd ../../ # Install jwt-cpp @@ -165,20 +166,37 @@ cmake --build --preset $sdk_configure_preset ### Build `.deb` packages -The SDK can be packaged as Debian development packages with CPack. The packaging build uses static libraries and produces the following packages: +The SDK can be packaged as Debian development packages with CPack. The complete packaging flow uses static libraries and produces the following packages: +- `yandex-googleapis-api-common-protos` — generated API Common Protos headers and static library, required by `libydb-cpp-dev`; - `libydb-cpp-dev` — core SDK static library, public headers and CMake package files; - `libydb-cpp-iam-dev` — IAM credentials plugin; - `libydb-cpp-otel-metrics-dev` — OpenTelemetry metrics plugin (includes vendored opentelemetry-cpp); - `libydb-cpp-otel-tracing-dev` — OpenTelemetry tracing plugin (requires `libydb-cpp-otel-metrics-dev` for OTel headers/libs). -The Debian packaging flow is intended for Ubuntu 24.04. Install the regular build dependencies first. OpenTelemetry plugins use the vendored `third_party/opentelemetry-cpp` submodule (v1.26.0, matching the YDB monorepo pin); initialize it with `git submodule update --init third_party/opentelemetry-cpp` before building. +The Debian packaging flow is intended for Ubuntu 24.04. Initialize the required submodules before building: + +```bash +git submodule update --init --recursive +``` + +The packaging helper first builds `yandex-googleapis-api-common-protos` from the +vendored API Common Protos using Ubuntu 24.04's `protobuf-compiler` and +`libprotobuf-dev`. It installs that package in the build container before +building the SDK, so both packages use the same distro protobuf ABI. +OpenTelemetry plugins use the vendored `third_party/opentelemetry-cpp` +submodule (v1.26.0, matching the YDB monorepo pin). -To build `.deb` packages directly with CPack: +To build the complete `.deb` package set with the same containerized flow used +by CI and release publishing (Docker is required): ```bash -cmake --preset package-deb-clang -cmake --build build-deb --target package -j$(nproc) +mkdir -p build-deb .deb-ccache +docker run --rm --network host \ + -e CCACHE_DIR=/source/.deb-ccache \ + -v "$PWD:/source" \ + ubuntu:24.04 \ + bash /source/scripts/build_cpack_deb_packages.sh /source/build-deb ``` The generated `.deb` files are placed into `build-deb/`. @@ -206,7 +224,7 @@ To smoke-test generated `.deb` packages with the sample consumer project: ### Install from GitHub releases Pre-built `.deb` packages for Ubuntu 24.04 (Noble) are attached to each -GitHub release. Download the assets and install them with `dpkg`: +GitHub release. Download the assets and install them with APT: ```bash # Replace with the desired release tag (e.g. v1.2.3) From ec3a40bdf33aceab01b1f3e6f7638fef5d765fbf Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Wed, 29 Jul 2026 18:45:51 +0300 Subject: [PATCH 54/56] fix tests and update versions --- .github/actions/prepare_vm/action.yaml | 6 +- .github/scripts/copy_sources.sh | 158 +- .github/workflows/import.yaml | 10 +- CMakeLists.txt | 10 + README.md | 12 +- cmake/public_headers.txt | 5 +- contrib/libs/libc_compat/CMakeLists.txt | 3 +- contrib/libs/libc_compat/README.md | 2 + contrib/libs/libc_compat/collate.h | 0 contrib/libs/libc_compat/glob.c | 1127 ++++++++++++++ contrib/libs/libc_compat/glob.h | 104 ++ contrib/libs/libc_compat/ifaddrs.c | 663 +++++++++ .../{ => memfd_create}/memfd_create.c | 0 .../libs/libc_compat/memfd_create/sys/mman.h | 16 + .../{include/windows => queue}/sys/queue.h | 0 contrib/libs/libc_compat/unistd.h | 3 + contrib/libs/libc_compat/update.sh | 60 + contrib/libs/lzmasdk/7zVersion.h | 27 + include/ydb-cpp-sdk/stlfwd.h | 8 + library/cpp/CMakeLists.txt | 2 + library/cpp/blockcodecs/codecs.h | 2 +- library/cpp/blockcodecs/codecs/zstd/README.md | 40 + library/cpp/blockcodecs/codecs/zstd/zstd.cpp | 29 +- library/cpp/blockcodecs/codecs_ut.cpp | 3 +- library/cpp/blockcodecs/core/codecs.h | 7 + library/cpp/blockcodecs/fuzz/main.cpp | 1 - library/cpp/cache/cache.h | 8 +- library/cpp/cache/thread_safe_cache.h | 19 +- library/cpp/cache/ut/cache_ut.cpp | 51 + library/cpp/cgiparam/cgiparam.cpp | 18 +- library/cpp/cgiparam/cgiparam.h | 29 +- library/cpp/cgiparam/cgiparam_ut.cpp | 2 + library/cpp/charset/wide_ut.cpp | 8 +- library/cpp/colorizer/ut/colorizer_ut.cpp | 2 +- .../cpp/containers/cow_string/CMakeLists.txt | 30 + library/cpp/containers/cow_string/README.md | 9 + .../cpp/containers/cow_string/cow_string.cpp | 280 ++++ .../cpp/containers/cow_string/cow_string.h | 1016 +++++++++++++ .../containers/cow_string/cow_string_ut.cpp | 1296 +++++++++++++++++ library/cpp/containers/cow_string/output.cpp | 46 + library/cpp/containers/cow_string/reverse.cpp | 32 + library/cpp/containers/cow_string/reverse.h | 16 + library/cpp/containers/cow_string/str_stl.h | 67 + library/cpp/containers/cow_string/subst.cpp | 182 +++ library/cpp/containers/cow_string/subst.h | 31 + .../ut_medium/cow_string_medium_ut.cpp | 55 + .../cpp/containers/cow_string/ysaveload.cpp | 1 + library/cpp/containers/cow_string/ysaveload.h | 9 + .../disjoint_interval_tree.h | 3 +- .../ut/disjoint_interval_tree_ut.cpp | 19 + library/cpp/containers/paged_vector/README.md | 99 ++ .../containers/paged_vector/paged_vector.h | 465 +++--- .../paged_vector/ut/paged_vector_ut.cpp | 593 +++++++- .../cpp/containers/stack_vector/stack_vec.h | 2 +- library/cpp/coroutine/engine/coroutine_ut.cpp | 11 +- library/cpp/cppparser/parser.cpp | 34 + library/cpp/digest/md5/md5.cpp | 1 - library/cpp/digest/md5/md5.h | 3 + library/cpp/digest/md5/md5_ut.cpp | 2 +- library/cpp/digest/murmur/murmur.h | 2 +- library/cpp/getopt/last_getopt_demo/demo.cpp | 4 +- .../cpp/getopt/small/completion_generator.cpp | 8 + library/cpp/getopt/small/last_getopt_opt.h | 63 +- library/cpp/getopt/small/last_getopt_opts.cpp | 8 +- library/cpp/getopt/small/last_getopt_opts.h | 54 +- .../getopt/small/last_getopt_parse_result.cpp | 58 +- .../getopt/small/last_getopt_parse_result.h | 45 +- .../cpp/getopt/small/last_getopt_parser.cpp | 7 +- library/cpp/getopt/small/last_getopt_parser.h | 2 +- library/cpp/getopt/small/modchooser.cpp | 33 +- library/cpp/getopt/small/modchooser.h | 15 +- library/cpp/getopt/ut/CMakeLists.txt | 56 +- library/cpp/getopt/ut/last_getopt_ut.cpp | 197 ++- library/cpp/getopt/ut/modchooser_ut.cpp | 50 + library/cpp/html/escape/CMakeLists.txt | 25 + library/cpp/html/escape/escape.cpp | 66 + library/cpp/html/escape/escape.h | 9 + library/cpp/html/escape/ut/escape_ut.cpp | 16 + library/cpp/http/misc/httpcodes.h | 1 + library/cpp/http/misc/httpreqdata.cpp | 2 + library/cpp/http/misc/httpreqdata.h | 15 +- library/cpp/http/misc/parsed_request.cpp | 10 +- library/cpp/http/misc/parsed_request.h | 6 +- library/cpp/http/server/conn.cpp | 81 +- library/cpp/http/server/conn.h | 33 +- library/cpp/http/server/http.cpp | 12 +- library/cpp/http/server/http.h | 2 + library/cpp/http/server/http_ex.cpp | 4 +- library/cpp/http/server/http_ut.cpp | 109 ++ library/cpp/http/simple/http_client.cpp | 113 +- library/cpp/http/simple/http_client.h | 71 +- library/cpp/http/simple/http_client_options.h | 21 + library/cpp/http/simple/ut/http_ut.cpp | 42 + library/cpp/int128/int128.h | 13 +- library/cpp/int128/ut/i128_find_ut.cpp | 16 + library/cpp/iterator/enumerate.h | 39 +- library/cpp/iterator/filtering.h | 5 +- library/cpp/iterator/mapped.h | 5 +- library/cpp/iterator/ut/functools_ut.cpp | 14 +- library/cpp/iterator/ut/mapped_ut.cpp | 17 + library/cpp/json/common/defs.h | 3 +- library/cpp/json/converter/converter.h | 55 +- .../cpp/json/converter/ut/test_conversion.cpp | 11 +- .../cpp/json/easy_parse/json_easy_parser.cpp | 78 +- .../cpp/json/easy_parse/json_easy_parser.h | 2 +- .../json/easy_parse/json_easy_parser_impl.h | 4 +- library/cpp/json/fast_sax/parser.h | 2 +- library/cpp/json/fuzzy_test/main.cpp | 6 +- library/cpp/json/json_prettifier.cpp | 17 +- library/cpp/json/json_prettifier.h | 2 +- library/cpp/json/json_reader.cpp | 33 +- library/cpp/json/json_reader.h | 26 +- library/cpp/json/json_value.h | 2 +- library/cpp/json/json_writer.cpp | 19 +- library/cpp/json/json_writer.h | 2 +- library/cpp/json/rapidjson_helpers.h | 6 +- library/cpp/json/ut/json_prettifier_ut.cpp | 2 +- library/cpp/json/ut/json_reader_fast_ut.cpp | 22 +- library/cpp/json/ut/json_reader_nan_ut.cpp | 27 + library/cpp/json/ut/json_reader_ut.cpp | 8 +- library/cpp/json/ut/json_saveload_ut.cpp | 3 +- library/cpp/json/ut/json_writer_ut.cpp | 4 +- library/cpp/json/writer/fwd.h | 5 + library/cpp/json/writer/json.cpp | 35 +- library/cpp/json/writer/json.h | 54 +- library/cpp/json/writer/json_ut.cpp | 2 +- library/cpp/json/writer/json_value.cpp | 271 ++-- library/cpp/json/writer/json_value.h | 103 +- library/cpp/json/writer/json_value_ut.cpp | 63 +- library/cpp/logger/backend_creator.cpp | 2 +- library/cpp/logger/rotating_file.cpp | 22 +- library/cpp/logger/sync_page_cache_file.cpp | 84 +- library/cpp/logger/sync_page_cache_file.h | 9 +- library/cpp/mime/types/mime.cpp | 6 +- library/cpp/mime/types/mime.h | 2 + .../monlib/dynamic_counters/CMakeLists.txt | 1 + .../cpp/monlib/dynamic_counters/counters.cpp | 12 + .../cpp/monlib/dynamic_counters/counters.h | 3 +- .../monlib/dynamic_counters/counters_ut.cpp | 26 + .../cpp/monlib/dynamic_counters/encode_ut.cpp | 31 + library/cpp/monlib/dynamic_counters/page.cpp | 151 +- library/cpp/monlib/dynamic_counters/page.h | 10 +- .../encode/buffered/buffered_encoder_base.cpp | 6 + .../encode/buffered/buffered_encoder_base.h | 3 + .../monlib/encode/buffered/string_pool.cpp | 2 + .../cpp/monlib/encode/buffered/string_pool.h | 7 + library/cpp/monlib/encode/fake/fake.cpp | 3 + library/cpp/monlib/encode/format.cpp | 15 + library/cpp/monlib/encode/format.h | 10 + library/cpp/monlib/encode/format_ut.cpp | 42 + .../cpp/monlib/encode/json/json_decoder.cpp | 32 +- .../cpp/monlib/encode/json/json_encoder.cpp | 27 +- library/cpp/monlib/encode/json/json_ut.cpp | 18 +- .../json/ut/expected_buffered_memOnly.json | 95 ++ .../encode/json/ut/expected_memOnly.json | 95 ++ .../cpp/monlib/encode/json/ut/metrics.json | 1 + .../cpp/monlib/encode/prometheus/prometheus.h | 8 + .../encode/prometheus/prometheus_decoder.cpp | 31 +- .../prometheus/prometheus_decoder_ut.cpp | 118 +- .../encode/prometheus/prometheus_encoder.cpp | 14 +- .../prometheus/prometheus_encoder_ut.cpp | 101 -- .../encode/protobuf/protobuf_encoder.cpp | 5 + .../encode/protobuf/protos/samples.proto | 3 +- library/cpp/monlib/encode/spack/spack_v1.h | 10 +- .../monlib/encode/spack/spack_v1_decoder.cpp | 77 +- .../monlib/encode/spack/spack_v1_encoder.cpp | 62 +- .../cpp/monlib/encode/spack/spack_v1_ut.cpp | 423 +++++- library/cpp/monlib/encode/unistat/unistat.h | 16 +- .../monlib/encode/unistat/unistat_decoder.cpp | 50 +- .../cpp/monlib/encode/unistat/unistat_ut.cpp | 47 + library/cpp/monlib/metrics/ewma.cpp | 2 +- library/cpp/monlib/metrics/fake.h | 16 +- .../cpp/monlib/metrics/histogram_collector.h | 13 +- .../metrics/histogram_collector_explicit.cpp | 19 +- .../histogram_collector_exponential.cpp | 22 +- .../metrics/histogram_collector_linear.cpp | 19 +- .../cpp/monlib/metrics/histogram_snapshot.h | 1 + library/cpp/monlib/metrics/labels.cpp | 4 +- library/cpp/monlib/metrics/metric.h | 26 +- library/cpp/monlib/metrics/metric_consumer.h | 4 + .../cpp/monlib/metrics/metric_registry.cpp | 258 ++-- library/cpp/monlib/metrics/metric_registry.h | 168 ++- .../cpp/monlib/metrics/metric_registry_ut.cpp | 72 + .../cpp/monlib/metrics/metric_sub_registry.h | 49 + library/cpp/monlib/service/format.h | 15 +- library/cpp/monlib/service/monservice.h | 1 + .../monlib/service/pages/index_mon_page.cpp | 10 +- .../cpp/monlib/service/pages/index_mon_page.h | 1 + library/cpp/monlib/service/pages/mon_page.h | 1 + library/cpp/monlib/service/pages/templates.h | 1 + library/cpp/openssl/init/init.cpp | 68 +- library/cpp/openssl/init/init.h | 3 - library/cpp/openssl/io/stream.cpp | 44 +- library/cpp/openssl/io/stream.h | 7 + library/cpp/openssl/io/ut/builtin_ut.cpp | 37 +- library/cpp/resource/CMakeLists.txt | 1 + library/cpp/resource/README.md | 6 +- library/cpp/resource/registry.cpp | 82 +- library/cpp/resource/registry.h | 8 +- library/cpp/streams/brotli/CMakeLists.txt | 1 - library/cpp/streams/brotli/brotli.cpp | 62 +- library/cpp/streams/brotli/brotli.h | 17 +- library/cpp/streams/brotli/brotli_ut.cpp | 37 +- library/cpp/streams/brotli/const.h | 10 + library/cpp/streams/brotli/dictionary.h | 54 + library/cpp/streams/lzma/lzma_ut.cpp | 2 +- library/cpp/string_utils/quote/CMakeLists.txt | 2 +- library/cpp/string_utils/quote/quote.cpp | 79 +- library/cpp/string_utils/quote/quote.h | 2 +- .../string_utils/quote/{ => ut}/quote_ut.cpp | 2 +- library/cpp/string_utils/url/url.cpp | 72 +- library/cpp/string_utils/url/url.h | 46 +- library/cpp/svnversion/test/main.cpp | 1 + library/cpp/testing/common/env.cpp | 39 +- library/cpp/testing/common/env.h | 1 - library/cpp/testing/common/network.cpp | 16 +- library/cpp/testing/common/ut/env_ut.cpp | 18 - library/cpp/testing/common/ut/network_ut.cpp | 85 +- library/cpp/testing/hook/yt_initialize_hook.h | 6 + library/cpp/testing/unittest/gtest.h | 2 + library/cpp/testing/unittest/registar.cpp | 14 +- library/cpp/testing/unittest/registar.h | 143 +- library/cpp/testing/unittest/utmain.cpp | 35 +- library/cpp/threading/cancellation/README.md | 112 ++ .../cancellation/cancellation_token.cpp | 1 + .../cancellation/cancellation_token.h | 137 ++ library/cpp/threading/cancellation/ut.cpp | 56 + library/cpp/threading/chunk_queue/queue.h | 123 +- library/cpp/threading/equeue/equeue.cpp | 15 +- library/cpp/threading/equeue/equeue.h | 14 +- library/cpp/threading/equeue/equeue_ut.cpp | 49 +- library/cpp/threading/equeue/fast/equeue.h | 29 +- .../threading/future/core/coroutine_traits.h | 112 +- .../cpp/threading/future/core/future-inl.h | 65 +- library/cpp/threading/future/core/future.h | 6 +- library/cpp/threading/future/future_ut.cpp | 6 + .../ut_gtest/coroutine_traits_adl_ut.cpp | 24 + .../threading/future/ut_gtest/simple_task.h | 47 + library/cpp/uri/common.h | 4 +- library/cpp/uri/uri_ut.cpp | 6 +- library/cpp/yson/varint.cpp | 1 + library/cpp/yt/CMakeLists.txt | 34 +- library/cpp/yt/assert/assert.cpp | 7 +- library/cpp/yt/assert/assert.h | 59 +- library/cpp/yt/coding/bit_io-inl.h | 69 + library/cpp/yt/coding/bit_io.h | 63 + library/cpp/yt/coding/interpolative-inl.h | 182 +++ library/cpp/yt/coding/interpolative.h | 52 + library/cpp/yt/coding/unittests/bit_io_ut.cpp | 69 + .../yt/coding/unittests/interpolative_ut.cpp | 221 +++ library/cpp/yt/coding/unittests/varint_ut.cpp | 76 +- library/cpp/yt/coding/varint-inl.h | 43 +- library/cpp/yt/coding/zig_zag-inl.h | 1 - .../compact_containers/compact_flat_map-inl.h | 241 +++ .../compact_flat_map.h | 80 +- .../compact_containers/compact_flat_set-inl.h | 251 ++++ .../yt/compact_containers/compact_flat_set.h | 111 ++ .../compact_heap-inl.h | 0 .../compact_heap.h | 0 .../compact_queue-inl.h | 0 .../compact_queue.h | 0 .../compact_set-inl.h | 52 +- .../compact_set.h | 20 +- .../compact_vector-inl.h | 6 +- .../compact_vector.h | 6 +- .../unittests/compact_flat_map_ut.cpp | 2 +- .../unittests/compact_flat_set_ut.cpp | 333 +++++ .../unittests/compact_heap_ut.cpp | 2 +- .../unittests/compact_queue_ut.cpp | 2 +- .../unittests/compact_set_ut.cpp | 2 +- .../unittests/compact_vector_ut.cpp | 32 +- library/cpp/yt/containers/default_map-inl.h | 58 + library/cpp/yt/containers/default_map.h | 64 + .../yt/containers/enum_indexed_array-inl.h | 16 +- .../cpp/yt/containers/enum_indexed_array.h | 18 +- library/cpp/yt/containers/expiring_set-inl.h | 88 ++ library/cpp/yt/containers/expiring_set.h | 56 + library/cpp/yt/containers/non_empty-inl.h | 193 +++ library/cpp/yt/containers/non_empty.h | 91 ++ .../cpp/yt/containers/ordered_hash_map-inl.h | 153 ++ library/cpp/yt/containers/ordered_hash_map.h | 94 ++ library/cpp/yt/containers/ring_queue.h | 382 +++++ .../cpp/yt/containers/sentinel_optional-inl.h | 118 ++ library/cpp/yt/containers/sentinel_optional.h | 128 ++ library/cpp/yt/containers/slot_map-inl.h | 96 ++ library/cpp/yt/containers/slot_map.h | 77 + .../cpp/yt/containers/static_ring_queue-inl.h | 69 + library/cpp/yt/containers/static_ring_queue.h | 29 + .../containers/unittests/default_map_ut.cpp | 27 + .../containers/unittests/expiring_set_ut.cpp | 132 ++ .../yt/containers/unittests/non_empty_ut.cpp | 109 ++ .../unittests/ordered_hash_map_ut.cpp | 149 ++ .../yt/containers/unittests/ring_queue_ut.cpp | 134 ++ .../unittests/sentinel_optional_ut.cpp | 291 ++++ .../yt/containers/unittests/slot_map_ut.cpp | 206 +++ .../unittests/static_ring_queue_ut.cpp | 130 ++ library/cpp/yt/exception/attributes.h | 6 +- library/cpp/yt/exception/exception.cpp | 6 +- library/cpp/yt/exception/exception.h | 10 +- library/cpp/yt/malloc/malloc.cpp | 9 +- library/cpp/yt/malloc/malloc.h | 5 + library/cpp/yt/memory/allocation_tags.h | 16 + .../cpp/yt/memory/allocation_tags_hooks.cpp | 27 + library/cpp/yt/memory/allocation_tags_hooks.h | 23 + library/cpp/yt/memory/atomic-inl.h | 29 + library/cpp/yt/memory/atomic.h | 23 + .../cpp/yt/memory/atomic_intrusive_ptr-inl.h | 121 +- library/cpp/yt/memory/atomic_intrusive_ptr.h | 37 +- library/cpp/yt/memory/blob.cpp | 20 +- library/cpp/yt/memory/blob.h | 2 +- .../cpp/yt/memory/chunked_memory_allocator.h | 4 +- .../cpp/yt/memory/chunked_memory_pool-inl.h | 33 +- library/cpp/yt/memory/chunked_memory_pool.cpp | 3 + library/cpp/yt/memory/chunked_memory_pool.h | 2 +- .../cpp/yt/memory/chunked_output_stream.cpp | 31 +- library/cpp/yt/memory/chunked_output_stream.h | 8 +- library/cpp/yt/memory/erased_storage-inl.h | 28 +- library/cpp/yt/memory/erased_storage.h | 37 +- .../yt/memory/exact_ref_counted_cast-inl.h | 46 + .../cpp/yt/memory/exact_ref_counted_cast.h | 36 + library/cpp/yt/memory/free_list-inl.h | 68 +- library/cpp/yt/memory/free_list.h | 44 +- library/cpp/yt/memory/intrusive_ptr.h | 143 +- .../memory/leaky_ref_counted_singleton-inl.h | 3 - library/cpp/yt/memory/memory_tag-inl.h | 2 +- library/cpp/yt/memory/memory_tag.h | 2 +- library/cpp/yt/memory/new-inl.h | 114 +- library/cpp/yt/memory/new.cpp | 18 + library/cpp/yt/memory/new.h | 34 +- library/cpp/yt/memory/non_null_ptr-inl.h | 56 + library/cpp/yt/memory/non_null_ptr.h | 84 ++ library/cpp/yt/memory/poison-inl.h | 77 + library/cpp/yt/memory/poison.cpp | 64 + library/cpp/yt/memory/poison.h | 53 + library/cpp/yt/memory/public.h | 1 + library/cpp/yt/memory/range.h | 196 +-- library/cpp/yt/memory/ref-inl.h | 53 +- library/cpp/yt/memory/ref.cpp | 198 ++- library/cpp/yt/memory/ref.h | 88 +- library/cpp/yt/memory/ref_counted-inl.h | 105 +- library/cpp/yt/memory/ref_counted.h | 37 +- library/cpp/yt/memory/ref_tracked-inl.h | 8 +- library/cpp/yt/memory/ref_tracked.h | 19 +- library/cpp/yt/memory/shared_range.h | 35 +- .../yt/memory/simple_memory_usage_tracker.cpp | 117 ++ .../yt/memory/simple_memory_usage_tracker.h | 57 + library/cpp/yt/memory/tagged_ptr-inl.h | 10 +- library/cpp/yt/memory/tagged_ptr.h | 21 +- library/cpp/yt/memory/type_erasure.h | 428 ++++++ library/cpp/yt/memory/type_erasure_detail.h | 729 ++++++++++ .../unittests/atomic_intrusive_ptr_ut.cpp | 129 +- library/cpp/yt/memory/unittests/atomic_ut.cpp | 175 +++ .../unittests/chunked_memory_pool_ut.cpp | 2 +- .../yt/memory/unittests/erased_storage_ut.cpp | 22 + .../unittests/exact_ref_counted_cast_ut.cpp | 102 ++ .../cpp/yt/memory/unittests/free_list_ut.cpp | 112 +- .../yt/memory/unittests/function_view_ut.cpp | 2 +- .../yt/memory/unittests/intrusive_ptr_ut.cpp | 92 ++ .../yt/memory/unittests/non_null_ptr_ut.cpp | 139 ++ .../range_protobuf_repeated_field_ut.cpp | 76 + library/cpp/yt/memory/unittests/ref_ut.cpp | 87 +- .../yt/memory/unittests/type_erasure_ut.cpp | 361 +++++ .../cpp/yt/memory/unittests/weak_ptr_ut.cpp | 93 +- library/cpp/yt/memory/weak_ptr-inl.h | 356 +++++ library/cpp/yt/memory/weak_ptr.h | 299 +--- library/cpp/yt/misc/arcadia_enum-inl.h | 14 +- library/cpp/yt/misc/cast-inl.h | 88 +- library/cpp/yt/misc/cast.h | 16 +- library/cpp/yt/misc/compare-inl.h | 71 + library/cpp/yt/misc/compare.h | 28 + library/cpp/yt/misc/concepts.h | 19 +- library/cpp/yt/misc/enum-inl.h | 127 +- library/cpp/yt/misc/enum.h | 78 +- library/cpp/yt/misc/global.h | 2 +- library/cpp/yt/misc/guid-inl.h | 9 + library/cpp/yt/misc/guid.cpp | 87 +- library/cpp/yt/misc/hash-inl.h | 73 +- library/cpp/yt/misc/hash.h | 22 +- library/cpp/yt/misc/numeric_helpers-inl.h | 59 + library/cpp/yt/misc/numeric_helpers.h | 31 + library/cpp/yt/misc/port.h | 29 +- library/cpp/yt/misc/preprocessor.h | 8 + library/cpp/yt/misc/property.h | 36 +- library/cpp/yt/misc/range_formatters-inl.h | 31 + library/cpp/yt/misc/range_formatters.h | 27 + library/cpp/yt/misc/range_helpers-inl.h | 165 +++ library/cpp/yt/misc/range_helpers.h | 70 + library/cpp/yt/misc/source_location-inl.h | 20 +- library/cpp/yt/misc/source_location.cpp | 33 - library/cpp/yt/misc/source_location.h | 27 +- library/cpp/yt/misc/static_initializer.h | 19 + library/cpp/yt/misc/strong_typedef-fwd.h | 17 + library/cpp/yt/misc/strong_typedef-inl.h | 124 +- library/cpp/yt/misc/strong_typedef.h | 34 +- library/cpp/yt/misc/tag_invoke.h | 95 ++ library/cpp/yt/misc/tag_invoke_cpo.h | 25 + library/cpp/yt/misc/typeid-inl.h | 49 + library/cpp/yt/misc/typeid.h | 27 + library/cpp/yt/misc/unittests/cast_ut.cpp | 113 ++ library/cpp/yt/misc/unittests/compare_ut.cpp | 62 + library/cpp/yt/misc/unittests/enum_ut.cpp | 71 +- library/cpp/yt/misc/unittests/hash_ut.cpp | 27 + .../cpp/yt/misc/unittests/preprocessor_ut.cpp | 13 + .../yt/misc/unittests/range_helpers_ut.cpp | 127 ++ .../yt/misc/unittests/tag_invoke_cpo_ut.cpp | 107 ++ .../yt/misc/unittests/tag_invoke_impl_ut.cpp | 72 + .../cpp/yt/misc/unittests/typeid_sample.cpp | 14 + library/cpp/yt/misc/unittests/typeid_sample.h | 17 + library/cpp/yt/misc/unittests/typeid_ut.cpp | 25 + library/cpp/yt/misc/variant-inl.h | 50 - library/cpp/yt/misc/variant.cpp | 16 - library/cpp/yt/misc/variant.h | 13 - .../small_containers/compact_flat_map-inl.h | 244 ---- library/cpp/yt/string/enum-inl.h | 137 +- library/cpp/yt/string/enum.cpp | 75 +- library/cpp/yt/string/enum.h | 15 +- library/cpp/yt/string/format-inl.h | 752 +++++++--- library/cpp/yt/string/format.cpp | 7 +- library/cpp/yt/string/format.h | 123 +- library/cpp/yt/string/format_analyser-inl.h | 112 -- library/cpp/yt/string/format_analyser.h | 137 +- library/cpp/yt/string/format_arg.h | 2 +- library/cpp/yt/string/format_string-inl.h | 16 +- library/cpp/yt/string/format_string.h | 15 +- library/cpp/yt/string/raw_formatter.h | 27 +- library/cpp/yt/string/stream.cpp | 144 ++ library/cpp/yt/string/stream.h | 107 ++ library/cpp/yt/string/string-inl.h | 54 +- library/cpp/yt/string/string.cpp | 106 +- library/cpp/yt/string/string.h | 107 +- library/cpp/yt/string/string_builder-inl.h | 115 ++ library/cpp/yt/string/string_builder.cpp | 30 + library/cpp/yt/string/string_builder.h | 78 +- library/cpp/yt/string/unittests/enum_ut.cpp | 48 +- library/cpp/yt/string/unittests/format_ut.cpp | 153 +- library/cpp/yt/string/unittests/guid_ut.cpp | 4 +- library/cpp/yt/string/unittests/stream_ut.cpp | 66 + library/cpp/yt/string/unittests/string_ut.cpp | 17 +- library/cpp/yt/system/benchmarks/cpu_id.cpp | 33 + library/cpp/yt/system/benchmarks/process.cpp | 35 + library/cpp/yt/system/benchmarks/thread.cpp | 55 + library/cpp/yt/system/benchmarks/tscp.cpp | 31 + library/cpp/yt/system/cpu_id-inl.h | 41 + library/cpp/yt/system/cpu_id.cpp | 44 + library/cpp/yt/system/cpu_id.h | 31 + library/cpp/yt/system/env-inl.h | 32 + library/cpp/yt/system/env.cpp | 76 + library/cpp/yt/system/env.h | 34 + library/cpp/yt/system/exit-inl.h | 32 + library/cpp/yt/system/exit.cpp | 28 + library/cpp/yt/system/exit.h | 49 + library/cpp/yt/system/handle_eintr-inl.h | 34 + library/cpp/yt/system/handle_eintr.h | 16 + library/cpp/yt/system/proc.h | 32 + library/cpp/yt/system/process_id-inl.h | 35 + library/cpp/yt/system/process_id.cpp | 37 + library/cpp/yt/system/process_id.h | 22 + library/cpp/yt/system/thread_id-inl.h | 50 + library/cpp/yt/system/thread_id.cpp | 44 + library/cpp/yt/system/thread_id.h | 28 + .../cpp/yt/{misc => system}/thread_name.cpp | 2 +- library/cpp/yt/{misc => system}/thread_name.h | 2 +- library/cpp/yt/system/tscp-inl.h | 71 + library/cpp/yt/system/tscp.h | 45 + library/cpp/yt/system/unittests/cpu_id_ut.cpp | 62 + library/cpp/yt/system/unittests/env_ut.cpp | 81 ++ library/cpp/yt/ya_cpp.make.inc | 15 +- library/cpp/yt/yson_string/convert.cpp | 19 + library/cpp/yt/yson_string/convert.h | 5 + library/cpp/yt/yson_string/string-inl.h | 16 + library/cpp/yt/yson_string/string.cpp | 55 +- library/cpp/yt/yson_string/string.h | 32 +- .../yt/yson_string/unittests/saveload_ut.cpp | 2 +- scripts/test_copy_sources_plugins.sh | 68 + .../impl/observability/metric_buffer.cpp | 98 +- .../client/observability/metric_buffer_ut.cpp | 13 +- tools/enum_parser/enum_parser/main.cpp | 4 +- .../enum_serialization_runtime/README.md | 2 +- .../enum_serialization_runtime/enum_runtime.h | 12 + tools/enum_parser/parse_enum/parse_enum.cpp | 14 + .../enum_parser/parse_enum/parse_enum_ut.cpp | 58 + .../parse_enum/ut/digit_separator.h | 20 + tools/rescompiler/main.cpp | 114 +- util/CMakeLists.txt | 30 +- util/README.md | 17 +- util/charset/unicode_table.h | 8 +- util/charset/unidata.h | 7 +- util/charset/utf8.cpp | 50 +- util/charset/utf8.h | 155 +- util/charset/utf8_ut.cpp | 62 +- util/charset/wide.cpp | 16 +- util/charset/wide.h | 132 +- util/charset/wide_sse41.cpp | 28 +- util/charset/wide_ut.cpp | 22 +- util/datetime/base.cpp | 5 +- util/datetime/base.h | 174 ++- util/datetime/base.pxd | 23 +- util/datetime/base_ut.cpp | 75 +- util/datetime/benchmark/gmtime_r/main.cpp | 100 +- util/datetime/cputimer_ut.cpp | 2 +- util/datetime/parser.h | 98 +- util/datetime/parser_deprecated_ut.cpp | 4 +- util/datetime/parser_ut.cpp | 6 +- util/datetime/process_uptime_ut.cpp | 2 +- util/datetime/systime.cpp | 218 ++- util/datetime/systime.h | 15 +- util/datetime/uptime.cpp | 23 +- util/datetime/uptime_ut.cpp | 2 +- util/digest/city.cpp | 62 + util/digest/city.h | 2 +- util/digest/city_streaming.h | 21 + util/digest/city_ut.cpp | 2 +- util/digest/fnv.h | 2 +- util/digest/fnv_ut.cpp | 2 +- util/digest/murmur.cpp | 2 +- util/digest/murmur.h | 2 +- util/draft/date_ut.cpp | 2 +- util/draft/datetime.cpp | 10 +- util/draft/datetime.h | 4 +- util/draft/datetime_ut.cpp | 6 +- util/draft/enum.h | 33 +- util/draft/holder_vector.h | 6 +- util/draft/holder_vector_ut.cpp | 2 +- util/draft/matrix.h | 3 +- util/draft/memory.h | 3 +- util/draft/memory_ut.cpp | 2 +- util/folder/dirent_win.c | 10 +- util/folder/dirut.cpp | 155 +- util/folder/dirut.h | 6 +- util/folder/dirut_ut.cpp | 2 +- util/folder/fts.h | 2 +- util/folder/fts_ut.cpp | 2 +- util/folder/iterator.h | 1 + util/folder/iterator_ut.cpp | 2 +- util/folder/path.cpp | 11 +- util/folder/path.h | 2 +- util/folder/path_ut.cpp | 19 +- util/folder/pathsplit.cpp | 2 +- util/folder/pathsplit.h | 4 +- util/folder/pathsplit_ut.cpp | 4 +- util/folder/tempdir.cpp | 8 + util/folder/tempdir.h | 6 + util/folder/tempdir_ut.cpp | 13 + util/generic/adaptor.h | 4 +- util/generic/adaptor_ut.cpp | 2 +- util/generic/algorithm.h | 46 +- util/generic/algorithm_ut.cpp | 20 +- util/generic/array_ref.h | 79 +- util/generic/array_ref_ut.cpp | 52 +- util/generic/array_size.h | 2 +- util/generic/array_size_ut.cpp | 2 +- util/generic/bitmap.h | 121 +- util/generic/bitmap_ut.cpp | 2 +- util/generic/bitops.cpp | 4 +- util/generic/bitops.h | 16 +- util/generic/bt_exception.cpp | 1 - util/generic/bt_exception.h | 24 - util/generic/buffer.cpp | 5 + util/generic/buffer.h | 8 +- util/generic/buffer_ut.cpp | 20 +- util/generic/cast.h | 27 +- util/generic/deque.h | 1 + util/generic/deque_ut.cpp | 26 +- util/generic/enum_cast.cpp | 12 + util/generic/enum_cast.h | 46 + util/generic/enum_cast_ut.cpp | 49 + util/generic/enum_cast_ut.h | 28 + util/generic/enum_range.cpp | 1 - util/generic/enum_range.h | 72 - util/generic/enum_range_ut.cpp | 237 --- util/generic/explicit_type.h | 2 +- util/generic/explicit_type_ut.cpp | 2 +- util/generic/flags.h | 4 +- util/generic/flags_ut.cpp | 10 +- util/generic/function.h | 23 +- util/generic/function_ref_ut.cpp | 5 +- util/generic/function_ut.cpp | 2 +- util/generic/fwd.h | 24 +- util/generic/guid.cpp | 2 +- util/generic/guid_ut.cpp | 6 +- util/generic/hash.h | 2 + util/generic/hash.pxd | 12 - util/generic/hash_multi_map.h | 2 + util/generic/hash_primes.cpp | 2 + util/generic/hash_primes.h | 2 +- util/generic/hash_primes_ut.cpp | 2 +- util/generic/hash_set.h | 24 +- util/generic/hash_set.pxd | 6 - util/generic/hash_table.h | 160 +- util/generic/hash_ut.cpp | 116 +- util/generic/intrlist.h | 16 +- util/generic/intrlist_ut.cpp | 123 +- util/generic/is_in.h | 2 +- util/generic/is_in_ut.cpp | 2 +- util/generic/iterator.h | 8 + util/generic/iterator_range_ut.cpp | 2 +- util/generic/iterator_ut.cpp | 4 +- util/generic/lazy_value.h | 6 +- util/generic/lazy_value_ut.cpp | 2 +- util/generic/list_ut.cpp | 2 +- util/generic/map_ut.cpp | 4 +- util/generic/mapfindptr_ut.cpp | 6 +- util/generic/maybe.h | 36 +- util/generic/maybe_traits.h | 37 +- util/generic/maybe_ut.cpp | 12 +- util/generic/mem_copy.h | 2 +- util/generic/mem_copy_ut.cpp | 4 +- util/generic/noncopyable.h | 2 +- util/generic/objects_counter_ut.cpp | 2 +- util/generic/overloaded.h | 6 +- util/generic/overloaded_ut.cpp | 4 +- util/generic/ptr.cpp | 5 + util/generic/ptr.h | 89 +- util/generic/ptr.pxd | 3 +- util/generic/ptr_ut.cpp | 61 +- util/generic/queue.h | 17 +- util/generic/queue_ut.cpp | 40 +- util/generic/reserve.h | 4 +- util/generic/scope.h | 15 +- util/generic/scope_ut.cpp | 2 +- util/generic/serialized_enum.h | 8 +- util/generic/serialized_enum_ut.cpp | 2 +- util/generic/set_ut.cpp | 8 +- util/generic/singleton.cpp | 4 +- util/generic/singleton.h | 2 +- util/generic/singleton_ut.cpp | 2 +- util/generic/size_literals.h | 28 +- util/generic/size_literals_ut.cpp | 2 +- util/generic/stack_ut.cpp | 2 +- util/generic/store_policy.h | 4 +- util/generic/store_policy_ut.cpp | 2 +- util/generic/strbase.h | 22 +- util/generic/strbuf.h | 46 +- util/generic/strbuf_ut.cpp | 47 +- util/generic/string.h | 324 +++-- util/generic/string.pxd | 2 - util/generic/string_hash.h | 2 +- util/generic/string_transparent_hash_ut.cpp | 2 +- util/generic/string_ut.cpp | 52 +- util/generic/string_ut.h | 17 +- util/generic/typelist.h | 39 +- util/generic/typelist_ut.cpp | 39 + util/generic/typetraits.h | 16 +- util/generic/typetraits_ut.cpp | 10 +- util/generic/utility.h | 12 +- util/generic/va_args_ut.cpp | 4 +- util/generic/vector.h | 1 + util/generic/vector_ut.cpp | 14 +- util/generic/xrange.h | 2 +- util/generic/xrange_ut.cpp | 2 +- util/generic/yexception.cpp | 6 +- util/generic/yexception.h | 53 +- util/generic/yexception_ut.cpp | 28 +- util/generic/ylimits.h | 4 +- util/generic/ymath.cpp | 6 +- util/generic/ymath.h | 4 +- util/linters.make.inc | 3 + util/memory/addstorage.h | 2 +- util/memory/addstorage_ut.cpp | 2 +- util/memory/blob.h | 8 +- util/memory/blob_ut.cpp | 2 +- util/memory/mmapalloc.cpp | 2 +- util/memory/pool.h | 23 + util/memory/pool_ut.cpp | 38 + util/memory/segmented_string_pool.h | 30 +- util/memory/segpool_alloc.h | 6 +- util/memory/tempbuf.cpp | 2 +- util/network/address.cpp | 2 +- util/network/address.h | 10 +- util/network/address_ut.cpp | 2 +- util/network/endpoint.h | 2 +- util/network/endpoint_ut.cpp | 2 +- util/network/hostip.cpp | 9 +- util/network/hostip.h | 2 +- util/network/init.cpp | 2 +- util/network/interface.cpp | 4 +- util/network/interface.h | 2 +- util/network/nonblock.cpp | 6 +- util/network/nonblock.h | 4 +- util/network/pair.cpp | 12 +- util/network/poller.cpp | 2 +- util/network/poller_ut.cpp | 2 +- util/network/pollerimpl.h | 2 +- util/network/sock.h | 54 +- util/network/sock_ut.cpp | 2 +- util/network/socket.cpp | 31 +- util/network/socket.h | 6 +- util/network/socket_ut.cpp | 6 +- util/random/common_ops.h | 4 +- util/random/common_ops_ut.cpp | 2 +- util/random/easy.h | 2 +- util/random/easy_ut.cpp | 2 +- util/random/entropy.cpp | 14 +- util/random/entropy_ut.cpp | 17 +- util/random/fast.h | 2 +- util/random/fast_ut.cpp | 2 +- util/random/init_atfork.cpp | 2 +- util/random/lcg_engine.cpp | 2 +- util/random/lcg_engine.h | 8 +- util/random/mersenne.h | 2 +- util/random/mersenne32.h | 2 +- util/random/mersenne64.h | 2 +- util/random/mersenne_ut.cpp | 2 +- util/random/normal.cpp | 2 +- util/random/normal_ut.cpp | 2 +- util/random/random.cpp | 2 +- util/random/random_ut.cpp | 2 +- util/random/shuffle_ut.cpp | 2 +- util/str_stl.h | 27 +- util/stream/aligned.h | 4 +- util/stream/aligned_ut.cpp | 2 +- util/stream/buffer.cpp | 2 +- util/stream/buffer_ut.cpp | 2 +- util/stream/buffered.cpp | 4 +- util/stream/buffered.h | 2 +- util/stream/buffered_ut.cpp | 22 +- util/stream/debug.cpp | 50 - util/stream/debug.h | 53 - util/stream/direct_io.h | 4 +- util/stream/direct_io_ut.cpp | 2 +- util/stream/file_ut.cpp | 2 +- util/stream/format.cpp | 2 +- util/stream/format.h | 23 +- util/stream/format_std_ut.cpp | 2 +- util/stream/format_ut.cpp | 2 +- util/stream/fwd.h | 2 +- util/stream/hex_ut.cpp | 2 +- util/stream/holder.h | 2 +- util/stream/input.cpp | 2 +- util/stream/input.h | 38 +- util/stream/input_ut.cpp | 2 +- util/stream/ios_ut.cpp | 2 +- util/stream/labeled_ut.cpp | 2 +- util/stream/length.h | 6 +- util/stream/length_ut.cpp | 2 +- util/stream/mem.h | 4 +- util/stream/mem_ut.cpp | 2 +- util/stream/multi.h | 2 +- util/stream/multi_ut.cpp | 2 +- util/stream/null.h | 2 +- util/stream/output.cpp | 70 +- util/stream/output.h | 63 +- util/stream/output_ut.cpp | 34 + util/stream/printf.h | 4 +- util/stream/printf_ut.cpp | 2 +- util/stream/str.cpp | 2 + util/stream/str.pxd | 1 - util/stream/str_ut.cpp | 2 +- util/stream/str_ut.pyx | 5 +- util/stream/tee.h | 2 +- util/stream/tempbuf.cpp | 2 +- util/stream/tokenizer_ut.cpp | 2 +- util/stream/trace.h | 16 +- util/stream/walk_ut.cpp | 2 +- util/stream/zerocopy_output_ut.cpp | 2 +- util/stream/zlib.cpp | 12 +- util/stream/zlib.h | 2 +- util/stream/zlib_ut.cpp | 2 +- util/string/ascii.cpp | 40 - util/string/ascii.h | 85 +- util/string/ascii_ut.cpp | 2 +- util/string/builder.h | 2 +- util/string/builder_ut.cpp | 2 +- util/string/cast.cpp | 4 +- util/string/cast.h | 11 +- util/string/cast_ut.cpp | 13 +- util/string/escape.cpp | 4 +- util/string/escape_ut.cpp | 4 +- util/string/hex.cpp | 6 +- util/string/hex_ut.cpp | 2 +- util/string/join.h | 31 +- util/string/join_ut.cpp | 10 +- util/string/printf_ut.cpp | 2 +- util/string/split.cpp | 2 + util/string/split.h | 90 +- util/string/split_ut.cpp | 58 +- util/string/strip.h | 110 +- util/string/strip_ut.cpp | 83 +- util/string/strspn_ut.cpp | 2 +- util/string/subst.h | 43 +- util/string/subst_ut.cpp | 2 +- util/string/type.cpp | 2 +- util/string/type_ut.cpp | 2 +- util/string/util.h | 34 +- util/string/vector.h | 10 +- util/string/vector_ut.cpp | 2 +- util/system/atexit.cpp | 2 +- util/system/atexit_ut.cpp | 4 +- util/system/backtrace.cpp | 24 +- util/system/backtrace.h | 5 + util/system/backtrace_ut.cpp | 2 +- util/system/byteorder.h | 6 +- util/system/compat.h | 4 +- util/system/compat_ut.cpp | 2 +- util/system/compiler.h | 274 +++- util/system/compiler_ut.c | 2 + util/system/compiler_ut.cpp | 13 +- util/system/condvar.cpp | 4 +- util/system/condvar.h | 2 +- util/system/context.cpp | 16 +- util/system/context_ut.cpp | 2 +- util/system/cpu_id.cpp | 16 +- util/system/cpu_id.h | 4 +- util/system/cpu_id_ut.cpp | 14 +- util/system/daemon.h | 2 +- util/system/daemon_ut.cpp | 2 +- util/system/datetime.cpp | 2 +- util/system/datetime.h | 4 +- util/system/defaults.c | 2 - util/system/defaults.h | 2 +- util/system/defaults_ut.c | 2 + util/system/demangle_impl.h | 2 +- util/system/direct_io.cpp | 2 +- util/system/direct_io_ut.cpp | 6 +- util/system/dynlib.cpp | 6 +- util/system/dynlib.h | 16 +- util/system/env.cpp | 89 +- util/system/env.h | 74 +- util/system/env_ut.cpp | 41 +- util/system/err.cpp | 2 + util/system/error.cpp | 11 +- util/system/event.cpp | 2 +- util/system/event.h | 2 +- util/system/event_ut.cpp | 4 +- util/system/execpath.cpp | 3 +- util/system/execpath_ut.cpp | 2 +- util/system/fasttime.cpp | 2 +- util/system/file.cpp | 6 +- util/system/file.h | 32 +- util/system/file_lock.cpp | 2 +- util/system/file_ut.cpp | 125 +- util/system/filemap.cpp | 8 +- util/system/filemap.h | 17 +- util/system/filemap_ut.cpp | 6 +- util/system/flock.h | 1 - util/system/flock_ut.cpp | 2 +- util/system/fs.cpp | 2 +- util/system/fs.h | 2 +- util/system/fs_ut.cpp | 18 +- util/system/fs_win.cpp | 18 +- util/system/fs_win.h | 2 +- util/system/fs_win_ut.cpp | 5 +- util/system/fstat.cpp | 9 +- util/system/fstat_ut.cpp | 4 +- util/system/guard.h | 2 +- util/system/hi_lo.h | 2 +- util/system/hi_lo_ut.cpp | 2 +- util/system/hostname.cpp | 2 +- util/system/hostname_ut.cpp | 2 +- util/system/hp_timer.cpp | 2 +- util/system/hp_timer.h | 2 +- util/system/info.cpp | 23 +- util/system/info.h | 2 +- util/system/interrupt_signals_ut.cpp | 2 +- util/system/madvise.cpp | 2 +- util/system/mem_info.cpp | 8 +- util/system/mem_info.h | 2 +- util/system/mincore.h | 2 +- util/system/mincore_ut.cpp | 2 +- util/system/mktemp_ut.cpp | 2 +- util/system/mlock.cpp | 18 +- util/system/mlock.h | 8 +- util/system/mutex.h | 8 +- util/system/nice_ut.cpp | 2 +- util/system/pipe.h | 2 +- util/system/pipe_ut.cpp | 2 +- util/system/platform.h | 43 +- util/system/progname.cpp | 2 +- util/system/progname_ut.cpp | 2 +- util/system/protect.cpp | 3 +- util/system/rusage.cpp | 3 +- util/system/rusage_ut.cpp | 2 +- util/system/rwlock.cpp | 2 +- util/system/sanitizers.cpp | 2 +- util/system/sanitizers.h | 2 +- util/system/sem.cpp | 22 +- util/system/sem.h | 12 +- util/system/shellcommand.cpp | 51 +- util/system/shellcommand.h | 10 +- util/system/shellcommand_ut.cpp | 4 +- util/system/shmat.cpp | 22 +- util/system/shmat_ut.cpp | 2 +- util/system/spin_wait.cpp | 4 +- util/system/spinlock.h | 24 +- util/system/spinlock_ut.cpp | 2 +- util/system/src_location_ut.cpp | 2 +- util/system/src_root.h | 2 +- util/system/src_root_ut.cpp | 2 +- util/system/sys_alloc.h | 9 +- util/system/sysstat.cpp | 3 +- util/system/tempfile.cpp | 13 + util/system/tempfile.h | 22 +- util/system/tempfile_ut.cpp | 4 +- util/system/thread.cpp | 21 +- util/system/thread_ut.cpp | 2 +- util/system/tls.cpp | 8 +- util/system/tls.h | 20 +- util/system/tls_ut.cpp | 2 +- util/system/type_name.cpp | 20 +- util/system/type_name_ut.cpp | 6 +- util/system/types.cpp | 4 + util/system/types_ut.cpp | 2 +- util/system/unaligned_mem_ut.cpp | 4 +- util/system/user.cpp | 5 +- util/system/user_ut.cpp | 2 +- util/system/ut/stdin_osfhandle/main.cpp | 3 +- util/system/valgrind.h | 2 +- util/system/yassert.cpp | 18 +- util/system/yassert.h | 4 +- util/system/yassert_ut.cpp | 2 +- util/tests/ya_util_tests.inc | 7 +- util/thread/factory.cpp | 2 +- util/thread/lfqueue.h | 24 +- util/thread/lfqueue_ut.cpp | 2 +- util/thread/lfstack.h | 15 +- util/thread/lfstack_ut.cpp | 2 +- util/thread/pool.cpp | 39 +- util/thread/pool.h | 10 +- util/thread/pool_ut.cpp | 2 +- util/thread/singleton.h | 2 +- util/thread/singleton_ut.cpp | 4 +- util/ysafeptr.cpp | 7 + util/ysafeptr.h | 45 +- util/ysafeptr_ut.cpp | 128 ++ util/ysaveload.h | 25 +- util/ysaveload_ut.cpp | 229 ++- 926 files changed, 31320 insertions(+), 6658 deletions(-) create mode 100644 contrib/libs/libc_compat/collate.h create mode 100644 contrib/libs/libc_compat/glob.c create mode 100644 contrib/libs/libc_compat/glob.h create mode 100644 contrib/libs/libc_compat/ifaddrs.c rename contrib/libs/libc_compat/{ => memfd_create}/memfd_create.c (100%) create mode 100644 contrib/libs/libc_compat/memfd_create/sys/mman.h rename contrib/libs/libc_compat/{include/windows => queue}/sys/queue.h (100%) create mode 100644 contrib/libs/libc_compat/unistd.h create mode 100644 contrib/libs/libc_compat/update.sh create mode 100644 contrib/libs/lzmasdk/7zVersion.h create mode 100644 library/cpp/blockcodecs/codecs/zstd/README.md create mode 100644 library/cpp/containers/cow_string/CMakeLists.txt create mode 100644 library/cpp/containers/cow_string/README.md create mode 100644 library/cpp/containers/cow_string/cow_string.cpp create mode 100644 library/cpp/containers/cow_string/cow_string.h create mode 100644 library/cpp/containers/cow_string/cow_string_ut.cpp create mode 100644 library/cpp/containers/cow_string/output.cpp create mode 100644 library/cpp/containers/cow_string/reverse.cpp create mode 100644 library/cpp/containers/cow_string/reverse.h create mode 100644 library/cpp/containers/cow_string/str_stl.h create mode 100644 library/cpp/containers/cow_string/subst.cpp create mode 100644 library/cpp/containers/cow_string/subst.h create mode 100644 library/cpp/containers/cow_string/ut_medium/cow_string_medium_ut.cpp create mode 100644 library/cpp/containers/cow_string/ysaveload.cpp create mode 100644 library/cpp/containers/cow_string/ysaveload.h create mode 100644 library/cpp/containers/paged_vector/README.md create mode 100644 library/cpp/html/escape/CMakeLists.txt create mode 100644 library/cpp/html/escape/escape.cpp create mode 100644 library/cpp/html/escape/escape.h create mode 100644 library/cpp/html/escape/ut/escape_ut.cpp create mode 100644 library/cpp/int128/ut/i128_find_ut.cpp create mode 100644 library/cpp/json/ut/json_reader_nan_ut.cpp create mode 100644 library/cpp/json/writer/fwd.h create mode 100644 library/cpp/monlib/encode/json/ut/expected_buffered_memOnly.json create mode 100644 library/cpp/monlib/encode/json/ut/expected_memOnly.json delete mode 100644 library/cpp/openssl/init/init.h create mode 100644 library/cpp/streams/brotli/const.h create mode 100644 library/cpp/streams/brotli/dictionary.h rename library/cpp/string_utils/quote/{ => ut}/quote_ut.cpp (99%) create mode 100644 library/cpp/testing/hook/yt_initialize_hook.h create mode 100644 library/cpp/threading/cancellation/README.md create mode 100644 library/cpp/threading/cancellation/cancellation_token.cpp create mode 100644 library/cpp/threading/cancellation/cancellation_token.h create mode 100644 library/cpp/threading/cancellation/ut.cpp create mode 100644 library/cpp/threading/future/ut_gtest/coroutine_traits_adl_ut.cpp create mode 100644 library/cpp/threading/future/ut_gtest/simple_task.h create mode 100644 library/cpp/yt/coding/bit_io-inl.h create mode 100644 library/cpp/yt/coding/bit_io.h create mode 100644 library/cpp/yt/coding/interpolative-inl.h create mode 100644 library/cpp/yt/coding/interpolative.h create mode 100644 library/cpp/yt/coding/unittests/bit_io_ut.cpp create mode 100644 library/cpp/yt/coding/unittests/interpolative_ut.cpp create mode 100644 library/cpp/yt/compact_containers/compact_flat_map-inl.h rename library/cpp/yt/{small_containers => compact_containers}/compact_flat_map.h (55%) create mode 100644 library/cpp/yt/compact_containers/compact_flat_set-inl.h create mode 100644 library/cpp/yt/compact_containers/compact_flat_set.h rename library/cpp/yt/{small_containers => compact_containers}/compact_heap-inl.h (100%) rename library/cpp/yt/{small_containers => compact_containers}/compact_heap.h (100%) rename library/cpp/yt/{small_containers => compact_containers}/compact_queue-inl.h (100%) rename library/cpp/yt/{small_containers => compact_containers}/compact_queue.h (100%) rename library/cpp/yt/{small_containers => compact_containers}/compact_set-inl.h (86%) rename library/cpp/yt/{small_containers => compact_containers}/compact_set.h (82%) rename library/cpp/yt/{small_containers => compact_containers}/compact_vector-inl.h (99%) rename library/cpp/yt/{small_containers => compact_containers}/compact_vector.h (98%) rename library/cpp/yt/{small_containers => compact_containers}/unittests/compact_flat_map_ut.cpp (99%) create mode 100644 library/cpp/yt/compact_containers/unittests/compact_flat_set_ut.cpp rename library/cpp/yt/{small_containers => compact_containers}/unittests/compact_heap_ut.cpp (97%) rename library/cpp/yt/{small_containers => compact_containers}/unittests/compact_queue_ut.cpp (97%) rename library/cpp/yt/{small_containers => compact_containers}/unittests/compact_set_ut.cpp (98%) rename library/cpp/yt/{small_containers => compact_containers}/unittests/compact_vector_ut.cpp (97%) create mode 100644 library/cpp/yt/containers/default_map-inl.h create mode 100644 library/cpp/yt/containers/default_map.h create mode 100644 library/cpp/yt/containers/expiring_set-inl.h create mode 100644 library/cpp/yt/containers/expiring_set.h create mode 100644 library/cpp/yt/containers/non_empty-inl.h create mode 100644 library/cpp/yt/containers/non_empty.h create mode 100644 library/cpp/yt/containers/ordered_hash_map-inl.h create mode 100644 library/cpp/yt/containers/ordered_hash_map.h create mode 100644 library/cpp/yt/containers/ring_queue.h create mode 100644 library/cpp/yt/containers/sentinel_optional-inl.h create mode 100644 library/cpp/yt/containers/sentinel_optional.h create mode 100644 library/cpp/yt/containers/slot_map-inl.h create mode 100644 library/cpp/yt/containers/slot_map.h create mode 100644 library/cpp/yt/containers/static_ring_queue-inl.h create mode 100644 library/cpp/yt/containers/static_ring_queue.h create mode 100644 library/cpp/yt/containers/unittests/default_map_ut.cpp create mode 100644 library/cpp/yt/containers/unittests/expiring_set_ut.cpp create mode 100644 library/cpp/yt/containers/unittests/non_empty_ut.cpp create mode 100644 library/cpp/yt/containers/unittests/ordered_hash_map_ut.cpp create mode 100644 library/cpp/yt/containers/unittests/ring_queue_ut.cpp create mode 100644 library/cpp/yt/containers/unittests/sentinel_optional_ut.cpp create mode 100644 library/cpp/yt/containers/unittests/slot_map_ut.cpp create mode 100644 library/cpp/yt/containers/unittests/static_ring_queue_ut.cpp create mode 100644 library/cpp/yt/memory/allocation_tags.h create mode 100644 library/cpp/yt/memory/allocation_tags_hooks.cpp create mode 100644 library/cpp/yt/memory/allocation_tags_hooks.h create mode 100644 library/cpp/yt/memory/atomic-inl.h create mode 100644 library/cpp/yt/memory/atomic.h create mode 100644 library/cpp/yt/memory/exact_ref_counted_cast-inl.h create mode 100644 library/cpp/yt/memory/exact_ref_counted_cast.h create mode 100644 library/cpp/yt/memory/new.cpp create mode 100644 library/cpp/yt/memory/non_null_ptr-inl.h create mode 100644 library/cpp/yt/memory/non_null_ptr.h create mode 100644 library/cpp/yt/memory/poison-inl.h create mode 100644 library/cpp/yt/memory/poison.cpp create mode 100644 library/cpp/yt/memory/poison.h create mode 100644 library/cpp/yt/memory/simple_memory_usage_tracker.cpp create mode 100644 library/cpp/yt/memory/simple_memory_usage_tracker.h create mode 100644 library/cpp/yt/memory/type_erasure.h create mode 100644 library/cpp/yt/memory/type_erasure_detail.h create mode 100644 library/cpp/yt/memory/unittests/atomic_ut.cpp create mode 100644 library/cpp/yt/memory/unittests/exact_ref_counted_cast_ut.cpp create mode 100644 library/cpp/yt/memory/unittests/non_null_ptr_ut.cpp create mode 100644 library/cpp/yt/memory/unittests/range_protobuf_repeated_field_ut.cpp create mode 100644 library/cpp/yt/memory/unittests/type_erasure_ut.cpp create mode 100644 library/cpp/yt/memory/weak_ptr-inl.h create mode 100644 library/cpp/yt/misc/compare-inl.h create mode 100644 library/cpp/yt/misc/compare.h create mode 100644 library/cpp/yt/misc/numeric_helpers-inl.h create mode 100644 library/cpp/yt/misc/numeric_helpers.h create mode 100644 library/cpp/yt/misc/range_formatters-inl.h create mode 100644 library/cpp/yt/misc/range_formatters.h create mode 100644 library/cpp/yt/misc/range_helpers-inl.h create mode 100644 library/cpp/yt/misc/range_helpers.h create mode 100644 library/cpp/yt/misc/static_initializer.h create mode 100644 library/cpp/yt/misc/strong_typedef-fwd.h create mode 100644 library/cpp/yt/misc/tag_invoke.h create mode 100644 library/cpp/yt/misc/tag_invoke_cpo.h create mode 100644 library/cpp/yt/misc/typeid-inl.h create mode 100644 library/cpp/yt/misc/typeid.h create mode 100644 library/cpp/yt/misc/unittests/cast_ut.cpp create mode 100644 library/cpp/yt/misc/unittests/compare_ut.cpp create mode 100644 library/cpp/yt/misc/unittests/hash_ut.cpp create mode 100644 library/cpp/yt/misc/unittests/range_helpers_ut.cpp create mode 100644 library/cpp/yt/misc/unittests/tag_invoke_cpo_ut.cpp create mode 100644 library/cpp/yt/misc/unittests/tag_invoke_impl_ut.cpp create mode 100644 library/cpp/yt/misc/unittests/typeid_sample.cpp create mode 100644 library/cpp/yt/misc/unittests/typeid_sample.h create mode 100644 library/cpp/yt/misc/unittests/typeid_ut.cpp delete mode 100644 library/cpp/yt/misc/variant-inl.h delete mode 100644 library/cpp/yt/misc/variant.cpp delete mode 100644 library/cpp/yt/small_containers/compact_flat_map-inl.h delete mode 100644 library/cpp/yt/string/format_analyser-inl.h create mode 100644 library/cpp/yt/string/stream.cpp create mode 100644 library/cpp/yt/string/stream.h create mode 100644 library/cpp/yt/string/string_builder-inl.h create mode 100644 library/cpp/yt/string/string_builder.cpp create mode 100644 library/cpp/yt/string/unittests/stream_ut.cpp create mode 100644 library/cpp/yt/system/benchmarks/cpu_id.cpp create mode 100644 library/cpp/yt/system/benchmarks/process.cpp create mode 100644 library/cpp/yt/system/benchmarks/thread.cpp create mode 100644 library/cpp/yt/system/benchmarks/tscp.cpp create mode 100644 library/cpp/yt/system/cpu_id-inl.h create mode 100644 library/cpp/yt/system/cpu_id.cpp create mode 100644 library/cpp/yt/system/cpu_id.h create mode 100644 library/cpp/yt/system/env-inl.h create mode 100644 library/cpp/yt/system/env.cpp create mode 100644 library/cpp/yt/system/env.h create mode 100644 library/cpp/yt/system/exit-inl.h create mode 100644 library/cpp/yt/system/exit.cpp create mode 100644 library/cpp/yt/system/exit.h create mode 100644 library/cpp/yt/system/handle_eintr-inl.h create mode 100644 library/cpp/yt/system/handle_eintr.h create mode 100644 library/cpp/yt/system/proc.h create mode 100644 library/cpp/yt/system/process_id-inl.h create mode 100644 library/cpp/yt/system/process_id.cpp create mode 100644 library/cpp/yt/system/process_id.h create mode 100644 library/cpp/yt/system/thread_id-inl.h create mode 100644 library/cpp/yt/system/thread_id.cpp create mode 100644 library/cpp/yt/system/thread_id.h rename library/cpp/yt/{misc => system}/thread_name.cpp (96%) rename library/cpp/yt/{misc => system}/thread_name.h (93%) create mode 100644 library/cpp/yt/system/tscp-inl.h create mode 100644 library/cpp/yt/system/tscp.h create mode 100644 library/cpp/yt/system/unittests/cpu_id_ut.cpp create mode 100644 library/cpp/yt/system/unittests/env_ut.cpp create mode 100644 tools/enum_parser/parse_enum/ut/digit_separator.h create mode 100644 util/digest/city_streaming.h create mode 100644 util/folder/tempdir_ut.cpp delete mode 100644 util/generic/bt_exception.cpp delete mode 100644 util/generic/bt_exception.h create mode 100644 util/generic/enum_cast.cpp create mode 100644 util/generic/enum_cast.h create mode 100644 util/generic/enum_cast_ut.cpp create mode 100644 util/generic/enum_cast_ut.h delete mode 100644 util/generic/enum_range.cpp delete mode 100644 util/generic/enum_range.h delete mode 100644 util/generic/enum_range_ut.cpp create mode 100644 util/linters.make.inc delete mode 100644 util/stream/debug.cpp delete mode 100644 util/stream/debug.h create mode 100644 util/stream/output_ut.cpp create mode 100644 util/system/compiler_ut.c delete mode 100644 util/system/defaults.c create mode 100644 util/system/defaults_ut.c create mode 100644 util/ysafeptr_ut.cpp diff --git a/.github/actions/prepare_vm/action.yaml b/.github/actions/prepare_vm/action.yaml index 1fff4159de..2e6f36d43e 100644 --- a/.github/actions/prepare_vm/action.yaml +++ b/.github/actions/prepare_vm/action.yaml @@ -39,9 +39,9 @@ runs: wget https://apt.llvm.org/llvm.sh chmod u+x llvm.sh - sudo ./llvm.sh 17 - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 10000 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 10000 + sudo ./llvm.sh 18 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 10000 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 10000 # Install abseil-cpp wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz diff --git a/.github/scripts/copy_sources.sh b/.github/scripts/copy_sources.sh index 9813b2434d..28f5ca8c8a 100755 --- a/.github/scripts/copy_sources.sh +++ b/.github/scripts/copy_sources.sh @@ -1,10 +1,158 @@ #!/bin/bash +set -euo pipefail tmp_dir=$(mktemp -d) +sync_upstream_tree() { + local upstream_repo=$1 + local oss_repo=$2 + local destination_root=$3 + local tree=$4 + local mode=${5:-full} + local previous_commit + local current_commit + local merge_dir + local status + local path + local new_path + local local_file + local local_target + local base_file + local upstream_file + local merged_file + local conflicts=0 + + previous_commit=$(cat "$oss_repo/.github/last_commit.txt") + current_commit=$(git -C "$upstream_repo" rev-parse HEAD) + merge_dir=$(mktemp -d "$tmp_dir/upstream-merge.XXXXXX") + + if ! git -C "$upstream_repo" merge-base --is-ancestor "$previous_commit" "$current_commit"; then + echo "Cannot sync $tree: $previous_commit is not an ancestor of $current_commit" >&2 + return 1 + fi + + while IFS=$'\t' read -r status path new_path; do + [ -n "$path" ] || continue + + # Standalone builds use CMake and deliberately do not import Arcadia build files. + if [ "$(basename "$path")" = "ya.make" ] || + { [ -n "${new_path:-}" ] && [ "$(basename "$new_path")" = "ya.make" ]; }; then + continue + fi + + local_file="$destination_root/$path" + base_file="$merge_dir/base" + upstream_file="$merge_dir/upstream" + merged_file="$merge_dir/merged" + + case "$status" in + R*) + local_target="$destination_root/$new_path" + if [ ! -f "$local_file" ]; then + if [ "$mode" = "managed" ]; then + continue + fi + echo "Cannot rename upstream file missing locally: $path" >&2 + conflicts=$((conflicts + 1)) + continue + fi + if [ -e "$local_target" ] && [ "$local_target" != "$local_file" ]; then + echo "Cannot rename upstream file onto existing local file: $new_path" >&2 + conflicts=$((conflicts + 1)) + continue + fi + + git -C "$upstream_repo" show "$previous_commit:$path" > "$base_file" + git -C "$upstream_repo" show "$current_commit:$new_path" > "$upstream_file" + mkdir -p "$(dirname "$local_target")" + + if cmp -s "$local_file" "$base_file"; then + cp "$upstream_file" "$local_target" + elif git merge-file -p "$local_file" "$base_file" "$upstream_file" > "$merged_file"; then + cp "$merged_file" "$local_target" + else + echo "Cannot merge renamed upstream and standalone changes: $path -> $new_path" >&2 + conflicts=$((conflicts + 1)) + continue + fi + + if [ "$local_target" != "$local_file" ]; then + rm "$local_file" + fi + ;; + A) + if [ "$mode" = "managed" ] && [ ! -d "$(dirname "$local_file")" ]; then + continue + fi + if [ ! -e "$local_file" ]; then + mkdir -p "$(dirname "$local_file")" + git -C "$upstream_repo" show "$current_commit:$path" > "$local_file" + else + git -C "$upstream_repo" show "$current_commit:$path" > "$upstream_file" + if ! cmp -s "$local_file" "$upstream_file"; then + echo "Cannot import added upstream file modified locally: $path" >&2 + conflicts=$((conflicts + 1)) + fi + fi + ;; + D) + if [ -e "$local_file" ]; then + git -C "$upstream_repo" show "$previous_commit:$path" > "$base_file" + if cmp -s "$local_file" "$base_file"; then + rm "$local_file" + else + echo "Cannot delete upstream file modified locally: $path" >&2 + conflicts=$((conflicts + 1)) + fi + fi + ;; + M) + if [ ! -f "$local_file" ]; then + if [ "$mode" = "managed" ]; then + continue + fi + echo "Cannot update upstream file missing locally: $path" >&2 + conflicts=$((conflicts + 1)) + continue + fi + + git -C "$upstream_repo" show "$previous_commit:$path" > "$base_file" + git -C "$upstream_repo" show "$current_commit:$path" > "$upstream_file" + + if cmp -s "$local_file" "$upstream_file"; then + continue + fi + + if cmp -s "$local_file" "$base_file"; then + cp "$upstream_file" "$local_file" + continue + fi + + if git merge-file -p "$local_file" "$base_file" "$upstream_file" > "$merged_file"; then + cp "$merged_file" "$local_file" + else + echo "Cannot merge upstream and standalone changes: $path" >&2 + conflicts=$((conflicts + 1)) + fi + ;; + *) + echo "Unsupported upstream change '$status' for $path" >&2 + conflicts=$((conflicts + 1)) + ;; + esac + done < <(git -C "$upstream_repo" diff --name-status --find-renames "$previous_commit..$current_commit" -- "$tree") + + rm -rf "$merge_dir" + + if [ "$conflicts" -ne 0 ]; then + echo "Failed to import $tree: $conflicts conflicting change(s)" >&2 + return 1 + fi +} + echo "Copying sources..." -cp -r $1/ydb/public/sdk/cpp/* $tmp_dir +cp -r "$1"/ydb/public/sdk/cpp/* "$tmp_dir" echo "tmp_dir: $tmp_dir" rm -r $tmp_dir/src/client/arrow @@ -48,6 +196,13 @@ cp -r $2/scripts $tmp_dir cp -r $2/third_party $tmp_dir cp -r $2/tools $tmp_dir +sync_upstream_tree "$1" "$2" "$tmp_dir" util +sync_upstream_tree "$1" "$2" "$tmp_dir" library/cpp managed +sync_upstream_tree "$1" "$2" "$tmp_dir" contrib/libs/libc_compat managed +sync_upstream_tree "$1" "$2" "$tmp_dir" contrib/libs/lzmasdk managed +sync_upstream_tree "$1" "$2" "$tmp_dir" tools/enum_parser managed +sync_upstream_tree "$1" "$2" "$tmp_dir" tools/rescompiler managed + cp $2/.gitignore $tmp_dir cp $2/.gitmodules $tmp_dir cp $2/CMakePresets.json $tmp_dir @@ -63,6 +218,7 @@ for oss_test_dir in slo_workloads deb_package; do done cp $2/include/ydb-cpp-sdk/type_switcher.h $tmp_dir/include/ydb-cpp-sdk/type_switcher.h +cp $2/include/ydb-cpp-sdk/stlfwd.h $tmp_dir/include/ydb-cpp-sdk/stlfwd.h cp $2/src/version.h $tmp_dir/src/version.h cd $2 diff --git a/.github/workflows/import.yaml b/.github/workflows/import.yaml index d1fe9f0074..ba9e7dc717 100644 --- a/.github/workflows/import.yaml +++ b/.github/workflows/import.yaml @@ -32,7 +32,15 @@ jobs: LAST_COMMIT=$(cat ydb-cpp-sdk/.github/last_commit.txt) - COMMITS=$(git -C ydb log --format="%H" --reverse $LAST_COMMIT..main -- ydb/public/sdk/cpp ydb/public/api) + COMMITS=$(git -C ydb log --format="%H" --reverse $LAST_COMMIT..main -- \ + ydb/public/sdk/cpp \ + ydb/public/api \ + util \ + library/cpp \ + contrib/libs/libc_compat \ + contrib/libs/lzmasdk \ + tools/enum_parser \ + tools/rescompiler) if [ -z "$COMMITS" ]; then echo "No new commits to import" exit 0 diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e9fa4883d..72b66aad07 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,13 @@ if (YDB_SDK_INSTALL) get_property(PackageIncludeDirs GLOBAL PROPERTY YDB_CPP_${PackageProp}_INCLUDE_DIRS) list(REMOVE_DUPLICATES PackageSources) + if (NOT TargetName STREQUAL "libydb-cpp") + get_property(CoreSources GLOBAL PROPERTY YDB_CPP_CORE_SOURCES) + if (CoreSources) + list(REMOVE_DUPLICATES CoreSources) + list(REMOVE_ITEM PackageSources ${CoreSources}) + endif() + endif() if (PackageInternalDeps) list(REMOVE_DUPLICATES PackageInternalDeps) endif() @@ -149,6 +156,9 @@ if (YDB_SDK_INSTALL) set_target_properties(${TargetName} PROPERTIES OUTPUT_NAME ydb-cpp) endif() target_compile_definitions(${TargetName} PUBLIC YDB_SDK_OSS ${PackagePublicDefs}) + if (NOT TargetName STREQUAL "libydb-cpp") + target_link_libraries(${TargetName} PUBLIC libydb-cpp) + endif() target_link_libraries(${TargetName} PUBLIC ${PackagePublicDeps}) if (PackageInternalDeps) add_dependencies(${TargetName} ${PackageInternalDeps}) diff --git a/README.md b/README.md index 13a29bace5..41d1790d19 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ### Prerequisites - cmake 3.22+ -- clang 17+ +- clang 18+ - git 2.20+ - ninja 1.10+ - ragel @@ -44,8 +44,9 @@ ### Install dependencies -The standalone dependency bundle uses gRPC 1.60.2 to match the imported YDB sources. -Its protobuf and Abseil pins match the dependency set published with that gRPC release: +The standalone dependency bundle is compatible with Ubuntu 24.04 and uses +gRPC 1.60.2 to match the imported YDB sources. Its protobuf and Abseil pins +match the dependency set published with that gRPC release: | Dependency | Version | |------------|---------| @@ -53,6 +54,9 @@ Its protobuf and Abseil pins match the dependency set published with that gRPC r | protobuf | 25.0 | | gRPC | 1.60.2 | +These pins are shared by regular CI builds, SLO workload images, and the +development container. + ```bash sudo apt-get -y update sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ @@ -61,7 +65,7 @@ sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-de wget https://apt.llvm.org/llvm.sh chmod u+x llvm.sh -sudo ./llvm.sh 17 +sudo ./llvm.sh 18 # Install abseil-cpp wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz diff --git a/cmake/public_headers.txt b/cmake/public_headers.txt index d926140d27..9591d3df57 100644 --- a/cmake/public_headers.txt +++ b/cmake/public_headers.txt @@ -100,7 +100,6 @@ util/generic/algorithm.h util/generic/array_size.h util/generic/array_ref.h util/generic/bitops.h -util/generic/bt_exception.h util/generic/buffer.h util/generic/cast.h util/generic/deque.h @@ -113,6 +112,7 @@ util/generic/hash_table.h util/generic/hash.h util/generic/intrlist.h util/generic/is_in.h +util/generic/iterator.h util/generic/iterator_range.h util/generic/list.h util/generic/map.h @@ -149,7 +149,6 @@ util/network/ip.h util/network/sock.h util/network/socket.h util/random/random.h -util/stream/debug.h util/stream/fwd.h util/stream/input.h util/stream/labeled.h @@ -204,4 +203,4 @@ util/thread/factory.h util/thread/fwd.h util/thread/pool.h util/str_stl.h -util/ysaveload.h \ No newline at end of file +util/ysaveload.h diff --git a/contrib/libs/libc_compat/CMakeLists.txt b/contrib/libs/libc_compat/CMakeLists.txt index ddce16d2ce..e296c7c8cf 100644 --- a/contrib/libs/libc_compat/CMakeLists.txt +++ b/contrib/libs/libc_compat/CMakeLists.txt @@ -5,13 +5,12 @@ target_compile_options(contrib-libs-libc_compat PRIVATE ) target_sources(contrib-libs-libc_compat PRIVATE - ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/string.c + ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/string.c ) if(NOT APPLE) target_sources(contrib-libs-libc_compat PRIVATE ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/explicit_bzero.c - ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/memfd_create.c ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/strlcat.c ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/strlcpy.c ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/reallocarray/reallocarray.c diff --git a/contrib/libs/libc_compat/README.md b/contrib/libs/libc_compat/README.md index fe7a22fbb1..4f6a3b85f3 100644 --- a/contrib/libs/libc_compat/README.md +++ b/contrib/libs/libc_compat/README.md @@ -9,6 +9,8 @@ During development one can make use of the following mapping of `OS_SDK` into gl | Ubuntu | glibc | | ------ | ----- | +| 24.04 | 2.39 | +| 22.04 | 2.35 | | 20.04 | 2.30 | | 18.04 | 2.27 | | 16.04 | 2.23 | diff --git a/contrib/libs/libc_compat/collate.h b/contrib/libs/libc_compat/collate.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/libs/libc_compat/glob.c b/contrib/libs/libc_compat/glob.c new file mode 100644 index 0000000000..00be637158 --- /dev/null +++ b/contrib/libs/libc_compat/glob.c @@ -0,0 +1,1127 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1989, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Guido van Rossum. + * + * Copyright (c) 2011 The FreeBSD Foundation + * All rights reserved. + * Portions of this software were developed by David Chisnall + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__SCCSID("@(#)glob.c 8.3 (Berkeley) 10/13/93"); +__FBSDID("$FreeBSD$"); + +/* + * glob(3) -- a superset of the one defined in POSIX 1003.2. + * + * The [!...] convention to negate a range is supported (SysV, Posix, ksh). + * + * Optional extra services, controlled by flags not defined by POSIX: + * + * GLOB_QUOTE: + * Escaping convention: \ inhibits any special meaning the following + * character might have (except \ at end of string is retained). + * GLOB_MAGCHAR: + * Set in gl_flags if pattern contained a globbing character. + * GLOB_NOMAGIC: + * Same as GLOB_NOCHECK, but it will only append pattern if it did + * not contain any magic characters. [Used in csh style globbing] + * GLOB_ALTDIRFUNC: + * Use alternately specified directory access functions. + * GLOB_TILDE: + * expand ~user/foo to the /home/dir/of/user/foo + * GLOB_BRACE: + * expand {1,2}{a,b} to 1a 1b 2a 2b + * gl_matchc: + * Number of matches in the current invocation of glob. + */ + +/* + * Some notes on multibyte character support: + * 1. Patterns with illegal byte sequences match nothing - even if + * GLOB_NOCHECK is specified. + * 2. Illegal byte sequences in filenames are handled by treating them as + * single-byte characters with a values of such bytes of the sequence + * cast to wchar_t. + * 3. State-dependent encodings are not currently supported. + */ + +#include +#include + +#include +#include +#include +#include "glob.h" +#include +#include +#include +#include +#include "stdlib.h" +#include +#include "unistd.h" +#include + +#ifdef USE_LOCALE_COLLATE +#include "collate.h" +#endif + +/* + * glob(3) expansion limits. Stop the expansion if any of these limits + * is reached. This caps the runtime in the face of DoS attacks. See + * also CVE-2010-2632 + */ +#define GLOB_LIMIT_BRACE 128 /* number of brace calls */ +#define GLOB_LIMIT_PATH 65536 /* number of path elements */ +#define GLOB_LIMIT_READDIR 16384 /* number of readdirs */ +#define GLOB_LIMIT_STAT 1024 /* number of stat system calls */ +#define GLOB_LIMIT_STRING ARG_MAX /* maximum total size for paths */ + +struct glob_limit { + size_t l_brace_cnt; + size_t l_path_lim; + size_t l_readdir_cnt; + size_t l_stat_cnt; + size_t l_string_cnt; +}; + +#define DOT L'.' +#define EOS L'\0' +#define LBRACKET L'[' +#define NOT L'!' +#define QUESTION L'?' +#define QUOTE L'\\' +#define RANGE L'-' +#define RBRACKET L']' +#define SEP L'/' +#define STAR L'*' +#define TILDE L'~' +#define LBRACE L'{' +#define RBRACE L'}' +#define COMMA L',' + +#define M_QUOTE 0x8000000000ULL +#define M_PROTECT 0x4000000000ULL +#define M_MASK 0xffffffffffULL +#define M_CHAR 0x00ffffffffULL + +typedef uint_fast64_t Char; + +#define CHAR(c) ((Char)((c)&M_CHAR)) +#define META(c) ((Char)((c)|M_QUOTE)) +#define UNPROT(c) ((c) & ~M_PROTECT) +#define M_ALL META(L'*') +#define M_END META(L']') +#define M_NOT META(L'!') +#define M_ONE META(L'?') +#define M_RNG META(L'-') +#define M_SET META(L'[') +#define ismeta(c) (((c)&M_QUOTE) != 0) +#ifdef DEBUG +#define isprot(c) (((c)&M_PROTECT) != 0) +#endif + +static int compare(const void *, const void *); +static int g_Ctoc(const Char *, char *, size_t); +static int g_lstat(Char *, struct stat *, glob_t *); +static DIR *g_opendir(Char *, glob_t *); +static const Char *g_strchr(const Char *, wchar_t); +#ifdef notdef +static Char *g_strcat(Char *, const Char *); +#endif +static int g_stat(Char *, struct stat *, glob_t *); +static int glob0(const Char *, glob_t *, struct glob_limit *, + const char *); +static int glob1(Char *, glob_t *, struct glob_limit *); +static int glob2(Char *, Char *, Char *, Char *, glob_t *, + struct glob_limit *); +static int glob3(Char *, Char *, Char *, Char *, Char *, glob_t *, + struct glob_limit *); +static int globextend(const Char *, glob_t *, struct glob_limit *, + const char *); +static const Char * + globtilde(const Char *, Char *, size_t, glob_t *); +static int globexp0(const Char *, glob_t *, struct glob_limit *, + const char *); +static int globexp1(const Char *, glob_t *, struct glob_limit *); +static int globexp2(const Char *, const Char *, glob_t *, + struct glob_limit *); +static int globfinal(glob_t *, struct glob_limit *, size_t, + const char *); +static int match(Char *, Char *, Char *); +static int err_nomatch(glob_t *, struct glob_limit *, const char *); +static int err_aborted(glob_t *, int, char *); +#ifdef DEBUG +static void qprintf(const char *, Char *); +#endif + +int +glob(const char * __restrict pattern, int flags, + int (*errfunc)(const char *, int), glob_t * __restrict pglob) +{ + struct glob_limit limit = { 0, 0, 0, 0, 0 }; + const char *patnext; + Char *bufnext, *bufend, patbuf[MAXPATHLEN], prot; + mbstate_t mbs; + wchar_t wc; + size_t clen; + int too_long; + + patnext = pattern; + if (!(flags & GLOB_APPEND)) { + pglob->gl_pathc = 0; + pglob->gl_pathv = NULL; + if (!(flags & GLOB_DOOFFS)) + pglob->gl_offs = 0; + } + if (flags & GLOB_LIMIT) { + limit.l_path_lim = pglob->gl_matchc; + if (limit.l_path_lim == 0) + limit.l_path_lim = GLOB_LIMIT_PATH; + } + pglob->gl_flags = flags & ~GLOB_MAGCHAR; + pglob->gl_errfunc = errfunc; + pglob->gl_matchc = 0; + + bufnext = patbuf; + bufend = bufnext + MAXPATHLEN - 1; + too_long = 1; + if (flags & GLOB_NOESCAPE) { + memset(&mbs, 0, sizeof(mbs)); + while (bufnext <= bufend) { + clen = mbrtowc(&wc, patnext, MB_LEN_MAX, &mbs); + if (clen == (size_t)-1 || clen == (size_t)-2) + return (err_nomatch(pglob, &limit, pattern)); + else if (clen == 0) { + too_long = 0; + break; + } + *bufnext++ = wc; + patnext += clen; + } + } else { + /* Protect the quoted characters. */ + memset(&mbs, 0, sizeof(mbs)); + while (bufnext <= bufend) { + if (*patnext == '\\') { + if (*++patnext == '\0') { + *bufnext++ = QUOTE; + continue; + } + prot = M_PROTECT; + } else + prot = 0; + clen = mbrtowc(&wc, patnext, MB_LEN_MAX, &mbs); + if (clen == (size_t)-1 || clen == (size_t)-2) + return (err_nomatch(pglob, &limit, pattern)); + else if (clen == 0) { + too_long = 0; + break; + } + *bufnext++ = wc | prot; + patnext += clen; + } + } + if (too_long) + return (err_nomatch(pglob, &limit, pattern)); + *bufnext = EOS; + + if (flags & GLOB_BRACE) + return (globexp0(patbuf, pglob, &limit, pattern)); + else + return (glob0(patbuf, pglob, &limit, pattern)); +} + +static int +globexp0(const Char *pattern, glob_t *pglob, struct glob_limit *limit, + const char *origpat) { + int rv; + size_t oldpathc; + + /* Protect a single {}, for find(1), like csh */ + if (pattern[0] == LBRACE && pattern[1] == RBRACE && pattern[2] == EOS) { + if ((pglob->gl_flags & GLOB_LIMIT) && + limit->l_brace_cnt++ >= GLOB_LIMIT_BRACE) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + return (glob0(pattern, pglob, limit, origpat)); + } + + oldpathc = pglob->gl_pathc; + + if ((rv = globexp1(pattern, pglob, limit)) != 0) + return rv; + + return (globfinal(pglob, limit, oldpathc, origpat)); +} + +/* + * Expand recursively a glob {} pattern. When there is no more expansion + * invoke the standard globbing routine to glob the rest of the magic + * characters + */ +static int +globexp1(const Char *pattern, glob_t *pglob, struct glob_limit *limit) +{ + const Char* ptr; + + if ((ptr = g_strchr(pattern, LBRACE)) != NULL) { + if ((pglob->gl_flags & GLOB_LIMIT) && + limit->l_brace_cnt++ >= GLOB_LIMIT_BRACE) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + return (globexp2(ptr, pattern, pglob, limit)); + } + + return (glob0(pattern, pglob, limit, NULL)); +} + + +/* + * Recursive brace globbing helper. Tries to expand a single brace. + * If it succeeds then it invokes globexp1 with the new pattern. + * If it fails then it tries to glob the rest of the pattern and returns. + */ +static int +globexp2(const Char *ptr, const Char *pattern, glob_t *pglob, + struct glob_limit *limit) +{ + int i, rv; + Char *lm, *ls; + const Char *pe, *pm, *pm1, *pl; + Char patbuf[MAXPATHLEN]; + + /* copy part up to the brace */ + for (lm = patbuf, pm = pattern; pm != ptr; *lm++ = *pm++) + continue; + *lm = EOS; + ls = lm; + + /* Find the balanced brace */ + for (i = 0, pe = ++ptr; *pe != EOS; pe++) + if (*pe == LBRACKET) { + /* Ignore everything between [] */ + for (pm = pe++; *pe != RBRACKET && *pe != EOS; pe++) + continue; + if (*pe == EOS) { + /* + * We could not find a matching RBRACKET. + * Ignore and just look for RBRACE + */ + pe = pm; + } + } + else if (*pe == LBRACE) + i++; + else if (*pe == RBRACE) { + if (i == 0) + break; + i--; + } + + /* Non matching braces; just glob the pattern */ + if (i != 0 || *pe == EOS) + return (glob0(pattern, pglob, limit, NULL)); + + for (i = 0, pl = pm = ptr; pm <= pe; pm++) + switch (*pm) { + case LBRACKET: + /* Ignore everything between [] */ + for (pm1 = pm++; *pm != RBRACKET && *pm != EOS; pm++) + continue; + if (*pm == EOS) { + /* + * We could not find a matching RBRACKET. + * Ignore and just look for RBRACE + */ + pm = pm1; + } + break; + + case LBRACE: + i++; + break; + + case RBRACE: + if (i) { + i--; + break; + } + /* FALLTHROUGH */ + case COMMA: + if (i && *pm == COMMA) + break; + else { + /* Append the current string */ + for (lm = ls; (pl < pm); *lm++ = *pl++) + continue; + /* + * Append the rest of the pattern after the + * closing brace + */ + for (pl = pe + 1; (*lm++ = *pl++) != EOS;) + continue; + + /* Expand the current pattern */ +#ifdef DEBUG + qprintf("globexp2:", patbuf); +#endif + rv = globexp1(patbuf, pglob, limit); + if (rv) + return (rv); + + /* move after the comma, to the next string */ + pl = pm + 1; + } + break; + + default: + break; + } + return (0); +} + + + +/* + * expand tilde from the passwd file. + */ +static const Char * +globtilde(const Char *pattern, Char *patbuf, size_t patbuf_len, glob_t *pglob) +{ + struct passwd *pwd; + char *h, *sc; + const Char *p; + Char *b, *eb; + wchar_t wc; + wchar_t wbuf[MAXPATHLEN]; + wchar_t *wbufend, *dc; + size_t clen; + mbstate_t mbs; + int too_long; + + if (*pattern != TILDE || !(pglob->gl_flags & GLOB_TILDE)) + return (pattern); + + /* + * Copy up to the end of the string or / + */ + eb = &patbuf[patbuf_len - 1]; + for (p = pattern + 1, b = patbuf; + b < eb && *p != EOS && UNPROT(*p) != SEP; *b++ = *p++) + continue; + + if (*p != EOS && UNPROT(*p) != SEP) + return (NULL); + + *b = EOS; + h = NULL; + + if (patbuf[0] == EOS) { + /* + * handle a plain ~ or ~/ by expanding $HOME first (iff + * we're not running setuid or setgid) and then trying + * the password file + */ + if (issetugid() != 0 || + (h = getenv("HOME")) == NULL) { + if (((h = getlogin()) != NULL && + (pwd = getpwnam(h)) != NULL) || + (pwd = getpwuid(getuid())) != NULL) + h = pwd->pw_dir; + else + return (pattern); + } + } + else { + /* + * Expand a ~user + */ + if (g_Ctoc(patbuf, (char *)wbuf, sizeof(wbuf))) + return (NULL); + if ((pwd = getpwnam((char *)wbuf)) == NULL) + return (pattern); + else + h = pwd->pw_dir; + } + + /* Copy the home directory */ + dc = wbuf; + sc = h; + wbufend = wbuf + MAXPATHLEN - 1; + too_long = 1; + memset(&mbs, 0, sizeof(mbs)); + while (dc <= wbufend) { + clen = mbrtowc(&wc, sc, MB_LEN_MAX, &mbs); + if (clen == (size_t)-1 || clen == (size_t)-2) { + /* XXX See initial comment #2. */ + wc = (unsigned char)*sc; + clen = 1; + memset(&mbs, 0, sizeof(mbs)); + } + if ((*dc++ = wc) == EOS) { + too_long = 0; + break; + } + sc += clen; + } + if (too_long) + return (NULL); + + dc = wbuf; + for (b = patbuf; b < eb && *dc != EOS; *b++ = *dc++ | M_PROTECT) + continue; + if (*dc != EOS) + return (NULL); + + /* Append the rest of the pattern */ + if (*p != EOS) { + too_long = 1; + while (b <= eb) { + if ((*b++ = *p++) == EOS) { + too_long = 0; + break; + } + } + if (too_long) + return (NULL); + } else + *b = EOS; + + return (patbuf); +} + + +/* + * The main glob() routine: compiles the pattern (optionally processing + * quotes), calls glob1() to do the real pattern matching, and finally + * sorts the list (unless unsorted operation is requested). Returns 0 + * if things went well, nonzero if errors occurred. + */ +static int +glob0(const Char *pattern, glob_t *pglob, struct glob_limit *limit, + const char *origpat) { + const Char *qpatnext; + int err; + size_t oldpathc; + Char *bufnext, c, patbuf[MAXPATHLEN]; + + qpatnext = globtilde(pattern, patbuf, MAXPATHLEN, pglob); + if (qpatnext == NULL) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + oldpathc = pglob->gl_pathc; + bufnext = patbuf; + + /* We don't need to check for buffer overflow any more. */ + while ((c = *qpatnext++) != EOS) { + switch (c) { + case LBRACKET: + c = *qpatnext; + if (c == NOT) + ++qpatnext; + if (*qpatnext == EOS || + g_strchr(qpatnext+1, RBRACKET) == NULL) { + *bufnext++ = LBRACKET; + if (c == NOT) + --qpatnext; + break; + } + *bufnext++ = M_SET; + if (c == NOT) + *bufnext++ = M_NOT; + c = *qpatnext++; + do { + *bufnext++ = CHAR(c); + if (*qpatnext == RANGE && + (c = qpatnext[1]) != RBRACKET) { + *bufnext++ = M_RNG; + *bufnext++ = CHAR(c); + qpatnext += 2; + } + } while ((c = *qpatnext++) != RBRACKET); + pglob->gl_flags |= GLOB_MAGCHAR; + *bufnext++ = M_END; + break; + case QUESTION: + pglob->gl_flags |= GLOB_MAGCHAR; + *bufnext++ = M_ONE; + break; + case STAR: + pglob->gl_flags |= GLOB_MAGCHAR; + /* collapse adjacent stars to one, + * to ensure "**" at the end continues to match the + * empty string + */ + if (bufnext == patbuf || bufnext[-1] != M_ALL) + *bufnext++ = M_ALL; + break; + default: + *bufnext++ = CHAR(c); + break; + } + } + *bufnext = EOS; +#ifdef DEBUG + qprintf("glob0:", patbuf); +#endif + + if ((err = glob1(patbuf, pglob, limit)) != 0) + return(err); + + if (origpat != NULL) + return (globfinal(pglob, limit, oldpathc, origpat)); + + return (0); +} + +static int +globfinal(glob_t *pglob, struct glob_limit *limit, size_t oldpathc, + const char *origpat) { + if (pglob->gl_pathc == oldpathc) + return (err_nomatch(pglob, limit, origpat)); + + if (!(pglob->gl_flags & GLOB_NOSORT)) + qsort(pglob->gl_pathv + pglob->gl_offs + oldpathc, + pglob->gl_pathc - oldpathc, sizeof(char *), compare); + + return (0); +} + +static int +compare(const void *p, const void *q) +{ + return (strcoll(*(char **)p, *(char **)q)); +} + +static int +glob1(Char *pattern, glob_t *pglob, struct glob_limit *limit) +{ + Char pathbuf[MAXPATHLEN]; + + /* A null pathname is invalid -- POSIX 1003.1 sect. 2.4. */ + if (*pattern == EOS) + return (0); + return (glob2(pathbuf, pathbuf, pathbuf + MAXPATHLEN - 1, + pattern, pglob, limit)); +} + +/* + * The functions glob2 and glob3 are mutually recursive; there is one level + * of recursion for each segment in the pattern that contains one or more + * meta characters. + */ +static int +glob2(Char *pathbuf, Char *pathend, Char *pathend_last, Char *pattern, + glob_t *pglob, struct glob_limit *limit) +{ + struct stat sb; + Char *p, *q; + int anymeta; + + /* + * Loop over pattern segments until end of pattern or until + * segment with meta character found. + */ + for (anymeta = 0;;) { + if (*pattern == EOS) { /* End of pattern? */ + *pathend = EOS; + if (g_lstat(pathbuf, &sb, pglob)) + return (0); + + if ((pglob->gl_flags & GLOB_LIMIT) && + limit->l_stat_cnt++ >= GLOB_LIMIT_STAT) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + if ((pglob->gl_flags & GLOB_MARK) && + UNPROT(pathend[-1]) != SEP && + (S_ISDIR(sb.st_mode) || + (S_ISLNK(sb.st_mode) && + g_stat(pathbuf, &sb, pglob) == 0 && + S_ISDIR(sb.st_mode)))) { + if (pathend + 1 > pathend_last) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + *pathend++ = SEP; + *pathend = EOS; + } + ++pglob->gl_matchc; + return (globextend(pathbuf, pglob, limit, NULL)); + } + + /* Find end of next segment, copy tentatively to pathend. */ + q = pathend; + p = pattern; + while (*p != EOS && UNPROT(*p) != SEP) { + if (ismeta(*p)) + anymeta = 1; + if (q + 1 > pathend_last) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + *q++ = *p++; + } + + if (!anymeta) { /* No expansion, do next segment. */ + pathend = q; + pattern = p; + while (UNPROT(*pattern) == SEP) { + if (pathend + 1 > pathend_last) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + *pathend++ = *pattern++; + } + } else /* Need expansion, recurse. */ + return (glob3(pathbuf, pathend, pathend_last, pattern, + p, pglob, limit)); + } + /* NOTREACHED */ +} + +static int +glob3(Char *pathbuf, Char *pathend, Char *pathend_last, + Char *pattern, Char *restpattern, + glob_t *pglob, struct glob_limit *limit) +{ + struct dirent *dp; + DIR *dirp; + int err, too_long, saverrno, saverrno2; + char buf[MAXPATHLEN + MB_LEN_MAX - 1]; + + struct dirent *(*readdirfunc)(DIR *); + + if (pathend > pathend_last) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + *pathend = EOS; + if (pglob->gl_errfunc != NULL && + g_Ctoc(pathbuf, buf, sizeof(buf))) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + + saverrno = errno; + errno = 0; + if ((dirp = g_opendir(pathbuf, pglob)) == NULL) { + if (errno == ENOENT || errno == ENOTDIR) + return (0); + err = err_aborted(pglob, errno, buf); + if (errno == 0) + errno = saverrno; + return (err); + } + + err = 0; + + /* pglob->gl_readdir takes a void *, fix this manually */ + if (pglob->gl_flags & GLOB_ALTDIRFUNC) + readdirfunc = (struct dirent *(*)(DIR *))pglob->gl_readdir; + else + readdirfunc = readdir; + + errno = 0; + /* Search directory for matching names. */ + while ((dp = (*readdirfunc)(dirp)) != NULL) { + char *sc; + Char *dc; + wchar_t wc; + size_t clen; + mbstate_t mbs; + + if ((pglob->gl_flags & GLOB_LIMIT) && + limit->l_readdir_cnt++ >= GLOB_LIMIT_READDIR) { + errno = E2BIG; + err = GLOB_NOSPACE; + break; + } + + /* Initial DOT must be matched literally. */ + if (dp->d_name[0] == '.' && UNPROT(*pattern) != DOT) { + errno = 0; + continue; + } + memset(&mbs, 0, sizeof(mbs)); + dc = pathend; + sc = dp->d_name; + too_long = 1; + while (dc <= pathend_last) { + clen = mbrtowc(&wc, sc, MB_LEN_MAX, &mbs); + if (clen == (size_t)-1 || clen == (size_t)-2) { + /* XXX See initial comment #2. */ + wc = (unsigned char)*sc; + clen = 1; + memset(&mbs, 0, sizeof(mbs)); + } + if ((*dc++ = wc) == EOS) { + too_long = 0; + break; + } + sc += clen; + } + if (too_long && (err = err_aborted(pglob, ENAMETOOLONG, + buf))) { + errno = ENAMETOOLONG; + break; + } + if (too_long || !match(pathend, pattern, restpattern)) { + *pathend = EOS; + errno = 0; + continue; + } + if (errno == 0) + errno = saverrno; + err = glob2(pathbuf, --dc, pathend_last, restpattern, + pglob, limit); + if (err) + break; + errno = 0; + } + + saverrno2 = errno; + if (pglob->gl_flags & GLOB_ALTDIRFUNC) + (*pglob->gl_closedir)(dirp); + else + closedir(dirp); + errno = saverrno2; + + if (err) + return (err); + + if (dp == NULL && errno != 0 && + (err = err_aborted(pglob, errno, buf))) + return (err); + + if (errno == 0) + errno = saverrno; + return (0); +} + + +/* + * Extend the gl_pathv member of a glob_t structure to accommodate a new item, + * add the new item, and update gl_pathc. + * + * This assumes the BSD realloc, which only copies the block when its size + * crosses a power-of-two boundary; for v7 realloc, this would cause quadratic + * behavior. + * + * Return 0 if new item added, error code if memory couldn't be allocated. + * + * Invariant of the glob_t structure: + * Either gl_pathc is zero and gl_pathv is NULL; or gl_pathc > 0 and + * gl_pathv points to (gl_offs + gl_pathc + 1) items. + */ +static int +globextend(const Char *path, glob_t *pglob, struct glob_limit *limit, + const char *origpat) +{ + char **pathv; + size_t i, newn, len; + char *copy; + const Char *p; + + if ((pglob->gl_flags & GLOB_LIMIT) && + pglob->gl_matchc > limit->l_path_lim) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + + newn = 2 + pglob->gl_pathc + pglob->gl_offs; + /* reallocarray(NULL, newn, size) is equivalent to malloc(newn*size). */ + pathv = reallocarray(pglob->gl_pathv, newn, sizeof(*pathv)); + if (pathv == NULL) + return (GLOB_NOSPACE); + + if (pglob->gl_pathv == NULL && pglob->gl_offs > 0) { + /* first time around -- clear initial gl_offs items */ + pathv += pglob->gl_offs; + for (i = pglob->gl_offs + 1; --i > 0; ) + *--pathv = NULL; + } + pglob->gl_pathv = pathv; + + if (origpat != NULL) + copy = strdup(origpat); + else { + for (p = path; *p++ != EOS;) + continue; + len = MB_CUR_MAX * (size_t)(p - path); /* XXX overallocation */ + if ((copy = malloc(len)) != NULL) { + if (g_Ctoc(path, copy, len)) { + free(copy); + errno = E2BIG; + return (GLOB_NOSPACE); + } + } + } + if (copy != NULL) { + limit->l_string_cnt += strlen(copy) + 1; + if ((pglob->gl_flags & GLOB_LIMIT) && + limit->l_string_cnt >= GLOB_LIMIT_STRING) { + free(copy); + errno = E2BIG; + return (GLOB_NOSPACE); + } + pathv[pglob->gl_offs + pglob->gl_pathc++] = copy; + } + pathv[pglob->gl_offs + pglob->gl_pathc] = NULL; + return (copy == NULL ? GLOB_NOSPACE : 0); +} + +/* + * pattern matching function for filenames. + */ +static int +match(Char *name, Char *pat, Char *patend) +{ + int ok, negate_range; + Char c, k, *nextp, *nextn; +#ifdef USE_LOCALE_COLLATE + struct xlocale_collate *table = + (struct xlocale_collate*)__get_locale()->components[XLC_COLLATE]; +#endif + + nextn = NULL; + nextp = NULL; + + while (1) { + while (pat < patend) { + c = *pat++; + switch (c & M_MASK) { + case M_ALL: + if (pat == patend) + return (1); + if (*name == EOS) + return (0); + nextn = name + 1; + nextp = pat - 1; + break; + case M_ONE: + if (*name++ == EOS) + goto fail; + break; + case M_SET: + ok = 0; + if ((k = *name++) == EOS) + goto fail; + negate_range = ((*pat & M_MASK) == M_NOT); + if (negate_range != 0) + ++pat; + while (((c = *pat++) & M_MASK) != M_END) + if ((*pat & M_MASK) == M_RNG) { +#ifdef USE_LOCALE_COLLATE + if (table->__collate_load_error ? + CHAR(c) <= CHAR(k) && + CHAR(k) <= CHAR(pat[1]) : + __wcollate_range_cmp(CHAR(c), + CHAR(k)) <= 0 && + __wcollate_range_cmp(CHAR(k), + CHAR(pat[1])) <= 0) +#else + if (c <= k && k <= pat[1]) +#endif + ok = 1; + pat += 2; + } else if (c == k) + ok = 1; + if (ok == negate_range) + goto fail; + break; + default: + if (*name++ != c) + goto fail; + break; + } + } + if (*name == EOS) + return (1); + + fail: + if (nextn == NULL) + break; + pat = nextp; + name = nextn; + } + return (0); +} + +/* Free allocated data belonging to a glob_t structure. */ +void +globfree(glob_t *pglob) +{ + size_t i; + char **pp; + + if (pglob->gl_pathv != NULL) { + pp = pglob->gl_pathv + pglob->gl_offs; + for (i = pglob->gl_pathc; i--; ++pp) + if (*pp) + free(*pp); + free(pglob->gl_pathv); + pglob->gl_pathv = NULL; + } +} + +static DIR * +g_opendir(Char *str, glob_t *pglob) +{ + char buf[MAXPATHLEN + MB_LEN_MAX - 1]; + + if (*str == EOS) + strcpy(buf, "."); + else { + if (g_Ctoc(str, buf, sizeof(buf))) { + errno = ENAMETOOLONG; + return (NULL); + } + } + + if (pglob->gl_flags & GLOB_ALTDIRFUNC) + return ((*pglob->gl_opendir)(buf)); + + return (opendir(buf)); +} + +static int +g_lstat(Char *fn, struct stat *sb, glob_t *pglob) +{ + char buf[MAXPATHLEN + MB_LEN_MAX - 1]; + + if (g_Ctoc(fn, buf, sizeof(buf))) { + errno = ENAMETOOLONG; + return (-1); + } + if (pglob->gl_flags & GLOB_ALTDIRFUNC) + return((*pglob->gl_lstat)(buf, sb)); + return (lstat(buf, sb)); +} + +static int +g_stat(Char *fn, struct stat *sb, glob_t *pglob) +{ + char buf[MAXPATHLEN + MB_LEN_MAX - 1]; + + if (g_Ctoc(fn, buf, sizeof(buf))) { + errno = ENAMETOOLONG; + return (-1); + } + if (pglob->gl_flags & GLOB_ALTDIRFUNC) + return ((*pglob->gl_stat)(buf, sb)); + return (stat(buf, sb)); +} + +static const Char * +g_strchr(const Char *str, wchar_t ch) +{ + + do { + if (*str == ch) + return (str); + } while (*str++); + return (NULL); +} + +static int +g_Ctoc(const Char *str, char *buf, size_t len) +{ + mbstate_t mbs; + size_t clen; + + memset(&mbs, 0, sizeof(mbs)); + while (len >= MB_CUR_MAX) { + clen = wcrtomb(buf, CHAR(*str), &mbs); + if (clen == (size_t)-1) { + /* XXX See initial comment #2. */ + *buf = (char)CHAR(*str); + clen = 1; + memset(&mbs, 0, sizeof(mbs)); + } + if (CHAR(*str) == EOS) + return (0); + str++; + buf += clen; + len -= clen; + } + return (1); +} + +static int +err_nomatch(glob_t *pglob, struct glob_limit *limit, const char *origpat) { + /* + * If there was no match we are going to append the origpat + * if GLOB_NOCHECK was specified or if GLOB_NOMAGIC was specified + * and the origpat did not contain any magic characters + * GLOB_NOMAGIC is there just for compatibility with csh. + */ + if ((pglob->gl_flags & GLOB_NOCHECK) || + ((pglob->gl_flags & GLOB_NOMAGIC) && + !(pglob->gl_flags & GLOB_MAGCHAR))) + return (globextend(NULL, pglob, limit, origpat)); + return (GLOB_NOMATCH); +} + +static int +err_aborted(glob_t *pglob, int err, char *buf) { + if ((pglob->gl_errfunc != NULL && pglob->gl_errfunc(buf, err)) || + (pglob->gl_flags & GLOB_ERR)) + return (GLOB_ABORTED); + return (0); +} + +#ifdef DEBUG +static void +qprintf(const char *str, Char *s) +{ + Char *p; + + (void)printf("%s\n", str); + if (s != NULL) { + for (p = s; *p != EOS; p++) + (void)printf("%c", (char)CHAR(*p)); + (void)printf("\n"); + for (p = s; *p != EOS; p++) + (void)printf("%c", (isprot(*p) ? '\\' : ' ')); + (void)printf("\n"); + for (p = s; *p != EOS; p++) + (void)printf("%c", (ismeta(*p) ? '_' : ' ')); + (void)printf("\n"); + } +} +#endif diff --git a/contrib/libs/libc_compat/glob.h b/contrib/libs/libc_compat/glob.h new file mode 100644 index 0000000000..783ba51042 --- /dev/null +++ b/contrib/libs/libc_compat/glob.h @@ -0,0 +1,104 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1989, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Guido van Rossum. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)glob.h 8.1 (Berkeley) 6/2/93 + * $FreeBSD$ + */ + +#ifndef _GLOB_H_ +#define _GLOB_H_ + +#include +#include + + +struct stat; +typedef struct { + size_t gl_pathc; /* Count of total paths so far. */ + size_t gl_matchc; /* Count of paths matching pattern. */ + size_t gl_offs; /* Reserved at beginning of gl_pathv. */ + int gl_flags; /* Copy of flags parameter to glob. */ + char **gl_pathv; /* List of paths matching pattern. */ + /* Copy of errfunc parameter to glob. */ + int (*gl_errfunc)(const char *, int); + + /* + * Alternate filesystem access methods for glob; replacement + * versions of closedir(3), readdir(3), opendir(3), stat(2) + * and lstat(2). + */ + void (*gl_closedir)(void *); + struct dirent *(*gl_readdir)(void *); + void *(*gl_opendir)(const char *); + int (*gl_lstat)(const char *, struct stat *); + int (*gl_stat)(const char *, struct stat *); +} glob_t; + +// #if __POSIX_VISIBLE >= 199209 +/* Believed to have been introduced in 1003.2-1992 */ +#define GLOB_APPEND 0x0001 /* Append to output from previous call. */ +#define GLOB_DOOFFS 0x0002 /* Use gl_offs. */ +#define GLOB_ERR 0x0004 /* Return on error. */ +#define GLOB_MARK 0x0008 /* Append / to matching directories. */ +#define GLOB_NOCHECK 0x0010 /* Return pattern itself if nothing matches. */ +#define GLOB_NOSORT 0x0020 /* Don't sort. */ +#define GLOB_NOESCAPE 0x2000 /* Disable backslash escaping. */ + +/* Error values returned by glob(3) */ +#define GLOB_NOSPACE (-1) /* Malloc call failed. */ +#define GLOB_ABORTED (-2) /* Unignored error. */ +#define GLOB_NOMATCH (-3) /* No match and GLOB_NOCHECK was not set. */ +#define GLOB_NOSYS (-4) /* Obsolete: source comptability only. */ +// #endif /* __POSIX_VISIBLE >= 199209 */ + +// #if __BSD_VISIBLE +#define GLOB_ALTDIRFUNC 0x0040 /* Use alternately specified directory funcs. */ +#define GLOB_BRACE 0x0080 /* Expand braces ala csh. */ +#define GLOB_MAGCHAR 0x0100 /* Pattern had globbing characters. */ +#define GLOB_NOMAGIC 0x0200 /* GLOB_NOCHECK without magic chars (csh). */ +#define GLOB_QUOTE 0x0400 /* Quote special chars with \. */ +#define GLOB_TILDE 0x0800 /* Expand tilde names from the passwd file. */ +#define GLOB_LIMIT 0x1000 /* limit number of returned paths */ + +/* source compatibility, these are the old names */ +#define GLOB_MAXPATH GLOB_LIMIT +#define GLOB_ABEND GLOB_ABORTED +// #endif /* __BSD_VISIBLE */ + +__BEGIN_DECLS +int glob(const char * __restrict, int, + int (*)(const char *, int), glob_t * __restrict); +void globfree(glob_t *); +__END_DECLS + +#endif /* !_GLOB_H_ */ diff --git a/contrib/libs/libc_compat/ifaddrs.c b/contrib/libs/libc_compat/ifaddrs.c new file mode 100644 index 0000000000..c59d8bc745 --- /dev/null +++ b/contrib/libs/libc_compat/ifaddrs.c @@ -0,0 +1,663 @@ +/* +Copyright (c) 2013, Kenneth MacKay +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "ifaddrs.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct NetlinkList +{ + struct NetlinkList *m_next; + struct nlmsghdr *m_data; + unsigned int m_size; +} NetlinkList; + +static int netlink_socket(void) +{ + int l_socket = socket(PF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if(l_socket < 0) + { + return -1; + } + + struct sockaddr_nl l_addr; + memset(&l_addr, 0, sizeof(l_addr)); + l_addr.nl_family = AF_NETLINK; + if(bind(l_socket, (struct sockaddr *)&l_addr, sizeof(l_addr)) < 0) + { + close(l_socket); + return -1; + } + + return l_socket; +} + +static int netlink_send(int p_socket, int p_request) +{ + struct + { + struct nlmsghdr m_hdr; + struct rtgenmsg m_msg; + } l_data; + + memset(&l_data, 0, sizeof(l_data)); + + l_data.m_hdr.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtgenmsg)); + l_data.m_hdr.nlmsg_type = p_request; + l_data.m_hdr.nlmsg_flags = NLM_F_ROOT | NLM_F_MATCH | NLM_F_REQUEST; + l_data.m_hdr.nlmsg_pid = 0; + l_data.m_hdr.nlmsg_seq = p_socket; + l_data.m_msg.rtgen_family = AF_UNSPEC; + + struct sockaddr_nl l_addr; + memset(&l_addr, 0, sizeof(l_addr)); + l_addr.nl_family = AF_NETLINK; + return (sendto(p_socket, &l_data.m_hdr, l_data.m_hdr.nlmsg_len, 0, (struct sockaddr *)&l_addr, sizeof(l_addr))); +} + +static int netlink_recv(int p_socket, void *p_buffer, size_t p_len) +{ + struct msghdr l_msg; + struct iovec l_iov = { p_buffer, p_len }; + struct sockaddr_nl l_addr; + + for(;;) + { + l_msg.msg_name = (void *)&l_addr; + l_msg.msg_namelen = sizeof(l_addr); + l_msg.msg_iov = &l_iov; + l_msg.msg_iovlen = 1; + l_msg.msg_control = NULL; + l_msg.msg_controllen = 0; + l_msg.msg_flags = 0; + int l_result = recvmsg(p_socket, &l_msg, 0); + + if(l_result < 0) + { + if(errno == EINTR) + { + continue; + } + return -2; + } + + if(l_msg.msg_flags & MSG_TRUNC) + { // buffer was too small + return -1; + } + return l_result; + } +} + +static struct nlmsghdr *getNetlinkResponse(int p_socket, int *p_size, int *p_done) +{ + size_t l_size = 4096; + void *l_buffer = NULL; + + for(;;) + { + free(l_buffer); + l_buffer = malloc(l_size); + if (l_buffer == NULL) + { + return NULL; + } + + int l_read = netlink_recv(p_socket, l_buffer, l_size); + *p_size = l_read; + if(l_read == -2) + { + free(l_buffer); + return NULL; + } + if(l_read >= 0) + { + pid_t l_pid = getpid(); + struct nlmsghdr *l_hdr; + for(l_hdr = (struct nlmsghdr *)l_buffer; NLMSG_OK(l_hdr, (unsigned int)l_read); l_hdr = (struct nlmsghdr *)NLMSG_NEXT(l_hdr, l_read)) + { + if((pid_t)l_hdr->nlmsg_pid != l_pid || (int)l_hdr->nlmsg_seq != p_socket) + { + continue; + } + + if(l_hdr->nlmsg_type == NLMSG_DONE) + { + *p_done = 1; + break; + } + + if(l_hdr->nlmsg_type == NLMSG_ERROR) + { + free(l_buffer); + return NULL; + } + } + return l_buffer; + } + + l_size *= 2; + } +} + +static NetlinkList *newListItem(struct nlmsghdr *p_data, unsigned int p_size) +{ + NetlinkList *l_item = malloc(sizeof(NetlinkList)); + if (l_item == NULL) + { + return NULL; + } + + l_item->m_next = NULL; + l_item->m_data = p_data; + l_item->m_size = p_size; + return l_item; +} + +static void freeResultList(NetlinkList *p_list) +{ + NetlinkList *l_cur; + while(p_list) + { + l_cur = p_list; + p_list = p_list->m_next; + free(l_cur->m_data); + free(l_cur); + } +} + +static NetlinkList *getResultList(int p_socket, int p_request) +{ + if(netlink_send(p_socket, p_request) < 0) + { + return NULL; + } + + NetlinkList *l_list = NULL; + NetlinkList *l_end = NULL; + int l_size; + int l_done = 0; + while(!l_done) + { + struct nlmsghdr *l_hdr = getNetlinkResponse(p_socket, &l_size, &l_done); + if(!l_hdr) + { // error + freeResultList(l_list); + return NULL; + } + + NetlinkList *l_item = newListItem(l_hdr, l_size); + if (!l_item) + { + freeResultList(l_list); + return NULL; + } + if(!l_list) + { + l_list = l_item; + } + else + { + l_end->m_next = l_item; + } + l_end = l_item; + } + return l_list; +} + +static size_t maxSize(size_t a, size_t b) +{ + return (a > b ? a : b); +} + +static size_t calcAddrLen(sa_family_t p_family, int p_dataSize) +{ + switch(p_family) + { + case AF_INET: + return sizeof(struct sockaddr_in); + case AF_INET6: + return sizeof(struct sockaddr_in6); + case AF_PACKET: + return maxSize(sizeof(struct sockaddr_ll), offsetof(struct sockaddr_ll, sll_addr) + p_dataSize); + default: + return maxSize(sizeof(struct sockaddr), offsetof(struct sockaddr, sa_data) + p_dataSize); + } +} + +static void makeSockaddr(sa_family_t p_family, struct sockaddr *p_dest, void *p_data, size_t p_size) +{ + switch(p_family) + { + case AF_INET: + memcpy(&((struct sockaddr_in*)p_dest)->sin_addr, p_data, p_size); + break; + case AF_INET6: + memcpy(&((struct sockaddr_in6*)p_dest)->sin6_addr, p_data, p_size); + break; + case AF_PACKET: + memcpy(((struct sockaddr_ll*)p_dest)->sll_addr, p_data, p_size); + ((struct sockaddr_ll*)p_dest)->sll_halen = p_size; + break; + default: + memcpy(p_dest->sa_data, p_data, p_size); + break; + } + p_dest->sa_family = p_family; +} + +static void addToEnd(struct ifaddrs **p_resultList, struct ifaddrs *p_entry) +{ + if(!*p_resultList) + { + *p_resultList = p_entry; + } + else + { + struct ifaddrs *l_cur = *p_resultList; + while(l_cur->ifa_next) + { + l_cur = l_cur->ifa_next; + } + l_cur->ifa_next = p_entry; + } +} + +static int interpretLink(struct nlmsghdr *p_hdr, struct ifaddrs **p_resultList) +{ + struct ifinfomsg *l_info = (struct ifinfomsg *)NLMSG_DATA(p_hdr); + + size_t l_nameSize = 0; + size_t l_addrSize = 0; + size_t l_dataSize = 0; + + size_t l_rtaSize = NLMSG_PAYLOAD(p_hdr, sizeof(struct ifinfomsg)); + struct rtattr *l_rta; + for(l_rta = IFLA_RTA(l_info); RTA_OK(l_rta, l_rtaSize); l_rta = RTA_NEXT(l_rta, l_rtaSize)) + { + void *l_rtaData = RTA_DATA(l_rta); + size_t l_rtaDataSize = RTA_PAYLOAD(l_rta); + switch(l_rta->rta_type) + { + case IFLA_ADDRESS: + case IFLA_BROADCAST: + l_addrSize += NLMSG_ALIGN(calcAddrLen(AF_PACKET, l_rtaDataSize)); + break; + case IFLA_IFNAME: + l_nameSize += NLMSG_ALIGN(l_rtaSize + 1); + break; + case IFLA_STATS: + l_dataSize += NLMSG_ALIGN(l_rtaSize); + break; + default: + break; + } + } + + struct ifaddrs *l_entry = malloc(sizeof(struct ifaddrs) + sizeof(int) + l_nameSize + l_addrSize + l_dataSize); + if (l_entry == NULL) + { + return -1; + } + memset(l_entry, 0, sizeof(struct ifaddrs)); + l_entry->ifa_name = ""; + + char *l_index = ((char *)l_entry) + sizeof(struct ifaddrs); + char *l_name = l_index + sizeof(int); + char *l_addr = l_name + l_nameSize; + char *l_data = l_addr + l_addrSize; + + // save the interface index so we can look it up when handling the addresses. + memcpy(l_index, &l_info->ifi_index, sizeof(int)); + + l_entry->ifa_flags = l_info->ifi_flags; + + l_rtaSize = NLMSG_PAYLOAD(p_hdr, sizeof(struct ifinfomsg)); + for(l_rta = IFLA_RTA(l_info); RTA_OK(l_rta, l_rtaSize); l_rta = RTA_NEXT(l_rta, l_rtaSize)) + { + void *l_rtaData = RTA_DATA(l_rta); + size_t l_rtaDataSize = RTA_PAYLOAD(l_rta); + switch(l_rta->rta_type) + { + case IFLA_ADDRESS: + case IFLA_BROADCAST: + { + size_t l_addrLen = calcAddrLen(AF_PACKET, l_rtaDataSize); + makeSockaddr(AF_PACKET, (struct sockaddr *)l_addr, l_rtaData, l_rtaDataSize); + ((struct sockaddr_ll *)l_addr)->sll_ifindex = l_info->ifi_index; + ((struct sockaddr_ll *)l_addr)->sll_hatype = l_info->ifi_type; + if(l_rta->rta_type == IFLA_ADDRESS) + { + l_entry->ifa_addr = (struct sockaddr *)l_addr; + } + else + { + l_entry->ifa_broadaddr = (struct sockaddr *)l_addr; + } + l_addr += NLMSG_ALIGN(l_addrLen); + break; + } + case IFLA_IFNAME: + strncpy(l_name, l_rtaData, l_rtaDataSize); + l_name[l_rtaDataSize] = '\0'; + l_entry->ifa_name = l_name; + break; + case IFLA_STATS: + memcpy(l_data, l_rtaData, l_rtaDataSize); + l_entry->ifa_data = l_data; + break; + default: + break; + } + } + + addToEnd(p_resultList, l_entry); + return 0; +} + +static struct ifaddrs *findInterface(int p_index, struct ifaddrs **p_links, int p_numLinks) +{ + int l_num = 0; + struct ifaddrs *l_cur = *p_links; + while(l_cur && l_num < p_numLinks) + { + char *l_indexPtr = ((char *)l_cur) + sizeof(struct ifaddrs); + int l_index; + memcpy(&l_index, l_indexPtr, sizeof(int)); + if(l_index == p_index) + { + return l_cur; + } + + l_cur = l_cur->ifa_next; + ++l_num; + } + return NULL; +} + +static int interpretAddr(struct nlmsghdr *p_hdr, struct ifaddrs **p_resultList, int p_numLinks) +{ + struct ifaddrmsg *l_info = (struct ifaddrmsg *)NLMSG_DATA(p_hdr); + struct ifaddrs *l_interface = findInterface(l_info->ifa_index, p_resultList, p_numLinks); + + if(l_info->ifa_family == AF_PACKET) + { + return 0; + } + + size_t l_nameSize = 0; + size_t l_addrSize = 0; + + int l_addedNetmask = 0; + + size_t l_rtaSize = NLMSG_PAYLOAD(p_hdr, sizeof(struct ifaddrmsg)); + struct rtattr *l_rta; + for(l_rta = IFA_RTA(l_info); RTA_OK(l_rta, l_rtaSize); l_rta = RTA_NEXT(l_rta, l_rtaSize)) + { + void *l_rtaData = RTA_DATA(l_rta); + size_t l_rtaDataSize = RTA_PAYLOAD(l_rta); + + switch(l_rta->rta_type) + { + case IFA_ADDRESS: + case IFA_LOCAL: + if((l_info->ifa_family == AF_INET || l_info->ifa_family == AF_INET6) && !l_addedNetmask) + { // make room for netmask + l_addrSize += NLMSG_ALIGN(calcAddrLen(l_info->ifa_family, l_rtaDataSize)); + l_addedNetmask = 1; + } + case IFA_BROADCAST: + l_addrSize += NLMSG_ALIGN(calcAddrLen(l_info->ifa_family, l_rtaDataSize)); + break; + case IFA_LABEL: + l_nameSize += NLMSG_ALIGN(l_rtaSize + 1); + break; + default: + break; + } + } + + struct ifaddrs *l_entry = malloc(sizeof(struct ifaddrs) + l_nameSize + l_addrSize); + if (l_entry == NULL) + { + return -1; + } + memset(l_entry, 0, sizeof(struct ifaddrs)); + l_entry->ifa_name = (l_interface ? l_interface->ifa_name : ""); + + char *l_name = ((char *)l_entry) + sizeof(struct ifaddrs); + char *l_addr = l_name + l_nameSize; + + l_entry->ifa_flags = l_info->ifa_flags; + if(l_interface) + { + l_entry->ifa_flags |= l_interface->ifa_flags; + } + + l_rtaSize = NLMSG_PAYLOAD(p_hdr, sizeof(struct ifaddrmsg)); + for(l_rta = IFA_RTA(l_info); RTA_OK(l_rta, l_rtaSize); l_rta = RTA_NEXT(l_rta, l_rtaSize)) + { + void *l_rtaData = RTA_DATA(l_rta); + size_t l_rtaDataSize = RTA_PAYLOAD(l_rta); + switch(l_rta->rta_type) + { + case IFA_ADDRESS: + case IFA_BROADCAST: + case IFA_LOCAL: + { + size_t l_addrLen = calcAddrLen(l_info->ifa_family, l_rtaDataSize); + makeSockaddr(l_info->ifa_family, (struct sockaddr *)l_addr, l_rtaData, l_rtaDataSize); + if(l_info->ifa_family == AF_INET6) + { + if(IN6_IS_ADDR_LINKLOCAL((struct in6_addr *)l_rtaData) || IN6_IS_ADDR_MC_LINKLOCAL((struct in6_addr *)l_rtaData)) + { + ((struct sockaddr_in6 *)l_addr)->sin6_scope_id = l_info->ifa_index; + } + } + + if(l_rta->rta_type == IFA_ADDRESS) + { // apparently in a point-to-point network IFA_ADDRESS contains the dest address and IFA_LOCAL contains the local address + if(l_entry->ifa_addr) + { + l_entry->ifa_dstaddr = (struct sockaddr *)l_addr; + } + else + { + l_entry->ifa_addr = (struct sockaddr *)l_addr; + } + } + else if(l_rta->rta_type == IFA_LOCAL) + { + if(l_entry->ifa_addr) + { + l_entry->ifa_dstaddr = l_entry->ifa_addr; + } + l_entry->ifa_addr = (struct sockaddr *)l_addr; + } + else + { + l_entry->ifa_broadaddr = (struct sockaddr *)l_addr; + } + l_addr += NLMSG_ALIGN(l_addrLen); + break; + } + case IFA_LABEL: + strncpy(l_name, l_rtaData, l_rtaDataSize); + l_name[l_rtaDataSize] = '\0'; + l_entry->ifa_name = l_name; + break; + default: + break; + } + } + + if(l_entry->ifa_addr && (l_entry->ifa_addr->sa_family == AF_INET || l_entry->ifa_addr->sa_family == AF_INET6)) + { + unsigned l_maxPrefix = (l_entry->ifa_addr->sa_family == AF_INET ? 32 : 128); + unsigned l_prefix = (l_info->ifa_prefixlen > l_maxPrefix ? l_maxPrefix : l_info->ifa_prefixlen); + char l_mask[16] = {0}; + unsigned i; + for(i=0; i<(l_prefix/8); ++i) + { + l_mask[i] = 0xff; + } + if(l_prefix % 8) + { + l_mask[i] = 0xff << (8 - (l_prefix % 8)); + } + + makeSockaddr(l_entry->ifa_addr->sa_family, (struct sockaddr *)l_addr, l_mask, l_maxPrefix / 8); + l_entry->ifa_netmask = (struct sockaddr *)l_addr; + } + + addToEnd(p_resultList, l_entry); + return 0; +} + +static int interpretLinks(int p_socket, NetlinkList *p_netlinkList, struct ifaddrs **p_resultList) +{ + int l_numLinks = 0; + pid_t l_pid = getpid(); + for(; p_netlinkList; p_netlinkList = p_netlinkList->m_next) + { + unsigned int l_nlsize = p_netlinkList->m_size; + struct nlmsghdr *l_hdr; + for(l_hdr = p_netlinkList->m_data; NLMSG_OK(l_hdr, l_nlsize); l_hdr = NLMSG_NEXT(l_hdr, l_nlsize)) + { + if((pid_t)l_hdr->nlmsg_pid != l_pid || (int)l_hdr->nlmsg_seq != p_socket) + { + continue; + } + + if(l_hdr->nlmsg_type == NLMSG_DONE) + { + break; + } + + if(l_hdr->nlmsg_type == RTM_NEWLINK) + { + if(interpretLink(l_hdr, p_resultList) == -1) + { + return -1; + } + ++l_numLinks; + } + } + } + return l_numLinks; +} + +static int interpretAddrs(int p_socket, NetlinkList *p_netlinkList, struct ifaddrs **p_resultList, int p_numLinks) +{ + pid_t l_pid = getpid(); + for(; p_netlinkList; p_netlinkList = p_netlinkList->m_next) + { + unsigned int l_nlsize = p_netlinkList->m_size; + struct nlmsghdr *l_hdr; + for(l_hdr = p_netlinkList->m_data; NLMSG_OK(l_hdr, l_nlsize); l_hdr = NLMSG_NEXT(l_hdr, l_nlsize)) + { + if((pid_t)l_hdr->nlmsg_pid != l_pid || (int)l_hdr->nlmsg_seq != p_socket) + { + continue; + } + + if(l_hdr->nlmsg_type == NLMSG_DONE) + { + break; + } + + if(l_hdr->nlmsg_type == RTM_NEWADDR) + { + if (interpretAddr(l_hdr, p_resultList, p_numLinks) == -1) + { + return -1; + } + } + } + } + return 0; +} + +int getifaddrs(struct ifaddrs **ifap) +{ + if(!ifap) + { + return -1; + } + *ifap = NULL; + + int l_socket = netlink_socket(); + if(l_socket < 0) + { + return -1; + } + + NetlinkList *l_linkResults = getResultList(l_socket, RTM_GETLINK); + if(!l_linkResults) + { + close(l_socket); + return -1; + } + + NetlinkList *l_addrResults = getResultList(l_socket, RTM_GETADDR); + if(!l_addrResults) + { + close(l_socket); + freeResultList(l_linkResults); + return -1; + } + + int l_result = 0; + int l_numLinks = interpretLinks(l_socket, l_linkResults, ifap); + if(l_numLinks == -1 || interpretAddrs(l_socket, l_addrResults, ifap, l_numLinks) == -1) + { + l_result = -1; + } + + freeResultList(l_linkResults); + freeResultList(l_addrResults); + close(l_socket); + return l_result; +} + +void freeifaddrs(struct ifaddrs *ifa) +{ + struct ifaddrs *l_cur; + while(ifa) + { + l_cur = ifa; + ifa = ifa->ifa_next; + free(l_cur); + } +} diff --git a/contrib/libs/libc_compat/memfd_create.c b/contrib/libs/libc_compat/memfd_create/memfd_create.c similarity index 100% rename from contrib/libs/libc_compat/memfd_create.c rename to contrib/libs/libc_compat/memfd_create/memfd_create.c diff --git a/contrib/libs/libc_compat/memfd_create/sys/mman.h b/contrib/libs/libc_compat/memfd_create/sys/mman.h new file mode 100644 index 0000000000..d36e9bdbe4 --- /dev/null +++ b/contrib/libs/libc_compat/memfd_create/sys/mman.h @@ -0,0 +1,16 @@ +#pragma once + +#include_next + +#ifdef __cplusplus +extern "C" { +#endif + +#define MFD_CLOEXEC 0x0001U +#define MADV_WIPEONFORK 18 + +int memfd_create(const char *name, unsigned flags); + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/contrib/libs/libc_compat/include/windows/sys/queue.h b/contrib/libs/libc_compat/queue/sys/queue.h similarity index 100% rename from contrib/libs/libc_compat/include/windows/sys/queue.h rename to contrib/libs/libc_compat/queue/sys/queue.h diff --git a/contrib/libs/libc_compat/unistd.h b/contrib/libs/libc_compat/unistd.h new file mode 100644 index 0000000000..c62f5f29cd --- /dev/null +++ b/contrib/libs/libc_compat/unistd.h @@ -0,0 +1,3 @@ +#include + +#define issetugid() 0 diff --git a/contrib/libs/libc_compat/update.sh b/contrib/libs/libc_compat/update.sh new file mode 100644 index 0000000000..4cee160904 --- /dev/null +++ b/contrib/libs/libc_compat/update.sh @@ -0,0 +1,60 @@ +#!/bin/sh -e + +unweak() { + sed --in-place --expression 's/DEF_WEAK(.\+);//g' "$1" +} + +get_string_method() { + curl "https://raw.githubusercontent.com/openbsd/src/master/lib/libc/string/$1" --output "$1" && unweak "$1" +} + +fix_tabs() { + sed --in-place --expression 's/\t/ /g' "$1" +} + +fix_decls() { + sed --in-place --expression 's/__BEGIN_DECLS/#ifdef __cplusplus\nextern "C" {\n#endif/g' "$1" + sed --in-place --expression 's/__END_DECLS/#ifdef __cplusplus\n} \/\/ extern "C"\n#endif/g' "$1" +} + +get_string_method "strlcpy.c" +get_string_method "strlcat.c" +get_string_method "strsep.c" +# strcasestr uses strncasecmp, which is platform dependent, so include local string.h +get_string_method "strcasestr.c" && sed --in-place 's/#include /#include "string.h"/g' "strcasestr.c" +get_string_method "memrchr.c" +get_string_method "stpcpy.c" + +mkdir -p include/windows/sys +curl "https://raw.githubusercontent.com/openbsd/src/master/sys/sys/queue.h" --output "include/windows/sys/queue.h" + +mkdir -p include/readpassphrase +curl "https://raw.githubusercontent.com/openbsd/src/master/include/readpassphrase.h" --output "include/readpassphrase/readpassphrase.h" && fix_decls "include/readpassphrase/readpassphrase.h" +curl "https://raw.githubusercontent.com/openbsd/src/master/lib/libc/gen/readpassphrase.c" --output "readpassphrase.c" && unweak "readpassphrase.c" && fix_tabs "readpassphrase.c" + +curl "https://raw.githubusercontent.com/freebsd/freebsd/master/include/glob.h" --output "glob.h" +curl "https://raw.githubusercontent.com/freebsd/freebsd/master/lib/libc/gen/glob.c" --output "glob.c" +curl "https://raw.githubusercontent.com/openbsd/src/master/lib/libc/stdlib/reallocarray.c" --output "reallocarray.c" && unweak "reallocarray.c" +> "collate.h" +> "stdlib.h" +> "unistd.h" + +mkdir -p include/uchar +curl "https://git.musl-libc.org/cgit/musl/plain/include/uchar.h" --output "include/uchar/uchar.h" +# TODO: provide c16rtomb, mbrtoc16, c32rtomb, mbrtoc32 implementations for uchar +# if any code actually needs them + +mkdir -p include/random/sys +curl "https://git.musl-libc.org/cgit/musl/plain/include/sys/random.h" --output "include/random/sys/random.h" +curl "https://git.musl-libc.org/cgit/musl/plain/src/linux/getrandom.c" --output "getrandom.c" +curl "https://git.musl-libc.org/cgit/musl/plain/src/linux/memfd_create.c" --output "memfd_create.c" + +# WARN: do not use github.com/morristech/android-ifaddrs, it is a long-ago abandoned fork +curl "https://raw.githubusercontent.com/oliviertilmans/android-ifaddrs/master/ifaddrs.c" --output "ifaddrs.c" +curl "https://raw.githubusercontent.com/oliviertilmans/android-ifaddrs/master/ifaddrs.h" --output "include/ifaddrs/ifaddrs.h" + +# apply patches if necessary +for patch in patches/*.patch; do + echo "Applying patch from $patch" + patch -p1 < $patch +done diff --git a/contrib/libs/lzmasdk/7zVersion.h b/contrib/libs/lzmasdk/7zVersion.h new file mode 100644 index 0000000000..0074c64be9 --- /dev/null +++ b/contrib/libs/lzmasdk/7zVersion.h @@ -0,0 +1,27 @@ +#define MY_VER_MAJOR 19 +#define MY_VER_MINOR 00 +#define MY_VER_BUILD 0 +#define MY_VERSION_NUMBERS "19.00" +#define MY_VERSION MY_VERSION_NUMBERS + +#ifdef MY_CPU_NAME + #define MY_VERSION_CPU MY_VERSION " (" MY_CPU_NAME ")" +#else + #define MY_VERSION_CPU MY_VERSION +#endif + +#define MY_DATE "2019-02-21" +#undef MY_COPYRIGHT +#undef MY_VERSION_COPYRIGHT_DATE +#define MY_AUTHOR_NAME "Igor Pavlov" +#define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2018 Igor Pavlov" + +#ifdef USE_COPYRIGHT_CR + #define MY_COPYRIGHT MY_COPYRIGHT_CR +#else + #define MY_COPYRIGHT MY_COPYRIGHT_PD +#endif + +#define MY_COPYRIGHT_DATE MY_COPYRIGHT " : " MY_DATE +#define MY_VERSION_COPYRIGHT_DATE MY_VERSION_CPU " : " MY_COPYRIGHT " : " MY_DATE diff --git a/include/ydb-cpp-sdk/stlfwd.h b/include/ydb-cpp-sdk/stlfwd.h index 77b97233f9..03b47eed96 100644 --- a/include/ydb-cpp-sdk/stlfwd.h +++ b/include/ydb-cpp-sdk/stlfwd.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -13,3 +14,10 @@ #include #include #include + +#ifdef __cpp_lib_format +namespace std { + template + struct formatter; +} +#endif diff --git a/library/cpp/CMakeLists.txt b/library/cpp/CMakeLists.txt index b3cfe65cee..b1f5ee7602 100644 --- a/library/cpp/CMakeLists.txt +++ b/library/cpp/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(case_insensitive_string) add_subdirectory(cgiparam) add_subdirectory(charset) add_subdirectory(colorizer) +add_subdirectory(containers/cow_string) add_subdirectory(containers/disjoint_interval_tree) add_subdirectory(containers/intrusive_rb_tree) add_subdirectory(containers/paged_vector) @@ -18,6 +19,7 @@ add_subdirectory(digest/lower_case) add_subdirectory(digest/md5) add_subdirectory(digest/murmur) add_subdirectory(getopt) +add_subdirectory(html/escape) add_subdirectory(http/fetch) add_subdirectory(http/io) add_subdirectory(http/misc) diff --git a/library/cpp/blockcodecs/codecs.h b/library/cpp/blockcodecs/codecs.h index fd499b54b0..43a9244465 100644 --- a/library/cpp/blockcodecs/codecs.h +++ b/library/cpp/blockcodecs/codecs.h @@ -1,3 +1,3 @@ #pragma once -#include +#include // IWYU pragma: export diff --git a/library/cpp/blockcodecs/codecs/zstd/README.md b/library/cpp/blockcodecs/codecs/zstd/README.md new file mode 100644 index 0000000000..98236f837d --- /dev/null +++ b/library/cpp/blockcodecs/codecs/zstd/README.md @@ -0,0 +1,40 @@ +Zstd codecs +============= + +This library registers zstd compression codecs as `zstd_1`, ..., `zstd_22`. +Fast levels are also registered as `zstd_fast_1`, ..., `zstd_fast_7`. + +Measured codec performance on every level. Values below are provided just for reference, exact numbers may vary depending on CPU model and type of data being compressed. + +| Codec | Comp. Ratio | Comp. Speed (MBps) | Decomp. Speed (MBps) | +|--------------|-------------|---------------------|----------------------| +| lz4 | 0.5876 | 913 | 4100 | +| zstd_fast_7 | 0.5783 | 1066 | 2887 | +| zstd_fast_6 | 0.5733 | 1050 | 2870 | +| zstd_fast_5 | 0.5528 | 942 | 2594 | +| zstd_fast_4 | 0.5529 | 918 | 2659 | +| zstd_fast_3 | 0.5408 | 885 | 2519 | +| zstd_fast_2 | 0.5132 | 769 | 2374 | +| zstd_fast_1 | 0.5119 | 707 | 2386 | +| zstd_1 | 0.4691 | 690 | 1692 | +| zstd_2 | 0.4083 | 467 | 1496 | +| zstd_3 | 0.3505 | 358 | 1801 | +| zstd_4 | 0.3356 | 310 | 1932 | +| zstd_5 | 0.3175 | 218 | 1832 | +| zstd_6 | 0.3168 | 173 | 1920 | +| zstd_7 | 0.3081 | 149 | 1965 | +| zstd_8 | 0.3077 | 125 | 2005 | +| zstd_9 | 0.272 | 128 | 2179 | +| zstd_10 | 0.2693 | 98.5 | 2228 | +| zstd_11 | 0.2684 | 78.6 | 2185 | +| zstd_12 | 0.2682 | 71.1 | 2231 | +| zstd_13 | 0.2687 | 27.8 | 2102 | +| zstd_14 | 0.2676 | 24 | 2024 | +| zstd_15 | 0.2663 | 18.8 | 2225 | +| zstd_16 | 0.257 | 15.2 | 2093 | +| zstd_17 | 0.2521 | 12.3 | 2072 | +| zstd_18 | 0.241 | 9.68 | 1696 | +| zstd_19 | 0.2395 | 7.97 | 1709 | +| zstd_20 | 0.2337 | 6.26 | 1520 | +| zstd_21 | 0.2255 | 5.22 | 1442 | +| zstd_22 | 0.2037 | 3.88 | 1490 | diff --git a/library/cpp/blockcodecs/codecs/zstd/zstd.cpp b/library/cpp/blockcodecs/codecs/zstd/zstd.cpp index bcbc400584..02133c9c04 100644 --- a/library/cpp/blockcodecs/codecs/zstd/zstd.cpp +++ b/library/cpp/blockcodecs/codecs/zstd/zstd.cpp @@ -5,36 +5,38 @@ #define ZSTD_STATIC_LINKING_ONLY #include +#include + using namespace NBlockCodecs; namespace { struct TZStd08Codec: public TAddLengthCodec { - inline TZStd08Codec(unsigned level) + TZStd08Codec(int level, TString name) : Level(level) - , MyName(TStringBuf("zstd08_") + ToString(Level)) + , MyName(std::move(name)) { } - static inline size_t CheckError(size_t ret, const char* what) { - if (ZSTD_isError(ret)) { + static size_t CheckError(size_t ret, const char* what) { + if (Y_UNLIKELY(ZSTD_isError(ret))) { ythrow yexception() << what << TStringBuf(" zstd error: ") << ZSTD_getErrorName(ret); } return ret; } - static inline size_t DoMaxCompressedLength(size_t l) noexcept { + static size_t DoMaxCompressedLength(size_t l) noexcept { return ZSTD_compressBound(l); } - inline size_t DoCompress(const TData& in, void* out) const { + size_t DoCompress(const TData& in, void* out) const { return CheckError(ZSTD_compress(out, DoMaxCompressedLength(in.size()), in.data(), in.size(), Level), "compress"); } - inline void DoDecompress(const TData& in, void* out, size_t dsize) const { + static void DoDecompress(const TData& in, void* out, size_t dsize) { const size_t res = CheckError(ZSTD_decompress(out, dsize, in.data(), in.size()), "decompress"); - if (res != dsize) { + if (Y_UNLIKELY(res != dsize)) { ythrow TDecompressError(dsize, res); } } @@ -43,15 +45,20 @@ namespace { return MyName; } - const unsigned Level; + const int Level; const TString MyName; }; struct TZStd08Registrar { TZStd08Registrar() { for (int i = 1; i <= ZSTD_maxCLevel(); ++i) { - RegisterCodec(MakeHolder(i)); - RegisterAlias("zstd_" + ToString(i), "zstd08_" + ToString(i)); + const TString name = "zstd08_"sv + ToString(i); + RegisterCodec(MakeHolder(i, name)); + RegisterAlias("zstd_"sv + ToString(i), name); + } + + for (int i = 1; i <= 7; ++i) { + RegisterCodec(MakeHolder(-i, "zstd_fast_"sv + ToString(i))); } } }; diff --git a/library/cpp/blockcodecs/codecs_ut.cpp b/library/cpp/blockcodecs/codecs_ut.cpp index bfe5a23690..e167816680 100644 --- a/library/cpp/blockcodecs/codecs_ut.cpp +++ b/library/cpp/blockcodecs/codecs_ut.cpp @@ -320,7 +320,8 @@ Y_UNIT_TEST_SUITE(TBlockCodecsTest) { "zstd08_1,zstd08_10,zstd08_11,zstd08_12,zstd08_13,zstd08_14,zstd08_15,zstd08_16,zstd08_17,zstd08_18," "zstd08_19,zstd08_2,zstd08_20,zstd08_21,zstd08_22,zstd08_3,zstd08_4,zstd08_5,zstd08_6,zstd08_7,zstd08_8," "zstd08_9,zstd_1,zstd_10,zstd_11,zstd_12,zstd_13,zstd_14,zstd_15,zstd_16,zstd_17,zstd_18,zstd_19,zstd_2," - "zstd_20,zstd_21,zstd_22,zstd_3,zstd_4,zstd_5,zstd_6,zstd_7,zstd_8,zstd_9"; + "zstd_20,zstd_21,zstd_22,zstd_3,zstd_4,zstd_5,zstd_6,zstd_7,zstd_8,zstd_9," + "zstd_fast_1,zstd_fast_2,zstd_fast_3,zstd_fast_4,zstd_fast_5,zstd_fast_6,zstd_fast_7"; UNIT_ASSERT_VALUES_EQUAL(ALL_CODECS, JoinSeq(",", ListAllCodecs())); } diff --git a/library/cpp/blockcodecs/core/codecs.h b/library/cpp/blockcodecs/core/codecs.h index 9c93c00274..61efc04744 100644 --- a/library/cpp/blockcodecs/core/codecs.h +++ b/library/cpp/blockcodecs/core/codecs.h @@ -25,8 +25,15 @@ namespace NBlockCodecs { : TStringBuf((const char*)t.Data(), t.Size()) { } + }; + template <> + inline TData::TData(const TString& t) + : TStringBuf((const char*)t.data(), t.size()) + { + } + struct TCodecError: public yexception { }; diff --git a/library/cpp/blockcodecs/fuzz/main.cpp b/library/cpp/blockcodecs/fuzz/main.cpp index a89fee7ab7..be4d684db2 100644 --- a/library/cpp/blockcodecs/fuzz/main.cpp +++ b/library/cpp/blockcodecs/fuzz/main.cpp @@ -1,5 +1,4 @@ #include -#include #include #include diff --git a/library/cpp/cache/cache.h b/library/cpp/cache/cache.h index 9fe98d5366..25906d1a02 100644 --- a/library/cpp/cache/cache.h +++ b/library/cpp/cache/cache.h @@ -465,7 +465,7 @@ class TLWList { size_t MaxSize; }; -template > +template > class TCache { typedef typename TListType::TItem TItem; typedef typename TItem::THash THash; @@ -708,7 +708,7 @@ struct TNoopDelete { } }; -template , typename TAllocator = std::allocator> +template , typename TAllocator = std::allocator::TItem>> class TLRUCache: public TCache, TDeleter, TAllocator> { using TListType = TLRUList; typedef TCache TBase; @@ -735,7 +735,7 @@ class TLRUCache: public TCache, class TSizeProvider = TUniformSizeProvider> +template , class TSizeProvider = TUniformSizeProvider> class TLFUCache: public TCache, TDeleter, TAllocator> { typedef TCache, TDeleter, TAllocator> TBase; using TListType = TLFUList; @@ -760,7 +760,7 @@ class TLFUCache: public TCache> +template ::TItem>> class TLWCache: public TCache, TDeleter, TAllocator> { typedef TCache, TDeleter, TAllocator> TBase; using TListType = TLWList; diff --git a/library/cpp/cache/thread_safe_cache.h b/library/cpp/cache/thread_safe_cache.h index e77d1a45fd..82b6806277 100644 --- a/library/cpp/cache/thread_safe_cache.h +++ b/library/cpp/cache/thread_safe_cache.h @@ -3,6 +3,7 @@ #include "cache.h" #include +#include #include namespace NPrivate { @@ -51,12 +52,20 @@ namespace NPrivate { const TPtr GetOrNull(TArgs... args) { Key key = Callbacks.GetKey(args...); - TReadGuard r(Mutex); - auto iter = Cache.Find(key); - if (iter == Cache.End()) { - return nullptr; + switch (GettersPromotionPolicy) { + case EGettersPromotionPolicy::Promoted: { + TWriteGuard r(Mutex); + if (auto iter = Cache.Find(key); iter != Cache.End()) + return iter.Value(); + } + break; + case EGettersPromotionPolicy::Unpromoted: { + TReadGuard r(Mutex); + if (auto iter = Cache.Find(key); iter != Cache.End()) + return iter.Value(); + } } - return iter.Value(); + return nullptr; } const TPtr Get(TArgs... args) const { diff --git a/library/cpp/cache/ut/cache_ut.cpp b/library/cpp/cache/ut/cache_ut.cpp index 16f29b29d1..c4c829cdb7 100644 --- a/library/cpp/cache/ut/cache_ut.cpp +++ b/library/cpp/cache/ut/cache_ut.cpp @@ -2,6 +2,10 @@ #include #include +#include +#include +#include + struct TStrokaWeighter { static size_t Weight(const TString& s) { return s.size(); @@ -536,6 +540,53 @@ Y_UNIT_TEST_SUITE(TThreadSafeCacheTest) { } } +Y_UNIT_TEST_SUITE(TThreadSafeLRUCacheMultiThreadTest) { + typedef TThreadSafeLRUCache TCache; + + class TSimpleCallbacks: public TCache::ICallbacks { + public: + TKey GetKey(ui32 i) const override { + return i; + } + TValue* CreateObject(ui32 i) const override { + Y_UNUSED(i); + return nullptr; + } + }; + + Y_UNIT_TEST(GetOrNullMultiThreadTest) { + const size_t poolSize = 8; + const size_t passCnt = 128; + const size_t tasksCnt = 128; + + TRWMutex lock; + TThreadPool pool; + TSimpleCallbacks callbacks; + TCache cache(callbacks, poolSize); + + for (size_t i = 0; i < poolSize; ++i) { + cache.Insert(i, MakeAtomicShared(ToString(i))); + } + + pool.Start(poolSize); + { + TWriteGuard wGruard(lock); + for (size_t i = 0; i < tasksCnt; ++i) { + pool.SafeAddFunc([&lock, &cache]() { + TReadGuard rGuard(lock); + for (size_t j = 0; j < passCnt; ++j) { + UNIT_ASSERT(cache.GetOrNull(RandomNumber(poolSize)) != nullptr); + } + }); + } + } // start race + pool.Stop(); + for (size_t i = 0; i < cache.Size(); ++i) { + UNIT_ASSERT(cache.GetOrNull(i) != nullptr); + } + } +} + Y_UNIT_TEST_SUITE(TThreadSafeCacheUnsafeTest) { typedef TThreadSafeCache TCache; diff --git a/library/cpp/cgiparam/cgiparam.cpp b/library/cpp/cgiparam/cgiparam.cpp index e2c9c0dbe5..503213719e 100644 --- a/library/cpp/cgiparam/cgiparam.cpp +++ b/library/cpp/cgiparam/cgiparam.cpp @@ -11,12 +11,22 @@ TCgiParameters::TCgiParameters(std::initializer_list } } -const TString& TCgiParameters::Get(const TStringBuf name, size_t numOfValue) const noexcept { +const TString& TCgiParameters::Get(const TStringBuf name, size_t numOfValue) const noexcept Y_LIFETIME_BOUND { const auto it = Find(name, numOfValue); return end() == it ? Default() : it->second; } +const TString& TCgiParameters::GetLast(const TStringBuf name) const noexcept { + if (auto it = this->upper_bound(name); it != this->begin()) { + --it; + if (it->first == name) { + return it->second; + } + } + return Default(); +} + bool TCgiParameters::Erase(const TStringBuf name, size_t pos) { const auto pair = equal_range(name); @@ -221,7 +231,7 @@ TString TCgiParameters::QuotedPrint(const char* safe) const { return res; } -TCgiParameters::const_iterator TCgiParameters::Find(const TStringBuf name, size_t pos) const noexcept { +TCgiParameters::const_iterator TCgiParameters::Find(const TStringBuf name, size_t pos) const noexcept Y_LIFETIME_BOUND { const auto pair = equal_range(name); for (auto it = pair.first; it != pair.second; ++it, --pos) { @@ -265,7 +275,7 @@ TQuickCgiParam::TQuickCgiParam(const TStringBuf cgiParamStr) { } } -const TStringBuf& TQuickCgiParam::Get(const TStringBuf name, size_t pos) const noexcept { +TStringBuf TQuickCgiParam::Get(const TStringBuf name, size_t pos) const noexcept Y_LIFETIME_BOUND { const auto pair = equal_range(name); for (auto it = pair.first; it != pair.second; ++it, --pos) { @@ -274,7 +284,7 @@ const TStringBuf& TQuickCgiParam::Get(const TStringBuf name, size_t pos) const n } } - return Default(); + return TStringBuf{}; } bool TQuickCgiParam::Has(const TStringBuf name, const TStringBuf value) const noexcept { diff --git a/library/cpp/cgiparam/cgiparam.h b/library/cpp/cgiparam/cgiparam.h index cbb212f6f0..104e3f401a 100644 --- a/library/cpp/cgiparam/cgiparam.h +++ b/library/cpp/cgiparam/cgiparam.h @@ -61,7 +61,7 @@ class TCgiParameters: public TMultiMap { } Y_PURE_FUNCTION - const_iterator Find(const TStringBuf name, size_t numOfValue = 0) const noexcept; + const_iterator Find(const TStringBuf name, size_t numOfValue = 0) const noexcept Y_LIFETIME_BOUND; Y_PURE_FUNCTION bool Has(const TStringBuf name, const TStringBuf value) const noexcept; @@ -76,23 +76,32 @@ class TCgiParameters: public TMultiMap { * @note The returned value is CGI-unescaped. */ Y_PURE_FUNCTION - const TString& Get(const TStringBuf name, size_t numOfValue = 0) const noexcept; + const TString& Get(const TStringBuf name, size_t numOfValue = 0) const noexcept Y_LIFETIME_BOUND; + + /// Returns the last value by name + /** + * @note The returned value is CGI-unescaped. + */ + Y_PURE_FUNCTION + const TString& GetLast(const TStringBuf name) const noexcept Y_LIFETIME_BOUND; void InsertEscaped(const TStringBuf name, const TStringBuf value); #if !defined(__GLIBCXX__) template - inline void InsertUnescaped(TName&& name, TValue&& value) { + inline TCgiParameters& InsertUnescaped(TName&& name, TValue&& value) { // TStringBuf use as TName or TValue is C++17 actually. // There is no pair constructor available in C++14 when required type // is not implicitly constructible from given type. // But libc++ pair allows this with C++14. emplace(std::forward(name), std::forward(value)); + return *this; } #else template - inline void InsertUnescaped(TName&& name, TValue&& value) { + inline TCgiParameters& InsertUnescaped(TName&& name, TValue&& value) { emplace(TString(name), TString(value)); + return *this; } #endif @@ -116,24 +125,24 @@ class TCgiParameters: public TMultiMap { bool Erase(const TStringBuf name, const TStringBuf val); bool ErasePattern(const TStringBuf name, const TStringBuf pat); - inline const char* FormField(const TStringBuf name, size_t numOfValue = 0) const { + inline const char* FormField(const TStringBuf name, size_t numOfValue = 0) const Y_LIFETIME_BOUND { const_iterator it = Find(name, numOfValue); if (it == end()) { return nullptr; } - return it->second.data(); + return it->second.c_str(); } - inline TStringBuf FormFieldBuf(const TStringBuf name, size_t numOfValue = 0) const { + inline TStringBuf FormFieldBuf(const TStringBuf name, size_t numOfValue = 0) const Y_LIFETIME_BOUND { const_iterator it = Find(name, numOfValue); if (it == end()) { - return nullptr; + return TStringBuf{}; } - return it->second.data(); + return it->second; } }; @@ -181,7 +190,7 @@ class TQuickCgiParam: public TMultiMap { } Y_PURE_FUNCTION - const TStringBuf& Get(const TStringBuf name, size_t numOfValue = 0) const noexcept; + TStringBuf Get(const TStringBuf name, size_t numOfValue = 0) const noexcept Y_LIFETIME_BOUND; private: TString UnescapeBuf; diff --git a/library/cpp/cgiparam/cgiparam_ut.cpp b/library/cpp/cgiparam/cgiparam_ut.cpp index de6d23882d..c82f0344d2 100644 --- a/library/cpp/cgiparam/cgiparam_ut.cpp +++ b/library/cpp/cgiparam/cgiparam_ut.cpp @@ -210,8 +210,10 @@ Y_UNIT_TEST_SUITE(TCgiParametersTest) { UNIT_ASSERT_VALUES_EQUAL(c.NumOfValues("b"), 1u); UNIT_ASSERT_VALUES_EQUAL(c.Get("b"), "b1"); + UNIT_ASSERT_VALUES_EQUAL(c.GetLast("b"), "b1"); UNIT_ASSERT_VALUES_EQUAL(c.Get("a", 0), "a1"); UNIT_ASSERT_VALUES_EQUAL(c.Get("a", 1), "a2"); + UNIT_ASSERT_VALUES_EQUAL(c.GetLast("a"), "a2"); UNIT_ASSERT_VALUES_EQUAL(c.Print(), "a=a1&a=a2&b=b1"); } diff --git a/library/cpp/charset/wide_ut.cpp b/library/cpp/charset/wide_ut.cpp index 93567161ba..4f5a666cb8 100644 --- a/library/cpp/charset/wide_ut.cpp +++ b/library/cpp/charset/wide_ut.cpp @@ -10,6 +10,8 @@ #include +extern const int TStringUseCow; + namespace { //! three UTF8 encoded russian letters (A, B, V) const char yandexCyrillicAlphabet[] = @@ -264,10 +266,12 @@ void TConversionTest::TestRecodeIntoString() { TUtf16String copy = sUnicode; // increase ref-counter wres = NDetail::Recode(UTF8Text, sUnicode, CODES_UTF8); UNIT_ASSERT(sUnicode == UnicodeText); // same content + if (TStringUseCow) { #ifndef TSTRING_IS_STD_STRING - UNIT_ASSERT(sUnicode.data() != wdata); // re-allocated (shared buffer supplied) - UNIT_ASSERT(sUnicode.data() == wres.data()); // same buffer + UNIT_ASSERT(sUnicode.data() != wdata); // re-allocated (shared buffer supplied) + UNIT_ASSERT(sUnicode.data() == wres.data()); // same buffer #endif + } UNIT_ASSERT(sUnicode.size() == wres.size()); // same content } diff --git a/library/cpp/colorizer/ut/colorizer_ut.cpp b/library/cpp/colorizer/ut/colorizer_ut.cpp index 20341440af..2512651861 100644 --- a/library/cpp/colorizer/ut/colorizer_ut.cpp +++ b/library/cpp/colorizer/ut/colorizer_ut.cpp @@ -12,7 +12,7 @@ Y_UNIT_TEST_SUITE(ColorizerTest) { UNIT_ASSERT_STRINGS_EQUAL(EscapeC(colors.BlueColor()), "\\x1B[22;34m"); UNIT_ASSERT_STRINGS_EQUAL(EscapeC(colors.ForeBlue()), "\\x1B[34m"); colors.Disable(); - UNIT_ASSERT(colors.BlueColor().Empty()); + UNIT_ASSERT(colors.BlueColor().empty()); } Y_UNIT_TEST(ResettingTest) { diff --git a/library/cpp/containers/cow_string/CMakeLists.txt b/library/cpp/containers/cow_string/CMakeLists.txt new file mode 100644 index 0000000000..64a9371437 --- /dev/null +++ b/library/cpp/containers/cow_string/CMakeLists.txt @@ -0,0 +1,30 @@ +if (YDB_SDK_TESTS) + add_ydb_test(NAME containers-cow_string-medium-ut + SOURCES + ut_medium/cow_string_medium_ut.cpp + LINK_LIBRARIES + containers-cow_string + cpp-testing-unittest_main + LABELS + medium + ) + target_include_directories(containers-cow_string-medium-ut PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +endif() + +_ydb_sdk_add_library(containers-cow_string) + +target_link_libraries(containers-cow_string + PUBLIC + yutil +) + +target_sources(containers-cow_string + PRIVATE + cow_string.cpp + output.cpp + reverse.cpp + subst.cpp + ysaveload.cpp +) + +_ydb_sdk_install_targets(TARGETS containers-cow_string) diff --git a/library/cpp/containers/cow_string/README.md b/library/cpp/containers/cow_string/README.md new file mode 100644 index 0000000000..d6aea7c734 --- /dev/null +++ b/library/cpp/containers/cow_string/README.md @@ -0,0 +1,9 @@ +## Copy-on-Write based string implementation + +Drop in replacement for TSring in the code which deeply relies on COW-semantic. + + * `#include ` main header of the library with the COW-string class itself + * `#include ` in-place strings reverse implementation + * `#include ` comparator/hashers/... template specialization allowing to use TCowString in tree-based or hash-based sets/maps. + * `#include ` TCowString implementation of the substitution function provided for TString in ``. + * `#include ` TCowString support of the `` serialization/deserialization. diff --git a/library/cpp/containers/cow_string/cow_string.cpp b/library/cpp/containers/cow_string/cow_string.cpp new file mode 100644 index 0000000000..87dc0ad99e --- /dev/null +++ b/library/cpp/containers/cow_string/cow_string.cpp @@ -0,0 +1,280 @@ +#include "cow_string.h" + +#include +#include +#include +#include + +#include + +template +static bool ModifySequence(TCharType*& p, const TCharType* const pe, F&& f) { + while (p != pe) { + const auto symbol = ReadSymbol(p, pe); + const auto modified = f(symbol); + if (symbol != modified) { + if (stopOnFirstModification) { + return true; + } + + WriteSymbol(modified, p); // also moves `p` forward + } else { + p = SkipSymbol(p, pe); + } + } + + return false; +} + +template +static bool ModifySequence(const TCharType*& p, const TCharType* const pe, TCharType*& out, F&& f) { + while (p != pe) { + const auto symbol = stopOnFirstModification ? ReadSymbol(p, pe) : ReadSymbolAndAdvance(p, pe); + const auto modified = f(symbol); + + if (stopOnFirstModification) { + if (symbol != modified) { + return true; + } + + p = SkipSymbol(p, pe); + } + + WriteSymbol(modified, out); + } + + return false; +} + +template +static void DetachAndFixPointers(TStringType& text, typename TStringType::value_type*& p, const typename TStringType::value_type*& pe) { + const auto pos = p - text.data(); + const auto count = pe - p; + p = text.Detach() + pos; + pe = p + count; +} + +template +static bool ModifyStringSymbolwise(TStringType& text, size_t pos, size_t count, F&& f) { + // TODO(yazevnul): this is done for consistency with `TUtf16String::to_lower` and friends + // at r2914050, maybe worth replacing them with asserts. Also see the same code in `ToTitle`. + pos = pos < text.size() ? pos : text.size(); + count = count < text.size() - pos ? count : text.size() - pos; + + // TUtf16String is refcounted and it's `data` method return pointer to the constant memory. + // To simplify the code we do a `const_cast`, though first write to the memory will be done only + // after we call `Detach()` and get pointer to a writable piece of memory. + auto* p = const_cast(text.data() + pos); + const auto* pe = text.data() + pos + count; + + if (ModifySequence(p, pe, f)) { + DetachAndFixPointers(text, p, pe); + ModifySequence(p, pe, f); + return true; + } + + return false; +} + +std::ostream& operator<<(std::ostream& os, const TCowString& s) { + return os.write(s.data(), s.size()); +} + +std::istream& operator>>(std::istream& is, TCowString& s) { + return is >> s.MutRef(); +} + +template <> +bool TBasicCowString>::to_lower(size_t pos, size_t n) { + return Transform([](size_t, char c) { return AsciiToLower(c); }, pos, n); +} + +template <> +bool TBasicCowString>::to_upper(size_t pos, size_t n) { + return Transform([](size_t, char c) { return AsciiToUpper(c); }, pos, n); +} + +template <> +bool TBasicCowString>::to_title(size_t pos, size_t n) { + if (n == 0) { + return false; + } + bool changed = to_upper(pos, 1); + return to_lower(pos + 1, n - 1) || changed; +} + +template <> +TUtf16CowString& +TBasicCowString>::AppendAscii(const ::TStringBuf& s) { + ReserveAndResize(size() + s.size()); + + auto dst = begin() + size() - s.size(); + + for (const char* src = s.data(); dst != end(); ++dst, ++src) { + *dst = static_cast(*src); + } + + return *this; +} + +template <> +TUtf16CowString& +TBasicCowString>::AppendUtf8(const ::TStringBuf& s) { + size_t oldSize = size(); + ReserveAndResize(size() + s.size() * 4); + size_t written = 0; + size_t pos = UTF8ToWideImpl(s.data(), s.size(), begin() + oldSize, written); + if (pos != s.size()) { + ythrow yexception() << "failed to decode UTF-8 string at pos " << pos << ::NDetail::InStringMsg(s.data(), s.size()); + } + resize(oldSize + written); + + return *this; +} + +template <> +bool TBasicCowString>::to_lower(size_t pos, size_t n) { + const auto f = [](const wchar32 s) { return ToLower(s); }; + return ModifyStringSymbolwise(*this, pos, n, f); +} + +template <> +bool TBasicCowString>::to_upper(size_t pos, size_t n) { + const auto f = [](const wchar32 s) { return ToUpper(s); }; + return ModifyStringSymbolwise(*this, pos, n, f); +} + +template <> +bool TBasicCowString>::to_title(size_t pos, size_t nn) { + if (!*this) { + return false; + } + + pos = pos < this->size() ? pos : this->size(); + nn = nn < this->size() - pos ? nn : this->size() - pos; + + const auto toLower = [](const wchar32 s) { return ToLower(s); }; + + auto* p = const_cast(this->data() + pos); + const auto* pe = this->data() + pos + nn; + + const auto firstSymbol = ReadSymbol(p, pe); + if (firstSymbol == ToTitle(firstSymbol)) { + p = SkipSymbol(p, pe); + if (ModifySequence(p, pe, toLower)) { + DetachAndFixPointers(*this, p, pe); + ModifySequence(p, pe, toLower); + return true; + } + } else { + DetachAndFixPointers(*this, p, pe); + WriteSymbol(ToTitle(ReadSymbol(p, pe)), p); // also moves `p` forward + ModifySequence(p, pe, toLower); + return true; + } + + return false; +} + +template <> +TUtf32CowString& +TBasicCowString>::AppendAscii(const ::TStringBuf& s) { + ReserveAndResize(size() + s.size()); + + auto dst = begin() + size() - s.size(); + + for (const char* src = s.data(); dst != end(); ++dst, ++src) { + *dst = static_cast(*src); + } + + return *this; +} + +template <> +TBasicCowString>& +TBasicCowString>::AppendUtf16(const ::TWtringBuf& s) { + const size_t oldSize = size(); + ReserveAndResize(size() + WideToUTF8BufferSize(s.size())); + + size_t written = 0; + WideToUTF8(s.data(), s.size(), begin() + oldSize, written); + + resize(oldSize + written); + + return *this; +} + +template <> +TUtf32CowString& +TBasicCowString>::AppendUtf8(const ::TStringBuf& s) { + size_t oldSize = size(); + ReserveAndResize(size() + s.size() * 4); + size_t written = 0; + size_t pos = UTF8ToWideImpl(s.data(), s.size(), begin() + oldSize, written); + if (pos != s.size()) { + ythrow yexception() << "failed to decode UTF-8 string at pos " << pos << ::NDetail::InStringMsg(s.data(), s.size()); + } + resize(oldSize + written); + + return *this; +} + +template <> +TUtf32CowString& +TBasicCowString>::AppendUtf16(const ::TWtringBuf& s) { + size_t oldSize = size(); + ReserveAndResize(size() + s.size() * 2); + + wchar32* oldEnd = begin() + oldSize; + wchar32* end = oldEnd; + NDetail::UTF16ToUTF32ImplScalar(s.data(), s.data() + s.size(), end); + size_t written = end - oldEnd; + + resize(oldSize + written); + + return *this; +} + +template <> +bool TBasicCowString>::to_lower(size_t pos, size_t n) { + const auto f = [](const wchar32 s) { return ToLower(s); }; + return ModifyStringSymbolwise(*this, pos, n, f); +} + +template <> +bool TBasicCowString>::to_upper(size_t pos, size_t n) { + const auto f = [](const wchar32 s) { return ToUpper(s); }; + return ModifyStringSymbolwise(*this, pos, n, f); +} + +template <> +bool TBasicCowString>::to_title(size_t pos, size_t n) { + if (!*this) { + return false; + } + + pos = pos < this->size() ? pos : this->size(); + n = n < this->size() - pos ? n : this->size() - pos; + + const auto toLower = [](const wchar32 s) { return ToLower(s); }; + + auto* p = const_cast(this->data() + pos); + const auto* pe = this->data() + pos + n; + + const auto firstSymbol = *p; + if (firstSymbol == ToTitle(firstSymbol)) { + p += 1; + if (ModifySequence(p, pe, toLower)) { + DetachAndFixPointers(*this, p, pe); + ModifySequence(p, pe, toLower); + return true; + } + } else { + DetachAndFixPointers(*this, p, pe); + WriteSymbol(ToTitle(ReadSymbol(p, pe)), p); // also moves `p` forward + ModifySequence(p, pe, toLower); + return true; + } + + return false; +} diff --git a/library/cpp/containers/cow_string/cow_string.h b/library/cpp/containers/cow_string/cow_string.h new file mode 100644 index 0000000000..34dcff81e8 --- /dev/null +++ b/library/cpp/containers/cow_string/cow_string.h @@ -0,0 +1,1016 @@ +#pragma once + +#include + +template > +class Y_EMPTY_BASES TBasicCowString: public TStringBase, TCharType, TTraits>, + public TStdStringCompatibilityBase, TCharType, TTraits> { +public: + // TODO: Move to private section + using TBase = TStringBase; + using TStringType = std::basic_string; + using TStdStr = TStdString; + using TStorage = TIntrusivePtr>; + using reference = TBasicCharRef; + using char_type = TCharType; // TODO: DROP + using value_type = TCharType; + using traits_type = TTraits; + + using iterator = TCharType*; + using reverse_iterator = std::reverse_iterator; + using typename TBase::const_iterator; + using typename TBase::const_reference; + using typename TBase::const_reverse_iterator; + + struct TUninitialized { + explicit TUninitialized(size_t size) + : Size(size) + { + } + + size_t Size; + }; + + size_t max_size() noexcept { + static size_t res = TStringType().max_size(); + + return res; + } + +protected: + TStorage S_; + + template + static TStorage Construct(A&&... a) { + return {new TStdStr(std::forward(a)...), typename TStorage::TNoIncrement()}; + } + + static TStorage Construct() noexcept { + return TStdStr::NullStr(); + } + + TStdStr& StdStr() noexcept { + return *S_; + } + + const TStdStr& StdStr() const noexcept { + return *S_; + } + + /** + * Makes a distinct copy of this string. `IsDetached()` is always true after this call. + * + * @throw std::length_error + */ + void Clone() { + Construct(StdStr()).Swap(S_); + } + + size_t RefCount() const noexcept { + return S_.RefCount(); + } + +public: + inline const TStringType& ConstRef() const Y_LIFETIME_BOUND { + return StdStr(); + } + + inline TStringType& MutRef() Y_LIFETIME_BOUND { + Detach(); + + return StdStr(); + } + + inline const_reference operator[](size_t pos) const noexcept Y_LIFETIME_BOUND { + Y_ASSERT(pos <= length()); + + return this->data()[pos]; + } + + inline reference operator[](size_t pos) noexcept Y_LIFETIME_BOUND { + Y_ASSERT(pos <= length()); + + return reference(*this, pos); + } + + using TBase::back; + + inline reference back() noexcept Y_LIFETIME_BOUND { + Y_ASSERT(!this->empty()); + + if (Y_UNLIKELY(this->empty())) { + return reference(*this, 0); + } + + return reference(*this, length() - 1); + } + + using TBase::front; + + inline reference front() noexcept Y_LIFETIME_BOUND { + Y_ASSERT(!this->empty()); + + return reference(*this, 0); + } + + inline size_t length() const noexcept { + return ConstRef().length(); + } + + inline const TCharType* data() const noexcept Y_LIFETIME_BOUND { + return ConstRef().data(); + } + + inline const TCharType* c_str() const noexcept Y_LIFETIME_BOUND { + return ConstRef().c_str(); + } + + // ~~~ STL compatible method to obtain data pointer ~~~ + iterator begin() Y_LIFETIME_BOUND { + return &*MutRef().begin(); + } + + iterator end() Y_LIFETIME_BOUND { + return &*MutRef().end(); + } + + reverse_iterator rbegin() Y_LIFETIME_BOUND { + return reverse_iterator(end()); + } + + reverse_iterator rend() Y_LIFETIME_BOUND { + return reverse_iterator(begin()); + } + + const_iterator begin() const noexcept Y_LIFETIME_BOUND { + return TBase::begin(); + } + const_iterator cbegin() const noexcept Y_LIFETIME_BOUND { + return TBase::cbegin(); + } + + const_iterator cend() const noexcept Y_LIFETIME_BOUND { + return TBase::cend(); + } + + const_reverse_iterator crbegin() const noexcept Y_LIFETIME_BOUND { + return TBase::crbegin(); + } + + const_reverse_iterator crend() const noexcept Y_LIFETIME_BOUND { + return TBase::crend(); + } + + const_iterator end() const noexcept Y_LIFETIME_BOUND { + return TBase::end(); + } + + const_reverse_iterator rbegin() const noexcept Y_LIFETIME_BOUND { + return TBase::rbegin(); + } + + const_reverse_iterator rend() const noexcept Y_LIFETIME_BOUND { + return TBase::rend(); + } + + inline size_t capacity() const noexcept { + if (S_->IsNull()) { + return 0; + } + + return S_->capacity(); + } + + TCharType* Detach() Y_LIFETIME_BOUND { + if (Y_UNLIKELY(!IsDetached())) { + Clone(); + } + + return (TCharType*)S_->data(); + } + + bool IsDetached() const { + return 1 == RefCount(); + } + + // ~~~ Size and capacity ~~~ + TBasicCowString& resize(size_t n, TCharType c = ' ') Y_LIFETIME_BOUND { // remove or append + MutRef().resize(n, c); + + return *this; + } + + // ~~~ Constructor ~~~ : FAMILY0(,TBasicCowString) + TBasicCowString() noexcept + : S_(Construct()) + { + } + + inline explicit TBasicCowString(::NDetail::TReserveTag rt) + : S_(Construct<>()) + { + reserve(rt.Capacity); + } + + inline TBasicCowString(const TBasicCowString& s) + : S_(s.S_) + { + } + + inline TBasicCowString(TBasicCowString&& s) noexcept + : S_(Construct()) + { + s.swap(*this); + } + + template + explicit inline TBasicCowString(const std::basic_string& s) + : TBasicCowString(s.data(), s.size()) + { + } + + template + inline TBasicCowString(std::basic_string&& s) + : S_(s.empty() ? Construct() : Construct(std::move(s))) + { + } + + TBasicCowString(const TBasicCowString& s, size_t pos, size_t n) + : S_(n ? Construct(s, pos, n) : Construct()) + { + } + + TBasicCowString(const TCharType* pc) + : TBasicCowString(pc, TBase::StrLen(pc)) + { + } + TBasicCowString(std::nullptr_t) = delete; + + TBasicCowString(const TCharType* pc, size_t n) + : S_(n ? Construct(pc, n) : Construct()) + { + } + TBasicCowString(std::nullptr_t, size_t) = delete; + + TBasicCowString(const TCharType* pc, size_t pos, size_t n) + : TBasicCowString(pc + pos, n) + { + } + + explicit TBasicCowString(TExplicitType c) + : TBasicCowString(&c.Value(), 1) + { + } + explicit TBasicCowString(const reference& c) + : TBasicCowString(&c, 1) + { + } + + TBasicCowString(size_t n, TCharType c) + : S_(Construct(n, c)) + { + } + + /** + * Constructs an uninitialized string of size `uninitialized.Size`. The proper + * way to use this ctor is via `TBasicCowString::Uninitialized` factory function. + * + * @throw std::length_error + */ + TBasicCowString(TUninitialized uninitialized) + : S_(Construct<>()) + { + ReserveAndResize(uninitialized.Size); + } + + TBasicCowString(const TCharType* b, const TCharType* e) + : TBasicCowString(b, NonNegativeDistance(b, e)) + { + } + + explicit TBasicCowString(const TBasicStringBuf s) + : TBasicCowString(s.data(), s.size()) + { + } + + template + explicit inline TBasicCowString(const std::basic_string_view& s) + : TBasicCowString(s.data(), s.size()) + { + } + + /** + * WARN: + * Certain invocations of this method will result in link-time error. + * You are free to implement corresponding methods in string.cpp if you need them. + */ + static TBasicCowString FromAscii(const ::TStringBuf& s) { + return TBasicCowString().AppendAscii(s); + } + + static TBasicCowString FromUtf8(const ::TStringBuf& s) { + return TBasicCowString().AppendUtf8(s); + } + + static TBasicCowString FromUtf16(const ::TWtringBuf& s) { + return TBasicCowString().AppendUtf16(s); + } + + static TBasicCowString Uninitialized(size_t n) { + return TBasicCowString(TUninitialized(n)); + } + +private: + using TJoinHelper = TStringJoinHelper; + + template + static inline TBasicCowString JoinImpl(const R&... r) { + TBasicCowString s{TUninitialized{TJoinHelper::SumLength(r...)}}; + TJoinHelper::CopyAll((TCharType*)s.data(), r...); + return s; + } + +public: + Y_REINITIALIZES_OBJECT inline void clear() noexcept { + if (IsDetached()) { + S_->clear(); + + return; + } + + Construct().Swap(S_); + } + + template + static inline TBasicCowString Join(const R&... r) { + return JoinImpl(typename TJoinHelper::template TJoinParam(r)...); + } + + // ~~~ Assignment ~~~ : FAMILY0(TBasicCowString&, assign); + TBasicCowString& assign(size_t size, TCharType ch) Y_LIFETIME_BOUND { + ReserveAndResize(size); + std::fill(begin(), end(), ch); + return *this; + } + + TBasicCowString& assign(const TBasicCowString& s) Y_LIFETIME_BOUND { + TBasicCowString(s).swap(*this); + + return *this; + } + + TBasicCowString& assign(const TBasicCowString& s, size_t pos, size_t n) Y_LIFETIME_BOUND { + return assign(TBasicCowString(s, pos, n)); + } + + TBasicCowString& assign(const TCharType* pc) Y_LIFETIME_BOUND { + return assign(pc, TBase::StrLen(pc)); + } + + TBasicCowString& assign(TCharType ch) Y_LIFETIME_BOUND { + return assign(&ch, 1); + } + + TBasicCowString& assign(const TCharType* pc, size_t len) Y_LIFETIME_BOUND { +#if defined(address_sanitizer_enabled) || defined(thread_sanitizer_enabled) + pc = (const TCharType*)HidePointerOrigin((void*)pc); +#endif + if (IsDetached()) { + MutRef().assign(pc, len); + } else { + TBasicCowString(pc, len).swap(*this); + } + + return *this; + } + + TBasicCowString& assign(const TCharType* first, const TCharType* last) Y_LIFETIME_BOUND { + return assign(first, NonNegativeDistance(first, last)); + } + + TBasicCowString& assign(const TCharType* pc, size_t pos, size_t n) Y_LIFETIME_BOUND { + return assign(pc + pos, n); + } + + TBasicCowString& assign(const TBasicStringBuf s) Y_LIFETIME_BOUND { + return assign(s.data(), s.size()); + } + + TBasicCowString& assign(const TBasicStringBuf s, size_t spos, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + return assign(s.SubString(spos, sn)); + } + + inline TBasicCowString& AssignNoAlias(const TCharType* pc, size_t len) Y_LIFETIME_BOUND { + return assign(pc, len); + } + + inline TBasicCowString& AssignNoAlias(const TCharType* b, const TCharType* e) Y_LIFETIME_BOUND { + return AssignNoAlias(b, e - b); + } + + TBasicCowString& AssignNoAlias(const TBasicStringBuf s) Y_LIFETIME_BOUND { + return AssignNoAlias(s.data(), s.size()); + } + + TBasicCowString& AssignNoAlias(const TBasicStringBuf s, size_t spos, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + return AssignNoAlias(s.SubString(spos, sn)); + } + + /** + * WARN: + * Certain invocations of this method will result in link-time error. + * You are free to implement corresponding methods in string.cpp if you need them. + */ + auto AssignAscii(const ::TStringBuf& s) { + clear(); + return AppendAscii(s); + } + + auto AssignUtf8(const ::TStringBuf& s) { + clear(); + return AppendUtf8(s); + } + + auto AssignUtf16(const ::TWtringBuf& s) { + clear(); + return AppendUtf16(s); + } + + TBasicCowString& operator=(const TBasicCowString& s) Y_LIFETIME_BOUND { + return assign(s); + } + + TBasicCowString& operator=(TBasicCowString&& s) noexcept Y_LIFETIME_BOUND { + swap(s); + return *this; + } + + template + TBasicCowString& operator=(std::basic_string&& s) noexcept Y_LIFETIME_BOUND { + TBasicCowString(std::move(s)).swap(*this); + + return *this; + } + + TBasicCowString& operator=(const TBasicStringBuf s) Y_LIFETIME_BOUND { + return assign(s); + } + + TBasicCowString& operator=(std::initializer_list il) Y_LIFETIME_BOUND { + return assign(il.begin(), il.end()); + } + + TBasicCowString& operator=(const TCharType* s) Y_LIFETIME_BOUND { + return assign(s); + } + TBasicCowString& operator=(std::nullptr_t) Y_LIFETIME_BOUND = delete; + + TBasicCowString& operator=(TExplicitType ch) Y_LIFETIME_BOUND { + return assign(ch); + } + + inline void reserve(size_t len) { + MutRef().reserve(len); + } + + // ~~~ Appending ~~~ : FAMILY0(TBasicCowString&, append); + inline TBasicCowString& append(size_t count, TCharType ch) Y_LIFETIME_BOUND { + MutRef().append(count, ch); + + return *this; + } + + inline TBasicCowString& append(const TBasicCowString& s) Y_LIFETIME_BOUND { + MutRef().append(s.ConstRef()); + + return *this; + } + + inline TBasicCowString& append(const TBasicCowString& s, size_t pos, size_t n) Y_LIFETIME_BOUND { + MutRef().append(s.ConstRef(), pos, n); + + return *this; + } + + inline TBasicCowString& append(const TCharType* pc) Y_LIFETIME_BOUND { + MutRef().append(pc); + + return *this; + } + + inline TBasicCowString& append(TCharType c) Y_LIFETIME_BOUND { + MutRef().push_back(c); + + return *this; + } + + inline TBasicCowString& append(const TCharType* first, const TCharType* last) Y_LIFETIME_BOUND { + MutRef().append(first, last); + + return *this; + } + + inline TBasicCowString& append(const TCharType* pc, size_t len) Y_LIFETIME_BOUND { + MutRef().append(pc, len); + + return *this; + } + + inline void ReserveAndResize(size_t len) { + ::ResizeUninitialized(MutRef(), len); + } + + TBasicCowString& AppendNoAlias(const TCharType* pc, size_t len) Y_LIFETIME_BOUND { + if (len) { + auto s = this->size(); + + ReserveAndResize(s + len); + memcpy(&*(begin() + s), pc, len * sizeof(*pc)); + } + + return *this; + } + + TBasicCowString& AppendNoAlias(const TBasicStringBuf s) Y_LIFETIME_BOUND { + return AppendNoAlias(s.data(), s.size()); + } + + TBasicCowString& AppendNoAlias(const TBasicStringBuf s, size_t spos, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + return AppendNoAlias(s.SubString(spos, sn)); + } + + TBasicCowString& append(const TBasicStringBuf s) Y_LIFETIME_BOUND { + return append(s.data(), s.size()); + } + + TBasicCowString& append(const TBasicStringBuf s, size_t spos, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + return append(s.SubString(spos, sn)); + } + + TBasicCowString& append(const TCharType* pc, size_t pos, size_t n, size_t pc_len = TBase::npos) Y_LIFETIME_BOUND { + return append(pc + pos, Min(n, pc_len - pos)); + } + + /** + * WARN: + * Certain invocations of this method will result in link-time error. + * You are free to implement corresponding methods in string.cpp if you need them. + */ + TBasicCowString& AppendAscii(const ::TStringBuf& s) Y_LIFETIME_BOUND; + + TBasicCowString& AppendUtf8(const ::TStringBuf& s) Y_LIFETIME_BOUND; + + TBasicCowString& AppendUtf16(const ::TWtringBuf& s) Y_LIFETIME_BOUND; + + inline void push_back(TCharType c) { + // TODO + append(c); + } + + template + TBasicCowString& operator+=(const T& s) Y_LIFETIME_BOUND { + return append(s); + } + + template + friend TBasicCowString operator*(const TBasicCowString& s, T count) { + static_assert(std::is_integral::value, "Integral type required."); + + TBasicCowString result; + + if (count > 0) { + result.reserve(s.length() * count); + } + + for (T i = 0; i < count; ++i) { + result += s; + } + + return result; + } + + template + TBasicCowString& operator*=(T count) Y_LIFETIME_BOUND { + static_assert(std::is_integral::value, "Integral type required."); + + TBasicCowString temp; + + if (count > 0) { + temp.reserve(length() * count); + } + + for (T i = 0; i < count; ++i) { + temp += *this; + } + + swap(temp); + + return *this; + } + + operator const TStringType&() const noexcept Y_LIFETIME_BOUND { + return this->ConstRef(); + } + + /* We have operator casting TString to `const std::string&` but we explicitly don't support + * casting TString to `std::string&` since such casting requires detaching TString and therefore + * modifies TString object. Sometimes compiler might call `operator std::string&` + * implicitly and it might lead to problems. Check IGNIETFERRO-2155 for details. + */ + template >> + operator T&() & Y_LIFETIME_BOUND requires false { + return this->MutRef(); + } + + /* + * Following overloads of "operator+" aim to choose the cheapest implementation depending on + * summand types: lvalues, detached rvalues, shared rvalues. + * + * General idea is to use the detached-rvalue argument (left of right) to store the result + * wherever possible. If a buffer in rvalue is large enough this saves a re-allocation. If + * both arguments are rvalues we check which one is detached. If both of them are detached then + * the left argument is obviously preferrable because you won't need to shift the data. + * + * If an rvalue is shared then it's basically the same as lvalue because you cannot use its + * buffer to store the sum. However, we rely on the fact that append() and prepend() are already + * optimized for the shared case and detach the string into the buffer large enough to store + * the sum (compared to the detach+reallocation). This way, if we have only one rvalue argument + * (left or right) then we simply append/prepend into it, without checking if it's detached or + * not. This will be checked inside ReserveAndResize anyway. + * + * If both arguments cannot be used to store the sum (e.g. two lvalues) then we fall back to the + * Join function that constructs a resulting string in the new buffer with the minimum overhead: + * malloc + memcpy + memcpy. + */ + + friend TBasicCowString operator+(TBasicCowString&& s1, const TBasicCowString& s2) Y_WARN_UNUSED_RESULT { + s1 += s2; + return std::move(s1); + } + + friend TBasicCowString operator+(const TBasicCowString& s1, TBasicCowString&& s2) Y_WARN_UNUSED_RESULT { + s2.prepend(s1); + return std::move(s2); + } + + friend TBasicCowString operator+(TBasicCowString&& s1, TBasicCowString&& s2) Y_WARN_UNUSED_RESULT { +#if 0 + if (!s1.IsDetached() && s2.IsDetached()) { + s2.prepend(s1); + return std::move(s2); + } +#endif + s1 += s2; + return std::move(s1); + } + + friend TBasicCowString operator+(TBasicCowString&& s1, const TBasicStringBuf s2) Y_WARN_UNUSED_RESULT { + s1 += s2; + return std::move(s1); + } + + friend TBasicCowString operator+(TBasicCowString&& s1, const TCharType* s2) Y_WARN_UNUSED_RESULT { + s1 += s2; + return std::move(s1); + } + + friend TBasicCowString operator+(TBasicCowString&& s1, TCharType s2) Y_WARN_UNUSED_RESULT { + s1 += s2; + return std::move(s1); + } + + friend TBasicCowString operator+(TExplicitType ch, const TBasicCowString& s) Y_WARN_UNUSED_RESULT { + return Join(TCharType(ch), s); + } + + friend TBasicCowString operator+(TExplicitType ch, TBasicCowString&& s) Y_WARN_UNUSED_RESULT { + s.prepend(ch); + return std::move(s); + } + + friend TBasicCowString operator+(const TBasicCowString& s1, const TBasicCowString& s2) Y_WARN_UNUSED_RESULT { + return Join(s1, s2); + } + + friend TBasicCowString operator+(const TBasicCowString& s1, const TBasicStringBuf s2) Y_WARN_UNUSED_RESULT { + return Join(s1, s2); + } + + friend TBasicCowString operator+(const TBasicCowString& s1, const TCharType* s2) Y_WARN_UNUSED_RESULT { + return Join(s1, s2); + } + + friend TBasicCowString operator+(const TBasicCowString& s1, TCharType s2) Y_WARN_UNUSED_RESULT { + return Join(s1, TBasicStringBuf(&s2, 1)); + } + + friend TBasicCowString operator+(const TCharType* s1, TBasicCowString&& s2) Y_WARN_UNUSED_RESULT { + s2.prepend(s1); + return std::move(s2); + } + + friend TBasicCowString operator+(const TBasicStringBuf s1, TBasicCowString&& s2) Y_WARN_UNUSED_RESULT { + s2.prepend(s1); + return std::move(s2); + } + + friend TBasicCowString operator+(const TBasicStringBuf s1, const TBasicCowString& s2) Y_WARN_UNUSED_RESULT { + return Join(s1, s2); + } + + friend TBasicCowString operator+(const TCharType* s1, const TBasicCowString& s2) Y_WARN_UNUSED_RESULT { + return Join(s1, s2); + } + + friend TBasicCowString operator+(std::basic_string l, TBasicCowString r) { + return std::move(l) + r.ConstRef(); + } + + friend TBasicCowString operator+(TBasicCowString l, std::basic_string r) { + return l.ConstRef() + std::move(r); + } + + // ~~~ Prepending ~~~ : FAMILY0(TBasicCowString&, prepend); + TBasicCowString& prepend(const TBasicCowString& s) Y_LIFETIME_BOUND { + MutRef().insert(0, s.ConstRef()); + + return *this; + } + + TBasicCowString& prepend(const TBasicCowString& s, size_t pos, size_t n) Y_LIFETIME_BOUND { + MutRef().insert(0, s.ConstRef(), pos, n); + + return *this; + } + + TBasicCowString& prepend(const TCharType* pc) Y_LIFETIME_BOUND { + MutRef().insert(0, pc); + + return *this; + } + + TBasicCowString& prepend(size_t n, TCharType c) Y_LIFETIME_BOUND { + MutRef().insert(size_t(0), n, c); + + return *this; + } + + TBasicCowString& prepend(TCharType c) Y_LIFETIME_BOUND { + MutRef().insert(size_t(0), 1, c); + + return *this; + } + + TBasicCowString& prepend(const TBasicStringBuf s, size_t spos = 0, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + return insert(0, s, spos, sn); + } + + // ~~~ Insertion ~~~ : FAMILY1(TBasicCowString&, insert, size_t pos); + TBasicCowString& insert(size_t pos, const TBasicCowString& s) Y_LIFETIME_BOUND { + MutRef().insert(pos, s.ConstRef()); + + return *this; + } + + TBasicCowString& insert(size_t pos, const TBasicCowString& s, size_t pos1, size_t n1) Y_LIFETIME_BOUND { + MutRef().insert(pos, s.ConstRef(), pos1, n1); + + return *this; + } + + TBasicCowString& insert(size_t pos, const TCharType* pc) Y_LIFETIME_BOUND { + MutRef().insert(pos, pc); + + return *this; + } + + TBasicCowString& insert(size_t pos, const TCharType* pc, size_t len) Y_LIFETIME_BOUND { + MutRef().insert(pos, pc, len); + + return *this; + } + + TBasicCowString& insert(const_iterator pos, const_iterator b, const_iterator e) Y_LIFETIME_BOUND { + return insert(this->off(pos), b, e - b); + } + + TBasicCowString& insert(size_t pos, size_t n, TCharType c) Y_LIFETIME_BOUND { + MutRef().insert(pos, n, c); + + return *this; + } + + TBasicCowString& insert(const_iterator pos, size_t len, TCharType ch) Y_LIFETIME_BOUND { + return this->insert(this->off(pos), len, ch); + } + + TBasicCowString& insert(const_iterator pos, TCharType ch) Y_LIFETIME_BOUND { + return this->insert(pos, 1, ch); + } + + TBasicCowString& insert(size_t pos, const TBasicStringBuf s, size_t spos = 0, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + MutRef().insert(pos, s, spos, sn); + + return *this; + } + + // ~~~ Removing ~~~ + TBasicCowString& remove(size_t pos, size_t n) Y_LIFETIME_BOUND { + if (pos < length()) { + MutRef().erase(pos, n); + } + + return *this; + } + + TBasicCowString& remove(size_t pos = 0) Y_LIFETIME_BOUND { + if (pos < length()) { + MutRef().erase(pos); + } + + return *this; + } + + TBasicCowString& erase(size_t pos = 0, size_t n = TBase::npos) Y_LIFETIME_BOUND { + MutRef().erase(pos, n); + + return *this; + } + + TBasicCowString& erase(const_iterator b, const_iterator e) Y_LIFETIME_BOUND { + return erase(this->off(b), e - b); + } + + TBasicCowString& erase(const_iterator i) Y_LIFETIME_BOUND { + return erase(i, i + 1); + } + + TBasicCowString& pop_back() Y_LIFETIME_BOUND { + Y_ASSERT(!this->empty()); + + MutRef().pop_back(); + + return *this; + } + + // ~~~ replacement ~~~ : FAMILY2(TBasicCowString&, replace, size_t pos, size_t n); + TBasicCowString& replace(size_t pos, size_t n, const TBasicCowString& s) Y_LIFETIME_BOUND { + MutRef().replace(pos, n, s.ConstRef()); + + return *this; + } + + TBasicCowString& replace(size_t pos, size_t n, const TBasicCowString& s, size_t pos1, size_t n1) Y_LIFETIME_BOUND { + MutRef().replace(pos, n, s.ConstRef(), pos1, n1); + + return *this; + } + + TBasicCowString& replace(size_t pos, size_t n, const TCharType* pc) Y_LIFETIME_BOUND { + MutRef().replace(pos, n, pc); + + return *this; + } + + TBasicCowString& replace(size_t pos, size_t n, const TCharType* s, size_t len) Y_LIFETIME_BOUND { + MutRef().replace(pos, n, s, len); + + return *this; + } + + TBasicCowString& replace(size_t pos, size_t n, const TCharType* s, size_t spos, size_t sn) Y_LIFETIME_BOUND { + MutRef().replace(pos, n, s + spos, sn - spos); + + return *this; + } + + TBasicCowString& replace(size_t pos, size_t n1, size_t n2, TCharType c) Y_LIFETIME_BOUND { + MutRef().replace(pos, n1, n2, c); + + return *this; + } + + TBasicCowString& replace(size_t pos, size_t n, const TBasicStringBuf s, size_t spos = 0, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + MutRef().replace(pos, n, s, spos, sn); + + return *this; + } + + void swap(TBasicCowString& s) noexcept { + S_.Swap(s.S_); + } + + /** + * @returns String suitable for debug printing (like Python's `repr()`). + * Format of the string is unspecified and may be changed over time. + */ + TBasicCowString Quote() const { + extern TBasicCowString EscapeC(const TBasicCowString&); + + return TBasicCowString() + '"' + EscapeC(*this) + '"'; + } + + /** + * Modifies the case of the string, depending on the operation. + * @return false if no changes have been made. + * + * @warning when the value_type is char, these methods will not work with non-ASCII letters. + */ + bool to_lower(size_t pos = 0, size_t n = TBase::npos); + bool to_upper(size_t pos = 0, size_t n = TBase::npos); + bool to_title(size_t pos = 0, size_t n = TBase::npos); + + constexpr const TCharType* Data() const noexcept = delete; + constexpr size_t Size() noexcept = delete; + Y_PURE_FUNCTION constexpr bool Empty() const noexcept = delete; + +public: + /** + * Modifies the substring of length `n` starting from `pos`, applying `f` to each position and symbol. + * + * @return false if no changes have been made. + */ + template + bool Transform(T&& f, size_t pos = 0, size_t n = TBase::npos) { + size_t len = length(); + + if (pos > len) { + pos = len; + } + + if (n > len - pos) { + n = len - pos; + } + + bool changed = false; + + for (size_t i = pos; i != pos + n; ++i) { + auto c = f(i, data()[i]); + if (c != data()[i]) { + if (!changed) { + Detach(); + changed = true; + } + + begin()[i] = c; + } + } + + return changed; + } +}; + +using TCowString = TBasicCowString; +using TUtf16CowString = TBasicCowString; +using TUtf32CowString = TBasicCowString; + +std::ostream& operator<<(std::ostream&, const TCowString&); +std::istream& operator>>(std::istream&, TCowString&); + +template +TBasicCowString to_lower(const TBasicCowString& s) { + TBasicCowString ret(s); + ret.to_lower(); + return ret; +} + +template +TBasicCowString to_upper(const TBasicCowString& s) { + TBasicCowString ret(s); + ret.to_upper(); + return ret; +} + +template +TBasicCowString to_title(const TBasicCowString& s) { + TBasicCowString ret(s); + ret.to_title(); + return ret; +} + +namespace std { + template <> + struct hash { + using argument_type = TCowString; + using result_type = size_t; + inline result_type operator()(argument_type const& s) const noexcept { + return NHashPrivate::ComputeStringHash(s.data(), s.size()); + } + }; +} // namespace std + +// interop +template +auto& MutRef(TBasicCowString& s Y_LIFETIME_BOUND) { + return s.MutRef(); +} + +template +const auto& ConstRef(const TBasicCowString& s Y_LIFETIME_BOUND) noexcept { + return s.ConstRef(); +} + +template +void ResizeUninitialized(TBasicCowString& s, size_t len) { + s.ReserveAndResize(len); +} diff --git a/library/cpp/containers/cow_string/cow_string_ut.cpp b/library/cpp/containers/cow_string/cow_string_ut.cpp new file mode 100644 index 0000000000..1be9d607ed --- /dev/null +++ b/library/cpp/containers/cow_string/cow_string_ut.cpp @@ -0,0 +1,1296 @@ +#include + +#include +#include +#include +#include + +#include +#include "util/generic/deque.h" +#include "util/generic/strbuf.h" +#include "util/generic/string_ut.h" +#include "util/generic/vector.h" +#include "util/generic/yexception.h" +#include +#include + +#include +#include +#include +#include + +static_assert(sizeof(TCowString) == sizeof(const char*), "expect sizeof(TCowString) == sizeof(const char*)"); + +class TStringTestZero: public TTestBase { + UNIT_TEST_SUITE(TStringTestZero); + UNIT_TEST(TestZero); + UNIT_TEST_SUITE_END(); + +public: + void TestZero() { + const char data[] = "abc\0def\0"; + TCowString s(data, sizeof(data)); + UNIT_ASSERT(s.size() == sizeof(data)); + UNIT_ASSERT(s.StartsWith(s)); + UNIT_ASSERT(s.EndsWith(s)); + UNIT_ASSERT(s.Contains('\0')); + + const char raw_def[] = "def"; + const char raw_zero[] = "\0"; + TCowString def(raw_def, sizeof(raw_def) - 1); + TCowString zero(raw_zero, sizeof(raw_zero) - 1); + UNIT_ASSERT_EQUAL(4, s.find(raw_def)); + UNIT_ASSERT_EQUAL(4, s.find(def)); + UNIT_ASSERT_EQUAL(4, s.find_first_of(raw_def)); + UNIT_ASSERT_EQUAL(3, s.find_first_of(zero)); + UNIT_ASSERT_EQUAL(7, s.find_first_not_of(def, 4)); + + const char nonSubstring[] = "def\0ghi"; + UNIT_ASSERT_EQUAL(TCowString::npos, s.find(TCowString(nonSubstring, sizeof(nonSubstring)))); + + TCowString copy = s; + copy.replace(copy.size() - 1, 1, "z"); + UNIT_ASSERT(s != copy); + copy.replace(copy.size() - 1, 1, "\0", 0, 1); + UNIT_ASSERT(s == copy); + + TCowString prefix(data, 5); + UNIT_ASSERT(s.StartsWith(prefix)); + UNIT_ASSERT(s != prefix); + UNIT_ASSERT(s > prefix); + UNIT_ASSERT(s > s.data()); + UNIT_ASSERT(s == TCowString(s.data(), s.size())); + UNIT_ASSERT(data < s); + + s.remove(5); + UNIT_ASSERT(s == prefix); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TStringTestZero); + +template +class TStringStdTestImpl { + using TChar = typename TStringType::char_type; + using TTraits = typename TStringType::traits_type; + using TView = std::basic_string_view; + + TTestData Data_; + +protected: + void Constructor() { + UNIT_ASSERT_EXCEPTION(TStringType((size_t)-1, *Data_.a()), std::length_error); + } + + void reserve() { +#if 0 + TStringType s; + UNIT_ASSERT_EXCEPTION(s.reserve(s.max_size() + 1), std::length_error); + + // Non-shared behaviour - never shrink + + s.reserve(256); + const auto* data = s.data(); + + UNIT_ASSERT(s.capacity() >= 256); + + s.reserve(128); + + UNIT_ASSERT(s.capacity() >= 256 && s.data() == data); + + s.resize(64, 'x'); + s.reserve(10); + + UNIT_ASSERT(s.capacity() >= 256 && s.data() == data); + + // Shared behaviour - always reallocate, just as much as requisted + + TStringType holder = s; + + UNIT_ASSERT(s.capacity() >= 256); + + s.reserve(128); + + UNIT_ASSERT(s.capacity() >= 128 && s.capacity() < 256 && s.data() != data); + UNIT_ASSERT(s.IsDetached()); + + s.resize(64, 'x'); + data = s.data(); + holder = s; + + s.reserve(10); + + UNIT_ASSERT(s.capacity() >= 64 && s.capacity() < 128 && s.data() != data); + UNIT_ASSERT(s.IsDetached()); +#endif + } + + void short_string() { + TStringType const ref_short_str1(Data_.str1()), ref_short_str2(Data_.str2()); + TStringType short_str1(ref_short_str1), short_str2(ref_short_str2); + TStringType const ref_long_str1(Data_.str__________________________________________________1()); + TStringType const ref_long_str2(Data_.str__________________________________________________2()); + TStringType long_str1(ref_long_str1), long_str2(ref_long_str2); + + UNIT_ASSERT(short_str1 == ref_short_str1); + UNIT_ASSERT(long_str1 == ref_long_str1); + + { + TStringType str1(short_str1); + str1 = long_str1; + UNIT_ASSERT(str1 == ref_long_str1); + } + + { + TStringType str1(long_str1); + str1 = short_str1; + UNIT_ASSERT(str1 == ref_short_str1); + } + + { + short_str1.swap(short_str2); + UNIT_ASSERT((short_str1 == ref_short_str2) && (short_str2 == ref_short_str1)); + short_str1.swap(short_str2); + } + + { + long_str1.swap(long_str2); + UNIT_ASSERT((long_str1 == ref_long_str2) && (long_str2 == ref_long_str1)); + long_str1.swap(long_str2); + } + + { + short_str1.swap(long_str1); + UNIT_ASSERT((short_str1 == ref_long_str1) && (long_str1 == ref_short_str1)); + short_str1.swap(long_str1); + } + + { + long_str1.swap(short_str1); + UNIT_ASSERT((short_str1 == ref_long_str1) && (long_str1 == ref_short_str1)); + long_str1.swap(short_str1); + } + + { + // This is to test move constructor + TVector str_vect; + + str_vect.push_back(short_str1); + str_vect.push_back(long_str1); + str_vect.push_back(short_str2); + str_vect.push_back(long_str2); + + UNIT_ASSERT(str_vect[0] == ref_short_str1); + UNIT_ASSERT(str_vect[1] == ref_long_str1); + UNIT_ASSERT(str_vect[2] == ref_short_str2); + UNIT_ASSERT(str_vect[3] == ref_long_str2); + } + } + + void erase() { + TChar const* c_str = Data_.Hello_World(); + TStringType str(c_str); + UNIT_ASSERT(str == c_str); + + str.erase(str.begin() + 1, str.end() - 1); // Erase all but first and last. + + size_t i; + for (i = 0; i < str.size(); ++i) { + switch (i) { + case 0: + UNIT_ASSERT(str[i] == *Data_.H()); + break; + + case 1: + UNIT_ASSERT(str[i] == *Data_.d()); + break; + + default: + UNIT_ASSERT(false); + } + } + + str.insert(1, c_str); + str.erase(str.begin()); // Erase first element. + str.erase(str.end() - 1); // Erase last element. + UNIT_ASSERT(str == c_str); + str.clear(); // Erase all. + UNIT_ASSERT(str.empty()); + + str = c_str; + UNIT_ASSERT(str == c_str); + + str.erase(1, str.size() - 1); // Erase all but first and last. + for (i = 0; i < str.size(); i++) { + switch (i) { + case 0: + UNIT_ASSERT(str[i] == *Data_.H()); + break; + + case 1: + UNIT_ASSERT(str[i] == *Data_.d()); + break; + + default: + UNIT_ASSERT(false); + } + } + + str.erase(1); + UNIT_ASSERT(str == Data_.H()); + } + + void data() { + TStringType xx; + + // ISO-IEC-14882:1998(E), 21.3.6, paragraph 3 + UNIT_ASSERT(xx.data() != nullptr); + } + + void c_str() { + TStringType low(Data_._2004_01_01()); + TStringType xx; + TStringType yy; + + // ISO-IEC-14882:1998(E), 21.3.6, paragraph 1 + UNIT_ASSERT(*(yy.c_str()) == 0); + + // Blocks A and B should follow each other. + // Block A: + xx = Data_._123456(); + xx += low; + UNIT_ASSERT(xx.c_str() == TView(Data_._1234562004_01_01())); + // End of block A + + // Block B: + xx = Data_._1234(); + xx += Data_._5(); + UNIT_ASSERT(xx.c_str() == TView(Data_._12345())); + // End of block B + } + + void null_char_of_empty() { + const TStringType s; + + // NOTE: https://a.yandex-team.ru/arcadia/junk/grechnik/test_string?rev=r12602052 + i64 i = s[s.size()]; + UNIT_ASSERT_VALUES_EQUAL(i, 0); + } + + void null_char() { + // ISO/IEC 14882:1998(E), ISO/IEC 14882:2003(E), 21.3.4 ('... the const version') + const TStringType s(Data_._123456()); + + UNIT_ASSERT(s[s.size()] == 0); + } + + // Allowed since C++17, see http://www.open-std.org/jtc1/sc22/wg21/docs/lwg-defects.html#2475 + void null_char_assignment_to_subscript_of_empty() { + TStringType s; + + using reference = typename TStringType::reference; + reference trailing_zero = s[s.size()]; + trailing_zero = 0; + UNIT_ASSERT(trailing_zero == 0); + } + + // Allowed since C++17, see http://www.open-std.org/jtc1/sc22/wg21/docs/lwg-defects.html#2475 + void null_char_assignment_to_subscript_of_nonempty() { + TStringType s(Data_._123456()); + + using reference = typename TStringType::reference; + reference trailing_zero = s[s.size()]; + trailing_zero = 0; + UNIT_ASSERT(trailing_zero == 0); + } + + // Dereferencing string end() is not allowed by C++ standard as of C++20, avoid using in real code. + void null_char_assignment_to_end_of_empty() { + TStringType s; + + volatile auto& trailing_zero = *(s.begin() + s.size()); + trailing_zero = 0; + UNIT_ASSERT(trailing_zero == 0); + } + + // Dereferencing string end() is not allowed by C++ standard as of C++20, avoid using in real code. + void null_char_assignment_to_end_of_nonempty() { + TStringType s(Data_._123456()); + + volatile auto& trailing_zero = *(s.begin() + s.size()); + trailing_zero = 0; + UNIT_ASSERT(trailing_zero == 0); + } + + void insert() { + TStringType strorg = Data_.This_is_test_string_for_string_calls(); + TStringType str; + + // In case of reallocation there is no auto reference problem + // so we reserve a big enough TStringType to be sure to test this + // particular point. + + str.reserve(100); + str = strorg; + + // test self insertion: + str.insert(10, str.c_str() + 5, 15); + UNIT_ASSERT(str == Data_.This_is_teis_test_string_st_string_for_string_calls()); + + str = strorg; + str.insert(15, str.c_str() + 5, 25); + UNIT_ASSERT(str == Data_.This_is_test_stis_test_string_for_stringring_for_string_calls()); + + str = strorg; + str.insert(0, str.c_str() + str.size() - 4, 4); + UNIT_ASSERT(str == Data_.allsThis_is_test_string_for_string_calls()); + + str = strorg; + str.insert(0, str.c_str() + str.size() / 2 - 1, str.size() / 2 + 1); + UNIT_ASSERT(str == Data_.ng_for_string_callsThis_is_test_string_for_string_calls()); + + str = strorg; + typename TStringType::iterator b = str.begin(); + typename TStringType::const_iterator s = str.begin() + str.size() / 2 - 1; + typename TStringType::const_iterator e = str.end(); + str.insert(b, s, e); + UNIT_ASSERT(str == Data_.ng_for_string_callsThis_is_test_string_for_string_calls()); + +#if 0 + // AV + str = strorg; + str.insert(str.begin(), str.begin() + str.size() / 2 - 1, str.end()); + UNIT_ASSERT(str == Data.ng_for_string_callsThis_is_test_string_for_string_calls()); +#endif + + TStringType str0; + str0.insert(str0.begin(), 5, *Data_._0()); + UNIT_ASSERT(str0 == Data_._00000()); + + TStringType str1; + { + typename TStringType::size_type pos = 0, nb = 2; + str1.insert(pos, nb, *Data_._1()); + } + UNIT_ASSERT(str1 == Data_._11()); + + str0.insert(0, str1); + UNIT_ASSERT(str0 == Data_._1100000()); + + TStringType str2(Data_._2345()); + str0.insert(str0.size(), str2, 1, 2); + UNIT_ASSERT(str0 == Data_._110000034()); + + str1.insert(str1.begin() + 1, 2, *Data_._2()); + UNIT_ASSERT(str1 == Data_._1221()); + + str1.insert(2, Data_._333333(), 3); + UNIT_ASSERT(str1 == Data_._1233321()); + + str1.insert(4, Data_._4444()); + UNIT_ASSERT(str1 == Data_._12334444321()); + + str1.insert(str1.begin() + 6, *Data_._5()); + UNIT_ASSERT(str1 == Data_._123344544321()); + } + + void resize() { + TStringType s; + + s.resize(0); + + UNIT_ASSERT(*s.c_str() == 0); + + s = Data_._1234567(); + + s.resize(0); + UNIT_ASSERT(*s.c_str() == 0); + + s = Data_._1234567(); + s.resize(1); + UNIT_ASSERT(s.size() == 1); + UNIT_ASSERT(*s.c_str() == *Data_._1()); + UNIT_ASSERT(*(s.c_str() + 1) == 0); + + s = Data_._1234567(); +#if 0 + s.resize(10); +#else + s.resize(10, 0); +#endif + UNIT_ASSERT(s.size() == 10); + UNIT_ASSERT(s[6] == *Data_._7()); + UNIT_ASSERT(s[7] == 0); + UNIT_ASSERT(s[8] == 0); + UNIT_ASSERT(s[9] == 0); + } + + void find() { + TStringType s(Data_.one_two_three_one_two_three()); + + UNIT_ASSERT(s.find(Data_.one()) == 0); + UNIT_ASSERT(s.find(*Data_.t()) == 4); + UNIT_ASSERT(s.find(*Data_.t(), 5) == 8); + + UNIT_ASSERT(s.find(Data_.four()) == TStringType::npos); + UNIT_ASSERT(s.find(Data_.one(), TStringType::npos) == TStringType::npos); + UNIT_ASSERT(s.find_first_of(Data_.abcde()) == 2); + UNIT_ASSERT(s.find_first_not_of(Data_.enotw_()) == 9); + } + + void capacity() { + TStringType s; + + UNIT_ASSERT(s.capacity() < s.max_size()); + UNIT_ASSERT(s.capacity() >= s.size()); + + for (int i = 0; i < 18; ++i) { + s += ' '; + + UNIT_ASSERT(s.capacity() > 0); + UNIT_ASSERT(s.capacity() < s.max_size()); + UNIT_ASSERT(s.capacity() >= s.size()); + } + } + + void assign() { + TStringType s; + TChar const* cstr = Data_.test_string_for_assign(); + + s.assign(cstr, cstr + 22); + UNIT_ASSERT(s == Data_.test_string_for_assign()); + + TStringType s2(Data_.other_test_string()); + s.assign(s2); + UNIT_ASSERT(s == s2); + + static TStringType str1; + static TStringType str2; + + // short TStringType optim: + str1 = Data_._123456(); + // longer than short TStringType: + str2 = Data_._1234567890123456789012345678901234567890(); + + UNIT_ASSERT(str1[5] == *Data_._6()); + UNIT_ASSERT(str2[29] == *Data_._0()); + } + + void copy() { + TStringType s(Data_.foo()); + TChar dest[4]; + dest[0] = dest[1] = dest[2] = dest[3] = 1; + s.copy(dest, 4); + int pos = 0; + UNIT_ASSERT(dest[pos++] == *Data_.f()); + UNIT_ASSERT(dest[pos++] == *Data_.o()); + UNIT_ASSERT(dest[pos++] == *Data_.o()); + UNIT_ASSERT(dest[pos++] == 1); + + dest[0] = dest[1] = dest[2] = dest[3] = 1; + s.copy(dest, 4, 2); + pos = 0; + UNIT_ASSERT(dest[pos++] == *Data_.o()); + UNIT_ASSERT(dest[pos++] == 1); + + UNIT_ASSERT_EXCEPTION(s.copy(dest, 4, 5), std::out_of_range); + } + + void cbegin_cend() { + const char helloThere[] = "Hello there"; + TCowString s = helloThere; + size_t index = 0; + for (auto it = s.cbegin(); s.cend() != it; ++it, ++index) { + UNIT_ASSERT_VALUES_EQUAL(helloThere[index], *it); + } + } + + void compare() { + TStringType str1(Data_.abcdef()); + TStringType str2; + + str2 = Data_.abcdef(); + UNIT_ASSERT(str1.compare(str2) == 0); + UNIT_ASSERT(str1.compare(str2.data(), str2.size()) == 0); + str2 = Data_.abcde(); + UNIT_ASSERT(str1.compare(str2) > 0); + UNIT_ASSERT(str1.compare(str2.data(), str2.size()) > 0); + str2 = Data_.abcdefg(); + UNIT_ASSERT(str1.compare(str2) < 0); + UNIT_ASSERT(str1.compare(str2.data(), str2.size()) < 0); + + UNIT_ASSERT(str1.compare(Data_.abcdef()) == 0); + UNIT_ASSERT(str1.compare(Data_.abcde()) > 0); + UNIT_ASSERT(str1.compare(Data_.abcdefg()) < 0); + + str2 = Data_.cde(); + UNIT_ASSERT(str1.compare(2, 3, str2) == 0); + str2 = Data_.cd(); + UNIT_ASSERT(str1.compare(2, 3, str2) > 0); + str2 = Data_.cdef(); + UNIT_ASSERT(str1.compare(2, 3, str2) < 0); + + str2 = Data_.abcdef(); + UNIT_ASSERT(str1.compare(2, 3, str2, 2, 3) == 0); + UNIT_ASSERT(str1.compare(2, 3, str2, 2, 2) > 0); + UNIT_ASSERT(str1.compare(2, 3, str2, 2, 4) < 0); + + UNIT_ASSERT(str1.compare(2, 3, Data_.cdefgh(), 3) == 0); + UNIT_ASSERT(str1.compare(2, 3, Data_.cdefgh(), 2) > 0); + UNIT_ASSERT(str1.compare(2, 3, Data_.cdefgh(), 4) < 0); + } + + void find_last_of() { + // 21.3.6.4 + TStringType s(Data_.one_two_three_one_two_three()); + + UNIT_ASSERT(s.find_last_of(Data_.abcde()) == 26); + UNIT_ASSERT(s.find_last_of(TStringType(Data_.abcde())) == 26); + + TStringType test(Data_.aba()); + + UNIT_ASSERT(test.find_last_of(Data_.a(), 2, 1) == 2); + UNIT_ASSERT(test.find_last_of(Data_.a(), 1, 1) == 0); + UNIT_ASSERT(test.find_last_of(Data_.a(), 0, 1) == 0); + + UNIT_ASSERT(test.find_last_of(*Data_.a(), 2) == 2); + UNIT_ASSERT(test.find_last_of(*Data_.a(), 1) == 0); + UNIT_ASSERT(test.find_last_of(*Data_.a(), 0) == 0); + } +#if 0 + void rfind() { + // 21.3.6.2 + TStringType s(Data.one_two_three_one_two_three()); + + UNIT_ASSERT(s.rfind(Data.two()) == 18); + UNIT_ASSERT(s.rfind(Data.two(), 0) == TStringType::npos); + UNIT_ASSERT(s.rfind(Data.two(), 11) == 4); + UNIT_ASSERT(s.rfind(*Data.w()) == 19); + + TStringType test(Data.aba()); + + UNIT_ASSERT(test.rfind(Data.a(), 2, 1) == 2); + UNIT_ASSERT(test.rfind(Data.a(), 1, 1) == 0); + UNIT_ASSERT(test.rfind(Data.a(), 0, 1) == 0); + + UNIT_ASSERT(test.rfind(*Data.a(), 2) == 2); + UNIT_ASSERT(test.rfind(*Data.a(), 1) == 0); + UNIT_ASSERT(test.rfind(*Data.a(), 0) == 0); + } +#endif + void find_last_not_of() { + // 21.3.6.6 + TStringType s(Data_.one_two_three_one_two_three()); + + UNIT_ASSERT(s.find_last_not_of(Data_.ehortw_()) == 15); + + TStringType test(Data_.aba()); + + UNIT_ASSERT(test.find_last_not_of(Data_.a(), 2, 1) == 1); + UNIT_ASSERT(test.find_last_not_of(Data_.b(), 2, 1) == 2); + UNIT_ASSERT(test.find_last_not_of(Data_.a(), 1, 1) == 1); + UNIT_ASSERT(test.find_last_not_of(Data_.b(), 1, 1) == 0); + UNIT_ASSERT(test.find_last_not_of(Data_.a(), 0, 1) == TStringType::npos); + UNIT_ASSERT(test.find_last_not_of(Data_.b(), 0, 1) == 0); + + UNIT_ASSERT(test.find_last_not_of(*Data_.a(), 2) == 1); + UNIT_ASSERT(test.find_last_not_of(*Data_.b(), 2) == 2); + UNIT_ASSERT(test.find_last_not_of(*Data_.a(), 1) == 1); + UNIT_ASSERT(test.find_last_not_of(*Data_.b(), 1) == 0); + UNIT_ASSERT(test.find_last_not_of(*Data_.a(), 0) == TStringType::npos); + UNIT_ASSERT(test.find_last_not_of(*Data_.b(), 0) == 0); + } +#if 0 + void replace() { + // This test case is for the non template basic_TString::replace method, + // this is why we play with the const iterators and reference to guaranty + // that the right method is called. + + const TStringType v(Data._78()); + TStringType s(Data._123456()); + TStringType const& cs = s; + + typename TStringType::iterator i = s.begin() + 1; + s.replace(i, i + 3, v.begin(), v.end()); + UNIT_ASSERT(s == Data._17856()); + + s = Data._123456(); + i = s.begin() + 1; + s.replace(i, i + 1, v.begin(), v.end()); + UNIT_ASSERT(s == Data._1783456()); + + s = Data._123456(); + i = s.begin() + 1; + typename TStringType::const_iterator ci = s.begin() + 1; + s.replace(i, i + 3, ci + 3, cs.end()); + UNIT_ASSERT(s == Data._15656()); + + s = Data._123456(); + i = s.begin() + 1; + ci = s.begin() + 1; + s.replace(i, i + 3, ci, ci + 2); + UNIT_ASSERT(s == Data._12356()); + + s = Data._123456(); + i = s.begin() + 1; + ci = s.begin() + 1; + s.replace(i, i + 3, ci + 1, cs.end()); + UNIT_ASSERT(s == Data._1345656()); + + s = Data._123456(); + i = s.begin(); + ci = s.begin() + 1; + s.replace(i, i, ci, ci + 1); + UNIT_ASSERT(s == Data._2123456()); + + s = Data._123456(); + s.replace(s.begin() + 4, s.end(), cs.begin(), cs.end()); + UNIT_ASSERT(s == Data._1234123456()); + + // This is the test for the template replace method. + + s = Data._123456(); + typename TStringType::iterator b = s.begin() + 4; + typename TStringType::iterator e = s.end(); + typename TStringType::const_iterator rb = s.begin(); + typename TStringType::const_iterator re = s.end(); + s.replace(b, e, rb, re); + UNIT_ASSERT(s == Data._1234123456()); + + s = Data._123456(); + s.replace(s.begin() + 4, s.end(), s.begin(), s.end()); + UNIT_ASSERT(s == Data._1234123456()); + + TStringType strorg(Data.This_is_test_StringT_for_StringT_calls()); + TStringType str = strorg; + str.replace(5, 15, str.c_str(), 10); + UNIT_ASSERT(str == Data.This_This_is_tefor_StringT_calls()); + + str = strorg; + str.replace(5, 5, str.c_str(), 10); + UNIT_ASSERT(str == Data.This_This_is_test_StringT_for_StringT_calls()); + + #if !defined(STLPORT) || defined(_STLP_MEMBER_TEMPLATES) + deque cdeque; + cdeque.push_back(*Data.I()); + str.replace(str.begin(), str.begin() + 11, cdeque.begin(), cdeque.end()); + UNIT_ASSERT(str == Data.Is_test_StringT_for_StringT_calls()); + #endif + } +#endif +}; // TStringStdTestImpl + +class TStringTest: public TTestBase, private TStringTestImpl> { +public: + UNIT_TEST_SUITE(TStringTest); + UNIT_TEST(TestMaxSize); + UNIT_TEST(TestConstructors); + UNIT_TEST(TestReplace); + UNIT_TEST(TestRefCount); + UNIT_TEST(TestFind); + UNIT_TEST(TestContains); + UNIT_TEST(TestOperators); + UNIT_TEST(TestMulOperators); + UNIT_TEST(TestFuncs); + UNIT_TEST(TestUtils); + UNIT_TEST(TestEmpty); + UNIT_TEST(TestJoin); + UNIT_TEST(TestCopy); + UNIT_TEST(TestStrCpy); + UNIT_TEST(TestPrefixSuffix); + UNIT_TEST(TestCharRef); + UNIT_TEST(TestBack) + UNIT_TEST(TestFront) + UNIT_TEST(TestIterators); + UNIT_TEST(TestReverseIterators); + UNIT_TEST(TestAppendUtf16) + UNIT_TEST(TestFillingAssign) + UNIT_TEST(TestStdStreamApi) + // UNIT_TEST(TestOperatorsCI); must fail + UNIT_TEST_SUITE_END(); + + void TestAppendUtf16() { + TCowString appended = TCowString("А роза упала").AppendUtf16(u" на лапу Азора"); + UNIT_ASSERT(appended == "А роза упала на лапу Азора"); + } + + void TestFillingAssign() { + TCowString s("abc"); + s.assign(5, 'a'); + UNIT_ASSERT_VALUES_EQUAL(s, "aaaaa"); + } + + void TestStdStreamApi() { + const TCowString data = "abracadabra"; + std::stringstream ss; + ss << data; + + UNIT_ASSERT_VALUES_EQUAL(data, ss.str()); + + ss << '\n' + << data << std::endl; + + TCowString read = "xxx"; + ss >> read; + UNIT_ASSERT_VALUES_EQUAL(read, data); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TStringTest); + +class TWideStringTest: public TTestBase, private TStringTestImpl> { +public: + UNIT_TEST_SUITE(TWideStringTest); + UNIT_TEST(TestConstructors); + UNIT_TEST(TestReplace); + UNIT_TEST(TestRefCount); + UNIT_TEST(TestFind); + UNIT_TEST(TestContains); + UNIT_TEST(TestOperators); + UNIT_TEST(TestLetOperator) + UNIT_TEST(TestMulOperators); + UNIT_TEST(TestFuncs); + UNIT_TEST(TestUtils); + UNIT_TEST(TestEmpty); + UNIT_TEST(TestJoin); + UNIT_TEST(TestCopy); + UNIT_TEST(TestStrCpy); + UNIT_TEST(TestPrefixSuffix); + UNIT_TEST(TestCharRef); + UNIT_TEST(TestBack); + UNIT_TEST(TestFront) + UNIT_TEST(TestDecodingMethods); + UNIT_TEST(TestIterators); + UNIT_TEST(TestReverseIterators); + UNIT_TEST(TestStringLiterals); + UNIT_TEST_SUITE_END(); + +private: + void TestDecodingMethods() { + UNIT_ASSERT(TUtf16CowString::FromAscii("").empty()); + UNIT_ASSERT(TUtf16CowString::FromAscii("abc") == ASCIIToWide("abc")); + +#if 0 // no wide convertions support + const char* text = "123kx83abcd ej)#$%ddja&%J&"; + TUtf16CowString wtext = ASCIIToWide(text); + + UNIT_ASSERT(wtext == TUtf16CowString::FromAscii(text)); + + TCowString strtext(text); + UNIT_ASSERT(wtext == TUtf16CowString::FromAscii(strtext)); + + TStringBuf strbuftext(text); + UNIT_ASSERT(wtext == TUtf16CowString::FromAscii(strbuftext)); + + UNIT_ASSERT(wtext.substr(5) == TUtf16CowString::FromAscii(text + 5)); + + const wchar16 wideCyrillicAlphabet[] = { + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, + 0x00}; + + TUtf16CowString strWide(wideCyrillicAlphabet); + TCowString strUtf8 = WideToUTF8(strWide); + + UNIT_ASSERT(strWide == TUtf16CowString::FromUtf8(strUtf8.c_str())); + UNIT_ASSERT(strWide == TUtf16CowString::FromUtf8(strUtf8)); + UNIT_ASSERT(strWide == TUtf16CowString::FromUtf8(TStringBuf(strUtf8))); + + // assign + + TUtf16CowString s1; + s1.AssignAscii("1234"); + UNIT_ASSERT(s1 == ASCIIToWide("1234")); + + s1.AssignUtf8(strUtf8); + UNIT_ASSERT(s1 == strWide); + + s1.AssignAscii(text); + UNIT_ASSERT(s1 == wtext); + + // append + + TUtf16CowString s2; + TUtf16CowString testAppend = strWide; + s2.AppendUtf8(strUtf8); + UNIT_ASSERT(testAppend == s2); + + testAppend += ' '; + s2.AppendAscii(" "); + UNIT_ASSERT(testAppend == s2); + + testAppend += '_'; + s2.AppendUtf8("_"); + UNIT_ASSERT(testAppend == s2); + + testAppend += wtext; + s2.AppendAscii(text); + UNIT_ASSERT(testAppend == s2); + + testAppend += wtext; + s2.AppendUtf8(text); + UNIT_ASSERT(testAppend == s2); +#endif + } + + void TestLetOperator() { + TUtf16CowString str; + + str = wchar16('X'); + UNIT_ASSERT(str == TUtf16CowString::FromAscii("X")); + + const TUtf16CowString hello = TUtf16CowString::FromAscii("hello"); + str = hello.data(); + UNIT_ASSERT(str == hello); + + str = hello; + UNIT_ASSERT(str == hello); + } + + void TestStringLiterals() { + TUtf16CowString s1 = u"hello"; + UNIT_ASSERT_VALUES_EQUAL(s1, TUtf16CowString::FromAscii("hello")); + + TUtf16CowString s2 = u"привет"; + UNIT_ASSERT_VALUES_EQUAL(s2, TUtf16CowString::FromUtf8("привет")); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TWideStringTest); + +class TUtf32StringTest: public TTestBase, private TStringTestImpl> { +public: + UNIT_TEST_SUITE(TUtf32StringTest); + UNIT_TEST(TestConstructors); + UNIT_TEST(TestReplace); + UNIT_TEST(TestRefCount); + UNIT_TEST(TestFind); + UNIT_TEST(TestContains); + UNIT_TEST(TestOperators); + UNIT_TEST(TestLetOperator) + UNIT_TEST(TestMulOperators); + UNIT_TEST(TestFuncs); + UNIT_TEST(TestUtils); + UNIT_TEST(TestEmpty); + UNIT_TEST(TestJoin); + UNIT_TEST(TestCopy); + UNIT_TEST(TestStrCpy); + UNIT_TEST(TestPrefixSuffix); + UNIT_TEST(TestCharRef); + UNIT_TEST(TestBack); + UNIT_TEST(TestFront) + UNIT_TEST(TestDecodingMethods); + UNIT_TEST(TestDecodingMethodsMixedStr); + UNIT_TEST(TestIterators); + UNIT_TEST(TestReverseIterators); + UNIT_TEST(TestStringLiterals); + UNIT_TEST_SUITE_END(); + +private: + void TestDecodingMethods() { + UNIT_ASSERT(TUtf32CowString::FromAscii("").empty()); + UNIT_ASSERT(TUtf32CowString::FromAscii("abc") == ASCIIToUTF32("abc")); + +#if 0 // no wide convertions support + const char* text = "123kx83abcd ej)#$%ddja&%J&"; + TUtf32CowString wtext = ASCIIToUTF32(text); + + UNIT_ASSERT(wtext == TUtf32CowString::FromAscii(text)); + + TCowString strtext(text); + UNIT_ASSERT(wtext == TUtf32CowString::FromAscii(strtext)); + + TStringBuf strbuftext(text); + UNIT_ASSERT(wtext == TUtf32CowString::FromAscii(strbuftext)); + + UNIT_ASSERT(wtext.substr(5) == TUtf32CowString::FromAscii(text + 5)); + + const wchar32 wideCyrillicAlphabet[] = { + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, + 0x00}; + + TUtf32CowString strWide(wideCyrillicAlphabet); + TCowString strUtf8 = WideToUTF8(strWide); + + UNIT_ASSERT(strWide == TUtf32CowString::FromUtf8(strUtf8.c_str())); + UNIT_ASSERT(strWide == TUtf32CowString::FromUtf8(strUtf8)); + UNIT_ASSERT(strWide == TUtf32CowString::FromUtf8(TStringBuf(strUtf8))); + + // assign + + TUtf32CowString s1; + s1.AssignAscii("1234"); + UNIT_ASSERT(s1 == ASCIIToUTF32("1234")); + + s1.AssignUtf8(strUtf8); + UNIT_ASSERT(s1 == strWide); + + s1.AssignAscii(text); + UNIT_ASSERT(s1 == wtext); + + // append + + TUtf32CowString s2; + TUtf32CowString testAppend = strWide; + s2.AppendUtf8(strUtf8); + UNIT_ASSERT(testAppend == s2); + + testAppend += ' '; + s2.AppendAscii(" "); + UNIT_ASSERT(testAppend == s2); + + testAppend += '_'; + s2.AppendUtf8("_"); + UNIT_ASSERT(testAppend == s2); + + testAppend += wtext; + s2.AppendAscii(text); + UNIT_ASSERT(testAppend == s2); + + testAppend += wtext; + s2.AppendUtf8(text); + + UNIT_ASSERT(testAppend == s2); +#endif + } + + void TestDecodingMethodsMixedStr() { + UNIT_ASSERT(TUtf32CowString::FromAscii("").empty()); + UNIT_ASSERT(TUtf32CowString::FromAscii("abc") == ASCIIToUTF32("abc")); + +#if 0 // no wide convertions support + const char* text = "123kx83abcd ej)#$%ddja&%J&"; + TUtf32CowString wtext = ASCIIToUTF32(text); + + UNIT_ASSERT(wtext == TUtf32CowString::FromAscii(text)); + + TCowString strtext(text); + UNIT_ASSERT(wtext == TUtf32CowString::FromAscii(strtext)); + + TStringBuf strbuftext(text); + UNIT_ASSERT(wtext == TUtf32CowString::FromAscii(strbuftext)); + + UNIT_ASSERT(wtext.substr(5) == TUtf32CowString::FromAscii(text + 5)); + + const wchar32 cyrilicAndLatinWide[] = { + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, + wchar32('z'), + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, + wchar32('z'), + 0x00}; + + TUtf32CowString strWide(cyrilicAndLatinWide); + TCowString strUtf8 = WideToUTF8(strWide); + + UNIT_ASSERT(strWide == TUtf32CowString::FromUtf8(strUtf8.c_str())); + UNIT_ASSERT(strWide == TUtf32CowString::FromUtf8(strUtf8)); + UNIT_ASSERT(strWide == UTF8ToUTF32(strUtf8)); + UNIT_ASSERT(strWide == UTF8ToUTF32(strUtf8)); + UNIT_ASSERT(strWide == TUtf32CowString::FromUtf8(TStringBuf(strUtf8))); + + // assign + + TUtf32CowString s1; + s1.AssignAscii("1234"); + UNIT_ASSERT(s1 == ASCIIToUTF32("1234")); + + s1.AssignUtf8(strUtf8); + UNIT_ASSERT(s1 == strWide); + + s1.AssignAscii(text); + UNIT_ASSERT(s1 == wtext); + + // append + + TUtf32CowString s2; + TUtf32CowString testAppend = strWide; + s2.AppendUtf16(UTF8ToWide(strUtf8)); + UNIT_ASSERT(testAppend == s2); + + testAppend += ' '; + s2.AppendAscii(" "); + UNIT_ASSERT(testAppend == s2); + + testAppend += '_'; + s2.AppendUtf8("_"); + UNIT_ASSERT(testAppend == s2); + + testAppend += wtext; + s2.AppendAscii(text); + UNIT_ASSERT(testAppend == s2); + + testAppend += wtext; + s2.AppendUtf8(text); + + UNIT_ASSERT(testAppend == s2); +#endif + } + + void TestLetOperator() { + TUtf32CowString str; + + str = wchar32('X'); + UNIT_ASSERT(str == TUtf32CowString::FromAscii("X")); + + const TUtf32CowString hello = TUtf32CowString::FromAscii("hello"); + str = hello.data(); + UNIT_ASSERT(str == hello); + + str = hello; + UNIT_ASSERT(str == hello); + } + + void TestStringLiterals() { + TUtf32CowString s1 = U"hello"; + UNIT_ASSERT_VALUES_EQUAL(s1, TUtf32CowString::FromAscii("hello")); + + TUtf32CowString s2 = U"привет"; + UNIT_ASSERT_VALUES_EQUAL(s2, TUtf32CowString::FromUtf8("привет")); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TUtf32StringTest); + +class TStringStdTest: public TTestBase, private TStringStdTestImpl> { +public: + UNIT_TEST_SUITE(TStringStdTest); + UNIT_TEST(Constructor); + UNIT_TEST(reserve); + UNIT_TEST(short_string); + UNIT_TEST(erase); + UNIT_TEST(data); + UNIT_TEST(c_str); + UNIT_TEST(null_char_of_empty); + UNIT_TEST(null_char); + UNIT_TEST(null_char_assignment_to_subscript_of_empty); + UNIT_TEST(null_char_assignment_to_subscript_of_nonempty); + UNIT_TEST(null_char_assignment_to_end_of_empty); + UNIT_TEST(null_char_assignment_to_end_of_nonempty); + UNIT_TEST(insert); + UNIT_TEST(resize); + UNIT_TEST(find); + UNIT_TEST(capacity); + UNIT_TEST(assign); + UNIT_TEST(copy); + UNIT_TEST(cbegin_cend); + UNIT_TEST(compare); + UNIT_TEST(find_last_of); +#if 0 + UNIT_TEST(rfind); + UNIT_TEST(replace); +#endif + UNIT_TEST(find_last_not_of); + UNIT_TEST_SUITE_END(); +}; + +UNIT_TEST_SUITE_REGISTRATION(TStringStdTest); + +class TWideStringStdTest: public TTestBase, private TStringStdTestImpl> { +public: + UNIT_TEST_SUITE(TWideStringStdTest); + UNIT_TEST(Constructor); + UNIT_TEST(reserve); + UNIT_TEST(short_string); + UNIT_TEST(erase); + UNIT_TEST(data); + UNIT_TEST(c_str); + UNIT_TEST(null_char_of_empty); + UNIT_TEST(null_char); + UNIT_TEST(null_char_assignment_to_subscript_of_empty); + UNIT_TEST(null_char_assignment_to_subscript_of_nonempty); + UNIT_TEST(null_char_assignment_to_end_of_empty); + UNIT_TEST(null_char_assignment_to_end_of_nonempty); + UNIT_TEST(insert); + UNIT_TEST(resize); + UNIT_TEST(find); + UNIT_TEST(capacity); + UNIT_TEST(assign); + UNIT_TEST(copy); + UNIT_TEST(cbegin_cend); + UNIT_TEST(compare); + UNIT_TEST(find_last_of); +#if 0 + UNIT_TEST(rfind); + UNIT_TEST(replace); +#endif + UNIT_TEST(find_last_not_of); + UNIT_TEST_SUITE_END(); +}; + +UNIT_TEST_SUITE_REGISTRATION(TWideStringStdTest); + +Y_UNIT_TEST_SUITE(TCowStringSerializationTest) { + TCowString SerializeThereAndBack(const TCowString& value) { + std::array buf; + TMemoryWriteBuffer out{buf.data(), buf.size()}; + Save(&out, value); + + TMemoryInput in{buf.data(), out.Len()}; + TCowString deserialized; + Load(&in, deserialized); + return deserialized; + } + + Y_UNIT_TEST(EmptyStringSerializationTest) { + TCowString nothing{}; + TCowString deserialized = SerializeThereAndBack(nothing); + + UNIT_ASSERT_VALUES_EQUAL(nothing, deserialized); + } + + Y_UNIT_TEST(RegularStringSerializationTest) { + TCowString abra = "cadabra"; + TCowString deserialized = SerializeThereAndBack(abra); + + UNIT_ASSERT_VALUES_EQUAL(abra, deserialized); + } +} // Y_UNIT_TEST_SUITE(TCowStringSerializationTest) + +Y_UNIT_TEST_SUITE(TStringConversionTest) { + Y_UNIT_TEST(ConversionToStdStringTest) { + TCowString abra = "cadabra"; + std::string stdAbra = abra; + UNIT_ASSERT_VALUES_EQUAL(stdAbra, "cadabra"); + } + + Y_UNIT_TEST(ConversionToStdStringViewTest) { + TCowString abra = "cadabra"; + std::string_view stdAbra = abra; + UNIT_ASSERT_VALUES_EQUAL(stdAbra, "cadabra"); + } +} // Y_UNIT_TEST_SUITE(TStringConversionTest) + +Y_UNIT_TEST_SUITE(HashFunctorTests) { + Y_UNIT_TEST(TestTransparency) { + THash h; + const char* ptr = "a"; + const TStringBuf strbuf = ptr; + const TCowString str = ptr; + const std::string stdStr = ptr; + UNIT_ASSERT_VALUES_EQUAL(h(ptr), h(strbuf)); + UNIT_ASSERT_VALUES_EQUAL(h(ptr), h(str)); + UNIT_ASSERT_VALUES_EQUAL(h(ptr), h(stdStr)); + } +} // Y_UNIT_TEST_SUITE(HashFunctorTests) + +Y_UNIT_TEST_SUITE(StdNonConformant) { + Y_UNIT_TEST(TestEraseNoThrow) { + TCowString x; + + LegacyErase(x, 10); + } + + Y_UNIT_TEST(TestReplaceNoThrow) { + TCowString x; + + LegacyReplace(x, 0, 0, "1"); + + UNIT_ASSERT_VALUES_EQUAL(x, "1"); + + LegacyReplace(x, 10, 0, "1"); + + UNIT_ASSERT_VALUES_EQUAL(x, "1"); + } + + Y_UNIT_TEST(TestNoAlias) { + TCowString s = "x"; + + s.AppendNoAlias("abc", 3); + + UNIT_ASSERT_VALUES_EQUAL(s, "xabc"); + UNIT_ASSERT_VALUES_EQUAL(TCowString(s.c_str()), "xabc"); + } +} // Y_UNIT_TEST_SUITE(StdNonConformant) + +Y_UNIT_TEST_SUITE(Interop) { + static void Mutate(std::string& s) { + s += "y"; + } + + static void Mutate(TCowString& s) { + Mutate(MutRef(s)); + } + + Y_UNIT_TEST(TestMutate) { + TCowString x = "x"; + + Mutate(x); + + UNIT_ASSERT_VALUES_EQUAL(x, "xy"); + } + + static std::string TransformStd(const std::string& s) { + return s + "y"; + } + + static TCowString Transform(const TCowString& s) { + return TransformStd(s); + } + + Y_UNIT_TEST(TestTransform) { + UNIT_ASSERT_VALUES_EQUAL(Transform(TCowString("x")), "xy"); + } + + Y_UNIT_TEST(TestTemp) { + UNIT_ASSERT_VALUES_EQUAL("x" + ConstRef(TCowString("y")), "xy"); + } + + static void ComparePointers(const std::string& s, const void* expected, TStringBuf descr) { + UNIT_ASSERT_VALUES_EQUAL_C(static_cast(s.c_str()), expected, descr); + } + + Y_UNIT_TEST(TestConstShared) { + TCowString s(600, 'a'); + const void* stringStart = s.c_str(); + ComparePointers(s, stringStart, "unique"); + TCowString shared{s}; + ComparePointers(s, stringStart, "shared"); // converting a TCowString to a `const std::string&` should not cause data cloning + } +} // Y_UNIT_TEST_SUITE(Interop) + +Y_UNIT_TEST_SUITE(CowPitfalls) { + template + static TString CopyStringViaBeginEndIterators(T& string, bool reverse) { + decltype(string.begin()) b; + decltype(string.end()) e; + if (!reverse) { + b = string.begin(); + e = string.end(); + } else { + e = string.end(); + b = string.begin(); + } + return TString{b, e}; + } + + Y_UNIT_TEST(IteratorCallOrder) { + const TString ref(600, 'a'); + for (const bool reverse : {false, true}) { + TCowString s = {ref.begin(), ref.end()}; + // sanity check + UNIT_ASSERT_VALUES_EQUAL_C(CopyStringViaBeginEndIterators(s, reverse), TStringBuf(ref), LabeledOutput(reverse)); + UNIT_ASSERT_VALUES_EQUAL_C(CopyStringViaBeginEndIterators(s, reverse), TStringBuf(ref), LabeledOutput(reverse)); + // test + TCowString copy = s; + UNIT_ASSERT_VALUES_EQUAL_C(CopyStringViaBeginEndIterators(s, reverse), TStringBuf(ref), LabeledOutput(reverse)); + UNIT_ASSERT_VALUES_EQUAL_C(CopyStringViaBeginEndIterators(s, reverse), TStringBuf(ref), LabeledOutput(reverse)); + } + } + + Y_UNIT_TEST(RangeFor) { + TCowString str; + str.resize(200); + TCowString copy = str; + for (auto& c : str) { + c = 'x'; + } + UNIT_ASSERT_VALUES_EQUAL(str, TString(200, 'x')); + } +} // Y_UNIT_TEST_SUITE(CowPitfalls) diff --git a/library/cpp/containers/cow_string/output.cpp b/library/cpp/containers/cow_string/output.cpp new file mode 100644 index 0000000000..e0b4924ad3 --- /dev/null +++ b/library/cpp/containers/cow_string/output.cpp @@ -0,0 +1,46 @@ +#include "cow_string.h" + +#include +#include +#include + +constexpr size_t MAX_UTF8_BYTES = 4; // UTF-8-encoded code point takes between 1 and 4 bytes + +template +static void WriteString(IOutputStream& o, const TCharType* w, size_t n) { + const size_t buflen = (n * MAX_UTF8_BYTES); // * 4 because the conversion functions can convert unicode character into maximum 4 bytes of UTF8 + TTempBuf buffer(buflen + 1); + size_t written = 0; + WideToUTF8(w, n, buffer.Data(), written); + o.Write(buffer.Data(), written); +} + +template <> +void Out(IOutputStream& o, const TCowString& p) { + o.Write(p.data(), p.size()); +} + +template <> +void Out(IOutputStream& o, const TUtf16CowString& w) { + WriteString(o, w.c_str(), w.size()); +} + +template <> +void Out(IOutputStream& o, const TUtf32CowString& w) { + WriteString(o, w.c_str(), w.size()); +} + +template <> +void Out>(IOutputStream& o, const TBasicCharRef& c) { + o << static_cast(c); +} + +template <> +void Out>(IOutputStream& o, const TBasicCharRef& c) { + o << static_cast(c); +} + +template <> +void Out>(IOutputStream& o, const TBasicCharRef& c) { + o << static_cast(c); +} diff --git a/library/cpp/containers/cow_string/reverse.cpp b/library/cpp/containers/cow_string/reverse.cpp new file mode 100644 index 0000000000..b5bd10d250 --- /dev/null +++ b/library/cpp/containers/cow_string/reverse.cpp @@ -0,0 +1,32 @@ +#include "reverse.h" + +#include +#include + +#include + +void ReverseInPlace(TCowString& string) { + auto* begin = string.begin(); + std::reverse(begin, begin + string.size()); +} + +void ReverseInPlace(TUtf16CowString& string) { + auto* begin = string.begin(); + const auto len = string.size(); + auto* end = begin + string.size(); + + TVector buffer(len); + wchar16* rbegin = buffer.data() + len; + for (wchar16* p = begin; p < end;) { + const size_t symbolSize = W16SymbolSize(p, end); + rbegin -= symbolSize; + std::copy(p, p + symbolSize, rbegin); + p += symbolSize; + } + std::copy(buffer.begin(), buffer.end(), begin); +} + +void ReverseInPlace(TUtf32CowString& string) { + auto* begin = string.begin(); + std::reverse(begin, begin + string.size()); +} diff --git a/library/cpp/containers/cow_string/reverse.h b/library/cpp/containers/cow_string/reverse.h new file mode 100644 index 0000000000..d27b0b4fed --- /dev/null +++ b/library/cpp/containers/cow_string/reverse.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +void ReverseInPlace(TCowString& string); + +/** NB. UTF-16 is variable-length encoding because of the surrogate pairs. + * This function takes this into account and treats a surrogate pair as a single symbol. + * Ex. if [C D] is a surrogate pair, + * A B [C D] E + * will become + * E [C D] B A + */ +void ReverseInPlace(TUtf16CowString& string); + +void ReverseInPlace(TUtf32CowString& string); diff --git a/library/cpp/containers/cow_string/str_stl.h b/library/cpp/containers/cow_string/str_stl.h new file mode 100644 index 0000000000..d8256a6e10 --- /dev/null +++ b/library/cpp/containers/cow_string/str_stl.h @@ -0,0 +1,67 @@ +#pragma once + +#include + +template <> +struct hash: ::NHashPrivate::TStringHash { +}; + +template <> +struct hash: ::NHashPrivate::TStringHash { +}; + +template <> +struct hash: ::NHashPrivate::TStringHash { +}; + +template <> +struct TEqualTo: public TEqualTo { + using is_transparent = void; +}; + +template <> +struct TEqualTo: public TEqualTo { + using is_transparent = void; +}; + +template <> +struct TEqualTo: public TEqualTo { + using is_transparent = void; +}; + +template <> +struct TCIEqualTo { + inline bool operator()(const TCowString& a, const TCowString& b) const { + return a.size() == b.size() && strnicmp(a.data(), b.data(), a.size()) == 0; + } +}; + +template <> +struct TLess: public TLess { + using is_transparent = void; +}; + +template <> +struct TLess: public TLess { + using is_transparent = void; +}; + +template <> +struct TLess: public TLess { + using is_transparent = void; +}; + +template <> +struct TGreater: public TGreater { + using is_transparent = void; +}; + +template <> +struct TGreater: public TGreater { + using is_transparent = void; +}; + +template <> +struct TGreater: public TGreater { + using is_transparent = void; +}; diff --git a/library/cpp/containers/cow_string/subst.cpp b/library/cpp/containers/cow_string/subst.cpp new file mode 100644 index 0000000000..d4e9ff3395 --- /dev/null +++ b/library/cpp/containers/cow_string/subst.cpp @@ -0,0 +1,182 @@ +#include "subst.h" + +#include +#include +#include + +#include +#include + +// a bit of template magic (to be fast and unreadable) +template +static Y_FORCE_INLINE void MoveBlock(typename TStringType::value_type* ptr, size_t& srcPos, size_t& dstPos, const size_t off, const TTo to, const size_t toSize) { + const size_t unchangedSize = off - srcPos; + if (dstPos < srcPos) { + for (size_t i = 0; i < unchangedSize; ++i) { + ptr[dstPos++] = ptr[srcPos++]; + } + } else { + dstPos += unchangedSize; + srcPos += unchangedSize; + } + + if (Main) { + for (size_t i = 0; i < toSize; ++i) { + ptr[dstPos++] = to[i]; + } + } +} + +template +static bool IsIntersect(const T& a, const U& b) noexcept { + if (b.data() < a.data()) { + return IsIntersect(b, a); + } + + return !a.empty() && !b.empty() && + ((a.data() <= b.data() && b.data() < a.data() + a.size()) || + (a.data() < b.data() + b.size() && b.data() + b.size() <= a.data() + a.size())); +} + +/** + * Replaces all occurences of substring @c from in string @c s to string @c to. + * Uses two separate implementations (inplace for shrink and append for grow case) + * See IGNIETFERRO-394 + **/ +template > +static inline size_t SubstGlobalImpl(TStringType& s, const TStringViewType from, const TStringViewType to, size_t fromPos = 0) { + if (from.empty()) { + return 0; + } + + Y_ASSERT(!IsIntersect(s, from)); + Y_ASSERT(!IsIntersect(s, to)); + + const size_t fromSize = from.size(); + const size_t toSize = to.size(); + size_t replacementsCount = 0; + size_t off = fromPos; + size_t srcPos = 0; + + if (toSize > fromSize) { + // string will grow: append to another string + TStringType result; + for (; (off = TStringViewType(s).find(from, off)) != TStringType::npos; off += fromSize) { + if (!replacementsCount) { + // first replacement occured, we can prepare result string + result.reserve(s.size() + s.size() / 3); + } + result.append(s.begin() + srcPos, s.begin() + off); + result.append(to.data(), to.size()); + srcPos = off + fromSize; + ++replacementsCount; + } + if (replacementsCount) { + // append tail + result.append(s.begin() + srcPos, s.end()); + s = std::move(result); + } + return replacementsCount; + } + + // string will not grow: use inplace algo + size_t dstPos = 0; + typename TStringType::value_type* ptr = &*s.begin(); + for (; (off = TStringViewType(s).find(from, off)) != TStringType::npos; off += fromSize) { + Y_ASSERT(dstPos <= srcPos); + MoveBlock(ptr, srcPos, dstPos, off, to, toSize); + srcPos = off + fromSize; + ++replacementsCount; + } + + if (replacementsCount) { + // append tail + MoveBlock(ptr, srcPos, dstPos, s.size(), to, toSize); + s.resize(dstPos); + } + return replacementsCount; +} + +/// Replaces all occurences of the 'from' symbol in a string to the 'to' symbol. +template +inline size_t SubstCharGlobalImpl(TStringType& s, typename TStringType::value_type from, typename TStringType::value_type to, size_t fromPos = 0) { + if (fromPos >= s.size()) { + return 0; + } + + size_t result = 0; + fromPos = s.find(from, fromPos); + + // s.begin() might cause memory copying, so call it only if needed + if (fromPos != TStringType::npos) { + auto* it = &*s.begin() + fromPos; + *it = to; + ++result; + // at this point string is copied and it's safe to use constant s.end() to iterate + const auto* const sEnd = &*s.end(); + // unrolled loop goes first because it is more likely that `it` will be properly aligned + for (const auto* const end = sEnd - (sEnd - it) % 4; it < end;) { + if (*it == from) { + *it = to; + ++result; + } + ++it; + if (*it == from) { + *it = to; + ++result; + } + ++it; + if (*it == from) { + *it = to; + ++result; + } + ++it; + if (*it == from) { + *it = to; + ++result; + } + ++it; + } + for (; it < sEnd; ++it) { + if (*it == from) { + *it = to; + ++result; + } + } + } + + return result; +} + +/* Standard says that `char16_t` is a distinct type and has same size, signedness and alignment as + * `std::uint_least16_t`, so we check if `char16_t` has same signedness and size as `wchar16` to be + * sure that we can make safe casts between values of these types and pointers. + */ +static_assert(sizeof(wchar16) == sizeof(char16_t), ""); +static_assert(sizeof(wchar32) == sizeof(char32_t), ""); +static_assert(std::is_unsigned::value == std::is_unsigned::value, ""); +static_assert(std::is_unsigned::value == std::is_unsigned::value, ""); + +size_t SubstGlobal(TCowString& text, const TStringBuf what, const TStringBuf with, size_t from) { + return SubstGlobalImpl(text, what, with, from); +} + +size_t SubstGlobal(TUtf16CowString& text, const TWtringBuf what, const TWtringBuf with, size_t from) { + return SubstGlobalImpl(text, what, with, from); +} + +size_t SubstGlobal(TUtf32CowString& text, const TUtf32StringBuf what, const TUtf32StringBuf with, size_t from) { + return SubstGlobalImpl(text, what, with, from); +} + +size_t SubstGlobal(TCowString& text, char what, char with, size_t from) { + return SubstCharGlobalImpl(text, what, with, from); +} + +size_t SubstGlobal(TUtf16CowString& text, wchar16 what, wchar16 with, size_t from) { + return SubstCharGlobalImpl(text, (char16_t)what, (char16_t)with, from); +} + +size_t SubstGlobal(TUtf32CowString& text, wchar32 what, wchar32 with, size_t from) { + return SubstCharGlobalImpl(text, (char32_t)what, (char32_t)with, from); +} diff --git a/library/cpp/containers/cow_string/subst.h b/library/cpp/containers/cow_string/subst.h new file mode 100644 index 0000000000..6090ba54b2 --- /dev/null +++ b/library/cpp/containers/cow_string/subst.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include + +/* Replace all occurences of substring `what` with string `with` starting from position `from`. + * + * @param text String to modify. + * @param what Substring to replace. + * @param with Substring to use as replacement. + * @param from Position at with to start replacement. + * + * @return Number of replacements occured. + */ +size_t SubstGlobal(TCowString& text, TStringBuf what, TStringBuf with, size_t from = 0); +size_t SubstGlobal(TUtf16CowString& text, TWtringBuf what, TWtringBuf with, size_t from = 0); +size_t SubstGlobal(TUtf32CowString& text, TUtf32StringBuf what, TUtf32StringBuf with, size_t from = 0); + +/* Replace all occurences of character `what` with character `with` starting from position `from`. + * + * @param text String to modify. + * @param what Character to replace. + * @param with Character to use as replacement. + * @param from Position at with to start replacement. + * + * @return Number of replacements occured. + */ +size_t SubstGlobal(TCowString& text, char what, char with, size_t from = 0); +size_t SubstGlobal(TUtf16CowString& text, wchar16 what, wchar16 with, size_t from = 0); +size_t SubstGlobal(TUtf32CowString& text, wchar32 what, wchar32 with, size_t from = 0); diff --git a/library/cpp/containers/cow_string/ut_medium/cow_string_medium_ut.cpp b/library/cpp/containers/cow_string/ut_medium/cow_string_medium_ut.cpp new file mode 100644 index 0000000000..a9a37db776 --- /dev/null +++ b/library/cpp/containers/cow_string/ut_medium/cow_string_medium_ut.cpp @@ -0,0 +1,55 @@ +#include + +#include + +#include +#include +#include +#include + +#include +#include + +static_assert(sizeof(TCowString) == sizeof(const char*), "expect sizeof(TCowString) == sizeof(const char*)"); + +Y_UNIT_TEST_SUITE(CowPitfalls) { + Y_UNIT_TEST(ParallelDetach) { + // best results with thread-sanitizer + std::vector> threads; + TCowString a = "the string"; + TCowString b = a; + auto makeRefToA = [&a, &b]() { + b = a; // make second reference to the same string + }; + constexpr int nThreads = 8; +#ifdef _tsan_enabled_ + constexpr i64 retries = 1'000; +#else + constexpr i64 retries = 1'000'000; +#endif + std::barrier iterationSyncPoint(nThreads, makeRefToA); + std::atomic totalLen = 0; + auto addLen = [](std::string a, std::atomic& len) { + len += a.length(); + }; + auto workload = [&a, &addLen, &totalLen, &iterationSyncPoint]() { + std::atomic len = 0; + for (i64 j = 0; j < retries; ++j) { + addLen(a, len); // possibility of bad implicit conversion + iterationSyncPoint.arrive_and_wait(); + } + totalLen += len.load(); + }; + for (int i = 0; i < nThreads; ++i) { + threads.push_back(std::make_unique(workload)); + } + for (auto& t : threads) { + t->Start(); + } + for (auto& t : threads) { + t->Join(); + } + UNIT_ASSERT_VALUES_EQUAL(totalLen.load(), b.size() * nThreads * retries); + } + +} // Y_UNIT_TEST_SUITE(CowPitfalls) diff --git a/library/cpp/containers/cow_string/ysaveload.cpp b/library/cpp/containers/cow_string/ysaveload.cpp new file mode 100644 index 0000000000..57555fc1d9 --- /dev/null +++ b/library/cpp/containers/cow_string/ysaveload.cpp @@ -0,0 +1 @@ +#include "ysaveload.h" diff --git a/library/cpp/containers/cow_string/ysaveload.h b/library/cpp/containers/cow_string/ysaveload.h new file mode 100644 index 0000000000..a4da801c40 --- /dev/null +++ b/library/cpp/containers/cow_string/ysaveload.h @@ -0,0 +1,9 @@ +#pragma once + +#include "cow_string.h" + +#include + +template <> +class TSerializer: public TVectorSerializer { +}; diff --git a/library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h b/library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h index f0c6644d4b..c5bfb659ac 100644 --- a/library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h +++ b/library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h @@ -115,7 +115,8 @@ class TDisjointIntervalTree { if (containingBegin->first < begin && begin < containingBegin->second) { // Contains begin. if (containingBegin->second > end) { // Contains end. const T prevEnd = containingBegin->second; - Y_ASSERT(containingBegin->second - begin <= NumElements); + Y_ASSERT(containingBegin->second >= begin); + Y_ASSERT(static_cast(containingBegin->second - begin) <= NumElements); Y_ASSERT(containingBegin->second - containingBegin->first > end - begin); containingBegin->second = begin; diff --git a/library/cpp/containers/disjoint_interval_tree/ut/disjoint_interval_tree_ut.cpp b/library/cpp/containers/disjoint_interval_tree/ut/disjoint_interval_tree_ut.cpp index 508a82459a..69278c31f7 100644 --- a/library/cpp/containers/disjoint_interval_tree/ut/disjoint_interval_tree_ut.cpp +++ b/library/cpp/containers/disjoint_interval_tree/ut/disjoint_interval_tree_ut.cpp @@ -288,4 +288,23 @@ Y_UNIT_TEST_SUITE(DisjointIntervalTreeTest) { UNIT_ASSERT(!tree.Intersects(15, 18)); } } + + Y_UNIT_TEST(TestI64) { + { + TDisjointIntervalTree tree; + tree.InsertInterval(-5, 10); + UNIT_ASSERT_VALUES_EQUAL(tree.EraseInterval(-2, 4), 6); + UNIT_ASSERT_VALUES_EQUAL(tree.GetNumIntervals(), 2); + UNIT_ASSERT_VALUES_EQUAL(tree.GetNumElements(), 9); + + UNIT_ASSERT_VALUES_EQUAL(tree.EraseInterval(-5, -2), 3); + UNIT_ASSERT_VALUES_EQUAL(tree.GetNumIntervals(), 1); + UNIT_ASSERT_VALUES_EQUAL(tree.GetNumElements(), 6); + + UNIT_ASSERT_VALUES_EQUAL(tree.EraseInterval(4, 10), 6); + UNIT_ASSERT_VALUES_EQUAL(tree.GetNumIntervals(), 0); + UNIT_ASSERT_VALUES_EQUAL(tree.GetNumElements(), 0); + UNIT_ASSERT(tree.Empty()); + } + } } diff --git a/library/cpp/containers/paged_vector/README.md b/library/cpp/containers/paged_vector/README.md new file mode 100644 index 0000000000..ae49c0c652 --- /dev/null +++ b/library/cpp/containers/paged_vector/README.md @@ -0,0 +1,99 @@ +# TPagedVector + +`NPagedVector::TPagedVector` is a dynamic sequence container implemented as a 2-level radix tree: elements are stored in fixed-size, individually heap-allocated pages, and a top-level vector holds pointers to those pages. + +```cpp +#include + +namespace NPagedVector { + template + class TPagedVector; +} +``` + +- `T` — element type. +- `PageSize` — number of elements per page (default: `1u << 20u` = 1,048,576 elements). + +## Why use it instead of TVector / std::vector? + +- **No reallocation of elements.** Growth allocates a new page instead of reallocating and moving the entire buffer. Elements are never moved on `push_back`/`emplace_back`, so references and pointers to existing elements remain valid when appending (iterators are offset-based and also stay usable). +- **No large contiguous allocations.** Memory is requested in page-size chunks, which is friendlier to the allocator for very large containers. +- **Cheaper worst-case append.** `push_back` never triggers an O(n) copy; the cost is at most one page allocation. + +The trade-off is that storage is not contiguous (no `data()`), and indexing does one extra pointer dereference (`idx / PageSize`, `idx % PageSize`). + +## API overview + +The interface mirrors a subset of `std::vector`: + +| Category | Members | +|---|---| +| Construction | default, copy, move, `TPagedVector(TIter b, TIter e)` | +| Assignment | copy, move, `swap()` | +| Element access | `operator[]`, `at()` (throws `std::out_of_range`), `front()`, `back()` | +| Iterators | `begin()/end()`, `rbegin()/rend()` + const versions; random-access iterators | +| Capacity | `size()`, `empty()`, `explicit operator bool()` (true when non-empty) | +| Modifiers | `push_back()`, `emplace_back()` (returns a reference), `pop_back()`, `append(b, e)`, `erase(it)`, `erase(b, e)`, `resize()`, `clear()` | +| Iteration helpers | `ForEach(fn)`, `ForEachReverse(fn)` | +| Comparison | `operator==`, `operator<` (lexicographical) | + +Notable differences from `std::vector`: + +- No `reserve()`/`capacity()`/`shrink_to_fit()` and no `data()` — storage is paged, not contiguous. + +## Iterators + +Iterators are random-access and are implemented as an *(owner pointer, offset)* pair. Consequences: + +- Iterators are not invalidated by `push_back`/`emplace_back` (an `end()` iterator taken earlier keeps pointing to the same logical position). +- Dereferencing goes through the vector, so an iterator is only valid while its source container is alive. +- To get the current index of an element from an iterator, call `it.GetIndex()` — it returns the offset of the pointed-to element within the container (equivalent to `it - begin()`). + +## Iteration helpers + +```cpp +template +void ForEach(Function fn) const; + +template +void ForEachReverse(Function fn) const; +``` + +`ForEach` applies `fn` to every element **from the first to the last**; `ForEachReverse` applies `fn` **from the last to the first**. + +These are faster than iterating with `begin()/end()` or `rbegin()/rend()`: they walk the pages directly through raw pointers, avoiding the two levels of indirection that the offset-based iterators go through on each dereference. This matters for containers with a large `PageSize` (the default is 1M elements per page), where the inner per-page loop is tight. + +```cpp +TPagedVector v; +// ... fill v ... + +long long sum = 0; +v.ForEach([&](int x) { sum += x; }); + +// process elements back-to-front, e.g. for a stack-like traversal +v.ForEachReverse([&](int x) { + // ... +}); +``` + +Notes: + +- The order is well-defined and contiguous: `ForEach` visits element `0, 1, ..., size()-1`; `ForEachReverse` visits `size()-1, ..., 1, 0`. +- Both are O(n) and do not allocate. + +## Complexity + +| Operation | Complexity | +|---|---| +| `operator[]` / `at()` | O(1) | +| `push_back` / `emplace_back` | O(1) amortized (page allocation at most every `PageSize` appends) | +| `pop_back` | O(1) | +| `erase` | O(n) — shifts all following elements | +| `clear` | O(n) for non-trivially destructible `T`, O(pages) otherwise | +| `ForEach` / `ForEachReverse` | O(n), no allocations | + +## Notes + +- Pages are allocated as raw storage; elements are constructed in place and destroyed explicitly, so non-trivially destructible types are handled correctly. +- Destruction of trivially destructible types is skipped entirely, making `clear()` and the destructor fast for POD-like types. +- The copy constructor is exception-safe: on a throw during copying, already-constructed elements are destroyed. diff --git a/library/cpp/containers/paged_vector/paged_vector.h b/library/cpp/containers/paged_vector/paged_vector.h index 3a2c58caf7..43073852ab 100644 --- a/library/cpp/containers/paged_vector/paged_vector.h +++ b/library/cpp/containers/paged_vector/paged_vector.h @@ -4,90 +4,91 @@ #include #include +#include #include namespace NPagedVector { - template > + template class TPagedVector; namespace NPrivate { - template + template struct TPagedVectorIterator { private: - friend class TPagedVector; - typedef TPagedVector TVec; - typedef TPagedVectorIterator TSelf; - size_t Offset; - TVec* Vector; + friend class TPagedVector; + using TVec = TPagedVector; + using TSelf = TPagedVectorIterator; + size_t Index_; + TVec* Vector_; - template + template friend struct TPagedVectorIterator; public: TPagedVectorIterator() - : Offset() - , Vector() + : Index_() + , Vector_() { } - TPagedVectorIterator(TVec* vector, size_t offset) - : Offset(offset) - , Vector(vector) + TPagedVectorIterator(TVec* vector, size_t index) + : Index_(index) + , Vector_(vector) { } - template - TPagedVectorIterator(const TPagedVectorIterator& it) - : Offset(it.Offset) - , Vector(it.Vector) + template + TPagedVectorIterator(const TPagedVectorIterator& it) + : Index_(it.Index_) + , Vector_(it.Vector_) { } T& operator*() const { - return (*Vector)[Offset]; + return (*Vector_)[Index_]; } T* operator->() const { return &(**this); } - template - bool operator==(const TPagedVectorIterator& it) const { - return Offset == it.Offset; + template + bool operator==(const TPagedVectorIterator& it) const { + return Index_ == it.Index_; } - template - bool operator!=(const TPagedVectorIterator& it) const { + template + bool operator!=(const TPagedVectorIterator& it) const { return !(*this == it); } - template - bool operator<(const TPagedVectorIterator& it) const { - return Offset < it.Offset; + template + bool operator<(const TPagedVectorIterator& it) const { + return Index_ < it.Index_; } - template - bool operator<=(const TPagedVectorIterator& it) const { - return Offset <= it.Offset; + template + bool operator<=(const TPagedVectorIterator& it) const { + return Index_ <= it.Index_; } - template - bool operator>(const TPagedVectorIterator& it) const { + template + bool operator>(const TPagedVectorIterator& it) const { return !(*this <= it); } - template - bool operator>=(const TPagedVectorIterator& it) const { + template + bool operator>=(const TPagedVectorIterator& it) const { return !(*this < it); } - template - ptrdiff_t operator-(const TPagedVectorIterator& it) const { - return Offset - it.Offset; + template + ptrdiff_t operator-(const TPagedVectorIterator& it) const { + return Index_ - it.Index_; } TSelf& operator+=(ptrdiff_t off) { - Offset += off; + Index_ += off; return *this; } @@ -125,51 +126,121 @@ namespace NPagedVector { return this->operator+(-off); } - size_t GetOffset() const { - return Offset; + [[nodiscard]] size_t GetIndex() const { + return Index_; } }; - } -} + } // namespace NPrivate +} // namespace NPagedVector namespace std { - template - struct iterator_traits> { - typedef ptrdiff_t difference_type; - typedef T value_type; - typedef T* pointer; - typedef T& reference; - typedef random_access_iterator_tag iterator_category; + template + struct iterator_traits> { + using difference_type = ptrdiff_t; + using value_type = T; + using pointer = T*; + using reference = T&; + using iterator_category = random_access_iterator_tag; }; -} +} // namespace std namespace NPagedVector { - //2-level radix tree - template - class TPagedVector: private TVector>, A> { + // 2-level radix tree + template + class TPagedVector { static_assert(PageSize, "expect PageSize"); - typedef TVector TPage; - typedef TVector, A> TPages; - typedef TPagedVector TSelf; + class alignas(T) TPage { + alignas(T) std::array Data_; + + public: + T* data() { + return reinterpret_cast(Data_.data()); + } + + const T* data() const { + return reinterpret_cast(Data_.data()); + } + + T& operator[](size_t idx) { + return *(data() + idx); + } + + const T& operator[](size_t idx) const { + return *(data() + idx); + } + }; + + using TPages = TVector>; + using TSelf = TPagedVector; + + TPages Pages_; + size_t CurrentPageSize_ = 0; public: - typedef NPrivate::TPagedVectorIterator iterator; - typedef NPrivate::TPagedVectorIterator const_iterator; - typedef std::reverse_iterator reverse_iterator; - typedef std::reverse_iterator const_reverse_iterator; - typedef T value_type; - typedef value_type& reference; - typedef const value_type& const_reference; + using iterator = NPrivate::TPagedVectorIterator; + using const_iterator = NPrivate::TPagedVectorIterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + using value_type = T; + using reference = value_type&; + using const_reference = const value_type&; TPagedVector() = default; + TPagedVector(TPagedVector&& other) noexcept + : Pages_(std::move(other.Pages_)) + , CurrentPageSize_(other.CurrentPageSize_) + { + other.CurrentPageSize_ = 0; + } + + TPagedVector(const TPagedVector& other) { + Pages_.reserve(other.Pages_.size()); + try { + for (auto& ptr : other.Pages_) { + auto& newPage = *Pages_.emplace_back(MakeHolder()); + CurrentPageSize_ = 0; + const size_t copyCount = Pages_.size() == other.Pages_.size() + ? other.CurrentPageSize_ + : PageSize; + + std::uninitialized_copy_n(ptr->data(), copyCount, newPage.data()); + CurrentPageSize_ = copyCount; + } + } catch (...) { + clear(); + throw; + } + } + + ~TPagedVector() { + clear(); + } template TPagedVector(TIter b, TIter e) { append(b, e); } + TPagedVector& operator=(const TPagedVector& other) { + if (this != &other) { + TPagedVector tmp(other); + swap(tmp); + } + return *this; + } + + TPagedVector& operator=(TPagedVector&& other) noexcept { + if (this != &other) { + clear(); + Pages_ = std::move(other.Pages_); + CurrentPageSize_ = other.CurrentPageSize_; + other.CurrentPageSize_ = 0; + } + return *this; + } + iterator begin() { return iterator(this, 0); } @@ -203,7 +274,53 @@ namespace NPagedVector { } void swap(TSelf& v) { - TPages::swap((TPages&)v); + Pages_.swap(v.Pages_); + std::swap(CurrentPageSize_, v.CurrentPageSize_); + } + + // Fast iteration over all elements. + template + void ForEach(Function fn) const { + if (Pages_.empty()) { + return; + } + + const auto currentPageIt = Pages_.end() - 1; + for (auto it = Pages_.begin(); it != currentPageIt; ++it) { + const TPage& page = **it; + for (size_t i = 0; i < PageSize; ++i) { + fn(page[i]); + } + } + + const TPage& currentPage = **currentPageIt; + + for (size_t i = 0; i < CurrentPageSize_; ++i) { + fn(currentPage[i]); + } + } + + // Fast iteration over all elements in reverse order. + template + void ForEachReverse(Function fn) const { + if (Pages_.empty()) { + return; + } + + const TPage& currentPage = *Pages_.back(); + + for (size_t i = CurrentPageSize_; i > 0;) { + --i; + fn(currentPage[i]); + } + + for (auto it = Pages_.rbegin() + 1; it != Pages_.rend(); ++it) { + const TPage& page = **it; + for (size_t i = PageSize; i > 0;) { + --i; + fn(page[i]); + } + } } private: @@ -215,110 +332,99 @@ namespace NPagedVector { return idx % PageSize; } - static size_t Index(size_t pnum, size_t poff) { - return pnum * PageSize + poff; - } - TPage& PageAt(size_t pnum) const { - return *TPages::at(pnum); + return *Pages_.at(pnum); } TPage& CurrentPage() const { - return *TPages::back(); - } - - size_t CurrentPageSize() const { - return TPages::empty() ? 0 : CurrentPage().size(); + return *Pages_.back(); } size_t NPages() const { - return TPages::size(); + return Pages_.size(); } void AllocateNewPage() { - TPages::push_back(new TPage()); - CurrentPage().reserve(PageSize); - } - - void MakeNewPage() { - AllocateNewPage(); - CurrentPage().resize(PageSize); + Pages_.emplace_back(MakeHolder()); + CurrentPageSize_ = 0; } void PrepareAppend() { - if (TPages::empty() || CurrentPage().size() + 1 > PageSize) + if (Pages_.empty() || CurrentPageSize_ >= PageSize) { AllocateNewPage(); + } } public: size_t size() const { - return empty() ? 0 : (NPages() - 1) * PageSize + CurrentPage().size(); + return Pages_.empty() ? 0 : (NPages() - 1) * PageSize + CurrentPageSize_; } bool empty() const { - return TPages::empty() || (1 == NPages() && CurrentPage().empty()); + return Pages_.empty() || (1 == NPages() && CurrentPageSize_ == 0); } explicit operator bool() const noexcept { return !empty(); } - template + template reference emplace_back(Args&&... args) { PrepareAppend(); - return CurrentPage().emplace_back(std::forward(args)...); + T* ptr = new (CurrentPage().data() + CurrentPageSize_) T(std::forward(args)...); + ++CurrentPageSize_; + return *ptr; } void push_back(const_reference t) { PrepareAppend(); - CurrentPage().push_back(t); + new (CurrentPage().data() + CurrentPageSize_) T(t); + ++CurrentPageSize_; } void pop_back() { - if (CurrentPage().empty()) - TPages::pop_back(); - CurrentPage().pop_back(); + Y_ASSERT(!empty()); + if (CurrentPageSize_ == 0) { + Pages_.pop_back(); + CurrentPageSize_ = PageSize; + } + --CurrentPageSize_; + if constexpr (!std::is_trivially_destructible_v) { + CurrentPage()[CurrentPageSize_].~T(); + } } template void append(TIter b, TIter e) { - size_t sz = e - b; - size_t sz1 = Min(sz, PageSize - CurrentPageSize()); - size_t sz2 = (sz - sz1) / PageSize; - size_t sz3 = (sz - sz1) % PageSize; - - if (sz1) { - PrepareAppend(); - TPage& p = CurrentPage(); - p.insert(p.end(), b, b + sz1); - } - - for (size_t i = 0; i < sz2; ++i) { - AllocateNewPage(); - TPage& p = CurrentPage(); - p.insert(p.end(), b + sz1 + i * PageSize, b + sz1 + (i + 1) * PageSize); - } - - if (sz3) { - AllocateNewPage(); - TPage& p = CurrentPage(); - p.insert(p.end(), b + sz1 + sz2 * PageSize, e); + for (TIter it = b; it != e; ++it) { + push_back(*it); } } iterator erase(iterator it) { - size_t pnum = PageNumber(it.Offset); - size_t pidx = InPageIndex(it.Offset); - - if (CurrentPage().empty()) - TPages::pop_back(); + if (CurrentPageSize_ == 0) { + Pages_.pop_back(); + CurrentPageSize_ = Pages_.empty() ? 0 : PageSize; + } - for (size_t p = NPages() - 1; p > pnum; --p) { - PageAt(p - 1).push_back(PageAt(p).front()); - PageAt(p).erase(PageAt(p).begin()); + size_t pidx = InPageIndex(it.Index_); + for (size_t pnum = PageNumber(it.Index_);; ++pnum) { + TPage& page = *Pages_[pnum]; + if (pnum + 1 == Pages_.size()) { + std::shift_left(page.data() + pidx, page.data() + CurrentPageSize_, 1); + --CurrentPageSize_; + if constexpr (!std::is_trivially_destructible_v) { + page[CurrentPageSize_].~T(); + } + break; + } + + std::shift_left(page.data() + pidx, page.data() + PageSize, 1); + TPage& nextPage = *Pages_[pnum + 1]; + page[PageSize - 1] = std::move(nextPage[0]); + pidx = 0; } - PageAt(pnum).erase(PageAt(pnum).begin() + pidx); return it; } @@ -332,86 +438,96 @@ namespace NPagedVector { return b; } - iterator insert(iterator it, const value_type& v) { - size_t pnum = PageNumber(it.Offset); - size_t pidx = InPageIndex(it.Offset); - - PrepareAppend(); - - for (size_t p = NPages() - 1; p > pnum; --p) { - PageAt(p).insert(PageAt(p).begin(), PageAt(p - 1).back()); - PageAt(p - 1).pop_back(); - } - - PageAt(pnum).insert(PageAt(pnum).begin() + pidx, v); - return it; - } - - template - void insert(iterator it, TIter b, TIter e) { - // todo : suboptimal! - for (; b != e; ++b, ++it) - it = insert(it, *b); - } - reference front() { - return TPages::front()->front(); + Y_ASSERT(CurrentPageSize_ > 0 || Pages_.size() > 1); + return (*Pages_.front())[0]; } const_reference front() const { - return TPages::front()->front(); + Y_ASSERT(CurrentPageSize_ > 0 || Pages_.size() > 1); + return (*Pages_.front())[0]; } reference back() { - return CurrentPage().back(); + if (CurrentPageSize_ > 0) { + return CurrentPage()[CurrentPageSize_ - 1]; + } else { + Y_ASSERT(Pages_.size() >= 2); + return (**(Pages_.end() - 2))[PageSize - 1]; + } } const_reference back() const { - return CurrentPage().back(); + if (CurrentPageSize_ > 0) { + return CurrentPage()[CurrentPageSize_ - 1]; + } else { + Y_ASSERT(Pages_.size() >= 2); + return (**(Pages_.end() - 2))[PageSize - 1]; + } } void clear() { - TPages::clear(); + if constexpr (std::is_trivially_destructible_v) { + Pages_.clear(); + CurrentPageSize_ = 0; + } else { + while (!Pages_.empty()) { + TPage& page = CurrentPage(); + while (CurrentPageSize_ > 0) { + --CurrentPageSize_; + page[CurrentPageSize_].~T(); + } + Pages_.pop_back(); + CurrentPageSize_ = Pages_.empty() ? 0 : PageSize; + } + } } void resize(size_t sz) { - if (sz == size()) + size_t curSize = size(); + if (sz == curSize) { return; + } - const size_t npages = NPages(); - const size_t newwholepages = sz / PageSize; - const size_t pagepart = sz % PageSize; - const size_t newpages = newwholepages + bool(pagepart); - - if (npages && newwholepages >= npages) - CurrentPage().resize(PageSize); - - if (newpages < npages) - TPages::resize(newpages); - else - for (size_t i = npages; i < newpages; ++i) - MakeNewPage(); - - if (pagepart) - CurrentPage().resize(pagepart); - - Y_ABORT_UNLESS(sz == size(), "%" PRIu64 " %" PRIu64, (ui64)sz, (ui64)size()); + if (sz < curSize) { + while (sz < curSize) { + pop_back(); + --curSize; + } + } else { + while (sz > curSize) { + emplace_back(); + ++curSize; + } + } } reference at(size_t idx) { - return TPages::at(PageNumber(idx))->at(InPageIndex(idx)); + if (idx >= size()) { + throw std::out_of_range("TPagedVector::at() - index out of range"); + } + const size_t pnum = PageNumber(idx); + const size_t inPageIdx = InPageIndex(idx); + + return (*Pages_[pnum])[inPageIdx]; } const_reference at(size_t idx) const { - return TPages::at(PageNumber(idx))->at(InPageIndex(idx)); + if (idx >= size()) { + throw std::out_of_range("TPagedVector::at() - index out of range"); + } + const size_t pnum = PageNumber(idx); + const size_t inPageIdx = InPageIndex(idx); + + return (*Pages_[pnum])[inPageIdx]; } reference operator[](size_t idx) { - return TPages::operator[](PageNumber(idx))->operator[](InPageIndex(idx)); + return Pages_.operator[](PageNumber(idx))->operator[](InPageIndex(idx)); } const_reference operator[](size_t idx) const { - return TPages::operator[](PageNumber(idx))->operator[](InPageIndex(idx)); + return Pages_.operator[](PageNumber(idx))->operator[](InPageIndex(idx)); } friend bool operator==(const TSelf& a, const TSelf& b) { @@ -424,10 +540,9 @@ namespace NPagedVector { }; namespace NPrivate { - typedef std::is_same::iterator>::iterator_category> - TIteratorCheck; + using TIteratorCheck = std::is_same::iterator>::iterator_category>; static_assert(TIteratorCheck::value, "expect TIteratorCheck::Result"); - } + } // namespace NPrivate -} +} // namespace NPagedVector diff --git a/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp b/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp index d059ce34ec..b0f39f748f 100644 --- a/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp +++ b/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp @@ -12,13 +12,25 @@ class TPagedVectorTest: public TTestBase { UNIT_TEST(Test4) UNIT_TEST(Test5) UNIT_TEST(Test6) - UNIT_TEST(Test7) UNIT_TEST(TestAt) UNIT_TEST(TestAutoRef) UNIT_TEST(TestIterators) UNIT_TEST(TestEmplaceBack1) UNIT_TEST(TestEmplaceBack2) - //UNIT_TEST(TestEbo) + UNIT_TEST(TestCopyConstructor) + UNIT_TEST(TestCopyAssignment) + UNIT_TEST(TestMoveConstructor) + UNIT_TEST(TestMoveAssignment) + UNIT_TEST(TestCopyConstructorString) + UNIT_TEST(TestCopyAssignmentString) + UNIT_TEST(TestMoveConstructorString) + UNIT_TEST(TestMoveAssignmentString) + UNIT_TEST(TestEmplaceBackNoncopyable) + UNIT_TEST(TestClear) + UNIT_TEST(TestBack) + UNIT_TEST(TestIterator) + UNIT_TEST(TestForEach) + UNIT_TEST(TestForEachReverse) UNIT_TEST_SUITE_END(); private: @@ -122,8 +134,9 @@ class TPagedVectorTest: public TTestBase { TPagedVector v2; v2.resize(v1.size()); - for (size_t i = 0; i < v1.size(); ++i) + for (size_t i = 0; i < v1.size(); ++i) { v2[i] = v1[i]; + } v2[1] = 'o'; // Replace second character. @@ -215,44 +228,6 @@ class TPagedVectorTest: public TTestBase { UNIT_ASSERT(v[1] == 25); } - void Test7() { - int array1[] = {1, 4, 25}; - int array2[] = {9, 16}; - - typedef NPagedVector::TPagedVector TVectorType; - - TVectorType v(array1, array1 + 3); - TVectorType::iterator vit; - vit = v.insert(v.begin(), 0); // Insert before first element. - UNIT_ASSERT_VALUES_EQUAL(*vit, 0); - - vit = v.insert(v.end(), 36); // Insert after last element. - UNIT_ASSERT(*vit == 36); - - UNIT_ASSERT(v.size() == 5); - UNIT_ASSERT(v[0] == 0); - UNIT_ASSERT(v[1] == 1); - UNIT_ASSERT(v[2] == 4); - UNIT_ASSERT(v[3] == 25); - UNIT_ASSERT(v[4] == 36); - - // Insert contents of array2 before fourth element. - v.insert(v.begin() + 3, array2, array2 + 2); - - UNIT_ASSERT(v.size() == 7); - - UNIT_ASSERT(v[0] == 0); - UNIT_ASSERT(v[1] == 1); - UNIT_ASSERT(v[2] == 4); - UNIT_ASSERT(v[3] == 9); - UNIT_ASSERT(v[4] == 16); - UNIT_ASSERT(v[5] == 25); - UNIT_ASSERT(v[6] == 36); - - v.clear(); - UNIT_ASSERT(v.empty()); - } - void TestAt() { using NPagedVector::TPagedVector; TPagedVector v; @@ -332,14 +307,14 @@ class TPagedVectorTest: public TTestBase { UNIT_ASSERT(vint.rbegin() == vint.rbegin()); // Not Standard: - //UNIT_ASSERT(vint.rbegin() == crvint.rbegin()); - //UNIT_ASSERT(crvint.rbegin() == vint.rbegin()); + // UNIT_ASSERT(vint.rbegin() == crvint.rbegin()); + // UNIT_ASSERT(crvint.rbegin() == vint.rbegin()); UNIT_ASSERT(crvint.rbegin() == crvint.rbegin()); UNIT_ASSERT(vint.rbegin() != vint.rend()); // Not Standard: - //UNIT_ASSERT(vint.rbegin() != crvint.rend()); - //UNIT_ASSERT(crvint.rbegin() != vint.rend()); + // UNIT_ASSERT(vint.rbegin() != crvint.rend()); + // UNIT_ASSERT(crvint.rbegin() != vint.rend()); UNIT_ASSERT(crvint.rbegin() != crvint.rend()); } @@ -376,37 +351,519 @@ class TPagedVectorTest: public TTestBase { } } - /* This test check a potential issue with empty base class - * optimization. Some compilers (VC6) do not implement it - * correctly resulting ina wrong behavior. */ - void TestEbo() { + void TestCopyConstructor() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(i); + } + + TPagedVector copied(v); + + UNIT_ASSERT_VALUES_EQUAL(copied.size(), 10u); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 10u); + + for (int i = 0; i < 10; ++i) { + // values are the same + UNIT_ASSERT_VALUES_EQUAL(v[i], i); + UNIT_ASSERT_VALUES_EQUAL(copied[i], i); + + // but pointers are different (the elements have been copied) + UNIT_ASSERT_VALUES_UNEQUAL(&copied[i], &v[i]); + } + + // Modifying the copy must not affect the original. + copied[0] = 999; + UNIT_ASSERT_VALUES_EQUAL(v[0], 0); + } + + void TestCopyAssignment() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(i); + } + + TPagedVector assigned; + assigned.push_back(999); + assigned = v; + + // The source vector should remain unchanged after copy. + UNIT_ASSERT_VALUES_EQUAL(v.size(), 10u); + + UNIT_ASSERT_VALUES_EQUAL(assigned.size(), 10u); + for (int i = 0; i < 10; ++i) { + // values are the same + UNIT_ASSERT_VALUES_EQUAL(v[i], i); + UNIT_ASSERT_VALUES_EQUAL(assigned[i], i); + + // but pointers are different (the elements have been copied) + UNIT_ASSERT_VALUES_UNEQUAL(&assigned[i], &v[i]); + } + + // Modifying the assigned vector must not affect the original. + assigned[0] = 999; + UNIT_ASSERT_VALUES_EQUAL(v[0], 0); + } + + void TestMoveConstructor() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(i); + } + + auto orig_ptr = &v[5]; + + TPagedVector moved(std::move(v)); + + UNIT_ASSERT_VALUES_EQUAL(moved.size(), 10u); + + // the move must keep original element pointers + UNIT_ASSERT_VALUES_EQUAL(orig_ptr, &moved[5]); + + for (int i = 0; i < 10; ++i) { + UNIT_ASSERT_VALUES_EQUAL(moved[i], i); + } + + // After move, the source vector should be empty. + UNIT_ASSERT(v.empty()); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 0u); + } + + void TestMoveAssignment() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(i); + } + + auto orig_ptr = &v[7]; + + TPagedVector assigned; + assigned.push_back(999); + assigned = std::move(v); + + UNIT_ASSERT_VALUES_EQUAL(assigned.size(), 10u); + + // the move must keep original element pointers + UNIT_ASSERT_VALUES_EQUAL(orig_ptr, &assigned[7]); + + for (int i = 0; i < 10; ++i) { + UNIT_ASSERT_VALUES_EQUAL(assigned[i], i); + } + + // After move, the source vector should be empty. + UNIT_ASSERT(v.empty()); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 0u); + } + + void TestCopyConstructorString() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(ToString(i)); + } + + TPagedVector copied(v); + + UNIT_ASSERT_VALUES_EQUAL(copied.size(), 10u); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 10u); + + for (int i = 0; i < 10; ++i) { + // values are the same + UNIT_ASSERT_VALUES_EQUAL(v[i], ToString(i)); + UNIT_ASSERT_VALUES_EQUAL(copied[i], ToString(i)); + + // but pointers are different (the elements have been copied, not moved) + UNIT_ASSERT_VALUES_UNEQUAL(&copied[i], &v[i]); + } + + // Modifying the copy must not affect the original (deep copy semantics). + copied[0] = "modified"; + UNIT_ASSERT_VALUES_EQUAL(v[0], "0"); + } + + void TestCopyAssignmentString() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(ToString(i)); + } + + TPagedVector assigned; + assigned.push_back("old"); + assigned = v; + + // The source vector should remain unchanged after copy. + UNIT_ASSERT_VALUES_EQUAL(v.size(), 10u); + + UNIT_ASSERT_VALUES_EQUAL(assigned.size(), 10u); + for (int i = 0; i < 10; ++i) { + // values are the same + UNIT_ASSERT_VALUES_EQUAL(v[i], ToString(i)); + UNIT_ASSERT_VALUES_EQUAL(assigned[i], ToString(i)); + + // but pointers are different (the elements have been copied, not moved) + UNIT_ASSERT_VALUES_UNEQUAL(&assigned[i], &v[i]); + } + + // Modifying the assigned vector must not affect the original. + assigned[0] = "modified"; + UNIT_ASSERT_VALUES_EQUAL(v[0], "0"); + } + + void TestMoveConstructorString() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(ToString(i)); + } + + auto orig_ptr = &v[5]; + + TPagedVector moved(std::move(v)); + + UNIT_ASSERT_VALUES_EQUAL(moved.size(), 10u); + + // the move must keep original element pointers (pages are stolen, not copied) + UNIT_ASSERT_VALUES_EQUAL(orig_ptr, &moved[5]); + + for (int i = 0; i < 10; ++i) { + UNIT_ASSERT_VALUES_EQUAL(moved[i], ToString(i)); + } + + // After move, the source vector should be empty. + UNIT_ASSERT(v.empty()); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 0u); + } + + void TestMoveAssignmentString() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(ToString(i)); + } + + auto orig_ptr = &v[7]; + + TPagedVector assigned; + assigned.push_back("old"); + assigned = std::move(v); + + UNIT_ASSERT_VALUES_EQUAL(assigned.size(), 10u); + + // the move must keep original element pointers (pages are stolen, not copied) + UNIT_ASSERT_VALUES_EQUAL(orig_ptr, &assigned[7]); + + for (int i = 0; i < 10; ++i) { + UNIT_ASSERT_VALUES_EQUAL(assigned[i], ToString(i)); + } + + // After move, the source vector should be empty. + UNIT_ASSERT(v.empty()); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 0u); + } + + struct TNonCopyableTestClass { + const TString Str; + TNonCopyableTestClass(const TString s) + : Str(s) + { + } + }; + + void TestEmplaceBackNoncopyable() { + using NPagedVector::TPagedVector; + TPagedVector v; + + for (int i = 0; i < 19; ++i) { + v.emplace_back(ToString(i)); + } + + for (int i = 0; i < 19; ++i) { + UNIT_ASSERT_VALUES_EQUAL(v[i].Str, ToString(i)); + } + + v.pop_back(); + v.pop_back(); + v.emplace_back("Hello world"); + UNIT_ASSERT_VALUES_EQUAL(v[17].Str, "Hello world"); + } + + void TestClear() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 4; ++i) { + v.push_back(ToString(i)); + } + + v.pop_back(); + v.clear(); + + UNIT_ASSERT(v.empty()); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 0u); + } + + void TestBack() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 4; ++i) { + v.push_back(ToString(i)); + } + + UNIT_ASSERT_VALUES_EQUAL(v.back(), "3"); + v.pop_back(); + UNIT_ASSERT_VALUES_EQUAL(v.back(), "2"); + } + + void TestIterator() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 11; ++i) { + v.push_back(ToString(i)); + } + + v.emplace_back("Hello"); + v.emplace_back("world"); + + auto it = v.begin(); + + UNIT_ASSERT_VALUES_EQUAL(it.GetIndex(), 0); + UNIT_ASSERT_VALUES_EQUAL(*it, "0"); + + ++it; + + UNIT_ASSERT_VALUES_EQUAL(it.GetIndex(), 1); + UNIT_ASSERT_VALUES_EQUAL(*it, "1"); + + it += 5; + + UNIT_ASSERT_VALUES_EQUAL(it.GetIndex(), 6); + UNIT_ASSERT_VALUES_EQUAL(*it, "6"); + + it = v.erase(it); + + UNIT_ASSERT_VALUES_EQUAL(it.GetIndex(), 6); + UNIT_ASSERT_VALUES_EQUAL(*it, "7"); + } + + void TestForEach() { using NPagedVector::TPagedVector; - // We use heap memory as test failure can corrupt vector internal - // representation making executable crash on vector destructor invocation. - // We prefer a simple memory leak, internal corruption should be reveal - // by size or capacity checks. - typedef TPagedVector V; - V* pv1 = new V; - pv1->resize(1); - pv1->at(0) = 1; + // Empty vector: the callback must not be invoked at all. + { + TPagedVector v; + size_t calls = 0; + v.ForEach([&](int) { + ++calls; + }); + UNIT_ASSERT_VALUES_EQUAL(calls, 0u); + } - V* pv2 = new V; + // Single element: the only element is visited once. + { + TPagedVector v; + v.push_back(42); + TVector visited; + v.ForEach([&](int x) { + visited.push_back(x); + }); + TVector expected{42}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } - pv2->resize(10); - for (int i = 0; i < 10; ++i) - pv2->at(i) = 2; + // Several elements within a single (partially filled) page. + { + TPagedVector v; + for (int i = 0; i < 2; ++i) { + v.push_back(i); + } + TVector visited; + int expectedElement = 0; + v.ForEach([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + ++expectedElement; + visited.push_back(x); + }); + TVector expected{0, 1}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } - pv1->swap(*pv2); + // A single exactly full page (3 elements): the visit order must be + // strictly forward. + { + TPagedVector v; + for (int i = 0; i < 3; ++i) { + v.push_back(i); + } + TVector visited; + int expectedElement = 0; + v.ForEach([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + ++expectedElement; + visited.push_back(x); + }); + TVector expected{0, 1, 2}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } - UNIT_ASSERT(pv1->size() == 10); - UNIT_ASSERT((*pv1)[5] == 2); + // Multiple pages with a partially filled last page: the visit order + // must be strictly forward (from the first element to the last). + { + TPagedVector v; + const int n = 10; // spans 4 pages of size 3: [0..2][3..5][6..8][9] + for (int i = 0; i < n; ++i) { + v.push_back(i); + } + TVector visited; + visited.reserve(n); + int expectedElement = 0; + v.ForEach([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + ++expectedElement; + visited.push_back(x); + }); + TVector expected; + expected.reserve(n); + for (int i = 0; i < n; ++i) { + expected.push_back(i); + } + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } - UNIT_ASSERT(pv2->size() == 1); - UNIT_ASSERT((*pv2)[0] == 1); + // Exactly full pages (no partial tail): every element is visited, + // last page is completely filled. + { + TPagedVector v; + const int n = 9; // exactly 3 full pages of size 3 + for (int i = 0; i < n; ++i) { + v.push_back(i); + } + TVector visited; + visited.reserve(n); + int expectedElement = 0; + v.ForEach([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + ++expectedElement; + visited.push_back(x); + }); + TVector expected; + expected.reserve(n); + for (int i = 0; i < n; ++i) { + expected.push_back(i); + } + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + } - delete pv2; - delete pv1; + void TestForEachReverse() { + using NPagedVector::TPagedVector; + + // Empty vector: the callback must not be invoked at all. + { + TPagedVector v; + size_t calls = 0; + v.ForEachReverse([&](int) { + ++calls; + }); + UNIT_ASSERT_VALUES_EQUAL(calls, 0u); + } + + // Single element: the only element is visited once. + { + TPagedVector v; + v.push_back(42); + TVector visited; + v.ForEachReverse([&](int x) { + visited.push_back(x); + }); + TVector expected{42}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // Several elements within a single (partially filled) page. + { + TPagedVector v; + for (int i = 0; i < 2; ++i) { + v.push_back(i); + } + TVector visited; + int expectedElement = 1; + v.ForEachReverse([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + --expectedElement; + visited.push_back(x); + }); + TVector expected{1, 0}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // A single exactly full page (3 elements): the visit order must be + // strictly reverse. + { + TPagedVector v; + for (int i = 0; i < 3; ++i) { + v.push_back(i); + } + TVector visited; + int expectedElement = 2; + v.ForEachReverse([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + --expectedElement; + visited.push_back(x); + }); + TVector expected{2, 1, 0}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // Multiple pages with a partially filled last page: the visit order + // must be strictly reverse (from the last element to the first). + { + TPagedVector v; + const int n = 10; // spans 4 pages of size 3: [0..2][3..5][6..8][9] + for (int i = 0; i < n; ++i) { + v.push_back(i); + } + TVector visited; + visited.reserve(n); + int expectedElement = n - 1; + v.ForEachReverse([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + --expectedElement; + visited.push_back(x); + }); + TVector expected; + expected.reserve(n); + for (int i = n - 1; i >= 0; --i) { + expected.push_back(i); + } + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // Exactly full pages (no partial tail): every element is visited, + // last page is completely filled. + { + TPagedVector v; + const int n = 9; // exactly 3 full pages of size 3 + for (int i = 0; i < n; ++i) { + v.push_back(i); + } + TVector visited; + visited.reserve(n); + int expectedElement = n - 1; + v.ForEachReverse([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + --expectedElement; + visited.push_back(x); + }); + TVector expected; + expected.reserve(n); + for (int i = n - 1; i >= 0; --i) { + expected.push_back(i); + } + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } } }; diff --git a/library/cpp/containers/stack_vector/stack_vec.h b/library/cpp/containers/stack_vector/stack_vec.h index b6036059b2..f2641b5d07 100644 --- a/library/cpp/containers/stack_vector/stack_vec.h +++ b/library/cpp/containers/stack_vector/stack_vec.h @@ -82,7 +82,7 @@ namespace NPrivate { } private: - std::aligned_storage_t StackBasedStorage[CountOnStack]; + alignas(T) char StackBasedStorage[CountOnStack][sizeof(T)]; bool IsStorageUsed = false; private: diff --git a/library/cpp/coroutine/engine/coroutine_ut.cpp b/library/cpp/coroutine/engine/coroutine_ut.cpp index de56d0ed2b..20176fdb91 100644 --- a/library/cpp/coroutine/engine/coroutine_ut.cpp +++ b/library/cpp/coroutine/engine/coroutine_ut.cpp @@ -2,7 +2,6 @@ #include "condvar.h" #include "network.h" -#include #include #include @@ -13,6 +12,8 @@ #include #include +#include + // TODO (velavokr): BALANCER-1345 add more tests on pollers class TCoroTest: public TTestBase { @@ -112,7 +113,7 @@ void TCoroTest::TestException() { auto f2 = [&unc, &f2run](TCont*) { f2run = true; - unc = std::uncaught_exception(); + unc = std::uncaught_exceptions(); // check segfault try { @@ -162,11 +163,11 @@ void TCoroTest::TestSimpleX1() { void TCoroTest::TestSimpleX1MultiThread() { TVector> threads; const size_t nThreads = 0; - TAtomic c = 0; + std::atomic c = 0; for (size_t i = 0; i < nThreads; ++i) { threads.push_back(MakeHolder([&]() { TestSimpleX1(); - AtomicIncrement(c); + ++c; })); } @@ -178,7 +179,7 @@ void TCoroTest::TestSimpleX1MultiThread() { t->Join(); } - UNIT_ASSERT_EQUAL(c, nThreads); + UNIT_ASSERT_EQUAL(c.load(), nThreads); } struct TTestObject { diff --git a/library/cpp/cppparser/parser.cpp b/library/cpp/cppparser/parser.cpp index 3bd968b459..70fb6a8735 100644 --- a/library/cpp/cppparser/parser.cpp +++ b/library/cpp/cppparser/parser.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -127,6 +128,10 @@ class TCppSaxParser::TImpl { break; case '\'': + if (QuoteCharIsADigitSeparator()) { + Text_.Data += ch; + break; + } Action(ch); State_ = Character; @@ -356,6 +361,35 @@ class TCppSaxParser::TImpl { } } + // digit separator in integral literal (ex. 73'709'550'592) + bool QuoteCharIsADigitSeparator() const { + const TStringBuf data = Text_.Data; + if (data.empty()) { + return false; + } + if (!IsAsciiHex(data.back())) { + return false; + } + // check for char literal prefix (ex. `u8'$'`) + static constexpr TStringBuf literalPrefixes[] { + "u8", + "u", + "U", + "L", + }; + for (const TStringBuf& literalPrefix : literalPrefixes) { + if (TStringBuf prev; data.BeforeSuffix(literalPrefix, prev)) { + if (!prev.empty() && (IsAsciiAlnum(prev.back()) || prev.back() == '_' || prev.back() == '$')) { + // some macro name ends with an `u8` sequence + continue; + } + // it is a prefixed character literal + return false; + } + } + return true; + } + inline void Action(char ch) { Action(); Text_.Data += ch; diff --git a/library/cpp/digest/md5/md5.cpp b/library/cpp/digest/md5/md5.cpp index de49749808..c583a08170 100644 --- a/library/cpp/digest/md5/md5.cpp +++ b/library/cpp/digest/md5/md5.cpp @@ -12,7 +12,6 @@ namespace { constexpr size_t MD5_BLOCK_LENGTH = 64; constexpr size_t MD5_PADDING_SHIFT = 56; - constexpr size_t MD5_HEX_DIGEST_LENGTH = 32; struct TMd5Stream: public IOutputStream { inline TMd5Stream(MD5* md5) diff --git a/library/cpp/digest/md5/md5.h b/library/cpp/digest/md5/md5.h index 2c17aa0518..b6aecb8f46 100644 --- a/library/cpp/digest/md5/md5.h +++ b/library/cpp/digest/md5/md5.h @@ -6,6 +6,9 @@ class IInputStream; class MD5 { +public: + static constexpr size_t MD5_HEX_DIGEST_LENGTH = 32; + public: MD5() { Init(); diff --git a/library/cpp/digest/md5/md5_ut.cpp b/library/cpp/digest/md5/md5_ut.cpp index 1c3e4ad0a9..7fb7f58a99 100644 --- a/library/cpp/digest/md5/md5_ut.cpp +++ b/library/cpp/digest/md5/md5_ut.cpp @@ -14,7 +14,7 @@ Y_UNIT_TEST_SUITE(TMD5Test) { r.Update((const unsigned char*)b, 15); r.Update((const unsigned char*)b + 15, strlen(b) - 15); - char rs[33]; + char rs[MD5::MD5_HEX_DIGEST_LENGTH + 1]; TString s(r.End(rs)); s.to_lower(); diff --git a/library/cpp/digest/murmur/murmur.h b/library/cpp/digest/murmur/murmur.h index cbf2886412..8ca20b378e 100644 --- a/library/cpp/digest/murmur/murmur.h +++ b/library/cpp/digest/murmur/murmur.h @@ -37,7 +37,7 @@ class TMurmurHash2A { using TValue = typename TTraits::TValue; public: - inline TMurmurHash2A(TValue seed = 0) + inline explicit TMurmurHash2A(TValue seed = 0) : Hash(seed) { } diff --git a/library/cpp/getopt/last_getopt_demo/demo.cpp b/library/cpp/getopt/last_getopt_demo/demo.cpp index a0e82a936c..3e98db97d4 100644 --- a/library/cpp/getopt/last_getopt_demo/demo.cpp +++ b/library/cpp/getopt/last_getopt_demo/demo.cpp @@ -121,8 +121,8 @@ class TMain: public TMainClassArgs { }) .Completer(NLastGetopt::NComp::File()); - // These two options can't be together. - opts.MutuallyExclusive("post-file", "post-data"); + // These options can not appear together. + opts.MutuallyExclusive("method", "post-file", "post-data"); opts.AddLongOption("header") .RequiredArgument("header-line") diff --git a/library/cpp/getopt/small/completion_generator.cpp b/library/cpp/getopt/small/completion_generator.cpp index d893afb40a..5e0e55ed38 100644 --- a/library/cpp/getopt/small/completion_generator.cpp +++ b/library/cpp/getopt/small/completion_generator.cpp @@ -63,6 +63,14 @@ namespace NLastGetopt { L; manager.GenerateZsh(out); + // When the completion file is autoloaded by `compinit` from `$fpath`, + // zsh treats the file content as the body of function `_`. + // On first invocation that body merely (re)defines `_` and + // its helpers, so completion would not actually run until the second + // TAB. Calling the redefined function here makes it work on the very + // first TAB and is also harmless when the script is `source`d. + L << "_" << command << " \"$@\""; + out.Print(stream); } diff --git a/library/cpp/getopt/small/last_getopt_opt.h b/library/cpp/getopt/small/last_getopt_opt.h index 8754ebb7ee..67a937bdb1 100644 --- a/library/cpp/getopt/small/last_getopt_opt.h +++ b/library/cpp/getopt/small/last_getopt_opt.h @@ -13,7 +13,8 @@ #include #include -#include +#include +#include namespace NLastGetopt { enum EHasArg { @@ -23,6 +24,11 @@ namespace NLastGetopt { DEFAULT_HAS_ARG = REQUIRED_ARGUMENT }; + template + concept ArgTagConcept = + std::is_enum_v> || + std::is_same_v, ui32>; + /** * NLastGetopt::TOpt is a storage of data about exactly one program option. * The data is: parse politics and help information. @@ -36,12 +42,12 @@ namespace NLastGetopt { * argument parse politics: no/optional/required/ * option existence: required or optional * handlers. See detailed documentation: - * default value: if the option has argument, but the option is ommited, + * default value: if the option has argument, but the option is omitted, * then the is used as the value of the argument * optional value: if the option has optional-argument, the option is present in parsed string, * but the argument is omitted, then - * in case of "not given , omited optional argument" the is used - * user value: allows to store arbitary pointer for handlers + * in case of "not given , omitted optional argument" the is used + * user value: allows to store arbitrary pointer for handlers */ class TOpt { public: @@ -94,7 +100,7 @@ namespace NLastGetopt { /** * Checks if given string can be a long name * @param name string to check - * @param c if given, the first bad charecter will be saved in c + * @param c if given, the first bad character will be saved in c */ static bool IsAllowedLongName(const TString& name, unsigned char* c = nullptr); @@ -798,6 +804,9 @@ namespace NLastGetopt { * argument name (title) */ struct TFreeArgSpec { + template + using TTagger = std::function; + TFreeArgSpec() = default; TFreeArgSpec(const TString& title, const TString& help = TString(), bool optional = false) : Title_(title) @@ -809,6 +818,7 @@ namespace NLastGetopt { TString Title_; TString Help_; TString CompletionArgHelp_; + TTagger Tagger_; bool Optional_ = false; NComp::ICompleterPtr Completer_ = nullptr; @@ -891,5 +901,48 @@ namespace NLastGetopt { Completer_ = std::move(completer); return *this; } + + /** + * Set a tagger that can compute tag dynamically for each argument value. + */ + TFreeArgSpec& SetTag(TTagger&& tagger) { + Tagger_ = std::forward>(tagger); + return *this; + } + + /** + * Set a static tag for all arguments described by this spec. + */ + template + TFreeArgSpec& SetTag(E tag) { + Tagger_ = [tag](const TString&) -> ui32 { + return static_cast(tag); + }; + return *this; + } + + /** + * Set a tagger that can compute tag dynamically for each argument value. + */ + template + TFreeArgSpec& SetTag(TTagger&& tagger) { + Tagger_ = [tagger](const TString& value) -> ui32 { + return static_cast(tagger(value)); + }; + return *this; + } + + /** + * Compute tag for argument value at given position. + */ + ui32 GetTag(const TString& value) const { + if (Tagger_) { + ui32 tag = Tagger_(value); + if (tag) { + return tag; + } + } + return 0; + } }; } diff --git a/library/cpp/getopt/small/last_getopt_opts.cpp b/library/cpp/getopt/small/last_getopt_opts.cpp index 984927a038..b656e607e8 100644 --- a/library/cpp/getopt/small/last_getopt_opts.cpp +++ b/library/cpp/getopt/small/last_getopt_opts.cpp @@ -213,7 +213,7 @@ namespace NLastGetopt { if (FreeArgsMax_ < FreeArgsMin_) { ythrow TConfException() << "FreeArgsMax must be >= FreeArgsMin"; } - if (!FreeArgSpecs_.empty() && FreeArgSpecs_.rbegin()->first >= FreeArgsMax_) { + if (!FreeArgSpecs_.empty() && GetTrailingArgsIndex() > FreeArgsMax_) { ythrow TConfException() << "Described args count is greater than FreeArgsMax. Either increase FreeArgsMax or remove unreachable descriptions"; } } @@ -335,7 +335,7 @@ namespace NLastGetopt { } os << "[OPTIONS]"; - ui32 numDescribedFlags = FreeArgSpecs_.empty() ? 0 : FreeArgSpecs_.rbegin()->first + 1; + ui32 numDescribedFlags = GetTrailingArgsIndex(); ui32 numArgsToShow = Max(FreeArgsMin_, FreeArgsMax_ == UNLIMITED_ARGS ? numDescribedFlags : FreeArgsMax_); for (ui32 i = 0, nonOptionalFlagsPrinted = 0; i < numArgsToShow; ++i) { @@ -513,8 +513,8 @@ namespace NLastGetopt { } os << colors.OldColor() << Endl; - const size_t limit = FreeArgSpecs_.empty() ? 0 : FreeArgSpecs_.rbegin()->first; - for (size_t i = 0; i <= limit; ++i) { + const size_t limit = GetTrailingArgsIndex(); + for (size_t i = 0; i < limit; ++i) { if (!FreeArgSpecs_.contains(i)) { continue; } diff --git a/library/cpp/getopt/small/last_getopt_opts.h b/library/cpp/getopt/small/last_getopt_opts.h index 718dbfcb89..868477ec79 100644 --- a/library/cpp/getopt/small/last_getopt_opts.h +++ b/library/cpp/getopt/small/last_getopt_opts.h @@ -28,7 +28,7 @@ namespace NLastGetopt { * the special string " -- " will be treated as end of named * options: all options after it will be parsed as free args * if PERMUTE is choosen, arguments will be rearranged in correct order, - * if RETURN_IN_ORDER is choosen, all free args will be ommited (TODO: looks very strange) + * if RETURN_IN_ORDER is choosen, all free args will be omitted (TODO: looks very strange) * - Using '+' as a prefix instead '--' for long names * - Using "-" as a prefix for both short and long names * - Allowing unknown options @@ -49,7 +49,7 @@ namespace NLastGetopt { bool AllowSingleDashForLong_ = false; // bool AllowPlusForLong_ = false; // using '+' instead '--' for long options - //Allows unknwon options: + //Allows unknown options: bool AllowUnknownCharOptions_ = false; bool AllowUnknownLongOptions_ = false; @@ -78,7 +78,7 @@ namespace NLastGetopt { /** * Constructs TOpts from string as in getopt(3) and - * additionally adds help option (for '?') and svn-verstion option (for 'V') + * additionally adds help option (for '?') and svn-version option (for 'V') */ static TOpts Default(const TStringBuf& optstring = TStringBuf()) { TOpts opts(optstring); @@ -92,7 +92,7 @@ namespace NLastGetopt { * Throws TConfException if validation failed. * Check consist of: * -not intersecting of names - * -compability of settings, that responsable for freeArgs parsing + * -compatibility of settings, that responsible for freeArgs parsing */ void Validate() const; @@ -398,9 +398,22 @@ namespace NLastGetopt { * Note: don't use this on options with default values. If option with default value wasn't specified, * parser will run handlers for default value, thus triggering a false-positive exclusivity check. */ - template - void MutuallyExclusive(T1&& opt1, T2&& opt2) { - MutuallyExclusiveOpt(GetOption(std::forward(opt1)), GetOption(std::forward(opt2))); + template + void MutuallyExclusive(Opt1&& name1, Opt2&& name2) { + TOpt& opt1 = GetOption(name1); + TOpt& opt2 = GetOption(name2); + MutuallyExclusiveOpt(opt1, opt2); + } + + template + void MutuallyExclusive(Opt1&& name1, OtherOpts&& ...otherNames) { + TOpt& opt1 = GetOption(name1); + std::array otherNamesArr{otherNames...}; + for (const auto& otherName: otherNamesArr) { + TOpt& otherOpt = GetOption(otherName); + MutuallyExclusiveOpt(opt1, otherOpt); + } + MutuallyExclusive(std::forward(otherNames)...); } /** @@ -449,6 +462,24 @@ namespace NLastGetopt { AddSection("Examples", std::move(examples)); } + /** + * Add section with examples. + * + * @param examples text of this section + */ + void SetExamples(std::string_view examples) { + SetExamples(TString(examples)); + } + + /** + * Add section with examples. + * + * @param examples text of this section + */ + void SetExamples(const char* examples) { + SetExamples(TString(examples)); + } + /** * Set minimal number of free args * @@ -490,6 +521,13 @@ namespace NLastGetopt { return FreeArgSpecs_; } + /** + * Get index from where trailing arguments start + */ + ui32 GetTrailingArgsIndex() const { + return FreeArgSpecs_.empty() ? 0 : FreeArgSpecs_.rbegin()->first + 1; + } + /** * Set exact expected number of free args * @@ -529,7 +567,7 @@ namespace NLastGetopt { /** * Legacy, don't use. Same as `SetTrailingArgTitle`. - * Older versions of lastgetopt didn't have destinction between default title and title + * Older versions of lastgetopt didn't have distinction between default title and title * for the trailing argument. */ void SetFreeArgDefaultTitle(const TString& title, const TString& help = TString()) { diff --git a/library/cpp/getopt/small/last_getopt_parse_result.cpp b/library/cpp/getopt/small/last_getopt_parse_result.cpp index 60effba41b..016fe34714 100644 --- a/library/cpp/getopt/small/last_getopt_parse_result.cpp +++ b/library/cpp/getopt/small/last_getopt_parse_result.cpp @@ -9,6 +9,44 @@ namespace NLastGetopt { return nullptr; } + void TOptsParseResult::BuildTaggedFreeArgs(const TOpts* options) { + TaggedFreeArgs_.clear(); + + if (!Parser_) { + return; + } + + const size_t freeArgsPos = GetFreeArgsPos(); + for (size_t argPos = freeArgsPos; argPos < Parser_->Argc_; ++argPos) { + size_t index = argPos - freeArgsPos; + + TString value = Parser_->Argv_[argPos]; + ui32 tag = 0; + + if (options) { + const TFreeArgSpec* spec = nullptr; + auto it = options->FreeArgSpecs_.find(index); + if (it != options->FreeArgSpecs_.end()) { + spec = &it->second; + } else if (options->FreeArgsMax_ == TOpts::UNLIMITED_ARGS) { + ui32 trailingArgsIndex = options->GetTrailingArgsIndex(); + if (index >= trailingArgsIndex) { + spec = &options->TrailingArgSpec_; + } + } + + if (spec) { + tag = spec->GetTag(value); + } + } + + TaggedFreeArgs_.push_back(TTaggedArg { + .Value = value, + .Tag = tag + }); + } + } + const TOptParseResult* TOptsParseResult::FindOptParseResult(const TOpt* opt, bool includeDefault) const { const TOptParseResult* r = FindParseResult(Opts_, opt); if (nullptr == r && includeDefault) @@ -99,6 +137,14 @@ namespace NLastGetopt { return Parser_->ProgramName_; } + void TOptsParseResult::SetProgramSubcommandPath(const TVector& parts) { + ProgramSubcommandPath_ = parts; + } + + const TVector& TOptsParseResult::GetProgramSubcommandPath() const { + return ProgramSubcommandPath_; + } + void TOptsParseResult::PrintUsage(IOutputStream& os) const { Parser_->Opts_->PrintUsage(Parser_->ProgramName_, os); } @@ -108,15 +154,16 @@ namespace NLastGetopt { } TVector TOptsParseResult::GetFreeArgs() const { - TVector v; - for (size_t i = GetFreeArgsPos(); i < Parser_->Argc_; ++i) { - v.push_back(Parser_->Argv_[i]); + TVector args; + args.reserve(TaggedFreeArgs_.size()); + for (const auto& arg : TaggedFreeArgs_) { + args.push_back(arg.Value); } - return v; + return args; } size_t TOptsParseResult::GetFreeArgCount() const { - return Parser_->Argc_ - GetFreeArgsPos(); + return TaggedFreeArgs_.size(); } void FindUserTypos(const TString& arg, const TOpts* options) { @@ -142,6 +189,7 @@ namespace NLastGetopt { } Y_ENSURE(options); + BuildTaggedFreeArgs(options); const auto freeArgs = GetFreeArgs(); for (size_t i = 0; i < freeArgs.size(); ++i) { if (i >= options->ArgBindings_.size()) { diff --git a/library/cpp/getopt/small/last_getopt_parse_result.h b/library/cpp/getopt/small/last_getopt_parse_result.h index c6e768c461..6574667aed 100644 --- a/library/cpp/getopt/small/last_getopt_parse_result.h +++ b/library/cpp/getopt/small/last_getopt_parse_result.h @@ -3,7 +3,14 @@ #include "last_getopt_opts.h" #include "last_getopt_parser.h" +#include + namespace NLastGetopt { + struct TTaggedArg { + TString Value; + ui32 Tag = 0; + }; + /** * NLastGetopt::TOptParseResult contains all arguments for exactly one TOpt, * that have been fetched during parsing @@ -73,6 +80,8 @@ namespace NLastGetopt { TdVec Opts_; //Parsing result for all options, that have been explicitly defined in argc/argv TdVec OptsDef_; //Parsing result for options, that have been defined by default values only + TVector ProgramSubcommandPath_; + TVector TaggedFreeArgs_; private: TOptParseResult& OptParseResult(); @@ -87,6 +96,8 @@ namespace NLastGetopt { */ static const TOptParseResult* FindParseResult(const TdVec& vec, const TOpt* opt); + void BuildTaggedFreeArgs(const TOpts* options); + protected: /** * Performs parsing of comand line arguments. @@ -161,6 +172,8 @@ namespace NLastGetopt { * @return argv[0] */ TString GetProgramName() const; + void SetProgramSubcommandPath(const TVector& parts); + const TVector& GetProgramSubcommandPath() const; /** * Print usage string. @@ -182,6 +195,22 @@ namespace NLastGetopt { */ TVector GetFreeArgs() const; + template + TVector GetFreeArgs(E tag) const { + TVector args; + ui32 ui32Tag = static_cast(tag); + for (const auto& arg : TaggedFreeArgs_) { + if (arg.Tag == ui32Tag) { + args.push_back(arg.Value); + } + } + return args; + } + + const TVector& GetTaggedFreeArgs() const { + return TaggedFreeArgs_; + } + /** * @return true if given option exist in results of parsing * @@ -192,7 +221,7 @@ namespace NLastGetopt { bool Has(const TOpt* opt, bool includeDefault = false) const; /** - * @return nil terminated string on the last fetched argument of givne option + * @return nil terminated string on the last fetched argument of given option * * @param opt ptr on required object * @param includeDefault search in results obtained from default values @@ -200,7 +229,7 @@ namespace NLastGetopt { const char* Get(const TOpt* opt, bool includeDefault = true) const; /** - * @return nil terminated string on the last fetched argument of givne option + * @return nil terminated string on the last fetched argument of given option * if option haven't been fetched, given defaultValue will be returned * * @param opt ptr on required object @@ -218,7 +247,7 @@ namespace NLastGetopt { bool Has(const TString& name, bool includeDefault = false) const; /** - * @return nil terminated string on the last fetched argument of givne option + * @return nil terminated string on the last fetched argument of given option * * @param name long name of required object * @param includeDefault search in results obtained from default values @@ -226,7 +255,7 @@ namespace NLastGetopt { const char* Get(const TString& name, bool includeDefault = true) const; /** - * @return nil terminated string on the last fetched argument of givne option + * @return nil terminated string on the last fetched argument of given option * if option haven't been fetched, given defaultValue will be returned * * @param name long name of required object @@ -244,7 +273,7 @@ namespace NLastGetopt { bool Has(char name, bool includeDefault = false) const; /** - * @return nil terminated string on the last fetched argument of givne option + * @return nil terminated string on the last fetched argument of given option * * @param c short name of required object * @param includeDefault search in results obtained from default values @@ -252,7 +281,7 @@ namespace NLastGetopt { const char* Get(char name, bool includeDefault = true) const; /** - * @return nil terminated string on the last fetched argument of givne option + * @return nil terminated string on the last fetched argument of given option * if option haven't been fetched, given defaultValue will be returned * * @param c short name of required object @@ -261,7 +290,7 @@ namespace NLastGetopt { const char* GetOrElse(char name, const char* defaultValue) const; /** - * for givne option return parsed value of the last fetched argument + * for given option return parsed value of the last fetched argument * if option haven't been fetched, HandleError action is called * * @param opt required option (one of: ptr, short name, long name). @@ -280,7 +309,7 @@ namespace NLastGetopt { } /** - * for givne option return parsed value of the last fetched argument + * for given option return parsed value of the last fetched argument * if option haven't been fetched, given defaultValue will be returned * * @param opt required option (one of: ptr, short name, long name). diff --git a/library/cpp/getopt/small/last_getopt_parser.cpp b/library/cpp/getopt/small/last_getopt_parser.cpp index 911c76f342..98c3951a1c 100644 --- a/library/cpp/getopt/small/last_getopt_parser.cpp +++ b/library/cpp/getopt/small/last_getopt_parser.cpp @@ -170,9 +170,14 @@ namespace NLastGetopt { bool TOptsParser::ParseOptParam(const TOpt* opt, size_t pos) { Y_ASSERT(opt); - if (opt->GetHasArg() == NO_ARGUMENT || opt->IsEqParseOnly()) { + if (opt->GetHasArg() == NO_ARGUMENT || + opt->GetHasArg() == OPTIONAL_ARGUMENT && opt->IsEqParseOnly()) { return Commit(opt, nullptr, pos, 0); } + if (opt->IsEqParseOnly()) { + Y_ASSERT(opt->GetHasArg() == REQUIRED_ARGUMENT); + throw TUsageException() << "option " << opt->ToShortString() << " requires an argument but only accepts it over ="; + } if (pos == Argc_) { if (opt->GetHasArg() == REQUIRED_ARGUMENT) throw TUsageException() << "option " << opt->ToShortString() << " must have arg"; diff --git a/library/cpp/getopt/small/last_getopt_parser.h b/library/cpp/getopt/small/last_getopt_parser.h index 2cf8a6c308..dca2355150 100644 --- a/library/cpp/getopt/small/last_getopt_parser.h +++ b/library/cpp/getopt/small/last_getopt_parser.h @@ -46,7 +46,7 @@ namespace NLastGetopt { bool GotMinusMinus_; //true if "--" have been seen in argv protected: - const TOpt* CurrentOpt_; // ptr on the last meeted option + const TOpt* CurrentOpt_; // ptr on the last met option TStringBuf CurrentValue_; // the value of the last met argument (corresponding to CurrentOpt_) private: diff --git a/library/cpp/getopt/small/modchooser.cpp b/library/cpp/getopt/small/modchooser.cpp index 3b3b0b7751..903061dd8d 100644 --- a/library/cpp/getopt/small/modchooser.cpp +++ b/library/cpp/getopt/small/modchooser.cpp @@ -58,6 +58,14 @@ class ClassWrapper: public TMainClass { TMainClassV* Main; }; +void TMainClass::SetSubcommandPath(TVector parts) { + SubcommandPath_ = std::move(parts); +} + +const TVector& TMainClass::GetSubcommandPath() const { + return SubcommandPath_; +} + TModChooser::TMode::TMode(const TString& name, TMainClass* main, const TString& descr, bool hidden, bool noCompletion) : Name(name) , Main(main) @@ -161,6 +169,14 @@ void TModChooser::AddCompletions(TString progName, const TString& name, bool hid } } +void TModChooser::SetSubcommandPath(const TVector& subcommandPath) const { + SubcommandPath_ = subcommandPath; +} + +const TVector& TModChooser::GetSubcommandPath() const { + return SubcommandPath_; +} + int TModChooser::Run(const int argc, const char** argv) const { Y_ENSURE(argc, "Can't run TModChooser with empty list of arguments."); @@ -202,6 +218,10 @@ int TModChooser::Run(const int argc, const char** argv) const { return 1; } + TVector subcommandPath = SubcommandPath_; + subcommandPath.push_back(modeIter->second->Name); + modeIter->second->Main->SetSubcommandPath(std::move(subcommandPath)); + if (shiftArgs) { TString firstArg; TVector nargv(Reserve(argc)); @@ -329,7 +349,11 @@ bool TModChooser::IsSvnRevisionOptionDisabled() const { } int TMainClassArgs::Run(int argc, const char** argv) { - return DoRun(NLastGetopt::TOptsParseResult(&GetOptions(), argc, argv)); + NLastGetopt::TOptsParseResult res(&GetOptions(), argc, argv); + if (!GetSubcommandPath().empty()) { + res.SetProgramSubcommandPath(GetSubcommandPath()); + } + return DoRun(std::move(res)); } const NLastGetopt::TOpts& TMainClassArgs::GetOptions() { @@ -355,10 +379,11 @@ int TMainClassModes::operator()(const int argc, const char** argv) { int TMainClassModes::Run(int argc, const char** argv) { auto& chooser = GetSubModes(); + chooser.SetSubcommandPath(GetSubcommandPath()); return chooser.Run(argc, argv); } -const TModChooser& TMainClassModes::GetSubModes() { +TModChooser& TMainClassModes::GetSubModes() { if (Modes_.Empty()) { Modes_.ConstructInPlace(); RegisterModes(Modes_.GetRef()); @@ -367,6 +392,10 @@ const TModChooser& TMainClassModes::GetSubModes() { return Modes_.GetRef(); } +const TModChooser& TMainClassModes::GetSubModes() const { + return const_cast(this)->GetSubModes(); +} + void TMainClassModes::RegisterModes(TModChooser& modes) { modes.SetModesHelpOption("-h"); } diff --git a/library/cpp/getopt/small/modchooser.h b/library/cpp/getopt/small/modchooser.h index d41ae78005..81f0629015 100644 --- a/library/cpp/getopt/small/modchooser.h +++ b/library/cpp/getopt/small/modchooser.h @@ -28,6 +28,13 @@ class TMainClass { public: virtual int operator()(int argc, const char** argv) = 0; virtual ~TMainClass() = default; + + void SetSubcommandPath(TVector parts); + + const TVector& GetSubcommandPath() const; + +protected: + TVector SubcommandPath_; }; //! Function to handle '--version' parameter @@ -89,6 +96,9 @@ class TModChooser { void AddCompletions(TString progName, const TString& name = "completion", bool hidden = false, bool noCompletion = false); + void SetSubcommandPath(const TVector& subcommandPath) const; + const TVector& GetSubcommandPath() const; + /*! Run appropriate mode. * * In this method following things happen: @@ -184,6 +194,8 @@ class TModChooser { * then help message will be printed to stdout */ bool HelpAlwaysToStdErr{true}; + + mutable TVector SubcommandPath_; }; //! Mode class that allows introspecting its console arguments. @@ -219,7 +231,8 @@ class TMainClassModes: public TMainClass { int Run(int argc, const char** argv); //! Get sub-modes for this mode. - const TModChooser& GetSubModes(); + TModChooser& GetSubModes(); + const TModChooser& GetSubModes() const; protected: //! Fill given modchooser with sub-modes. diff --git a/library/cpp/getopt/ut/CMakeLists.txt b/library/cpp/getopt/ut/CMakeLists.txt index ee1a898a7c..09465ddc74 100644 --- a/library/cpp/getopt/ut/CMakeLists.txt +++ b/library/cpp/getopt/ut/CMakeLists.txt @@ -1,65 +1,11 @@ -add_ydb_test(NAME getopt-last_getopt_ut +add_ydb_test(NAME getopt-ut SOURCES last_getopt_ut.cpp - LINK_LIBRARIES - getopt - cpp-testing-unittest_main - LABELS - unit -) - -add_ydb_test(NAME getopt-modchooser_ut - SOURCES modchooser_ut.cpp - LINK_LIBRARIES - getopt - cpp-testing-unittest_main - LABELS - unit -) - -add_ydb_test(NAME getopt-opt2_ut - SOURCES opt2_ut.cpp - LINK_LIBRARIES - getopt - cpp-testing-unittest_main - LABELS - unit -) - -add_ydb_test(NAME getopt-opt_ut - SOURCES opt_ut.cpp - LINK_LIBRARIES - getopt - cpp-testing-unittest_main - LABELS - unit -) - -add_ydb_test(NAME getopt-posix_getopt_ut - SOURCES posix_getopt_ut.cpp - LINK_LIBRARIES - getopt - cpp-testing-unittest_main - LABELS - unit -) - -add_ydb_test(NAME getopt-wrap_ut - SOURCES wrap.cpp - LINK_LIBRARIES - getopt-small - cpp-testing-unittest_main - LABELS - unit -) - -add_ydb_test(NAME getopt-ygetopt_ut - SOURCES ygetopt_ut.cpp LINK_LIBRARIES getopt diff --git a/library/cpp/getopt/ut/last_getopt_ut.cpp b/library/cpp/getopt/ut/last_getopt_ut.cpp index b517ea359d..31fc923d31 100644 --- a/library/cpp/getopt/ut/last_getopt_ut.cpp +++ b/library/cpp/getopt/ut/last_getopt_ut.cpp @@ -28,8 +28,6 @@ namespace { Init(opts, (int)Argv_.size(), Argv_.data()); } }; - - using V = TVector; } struct TOptsParserTester { @@ -137,7 +135,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { TOptsNoDefault opts; opts.AddLongOption("from"); opts.AddLongOption("to"); - TOptsParseResultTestWrapper r(&opts, V({"copy", "--from=/", "--to=/etc"})); + TOptsParseResultTestWrapper r(&opts, {"copy", "--from=/", "--to=/etc"}); UNIT_ASSERT_VALUES_EQUAL("copy", r.GetProgramName()); UNIT_ASSERT_VALUES_EQUAL("/", r.Get("from")); @@ -154,7 +152,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddCharOption('R', NO_ARGUMENT); opts.AddCharOption('l', NO_ARGUMENT); opts.AddCharOption('h', NO_ARGUMENT); - TOptsParseResultTestWrapper r(&opts, V({"cp", "/etc", "-Rl", "/tmp/etc"})); + TOptsParseResultTestWrapper r(&opts, {"cp", "/etc", "-Rl", "/tmp/etc"}); UNIT_ASSERT(r.Has('R')); UNIT_ASSERT(r.Has('l')); UNIT_ASSERT(!r.Has('h')); @@ -168,20 +166,36 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { Y_UNIT_TEST(TestFreeArgs) { TOptsNoDefault opts; opts.SetFreeArgsNum(1, 3); - TOptsParseResultTestWrapper r11(&opts, V({"cp", "/etc"})); - TOptsParseResultTestWrapper r12(&opts, V({"cp", "/etc", "/tmp/etc"})); - TOptsParseResultTestWrapper r13(&opts, V({"cp", "/etc", "/tmp/etc", "verbose"})); + TOptsParseResultTestWrapper r11(&opts, {"cp", "/etc"}); + TOptsParseResultTestWrapper r12(&opts, {"cp", "/etc", "/tmp/etc"}); + TOptsParseResultTestWrapper r13(&opts, {"cp", "/etc", "/tmp/etc", "verbose"}); UNIT_ASSERT_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"cp", "/etc", "/tmp/etc", "verbose", "nosymlink"})), + TOptsParseResultTestWrapper(&opts, {"cp", "/etc", "/tmp/etc", "verbose", "nosymlink"}), yexception); UNIT_ASSERT_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"cp"})), + TOptsParseResultTestWrapper(&opts, {"cp"}), yexception); opts.SetFreeArgsNum(2); - TOptsParseResultTestWrapper r22(&opts, V({"cp", "/etc", "/var/tmp"})); + TOptsParseResultTestWrapper r22(&opts, {"cp", "/etc", "/var/tmp"}); + } + + Y_UNIT_TEST(TestProgramSubcommandPathSetter) { + TOptsNoDefault opts; + TOptsParseResultTestWrapper r(&opts, {"tool"}); + const TVector parts = {"tool", "sub", "command"}; + r.SetProgramSubcommandPath(parts); + UNIT_ASSERT_VALUES_EQUAL(parts, r.GetProgramSubcommandPath()); + } + + Y_UNIT_TEST(TestProgramCanonicalNameCompat) { + TOptsNoDefault opts; + TOptsParseResultTestWrapper r(&opts, {"tool"}); + r.SetProgramSubcommandPath({"tool", "outer", "inner"}); + const TVector expected = {"tool", "outer", "inner"}; + UNIT_ASSERT_VALUES_EQUAL(expected, r.GetProgramSubcommandPath()); } Y_UNIT_TEST(TestCharOptionsRequiredOptional) { @@ -191,7 +205,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddCharOption('x', REQUIRED_ARGUMENT); opts.AddCharOption('y', REQUIRED_ARGUMENT); opts.AddCharOption('l', NO_ARGUMENT); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "-ld11", "-e", "22", "-lllx33", "-y", "44"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "-ld11", "-e", "22", "-lllx33", "-y", "44"}); UNIT_ASSERT_VALUES_EQUAL("11", r.Get('d')); UNIT_ASSERT_VALUES_EQUAL("22", r.Get('e')); UNIT_ASSERT_VALUES_EQUAL("33", r.Get('x')); @@ -449,6 +463,19 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { tester.AcceptEndOfFreeArgs(); } + Y_UNIT_TEST(TestEqParseOnlyRequiredArgument) { + TOptsNoDefault opts; + + opts.AddLongOption("eq-only").RequiredArgument().DisableSpaceParse(); + + TOptsParseResultTestWrapper res(&opts, {"cmd", "--eq-only=value"}); + UNIT_ASSERT_EQUAL(res.Get("eq-only"), "value"sv); + + UNIT_ASSERT_EXCEPTION( + TOptsParseResultTestWrapper(&opts, {"cmd", "--eq-only", "value"}), + TUsageException); + } + Y_UNIT_TEST(TestStoreResult) { TOptsNoDefault opts; TString data; @@ -461,7 +488,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption("optional-number-0").StoreResult(&optionalNumber0); opts.AddLongOption("optional-string-1").StoreResult(&optionalString1); opts.AddLongOption("optional-number-1").StoreResult(&optionalNumber1); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--data=jjhh", "-n", "11", "--optional-number-1=8", "--optional-string-1=os1"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--data=jjhh", "-n", "11", "--optional-number-1=8", "--optional-string-1=os1"}); UNIT_ASSERT_VALUES_EQUAL("jjhh", data); UNIT_ASSERT_VALUES_EQUAL(11, number); UNIT_ASSERT(!optionalString0.Defined()); @@ -480,7 +507,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption('b', "beta").NoArgument().StoreValue(&b, 24); opts.AddLongOption('e', "enum").NoArgument().StoreValue(&e, REQUIRED_ARGUMENT).StoreValue(&c, 12345); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "-a", "-e"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "-a", "-e"}); UNIT_ASSERT_VALUES_EQUAL(42, a); UNIT_ASSERT_VALUES_EQUAL(0, b); @@ -497,7 +524,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddCharOption('c').StoreTrue(&c); opts.AddCharOption('d').StoreTrue(&d); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "-a", "-c"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "-a", "-c"}); UNIT_ASSERT(a); UNIT_ASSERT(!b); @@ -510,7 +537,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption("path").DefaultValue("/etc"); int value = 42; opts.AddLongOption("value").StoreResult(&value).DefaultValue(32); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "dfdf"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "dfdf"}); UNIT_ASSERT_VALUES_EQUAL("/etc", r.Get("path")); UNIT_ASSERT_VALUES_EQUAL(32, value); } @@ -519,7 +546,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { TOptsNoDefault opts; TVector vals; opts.AddLongOption('s', "split").SplitHandler(&vals, ','); - TOptsParseResultTestWrapper r(&opts, V({"prog", "--split=a,b,c"})); + TOptsParseResultTestWrapper r(&opts, {"prog", "--split=a,b,c"}); UNIT_ASSERT_EQUAL(vals.size(), 3); UNIT_ASSERT_EQUAL(vals[0], "a"); UNIT_ASSERT_EQUAL(vals[1], "b"); @@ -530,7 +557,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { TOptsNoDefault opts; TVector vals; opts.AddLongOption('s', "split").RangeSplitHandler(&vals, ',', '-'); - TOptsParseResultTestWrapper r(&opts, V({"prog", "--split=1,8-10", "--split=12-14"})); + TOptsParseResultTestWrapper r(&opts, {"prog", "--split=1,8-10", "--split=12-14"}); UNIT_ASSERT_EQUAL(vals.size(), 7); UNIT_ASSERT_EQUAL(vals[0], 1); UNIT_ASSERT_EQUAL(vals[1], 8); @@ -557,15 +584,15 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { // test 'not required' // makes sure that the problem will only be in 'required' - TOptsParseResultTestWrapper r1(&opts, V({"cmd"})); + TOptsParseResultTestWrapper r1(&opts, {"cmd"}); // test 'required' opt_d.Required(); UNIT_ASSERT_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"cmd"})), + TOptsParseResultTestWrapper(&opts, {"cmd"}), TUsageException); - TOptsParseResultTestWrapper r3(&opts, V({"cmd", "-d11"})); + TOptsParseResultTestWrapper r3(&opts, {"cmd", "-d11"}); UNIT_ASSERT_VALUES_EQUAL("11", r3.Get('d')); } @@ -581,12 +608,13 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { *Flag = true; } }; + Y_UNIT_TEST(TestHandlers) { { TOptsNoDefault opts; bool flag = false; opts.AddLongOption("flag").Handler0(HandlerStoreTrue(&flag)).NoArgument(); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--flag"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--flag"}); UNIT_ASSERT(flag); } { @@ -598,11 +626,11 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption("flag3").RequiredArgument().StoreMappedResult(&fval, (double (*)(double))fabs); opts.AddLongOption("flag4").RequiredArgument().StoreMappedResult(&fval, (double (*)(double))sqrt); UNIT_ASSERT_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"cmd", "--flag3", "-2.0", "--flag1", "-1"})), + TOptsParseResultTestWrapper(&opts, {"cmd", "--flag3", "-2.0", "--flag1", "-1"}), yexception); UNIT_ASSERT_VALUES_EQUAL(uval, 5u); UNIT_ASSERT_VALUES_EQUAL(fval, 2.0); - TOptsParseResultTestWrapper r1(&opts, V({"cmd", "--flag4", "9.0", "--flag2", "-1"})); + TOptsParseResultTestWrapper r1(&opts, {"cmd", "--flag4", "9.0", "--flag2", "-1"}); UNIT_ASSERT_VALUES_EQUAL(uval, Max()); UNIT_ASSERT_VALUES_EQUAL(fval, 3.0); } @@ -742,7 +770,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption("size").AppendTo(&ints); opts.AddLongOption("value").AppendTo(&strings); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--size=17", "--size=19", "--value=v1", "--value=v2"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--size=17", "--size=19", "--value=v1", "--value=v2"}); UNIT_ASSERT_VALUES_EQUAL(size_t(2), ints.size()); UNIT_ASSERT_VALUES_EQUAL(17, ints.at(0)); @@ -759,7 +787,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { TOptsNoDefault opts; opts.AddLongOption("path").EmplaceTo(&richPaths); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--path=//cool", "--path=//nice"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--path=//cool", "--path=//nice"}); UNIT_ASSERT_VALUES_EQUAL(size_t(2), richPaths.size()); UNIT_ASSERT_VALUES_EQUAL("//cool", std::get<0>(richPaths.at(0))); @@ -772,7 +800,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { TOptsNoDefault opts; opts.AddLongOption("set").KVHandler([&keyvals](TString k, TString v) { keyvals << k << ":" << v << ","; }); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--set", "x=1", "--set", "y=2", "--set=z=3"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--set", "x=1", "--set", "y=2", "--set=z=3"}); UNIT_ASSERT_VALUES_EQUAL(keyvals, "x:1,y:2,z:3,"); } @@ -784,25 +812,25 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { { gSimpleFlag = false; - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--abstract"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--abstract"}); UNIT_ASSERT(!flag); UNIT_ASSERT(!gSimpleFlag); } { - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--abstract", "--global", "-t"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--abstract", "--global", "-t"}); UNIT_ASSERT(flag); UNIT_ASSERT(gSimpleFlag); } { UNIT_ASSERT_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"cmd", "--true"})), + TOptsParseResultTestWrapper(&opts, {"cmd", "--true"}), TUsageException); } { - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--abstract", "--buffer=512"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--abstract", "--buffer=512"}); UNIT_ASSERT(r.Has('b')); UNIT_ASSERT_VALUES_EQUAL(r.Get('b', 0), "512"); } @@ -826,12 +854,81 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { int number = 0; opts.AddFreeArgBinding("data", data); opts.AddFreeArgBinding("number", number); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "hello", "25"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "hello", "25"}); UNIT_ASSERT_VALUES_EQUAL("hello", data); UNIT_ASSERT_VALUES_EQUAL(25, number); UNIT_ASSERT_VALUES_EQUAL(2, r.GetFreeArgCount()); } + Y_UNIT_TEST(TestFreeArgsTaggedAccess) { + enum class EFreeArgTag { + Unknown, + Src, + Dst, + }; + + TOptsNoDefault opts; + opts.SetFreeArgsNum(2); + opts.GetFreeArgSpec(0).SetTag(EFreeArgTag::Src); + opts.GetFreeArgSpec(1).SetTag(EFreeArgTag::Dst); + + TOptsParseResultTestWrapper r(&opts, {"cmd", "input.txt", "output.txt"}); + + const auto allArgs = r.GetFreeArgs(); + UNIT_ASSERT_VALUES_EQUAL(2u, allArgs.size()); + + const auto srcArgs = r.GetFreeArgs(EFreeArgTag::Src); + UNIT_ASSERT_VALUES_EQUAL(1u, srcArgs.size()); + UNIT_ASSERT_VALUES_EQUAL("input.txt", srcArgs.front()); + + const auto dstArgs = r.GetFreeArgs(EFreeArgTag::Dst); + UNIT_ASSERT_VALUES_EQUAL(1u, dstArgs.size()); + UNIT_ASSERT_VALUES_EQUAL("output.txt", dstArgs.front()); + + UNIT_ASSERT(r.GetFreeArgs(EFreeArgTag::Unknown).empty()); + } + + Y_UNIT_TEST(TestTrailingFreeArgsTagger) { + enum class EFreeArgTag { + Unknown, + Primary, + Logs, + Temp, + }; + + TOptsNoDefault opts; + opts.SetFreeArgsMin(1); + opts.SetFreeArgsMax(TOpts::UNLIMITED_ARGS); + + opts.GetFreeArgSpec(0).SetTag(EFreeArgTag::Primary); + opts.GetTrailingArgSpec().SetTag([](const TString& value) { + if (value.EndsWith(".log")) { + return EFreeArgTag::Logs; + } + if (value.EndsWith(".tmp")) { + return EFreeArgTag::Temp; + } + return EFreeArgTag::Unknown; + }); + + TOptsParseResultTestWrapper r(&opts, {"cmd", "config.yaml", "db.log", "cache.tmp", "service.log"}); + + const auto primary = r.GetFreeArgs(EFreeArgTag::Primary); + UNIT_ASSERT_VALUES_EQUAL(1u, primary.size()); + UNIT_ASSERT_VALUES_EQUAL("config.yaml", primary.front()); + + const auto logs = r.GetFreeArgs(EFreeArgTag::Logs); + UNIT_ASSERT_VALUES_EQUAL(2u, logs.size()); + UNIT_ASSERT_VALUES_EQUAL("db.log", logs[0]); + UNIT_ASSERT_VALUES_EQUAL("service.log", logs[1]); + + const auto temp = r.GetFreeArgs(EFreeArgTag::Temp); + UNIT_ASSERT_VALUES_EQUAL(1u, temp.size()); + UNIT_ASSERT_VALUES_EQUAL("cache.tmp", temp.front()); + + UNIT_ASSERT_VALUES_EQUAL(4u, r.GetFreeArgCount()); + } + Y_UNIT_TEST(TestCheckUserTypos) { { TOptsNoDefault opts; @@ -840,10 +937,10 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption("to"); UNIT_ASSERT_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"copy", "-from", "/home", "--to=/etc"})), + TOptsParseResultTestWrapper(&opts, {"copy", "-from", "/home", "--to=/etc"}), TUsageException); UNIT_ASSERT_NO_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"copy", "--from", "from", "--to=/etc"}))); + TOptsParseResultTestWrapper(&opts, {"copy", "--from", "from", "--to=/etc"})); } { @@ -853,7 +950,39 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption('r', "read", ""); opts.AddLongOption("fr"); UNIT_ASSERT_NO_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"copy", "-fr"}))); + TOptsParseResultTestWrapper(&opts, {"copy", "-fr"})); } } + + Y_UNIT_TEST(TestMutuallyExclusive) { + // FIXME: somehow MutuallyExclusive() does not work without SetFlag() + bool flag; + TOptsNoDefault opts; + opts.AddLongOption("do").SetFlag(&flag); + opts.AddLongOption("dont").SetFlag(&flag); + opts.AddLongOption("maybe-do-maybe-dont").SetFlag(&flag); + + opts.MutuallyExclusive("do", "dont", "maybe-do-maybe-dont"); + + UNIT_ASSERT_EXCEPTION( + TOptsParseResultTestWrapper(&opts, {"--do", "--dont"}), + TUsageException + ); + UNIT_ASSERT_EXCEPTION( + TOptsParseResultTestWrapper(&opts, {"--dont", "--maybe-do-maybe-dont"}), + TUsageException + ); + UNIT_ASSERT_EXCEPTION( + TOptsParseResultTestWrapper(&opts, {"--do", "--maybe-do-maybe-dont"}), + TUsageException + ); + UNIT_ASSERT_EXCEPTION( + TOptsParseResultTestWrapper(&opts, {"-d", "-n"}), + TUsageException + ); + UNIT_ASSERT_EXCEPTION( + TOptsParseResultTestWrapper(&opts, {"--do", "--dont", "--maybe-do-maybe-dont"}), + TUsageException + ); + } } diff --git a/library/cpp/getopt/ut/modchooser_ut.cpp b/library/cpp/getopt/ut/modchooser_ut.cpp index a14c8a5853..24776f47fc 100644 --- a/library/cpp/getopt/ut/modchooser_ut.cpp +++ b/library/cpp/getopt/ut/modchooser_ut.cpp @@ -39,6 +39,42 @@ static const F_PTR FUNCTIONS[] = {One, Two, Three, Four, Five}; static const char* NAMES[] = {"one", "two", "three", "four", "five"}; static_assert(Y_ARRAY_SIZE(FUNCTIONS) == Y_ARRAY_SIZE(NAMES), "Incorrect input tests data"); +class TRecordingAction: public TMainClassArgs { +public: + int DoRun(NLastGetopt::TOptsParseResult&& /*res*/) override { + CapturedSubcommandPath = GetSubcommandPath(); + return 0; + } + + void RegisterOptions(NLastGetopt::TOpts& opts) override { + opts.SetFreeArgsMax(2); + opts.AddLongOption("options-flag") + .Optional() + .NoArgument() + .StoreTrue(&OptionsFlag); + } + +public: + bool OptionsFlag = false; + TVector CapturedSubcommandPath; +}; + +class TOuterModes: public TMainClassModes { +public: + explicit TOuterModes(TMainClass* inner) + : Inner_(inner) + { + } + +protected: + void RegisterModes(TModChooser& modes) override { + modes.AddMode("inner", Inner_, "inner"); + } + +private: + TMainClass* Inner_; +}; + Y_UNIT_TEST_SUITE(TModChooserTest) { Y_UNIT_TEST(TestModesSimpleRunner) { TModChooser chooser; @@ -68,4 +104,18 @@ Y_UNIT_TEST_SUITE(TModChooserTest) { chooser.Run(argc, argv); } + + Y_UNIT_TEST(TestSubcommandPathPropagation) { + TRecordingAction innerAction; + TOuterModes outer(&innerAction); + TModChooser chooser; + chooser.AddMode("outer", &outer, "outer"); + + const char* argv[] = {"UNITTEST", "outer", "inner", "--options-flag", "free-arg1", "free-arg2", nullptr}; + UNIT_ASSERT_NO_EXCEPTION(chooser.Run(6, argv)); + + const TVector expected = {"outer", "inner"}; + UNIT_ASSERT_EQUAL(innerAction.OptionsFlag, true); + UNIT_ASSERT_VALUES_EQUAL(expected, innerAction.CapturedSubcommandPath); + } } diff --git a/library/cpp/html/escape/CMakeLists.txt b/library/cpp/html/escape/CMakeLists.txt new file mode 100644 index 0000000000..ddb97a1425 --- /dev/null +++ b/library/cpp/html/escape/CMakeLists.txt @@ -0,0 +1,25 @@ +if (YDB_SDK_TESTS) + add_ydb_test(NAME html-escape-ut + SOURCES + ut/escape_ut.cpp + LINK_LIBRARIES + html-escape + cpp-testing-unittest_main + LABELS + unit + ) +endif() + +_ydb_sdk_add_library(html-escape) + +target_link_libraries(html-escape + PUBLIC + yutil +) + +target_sources(html-escape + PRIVATE + escape.cpp +) + +_ydb_sdk_install_targets(TARGETS html-escape) diff --git a/library/cpp/html/escape/escape.cpp b/library/cpp/html/escape/escape.cpp new file mode 100644 index 0000000000..5b8ed60f04 --- /dev/null +++ b/library/cpp/html/escape/escape.cpp @@ -0,0 +1,66 @@ +#include "escape.h" + +#include +#include + +namespace NHtml { + namespace { + struct TReplace { + char Char; + bool ForText; + TStringBuf Entity; + }; + + TReplace Escapable[] = { + {'"', false, TStringBuf(""")}, + {'&', true, TStringBuf("&")}, + {'<', true, TStringBuf("<")}, + {'>', true, TStringBuf(">")}, + }; + + TString EscapeImpl(const TString& value, bool isText) { + auto ci = value.begin(); + // Looking for escapable characters. + for (; ci != value.end(); ++ci) { + for (size_t i = (isText ? 1 : 0); i < Y_ARRAY_SIZE(Escapable); ++i) { + if (*ci == Escapable[i].Char) { + goto escape; + } + } + } + + // There is no escapable characters, so return original value. + return value; + + escape: + TString tmp = TString(value.begin(), ci); + + for (; ci != value.end(); ++ci) { + size_t i = (isText ? 1 : 0); + + for (; i < Y_ARRAY_SIZE(Escapable); ++i) { + if (*ci == Escapable[i].Char) { + tmp += Escapable[i].Entity; + break; + } + } + + if (i == Y_ARRAY_SIZE(Escapable)) { + tmp += *ci; + } + } + + return tmp; + } + + } + + TString EscapeAttributeValue(const TString& value) { + return EscapeImpl(value, false); + } + + TString EscapeText(const TString& value) { + return EscapeImpl(value, true); + } + +} diff --git a/library/cpp/html/escape/escape.h b/library/cpp/html/escape/escape.h new file mode 100644 index 0000000000..1c45fc5193 --- /dev/null +++ b/library/cpp/html/escape/escape.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace NHtml { + TString EscapeAttributeValue(const TString& value); + TString EscapeText(const TString& value); + +} diff --git a/library/cpp/html/escape/ut/escape_ut.cpp b/library/cpp/html/escape/ut/escape_ut.cpp new file mode 100644 index 0000000000..cd7b955138 --- /dev/null +++ b/library/cpp/html/escape/ut/escape_ut.cpp @@ -0,0 +1,16 @@ +#include +#include + +using namespace NHtml; + +Y_UNIT_TEST_SUITE(TEscapeHtml) { + Y_UNIT_TEST(Escape) { + UNIT_ASSERT_EQUAL(EscapeText("in & out"), "in & out"); + UNIT_ASSERT_EQUAL(EscapeText("&&"), "&&"); + UNIT_ASSERT_EQUAL(EscapeText("&"), "&amp;"); + + UNIT_ASSERT_EQUAL(EscapeText("