Import YDB C++ SDK 44

.devcontainer/Dockerfile

@@ -44,17 +44,18 @@ RUN wget -O abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz https://github.com/abseil/ab
     cmake --install . --config Release --prefix ${ABSEIL_CPP_INSTALL_DIR}
 
 # Install protobuf
-ENV PROTOBUF_VERSION=3.21.12
+ENV PROTOBUF_VERSION=25.0
 ENV PROTOBUF_INSTALL_DIR=~/ydb_deps/protobuf
 RUN wget -O protobuf-${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \
     tar -xvzf protobuf-${PROTOBUF_VERSION}.tar.gz && cd protobuf-${PROTOBUF_VERSION} && \
     mkdir build && cd build && \
-    cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \
+    cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR}" \
+    -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \
     cmake --build . --config Release && \
     cmake --install . --config Release --prefix ${PROTOBUF_INSTALL_DIR}
 
 # Install grpc
-ENV GRPC_VERSION=1.54.3
+ENV GRPC_VERSION=1.60.2
 ENV GRPC_INSTALL_DIR=~/ydb_deps/grpc
 RUN wget -O grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v${GRPC_VERSION}.tar.gz && \
     tar -xvzf grpc-${GRPC_VERSION}.tar.gz && cd grpc-${GRPC_VERSION} && \

.github/actions/prepare_vm/action.yaml

@@ -39,9 +39,9 @@ runs:
 
         wget https://apt.llvm.org/llvm.sh
         chmod u+x llvm.sh
-        sudo ./llvm.sh 16
-        sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-16 10000
-        sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-16 10000
+        sudo ./llvm.sh 18
+        sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 10000
+        sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 10000
 
         # Install abseil-cpp
         wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz
@@ -54,20 +54,21 @@ runs:
         cd ../../
 
         # Install protobuf
-        wget -O protobuf-3.21.12.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz
-        tar -xvzf protobuf-3.21.12.tar.gz
-        cd protobuf-3.21.12
+        wget -O protobuf-25.0.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v25.0.tar.gz
+        tar -xvzf protobuf-25.0.tar.gz
+        cd protobuf-25.0
         mkdir build && cd build
-        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON ..
+        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="${HOME}/ydb_deps/absl" -DCMAKE_BUILD_TYPE=Release \
+          -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package ..
         cmake --build . --config Release
         cmake --install . --config Release --prefix ~/ydb_deps/protobuf
         cd ../../
 
         # Install gRPC
-        wget -O grpc-1.54.3.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz
-        tar -xvzf grpc-1.54.3.tar.gz && cd grpc-1.54.3
+        wget -O grpc-1.60.2.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.60.2.tar.gz
+        tar -xvzf grpc-1.60.2.tar.gz && cd grpc-1.60.2
         mkdir build && cd build
-        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="~/ydb_deps/absl;~/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \
+        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="${HOME}/ydb_deps/absl;${HOME}/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \
           -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \
           -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \
           -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \
@@ -88,6 +89,6 @@ runs:
 
         # Clean up
         ccache -s
-        sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-3.21.12.tar.gz grpc-1.54.3.tar.gz \
+        sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-25.0.tar.gz grpc-1.60.2.tar.gz \
           brotli-1.1.0.tar.gz abseil-cpp-20230802.0 \
-          protobuf-3.21.12 grpc-1.54.3 brotli-1.1.0
+          protobuf-25.0 grpc-1.60.2 brotli-1.1.0

.github/import_generation.txt

@@ -1 +1 @@
-44
+45

.github/last_commit.txt

@@ -1 +1 @@
-fb25fb6453264260a02c6ec49492382fcaf801ce
+1e3f62319678deefbb65acd4a3894eb0badcbb0d

.github/scripts/copy_sources.sh

@@ -1,10 +1,158 @@
 #!/bin/bash
+set -euo pipefail
 
 tmp_dir=$(mktemp -d)
 
+sync_upstream_tree() {
+    local upstream_repo=$1
+    local oss_repo=$2
+    local destination_root=$3
+    local tree=$4
+    local mode=${5:-full}
+    local previous_commit
+    local current_commit
+    local merge_dir
+    local status
+    local path
+    local new_path
+    local local_file
+    local local_target
+    local base_file
+    local upstream_file
+    local merged_file
+    local conflicts=0
+
+    previous_commit=$(cat "$oss_repo/.github/last_commit.txt")
+    current_commit=$(git -C "$upstream_repo" rev-parse HEAD)
+    merge_dir=$(mktemp -d "$tmp_dir/upstream-merge.XXXXXX")
+
+    if ! git -C "$upstream_repo" merge-base --is-ancestor "$previous_commit" "$current_commit"; then
+        echo "Cannot sync $tree: $previous_commit is not an ancestor of $current_commit" >&2
+        return 1
+    fi
+
+    while IFS=$'\t' read -r status path new_path; do
+        [ -n "$path" ] || continue
+
+        # Standalone builds use CMake and deliberately do not import Arcadia build files.
+        if [ "$(basename "$path")" = "ya.make" ] ||
+            { [ -n "${new_path:-}" ] && [ "$(basename "$new_path")" = "ya.make" ]; }; then
+            continue
+        fi
+
+        local_file="$destination_root/$path"
+        base_file="$merge_dir/base"
+        upstream_file="$merge_dir/upstream"
+        merged_file="$merge_dir/merged"
+
+        case "$status" in
+            R*)
+                local_target="$destination_root/$new_path"
+                if [ ! -f "$local_file" ]; then
+                    if [ "$mode" = "managed" ]; then
+                        continue
+                    fi
+                    echo "Cannot rename upstream file missing locally: $path" >&2
+                    conflicts=$((conflicts + 1))
+                    continue
+                fi
+                if [ -e "$local_target" ] && [ "$local_target" != "$local_file" ]; then
+                    echo "Cannot rename upstream file onto existing local file: $new_path" >&2
+                    conflicts=$((conflicts + 1))
+                    continue
+                fi
+
+                git -C "$upstream_repo" show "$previous_commit:$path" > "$base_file"
+                git -C "$upstream_repo" show "$current_commit:$new_path" > "$upstream_file"
+                mkdir -p "$(dirname "$local_target")"
+
+                if cmp -s "$local_file" "$base_file"; then
+                    cp "$upstream_file" "$local_target"
+                elif git merge-file -p "$local_file" "$base_file" "$upstream_file" > "$merged_file"; then
+                    cp "$merged_file" "$local_target"
+                else
+                    echo "Cannot merge renamed upstream and standalone changes: $path -> $new_path" >&2
+                    conflicts=$((conflicts + 1))
+                    continue
+                fi
+
+                if [ "$local_target" != "$local_file" ]; then
+                    rm "$local_file"
+                fi
+                ;;
+            A)
+                if [ "$mode" = "managed" ] && [ ! -d "$(dirname "$local_file")" ]; then
+                    continue
+                fi
+                if [ ! -e "$local_file" ]; then
+                    mkdir -p "$(dirname "$local_file")"
+                    git -C "$upstream_repo" show "$current_commit:$path" > "$local_file"
+                else
+                    git -C "$upstream_repo" show "$current_commit:$path" > "$upstream_file"
+                    if ! cmp -s "$local_file" "$upstream_file"; then
+                        echo "Cannot import added upstream file modified locally: $path" >&2
+                        conflicts=$((conflicts + 1))
+                    fi
+                fi
+                ;;
+            D)
+                if [ -e "$local_file" ]; then
+                    git -C "$upstream_repo" show "$previous_commit:$path" > "$base_file"
+                    if cmp -s "$local_file" "$base_file"; then
+                        rm "$local_file"
+                    else
+                        echo "Cannot delete upstream file modified locally: $path" >&2
+                        conflicts=$((conflicts + 1))
+                    fi
+                fi
+                ;;
+            M)
+                if [ ! -f "$local_file" ]; then
+                    if [ "$mode" = "managed" ]; then
+                        continue
+                    fi
+                    echo "Cannot update upstream file missing locally: $path" >&2
+                    conflicts=$((conflicts + 1))
+                    continue
+                fi
+
+                git -C "$upstream_repo" show "$previous_commit:$path" > "$base_file"
+                git -C "$upstream_repo" show "$current_commit:$path" > "$upstream_file"
+
+                if cmp -s "$local_file" "$upstream_file"; then
+                    continue
+                fi
+
+                if cmp -s "$local_file" "$base_file"; then
+                    cp "$upstream_file" "$local_file"
+                    continue
+                fi
+
+                if git merge-file -p "$local_file" "$base_file" "$upstream_file" > "$merged_file"; then
+                    cp "$merged_file" "$local_file"
+                else
+                    echo "Cannot merge upstream and standalone changes: $path" >&2
+                    conflicts=$((conflicts + 1))
+                fi
+                ;;
+            *)
+                echo "Unsupported upstream change '$status' for $path" >&2
+                conflicts=$((conflicts + 1))
+                ;;
+        esac
+    done < <(git -C "$upstream_repo" diff --name-status --find-renames "$previous_commit..$current_commit" -- "$tree")
+
+    rm -rf "$merge_dir"
+
+    if [ "$conflicts" -ne 0 ]; then
+        echo "Failed to import $tree: $conflicts conflicting change(s)" >&2
+        return 1
+    fi
+}
+
 echo "Copying sources..."
 
-cp -r $1/ydb/public/sdk/cpp/* $tmp_dir
+cp -r "$1"/ydb/public/sdk/cpp/* "$tmp_dir"
 echo "tmp_dir: $tmp_dir"
 
 rm -r $tmp_dir/src/client/arrow
@@ -48,6 +196,13 @@ cp -r $2/scripts $tmp_dir
 cp -r $2/third_party $tmp_dir
 cp -r $2/tools $tmp_dir
 
+sync_upstream_tree "$1" "$2" "$tmp_dir" util
+sync_upstream_tree "$1" "$2" "$tmp_dir" library/cpp managed
+sync_upstream_tree "$1" "$2" "$tmp_dir" contrib/libs/libc_compat managed
+sync_upstream_tree "$1" "$2" "$tmp_dir" contrib/libs/lzmasdk managed
+sync_upstream_tree "$1" "$2" "$tmp_dir" tools/enum_parser managed
+sync_upstream_tree "$1" "$2" "$tmp_dir" tools/rescompiler managed
+
 cp $2/.gitignore $tmp_dir
 cp $2/.gitmodules $tmp_dir
 cp $2/CMakePresets.json $tmp_dir
@@ -63,6 +218,7 @@ for oss_test_dir in slo_workloads deb_package; do
 done
 
 cp $2/include/ydb-cpp-sdk/type_switcher.h $tmp_dir/include/ydb-cpp-sdk/type_switcher.h
+cp $2/include/ydb-cpp-sdk/stlfwd.h $tmp_dir/include/ydb-cpp-sdk/stlfwd.h
 cp $2/src/version.h $tmp_dir/src/version.h
 
 cd $2

.github/workflows/coverage.yml

@@ -61,7 +61,6 @@ jobs:
           key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
           restore-keys: |
             ${{ steps.coverage-cache-key.outputs.prefix }}-
-            ubuntu-22.04-coverage-gcc-
 
       - name: Install dependencies
         uses: ./.github/actions/prepare_vm
@@ -84,6 +83,8 @@ jobs:
 
           find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete
           rm -rf build/coverage
+          cmake -E remove_directory build/util/system/testing_out_stuff
+          cmake -E make_directory build/util/system/testing_out_stuff
 
           ctest -j1 --preset coverage-all -E "${EXCLUDE_REGEX}" --output-on-failure
 
@@ -121,6 +122,7 @@ jobs:
         run: |
           find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete
           rm -rf build/coverage
+          cmake -E remove_directory build/util/system/testing_out_stuff
 
       - name: Save coverage build cache
         if: github.event_name != 'pull_request' && success() && steps.coverage-build-cache.outputs.cache-hit != 'true'

.github/workflows/import.yaml

@@ -32,7 +32,15 @@ jobs:
 
           LAST_COMMIT=$(cat ydb-cpp-sdk/.github/last_commit.txt)
 
-          COMMITS=$(git -C ydb log --format="%H" --reverse $LAST_COMMIT..main -- ydb/public/sdk/cpp ydb/public/api)
+          COMMITS=$(git -C ydb log --format="%H" --reverse $LAST_COMMIT..main -- \
+            ydb/public/sdk/cpp \
+            ydb/public/api \
+            util \
+            library/cpp \
+            contrib/libs/libc_compat \
+            contrib/libs/lzmasdk \
+            tools/enum_parser \
+            tools/rescompiler)
           if [ -z "$COMMITS" ]; then
             echo "No new commits to import"
             exit 0

.github/workflows/warmup_cache.yaml

@@ -64,7 +64,6 @@ jobs:
         key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
         restore-keys: |
           ${{ steps.coverage-cache-key.outputs.prefix }}-
-          ubuntu-22.04-coverage-gcc-
     - name: Install dependencies
       uses: ./.github/actions/prepare_vm
     - name: Configure and build with coverage
@@ -75,6 +74,7 @@ jobs:
         cmake --build build -j"$(nproc)"
         find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete
         rm -rf build/coverage
+        cmake -E remove_directory build/util/system/testing_out_stuff
     - name: Save coverage build cache
       if: steps.coverage-build-cache.outputs.cache-hit != 'true'
       uses: actions/cache/save@v4

CHANGELOG.md

@@ -1,3 +1,15 @@
+## v3.21.0
+
+* Fixed Query SDK `CreateSession` metrics being recorded when reusing a session from the pool.
+
+* Added `TQueryClient::DeleteSession` to explicitly delete a query session by session id.
+
+* Driver now supports async credentials initialisation: the first request is delayed until they are ready.
+
+* Added a flag to support deferred session creation(when client timeout exceeded, the session is created in the backgroud)
+
+* Added a distributed lock primitive based on the coordination service, which implements basic_lockable concept.
+
 ## v3.20.0
 
 * Added automatic retries for unary methods of table and query clients(ExecuteQuery, ExecuteScript, BulkUpsert, ReadRows).

CMakeLists.txt

@@ -67,6 +67,7 @@ if (YDB_CPP_SDK_SLO_USE_INSTALLED_SDK)
 
   add_subdirectory(tools)
   add_subdirectory(contrib/libs)
+  add_subdirectory(src/library/string_utils/base64)
   add_subdirectory(library/cpp)
   add_subdirectory(util)
   add_subdirectory(tests/slo_workloads)
@@ -110,6 +111,13 @@ if (YDB_SDK_INSTALL)
     get_property(PackageIncludeDirs GLOBAL PROPERTY YDB_CPP_${PackageProp}_INCLUDE_DIRS)
 
     list(REMOVE_DUPLICATES PackageSources)
+    if (NOT TargetName STREQUAL "libydb-cpp")
+      get_property(CoreSources GLOBAL PROPERTY YDB_CPP_CORE_SOURCES)
+      if (CoreSources)
+        list(REMOVE_DUPLICATES CoreSources)
+        list(REMOVE_ITEM PackageSources ${CoreSources})
+      endif()
+    endif()
     if (PackageInternalDeps)
       list(REMOVE_DUPLICATES PackageInternalDeps)
     endif()
@@ -148,6 +156,9 @@ if (YDB_SDK_INSTALL)
       set_target_properties(${TargetName} PROPERTIES OUTPUT_NAME ydb-cpp)
     endif()
     target_compile_definitions(${TargetName} PUBLIC YDB_SDK_OSS ${PackagePublicDefs})
+    if (NOT TargetName STREQUAL "libydb-cpp")
+      target_link_libraries(${TargetName} PUBLIC libydb-cpp)
+    endif()
     target_link_libraries(${TargetName} PUBLIC ${PackagePublicDeps})
     if (PackageInternalDeps)
       add_dependencies(${TargetName} ${PackageInternalDeps})