simplify the dependency management

.devcontainer/Dockerfile

@@ -1,108 +1,10 @@
-FROM --platform=linux/amd64 mcr.microsoft.com/devcontainers/cpp:1-ubuntu-22.04
+FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04
 
-# Install software-properties-common for add-apt-repository
-RUN apt-get -y update && apt-get -y install software-properties-common && add-apt-repository ppa:ubuntu-toolchain-r/test
+# CPM supplies project libraries at configure time. The image contains only
+# platform prerequisites and developer tools.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      ca-certificates ccache clang cmake gdb git libidn11-dev libssl-dev lld ninja-build \
+      pkg-config python3 ragel yasm \
+    && rm -rf /var/lib/apt/lists/*
 
-# Install C++ tools and libraries
-RUN apt-get -y update && apt-get -y install \
-    git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \
-    rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \
-    libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# Install CMake
-ENV CMAKE_VERSION=3.27.7
-RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh \
-    -q -O cmake-install.sh \
-    && chmod u+x cmake-install.sh \
-    && ./cmake-install.sh --skip-license --prefix=/usr/local \
-    && rm cmake-install.sh
-
-# Install LLVM
-ENV LLVM_VERSION=16
-RUN wget https://apt.llvm.org/llvm.sh && \
-    chmod u+x llvm.sh && \
-    ./llvm.sh ${LLVM_VERSION}
-
-# Update alternatives to use clang-16 by default
-RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 10000 && \
-    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-${LLVM_VERSION} 10000 && \
-    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM_VERSION} 10000
-
-# Update alternatives to use gcc-13 by default
-RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000 && \
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000
-
-# Install abseil-cpp
-ENV ABSEIL_CPP_VERSION=20230802.0
-ENV ABSEIL_CPP_INSTALL_DIR=~/ydb_deps/absl
-RUN wget -O abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSEIL_CPP_VERSION}.tar.gz && \
-    tar -xvzf abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz && cd abseil-cpp-${ABSEIL_CPP_VERSION} && \
-    mkdir build && cd build && \
-    cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. && \
-    cmake --build . --config Release && \
-    cmake --install . --config Release --prefix ${ABSEIL_CPP_INSTALL_DIR}
-
-# Install protobuf
-ENV PROTOBUF_VERSION=25.0
-ENV PROTOBUF_INSTALL_DIR=~/ydb_deps/protobuf
-RUN wget -O protobuf-${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \
-    tar -xvzf protobuf-${PROTOBUF_VERSION}.tar.gz && cd protobuf-${PROTOBUF_VERSION} && \
-    mkdir build && cd build && \
-    cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR}" \
-    -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \
-    cmake --build . --config Release && \
-    cmake --install . --config Release --prefix ${PROTOBUF_INSTALL_DIR}
-
-# Install grpc
-ENV GRPC_VERSION=1.60.2
-ENV GRPC_INSTALL_DIR=~/ydb_deps/grpc
-RUN wget -O grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v${GRPC_VERSION}.tar.gz && \
-    tar -xvzf grpc-${GRPC_VERSION}.tar.gz && cd grpc-${GRPC_VERSION} && \
-    mkdir build && cd build && \
-    cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR};${PROTOBUF_INSTALL_DIR}" \
-    -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \
-    -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \
-    -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \
-    -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \
-    -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \
-    -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF .. && \
-    cmake --build . --config Release && \
-    cmake --install . --config Release --prefix ${GRPC_INSTALL_DIR}
-
-# Install base64
-ENV BASE64_VERSION=0.5.2
-RUN wget -O base64-${BASE64_VERSION}.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v${BASE64_VERSION}.tar.gz && \
-    tar -xvzf base64-${BASE64_VERSION}.tar.gz && cd base64-${BASE64_VERSION} && \
-    mkdir build && cd build && \
-    cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \
-    cmake --build . --config Release --target install
-
-# Install brotli
-ENV BROTLI_VERSION=1.1.0
-RUN wget -O brotli-${BROTLI_VERSION}.tar.gz https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz && \
-    tar -xvzf brotli-${BROTLI_VERSION}.tar.gz && cd brotli-${BROTLI_VERSION} && \
-    mkdir build && cd build && \
-    cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \
-    cmake --build . --config Release --target install
-
-# Install jwt-cpp
-ENV JWT_CPP_VERSION=0.7.0
-RUN wget -O jwt-cpp-${JWT_CPP_VERSION}.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v${JWT_CPP_VERSION}.tar.gz && \
-    tar -xvzf jwt-cpp-${JWT_CPP_VERSION}.tar.gz && cd jwt-cpp-${JWT_CPP_VERSION} && \
-    mkdir build && cd build && \
-    cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \
-    cmake --build . --config Release --target install
-
-# Install ccache 4.8.1 or above
-ENV CCACHE_VERSION=4.8.1
-RUN wget https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz \
-    && tar -xf ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz \
-    && cp ccache-${CCACHE_VERSION}-linux-x86_64/ccache /usr/local/bin/ \
-    && rm -rf ccache-${CCACHE_VERSION}-linux-x86_64 ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz
-
-# Install ydb cli 2.16.0
-ENV YDB_VERSION=2.16.0
-RUN wget https://storage.yandexcloud.net/yandexcloud-ydb/release/${YDB_VERSION}/linux/amd64/ydb && \
-    chmod u+x ydb && \
-    mv ydb /usr/local/bin/
+ENV CPM_SOURCE_CACHE=/workspaces/.cache/cpm

.devcontainer/configure.sh

@@ -1,8 +1,6 @@
 #!/bin/sh
 
-mkdir -p build
-git submodule update --init --recursive
-ccache -o cache_dir=/root/.ccache
+ccache --set-config=cache_dir=/root/.ccache
 cmake --preset release-test-clang
 
 if which ydb > /dev/null 2>&1; then

.devcontainer/devcontainer.json

@@ -24,7 +24,7 @@
 		8765
 	],
 	// Use 'initializeCommand' to run commands before the container is created.
-	"initializeCommand": "mkdir -p ${localEnv:HOME}/.ccache && cd \"${localWorkspaceFolder}\" && git config --local user.email \"$(git config user.email)\" && git config --local user.name \"$(git config user.name)\"",
+	"initializeCommand": "mkdir -p ${localEnv:HOME}/.ccache ${localEnv:HOME}/.cache/ydb-cpp-sdk-cpm && cd \"${localWorkspaceFolder}\" && git config --local user.email \"$(git config user.email)\" && git config --local user.name \"$(git config user.name)\"",
 	// Use 'postStartCommand' to run commands after the container is started.
 	"postStartCommand": ".devcontainer/configure.sh",
 	// Configure tool-specific properties.
@@ -40,7 +40,8 @@
 		}
 	},
 	"mounts": [
-		"source=${localEnv:HOME}/.ccache,target=/root/.ccache,type=bind,consistency=cached"
+		"source=${localEnv:HOME}/.ccache,target=/root/.ccache,type=bind,consistency=cached",
+		"source=${localEnv:HOME}/.cache/ydb-cpp-sdk-cpm,target=/workspaces/.cache/cpm,type=bind,consistency=cached"
 	],
 	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
 	"remoteUser": "root"

.github/actions/gcovr/run_gcovr.sh

@@ -12,6 +12,7 @@ gcovr_args=(
   --root "${SOURCE_ROOT}"
   --object-directory "${BUILD_DIR}"
   --gcov-executable "${GCOV_EXECUTABLE}"
+  --gcov-ignore-parse-errors negative_hits.warn_once_per_file
   --print-summary
   --json-summary "${OUTPUT_DIR}/summary.json"
   --html --html-details
@@ -21,7 +22,6 @@ gcovr_args=(
   --exclude '.*tests/.*'
   --merge-lines
   --exclude '.*/_deps/.*'
-  --merge-lines
   --filter 'src/'
   --filter 'include/ydb-cpp-sdk/'
   --filter 'plugins/'

.github/actions/prepare_vm/action.yaml

@@ -1,94 +1,30 @@
 name: Prepare VM for YDB SDK build
-description: Install required packages
+description: Install platform prerequisites; CPM supplies project libraries
 
 inputs:
   mode:
-    description: "Setup scope: build installs full build dependencies; runtime installs only test runtime dependencies."
+    description: "Setup scope: build installs tools and headers; runtime installs shared runtime libraries."
     required: false
     default: build
 
 runs:
-  using: "composite"
+  using: composite
   steps:
     - name: Install CMake and Ninja
+      if: ${{ inputs.mode == 'build' }}
       uses: lukka/get-cmake@v3.27.7
-    - name: Install runtime dependencies
-      shell: bash
-      run: |
-        sudo add-apt-repository ppa:ubuntu-toolchain-r/test
-        sudo apt-get -y update
-        sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \
-          rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \
-          libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13
 
-        sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000
-        sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000
-
-    - name: Install build dependencies
-      if: ${{ inputs.mode == 'build' }}
+    - name: Install platform prerequisites
       shell: bash
       run: |
-        # Install ccache
-        (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \
-        sudo tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache)
-
-        ccache -o base_dir=`realpath ..`
-        ccache -o cache_dir=~/.ccache
-        ccache -z
-        export ENABLE_CCACHE="-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
-
-        wget https://apt.llvm.org/llvm.sh
-        chmod u+x llvm.sh
-        sudo ./llvm.sh 18
-        sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 10000
-        sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 10000
-
-        # Install abseil-cpp
-        wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz
-        tar -xvzf abseil-cpp-20230802.0.tar.gz
-        cd abseil-cpp-20230802.0
-        mkdir build && cd build
-        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON ..
-        cmake --build . --config Release
-        cmake --install . --config Release --prefix ~/ydb_deps/absl
-        cd ../../
-
-        # Install protobuf
-        wget -O protobuf-25.0.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v25.0.tar.gz
-        tar -xvzf protobuf-25.0.tar.gz
-        cd protobuf-25.0
-        mkdir build && cd build
-        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="${HOME}/ydb_deps/absl" -DCMAKE_BUILD_TYPE=Release \
-          -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package ..
-        cmake --build . --config Release
-        cmake --install . --config Release --prefix ~/ydb_deps/protobuf
-        cd ../../
-
-        # Install gRPC
-        wget -O grpc-1.60.2.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.60.2.tar.gz
-        tar -xvzf grpc-1.60.2.tar.gz && cd grpc-1.60.2
-        mkdir build && cd build
-        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="${HOME}/ydb_deps/absl;${HOME}/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \
-          -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \
-          -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \
-          -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \
-          -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \
-          -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF ..
-        cmake --build . --config Release
-        cmake --install . --config Release --prefix ~/ydb_deps/grpc
-        cd ../../
-
-        # Install brotli
-        wget -O brotli-1.1.0.tar.gz https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz
-        tar -xvzf brotli-1.1.0.tar.gz && cd brotli-1.1.0
-        mkdir build && cd build
-        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release ..
-        cmake --build . --config Release
-        cmake --install . --config Release --prefix ~/ydb_deps/brotli
-        cd ../../
-
-        # Clean up
-        ccache -s
-        sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-25.0.tar.gz grpc-1.60.2.tar.gz \
-          brotli-1.1.0.tar.gz abseil-cpp-20230802.0 \
-          protobuf-25.0 grpc-1.60.2 brotli-1.1.0
+        sudo apt-get -y update
+        if [[ '${{ inputs.mode }}' == 'build' ]]; then
+          sudo apt-get -y install \
+            ccache clang g++ gcc git libidn11-dev libssl-dev lld ninja-build \
+            pkg-config python3 ragel yasm
+          ccache --set-config=base_dir="$(realpath ..)"
+          ccache --set-config=cache_dir="$HOME/.ccache"
+          ccache --zero-stats
+        else
+          sudo apt-get -y install libidn12 libssl3 libstdc++6
+        fi

.github/scripts/copy_sources.sh

@@ -204,7 +204,6 @@ sync_upstream_tree "$1" "$2" "$tmp_dir" tools/enum_parser managed
 sync_upstream_tree "$1" "$2" "$tmp_dir" tools/rescompiler managed
 
 cp $2/.gitignore $tmp_dir
-cp $2/.gitmodules $tmp_dir
 cp $2/CMakePresets.json $tmp_dir
 cp $2/CMakeLists.txt $tmp_dir
 cp $2/LICENSE $tmp_dir

.github/workflows/coverage.yml

@@ -15,7 +15,9 @@ concurrency:
 
 jobs:
   coverage:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
+    env:
+      CPM_SOURCE_CACHE: ${{ github.workspace }}/.cache/cpm
     permissions:
       contents: read
       id-token: write
@@ -38,29 +40,21 @@ jobs:
         uses: actions/checkout@v4
         if: github.event.pull_request.head.sha != ''
         with:
-          submodules: true
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Checkout
         uses: actions/checkout@v4
         if: github.event.pull_request.head.sha == ''
-        with:
-          submodules: true
-
-      - name: Prepare coverage cache key
-        id: coverage-cache-key
-        shell: bash
-        run: |
-          echo "prefix=ubuntu-22.04-coverage-gcc-${{ hashFiles('CMakePresets.json', '**/CMakeLists.txt', 'cmake/**') }}" >> "$GITHUB_OUTPUT"
 
-      - name: Restore coverage build cache
-        id: coverage-build-cache
-        uses: actions/cache/restore@v4
+      - name: Restore CPM and compiler caches
+        uses: actions/cache@v4
         with:
-          path: build
-          key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            .cache/cpm
+            ~/.ccache
+          key: coverage-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }}-${{ github.run_id }}
           restore-keys: |
-            ${{ steps.coverage-cache-key.outputs.prefix }}-
+            coverage-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }}-
 
       - name: Install dependencies
         uses: ./.github/actions/prepare_vm
@@ -95,7 +89,7 @@ jobs:
         with:
           build-directory: build
           source-root: ${{ github.workspace }}
-          gcov-executable: gcov-13
+          gcov-executable: gcov
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
@@ -115,18 +109,3 @@ jobs:
           name: coverage-report
           path: build/coverage/
           retention-days: 14
-
-      - name: Clean coverage data before cache save
-        if: github.event_name != 'pull_request' && success()
-        shell: bash
-        run: |
-          find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete
-          rm -rf build/coverage
-          cmake -E remove_directory build/util/system/testing_out_stuff
-
-      - name: Save coverage build cache
-        if: github.event_name != 'pull_request' && success() && steps.coverage-build-cache.outputs.cache-hit != 'true'
-        uses: actions/cache/save@v4
-        with:
-          path: build
-          key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}