diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a30aaa6147b..f3609230636 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -44,17 +44,18 @@ RUN wget -O abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz https://github.com/abseil/ab cmake --install . --config Release --prefix ${ABSEIL_CPP_INSTALL_DIR} # Install protobuf -ENV PROTOBUF_VERSION=3.21.12 +ENV PROTOBUF_VERSION=25.0 ENV PROTOBUF_INSTALL_DIR=~/ydb_deps/protobuf RUN wget -O protobuf-${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \ tar -xvzf protobuf-${PROTOBUF_VERSION}.tar.gz && cd protobuf-${PROTOBUF_VERSION} && \ mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ + cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ cmake --build . --config Release && \ cmake --install . --config Release --prefix ${PROTOBUF_INSTALL_DIR} # Install grpc -ENV GRPC_VERSION=1.54.3 +ENV GRPC_VERSION=1.60.2 ENV GRPC_INSTALL_DIR=~/ydb_deps/grpc RUN wget -O grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v${GRPC_VERSION}.tar.gz && \ tar -xvzf grpc-${GRPC_VERSION}.tar.gz && cd grpc-${GRPC_VERSION} && \ diff --git a/.github/actions/prepare_vm/action.yaml b/.github/actions/prepare_vm/action.yaml index 35819ab6309..2e6f36d43ea 100644 --- a/.github/actions/prepare_vm/action.yaml +++ b/.github/actions/prepare_vm/action.yaml @@ -39,9 +39,9 @@ runs: wget https://apt.llvm.org/llvm.sh chmod u+x llvm.sh - sudo ./llvm.sh 16 - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-16 10000 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-16 10000 + sudo ./llvm.sh 18 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 10000 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 10000 # Install abseil-cpp wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz @@ -54,20 +54,21 @@ runs: cd ../../ # Install protobuf - wget -O protobuf-3.21.12.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz - tar -xvzf protobuf-3.21.12.tar.gz - cd protobuf-3.21.12 + wget -O protobuf-25.0.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v25.0.tar.gz + tar -xvzf protobuf-25.0.tar.gz + cd protobuf-25.0 mkdir build && cd build - cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON .. + cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="${HOME}/ydb_deps/absl" -DCMAKE_BUILD_TYPE=Release \ + -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. cmake --build . --config Release cmake --install . --config Release --prefix ~/ydb_deps/protobuf cd ../../ # Install gRPC - wget -O grpc-1.54.3.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz - tar -xvzf grpc-1.54.3.tar.gz && cd grpc-1.54.3 + wget -O grpc-1.60.2.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.60.2.tar.gz + tar -xvzf grpc-1.60.2.tar.gz && cd grpc-1.60.2 mkdir build && cd build - cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="~/ydb_deps/absl;~/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ + cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="${HOME}/ydb_deps/absl;${HOME}/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ @@ -88,6 +89,6 @@ runs: # Clean up ccache -s - sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-3.21.12.tar.gz grpc-1.54.3.tar.gz \ + sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-25.0.tar.gz grpc-1.60.2.tar.gz \ brotli-1.1.0.tar.gz abseil-cpp-20230802.0 \ - protobuf-3.21.12 grpc-1.54.3 brotli-1.1.0 + protobuf-25.0 grpc-1.60.2 brotli-1.1.0 diff --git a/.github/import_generation.txt b/.github/import_generation.txt index c739b42c4d2..ea90ee31980 100644 --- a/.github/import_generation.txt +++ b/.github/import_generation.txt @@ -1 +1 @@ -44 +45 diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 3a09c931bbf..747fec7e66d 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -fb25fb6453264260a02c6ec49492382fcaf801ce +1e3f62319678deefbb65acd4a3894eb0badcbb0d diff --git a/.github/scripts/copy_sources.sh b/.github/scripts/copy_sources.sh index 9813b2434d2..28f5ca8c8ab 100755 --- a/.github/scripts/copy_sources.sh +++ b/.github/scripts/copy_sources.sh @@ -1,10 +1,158 @@ #!/bin/bash +set -euo pipefail tmp_dir=$(mktemp -d) +sync_upstream_tree() { + local upstream_repo=$1 + local oss_repo=$2 + local destination_root=$3 + local tree=$4 + local mode=${5:-full} + local previous_commit + local current_commit + local merge_dir + local status + local path + local new_path + local local_file + local local_target + local base_file + local upstream_file + local merged_file + local conflicts=0 + + previous_commit=$(cat "$oss_repo/.github/last_commit.txt") + current_commit=$(git -C "$upstream_repo" rev-parse HEAD) + merge_dir=$(mktemp -d "$tmp_dir/upstream-merge.XXXXXX") + + if ! git -C "$upstream_repo" merge-base --is-ancestor "$previous_commit" "$current_commit"; then + echo "Cannot sync $tree: $previous_commit is not an ancestor of $current_commit" >&2 + return 1 + fi + + while IFS=$'\t' read -r status path new_path; do + [ -n "$path" ] || continue + + # Standalone builds use CMake and deliberately do not import Arcadia build files. + if [ "$(basename "$path")" = "ya.make" ] || + { [ -n "${new_path:-}" ] && [ "$(basename "$new_path")" = "ya.make" ]; }; then + continue + fi + + local_file="$destination_root/$path" + base_file="$merge_dir/base" + upstream_file="$merge_dir/upstream" + merged_file="$merge_dir/merged" + + case "$status" in + R*) + local_target="$destination_root/$new_path" + if [ ! -f "$local_file" ]; then + if [ "$mode" = "managed" ]; then + continue + fi + echo "Cannot rename upstream file missing locally: $path" >&2 + conflicts=$((conflicts + 1)) + continue + fi + if [ -e "$local_target" ] && [ "$local_target" != "$local_file" ]; then + echo "Cannot rename upstream file onto existing local file: $new_path" >&2 + conflicts=$((conflicts + 1)) + continue + fi + + git -C "$upstream_repo" show "$previous_commit:$path" > "$base_file" + git -C "$upstream_repo" show "$current_commit:$new_path" > "$upstream_file" + mkdir -p "$(dirname "$local_target")" + + if cmp -s "$local_file" "$base_file"; then + cp "$upstream_file" "$local_target" + elif git merge-file -p "$local_file" "$base_file" "$upstream_file" > "$merged_file"; then + cp "$merged_file" "$local_target" + else + echo "Cannot merge renamed upstream and standalone changes: $path -> $new_path" >&2 + conflicts=$((conflicts + 1)) + continue + fi + + if [ "$local_target" != "$local_file" ]; then + rm "$local_file" + fi + ;; + A) + if [ "$mode" = "managed" ] && [ ! -d "$(dirname "$local_file")" ]; then + continue + fi + if [ ! -e "$local_file" ]; then + mkdir -p "$(dirname "$local_file")" + git -C "$upstream_repo" show "$current_commit:$path" > "$local_file" + else + git -C "$upstream_repo" show "$current_commit:$path" > "$upstream_file" + if ! cmp -s "$local_file" "$upstream_file"; then + echo "Cannot import added upstream file modified locally: $path" >&2 + conflicts=$((conflicts + 1)) + fi + fi + ;; + D) + if [ -e "$local_file" ]; then + git -C "$upstream_repo" show "$previous_commit:$path" > "$base_file" + if cmp -s "$local_file" "$base_file"; then + rm "$local_file" + else + echo "Cannot delete upstream file modified locally: $path" >&2 + conflicts=$((conflicts + 1)) + fi + fi + ;; + M) + if [ ! -f "$local_file" ]; then + if [ "$mode" = "managed" ]; then + continue + fi + echo "Cannot update upstream file missing locally: $path" >&2 + conflicts=$((conflicts + 1)) + continue + fi + + git -C "$upstream_repo" show "$previous_commit:$path" > "$base_file" + git -C "$upstream_repo" show "$current_commit:$path" > "$upstream_file" + + if cmp -s "$local_file" "$upstream_file"; then + continue + fi + + if cmp -s "$local_file" "$base_file"; then + cp "$upstream_file" "$local_file" + continue + fi + + if git merge-file -p "$local_file" "$base_file" "$upstream_file" > "$merged_file"; then + cp "$merged_file" "$local_file" + else + echo "Cannot merge upstream and standalone changes: $path" >&2 + conflicts=$((conflicts + 1)) + fi + ;; + *) + echo "Unsupported upstream change '$status' for $path" >&2 + conflicts=$((conflicts + 1)) + ;; + esac + done < <(git -C "$upstream_repo" diff --name-status --find-renames "$previous_commit..$current_commit" -- "$tree") + + rm -rf "$merge_dir" + + if [ "$conflicts" -ne 0 ]; then + echo "Failed to import $tree: $conflicts conflicting change(s)" >&2 + return 1 + fi +} + echo "Copying sources..." -cp -r $1/ydb/public/sdk/cpp/* $tmp_dir +cp -r "$1"/ydb/public/sdk/cpp/* "$tmp_dir" echo "tmp_dir: $tmp_dir" rm -r $tmp_dir/src/client/arrow @@ -48,6 +196,13 @@ cp -r $2/scripts $tmp_dir cp -r $2/third_party $tmp_dir cp -r $2/tools $tmp_dir +sync_upstream_tree "$1" "$2" "$tmp_dir" util +sync_upstream_tree "$1" "$2" "$tmp_dir" library/cpp managed +sync_upstream_tree "$1" "$2" "$tmp_dir" contrib/libs/libc_compat managed +sync_upstream_tree "$1" "$2" "$tmp_dir" contrib/libs/lzmasdk managed +sync_upstream_tree "$1" "$2" "$tmp_dir" tools/enum_parser managed +sync_upstream_tree "$1" "$2" "$tmp_dir" tools/rescompiler managed + cp $2/.gitignore $tmp_dir cp $2/.gitmodules $tmp_dir cp $2/CMakePresets.json $tmp_dir @@ -63,6 +218,7 @@ for oss_test_dir in slo_workloads deb_package; do done cp $2/include/ydb-cpp-sdk/type_switcher.h $tmp_dir/include/ydb-cpp-sdk/type_switcher.h +cp $2/include/ydb-cpp-sdk/stlfwd.h $tmp_dir/include/ydb-cpp-sdk/stlfwd.h cp $2/src/version.h $tmp_dir/src/version.h cd $2 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 6fb34b974e5..c6d978e0df8 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -61,7 +61,6 @@ jobs: key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | ${{ steps.coverage-cache-key.outputs.prefix }}- - ubuntu-22.04-coverage-gcc- - name: Install dependencies uses: ./.github/actions/prepare_vm @@ -84,6 +83,8 @@ jobs: find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete rm -rf build/coverage + cmake -E remove_directory build/util/system/testing_out_stuff + cmake -E make_directory build/util/system/testing_out_stuff ctest -j1 --preset coverage-all -E "${EXCLUDE_REGEX}" --output-on-failure @@ -121,6 +122,7 @@ jobs: run: | find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete rm -rf build/coverage + cmake -E remove_directory build/util/system/testing_out_stuff - name: Save coverage build cache if: github.event_name != 'pull_request' && success() && steps.coverage-build-cache.outputs.cache-hit != 'true' diff --git a/.github/workflows/import.yaml b/.github/workflows/import.yaml index d1fe9f00746..ba9e7dc7177 100644 --- a/.github/workflows/import.yaml +++ b/.github/workflows/import.yaml @@ -32,7 +32,15 @@ jobs: LAST_COMMIT=$(cat ydb-cpp-sdk/.github/last_commit.txt) - COMMITS=$(git -C ydb log --format="%H" --reverse $LAST_COMMIT..main -- ydb/public/sdk/cpp ydb/public/api) + COMMITS=$(git -C ydb log --format="%H" --reverse $LAST_COMMIT..main -- \ + ydb/public/sdk/cpp \ + ydb/public/api \ + util \ + library/cpp \ + contrib/libs/libc_compat \ + contrib/libs/lzmasdk \ + tools/enum_parser \ + tools/rescompiler) if [ -z "$COMMITS" ]; then echo "No new commits to import" exit 0 diff --git a/.github/workflows/warmup_cache.yaml b/.github/workflows/warmup_cache.yaml index be353eeea69..e5864270b4d 100644 --- a/.github/workflows/warmup_cache.yaml +++ b/.github/workflows/warmup_cache.yaml @@ -64,7 +64,6 @@ jobs: key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | ${{ steps.coverage-cache-key.outputs.prefix }}- - ubuntu-22.04-coverage-gcc- - name: Install dependencies uses: ./.github/actions/prepare_vm - name: Configure and build with coverage @@ -75,6 +74,7 @@ jobs: cmake --build build -j"$(nproc)" find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete rm -rf build/coverage + cmake -E remove_directory build/util/system/testing_out_stuff - name: Save coverage build cache if: steps.coverage-build-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index f9d4fe04a0c..9560323d428 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## v3.21.0 + +* Fixed Query SDK `CreateSession` metrics being recorded when reusing a session from the pool. + +* Added `TQueryClient::DeleteSession` to explicitly delete a query session by session id. + +* Driver now supports async credentials initialisation: the first request is delayed until they are ready. + +* Added a flag to support deferred session creation(when client timeout exceeded, the session is created in the backgroud) + +* Added a distributed lock primitive based on the coordination service, which implements basic_lockable concept. + ## v3.20.0 * Added automatic retries for unary methods of table and query clients(ExecuteQuery, ExecuteScript, BulkUpsert, ReadRows). diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bec181f06e..72b66aad079 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,6 +67,7 @@ if (YDB_CPP_SDK_SLO_USE_INSTALLED_SDK) add_subdirectory(tools) add_subdirectory(contrib/libs) + add_subdirectory(src/library/string_utils/base64) add_subdirectory(library/cpp) add_subdirectory(util) add_subdirectory(tests/slo_workloads) @@ -110,6 +111,13 @@ if (YDB_SDK_INSTALL) get_property(PackageIncludeDirs GLOBAL PROPERTY YDB_CPP_${PackageProp}_INCLUDE_DIRS) list(REMOVE_DUPLICATES PackageSources) + if (NOT TargetName STREQUAL "libydb-cpp") + get_property(CoreSources GLOBAL PROPERTY YDB_CPP_CORE_SOURCES) + if (CoreSources) + list(REMOVE_DUPLICATES CoreSources) + list(REMOVE_ITEM PackageSources ${CoreSources}) + endif() + endif() if (PackageInternalDeps) list(REMOVE_DUPLICATES PackageInternalDeps) endif() @@ -148,6 +156,9 @@ if (YDB_SDK_INSTALL) set_target_properties(${TargetName} PROPERTIES OUTPUT_NAME ydb-cpp) endif() target_compile_definitions(${TargetName} PUBLIC YDB_SDK_OSS ${PackagePublicDefs}) + if (NOT TargetName STREQUAL "libydb-cpp") + target_link_libraries(${TargetName} PUBLIC libydb-cpp) + endif() target_link_libraries(${TargetName} PUBLIC ${PackagePublicDeps}) if (PackageInternalDeps) add_dependencies(${TargetName} ${PackageInternalDeps}) diff --git a/README.md b/README.md index affad1c8443..41d1790d193 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ### Prerequisites - cmake 3.22+ -- clang 16+ +- clang 18+ - git 2.20+ - ninja 1.10+ - ragel @@ -44,6 +44,19 @@ ### Install dependencies +The standalone dependency bundle is compatible with Ubuntu 24.04 and uses +gRPC 1.60.2 to match the imported YDB sources. Its protobuf and Abseil pins +match the dependency set published with that gRPC release: + +| Dependency | Version | +|------------|---------| +| Abseil | 20230802.0 | +| protobuf | 25.0 | +| gRPC | 1.60.2 | + +These pins are shared by regular CI builds, SLO workload images, and the +development container. + ```bash sudo apt-get -y update sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ @@ -52,7 +65,7 @@ sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-de wget https://apt.llvm.org/llvm.sh chmod u+x llvm.sh -sudo ./llvm.sh 16 +sudo ./llvm.sh 18 # Install abseil-cpp wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz @@ -65,20 +78,21 @@ cmake --install . --config Release --prefix ~/ydb_deps/absl cd ../../ # Install protobuf -wget -O protobuf-3.21.12.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz -tar -xvzf protobuf-3.21.12.tar.gz -cd protobuf-3.21.12 +wget -O protobuf-25.0.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v25.0.tar.gz +tar -xvzf protobuf-25.0.tar.gz +cd protobuf-25.0 mkdir build && cd build -cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON .. +cmake -G Ninja -DCMAKE_PREFIX_PATH="$HOME/ydb_deps/absl" -DCMAKE_BUILD_TYPE=Release \ + -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. cmake --build . --config Release cmake --install . --config Release --prefix ~/ydb_deps/protobuf cd ../../ # Install gRPC -wget -O grpc-1.54.3.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.54.3.tar.gz -tar -xvzf grpc-1.54.3.tar.gz && cd grpc-1.54.3 +wget -O grpc-1.60.2.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.60.2.tar.gz +tar -xvzf grpc-1.60.2.tar.gz && cd grpc-1.60.2 mkdir build && cd build -cmake -G Ninja -DCMAKE_PREFIX_PATH="~/ydb_deps/absl;~/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ +cmake -G Ninja -DCMAKE_PREFIX_PATH="$HOME/ydb_deps/absl;$HOME/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ @@ -101,9 +115,10 @@ cd ../../ wget -O brotli-1.1.0.tar.gz https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz tar -xvzf brotli-1.1.0.tar.gz && cd brotli-1.1.0 mkdir build && cd build -cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. +cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$HOME/ydb_deps/brotli" .. cmake --build . --config Release -cmake --install . --config Release --prefix ~/ydb_deps/brotli +cmake --install . --config Release cd ../../ # Install jwt-cpp @@ -155,20 +170,37 @@ cmake --build --preset $sdk_configure_preset ### Build `.deb` packages -The SDK can be packaged as Debian development packages with CPack. The packaging build uses static libraries and produces the following packages: +The SDK can be packaged as Debian development packages with CPack. The complete packaging flow uses static libraries and produces the following packages: +- `yandex-googleapis-api-common-protos` — generated API Common Protos headers and static library, required by `libydb-cpp-dev`; - `libydb-cpp-dev` — core SDK static library, public headers and CMake package files; - `libydb-cpp-iam-dev` — IAM credentials plugin; - `libydb-cpp-otel-metrics-dev` — OpenTelemetry metrics plugin (includes vendored opentelemetry-cpp); - `libydb-cpp-otel-tracing-dev` — OpenTelemetry tracing plugin (requires `libydb-cpp-otel-metrics-dev` for OTel headers/libs). -The Debian packaging flow is intended for Ubuntu 24.04. Install the regular build dependencies first. OpenTelemetry plugins use the vendored `third_party/opentelemetry-cpp` submodule (v1.26.0, matching the YDB monorepo pin); initialize it with `git submodule update --init third_party/opentelemetry-cpp` before building. +The Debian packaging flow is intended for Ubuntu 24.04. Initialize the required submodules before building: + +```bash +git submodule update --init --recursive +``` + +The packaging helper first builds `yandex-googleapis-api-common-protos` from the +vendored API Common Protos using Ubuntu 24.04's `protobuf-compiler` and +`libprotobuf-dev`. It installs that package in the build container before +building the SDK, so both packages use the same distro protobuf ABI. +OpenTelemetry plugins use the vendored `third_party/opentelemetry-cpp` +submodule (v1.26.0, matching the YDB monorepo pin). -To build `.deb` packages directly with CPack: +To build the complete `.deb` package set with the same containerized flow used +by CI and release publishing (Docker is required): ```bash -cmake --preset package-deb-clang -cmake --build build-deb --target package -j$(nproc) +mkdir -p build-deb .deb-ccache +docker run --rm --network host \ + -e CCACHE_DIR=/source/.deb-ccache \ + -v "$PWD:/source" \ + ubuntu:24.04 \ + bash /source/scripts/build_cpack_deb_packages.sh /source/build-deb ``` The generated `.deb` files are placed into `build-deb/`. @@ -196,7 +228,7 @@ To smoke-test generated `.deb` packages with the sample consumer project: ### Install from GitHub releases Pre-built `.deb` packages for Ubuntu 24.04 (Noble) are attached to each -GitHub release. Download the assets and install them with `dpkg`: +GitHub release. Download the assets and install them with APT: ```bash # Replace with the desired release tag (e.g. v1.2.3) diff --git a/allowed_peerdirs.txt b/allowed_peerdirs.txt new file mode 100644 index 00000000000..49259c99a74 --- /dev/null +++ b/allowed_peerdirs.txt @@ -0,0 +1,6 @@ +# Mirrors ydb-cpp-sdk/.github/scripts/copy_sources.sh. +ydb/public/sdk/cpp/ +ydb/public/api/ +library/cpp/ +contrib/ +util/ diff --git a/cmake/PackSDK.cmake b/cmake/PackSDK.cmake index 505d435d78b..c204e7a49e3 100644 --- a/cmake/PackSDK.cmake +++ b/cmake/PackSDK.cmake @@ -10,6 +10,7 @@ set(CPACK_PACKAGE_DESCRIPTION set(CPACK_PACKAGE_NAME "ydb-cpp-sdk") set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) set(CPACK_GENERATOR "DEB") +set(CPACK_PACKAGING_INSTALL_PREFIX "/usr/share/yandex") set(CPACK_PACKAGE_VERSION "${YDB_SDK_VERSION}") set(CPACK_PACKAGE_HOMEPAGE_URL "https://ydb.tech") set(CPACK_PACKAGE_CONTACT "YDB Team ") diff --git a/cmake/external_libs.cmake b/cmake/external_libs.cmake index 64723763a86..469df6000ca 100644 --- a/cmake/external_libs.cmake +++ b/cmake/external_libs.cmake @@ -1,7 +1,10 @@ find_package(IDN REQUIRED) find_package(Iconv REQUIRED) find_package(OpenSSL REQUIRED) -find_package(Protobuf REQUIRED) +find_package(Protobuf CONFIG QUIET) +if (NOT Protobuf_FOUND) + find_package(Protobuf MODULE REQUIRED) +endif() find_package(gRPC 1.41.0 REQUIRED) find_package(ZLIB REQUIRED) find_package(xxHash REQUIRED) diff --git a/cmake/protos_public_headers.txt b/cmake/protos_public_headers.txt index 8bf8cb074e3..468ca6d992c 100644 --- a/cmake/protos_public_headers.txt +++ b/cmake/protos_public_headers.txt @@ -1,7 +1,9 @@ src/api/grpc/draft/ydb_datastreams_v1.pb.h src/api/grpc/ydb_topic_v1.pb.h +src/api/protos/annotations/sensitive.pb.h src/api/protos/annotations/validation.pb.h src/api/protos/draft/datastreams.pb.h +src/api/protos/draft/field_transformation.pb.h src/api/protos/ydb_common.pb.h src/api/protos/ydb_federation_discovery.pb.h src/api/protos/ydb_operation.pb.h @@ -17,4 +19,4 @@ src/api/protos/ydb_issue_message.pb.h src/api/protos/ydb_export.pb.h src/api/protos/ydb_coordination.pb.h src/api/protos/ydb_status_codes.pb.h -src/api/protos/draft/ydb_replication.pb.h \ No newline at end of file +src/api/protos/draft/ydb_replication.pb.h diff --git a/cmake/public_headers.txt b/cmake/public_headers.txt index d926140d27d..9591d3df578 100644 --- a/cmake/public_headers.txt +++ b/cmake/public_headers.txt @@ -100,7 +100,6 @@ util/generic/algorithm.h util/generic/array_size.h util/generic/array_ref.h util/generic/bitops.h -util/generic/bt_exception.h util/generic/buffer.h util/generic/cast.h util/generic/deque.h @@ -113,6 +112,7 @@ util/generic/hash_table.h util/generic/hash.h util/generic/intrlist.h util/generic/is_in.h +util/generic/iterator.h util/generic/iterator_range.h util/generic/list.h util/generic/map.h @@ -149,7 +149,6 @@ util/network/ip.h util/network/sock.h util/network/socket.h util/random/random.h -util/stream/debug.h util/stream/fwd.h util/stream/input.h util/stream/labeled.h @@ -204,4 +203,4 @@ util/thread/factory.h util/thread/fwd.h util/thread/pool.h util/str_stl.h -util/ysaveload.h \ No newline at end of file +util/ysaveload.h diff --git a/cmake/ydb-cpp-sdk-config.cmake.in b/cmake/ydb-cpp-sdk-config.cmake.in index 5d53b62b283..c6817d87998 100644 --- a/cmake/ydb-cpp-sdk-config.cmake.in +++ b/cmake/ydb-cpp-sdk-config.cmake.in @@ -45,7 +45,10 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/Modules") find_package(IDN REQUIRED) find_package(Iconv REQUIRED) find_package(OpenSSL REQUIRED) -find_package(Protobuf REQUIRED) +find_package(Protobuf CONFIG QUIET) +if (NOT Protobuf_FOUND) + find_package(Protobuf MODULE REQUIRED) +endif() find_package(gRPC REQUIRED) find_package(ZLIB REQUIRED) find_package(xxHash REQUIRED) @@ -66,11 +69,13 @@ endif() if (@YDB_SDK_USE_RAPID_JSON@) find_package(RapidJSON REQUIRED) - add_library(RapidJSON::RapidJSON INTERFACE IMPORTED) + if (NOT TARGET RapidJSON::RapidJSON) + add_library(RapidJSON::RapidJSON INTERFACE IMPORTED) - target_include_directories(RapidJSON::RapidJSON INTERFACE - ${RAPIDJSON_INCLUDE_DIRS} - ) + target_include_directories(RapidJSON::RapidJSON INTERFACE + ${RAPIDJSON_INCLUDE_DIRS} + ) + endif() endif() if (@YDB_SDK_ENABLE_OTEL_METRICS@ OR @YDB_SDK_ENABLE_OTEL_TRACE@) diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index e4cddb46612..00000000000 --- a/codecov.yml +++ /dev/null @@ -1,25 +0,0 @@ -coverage: - status: - project: - default: - target: auto # do not drop below the current level... - threshold: 1% # ...by more than 1% - patch: - default: - target: 80% # new/changed code in the PR diff must be >= 80% covered - -# Cross-repo uniform metric: every YDB SDK repo defines this same component_id, -# so native-SDK coverage is queryable identically via the Codecov API -# (?component_id=native-sdk), regardless of how each repo tags its uploads. -# -# The SDK proper lives in src/ (implementation) and include/ (public headers). -# The vendored Arcadia libraries (util/, library/, contrib/, third_party/) and -# examples/tests/tools are not part of the native SDK and are pinned out by listing -# only the SDK paths. -component_management: - individual_components: - - component_id: native-sdk - name: Native SDK - paths: - - "src/**" - - "include/**" diff --git a/contrib/libs/libc_compat/CMakeLists.txt b/contrib/libs/libc_compat/CMakeLists.txt index ddce16d2cef..e296c7c8cf1 100644 --- a/contrib/libs/libc_compat/CMakeLists.txt +++ b/contrib/libs/libc_compat/CMakeLists.txt @@ -5,13 +5,12 @@ target_compile_options(contrib-libs-libc_compat PRIVATE ) target_sources(contrib-libs-libc_compat PRIVATE - ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/string.c + ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/string.c ) if(NOT APPLE) target_sources(contrib-libs-libc_compat PRIVATE ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/explicit_bzero.c - ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/memfd_create.c ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/strlcat.c ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/strlcpy.c ${YDB_SDK_SOURCE_DIR}/contrib/libs/libc_compat/reallocarray/reallocarray.c diff --git a/contrib/libs/libc_compat/README.md b/contrib/libs/libc_compat/README.md index fe7a22fbb1e..4f6a3b85f3c 100644 --- a/contrib/libs/libc_compat/README.md +++ b/contrib/libs/libc_compat/README.md @@ -9,6 +9,8 @@ During development one can make use of the following mapping of `OS_SDK` into gl | Ubuntu | glibc | | ------ | ----- | +| 24.04 | 2.39 | +| 22.04 | 2.35 | | 20.04 | 2.30 | | 18.04 | 2.27 | | 16.04 | 2.23 | diff --git a/contrib/libs/libc_compat/collate.h b/contrib/libs/libc_compat/collate.h new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/libs/libc_compat/glob.c b/contrib/libs/libc_compat/glob.c new file mode 100644 index 00000000000..00be6371585 --- /dev/null +++ b/contrib/libs/libc_compat/glob.c @@ -0,0 +1,1127 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1989, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Guido van Rossum. + * + * Copyright (c) 2011 The FreeBSD Foundation + * All rights reserved. + * Portions of this software were developed by David Chisnall + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__SCCSID("@(#)glob.c 8.3 (Berkeley) 10/13/93"); +__FBSDID("$FreeBSD$"); + +/* + * glob(3) -- a superset of the one defined in POSIX 1003.2. + * + * The [!...] convention to negate a range is supported (SysV, Posix, ksh). + * + * Optional extra services, controlled by flags not defined by POSIX: + * + * GLOB_QUOTE: + * Escaping convention: \ inhibits any special meaning the following + * character might have (except \ at end of string is retained). + * GLOB_MAGCHAR: + * Set in gl_flags if pattern contained a globbing character. + * GLOB_NOMAGIC: + * Same as GLOB_NOCHECK, but it will only append pattern if it did + * not contain any magic characters. [Used in csh style globbing] + * GLOB_ALTDIRFUNC: + * Use alternately specified directory access functions. + * GLOB_TILDE: + * expand ~user/foo to the /home/dir/of/user/foo + * GLOB_BRACE: + * expand {1,2}{a,b} to 1a 1b 2a 2b + * gl_matchc: + * Number of matches in the current invocation of glob. + */ + +/* + * Some notes on multibyte character support: + * 1. Patterns with illegal byte sequences match nothing - even if + * GLOB_NOCHECK is specified. + * 2. Illegal byte sequences in filenames are handled by treating them as + * single-byte characters with a values of such bytes of the sequence + * cast to wchar_t. + * 3. State-dependent encodings are not currently supported. + */ + +#include +#include + +#include +#include +#include +#include "glob.h" +#include +#include +#include +#include +#include "stdlib.h" +#include +#include "unistd.h" +#include + +#ifdef USE_LOCALE_COLLATE +#include "collate.h" +#endif + +/* + * glob(3) expansion limits. Stop the expansion if any of these limits + * is reached. This caps the runtime in the face of DoS attacks. See + * also CVE-2010-2632 + */ +#define GLOB_LIMIT_BRACE 128 /* number of brace calls */ +#define GLOB_LIMIT_PATH 65536 /* number of path elements */ +#define GLOB_LIMIT_READDIR 16384 /* number of readdirs */ +#define GLOB_LIMIT_STAT 1024 /* number of stat system calls */ +#define GLOB_LIMIT_STRING ARG_MAX /* maximum total size for paths */ + +struct glob_limit { + size_t l_brace_cnt; + size_t l_path_lim; + size_t l_readdir_cnt; + size_t l_stat_cnt; + size_t l_string_cnt; +}; + +#define DOT L'.' +#define EOS L'\0' +#define LBRACKET L'[' +#define NOT L'!' +#define QUESTION L'?' +#define QUOTE L'\\' +#define RANGE L'-' +#define RBRACKET L']' +#define SEP L'/' +#define STAR L'*' +#define TILDE L'~' +#define LBRACE L'{' +#define RBRACE L'}' +#define COMMA L',' + +#define M_QUOTE 0x8000000000ULL +#define M_PROTECT 0x4000000000ULL +#define M_MASK 0xffffffffffULL +#define M_CHAR 0x00ffffffffULL + +typedef uint_fast64_t Char; + +#define CHAR(c) ((Char)((c)&M_CHAR)) +#define META(c) ((Char)((c)|M_QUOTE)) +#define UNPROT(c) ((c) & ~M_PROTECT) +#define M_ALL META(L'*') +#define M_END META(L']') +#define M_NOT META(L'!') +#define M_ONE META(L'?') +#define M_RNG META(L'-') +#define M_SET META(L'[') +#define ismeta(c) (((c)&M_QUOTE) != 0) +#ifdef DEBUG +#define isprot(c) (((c)&M_PROTECT) != 0) +#endif + +static int compare(const void *, const void *); +static int g_Ctoc(const Char *, char *, size_t); +static int g_lstat(Char *, struct stat *, glob_t *); +static DIR *g_opendir(Char *, glob_t *); +static const Char *g_strchr(const Char *, wchar_t); +#ifdef notdef +static Char *g_strcat(Char *, const Char *); +#endif +static int g_stat(Char *, struct stat *, glob_t *); +static int glob0(const Char *, glob_t *, struct glob_limit *, + const char *); +static int glob1(Char *, glob_t *, struct glob_limit *); +static int glob2(Char *, Char *, Char *, Char *, glob_t *, + struct glob_limit *); +static int glob3(Char *, Char *, Char *, Char *, Char *, glob_t *, + struct glob_limit *); +static int globextend(const Char *, glob_t *, struct glob_limit *, + const char *); +static const Char * + globtilde(const Char *, Char *, size_t, glob_t *); +static int globexp0(const Char *, glob_t *, struct glob_limit *, + const char *); +static int globexp1(const Char *, glob_t *, struct glob_limit *); +static int globexp2(const Char *, const Char *, glob_t *, + struct glob_limit *); +static int globfinal(glob_t *, struct glob_limit *, size_t, + const char *); +static int match(Char *, Char *, Char *); +static int err_nomatch(glob_t *, struct glob_limit *, const char *); +static int err_aborted(glob_t *, int, char *); +#ifdef DEBUG +static void qprintf(const char *, Char *); +#endif + +int +glob(const char * __restrict pattern, int flags, + int (*errfunc)(const char *, int), glob_t * __restrict pglob) +{ + struct glob_limit limit = { 0, 0, 0, 0, 0 }; + const char *patnext; + Char *bufnext, *bufend, patbuf[MAXPATHLEN], prot; + mbstate_t mbs; + wchar_t wc; + size_t clen; + int too_long; + + patnext = pattern; + if (!(flags & GLOB_APPEND)) { + pglob->gl_pathc = 0; + pglob->gl_pathv = NULL; + if (!(flags & GLOB_DOOFFS)) + pglob->gl_offs = 0; + } + if (flags & GLOB_LIMIT) { + limit.l_path_lim = pglob->gl_matchc; + if (limit.l_path_lim == 0) + limit.l_path_lim = GLOB_LIMIT_PATH; + } + pglob->gl_flags = flags & ~GLOB_MAGCHAR; + pglob->gl_errfunc = errfunc; + pglob->gl_matchc = 0; + + bufnext = patbuf; + bufend = bufnext + MAXPATHLEN - 1; + too_long = 1; + if (flags & GLOB_NOESCAPE) { + memset(&mbs, 0, sizeof(mbs)); + while (bufnext <= bufend) { + clen = mbrtowc(&wc, patnext, MB_LEN_MAX, &mbs); + if (clen == (size_t)-1 || clen == (size_t)-2) + return (err_nomatch(pglob, &limit, pattern)); + else if (clen == 0) { + too_long = 0; + break; + } + *bufnext++ = wc; + patnext += clen; + } + } else { + /* Protect the quoted characters. */ + memset(&mbs, 0, sizeof(mbs)); + while (bufnext <= bufend) { + if (*patnext == '\\') { + if (*++patnext == '\0') { + *bufnext++ = QUOTE; + continue; + } + prot = M_PROTECT; + } else + prot = 0; + clen = mbrtowc(&wc, patnext, MB_LEN_MAX, &mbs); + if (clen == (size_t)-1 || clen == (size_t)-2) + return (err_nomatch(pglob, &limit, pattern)); + else if (clen == 0) { + too_long = 0; + break; + } + *bufnext++ = wc | prot; + patnext += clen; + } + } + if (too_long) + return (err_nomatch(pglob, &limit, pattern)); + *bufnext = EOS; + + if (flags & GLOB_BRACE) + return (globexp0(patbuf, pglob, &limit, pattern)); + else + return (glob0(patbuf, pglob, &limit, pattern)); +} + +static int +globexp0(const Char *pattern, glob_t *pglob, struct glob_limit *limit, + const char *origpat) { + int rv; + size_t oldpathc; + + /* Protect a single {}, for find(1), like csh */ + if (pattern[0] == LBRACE && pattern[1] == RBRACE && pattern[2] == EOS) { + if ((pglob->gl_flags & GLOB_LIMIT) && + limit->l_brace_cnt++ >= GLOB_LIMIT_BRACE) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + return (glob0(pattern, pglob, limit, origpat)); + } + + oldpathc = pglob->gl_pathc; + + if ((rv = globexp1(pattern, pglob, limit)) != 0) + return rv; + + return (globfinal(pglob, limit, oldpathc, origpat)); +} + +/* + * Expand recursively a glob {} pattern. When there is no more expansion + * invoke the standard globbing routine to glob the rest of the magic + * characters + */ +static int +globexp1(const Char *pattern, glob_t *pglob, struct glob_limit *limit) +{ + const Char* ptr; + + if ((ptr = g_strchr(pattern, LBRACE)) != NULL) { + if ((pglob->gl_flags & GLOB_LIMIT) && + limit->l_brace_cnt++ >= GLOB_LIMIT_BRACE) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + return (globexp2(ptr, pattern, pglob, limit)); + } + + return (glob0(pattern, pglob, limit, NULL)); +} + + +/* + * Recursive brace globbing helper. Tries to expand a single brace. + * If it succeeds then it invokes globexp1 with the new pattern. + * If it fails then it tries to glob the rest of the pattern and returns. + */ +static int +globexp2(const Char *ptr, const Char *pattern, glob_t *pglob, + struct glob_limit *limit) +{ + int i, rv; + Char *lm, *ls; + const Char *pe, *pm, *pm1, *pl; + Char patbuf[MAXPATHLEN]; + + /* copy part up to the brace */ + for (lm = patbuf, pm = pattern; pm != ptr; *lm++ = *pm++) + continue; + *lm = EOS; + ls = lm; + + /* Find the balanced brace */ + for (i = 0, pe = ++ptr; *pe != EOS; pe++) + if (*pe == LBRACKET) { + /* Ignore everything between [] */ + for (pm = pe++; *pe != RBRACKET && *pe != EOS; pe++) + continue; + if (*pe == EOS) { + /* + * We could not find a matching RBRACKET. + * Ignore and just look for RBRACE + */ + pe = pm; + } + } + else if (*pe == LBRACE) + i++; + else if (*pe == RBRACE) { + if (i == 0) + break; + i--; + } + + /* Non matching braces; just glob the pattern */ + if (i != 0 || *pe == EOS) + return (glob0(pattern, pglob, limit, NULL)); + + for (i = 0, pl = pm = ptr; pm <= pe; pm++) + switch (*pm) { + case LBRACKET: + /* Ignore everything between [] */ + for (pm1 = pm++; *pm != RBRACKET && *pm != EOS; pm++) + continue; + if (*pm == EOS) { + /* + * We could not find a matching RBRACKET. + * Ignore and just look for RBRACE + */ + pm = pm1; + } + break; + + case LBRACE: + i++; + break; + + case RBRACE: + if (i) { + i--; + break; + } + /* FALLTHROUGH */ + case COMMA: + if (i && *pm == COMMA) + break; + else { + /* Append the current string */ + for (lm = ls; (pl < pm); *lm++ = *pl++) + continue; + /* + * Append the rest of the pattern after the + * closing brace + */ + for (pl = pe + 1; (*lm++ = *pl++) != EOS;) + continue; + + /* Expand the current pattern */ +#ifdef DEBUG + qprintf("globexp2:", patbuf); +#endif + rv = globexp1(patbuf, pglob, limit); + if (rv) + return (rv); + + /* move after the comma, to the next string */ + pl = pm + 1; + } + break; + + default: + break; + } + return (0); +} + + + +/* + * expand tilde from the passwd file. + */ +static const Char * +globtilde(const Char *pattern, Char *patbuf, size_t patbuf_len, glob_t *pglob) +{ + struct passwd *pwd; + char *h, *sc; + const Char *p; + Char *b, *eb; + wchar_t wc; + wchar_t wbuf[MAXPATHLEN]; + wchar_t *wbufend, *dc; + size_t clen; + mbstate_t mbs; + int too_long; + + if (*pattern != TILDE || !(pglob->gl_flags & GLOB_TILDE)) + return (pattern); + + /* + * Copy up to the end of the string or / + */ + eb = &patbuf[patbuf_len - 1]; + for (p = pattern + 1, b = patbuf; + b < eb && *p != EOS && UNPROT(*p) != SEP; *b++ = *p++) + continue; + + if (*p != EOS && UNPROT(*p) != SEP) + return (NULL); + + *b = EOS; + h = NULL; + + if (patbuf[0] == EOS) { + /* + * handle a plain ~ or ~/ by expanding $HOME first (iff + * we're not running setuid or setgid) and then trying + * the password file + */ + if (issetugid() != 0 || + (h = getenv("HOME")) == NULL) { + if (((h = getlogin()) != NULL && + (pwd = getpwnam(h)) != NULL) || + (pwd = getpwuid(getuid())) != NULL) + h = pwd->pw_dir; + else + return (pattern); + } + } + else { + /* + * Expand a ~user + */ + if (g_Ctoc(patbuf, (char *)wbuf, sizeof(wbuf))) + return (NULL); + if ((pwd = getpwnam((char *)wbuf)) == NULL) + return (pattern); + else + h = pwd->pw_dir; + } + + /* Copy the home directory */ + dc = wbuf; + sc = h; + wbufend = wbuf + MAXPATHLEN - 1; + too_long = 1; + memset(&mbs, 0, sizeof(mbs)); + while (dc <= wbufend) { + clen = mbrtowc(&wc, sc, MB_LEN_MAX, &mbs); + if (clen == (size_t)-1 || clen == (size_t)-2) { + /* XXX See initial comment #2. */ + wc = (unsigned char)*sc; + clen = 1; + memset(&mbs, 0, sizeof(mbs)); + } + if ((*dc++ = wc) == EOS) { + too_long = 0; + break; + } + sc += clen; + } + if (too_long) + return (NULL); + + dc = wbuf; + for (b = patbuf; b < eb && *dc != EOS; *b++ = *dc++ | M_PROTECT) + continue; + if (*dc != EOS) + return (NULL); + + /* Append the rest of the pattern */ + if (*p != EOS) { + too_long = 1; + while (b <= eb) { + if ((*b++ = *p++) == EOS) { + too_long = 0; + break; + } + } + if (too_long) + return (NULL); + } else + *b = EOS; + + return (patbuf); +} + + +/* + * The main glob() routine: compiles the pattern (optionally processing + * quotes), calls glob1() to do the real pattern matching, and finally + * sorts the list (unless unsorted operation is requested). Returns 0 + * if things went well, nonzero if errors occurred. + */ +static int +glob0(const Char *pattern, glob_t *pglob, struct glob_limit *limit, + const char *origpat) { + const Char *qpatnext; + int err; + size_t oldpathc; + Char *bufnext, c, patbuf[MAXPATHLEN]; + + qpatnext = globtilde(pattern, patbuf, MAXPATHLEN, pglob); + if (qpatnext == NULL) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + oldpathc = pglob->gl_pathc; + bufnext = patbuf; + + /* We don't need to check for buffer overflow any more. */ + while ((c = *qpatnext++) != EOS) { + switch (c) { + case LBRACKET: + c = *qpatnext; + if (c == NOT) + ++qpatnext; + if (*qpatnext == EOS || + g_strchr(qpatnext+1, RBRACKET) == NULL) { + *bufnext++ = LBRACKET; + if (c == NOT) + --qpatnext; + break; + } + *bufnext++ = M_SET; + if (c == NOT) + *bufnext++ = M_NOT; + c = *qpatnext++; + do { + *bufnext++ = CHAR(c); + if (*qpatnext == RANGE && + (c = qpatnext[1]) != RBRACKET) { + *bufnext++ = M_RNG; + *bufnext++ = CHAR(c); + qpatnext += 2; + } + } while ((c = *qpatnext++) != RBRACKET); + pglob->gl_flags |= GLOB_MAGCHAR; + *bufnext++ = M_END; + break; + case QUESTION: + pglob->gl_flags |= GLOB_MAGCHAR; + *bufnext++ = M_ONE; + break; + case STAR: + pglob->gl_flags |= GLOB_MAGCHAR; + /* collapse adjacent stars to one, + * to ensure "**" at the end continues to match the + * empty string + */ + if (bufnext == patbuf || bufnext[-1] != M_ALL) + *bufnext++ = M_ALL; + break; + default: + *bufnext++ = CHAR(c); + break; + } + } + *bufnext = EOS; +#ifdef DEBUG + qprintf("glob0:", patbuf); +#endif + + if ((err = glob1(patbuf, pglob, limit)) != 0) + return(err); + + if (origpat != NULL) + return (globfinal(pglob, limit, oldpathc, origpat)); + + return (0); +} + +static int +globfinal(glob_t *pglob, struct glob_limit *limit, size_t oldpathc, + const char *origpat) { + if (pglob->gl_pathc == oldpathc) + return (err_nomatch(pglob, limit, origpat)); + + if (!(pglob->gl_flags & GLOB_NOSORT)) + qsort(pglob->gl_pathv + pglob->gl_offs + oldpathc, + pglob->gl_pathc - oldpathc, sizeof(char *), compare); + + return (0); +} + +static int +compare(const void *p, const void *q) +{ + return (strcoll(*(char **)p, *(char **)q)); +} + +static int +glob1(Char *pattern, glob_t *pglob, struct glob_limit *limit) +{ + Char pathbuf[MAXPATHLEN]; + + /* A null pathname is invalid -- POSIX 1003.1 sect. 2.4. */ + if (*pattern == EOS) + return (0); + return (glob2(pathbuf, pathbuf, pathbuf + MAXPATHLEN - 1, + pattern, pglob, limit)); +} + +/* + * The functions glob2 and glob3 are mutually recursive; there is one level + * of recursion for each segment in the pattern that contains one or more + * meta characters. + */ +static int +glob2(Char *pathbuf, Char *pathend, Char *pathend_last, Char *pattern, + glob_t *pglob, struct glob_limit *limit) +{ + struct stat sb; + Char *p, *q; + int anymeta; + + /* + * Loop over pattern segments until end of pattern or until + * segment with meta character found. + */ + for (anymeta = 0;;) { + if (*pattern == EOS) { /* End of pattern? */ + *pathend = EOS; + if (g_lstat(pathbuf, &sb, pglob)) + return (0); + + if ((pglob->gl_flags & GLOB_LIMIT) && + limit->l_stat_cnt++ >= GLOB_LIMIT_STAT) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + if ((pglob->gl_flags & GLOB_MARK) && + UNPROT(pathend[-1]) != SEP && + (S_ISDIR(sb.st_mode) || + (S_ISLNK(sb.st_mode) && + g_stat(pathbuf, &sb, pglob) == 0 && + S_ISDIR(sb.st_mode)))) { + if (pathend + 1 > pathend_last) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + *pathend++ = SEP; + *pathend = EOS; + } + ++pglob->gl_matchc; + return (globextend(pathbuf, pglob, limit, NULL)); + } + + /* Find end of next segment, copy tentatively to pathend. */ + q = pathend; + p = pattern; + while (*p != EOS && UNPROT(*p) != SEP) { + if (ismeta(*p)) + anymeta = 1; + if (q + 1 > pathend_last) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + *q++ = *p++; + } + + if (!anymeta) { /* No expansion, do next segment. */ + pathend = q; + pattern = p; + while (UNPROT(*pattern) == SEP) { + if (pathend + 1 > pathend_last) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + *pathend++ = *pattern++; + } + } else /* Need expansion, recurse. */ + return (glob3(pathbuf, pathend, pathend_last, pattern, + p, pglob, limit)); + } + /* NOTREACHED */ +} + +static int +glob3(Char *pathbuf, Char *pathend, Char *pathend_last, + Char *pattern, Char *restpattern, + glob_t *pglob, struct glob_limit *limit) +{ + struct dirent *dp; + DIR *dirp; + int err, too_long, saverrno, saverrno2; + char buf[MAXPATHLEN + MB_LEN_MAX - 1]; + + struct dirent *(*readdirfunc)(DIR *); + + if (pathend > pathend_last) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + *pathend = EOS; + if (pglob->gl_errfunc != NULL && + g_Ctoc(pathbuf, buf, sizeof(buf))) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + + saverrno = errno; + errno = 0; + if ((dirp = g_opendir(pathbuf, pglob)) == NULL) { + if (errno == ENOENT || errno == ENOTDIR) + return (0); + err = err_aborted(pglob, errno, buf); + if (errno == 0) + errno = saverrno; + return (err); + } + + err = 0; + + /* pglob->gl_readdir takes a void *, fix this manually */ + if (pglob->gl_flags & GLOB_ALTDIRFUNC) + readdirfunc = (struct dirent *(*)(DIR *))pglob->gl_readdir; + else + readdirfunc = readdir; + + errno = 0; + /* Search directory for matching names. */ + while ((dp = (*readdirfunc)(dirp)) != NULL) { + char *sc; + Char *dc; + wchar_t wc; + size_t clen; + mbstate_t mbs; + + if ((pglob->gl_flags & GLOB_LIMIT) && + limit->l_readdir_cnt++ >= GLOB_LIMIT_READDIR) { + errno = E2BIG; + err = GLOB_NOSPACE; + break; + } + + /* Initial DOT must be matched literally. */ + if (dp->d_name[0] == '.' && UNPROT(*pattern) != DOT) { + errno = 0; + continue; + } + memset(&mbs, 0, sizeof(mbs)); + dc = pathend; + sc = dp->d_name; + too_long = 1; + while (dc <= pathend_last) { + clen = mbrtowc(&wc, sc, MB_LEN_MAX, &mbs); + if (clen == (size_t)-1 || clen == (size_t)-2) { + /* XXX See initial comment #2. */ + wc = (unsigned char)*sc; + clen = 1; + memset(&mbs, 0, sizeof(mbs)); + } + if ((*dc++ = wc) == EOS) { + too_long = 0; + break; + } + sc += clen; + } + if (too_long && (err = err_aborted(pglob, ENAMETOOLONG, + buf))) { + errno = ENAMETOOLONG; + break; + } + if (too_long || !match(pathend, pattern, restpattern)) { + *pathend = EOS; + errno = 0; + continue; + } + if (errno == 0) + errno = saverrno; + err = glob2(pathbuf, --dc, pathend_last, restpattern, + pglob, limit); + if (err) + break; + errno = 0; + } + + saverrno2 = errno; + if (pglob->gl_flags & GLOB_ALTDIRFUNC) + (*pglob->gl_closedir)(dirp); + else + closedir(dirp); + errno = saverrno2; + + if (err) + return (err); + + if (dp == NULL && errno != 0 && + (err = err_aborted(pglob, errno, buf))) + return (err); + + if (errno == 0) + errno = saverrno; + return (0); +} + + +/* + * Extend the gl_pathv member of a glob_t structure to accommodate a new item, + * add the new item, and update gl_pathc. + * + * This assumes the BSD realloc, which only copies the block when its size + * crosses a power-of-two boundary; for v7 realloc, this would cause quadratic + * behavior. + * + * Return 0 if new item added, error code if memory couldn't be allocated. + * + * Invariant of the glob_t structure: + * Either gl_pathc is zero and gl_pathv is NULL; or gl_pathc > 0 and + * gl_pathv points to (gl_offs + gl_pathc + 1) items. + */ +static int +globextend(const Char *path, glob_t *pglob, struct glob_limit *limit, + const char *origpat) +{ + char **pathv; + size_t i, newn, len; + char *copy; + const Char *p; + + if ((pglob->gl_flags & GLOB_LIMIT) && + pglob->gl_matchc > limit->l_path_lim) { + errno = E2BIG; + return (GLOB_NOSPACE); + } + + newn = 2 + pglob->gl_pathc + pglob->gl_offs; + /* reallocarray(NULL, newn, size) is equivalent to malloc(newn*size). */ + pathv = reallocarray(pglob->gl_pathv, newn, sizeof(*pathv)); + if (pathv == NULL) + return (GLOB_NOSPACE); + + if (pglob->gl_pathv == NULL && pglob->gl_offs > 0) { + /* first time around -- clear initial gl_offs items */ + pathv += pglob->gl_offs; + for (i = pglob->gl_offs + 1; --i > 0; ) + *--pathv = NULL; + } + pglob->gl_pathv = pathv; + + if (origpat != NULL) + copy = strdup(origpat); + else { + for (p = path; *p++ != EOS;) + continue; + len = MB_CUR_MAX * (size_t)(p - path); /* XXX overallocation */ + if ((copy = malloc(len)) != NULL) { + if (g_Ctoc(path, copy, len)) { + free(copy); + errno = E2BIG; + return (GLOB_NOSPACE); + } + } + } + if (copy != NULL) { + limit->l_string_cnt += strlen(copy) + 1; + if ((pglob->gl_flags & GLOB_LIMIT) && + limit->l_string_cnt >= GLOB_LIMIT_STRING) { + free(copy); + errno = E2BIG; + return (GLOB_NOSPACE); + } + pathv[pglob->gl_offs + pglob->gl_pathc++] = copy; + } + pathv[pglob->gl_offs + pglob->gl_pathc] = NULL; + return (copy == NULL ? GLOB_NOSPACE : 0); +} + +/* + * pattern matching function for filenames. + */ +static int +match(Char *name, Char *pat, Char *patend) +{ + int ok, negate_range; + Char c, k, *nextp, *nextn; +#ifdef USE_LOCALE_COLLATE + struct xlocale_collate *table = + (struct xlocale_collate*)__get_locale()->components[XLC_COLLATE]; +#endif + + nextn = NULL; + nextp = NULL; + + while (1) { + while (pat < patend) { + c = *pat++; + switch (c & M_MASK) { + case M_ALL: + if (pat == patend) + return (1); + if (*name == EOS) + return (0); + nextn = name + 1; + nextp = pat - 1; + break; + case M_ONE: + if (*name++ == EOS) + goto fail; + break; + case M_SET: + ok = 0; + if ((k = *name++) == EOS) + goto fail; + negate_range = ((*pat & M_MASK) == M_NOT); + if (negate_range != 0) + ++pat; + while (((c = *pat++) & M_MASK) != M_END) + if ((*pat & M_MASK) == M_RNG) { +#ifdef USE_LOCALE_COLLATE + if (table->__collate_load_error ? + CHAR(c) <= CHAR(k) && + CHAR(k) <= CHAR(pat[1]) : + __wcollate_range_cmp(CHAR(c), + CHAR(k)) <= 0 && + __wcollate_range_cmp(CHAR(k), + CHAR(pat[1])) <= 0) +#else + if (c <= k && k <= pat[1]) +#endif + ok = 1; + pat += 2; + } else if (c == k) + ok = 1; + if (ok == negate_range) + goto fail; + break; + default: + if (*name++ != c) + goto fail; + break; + } + } + if (*name == EOS) + return (1); + + fail: + if (nextn == NULL) + break; + pat = nextp; + name = nextn; + } + return (0); +} + +/* Free allocated data belonging to a glob_t structure. */ +void +globfree(glob_t *pglob) +{ + size_t i; + char **pp; + + if (pglob->gl_pathv != NULL) { + pp = pglob->gl_pathv + pglob->gl_offs; + for (i = pglob->gl_pathc; i--; ++pp) + if (*pp) + free(*pp); + free(pglob->gl_pathv); + pglob->gl_pathv = NULL; + } +} + +static DIR * +g_opendir(Char *str, glob_t *pglob) +{ + char buf[MAXPATHLEN + MB_LEN_MAX - 1]; + + if (*str == EOS) + strcpy(buf, "."); + else { + if (g_Ctoc(str, buf, sizeof(buf))) { + errno = ENAMETOOLONG; + return (NULL); + } + } + + if (pglob->gl_flags & GLOB_ALTDIRFUNC) + return ((*pglob->gl_opendir)(buf)); + + return (opendir(buf)); +} + +static int +g_lstat(Char *fn, struct stat *sb, glob_t *pglob) +{ + char buf[MAXPATHLEN + MB_LEN_MAX - 1]; + + if (g_Ctoc(fn, buf, sizeof(buf))) { + errno = ENAMETOOLONG; + return (-1); + } + if (pglob->gl_flags & GLOB_ALTDIRFUNC) + return((*pglob->gl_lstat)(buf, sb)); + return (lstat(buf, sb)); +} + +static int +g_stat(Char *fn, struct stat *sb, glob_t *pglob) +{ + char buf[MAXPATHLEN + MB_LEN_MAX - 1]; + + if (g_Ctoc(fn, buf, sizeof(buf))) { + errno = ENAMETOOLONG; + return (-1); + } + if (pglob->gl_flags & GLOB_ALTDIRFUNC) + return ((*pglob->gl_stat)(buf, sb)); + return (stat(buf, sb)); +} + +static const Char * +g_strchr(const Char *str, wchar_t ch) +{ + + do { + if (*str == ch) + return (str); + } while (*str++); + return (NULL); +} + +static int +g_Ctoc(const Char *str, char *buf, size_t len) +{ + mbstate_t mbs; + size_t clen; + + memset(&mbs, 0, sizeof(mbs)); + while (len >= MB_CUR_MAX) { + clen = wcrtomb(buf, CHAR(*str), &mbs); + if (clen == (size_t)-1) { + /* XXX See initial comment #2. */ + *buf = (char)CHAR(*str); + clen = 1; + memset(&mbs, 0, sizeof(mbs)); + } + if (CHAR(*str) == EOS) + return (0); + str++; + buf += clen; + len -= clen; + } + return (1); +} + +static int +err_nomatch(glob_t *pglob, struct glob_limit *limit, const char *origpat) { + /* + * If there was no match we are going to append the origpat + * if GLOB_NOCHECK was specified or if GLOB_NOMAGIC was specified + * and the origpat did not contain any magic characters + * GLOB_NOMAGIC is there just for compatibility with csh. + */ + if ((pglob->gl_flags & GLOB_NOCHECK) || + ((pglob->gl_flags & GLOB_NOMAGIC) && + !(pglob->gl_flags & GLOB_MAGCHAR))) + return (globextend(NULL, pglob, limit, origpat)); + return (GLOB_NOMATCH); +} + +static int +err_aborted(glob_t *pglob, int err, char *buf) { + if ((pglob->gl_errfunc != NULL && pglob->gl_errfunc(buf, err)) || + (pglob->gl_flags & GLOB_ERR)) + return (GLOB_ABORTED); + return (0); +} + +#ifdef DEBUG +static void +qprintf(const char *str, Char *s) +{ + Char *p; + + (void)printf("%s\n", str); + if (s != NULL) { + for (p = s; *p != EOS; p++) + (void)printf("%c", (char)CHAR(*p)); + (void)printf("\n"); + for (p = s; *p != EOS; p++) + (void)printf("%c", (isprot(*p) ? '\\' : ' ')); + (void)printf("\n"); + for (p = s; *p != EOS; p++) + (void)printf("%c", (ismeta(*p) ? '_' : ' ')); + (void)printf("\n"); + } +} +#endif diff --git a/contrib/libs/libc_compat/glob.h b/contrib/libs/libc_compat/glob.h new file mode 100644 index 00000000000..783ba51042d --- /dev/null +++ b/contrib/libs/libc_compat/glob.h @@ -0,0 +1,104 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1989, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Guido van Rossum. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)glob.h 8.1 (Berkeley) 6/2/93 + * $FreeBSD$ + */ + +#ifndef _GLOB_H_ +#define _GLOB_H_ + +#include +#include + + +struct stat; +typedef struct { + size_t gl_pathc; /* Count of total paths so far. */ + size_t gl_matchc; /* Count of paths matching pattern. */ + size_t gl_offs; /* Reserved at beginning of gl_pathv. */ + int gl_flags; /* Copy of flags parameter to glob. */ + char **gl_pathv; /* List of paths matching pattern. */ + /* Copy of errfunc parameter to glob. */ + int (*gl_errfunc)(const char *, int); + + /* + * Alternate filesystem access methods for glob; replacement + * versions of closedir(3), readdir(3), opendir(3), stat(2) + * and lstat(2). + */ + void (*gl_closedir)(void *); + struct dirent *(*gl_readdir)(void *); + void *(*gl_opendir)(const char *); + int (*gl_lstat)(const char *, struct stat *); + int (*gl_stat)(const char *, struct stat *); +} glob_t; + +// #if __POSIX_VISIBLE >= 199209 +/* Believed to have been introduced in 1003.2-1992 */ +#define GLOB_APPEND 0x0001 /* Append to output from previous call. */ +#define GLOB_DOOFFS 0x0002 /* Use gl_offs. */ +#define GLOB_ERR 0x0004 /* Return on error. */ +#define GLOB_MARK 0x0008 /* Append / to matching directories. */ +#define GLOB_NOCHECK 0x0010 /* Return pattern itself if nothing matches. */ +#define GLOB_NOSORT 0x0020 /* Don't sort. */ +#define GLOB_NOESCAPE 0x2000 /* Disable backslash escaping. */ + +/* Error values returned by glob(3) */ +#define GLOB_NOSPACE (-1) /* Malloc call failed. */ +#define GLOB_ABORTED (-2) /* Unignored error. */ +#define GLOB_NOMATCH (-3) /* No match and GLOB_NOCHECK was not set. */ +#define GLOB_NOSYS (-4) /* Obsolete: source comptability only. */ +// #endif /* __POSIX_VISIBLE >= 199209 */ + +// #if __BSD_VISIBLE +#define GLOB_ALTDIRFUNC 0x0040 /* Use alternately specified directory funcs. */ +#define GLOB_BRACE 0x0080 /* Expand braces ala csh. */ +#define GLOB_MAGCHAR 0x0100 /* Pattern had globbing characters. */ +#define GLOB_NOMAGIC 0x0200 /* GLOB_NOCHECK without magic chars (csh). */ +#define GLOB_QUOTE 0x0400 /* Quote special chars with \. */ +#define GLOB_TILDE 0x0800 /* Expand tilde names from the passwd file. */ +#define GLOB_LIMIT 0x1000 /* limit number of returned paths */ + +/* source compatibility, these are the old names */ +#define GLOB_MAXPATH GLOB_LIMIT +#define GLOB_ABEND GLOB_ABORTED +// #endif /* __BSD_VISIBLE */ + +__BEGIN_DECLS +int glob(const char * __restrict, int, + int (*)(const char *, int), glob_t * __restrict); +void globfree(glob_t *); +__END_DECLS + +#endif /* !_GLOB_H_ */ diff --git a/contrib/libs/libc_compat/ifaddrs.c b/contrib/libs/libc_compat/ifaddrs.c new file mode 100644 index 00000000000..c59d8bc7455 --- /dev/null +++ b/contrib/libs/libc_compat/ifaddrs.c @@ -0,0 +1,663 @@ +/* +Copyright (c) 2013, Kenneth MacKay +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "ifaddrs.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct NetlinkList +{ + struct NetlinkList *m_next; + struct nlmsghdr *m_data; + unsigned int m_size; +} NetlinkList; + +static int netlink_socket(void) +{ + int l_socket = socket(PF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if(l_socket < 0) + { + return -1; + } + + struct sockaddr_nl l_addr; + memset(&l_addr, 0, sizeof(l_addr)); + l_addr.nl_family = AF_NETLINK; + if(bind(l_socket, (struct sockaddr *)&l_addr, sizeof(l_addr)) < 0) + { + close(l_socket); + return -1; + } + + return l_socket; +} + +static int netlink_send(int p_socket, int p_request) +{ + struct + { + struct nlmsghdr m_hdr; + struct rtgenmsg m_msg; + } l_data; + + memset(&l_data, 0, sizeof(l_data)); + + l_data.m_hdr.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtgenmsg)); + l_data.m_hdr.nlmsg_type = p_request; + l_data.m_hdr.nlmsg_flags = NLM_F_ROOT | NLM_F_MATCH | NLM_F_REQUEST; + l_data.m_hdr.nlmsg_pid = 0; + l_data.m_hdr.nlmsg_seq = p_socket; + l_data.m_msg.rtgen_family = AF_UNSPEC; + + struct sockaddr_nl l_addr; + memset(&l_addr, 0, sizeof(l_addr)); + l_addr.nl_family = AF_NETLINK; + return (sendto(p_socket, &l_data.m_hdr, l_data.m_hdr.nlmsg_len, 0, (struct sockaddr *)&l_addr, sizeof(l_addr))); +} + +static int netlink_recv(int p_socket, void *p_buffer, size_t p_len) +{ + struct msghdr l_msg; + struct iovec l_iov = { p_buffer, p_len }; + struct sockaddr_nl l_addr; + + for(;;) + { + l_msg.msg_name = (void *)&l_addr; + l_msg.msg_namelen = sizeof(l_addr); + l_msg.msg_iov = &l_iov; + l_msg.msg_iovlen = 1; + l_msg.msg_control = NULL; + l_msg.msg_controllen = 0; + l_msg.msg_flags = 0; + int l_result = recvmsg(p_socket, &l_msg, 0); + + if(l_result < 0) + { + if(errno == EINTR) + { + continue; + } + return -2; + } + + if(l_msg.msg_flags & MSG_TRUNC) + { // buffer was too small + return -1; + } + return l_result; + } +} + +static struct nlmsghdr *getNetlinkResponse(int p_socket, int *p_size, int *p_done) +{ + size_t l_size = 4096; + void *l_buffer = NULL; + + for(;;) + { + free(l_buffer); + l_buffer = malloc(l_size); + if (l_buffer == NULL) + { + return NULL; + } + + int l_read = netlink_recv(p_socket, l_buffer, l_size); + *p_size = l_read; + if(l_read == -2) + { + free(l_buffer); + return NULL; + } + if(l_read >= 0) + { + pid_t l_pid = getpid(); + struct nlmsghdr *l_hdr; + for(l_hdr = (struct nlmsghdr *)l_buffer; NLMSG_OK(l_hdr, (unsigned int)l_read); l_hdr = (struct nlmsghdr *)NLMSG_NEXT(l_hdr, l_read)) + { + if((pid_t)l_hdr->nlmsg_pid != l_pid || (int)l_hdr->nlmsg_seq != p_socket) + { + continue; + } + + if(l_hdr->nlmsg_type == NLMSG_DONE) + { + *p_done = 1; + break; + } + + if(l_hdr->nlmsg_type == NLMSG_ERROR) + { + free(l_buffer); + return NULL; + } + } + return l_buffer; + } + + l_size *= 2; + } +} + +static NetlinkList *newListItem(struct nlmsghdr *p_data, unsigned int p_size) +{ + NetlinkList *l_item = malloc(sizeof(NetlinkList)); + if (l_item == NULL) + { + return NULL; + } + + l_item->m_next = NULL; + l_item->m_data = p_data; + l_item->m_size = p_size; + return l_item; +} + +static void freeResultList(NetlinkList *p_list) +{ + NetlinkList *l_cur; + while(p_list) + { + l_cur = p_list; + p_list = p_list->m_next; + free(l_cur->m_data); + free(l_cur); + } +} + +static NetlinkList *getResultList(int p_socket, int p_request) +{ + if(netlink_send(p_socket, p_request) < 0) + { + return NULL; + } + + NetlinkList *l_list = NULL; + NetlinkList *l_end = NULL; + int l_size; + int l_done = 0; + while(!l_done) + { + struct nlmsghdr *l_hdr = getNetlinkResponse(p_socket, &l_size, &l_done); + if(!l_hdr) + { // error + freeResultList(l_list); + return NULL; + } + + NetlinkList *l_item = newListItem(l_hdr, l_size); + if (!l_item) + { + freeResultList(l_list); + return NULL; + } + if(!l_list) + { + l_list = l_item; + } + else + { + l_end->m_next = l_item; + } + l_end = l_item; + } + return l_list; +} + +static size_t maxSize(size_t a, size_t b) +{ + return (a > b ? a : b); +} + +static size_t calcAddrLen(sa_family_t p_family, int p_dataSize) +{ + switch(p_family) + { + case AF_INET: + return sizeof(struct sockaddr_in); + case AF_INET6: + return sizeof(struct sockaddr_in6); + case AF_PACKET: + return maxSize(sizeof(struct sockaddr_ll), offsetof(struct sockaddr_ll, sll_addr) + p_dataSize); + default: + return maxSize(sizeof(struct sockaddr), offsetof(struct sockaddr, sa_data) + p_dataSize); + } +} + +static void makeSockaddr(sa_family_t p_family, struct sockaddr *p_dest, void *p_data, size_t p_size) +{ + switch(p_family) + { + case AF_INET: + memcpy(&((struct sockaddr_in*)p_dest)->sin_addr, p_data, p_size); + break; + case AF_INET6: + memcpy(&((struct sockaddr_in6*)p_dest)->sin6_addr, p_data, p_size); + break; + case AF_PACKET: + memcpy(((struct sockaddr_ll*)p_dest)->sll_addr, p_data, p_size); + ((struct sockaddr_ll*)p_dest)->sll_halen = p_size; + break; + default: + memcpy(p_dest->sa_data, p_data, p_size); + break; + } + p_dest->sa_family = p_family; +} + +static void addToEnd(struct ifaddrs **p_resultList, struct ifaddrs *p_entry) +{ + if(!*p_resultList) + { + *p_resultList = p_entry; + } + else + { + struct ifaddrs *l_cur = *p_resultList; + while(l_cur->ifa_next) + { + l_cur = l_cur->ifa_next; + } + l_cur->ifa_next = p_entry; + } +} + +static int interpretLink(struct nlmsghdr *p_hdr, struct ifaddrs **p_resultList) +{ + struct ifinfomsg *l_info = (struct ifinfomsg *)NLMSG_DATA(p_hdr); + + size_t l_nameSize = 0; + size_t l_addrSize = 0; + size_t l_dataSize = 0; + + size_t l_rtaSize = NLMSG_PAYLOAD(p_hdr, sizeof(struct ifinfomsg)); + struct rtattr *l_rta; + for(l_rta = IFLA_RTA(l_info); RTA_OK(l_rta, l_rtaSize); l_rta = RTA_NEXT(l_rta, l_rtaSize)) + { + void *l_rtaData = RTA_DATA(l_rta); + size_t l_rtaDataSize = RTA_PAYLOAD(l_rta); + switch(l_rta->rta_type) + { + case IFLA_ADDRESS: + case IFLA_BROADCAST: + l_addrSize += NLMSG_ALIGN(calcAddrLen(AF_PACKET, l_rtaDataSize)); + break; + case IFLA_IFNAME: + l_nameSize += NLMSG_ALIGN(l_rtaSize + 1); + break; + case IFLA_STATS: + l_dataSize += NLMSG_ALIGN(l_rtaSize); + break; + default: + break; + } + } + + struct ifaddrs *l_entry = malloc(sizeof(struct ifaddrs) + sizeof(int) + l_nameSize + l_addrSize + l_dataSize); + if (l_entry == NULL) + { + return -1; + } + memset(l_entry, 0, sizeof(struct ifaddrs)); + l_entry->ifa_name = ""; + + char *l_index = ((char *)l_entry) + sizeof(struct ifaddrs); + char *l_name = l_index + sizeof(int); + char *l_addr = l_name + l_nameSize; + char *l_data = l_addr + l_addrSize; + + // save the interface index so we can look it up when handling the addresses. + memcpy(l_index, &l_info->ifi_index, sizeof(int)); + + l_entry->ifa_flags = l_info->ifi_flags; + + l_rtaSize = NLMSG_PAYLOAD(p_hdr, sizeof(struct ifinfomsg)); + for(l_rta = IFLA_RTA(l_info); RTA_OK(l_rta, l_rtaSize); l_rta = RTA_NEXT(l_rta, l_rtaSize)) + { + void *l_rtaData = RTA_DATA(l_rta); + size_t l_rtaDataSize = RTA_PAYLOAD(l_rta); + switch(l_rta->rta_type) + { + case IFLA_ADDRESS: + case IFLA_BROADCAST: + { + size_t l_addrLen = calcAddrLen(AF_PACKET, l_rtaDataSize); + makeSockaddr(AF_PACKET, (struct sockaddr *)l_addr, l_rtaData, l_rtaDataSize); + ((struct sockaddr_ll *)l_addr)->sll_ifindex = l_info->ifi_index; + ((struct sockaddr_ll *)l_addr)->sll_hatype = l_info->ifi_type; + if(l_rta->rta_type == IFLA_ADDRESS) + { + l_entry->ifa_addr = (struct sockaddr *)l_addr; + } + else + { + l_entry->ifa_broadaddr = (struct sockaddr *)l_addr; + } + l_addr += NLMSG_ALIGN(l_addrLen); + break; + } + case IFLA_IFNAME: + strncpy(l_name, l_rtaData, l_rtaDataSize); + l_name[l_rtaDataSize] = '\0'; + l_entry->ifa_name = l_name; + break; + case IFLA_STATS: + memcpy(l_data, l_rtaData, l_rtaDataSize); + l_entry->ifa_data = l_data; + break; + default: + break; + } + } + + addToEnd(p_resultList, l_entry); + return 0; +} + +static struct ifaddrs *findInterface(int p_index, struct ifaddrs **p_links, int p_numLinks) +{ + int l_num = 0; + struct ifaddrs *l_cur = *p_links; + while(l_cur && l_num < p_numLinks) + { + char *l_indexPtr = ((char *)l_cur) + sizeof(struct ifaddrs); + int l_index; + memcpy(&l_index, l_indexPtr, sizeof(int)); + if(l_index == p_index) + { + return l_cur; + } + + l_cur = l_cur->ifa_next; + ++l_num; + } + return NULL; +} + +static int interpretAddr(struct nlmsghdr *p_hdr, struct ifaddrs **p_resultList, int p_numLinks) +{ + struct ifaddrmsg *l_info = (struct ifaddrmsg *)NLMSG_DATA(p_hdr); + struct ifaddrs *l_interface = findInterface(l_info->ifa_index, p_resultList, p_numLinks); + + if(l_info->ifa_family == AF_PACKET) + { + return 0; + } + + size_t l_nameSize = 0; + size_t l_addrSize = 0; + + int l_addedNetmask = 0; + + size_t l_rtaSize = NLMSG_PAYLOAD(p_hdr, sizeof(struct ifaddrmsg)); + struct rtattr *l_rta; + for(l_rta = IFA_RTA(l_info); RTA_OK(l_rta, l_rtaSize); l_rta = RTA_NEXT(l_rta, l_rtaSize)) + { + void *l_rtaData = RTA_DATA(l_rta); + size_t l_rtaDataSize = RTA_PAYLOAD(l_rta); + + switch(l_rta->rta_type) + { + case IFA_ADDRESS: + case IFA_LOCAL: + if((l_info->ifa_family == AF_INET || l_info->ifa_family == AF_INET6) && !l_addedNetmask) + { // make room for netmask + l_addrSize += NLMSG_ALIGN(calcAddrLen(l_info->ifa_family, l_rtaDataSize)); + l_addedNetmask = 1; + } + case IFA_BROADCAST: + l_addrSize += NLMSG_ALIGN(calcAddrLen(l_info->ifa_family, l_rtaDataSize)); + break; + case IFA_LABEL: + l_nameSize += NLMSG_ALIGN(l_rtaSize + 1); + break; + default: + break; + } + } + + struct ifaddrs *l_entry = malloc(sizeof(struct ifaddrs) + l_nameSize + l_addrSize); + if (l_entry == NULL) + { + return -1; + } + memset(l_entry, 0, sizeof(struct ifaddrs)); + l_entry->ifa_name = (l_interface ? l_interface->ifa_name : ""); + + char *l_name = ((char *)l_entry) + sizeof(struct ifaddrs); + char *l_addr = l_name + l_nameSize; + + l_entry->ifa_flags = l_info->ifa_flags; + if(l_interface) + { + l_entry->ifa_flags |= l_interface->ifa_flags; + } + + l_rtaSize = NLMSG_PAYLOAD(p_hdr, sizeof(struct ifaddrmsg)); + for(l_rta = IFA_RTA(l_info); RTA_OK(l_rta, l_rtaSize); l_rta = RTA_NEXT(l_rta, l_rtaSize)) + { + void *l_rtaData = RTA_DATA(l_rta); + size_t l_rtaDataSize = RTA_PAYLOAD(l_rta); + switch(l_rta->rta_type) + { + case IFA_ADDRESS: + case IFA_BROADCAST: + case IFA_LOCAL: + { + size_t l_addrLen = calcAddrLen(l_info->ifa_family, l_rtaDataSize); + makeSockaddr(l_info->ifa_family, (struct sockaddr *)l_addr, l_rtaData, l_rtaDataSize); + if(l_info->ifa_family == AF_INET6) + { + if(IN6_IS_ADDR_LINKLOCAL((struct in6_addr *)l_rtaData) || IN6_IS_ADDR_MC_LINKLOCAL((struct in6_addr *)l_rtaData)) + { + ((struct sockaddr_in6 *)l_addr)->sin6_scope_id = l_info->ifa_index; + } + } + + if(l_rta->rta_type == IFA_ADDRESS) + { // apparently in a point-to-point network IFA_ADDRESS contains the dest address and IFA_LOCAL contains the local address + if(l_entry->ifa_addr) + { + l_entry->ifa_dstaddr = (struct sockaddr *)l_addr; + } + else + { + l_entry->ifa_addr = (struct sockaddr *)l_addr; + } + } + else if(l_rta->rta_type == IFA_LOCAL) + { + if(l_entry->ifa_addr) + { + l_entry->ifa_dstaddr = l_entry->ifa_addr; + } + l_entry->ifa_addr = (struct sockaddr *)l_addr; + } + else + { + l_entry->ifa_broadaddr = (struct sockaddr *)l_addr; + } + l_addr += NLMSG_ALIGN(l_addrLen); + break; + } + case IFA_LABEL: + strncpy(l_name, l_rtaData, l_rtaDataSize); + l_name[l_rtaDataSize] = '\0'; + l_entry->ifa_name = l_name; + break; + default: + break; + } + } + + if(l_entry->ifa_addr && (l_entry->ifa_addr->sa_family == AF_INET || l_entry->ifa_addr->sa_family == AF_INET6)) + { + unsigned l_maxPrefix = (l_entry->ifa_addr->sa_family == AF_INET ? 32 : 128); + unsigned l_prefix = (l_info->ifa_prefixlen > l_maxPrefix ? l_maxPrefix : l_info->ifa_prefixlen); + char l_mask[16] = {0}; + unsigned i; + for(i=0; i<(l_prefix/8); ++i) + { + l_mask[i] = 0xff; + } + if(l_prefix % 8) + { + l_mask[i] = 0xff << (8 - (l_prefix % 8)); + } + + makeSockaddr(l_entry->ifa_addr->sa_family, (struct sockaddr *)l_addr, l_mask, l_maxPrefix / 8); + l_entry->ifa_netmask = (struct sockaddr *)l_addr; + } + + addToEnd(p_resultList, l_entry); + return 0; +} + +static int interpretLinks(int p_socket, NetlinkList *p_netlinkList, struct ifaddrs **p_resultList) +{ + int l_numLinks = 0; + pid_t l_pid = getpid(); + for(; p_netlinkList; p_netlinkList = p_netlinkList->m_next) + { + unsigned int l_nlsize = p_netlinkList->m_size; + struct nlmsghdr *l_hdr; + for(l_hdr = p_netlinkList->m_data; NLMSG_OK(l_hdr, l_nlsize); l_hdr = NLMSG_NEXT(l_hdr, l_nlsize)) + { + if((pid_t)l_hdr->nlmsg_pid != l_pid || (int)l_hdr->nlmsg_seq != p_socket) + { + continue; + } + + if(l_hdr->nlmsg_type == NLMSG_DONE) + { + break; + } + + if(l_hdr->nlmsg_type == RTM_NEWLINK) + { + if(interpretLink(l_hdr, p_resultList) == -1) + { + return -1; + } + ++l_numLinks; + } + } + } + return l_numLinks; +} + +static int interpretAddrs(int p_socket, NetlinkList *p_netlinkList, struct ifaddrs **p_resultList, int p_numLinks) +{ + pid_t l_pid = getpid(); + for(; p_netlinkList; p_netlinkList = p_netlinkList->m_next) + { + unsigned int l_nlsize = p_netlinkList->m_size; + struct nlmsghdr *l_hdr; + for(l_hdr = p_netlinkList->m_data; NLMSG_OK(l_hdr, l_nlsize); l_hdr = NLMSG_NEXT(l_hdr, l_nlsize)) + { + if((pid_t)l_hdr->nlmsg_pid != l_pid || (int)l_hdr->nlmsg_seq != p_socket) + { + continue; + } + + if(l_hdr->nlmsg_type == NLMSG_DONE) + { + break; + } + + if(l_hdr->nlmsg_type == RTM_NEWADDR) + { + if (interpretAddr(l_hdr, p_resultList, p_numLinks) == -1) + { + return -1; + } + } + } + } + return 0; +} + +int getifaddrs(struct ifaddrs **ifap) +{ + if(!ifap) + { + return -1; + } + *ifap = NULL; + + int l_socket = netlink_socket(); + if(l_socket < 0) + { + return -1; + } + + NetlinkList *l_linkResults = getResultList(l_socket, RTM_GETLINK); + if(!l_linkResults) + { + close(l_socket); + return -1; + } + + NetlinkList *l_addrResults = getResultList(l_socket, RTM_GETADDR); + if(!l_addrResults) + { + close(l_socket); + freeResultList(l_linkResults); + return -1; + } + + int l_result = 0; + int l_numLinks = interpretLinks(l_socket, l_linkResults, ifap); + if(l_numLinks == -1 || interpretAddrs(l_socket, l_addrResults, ifap, l_numLinks) == -1) + { + l_result = -1; + } + + freeResultList(l_linkResults); + freeResultList(l_addrResults); + close(l_socket); + return l_result; +} + +void freeifaddrs(struct ifaddrs *ifa) +{ + struct ifaddrs *l_cur; + while(ifa) + { + l_cur = ifa; + ifa = ifa->ifa_next; + free(l_cur); + } +} diff --git a/contrib/libs/libc_compat/memfd_create.c b/contrib/libs/libc_compat/memfd_create/memfd_create.c similarity index 100% rename from contrib/libs/libc_compat/memfd_create.c rename to contrib/libs/libc_compat/memfd_create/memfd_create.c diff --git a/contrib/libs/libc_compat/memfd_create/sys/mman.h b/contrib/libs/libc_compat/memfd_create/sys/mman.h new file mode 100644 index 00000000000..d36e9bdbe40 --- /dev/null +++ b/contrib/libs/libc_compat/memfd_create/sys/mman.h @@ -0,0 +1,16 @@ +#pragma once + +#include_next + +#ifdef __cplusplus +extern "C" { +#endif + +#define MFD_CLOEXEC 0x0001U +#define MADV_WIPEONFORK 18 + +int memfd_create(const char *name, unsigned flags); + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/contrib/libs/libc_compat/include/windows/sys/queue.h b/contrib/libs/libc_compat/queue/sys/queue.h similarity index 100% rename from contrib/libs/libc_compat/include/windows/sys/queue.h rename to contrib/libs/libc_compat/queue/sys/queue.h diff --git a/contrib/libs/libc_compat/unistd.h b/contrib/libs/libc_compat/unistd.h new file mode 100644 index 00000000000..c62f5f29cdd --- /dev/null +++ b/contrib/libs/libc_compat/unistd.h @@ -0,0 +1,3 @@ +#include + +#define issetugid() 0 diff --git a/contrib/libs/libc_compat/update.sh b/contrib/libs/libc_compat/update.sh new file mode 100644 index 00000000000..4cee1609048 --- /dev/null +++ b/contrib/libs/libc_compat/update.sh @@ -0,0 +1,60 @@ +#!/bin/sh -e + +unweak() { + sed --in-place --expression 's/DEF_WEAK(.\+);//g' "$1" +} + +get_string_method() { + curl "https://raw.githubusercontent.com/openbsd/src/master/lib/libc/string/$1" --output "$1" && unweak "$1" +} + +fix_tabs() { + sed --in-place --expression 's/\t/ /g' "$1" +} + +fix_decls() { + sed --in-place --expression 's/__BEGIN_DECLS/#ifdef __cplusplus\nextern "C" {\n#endif/g' "$1" + sed --in-place --expression 's/__END_DECLS/#ifdef __cplusplus\n} \/\/ extern "C"\n#endif/g' "$1" +} + +get_string_method "strlcpy.c" +get_string_method "strlcat.c" +get_string_method "strsep.c" +# strcasestr uses strncasecmp, which is platform dependent, so include local string.h +get_string_method "strcasestr.c" && sed --in-place 's/#include /#include "string.h"/g' "strcasestr.c" +get_string_method "memrchr.c" +get_string_method "stpcpy.c" + +mkdir -p include/windows/sys +curl "https://raw.githubusercontent.com/openbsd/src/master/sys/sys/queue.h" --output "include/windows/sys/queue.h" + +mkdir -p include/readpassphrase +curl "https://raw.githubusercontent.com/openbsd/src/master/include/readpassphrase.h" --output "include/readpassphrase/readpassphrase.h" && fix_decls "include/readpassphrase/readpassphrase.h" +curl "https://raw.githubusercontent.com/openbsd/src/master/lib/libc/gen/readpassphrase.c" --output "readpassphrase.c" && unweak "readpassphrase.c" && fix_tabs "readpassphrase.c" + +curl "https://raw.githubusercontent.com/freebsd/freebsd/master/include/glob.h" --output "glob.h" +curl "https://raw.githubusercontent.com/freebsd/freebsd/master/lib/libc/gen/glob.c" --output "glob.c" +curl "https://raw.githubusercontent.com/openbsd/src/master/lib/libc/stdlib/reallocarray.c" --output "reallocarray.c" && unweak "reallocarray.c" +> "collate.h" +> "stdlib.h" +> "unistd.h" + +mkdir -p include/uchar +curl "https://git.musl-libc.org/cgit/musl/plain/include/uchar.h" --output "include/uchar/uchar.h" +# TODO: provide c16rtomb, mbrtoc16, c32rtomb, mbrtoc32 implementations for uchar +# if any code actually needs them + +mkdir -p include/random/sys +curl "https://git.musl-libc.org/cgit/musl/plain/include/sys/random.h" --output "include/random/sys/random.h" +curl "https://git.musl-libc.org/cgit/musl/plain/src/linux/getrandom.c" --output "getrandom.c" +curl "https://git.musl-libc.org/cgit/musl/plain/src/linux/memfd_create.c" --output "memfd_create.c" + +# WARN: do not use github.com/morristech/android-ifaddrs, it is a long-ago abandoned fork +curl "https://raw.githubusercontent.com/oliviertilmans/android-ifaddrs/master/ifaddrs.c" --output "ifaddrs.c" +curl "https://raw.githubusercontent.com/oliviertilmans/android-ifaddrs/master/ifaddrs.h" --output "include/ifaddrs/ifaddrs.h" + +# apply patches if necessary +for patch in patches/*.patch; do + echo "Applying patch from $patch" + patch -p1 < $patch +done diff --git a/contrib/libs/lzmasdk/7zVersion.h b/contrib/libs/lzmasdk/7zVersion.h new file mode 100644 index 00000000000..0074c64be96 --- /dev/null +++ b/contrib/libs/lzmasdk/7zVersion.h @@ -0,0 +1,27 @@ +#define MY_VER_MAJOR 19 +#define MY_VER_MINOR 00 +#define MY_VER_BUILD 0 +#define MY_VERSION_NUMBERS "19.00" +#define MY_VERSION MY_VERSION_NUMBERS + +#ifdef MY_CPU_NAME + #define MY_VERSION_CPU MY_VERSION " (" MY_CPU_NAME ")" +#else + #define MY_VERSION_CPU MY_VERSION +#endif + +#define MY_DATE "2019-02-21" +#undef MY_COPYRIGHT +#undef MY_VERSION_COPYRIGHT_DATE +#define MY_AUTHOR_NAME "Igor Pavlov" +#define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2018 Igor Pavlov" + +#ifdef USE_COPYRIGHT_CR + #define MY_COPYRIGHT MY_COPYRIGHT_CR +#else + #define MY_COPYRIGHT MY_COPYRIGHT_PD +#endif + +#define MY_COPYRIGHT_DATE MY_COPYRIGHT " : " MY_DATE +#define MY_VERSION_COPYRIGHT_DATE MY_VERSION_CPU " : " MY_COPYRIGHT " : " MY_DATE diff --git a/include/ydb-cpp-sdk/client/coordination/coordination.h b/include/ydb-cpp-sdk/client/coordination/coordination.h index 4b46428ca9d..45bda4be922 100644 --- a/include/ydb-cpp-sdk/client/coordination/coordination.h +++ b/include/ydb-cpp-sdk/client/coordination/coordination.h @@ -291,6 +291,9 @@ struct TDescribeSemaphoreSettings { //////////////////////////////////////////////////////////////////////////////// +class TDistributedLock; +struct TDistributedLockSettings; + class TClient { public: TClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); @@ -323,6 +326,7 @@ class TSessionContext; class TSession { friend class TSessionContext; + friend class TDistributedLock; public: TSession() = default; @@ -360,9 +364,13 @@ class TSession { TAsyncResult DeleteSemaphore(const std::string& name, bool force = false); + TDistributedLock CreateDistributedLock(const TDistributedLockSettings& settings); + private: explicit TSession(TSessionContext* context); + std::shared_ptr SubscribeSessionLost(std::function callback); + private: class TImpl; std::shared_ptr Impl_; diff --git a/include/ydb-cpp-sdk/client/coordination/distributed_lock.h b/include/ydb-cpp-sdk/client/coordination/distributed_lock.h new file mode 100644 index 00000000000..56edbdac1f3 --- /dev/null +++ b/include/ydb-cpp-sdk/client/coordination/distributed_lock.h @@ -0,0 +1,42 @@ +#pragma once +#include +#include +namespace NYdb { +namespace NCoordination { + struct TYdbLockException : public TYdbException { + TYdbLockException(const std::string& message) : TYdbException(message) {} + }; + struct TDistributedLockSettings { + using TSelf = TDistributedLockSettings; + FLUENT_SETTING(std::string, Name); + FLUENT_SETTING_DEFAULT(TDuration, Timeout, TDuration::Seconds(5)); + }; + // Distributed exclusive lock backed by a YDB coordination semaphore. + // Satisfies BasicLockable (lock/unlock) for std::lock_guard; not a blocking Lockable. + class TDistributedLock { + public: + TDistributedLock(TSession session, const TDistributedLockSettings& settings); + ~TDistributedLock(); + TDistributedLock(const TDistributedLock&) = delete; + TDistributedLock& operator=(const TDistributedLock&) = delete; + TDistributedLock(TDistributedLock&&) = delete; + TDistributedLock& operator=(TDistributedLock&&) = delete; + // Throws TYdbLockException on session start failure, acquire timeout, transport + // error, or contention timeout (timeout bounds the acquire wait). + void lock(); + // Same as lock() + void Acquire(); + // noexcept. Undefined behavior if called when the lock is not held (same as std::mutex). + void unlock() noexcept; + // Same as unlock() + void Release() noexcept; + // noexcept. Returns false on any failure without throwing. + bool try_lock() noexcept; + // Signals lock loss for the current hold; refreshed on successful acquire — call again after re-lock. + std::stop_token getStopToken() const; + private: + struct TImpl; + std::unique_ptr impl_; + }; +} +} diff --git a/include/ydb-cpp-sdk/client/export/export.h b/include/ydb-cpp-sdk/client/export/export.h index 35cec0b9437..02f73af6097 100644 --- a/include/ydb-cpp-sdk/client/export/export.h +++ b/include/ydb-cpp-sdk/client/export/export.h @@ -4,6 +4,8 @@ #include #include +#include + namespace NYdb::inline V3 { namespace NExport { @@ -32,6 +34,15 @@ struct TEncryptionAlgorithm { static const std::string CHACHA_20_POLY_1305; }; +struct TYdbDumpFormat { +}; + +struct TParquetFormat { + using TSelf = TParquetFormat; + + FLUENT_SETTING_DEFAULT(uint32_t, RowGroupSize, 10000); +}; + /// YT struct TExportToYtSettings : public TOperationRequestSettings { struct TItem { @@ -104,6 +115,9 @@ struct TExportToS3Settings : public TOperationRequestSettings; + FLUENT_SETTING(FormatVariant, Format); + TSelf& SymmetricEncryption(const std::string& algorithm, const std::string& key) { EncryptionAlgorithm_ = algorithm; SymmetricKey_ = key; diff --git a/include/ydb-cpp-sdk/client/iam/common/generic_provider.h b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h index 37d3184e4b6..e7deca3f55e 100644 --- a/include/ydb-cpp-sdk/client/iam/common/generic_provider.h +++ b/include/ydb-cpp-sdk/client/iam/common/generic_provider.h @@ -14,17 +14,50 @@ #include #include +#include #include #include #include +#include namespace NYdb::inline V3 { +using NCredentials::NDetail::TOwningFacilityCredentialsProvider; + constexpr std::chrono::milliseconds BACKOFF_START{50}; constexpr std::chrono::milliseconds BACKOFF_MAX{10000}; constexpr std::chrono::milliseconds PERIODIC_TICK{100}; constexpr std::chrono::milliseconds MINIMUM_REFRESH_INTERVAL{100}; +// Implementation detail for the IAM factory templates below. Symbols in NDetail are not part of +// the public YDB C++ SDK API and may change or be removed without notice. +namespace NIam::NDetail { + +template +std::string MakeClientIdentity( + const char* factoryType, + const TIamEndpoint& params, + const TExtraValues&... extraValues) +{ + TStringBuilder identity; + const auto append = [&identity](const auto& value) { + const std::string serialized = TStringBuilder() << value; + identity << serialized.size() << ':' << serialized; + }; + + append(factoryType); + append(params.Endpoint); + append(params.RefreshPeriod); + append(params.RequestTimeout); + append(params.EnableSsl); + append(params.CaCerts); + (append(extraValues), ...); + + return identity; +} + +} // namespace NIam::NDetail + // This file contains internal generic implementation of IAM credentials providers. // DO NOT USE THIS CLASS DIRECTLY. Use specialized factory methods for specific cases. template @@ -46,17 +79,16 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { std::weak_ptr responseFacility, TCredentialsProviderPtr authTokenProvider) : Rpc_(rpc) - , Ticket_("") , NextTicketUpdate_(SysTimePoint{}) , IamEndpoint_(iamEndpoint) , RequestFiller_(requestFiller) , Context_(std::nullopt) - , LastRequestError_("") , NeedStop_(false) , BackoffTimeout_(BACKOFF_START) , Lock_() , ResponseFacility_(std::move(responseFacility)) , AuthTokenProvider_(authTokenProvider) + , AuthInfo_(NThreading::NewPromise()) { std::shared_ptr creds = nullptr; if (IamEndpoint_.EnableSsl) { @@ -79,53 +111,53 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { void StartPeriodicTask() { auto facility = ResponseFacility_.lock(); if (!facility) { + Fail("IAM-token provider response facility is not available"); return; } std::weak_ptr weakSelf = TGrpcIamCredentialsProvider::TImpl::weak_from_this(); - facility->AddPeriodicTask( - [weakSelf](NYdb::NIssue::TIssues&&, EStatus status) { - auto self = weakSelf.lock(); - if (!self || status != EStatus::SUCCESS) { - return false; - } - return self->OnPeriodicTick(); - }, - PERIODIC_TICK - ); - } - - std::string GetTicket() { - std::lock_guard guard(Lock_); - if (Ticket_.empty()) { - ythrow yexception() << "IAM-token not ready yet. " << LastRequestError_; + try { + facility->AddPeriodicTask( + [weakSelf](NYdb::NIssue::TIssues&&, EStatus status) { + auto self = weakSelf.lock(); + if (!self) { + return false; + } + if (status != EStatus::SUCCESS) { + self->Fail(TStringBuilder() + << "IAM-token provider periodic task failed with status " + << static_cast(status)); + return false; + } + return self->OnPeriodicTick(); + }, + PERIODIC_TICK + ); + } catch (...) { + Fail(TStringBuilder() + << "Failed to start IAM-token provider periodic task: " + << CurrentExceptionMessage()); } - return Ticket_; } - void WaitForToken() { - std::unique_lock guard(Lock_); - TokenReady_.wait_for(guard, - std::chrono::microseconds(2 * IamEndpoint_.RequestTimeout.MicroSeconds()), - [this]() { - return NeedStop_ || !Ticket_.empty(); - } - ); + NThreading::TFuture GetAuthInfoAsync() { + std::lock_guard guard(Lock_); + return AuthInfo_.GetFuture(); } void Stop() { + NThreading::TPromise promise; { std::unique_lock guard(Lock_); - if (NeedStop_) { - return; - } NeedStop_ = true; - TokenReady_.notify_all(); + promise = AuthInfo_; if (Context_.has_value()) { Context_->TryCancel(); } ContextReady_.wait(guard, [this]() { return !Context_.has_value(); }); } + promise.TrySetException(std::make_exception_ptr( + yexception() << "IAM-token provider stopped before token was ready")); Stub_.reset(); Channel_.reset(); } @@ -133,6 +165,19 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { private: using SysDuration = SysClock::duration; + void Fail(std::string error) { + NThreading::TPromise promise; + { + std::lock_guard guard(Lock_); + NeedStop_ = true; + promise = AuthInfo_; + if (Context_) { + Context_->TryCancel(); + } + } + promise.TrySetException(std::make_exception_ptr(yexception() << error)); + } + static SysDuration ToBoundedSysDuration(const TDuration& d) { return std::chrono::duration_cast(TDeadline::SafeDurationCast(d)); } @@ -178,8 +223,11 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } if (auto self = weakSelf.lock()) { - std::lock_guard guard(self->Lock_); - self->ResetContextImpl(); + { + std::lock_guard guard(self->Lock_); + self->ResetContextImpl(); + } + self->Fail("IAM-token provider response facility is not available"); } }; @@ -187,33 +235,36 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { try { RequestFiller_(req); + Rpc_(Stub_.get(), &*Context_, &req, response.get(), std::move(cb)); } catch (...) { - const auto now = SysClock::now(); std::lock_guard guard(Lock_); - LastRequestError_ = TStringBuilder() - << "Last request error was at " << FormatSysTimeUtcIsoMicros(now) - << ". Failed to prepare IAM request: " << CurrentExceptionMessage(); ResetContextImpl(); RescheduleOnFailure(); - return; } - - Rpc_(Stub_.get(), &*Context_, &req, response.get(), std::move(cb)); } - void FillContext(std::unique_lock& guard) { + bool FillContext(std::unique_lock& guard) { std::optional authToken; if (AuthTokenProvider_) { guard.unlock(); try { - authToken = AuthTokenProvider_->GetAuthInfo(); + if (!AuthTokenInfo_.Initialized()) { + AuthTokenInfo_ = AuthTokenProvider_->GetAuthInfoAsync(); + } + if (!AuthTokenInfo_.IsReady()) { + guard.lock(); + return false; + } + authToken = AuthTokenInfo_.GetValue(); + AuthTokenInfo_ = {}; } catch (...) { + AuthTokenInfo_ = {}; guard.lock(); throw; } guard.lock(); if (NeedStop_) { - return; + return false; } } @@ -227,6 +278,7 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { if (authToken) { context.AddMetadata("authorization", "Bearer " + *authToken); } + return true; } void ResetContextImpl() { @@ -242,6 +294,9 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } bool OnPeriodicTick() { + std::optional terminalError; + bool updateTicket = false; + bool authPending = false; { std::unique_lock guard(Lock_); if (NeedStop_) { @@ -250,12 +305,14 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { if (Context_.has_value() || SysClock::now() < NextTicketUpdate_) { return true; } + if (AuthInfo_.GetFuture().IsReady()) { + AuthInfo_ = NThreading::NewPromise(); + } try { - FillContext(guard); + authPending = !FillContext(guard); } catch (...) { - const auto now = SysClock::now(); - LastRequestError_ = TStringBuilder() - << "Last request error was at " << FormatSysTimeUtcIsoMicros(now) + terminalError = TStringBuilder() + << "Last request error was at " << FormatSysTimeUtcIsoMicros(SysClock::now()) << ". Failed to prepare IAM request context: " << CurrentExceptionMessage(); ResetContextImpl(); } @@ -264,36 +321,68 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { return false; } if (!Context_.has_value()) { - RescheduleOnFailure(); - return true; + if (!authPending && !terminalError) { + RescheduleOnFailure(); + } + } else { + updateTicket = true; } } - UpdateTicket(); + if (terminalError) { + Fail(*terminalError); + return false; + } + if (updateTicket) { + UpdateTicket(); + } return true; } void ProcessIamResponse(grpc::Status&& status, TResponse&& result) { - std::lock_guard guard(Lock_); + std::optional token; + std::optional terminalError; + NThreading::TPromise promise; - if (!status.ok()) { - LastRequestError_ = TStringBuilder() - << "Last request error was at " << FormatSysTimeUtcIsoMicros(SysClock::now()) - << ". GrpcStatusCode: " << static_cast(status.error_code()) - << " Message: \"" << status.error_message() - << "\" iam-endpoint: \"" << IamEndpoint_.Endpoint << "\""; + { + std::lock_guard guard(Lock_); - RescheduleOnFailure(); - } else { - LastRequestError_ = ""; - Ticket_ = result.iam_token(); + if (!status.ok()) { + const std::string error = TStringBuilder() + << "Last request error was at " << FormatSysTimeUtcIsoMicros(SysClock::now()) + << ". GrpcStatusCode: " << static_cast(status.error_code()) + << " Message: \"" << status.error_message() + << "\" iam-endpoint: \"" << IamEndpoint_.Endpoint << "\""; + + if (IsRetryable(status.error_code())) { + RescheduleOnFailure(); + } else { + terminalError = error; + } + } else if (result.iam_token().empty()) { + terminalError = "IAM-token service returned an empty token"; + } else { + token = result.iam_token(); + promise = AuthInfo_; + + const SysTimePoint expiresAt = SysClock::from_time_t(result.expires_at().seconds()); + RescheduleOnSuccess(expiresAt); + } - const SysTimePoint expiresAt = SysClock::from_time_t(result.expires_at().seconds()); - RescheduleOnSuccess(expiresAt); + ResetContextImpl(); + } - TokenReady_.notify_all(); + if (token) { + promise.TrySetValue(std::move(*token)); + } else if (terminalError) { + Fail(*terminalError); } + } - ResetContextImpl(); + static bool IsRetryable(grpc::StatusCode code) { + return code == grpc::StatusCode::CANCELLED || code == grpc::StatusCode::UNKNOWN || + code == grpc::StatusCode::DEADLINE_EXCEEDED || code == grpc::StatusCode::RESOURCE_EXHAUSTED || + code == grpc::StatusCode::ABORTED || code == grpc::StatusCode::INTERNAL || + code == grpc::StatusCode::UNAVAILABLE; } void RescheduleOnFailure() { // call with Lock_ @@ -321,19 +410,18 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { std::shared_ptr Stub_; TAsyncRpc Rpc_; - std::string Ticket_; SysTimePoint NextTicketUpdate_; const TIamEndpoint IamEndpoint_; const TRequestFiller RequestFiller_; std::optional Context_; std::condition_variable ContextReady_; - std::condition_variable TokenReady_; - std::string LastRequestError_; bool NeedStop_; std::chrono::milliseconds BackoffTimeout_; std::mutex Lock_; std::weak_ptr ResponseFacility_; TCredentialsProviderPtr AuthTokenProvider_; + NThreading::TFuture AuthTokenInfo_; + NThreading::TPromise AuthInfo_; }; public: @@ -345,7 +433,6 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { : Impl_(std::make_shared(endpoint, requestFiller, rpc, std::move(responseFacility), authTokenProvider)) { Impl_->StartPeriodicTask(); - Impl_->WaitForToken(); } ~TGrpcIamCredentialsProvider() { @@ -353,42 +440,19 @@ class TGrpcIamCredentialsProvider : public ICredentialsProvider { } std::string GetAuthInfo() const override { - return Impl_->GetTicket(); - } - - bool IsValid() const override { - return true; + return GetAuthInfoAsync().GetValueSync(); } -private: - std::shared_ptr Impl_; -}; - -// Adapter that keeps a self-owned ICoreFacility alive for the lifetime of an inner credentials -// provider. Used by deprecated no-arg ICredentialsProviderFactory::CreateProvider() paths where -// the caller hasn't supplied a facility. -class TOwningFacilityCredentialsProvider : public ICredentialsProvider { -public: - TOwningFacilityCredentialsProvider(std::shared_ptr facility, - TCredentialsProviderPtr inner) - : Facility_(std::move(facility)) - , Inner_(std::move(inner)) - {} - - std::string GetAuthInfo() const override { - return Inner_->GetAuthInfo(); + NThreading::TFuture GetAuthInfoAsync() const override { + return Impl_->GetAuthInfoAsync(); } bool IsValid() const override { - return Inner_->IsValid(); + return true; } private: - // Field declaration order matters: Inner_ is destroyed first so that its Stop() can still - // drive the facility's queue (cancel the in-flight gRPC context, drain the response callback), - // and only then is Facility_ destroyed. - std::shared_ptr Facility_; - TCredentialsProviderPtr Inner_; + std::shared_ptr Impl_; }; template @@ -400,7 +464,7 @@ class TIamJwtCredentialsProvider : public TGrpcIamCredentialsProvider cb) { stub->async()->Create(context, request, response, std::move(cb)); - }, std::move(responseFacility), nullptr) {} + }, std::move(responseFacility)) {} }; template @@ -412,7 +476,7 @@ class TIamOAuthCredentialsProvider : public TGrpcIamCredentialsProvider cb) { stub->async()->Create(context, request, response, std::move(cb)); - }, std::move(responseFacility), nullptr) {} + }, std::move(responseFacility)) {} }; template @@ -424,16 +488,32 @@ class TIamJwtCredentialsProviderFactory : public ICredentialsProviderFactory { // that don't have access to an ICoreFacility. Spins up a private TSimpleCoreFacility and ties // its lifetime to the returned provider via TOwningFacilityCredentialsProvider. TCredentialsProviderPtr CreateProvider() const final { - auto facility = CreateSimpleCoreFacility(); - auto inner = std::make_shared>( - Params_, std::weak_ptr(facility)); - return std::make_shared(std::move(facility), std::move(inner)); + return NCredentials::NDetail::GetOrCreateCachedProvider( + GetClientIdentity(), + [this] { + auto facility = CreateSimpleCoreFacility(); + auto inner = std::make_shared>( + Params_, std::weak_ptr(facility)); + return std::make_shared( + std::move(facility), std::move(inner)); + }); } TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const override { return std::make_shared>(Params_, std::move(facility)); } + std::string GetClientIdentity() const override final { + return NIam::NDetail::MakeClientIdentity( + "TIamJwtCredentialsProviderFactory", + Params_, + TService::service_full_name(), + Params_.JwtParams.AccountId, + Params_.JwtParams.KeyId, + Params_.JwtParams.PubKey, + Params_.JwtParams.PrivKey); + } + private: TIamJwtParams Params_; }; @@ -445,16 +525,29 @@ class TIamOAuthCredentialsProviderFactory : public ICredentialsProviderFactory { // Deprecated. Kept for backward compatibility — see comment on TIamJwtCredentialsProviderFactory. TCredentialsProviderPtr CreateProvider() const final { - auto facility = CreateSimpleCoreFacility(); - auto inner = std::make_shared>( - Params_, std::weak_ptr(facility)); - return std::make_shared(std::move(facility), std::move(inner)); + return NCredentials::NDetail::GetOrCreateCachedProvider( + GetClientIdentity(), + [this] { + auto facility = CreateSimpleCoreFacility(); + auto inner = std::make_shared>( + Params_, std::weak_ptr(facility)); + return std::make_shared( + std::move(facility), std::move(inner)); + }); } TCredentialsProviderPtr CreateProvider(std::weak_ptr facility) const override { return std::make_shared>(Params_, std::move(facility)); } + std::string GetClientIdentity() const override final { + return NIam::NDetail::MakeClientIdentity( + "TIamOAuthCredentialsProviderFactory", + Params_, + TService::service_full_name(), + Params_.OAuthToken); + } + private: TIamOAuth Params_; }; diff --git a/include/ydb-cpp-sdk/client/proto/accessor.h b/include/ydb-cpp-sdk/client/proto/accessor.h index b3b9c2cfef8..da3da5707ca 100644 --- a/include/ydb-cpp-sdk/client/proto/accessor.h +++ b/include/ydb-cpp-sdk/client/proto/accessor.h @@ -58,6 +58,9 @@ class TProtoAccessor { static NTable::TIndexDescription FromProto(const Ydb::Table::TableIndex& tableIndex); static NTable::TIndexDescription FromProto(const Ydb::Table::TableIndexDescription& tableIndexDesc); + static NTable::TMultiColumnStatisticsDescription FromProto(const Ydb::Table::TableMultiColumnStatistics& tableMultiColumnStatistics); + static NTable::TMultiColumnStatisticsDescription FromProto(const Ydb::Table::TableMultiColumnStatisticsDescription& tableMultiColumnStatisticsDesc); + static NTable::TChangefeedDescription FromProto(const Ydb::Table::Changefeed& changefeed); static NTable::TChangefeedDescription FromProto(const Ydb::Table::ChangefeedDescription& changefeed); @@ -73,6 +76,8 @@ class TProtoAccessor { static Ydb::Export::ExportToS3Settings::StorageClass GetProto(NExport::TExportToS3Settings::EStorageClass value); static NExport::TExportToS3Settings::EStorageClass FromProto(Ydb::Export::ExportToS3Settings::StorageClass value); static NExport::EExportProgress FromProto(Ydb::Export::ExportProgress::Progress value); + static NExport::TYdbDumpFormat FromProto(const Ydb::Export::YdbDumpFormat& value); + static NExport::TParquetFormat FromProto(const Ydb::Export::ParquetFormat& value); static NImport::EImportProgress FromProto(Ydb::Import::ImportProgress::Progress value); static Ydb::Import::ImportFromS3Settings::IndexPopulationMode GetProto(NImport::EIndexPopulationMode value); static NImport::EIndexPopulationMode FromProto(Ydb::Import::ImportFromS3Settings::IndexPopulationMode value); diff --git a/include/ydb-cpp-sdk/client/query/client.h b/include/ydb-cpp-sdk/client/query/client.h index 3b684dfb53d..b0a87eb71e3 100644 --- a/include/ydb-cpp-sdk/client/query/client.h +++ b/include/ydb-cpp-sdk/client/query/client.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -51,6 +52,10 @@ struct TSessionPoolSettings { // Min number of session in session pool. // Sessions will not be closed by CloseIdleThreshold if the number of sessions less then this limit. FLUENT_SETTING_DEFAULT(uint32_t, MinPoolSize, 10); + + // Create session in the background even after client timeout. + // This is useful for applications with short session timeouts. + FLUENT_SETTING_DEFAULT(bool, UseDeferredSessionCreation, false); }; struct TClientSettings : public TCommonClientSettingsBase { @@ -127,6 +132,8 @@ class TQueryClient { TAsyncCreateSessionResult GetSession(const TCreateSessionSettings& settings = TCreateSessionSettings()); + TAsyncStatus DeleteSession(const std::string& sessionId, const TDeleteSessionSettings& settings = TDeleteSessionSettings()); + //! Returns number of active sessions given via session pool int64_t GetActiveSessionCount() const; @@ -287,19 +294,25 @@ class TExecuteQueryPart : public TStreamPartStatus { const std::optional& GetTransaction() const { return Transaction_; } - TExecuteQueryPart(TStatus&& status, std::optional&& queryStats, std::optional&& tx) + const std::optional& GetCommitTimestamp() const { return CommitTimestamp_; } + + TExecuteQueryPart(TStatus&& status, std::optional&& queryStats, std::optional&& tx, + std::optional&& commitTimestamp = {}) : TStreamPartStatus(std::move(status)) , Stats_(std::move(queryStats)) , Transaction_(std::move(tx)) + , CommitTimestamp_(std::move(commitTimestamp)) {} TExecuteQueryPart(TStatus&& status, TResultSet&& resultSet, int64_t resultSetIndex, - std::optional&& queryStats, std::optional&& tx) + std::optional&& queryStats, std::optional&& tx, + std::optional&& commitTimestamp = {}) : TStreamPartStatus(std::move(status)) , ResultSet_(std::move(resultSet)) , ResultSetIndex_(resultSetIndex) , Stats_(std::move(queryStats)) , Transaction_(std::move(tx)) + , CommitTimestamp_(std::move(commitTimestamp)) {} private: @@ -307,6 +320,7 @@ class TExecuteQueryPart : public TStreamPartStatus { int64_t ResultSetIndex_ = 0; std::optional Stats_; std::optional Transaction_; + std::optional CommitTimestamp_; }; class TExecuteQueryResult : public TStatus { @@ -319,22 +333,27 @@ class TExecuteQueryResult : public TStatus { std::optional GetTransaction() const {return Transaction_; } + const std::optional& GetCommitTimestamp() const { return CommitTimestamp_; } + TExecuteQueryResult(TStatus&& status) : TStatus(std::move(status)) {} TExecuteQueryResult(TStatus&& status, std::vector&& resultSets, - std::optional&& stats, std::optional&& tx) + std::optional&& stats, std::optional&& tx, + std::optional&& commitTimestamp = {}) : TStatus(std::move(status)) , ResultSets_(std::move(resultSets)) , Stats_(std::move(stats)) , Transaction_(std::move(tx)) + , CommitTimestamp_(std::move(commitTimestamp)) {} private: std::vector ResultSets_; std::optional Stats_; std::optional Transaction_; + std::optional CommitTimestamp_; }; } // namespace NYdb::NQuery diff --git a/include/ydb-cpp-sdk/client/query/fwd.h b/include/ydb-cpp-sdk/client/query/fwd.h index 9f88fdc3c00..455cca65edf 100644 --- a/include/ydb-cpp-sdk/client/query/fwd.h +++ b/include/ydb-cpp-sdk/client/query/fwd.h @@ -9,6 +9,7 @@ struct TExecuteQuerySettings; struct TBeginTxSettings; struct TCommitTxSettings; struct TRollbackTxSettings; +struct TDeleteSessionSettings; struct TExecuteScriptSettings; struct TFetchScriptResultsSettings; struct TTxOnlineSettings; diff --git a/include/ydb-cpp-sdk/client/query/query.h b/include/ydb-cpp-sdk/client/query/query.h index 23fc18c5558..fa0c557f038 100644 --- a/include/ydb-cpp-sdk/client/query/query.h +++ b/include/ydb-cpp-sdk/client/query/query.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -122,12 +123,21 @@ struct TExecuteQuerySettings : public TRequestSettings { struct TBeginTxSettings : public TRequestSettings {}; struct TCommitTxSettings : public TRequestSettings {}; struct TRollbackTxSettings : public TRequestSettings {}; +struct TDeleteSessionSettings : public TRequestSettings { + FLUENT_SETTING_OPTIONAL(TRetryOperationSettings, RetrySettings); +}; class TCommitTransactionResult : public TStatus { public: TCommitTransactionResult(TStatus&& status); + TCommitTransactionResult(TStatus&& status, std::optional&& commitTimestamp); + + const std::optional& GetCommitTimestamp() const { return CommitTimestamp_; } + +private: + std::optional CommitTimestamp_; }; using TAsyncBeginTransactionResult = NThreading::TFuture; diff --git a/include/ydb-cpp-sdk/client/scheme/scheme.h b/include/ydb-cpp-sdk/client/scheme/scheme.h index 6ef6941b895..517bf7d2a5f 100644 --- a/include/ydb-cpp-sdk/client/scheme/scheme.h +++ b/include/ydb-cpp-sdk/client/scheme/scheme.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace Ydb { class VirtualTimestamp; @@ -57,25 +58,6 @@ enum class ESchemeEntryType : i32 { Secret = 26, }; -struct TVirtualTimestamp { - uint64_t PlanStep = 0; - uint64_t TxId = 0; - - TVirtualTimestamp() = default; - TVirtualTimestamp(uint64_t planStep, uint64_t txId); - TVirtualTimestamp(const ::Ydb::VirtualTimestamp& proto); - - std::string ToString() const; - void Out(IOutputStream& out) const; - - bool operator<(const TVirtualTimestamp& rhs) const; - bool operator<=(const TVirtualTimestamp& rhs) const; - bool operator>(const TVirtualTimestamp& rhs) const; - bool operator>=(const TVirtualTimestamp& rhs) const; - bool operator==(const TVirtualTimestamp& rhs) const; - bool operator!=(const TVirtualTimestamp& rhs) const; -}; - struct TSchemeEntry { std::string Name; std::string Owner; diff --git a/include/ydb-cpp-sdk/client/table/fwd.h b/include/ydb-cpp-sdk/client/table/fwd.h index 8315bce41e4..11b0cbfcb96 100644 --- a/include/ydb-cpp-sdk/client/table/fwd.h +++ b/include/ydb-cpp-sdk/client/table/fwd.h @@ -13,6 +13,7 @@ struct TPartitionStats; struct TSequenceDescription; class TChangefeedDescription; class TIndexDescription; +class TMultiColumnStatisticsDescription; class TColumnFamilyDescription; class TTableDescription; diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index fa892c3169f..979dd631a92 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -41,6 +41,8 @@ class TtlSettings; class TtlTier; class TableIndex; class TableIndexDescription; +class TableMultiColumnStatistics; +class TableMultiColumnStatisticsDescription; class ValueSinceUnixEpochModeSettings; class EvictionToExternalStorageSettings; class CompactItem; @@ -389,6 +391,7 @@ struct TFulltextIndexSettings { Whitespace, Standard, Keyword, + Alphanumeric, }; struct TAnalyzers { @@ -623,6 +626,23 @@ class TAnalyzeOperation : public TOperation { TMetadata Metadata_; }; +class TSetNotNullOperation : public TOperation { +public: + using TOperation::TOperation; + TSetNotNullOperation(TStatus&& status, Ydb::Operations::Operation&& operation); + + struct TMetadata { + ESetNotNullState State = ESetNotNullState::Unspecified; + float Progress = 0; + std::string Path; + std::vector Columns; + }; + + const TMetadata& Metadata() const; +private: + TMetadata Metadata_; +}; + //////////////////////////////////////////////////////////////////////////////// //! Represents changefeed description @@ -1016,6 +1036,44 @@ enum class EStoreType { Column = 1 }; +enum class EMultiColumnStatisticsType { + Unknown = 0, + CountMinSketch = 1, +}; + +//! Represents multi-column table statistics description +class TMultiColumnStatisticsDescription { + friend class NYdb::TProtoAccessor; + +public: + TMultiColumnStatisticsDescription( + const std::string& name, + const std::vector& columns, + const std::vector& types + ); + + const std::string& GetName() const; + const std::vector& GetColumns() const; + const std::vector& GetTypes() const; + + void SerializeTo(Ydb::Table::TableMultiColumnStatistics& proto) const; + +private: + explicit TMultiColumnStatisticsDescription(const Ydb::Table::TableMultiColumnStatistics& proto); + explicit TMultiColumnStatisticsDescription(const Ydb::Table::TableMultiColumnStatisticsDescription& proto); + + template + static TMultiColumnStatisticsDescription FromProto(const TProto& proto); + +private: + std::string Name_; + std::vector Columns_; + std::vector Types_; +}; + +bool operator==(const TMultiColumnStatisticsDescription& lhs, const TMultiColumnStatisticsDescription& rhs); +bool operator!=(const TMultiColumnStatisticsDescription& lhs, const TMultiColumnStatisticsDescription& rhs); + //! Represents table description class TTableDescription { friend class TTableBuilder; @@ -1031,6 +1089,7 @@ class TTableDescription { std::vector GetColumns() const; std::vector GetTableColumns() const; std::vector GetIndexDescriptions() const; + std::vector GetMultiColumnStatisticsDescriptions() const; std::vector GetChangefeedDescriptions() const; std::optional GetTtlSettings() const; // Deprecated. Use GetTtlSettings() instead @@ -1124,6 +1183,9 @@ class TTableDescription { void AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); void AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); + // multi-column statistics + void AddMultiColumnStatistics(const TMultiColumnStatisticsDescription& statisticsDescription); + void SetTtlSettings(TTtlSettings&& settings); void SetTtlSettings(const TTtlSettings& settings); @@ -1379,6 +1441,9 @@ class TTableBuilder { TTableBuilder& AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); TTableBuilder& AddSecondaryIndex(const std::string& indexName, const std::string& indexColumn); + // multi-column statistics + TTableBuilder& AddMultiColumnStatistics(const TMultiColumnStatisticsDescription& statisticsDescription); + TTableBuilder& SetTtlSettings(TTtlSettings&& settings); TTableBuilder& SetTtlSettings(const TTtlSettings& settings); TTableBuilder& SetTtlSettings(const std::string& columnName, const TDuration& expireAfter = TDuration::Zero()); diff --git a/include/ydb-cpp-sdk/client/table/table_enum.h b/include/ydb-cpp-sdk/client/table/table_enum.h index 6ae6d50e41f..bc195a48022 100644 --- a/include/ydb-cpp-sdk/client/table/table_enum.h +++ b/include/ydb-cpp-sdk/client/table/table_enum.h @@ -52,6 +52,15 @@ enum class ECompactState { Cancelled = 3, }; +enum class ESetNotNullState { + Unspecified = 0, + Preparing = 1, + Validating = 2, + Applying = 3, + Done = 4, + Cancelled = 5, +}; + enum class EAnalyzeState { Unspecified = 0, Enqueued = 1, diff --git a/include/ydb-cpp-sdk/client/test_shard/test_shard.h b/include/ydb-cpp-sdk/client/test_shard/test_shard.h new file mode 100644 index 00000000000..7855f1d5bb6 --- /dev/null +++ b/include/ydb-cpp-sdk/client/test_shard/test_shard.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace NYdb::NTestShardSet { + +struct TCreateTestShardSetSettings : public TOperationRequestSettings {}; + +struct TDeleteTestShardSetSettings : public TOperationRequestSettings {}; + +class TCreateTestShardSetResult : public TStatus { +public: + TCreateTestShardSetResult(TStatus&& status, std::vector tabletIds) + : TStatus(std::move(status)) + , TabletIds_(std::move(tabletIds)) + {} + + const std::vector& GetTabletIds() const { + return TabletIds_; + } + +private: + std::vector TabletIds_; +}; + +using TAsyncCreateTestShardSetResult = NThreading::TFuture; + +class TTestShardSetClient { +public: + explicit TTestShardSetClient(const TDriver& driver, const TCommonClientSettings& settings = {}); + ~TTestShardSetClient(); + + TAsyncCreateTestShardSetResult CreateTestShardSet( + const std::string& path, + const std::vector& channels, + uint32_t count, + const std::string& config, + const TCreateTestShardSetSettings& settings = {}); + + TAsyncStatus DeleteTestShardSet( + const std::string& path, + const TDeleteTestShardSetSettings& settings = {}); + +private: + class TImpl; + std::unique_ptr Impl_; +}; + +} // namespace NYdb::NTestShardSet diff --git a/include/ydb-cpp-sdk/client/topic/codecs.h b/include/ydb-cpp-sdk/client/topic/codecs.h index 00c4e8a77f3..2fc66a8f2b5 100644 --- a/include/ydb-cpp-sdk/client/topic/codecs.h +++ b/include/ydb-cpp-sdk/client/topic/codecs.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -43,6 +44,7 @@ struct TWriteBlockCompression { ECodec Codec = ECodec::RAW; std::vector& Payloads; const std::vector& CreatedAt; + const std::vector>& MessageKeys; TBuffer& Data; ui32& CodecID; bool& Compressed; diff --git a/include/ydb-cpp-sdk/client/topic/read_events.h b/include/ydb-cpp-sdk/client/topic/read_events.h index d9b2369187a..4099fb010be 100644 --- a/include/ydb-cpp-sdk/client/topic/read_events.h +++ b/include/ydb-cpp-sdk/client/topic/read_events.h @@ -6,6 +6,7 @@ #include +#include namespace NYdb::inline V3::NTopic { diff --git a/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h b/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h index 6e4d48dba74..0a6b1d5eeb0 100644 --- a/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h +++ b/include/ydb-cpp-sdk/client/types/core_facility/core_facility.h @@ -19,7 +19,7 @@ class ICoreFacility { // Add task to execute periodicaly // Task should return false to stop execution virtual void AddPeriodicTask(TPeriodicCb&& cb, TDeadline::Duration period) = 0; - // Post task on SDK response executor. + // Post task on SDK response executor, never inline. virtual void PostToResponseQueue(TPostTaskCb&& f) = 0; }; diff --git a/include/ydb-cpp-sdk/client/types/credentials/credentials.h b/include/ydb-cpp-sdk/client/types/credentials/credentials.h index 054db38a46e..f84ca505a13 100644 --- a/include/ydb-cpp-sdk/client/types/credentials/credentials.h +++ b/include/ydb-cpp-sdk/client/types/credentials/credentials.h @@ -2,8 +2,13 @@ #include +#include +#include + +#include #include #include +#include namespace NYdb::inline V3 { @@ -12,16 +17,67 @@ class ICredentialsProvider { virtual ~ICredentialsProvider() = default; virtual std::string GetAuthInfo() const = 0; virtual bool IsValid() const = 0; + virtual NThreading::TFuture GetAuthInfoAsync() const { + try { + return NThreading::MakeFuture(GetAuthInfo()); + } catch (...) { + return NThreading::MakeErrorFuture(std::current_exception()); + } + } }; using TCredentialsProviderPtr = std::shared_ptr; - class ICoreFacility; + +// Implementation detail for SDK credentials factories. Symbols in NCredentials::NDetail are not +// part of the public YDB C++ SDK API and may change or be removed without notice. +namespace NCredentials::NDetail { + +using TCredentialsProviderCreator = std::function; + +class TOwningFacilityCredentialsProvider final : public ICredentialsProvider { +public: + TOwningFacilityCredentialsProvider(std::shared_ptr facility, + TCredentialsProviderPtr inner, + bool forwardAsync = false) + : Facility_(std::move(facility)) + , Inner_(std::move(inner)) + , ForwardAsync_(forwardAsync) + {} + + std::string GetAuthInfo() const override { + return Inner_->GetAuthInfo(); + } + + NThreading::TFuture GetAuthInfoAsync() const override { + return ForwardAsync_ ? Inner_->GetAuthInfoAsync() : ICredentialsProvider::GetAuthInfoAsync(); + } + + bool IsValid() const override { + return Inner_->IsValid(); + } + +private: + // Reverse destruction keeps Facility_ alive while Inner_ stops. + std::shared_ptr Facility_; + TCredentialsProviderPtr Inner_; + const bool ForwardAsync_; +}; + +// Process-wide weak cache for no-argument factory paths whose providers own their facilities. +// Facility-bound providers must not use it: their callbacks belong to the supplied facility. +TCredentialsProviderPtr GetOrCreateCachedProvider( + const std::string& identity, + TCredentialsProviderCreator createProvider); + +} // namespace NCredentials::NDetail + class ICredentialsProviderFactory { public: virtual ~ICredentialsProviderFactory() = default; // deprecated, use CreateProvider(std::weak_ptr facility) instead virtual TCredentialsProviderPtr CreateProvider() const = 0; + // The facility must outlive the returned provider. virtual TCredentialsProviderPtr CreateProvider([[maybe_unused]] std::weak_ptr facility) const { return CreateProvider(); } diff --git a/include/ydb-cpp-sdk/client/types/virtual_timestamp.h b/include/ydb-cpp-sdk/client/types/virtual_timestamp.h new file mode 100644 index 00000000000..e68de28f7c6 --- /dev/null +++ b/include/ydb-cpp-sdk/client/types/virtual_timestamp.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +#include + +#include + +namespace NYdb::inline V3 { +namespace NScheme { + +struct TVirtualTimestamp { + uint64_t PlanStep = 0; + uint64_t TxId = 0; + + TVirtualTimestamp() = default; + TVirtualTimestamp(uint64_t planStep, uint64_t txId); + TVirtualTimestamp(const ::Ydb::VirtualTimestamp& proto); + + std::string ToString() const; + void Out(IOutputStream& out) const; + + bool operator<(const TVirtualTimestamp& rhs) const; + bool operator<=(const TVirtualTimestamp& rhs) const; + bool operator>(const TVirtualTimestamp& rhs) const; + bool operator>=(const TVirtualTimestamp& rhs) const; + bool operator==(const TVirtualTimestamp& rhs) const; + bool operator!=(const TVirtualTimestamp& rhs) const; +}; + +} // namespace NScheme +} // namespace NYdb diff --git a/include/ydb-cpp-sdk/library/operation_id/operation_id.h b/include/ydb-cpp-sdk/library/operation_id/operation_id.h index 57b69eb2672..ec307262690 100644 --- a/include/ydb-cpp-sdk/library/operation_id/operation_id.h +++ b/include/ydb-cpp-sdk/library/operation_id/operation_id.h @@ -31,6 +31,7 @@ class TOperationId { COMPACTION = 13, FULL_BACKUP = 14, ANALYZE = 15, + SET_NOT_NULL = 16, }; struct TData { diff --git a/include/ydb-cpp-sdk/stlfwd.h b/include/ydb-cpp-sdk/stlfwd.h index 77b97233f95..7375a154e7f 100644 --- a/include/ydb-cpp-sdk/stlfwd.h +++ b/include/ydb-cpp-sdk/stlfwd.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -13,3 +14,7 @@ #include #include #include + +#ifdef __cpp_lib_format +#include +#endif diff --git a/library/cpp/CMakeLists.txt b/library/cpp/CMakeLists.txt index b3cfe65ceee..b1f5ee76025 100644 --- a/library/cpp/CMakeLists.txt +++ b/library/cpp/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(case_insensitive_string) add_subdirectory(cgiparam) add_subdirectory(charset) add_subdirectory(colorizer) +add_subdirectory(containers/cow_string) add_subdirectory(containers/disjoint_interval_tree) add_subdirectory(containers/intrusive_rb_tree) add_subdirectory(containers/paged_vector) @@ -18,6 +19,7 @@ add_subdirectory(digest/lower_case) add_subdirectory(digest/md5) add_subdirectory(digest/murmur) add_subdirectory(getopt) +add_subdirectory(html/escape) add_subdirectory(http/fetch) add_subdirectory(http/io) add_subdirectory(http/misc) diff --git a/library/cpp/blockcodecs/codecs.h b/library/cpp/blockcodecs/codecs.h index fd499b54b0d..43a9244465e 100644 --- a/library/cpp/blockcodecs/codecs.h +++ b/library/cpp/blockcodecs/codecs.h @@ -1,3 +1,3 @@ #pragma once -#include +#include // IWYU pragma: export diff --git a/library/cpp/blockcodecs/codecs/zstd/README.md b/library/cpp/blockcodecs/codecs/zstd/README.md new file mode 100644 index 00000000000..98236f837d7 --- /dev/null +++ b/library/cpp/blockcodecs/codecs/zstd/README.md @@ -0,0 +1,40 @@ +Zstd codecs +============= + +This library registers zstd compression codecs as `zstd_1`, ..., `zstd_22`. +Fast levels are also registered as `zstd_fast_1`, ..., `zstd_fast_7`. + +Measured codec performance on every level. Values below are provided just for reference, exact numbers may vary depending on CPU model and type of data being compressed. + +| Codec | Comp. Ratio | Comp. Speed (MBps) | Decomp. Speed (MBps) | +|--------------|-------------|---------------------|----------------------| +| lz4 | 0.5876 | 913 | 4100 | +| zstd_fast_7 | 0.5783 | 1066 | 2887 | +| zstd_fast_6 | 0.5733 | 1050 | 2870 | +| zstd_fast_5 | 0.5528 | 942 | 2594 | +| zstd_fast_4 | 0.5529 | 918 | 2659 | +| zstd_fast_3 | 0.5408 | 885 | 2519 | +| zstd_fast_2 | 0.5132 | 769 | 2374 | +| zstd_fast_1 | 0.5119 | 707 | 2386 | +| zstd_1 | 0.4691 | 690 | 1692 | +| zstd_2 | 0.4083 | 467 | 1496 | +| zstd_3 | 0.3505 | 358 | 1801 | +| zstd_4 | 0.3356 | 310 | 1932 | +| zstd_5 | 0.3175 | 218 | 1832 | +| zstd_6 | 0.3168 | 173 | 1920 | +| zstd_7 | 0.3081 | 149 | 1965 | +| zstd_8 | 0.3077 | 125 | 2005 | +| zstd_9 | 0.272 | 128 | 2179 | +| zstd_10 | 0.2693 | 98.5 | 2228 | +| zstd_11 | 0.2684 | 78.6 | 2185 | +| zstd_12 | 0.2682 | 71.1 | 2231 | +| zstd_13 | 0.2687 | 27.8 | 2102 | +| zstd_14 | 0.2676 | 24 | 2024 | +| zstd_15 | 0.2663 | 18.8 | 2225 | +| zstd_16 | 0.257 | 15.2 | 2093 | +| zstd_17 | 0.2521 | 12.3 | 2072 | +| zstd_18 | 0.241 | 9.68 | 1696 | +| zstd_19 | 0.2395 | 7.97 | 1709 | +| zstd_20 | 0.2337 | 6.26 | 1520 | +| zstd_21 | 0.2255 | 5.22 | 1442 | +| zstd_22 | 0.2037 | 3.88 | 1490 | diff --git a/library/cpp/blockcodecs/codecs/zstd/zstd.cpp b/library/cpp/blockcodecs/codecs/zstd/zstd.cpp index bcbc4005840..02133c9c04d 100644 --- a/library/cpp/blockcodecs/codecs/zstd/zstd.cpp +++ b/library/cpp/blockcodecs/codecs/zstd/zstd.cpp @@ -5,36 +5,38 @@ #define ZSTD_STATIC_LINKING_ONLY #include +#include + using namespace NBlockCodecs; namespace { struct TZStd08Codec: public TAddLengthCodec { - inline TZStd08Codec(unsigned level) + TZStd08Codec(int level, TString name) : Level(level) - , MyName(TStringBuf("zstd08_") + ToString(Level)) + , MyName(std::move(name)) { } - static inline size_t CheckError(size_t ret, const char* what) { - if (ZSTD_isError(ret)) { + static size_t CheckError(size_t ret, const char* what) { + if (Y_UNLIKELY(ZSTD_isError(ret))) { ythrow yexception() << what << TStringBuf(" zstd error: ") << ZSTD_getErrorName(ret); } return ret; } - static inline size_t DoMaxCompressedLength(size_t l) noexcept { + static size_t DoMaxCompressedLength(size_t l) noexcept { return ZSTD_compressBound(l); } - inline size_t DoCompress(const TData& in, void* out) const { + size_t DoCompress(const TData& in, void* out) const { return CheckError(ZSTD_compress(out, DoMaxCompressedLength(in.size()), in.data(), in.size(), Level), "compress"); } - inline void DoDecompress(const TData& in, void* out, size_t dsize) const { + static void DoDecompress(const TData& in, void* out, size_t dsize) { const size_t res = CheckError(ZSTD_decompress(out, dsize, in.data(), in.size()), "decompress"); - if (res != dsize) { + if (Y_UNLIKELY(res != dsize)) { ythrow TDecompressError(dsize, res); } } @@ -43,15 +45,20 @@ namespace { return MyName; } - const unsigned Level; + const int Level; const TString MyName; }; struct TZStd08Registrar { TZStd08Registrar() { for (int i = 1; i <= ZSTD_maxCLevel(); ++i) { - RegisterCodec(MakeHolder(i)); - RegisterAlias("zstd_" + ToString(i), "zstd08_" + ToString(i)); + const TString name = "zstd08_"sv + ToString(i); + RegisterCodec(MakeHolder(i, name)); + RegisterAlias("zstd_"sv + ToString(i), name); + } + + for (int i = 1; i <= 7; ++i) { + RegisterCodec(MakeHolder(-i, "zstd_fast_"sv + ToString(i))); } } }; diff --git a/library/cpp/blockcodecs/codecs_ut.cpp b/library/cpp/blockcodecs/codecs_ut.cpp index bfe5a236909..e167816680e 100644 --- a/library/cpp/blockcodecs/codecs_ut.cpp +++ b/library/cpp/blockcodecs/codecs_ut.cpp @@ -320,7 +320,8 @@ Y_UNIT_TEST_SUITE(TBlockCodecsTest) { "zstd08_1,zstd08_10,zstd08_11,zstd08_12,zstd08_13,zstd08_14,zstd08_15,zstd08_16,zstd08_17,zstd08_18," "zstd08_19,zstd08_2,zstd08_20,zstd08_21,zstd08_22,zstd08_3,zstd08_4,zstd08_5,zstd08_6,zstd08_7,zstd08_8," "zstd08_9,zstd_1,zstd_10,zstd_11,zstd_12,zstd_13,zstd_14,zstd_15,zstd_16,zstd_17,zstd_18,zstd_19,zstd_2," - "zstd_20,zstd_21,zstd_22,zstd_3,zstd_4,zstd_5,zstd_6,zstd_7,zstd_8,zstd_9"; + "zstd_20,zstd_21,zstd_22,zstd_3,zstd_4,zstd_5,zstd_6,zstd_7,zstd_8,zstd_9," + "zstd_fast_1,zstd_fast_2,zstd_fast_3,zstd_fast_4,zstd_fast_5,zstd_fast_6,zstd_fast_7"; UNIT_ASSERT_VALUES_EQUAL(ALL_CODECS, JoinSeq(",", ListAllCodecs())); } diff --git a/library/cpp/blockcodecs/core/codecs.h b/library/cpp/blockcodecs/core/codecs.h index 9c93c002748..61efc04744a 100644 --- a/library/cpp/blockcodecs/core/codecs.h +++ b/library/cpp/blockcodecs/core/codecs.h @@ -25,8 +25,15 @@ namespace NBlockCodecs { : TStringBuf((const char*)t.Data(), t.Size()) { } + }; + template <> + inline TData::TData(const TString& t) + : TStringBuf((const char*)t.data(), t.size()) + { + } + struct TCodecError: public yexception { }; diff --git a/library/cpp/blockcodecs/fuzz/main.cpp b/library/cpp/blockcodecs/fuzz/main.cpp index a89fee7ab73..be4d684db2d 100644 --- a/library/cpp/blockcodecs/fuzz/main.cpp +++ b/library/cpp/blockcodecs/fuzz/main.cpp @@ -1,5 +1,4 @@ #include -#include #include #include diff --git a/library/cpp/cache/cache.h b/library/cpp/cache/cache.h index 9fe98d53666..25906d1a024 100644 --- a/library/cpp/cache/cache.h +++ b/library/cpp/cache/cache.h @@ -465,7 +465,7 @@ class TLWList { size_t MaxSize; }; -template > +template > class TCache { typedef typename TListType::TItem TItem; typedef typename TItem::THash THash; @@ -708,7 +708,7 @@ struct TNoopDelete { } }; -template , typename TAllocator = std::allocator> +template , typename TAllocator = std::allocator::TItem>> class TLRUCache: public TCache, TDeleter, TAllocator> { using TListType = TLRUList; typedef TCache TBase; @@ -735,7 +735,7 @@ class TLRUCache: public TCache, class TSizeProvider = TUniformSizeProvider> +template , class TSizeProvider = TUniformSizeProvider> class TLFUCache: public TCache, TDeleter, TAllocator> { typedef TCache, TDeleter, TAllocator> TBase; using TListType = TLFUList; @@ -760,7 +760,7 @@ class TLFUCache: public TCache> +template ::TItem>> class TLWCache: public TCache, TDeleter, TAllocator> { typedef TCache, TDeleter, TAllocator> TBase; using TListType = TLWList; diff --git a/library/cpp/cache/thread_safe_cache.h b/library/cpp/cache/thread_safe_cache.h index e77d1a45fd8..82b68062777 100644 --- a/library/cpp/cache/thread_safe_cache.h +++ b/library/cpp/cache/thread_safe_cache.h @@ -3,6 +3,7 @@ #include "cache.h" #include +#include #include namespace NPrivate { @@ -51,12 +52,20 @@ namespace NPrivate { const TPtr GetOrNull(TArgs... args) { Key key = Callbacks.GetKey(args...); - TReadGuard r(Mutex); - auto iter = Cache.Find(key); - if (iter == Cache.End()) { - return nullptr; + switch (GettersPromotionPolicy) { + case EGettersPromotionPolicy::Promoted: { + TWriteGuard r(Mutex); + if (auto iter = Cache.Find(key); iter != Cache.End()) + return iter.Value(); + } + break; + case EGettersPromotionPolicy::Unpromoted: { + TReadGuard r(Mutex); + if (auto iter = Cache.Find(key); iter != Cache.End()) + return iter.Value(); + } } - return iter.Value(); + return nullptr; } const TPtr Get(TArgs... args) const { diff --git a/library/cpp/cache/ut/cache_ut.cpp b/library/cpp/cache/ut/cache_ut.cpp index 16f29b29d1f..c4c829cdb7d 100644 --- a/library/cpp/cache/ut/cache_ut.cpp +++ b/library/cpp/cache/ut/cache_ut.cpp @@ -2,6 +2,10 @@ #include #include +#include +#include +#include + struct TStrokaWeighter { static size_t Weight(const TString& s) { return s.size(); @@ -536,6 +540,53 @@ Y_UNIT_TEST_SUITE(TThreadSafeCacheTest) { } } +Y_UNIT_TEST_SUITE(TThreadSafeLRUCacheMultiThreadTest) { + typedef TThreadSafeLRUCache TCache; + + class TSimpleCallbacks: public TCache::ICallbacks { + public: + TKey GetKey(ui32 i) const override { + return i; + } + TValue* CreateObject(ui32 i) const override { + Y_UNUSED(i); + return nullptr; + } + }; + + Y_UNIT_TEST(GetOrNullMultiThreadTest) { + const size_t poolSize = 8; + const size_t passCnt = 128; + const size_t tasksCnt = 128; + + TRWMutex lock; + TThreadPool pool; + TSimpleCallbacks callbacks; + TCache cache(callbacks, poolSize); + + for (size_t i = 0; i < poolSize; ++i) { + cache.Insert(i, MakeAtomicShared(ToString(i))); + } + + pool.Start(poolSize); + { + TWriteGuard wGruard(lock); + for (size_t i = 0; i < tasksCnt; ++i) { + pool.SafeAddFunc([&lock, &cache]() { + TReadGuard rGuard(lock); + for (size_t j = 0; j < passCnt; ++j) { + UNIT_ASSERT(cache.GetOrNull(RandomNumber(poolSize)) != nullptr); + } + }); + } + } // start race + pool.Stop(); + for (size_t i = 0; i < cache.Size(); ++i) { + UNIT_ASSERT(cache.GetOrNull(i) != nullptr); + } + } +} + Y_UNIT_TEST_SUITE(TThreadSafeCacheUnsafeTest) { typedef TThreadSafeCache TCache; diff --git a/library/cpp/cgiparam/cgiparam.cpp b/library/cpp/cgiparam/cgiparam.cpp index e2c9c0dbe55..503213719e5 100644 --- a/library/cpp/cgiparam/cgiparam.cpp +++ b/library/cpp/cgiparam/cgiparam.cpp @@ -11,12 +11,22 @@ TCgiParameters::TCgiParameters(std::initializer_list } } -const TString& TCgiParameters::Get(const TStringBuf name, size_t numOfValue) const noexcept { +const TString& TCgiParameters::Get(const TStringBuf name, size_t numOfValue) const noexcept Y_LIFETIME_BOUND { const auto it = Find(name, numOfValue); return end() == it ? Default() : it->second; } +const TString& TCgiParameters::GetLast(const TStringBuf name) const noexcept { + if (auto it = this->upper_bound(name); it != this->begin()) { + --it; + if (it->first == name) { + return it->second; + } + } + return Default(); +} + bool TCgiParameters::Erase(const TStringBuf name, size_t pos) { const auto pair = equal_range(name); @@ -221,7 +231,7 @@ TString TCgiParameters::QuotedPrint(const char* safe) const { return res; } -TCgiParameters::const_iterator TCgiParameters::Find(const TStringBuf name, size_t pos) const noexcept { +TCgiParameters::const_iterator TCgiParameters::Find(const TStringBuf name, size_t pos) const noexcept Y_LIFETIME_BOUND { const auto pair = equal_range(name); for (auto it = pair.first; it != pair.second; ++it, --pos) { @@ -265,7 +275,7 @@ TQuickCgiParam::TQuickCgiParam(const TStringBuf cgiParamStr) { } } -const TStringBuf& TQuickCgiParam::Get(const TStringBuf name, size_t pos) const noexcept { +TStringBuf TQuickCgiParam::Get(const TStringBuf name, size_t pos) const noexcept Y_LIFETIME_BOUND { const auto pair = equal_range(name); for (auto it = pair.first; it != pair.second; ++it, --pos) { @@ -274,7 +284,7 @@ const TStringBuf& TQuickCgiParam::Get(const TStringBuf name, size_t pos) const n } } - return Default(); + return TStringBuf{}; } bool TQuickCgiParam::Has(const TStringBuf name, const TStringBuf value) const noexcept { diff --git a/library/cpp/cgiparam/cgiparam.h b/library/cpp/cgiparam/cgiparam.h index cbb212f6f06..104e3f401a8 100644 --- a/library/cpp/cgiparam/cgiparam.h +++ b/library/cpp/cgiparam/cgiparam.h @@ -61,7 +61,7 @@ class TCgiParameters: public TMultiMap { } Y_PURE_FUNCTION - const_iterator Find(const TStringBuf name, size_t numOfValue = 0) const noexcept; + const_iterator Find(const TStringBuf name, size_t numOfValue = 0) const noexcept Y_LIFETIME_BOUND; Y_PURE_FUNCTION bool Has(const TStringBuf name, const TStringBuf value) const noexcept; @@ -76,23 +76,32 @@ class TCgiParameters: public TMultiMap { * @note The returned value is CGI-unescaped. */ Y_PURE_FUNCTION - const TString& Get(const TStringBuf name, size_t numOfValue = 0) const noexcept; + const TString& Get(const TStringBuf name, size_t numOfValue = 0) const noexcept Y_LIFETIME_BOUND; + + /// Returns the last value by name + /** + * @note The returned value is CGI-unescaped. + */ + Y_PURE_FUNCTION + const TString& GetLast(const TStringBuf name) const noexcept Y_LIFETIME_BOUND; void InsertEscaped(const TStringBuf name, const TStringBuf value); #if !defined(__GLIBCXX__) template - inline void InsertUnescaped(TName&& name, TValue&& value) { + inline TCgiParameters& InsertUnescaped(TName&& name, TValue&& value) { // TStringBuf use as TName or TValue is C++17 actually. // There is no pair constructor available in C++14 when required type // is not implicitly constructible from given type. // But libc++ pair allows this with C++14. emplace(std::forward(name), std::forward(value)); + return *this; } #else template - inline void InsertUnescaped(TName&& name, TValue&& value) { + inline TCgiParameters& InsertUnescaped(TName&& name, TValue&& value) { emplace(TString(name), TString(value)); + return *this; } #endif @@ -116,24 +125,24 @@ class TCgiParameters: public TMultiMap { bool Erase(const TStringBuf name, const TStringBuf val); bool ErasePattern(const TStringBuf name, const TStringBuf pat); - inline const char* FormField(const TStringBuf name, size_t numOfValue = 0) const { + inline const char* FormField(const TStringBuf name, size_t numOfValue = 0) const Y_LIFETIME_BOUND { const_iterator it = Find(name, numOfValue); if (it == end()) { return nullptr; } - return it->second.data(); + return it->second.c_str(); } - inline TStringBuf FormFieldBuf(const TStringBuf name, size_t numOfValue = 0) const { + inline TStringBuf FormFieldBuf(const TStringBuf name, size_t numOfValue = 0) const Y_LIFETIME_BOUND { const_iterator it = Find(name, numOfValue); if (it == end()) { - return nullptr; + return TStringBuf{}; } - return it->second.data(); + return it->second; } }; @@ -181,7 +190,7 @@ class TQuickCgiParam: public TMultiMap { } Y_PURE_FUNCTION - const TStringBuf& Get(const TStringBuf name, size_t numOfValue = 0) const noexcept; + TStringBuf Get(const TStringBuf name, size_t numOfValue = 0) const noexcept Y_LIFETIME_BOUND; private: TString UnescapeBuf; diff --git a/library/cpp/cgiparam/cgiparam_ut.cpp b/library/cpp/cgiparam/cgiparam_ut.cpp index de6d23882dc..c82f0344d2a 100644 --- a/library/cpp/cgiparam/cgiparam_ut.cpp +++ b/library/cpp/cgiparam/cgiparam_ut.cpp @@ -210,8 +210,10 @@ Y_UNIT_TEST_SUITE(TCgiParametersTest) { UNIT_ASSERT_VALUES_EQUAL(c.NumOfValues("b"), 1u); UNIT_ASSERT_VALUES_EQUAL(c.Get("b"), "b1"); + UNIT_ASSERT_VALUES_EQUAL(c.GetLast("b"), "b1"); UNIT_ASSERT_VALUES_EQUAL(c.Get("a", 0), "a1"); UNIT_ASSERT_VALUES_EQUAL(c.Get("a", 1), "a2"); + UNIT_ASSERT_VALUES_EQUAL(c.GetLast("a"), "a2"); UNIT_ASSERT_VALUES_EQUAL(c.Print(), "a=a1&a=a2&b=b1"); } diff --git a/library/cpp/charset/wide_ut.cpp b/library/cpp/charset/wide_ut.cpp index 93567161bab..4f5a666cb8e 100644 --- a/library/cpp/charset/wide_ut.cpp +++ b/library/cpp/charset/wide_ut.cpp @@ -10,6 +10,8 @@ #include +extern const int TStringUseCow; + namespace { //! three UTF8 encoded russian letters (A, B, V) const char yandexCyrillicAlphabet[] = @@ -264,10 +266,12 @@ void TConversionTest::TestRecodeIntoString() { TUtf16String copy = sUnicode; // increase ref-counter wres = NDetail::Recode(UTF8Text, sUnicode, CODES_UTF8); UNIT_ASSERT(sUnicode == UnicodeText); // same content + if (TStringUseCow) { #ifndef TSTRING_IS_STD_STRING - UNIT_ASSERT(sUnicode.data() != wdata); // re-allocated (shared buffer supplied) - UNIT_ASSERT(sUnicode.data() == wres.data()); // same buffer + UNIT_ASSERT(sUnicode.data() != wdata); // re-allocated (shared buffer supplied) + UNIT_ASSERT(sUnicode.data() == wres.data()); // same buffer #endif + } UNIT_ASSERT(sUnicode.size() == wres.size()); // same content } diff --git a/library/cpp/colorizer/ut/colorizer_ut.cpp b/library/cpp/colorizer/ut/colorizer_ut.cpp index 20341440aff..2512651861e 100644 --- a/library/cpp/colorizer/ut/colorizer_ut.cpp +++ b/library/cpp/colorizer/ut/colorizer_ut.cpp @@ -12,7 +12,7 @@ Y_UNIT_TEST_SUITE(ColorizerTest) { UNIT_ASSERT_STRINGS_EQUAL(EscapeC(colors.BlueColor()), "\\x1B[22;34m"); UNIT_ASSERT_STRINGS_EQUAL(EscapeC(colors.ForeBlue()), "\\x1B[34m"); colors.Disable(); - UNIT_ASSERT(colors.BlueColor().Empty()); + UNIT_ASSERT(colors.BlueColor().empty()); } Y_UNIT_TEST(ResettingTest) { diff --git a/library/cpp/containers/cow_string/CMakeLists.txt b/library/cpp/containers/cow_string/CMakeLists.txt new file mode 100644 index 00000000000..64a93714373 --- /dev/null +++ b/library/cpp/containers/cow_string/CMakeLists.txt @@ -0,0 +1,30 @@ +if (YDB_SDK_TESTS) + add_ydb_test(NAME containers-cow_string-medium-ut + SOURCES + ut_medium/cow_string_medium_ut.cpp + LINK_LIBRARIES + containers-cow_string + cpp-testing-unittest_main + LABELS + medium + ) + target_include_directories(containers-cow_string-medium-ut PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +endif() + +_ydb_sdk_add_library(containers-cow_string) + +target_link_libraries(containers-cow_string + PUBLIC + yutil +) + +target_sources(containers-cow_string + PRIVATE + cow_string.cpp + output.cpp + reverse.cpp + subst.cpp + ysaveload.cpp +) + +_ydb_sdk_install_targets(TARGETS containers-cow_string) diff --git a/library/cpp/containers/cow_string/README.md b/library/cpp/containers/cow_string/README.md new file mode 100644 index 00000000000..d6aea7c734c --- /dev/null +++ b/library/cpp/containers/cow_string/README.md @@ -0,0 +1,9 @@ +## Copy-on-Write based string implementation + +Drop in replacement for TSring in the code which deeply relies on COW-semantic. + + * `#include ` main header of the library with the COW-string class itself + * `#include ` in-place strings reverse implementation + * `#include ` comparator/hashers/... template specialization allowing to use TCowString in tree-based or hash-based sets/maps. + * `#include ` TCowString implementation of the substitution function provided for TString in ``. + * `#include ` TCowString support of the `` serialization/deserialization. diff --git a/library/cpp/containers/cow_string/cow_string.cpp b/library/cpp/containers/cow_string/cow_string.cpp new file mode 100644 index 00000000000..87dc0ad99ec --- /dev/null +++ b/library/cpp/containers/cow_string/cow_string.cpp @@ -0,0 +1,280 @@ +#include "cow_string.h" + +#include +#include +#include +#include + +#include + +template +static bool ModifySequence(TCharType*& p, const TCharType* const pe, F&& f) { + while (p != pe) { + const auto symbol = ReadSymbol(p, pe); + const auto modified = f(symbol); + if (symbol != modified) { + if (stopOnFirstModification) { + return true; + } + + WriteSymbol(modified, p); // also moves `p` forward + } else { + p = SkipSymbol(p, pe); + } + } + + return false; +} + +template +static bool ModifySequence(const TCharType*& p, const TCharType* const pe, TCharType*& out, F&& f) { + while (p != pe) { + const auto symbol = stopOnFirstModification ? ReadSymbol(p, pe) : ReadSymbolAndAdvance(p, pe); + const auto modified = f(symbol); + + if (stopOnFirstModification) { + if (symbol != modified) { + return true; + } + + p = SkipSymbol(p, pe); + } + + WriteSymbol(modified, out); + } + + return false; +} + +template +static void DetachAndFixPointers(TStringType& text, typename TStringType::value_type*& p, const typename TStringType::value_type*& pe) { + const auto pos = p - text.data(); + const auto count = pe - p; + p = text.Detach() + pos; + pe = p + count; +} + +template +static bool ModifyStringSymbolwise(TStringType& text, size_t pos, size_t count, F&& f) { + // TODO(yazevnul): this is done for consistency with `TUtf16String::to_lower` and friends + // at r2914050, maybe worth replacing them with asserts. Also see the same code in `ToTitle`. + pos = pos < text.size() ? pos : text.size(); + count = count < text.size() - pos ? count : text.size() - pos; + + // TUtf16String is refcounted and it's `data` method return pointer to the constant memory. + // To simplify the code we do a `const_cast`, though first write to the memory will be done only + // after we call `Detach()` and get pointer to a writable piece of memory. + auto* p = const_cast(text.data() + pos); + const auto* pe = text.data() + pos + count; + + if (ModifySequence(p, pe, f)) { + DetachAndFixPointers(text, p, pe); + ModifySequence(p, pe, f); + return true; + } + + return false; +} + +std::ostream& operator<<(std::ostream& os, const TCowString& s) { + return os.write(s.data(), s.size()); +} + +std::istream& operator>>(std::istream& is, TCowString& s) { + return is >> s.MutRef(); +} + +template <> +bool TBasicCowString>::to_lower(size_t pos, size_t n) { + return Transform([](size_t, char c) { return AsciiToLower(c); }, pos, n); +} + +template <> +bool TBasicCowString>::to_upper(size_t pos, size_t n) { + return Transform([](size_t, char c) { return AsciiToUpper(c); }, pos, n); +} + +template <> +bool TBasicCowString>::to_title(size_t pos, size_t n) { + if (n == 0) { + return false; + } + bool changed = to_upper(pos, 1); + return to_lower(pos + 1, n - 1) || changed; +} + +template <> +TUtf16CowString& +TBasicCowString>::AppendAscii(const ::TStringBuf& s) { + ReserveAndResize(size() + s.size()); + + auto dst = begin() + size() - s.size(); + + for (const char* src = s.data(); dst != end(); ++dst, ++src) { + *dst = static_cast(*src); + } + + return *this; +} + +template <> +TUtf16CowString& +TBasicCowString>::AppendUtf8(const ::TStringBuf& s) { + size_t oldSize = size(); + ReserveAndResize(size() + s.size() * 4); + size_t written = 0; + size_t pos = UTF8ToWideImpl(s.data(), s.size(), begin() + oldSize, written); + if (pos != s.size()) { + ythrow yexception() << "failed to decode UTF-8 string at pos " << pos << ::NDetail::InStringMsg(s.data(), s.size()); + } + resize(oldSize + written); + + return *this; +} + +template <> +bool TBasicCowString>::to_lower(size_t pos, size_t n) { + const auto f = [](const wchar32 s) { return ToLower(s); }; + return ModifyStringSymbolwise(*this, pos, n, f); +} + +template <> +bool TBasicCowString>::to_upper(size_t pos, size_t n) { + const auto f = [](const wchar32 s) { return ToUpper(s); }; + return ModifyStringSymbolwise(*this, pos, n, f); +} + +template <> +bool TBasicCowString>::to_title(size_t pos, size_t nn) { + if (!*this) { + return false; + } + + pos = pos < this->size() ? pos : this->size(); + nn = nn < this->size() - pos ? nn : this->size() - pos; + + const auto toLower = [](const wchar32 s) { return ToLower(s); }; + + auto* p = const_cast(this->data() + pos); + const auto* pe = this->data() + pos + nn; + + const auto firstSymbol = ReadSymbol(p, pe); + if (firstSymbol == ToTitle(firstSymbol)) { + p = SkipSymbol(p, pe); + if (ModifySequence(p, pe, toLower)) { + DetachAndFixPointers(*this, p, pe); + ModifySequence(p, pe, toLower); + return true; + } + } else { + DetachAndFixPointers(*this, p, pe); + WriteSymbol(ToTitle(ReadSymbol(p, pe)), p); // also moves `p` forward + ModifySequence(p, pe, toLower); + return true; + } + + return false; +} + +template <> +TUtf32CowString& +TBasicCowString>::AppendAscii(const ::TStringBuf& s) { + ReserveAndResize(size() + s.size()); + + auto dst = begin() + size() - s.size(); + + for (const char* src = s.data(); dst != end(); ++dst, ++src) { + *dst = static_cast(*src); + } + + return *this; +} + +template <> +TBasicCowString>& +TBasicCowString>::AppendUtf16(const ::TWtringBuf& s) { + const size_t oldSize = size(); + ReserveAndResize(size() + WideToUTF8BufferSize(s.size())); + + size_t written = 0; + WideToUTF8(s.data(), s.size(), begin() + oldSize, written); + + resize(oldSize + written); + + return *this; +} + +template <> +TUtf32CowString& +TBasicCowString>::AppendUtf8(const ::TStringBuf& s) { + size_t oldSize = size(); + ReserveAndResize(size() + s.size() * 4); + size_t written = 0; + size_t pos = UTF8ToWideImpl(s.data(), s.size(), begin() + oldSize, written); + if (pos != s.size()) { + ythrow yexception() << "failed to decode UTF-8 string at pos " << pos << ::NDetail::InStringMsg(s.data(), s.size()); + } + resize(oldSize + written); + + return *this; +} + +template <> +TUtf32CowString& +TBasicCowString>::AppendUtf16(const ::TWtringBuf& s) { + size_t oldSize = size(); + ReserveAndResize(size() + s.size() * 2); + + wchar32* oldEnd = begin() + oldSize; + wchar32* end = oldEnd; + NDetail::UTF16ToUTF32ImplScalar(s.data(), s.data() + s.size(), end); + size_t written = end - oldEnd; + + resize(oldSize + written); + + return *this; +} + +template <> +bool TBasicCowString>::to_lower(size_t pos, size_t n) { + const auto f = [](const wchar32 s) { return ToLower(s); }; + return ModifyStringSymbolwise(*this, pos, n, f); +} + +template <> +bool TBasicCowString>::to_upper(size_t pos, size_t n) { + const auto f = [](const wchar32 s) { return ToUpper(s); }; + return ModifyStringSymbolwise(*this, pos, n, f); +} + +template <> +bool TBasicCowString>::to_title(size_t pos, size_t n) { + if (!*this) { + return false; + } + + pos = pos < this->size() ? pos : this->size(); + n = n < this->size() - pos ? n : this->size() - pos; + + const auto toLower = [](const wchar32 s) { return ToLower(s); }; + + auto* p = const_cast(this->data() + pos); + const auto* pe = this->data() + pos + n; + + const auto firstSymbol = *p; + if (firstSymbol == ToTitle(firstSymbol)) { + p += 1; + if (ModifySequence(p, pe, toLower)) { + DetachAndFixPointers(*this, p, pe); + ModifySequence(p, pe, toLower); + return true; + } + } else { + DetachAndFixPointers(*this, p, pe); + WriteSymbol(ToTitle(ReadSymbol(p, pe)), p); // also moves `p` forward + ModifySequence(p, pe, toLower); + return true; + } + + return false; +} diff --git a/library/cpp/containers/cow_string/cow_string.h b/library/cpp/containers/cow_string/cow_string.h new file mode 100644 index 00000000000..34dcff81e8a --- /dev/null +++ b/library/cpp/containers/cow_string/cow_string.h @@ -0,0 +1,1016 @@ +#pragma once + +#include + +template > +class Y_EMPTY_BASES TBasicCowString: public TStringBase, TCharType, TTraits>, + public TStdStringCompatibilityBase, TCharType, TTraits> { +public: + // TODO: Move to private section + using TBase = TStringBase; + using TStringType = std::basic_string; + using TStdStr = TStdString; + using TStorage = TIntrusivePtr>; + using reference = TBasicCharRef; + using char_type = TCharType; // TODO: DROP + using value_type = TCharType; + using traits_type = TTraits; + + using iterator = TCharType*; + using reverse_iterator = std::reverse_iterator; + using typename TBase::const_iterator; + using typename TBase::const_reference; + using typename TBase::const_reverse_iterator; + + struct TUninitialized { + explicit TUninitialized(size_t size) + : Size(size) + { + } + + size_t Size; + }; + + size_t max_size() noexcept { + static size_t res = TStringType().max_size(); + + return res; + } + +protected: + TStorage S_; + + template + static TStorage Construct(A&&... a) { + return {new TStdStr(std::forward(a)...), typename TStorage::TNoIncrement()}; + } + + static TStorage Construct() noexcept { + return TStdStr::NullStr(); + } + + TStdStr& StdStr() noexcept { + return *S_; + } + + const TStdStr& StdStr() const noexcept { + return *S_; + } + + /** + * Makes a distinct copy of this string. `IsDetached()` is always true after this call. + * + * @throw std::length_error + */ + void Clone() { + Construct(StdStr()).Swap(S_); + } + + size_t RefCount() const noexcept { + return S_.RefCount(); + } + +public: + inline const TStringType& ConstRef() const Y_LIFETIME_BOUND { + return StdStr(); + } + + inline TStringType& MutRef() Y_LIFETIME_BOUND { + Detach(); + + return StdStr(); + } + + inline const_reference operator[](size_t pos) const noexcept Y_LIFETIME_BOUND { + Y_ASSERT(pos <= length()); + + return this->data()[pos]; + } + + inline reference operator[](size_t pos) noexcept Y_LIFETIME_BOUND { + Y_ASSERT(pos <= length()); + + return reference(*this, pos); + } + + using TBase::back; + + inline reference back() noexcept Y_LIFETIME_BOUND { + Y_ASSERT(!this->empty()); + + if (Y_UNLIKELY(this->empty())) { + return reference(*this, 0); + } + + return reference(*this, length() - 1); + } + + using TBase::front; + + inline reference front() noexcept Y_LIFETIME_BOUND { + Y_ASSERT(!this->empty()); + + return reference(*this, 0); + } + + inline size_t length() const noexcept { + return ConstRef().length(); + } + + inline const TCharType* data() const noexcept Y_LIFETIME_BOUND { + return ConstRef().data(); + } + + inline const TCharType* c_str() const noexcept Y_LIFETIME_BOUND { + return ConstRef().c_str(); + } + + // ~~~ STL compatible method to obtain data pointer ~~~ + iterator begin() Y_LIFETIME_BOUND { + return &*MutRef().begin(); + } + + iterator end() Y_LIFETIME_BOUND { + return &*MutRef().end(); + } + + reverse_iterator rbegin() Y_LIFETIME_BOUND { + return reverse_iterator(end()); + } + + reverse_iterator rend() Y_LIFETIME_BOUND { + return reverse_iterator(begin()); + } + + const_iterator begin() const noexcept Y_LIFETIME_BOUND { + return TBase::begin(); + } + const_iterator cbegin() const noexcept Y_LIFETIME_BOUND { + return TBase::cbegin(); + } + + const_iterator cend() const noexcept Y_LIFETIME_BOUND { + return TBase::cend(); + } + + const_reverse_iterator crbegin() const noexcept Y_LIFETIME_BOUND { + return TBase::crbegin(); + } + + const_reverse_iterator crend() const noexcept Y_LIFETIME_BOUND { + return TBase::crend(); + } + + const_iterator end() const noexcept Y_LIFETIME_BOUND { + return TBase::end(); + } + + const_reverse_iterator rbegin() const noexcept Y_LIFETIME_BOUND { + return TBase::rbegin(); + } + + const_reverse_iterator rend() const noexcept Y_LIFETIME_BOUND { + return TBase::rend(); + } + + inline size_t capacity() const noexcept { + if (S_->IsNull()) { + return 0; + } + + return S_->capacity(); + } + + TCharType* Detach() Y_LIFETIME_BOUND { + if (Y_UNLIKELY(!IsDetached())) { + Clone(); + } + + return (TCharType*)S_->data(); + } + + bool IsDetached() const { + return 1 == RefCount(); + } + + // ~~~ Size and capacity ~~~ + TBasicCowString& resize(size_t n, TCharType c = ' ') Y_LIFETIME_BOUND { // remove or append + MutRef().resize(n, c); + + return *this; + } + + // ~~~ Constructor ~~~ : FAMILY0(,TBasicCowString) + TBasicCowString() noexcept + : S_(Construct()) + { + } + + inline explicit TBasicCowString(::NDetail::TReserveTag rt) + : S_(Construct<>()) + { + reserve(rt.Capacity); + } + + inline TBasicCowString(const TBasicCowString& s) + : S_(s.S_) + { + } + + inline TBasicCowString(TBasicCowString&& s) noexcept + : S_(Construct()) + { + s.swap(*this); + } + + template + explicit inline TBasicCowString(const std::basic_string& s) + : TBasicCowString(s.data(), s.size()) + { + } + + template + inline TBasicCowString(std::basic_string&& s) + : S_(s.empty() ? Construct() : Construct(std::move(s))) + { + } + + TBasicCowString(const TBasicCowString& s, size_t pos, size_t n) + : S_(n ? Construct(s, pos, n) : Construct()) + { + } + + TBasicCowString(const TCharType* pc) + : TBasicCowString(pc, TBase::StrLen(pc)) + { + } + TBasicCowString(std::nullptr_t) = delete; + + TBasicCowString(const TCharType* pc, size_t n) + : S_(n ? Construct(pc, n) : Construct()) + { + } + TBasicCowString(std::nullptr_t, size_t) = delete; + + TBasicCowString(const TCharType* pc, size_t pos, size_t n) + : TBasicCowString(pc + pos, n) + { + } + + explicit TBasicCowString(TExplicitType c) + : TBasicCowString(&c.Value(), 1) + { + } + explicit TBasicCowString(const reference& c) + : TBasicCowString(&c, 1) + { + } + + TBasicCowString(size_t n, TCharType c) + : S_(Construct(n, c)) + { + } + + /** + * Constructs an uninitialized string of size `uninitialized.Size`. The proper + * way to use this ctor is via `TBasicCowString::Uninitialized` factory function. + * + * @throw std::length_error + */ + TBasicCowString(TUninitialized uninitialized) + : S_(Construct<>()) + { + ReserveAndResize(uninitialized.Size); + } + + TBasicCowString(const TCharType* b, const TCharType* e) + : TBasicCowString(b, NonNegativeDistance(b, e)) + { + } + + explicit TBasicCowString(const TBasicStringBuf s) + : TBasicCowString(s.data(), s.size()) + { + } + + template + explicit inline TBasicCowString(const std::basic_string_view& s) + : TBasicCowString(s.data(), s.size()) + { + } + + /** + * WARN: + * Certain invocations of this method will result in link-time error. + * You are free to implement corresponding methods in string.cpp if you need them. + */ + static TBasicCowString FromAscii(const ::TStringBuf& s) { + return TBasicCowString().AppendAscii(s); + } + + static TBasicCowString FromUtf8(const ::TStringBuf& s) { + return TBasicCowString().AppendUtf8(s); + } + + static TBasicCowString FromUtf16(const ::TWtringBuf& s) { + return TBasicCowString().AppendUtf16(s); + } + + static TBasicCowString Uninitialized(size_t n) { + return TBasicCowString(TUninitialized(n)); + } + +private: + using TJoinHelper = TStringJoinHelper; + + template + static inline TBasicCowString JoinImpl(const R&... r) { + TBasicCowString s{TUninitialized{TJoinHelper::SumLength(r...)}}; + TJoinHelper::CopyAll((TCharType*)s.data(), r...); + return s; + } + +public: + Y_REINITIALIZES_OBJECT inline void clear() noexcept { + if (IsDetached()) { + S_->clear(); + + return; + } + + Construct().Swap(S_); + } + + template + static inline TBasicCowString Join(const R&... r) { + return JoinImpl(typename TJoinHelper::template TJoinParam(r)...); + } + + // ~~~ Assignment ~~~ : FAMILY0(TBasicCowString&, assign); + TBasicCowString& assign(size_t size, TCharType ch) Y_LIFETIME_BOUND { + ReserveAndResize(size); + std::fill(begin(), end(), ch); + return *this; + } + + TBasicCowString& assign(const TBasicCowString& s) Y_LIFETIME_BOUND { + TBasicCowString(s).swap(*this); + + return *this; + } + + TBasicCowString& assign(const TBasicCowString& s, size_t pos, size_t n) Y_LIFETIME_BOUND { + return assign(TBasicCowString(s, pos, n)); + } + + TBasicCowString& assign(const TCharType* pc) Y_LIFETIME_BOUND { + return assign(pc, TBase::StrLen(pc)); + } + + TBasicCowString& assign(TCharType ch) Y_LIFETIME_BOUND { + return assign(&ch, 1); + } + + TBasicCowString& assign(const TCharType* pc, size_t len) Y_LIFETIME_BOUND { +#if defined(address_sanitizer_enabled) || defined(thread_sanitizer_enabled) + pc = (const TCharType*)HidePointerOrigin((void*)pc); +#endif + if (IsDetached()) { + MutRef().assign(pc, len); + } else { + TBasicCowString(pc, len).swap(*this); + } + + return *this; + } + + TBasicCowString& assign(const TCharType* first, const TCharType* last) Y_LIFETIME_BOUND { + return assign(first, NonNegativeDistance(first, last)); + } + + TBasicCowString& assign(const TCharType* pc, size_t pos, size_t n) Y_LIFETIME_BOUND { + return assign(pc + pos, n); + } + + TBasicCowString& assign(const TBasicStringBuf s) Y_LIFETIME_BOUND { + return assign(s.data(), s.size()); + } + + TBasicCowString& assign(const TBasicStringBuf s, size_t spos, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + return assign(s.SubString(spos, sn)); + } + + inline TBasicCowString& AssignNoAlias(const TCharType* pc, size_t len) Y_LIFETIME_BOUND { + return assign(pc, len); + } + + inline TBasicCowString& AssignNoAlias(const TCharType* b, const TCharType* e) Y_LIFETIME_BOUND { + return AssignNoAlias(b, e - b); + } + + TBasicCowString& AssignNoAlias(const TBasicStringBuf s) Y_LIFETIME_BOUND { + return AssignNoAlias(s.data(), s.size()); + } + + TBasicCowString& AssignNoAlias(const TBasicStringBuf s, size_t spos, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + return AssignNoAlias(s.SubString(spos, sn)); + } + + /** + * WARN: + * Certain invocations of this method will result in link-time error. + * You are free to implement corresponding methods in string.cpp if you need them. + */ + auto AssignAscii(const ::TStringBuf& s) { + clear(); + return AppendAscii(s); + } + + auto AssignUtf8(const ::TStringBuf& s) { + clear(); + return AppendUtf8(s); + } + + auto AssignUtf16(const ::TWtringBuf& s) { + clear(); + return AppendUtf16(s); + } + + TBasicCowString& operator=(const TBasicCowString& s) Y_LIFETIME_BOUND { + return assign(s); + } + + TBasicCowString& operator=(TBasicCowString&& s) noexcept Y_LIFETIME_BOUND { + swap(s); + return *this; + } + + template + TBasicCowString& operator=(std::basic_string&& s) noexcept Y_LIFETIME_BOUND { + TBasicCowString(std::move(s)).swap(*this); + + return *this; + } + + TBasicCowString& operator=(const TBasicStringBuf s) Y_LIFETIME_BOUND { + return assign(s); + } + + TBasicCowString& operator=(std::initializer_list il) Y_LIFETIME_BOUND { + return assign(il.begin(), il.end()); + } + + TBasicCowString& operator=(const TCharType* s) Y_LIFETIME_BOUND { + return assign(s); + } + TBasicCowString& operator=(std::nullptr_t) Y_LIFETIME_BOUND = delete; + + TBasicCowString& operator=(TExplicitType ch) Y_LIFETIME_BOUND { + return assign(ch); + } + + inline void reserve(size_t len) { + MutRef().reserve(len); + } + + // ~~~ Appending ~~~ : FAMILY0(TBasicCowString&, append); + inline TBasicCowString& append(size_t count, TCharType ch) Y_LIFETIME_BOUND { + MutRef().append(count, ch); + + return *this; + } + + inline TBasicCowString& append(const TBasicCowString& s) Y_LIFETIME_BOUND { + MutRef().append(s.ConstRef()); + + return *this; + } + + inline TBasicCowString& append(const TBasicCowString& s, size_t pos, size_t n) Y_LIFETIME_BOUND { + MutRef().append(s.ConstRef(), pos, n); + + return *this; + } + + inline TBasicCowString& append(const TCharType* pc) Y_LIFETIME_BOUND { + MutRef().append(pc); + + return *this; + } + + inline TBasicCowString& append(TCharType c) Y_LIFETIME_BOUND { + MutRef().push_back(c); + + return *this; + } + + inline TBasicCowString& append(const TCharType* first, const TCharType* last) Y_LIFETIME_BOUND { + MutRef().append(first, last); + + return *this; + } + + inline TBasicCowString& append(const TCharType* pc, size_t len) Y_LIFETIME_BOUND { + MutRef().append(pc, len); + + return *this; + } + + inline void ReserveAndResize(size_t len) { + ::ResizeUninitialized(MutRef(), len); + } + + TBasicCowString& AppendNoAlias(const TCharType* pc, size_t len) Y_LIFETIME_BOUND { + if (len) { + auto s = this->size(); + + ReserveAndResize(s + len); + memcpy(&*(begin() + s), pc, len * sizeof(*pc)); + } + + return *this; + } + + TBasicCowString& AppendNoAlias(const TBasicStringBuf s) Y_LIFETIME_BOUND { + return AppendNoAlias(s.data(), s.size()); + } + + TBasicCowString& AppendNoAlias(const TBasicStringBuf s, size_t spos, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + return AppendNoAlias(s.SubString(spos, sn)); + } + + TBasicCowString& append(const TBasicStringBuf s) Y_LIFETIME_BOUND { + return append(s.data(), s.size()); + } + + TBasicCowString& append(const TBasicStringBuf s, size_t spos, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + return append(s.SubString(spos, sn)); + } + + TBasicCowString& append(const TCharType* pc, size_t pos, size_t n, size_t pc_len = TBase::npos) Y_LIFETIME_BOUND { + return append(pc + pos, Min(n, pc_len - pos)); + } + + /** + * WARN: + * Certain invocations of this method will result in link-time error. + * You are free to implement corresponding methods in string.cpp if you need them. + */ + TBasicCowString& AppendAscii(const ::TStringBuf& s) Y_LIFETIME_BOUND; + + TBasicCowString& AppendUtf8(const ::TStringBuf& s) Y_LIFETIME_BOUND; + + TBasicCowString& AppendUtf16(const ::TWtringBuf& s) Y_LIFETIME_BOUND; + + inline void push_back(TCharType c) { + // TODO + append(c); + } + + template + TBasicCowString& operator+=(const T& s) Y_LIFETIME_BOUND { + return append(s); + } + + template + friend TBasicCowString operator*(const TBasicCowString& s, T count) { + static_assert(std::is_integral::value, "Integral type required."); + + TBasicCowString result; + + if (count > 0) { + result.reserve(s.length() * count); + } + + for (T i = 0; i < count; ++i) { + result += s; + } + + return result; + } + + template + TBasicCowString& operator*=(T count) Y_LIFETIME_BOUND { + static_assert(std::is_integral::value, "Integral type required."); + + TBasicCowString temp; + + if (count > 0) { + temp.reserve(length() * count); + } + + for (T i = 0; i < count; ++i) { + temp += *this; + } + + swap(temp); + + return *this; + } + + operator const TStringType&() const noexcept Y_LIFETIME_BOUND { + return this->ConstRef(); + } + + /* We have operator casting TString to `const std::string&` but we explicitly don't support + * casting TString to `std::string&` since such casting requires detaching TString and therefore + * modifies TString object. Sometimes compiler might call `operator std::string&` + * implicitly and it might lead to problems. Check IGNIETFERRO-2155 for details. + */ + template >> + operator T&() & Y_LIFETIME_BOUND requires false { + return this->MutRef(); + } + + /* + * Following overloads of "operator+" aim to choose the cheapest implementation depending on + * summand types: lvalues, detached rvalues, shared rvalues. + * + * General idea is to use the detached-rvalue argument (left of right) to store the result + * wherever possible. If a buffer in rvalue is large enough this saves a re-allocation. If + * both arguments are rvalues we check which one is detached. If both of them are detached then + * the left argument is obviously preferrable because you won't need to shift the data. + * + * If an rvalue is shared then it's basically the same as lvalue because you cannot use its + * buffer to store the sum. However, we rely on the fact that append() and prepend() are already + * optimized for the shared case and detach the string into the buffer large enough to store + * the sum (compared to the detach+reallocation). This way, if we have only one rvalue argument + * (left or right) then we simply append/prepend into it, without checking if it's detached or + * not. This will be checked inside ReserveAndResize anyway. + * + * If both arguments cannot be used to store the sum (e.g. two lvalues) then we fall back to the + * Join function that constructs a resulting string in the new buffer with the minimum overhead: + * malloc + memcpy + memcpy. + */ + + friend TBasicCowString operator+(TBasicCowString&& s1, const TBasicCowString& s2) Y_WARN_UNUSED_RESULT { + s1 += s2; + return std::move(s1); + } + + friend TBasicCowString operator+(const TBasicCowString& s1, TBasicCowString&& s2) Y_WARN_UNUSED_RESULT { + s2.prepend(s1); + return std::move(s2); + } + + friend TBasicCowString operator+(TBasicCowString&& s1, TBasicCowString&& s2) Y_WARN_UNUSED_RESULT { +#if 0 + if (!s1.IsDetached() && s2.IsDetached()) { + s2.prepend(s1); + return std::move(s2); + } +#endif + s1 += s2; + return std::move(s1); + } + + friend TBasicCowString operator+(TBasicCowString&& s1, const TBasicStringBuf s2) Y_WARN_UNUSED_RESULT { + s1 += s2; + return std::move(s1); + } + + friend TBasicCowString operator+(TBasicCowString&& s1, const TCharType* s2) Y_WARN_UNUSED_RESULT { + s1 += s2; + return std::move(s1); + } + + friend TBasicCowString operator+(TBasicCowString&& s1, TCharType s2) Y_WARN_UNUSED_RESULT { + s1 += s2; + return std::move(s1); + } + + friend TBasicCowString operator+(TExplicitType ch, const TBasicCowString& s) Y_WARN_UNUSED_RESULT { + return Join(TCharType(ch), s); + } + + friend TBasicCowString operator+(TExplicitType ch, TBasicCowString&& s) Y_WARN_UNUSED_RESULT { + s.prepend(ch); + return std::move(s); + } + + friend TBasicCowString operator+(const TBasicCowString& s1, const TBasicCowString& s2) Y_WARN_UNUSED_RESULT { + return Join(s1, s2); + } + + friend TBasicCowString operator+(const TBasicCowString& s1, const TBasicStringBuf s2) Y_WARN_UNUSED_RESULT { + return Join(s1, s2); + } + + friend TBasicCowString operator+(const TBasicCowString& s1, const TCharType* s2) Y_WARN_UNUSED_RESULT { + return Join(s1, s2); + } + + friend TBasicCowString operator+(const TBasicCowString& s1, TCharType s2) Y_WARN_UNUSED_RESULT { + return Join(s1, TBasicStringBuf(&s2, 1)); + } + + friend TBasicCowString operator+(const TCharType* s1, TBasicCowString&& s2) Y_WARN_UNUSED_RESULT { + s2.prepend(s1); + return std::move(s2); + } + + friend TBasicCowString operator+(const TBasicStringBuf s1, TBasicCowString&& s2) Y_WARN_UNUSED_RESULT { + s2.prepend(s1); + return std::move(s2); + } + + friend TBasicCowString operator+(const TBasicStringBuf s1, const TBasicCowString& s2) Y_WARN_UNUSED_RESULT { + return Join(s1, s2); + } + + friend TBasicCowString operator+(const TCharType* s1, const TBasicCowString& s2) Y_WARN_UNUSED_RESULT { + return Join(s1, s2); + } + + friend TBasicCowString operator+(std::basic_string l, TBasicCowString r) { + return std::move(l) + r.ConstRef(); + } + + friend TBasicCowString operator+(TBasicCowString l, std::basic_string r) { + return l.ConstRef() + std::move(r); + } + + // ~~~ Prepending ~~~ : FAMILY0(TBasicCowString&, prepend); + TBasicCowString& prepend(const TBasicCowString& s) Y_LIFETIME_BOUND { + MutRef().insert(0, s.ConstRef()); + + return *this; + } + + TBasicCowString& prepend(const TBasicCowString& s, size_t pos, size_t n) Y_LIFETIME_BOUND { + MutRef().insert(0, s.ConstRef(), pos, n); + + return *this; + } + + TBasicCowString& prepend(const TCharType* pc) Y_LIFETIME_BOUND { + MutRef().insert(0, pc); + + return *this; + } + + TBasicCowString& prepend(size_t n, TCharType c) Y_LIFETIME_BOUND { + MutRef().insert(size_t(0), n, c); + + return *this; + } + + TBasicCowString& prepend(TCharType c) Y_LIFETIME_BOUND { + MutRef().insert(size_t(0), 1, c); + + return *this; + } + + TBasicCowString& prepend(const TBasicStringBuf s, size_t spos = 0, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + return insert(0, s, spos, sn); + } + + // ~~~ Insertion ~~~ : FAMILY1(TBasicCowString&, insert, size_t pos); + TBasicCowString& insert(size_t pos, const TBasicCowString& s) Y_LIFETIME_BOUND { + MutRef().insert(pos, s.ConstRef()); + + return *this; + } + + TBasicCowString& insert(size_t pos, const TBasicCowString& s, size_t pos1, size_t n1) Y_LIFETIME_BOUND { + MutRef().insert(pos, s.ConstRef(), pos1, n1); + + return *this; + } + + TBasicCowString& insert(size_t pos, const TCharType* pc) Y_LIFETIME_BOUND { + MutRef().insert(pos, pc); + + return *this; + } + + TBasicCowString& insert(size_t pos, const TCharType* pc, size_t len) Y_LIFETIME_BOUND { + MutRef().insert(pos, pc, len); + + return *this; + } + + TBasicCowString& insert(const_iterator pos, const_iterator b, const_iterator e) Y_LIFETIME_BOUND { + return insert(this->off(pos), b, e - b); + } + + TBasicCowString& insert(size_t pos, size_t n, TCharType c) Y_LIFETIME_BOUND { + MutRef().insert(pos, n, c); + + return *this; + } + + TBasicCowString& insert(const_iterator pos, size_t len, TCharType ch) Y_LIFETIME_BOUND { + return this->insert(this->off(pos), len, ch); + } + + TBasicCowString& insert(const_iterator pos, TCharType ch) Y_LIFETIME_BOUND { + return this->insert(pos, 1, ch); + } + + TBasicCowString& insert(size_t pos, const TBasicStringBuf s, size_t spos = 0, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + MutRef().insert(pos, s, spos, sn); + + return *this; + } + + // ~~~ Removing ~~~ + TBasicCowString& remove(size_t pos, size_t n) Y_LIFETIME_BOUND { + if (pos < length()) { + MutRef().erase(pos, n); + } + + return *this; + } + + TBasicCowString& remove(size_t pos = 0) Y_LIFETIME_BOUND { + if (pos < length()) { + MutRef().erase(pos); + } + + return *this; + } + + TBasicCowString& erase(size_t pos = 0, size_t n = TBase::npos) Y_LIFETIME_BOUND { + MutRef().erase(pos, n); + + return *this; + } + + TBasicCowString& erase(const_iterator b, const_iterator e) Y_LIFETIME_BOUND { + return erase(this->off(b), e - b); + } + + TBasicCowString& erase(const_iterator i) Y_LIFETIME_BOUND { + return erase(i, i + 1); + } + + TBasicCowString& pop_back() Y_LIFETIME_BOUND { + Y_ASSERT(!this->empty()); + + MutRef().pop_back(); + + return *this; + } + + // ~~~ replacement ~~~ : FAMILY2(TBasicCowString&, replace, size_t pos, size_t n); + TBasicCowString& replace(size_t pos, size_t n, const TBasicCowString& s) Y_LIFETIME_BOUND { + MutRef().replace(pos, n, s.ConstRef()); + + return *this; + } + + TBasicCowString& replace(size_t pos, size_t n, const TBasicCowString& s, size_t pos1, size_t n1) Y_LIFETIME_BOUND { + MutRef().replace(pos, n, s.ConstRef(), pos1, n1); + + return *this; + } + + TBasicCowString& replace(size_t pos, size_t n, const TCharType* pc) Y_LIFETIME_BOUND { + MutRef().replace(pos, n, pc); + + return *this; + } + + TBasicCowString& replace(size_t pos, size_t n, const TCharType* s, size_t len) Y_LIFETIME_BOUND { + MutRef().replace(pos, n, s, len); + + return *this; + } + + TBasicCowString& replace(size_t pos, size_t n, const TCharType* s, size_t spos, size_t sn) Y_LIFETIME_BOUND { + MutRef().replace(pos, n, s + spos, sn - spos); + + return *this; + } + + TBasicCowString& replace(size_t pos, size_t n1, size_t n2, TCharType c) Y_LIFETIME_BOUND { + MutRef().replace(pos, n1, n2, c); + + return *this; + } + + TBasicCowString& replace(size_t pos, size_t n, const TBasicStringBuf s, size_t spos = 0, size_t sn = TBase::npos) Y_LIFETIME_BOUND { + MutRef().replace(pos, n, s, spos, sn); + + return *this; + } + + void swap(TBasicCowString& s) noexcept { + S_.Swap(s.S_); + } + + /** + * @returns String suitable for debug printing (like Python's `repr()`). + * Format of the string is unspecified and may be changed over time. + */ + TBasicCowString Quote() const { + extern TBasicCowString EscapeC(const TBasicCowString&); + + return TBasicCowString() + '"' + EscapeC(*this) + '"'; + } + + /** + * Modifies the case of the string, depending on the operation. + * @return false if no changes have been made. + * + * @warning when the value_type is char, these methods will not work with non-ASCII letters. + */ + bool to_lower(size_t pos = 0, size_t n = TBase::npos); + bool to_upper(size_t pos = 0, size_t n = TBase::npos); + bool to_title(size_t pos = 0, size_t n = TBase::npos); + + constexpr const TCharType* Data() const noexcept = delete; + constexpr size_t Size() noexcept = delete; + Y_PURE_FUNCTION constexpr bool Empty() const noexcept = delete; + +public: + /** + * Modifies the substring of length `n` starting from `pos`, applying `f` to each position and symbol. + * + * @return false if no changes have been made. + */ + template + bool Transform(T&& f, size_t pos = 0, size_t n = TBase::npos) { + size_t len = length(); + + if (pos > len) { + pos = len; + } + + if (n > len - pos) { + n = len - pos; + } + + bool changed = false; + + for (size_t i = pos; i != pos + n; ++i) { + auto c = f(i, data()[i]); + if (c != data()[i]) { + if (!changed) { + Detach(); + changed = true; + } + + begin()[i] = c; + } + } + + return changed; + } +}; + +using TCowString = TBasicCowString; +using TUtf16CowString = TBasicCowString; +using TUtf32CowString = TBasicCowString; + +std::ostream& operator<<(std::ostream&, const TCowString&); +std::istream& operator>>(std::istream&, TCowString&); + +template +TBasicCowString to_lower(const TBasicCowString& s) { + TBasicCowString ret(s); + ret.to_lower(); + return ret; +} + +template +TBasicCowString to_upper(const TBasicCowString& s) { + TBasicCowString ret(s); + ret.to_upper(); + return ret; +} + +template +TBasicCowString to_title(const TBasicCowString& s) { + TBasicCowString ret(s); + ret.to_title(); + return ret; +} + +namespace std { + template <> + struct hash { + using argument_type = TCowString; + using result_type = size_t; + inline result_type operator()(argument_type const& s) const noexcept { + return NHashPrivate::ComputeStringHash(s.data(), s.size()); + } + }; +} // namespace std + +// interop +template +auto& MutRef(TBasicCowString& s Y_LIFETIME_BOUND) { + return s.MutRef(); +} + +template +const auto& ConstRef(const TBasicCowString& s Y_LIFETIME_BOUND) noexcept { + return s.ConstRef(); +} + +template +void ResizeUninitialized(TBasicCowString& s, size_t len) { + s.ReserveAndResize(len); +} diff --git a/library/cpp/containers/cow_string/cow_string_ut.cpp b/library/cpp/containers/cow_string/cow_string_ut.cpp new file mode 100644 index 00000000000..1be9d607ed7 --- /dev/null +++ b/library/cpp/containers/cow_string/cow_string_ut.cpp @@ -0,0 +1,1296 @@ +#include + +#include +#include +#include +#include + +#include +#include "util/generic/deque.h" +#include "util/generic/strbuf.h" +#include "util/generic/string_ut.h" +#include "util/generic/vector.h" +#include "util/generic/yexception.h" +#include +#include + +#include +#include +#include +#include + +static_assert(sizeof(TCowString) == sizeof(const char*), "expect sizeof(TCowString) == sizeof(const char*)"); + +class TStringTestZero: public TTestBase { + UNIT_TEST_SUITE(TStringTestZero); + UNIT_TEST(TestZero); + UNIT_TEST_SUITE_END(); + +public: + void TestZero() { + const char data[] = "abc\0def\0"; + TCowString s(data, sizeof(data)); + UNIT_ASSERT(s.size() == sizeof(data)); + UNIT_ASSERT(s.StartsWith(s)); + UNIT_ASSERT(s.EndsWith(s)); + UNIT_ASSERT(s.Contains('\0')); + + const char raw_def[] = "def"; + const char raw_zero[] = "\0"; + TCowString def(raw_def, sizeof(raw_def) - 1); + TCowString zero(raw_zero, sizeof(raw_zero) - 1); + UNIT_ASSERT_EQUAL(4, s.find(raw_def)); + UNIT_ASSERT_EQUAL(4, s.find(def)); + UNIT_ASSERT_EQUAL(4, s.find_first_of(raw_def)); + UNIT_ASSERT_EQUAL(3, s.find_first_of(zero)); + UNIT_ASSERT_EQUAL(7, s.find_first_not_of(def, 4)); + + const char nonSubstring[] = "def\0ghi"; + UNIT_ASSERT_EQUAL(TCowString::npos, s.find(TCowString(nonSubstring, sizeof(nonSubstring)))); + + TCowString copy = s; + copy.replace(copy.size() - 1, 1, "z"); + UNIT_ASSERT(s != copy); + copy.replace(copy.size() - 1, 1, "\0", 0, 1); + UNIT_ASSERT(s == copy); + + TCowString prefix(data, 5); + UNIT_ASSERT(s.StartsWith(prefix)); + UNIT_ASSERT(s != prefix); + UNIT_ASSERT(s > prefix); + UNIT_ASSERT(s > s.data()); + UNIT_ASSERT(s == TCowString(s.data(), s.size())); + UNIT_ASSERT(data < s); + + s.remove(5); + UNIT_ASSERT(s == prefix); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TStringTestZero); + +template +class TStringStdTestImpl { + using TChar = typename TStringType::char_type; + using TTraits = typename TStringType::traits_type; + using TView = std::basic_string_view; + + TTestData Data_; + +protected: + void Constructor() { + UNIT_ASSERT_EXCEPTION(TStringType((size_t)-1, *Data_.a()), std::length_error); + } + + void reserve() { +#if 0 + TStringType s; + UNIT_ASSERT_EXCEPTION(s.reserve(s.max_size() + 1), std::length_error); + + // Non-shared behaviour - never shrink + + s.reserve(256); + const auto* data = s.data(); + + UNIT_ASSERT(s.capacity() >= 256); + + s.reserve(128); + + UNIT_ASSERT(s.capacity() >= 256 && s.data() == data); + + s.resize(64, 'x'); + s.reserve(10); + + UNIT_ASSERT(s.capacity() >= 256 && s.data() == data); + + // Shared behaviour - always reallocate, just as much as requisted + + TStringType holder = s; + + UNIT_ASSERT(s.capacity() >= 256); + + s.reserve(128); + + UNIT_ASSERT(s.capacity() >= 128 && s.capacity() < 256 && s.data() != data); + UNIT_ASSERT(s.IsDetached()); + + s.resize(64, 'x'); + data = s.data(); + holder = s; + + s.reserve(10); + + UNIT_ASSERT(s.capacity() >= 64 && s.capacity() < 128 && s.data() != data); + UNIT_ASSERT(s.IsDetached()); +#endif + } + + void short_string() { + TStringType const ref_short_str1(Data_.str1()), ref_short_str2(Data_.str2()); + TStringType short_str1(ref_short_str1), short_str2(ref_short_str2); + TStringType const ref_long_str1(Data_.str__________________________________________________1()); + TStringType const ref_long_str2(Data_.str__________________________________________________2()); + TStringType long_str1(ref_long_str1), long_str2(ref_long_str2); + + UNIT_ASSERT(short_str1 == ref_short_str1); + UNIT_ASSERT(long_str1 == ref_long_str1); + + { + TStringType str1(short_str1); + str1 = long_str1; + UNIT_ASSERT(str1 == ref_long_str1); + } + + { + TStringType str1(long_str1); + str1 = short_str1; + UNIT_ASSERT(str1 == ref_short_str1); + } + + { + short_str1.swap(short_str2); + UNIT_ASSERT((short_str1 == ref_short_str2) && (short_str2 == ref_short_str1)); + short_str1.swap(short_str2); + } + + { + long_str1.swap(long_str2); + UNIT_ASSERT((long_str1 == ref_long_str2) && (long_str2 == ref_long_str1)); + long_str1.swap(long_str2); + } + + { + short_str1.swap(long_str1); + UNIT_ASSERT((short_str1 == ref_long_str1) && (long_str1 == ref_short_str1)); + short_str1.swap(long_str1); + } + + { + long_str1.swap(short_str1); + UNIT_ASSERT((short_str1 == ref_long_str1) && (long_str1 == ref_short_str1)); + long_str1.swap(short_str1); + } + + { + // This is to test move constructor + TVector str_vect; + + str_vect.push_back(short_str1); + str_vect.push_back(long_str1); + str_vect.push_back(short_str2); + str_vect.push_back(long_str2); + + UNIT_ASSERT(str_vect[0] == ref_short_str1); + UNIT_ASSERT(str_vect[1] == ref_long_str1); + UNIT_ASSERT(str_vect[2] == ref_short_str2); + UNIT_ASSERT(str_vect[3] == ref_long_str2); + } + } + + void erase() { + TChar const* c_str = Data_.Hello_World(); + TStringType str(c_str); + UNIT_ASSERT(str == c_str); + + str.erase(str.begin() + 1, str.end() - 1); // Erase all but first and last. + + size_t i; + for (i = 0; i < str.size(); ++i) { + switch (i) { + case 0: + UNIT_ASSERT(str[i] == *Data_.H()); + break; + + case 1: + UNIT_ASSERT(str[i] == *Data_.d()); + break; + + default: + UNIT_ASSERT(false); + } + } + + str.insert(1, c_str); + str.erase(str.begin()); // Erase first element. + str.erase(str.end() - 1); // Erase last element. + UNIT_ASSERT(str == c_str); + str.clear(); // Erase all. + UNIT_ASSERT(str.empty()); + + str = c_str; + UNIT_ASSERT(str == c_str); + + str.erase(1, str.size() - 1); // Erase all but first and last. + for (i = 0; i < str.size(); i++) { + switch (i) { + case 0: + UNIT_ASSERT(str[i] == *Data_.H()); + break; + + case 1: + UNIT_ASSERT(str[i] == *Data_.d()); + break; + + default: + UNIT_ASSERT(false); + } + } + + str.erase(1); + UNIT_ASSERT(str == Data_.H()); + } + + void data() { + TStringType xx; + + // ISO-IEC-14882:1998(E), 21.3.6, paragraph 3 + UNIT_ASSERT(xx.data() != nullptr); + } + + void c_str() { + TStringType low(Data_._2004_01_01()); + TStringType xx; + TStringType yy; + + // ISO-IEC-14882:1998(E), 21.3.6, paragraph 1 + UNIT_ASSERT(*(yy.c_str()) == 0); + + // Blocks A and B should follow each other. + // Block A: + xx = Data_._123456(); + xx += low; + UNIT_ASSERT(xx.c_str() == TView(Data_._1234562004_01_01())); + // End of block A + + // Block B: + xx = Data_._1234(); + xx += Data_._5(); + UNIT_ASSERT(xx.c_str() == TView(Data_._12345())); + // End of block B + } + + void null_char_of_empty() { + const TStringType s; + + // NOTE: https://a.yandex-team.ru/arcadia/junk/grechnik/test_string?rev=r12602052 + i64 i = s[s.size()]; + UNIT_ASSERT_VALUES_EQUAL(i, 0); + } + + void null_char() { + // ISO/IEC 14882:1998(E), ISO/IEC 14882:2003(E), 21.3.4 ('... the const version') + const TStringType s(Data_._123456()); + + UNIT_ASSERT(s[s.size()] == 0); + } + + // Allowed since C++17, see http://www.open-std.org/jtc1/sc22/wg21/docs/lwg-defects.html#2475 + void null_char_assignment_to_subscript_of_empty() { + TStringType s; + + using reference = typename TStringType::reference; + reference trailing_zero = s[s.size()]; + trailing_zero = 0; + UNIT_ASSERT(trailing_zero == 0); + } + + // Allowed since C++17, see http://www.open-std.org/jtc1/sc22/wg21/docs/lwg-defects.html#2475 + void null_char_assignment_to_subscript_of_nonempty() { + TStringType s(Data_._123456()); + + using reference = typename TStringType::reference; + reference trailing_zero = s[s.size()]; + trailing_zero = 0; + UNIT_ASSERT(trailing_zero == 0); + } + + // Dereferencing string end() is not allowed by C++ standard as of C++20, avoid using in real code. + void null_char_assignment_to_end_of_empty() { + TStringType s; + + volatile auto& trailing_zero = *(s.begin() + s.size()); + trailing_zero = 0; + UNIT_ASSERT(trailing_zero == 0); + } + + // Dereferencing string end() is not allowed by C++ standard as of C++20, avoid using in real code. + void null_char_assignment_to_end_of_nonempty() { + TStringType s(Data_._123456()); + + volatile auto& trailing_zero = *(s.begin() + s.size()); + trailing_zero = 0; + UNIT_ASSERT(trailing_zero == 0); + } + + void insert() { + TStringType strorg = Data_.This_is_test_string_for_string_calls(); + TStringType str; + + // In case of reallocation there is no auto reference problem + // so we reserve a big enough TStringType to be sure to test this + // particular point. + + str.reserve(100); + str = strorg; + + // test self insertion: + str.insert(10, str.c_str() + 5, 15); + UNIT_ASSERT(str == Data_.This_is_teis_test_string_st_string_for_string_calls()); + + str = strorg; + str.insert(15, str.c_str() + 5, 25); + UNIT_ASSERT(str == Data_.This_is_test_stis_test_string_for_stringring_for_string_calls()); + + str = strorg; + str.insert(0, str.c_str() + str.size() - 4, 4); + UNIT_ASSERT(str == Data_.allsThis_is_test_string_for_string_calls()); + + str = strorg; + str.insert(0, str.c_str() + str.size() / 2 - 1, str.size() / 2 + 1); + UNIT_ASSERT(str == Data_.ng_for_string_callsThis_is_test_string_for_string_calls()); + + str = strorg; + typename TStringType::iterator b = str.begin(); + typename TStringType::const_iterator s = str.begin() + str.size() / 2 - 1; + typename TStringType::const_iterator e = str.end(); + str.insert(b, s, e); + UNIT_ASSERT(str == Data_.ng_for_string_callsThis_is_test_string_for_string_calls()); + +#if 0 + // AV + str = strorg; + str.insert(str.begin(), str.begin() + str.size() / 2 - 1, str.end()); + UNIT_ASSERT(str == Data.ng_for_string_callsThis_is_test_string_for_string_calls()); +#endif + + TStringType str0; + str0.insert(str0.begin(), 5, *Data_._0()); + UNIT_ASSERT(str0 == Data_._00000()); + + TStringType str1; + { + typename TStringType::size_type pos = 0, nb = 2; + str1.insert(pos, nb, *Data_._1()); + } + UNIT_ASSERT(str1 == Data_._11()); + + str0.insert(0, str1); + UNIT_ASSERT(str0 == Data_._1100000()); + + TStringType str2(Data_._2345()); + str0.insert(str0.size(), str2, 1, 2); + UNIT_ASSERT(str0 == Data_._110000034()); + + str1.insert(str1.begin() + 1, 2, *Data_._2()); + UNIT_ASSERT(str1 == Data_._1221()); + + str1.insert(2, Data_._333333(), 3); + UNIT_ASSERT(str1 == Data_._1233321()); + + str1.insert(4, Data_._4444()); + UNIT_ASSERT(str1 == Data_._12334444321()); + + str1.insert(str1.begin() + 6, *Data_._5()); + UNIT_ASSERT(str1 == Data_._123344544321()); + } + + void resize() { + TStringType s; + + s.resize(0); + + UNIT_ASSERT(*s.c_str() == 0); + + s = Data_._1234567(); + + s.resize(0); + UNIT_ASSERT(*s.c_str() == 0); + + s = Data_._1234567(); + s.resize(1); + UNIT_ASSERT(s.size() == 1); + UNIT_ASSERT(*s.c_str() == *Data_._1()); + UNIT_ASSERT(*(s.c_str() + 1) == 0); + + s = Data_._1234567(); +#if 0 + s.resize(10); +#else + s.resize(10, 0); +#endif + UNIT_ASSERT(s.size() == 10); + UNIT_ASSERT(s[6] == *Data_._7()); + UNIT_ASSERT(s[7] == 0); + UNIT_ASSERT(s[8] == 0); + UNIT_ASSERT(s[9] == 0); + } + + void find() { + TStringType s(Data_.one_two_three_one_two_three()); + + UNIT_ASSERT(s.find(Data_.one()) == 0); + UNIT_ASSERT(s.find(*Data_.t()) == 4); + UNIT_ASSERT(s.find(*Data_.t(), 5) == 8); + + UNIT_ASSERT(s.find(Data_.four()) == TStringType::npos); + UNIT_ASSERT(s.find(Data_.one(), TStringType::npos) == TStringType::npos); + UNIT_ASSERT(s.find_first_of(Data_.abcde()) == 2); + UNIT_ASSERT(s.find_first_not_of(Data_.enotw_()) == 9); + } + + void capacity() { + TStringType s; + + UNIT_ASSERT(s.capacity() < s.max_size()); + UNIT_ASSERT(s.capacity() >= s.size()); + + for (int i = 0; i < 18; ++i) { + s += ' '; + + UNIT_ASSERT(s.capacity() > 0); + UNIT_ASSERT(s.capacity() < s.max_size()); + UNIT_ASSERT(s.capacity() >= s.size()); + } + } + + void assign() { + TStringType s; + TChar const* cstr = Data_.test_string_for_assign(); + + s.assign(cstr, cstr + 22); + UNIT_ASSERT(s == Data_.test_string_for_assign()); + + TStringType s2(Data_.other_test_string()); + s.assign(s2); + UNIT_ASSERT(s == s2); + + static TStringType str1; + static TStringType str2; + + // short TStringType optim: + str1 = Data_._123456(); + // longer than short TStringType: + str2 = Data_._1234567890123456789012345678901234567890(); + + UNIT_ASSERT(str1[5] == *Data_._6()); + UNIT_ASSERT(str2[29] == *Data_._0()); + } + + void copy() { + TStringType s(Data_.foo()); + TChar dest[4]; + dest[0] = dest[1] = dest[2] = dest[3] = 1; + s.copy(dest, 4); + int pos = 0; + UNIT_ASSERT(dest[pos++] == *Data_.f()); + UNIT_ASSERT(dest[pos++] == *Data_.o()); + UNIT_ASSERT(dest[pos++] == *Data_.o()); + UNIT_ASSERT(dest[pos++] == 1); + + dest[0] = dest[1] = dest[2] = dest[3] = 1; + s.copy(dest, 4, 2); + pos = 0; + UNIT_ASSERT(dest[pos++] == *Data_.o()); + UNIT_ASSERT(dest[pos++] == 1); + + UNIT_ASSERT_EXCEPTION(s.copy(dest, 4, 5), std::out_of_range); + } + + void cbegin_cend() { + const char helloThere[] = "Hello there"; + TCowString s = helloThere; + size_t index = 0; + for (auto it = s.cbegin(); s.cend() != it; ++it, ++index) { + UNIT_ASSERT_VALUES_EQUAL(helloThere[index], *it); + } + } + + void compare() { + TStringType str1(Data_.abcdef()); + TStringType str2; + + str2 = Data_.abcdef(); + UNIT_ASSERT(str1.compare(str2) == 0); + UNIT_ASSERT(str1.compare(str2.data(), str2.size()) == 0); + str2 = Data_.abcde(); + UNIT_ASSERT(str1.compare(str2) > 0); + UNIT_ASSERT(str1.compare(str2.data(), str2.size()) > 0); + str2 = Data_.abcdefg(); + UNIT_ASSERT(str1.compare(str2) < 0); + UNIT_ASSERT(str1.compare(str2.data(), str2.size()) < 0); + + UNIT_ASSERT(str1.compare(Data_.abcdef()) == 0); + UNIT_ASSERT(str1.compare(Data_.abcde()) > 0); + UNIT_ASSERT(str1.compare(Data_.abcdefg()) < 0); + + str2 = Data_.cde(); + UNIT_ASSERT(str1.compare(2, 3, str2) == 0); + str2 = Data_.cd(); + UNIT_ASSERT(str1.compare(2, 3, str2) > 0); + str2 = Data_.cdef(); + UNIT_ASSERT(str1.compare(2, 3, str2) < 0); + + str2 = Data_.abcdef(); + UNIT_ASSERT(str1.compare(2, 3, str2, 2, 3) == 0); + UNIT_ASSERT(str1.compare(2, 3, str2, 2, 2) > 0); + UNIT_ASSERT(str1.compare(2, 3, str2, 2, 4) < 0); + + UNIT_ASSERT(str1.compare(2, 3, Data_.cdefgh(), 3) == 0); + UNIT_ASSERT(str1.compare(2, 3, Data_.cdefgh(), 2) > 0); + UNIT_ASSERT(str1.compare(2, 3, Data_.cdefgh(), 4) < 0); + } + + void find_last_of() { + // 21.3.6.4 + TStringType s(Data_.one_two_three_one_two_three()); + + UNIT_ASSERT(s.find_last_of(Data_.abcde()) == 26); + UNIT_ASSERT(s.find_last_of(TStringType(Data_.abcde())) == 26); + + TStringType test(Data_.aba()); + + UNIT_ASSERT(test.find_last_of(Data_.a(), 2, 1) == 2); + UNIT_ASSERT(test.find_last_of(Data_.a(), 1, 1) == 0); + UNIT_ASSERT(test.find_last_of(Data_.a(), 0, 1) == 0); + + UNIT_ASSERT(test.find_last_of(*Data_.a(), 2) == 2); + UNIT_ASSERT(test.find_last_of(*Data_.a(), 1) == 0); + UNIT_ASSERT(test.find_last_of(*Data_.a(), 0) == 0); + } +#if 0 + void rfind() { + // 21.3.6.2 + TStringType s(Data.one_two_three_one_two_three()); + + UNIT_ASSERT(s.rfind(Data.two()) == 18); + UNIT_ASSERT(s.rfind(Data.two(), 0) == TStringType::npos); + UNIT_ASSERT(s.rfind(Data.two(), 11) == 4); + UNIT_ASSERT(s.rfind(*Data.w()) == 19); + + TStringType test(Data.aba()); + + UNIT_ASSERT(test.rfind(Data.a(), 2, 1) == 2); + UNIT_ASSERT(test.rfind(Data.a(), 1, 1) == 0); + UNIT_ASSERT(test.rfind(Data.a(), 0, 1) == 0); + + UNIT_ASSERT(test.rfind(*Data.a(), 2) == 2); + UNIT_ASSERT(test.rfind(*Data.a(), 1) == 0); + UNIT_ASSERT(test.rfind(*Data.a(), 0) == 0); + } +#endif + void find_last_not_of() { + // 21.3.6.6 + TStringType s(Data_.one_two_three_one_two_three()); + + UNIT_ASSERT(s.find_last_not_of(Data_.ehortw_()) == 15); + + TStringType test(Data_.aba()); + + UNIT_ASSERT(test.find_last_not_of(Data_.a(), 2, 1) == 1); + UNIT_ASSERT(test.find_last_not_of(Data_.b(), 2, 1) == 2); + UNIT_ASSERT(test.find_last_not_of(Data_.a(), 1, 1) == 1); + UNIT_ASSERT(test.find_last_not_of(Data_.b(), 1, 1) == 0); + UNIT_ASSERT(test.find_last_not_of(Data_.a(), 0, 1) == TStringType::npos); + UNIT_ASSERT(test.find_last_not_of(Data_.b(), 0, 1) == 0); + + UNIT_ASSERT(test.find_last_not_of(*Data_.a(), 2) == 1); + UNIT_ASSERT(test.find_last_not_of(*Data_.b(), 2) == 2); + UNIT_ASSERT(test.find_last_not_of(*Data_.a(), 1) == 1); + UNIT_ASSERT(test.find_last_not_of(*Data_.b(), 1) == 0); + UNIT_ASSERT(test.find_last_not_of(*Data_.a(), 0) == TStringType::npos); + UNIT_ASSERT(test.find_last_not_of(*Data_.b(), 0) == 0); + } +#if 0 + void replace() { + // This test case is for the non template basic_TString::replace method, + // this is why we play with the const iterators and reference to guaranty + // that the right method is called. + + const TStringType v(Data._78()); + TStringType s(Data._123456()); + TStringType const& cs = s; + + typename TStringType::iterator i = s.begin() + 1; + s.replace(i, i + 3, v.begin(), v.end()); + UNIT_ASSERT(s == Data._17856()); + + s = Data._123456(); + i = s.begin() + 1; + s.replace(i, i + 1, v.begin(), v.end()); + UNIT_ASSERT(s == Data._1783456()); + + s = Data._123456(); + i = s.begin() + 1; + typename TStringType::const_iterator ci = s.begin() + 1; + s.replace(i, i + 3, ci + 3, cs.end()); + UNIT_ASSERT(s == Data._15656()); + + s = Data._123456(); + i = s.begin() + 1; + ci = s.begin() + 1; + s.replace(i, i + 3, ci, ci + 2); + UNIT_ASSERT(s == Data._12356()); + + s = Data._123456(); + i = s.begin() + 1; + ci = s.begin() + 1; + s.replace(i, i + 3, ci + 1, cs.end()); + UNIT_ASSERT(s == Data._1345656()); + + s = Data._123456(); + i = s.begin(); + ci = s.begin() + 1; + s.replace(i, i, ci, ci + 1); + UNIT_ASSERT(s == Data._2123456()); + + s = Data._123456(); + s.replace(s.begin() + 4, s.end(), cs.begin(), cs.end()); + UNIT_ASSERT(s == Data._1234123456()); + + // This is the test for the template replace method. + + s = Data._123456(); + typename TStringType::iterator b = s.begin() + 4; + typename TStringType::iterator e = s.end(); + typename TStringType::const_iterator rb = s.begin(); + typename TStringType::const_iterator re = s.end(); + s.replace(b, e, rb, re); + UNIT_ASSERT(s == Data._1234123456()); + + s = Data._123456(); + s.replace(s.begin() + 4, s.end(), s.begin(), s.end()); + UNIT_ASSERT(s == Data._1234123456()); + + TStringType strorg(Data.This_is_test_StringT_for_StringT_calls()); + TStringType str = strorg; + str.replace(5, 15, str.c_str(), 10); + UNIT_ASSERT(str == Data.This_This_is_tefor_StringT_calls()); + + str = strorg; + str.replace(5, 5, str.c_str(), 10); + UNIT_ASSERT(str == Data.This_This_is_test_StringT_for_StringT_calls()); + + #if !defined(STLPORT) || defined(_STLP_MEMBER_TEMPLATES) + deque cdeque; + cdeque.push_back(*Data.I()); + str.replace(str.begin(), str.begin() + 11, cdeque.begin(), cdeque.end()); + UNIT_ASSERT(str == Data.Is_test_StringT_for_StringT_calls()); + #endif + } +#endif +}; // TStringStdTestImpl + +class TStringTest: public TTestBase, private TStringTestImpl> { +public: + UNIT_TEST_SUITE(TStringTest); + UNIT_TEST(TestMaxSize); + UNIT_TEST(TestConstructors); + UNIT_TEST(TestReplace); + UNIT_TEST(TestRefCount); + UNIT_TEST(TestFind); + UNIT_TEST(TestContains); + UNIT_TEST(TestOperators); + UNIT_TEST(TestMulOperators); + UNIT_TEST(TestFuncs); + UNIT_TEST(TestUtils); + UNIT_TEST(TestEmpty); + UNIT_TEST(TestJoin); + UNIT_TEST(TestCopy); + UNIT_TEST(TestStrCpy); + UNIT_TEST(TestPrefixSuffix); + UNIT_TEST(TestCharRef); + UNIT_TEST(TestBack) + UNIT_TEST(TestFront) + UNIT_TEST(TestIterators); + UNIT_TEST(TestReverseIterators); + UNIT_TEST(TestAppendUtf16) + UNIT_TEST(TestFillingAssign) + UNIT_TEST(TestStdStreamApi) + // UNIT_TEST(TestOperatorsCI); must fail + UNIT_TEST_SUITE_END(); + + void TestAppendUtf16() { + TCowString appended = TCowString("А роза упала").AppendUtf16(u" на лапу Азора"); + UNIT_ASSERT(appended == "А роза упала на лапу Азора"); + } + + void TestFillingAssign() { + TCowString s("abc"); + s.assign(5, 'a'); + UNIT_ASSERT_VALUES_EQUAL(s, "aaaaa"); + } + + void TestStdStreamApi() { + const TCowString data = "abracadabra"; + std::stringstream ss; + ss << data; + + UNIT_ASSERT_VALUES_EQUAL(data, ss.str()); + + ss << '\n' + << data << std::endl; + + TCowString read = "xxx"; + ss >> read; + UNIT_ASSERT_VALUES_EQUAL(read, data); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TStringTest); + +class TWideStringTest: public TTestBase, private TStringTestImpl> { +public: + UNIT_TEST_SUITE(TWideStringTest); + UNIT_TEST(TestConstructors); + UNIT_TEST(TestReplace); + UNIT_TEST(TestRefCount); + UNIT_TEST(TestFind); + UNIT_TEST(TestContains); + UNIT_TEST(TestOperators); + UNIT_TEST(TestLetOperator) + UNIT_TEST(TestMulOperators); + UNIT_TEST(TestFuncs); + UNIT_TEST(TestUtils); + UNIT_TEST(TestEmpty); + UNIT_TEST(TestJoin); + UNIT_TEST(TestCopy); + UNIT_TEST(TestStrCpy); + UNIT_TEST(TestPrefixSuffix); + UNIT_TEST(TestCharRef); + UNIT_TEST(TestBack); + UNIT_TEST(TestFront) + UNIT_TEST(TestDecodingMethods); + UNIT_TEST(TestIterators); + UNIT_TEST(TestReverseIterators); + UNIT_TEST(TestStringLiterals); + UNIT_TEST_SUITE_END(); + +private: + void TestDecodingMethods() { + UNIT_ASSERT(TUtf16CowString::FromAscii("").empty()); + UNIT_ASSERT(TUtf16CowString::FromAscii("abc") == ASCIIToWide("abc")); + +#if 0 // no wide convertions support + const char* text = "123kx83abcd ej)#$%ddja&%J&"; + TUtf16CowString wtext = ASCIIToWide(text); + + UNIT_ASSERT(wtext == TUtf16CowString::FromAscii(text)); + + TCowString strtext(text); + UNIT_ASSERT(wtext == TUtf16CowString::FromAscii(strtext)); + + TStringBuf strbuftext(text); + UNIT_ASSERT(wtext == TUtf16CowString::FromAscii(strbuftext)); + + UNIT_ASSERT(wtext.substr(5) == TUtf16CowString::FromAscii(text + 5)); + + const wchar16 wideCyrillicAlphabet[] = { + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, + 0x00}; + + TUtf16CowString strWide(wideCyrillicAlphabet); + TCowString strUtf8 = WideToUTF8(strWide); + + UNIT_ASSERT(strWide == TUtf16CowString::FromUtf8(strUtf8.c_str())); + UNIT_ASSERT(strWide == TUtf16CowString::FromUtf8(strUtf8)); + UNIT_ASSERT(strWide == TUtf16CowString::FromUtf8(TStringBuf(strUtf8))); + + // assign + + TUtf16CowString s1; + s1.AssignAscii("1234"); + UNIT_ASSERT(s1 == ASCIIToWide("1234")); + + s1.AssignUtf8(strUtf8); + UNIT_ASSERT(s1 == strWide); + + s1.AssignAscii(text); + UNIT_ASSERT(s1 == wtext); + + // append + + TUtf16CowString s2; + TUtf16CowString testAppend = strWide; + s2.AppendUtf8(strUtf8); + UNIT_ASSERT(testAppend == s2); + + testAppend += ' '; + s2.AppendAscii(" "); + UNIT_ASSERT(testAppend == s2); + + testAppend += '_'; + s2.AppendUtf8("_"); + UNIT_ASSERT(testAppend == s2); + + testAppend += wtext; + s2.AppendAscii(text); + UNIT_ASSERT(testAppend == s2); + + testAppend += wtext; + s2.AppendUtf8(text); + UNIT_ASSERT(testAppend == s2); +#endif + } + + void TestLetOperator() { + TUtf16CowString str; + + str = wchar16('X'); + UNIT_ASSERT(str == TUtf16CowString::FromAscii("X")); + + const TUtf16CowString hello = TUtf16CowString::FromAscii("hello"); + str = hello.data(); + UNIT_ASSERT(str == hello); + + str = hello; + UNIT_ASSERT(str == hello); + } + + void TestStringLiterals() { + TUtf16CowString s1 = u"hello"; + UNIT_ASSERT_VALUES_EQUAL(s1, TUtf16CowString::FromAscii("hello")); + + TUtf16CowString s2 = u"привет"; + UNIT_ASSERT_VALUES_EQUAL(s2, TUtf16CowString::FromUtf8("привет")); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TWideStringTest); + +class TUtf32StringTest: public TTestBase, private TStringTestImpl> { +public: + UNIT_TEST_SUITE(TUtf32StringTest); + UNIT_TEST(TestConstructors); + UNIT_TEST(TestReplace); + UNIT_TEST(TestRefCount); + UNIT_TEST(TestFind); + UNIT_TEST(TestContains); + UNIT_TEST(TestOperators); + UNIT_TEST(TestLetOperator) + UNIT_TEST(TestMulOperators); + UNIT_TEST(TestFuncs); + UNIT_TEST(TestUtils); + UNIT_TEST(TestEmpty); + UNIT_TEST(TestJoin); + UNIT_TEST(TestCopy); + UNIT_TEST(TestStrCpy); + UNIT_TEST(TestPrefixSuffix); + UNIT_TEST(TestCharRef); + UNIT_TEST(TestBack); + UNIT_TEST(TestFront) + UNIT_TEST(TestDecodingMethods); + UNIT_TEST(TestDecodingMethodsMixedStr); + UNIT_TEST(TestIterators); + UNIT_TEST(TestReverseIterators); + UNIT_TEST(TestStringLiterals); + UNIT_TEST_SUITE_END(); + +private: + void TestDecodingMethods() { + UNIT_ASSERT(TUtf32CowString::FromAscii("").empty()); + UNIT_ASSERT(TUtf32CowString::FromAscii("abc") == ASCIIToUTF32("abc")); + +#if 0 // no wide convertions support + const char* text = "123kx83abcd ej)#$%ddja&%J&"; + TUtf32CowString wtext = ASCIIToUTF32(text); + + UNIT_ASSERT(wtext == TUtf32CowString::FromAscii(text)); + + TCowString strtext(text); + UNIT_ASSERT(wtext == TUtf32CowString::FromAscii(strtext)); + + TStringBuf strbuftext(text); + UNIT_ASSERT(wtext == TUtf32CowString::FromAscii(strbuftext)); + + UNIT_ASSERT(wtext.substr(5) == TUtf32CowString::FromAscii(text + 5)); + + const wchar32 wideCyrillicAlphabet[] = { + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, + 0x00}; + + TUtf32CowString strWide(wideCyrillicAlphabet); + TCowString strUtf8 = WideToUTF8(strWide); + + UNIT_ASSERT(strWide == TUtf32CowString::FromUtf8(strUtf8.c_str())); + UNIT_ASSERT(strWide == TUtf32CowString::FromUtf8(strUtf8)); + UNIT_ASSERT(strWide == TUtf32CowString::FromUtf8(TStringBuf(strUtf8))); + + // assign + + TUtf32CowString s1; + s1.AssignAscii("1234"); + UNIT_ASSERT(s1 == ASCIIToUTF32("1234")); + + s1.AssignUtf8(strUtf8); + UNIT_ASSERT(s1 == strWide); + + s1.AssignAscii(text); + UNIT_ASSERT(s1 == wtext); + + // append + + TUtf32CowString s2; + TUtf32CowString testAppend = strWide; + s2.AppendUtf8(strUtf8); + UNIT_ASSERT(testAppend == s2); + + testAppend += ' '; + s2.AppendAscii(" "); + UNIT_ASSERT(testAppend == s2); + + testAppend += '_'; + s2.AppendUtf8("_"); + UNIT_ASSERT(testAppend == s2); + + testAppend += wtext; + s2.AppendAscii(text); + UNIT_ASSERT(testAppend == s2); + + testAppend += wtext; + s2.AppendUtf8(text); + + UNIT_ASSERT(testAppend == s2); +#endif + } + + void TestDecodingMethodsMixedStr() { + UNIT_ASSERT(TUtf32CowString::FromAscii("").empty()); + UNIT_ASSERT(TUtf32CowString::FromAscii("abc") == ASCIIToUTF32("abc")); + +#if 0 // no wide convertions support + const char* text = "123kx83abcd ej)#$%ddja&%J&"; + TUtf32CowString wtext = ASCIIToUTF32(text); + + UNIT_ASSERT(wtext == TUtf32CowString::FromAscii(text)); + + TCowString strtext(text); + UNIT_ASSERT(wtext == TUtf32CowString::FromAscii(strtext)); + + TStringBuf strbuftext(text); + UNIT_ASSERT(wtext == TUtf32CowString::FromAscii(strbuftext)); + + UNIT_ASSERT(wtext.substr(5) == TUtf32CowString::FromAscii(text + 5)); + + const wchar32 cyrilicAndLatinWide[] = { + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, + wchar32('z'), + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, + wchar32('z'), + 0x00}; + + TUtf32CowString strWide(cyrilicAndLatinWide); + TCowString strUtf8 = WideToUTF8(strWide); + + UNIT_ASSERT(strWide == TUtf32CowString::FromUtf8(strUtf8.c_str())); + UNIT_ASSERT(strWide == TUtf32CowString::FromUtf8(strUtf8)); + UNIT_ASSERT(strWide == UTF8ToUTF32(strUtf8)); + UNIT_ASSERT(strWide == UTF8ToUTF32(strUtf8)); + UNIT_ASSERT(strWide == TUtf32CowString::FromUtf8(TStringBuf(strUtf8))); + + // assign + + TUtf32CowString s1; + s1.AssignAscii("1234"); + UNIT_ASSERT(s1 == ASCIIToUTF32("1234")); + + s1.AssignUtf8(strUtf8); + UNIT_ASSERT(s1 == strWide); + + s1.AssignAscii(text); + UNIT_ASSERT(s1 == wtext); + + // append + + TUtf32CowString s2; + TUtf32CowString testAppend = strWide; + s2.AppendUtf16(UTF8ToWide(strUtf8)); + UNIT_ASSERT(testAppend == s2); + + testAppend += ' '; + s2.AppendAscii(" "); + UNIT_ASSERT(testAppend == s2); + + testAppend += '_'; + s2.AppendUtf8("_"); + UNIT_ASSERT(testAppend == s2); + + testAppend += wtext; + s2.AppendAscii(text); + UNIT_ASSERT(testAppend == s2); + + testAppend += wtext; + s2.AppendUtf8(text); + + UNIT_ASSERT(testAppend == s2); +#endif + } + + void TestLetOperator() { + TUtf32CowString str; + + str = wchar32('X'); + UNIT_ASSERT(str == TUtf32CowString::FromAscii("X")); + + const TUtf32CowString hello = TUtf32CowString::FromAscii("hello"); + str = hello.data(); + UNIT_ASSERT(str == hello); + + str = hello; + UNIT_ASSERT(str == hello); + } + + void TestStringLiterals() { + TUtf32CowString s1 = U"hello"; + UNIT_ASSERT_VALUES_EQUAL(s1, TUtf32CowString::FromAscii("hello")); + + TUtf32CowString s2 = U"привет"; + UNIT_ASSERT_VALUES_EQUAL(s2, TUtf32CowString::FromUtf8("привет")); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TUtf32StringTest); + +class TStringStdTest: public TTestBase, private TStringStdTestImpl> { +public: + UNIT_TEST_SUITE(TStringStdTest); + UNIT_TEST(Constructor); + UNIT_TEST(reserve); + UNIT_TEST(short_string); + UNIT_TEST(erase); + UNIT_TEST(data); + UNIT_TEST(c_str); + UNIT_TEST(null_char_of_empty); + UNIT_TEST(null_char); + UNIT_TEST(null_char_assignment_to_subscript_of_empty); + UNIT_TEST(null_char_assignment_to_subscript_of_nonempty); + UNIT_TEST(null_char_assignment_to_end_of_empty); + UNIT_TEST(null_char_assignment_to_end_of_nonempty); + UNIT_TEST(insert); + UNIT_TEST(resize); + UNIT_TEST(find); + UNIT_TEST(capacity); + UNIT_TEST(assign); + UNIT_TEST(copy); + UNIT_TEST(cbegin_cend); + UNIT_TEST(compare); + UNIT_TEST(find_last_of); +#if 0 + UNIT_TEST(rfind); + UNIT_TEST(replace); +#endif + UNIT_TEST(find_last_not_of); + UNIT_TEST_SUITE_END(); +}; + +UNIT_TEST_SUITE_REGISTRATION(TStringStdTest); + +class TWideStringStdTest: public TTestBase, private TStringStdTestImpl> { +public: + UNIT_TEST_SUITE(TWideStringStdTest); + UNIT_TEST(Constructor); + UNIT_TEST(reserve); + UNIT_TEST(short_string); + UNIT_TEST(erase); + UNIT_TEST(data); + UNIT_TEST(c_str); + UNIT_TEST(null_char_of_empty); + UNIT_TEST(null_char); + UNIT_TEST(null_char_assignment_to_subscript_of_empty); + UNIT_TEST(null_char_assignment_to_subscript_of_nonempty); + UNIT_TEST(null_char_assignment_to_end_of_empty); + UNIT_TEST(null_char_assignment_to_end_of_nonempty); + UNIT_TEST(insert); + UNIT_TEST(resize); + UNIT_TEST(find); + UNIT_TEST(capacity); + UNIT_TEST(assign); + UNIT_TEST(copy); + UNIT_TEST(cbegin_cend); + UNIT_TEST(compare); + UNIT_TEST(find_last_of); +#if 0 + UNIT_TEST(rfind); + UNIT_TEST(replace); +#endif + UNIT_TEST(find_last_not_of); + UNIT_TEST_SUITE_END(); +}; + +UNIT_TEST_SUITE_REGISTRATION(TWideStringStdTest); + +Y_UNIT_TEST_SUITE(TCowStringSerializationTest) { + TCowString SerializeThereAndBack(const TCowString& value) { + std::array buf; + TMemoryWriteBuffer out{buf.data(), buf.size()}; + Save(&out, value); + + TMemoryInput in{buf.data(), out.Len()}; + TCowString deserialized; + Load(&in, deserialized); + return deserialized; + } + + Y_UNIT_TEST(EmptyStringSerializationTest) { + TCowString nothing{}; + TCowString deserialized = SerializeThereAndBack(nothing); + + UNIT_ASSERT_VALUES_EQUAL(nothing, deserialized); + } + + Y_UNIT_TEST(RegularStringSerializationTest) { + TCowString abra = "cadabra"; + TCowString deserialized = SerializeThereAndBack(abra); + + UNIT_ASSERT_VALUES_EQUAL(abra, deserialized); + } +} // Y_UNIT_TEST_SUITE(TCowStringSerializationTest) + +Y_UNIT_TEST_SUITE(TStringConversionTest) { + Y_UNIT_TEST(ConversionToStdStringTest) { + TCowString abra = "cadabra"; + std::string stdAbra = abra; + UNIT_ASSERT_VALUES_EQUAL(stdAbra, "cadabra"); + } + + Y_UNIT_TEST(ConversionToStdStringViewTest) { + TCowString abra = "cadabra"; + std::string_view stdAbra = abra; + UNIT_ASSERT_VALUES_EQUAL(stdAbra, "cadabra"); + } +} // Y_UNIT_TEST_SUITE(TStringConversionTest) + +Y_UNIT_TEST_SUITE(HashFunctorTests) { + Y_UNIT_TEST(TestTransparency) { + THash h; + const char* ptr = "a"; + const TStringBuf strbuf = ptr; + const TCowString str = ptr; + const std::string stdStr = ptr; + UNIT_ASSERT_VALUES_EQUAL(h(ptr), h(strbuf)); + UNIT_ASSERT_VALUES_EQUAL(h(ptr), h(str)); + UNIT_ASSERT_VALUES_EQUAL(h(ptr), h(stdStr)); + } +} // Y_UNIT_TEST_SUITE(HashFunctorTests) + +Y_UNIT_TEST_SUITE(StdNonConformant) { + Y_UNIT_TEST(TestEraseNoThrow) { + TCowString x; + + LegacyErase(x, 10); + } + + Y_UNIT_TEST(TestReplaceNoThrow) { + TCowString x; + + LegacyReplace(x, 0, 0, "1"); + + UNIT_ASSERT_VALUES_EQUAL(x, "1"); + + LegacyReplace(x, 10, 0, "1"); + + UNIT_ASSERT_VALUES_EQUAL(x, "1"); + } + + Y_UNIT_TEST(TestNoAlias) { + TCowString s = "x"; + + s.AppendNoAlias("abc", 3); + + UNIT_ASSERT_VALUES_EQUAL(s, "xabc"); + UNIT_ASSERT_VALUES_EQUAL(TCowString(s.c_str()), "xabc"); + } +} // Y_UNIT_TEST_SUITE(StdNonConformant) + +Y_UNIT_TEST_SUITE(Interop) { + static void Mutate(std::string& s) { + s += "y"; + } + + static void Mutate(TCowString& s) { + Mutate(MutRef(s)); + } + + Y_UNIT_TEST(TestMutate) { + TCowString x = "x"; + + Mutate(x); + + UNIT_ASSERT_VALUES_EQUAL(x, "xy"); + } + + static std::string TransformStd(const std::string& s) { + return s + "y"; + } + + static TCowString Transform(const TCowString& s) { + return TransformStd(s); + } + + Y_UNIT_TEST(TestTransform) { + UNIT_ASSERT_VALUES_EQUAL(Transform(TCowString("x")), "xy"); + } + + Y_UNIT_TEST(TestTemp) { + UNIT_ASSERT_VALUES_EQUAL("x" + ConstRef(TCowString("y")), "xy"); + } + + static void ComparePointers(const std::string& s, const void* expected, TStringBuf descr) { + UNIT_ASSERT_VALUES_EQUAL_C(static_cast(s.c_str()), expected, descr); + } + + Y_UNIT_TEST(TestConstShared) { + TCowString s(600, 'a'); + const void* stringStart = s.c_str(); + ComparePointers(s, stringStart, "unique"); + TCowString shared{s}; + ComparePointers(s, stringStart, "shared"); // converting a TCowString to a `const std::string&` should not cause data cloning + } +} // Y_UNIT_TEST_SUITE(Interop) + +Y_UNIT_TEST_SUITE(CowPitfalls) { + template + static TString CopyStringViaBeginEndIterators(T& string, bool reverse) { + decltype(string.begin()) b; + decltype(string.end()) e; + if (!reverse) { + b = string.begin(); + e = string.end(); + } else { + e = string.end(); + b = string.begin(); + } + return TString{b, e}; + } + + Y_UNIT_TEST(IteratorCallOrder) { + const TString ref(600, 'a'); + for (const bool reverse : {false, true}) { + TCowString s = {ref.begin(), ref.end()}; + // sanity check + UNIT_ASSERT_VALUES_EQUAL_C(CopyStringViaBeginEndIterators(s, reverse), TStringBuf(ref), LabeledOutput(reverse)); + UNIT_ASSERT_VALUES_EQUAL_C(CopyStringViaBeginEndIterators(s, reverse), TStringBuf(ref), LabeledOutput(reverse)); + // test + TCowString copy = s; + UNIT_ASSERT_VALUES_EQUAL_C(CopyStringViaBeginEndIterators(s, reverse), TStringBuf(ref), LabeledOutput(reverse)); + UNIT_ASSERT_VALUES_EQUAL_C(CopyStringViaBeginEndIterators(s, reverse), TStringBuf(ref), LabeledOutput(reverse)); + } + } + + Y_UNIT_TEST(RangeFor) { + TCowString str; + str.resize(200); + TCowString copy = str; + for (auto& c : str) { + c = 'x'; + } + UNIT_ASSERT_VALUES_EQUAL(str, TString(200, 'x')); + } +} // Y_UNIT_TEST_SUITE(CowPitfalls) diff --git a/library/cpp/containers/cow_string/output.cpp b/library/cpp/containers/cow_string/output.cpp new file mode 100644 index 00000000000..e0b4924ad31 --- /dev/null +++ b/library/cpp/containers/cow_string/output.cpp @@ -0,0 +1,46 @@ +#include "cow_string.h" + +#include +#include +#include + +constexpr size_t MAX_UTF8_BYTES = 4; // UTF-8-encoded code point takes between 1 and 4 bytes + +template +static void WriteString(IOutputStream& o, const TCharType* w, size_t n) { + const size_t buflen = (n * MAX_UTF8_BYTES); // * 4 because the conversion functions can convert unicode character into maximum 4 bytes of UTF8 + TTempBuf buffer(buflen + 1); + size_t written = 0; + WideToUTF8(w, n, buffer.Data(), written); + o.Write(buffer.Data(), written); +} + +template <> +void Out(IOutputStream& o, const TCowString& p) { + o.Write(p.data(), p.size()); +} + +template <> +void Out(IOutputStream& o, const TUtf16CowString& w) { + WriteString(o, w.c_str(), w.size()); +} + +template <> +void Out(IOutputStream& o, const TUtf32CowString& w) { + WriteString(o, w.c_str(), w.size()); +} + +template <> +void Out>(IOutputStream& o, const TBasicCharRef& c) { + o << static_cast(c); +} + +template <> +void Out>(IOutputStream& o, const TBasicCharRef& c) { + o << static_cast(c); +} + +template <> +void Out>(IOutputStream& o, const TBasicCharRef& c) { + o << static_cast(c); +} diff --git a/library/cpp/containers/cow_string/reverse.cpp b/library/cpp/containers/cow_string/reverse.cpp new file mode 100644 index 00000000000..b5bd10d250a --- /dev/null +++ b/library/cpp/containers/cow_string/reverse.cpp @@ -0,0 +1,32 @@ +#include "reverse.h" + +#include +#include + +#include + +void ReverseInPlace(TCowString& string) { + auto* begin = string.begin(); + std::reverse(begin, begin + string.size()); +} + +void ReverseInPlace(TUtf16CowString& string) { + auto* begin = string.begin(); + const auto len = string.size(); + auto* end = begin + string.size(); + + TVector buffer(len); + wchar16* rbegin = buffer.data() + len; + for (wchar16* p = begin; p < end;) { + const size_t symbolSize = W16SymbolSize(p, end); + rbegin -= symbolSize; + std::copy(p, p + symbolSize, rbegin); + p += symbolSize; + } + std::copy(buffer.begin(), buffer.end(), begin); +} + +void ReverseInPlace(TUtf32CowString& string) { + auto* begin = string.begin(); + std::reverse(begin, begin + string.size()); +} diff --git a/library/cpp/containers/cow_string/reverse.h b/library/cpp/containers/cow_string/reverse.h new file mode 100644 index 00000000000..d27b0b4fed6 --- /dev/null +++ b/library/cpp/containers/cow_string/reverse.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +void ReverseInPlace(TCowString& string); + +/** NB. UTF-16 is variable-length encoding because of the surrogate pairs. + * This function takes this into account and treats a surrogate pair as a single symbol. + * Ex. if [C D] is a surrogate pair, + * A B [C D] E + * will become + * E [C D] B A + */ +void ReverseInPlace(TUtf16CowString& string); + +void ReverseInPlace(TUtf32CowString& string); diff --git a/library/cpp/containers/cow_string/str_stl.h b/library/cpp/containers/cow_string/str_stl.h new file mode 100644 index 00000000000..d8256a6e10b --- /dev/null +++ b/library/cpp/containers/cow_string/str_stl.h @@ -0,0 +1,67 @@ +#pragma once + +#include + +template <> +struct hash: ::NHashPrivate::TStringHash { +}; + +template <> +struct hash: ::NHashPrivate::TStringHash { +}; + +template <> +struct hash: ::NHashPrivate::TStringHash { +}; + +template <> +struct TEqualTo: public TEqualTo { + using is_transparent = void; +}; + +template <> +struct TEqualTo: public TEqualTo { + using is_transparent = void; +}; + +template <> +struct TEqualTo: public TEqualTo { + using is_transparent = void; +}; + +template <> +struct TCIEqualTo { + inline bool operator()(const TCowString& a, const TCowString& b) const { + return a.size() == b.size() && strnicmp(a.data(), b.data(), a.size()) == 0; + } +}; + +template <> +struct TLess: public TLess { + using is_transparent = void; +}; + +template <> +struct TLess: public TLess { + using is_transparent = void; +}; + +template <> +struct TLess: public TLess { + using is_transparent = void; +}; + +template <> +struct TGreater: public TGreater { + using is_transparent = void; +}; + +template <> +struct TGreater: public TGreater { + using is_transparent = void; +}; + +template <> +struct TGreater: public TGreater { + using is_transparent = void; +}; diff --git a/library/cpp/containers/cow_string/subst.cpp b/library/cpp/containers/cow_string/subst.cpp new file mode 100644 index 00000000000..d4e9ff3395d --- /dev/null +++ b/library/cpp/containers/cow_string/subst.cpp @@ -0,0 +1,182 @@ +#include "subst.h" + +#include +#include +#include + +#include +#include + +// a bit of template magic (to be fast and unreadable) +template +static Y_FORCE_INLINE void MoveBlock(typename TStringType::value_type* ptr, size_t& srcPos, size_t& dstPos, const size_t off, const TTo to, const size_t toSize) { + const size_t unchangedSize = off - srcPos; + if (dstPos < srcPos) { + for (size_t i = 0; i < unchangedSize; ++i) { + ptr[dstPos++] = ptr[srcPos++]; + } + } else { + dstPos += unchangedSize; + srcPos += unchangedSize; + } + + if (Main) { + for (size_t i = 0; i < toSize; ++i) { + ptr[dstPos++] = to[i]; + } + } +} + +template +static bool IsIntersect(const T& a, const U& b) noexcept { + if (b.data() < a.data()) { + return IsIntersect(b, a); + } + + return !a.empty() && !b.empty() && + ((a.data() <= b.data() && b.data() < a.data() + a.size()) || + (a.data() < b.data() + b.size() && b.data() + b.size() <= a.data() + a.size())); +} + +/** + * Replaces all occurences of substring @c from in string @c s to string @c to. + * Uses two separate implementations (inplace for shrink and append for grow case) + * See IGNIETFERRO-394 + **/ +template > +static inline size_t SubstGlobalImpl(TStringType& s, const TStringViewType from, const TStringViewType to, size_t fromPos = 0) { + if (from.empty()) { + return 0; + } + + Y_ASSERT(!IsIntersect(s, from)); + Y_ASSERT(!IsIntersect(s, to)); + + const size_t fromSize = from.size(); + const size_t toSize = to.size(); + size_t replacementsCount = 0; + size_t off = fromPos; + size_t srcPos = 0; + + if (toSize > fromSize) { + // string will grow: append to another string + TStringType result; + for (; (off = TStringViewType(s).find(from, off)) != TStringType::npos; off += fromSize) { + if (!replacementsCount) { + // first replacement occured, we can prepare result string + result.reserve(s.size() + s.size() / 3); + } + result.append(s.begin() + srcPos, s.begin() + off); + result.append(to.data(), to.size()); + srcPos = off + fromSize; + ++replacementsCount; + } + if (replacementsCount) { + // append tail + result.append(s.begin() + srcPos, s.end()); + s = std::move(result); + } + return replacementsCount; + } + + // string will not grow: use inplace algo + size_t dstPos = 0; + typename TStringType::value_type* ptr = &*s.begin(); + for (; (off = TStringViewType(s).find(from, off)) != TStringType::npos; off += fromSize) { + Y_ASSERT(dstPos <= srcPos); + MoveBlock(ptr, srcPos, dstPos, off, to, toSize); + srcPos = off + fromSize; + ++replacementsCount; + } + + if (replacementsCount) { + // append tail + MoveBlock(ptr, srcPos, dstPos, s.size(), to, toSize); + s.resize(dstPos); + } + return replacementsCount; +} + +/// Replaces all occurences of the 'from' symbol in a string to the 'to' symbol. +template +inline size_t SubstCharGlobalImpl(TStringType& s, typename TStringType::value_type from, typename TStringType::value_type to, size_t fromPos = 0) { + if (fromPos >= s.size()) { + return 0; + } + + size_t result = 0; + fromPos = s.find(from, fromPos); + + // s.begin() might cause memory copying, so call it only if needed + if (fromPos != TStringType::npos) { + auto* it = &*s.begin() + fromPos; + *it = to; + ++result; + // at this point string is copied and it's safe to use constant s.end() to iterate + const auto* const sEnd = &*s.end(); + // unrolled loop goes first because it is more likely that `it` will be properly aligned + for (const auto* const end = sEnd - (sEnd - it) % 4; it < end;) { + if (*it == from) { + *it = to; + ++result; + } + ++it; + if (*it == from) { + *it = to; + ++result; + } + ++it; + if (*it == from) { + *it = to; + ++result; + } + ++it; + if (*it == from) { + *it = to; + ++result; + } + ++it; + } + for (; it < sEnd; ++it) { + if (*it == from) { + *it = to; + ++result; + } + } + } + + return result; +} + +/* Standard says that `char16_t` is a distinct type and has same size, signedness and alignment as + * `std::uint_least16_t`, so we check if `char16_t` has same signedness and size as `wchar16` to be + * sure that we can make safe casts between values of these types and pointers. + */ +static_assert(sizeof(wchar16) == sizeof(char16_t), ""); +static_assert(sizeof(wchar32) == sizeof(char32_t), ""); +static_assert(std::is_unsigned::value == std::is_unsigned::value, ""); +static_assert(std::is_unsigned::value == std::is_unsigned::value, ""); + +size_t SubstGlobal(TCowString& text, const TStringBuf what, const TStringBuf with, size_t from) { + return SubstGlobalImpl(text, what, with, from); +} + +size_t SubstGlobal(TUtf16CowString& text, const TWtringBuf what, const TWtringBuf with, size_t from) { + return SubstGlobalImpl(text, what, with, from); +} + +size_t SubstGlobal(TUtf32CowString& text, const TUtf32StringBuf what, const TUtf32StringBuf with, size_t from) { + return SubstGlobalImpl(text, what, with, from); +} + +size_t SubstGlobal(TCowString& text, char what, char with, size_t from) { + return SubstCharGlobalImpl(text, what, with, from); +} + +size_t SubstGlobal(TUtf16CowString& text, wchar16 what, wchar16 with, size_t from) { + return SubstCharGlobalImpl(text, (char16_t)what, (char16_t)with, from); +} + +size_t SubstGlobal(TUtf32CowString& text, wchar32 what, wchar32 with, size_t from) { + return SubstCharGlobalImpl(text, (char32_t)what, (char32_t)with, from); +} diff --git a/library/cpp/containers/cow_string/subst.h b/library/cpp/containers/cow_string/subst.h new file mode 100644 index 00000000000..6090ba54b25 --- /dev/null +++ b/library/cpp/containers/cow_string/subst.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include + +/* Replace all occurences of substring `what` with string `with` starting from position `from`. + * + * @param text String to modify. + * @param what Substring to replace. + * @param with Substring to use as replacement. + * @param from Position at with to start replacement. + * + * @return Number of replacements occured. + */ +size_t SubstGlobal(TCowString& text, TStringBuf what, TStringBuf with, size_t from = 0); +size_t SubstGlobal(TUtf16CowString& text, TWtringBuf what, TWtringBuf with, size_t from = 0); +size_t SubstGlobal(TUtf32CowString& text, TUtf32StringBuf what, TUtf32StringBuf with, size_t from = 0); + +/* Replace all occurences of character `what` with character `with` starting from position `from`. + * + * @param text String to modify. + * @param what Character to replace. + * @param with Character to use as replacement. + * @param from Position at with to start replacement. + * + * @return Number of replacements occured. + */ +size_t SubstGlobal(TCowString& text, char what, char with, size_t from = 0); +size_t SubstGlobal(TUtf16CowString& text, wchar16 what, wchar16 with, size_t from = 0); +size_t SubstGlobal(TUtf32CowString& text, wchar32 what, wchar32 with, size_t from = 0); diff --git a/library/cpp/containers/cow_string/ut_medium/cow_string_medium_ut.cpp b/library/cpp/containers/cow_string/ut_medium/cow_string_medium_ut.cpp new file mode 100644 index 00000000000..a9a37db776d --- /dev/null +++ b/library/cpp/containers/cow_string/ut_medium/cow_string_medium_ut.cpp @@ -0,0 +1,55 @@ +#include + +#include + +#include +#include +#include +#include + +#include +#include + +static_assert(sizeof(TCowString) == sizeof(const char*), "expect sizeof(TCowString) == sizeof(const char*)"); + +Y_UNIT_TEST_SUITE(CowPitfalls) { + Y_UNIT_TEST(ParallelDetach) { + // best results with thread-sanitizer + std::vector> threads; + TCowString a = "the string"; + TCowString b = a; + auto makeRefToA = [&a, &b]() { + b = a; // make second reference to the same string + }; + constexpr int nThreads = 8; +#ifdef _tsan_enabled_ + constexpr i64 retries = 1'000; +#else + constexpr i64 retries = 1'000'000; +#endif + std::barrier iterationSyncPoint(nThreads, makeRefToA); + std::atomic totalLen = 0; + auto addLen = [](std::string a, std::atomic& len) { + len += a.length(); + }; + auto workload = [&a, &addLen, &totalLen, &iterationSyncPoint]() { + std::atomic len = 0; + for (i64 j = 0; j < retries; ++j) { + addLen(a, len); // possibility of bad implicit conversion + iterationSyncPoint.arrive_and_wait(); + } + totalLen += len.load(); + }; + for (int i = 0; i < nThreads; ++i) { + threads.push_back(std::make_unique(workload)); + } + for (auto& t : threads) { + t->Start(); + } + for (auto& t : threads) { + t->Join(); + } + UNIT_ASSERT_VALUES_EQUAL(totalLen.load(), b.size() * nThreads * retries); + } + +} // Y_UNIT_TEST_SUITE(CowPitfalls) diff --git a/library/cpp/containers/cow_string/ysaveload.cpp b/library/cpp/containers/cow_string/ysaveload.cpp new file mode 100644 index 00000000000..57555fc1d9e --- /dev/null +++ b/library/cpp/containers/cow_string/ysaveload.cpp @@ -0,0 +1 @@ +#include "ysaveload.h" diff --git a/library/cpp/containers/cow_string/ysaveload.h b/library/cpp/containers/cow_string/ysaveload.h new file mode 100644 index 00000000000..a4da801c404 --- /dev/null +++ b/library/cpp/containers/cow_string/ysaveload.h @@ -0,0 +1,9 @@ +#pragma once + +#include "cow_string.h" + +#include + +template <> +class TSerializer: public TVectorSerializer { +}; diff --git a/library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h b/library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h index f0c6644d4be..c5bfb659acb 100644 --- a/library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h +++ b/library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h @@ -115,7 +115,8 @@ class TDisjointIntervalTree { if (containingBegin->first < begin && begin < containingBegin->second) { // Contains begin. if (containingBegin->second > end) { // Contains end. const T prevEnd = containingBegin->second; - Y_ASSERT(containingBegin->second - begin <= NumElements); + Y_ASSERT(containingBegin->second >= begin); + Y_ASSERT(static_cast(containingBegin->second - begin) <= NumElements); Y_ASSERT(containingBegin->second - containingBegin->first > end - begin); containingBegin->second = begin; diff --git a/library/cpp/containers/disjoint_interval_tree/ut/disjoint_interval_tree_ut.cpp b/library/cpp/containers/disjoint_interval_tree/ut/disjoint_interval_tree_ut.cpp index 508a82459af..69278c31f73 100644 --- a/library/cpp/containers/disjoint_interval_tree/ut/disjoint_interval_tree_ut.cpp +++ b/library/cpp/containers/disjoint_interval_tree/ut/disjoint_interval_tree_ut.cpp @@ -288,4 +288,23 @@ Y_UNIT_TEST_SUITE(DisjointIntervalTreeTest) { UNIT_ASSERT(!tree.Intersects(15, 18)); } } + + Y_UNIT_TEST(TestI64) { + { + TDisjointIntervalTree tree; + tree.InsertInterval(-5, 10); + UNIT_ASSERT_VALUES_EQUAL(tree.EraseInterval(-2, 4), 6); + UNIT_ASSERT_VALUES_EQUAL(tree.GetNumIntervals(), 2); + UNIT_ASSERT_VALUES_EQUAL(tree.GetNumElements(), 9); + + UNIT_ASSERT_VALUES_EQUAL(tree.EraseInterval(-5, -2), 3); + UNIT_ASSERT_VALUES_EQUAL(tree.GetNumIntervals(), 1); + UNIT_ASSERT_VALUES_EQUAL(tree.GetNumElements(), 6); + + UNIT_ASSERT_VALUES_EQUAL(tree.EraseInterval(4, 10), 6); + UNIT_ASSERT_VALUES_EQUAL(tree.GetNumIntervals(), 0); + UNIT_ASSERT_VALUES_EQUAL(tree.GetNumElements(), 0); + UNIT_ASSERT(tree.Empty()); + } + } } diff --git a/library/cpp/containers/paged_vector/README.md b/library/cpp/containers/paged_vector/README.md new file mode 100644 index 00000000000..ae49c0c6523 --- /dev/null +++ b/library/cpp/containers/paged_vector/README.md @@ -0,0 +1,99 @@ +# TPagedVector + +`NPagedVector::TPagedVector` is a dynamic sequence container implemented as a 2-level radix tree: elements are stored in fixed-size, individually heap-allocated pages, and a top-level vector holds pointers to those pages. + +```cpp +#include + +namespace NPagedVector { + template + class TPagedVector; +} +``` + +- `T` — element type. +- `PageSize` — number of elements per page (default: `1u << 20u` = 1,048,576 elements). + +## Why use it instead of TVector / std::vector? + +- **No reallocation of elements.** Growth allocates a new page instead of reallocating and moving the entire buffer. Elements are never moved on `push_back`/`emplace_back`, so references and pointers to existing elements remain valid when appending (iterators are offset-based and also stay usable). +- **No large contiguous allocations.** Memory is requested in page-size chunks, which is friendlier to the allocator for very large containers. +- **Cheaper worst-case append.** `push_back` never triggers an O(n) copy; the cost is at most one page allocation. + +The trade-off is that storage is not contiguous (no `data()`), and indexing does one extra pointer dereference (`idx / PageSize`, `idx % PageSize`). + +## API overview + +The interface mirrors a subset of `std::vector`: + +| Category | Members | +|---|---| +| Construction | default, copy, move, `TPagedVector(TIter b, TIter e)` | +| Assignment | copy, move, `swap()` | +| Element access | `operator[]`, `at()` (throws `std::out_of_range`), `front()`, `back()` | +| Iterators | `begin()/end()`, `rbegin()/rend()` + const versions; random-access iterators | +| Capacity | `size()`, `empty()`, `explicit operator bool()` (true when non-empty) | +| Modifiers | `push_back()`, `emplace_back()` (returns a reference), `pop_back()`, `append(b, e)`, `erase(it)`, `erase(b, e)`, `resize()`, `clear()` | +| Iteration helpers | `ForEach(fn)`, `ForEachReverse(fn)` | +| Comparison | `operator==`, `operator<` (lexicographical) | + +Notable differences from `std::vector`: + +- No `reserve()`/`capacity()`/`shrink_to_fit()` and no `data()` — storage is paged, not contiguous. + +## Iterators + +Iterators are random-access and are implemented as an *(owner pointer, offset)* pair. Consequences: + +- Iterators are not invalidated by `push_back`/`emplace_back` (an `end()` iterator taken earlier keeps pointing to the same logical position). +- Dereferencing goes through the vector, so an iterator is only valid while its source container is alive. +- To get the current index of an element from an iterator, call `it.GetIndex()` — it returns the offset of the pointed-to element within the container (equivalent to `it - begin()`). + +## Iteration helpers + +```cpp +template +void ForEach(Function fn) const; + +template +void ForEachReverse(Function fn) const; +``` + +`ForEach` applies `fn` to every element **from the first to the last**; `ForEachReverse` applies `fn` **from the last to the first**. + +These are faster than iterating with `begin()/end()` or `rbegin()/rend()`: they walk the pages directly through raw pointers, avoiding the two levels of indirection that the offset-based iterators go through on each dereference. This matters for containers with a large `PageSize` (the default is 1M elements per page), where the inner per-page loop is tight. + +```cpp +TPagedVector v; +// ... fill v ... + +long long sum = 0; +v.ForEach([&](int x) { sum += x; }); + +// process elements back-to-front, e.g. for a stack-like traversal +v.ForEachReverse([&](int x) { + // ... +}); +``` + +Notes: + +- The order is well-defined and contiguous: `ForEach` visits element `0, 1, ..., size()-1`; `ForEachReverse` visits `size()-1, ..., 1, 0`. +- Both are O(n) and do not allocate. + +## Complexity + +| Operation | Complexity | +|---|---| +| `operator[]` / `at()` | O(1) | +| `push_back` / `emplace_back` | O(1) amortized (page allocation at most every `PageSize` appends) | +| `pop_back` | O(1) | +| `erase` | O(n) — shifts all following elements | +| `clear` | O(n) for non-trivially destructible `T`, O(pages) otherwise | +| `ForEach` / `ForEachReverse` | O(n), no allocations | + +## Notes + +- Pages are allocated as raw storage; elements are constructed in place and destroyed explicitly, so non-trivially destructible types are handled correctly. +- Destruction of trivially destructible types is skipped entirely, making `clear()` and the destructor fast for POD-like types. +- The copy constructor is exception-safe: on a throw during copying, already-constructed elements are destroyed. diff --git a/library/cpp/containers/paged_vector/paged_vector.h b/library/cpp/containers/paged_vector/paged_vector.h index 3a2c58caf7b..43073852ab3 100644 --- a/library/cpp/containers/paged_vector/paged_vector.h +++ b/library/cpp/containers/paged_vector/paged_vector.h @@ -4,90 +4,91 @@ #include #include +#include #include namespace NPagedVector { - template > + template class TPagedVector; namespace NPrivate { - template + template struct TPagedVectorIterator { private: - friend class TPagedVector; - typedef TPagedVector TVec; - typedef TPagedVectorIterator TSelf; - size_t Offset; - TVec* Vector; + friend class TPagedVector; + using TVec = TPagedVector; + using TSelf = TPagedVectorIterator; + size_t Index_; + TVec* Vector_; - template + template friend struct TPagedVectorIterator; public: TPagedVectorIterator() - : Offset() - , Vector() + : Index_() + , Vector_() { } - TPagedVectorIterator(TVec* vector, size_t offset) - : Offset(offset) - , Vector(vector) + TPagedVectorIterator(TVec* vector, size_t index) + : Index_(index) + , Vector_(vector) { } - template - TPagedVectorIterator(const TPagedVectorIterator& it) - : Offset(it.Offset) - , Vector(it.Vector) + template + TPagedVectorIterator(const TPagedVectorIterator& it) + : Index_(it.Index_) + , Vector_(it.Vector_) { } T& operator*() const { - return (*Vector)[Offset]; + return (*Vector_)[Index_]; } T* operator->() const { return &(**this); } - template - bool operator==(const TPagedVectorIterator& it) const { - return Offset == it.Offset; + template + bool operator==(const TPagedVectorIterator& it) const { + return Index_ == it.Index_; } - template - bool operator!=(const TPagedVectorIterator& it) const { + template + bool operator!=(const TPagedVectorIterator& it) const { return !(*this == it); } - template - bool operator<(const TPagedVectorIterator& it) const { - return Offset < it.Offset; + template + bool operator<(const TPagedVectorIterator& it) const { + return Index_ < it.Index_; } - template - bool operator<=(const TPagedVectorIterator& it) const { - return Offset <= it.Offset; + template + bool operator<=(const TPagedVectorIterator& it) const { + return Index_ <= it.Index_; } - template - bool operator>(const TPagedVectorIterator& it) const { + template + bool operator>(const TPagedVectorIterator& it) const { return !(*this <= it); } - template - bool operator>=(const TPagedVectorIterator& it) const { + template + bool operator>=(const TPagedVectorIterator& it) const { return !(*this < it); } - template - ptrdiff_t operator-(const TPagedVectorIterator& it) const { - return Offset - it.Offset; + template + ptrdiff_t operator-(const TPagedVectorIterator& it) const { + return Index_ - it.Index_; } TSelf& operator+=(ptrdiff_t off) { - Offset += off; + Index_ += off; return *this; } @@ -125,51 +126,121 @@ namespace NPagedVector { return this->operator+(-off); } - size_t GetOffset() const { - return Offset; + [[nodiscard]] size_t GetIndex() const { + return Index_; } }; - } -} + } // namespace NPrivate +} // namespace NPagedVector namespace std { - template - struct iterator_traits> { - typedef ptrdiff_t difference_type; - typedef T value_type; - typedef T* pointer; - typedef T& reference; - typedef random_access_iterator_tag iterator_category; + template + struct iterator_traits> { + using difference_type = ptrdiff_t; + using value_type = T; + using pointer = T*; + using reference = T&; + using iterator_category = random_access_iterator_tag; }; -} +} // namespace std namespace NPagedVector { - //2-level radix tree - template - class TPagedVector: private TVector>, A> { + // 2-level radix tree + template + class TPagedVector { static_assert(PageSize, "expect PageSize"); - typedef TVector TPage; - typedef TVector, A> TPages; - typedef TPagedVector TSelf; + class alignas(T) TPage { + alignas(T) std::array Data_; + + public: + T* data() { + return reinterpret_cast(Data_.data()); + } + + const T* data() const { + return reinterpret_cast(Data_.data()); + } + + T& operator[](size_t idx) { + return *(data() + idx); + } + + const T& operator[](size_t idx) const { + return *(data() + idx); + } + }; + + using TPages = TVector>; + using TSelf = TPagedVector; + + TPages Pages_; + size_t CurrentPageSize_ = 0; public: - typedef NPrivate::TPagedVectorIterator iterator; - typedef NPrivate::TPagedVectorIterator const_iterator; - typedef std::reverse_iterator reverse_iterator; - typedef std::reverse_iterator const_reverse_iterator; - typedef T value_type; - typedef value_type& reference; - typedef const value_type& const_reference; + using iterator = NPrivate::TPagedVectorIterator; + using const_iterator = NPrivate::TPagedVectorIterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + using value_type = T; + using reference = value_type&; + using const_reference = const value_type&; TPagedVector() = default; + TPagedVector(TPagedVector&& other) noexcept + : Pages_(std::move(other.Pages_)) + , CurrentPageSize_(other.CurrentPageSize_) + { + other.CurrentPageSize_ = 0; + } + + TPagedVector(const TPagedVector& other) { + Pages_.reserve(other.Pages_.size()); + try { + for (auto& ptr : other.Pages_) { + auto& newPage = *Pages_.emplace_back(MakeHolder()); + CurrentPageSize_ = 0; + const size_t copyCount = Pages_.size() == other.Pages_.size() + ? other.CurrentPageSize_ + : PageSize; + + std::uninitialized_copy_n(ptr->data(), copyCount, newPage.data()); + CurrentPageSize_ = copyCount; + } + } catch (...) { + clear(); + throw; + } + } + + ~TPagedVector() { + clear(); + } template TPagedVector(TIter b, TIter e) { append(b, e); } + TPagedVector& operator=(const TPagedVector& other) { + if (this != &other) { + TPagedVector tmp(other); + swap(tmp); + } + return *this; + } + + TPagedVector& operator=(TPagedVector&& other) noexcept { + if (this != &other) { + clear(); + Pages_ = std::move(other.Pages_); + CurrentPageSize_ = other.CurrentPageSize_; + other.CurrentPageSize_ = 0; + } + return *this; + } + iterator begin() { return iterator(this, 0); } @@ -203,7 +274,53 @@ namespace NPagedVector { } void swap(TSelf& v) { - TPages::swap((TPages&)v); + Pages_.swap(v.Pages_); + std::swap(CurrentPageSize_, v.CurrentPageSize_); + } + + // Fast iteration over all elements. + template + void ForEach(Function fn) const { + if (Pages_.empty()) { + return; + } + + const auto currentPageIt = Pages_.end() - 1; + for (auto it = Pages_.begin(); it != currentPageIt; ++it) { + const TPage& page = **it; + for (size_t i = 0; i < PageSize; ++i) { + fn(page[i]); + } + } + + const TPage& currentPage = **currentPageIt; + + for (size_t i = 0; i < CurrentPageSize_; ++i) { + fn(currentPage[i]); + } + } + + // Fast iteration over all elements in reverse order. + template + void ForEachReverse(Function fn) const { + if (Pages_.empty()) { + return; + } + + const TPage& currentPage = *Pages_.back(); + + for (size_t i = CurrentPageSize_; i > 0;) { + --i; + fn(currentPage[i]); + } + + for (auto it = Pages_.rbegin() + 1; it != Pages_.rend(); ++it) { + const TPage& page = **it; + for (size_t i = PageSize; i > 0;) { + --i; + fn(page[i]); + } + } } private: @@ -215,110 +332,99 @@ namespace NPagedVector { return idx % PageSize; } - static size_t Index(size_t pnum, size_t poff) { - return pnum * PageSize + poff; - } - TPage& PageAt(size_t pnum) const { - return *TPages::at(pnum); + return *Pages_.at(pnum); } TPage& CurrentPage() const { - return *TPages::back(); - } - - size_t CurrentPageSize() const { - return TPages::empty() ? 0 : CurrentPage().size(); + return *Pages_.back(); } size_t NPages() const { - return TPages::size(); + return Pages_.size(); } void AllocateNewPage() { - TPages::push_back(new TPage()); - CurrentPage().reserve(PageSize); - } - - void MakeNewPage() { - AllocateNewPage(); - CurrentPage().resize(PageSize); + Pages_.emplace_back(MakeHolder()); + CurrentPageSize_ = 0; } void PrepareAppend() { - if (TPages::empty() || CurrentPage().size() + 1 > PageSize) + if (Pages_.empty() || CurrentPageSize_ >= PageSize) { AllocateNewPage(); + } } public: size_t size() const { - return empty() ? 0 : (NPages() - 1) * PageSize + CurrentPage().size(); + return Pages_.empty() ? 0 : (NPages() - 1) * PageSize + CurrentPageSize_; } bool empty() const { - return TPages::empty() || (1 == NPages() && CurrentPage().empty()); + return Pages_.empty() || (1 == NPages() && CurrentPageSize_ == 0); } explicit operator bool() const noexcept { return !empty(); } - template + template reference emplace_back(Args&&... args) { PrepareAppend(); - return CurrentPage().emplace_back(std::forward(args)...); + T* ptr = new (CurrentPage().data() + CurrentPageSize_) T(std::forward(args)...); + ++CurrentPageSize_; + return *ptr; } void push_back(const_reference t) { PrepareAppend(); - CurrentPage().push_back(t); + new (CurrentPage().data() + CurrentPageSize_) T(t); + ++CurrentPageSize_; } void pop_back() { - if (CurrentPage().empty()) - TPages::pop_back(); - CurrentPage().pop_back(); + Y_ASSERT(!empty()); + if (CurrentPageSize_ == 0) { + Pages_.pop_back(); + CurrentPageSize_ = PageSize; + } + --CurrentPageSize_; + if constexpr (!std::is_trivially_destructible_v) { + CurrentPage()[CurrentPageSize_].~T(); + } } template void append(TIter b, TIter e) { - size_t sz = e - b; - size_t sz1 = Min(sz, PageSize - CurrentPageSize()); - size_t sz2 = (sz - sz1) / PageSize; - size_t sz3 = (sz - sz1) % PageSize; - - if (sz1) { - PrepareAppend(); - TPage& p = CurrentPage(); - p.insert(p.end(), b, b + sz1); - } - - for (size_t i = 0; i < sz2; ++i) { - AllocateNewPage(); - TPage& p = CurrentPage(); - p.insert(p.end(), b + sz1 + i * PageSize, b + sz1 + (i + 1) * PageSize); - } - - if (sz3) { - AllocateNewPage(); - TPage& p = CurrentPage(); - p.insert(p.end(), b + sz1 + sz2 * PageSize, e); + for (TIter it = b; it != e; ++it) { + push_back(*it); } } iterator erase(iterator it) { - size_t pnum = PageNumber(it.Offset); - size_t pidx = InPageIndex(it.Offset); - - if (CurrentPage().empty()) - TPages::pop_back(); + if (CurrentPageSize_ == 0) { + Pages_.pop_back(); + CurrentPageSize_ = Pages_.empty() ? 0 : PageSize; + } - for (size_t p = NPages() - 1; p > pnum; --p) { - PageAt(p - 1).push_back(PageAt(p).front()); - PageAt(p).erase(PageAt(p).begin()); + size_t pidx = InPageIndex(it.Index_); + for (size_t pnum = PageNumber(it.Index_);; ++pnum) { + TPage& page = *Pages_[pnum]; + if (pnum + 1 == Pages_.size()) { + std::shift_left(page.data() + pidx, page.data() + CurrentPageSize_, 1); + --CurrentPageSize_; + if constexpr (!std::is_trivially_destructible_v) { + page[CurrentPageSize_].~T(); + } + break; + } + + std::shift_left(page.data() + pidx, page.data() + PageSize, 1); + TPage& nextPage = *Pages_[pnum + 1]; + page[PageSize - 1] = std::move(nextPage[0]); + pidx = 0; } - PageAt(pnum).erase(PageAt(pnum).begin() + pidx); return it; } @@ -332,86 +438,96 @@ namespace NPagedVector { return b; } - iterator insert(iterator it, const value_type& v) { - size_t pnum = PageNumber(it.Offset); - size_t pidx = InPageIndex(it.Offset); - - PrepareAppend(); - - for (size_t p = NPages() - 1; p > pnum; --p) { - PageAt(p).insert(PageAt(p).begin(), PageAt(p - 1).back()); - PageAt(p - 1).pop_back(); - } - - PageAt(pnum).insert(PageAt(pnum).begin() + pidx, v); - return it; - } - - template - void insert(iterator it, TIter b, TIter e) { - // todo : suboptimal! - for (; b != e; ++b, ++it) - it = insert(it, *b); - } - reference front() { - return TPages::front()->front(); + Y_ASSERT(CurrentPageSize_ > 0 || Pages_.size() > 1); + return (*Pages_.front())[0]; } const_reference front() const { - return TPages::front()->front(); + Y_ASSERT(CurrentPageSize_ > 0 || Pages_.size() > 1); + return (*Pages_.front())[0]; } reference back() { - return CurrentPage().back(); + if (CurrentPageSize_ > 0) { + return CurrentPage()[CurrentPageSize_ - 1]; + } else { + Y_ASSERT(Pages_.size() >= 2); + return (**(Pages_.end() - 2))[PageSize - 1]; + } } const_reference back() const { - return CurrentPage().back(); + if (CurrentPageSize_ > 0) { + return CurrentPage()[CurrentPageSize_ - 1]; + } else { + Y_ASSERT(Pages_.size() >= 2); + return (**(Pages_.end() - 2))[PageSize - 1]; + } } void clear() { - TPages::clear(); + if constexpr (std::is_trivially_destructible_v) { + Pages_.clear(); + CurrentPageSize_ = 0; + } else { + while (!Pages_.empty()) { + TPage& page = CurrentPage(); + while (CurrentPageSize_ > 0) { + --CurrentPageSize_; + page[CurrentPageSize_].~T(); + } + Pages_.pop_back(); + CurrentPageSize_ = Pages_.empty() ? 0 : PageSize; + } + } } void resize(size_t sz) { - if (sz == size()) + size_t curSize = size(); + if (sz == curSize) { return; + } - const size_t npages = NPages(); - const size_t newwholepages = sz / PageSize; - const size_t pagepart = sz % PageSize; - const size_t newpages = newwholepages + bool(pagepart); - - if (npages && newwholepages >= npages) - CurrentPage().resize(PageSize); - - if (newpages < npages) - TPages::resize(newpages); - else - for (size_t i = npages; i < newpages; ++i) - MakeNewPage(); - - if (pagepart) - CurrentPage().resize(pagepart); - - Y_ABORT_UNLESS(sz == size(), "%" PRIu64 " %" PRIu64, (ui64)sz, (ui64)size()); + if (sz < curSize) { + while (sz < curSize) { + pop_back(); + --curSize; + } + } else { + while (sz > curSize) { + emplace_back(); + ++curSize; + } + } } reference at(size_t idx) { - return TPages::at(PageNumber(idx))->at(InPageIndex(idx)); + if (idx >= size()) { + throw std::out_of_range("TPagedVector::at() - index out of range"); + } + const size_t pnum = PageNumber(idx); + const size_t inPageIdx = InPageIndex(idx); + + return (*Pages_[pnum])[inPageIdx]; } const_reference at(size_t idx) const { - return TPages::at(PageNumber(idx))->at(InPageIndex(idx)); + if (idx >= size()) { + throw std::out_of_range("TPagedVector::at() - index out of range"); + } + const size_t pnum = PageNumber(idx); + const size_t inPageIdx = InPageIndex(idx); + + return (*Pages_[pnum])[inPageIdx]; } reference operator[](size_t idx) { - return TPages::operator[](PageNumber(idx))->operator[](InPageIndex(idx)); + return Pages_.operator[](PageNumber(idx))->operator[](InPageIndex(idx)); } const_reference operator[](size_t idx) const { - return TPages::operator[](PageNumber(idx))->operator[](InPageIndex(idx)); + return Pages_.operator[](PageNumber(idx))->operator[](InPageIndex(idx)); } friend bool operator==(const TSelf& a, const TSelf& b) { @@ -424,10 +540,9 @@ namespace NPagedVector { }; namespace NPrivate { - typedef std::is_same::iterator>::iterator_category> - TIteratorCheck; + using TIteratorCheck = std::is_same::iterator>::iterator_category>; static_assert(TIteratorCheck::value, "expect TIteratorCheck::Result"); - } + } // namespace NPrivate -} +} // namespace NPagedVector diff --git a/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp b/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp index d059ce34ec1..b0f39f748fe 100644 --- a/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp +++ b/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp @@ -12,13 +12,25 @@ class TPagedVectorTest: public TTestBase { UNIT_TEST(Test4) UNIT_TEST(Test5) UNIT_TEST(Test6) - UNIT_TEST(Test7) UNIT_TEST(TestAt) UNIT_TEST(TestAutoRef) UNIT_TEST(TestIterators) UNIT_TEST(TestEmplaceBack1) UNIT_TEST(TestEmplaceBack2) - //UNIT_TEST(TestEbo) + UNIT_TEST(TestCopyConstructor) + UNIT_TEST(TestCopyAssignment) + UNIT_TEST(TestMoveConstructor) + UNIT_TEST(TestMoveAssignment) + UNIT_TEST(TestCopyConstructorString) + UNIT_TEST(TestCopyAssignmentString) + UNIT_TEST(TestMoveConstructorString) + UNIT_TEST(TestMoveAssignmentString) + UNIT_TEST(TestEmplaceBackNoncopyable) + UNIT_TEST(TestClear) + UNIT_TEST(TestBack) + UNIT_TEST(TestIterator) + UNIT_TEST(TestForEach) + UNIT_TEST(TestForEachReverse) UNIT_TEST_SUITE_END(); private: @@ -122,8 +134,9 @@ class TPagedVectorTest: public TTestBase { TPagedVector v2; v2.resize(v1.size()); - for (size_t i = 0; i < v1.size(); ++i) + for (size_t i = 0; i < v1.size(); ++i) { v2[i] = v1[i]; + } v2[1] = 'o'; // Replace second character. @@ -215,44 +228,6 @@ class TPagedVectorTest: public TTestBase { UNIT_ASSERT(v[1] == 25); } - void Test7() { - int array1[] = {1, 4, 25}; - int array2[] = {9, 16}; - - typedef NPagedVector::TPagedVector TVectorType; - - TVectorType v(array1, array1 + 3); - TVectorType::iterator vit; - vit = v.insert(v.begin(), 0); // Insert before first element. - UNIT_ASSERT_VALUES_EQUAL(*vit, 0); - - vit = v.insert(v.end(), 36); // Insert after last element. - UNIT_ASSERT(*vit == 36); - - UNIT_ASSERT(v.size() == 5); - UNIT_ASSERT(v[0] == 0); - UNIT_ASSERT(v[1] == 1); - UNIT_ASSERT(v[2] == 4); - UNIT_ASSERT(v[3] == 25); - UNIT_ASSERT(v[4] == 36); - - // Insert contents of array2 before fourth element. - v.insert(v.begin() + 3, array2, array2 + 2); - - UNIT_ASSERT(v.size() == 7); - - UNIT_ASSERT(v[0] == 0); - UNIT_ASSERT(v[1] == 1); - UNIT_ASSERT(v[2] == 4); - UNIT_ASSERT(v[3] == 9); - UNIT_ASSERT(v[4] == 16); - UNIT_ASSERT(v[5] == 25); - UNIT_ASSERT(v[6] == 36); - - v.clear(); - UNIT_ASSERT(v.empty()); - } - void TestAt() { using NPagedVector::TPagedVector; TPagedVector v; @@ -332,14 +307,14 @@ class TPagedVectorTest: public TTestBase { UNIT_ASSERT(vint.rbegin() == vint.rbegin()); // Not Standard: - //UNIT_ASSERT(vint.rbegin() == crvint.rbegin()); - //UNIT_ASSERT(crvint.rbegin() == vint.rbegin()); + // UNIT_ASSERT(vint.rbegin() == crvint.rbegin()); + // UNIT_ASSERT(crvint.rbegin() == vint.rbegin()); UNIT_ASSERT(crvint.rbegin() == crvint.rbegin()); UNIT_ASSERT(vint.rbegin() != vint.rend()); // Not Standard: - //UNIT_ASSERT(vint.rbegin() != crvint.rend()); - //UNIT_ASSERT(crvint.rbegin() != vint.rend()); + // UNIT_ASSERT(vint.rbegin() != crvint.rend()); + // UNIT_ASSERT(crvint.rbegin() != vint.rend()); UNIT_ASSERT(crvint.rbegin() != crvint.rend()); } @@ -376,37 +351,519 @@ class TPagedVectorTest: public TTestBase { } } - /* This test check a potential issue with empty base class - * optimization. Some compilers (VC6) do not implement it - * correctly resulting ina wrong behavior. */ - void TestEbo() { + void TestCopyConstructor() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(i); + } + + TPagedVector copied(v); + + UNIT_ASSERT_VALUES_EQUAL(copied.size(), 10u); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 10u); + + for (int i = 0; i < 10; ++i) { + // values are the same + UNIT_ASSERT_VALUES_EQUAL(v[i], i); + UNIT_ASSERT_VALUES_EQUAL(copied[i], i); + + // but pointers are different (the elements have been copied) + UNIT_ASSERT_VALUES_UNEQUAL(&copied[i], &v[i]); + } + + // Modifying the copy must not affect the original. + copied[0] = 999; + UNIT_ASSERT_VALUES_EQUAL(v[0], 0); + } + + void TestCopyAssignment() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(i); + } + + TPagedVector assigned; + assigned.push_back(999); + assigned = v; + + // The source vector should remain unchanged after copy. + UNIT_ASSERT_VALUES_EQUAL(v.size(), 10u); + + UNIT_ASSERT_VALUES_EQUAL(assigned.size(), 10u); + for (int i = 0; i < 10; ++i) { + // values are the same + UNIT_ASSERT_VALUES_EQUAL(v[i], i); + UNIT_ASSERT_VALUES_EQUAL(assigned[i], i); + + // but pointers are different (the elements have been copied) + UNIT_ASSERT_VALUES_UNEQUAL(&assigned[i], &v[i]); + } + + // Modifying the assigned vector must not affect the original. + assigned[0] = 999; + UNIT_ASSERT_VALUES_EQUAL(v[0], 0); + } + + void TestMoveConstructor() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(i); + } + + auto orig_ptr = &v[5]; + + TPagedVector moved(std::move(v)); + + UNIT_ASSERT_VALUES_EQUAL(moved.size(), 10u); + + // the move must keep original element pointers + UNIT_ASSERT_VALUES_EQUAL(orig_ptr, &moved[5]); + + for (int i = 0; i < 10; ++i) { + UNIT_ASSERT_VALUES_EQUAL(moved[i], i); + } + + // After move, the source vector should be empty. + UNIT_ASSERT(v.empty()); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 0u); + } + + void TestMoveAssignment() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(i); + } + + auto orig_ptr = &v[7]; + + TPagedVector assigned; + assigned.push_back(999); + assigned = std::move(v); + + UNIT_ASSERT_VALUES_EQUAL(assigned.size(), 10u); + + // the move must keep original element pointers + UNIT_ASSERT_VALUES_EQUAL(orig_ptr, &assigned[7]); + + for (int i = 0; i < 10; ++i) { + UNIT_ASSERT_VALUES_EQUAL(assigned[i], i); + } + + // After move, the source vector should be empty. + UNIT_ASSERT(v.empty()); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 0u); + } + + void TestCopyConstructorString() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(ToString(i)); + } + + TPagedVector copied(v); + + UNIT_ASSERT_VALUES_EQUAL(copied.size(), 10u); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 10u); + + for (int i = 0; i < 10; ++i) { + // values are the same + UNIT_ASSERT_VALUES_EQUAL(v[i], ToString(i)); + UNIT_ASSERT_VALUES_EQUAL(copied[i], ToString(i)); + + // but pointers are different (the elements have been copied, not moved) + UNIT_ASSERT_VALUES_UNEQUAL(&copied[i], &v[i]); + } + + // Modifying the copy must not affect the original (deep copy semantics). + copied[0] = "modified"; + UNIT_ASSERT_VALUES_EQUAL(v[0], "0"); + } + + void TestCopyAssignmentString() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(ToString(i)); + } + + TPagedVector assigned; + assigned.push_back("old"); + assigned = v; + + // The source vector should remain unchanged after copy. + UNIT_ASSERT_VALUES_EQUAL(v.size(), 10u); + + UNIT_ASSERT_VALUES_EQUAL(assigned.size(), 10u); + for (int i = 0; i < 10; ++i) { + // values are the same + UNIT_ASSERT_VALUES_EQUAL(v[i], ToString(i)); + UNIT_ASSERT_VALUES_EQUAL(assigned[i], ToString(i)); + + // but pointers are different (the elements have been copied, not moved) + UNIT_ASSERT_VALUES_UNEQUAL(&assigned[i], &v[i]); + } + + // Modifying the assigned vector must not affect the original. + assigned[0] = "modified"; + UNIT_ASSERT_VALUES_EQUAL(v[0], "0"); + } + + void TestMoveConstructorString() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(ToString(i)); + } + + auto orig_ptr = &v[5]; + + TPagedVector moved(std::move(v)); + + UNIT_ASSERT_VALUES_EQUAL(moved.size(), 10u); + + // the move must keep original element pointers (pages are stolen, not copied) + UNIT_ASSERT_VALUES_EQUAL(orig_ptr, &moved[5]); + + for (int i = 0; i < 10; ++i) { + UNIT_ASSERT_VALUES_EQUAL(moved[i], ToString(i)); + } + + // After move, the source vector should be empty. + UNIT_ASSERT(v.empty()); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 0u); + } + + void TestMoveAssignmentString() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 10; ++i) { + v.push_back(ToString(i)); + } + + auto orig_ptr = &v[7]; + + TPagedVector assigned; + assigned.push_back("old"); + assigned = std::move(v); + + UNIT_ASSERT_VALUES_EQUAL(assigned.size(), 10u); + + // the move must keep original element pointers (pages are stolen, not copied) + UNIT_ASSERT_VALUES_EQUAL(orig_ptr, &assigned[7]); + + for (int i = 0; i < 10; ++i) { + UNIT_ASSERT_VALUES_EQUAL(assigned[i], ToString(i)); + } + + // After move, the source vector should be empty. + UNIT_ASSERT(v.empty()); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 0u); + } + + struct TNonCopyableTestClass { + const TString Str; + TNonCopyableTestClass(const TString s) + : Str(s) + { + } + }; + + void TestEmplaceBackNoncopyable() { + using NPagedVector::TPagedVector; + TPagedVector v; + + for (int i = 0; i < 19; ++i) { + v.emplace_back(ToString(i)); + } + + for (int i = 0; i < 19; ++i) { + UNIT_ASSERT_VALUES_EQUAL(v[i].Str, ToString(i)); + } + + v.pop_back(); + v.pop_back(); + v.emplace_back("Hello world"); + UNIT_ASSERT_VALUES_EQUAL(v[17].Str, "Hello world"); + } + + void TestClear() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 4; ++i) { + v.push_back(ToString(i)); + } + + v.pop_back(); + v.clear(); + + UNIT_ASSERT(v.empty()); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 0u); + } + + void TestBack() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 4; ++i) { + v.push_back(ToString(i)); + } + + UNIT_ASSERT_VALUES_EQUAL(v.back(), "3"); + v.pop_back(); + UNIT_ASSERT_VALUES_EQUAL(v.back(), "2"); + } + + void TestIterator() { + using NPagedVector::TPagedVector; + TPagedVector v; + for (int i = 0; i < 11; ++i) { + v.push_back(ToString(i)); + } + + v.emplace_back("Hello"); + v.emplace_back("world"); + + auto it = v.begin(); + + UNIT_ASSERT_VALUES_EQUAL(it.GetIndex(), 0); + UNIT_ASSERT_VALUES_EQUAL(*it, "0"); + + ++it; + + UNIT_ASSERT_VALUES_EQUAL(it.GetIndex(), 1); + UNIT_ASSERT_VALUES_EQUAL(*it, "1"); + + it += 5; + + UNIT_ASSERT_VALUES_EQUAL(it.GetIndex(), 6); + UNIT_ASSERT_VALUES_EQUAL(*it, "6"); + + it = v.erase(it); + + UNIT_ASSERT_VALUES_EQUAL(it.GetIndex(), 6); + UNIT_ASSERT_VALUES_EQUAL(*it, "7"); + } + + void TestForEach() { using NPagedVector::TPagedVector; - // We use heap memory as test failure can corrupt vector internal - // representation making executable crash on vector destructor invocation. - // We prefer a simple memory leak, internal corruption should be reveal - // by size or capacity checks. - typedef TPagedVector V; - V* pv1 = new V; - pv1->resize(1); - pv1->at(0) = 1; + // Empty vector: the callback must not be invoked at all. + { + TPagedVector v; + size_t calls = 0; + v.ForEach([&](int) { + ++calls; + }); + UNIT_ASSERT_VALUES_EQUAL(calls, 0u); + } - V* pv2 = new V; + // Single element: the only element is visited once. + { + TPagedVector v; + v.push_back(42); + TVector visited; + v.ForEach([&](int x) { + visited.push_back(x); + }); + TVector expected{42}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } - pv2->resize(10); - for (int i = 0; i < 10; ++i) - pv2->at(i) = 2; + // Several elements within a single (partially filled) page. + { + TPagedVector v; + for (int i = 0; i < 2; ++i) { + v.push_back(i); + } + TVector visited; + int expectedElement = 0; + v.ForEach([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + ++expectedElement; + visited.push_back(x); + }); + TVector expected{0, 1}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } - pv1->swap(*pv2); + // A single exactly full page (3 elements): the visit order must be + // strictly forward. + { + TPagedVector v; + for (int i = 0; i < 3; ++i) { + v.push_back(i); + } + TVector visited; + int expectedElement = 0; + v.ForEach([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + ++expectedElement; + visited.push_back(x); + }); + TVector expected{0, 1, 2}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } - UNIT_ASSERT(pv1->size() == 10); - UNIT_ASSERT((*pv1)[5] == 2); + // Multiple pages with a partially filled last page: the visit order + // must be strictly forward (from the first element to the last). + { + TPagedVector v; + const int n = 10; // spans 4 pages of size 3: [0..2][3..5][6..8][9] + for (int i = 0; i < n; ++i) { + v.push_back(i); + } + TVector visited; + visited.reserve(n); + int expectedElement = 0; + v.ForEach([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + ++expectedElement; + visited.push_back(x); + }); + TVector expected; + expected.reserve(n); + for (int i = 0; i < n; ++i) { + expected.push_back(i); + } + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } - UNIT_ASSERT(pv2->size() == 1); - UNIT_ASSERT((*pv2)[0] == 1); + // Exactly full pages (no partial tail): every element is visited, + // last page is completely filled. + { + TPagedVector v; + const int n = 9; // exactly 3 full pages of size 3 + for (int i = 0; i < n; ++i) { + v.push_back(i); + } + TVector visited; + visited.reserve(n); + int expectedElement = 0; + v.ForEach([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + ++expectedElement; + visited.push_back(x); + }); + TVector expected; + expected.reserve(n); + for (int i = 0; i < n; ++i) { + expected.push_back(i); + } + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + } - delete pv2; - delete pv1; + void TestForEachReverse() { + using NPagedVector::TPagedVector; + + // Empty vector: the callback must not be invoked at all. + { + TPagedVector v; + size_t calls = 0; + v.ForEachReverse([&](int) { + ++calls; + }); + UNIT_ASSERT_VALUES_EQUAL(calls, 0u); + } + + // Single element: the only element is visited once. + { + TPagedVector v; + v.push_back(42); + TVector visited; + v.ForEachReverse([&](int x) { + visited.push_back(x); + }); + TVector expected{42}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // Several elements within a single (partially filled) page. + { + TPagedVector v; + for (int i = 0; i < 2; ++i) { + v.push_back(i); + } + TVector visited; + int expectedElement = 1; + v.ForEachReverse([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + --expectedElement; + visited.push_back(x); + }); + TVector expected{1, 0}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // A single exactly full page (3 elements): the visit order must be + // strictly reverse. + { + TPagedVector v; + for (int i = 0; i < 3; ++i) { + v.push_back(i); + } + TVector visited; + int expectedElement = 2; + v.ForEachReverse([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + --expectedElement; + visited.push_back(x); + }); + TVector expected{2, 1, 0}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // Multiple pages with a partially filled last page: the visit order + // must be strictly reverse (from the last element to the first). + { + TPagedVector v; + const int n = 10; // spans 4 pages of size 3: [0..2][3..5][6..8][9] + for (int i = 0; i < n; ++i) { + v.push_back(i); + } + TVector visited; + visited.reserve(n); + int expectedElement = n - 1; + v.ForEachReverse([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + --expectedElement; + visited.push_back(x); + }); + TVector expected; + expected.reserve(n); + for (int i = n - 1; i >= 0; --i) { + expected.push_back(i); + } + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // Exactly full pages (no partial tail): every element is visited, + // last page is completely filled. + { + TPagedVector v; + const int n = 9; // exactly 3 full pages of size 3 + for (int i = 0; i < n; ++i) { + v.push_back(i); + } + TVector visited; + visited.reserve(n); + int expectedElement = n - 1; + v.ForEachReverse([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + --expectedElement; + visited.push_back(x); + }); + TVector expected; + expected.reserve(n); + for (int i = n - 1; i >= 0; --i) { + expected.push_back(i); + } + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } } }; diff --git a/library/cpp/containers/stack_vector/stack_vec.h b/library/cpp/containers/stack_vector/stack_vec.h index b6036059b26..f2641b5d070 100644 --- a/library/cpp/containers/stack_vector/stack_vec.h +++ b/library/cpp/containers/stack_vector/stack_vec.h @@ -82,7 +82,7 @@ namespace NPrivate { } private: - std::aligned_storage_t StackBasedStorage[CountOnStack]; + alignas(T) char StackBasedStorage[CountOnStack][sizeof(T)]; bool IsStorageUsed = false; private: diff --git a/library/cpp/coroutine/engine/coroutine_ut.cpp b/library/cpp/coroutine/engine/coroutine_ut.cpp index de56d0ed2bb..20176fdb91f 100644 --- a/library/cpp/coroutine/engine/coroutine_ut.cpp +++ b/library/cpp/coroutine/engine/coroutine_ut.cpp @@ -2,7 +2,6 @@ #include "condvar.h" #include "network.h" -#include #include #include @@ -13,6 +12,8 @@ #include #include +#include + // TODO (velavokr): BALANCER-1345 add more tests on pollers class TCoroTest: public TTestBase { @@ -112,7 +113,7 @@ void TCoroTest::TestException() { auto f2 = [&unc, &f2run](TCont*) { f2run = true; - unc = std::uncaught_exception(); + unc = std::uncaught_exceptions(); // check segfault try { @@ -162,11 +163,11 @@ void TCoroTest::TestSimpleX1() { void TCoroTest::TestSimpleX1MultiThread() { TVector> threads; const size_t nThreads = 0; - TAtomic c = 0; + std::atomic c = 0; for (size_t i = 0; i < nThreads; ++i) { threads.push_back(MakeHolder([&]() { TestSimpleX1(); - AtomicIncrement(c); + ++c; })); } @@ -178,7 +179,7 @@ void TCoroTest::TestSimpleX1MultiThread() { t->Join(); } - UNIT_ASSERT_EQUAL(c, nThreads); + UNIT_ASSERT_EQUAL(c.load(), nThreads); } struct TTestObject { diff --git a/library/cpp/cppparser/parser.cpp b/library/cpp/cppparser/parser.cpp index 3bd968b4594..70fb6a87356 100644 --- a/library/cpp/cppparser/parser.cpp +++ b/library/cpp/cppparser/parser.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -127,6 +128,10 @@ class TCppSaxParser::TImpl { break; case '\'': + if (QuoteCharIsADigitSeparator()) { + Text_.Data += ch; + break; + } Action(ch); State_ = Character; @@ -356,6 +361,35 @@ class TCppSaxParser::TImpl { } } + // digit separator in integral literal (ex. 73'709'550'592) + bool QuoteCharIsADigitSeparator() const { + const TStringBuf data = Text_.Data; + if (data.empty()) { + return false; + } + if (!IsAsciiHex(data.back())) { + return false; + } + // check for char literal prefix (ex. `u8'$'`) + static constexpr TStringBuf literalPrefixes[] { + "u8", + "u", + "U", + "L", + }; + for (const TStringBuf& literalPrefix : literalPrefixes) { + if (TStringBuf prev; data.BeforeSuffix(literalPrefix, prev)) { + if (!prev.empty() && (IsAsciiAlnum(prev.back()) || prev.back() == '_' || prev.back() == '$')) { + // some macro name ends with an `u8` sequence + continue; + } + // it is a prefixed character literal + return false; + } + } + return true; + } + inline void Action(char ch) { Action(); Text_.Data += ch; diff --git a/library/cpp/digest/md5/md5.cpp b/library/cpp/digest/md5/md5.cpp index de497498082..c583a08170d 100644 --- a/library/cpp/digest/md5/md5.cpp +++ b/library/cpp/digest/md5/md5.cpp @@ -12,7 +12,6 @@ namespace { constexpr size_t MD5_BLOCK_LENGTH = 64; constexpr size_t MD5_PADDING_SHIFT = 56; - constexpr size_t MD5_HEX_DIGEST_LENGTH = 32; struct TMd5Stream: public IOutputStream { inline TMd5Stream(MD5* md5) diff --git a/library/cpp/digest/md5/md5.h b/library/cpp/digest/md5/md5.h index 2c17aa05181..b6aecb8f460 100644 --- a/library/cpp/digest/md5/md5.h +++ b/library/cpp/digest/md5/md5.h @@ -6,6 +6,9 @@ class IInputStream; class MD5 { +public: + static constexpr size_t MD5_HEX_DIGEST_LENGTH = 32; + public: MD5() { Init(); diff --git a/library/cpp/digest/md5/md5_ut.cpp b/library/cpp/digest/md5/md5_ut.cpp index 1c3e4ad0a9f..7fb7f58a996 100644 --- a/library/cpp/digest/md5/md5_ut.cpp +++ b/library/cpp/digest/md5/md5_ut.cpp @@ -14,7 +14,7 @@ Y_UNIT_TEST_SUITE(TMD5Test) { r.Update((const unsigned char*)b, 15); r.Update((const unsigned char*)b + 15, strlen(b) - 15); - char rs[33]; + char rs[MD5::MD5_HEX_DIGEST_LENGTH + 1]; TString s(r.End(rs)); s.to_lower(); diff --git a/library/cpp/digest/murmur/murmur.h b/library/cpp/digest/murmur/murmur.h index cbf28864128..8ca20b378e8 100644 --- a/library/cpp/digest/murmur/murmur.h +++ b/library/cpp/digest/murmur/murmur.h @@ -37,7 +37,7 @@ class TMurmurHash2A { using TValue = typename TTraits::TValue; public: - inline TMurmurHash2A(TValue seed = 0) + inline explicit TMurmurHash2A(TValue seed = 0) : Hash(seed) { } diff --git a/library/cpp/getopt/last_getopt_demo/demo.cpp b/library/cpp/getopt/last_getopt_demo/demo.cpp index a0e82a936cd..3e98db97d4f 100644 --- a/library/cpp/getopt/last_getopt_demo/demo.cpp +++ b/library/cpp/getopt/last_getopt_demo/demo.cpp @@ -121,8 +121,8 @@ class TMain: public TMainClassArgs { }) .Completer(NLastGetopt::NComp::File()); - // These two options can't be together. - opts.MutuallyExclusive("post-file", "post-data"); + // These options can not appear together. + opts.MutuallyExclusive("method", "post-file", "post-data"); opts.AddLongOption("header") .RequiredArgument("header-line") diff --git a/library/cpp/getopt/small/completion_generator.cpp b/library/cpp/getopt/small/completion_generator.cpp index d893afb40a9..5e0e55ed38b 100644 --- a/library/cpp/getopt/small/completion_generator.cpp +++ b/library/cpp/getopt/small/completion_generator.cpp @@ -63,6 +63,14 @@ namespace NLastGetopt { L; manager.GenerateZsh(out); + // When the completion file is autoloaded by `compinit` from `$fpath`, + // zsh treats the file content as the body of function `_`. + // On first invocation that body merely (re)defines `_` and + // its helpers, so completion would not actually run until the second + // TAB. Calling the redefined function here makes it work on the very + // first TAB and is also harmless when the script is `source`d. + L << "_" << command << " \"$@\""; + out.Print(stream); } diff --git a/library/cpp/getopt/small/last_getopt_opt.h b/library/cpp/getopt/small/last_getopt_opt.h index 8754ebb7eeb..67a937bdb13 100644 --- a/library/cpp/getopt/small/last_getopt_opt.h +++ b/library/cpp/getopt/small/last_getopt_opt.h @@ -13,7 +13,8 @@ #include #include -#include +#include +#include namespace NLastGetopt { enum EHasArg { @@ -23,6 +24,11 @@ namespace NLastGetopt { DEFAULT_HAS_ARG = REQUIRED_ARGUMENT }; + template + concept ArgTagConcept = + std::is_enum_v> || + std::is_same_v, ui32>; + /** * NLastGetopt::TOpt is a storage of data about exactly one program option. * The data is: parse politics and help information. @@ -36,12 +42,12 @@ namespace NLastGetopt { * argument parse politics: no/optional/required/ * option existence: required or optional * handlers. See detailed documentation: - * default value: if the option has argument, but the option is ommited, + * default value: if the option has argument, but the option is omitted, * then the is used as the value of the argument * optional value: if the option has optional-argument, the option is present in parsed string, * but the argument is omitted, then - * in case of "not given , omited optional argument" the is used - * user value: allows to store arbitary pointer for handlers + * in case of "not given , omitted optional argument" the is used + * user value: allows to store arbitrary pointer for handlers */ class TOpt { public: @@ -94,7 +100,7 @@ namespace NLastGetopt { /** * Checks if given string can be a long name * @param name string to check - * @param c if given, the first bad charecter will be saved in c + * @param c if given, the first bad character will be saved in c */ static bool IsAllowedLongName(const TString& name, unsigned char* c = nullptr); @@ -798,6 +804,9 @@ namespace NLastGetopt { * argument name (title) */ struct TFreeArgSpec { + template + using TTagger = std::function; + TFreeArgSpec() = default; TFreeArgSpec(const TString& title, const TString& help = TString(), bool optional = false) : Title_(title) @@ -809,6 +818,7 @@ namespace NLastGetopt { TString Title_; TString Help_; TString CompletionArgHelp_; + TTagger Tagger_; bool Optional_ = false; NComp::ICompleterPtr Completer_ = nullptr; @@ -891,5 +901,48 @@ namespace NLastGetopt { Completer_ = std::move(completer); return *this; } + + /** + * Set a tagger that can compute tag dynamically for each argument value. + */ + TFreeArgSpec& SetTag(TTagger&& tagger) { + Tagger_ = std::forward>(tagger); + return *this; + } + + /** + * Set a static tag for all arguments described by this spec. + */ + template + TFreeArgSpec& SetTag(E tag) { + Tagger_ = [tag](const TString&) -> ui32 { + return static_cast(tag); + }; + return *this; + } + + /** + * Set a tagger that can compute tag dynamically for each argument value. + */ + template + TFreeArgSpec& SetTag(TTagger&& tagger) { + Tagger_ = [tagger](const TString& value) -> ui32 { + return static_cast(tagger(value)); + }; + return *this; + } + + /** + * Compute tag for argument value at given position. + */ + ui32 GetTag(const TString& value) const { + if (Tagger_) { + ui32 tag = Tagger_(value); + if (tag) { + return tag; + } + } + return 0; + } }; } diff --git a/library/cpp/getopt/small/last_getopt_opts.cpp b/library/cpp/getopt/small/last_getopt_opts.cpp index 984927a0387..b656e607e85 100644 --- a/library/cpp/getopt/small/last_getopt_opts.cpp +++ b/library/cpp/getopt/small/last_getopt_opts.cpp @@ -213,7 +213,7 @@ namespace NLastGetopt { if (FreeArgsMax_ < FreeArgsMin_) { ythrow TConfException() << "FreeArgsMax must be >= FreeArgsMin"; } - if (!FreeArgSpecs_.empty() && FreeArgSpecs_.rbegin()->first >= FreeArgsMax_) { + if (!FreeArgSpecs_.empty() && GetTrailingArgsIndex() > FreeArgsMax_) { ythrow TConfException() << "Described args count is greater than FreeArgsMax. Either increase FreeArgsMax or remove unreachable descriptions"; } } @@ -335,7 +335,7 @@ namespace NLastGetopt { } os << "[OPTIONS]"; - ui32 numDescribedFlags = FreeArgSpecs_.empty() ? 0 : FreeArgSpecs_.rbegin()->first + 1; + ui32 numDescribedFlags = GetTrailingArgsIndex(); ui32 numArgsToShow = Max(FreeArgsMin_, FreeArgsMax_ == UNLIMITED_ARGS ? numDescribedFlags : FreeArgsMax_); for (ui32 i = 0, nonOptionalFlagsPrinted = 0; i < numArgsToShow; ++i) { @@ -513,8 +513,8 @@ namespace NLastGetopt { } os << colors.OldColor() << Endl; - const size_t limit = FreeArgSpecs_.empty() ? 0 : FreeArgSpecs_.rbegin()->first; - for (size_t i = 0; i <= limit; ++i) { + const size_t limit = GetTrailingArgsIndex(); + for (size_t i = 0; i < limit; ++i) { if (!FreeArgSpecs_.contains(i)) { continue; } diff --git a/library/cpp/getopt/small/last_getopt_opts.h b/library/cpp/getopt/small/last_getopt_opts.h index 718dbfcb893..868477ec79e 100644 --- a/library/cpp/getopt/small/last_getopt_opts.h +++ b/library/cpp/getopt/small/last_getopt_opts.h @@ -28,7 +28,7 @@ namespace NLastGetopt { * the special string " -- " will be treated as end of named * options: all options after it will be parsed as free args * if PERMUTE is choosen, arguments will be rearranged in correct order, - * if RETURN_IN_ORDER is choosen, all free args will be ommited (TODO: looks very strange) + * if RETURN_IN_ORDER is choosen, all free args will be omitted (TODO: looks very strange) * - Using '+' as a prefix instead '--' for long names * - Using "-" as a prefix for both short and long names * - Allowing unknown options @@ -49,7 +49,7 @@ namespace NLastGetopt { bool AllowSingleDashForLong_ = false; // bool AllowPlusForLong_ = false; // using '+' instead '--' for long options - //Allows unknwon options: + //Allows unknown options: bool AllowUnknownCharOptions_ = false; bool AllowUnknownLongOptions_ = false; @@ -78,7 +78,7 @@ namespace NLastGetopt { /** * Constructs TOpts from string as in getopt(3) and - * additionally adds help option (for '?') and svn-verstion option (for 'V') + * additionally adds help option (for '?') and svn-version option (for 'V') */ static TOpts Default(const TStringBuf& optstring = TStringBuf()) { TOpts opts(optstring); @@ -92,7 +92,7 @@ namespace NLastGetopt { * Throws TConfException if validation failed. * Check consist of: * -not intersecting of names - * -compability of settings, that responsable for freeArgs parsing + * -compatibility of settings, that responsible for freeArgs parsing */ void Validate() const; @@ -398,9 +398,22 @@ namespace NLastGetopt { * Note: don't use this on options with default values. If option with default value wasn't specified, * parser will run handlers for default value, thus triggering a false-positive exclusivity check. */ - template - void MutuallyExclusive(T1&& opt1, T2&& opt2) { - MutuallyExclusiveOpt(GetOption(std::forward(opt1)), GetOption(std::forward(opt2))); + template + void MutuallyExclusive(Opt1&& name1, Opt2&& name2) { + TOpt& opt1 = GetOption(name1); + TOpt& opt2 = GetOption(name2); + MutuallyExclusiveOpt(opt1, opt2); + } + + template + void MutuallyExclusive(Opt1&& name1, OtherOpts&& ...otherNames) { + TOpt& opt1 = GetOption(name1); + std::array otherNamesArr{otherNames...}; + for (const auto& otherName: otherNamesArr) { + TOpt& otherOpt = GetOption(otherName); + MutuallyExclusiveOpt(opt1, otherOpt); + } + MutuallyExclusive(std::forward(otherNames)...); } /** @@ -449,6 +462,24 @@ namespace NLastGetopt { AddSection("Examples", std::move(examples)); } + /** + * Add section with examples. + * + * @param examples text of this section + */ + void SetExamples(std::string_view examples) { + SetExamples(TString(examples)); + } + + /** + * Add section with examples. + * + * @param examples text of this section + */ + void SetExamples(const char* examples) { + SetExamples(TString(examples)); + } + /** * Set minimal number of free args * @@ -490,6 +521,13 @@ namespace NLastGetopt { return FreeArgSpecs_; } + /** + * Get index from where trailing arguments start + */ + ui32 GetTrailingArgsIndex() const { + return FreeArgSpecs_.empty() ? 0 : FreeArgSpecs_.rbegin()->first + 1; + } + /** * Set exact expected number of free args * @@ -529,7 +567,7 @@ namespace NLastGetopt { /** * Legacy, don't use. Same as `SetTrailingArgTitle`. - * Older versions of lastgetopt didn't have destinction between default title and title + * Older versions of lastgetopt didn't have distinction between default title and title * for the trailing argument. */ void SetFreeArgDefaultTitle(const TString& title, const TString& help = TString()) { diff --git a/library/cpp/getopt/small/last_getopt_parse_result.cpp b/library/cpp/getopt/small/last_getopt_parse_result.cpp index 60effba41b4..016fe347141 100644 --- a/library/cpp/getopt/small/last_getopt_parse_result.cpp +++ b/library/cpp/getopt/small/last_getopt_parse_result.cpp @@ -9,6 +9,44 @@ namespace NLastGetopt { return nullptr; } + void TOptsParseResult::BuildTaggedFreeArgs(const TOpts* options) { + TaggedFreeArgs_.clear(); + + if (!Parser_) { + return; + } + + const size_t freeArgsPos = GetFreeArgsPos(); + for (size_t argPos = freeArgsPos; argPos < Parser_->Argc_; ++argPos) { + size_t index = argPos - freeArgsPos; + + TString value = Parser_->Argv_[argPos]; + ui32 tag = 0; + + if (options) { + const TFreeArgSpec* spec = nullptr; + auto it = options->FreeArgSpecs_.find(index); + if (it != options->FreeArgSpecs_.end()) { + spec = &it->second; + } else if (options->FreeArgsMax_ == TOpts::UNLIMITED_ARGS) { + ui32 trailingArgsIndex = options->GetTrailingArgsIndex(); + if (index >= trailingArgsIndex) { + spec = &options->TrailingArgSpec_; + } + } + + if (spec) { + tag = spec->GetTag(value); + } + } + + TaggedFreeArgs_.push_back(TTaggedArg { + .Value = value, + .Tag = tag + }); + } + } + const TOptParseResult* TOptsParseResult::FindOptParseResult(const TOpt* opt, bool includeDefault) const { const TOptParseResult* r = FindParseResult(Opts_, opt); if (nullptr == r && includeDefault) @@ -99,6 +137,14 @@ namespace NLastGetopt { return Parser_->ProgramName_; } + void TOptsParseResult::SetProgramSubcommandPath(const TVector& parts) { + ProgramSubcommandPath_ = parts; + } + + const TVector& TOptsParseResult::GetProgramSubcommandPath() const { + return ProgramSubcommandPath_; + } + void TOptsParseResult::PrintUsage(IOutputStream& os) const { Parser_->Opts_->PrintUsage(Parser_->ProgramName_, os); } @@ -108,15 +154,16 @@ namespace NLastGetopt { } TVector TOptsParseResult::GetFreeArgs() const { - TVector v; - for (size_t i = GetFreeArgsPos(); i < Parser_->Argc_; ++i) { - v.push_back(Parser_->Argv_[i]); + TVector args; + args.reserve(TaggedFreeArgs_.size()); + for (const auto& arg : TaggedFreeArgs_) { + args.push_back(arg.Value); } - return v; + return args; } size_t TOptsParseResult::GetFreeArgCount() const { - return Parser_->Argc_ - GetFreeArgsPos(); + return TaggedFreeArgs_.size(); } void FindUserTypos(const TString& arg, const TOpts* options) { @@ -142,6 +189,7 @@ namespace NLastGetopt { } Y_ENSURE(options); + BuildTaggedFreeArgs(options); const auto freeArgs = GetFreeArgs(); for (size_t i = 0; i < freeArgs.size(); ++i) { if (i >= options->ArgBindings_.size()) { diff --git a/library/cpp/getopt/small/last_getopt_parse_result.h b/library/cpp/getopt/small/last_getopt_parse_result.h index c6e768c4617..6574667aede 100644 --- a/library/cpp/getopt/small/last_getopt_parse_result.h +++ b/library/cpp/getopt/small/last_getopt_parse_result.h @@ -3,7 +3,14 @@ #include "last_getopt_opts.h" #include "last_getopt_parser.h" +#include + namespace NLastGetopt { + struct TTaggedArg { + TString Value; + ui32 Tag = 0; + }; + /** * NLastGetopt::TOptParseResult contains all arguments for exactly one TOpt, * that have been fetched during parsing @@ -73,6 +80,8 @@ namespace NLastGetopt { TdVec Opts_; //Parsing result for all options, that have been explicitly defined in argc/argv TdVec OptsDef_; //Parsing result for options, that have been defined by default values only + TVector ProgramSubcommandPath_; + TVector TaggedFreeArgs_; private: TOptParseResult& OptParseResult(); @@ -87,6 +96,8 @@ namespace NLastGetopt { */ static const TOptParseResult* FindParseResult(const TdVec& vec, const TOpt* opt); + void BuildTaggedFreeArgs(const TOpts* options); + protected: /** * Performs parsing of comand line arguments. @@ -161,6 +172,8 @@ namespace NLastGetopt { * @return argv[0] */ TString GetProgramName() const; + void SetProgramSubcommandPath(const TVector& parts); + const TVector& GetProgramSubcommandPath() const; /** * Print usage string. @@ -182,6 +195,22 @@ namespace NLastGetopt { */ TVector GetFreeArgs() const; + template + TVector GetFreeArgs(E tag) const { + TVector args; + ui32 ui32Tag = static_cast(tag); + for (const auto& arg : TaggedFreeArgs_) { + if (arg.Tag == ui32Tag) { + args.push_back(arg.Value); + } + } + return args; + } + + const TVector& GetTaggedFreeArgs() const { + return TaggedFreeArgs_; + } + /** * @return true if given option exist in results of parsing * @@ -192,7 +221,7 @@ namespace NLastGetopt { bool Has(const TOpt* opt, bool includeDefault = false) const; /** - * @return nil terminated string on the last fetched argument of givne option + * @return nil terminated string on the last fetched argument of given option * * @param opt ptr on required object * @param includeDefault search in results obtained from default values @@ -200,7 +229,7 @@ namespace NLastGetopt { const char* Get(const TOpt* opt, bool includeDefault = true) const; /** - * @return nil terminated string on the last fetched argument of givne option + * @return nil terminated string on the last fetched argument of given option * if option haven't been fetched, given defaultValue will be returned * * @param opt ptr on required object @@ -218,7 +247,7 @@ namespace NLastGetopt { bool Has(const TString& name, bool includeDefault = false) const; /** - * @return nil terminated string on the last fetched argument of givne option + * @return nil terminated string on the last fetched argument of given option * * @param name long name of required object * @param includeDefault search in results obtained from default values @@ -226,7 +255,7 @@ namespace NLastGetopt { const char* Get(const TString& name, bool includeDefault = true) const; /** - * @return nil terminated string on the last fetched argument of givne option + * @return nil terminated string on the last fetched argument of given option * if option haven't been fetched, given defaultValue will be returned * * @param name long name of required object @@ -244,7 +273,7 @@ namespace NLastGetopt { bool Has(char name, bool includeDefault = false) const; /** - * @return nil terminated string on the last fetched argument of givne option + * @return nil terminated string on the last fetched argument of given option * * @param c short name of required object * @param includeDefault search in results obtained from default values @@ -252,7 +281,7 @@ namespace NLastGetopt { const char* Get(char name, bool includeDefault = true) const; /** - * @return nil terminated string on the last fetched argument of givne option + * @return nil terminated string on the last fetched argument of given option * if option haven't been fetched, given defaultValue will be returned * * @param c short name of required object @@ -261,7 +290,7 @@ namespace NLastGetopt { const char* GetOrElse(char name, const char* defaultValue) const; /** - * for givne option return parsed value of the last fetched argument + * for given option return parsed value of the last fetched argument * if option haven't been fetched, HandleError action is called * * @param opt required option (one of: ptr, short name, long name). @@ -280,7 +309,7 @@ namespace NLastGetopt { } /** - * for givne option return parsed value of the last fetched argument + * for given option return parsed value of the last fetched argument * if option haven't been fetched, given defaultValue will be returned * * @param opt required option (one of: ptr, short name, long name). diff --git a/library/cpp/getopt/small/last_getopt_parser.cpp b/library/cpp/getopt/small/last_getopt_parser.cpp index 911c76f3425..98c3951a1c7 100644 --- a/library/cpp/getopt/small/last_getopt_parser.cpp +++ b/library/cpp/getopt/small/last_getopt_parser.cpp @@ -170,9 +170,14 @@ namespace NLastGetopt { bool TOptsParser::ParseOptParam(const TOpt* opt, size_t pos) { Y_ASSERT(opt); - if (opt->GetHasArg() == NO_ARGUMENT || opt->IsEqParseOnly()) { + if (opt->GetHasArg() == NO_ARGUMENT || + opt->GetHasArg() == OPTIONAL_ARGUMENT && opt->IsEqParseOnly()) { return Commit(opt, nullptr, pos, 0); } + if (opt->IsEqParseOnly()) { + Y_ASSERT(opt->GetHasArg() == REQUIRED_ARGUMENT); + throw TUsageException() << "option " << opt->ToShortString() << " requires an argument but only accepts it over ="; + } if (pos == Argc_) { if (opt->GetHasArg() == REQUIRED_ARGUMENT) throw TUsageException() << "option " << opt->ToShortString() << " must have arg"; diff --git a/library/cpp/getopt/small/last_getopt_parser.h b/library/cpp/getopt/small/last_getopt_parser.h index 2cf8a6c308d..dca23551508 100644 --- a/library/cpp/getopt/small/last_getopt_parser.h +++ b/library/cpp/getopt/small/last_getopt_parser.h @@ -46,7 +46,7 @@ namespace NLastGetopt { bool GotMinusMinus_; //true if "--" have been seen in argv protected: - const TOpt* CurrentOpt_; // ptr on the last meeted option + const TOpt* CurrentOpt_; // ptr on the last met option TStringBuf CurrentValue_; // the value of the last met argument (corresponding to CurrentOpt_) private: diff --git a/library/cpp/getopt/small/modchooser.cpp b/library/cpp/getopt/small/modchooser.cpp index 3b3b0b7751e..903061dd8d9 100644 --- a/library/cpp/getopt/small/modchooser.cpp +++ b/library/cpp/getopt/small/modchooser.cpp @@ -58,6 +58,14 @@ class ClassWrapper: public TMainClass { TMainClassV* Main; }; +void TMainClass::SetSubcommandPath(TVector parts) { + SubcommandPath_ = std::move(parts); +} + +const TVector& TMainClass::GetSubcommandPath() const { + return SubcommandPath_; +} + TModChooser::TMode::TMode(const TString& name, TMainClass* main, const TString& descr, bool hidden, bool noCompletion) : Name(name) , Main(main) @@ -161,6 +169,14 @@ void TModChooser::AddCompletions(TString progName, const TString& name, bool hid } } +void TModChooser::SetSubcommandPath(const TVector& subcommandPath) const { + SubcommandPath_ = subcommandPath; +} + +const TVector& TModChooser::GetSubcommandPath() const { + return SubcommandPath_; +} + int TModChooser::Run(const int argc, const char** argv) const { Y_ENSURE(argc, "Can't run TModChooser with empty list of arguments."); @@ -202,6 +218,10 @@ int TModChooser::Run(const int argc, const char** argv) const { return 1; } + TVector subcommandPath = SubcommandPath_; + subcommandPath.push_back(modeIter->second->Name); + modeIter->second->Main->SetSubcommandPath(std::move(subcommandPath)); + if (shiftArgs) { TString firstArg; TVector nargv(Reserve(argc)); @@ -329,7 +349,11 @@ bool TModChooser::IsSvnRevisionOptionDisabled() const { } int TMainClassArgs::Run(int argc, const char** argv) { - return DoRun(NLastGetopt::TOptsParseResult(&GetOptions(), argc, argv)); + NLastGetopt::TOptsParseResult res(&GetOptions(), argc, argv); + if (!GetSubcommandPath().empty()) { + res.SetProgramSubcommandPath(GetSubcommandPath()); + } + return DoRun(std::move(res)); } const NLastGetopt::TOpts& TMainClassArgs::GetOptions() { @@ -355,10 +379,11 @@ int TMainClassModes::operator()(const int argc, const char** argv) { int TMainClassModes::Run(int argc, const char** argv) { auto& chooser = GetSubModes(); + chooser.SetSubcommandPath(GetSubcommandPath()); return chooser.Run(argc, argv); } -const TModChooser& TMainClassModes::GetSubModes() { +TModChooser& TMainClassModes::GetSubModes() { if (Modes_.Empty()) { Modes_.ConstructInPlace(); RegisterModes(Modes_.GetRef()); @@ -367,6 +392,10 @@ const TModChooser& TMainClassModes::GetSubModes() { return Modes_.GetRef(); } +const TModChooser& TMainClassModes::GetSubModes() const { + return const_cast(this)->GetSubModes(); +} + void TMainClassModes::RegisterModes(TModChooser& modes) { modes.SetModesHelpOption("-h"); } diff --git a/library/cpp/getopt/small/modchooser.h b/library/cpp/getopt/small/modchooser.h index d41ae780052..81f0629015e 100644 --- a/library/cpp/getopt/small/modchooser.h +++ b/library/cpp/getopt/small/modchooser.h @@ -28,6 +28,13 @@ class TMainClass { public: virtual int operator()(int argc, const char** argv) = 0; virtual ~TMainClass() = default; + + void SetSubcommandPath(TVector parts); + + const TVector& GetSubcommandPath() const; + +protected: + TVector SubcommandPath_; }; //! Function to handle '--version' parameter @@ -89,6 +96,9 @@ class TModChooser { void AddCompletions(TString progName, const TString& name = "completion", bool hidden = false, bool noCompletion = false); + void SetSubcommandPath(const TVector& subcommandPath) const; + const TVector& GetSubcommandPath() const; + /*! Run appropriate mode. * * In this method following things happen: @@ -184,6 +194,8 @@ class TModChooser { * then help message will be printed to stdout */ bool HelpAlwaysToStdErr{true}; + + mutable TVector SubcommandPath_; }; //! Mode class that allows introspecting its console arguments. @@ -219,7 +231,8 @@ class TMainClassModes: public TMainClass { int Run(int argc, const char** argv); //! Get sub-modes for this mode. - const TModChooser& GetSubModes(); + TModChooser& GetSubModes(); + const TModChooser& GetSubModes() const; protected: //! Fill given modchooser with sub-modes. diff --git a/library/cpp/getopt/ut/CMakeLists.txt b/library/cpp/getopt/ut/CMakeLists.txt index ee1a898a7ca..09465ddc741 100644 --- a/library/cpp/getopt/ut/CMakeLists.txt +++ b/library/cpp/getopt/ut/CMakeLists.txt @@ -1,65 +1,11 @@ -add_ydb_test(NAME getopt-last_getopt_ut +add_ydb_test(NAME getopt-ut SOURCES last_getopt_ut.cpp - LINK_LIBRARIES - getopt - cpp-testing-unittest_main - LABELS - unit -) - -add_ydb_test(NAME getopt-modchooser_ut - SOURCES modchooser_ut.cpp - LINK_LIBRARIES - getopt - cpp-testing-unittest_main - LABELS - unit -) - -add_ydb_test(NAME getopt-opt2_ut - SOURCES opt2_ut.cpp - LINK_LIBRARIES - getopt - cpp-testing-unittest_main - LABELS - unit -) - -add_ydb_test(NAME getopt-opt_ut - SOURCES opt_ut.cpp - LINK_LIBRARIES - getopt - cpp-testing-unittest_main - LABELS - unit -) - -add_ydb_test(NAME getopt-posix_getopt_ut - SOURCES posix_getopt_ut.cpp - LINK_LIBRARIES - getopt - cpp-testing-unittest_main - LABELS - unit -) - -add_ydb_test(NAME getopt-wrap_ut - SOURCES wrap.cpp - LINK_LIBRARIES - getopt-small - cpp-testing-unittest_main - LABELS - unit -) - -add_ydb_test(NAME getopt-ygetopt_ut - SOURCES ygetopt_ut.cpp LINK_LIBRARIES getopt diff --git a/library/cpp/getopt/ut/last_getopt_ut.cpp b/library/cpp/getopt/ut/last_getopt_ut.cpp index b517ea359d2..31fc923d31e 100644 --- a/library/cpp/getopt/ut/last_getopt_ut.cpp +++ b/library/cpp/getopt/ut/last_getopt_ut.cpp @@ -28,8 +28,6 @@ namespace { Init(opts, (int)Argv_.size(), Argv_.data()); } }; - - using V = TVector; } struct TOptsParserTester { @@ -137,7 +135,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { TOptsNoDefault opts; opts.AddLongOption("from"); opts.AddLongOption("to"); - TOptsParseResultTestWrapper r(&opts, V({"copy", "--from=/", "--to=/etc"})); + TOptsParseResultTestWrapper r(&opts, {"copy", "--from=/", "--to=/etc"}); UNIT_ASSERT_VALUES_EQUAL("copy", r.GetProgramName()); UNIT_ASSERT_VALUES_EQUAL("/", r.Get("from")); @@ -154,7 +152,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddCharOption('R', NO_ARGUMENT); opts.AddCharOption('l', NO_ARGUMENT); opts.AddCharOption('h', NO_ARGUMENT); - TOptsParseResultTestWrapper r(&opts, V({"cp", "/etc", "-Rl", "/tmp/etc"})); + TOptsParseResultTestWrapper r(&opts, {"cp", "/etc", "-Rl", "/tmp/etc"}); UNIT_ASSERT(r.Has('R')); UNIT_ASSERT(r.Has('l')); UNIT_ASSERT(!r.Has('h')); @@ -168,20 +166,36 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { Y_UNIT_TEST(TestFreeArgs) { TOptsNoDefault opts; opts.SetFreeArgsNum(1, 3); - TOptsParseResultTestWrapper r11(&opts, V({"cp", "/etc"})); - TOptsParseResultTestWrapper r12(&opts, V({"cp", "/etc", "/tmp/etc"})); - TOptsParseResultTestWrapper r13(&opts, V({"cp", "/etc", "/tmp/etc", "verbose"})); + TOptsParseResultTestWrapper r11(&opts, {"cp", "/etc"}); + TOptsParseResultTestWrapper r12(&opts, {"cp", "/etc", "/tmp/etc"}); + TOptsParseResultTestWrapper r13(&opts, {"cp", "/etc", "/tmp/etc", "verbose"}); UNIT_ASSERT_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"cp", "/etc", "/tmp/etc", "verbose", "nosymlink"})), + TOptsParseResultTestWrapper(&opts, {"cp", "/etc", "/tmp/etc", "verbose", "nosymlink"}), yexception); UNIT_ASSERT_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"cp"})), + TOptsParseResultTestWrapper(&opts, {"cp"}), yexception); opts.SetFreeArgsNum(2); - TOptsParseResultTestWrapper r22(&opts, V({"cp", "/etc", "/var/tmp"})); + TOptsParseResultTestWrapper r22(&opts, {"cp", "/etc", "/var/tmp"}); + } + + Y_UNIT_TEST(TestProgramSubcommandPathSetter) { + TOptsNoDefault opts; + TOptsParseResultTestWrapper r(&opts, {"tool"}); + const TVector parts = {"tool", "sub", "command"}; + r.SetProgramSubcommandPath(parts); + UNIT_ASSERT_VALUES_EQUAL(parts, r.GetProgramSubcommandPath()); + } + + Y_UNIT_TEST(TestProgramCanonicalNameCompat) { + TOptsNoDefault opts; + TOptsParseResultTestWrapper r(&opts, {"tool"}); + r.SetProgramSubcommandPath({"tool", "outer", "inner"}); + const TVector expected = {"tool", "outer", "inner"}; + UNIT_ASSERT_VALUES_EQUAL(expected, r.GetProgramSubcommandPath()); } Y_UNIT_TEST(TestCharOptionsRequiredOptional) { @@ -191,7 +205,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddCharOption('x', REQUIRED_ARGUMENT); opts.AddCharOption('y', REQUIRED_ARGUMENT); opts.AddCharOption('l', NO_ARGUMENT); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "-ld11", "-e", "22", "-lllx33", "-y", "44"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "-ld11", "-e", "22", "-lllx33", "-y", "44"}); UNIT_ASSERT_VALUES_EQUAL("11", r.Get('d')); UNIT_ASSERT_VALUES_EQUAL("22", r.Get('e')); UNIT_ASSERT_VALUES_EQUAL("33", r.Get('x')); @@ -449,6 +463,19 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { tester.AcceptEndOfFreeArgs(); } + Y_UNIT_TEST(TestEqParseOnlyRequiredArgument) { + TOptsNoDefault opts; + + opts.AddLongOption("eq-only").RequiredArgument().DisableSpaceParse(); + + TOptsParseResultTestWrapper res(&opts, {"cmd", "--eq-only=value"}); + UNIT_ASSERT_EQUAL(res.Get("eq-only"), "value"sv); + + UNIT_ASSERT_EXCEPTION( + TOptsParseResultTestWrapper(&opts, {"cmd", "--eq-only", "value"}), + TUsageException); + } + Y_UNIT_TEST(TestStoreResult) { TOptsNoDefault opts; TString data; @@ -461,7 +488,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption("optional-number-0").StoreResult(&optionalNumber0); opts.AddLongOption("optional-string-1").StoreResult(&optionalString1); opts.AddLongOption("optional-number-1").StoreResult(&optionalNumber1); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--data=jjhh", "-n", "11", "--optional-number-1=8", "--optional-string-1=os1"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--data=jjhh", "-n", "11", "--optional-number-1=8", "--optional-string-1=os1"}); UNIT_ASSERT_VALUES_EQUAL("jjhh", data); UNIT_ASSERT_VALUES_EQUAL(11, number); UNIT_ASSERT(!optionalString0.Defined()); @@ -480,7 +507,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption('b', "beta").NoArgument().StoreValue(&b, 24); opts.AddLongOption('e', "enum").NoArgument().StoreValue(&e, REQUIRED_ARGUMENT).StoreValue(&c, 12345); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "-a", "-e"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "-a", "-e"}); UNIT_ASSERT_VALUES_EQUAL(42, a); UNIT_ASSERT_VALUES_EQUAL(0, b); @@ -497,7 +524,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddCharOption('c').StoreTrue(&c); opts.AddCharOption('d').StoreTrue(&d); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "-a", "-c"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "-a", "-c"}); UNIT_ASSERT(a); UNIT_ASSERT(!b); @@ -510,7 +537,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption("path").DefaultValue("/etc"); int value = 42; opts.AddLongOption("value").StoreResult(&value).DefaultValue(32); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "dfdf"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "dfdf"}); UNIT_ASSERT_VALUES_EQUAL("/etc", r.Get("path")); UNIT_ASSERT_VALUES_EQUAL(32, value); } @@ -519,7 +546,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { TOptsNoDefault opts; TVector vals; opts.AddLongOption('s', "split").SplitHandler(&vals, ','); - TOptsParseResultTestWrapper r(&opts, V({"prog", "--split=a,b,c"})); + TOptsParseResultTestWrapper r(&opts, {"prog", "--split=a,b,c"}); UNIT_ASSERT_EQUAL(vals.size(), 3); UNIT_ASSERT_EQUAL(vals[0], "a"); UNIT_ASSERT_EQUAL(vals[1], "b"); @@ -530,7 +557,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { TOptsNoDefault opts; TVector vals; opts.AddLongOption('s', "split").RangeSplitHandler(&vals, ',', '-'); - TOptsParseResultTestWrapper r(&opts, V({"prog", "--split=1,8-10", "--split=12-14"})); + TOptsParseResultTestWrapper r(&opts, {"prog", "--split=1,8-10", "--split=12-14"}); UNIT_ASSERT_EQUAL(vals.size(), 7); UNIT_ASSERT_EQUAL(vals[0], 1); UNIT_ASSERT_EQUAL(vals[1], 8); @@ -557,15 +584,15 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { // test 'not required' // makes sure that the problem will only be in 'required' - TOptsParseResultTestWrapper r1(&opts, V({"cmd"})); + TOptsParseResultTestWrapper r1(&opts, {"cmd"}); // test 'required' opt_d.Required(); UNIT_ASSERT_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"cmd"})), + TOptsParseResultTestWrapper(&opts, {"cmd"}), TUsageException); - TOptsParseResultTestWrapper r3(&opts, V({"cmd", "-d11"})); + TOptsParseResultTestWrapper r3(&opts, {"cmd", "-d11"}); UNIT_ASSERT_VALUES_EQUAL("11", r3.Get('d')); } @@ -581,12 +608,13 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { *Flag = true; } }; + Y_UNIT_TEST(TestHandlers) { { TOptsNoDefault opts; bool flag = false; opts.AddLongOption("flag").Handler0(HandlerStoreTrue(&flag)).NoArgument(); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--flag"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--flag"}); UNIT_ASSERT(flag); } { @@ -598,11 +626,11 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption("flag3").RequiredArgument().StoreMappedResult(&fval, (double (*)(double))fabs); opts.AddLongOption("flag4").RequiredArgument().StoreMappedResult(&fval, (double (*)(double))sqrt); UNIT_ASSERT_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"cmd", "--flag3", "-2.0", "--flag1", "-1"})), + TOptsParseResultTestWrapper(&opts, {"cmd", "--flag3", "-2.0", "--flag1", "-1"}), yexception); UNIT_ASSERT_VALUES_EQUAL(uval, 5u); UNIT_ASSERT_VALUES_EQUAL(fval, 2.0); - TOptsParseResultTestWrapper r1(&opts, V({"cmd", "--flag4", "9.0", "--flag2", "-1"})); + TOptsParseResultTestWrapper r1(&opts, {"cmd", "--flag4", "9.0", "--flag2", "-1"}); UNIT_ASSERT_VALUES_EQUAL(uval, Max()); UNIT_ASSERT_VALUES_EQUAL(fval, 3.0); } @@ -742,7 +770,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption("size").AppendTo(&ints); opts.AddLongOption("value").AppendTo(&strings); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--size=17", "--size=19", "--value=v1", "--value=v2"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--size=17", "--size=19", "--value=v1", "--value=v2"}); UNIT_ASSERT_VALUES_EQUAL(size_t(2), ints.size()); UNIT_ASSERT_VALUES_EQUAL(17, ints.at(0)); @@ -759,7 +787,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { TOptsNoDefault opts; opts.AddLongOption("path").EmplaceTo(&richPaths); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--path=//cool", "--path=//nice"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--path=//cool", "--path=//nice"}); UNIT_ASSERT_VALUES_EQUAL(size_t(2), richPaths.size()); UNIT_ASSERT_VALUES_EQUAL("//cool", std::get<0>(richPaths.at(0))); @@ -772,7 +800,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { TOptsNoDefault opts; opts.AddLongOption("set").KVHandler([&keyvals](TString k, TString v) { keyvals << k << ":" << v << ","; }); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--set", "x=1", "--set", "y=2", "--set=z=3"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--set", "x=1", "--set", "y=2", "--set=z=3"}); UNIT_ASSERT_VALUES_EQUAL(keyvals, "x:1,y:2,z:3,"); } @@ -784,25 +812,25 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { { gSimpleFlag = false; - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--abstract"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--abstract"}); UNIT_ASSERT(!flag); UNIT_ASSERT(!gSimpleFlag); } { - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--abstract", "--global", "-t"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--abstract", "--global", "-t"}); UNIT_ASSERT(flag); UNIT_ASSERT(gSimpleFlag); } { UNIT_ASSERT_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"cmd", "--true"})), + TOptsParseResultTestWrapper(&opts, {"cmd", "--true"}), TUsageException); } { - TOptsParseResultTestWrapper r(&opts, V({"cmd", "--abstract", "--buffer=512"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "--abstract", "--buffer=512"}); UNIT_ASSERT(r.Has('b')); UNIT_ASSERT_VALUES_EQUAL(r.Get('b', 0), "512"); } @@ -826,12 +854,81 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { int number = 0; opts.AddFreeArgBinding("data", data); opts.AddFreeArgBinding("number", number); - TOptsParseResultTestWrapper r(&opts, V({"cmd", "hello", "25"})); + TOptsParseResultTestWrapper r(&opts, {"cmd", "hello", "25"}); UNIT_ASSERT_VALUES_EQUAL("hello", data); UNIT_ASSERT_VALUES_EQUAL(25, number); UNIT_ASSERT_VALUES_EQUAL(2, r.GetFreeArgCount()); } + Y_UNIT_TEST(TestFreeArgsTaggedAccess) { + enum class EFreeArgTag { + Unknown, + Src, + Dst, + }; + + TOptsNoDefault opts; + opts.SetFreeArgsNum(2); + opts.GetFreeArgSpec(0).SetTag(EFreeArgTag::Src); + opts.GetFreeArgSpec(1).SetTag(EFreeArgTag::Dst); + + TOptsParseResultTestWrapper r(&opts, {"cmd", "input.txt", "output.txt"}); + + const auto allArgs = r.GetFreeArgs(); + UNIT_ASSERT_VALUES_EQUAL(2u, allArgs.size()); + + const auto srcArgs = r.GetFreeArgs(EFreeArgTag::Src); + UNIT_ASSERT_VALUES_EQUAL(1u, srcArgs.size()); + UNIT_ASSERT_VALUES_EQUAL("input.txt", srcArgs.front()); + + const auto dstArgs = r.GetFreeArgs(EFreeArgTag::Dst); + UNIT_ASSERT_VALUES_EQUAL(1u, dstArgs.size()); + UNIT_ASSERT_VALUES_EQUAL("output.txt", dstArgs.front()); + + UNIT_ASSERT(r.GetFreeArgs(EFreeArgTag::Unknown).empty()); + } + + Y_UNIT_TEST(TestTrailingFreeArgsTagger) { + enum class EFreeArgTag { + Unknown, + Primary, + Logs, + Temp, + }; + + TOptsNoDefault opts; + opts.SetFreeArgsMin(1); + opts.SetFreeArgsMax(TOpts::UNLIMITED_ARGS); + + opts.GetFreeArgSpec(0).SetTag(EFreeArgTag::Primary); + opts.GetTrailingArgSpec().SetTag([](const TString& value) { + if (value.EndsWith(".log")) { + return EFreeArgTag::Logs; + } + if (value.EndsWith(".tmp")) { + return EFreeArgTag::Temp; + } + return EFreeArgTag::Unknown; + }); + + TOptsParseResultTestWrapper r(&opts, {"cmd", "config.yaml", "db.log", "cache.tmp", "service.log"}); + + const auto primary = r.GetFreeArgs(EFreeArgTag::Primary); + UNIT_ASSERT_VALUES_EQUAL(1u, primary.size()); + UNIT_ASSERT_VALUES_EQUAL("config.yaml", primary.front()); + + const auto logs = r.GetFreeArgs(EFreeArgTag::Logs); + UNIT_ASSERT_VALUES_EQUAL(2u, logs.size()); + UNIT_ASSERT_VALUES_EQUAL("db.log", logs[0]); + UNIT_ASSERT_VALUES_EQUAL("service.log", logs[1]); + + const auto temp = r.GetFreeArgs(EFreeArgTag::Temp); + UNIT_ASSERT_VALUES_EQUAL(1u, temp.size()); + UNIT_ASSERT_VALUES_EQUAL("cache.tmp", temp.front()); + + UNIT_ASSERT_VALUES_EQUAL(4u, r.GetFreeArgCount()); + } + Y_UNIT_TEST(TestCheckUserTypos) { { TOptsNoDefault opts; @@ -840,10 +937,10 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption("to"); UNIT_ASSERT_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"copy", "-from", "/home", "--to=/etc"})), + TOptsParseResultTestWrapper(&opts, {"copy", "-from", "/home", "--to=/etc"}), TUsageException); UNIT_ASSERT_NO_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"copy", "--from", "from", "--to=/etc"}))); + TOptsParseResultTestWrapper(&opts, {"copy", "--from", "from", "--to=/etc"})); } { @@ -853,7 +950,39 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.AddLongOption('r', "read", ""); opts.AddLongOption("fr"); UNIT_ASSERT_NO_EXCEPTION( - TOptsParseResultTestWrapper(&opts, V({"copy", "-fr"}))); + TOptsParseResultTestWrapper(&opts, {"copy", "-fr"})); } } + + Y_UNIT_TEST(TestMutuallyExclusive) { + // FIXME: somehow MutuallyExclusive() does not work without SetFlag() + bool flag; + TOptsNoDefault opts; + opts.AddLongOption("do").SetFlag(&flag); + opts.AddLongOption("dont").SetFlag(&flag); + opts.AddLongOption("maybe-do-maybe-dont").SetFlag(&flag); + + opts.MutuallyExclusive("do", "dont", "maybe-do-maybe-dont"); + + UNIT_ASSERT_EXCEPTION( + TOptsParseResultTestWrapper(&opts, {"--do", "--dont"}), + TUsageException + ); + UNIT_ASSERT_EXCEPTION( + TOptsParseResultTestWrapper(&opts, {"--dont", "--maybe-do-maybe-dont"}), + TUsageException + ); + UNIT_ASSERT_EXCEPTION( + TOptsParseResultTestWrapper(&opts, {"--do", "--maybe-do-maybe-dont"}), + TUsageException + ); + UNIT_ASSERT_EXCEPTION( + TOptsParseResultTestWrapper(&opts, {"-d", "-n"}), + TUsageException + ); + UNIT_ASSERT_EXCEPTION( + TOptsParseResultTestWrapper(&opts, {"--do", "--dont", "--maybe-do-maybe-dont"}), + TUsageException + ); + } } diff --git a/library/cpp/getopt/ut/modchooser_ut.cpp b/library/cpp/getopt/ut/modchooser_ut.cpp index a14c8a58535..24776f47fc6 100644 --- a/library/cpp/getopt/ut/modchooser_ut.cpp +++ b/library/cpp/getopt/ut/modchooser_ut.cpp @@ -39,6 +39,42 @@ static const F_PTR FUNCTIONS[] = {One, Two, Three, Four, Five}; static const char* NAMES[] = {"one", "two", "three", "four", "five"}; static_assert(Y_ARRAY_SIZE(FUNCTIONS) == Y_ARRAY_SIZE(NAMES), "Incorrect input tests data"); +class TRecordingAction: public TMainClassArgs { +public: + int DoRun(NLastGetopt::TOptsParseResult&& /*res*/) override { + CapturedSubcommandPath = GetSubcommandPath(); + return 0; + } + + void RegisterOptions(NLastGetopt::TOpts& opts) override { + opts.SetFreeArgsMax(2); + opts.AddLongOption("options-flag") + .Optional() + .NoArgument() + .StoreTrue(&OptionsFlag); + } + +public: + bool OptionsFlag = false; + TVector CapturedSubcommandPath; +}; + +class TOuterModes: public TMainClassModes { +public: + explicit TOuterModes(TMainClass* inner) + : Inner_(inner) + { + } + +protected: + void RegisterModes(TModChooser& modes) override { + modes.AddMode("inner", Inner_, "inner"); + } + +private: + TMainClass* Inner_; +}; + Y_UNIT_TEST_SUITE(TModChooserTest) { Y_UNIT_TEST(TestModesSimpleRunner) { TModChooser chooser; @@ -68,4 +104,18 @@ Y_UNIT_TEST_SUITE(TModChooserTest) { chooser.Run(argc, argv); } + + Y_UNIT_TEST(TestSubcommandPathPropagation) { + TRecordingAction innerAction; + TOuterModes outer(&innerAction); + TModChooser chooser; + chooser.AddMode("outer", &outer, "outer"); + + const char* argv[] = {"UNITTEST", "outer", "inner", "--options-flag", "free-arg1", "free-arg2", nullptr}; + UNIT_ASSERT_NO_EXCEPTION(chooser.Run(6, argv)); + + const TVector expected = {"outer", "inner"}; + UNIT_ASSERT_EQUAL(innerAction.OptionsFlag, true); + UNIT_ASSERT_VALUES_EQUAL(expected, innerAction.CapturedSubcommandPath); + } } diff --git a/library/cpp/html/escape/CMakeLists.txt b/library/cpp/html/escape/CMakeLists.txt new file mode 100644 index 00000000000..ddb97a14252 --- /dev/null +++ b/library/cpp/html/escape/CMakeLists.txt @@ -0,0 +1,25 @@ +if (YDB_SDK_TESTS) + add_ydb_test(NAME html-escape-ut + SOURCES + ut/escape_ut.cpp + LINK_LIBRARIES + html-escape + cpp-testing-unittest_main + LABELS + unit + ) +endif() + +_ydb_sdk_add_library(html-escape) + +target_link_libraries(html-escape + PUBLIC + yutil +) + +target_sources(html-escape + PRIVATE + escape.cpp +) + +_ydb_sdk_install_targets(TARGETS html-escape) diff --git a/library/cpp/html/escape/escape.cpp b/library/cpp/html/escape/escape.cpp new file mode 100644 index 00000000000..5b8ed60f045 --- /dev/null +++ b/library/cpp/html/escape/escape.cpp @@ -0,0 +1,66 @@ +#include "escape.h" + +#include +#include + +namespace NHtml { + namespace { + struct TReplace { + char Char; + bool ForText; + TStringBuf Entity; + }; + + TReplace Escapable[] = { + {'"', false, TStringBuf(""")}, + {'&', true, TStringBuf("&")}, + {'<', true, TStringBuf("<")}, + {'>', true, TStringBuf(">")}, + }; + + TString EscapeImpl(const TString& value, bool isText) { + auto ci = value.begin(); + // Looking for escapable characters. + for (; ci != value.end(); ++ci) { + for (size_t i = (isText ? 1 : 0); i < Y_ARRAY_SIZE(Escapable); ++i) { + if (*ci == Escapable[i].Char) { + goto escape; + } + } + } + + // There is no escapable characters, so return original value. + return value; + + escape: + TString tmp = TString(value.begin(), ci); + + for (; ci != value.end(); ++ci) { + size_t i = (isText ? 1 : 0); + + for (; i < Y_ARRAY_SIZE(Escapable); ++i) { + if (*ci == Escapable[i].Char) { + tmp += Escapable[i].Entity; + break; + } + } + + if (i == Y_ARRAY_SIZE(Escapable)) { + tmp += *ci; + } + } + + return tmp; + } + + } + + TString EscapeAttributeValue(const TString& value) { + return EscapeImpl(value, false); + } + + TString EscapeText(const TString& value) { + return EscapeImpl(value, true); + } + +} diff --git a/library/cpp/html/escape/escape.h b/library/cpp/html/escape/escape.h new file mode 100644 index 00000000000..1c45fc51933 --- /dev/null +++ b/library/cpp/html/escape/escape.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace NHtml { + TString EscapeAttributeValue(const TString& value); + TString EscapeText(const TString& value); + +} diff --git a/library/cpp/html/escape/ut/escape_ut.cpp b/library/cpp/html/escape/ut/escape_ut.cpp new file mode 100644 index 00000000000..cd7b9551385 --- /dev/null +++ b/library/cpp/html/escape/ut/escape_ut.cpp @@ -0,0 +1,16 @@ +#include +#include + +using namespace NHtml; + +Y_UNIT_TEST_SUITE(TEscapeHtml) { + Y_UNIT_TEST(Escape) { + UNIT_ASSERT_EQUAL(EscapeText("in & out"), "in & out"); + UNIT_ASSERT_EQUAL(EscapeText("&&"), "&&"); + UNIT_ASSERT_EQUAL(EscapeText("&"), "&amp;"); + + UNIT_ASSERT_EQUAL(EscapeText("