diff --git a/absl/log/CMakeLists.txt b/absl/log/CMakeLists.txt index e0a307b3aff..881a775a670 100644 --- a/absl/log/CMakeLists.txt +++ b/absl/log/CMakeLists.txt @@ -401,6 +401,21 @@ absl_cc_library( absl::span ) +absl_cc_test( + NAME + log_internal_append_truncated_test + SRCS + "internal/append_truncated_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_append_truncated + absl::span + GTest::gmock_main +) + # Public targets absl_cc_library( NAME diff --git a/absl/log/internal/BUILD.bazel b/absl/log/internal/BUILD.bazel index d9cb7b20f70..96f698b7cd0 100644 --- a/absl/log/internal/BUILD.bazel +++ b/absl/log/internal/BUILD.bazel @@ -235,6 +235,17 @@ cc_library( ], ) +cc_test( + name = "append_truncated_test", + srcs = ["append_truncated_test.cc"], + deps = [ + ":append_truncated", + "//absl/types:span", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + cc_library( name = "log_sink_set", srcs = ["log_sink_set.cc"], diff --git a/absl/log/internal/append_truncated.h b/absl/log/internal/append_truncated.h index d420a8b5c6a..7adcd2eadc4 100644 --- a/absl/log/internal/append_truncated.h +++ b/absl/log/internal/append_truncated.h @@ -16,6 +16,7 @@ #define ABSL_LOG_INTERNAL_APPEND_TRUNCATED_H_ #include +#include #include #include @@ -35,6 +36,14 @@ inline size_t AppendTruncated(absl::string_view src, absl::Span &dst) { dst.remove_prefix(src.size()); return src.size(); } +inline bool IsHighSurrogate(wchar_t wc) { + const uint32_t v = static_cast(wc); + return v >= 0xD800 && v <= 0xDBFF; +} +inline bool IsLowSurrogate(wchar_t wc) { + const uint32_t v = static_cast(wc); + return v >= 0xDC00 && v <= 0xDFFF; +} // Likewise, but it also takes a wide character string and transforms it into a // UTF-8 encoded byte string regardless of the current locale. // - On platforms where `wchar_t` is 2 bytes (e.g., Windows), the input is @@ -42,17 +51,32 @@ inline size_t AppendTruncated(absl::string_view src, absl::Span &dst) { // - On platforms where `wchar_t` is 4 bytes (e.g., Linux, macOS), the input // is treated as UTF-32. inline size_t AppendTruncated(std::wstring_view src, absl::Span &dst) { + constexpr wchar_t kReplacementCharacter = L'\uFFFD'; absl::strings_internal::ShiftState state; size_t total_bytes_written = 0; - for (const wchar_t wc : src) { - // If the destination buffer might not be large enough to write the next + for (size_t i = 0; i < src.size(); ++i) { + // A pending high surrogate already reserved the four bytes of the sequence + // it started, so the low surrogate completing it always fits. Otherwise, if + // the destination buffer might not be large enough to write the next // character, stop. - if (dst.size() < absl::strings_internal::kMaxEncodedUTF8Size) break; + if (!state.saw_high_surrogate && + dst.size() < absl::strings_internal::kMaxEncodedUTF8Size) { + break; + } + wchar_t wc = src[i]; + // `WideToUtf8()` encodes a surrogate pair over two calls, emitting the + // first two bytes of a four-byte sequence for the high surrogate and the + // remaining two for the low one. Unless the matching low surrogate follows + // immediately, those first two bytes would be left in `dst` as a partial + // sequence, so encode U+FFFD for the unpaired high surrogate instead. + if (IsHighSurrogate(wc) && + !(i + 1 < src.size() && IsLowSurrogate(src[i + 1]))) { + wc = kReplacementCharacter; + } size_t bytes_written = absl::strings_internal::WideToUtf8(wc, dst.data(), state); if (bytes_written == static_cast(-1)) { // Invalid character. Encode REPLACEMENT CHARACTER (U+FFFD) instead. - constexpr wchar_t kReplacementCharacter = L'\uFFFD'; bytes_written = absl::strings_internal::WideToUtf8(kReplacementCharacter, dst.data(), state); } diff --git a/absl/log/internal/append_truncated_test.cc b/absl/log/internal/append_truncated_test.cc new file mode 100644 index 00000000000..b002673cb2d --- /dev/null +++ b/absl/log/internal/append_truncated_test.cc @@ -0,0 +1,81 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/log/internal/append_truncated.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "absl/types/span.h" + +namespace { + +using ::absl::log_internal::AppendTruncated; + +// Runs `AppendTruncated(src, ...)` against a `capacity`-byte buffer and returns +// the bytes it wrote. +std::string Append(std::wstring_view src, size_t capacity) { + std::array buffer{}; + absl::Span dst(buffer.data(), capacity); + const size_t bytes_written = AppendTruncated(src, dst); + EXPECT_LE(bytes_written, capacity); + return std::string(buffer.data(), bytes_written); +} + +const std::string kReplacement = "\xEF\xBF\xBD"; // U+FFFD + +TEST(AppendTruncatedTest, EncodesSurrogatePair) { + // U+1F600, as a UTF-16 pair on platforms with a 2-byte wchar_t. + EXPECT_EQ(Append(L"\xD83D\xDE00", 32), "\xF0\x9F\x98\x80"); +} + +TEST(AppendTruncatedTest, TrailingUnpairedHighSurrogate) { + // The high surrogate encodes only the first two bytes of a four-byte + // sequence, so emitting it alone would leave a partial sequence behind. + EXPECT_EQ(Append(L"\xD800", 32), kReplacement); +} + +TEST(AppendTruncatedTest, HighSurrogateNotFollowedByLowSurrogate) { + const std::wstring high_then_ascii = std::wstring(1, wchar_t{0xD800}) + L"A"; + EXPECT_EQ(Append(high_then_ascii, 32), kReplacement + "A"); + EXPECT_EQ(Append(L"\xD800\xD801", 32), kReplacement + kReplacement); +} + +TEST(AppendTruncatedTest, IsolatedLowSurrogate) { + EXPECT_EQ(Append(L"\xDC00", 32), kReplacement); +} + +TEST(AppendTruncatedTest, SurrogatePairAtTruncationBoundary) { + const std::wstring src = + std::wstring(L"a") + wchar_t{0xD83D} + wchar_t{0xDE00}; + // Five bytes is exactly enough for "a" plus the four-byte sequence. + EXPECT_EQ(Append(src, 5), "a\xF0\x9F\x98\x80"); + EXPECT_EQ(Append(src, 6), "a\xF0\x9F\x98\x80"); + // Four bytes is not, so the pair is dropped rather than half-written. + EXPECT_EQ(Append(src, 4), "a"); +} + +TEST(AppendTruncatedTest, PlainCharactersAreUnaffected) { + EXPECT_EQ(Append(L"hello", 32), "hello"); + // Spell the wide literals with universal character names so the encoding + // does not depend on the source file code page. MSVC decodes narrow source + // bytes with the system code page unless /utf-8 is passed, which would make + // a literal "\u00e9\u4e2d" here map to the wrong wchar_t values. + EXPECT_EQ(Append(L"\u00E9\u4E2D", 32), "\xC3\xA9\xE4\xB8\xAD"); +} + +} // namespace