diff --git a/AGENTS.md b/AGENTS.md index 4f351115..8b95c617 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -229,6 +229,292 @@ Important files: When touching shared SIMD helpers, validate both parser and skip/on-demand tests. If possible, also test `--//:sonic_dispatch=dynamic` on x86. +## SIMD Architecture Organization - Detailed Analysis + +### Directory Structure Overview + +``` +include/sonic/internal/arch/ +├── simd_dispatch.h # Central dispatch macro definitions +├── simd_base.h # Base function dispatch (TrailingZeroes, etc.) +├── simd_quote.h # Quote/string function dispatch +├── simd_skip.h # Skip scanner function dispatch +├── simd_itoa.h # Integer-to-string function dispatch +├── simd_str2int.h # String-to-integer function dispatch +├── sonic_cpu_feature.h # CPU feature detection macros +├── common/ +│ ├── arm_common/ # Shared ARM implementations +│ │ ├── base.h # Shared bit manipulation (TrailingZeroes, etc.) +│ │ ├── simd.h # Shared to_bitmask() function +│ │ ├── quote.h # Shared Quote() implementation +│ │ ├── skip.inc.h # Shared skip implementations (template-based) +│ │ ├── itoa.h # Shared Utoa_8/Utoa_16 implementations +│ │ └── str2int.h # Shared simd_str2int (scalar fallback) +│ ├── x86_common/ # Shared x86 implementations +│ ├── quote_common.h # Scalar fallback for parseStringInplace/Quote +│ ├── quote_tables.h # Escape lookup tables (kEscapedMap, kQuoteTab*) +│ ├── skip_common.h # Shared EqBytes4, SkipLiteral, IsValidSeparator +│ └── unicode_common.h # Unicode handling (handle_unicode_codepoint, etc.) +├── neon/ # ARM NEON implementation +│ ├── base.h # Re-exports from arm_common +│ ├── simd.h # NEON-specific simd8/simd8x64 templates +│ ├── quote.h # NEON-specific parseStringInplace +│ ├── skip.h # NEON-specific SkipContainer, skip_space +│ ├── itoa.h # Re-exports from arm_common +│ ├── str2int.h # Re-exports from arm_common +│ └── unicode.h # NEON-specific StringBlock, GetNonSpaceBits +├── sve2-128/ # ARM SVE2-128 implementation +│ ├── base.h # Re-exports from arm_common +│ ├── simd.h # SVE2-specific type definitions (svuint8x16_t) +│ ├── quote.h # SVE2-specific parseStringInplace +│ ├── skip.h # SVE2-specific SkipContainer, skip_space +│ ├── itoa.h # Re-exports from arm_common +│ ├── str2int.h # SVE2-specific simd_str2int (optimized) +│ └── unicode.h # SVE2-specific StringBlock, GetNonSpaceBits +├── avx2/ # x86 AVX2 implementation +├── sse/ # x86 SSE implementation +└── x86_ifuncs/ # x86 dynamic dispatch (ifunc) +``` + +### Required Files Per Architecture Directory + +Each architecture directory (`neon/`, `sve2-128/`, `avx2/`, `sse/`) must contain these files: + +| File | Purpose | +|------|---------| +| `base.h` | Bit manipulation utilities (TrailingZeroes, ClearLowestBit, LeadingZeroes, CountOnes, PrefixXor, Xmemcpy, InlinedMemcmp, InlinedMemcmpEq) | +| `simd.h` | SIMD vector type definitions (`simd8`, `simd8x64`) with operator overloads, `to_bitmask()`, `eq()`, `lteq()` | +| `quote.h` | `parseStringInplace()` and `Quote()` functions | +| `skip.h` | `SkipContainer()`, `skip_space()`, `skip_space_safe()`, plus includes `skip.inc.h` for `GetNextToken()`, `SkipString()`, `GetStringBits()` | +| `itoa.h` | `Utoa_8()` and `Utoa_16()` for integer-to-string conversion | +| `str2int.h` | `simd_str2int()` for string-to-integer conversion | +| `unicode.h` | `StringBlock` struct with `Find()`, `HasQuoteFirst()`, `HasBackslash()`, `HasUnescaped()`, `GetNonSpaceBits()` | + +### Architecture Switching via Macro Definitions + +The entire dispatch mechanism is controlled by two key macros defined in `simd_dispatch.h:21-55`: + +**1. CPU Feature Detection (`sonic_cpu_feature.h:17-47`)** +```cpp +// x86 detection +#if defined(__SSE2__) +#define SONIC_HAVE_SSE +// ... SSE3, SSSE3, SSE4_1, SSE4_2, AVX, AVX2 +#endif + +// ARM detection +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#define SONIC_HAVE_NEON +#endif +#if defined(__ARM_FEATURE_SVE2) && (__ARM_FEATURE_SVE_BITS == 128) +#define SONIC_HAVE_SVE2_128 +#endif +``` + +**2. Static Dispatch (`simd_dispatch.h:25-42`)** +```cpp +#if defined(SONIC_STATIC_DISPATCH) +// x86 priority: AVX2 > SSE +#if defined(SONIC_HAVE_AVX2) +#define SONIC_USING_ARCH_FUNC(func) using avx2::func +#define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(avx2/file) +#elif defined(SONIC_HAVE_SSE) +#define SONIC_USING_ARCH_FUNC(func) using sse::func +#define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(sse/file) +#endif + +// ARM priority: SVE2-128 > NEON +#if defined(SONIC_HAVE_SVE2_128) +#define SONIC_USING_ARCH_FUNC(func) using sve2_128::func +#define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(sve2-128/file) +#elif defined(SONIC_HAVE_NEON) +#define SONIC_USING_ARCH_FUNC(func) using neon::func +#define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(neon/file) +#endif +``` + +**3. Dynamic Dispatch (`simd_dispatch.h:44-53`)** +```cpp +#elif defined(SONIC_DYNAMIC_DISPATCH) +#if defined(__x86_64__) +#define SONIC_USING_ARCH_FUNC(func) // Empty - uses ifunc resolution +#define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(x86_ifuncs/file) +#elif defined(SONIC_HAVE_NEON) +#define SONIC_USING_ARCH_FUNC(func) using neon::func +#define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(neon/file) +#endif +``` + +**4. Dispatch Header Pattern (e.g., `simd_base.h:17-36`)** +```cpp +#include "simd_dispatch.h" +#include INCLUDE_ARCH_FILE(base.h) // Includes e.g., "neon/base.h" + +namespace sonic_json { +namespace internal { +SONIC_USING_ARCH_FUNC(TrailingZeroes); // Becomes: using neon::TrailingZeroes; +SONIC_USING_ARCH_FUNC(ClearLowestBit); +// ... more functions +} // namespace internal +} // namespace sonic_json +``` + +### SIMD Function Naming and Organization + +**Namespace Hierarchy:** +``` +sonic_json::internal:: +├── common:: # Scalar fallbacks and cross-arch utilities +├── arm_common:: # Shared ARM implementations +├── x86_common:: # Shared x86 implementations +├── neon:: # NEON-specific implementations +├── sve2_128:: # SVE2-128-specific implementations +├── avx2:: # AVX2-specific implementations +├── sse:: # SSE-specific implementations +└── (direct) # Dispatched functions via using declarations +``` + +**Key Function Patterns:** + +1. **SIMD Vector Types** (`simd.h`): + - `simd8` - 16-byte vector (16 elements of type T) + - `simd8x64` - 64-byte block (4 x simd8 chunks) + - Methods: `load()`, `store()`, `to_bitmask()`, `eq()`, `lteq()`, `reduce_or()`, operator overloads + +2. **String Block Pattern** (`unicode.h`): + ```cpp + struct StringBlock { + static StringBlock Find(const uint8_t* src); // Load and analyze 16 bytes + template bool HasQuoteFirst() const; + bool HasBackslash() const; + bool HasUnescaped() const; + int QuoteIndex() const; + int BsIndex() const; + uint64_t bs_bits, quote_bits, unescaped_bits; // NEON: bitmasks + // OR for SVE2: unsigned bs_index, quote_index, unescaped_index; + }; + ``` + +3. **String Parsing** (`quote.h`): + ```cpp + template + sonic_force_inline size_t parseStringInplace(uint8_t*& src, SonicError& err); + ``` + +4. **Skip Functions** (`skip.h`): + ```cpp + sonic_force_inline bool SkipContainer(const uint8_t* data, size_t& pos, + size_t len, uint8_t left, uint8_t right); + sonic_force_inline uint8_t skip_space(const uint8_t* data, size_t& pos, + size_t&, uint64_t&); + ``` + +### CPU Feature Detection + +**Compile-time detection only** (no runtime CPUID for ARM): +- `sonic_cpu_feature.h` uses compiler predefined macros +- ARM: `__ARM_NEON`, `__ARM_FEATURE_SVE2`, `__ARM_FEATURE_SVE_BITS` +- x86: `__SSE2__`, `__AVX2__`, etc. +- No runtime feature detection for ARM (static dispatch only) +- x86 supports dynamic dispatch via `x86_ifuncs/` using GNU ifunc attribute + +**Build system integration:** +- CMake: `-DENABLE_SVE2_128=ON` enables SVE2 compiler flags +- Bazel: `--//:sonic_arch={arm|sve2|haswell|westmere}` +- Bazel: `--//:sonic_dispatch={static|dynamic}` + +### Common Code Organization + +**Three levels of code sharing:** + +1. **`common/*.h` - Architecture-independent:** + - `quote_common.h`: Scalar fallback `parseStringInplace()` and `Quote()` for x86 dynamic dispatch + - `skip_common.h`: `EqBytes4()`, `SkipLiteral()`, `IsValidSeparator()` - pure scalar + - `unicode_common.h`: `handle_unicode_codepoint()`, `codepoint_to_utf8()`, `hex_to_u32_nocheck()`, `GetEscaped()` + - `quote_tables.h`: `kEscapedMap[256]`, `kQuoteTabLowerCase[256]`, `kQuoteTabUpperCase[256]`, `kNeedEscaped[256]` + +2. **`common/arm_common/*.h` - ARM-specific shared code:** + - `base.h`: `TrailingZeroes()` → `__builtin_ctzll()`, `ClearLowestBit()`, `LeadingZeroes()` → `__builtin_clzll()`, `CountOnes()` → `__builtin_popcountll()`, `PrefixXor()`, `Xmemcpy()`, `InlinedMemcmp()`, `InlinedMemcmpEq()` + - `simd.h`: `to_bitmask(uint8x16_t)` using NEON `vshrn_n_u16` + - `quote.h`: `Quote()` using NEON intrinsics, shared by both NEON and SVE2-128 + - `skip.inc.h`: Template implementations of `GetStringBits()`, `GetNextToken()`, `SkipString()`, `skip_container()` - included with `#include` (not `using`) + - `itoa.h`: `UtoaNeon()`, `Utoa_8()`, `Utoa_16()` using NEON intrinsics + - `str2int.h`: Scalar fallback `simd_str2int()` (used by NEON, overridden by SVE2-128) + +3. **Architecture-specific files (`neon/`, `sve2-128/`):** + - Thin wrapper headers that `using` declarations from `arm_common` + - Architecture-specific optimizations where needed: + - NEON: Full `simd8` and `simd8x64` template implementations + - SVE2-128: Minimal `simd.h` (reuses NEON simd types via `../neon/simd.h` in skip.h), optimized `simd_str2int()` using SVE2 `svdot`, predicate-based `StringBlock` using `svmatch`/`svbrkb`/`svcntp` + +### Key Architectural Patterns + +**1. Re-export Pattern (most files):** +```cpp +// neon/base.h +#include "../common/arm_common/base.h" +namespace neon { +using arm_common::TrailingZeroes; +using arm_common::ClearLowestBit; +// ... +} +``` + +**2. Template Include Pattern (`skip.inc.h`):** +```cpp +// neon/skip.h +#include "../common/arm_common/skip.inc.h" // Injects template functions directly + +sonic_force_inline bool SkipContainer(...) { + return skip_container>(data, pos, len, left, right); +} +``` + +**3. Cross-architecture Reuse (SVE2-128 using NEON):** +```cpp +// sve2-128/skip.h +#include "../neon/simd.h" // Reuse NEON simd types for skip_container + +sonic_force_inline bool SkipContainer(...) { + // Use NEON implementation since it's faster for comparisons + return skip_container>(data, pos, len, left, right); +} +``` + +**4. Override Pattern (SVE2-128 str2int):** +```cpp +// sve2-128/str2int.h - does NOT include arm_common/str2int.h +// Instead provides optimized SVE2 implementation +sonic_force_inline uint64_t simd_str2int(const char* c, int& man_nd) { + // Uses svld1, svmatch, svdot, etc. +} +``` + +### NEON vs SVE2-128 Implementation Differences + +| Aspect | NEON | SVE2-128 | +|--------|------|----------| +| Vector type | `uint8x16_t` | `svuint8x16_t` (fixed 128-bit SVE) | +| Load | `vld1q_u8(ptr)` | `svld1_u8(svptrue_b8(), ptr)` | +| Store | `vst1q_u8(ptr, v)` | `svst1_u8(svptrue_b8(), ptr, v)` | +| Compare | `vceqq_u8(v, dup)` → bitmask via `to_bitmask()` | `svmatch(ptrue, v, dup)` → predicate | +| Bitmask format | `uint64_t` (4 bits per lane) | `svbool_t` (predicate) | +| Find first | `TrailingZeroes(bitmask) >> 2` | `svcntp_b8(ptrue, svbrkb_z(ptrue, pmatch))` | +| StringBlock | Stores 3x `uint64_t` bitmasks | Stores 3x `unsigned` indices (0-15) | +| str2int | Scalar loop from arm_common | Optimized SVE2 `svdot` implementation | +| SkipContainer | Uses `neon::simd8x64` | Also uses `neon::simd8x64` (faster for comparisons) | + +### Build Configuration Matrix + +| Bazel Flag | CMake Equivalent | Effect | +|------------|------------------|--------| +| `--//:sonic_arch=arm` | (default on ARM) | Enables NEON | +| `--//:sonic_arch=sve2` | `-DENABLE_SVE2_128=ON` | Enables SVE2-128 (higher priority than NEON) | +| `--//:sonic_arch=haswell` | (default on x86) | Enables AVX2 | +| `--//:sonic_arch=westmere` | (manual) | Enables SSE4.2 | +| `--//:sonic_dispatch=static` | (default) | Static dispatch via macros | +| `--//:sonic_dispatch=dynamic` | (manual) | Dynamic dispatch via ifunc (x86 only) | + ## Coding Style - Follow root `.clang-format` (`BasedOnStyle: Google`). diff --git a/BUILD.bazel b/BUILD.bazel index cc8030f8..194243c2 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -10,6 +10,8 @@ sse_copts = ['-msse', '-msse2', '-msse4.1', '-msse4.2','-mpclmul'] arm_copts = ['-march=armv8-a'] apple_silicon_copts = [ '-mcpu=apple-m1' ] sve2_copts = ['-march=armv8-a+sve2', '-msve-vector-bits=128'] +riscv64_copts = ['-march=rv64gc'] +riscv64_zbb_copts = ['-march=rv64gc_zbb'] default_copts = ['-march=native'] benchmark_copts = ['-O3', '-DNDEBUG', '-std=c++17'] static_dispatch_copts = [] @@ -29,6 +31,7 @@ filesystem_linkopts = select({ string_flag( name = "sonic_arch", build_setting_default = "default", + values = ["default", "haswell", "westmere", "arm", "sve2", "riscv64", "riscv64_zbb"], ) string_flag( @@ -60,6 +63,16 @@ config_setting( flag_values = {":sonic_arch": "sve2"}, ) +config_setting( + name = "riscv64_build", + flag_values = {":sonic_arch": "riscv64"}, +) + +config_setting( + name = "riscv64_zbb_build", + flag_values = {":sonic_arch": "riscv64_zbb"}, +) + config_setting( name = "apple_silicon_build", values = {"cpu": "darwin_arm64"}, @@ -103,6 +116,8 @@ auto_arch_copts = select({ "sse_build": sse_copts, "avx2_build": avx2_copts, "apple_silicon_build": apple_silicon_copts, + "riscv64_build": riscv64_copts, + "riscv64_zbb_build": riscv64_zbb_copts, "//conditions:default": default_copts, }) + common_copts diff --git a/CMakeLists.txt b/CMakeLists.txt index 020b9d2b..69bd55c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,7 @@ option(BUILD_UNITTEST "Build unittest." ON) option(BUILD_FUZZ "Build fuzz." OFF) option(BUILD_BENCH "Build benchmark." OFF) option(ENABLE_SVE2_128 "Build for Arm SVE2 with 128 bit vector size" OFF) +option(ENABLE_RISCV_ZBB "Enable RISC-V Zbb extension for bit manipulation optimizations" OFF) set(CMAKE_CXX_EXTENSIONS OFF) if(BUILD_UNITTEST) diff --git a/cmake/set_arch_flags.cmake b/cmake/set_arch_flags.cmake index 43948401..8c679989 100644 --- a/cmake/set_arch_flags.cmake +++ b/cmake/set_arch_flags.cmake @@ -12,6 +12,12 @@ function(set_arch_flags target arch) target_compile_options(${target} PRIVATE -march=armv8-a) endif() endif() + elseif(arch MATCHES "riscv64|riscv") + if(ENABLE_RISCV_ZBB) + target_compile_options(${target} PRIVATE -march=rv64gc_zbb) + else() + target_compile_options(${target} PRIVATE -march=rv64gc) + endif() else() message(FATAL_ERROR "Unsupported architecture: ${arch}") endif() diff --git a/include/sonic/allocator.h b/include/sonic/allocator.h index 6885502d..3858f20a 100644 --- a/include/sonic/allocator.h +++ b/include/sonic/allocator.h @@ -93,8 +93,9 @@ class SpinLock { __builtin_ia32_pause(); #elif defined(__aarch64__) || defined(_M_ARM64) asm volatile("yield"); +#elif defined(__riscv) + __asm__ volatile("pause"); #endif - #endif } } diff --git a/include/sonic/internal/arch/riscv/base.h b/include/sonic/internal/arch/riscv/base.h new file mode 100644 index 00000000..f7d37692 --- /dev/null +++ b/include/sonic/internal/arch/riscv/base.h @@ -0,0 +1,78 @@ +// Copyright 2018-2019 The simdjson authors + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file may have been modified by ByteDance authors. All ByteDance +// Modifications are Copyright 2022 ByteDance Authors. + +#pragma once + +#include + +#include +#include + +namespace sonic_json { +namespace internal { +namespace riscv { + +// We sometimes call trailing_zero on inputs that are zero, +// but the algorithms do not end up using the returned value. +// Sadly, sanitizers are not smart enough to figure it out. + +sonic_force_inline __attribute__((no_sanitize("undefined"))) int TrailingZeroes( + uint64_t input_num) { + return __builtin_ctzll(input_num); +} + +/* result might be undefined when input_num is zero */ +sonic_force_inline uint64_t ClearLowestBit(uint64_t input_num) { + return input_num & (input_num - 1); +} + +/* result might be undefined when input_num is zero */ +sonic_force_inline int LeadingZeroes(uint64_t input_num) { + return __builtin_clzll(input_num); +} + +sonic_force_inline long long int CountOnes(uint64_t input_num) { + return __builtin_popcountll(input_num); +} + +sonic_force_inline uint64_t PrefixXor(uint64_t bitmask) { + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; +} + +template +sonic_force_inline void Xmemcpy(void* dst_, const void* src_, size_t chunks) { + std::memcpy(dst_, src_, chunks * ChunkSize); +} + +sonic_force_inline bool InlinedMemcmpEq(const void* _a, const void* _b, + size_t s) { + return std::memcmp(_a, _b, s) == 0; +} + +sonic_force_inline int InlinedMemcmp(const void* _l, const void* _r, size_t s) { + return std::memcmp(_l, _r, s); +} + +} // namespace riscv +} // namespace internal +} // namespace sonic_json diff --git a/include/sonic/internal/arch/riscv/itoa.h b/include/sonic/internal/arch/riscv/itoa.h new file mode 100644 index 00000000..d8036ad0 --- /dev/null +++ b/include/sonic/internal/arch/riscv/itoa.h @@ -0,0 +1,85 @@ +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include + +namespace sonic_json { +namespace internal { +namespace riscv { + +// Scalar implementation of Utoa_4_helper: convert num {abcd} to digits +static sonic_force_inline void Utoa_4_helper_scalar(uint16_t num, + uint8_t digits[4]) { + digits[0] = static_cast((num / 1000) % 10); + digits[1] = static_cast((num / 100) % 10); + digits[2] = static_cast((num / 10) % 10); + digits[3] = static_cast(num % 10); +} + +// Convert num's each digit to bytes. +// num's digits as abcdefgh (high bits is 0 if not enough) +// The converted digits are { a, b, c, d, e, f, g, h } +static sonic_force_inline void Utoa_Scalar(uint32_t num, uint8_t digits[8]) { + uint16_t hi = static_cast(num % 10000); // {efgh} + uint16_t lo = static_cast(num / 10000); // {abcd} + + uint8_t lo_digits[4]; + uint8_t hi_digits[4]; + Utoa_4_helper_scalar(lo, lo_digits); + Utoa_4_helper_scalar(hi, hi_digits); + + digits[0] = lo_digits[0]; + digits[1] = lo_digits[1]; + digits[2] = lo_digits[2]; + digits[3] = lo_digits[3]; + digits[4] = hi_digits[0]; + digits[5] = hi_digits[1]; + digits[6] = hi_digits[2]; + digits[7] = hi_digits[3]; +} + +static sonic_force_inline char *Utoa_8(uint32_t val, char *out) { + uint8_t digits[8]; + Utoa_Scalar(val, digits); + for (int i = 0; i < 8; i++) { + out[i] = static_cast('0' + digits[i]); + } + return out + 8; +} + +static sonic_force_inline char *Utoa_16(uint64_t val, char *out) { + uint8_t digits_hi[8]; + uint8_t digits_lo[8]; + Utoa_Scalar(static_cast(val / 100000000), digits_hi); + Utoa_Scalar(static_cast(val % 100000000), digits_lo); + + for (int i = 0; i < 8; i++) { + out[i] = static_cast('0' + digits_hi[i]); + } + for (int i = 0; i < 8; i++) { + out[i + 8] = static_cast('0' + digits_lo[i]); + } + return out + 16; +} + +} // namespace riscv +} // namespace internal +} // namespace sonic_json diff --git a/include/sonic/internal/arch/riscv/quote.h b/include/sonic/internal/arch/riscv/quote.h new file mode 100644 index 00000000..05d8f0c6 --- /dev/null +++ b/include/sonic/internal/arch/riscv/quote.h @@ -0,0 +1,250 @@ +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#define VEC_LEN 16 + +#include +#include + +#include +#include + +#include "../common/quote_common.h" +#include "../common/quote_tables.h" +#include "base.h" +#include "simd.h" +#include "unicode.h" + +#ifndef PAGE_SIZE +#define PAGE_SIZE 4096 +#endif + +#ifdef __GNUC__ +#if defined(__SANITIZE_THREAD__) || defined(__SANITIZE_ADDRESS__) || \ + defined(__SANITIZE_LEAK__) || defined(__SANITIZE_UNDEFINED__) +#ifndef SONIC_USE_SANITIZE +#define SONIC_USE_SANITIZE +#endif +#endif +#endif + +#if defined(__clang__) +#if defined(__has_feature) +#if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer) || \ + __has_feature(memory_sanitizer) || \ + __has_feature(undefined_behavior_sanitizer) || \ + __has_feature(leak_sanitizer) +#ifndef SONIC_USE_SANITIZE +#define SONIC_USE_SANITIZE +#endif +#endif +#endif +#endif + +#define MOVE_N_CHARS(src, N) \ + { \ + (src) += (N); \ + nb -= (N); \ + dst += (N); \ + } + +namespace sonic_json { +namespace internal { +namespace riscv { + +using sonic_json::internal::common::handle_unicode_codepoint; + +template +static sonic_force_inline uint64_t CopyAndGetEscapMask128(const char* src, + char* dst) { + simd8 v = + simd8::load(reinterpret_cast(src)); + v.store(reinterpret_cast(dst)); + + simd8 bs = simd8::splat('\\'); + simd8 quote = simd8::splat('"'); + simd8 ctrl = simd8::splat('\x20'); + + simd8 m1 = (v == bs); + simd8 m2 = (v == quote); + simd8 m3 = (v < ctrl); + + simd8 mask = m1 | m2; + mask = mask | m3; + if constexpr (EscapeEmoji) { + simd8 emoji = simd8::splat(0xF0); + simd8 m_emoji = (v >= emoji); + mask = mask | m_emoji; + } + + return mask.to_bitmask(); +} + +template +sonic_static_inline char* Quote(const char* src, size_t nb, char* dst) { + constexpr bool EscapeEmoji = + serializeFlags & SerializeFlags::kSerializeEscapeEmoji; + *dst++ = '"'; + sonic_assert(nb < (1ULL << 32)); + uint64_t mm; + int cn; + + /* VEC_LEN byte loop */ + while (nb >= VEC_LEN) { + /* check for matches */ + if ((mm = CopyAndGetEscapMask128(src, dst)) != 0) { + cn = TrailingZeroes(mm) >> 2; + MOVE_N_CHARS(src, cn); + DoEscape(src, dst, nb); + } else { + /* move to next block */ + MOVE_N_CHARS(src, VEC_LEN); + } + } + + if (nb > 0) { + char tmp_src[64]; + const char* src_r; +#ifdef SONIC_USE_SANITIZE + if (0) { +#else + /* This code would cause address sanitizer report heap-buffer-overflow. */ + if (((size_t)(src) & (PAGE_SIZE - 1)) <= (PAGE_SIZE - 64)) { + src_r = src; +#endif + } else { + std::memcpy(tmp_src, src, nb); + src_r = tmp_src; + } + while (nb > 0) { + mm = CopyAndGetEscapMask128(src_r, dst) & + (0xFFFFFFFFFFFFFFFF >> ((VEC_LEN - nb) << 2)); + if (mm) { + cn = TrailingZeroes(mm) >> 2; + MOVE_N_CHARS(src_r, cn); + DoEscape(src_r, dst, nb); + } else { + dst += nb; + nb = 0; + } + } + } + + *dst++ = '"'; + return dst; +} + +template +sonic_force_inline size_t parseStringInplace(uint8_t*& src, SonicError& err) { +#define SONIC_REPEAT8(v) \ + { v v v v v v v v } + constexpr bool kAllowUnescapedControlChars = + (parseFlags & ParseFlags::kParseAllowUnescapedControlChars) != 0; + uint8_t* dst = src; + uint8_t* sdst = src; + while (1) { + find: + auto block = StringBlock::Find(src); + if (block.HasQuoteFirst()) { + int idx = block.QuoteIndex(); + src += idx; + *src++ = '\0'; + return src - sdst - 1; + } + if constexpr (!kAllowUnescapedControlChars) { + if (block.HasUnescaped()) { + err = kParseErrorUnEscaped; + return 0; + } + } + if (!block.HasBackslash()) { + src += VEC_LEN; + goto find; + } + + /* find out where the backspace is */ + auto bs_dist = block.BsIndex(); + src += bs_dist; + dst = src; + cont: + uint8_t escape_char = src[1]; + if (sonic_unlikely(escape_char == 'u')) { + if (!handle_unicode_codepoint(const_cast(&src), &dst)) { + err = kParseErrorEscapedUnicode; + return 0; + } + } else { + *dst = kEscapedMap[escape_char]; + if (sonic_unlikely(*dst == 0u)) { + err = kParseErrorEscapedFormat; + return 0; + } + src += 2; + dst += 1; + } + // fast path for continuous escaped chars + if (*src == '\\') { + bs_dist = 0; + goto cont; + } + + find_and_move: + // Copy the next n bytes, and find the backslash and quote in them. + simd8 v = simd8::load(src); + block = StringBlock::Find(v); + // If the next thing is the end quote, copy and return + if (block.HasQuoteFirst()) { + // we encountered quotes first. Move dst to point to quotes and exit + while (1) { + SONIC_REPEAT8(if (sonic_unlikely(*src == '"')) break; + else { *dst++ = *src++; }); + } + *dst = '\0'; + src++; + return dst - sdst; + } + if constexpr (!kAllowUnescapedControlChars) { + if (block.HasUnescaped()) { + err = kParseErrorUnEscaped; + return 0; + } + } + if (!block.HasBackslash()) { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + v.store(dst); + src += VEC_LEN; + dst += VEC_LEN; + goto find_and_move; + } + while (1) { + SONIC_REPEAT8(if (sonic_unlikely(*src == '\\')) break; + else { *dst++ = *src++; }); + } + goto cont; + } + sonic_assert(false); +#undef SONIC_REPEAT8 +} + +} // namespace riscv +} // namespace internal +} // namespace sonic_json + +#undef VEC_LEN +#undef MOVE_N_CHARS diff --git a/include/sonic/internal/arch/riscv/simd.h b/include/sonic/internal/arch/riscv/simd.h new file mode 100644 index 00000000..c7673131 --- /dev/null +++ b/include/sonic/internal/arch/riscv/simd.h @@ -0,0 +1,760 @@ +// Copyright 2018-2019 The simdjson authors + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file may have been modified by ByteDance authors. All ByteDance +// Modifications are Copyright 2022 ByteDance Authors. + +// RISC-V scalar simd facade with optional Zbb byte-search helpers. + +#pragma once + +#include + +#include +#include + +namespace sonic_json { +namespace internal { +namespace riscv { + +#if defined(__riscv) && defined(__riscv_xlen) && (__riscv_xlen == 64) && \ + defined(__riscv_zbb) +#define SONIC_RISCV_SIMD_HAS_ZBB 1 + +namespace riscv_zbb_detail { + +static constexpr uint64_t kByteMsb = 0x8080808080808080ULL; +static constexpr uint64_t kByteLsb = 0x0101010101010101ULL; +static constexpr uint64_t kMovemaskMultiplier = 0x0002040810204081ULL; + +sonic_force_inline uint64_t load_u64(const uint8_t* ptr) { + uint64_t value; + std::memcpy(&value, ptr, sizeof(value)); + return value; +} + +sonic_force_inline uint64_t orc_b(uint64_t value) { + uint64_t result; + __asm__ volatile("orc.b %0, %1" : "=r"(result) : "r"(value)); + return result; +} + +sonic_force_inline uint32_t byte_msb_to_mask(uint64_t value) { + return static_cast(((value & kByteMsb) * kMovemaskMultiplier) >> + 56); +} + +sonic_force_inline uint32_t nonzero_byte_mask(uint64_t value) { + return byte_msb_to_mask(orc_b(value)); +} + +sonic_force_inline uint32_t zero_byte_mask(uint64_t value) { + return byte_msb_to_mask(~orc_b(value)); +} + +sonic_force_inline uint32_t bool_mask16(const uint8_t* ptr) { + uint32_t low = nonzero_byte_mask(load_u64(ptr)); + uint32_t high = nonzero_byte_mask(load_u64(ptr + 8)); + return low | (high << 8); +} + +sonic_force_inline uint32_t eq_mask16(const uint8_t* ptr, uint8_t value) { + uint64_t repeated = kByteLsb * value; + uint32_t low = zero_byte_mask(load_u64(ptr) ^ repeated); + uint32_t high = zero_byte_mask(load_u64(ptr + 8) ^ repeated); + return low | (high << 8); +} + +sonic_force_inline uint32_t lteq_mask16(const uint8_t* ptr, uint8_t value) { + if (value == 0xFFu) { + return 0xFFFFu; + } + if (value == 0x1Fu) { + constexpr uint64_t high_three_bits = 0xE0E0E0E0E0E0E0E0ULL; + uint32_t low = zero_byte_mask(load_u64(ptr) & high_three_bits); + uint32_t high = zero_byte_mask(load_u64(ptr + 8) & high_three_bits); + return low | (high << 8); + } + if (value == 0x7Fu) { + uint32_t low = byte_msb_to_mask(~load_u64(ptr)); + uint32_t high = byte_msb_to_mask(~load_u64(ptr + 8)); + return low | (high << 8); + } + + uint32_t mask = 0; + for (int i = 0; i < 16; i++) { + if (ptr[i] <= value) mask |= (1u << i); + } + return mask; +} + +} // namespace riscv_zbb_detail + +#else +#define SONIC_RISCV_SIMD_HAS_ZBB 0 +#endif + +template +struct simd8; + +// SIMD byte mask type (returned by things like eq and gt) +template <> +struct simd8 { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + uint8_t data[16]; + + static sonic_force_inline simd8 splat(bool _value) { + simd8 r; + std::memset(r.data, _value ? 0xFF : 0x00, 16); + return r; + } + + sonic_force_inline simd8() { std::memset(data, 0, 16); } + sonic_force_inline simd8(bool _value) { + std::memset(data, _value ? 0xFF : 0x00, 16); + } + + // Construct from raw array + sonic_force_inline explicit simd8(const uint8_t d[16]) { + std::memcpy(data, d, 16); + } + + // We return uint64_t with 4 bits per byte (same as NEON movemask format) + // so that TrailingZeroes(x) >> 2 gives the correct byte index. + sonic_force_inline uint64_t to_bitmask() const { +#if SONIC_RISCV_SIMD_HAS_ZBB + uint32_t bits = riscv_zbb_detail::bool_mask16(data); + uint64_t mask = 0; + for (int i = 0; i < 16; i++) { + if (bits & (1u << i)) { + mask |= (0xFULL << (i * 4)); + } + } + return mask; +#else + uint64_t mask = 0; + for (int i = 0; i < 16; i++) { + if (data[i]) { + mask |= (0xFULL << (i * 4)); + } + } + return mask; +#endif + } + + sonic_force_inline bool any() const { +#if SONIC_RISCV_SIMD_HAS_ZBB + return (riscv_zbb_detail::load_u64(data) | + riscv_zbb_detail::load_u64(data + 8)) != 0; +#else + for (int i = 0; i < 16; i++) { + if (data[i]) return true; + } + return false; +#endif + } + + sonic_force_inline simd8 operator~() const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = ~data[i]; + return r; + } + + sonic_force_inline simd8 operator|(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] | other.data[i]; + return r; + } + sonic_force_inline simd8 operator&(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] & other.data[i]; + return r; + } + sonic_force_inline simd8 operator^(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] ^ other.data[i]; + return r; + } + sonic_force_inline simd8 bit_andnot(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] & ~other.data[i]; + return r; + } + sonic_force_inline simd8& operator|=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] |= other.data[i]; + return *this; + } + sonic_force_inline simd8& operator&=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] &= other.data[i]; + return *this; + } + sonic_force_inline simd8& operator^=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] ^= other.data[i]; + return *this; + } +}; + +// Free function for skip.inc.h compatibility (same as NEON's to_bitmask) +sonic_force_inline uint64_t to_bitmask(const simd8& v) { + return v.to_bitmask(); +} + +// Unsigned bytes +template <> +struct simd8 { + static const int SIZE = 16; + uint8_t data[16]; + + static sonic_force_inline simd8 splat(uint8_t _value) { + simd8 r; + std::memset(r.data, _value, 16); + return r; + } + static sonic_force_inline simd8 zero() { + simd8 r; + std::memset(r.data, 0, 16); + return r; + } + static sonic_force_inline simd8 load(const uint8_t* values) { + simd8 r; + std::memcpy(r.data, values, 16); + return r; + } + + sonic_force_inline simd8() { std::memset(data, 0, 16); } + sonic_force_inline simd8(const uint8_t values[16]) { + std::memcpy(data, values, 16); + } + sonic_force_inline simd8(uint8_t _value) { std::memset(data, _value, 16); } + // Conversion from bool mask + sonic_force_inline simd8(const simd8& other) { + std::memcpy(data, other.data, 16); + } + sonic_force_inline simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, + uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, + uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) { + data[0] = v0; + data[1] = v1; + data[2] = v2; + data[3] = v3; + data[4] = v4; + data[5] = v5; + data[6] = v6; + data[7] = v7; + data[8] = v8; + data[9] = v9; + data[10] = v10; + data[11] = v11; + data[12] = v12; + data[13] = v13; + data[14] = v14; + data[15] = v15; + } + + // Repeat 16 values as many times as necessary (usually for lookup tables) + static sonic_force_inline simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, + uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, + uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); + } + + sonic_force_inline void store(uint8_t dst[16]) const { + std::memcpy(dst, data, 16); + } + + // Saturated math + sonic_force_inline simd8 saturating_add( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) { + uint16_t sum = (uint16_t)data[i] + other.data[i]; + r.data[i] = sum > 255 ? 255 : (uint8_t)sum; + } + return r; + } + sonic_force_inline simd8 saturating_sub( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) { + r.data[i] = data[i] > other.data[i] ? data[i] - other.data[i] : 0; + } + return r; + } + + // Arithmetic + sonic_force_inline simd8 operator+( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] + other.data[i]; + return r; + } + sonic_force_inline simd8 operator-( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] - other.data[i]; + return r; + } + sonic_force_inline simd8& operator+=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] += other.data[i]; + return *this; + } + sonic_force_inline simd8& operator-=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] -= other.data[i]; + return *this; + } + + // Bitwise + sonic_force_inline simd8 operator|( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] | other.data[i]; + return r; + } + sonic_force_inline simd8 operator&( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] & other.data[i]; + return r; + } + sonic_force_inline simd8 operator^( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] ^ other.data[i]; + return r; + } + sonic_force_inline simd8 bit_andnot( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] & ~other.data[i]; + return r; + } + sonic_force_inline simd8 operator~() const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = ~data[i]; + return r; + } + sonic_force_inline simd8& operator|=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] |= other.data[i]; + return *this; + } + sonic_force_inline simd8& operator&=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] &= other.data[i]; + return *this; + } + sonic_force_inline simd8& operator^=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] ^= other.data[i]; + return *this; + } + + // Comparison + friend sonic_force_inline simd8 operator==(const simd8 lhs, + const simd8 rhs) { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (lhs.data[i] == rhs.data[i]) ? 0xFF : 0x00; + return r; + } + sonic_force_inline simd8 operator<=(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] <= other.data[i]) ? 0xFF : 0x00; + return r; + } + sonic_force_inline simd8 operator>=(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] >= other.data[i]) ? 0xFF : 0x00; + return r; + } + sonic_force_inline simd8 operator>(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] > other.data[i]) ? 0xFF : 0x00; + return r; + } + sonic_force_inline simd8 operator<(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] < other.data[i]) ? 0xFF : 0x00; + return r; + } + + // gt_bits / lt_bits (non-zero instead of all-1s) + sonic_force_inline simd8 gt_bits(const simd8 other) const { + return this->saturating_sub(other); + } + sonic_force_inline simd8 lt_bits(const simd8 other) const { + return other.saturating_sub(*this); + } + + // Max / min + sonic_force_inline uint8_t max_val() const { + uint8_t m = 0; + for (int i = 0; i < 16; i++) + if (data[i] > m) m = data[i]; + return m; + } + sonic_force_inline uint8_t min_val() const { + uint8_t m = 255; + for (int i = 0; i < 16; i++) + if (data[i] < m) m = data[i]; + return m; + } + sonic_force_inline simd8 max_val(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = data[i] > other.data[i] ? data[i] : other.data[i]; + return r; + } + sonic_force_inline simd8 min_val(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = data[i] < other.data[i] ? data[i] : other.data[i]; + return r; + } + + // Bit-specific operations + sonic_force_inline simd8 any_bits_set(simd8 bits) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] & bits.data[i]) ? 0xFF : 0x00; + return r; + } + sonic_force_inline bool any_bits_set_anywhere() const { + return max_val() != 0; + } + sonic_force_inline bool any_bits_set_anywhere(simd8 bits) const { + return (*this & bits).any_bits_set_anywhere(); + } + + template + sonic_force_inline simd8 shr() const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] >> N; + return r; + } + template + sonic_force_inline simd8 shl() const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] << N; + return r; + } + + // prev: shift in bytes from prev_chunk + template + sonic_force_inline simd8 prev( + const simd8 prev_chunk) const { + simd8 r; + for (int i = 0; i < N; i++) r.data[i] = prev_chunk.data[16 - N + i]; + for (int i = N; i < 16; i++) r.data[i] = data[i - N]; + return r; + } + + // lookup_16: table lookup, byte by byte + template + sonic_force_inline simd8 lookup_16(simd8 lookup_table) const { + simd8 r; + for (int i = 0; i < 16; i++) { + uint8_t idx = data[i] & 0x0F; + r.data[i] = lookup_table.data[idx]; + } + return r; + } + + template + sonic_force_inline simd8 lookup_16(L r0, L r1, L r2, L r3, L r4, L r5, + L r6, L r7, L r8, L r9, L r10, L r11, + L r12, L r13, L r14, L r15) const { + return lookup_16(simd8::repeat_16(r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, + r10, r11, r12, r13, r14, r15)); + } + + template + sonic_force_inline simd8 apply_lookup_16_to( + const simd8 original) { + simd8 r; + for (int i = 0; i < 16; i++) { + uint8_t idx = (uint8_t)original.data[i]; + r.data[i] = idx < 16 ? data[idx] : 0; + } + return r; + } +}; + +// Signed bytes +template <> +struct simd8 { + int8_t data[16]; + + static sonic_force_inline simd8 splat(int8_t _value) { + simd8 r; + std::memset(r.data, (uint8_t)_value, 16); + return r; + } + static sonic_force_inline simd8 zero() { + simd8 r; + std::memset(r.data, 0, 16); + return r; + } + static sonic_force_inline simd8 load(const int8_t values[16]) { + simd8 r; + std::memcpy(r.data, values, 16); + return r; + } + + sonic_force_inline simd8() { std::memset(data, 0, 16); } + sonic_force_inline simd8(int8_t _value) { + std::memset(data, (uint8_t)_value, 16); + } + sonic_force_inline simd8(const int8_t* values) { + std::memcpy(data, values, 16); + } + sonic_force_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, + int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, + int8_t v12, int8_t v13, int8_t v14, int8_t v15) { + data[0] = v0; + data[1] = v1; + data[2] = v2; + data[3] = v3; + data[4] = v4; + data[5] = v5; + data[6] = v6; + data[7] = v7; + data[8] = v8; + data[9] = v9; + data[10] = v10; + data[11] = v11; + data[12] = v12; + data[13] = v13; + data[14] = v14; + data[15] = v15; + } + + static sonic_force_inline simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, + int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, + int8_t v12, int8_t v13, int8_t v14, int8_t v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); + } + + sonic_force_inline void store(int8_t dst[16]) const { + std::memcpy(dst, data, 16); + } + + // Explicit conversion to uint8_t + sonic_force_inline explicit simd8(const simd8& other) { + std::memcpy(data, other.data, 16); + } + sonic_force_inline explicit operator simd8() const { + simd8 r; + std::memcpy(r.data, data, 16); + return r; + } + + // Bitwise + sonic_force_inline simd8 operator|(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] | other.data[i]; + return r; + } + + // Arithmetic + sonic_force_inline simd8 operator+(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] + other.data[i]; + return r; + } + sonic_force_inline simd8 operator-(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] - other.data[i]; + return r; + } + sonic_force_inline simd8& operator+=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] += other.data[i]; + return *this; + } + sonic_force_inline simd8& operator-=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] -= other.data[i]; + return *this; + } + + // Comparison + sonic_force_inline simd8 max_val(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = data[i] > other.data[i] ? data[i] : other.data[i]; + return r; + } + sonic_force_inline simd8 min_val(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = data[i] < other.data[i] ? data[i] : other.data[i]; + return r; + } + sonic_force_inline simd8 operator>(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] > other.data[i]) ? 0xFF : 0x00; + return r; + } + sonic_force_inline simd8 operator<(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] < other.data[i]) ? 0xFF : 0x00; + return r; + } + friend sonic_force_inline simd8 operator==(const simd8 lhs, + const simd8 rhs) { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (lhs.data[i] == rhs.data[i]) ? 0xFF : 0x00; + return r; + } + + template + sonic_force_inline simd8 prev(const simd8 prev_chunk) const { + simd8 r; + for (int i = 0; i < N; i++) r.data[i] = prev_chunk.data[16 - N + i]; + for (int i = N; i < 16; i++) r.data[i] = data[i - N]; + return r; + } + + template + sonic_force_inline simd8 lookup_16(simd8 lookup_table) const { + return lookup_table.apply_lookup_16_to(*this); + } + template + sonic_force_inline simd8 lookup_16(L r0, L r1, L r2, L r3, L r4, L r5, + L r6, L r7, L r8, L r9, L r10, L r11, + L r12, L r13, L r14, L r15) const { + return lookup_16(simd8::repeat_16(r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, + r10, r11, r12, r13, r14, r15)); + } + + template + sonic_force_inline simd8 apply_lookup_16_to(const simd8 original) { + simd8 r; + for (int i = 0; i < 16; i++) { + uint8_t idx = (uint8_t)original.data[i]; + r.data[i] = idx < 16 ? data[idx] : 0; + } + return r; + } +}; + +template +struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, + "RISC-V kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = + delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + sonic_force_inline simd8x64(const simd8 chunk0, const simd8 chunk1, + const simd8 chunk2, const simd8 chunk3) + : chunks{chunk0, chunk1, chunk2, chunk3} {} + sonic_force_inline simd8x64(const T ptr[64]) + : chunks{simd8::load((const uint8_t*)ptr), + simd8::load((const uint8_t*)(ptr + 16)), + simd8::load((const uint8_t*)(ptr + 32)), + simd8::load((const uint8_t*)(ptr + 48))} {} + + sonic_force_inline void store(T ptr[64]) const { + this->chunks[0].store((uint8_t*)(ptr)); + this->chunks[1].store((uint8_t*)(ptr + 16)); + this->chunks[2].store((uint8_t*)(ptr + 32)); + this->chunks[3].store((uint8_t*)(ptr + 48)); + } + + sonic_force_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | + (this->chunks[2] | this->chunks[3]); + } + + sonic_force_inline uint64_t to_bitmask() const { + // Each chunk's to_bitmask() returns 64 bits with 4 bits per byte. + // Compress to 1 bit per byte (16 bits per chunk) before combining. + auto compress = [](uint64_t mask) -> uint64_t { + uint64_t result = 0; + for (int i = 0; i < 16; i++) { + if (mask & (0xFULL << (i * 4))) { + result |= (1ULL << i); + } + } + return result; + }; + uint64_t r0 = compress(this->chunks[0].to_bitmask()); + uint64_t r1 = compress(this->chunks[1].to_bitmask()); + uint64_t r2 = compress(this->chunks[2].to_bitmask()); + uint64_t r3 = compress(this->chunks[3].to_bitmask()); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + + sonic_force_inline uint64_t eq(const T m) const { +#if SONIC_RISCV_SIMD_HAS_ZBB + const uint8_t byte = static_cast(m); + uint64_t r0 = riscv_zbb_detail::eq_mask16( + reinterpret_cast(this->chunks[0].data), byte); + uint64_t r1 = riscv_zbb_detail::eq_mask16( + reinterpret_cast(this->chunks[1].data), byte); + uint64_t r2 = riscv_zbb_detail::eq_mask16( + reinterpret_cast(this->chunks[2].data), byte); + uint64_t r3 = riscv_zbb_detail::eq_mask16( + reinterpret_cast(this->chunks[3].data), byte); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); +#else + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, + this->chunks[2] == mask, this->chunks[3] == mask) + .to_bitmask(); +#endif + } + + sonic_force_inline uint64_t lteq(const T m) const { +#if SONIC_RISCV_SIMD_HAS_ZBB + const uint8_t byte = static_cast(m); + uint64_t r0 = riscv_zbb_detail::lteq_mask16( + reinterpret_cast(this->chunks[0].data), byte); + uint64_t r1 = riscv_zbb_detail::lteq_mask16( + reinterpret_cast(this->chunks[1].data), byte); + uint64_t r2 = riscv_zbb_detail::lteq_mask16( + reinterpret_cast(this->chunks[2].data), byte); + uint64_t r3 = riscv_zbb_detail::lteq_mask16( + reinterpret_cast(this->chunks[3].data), byte); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); +#else + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] <= mask, this->chunks[1] <= mask, + this->chunks[2] <= mask, this->chunks[3] <= mask) + .to_bitmask(); +#endif + } +}; // struct simd8x64 + +} // namespace riscv +} // namespace internal +} // namespace sonic_json + +#undef SONIC_RISCV_SIMD_HAS_ZBB diff --git a/include/sonic/internal/arch/riscv/skip.h b/include/sonic/internal/arch/riscv/skip.h new file mode 100644 index 00000000..26fd2de3 --- /dev/null +++ b/include/sonic/internal/arch/riscv/skip.h @@ -0,0 +1,65 @@ +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#define VEC_LEN 16 + +#include +#include + +#include "base.h" +#include "simd.h" + +namespace sonic_json { +namespace internal { +namespace riscv { + +using sonic_json::internal::common::EqBytes4; +using sonic_json::internal::common::SkipLiteral; + +#include "skip.inc.h" + +sonic_force_inline bool SkipContainer(const uint8_t *data, size_t &pos, + size_t len, uint8_t left, uint8_t right) { + return skip_container>(data, pos, len, left, right); +} + +// TODO: optimize by removing bound checking. +sonic_force_inline uint8_t skip_space(const uint8_t *data, size_t &pos, + size_t &, uint64_t &) { + // fast path for single space + if (!IsSpace(data[pos++])) return data[pos - 1]; + if (!IsSpace(data[pos++])) return data[pos - 1]; + + // current pos is out of block + while (1) { + uint64_t nonspace = GetNonSpaceBits(data + pos); + if (nonspace) { + pos += TrailingZeroes(nonspace) >> 2; + return data[pos++]; + } else { + pos += 16; + } + } + sonic_assert(false && "!should not happen"); +} + +} // namespace riscv +} // namespace internal +} // namespace sonic_json + +#undef VEC_LEN diff --git a/include/sonic/internal/arch/riscv/skip.inc.h b/include/sonic/internal/arch/riscv/skip.inc.h new file mode 100644 index 00000000..4d3a4113 --- /dev/null +++ b/include/sonic/internal/arch/riscv/skip.inc.h @@ -0,0 +1,168 @@ +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VEC_LEN +#error "Define vector length firstly!" +#endif + +template +sonic_force_inline uint64_t GetStringBits(const uint8_t *data, + uint64_t &prev_instring, + uint64_t &prev_escaped) { + const T v(data); + uint64_t escaped = 0; + uint64_t bs_bits = v.eq('\\'); + if (bs_bits) { + escaped = common::GetEscaped<64>(prev_escaped, bs_bits); + } else { + escaped = prev_escaped; + prev_escaped = 0; + } + uint64_t quote_bits = v.eq('"') & ~escaped; + uint64_t in_string = PrefixXor(quote_bits) ^ prev_instring; + prev_instring = uint64_t(static_cast(in_string) >> 63); + return in_string; +} + +// GetNextToken find the next characters in tokens and update the position to +// it. +template +sonic_force_inline uint8_t GetNextToken(const uint8_t *data, size_t &pos, + size_t len, const char (&tokens)[N]) { + while (pos + VEC_LEN <= len) { + simd8 v = simd8::load(data + pos); + simd8 vor = simd8::splat(false); + for (size_t i = 0; i < N - 1; i++) { + simd8 vtmp = (v == simd8::splat((uint8_t)(tokens[i]))); + vor = vor | vtmp; + } + + uint64_t next = vor.to_bitmask(); + if (next) { + pos += (TrailingZeroes(next) >> 2); + return data[pos]; + } + pos += VEC_LEN; + } + while (pos < len) { + for (size_t i = 0; i < N - 1; i++) { + if (data[pos] == tokens[i]) { + return tokens[i]; + } + } + pos++; + } + return '\0'; +} + +// pos is the after the ending quote +sonic_force_inline int SkipString(const uint8_t *data, size_t &pos, + size_t len) { + const static int kEscaped = 2; + const static int kNormal = 1; + const static int kUnclosed = 0; + uint64_t quote_bits = 0; + uint64_t bs_bits = 0; + int ret = kNormal; + while (pos + VEC_LEN <= len) { + simd8 v = simd8::load(data + pos); + simd8 bs_cmp = (v == simd8::splat('\\')); + simd8 quote_cmp = (v == simd8::splat('"')); + bs_bits = bs_cmp.to_bitmask(); + quote_bits = quote_cmp.to_bitmask(); + if (((bs_bits - 1) & quote_bits) != 0) { + pos += (TrailingZeroes(quote_bits) >> 2) + 1; + return ret; + } + if (bs_bits) { + ret = kEscaped; + pos += ((TrailingZeroes(bs_bits) >> 2) + 2); + while (pos < len) { + if (data[pos] == '\\') { + pos += 2; + } else { + break; + } + } + } else { + pos += VEC_LEN; + } + } + while (pos < len) { + if (data[pos] == '\\') { + if (pos + 1 >= len) { + return kUnclosed; + } + ret = kEscaped; + pos += 2; + continue; + } + if (data[pos++] == '"') { + return ret; + } + }; + return kUnclosed; +} + +// return true if container is closed. +template +sonic_force_inline bool skip_container(const uint8_t *data, size_t &pos, + size_t len, uint8_t left, + uint8_t right) { + uint64_t prev_instring = 0, prev_escaped = 0, instring; + int rbrace_num = 0, lbrace_num = 0, last_lbrace_num; + const uint8_t *p; + while (pos + 64 <= len) { + p = data + pos; +#define SKIP_LOOP() \ + { \ + instring = GetStringBits(p, prev_instring, prev_escaped); \ + T v(p); \ + last_lbrace_num = lbrace_num; \ + uint64_t rbrace = v.eq(right) & ~instring; \ + uint64_t lbrace = v.eq(left) & ~instring; \ + /* traverse each '}' */ \ + while (rbrace > 0) { \ + rbrace_num++; \ + lbrace_num = last_lbrace_num + CountOnes((rbrace - 1) & lbrace); \ + bool is_closed = lbrace_num < rbrace_num; \ + if (is_closed) { \ + sonic_assert(rbrace_num == lbrace_num + 1); \ + pos += TrailingZeroes(rbrace) + 1; \ + return true; \ + } \ + rbrace &= (rbrace - 1); \ + } \ + lbrace_num = last_lbrace_num + CountOnes(lbrace); \ + } + SKIP_LOOP(); + pos += 64; + } + uint8_t buf[64] = {0}; + std::memcpy(buf, data + pos, len - pos); + p = buf; + SKIP_LOOP(); +#undef SKIP_LOOP + return false; +} + +sonic_force_inline uint8_t skip_space_safe(const uint8_t *data, size_t &pos, + size_t len, size_t &, uint64_t &) { + while (pos < len && IsSpace(data[pos++])) + ; + // if not found, still return the space chars + return data[pos - 1]; +} diff --git a/include/sonic/internal/arch/riscv/str2int.h b/include/sonic/internal/arch/riscv/str2int.h new file mode 100644 index 00000000..c9ddeb5c --- /dev/null +++ b/include/sonic/internal/arch/riscv/str2int.h @@ -0,0 +1,29 @@ +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../common/arm_common/str2int.h" + +namespace sonic_json { +namespace internal { +namespace riscv { + +using sonic_json::internal::arm_common::simd_str2int; + +} // namespace riscv +} // namespace internal +} // namespace sonic_json diff --git a/include/sonic/internal/arch/riscv/unicode.h b/include/sonic/internal/arch/riscv/unicode.h new file mode 100644 index 00000000..b5c823ba --- /dev/null +++ b/include/sonic/internal/arch/riscv/unicode.h @@ -0,0 +1,122 @@ +// Copyright 2018-2019 The simdjson authors + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file may have been modified by ByteDance authors. All ByteDance +// Modifications are Copyright 2022 ByteDance Authors. + +#pragma once + +#include + +#include +#include + +#include "../common/unicode_common.h" +#include "base.h" +#include "simd.h" + +namespace sonic_json { +namespace internal { +namespace riscv { + +using sonic_json::internal::common::handle_unicode_codepoint; + +struct StringBlock { + public: + sonic_force_inline static StringBlock Find(const uint8_t *src); + sonic_force_inline static StringBlock Find(simd8 &v); + template + sonic_force_inline bool HasQuoteFirst() const { + constexpr bool kAllowUnescapedControlChars = + (parseFlags & ParseFlags::kParseAllowUnescapedControlChars) != 0; + if constexpr (kAllowUnescapedControlChars) { + return (((bs_bits - 1) & quote_bits) != 0); + } else { + return (((bs_bits - 1) & quote_bits) != 0) && (!HasUnescaped()); + } + } + sonic_force_inline bool HasBackslash() const { + return ((quote_bits - 1) & bs_bits) != 0; + } + sonic_force_inline bool HasUnescaped() const { + return ((quote_bits - 1) & unescaped_bits) != 0; + } + sonic_force_inline int QuoteIndex() const { + return TrailingZeroes(quote_bits) >> 2; + } + sonic_force_inline int BsIndex() const { + return TrailingZeroes(bs_bits) >> 2; + } + + uint64_t bs_bits; + uint64_t quote_bits; + uint64_t unescaped_bits; +}; + +sonic_force_inline StringBlock StringBlock::Find(const uint8_t *src) { + simd8 v = simd8::load(src); + return StringBlock::Find(v); +} + +sonic_force_inline StringBlock StringBlock::Find(simd8 &v) { + simd8 bs = simd8::splat('\\'); + simd8 quote = simd8::splat('"'); + simd8 ctrl = simd8::splat('\x1f'); + + simd8 bs_cmp = (v == bs); + simd8 quote_cmp = (v == quote); + simd8 ctrl_cmp = (v <= ctrl); + + return { + bs_cmp.to_bitmask(), + quote_cmp.to_bitmask(), + ctrl_cmp.to_bitmask(), + }; +} + +sonic_force_inline uint64_t GetNonSpaceBits(const uint8_t *data) { + simd8 v = simd8::load(data); + simd8 space = simd8::splat(' '); + simd8 tab = simd8::splat('\t'); + simd8 nl = simd8::splat('\n'); + simd8 cr = simd8::splat('\r'); + + simd8 m1 = (v == space); + simd8 m2 = (v == tab); + simd8 m3 = (v == nl); + simd8 m4 = (v == cr); + + simd8 m5 = m1 | m2; + simd8 m6 = m3 | m4; + simd8 m7 = m5 | m6; + simd8 m8 = ~m7; + + return m8.to_bitmask(); +} + +sonic_force_inline bool IsAscii(const simd8x64 &input) { + simd8 bits = input.reduce_or(); + return bits.max_val() < 0x80u; +} + +sonic_force_inline simd8 MustBe2_3Continuation( + const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2 >= uint8_t(0xe0u); + simd8 is_fourth_byte = prev3 >= uint8_t(0xf0u); + return is_third_byte ^ is_fourth_byte; +} + +} // namespace riscv +} // namespace internal +} // namespace sonic_json diff --git a/include/sonic/internal/arch/simd_dispatch.h b/include/sonic/internal/arch/simd_dispatch.h index 0c474ce0..a8ee07f7 100644 --- a/include/sonic/internal/arch/simd_dispatch.h +++ b/include/sonic/internal/arch/simd_dispatch.h @@ -39,6 +39,9 @@ #elif defined(SONIC_HAVE_NEON) #define SONIC_USING_ARCH_FUNC(func) using neon::func #define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(neon/file) +#elif defined(SONIC_HAVE_RISCV) +#define SONIC_USING_ARCH_FUNC(func) using riscv::func +#define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(riscv/file) #endif #elif defined(SONIC_DYNAMIC_DISPATCH) @@ -50,6 +53,9 @@ #elif defined(SONIC_HAVE_NEON) #define SONIC_USING_ARCH_FUNC(func) using neon::func #define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(neon/file) +#elif defined(SONIC_HAVE_RISCV) +#define SONIC_USING_ARCH_FUNC(func) using riscv::func +#define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(riscv/file) #endif #endif diff --git a/include/sonic/internal/arch/sonic_cpu_feature.h b/include/sonic/internal/arch/sonic_cpu_feature.h index 21d2707f..17222fb9 100644 --- a/include/sonic/internal/arch/sonic_cpu_feature.h +++ b/include/sonic/internal/arch/sonic_cpu_feature.h @@ -37,11 +37,18 @@ #if defined(__AVX2__) #define SONIC_HAVE_AVX2 #endif -#else +#elif defined(__ARM_NEON) || defined(__ARM_NEON__) || \ + defined(__ARM_FEATURE_SVE2) #if defined(__ARM_NEON) || defined(__ARM_NEON__) #define SONIC_HAVE_NEON #endif #if defined(__ARM_FEATURE_SVE2) && (__ARM_FEATURE_SVE_BITS == 128) #define SONIC_HAVE_SVE2_128 #endif +#elif defined(__riscv) && defined(__riscv_xlen) && (__riscv_xlen == 64) +#define SONIC_IS_RISCV 1 +#if defined(__riscv_zbb) +#define SONIC_HAVE_RISCV_ZBB 1 +#endif +#define SONIC_HAVE_RISCV 1 #endif