Merge branch 'add_rvv_support' of https://github.com/anthony-zy/snappy into add_rvv_support
diff --git a/snappy-internal.h b/snappy-internal.h index 1c9d043..00b2db5 100644 --- a/snappy-internal.h +++ b/snappy-internal.h
@@ -49,10 +49,21 @@ #if SNAPPY_RVV_1 || SNAPPY_RVV_0_7 #define SNAPPY_HAVE_RVV 1 #include <riscv_vector.h> +#else +#define SNAPPY_HAVE_RVV 0 #endif +#ifdef SNAPPY_RVV_1 +#define VSETVL_E8M2 __riscv_vsetvl_e8m2 +#define VLE8_V_U8M2 __riscv_vle8_v_u8m2 +#define VSE8_V_U8M2 __riscv_vse8_v_u8m2 +#elif SNAPPY_RVV_0_7 +#define VSETVL_E8M2 vsetvl_e8m2 +#define VLE8_V_U8M2 vle8_v_u8m2 +#define VSE8_V_U8M2 vse8_v_u8m2 +#endif -#if SNAPPY_HAVE_SSSE3 || SNAPPY_HAVE_NEON || SNAPPY_HAVE_RVV +#if SNAPPY_HAVE_SSSE3 || SNAPPY_HAVE_NEON #define SNAPPY_HAVE_VECTOR_BYTE_SHUFFLE 1 #else #define SNAPPY_HAVE_VECTOR_BYTE_SHUFFLE 0 @@ -66,23 +77,8 @@ using V128 = __m128i; #elif SNAPPY_HAVE_NEON using V128 = uint8x16_t; -#elif SNAPPY_HAVE_RVV -using V128 = vuint8m1_t; #endif - -#ifdef SNAPPY_RVV_1 -#define VSETVL_E8M1 __riscv_vsetvl_e8m1 -#define VLE8_V_U8M1 __riscv_vle8_v_u8m1 -#define VSE8_V_U8M1 __riscv_vse8_v_u8m1 -#define VRGATHER_VV_U8M1 __riscv_vrgather_vv_u8m1 -#define VMV_V_X_U8M1 __riscv_vmv_v_x_u8m1 -#elif SNAPPY_RVV_0_7 -#define VSETVL_E8M1 vsetvl_e8m1 -#define VLE8_V_U8M1 vle8_v_u8m1 -#define VSE8_V_U8M1 vse8_v_u8m1 -#define VRGATHER_VV_U8M1 vrgather_vv_u8m1 -#define VMV_V_X_U8M1 vmv_v_x_u8m1 -#endif + // Load 128 bits of integer data. `src` must be 16-byte aligned. inline V128 V128_Load(const V128* src); @@ -132,31 +128,7 @@ inline V128 V128_DupChar(char c) { return vdupq_n_u8(c); } -#elif SNAPPY_HAVE_RVV -inline V128 V128_Load(const V128* src) { - size_t vl = VSETVL_E8M1(16); - return VLE8_V_U8M1(reinterpret_cast<const uint8_t*>(src), vl); -} -inline V128 V128_LoadU(const V128* src) { - size_t vl = VSETVL_E8M1(16); - return VLE8_V_U8M1(reinterpret_cast<const uint8_t*>(src), vl); -} - -inline void V128_StoreU(V128* dst, V128 val) { - size_t vl = VSETVL_E8M1(16); - VSE8_V_U8M1(reinterpret_cast<uint8_t*>(dst), val, vl); -} - -inline V128 V128_Shuffle(V128 input, V128 shuffle_mask) { - size_t vl = VSETVL_E8M1(16); - return VRGATHER_VV_U8M1(input, shuffle_mask, vl); -} - -inline V128 V128_DupChar(char c) { - size_t vl = VSETVL_E8M1(16); - return VMV_V_X_U8M1(static_cast<uint8_t>(c), vl); -} #endif #endif // SNAPPY_HAVE_VECTOR_BYTE_SHUFFLE
diff --git a/snappy.cc b/snappy.cc index 95b9874..9a64563 100644 --- a/snappy.cc +++ b/snappy.cc
@@ -282,18 +282,6 @@ // calling MakePatternMaskBytes(0, 6, index_sequence<16>()) and // MakePatternMaskBytes(16, 6, index_sequence<16>()) respectively. -// Selects the appropriate vector size based on the current architecture -// vuint8m1_t, RISC-V vector type with fixed 128-bit size -// (sizeof not used due to variable-length vector register in RVV) -#if defined(__SSE2__) || defined(SNAPPY_HAVE_SSSE3) -constexpr size_t kVectorSize = sizeof(V128); // __m128i -#elif defined(__ARM_NEON) || defined(SNAPPY_HAVE_NEON) -constexpr size_t kVectorSize = sizeof(uint8x16_t); // uint8x16_t -#elif defined(SNAPPY_HAVE_RVV) || defined(__riscv_vector) -constexpr size_t kVectorSize = 16; // vuint8m1_t -#else -#error "Unsupported architecture. Please define __SSE2__, __ARM_NEON, or SNAPPY_HAVE_RVV/__riscv_vector." -#endif template <size_t... indexes> inline constexpr std::array<char, sizeof...(indexes)> MakePatternMaskBytes( @@ -304,17 +292,17 @@ // Computes the shuffle control mask bytes array for given pattern-sizes and // returns an array. template <size_t... pattern_sizes_minus_one> -inline constexpr std::array<std::array<char, kVectorSize>, +inline constexpr std::array<std::array<char, sizeof(V128)>, sizeof...(pattern_sizes_minus_one)> MakePatternMaskBytesTable(int index_offset, index_sequence<pattern_sizes_minus_one...>) { return {MakePatternMaskBytes(index_offset, pattern_sizes_minus_one + 1, - make_index_sequence<kVectorSize>())...}; + make_index_sequence<sizeof(V128)>())...}; } // This is an array of shuffle control masks that can be used as the source // operand for PSHUFB to permute the contents of the destination XMM register // into a repeating byte pattern. -alignas(16) constexpr std::array<std::array<char, kVectorSize>, +alignas(16) constexpr std::array<std::array<char, sizeof(V128)>, 16> pattern_generation_masks = MakePatternMaskBytesTable( /*index_offset=*/0, @@ -325,7 +313,7 @@ // Basically, pattern_reshuffle_masks is a continuation of // pattern_generation_masks. It follows that, pattern_reshuffle_masks is same as // pattern_generation_masks for offsets 1, 2, 4, 8 and 16. -alignas(16) constexpr std::array<std::array<char, kVectorSize>, +alignas(16) constexpr std::array<std::array<char, sizeof(V128)>, 16> pattern_reshuffle_masks = MakePatternMaskBytesTable( /*index_offset=*/16, @@ -342,12 +330,6 @@ generation_mask); } // vuint8m1_t cannot be used as an element of std::pair -#if SNAPPY_HAVE_RVV -#define LoadPatternAndReshuffleMask(src, pattern_size) \ - V128 pattern = LoadPattern(src, pattern_size);\ - V128 reshuffle_mask = V128_Load(reinterpret_cast<const V128*>(\ - pattern_reshuffle_masks[pattern_size - 1].data())); -#else // Suppress -Wignored-attributes warning for __m128i in x86 SSE2 environment // warning: ignoring attributes on template argument 'snappy::internal::V128' {aka '__vector(2) long long int'} [-Wignored-attributes] @@ -355,7 +337,7 @@ #ifdef __SSE2__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-attributes" -#endif + SNAPPY_ATTRIBUTE_ALWAYS_INLINE static inline std::pair<V128 /* pattern */, V128 /* reshuffle_mask */> @@ -411,14 +393,10 @@ return true; } default: { - #if SNAPPY_HAVE_RVV - LoadPatternAndReshuffleMask(dst - offset, offset) - #else auto pattern_and_reshuffle_mask = LoadPatternAndReshuffleMask(dst - offset, offset); V128 pattern = pattern_and_reshuffle_mask.first; V128 reshuffle_mask = pattern_and_reshuffle_mask.second; - #endif for (int i = 0; i < 4; i++) { V128_StoreU(reinterpret_cast<V128*>(dst + 16 * i), pattern); pattern = V128_Shuffle(pattern, reshuffle_mask); @@ -526,14 +504,10 @@ // Typically, the op_limit is the gating factor so try to simplify the loop // based on that. if (SNAPPY_PREDICT_TRUE(op_limit <= buf_limit - 15)) { - #if SNAPPY_HAVE_RVV - LoadPatternAndReshuffleMask(src, pattern_size); - #else auto pattern_and_reshuffle_mask = LoadPatternAndReshuffleMask(src, pattern_size); V128 pattern = pattern_and_reshuffle_mask.first; V128 reshuffle_mask = pattern_and_reshuffle_mask.second; - #endif // There is at least one, and at most four 16-byte blocks. Writing four // conditionals instead of a loop allows FDO to layout the code with // respect to the actual probabilities of each length. @@ -556,14 +530,10 @@ } char* const op_end = buf_limit - 15; if (SNAPPY_PREDICT_TRUE(op < op_end)) { - #if SNAPPY_HAVE_RVV - LoadPatternAndReshuffleMask(src, pattern_size); - #else auto pattern_and_reshuffle_mask = LoadPatternAndReshuffleMask(src, pattern_size); V128 pattern = pattern_and_reshuffle_mask.first; V128 reshuffle_mask = pattern_and_reshuffle_mask.second; - #endif // This code path is relatively cold however so we save code size // by avoiding unrolling and vectorizing. // @@ -1288,36 +1258,27 @@ data = _mm256_lddqu_si256(static_cast<const __m256i *>(src) + 1); _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst) + 1, data); } - // RVV acceleration available on RISC-V when compiled with -march=rv64gcv -#elif defined(__riscv) & SNAPPY_HAVE_RVV - uint8_t* dst_u8 = (uint8_t*)dst; - const uint8_t* src_u8 = (const uint8_t*)src; - //overlap bwd copy - if (src_u8 < dst_u8 && dst_u8 < src_u8 + size) { - size_t offset = size; - while (offset > 0) { - size_t vl = VSETVL_E8M1(offset); - offset -= vl; - vuint8m1_t vec = VLE8_V_U8M1(src_u8 + offset, vl); - VSE8_V_U8M1(dst_u8 + offset, vec, vl); - } - } else { - size_t vl = VSETVL_E8M1(size); - // if size >vl,use the max_vlen copy - if (vl < size) { - size_t offset = 0; - while (offset < size) { - vl = VSETVL_E8M1(size - offset); - vuint8m1_t vec = VLE8_V_U8M1(src_u8 + offset, vl); - VSE8_V_U8M1(dst_u8 + offset, vec, vl); - offset += vl; - } - } else { - // Copy the rest - vuint8m1_t vec = VLE8_V_U8M1(src_u8, vl); - VSE8_V_U8M1(dst_u8, vec, vl); - } - } + // RVV acceleration available on RISC-V when compiled with -march=rv64gcv +#elif defined(__riscv) && SNAPPY_HAVE_RVV + // Cast pointers to the type we will operate on. + unsigned char* dst_ptr = (unsigned char*)dst; + const unsigned char* src_ptr = (const unsigned char*)src; + size_t remaining_bytes = size; + //Loop as long as there are bytes remaining to be copied. + while (remaining_bytes > 0) { + //Set vector configuration: e8 (8-bit elements), m2 (LMUL=2). + //Use e8m2 configuration to maximize throughput. + size_t vl = VSETVL_E8M2(remaining_bytes); + //Load data from the current source pointer. + vuint8m2_t vec = VLE8_V_U8M2(src_ptr, vl); + //Store data to the current destination pointer. + VSE8_V_U8M2(dst_ptr, vec, vl); + //Update pointers and the remaining count. + src_ptr += vl; + dst_ptr += vl; + remaining_bytes -= vl; + } + #else std::memmove(dst, src, kShortMemCopy); //Profiling shows that nearly all copies are short.