add RVV support and optmized uncompress speed
diff --git a/CMakeLists.txt b/CMakeLists.txt index cd71a47..490f5b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt
@@ -216,6 +216,31 @@ return 0; }" SNAPPY_HAVE_NEON) +#check RVV 1.0 need __riscv_ prefix +check_cxx_source_compiles(" + #include <riscv_vector.h> + #include <stdint.h> + #include <stddef.h> + int main() { + uint8_t val = 3, dup[8]; + size_t vl = __riscv_vsetvl_e8m1(8); + vuint8m1_t v = __riscv_vmv_v_x_u8m1(val, vl); + return 0; + }" SNAPPY_RVV_1) + + +#check RVV 0.7.1 not __riscv_ prefix +check_cxx_source_compiles(" + #include <riscv_vector.h> + #include <stdint.h> + #include <stddef.h> + int main() { + uint8_t val = 3, dup[8]; + size_t vl = vsetvl_e8m1(8); + vuint8m1_t v = vmv_v_x_u8m1(val, vl); + return 0; + }" SNAPPY_RVV_0_7) + include(CheckSymbolExists) check_symbol_exists("mmap" "sys/mman.h" HAVE_FUNC_MMAP) check_symbol_exists("sysconf" "unistd.h" HAVE_FUNC_SYSCONF)
diff --git a/cmake/config.h.in b/cmake/config.h.in index 3510c27..de80c5f 100644 --- a/cmake/config.h.in +++ b/cmake/config.h.in
@@ -58,6 +58,12 @@ /* Define to 1 if you target processors with NEON and have <arm_neon.h>. */ #cmakedefine01 SNAPPY_HAVE_NEON +/* Define to 1 if you target processors with RVV1.0 and have <riscv_vector.h>. */ +#cmakedefine01 SNAPPY_RVV_1 + +/* Define to 1 if you target processors with RVV0.7 and have <riscv_vector.h>. */ +#cmakedefine01 SNAPPY_RVV_0_7 + /* Define to 1 if you have <arm_neon.h> and <arm_acle.h> and want to optimize compression speed by using __crc32cw from <arm_acle.h>. */ #cmakedefine01 SNAPPY_HAVE_NEON_CRC32
diff --git a/snappy-internal.h b/snappy-internal.h index 53a5110..1c9d043 100644 --- a/snappy-internal.h +++ b/snappy-internal.h
@@ -46,7 +46,13 @@ #include <arm_neon.h> #endif -#if SNAPPY_HAVE_SSSE3 || SNAPPY_HAVE_NEON +#if SNAPPY_RVV_1 || SNAPPY_RVV_0_7 +#define SNAPPY_HAVE_RVV 1 +#include <riscv_vector.h> +#endif + + +#if SNAPPY_HAVE_SSSE3 || SNAPPY_HAVE_NEON || SNAPPY_HAVE_RVV #define SNAPPY_HAVE_VECTOR_BYTE_SHUFFLE 1 #else #define SNAPPY_HAVE_VECTOR_BYTE_SHUFFLE 0 @@ -60,8 +66,23 @@ using V128 = __m128i; #elif SNAPPY_HAVE_NEON using V128 = uint8x16_t; +#elif SNAPPY_HAVE_RVV +using V128 = vuint8m1_t; #endif +#ifdef SNAPPY_RVV_1 +#define VSETVL_E8M1 __riscv_vsetvl_e8m1 +#define VLE8_V_U8M1 __riscv_vle8_v_u8m1 +#define VSE8_V_U8M1 __riscv_vse8_v_u8m1 +#define VRGATHER_VV_U8M1 __riscv_vrgather_vv_u8m1 +#define VMV_V_X_U8M1 __riscv_vmv_v_x_u8m1 +#elif SNAPPY_RVV_0_7 +#define VSETVL_E8M1 vsetvl_e8m1 +#define VLE8_V_U8M1 vle8_v_u8m1 +#define VSE8_V_U8M1 vse8_v_u8m1 +#define VRGATHER_VV_U8M1 vrgather_vv_u8m1 +#define VMV_V_X_U8M1 vmv_v_x_u8m1 +#endif // Load 128 bits of integer data. `src` must be 16-byte aligned. inline V128 V128_Load(const V128* src); @@ -110,6 +131,32 @@ } inline V128 V128_DupChar(char c) { return vdupq_n_u8(c); } + +#elif SNAPPY_HAVE_RVV +inline V128 V128_Load(const V128* src) { + size_t vl = VSETVL_E8M1(16); + return VLE8_V_U8M1(reinterpret_cast<const uint8_t*>(src), vl); +} + +inline V128 V128_LoadU(const V128* src) { + size_t vl = VSETVL_E8M1(16); + return VLE8_V_U8M1(reinterpret_cast<const uint8_t*>(src), vl); +} + +inline void V128_StoreU(V128* dst, V128 val) { + size_t vl = VSETVL_E8M1(16); + VSE8_V_U8M1(reinterpret_cast<uint8_t*>(dst), val, vl); +} + +inline V128 V128_Shuffle(V128 input, V128 shuffle_mask) { + size_t vl = VSETVL_E8M1(16); + return VRGATHER_VV_U8M1(input, shuffle_mask, vl); +} + +inline V128 V128_DupChar(char c) { + size_t vl = VSETVL_E8M1(16); + return VMV_V_X_U8M1(static_cast<uint8_t>(c), vl); +} #endif #endif // SNAPPY_HAVE_VECTOR_BYTE_SHUFFLE
diff --git a/snappy.cc b/snappy.cc index 8dc3713..421b335 100644 --- a/snappy.cc +++ b/snappy.cc
@@ -281,6 +281,20 @@ // 4, 5, 0, 1, 2, 3, 4, 5, 0, 1}. These byte index sequences are generated by // calling MakePatternMaskBytes(0, 6, index_sequence<16>()) and // MakePatternMaskBytes(16, 6, index_sequence<16>()) respectively. + +// Selects the appropriate vector size based on the current architecture +// vuint8m1_t, RISC-V vector type with fixed 128-bit size +// (sizeof not used due to variable-length vector register in RVV) +#if defined(__SSE2__) || defined(SNAPPY_HAVE_SSSE3) +constexpr size_t kVectorSize = sizeof(V128); // __m128i +#elif defined(__ARM_NEON) || defined(SNAPPY_HAVE_NEON) +constexpr size_t kVectorSize = sizeof(uint8x16_t); // uint8x16_t +#elif defined(SNAPPY_HAVE_RVV) || defined(__riscv_vector) +constexpr size_t kVectorSize = 16; // vuint8m1_t +#else +#error "Unsupported architecture. Please define __SSE2__, __ARM_NEON, or SNAPPY_HAVE_RVV/__riscv_vector." +#endif + template <size_t... indexes> inline constexpr std::array<char, sizeof...(indexes)> MakePatternMaskBytes( int index_offset, int pattern_size, index_sequence<indexes...>) { @@ -290,19 +304,17 @@ // Computes the shuffle control mask bytes array for given pattern-sizes and // returns an array. template <size_t... pattern_sizes_minus_one> -inline constexpr std::array<std::array<char, sizeof(V128)>, +inline constexpr std::array<std::array<char, kVectorSize>, sizeof...(pattern_sizes_minus_one)> MakePatternMaskBytesTable(int index_offset, index_sequence<pattern_sizes_minus_one...>) { - return { - MakePatternMaskBytes(index_offset, pattern_sizes_minus_one + 1, - make_index_sequence</*indexes=*/sizeof(V128)>())...}; + return {MakePatternMaskBytes(index_offset, pattern_sizes_minus_one + 1, + make_index_sequence<kVectorSize>())...}; } - // This is an array of shuffle control masks that can be used as the source // operand for PSHUFB to permute the contents of the destination XMM register // into a repeating byte pattern. -alignas(16) constexpr std::array<std::array<char, sizeof(V128)>, +alignas(16) constexpr std::array<std::array<char, kVectorSize>, 16> pattern_generation_masks = MakePatternMaskBytesTable( /*index_offset=*/0, @@ -313,7 +325,7 @@ // Basically, pattern_reshuffle_masks is a continuation of // pattern_generation_masks. It follows that, pattern_reshuffle_masks is same as // pattern_generation_masks for offsets 1, 2, 4, 8 and 16. -alignas(16) constexpr std::array<std::array<char, sizeof(V128)>, +alignas(16) constexpr std::array<std::array<char, kVectorSize>, 16> pattern_reshuffle_masks = MakePatternMaskBytesTable( /*index_offset=*/16, @@ -329,6 +341,21 @@ return V128_Shuffle(V128_LoadU(reinterpret_cast<const V128*>(src)), generation_mask); } +// vuint8m1_t cannot be used as an element of std::pair +#if SNAPPY_HAVE_RVV +#define LoadPatternAndReshuffleMask(src, pattern_size) \ + V128 pattern = LoadPattern(src, pattern_size);\ + V128 reshuffle_mask = V128_Load(reinterpret_cast<const V128*>(\ + pattern_reshuffle_masks[pattern_size - 1].data())); +#else + +// Suppress -Wignored-attributes warning for __m128i in x86 SSE2 environment +// warning: ignoring attributes on template argument 'snappy::internal::V128' {aka '__vector(2) long long int'} [-Wignored-attributes] +// This occurs because __m128i has vector attributes (e.g., __attribute__((vector_size(16)))) that are ignored in template parameters. +#ifdef __SSE2__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-attributes" +#endif SNAPPY_ATTRIBUTE_ALWAYS_INLINE static inline std::pair<V128 /* pattern */, V128 /* reshuffle_mask */> @@ -345,7 +372,12 @@ pattern_reshuffle_masks[pattern_size - 1].data())); return {pattern, reshuffle_mask}; } +// Restore original diagnostic state in x86 SSE2 environment +#ifdef __SSE2__ +#pragma GCC diagnostic pop +#endif +#endif #endif // SNAPPY_HAVE_VECTOR_BYTE_SHUFFLE // Fallback for when we need to copy while extending the pattern, for example @@ -379,10 +411,14 @@ return true; } default: { + #if SNAPPY_HAVE_RVV + LoadPatternAndReshuffleMask(dst - offset, offset) + #else auto pattern_and_reshuffle_mask = LoadPatternAndReshuffleMask(dst - offset, offset); V128 pattern = pattern_and_reshuffle_mask.first; V128 reshuffle_mask = pattern_and_reshuffle_mask.second; + #endif for (int i = 0; i < 4; i++) { V128_StoreU(reinterpret_cast<V128*>(dst + 16 * i), pattern); pattern = V128_Shuffle(pattern, reshuffle_mask); @@ -490,11 +526,14 @@ // Typically, the op_limit is the gating factor so try to simplify the loop // based on that. if (SNAPPY_PREDICT_TRUE(op_limit <= buf_limit - 15)) { + #if SNAPPY_HAVE_RVV + LoadPatternAndReshuffleMask(src, pattern_size); + #else auto pattern_and_reshuffle_mask = LoadPatternAndReshuffleMask(src, pattern_size); V128 pattern = pattern_and_reshuffle_mask.first; V128 reshuffle_mask = pattern_and_reshuffle_mask.second; - + #endif // There is at least one, and at most four 16-byte blocks. Writing four // conditionals instead of a loop allows FDO to layout the code with // respect to the actual probabilities of each length. @@ -517,11 +556,14 @@ } char* const op_end = buf_limit - 15; if (SNAPPY_PREDICT_TRUE(op < op_end)) { + #if SNAPPY_HAVE_RVV + LoadPatternAndReshuffleMask(src, pattern_size); + #else auto pattern_and_reshuffle_mask = LoadPatternAndReshuffleMask(src, pattern_size); V128 pattern = pattern_and_reshuffle_mask.first; V128 reshuffle_mask = pattern_and_reshuffle_mask.second; - + #endif // This code path is relatively cold however so we save code size // by avoiding unrolling and vectorizing. // @@ -1247,13 +1289,41 @@ _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst) + 1, data); } #else - std::memmove(dst, src, kShortMemCopy); - // Profiling shows that nearly all copies are short. - if (SNAPPY_PREDICT_FALSE(size > kShortMemCopy)) { - std::memmove(dst + kShortMemCopy, - static_cast<const uint8_t*>(src) + kShortMemCopy, - 64 - kShortMemCopy); +#ifdef SNAPPY_HAVE_RVV +uint8_t* dst_u8 = (uint8_t*)dst; +const uint8_t* src_u8 = (const uint8_t*)src; +if (src_u8 < dst_u8 && dst_u8 < src_u8 + size) { //overlap bwd copy + size_t offset = size; + while (offset > 0) { + size_t vl = VSETVL_E8M1(offset); + offset -= vl; + vuint8m1_t vec = VLE8_V_U8M1(src_u8 + offset, vl); + VSE8_V_U8M1(dst_u8 + offset, vec, vl); } +} else { + size_t vl = VSETVL_E8M1(size); + if (vl < size) { // if size >vl,use the max_vlen copy + size_t offset = 0; + while (offset < size) { + vl = VSETVL_E8M1(size - offset); + vuint8m1_t vec = VLE8_V_U8M1(src_u8 + offset, vl); + VSE8_V_U8M1(dst_u8 + offset, vec, vl); + offset += vl; + } + } else { // copy the leaft + vuint8m1_t vec = VLE8_V_U8M1(src_u8, vl); + VSE8_V_U8M1(dst_u8, vec, vl); + } + } +#else +std::memmove(dst, src, kShortMemCopy); + //Profiling shows that nearly all copies are short. +if (SNAPPY_PREDICT_FALSE(size > kShortMemCopy)) { + std::memmove(dst + kShortMemCopy, + static_cast<const uint8_t*>(src) + kShortMemCopy, + 64 - kShortMemCopy);} + +#endif #endif }