Enable PCLMULQDQ and VPCLMULQDQ CRC32C acceleration on client and workstation x86_64 CPUs. PiperOrigin-RevId: 999948642
diff --git a/absl/base/internal/cpu_detect.cc b/absl/base/internal/cpu_detect.cc index 43b6c4b..c11eb4a 100644 --- a/absl/base/internal/cpu_detect.cc +++ b/absl/base/internal/cpu_detect.cc
@@ -132,15 +132,23 @@ switch (family) { case 6: // Most PentiumIII processors are in this category switch (model_num) { + case 0x25: // Westmere: Arrandale / Clarkdale case 0x2c: // Westmere: Gulftown + case 0x2f: // Westmere-EX return CpuType::kIntelWestmere; + case 0x2a: // Sandybridge (client) case 0x2d: // Sandybridge return CpuType::kIntelSandybridge; + case 0x3a: // Ivybridge (client) case 0x3e: // Ivybridge return CpuType::kIntelIvybridge; case 0x3c: // Haswell (client) case 0x3f: // Haswell + case 0x45: // Haswell-ULT + case 0x46: // Haswell (GT3e) return CpuType::kIntelHaswell; + case 0x3d: // Broadwell (client) + case 0x47: // Broadwell (H/C) case 0x4f: // Broadwell case 0x56: // BroadwellDE return CpuType::kIntelBroadwell; @@ -150,15 +158,27 @@ } else { // stepping >= 5 is cascadelake return CpuType::kIntelCascadelakeXeon; } + case 0x4e: // Skylake (mobile) case 0x5e: // Skylake (client) + case 0x8e: // Kaby Lake / Coffee Lake / Whiskey Lake / Amber Lake + case 0x9e: // Kaby Lake / Coffee Lake (desktop/H/Xeon E) + case 0xa5: // Comet Lake (desktop/H) + case 0xa6: // Comet Lake-U return CpuType::kIntelSkylake; - case 0x6a: // Ice Lake + case 0x6a: // Ice Lake-SP + case 0x6c: // Ice Lake-DE + case 0x7d: // Ice Lake-Y + case 0x7e: // Ice Lake-U + case 0x8c: // Tiger Lake-U + case 0x8d: // Tiger Lake-H + case 0xa7: // Rocket Lake (Cypress Cove) return CpuType::kIntelIcelake; case 0x8f: // Sapphire Rapids return CpuType::kIntelSapphirerapids; case 0xcf: // Emerald Rapids return CpuType::kIntelEmeraldrapids; - case 0xad: // Granite Rapids + case 0xad: // Granite Rapids-SP/AP + case 0xae: // Granite Rapids-D return CpuType::kIntelGraniterapids; default: return CpuType::kUnknown; @@ -189,49 +209,65 @@ // 20-27 (extended family) int family = (cpu_info[0] >> 8) & 0x0f; - int model_num = (cpu_info[0] >> 4) & 0x0f; int ext_family = (cpu_info[0] >> 20) & 0xff; int ext_model_num = (cpu_info[0] >> 16) & 0x0f; if (family == 0x0f) { family += ext_family; - model_num += (ext_model_num << 4); } switch (family) { case 0x17: - switch (model_num) { - case 0x0: // Stepping Ax - case 0x1: // Stepping Bx + switch (ext_model_num) { + case 0x0: // Zen 1: Naples / Summit Ridge / Whitehaven / Snowy Owl + case 0x1: // Zen 1 / Zen+: Raven Ridge / Picasso / Banded Kestrel + case 0x2: // Zen+: Dali return CpuType::kAmdNaples; - case 0x30: // Stepping Ax - case 0x31: // Stepping Bx + case 0x3: // Zen 2: Rome / Castle Peak + case 0x4: // Zen 2: Van Gogh + case 0x6: // Zen 2: Renoir / Grey Hawk + case 0x7: // Zen 2: Matisse + case 0x8: // Zen 2: Lucienne + case 0x9: // Zen 2: Mero + case 0xa: // Zen 2: Mendocino return CpuType::kAmdRome; default: return CpuType::kUnknown; } break; case 0x19: - switch (model_num) { - case 0x0: // Stepping Ax - case 0x1: // Stepping B0 + switch (ext_model_num) { + case 0x0: // Zen 3: Milan / Chagall (Threadripper PRO 5000) + case 0x2: // Zen 3: Vermeer (Ryzen 5000 desktop) + case 0x3: // Zen 3: Badami + case 0x5: // Zen 3: Cezanne / Barcelo (Ryzen 5000 mobile) return CpuType::kAmdMilan; - case 0x10: // Stepping A0 - case 0x11: // Stepping B0 + case 0x1: // Zen 4: Genoa / Storm Peak (Threadripper PRO 7000) + case 0x6: // Zen 4: Raphael (Ryzen 7000 desktop) / Dragon Range + case 0x7: // Zen 4: Phoenix / Hawk Point (Ryzen 7040/8040 mobile) return CpuType::kAmdGenoa; - case 0x44: // Stepping A0 + case 0x4: // Zen 3+: Rembrandt (Ryzen 6000 mobile / Ryzen V3000) return CpuType::kAmdRyzenV3000; - case 0xA0: // Stepping A0 + case 0xa: // Zen 4c: Bergamo / Siena return CpuType::kAmdSiena; default: return CpuType::kUnknown; } break; case 0x1A: - switch (model_num) { - case 0x2: + switch (ext_model_num) { + case 0x0: // Zen 5: Turin + case 0x1: // Zen 5c: Turin Dense + case 0x2: // Zen 5: Strix Point / Krackan Point + case 0x4: // Zen 5: Granite Ridge (Ryzen 9000) / Shimada Peak + case 0x6: // Zen 5: Fire Range + case 0x7: // Zen 5: Strix Halo return CpuType::kAmdTurin; - case 0x50: + case 0x5: // Zen 6: Venice + case 0x8: // Zen 6: Olympic Ridge + case 0x9: // Zen 6: Medusa Point + case 0xa: // Zen 6: Medusa Halo + case 0xc: // Zen 6: Venice Dense return CpuType::kAmdVenice; default: return CpuType::kUnknown; @@ -267,6 +303,33 @@ return (cpu_info[1] & (1 << 8)) != 0; } +bool SupportsAvx2() { + int cpu_info[4]; + __cpuid(cpu_info, 0); + if (cpu_info[0] < 7) { + return false; + } + __cpuid(cpu_info, 1); + // Check PCLMULQDQ (bit 1), OSXSAVE (bit 27), and AVX (bit 28) in ECX. + constexpr int kPclmulAvxOsxsave = (1 << 1) | (1 << 27) | (1 << 28); + if ((cpu_info[2] & kPclmulAvxOsxsave) != kPclmulAvxOsxsave) { + return false; + } + __cpuidex(cpu_info, 7, 0); + // Check AVX2 (EBX bit 5). + return (cpu_info[1] & (1 << 5)) != 0; +} + +bool SupportsVpclmulqdq() { + if (!SupportsAvx2()) { + return false; + } + int cpu_info[4]; + __cpuidex(cpu_info, 7, 0); + // Check VPCLMULQDQ (ECX bit 10). + return (cpu_info[2] & (1 << 10)) != 0; +} + #elif defined(__aarch64__) && defined(__linux__) #ifndef HWCAP_CPUID @@ -342,6 +405,10 @@ bool SupportsBmi2() { return false; } +bool SupportsAvx2() { return false; } + +bool SupportsVpclmulqdq() { return false; } + #elif defined(__aarch64__) && defined(__APPLE__) CpuType GetCpuType() { return CpuType::kUnknown; } @@ -390,6 +457,10 @@ bool SupportsBmi2() { return false; } +bool SupportsAvx2() { return false; } + +bool SupportsVpclmulqdq() { return false; } + #else CpuType GetCpuType() { return CpuType::kUnknown; } @@ -398,6 +469,10 @@ bool SupportsBmi2() { return false; } +bool SupportsAvx2() { return false; } + +bool SupportsVpclmulqdq() { return false; } + #endif // Returns how many hardware contexts per CPU exist. Note: AMD CPUs prior to Zen
diff --git a/absl/base/internal/cpu_detect.h b/absl/base/internal/cpu_detect.h index 06eb72a..9aef45e 100644 --- a/absl/base/internal/cpu_detect.h +++ b/absl/base/internal/cpu_detect.h
@@ -70,6 +70,12 @@ // Returns whether the host CPU supports BMI2 instructions. bool SupportsBmi2(); +// Returns whether the host CPU supports AVX2 and PCLMULQDQ instructions. +bool SupportsAvx2(); + +// Returns whether the host CPU supports 256-bit VPCLMULQDQ instructions. +bool SupportsVpclmulqdq(); + // Returns whether the host CPU supports simultaneous multithreading (SMT) and // if it is enabled. bool IsSMTEnabled();
diff --git a/absl/base/internal/cpu_detect_test.cc b/absl/base/internal/cpu_detect_test.cc index 80b8b87..6935447 100644 --- a/absl/base/internal/cpu_detect_test.cc +++ b/absl/base/internal/cpu_detect_test.cc
@@ -37,6 +37,34 @@ #endif } +TEST(CpuDetectTest, SupportsAvx2) { +#if defined(__x86_64__) || defined(_M_X64) +#if ABSL_HAVE_BUILTIN(__builtin_cpu_supports) && defined(__linux__) + EXPECT_EQ(SupportsAvx2(), __builtin_cpu_supports("avx2") != 0 && + __builtin_cpu_supports("pclmul") != 0); +#else + (void)SupportsAvx2(); +#endif +#else + EXPECT_FALSE(SupportsAvx2()); +#endif +} + +TEST(CpuDetectTest, SupportsVpclmulqdq) { +#if defined(__x86_64__) || defined(_M_X64) +#if ABSL_HAVE_BUILTIN(__builtin_cpu_supports) && defined(__linux__) + EXPECT_EQ(SupportsVpclmulqdq(), + __builtin_cpu_supports("vpclmulqdq") != 0 && + __builtin_cpu_supports("avx2") != 0 && + __builtin_cpu_supports("pclmul") != 0); +#else + (void)SupportsVpclmulqdq(); +#endif +#else + EXPECT_FALSE(SupportsVpclmulqdq()); +#endif +} + } // namespace } // namespace base_internal ABSL_NAMESPACE_END
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index 95221e4..d7eb523 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -43,6 +43,9 @@ using ::absl::base_internal::GetCpuType; #if defined(__aarch64__) using ::absl::base_internal::SupportsArmCRC32PMULL; +#else +using ::absl::base_internal::SupportsAvx2; +using ::absl::base_internal::SupportsVpclmulqdq; #endif #if defined(ABSL_INTERNAL_CAN_USE_SIMD_CRC32C) @@ -1153,6 +1156,18 @@ } #else default: +#if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && \ + (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && \ + (!defined(_MSC_VER) || defined(__clang__)) + if (SupportsVpclmulqdq()) { + return new CRC32AcceleratedX86ARMCombinedMultipleStreams< + 3, 3, PclmulStreamType::VPCLMUL>(); + } +#endif + if (SupportsAvx2()) { + return new CRC32AcceleratedX86ARMCombinedMultipleStreams< + 3, 2, PclmulStreamType::PCLMUL>(); + } // Something else, play it safe and assume slow PCLMULQDQ. return new CRC32AcceleratedX86ARMCombinedMultipleStreams< 3, 0, PclmulStreamType::NONE>();