Merge pull request #78 from easyaspi314/master
Add ARMv7/NEON support, fix tsc_timer.h and os_mac.cc/h from xpahos
diff --git a/Makefile b/Makefile
index a6a8599..9e82512 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-# We assume X64 unless HH_POWER or HH_AARCH64 are defined.
+# We assume X64 unless HH_POWER, HH_ARM, or HH_AARCH64 are defined.
override CPPFLAGS += -I.
override CXXFLAGS += -std=c++11 -Wall -O3 -fPIC -pthread
@@ -25,8 +25,17 @@
HIGHWAYHASH_TEST_OBJS := $(DISPATCHER_OBJS) obj/highwayhash_test_portable.o
VECTOR_TEST_OBJS := $(DISPATCHER_OBJS) obj/vector_test_portable.o
+# aarch64 and ARM use the same code, although ARM usually needs an extra flag for NEON.
+ifdef HH_ARM
+CXXFLAGS += -mfloat-abi=hard -march=armv7-a -mfpu=neon
+HH_AARCH64 = 1
+endif
+
ifdef HH_AARCH64
HH_X64 =
+HIGHWAYHASH_OBJS += obj/hh_neon.o
+HIGHWAYHASH_TEST_OBJS += obj/highwayhash_test_neon.o
+VECTOR_TEST_OBJS += obj/vector_test_neon.o
else
ifdef HH_POWER
HH_X64 =
@@ -72,6 +81,7 @@
bin/nanobenchmark_example: $(DISPATCHER_OBJS) obj/nanobenchmark.o
ifdef HH_X64
+# TODO: Portability: Have AVX2 be optional so benchmarking can be done on older machines.
obj/sip_tree_hash.o: CXXFLAGS+=-mavx2
# (Compiled from same source file with different compiler flags)
obj/highwayhash_test_avx2.o: CXXFLAGS+=-mavx2
diff --git a/highwayhash/arch_specific.cc b/highwayhash/arch_specific.cc
index 4949dfa..ea1ec7b 100644
--- a/highwayhash/arch_specific.cc
+++ b/highwayhash/arch_specific.cc
@@ -39,6 +39,8 @@
return "AVX2";
case HH_TARGET_VSX:
return "VSX";
+ case HH_TARGET_NEON:
+ return "NEON";
default:
return nullptr; // zero, multiple, or unknown bits
}
diff --git a/highwayhash/arch_specific.h b/highwayhash/arch_specific.h
index 6ea4041..e9d4285 100644
--- a/highwayhash/arch_specific.h
+++ b/highwayhash/arch_specific.h
@@ -52,12 +52,24 @@
#define HH_ARCH_X64 0
#endif
-#ifdef __aarch64__
+#if defined(__aarch64__) || defined(__arm64__)
#define HH_ARCH_AARCH64 1
#else
#define HH_ARCH_AARCH64 0
#endif
+#ifdef __arm__
+#define HH_ARCH_ARM 1
+#else
+#define HH_ARCH_ARM 0
+#endif
+
+#if defined(__ARM_NEON__) || defined(__ARM_NEON)
+#define HH_ARCH_NEON 1
+#else
+#define HH_ARCH_NEON 0
+#endif
+
#if defined(__powerpc64__) || defined(_M_PPC)
#define HH_ARCH_PPC 1
#else
@@ -82,6 +94,8 @@
#define HH_TARGET_NAME SSE41
#elif defined(__VSX__)
#define HH_TARGET_NAME VSX
+#elif HH_ARCH_NEON
+#define HH_TARGET_NAME NEON
#else
#define HH_TARGET_NAME Portable
#endif
@@ -112,6 +126,7 @@
#define HH_TARGET_SSE41 2
#define HH_TARGET_AVX2 4
#define HH_TARGET_VSX 8
+#define HH_TARGET_NEON 16
// Bit array for one or more HH_TARGET_*. Used to indicate which target(s) are
// supported or were called by InstructionSets::RunAll.
diff --git a/highwayhash/hh_buffer.h b/highwayhash/hh_buffer.h
index 5b1c83f..8a60f00 100644
--- a/highwayhash/hh_buffer.h
+++ b/highwayhash/hh_buffer.h
@@ -22,7 +22,11 @@
// dependencies must not define any function unless it is static inline and/or
// within namespace HH_TARGET_NAME. See arch_specific.h for details.
+#if HH_TARGET == HH_TARGET_NEON
+#include "highwayhash/vector_neon.h"
+#else
#include "highwayhash/vector128.h"
+#endif
// For auto-dependency generation, we need to include all headers but not their
// contents (otherwise compilation fails because -msse4.1 is not specified).
@@ -45,7 +49,11 @@
HH_INLINE V4x32U operator()(const V4x32U& size) const {
// Lane n is valid if size >= (n + 1) * 4; subtract one because we only have
// greater-than comparisons and don't want a negated mask.
+#if HH_TARGET == HH_TARGET_NEON
+ return V4x32U(vcgtq_u32(size, V4x32U(15, 11, 7, 3)));
+#else
return V4x32U(_mm_cmpgt_epi32(size, V4x32U(15, 11, 7, 3)));
+#endif
}
};
@@ -54,7 +62,11 @@
// "size" is 16..31; this is for loading the upper half of a packet, so
// effectively deduct 16 from size by changing the comparands.
HH_INLINE V4x32U operator()(const V4x32U& size) const {
+#if HH_TARGET == HH_TARGET_NEON
+ return V4x32U(vcgtq_u32(size, V4x32U(31, 27, 23, 19)));
+#else
return V4x32U(_mm_cmpgt_epi32(size, V4x32U(31, 27, 23, 19)));
+#endif
}
};
@@ -70,10 +82,12 @@
return prev | AndNot(mask, V4x32U(bytes4));
}
+#if HH_TARGET == HH_TARGET_AVX2
// Shifts "suffix" left by "prefix_len" = 0..15 bytes, clears upper bytes of
// "prefix", and returns the merged/concatenated bytes.
HH_INLINE V4x32U Concatenate(const V4x32U& prefix, const size_t prefix_len,
const V4x32U& suffix) {
+
static const uint64_t table[V16x8U::N][V2x64U::N] = {
{0x0706050403020100ull, 0x0F0E0D0C0B0A0908ull},
{0x06050403020100FFull, 0x0E0D0C0B0A090807ull},
@@ -95,7 +109,7 @@
const V2x64U shifted_suffix(_mm_shuffle_epi8(suffix, control));
return V4x32U(_mm_blendv_epi8(shifted_suffix, prefix, control));
}
-
+#endif
} // namespace HH_TARGET_NAME
} // namespace highwayhash
diff --git a/highwayhash/hh_neon.cc b/highwayhash/hh_neon.cc
new file mode 100644
index 0000000..4d6711b
--- /dev/null
+++ b/highwayhash/hh_neon.cc
@@ -0,0 +1,19 @@
+// Copyright 2017-2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// WARNING: this is a "restricted" source file; avoid including any headers
+// unless they are also restricted. See arch_specific.h for details.
+
+#define HH_TARGET_NAME NEON
+#include "highwayhash/highwayhash_target.cc"
diff --git a/highwayhash/hh_neon.h b/highwayhash/hh_neon.h
new file mode 100644
index 0000000..d0af81e
--- /dev/null
+++ b/highwayhash/hh_neon.h
@@ -0,0 +1,363 @@
+// Copyright 2015-2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HIGHWAYHASH_HH_NEON_H_
+#define HIGHWAYHASH_HH_NEON_H_
+
+// WARNING: this is a "restricted" header because it is included from
+// translation units compiled with different flags. This header and its
+// dependencies must not define any function unless it is static inline and/or
+// within namespace HH_TARGET_NAME. See arch_specific.h for details.
+
+#include "highwayhash/arch_specific.h"
+#include "highwayhash/compiler_specific.h"
+#include "highwayhash/hh_buffer.h"
+#include "highwayhash/hh_types.h"
+#include "highwayhash/load3.h"
+#include "highwayhash/vector_neon.h"
+
+// For auto-dependency generation, we need to include all headers but not their
+// contents.
+#ifndef HH_DISABLE_TARGET_SPECIFIC
+
+namespace highwayhash {
+
+
+// See vector_neon.h for why this namespace is necessary; matching it here makes
+// it easier use the vector_neon symbols, but requires textual inclusion.
+namespace HH_TARGET_NAME {
+
+// J-lanes tree hashing: see http://dx.doi.org/10.4236/jis.2014.53010
+// Uses the same method that SSE4.1 uses, only with NEON used instead.
+class HHStateNEON {
+ public:
+ explicit HH_INLINE HHStateNEON(const HHKey key) { Reset(key); }
+
+ HH_INLINE void Reset(const HHKey key) {
+ // "Nothing up my sleeve numbers"; see HHStateTAVX2.
+ const V2x64U init0L(0xa4093822299f31d0ull, 0xdbe6d5d5fe4cce2full);
+ const V2x64U init0H(0x243f6a8885a308d3ull, 0x13198a2e03707344ull);
+ const V2x64U init1L(0xc0acf169b5f18a8cull, 0x3bd39e10cb0ef593ull);
+ const V2x64U init1H(0x452821e638d01377ull, 0xbe5466cf34e90c6cull);
+ const V2x64U keyL = LoadUnaligned<V2x64U>(key + 0);
+ const V2x64U keyH = LoadUnaligned<V2x64U>(key + 2);
+ v0L = keyL ^ init0L;
+ v0H = keyH ^ init0H;
+ v1L = Rotate64By32(keyL) ^ init1L;
+ v1H = Rotate64By32(keyH) ^ init1H;
+ mul0L = init0L;
+ mul0H = init0H;
+ mul1L = init1L;
+ mul1H = init1H;
+ }
+
+ HH_INLINE void Update(const HHPacket& packet_bytes) {
+ const uint64_t* HH_RESTRICT packet =
+ reinterpret_cast<const uint64_t * HH_RESTRICT>(packet_bytes);
+ const V2x64U packetL = LoadUnaligned<V2x64U>(packet + 0);
+ const V2x64U packetH = LoadUnaligned<V2x64U>(packet + 2);
+ Update(packetH, packetL);
+ }
+
+ HH_INLINE void UpdateRemainder(const char* bytes, const size_t size_mod32) {
+ // 'Length padding' differentiates zero-valued inputs that have the same
+ // size/32. mod32 is sufficient because each Update behaves as if a
+ // counter were injected, because the state is large and mixed thoroughly.
+ const int32x4_t vsize_mod32(static_cast<int32_t>(size_mod32));
+ const int32x4_t shift_right_amt = vdupq_n_s32(static_cast<int32_t>(size_mod32) + (~32 + 1));
+ // Equivalent to storing size_mod32 in packet.
+ v0L += V2x64U(vreinterpretq_u64_s32(vsize_mod32));
+ v0H += V2x64U(vreinterpretq_u64_s32(vsize_mod32));
+
+ // Boosts the avalanche effect of mod32.
+ v1L = V2x64U(vreinterpretq_u64_u32(
+ vorrq_u32(
+ vshlq_u32(vreinterpretq_u32_u64(v1L), vsize_mod32),
+ vshlq_u32(vreinterpretq_u32_u64(v1L), shift_right_amt)
+ )
+ ));
+ v1H = V2x64U(vreinterpretq_u64_u32(
+ vorrq_u32(
+ vshlq_u32(vreinterpretq_u32_u64(v1H), vsize_mod32),
+ vshlq_u32(vreinterpretq_u32_u64(v1H), shift_right_amt)
+ )
+ ));
+
+ const size_t size_mod4 = size_mod32 & 3;
+ const char* HH_RESTRICT remainder = bytes + (size_mod32 & ~3);
+
+ if (HH_UNLIKELY(size_mod32 & 16)) { // 16..31 bytes left
+ const V2x64U packetL =
+ LoadUnaligned<V2x64U>(reinterpret_cast<const uint64_t*>(bytes));
+
+ V2x64U packetH = LoadMultipleOfFour(bytes + 16, size_mod32);
+
+ const uint32_t last4 =
+ Load3()(Load3::AllowReadBeforeAndReturn(), remainder, size_mod4);
+
+ // The upper four bytes of packetH are zero, so insert there.
+ packetH = V2x64U(
+ vreinterpretq_u64_u32(
+ vsetq_lane_u32(last4, vreinterpretq_u32_u64(packetH), 3)
+ )
+ );
+ Update(packetH, packetL);
+ } else { // size_mod32 < 16
+ const V2x64U packetL = LoadMultipleOfFour(bytes, size_mod32);
+
+ const uint64_t last4 =
+ Load3()(Load3::AllowUnordered(), remainder, size_mod4);
+
+ // Rather than insert into packetL[3], it is faster to initialize
+ // the otherwise empty packetH.
+ HH_ALIGNAS(16) uint64_t tmp[2] = { last4, 0 };
+ const V2x64U packetH(vld1q_u64(tmp));
+ Update(packetH, packetL);
+ }
+ }
+
+ HH_INLINE void Finalize(HHResult64* HH_RESTRICT result) {
+ // Mix together all lanes.
+ for (int n = 0; n < 4; n++) {
+ PermuteAndUpdate();
+ }
+
+ const V2x64U sum0 = v0L + mul0L;
+ const V2x64U sum1 = v1L + mul1L;
+ const V2x64U hash = sum0 + sum1;
+ vst1q_low_u64(reinterpret_cast<uint64_t*>(result), hash);
+ }
+
+ HH_INLINE void Finalize(HHResult128* HH_RESTRICT result) {
+ for (int n = 0; n < 6; n++) {
+ PermuteAndUpdate();
+ }
+
+ const V2x64U sum0 = v0L + mul0L;
+ const V2x64U sum1 = v1H + mul1H;
+ const V2x64U hash = sum0 + sum1;
+ StoreUnaligned(hash, &(*result)[0]);
+ }
+
+ HH_INLINE void Finalize(HHResult256* HH_RESTRICT result) {
+ for (int n = 0; n < 10; n++) {
+ PermuteAndUpdate();
+ }
+
+ const V2x64U sum0L = v0L + mul0L;
+ const V2x64U sum1L = v1L + mul1L;
+ const V2x64U sum0H = v0H + mul0H;
+ const V2x64U sum1H = v1H + mul1H;
+ const V2x64U hashL = ModularReduction(sum1L, sum0L);
+ const V2x64U hashH = ModularReduction(sum1H, sum0H);
+ StoreUnaligned(hashL, &(*result)[0]);
+ StoreUnaligned(hashH, &(*result)[2]);
+ }
+
+ static HH_INLINE void ZeroInitialize(char* HH_RESTRICT buffer_bytes) {
+ for (size_t i = 0; i < sizeof(HHPacket); ++i) {
+ buffer_bytes[i] = 0;
+ }
+ }
+
+ static HH_INLINE void CopyPartial(const char* HH_RESTRICT from,
+ const size_t size_mod32,
+ char* HH_RESTRICT buffer) {
+ for (size_t i = 0; i < size_mod32; ++i) {
+ buffer[i] = from[i];
+ }
+ }
+
+ static HH_INLINE void AppendPartial(const char* HH_RESTRICT from,
+ const size_t size_mod32,
+ char* HH_RESTRICT buffer,
+ const size_t buffer_valid) {
+ for (size_t i = 0; i < size_mod32; ++i) {
+ buffer[buffer_valid + i] = from[i];
+ }
+ }
+
+ HH_INLINE void AppendAndUpdate(const char* HH_RESTRICT from,
+ const size_t size_mod32,
+ const char* HH_RESTRICT buffer,
+ const size_t buffer_valid) {
+ HHPacket tmp HH_ALIGNAS(32);
+ for (size_t i = 0; i < buffer_valid; ++i) {
+ tmp[i] = buffer[i];
+ }
+ for (size_t i = 0; i < size_mod32; ++i) {
+ tmp[buffer_valid + i] = from[i];
+ }
+ Update(tmp);
+ }
+
+ private:
+ // Swap 32-bit halves of each lane (caller swaps 128-bit halves)
+ static HH_INLINE V2x64U Rotate64By32(const V2x64U& v) {
+ return V2x64U(
+ vreinterpretq_u64_u32(
+ vrev64q_u32(
+ vreinterpretq_u32_u64(v)
+ )
+ )
+ );
+ }
+
+ // Rotates 32-bit lanes by "count" bits.
+ static HH_INLINE void Rotate32By(V2x64U* HH_RESTRICT vH,
+ V2x64U* HH_RESTRICT vL,
+ const uint64_t count) {
+ // WARNING: the shift count is 64 bits, so we can't reuse vsize_mod32,
+ // which is broadcast into 32-bit lanes.
+ *vL = V2x64U((V4x32U(*vL) << int(count)).ShiftRightInsert(*vL, 32-count));
+ *vH = V2x64U((V4x32U(*vH) << int(count)).ShiftRightInsert(*vH, 32-count));
+ }
+
+ static HH_INLINE V2x64U ZipperMerge(const V2x64U& v) {
+ // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to
+ // varying degrees. In descending order of goodness, bytes
+ // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32.
+ // As expected, the upper and lower bytes are much worse.
+ // For each 64-bit lane, our objectives are:
+ // 1) maximizing and equalizing total goodness across each lane's bytes;
+ // 2) mixing with bytes from the neighboring lane;
+ // 3) placing the worst bytes in the upper 32 bits because those will not
+ // be used in the next 32x32 multiplication.
+
+ // The positions of each byte in the new vector.
+ const uint8_t shuffle_positions[] = {
+ 3, 12, 2, 5, 14, 1, 15, 0,
+ 11, 4, 10, 13, 9, 6, 8, 7
+ };
+ const uint8x16_t tbl = vld1q_u8(shuffle_positions);
+
+ // Note: vqtbl1q_u8 is polyfilled for ARMv7a in vector_neon.h.
+ return V2x64U(
+ vreinterpretq_u64_u8(
+ vqtbl1q_u8(
+ vreinterpretq_u8_u64(v),
+ tbl
+ )
+ )
+ );
+ }
+
+ HH_INLINE void Update(const V2x64U& packetH, const V2x64U& packetL) {
+ v1L += packetL;
+ v1H += packetH;
+ v1L += mul0L;
+ v1H += mul0H;
+ // mul0L ^= (v1L & 0xFFFFFFFF) * (v0L >> 32);
+ mul0L ^= V2x64U(vmull_u32(vmovn_u64(v1L), vshrn_n_u64(v0L, 32)));
+ // mul0H ^= (v1H & 0xFFFFFFFF) * (v0H >> 32);
+ mul0H ^= V2x64U(vmull_u32(vmovn_u64(v1H), vshrn_n_u64(v0H, 32)));
+ v0L += mul1L;
+ v0H += mul1H;
+ // mul1L ^= (v0L & 0xFFFFFFFF) * (v1L >> 32);
+ mul1L ^= V2x64U(vmull_u32(vmovn_u64(v0L), vshrn_n_u64(v1L, 32)));
+ // mul1H ^= (v0H & 0xFFFFFFFF) * (v1H >> 32);
+ mul1H ^= V2x64U(vmull_u32(vmovn_u64(v0H), vshrn_n_u64(v1H, 32)));
+ v0L += ZipperMerge(v1L);
+ v0H += ZipperMerge(v1H);
+ v1L += ZipperMerge(v0L);
+ v1H += ZipperMerge(v0H);
+ }
+
+ HH_INLINE void PermuteAndUpdate() {
+ // It is slightly better to permute v0 than v1; it will be added to v1.
+ Update(Rotate64By32(v0L), Rotate64By32(v0H));
+ }
+
+ // Returns zero-initialized vector with the lower "size" = 0, 4, 8 or 12
+ // bytes loaded from "bytes". Serves as a replacement for AVX2 maskload_epi32.
+ static HH_INLINE V2x64U LoadMultipleOfFour(const char* bytes,
+ const size_t size) {
+ const uint32_t* words = reinterpret_cast<const uint32_t*>(bytes);
+ // Mask of 1-bits where the final 4 bytes should be inserted (replacement
+ // for variable shift/insert using broadcast+blend).
+ alignas(16) const uint64_t mask_pattern[2] = { 0xFFFFFFFFULL, 0 };
+ V2x64U mask4(vld1q_u64(mask_pattern)); // 'insert' into lane 0
+ V2x64U ret(vdupq_n_u64(0));
+ if (size & 8) {
+ ret = V2x64U(vld1q_low_u64(reinterpret_cast<const uint64_t*>(words)));
+ // mask4 = 0 ~0 0 0 ('insert' into lane 2)
+ mask4 = V2x64U(vshlq_n_u128(mask4, 8));
+ words += 2;
+ }
+ // Final 4 (possibly after the 8 above); 'insert' into lane 0 or 2 of ret.
+ if (size & 4) {
+ // = 0 word2 0 word2; mask4 will select which lane to keep.
+ const V2x64U broadcast(vreinterpretq_u64_u32(vdupq_n_u32(words[0])));
+ // (slightly faster than blendv_epi8)
+ ret |= V2x64U(broadcast & mask4);
+ }
+ return ret;
+ }
+
+ // XORs x << 1 and x << 2 into *out after clearing the upper two bits of x.
+ // Bit shifts are only possible on independent 64-bit lanes. We therefore
+ // insert the upper bits of x[0] that were lost into x[1].
+ // Thanks to D. Lemire for helpful comments!
+ static HH_INLINE void XorByShift128Left12(const V2x64U& x,
+ V2x64U* HH_RESTRICT out) {
+ const V4x32U zero(vdupq_n_u32(0));
+ const V2x64U sign_bit128(vreinterpretq_u64_u32(vsetq_lane_u32(0x80000000u, zero, 3)));
+ const V2x64U top_bits2 = x >> (64 - 2);
+ HH_COMPILER_FENCE;
+ const V2x64U shifted1_unmasked = x + x; // (avoids needing port0)
+
+ // Only the lower half of top_bits1 will be used, so we
+ // can compute it before clearing the upper two bits of x.
+ const V2x64U top_bits1 = x >> (64 - 1);
+ const V2x64U shifted2 = shifted1_unmasked + shifted1_unmasked;
+ HH_COMPILER_FENCE;
+
+ const V2x64U new_low_bits2(vshlq_n_u128(top_bits2, 8));
+ *out ^= shifted2;
+ // The result must be as if the upper two bits of the input had been clear,
+ // otherwise we're no longer computing a reduction.
+ const V2x64U shifted1 = AndNot(sign_bit128, shifted1_unmasked);
+ HH_COMPILER_FENCE;
+
+ const V2x64U new_low_bits1(vshlq_n_u128(top_bits1, 8));
+ *out ^= new_low_bits2;
+ *out ^= shifted1;
+ *out ^= new_low_bits1;
+ }
+
+ // Modular reduction by the irreducible polynomial (x^128 + x^2 + x).
+ // Input: a 256-bit number a3210.
+ static HH_INLINE V2x64U ModularReduction(const V2x64U& a32_unmasked,
+ const V2x64U& a10) {
+ // See Lemire, https://arxiv.org/pdf/1503.03465v8.pdf.
+ V2x64U out = a10;
+ XorByShift128Left12(a32_unmasked, &out);
+ return out;
+ }
+
+ V2x64U v0L;
+ V2x64U v0H;
+ V2x64U v1L;
+ V2x64U v1H;
+ V2x64U mul0L;
+ V2x64U mul0H;
+ V2x64U mul1L;
+ V2x64U mul1H;
+};
+
+} // namespace HH_TARGET_NAME
+} // namespace highwayhash
+
+#endif // HH_DISABLE_TARGET_SPECIFIC
+#endif // HIGHWAYHASH_HH_NEON_H_
diff --git a/highwayhash/highwayhash.h b/highwayhash/highwayhash.h
index a238ef9..fe21ba1 100644
--- a/highwayhash/highwayhash.h
+++ b/highwayhash/highwayhash.h
@@ -45,6 +45,8 @@
#include "highwayhash/hh_sse41.h"
#elif HH_TARGET == HH_TARGET_VSX
#include "highwayhash/hh_vsx.h"
+#elif HH_TARGET == HH_TARGET_NEON
+#include "highwayhash/hh_neon.h"
#elif HH_TARGET == HH_TARGET_Portable
#include "highwayhash/hh_portable.h"
#else
diff --git a/highwayhash/highwayhash_test_neon.cc b/highwayhash/highwayhash_test_neon.cc
new file mode 100644
index 0000000..2c714e0
--- /dev/null
+++ b/highwayhash/highwayhash_test_neon.cc
@@ -0,0 +1,19 @@
+// Copyright 2017-2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// WARNING: this is a "restricted" source file; avoid including any headers
+// unless they are also restricted. See arch_specific.h for details.
+
+#define HH_TARGET_NAME NEON
+#include "highwayhash/highwayhash_test_target.cc"
diff --git a/highwayhash/instruction_sets.h b/highwayhash/instruction_sets.h
index ab6fbbc..aa7bd6b 100644
--- a/highwayhash/instruction_sets.h
+++ b/highwayhash/instruction_sets.h
@@ -40,6 +40,10 @@
static HH_INLINE TargetBits Supported() {
return HH_TARGET_VSX | HH_TARGET_Portable;
}
+#elif HH_ARCH_NEON
+ static HH_INLINE TargetBits Supported() {
+ return HH_TARGET_NEON | HH_TARGET_Portable;
+ }
#else
static HH_INLINE TargetBits Supported() { return HH_TARGET_Portable; }
#endif
@@ -66,6 +70,12 @@
Func<HH_TARGET_VSX>()(std::forward<Args>(args)...);
return HH_TARGET_VSX;
}
+#elif HH_ARCH_NEON
+ const TargetBits supported = Supported();
+ if (supported & HH_TARGET_NEON) {
+ Func<HH_TARGET_NEON>()(std::forward<Args>(args)...);
+ return HH_TARGET_NEON;
+ }
#endif
// No matching HH_ARCH or no supported HH_TARGET:
@@ -90,6 +100,11 @@
if (supported & HH_TARGET_VSX) {
Func<HH_TARGET_VSX>()(std::forward<Args>(args)...);
}
+
+#elif HH_ARCH_NEON
+ if (supported & HH_TARGET_NEON) {
+ Func<HH_TARGET_NEON>()(std::forward<Args>(args)...);
+ }
#endif
Func<HH_TARGET_Portable>()(std::forward<Args>(args)...);
diff --git a/highwayhash/os_mac.cc b/highwayhash/os_mac.cc
new file mode 100644
index 0000000..10d24e7
--- /dev/null
+++ b/highwayhash/os_mac.cc
@@ -0,0 +1,45 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Created by Alexander Gryanko on 16/09/2017.
+
+#include "os_mac.h"
+
+int mac_getaffinity(cpu_set_t* set) {
+ int64_t core_count = 0;
+ size_t core_count_size = sizeof(core_count); // size is a pointer
+ const int err = sysctlbyname(SYSCTL_CORE_COUNT, &core_count,
+ &core_count_size, NULL, 0);
+ if (err != 0)
+ return err;
+
+ CPU_ZERO(set);
+ for (int64_t i = 0; i < core_count; ++i) {
+ CPU_SET(i, set);
+ }
+
+ return 0;
+}
+
+int mac_setaffinity(cpu_set_t* set) {
+ thread_port_t thread = pthread_mach_thread_np(pthread_self());
+
+ uint16_t current_core;
+ for (current_core = 0; current_core < NR_CPUS; ++current_core) {
+ if (CPU_ISSET(current_core, set)) break;
+ }
+ thread_affinity_policy_data_t policy = { current_core };
+ return thread_policy_set(thread, THREAD_AFFINITY_POLICY,
+ (thread_policy_t)&policy, 1);
+}
diff --git a/highwayhash/os_mac.h b/highwayhash/os_mac.h
new file mode 100644
index 0000000..379c500
--- /dev/null
+++ b/highwayhash/os_mac.h
@@ -0,0 +1,62 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Created by Alexander Gryanko on 16/09/2017.
+
+#ifndef HIGHWAYHASH_OS_MAC_H_
+#define HIGHWAYHASH_OS_MAC_H_
+
+#ifndef HH_DISABLE_TARGET_SPECIFIC
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <sys/sysctl.h>
+
+#include <mach/mach_types.h>
+#include <mach/thread_act.h>
+#include <pthread.h>
+
+typedef unsigned long int cpu_mask;
+
+#define SYSCTL_CORE_COUNT "machdep.cpu.thread_count"
+#define NR_CPUS 512 // from the linux kernel limit
+#define NR_CPUBITS (8 * sizeof(cpu_mask))
+
+struct cpu_set_t {
+ cpu_mask bits[NR_CPUS / NR_CPUBITS];
+};
+
+static inline void CPU_ZERO(size_t setsize, cpu_set_t* set) {
+ memset(set, 0, setsize);
+}
+
+static inline int CPU_ISSET(int cpu, const cpu_set_t* set) {
+ if (cpu < NR_CPUS) {
+ return (set->bits[cpu / NR_CPUBITS] & 1L << (cpu % NR_CPUBITS)) != 0;
+ }
+ return 0;
+}
+
+static inline void CPU_SET(int cpu, cpu_set_t* set) {
+ if (cpu < NR_CPUS) {
+ set->bits[cpu / NR_CPUBITS] |= 1L << (cpu % NR_CPUBITS);
+ }
+}
+
+int mac_getaffinity(cpu_set_t* set);
+int mac_setaffinity(cpu_set_t* set);
+
+#endif // !HH_DISABLE_TARGET_SPECIFIC
+#endif // HIGHWAYHASH_OS_MAC_H_
diff --git a/highwayhash/os_specific.cc b/highwayhash/os_specific.cc
index 6e81e13..9cca726 100644
--- a/highwayhash/os_specific.cc
+++ b/highwayhash/os_specific.cc
@@ -40,8 +40,9 @@
#define OS_LINUX 0
#endif
-#ifdef __MACH__
+#if defined(__APPLE__) || defined(__MACH__) // __MACH__ also defined for GNU/Hurd
#define OS_MAC 1
+#include "os_mac.cc"
#include <mach/mach.h>
#include <mach/mach_time.h>
#else
@@ -101,7 +102,7 @@
#elif OS_LINUX
// omit: SCHED_RR and SCHED_FIFO with sched_priority max, max-1 and max/2
// lead to 2-3x runtime and higher variability!
-#elif OS_FREEBSD
+#elif OS_FREEBSD || OS_MAC
#else
#error "port"
#endif
@@ -110,7 +111,7 @@
struct ThreadAffinity {
#if OS_WIN
DWORD_PTR mask;
-#elif OS_LINUX
+#elif OS_LINUX || OS_MAC
cpu_set_t set;
#elif OS_FREEBSD
cpuset_t set;
@@ -134,6 +135,9 @@
const int err = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid,
sizeof(cpuset_t), &affinity->set);
CHECK(err == 0);
+#elif OS_MAC
+ const int err = mac_getaffinity(&affinity->set);
+ CHECK(err == 0);
#endif
return affinity;
}
@@ -165,6 +169,9 @@
const int err = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid,
sizeof(cpuset_t), &affinity->set);
CHECK(err == 0);
+#elif OS_MAC
+ const int err = mac_setaffinity(&affinity->set);
+ CHECK(err == 0);
#else
#error "port"
#endif
@@ -192,6 +199,12 @@
cpus.push_back(cpu);
}
}
+#elif OS_MAC
+ for (int cpu = 0; cpu < sizeof(cpu_set_t) * 8; ++cpu) {
+ if (CPU_ISSET(cpu, &affinity->set)) {
+ cpus.push_back(cpu);
+ }
+ }
#else
#error "port"
#endif
@@ -208,6 +221,9 @@
#elif OS_FREEBSD
CPU_ZERO(&affinity.set);
CPU_SET(cpu, &affinity.set);
+#elif OS_MAC
+ CPU_ZERO(&affinity.set);
+ CPU_SET(cpu, &affinity.set);
#else
#error "port"
#endif
diff --git a/highwayhash/tsc_timer.h b/highwayhash/tsc_timer.h
index 3d969b1..e8119b5 100644
--- a/highwayhash/tsc_timer.h
+++ b/highwayhash/tsc_timer.h
@@ -28,6 +28,8 @@
#include <intrin.h>
#endif
+#include <ctime>
+
namespace highwayhash {
// Start/Stop return absolute timestamps and must be placed immediately before
@@ -114,7 +116,7 @@
// "cc" = flags modified by SHL.
: "rdx", "memory", "cc");
#else
-#error "Port"
+ t = static_cast<uint64_t>(clock());
#endif
return t;
}
@@ -145,7 +147,7 @@
// "cc" = flags modified by SHL.
: "rcx", "rdx", "memory", "cc");
#else
-#error "Port"
+ t = static_cast<uint64_t>(clock());
#endif
return t;
}
diff --git a/highwayhash/vector_neon.h b/highwayhash/vector_neon.h
new file mode 100644
index 0000000..13db961
--- /dev/null
+++ b/highwayhash/vector_neon.h
@@ -0,0 +1,1030 @@
+// Copyright 2016-2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HIGHWAYHASH_VECTOR128_NEON_H_
+#define HIGHWAYHASH_VECTOR128_NEON_H_
+
+// Defines SIMD vector classes ("V2x64U") with overloaded arithmetic operators:
+// const V2x64U masked_sum = (a + b) & m;
+// This is shorter and more readable than compiler intrinsics:
+// const uint64x2_t masked_sum = vandq_u64(vaddq_u64(a, b), m);
+// There is typically no runtime cost for these abstractions.
+//
+// The naming convention is VNxBBT where N is the number of lanes, BB the
+// number of bits per lane and T is the lane type: unsigned integer (U),
+// signed integer (I), or floating-point (F). (Note: Floating point vectors are
+// currently disabled).
+
+// WARNING: this is a "restricted" header because it is included from
+// translation units compiled with different flags. This header and its
+// dependencies must not define any function unless it is static inline and/or
+// within namespace HH_TARGET_NAME. See arch_specific.h for details.
+
+#include "highwayhash/arch_specific.h"
+#include "highwayhash/compiler_specific.h"
+
+// For auto-dependency generation, we need to include all headers but not their
+// contents.
+#ifndef HH_DISABLE_TARGET_SPECIFIC
+#include <arm_neon.h> // NEON
+
+namespace highwayhash {
+// To prevent ODR violations when including this from multiple translation
+// units (TU) that are compiled with different flags, the contents must reside
+// in a namespace whose name is unique to the TU. NOTE: this behavior is
+// incompatible with precompiled modules and requires textual inclusion instead.
+namespace HH_TARGET_NAME {
+
+// Polyfills for ARMv7. ARMv7 lacks a few important instructions which were added
+// in aarch64, so we simulate them with these polyfills.
+#if !defined(__aarch64__) && !defined(__arm64__)
+#ifndef vqtbl1q_u8
+// aarch64 allows a 128-bit lookup table with vtbl. ARMv7 needs to do two
+// lookups for the same effect.
+HH_INLINE uint8x16_t vqtbl1q_u8 (uint8x16_t t, uint8x16_t idx)
+{
+ // Prevents scalarizing by GCC.
+ // NOT PORTABLE TO AARCH64! aarch64 uses two separate vectors instead of packing
+ // them, so this reinterpret_cast would fail!
+ uint8x8x2_t split = *reinterpret_cast<const uint8x8x2_t *>(&t);
+ return vcombine_u8(
+ vtbl2_u8(split, vget_low_u8(idx)),
+ vtbl2_u8(split, vget_high_u8(idx))
+ );
+}
+#endif
+#ifndef vnegq_s64
+// ARMv7 lacks this for some weird reason.
+HH_INLINE int64x2_t vnegq_s64(int64x2_t v)
+{
+ const int64x2_t zero = vdupq_n_u64(0);
+ return vsubq_s64(zero, v);
+}
+#endif
+#ifndef vceqq_u64
+HH_INLINE uint64x2_t vceqq_u64(uint64x2_t v1, uint64x2_t v2)
+{
+ uint32x4_t comparison = vceqq_u32(vreinterpretq_u32_u64(v1), vreinterpretq_u32_u64(v2));
+ return vreinterpretq_u64_u32(vandq_u32(comparison, vrev64q_u32(comparison)));
+}
+#endif // vnegq_s64
+#endif // !__aarch64__ && !__arm64__
+
+// Pseudo-instructions.
+// _mm_storel_epi64
+HH_INLINE void vst1q_low_u64(uint64_t* a, uint64x2_t b)
+{
+ uint64x1_t lo = vget_low_u64(b);
+ vst1_u8(reinterpret_cast<uint8_t*>(a), vreinterpret_u8_u64(lo));
+}
+// _mm_loadl_epi64
+HH_INLINE uint64x2_t vld1q_low_u64(const uint64_t* p)
+{
+ return vcombine_u64(
+ vld1_u64(p),
+ vdup_n_u64(0)
+ );
+}
+// _mm_slli_si128 (almost)
+#define vshlq_n_u128(a, imm) ( \
+ vreinterpretq_u64_u8( \
+ vextq_u8( \
+ vdupq_n_u8(0), \
+ vreinterpretq_u8_u64(a), \
+ 16 - (imm) \
+ ) \
+ ) \
+)
+
+// Adapted from xsimd.
+// arm_neon.h requires literals for their parameters in many
+// functions, such as vshrq_n_u64, and it will complain even when
+// the value is known at compile-time.
+#define EXPAND(...) __VA_ARGS__
+
+#define CASE(op, i, ...) \
+ case i: v_ = op(__VA_ARGS__, i);
+
+#define INTRINSIC_REPEAT_8_0(op, addx, ...) \
+ CASE(EXPAND(op), 1 + addx, __VA_ARGS__); \
+ CASE(EXPAND(op), 2 + addx, __VA_ARGS__); \
+ CASE(EXPAND(op), 3 + addx, __VA_ARGS__); \
+ CASE(EXPAND(op), 4 + addx, __VA_ARGS__); \
+ CASE(EXPAND(op), 5 + addx, __VA_ARGS__); \
+ CASE(EXPAND(op), 6 + addx, __VA_ARGS__); \
+ CASE(EXPAND(op), 7 + addx, __VA_ARGS__);
+
+#define INTRINSIC_REPEAT_8_N(op, addx, ...) \
+ CASE(EXPAND(op), 0 + addx, __VA_ARGS__); \
+ INTRINSIC_REPEAT_8_0(op, addx, __VA_ARGS__);
+
+#define INTRINSIC_REPEAT_8(op, ...) \
+ INTRINSIC_REPEAT_8_0(op, 0, __VA_ARGS__);
+
+#define INTRINSIC_REPEAT_16_0(op, addx,...) \
+ INTRINSIC_REPEAT_8_0(op, 0 + addx, __VA_ARGS__); \
+ INTRINSIC_REPEAT_8_N(op, 8 + addx, __VA_ARGS__);
+
+#define INTRINSIC_REPEAT_16_N(op, addx, ...) \
+ INTRINSIC_REPEAT_8_N(op, 0 + addx, __VA_ARGS__); \
+ INTRINSIC_REPEAT_8_N(op, 8 + addx, __VA_ARGS__);
+
+#define INTRINSIC_EACH_16(op, ...) \
+ INTRINSIC_REPEAT_16_0(op, 0, __VA_ARGS__);
+
+#define INTRINSIC_REPEAT_32_0(op, addx, ...) \
+ INTRINSIC_REPEAT_16_0(op, 0 + addx, __VA_ARGS__); \
+ INTRINSIC_REPEAT_16_N(op, 16 + addx, __VA_ARGS__);
+
+#define INTRINSIC_REPEAT_32_N(op, addx, ...) \
+ INTRINSIC_REPEAT_16_N(op, 0 + addx, __VA_ARGS__); \
+ INTRINSIC_REPEAT_16_N(op, 16 + addx, __VA_ARGS__);
+
+#define INTRINSIC_EACH_32(op, ...) \
+ INTRINSIC_REPEAT_32_0(op, 0, __VA_ARGS__);
+
+#define INTRINSIC_EACH_64(op, ...) \
+ INTRINSIC_REPEAT_32_0(op, 0, __VA_ARGS__); \
+ INTRINSIC_REPEAT_32_N(op, 32, __VA_ARGS__);
+
+// Primary template for 128-bit SSE4.1 vectors; only specializations are used.
+template <typename T>
+class V128 {};
+
+template <>
+class V128<uint8_t> {
+ public:
+ using Intrinsic = uint8x16_t;
+ using T = uint8_t;
+ static constexpr size_t N = 16;
+
+ // Leaves v_ uninitialized - typically used for output parameters.
+ HH_INLINE V128() {}
+
+ // Sets all lanes to the same value.
+ HH_INLINE explicit V128(T i) : v_(vdupq_n_u8(i)) {}
+
+ // Copy from other vector.
+ HH_INLINE explicit V128(const V128& other) : v_(other.v_) {}
+
+ // C-style cast because vector casts are stupid on NEON.
+ template <typename U>
+ HH_INLINE explicit V128(const V128<U>& other) : v_((const uint8x16_t)(other)) {}
+ HH_INLINE V128& operator=(const V128& other) {
+ v_ = other.v_;
+ return *this;
+ }
+
+ // Convert from/to intrinsics.
+ HH_INLINE V128(const Intrinsic& v) : v_(v) {}
+
+ HH_INLINE V128& operator=(const Intrinsic& v) {
+ v_ = v;
+ return *this;
+ }
+ HH_INLINE operator Intrinsic() const { return v_; }
+
+ // There are no greater-than comparison instructions for unsigned T.
+ HH_INLINE V128 operator==(const V128& other) const {
+ return V128(vceqq_u8(v_, other.v_));
+ }
+
+ HH_INLINE V128& operator+=(const V128& other) {
+ v_ = vaddq_u8(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator-=(const V128& other) {
+ v_ = vsubq_u8(v_, other.v_);
+ return *this;
+ }
+
+ HH_INLINE V128& operator&=(const V128& other) {
+ v_ = vandq_u8(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator|=(const V128& other) {
+ v_ = vorrq_u8(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator^=(const V128& other) {
+ v_ = veorq_u8(v_, other.v_);
+ return *this;
+ }
+
+ HH_INLINE V128& AndNot(const Intrinsic &neg_mask) {
+ v_ = vbicq_u8(v_, neg_mask);
+ return *this;
+ }
+ private:
+ Intrinsic v_;
+};
+
+template <>
+class V128<uint16_t> {
+ public:
+ using Intrinsic = uint16x8_t;
+ using T = uint16_t;
+ static constexpr size_t N = 8;
+
+ // Leaves v_ uninitialized - typically used for output parameters.
+ HH_INLINE V128() {}
+
+ // Lane 0 (p_0) is the lowest.
+ HH_INLINE V128(T p_7, T p_6, T p_5, T p_4, T p_3, T p_2, T p_1, T p_0) {
+ alignas(16) const uint16_t data[8] = {
+ p_0, p_1, p_2, p_3, p_4, p_5, p_6, p_7
+ };
+ v_ = vld1q_u16(data);
+ }
+
+ // Broadcasts i to all lanes (usually by loading from memory).
+ HH_INLINE explicit V128(T i) : v_(vdupq_n_u16(i)) {}
+
+ // Copy from other vector.
+ HH_INLINE explicit V128(const V128& other) : v_(other.v_) {}
+ template <typename U>
+ HH_INLINE explicit V128(const V128<U>& other) : v_(other) {}
+ HH_INLINE V128& operator=(const V128& other) {
+ v_ = other.v_;
+ return *this;
+ }
+
+ // Convert from/to intrinsics.
+ HH_INLINE V128(const Intrinsic& v) : v_(v) {}
+ HH_INLINE V128& operator=(const Intrinsic& v) {
+ v_ = v;
+ return *this;
+ }
+ HH_INLINE operator Intrinsic() const { return v_; }
+
+ // There are no greater-than comparison instructions for unsigned T.
+ HH_INLINE V128 operator==(const V128& other) const {
+ return V128(vceqq_u16(v_, other.v_));
+ }
+
+ HH_INLINE V128& operator+=(const V128& other) {
+ v_ = vaddq_u16(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator-=(const V128& other) {
+ v_ = vsubq_u16(v_, other.v_);
+ return *this;
+ }
+
+ HH_INLINE V128& operator&=(const V128& other) {
+ v_ = vandq_u16(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator|=(const V128& other) {
+ v_ = vorrq_u16(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator^=(const V128& other) {
+ v_ = veorq_u16(v_, other.v_);
+ return *this;
+ }
+
+ HH_INLINE V128& operator<<=(const int count) {
+ switch (count) {
+ INTRINSIC_EACH_16(vshlq_n_u16, v_)
+ }
+ return *this;
+ }
+ HH_INLINE V128& operator<<=(const Intrinsic& count) {
+ v_ = vshlq_u16(v_, vreinterpretq_s16_u16(count));
+ return *this;
+ }
+
+ HH_INLINE V128& operator>>=(const int count) {
+ switch (count) {
+ INTRINSIC_EACH_16(vshrq_n_u16, v_)
+ }
+ return *this;
+ }
+ HH_INLINE V128& operator>>=(const Intrinsic& count) {
+ v_ = vshlq_u16(v_, vnegq_s16(vreinterpretq_s16_u16(count)));
+ return *this;
+ }
+
+ HH_INLINE V128& ShiftRightInsert(const Intrinsic &value, const int count) {
+ switch (count) {
+ INTRINSIC_EACH_16(vsriq_n_u16, v_, value)
+ }
+ return *this;
+ }
+
+ HH_INLINE V128& AndNot(const Intrinsic &neg_mask) {
+ v_ = vbicq_u16(v_, neg_mask);
+ return *this;
+ }
+ private:
+ Intrinsic v_;
+};
+
+template <>
+class V128<uint32_t> {
+ public:
+ using Intrinsic = uint32x4_t;
+ using T = uint32_t;
+ static constexpr size_t N = 4;
+
+ // Leaves v_ uninitialized - typically used for output parameters.
+ HH_INLINE V128() {}
+
+ // Lane 0 (p_0) is the lowest.
+ HH_INLINE V128(T p_3, T p_2, T p_1, T p_0) {
+ alignas(16) const T data[4] = {
+ p_0, p_1, p_2, p_3
+ };
+ v_ = vld1q_u32(data);
+ }
+
+ // Broadcasts i to all lanes (usually by loading from memory).
+ HH_INLINE explicit V128(T i) : v_(vdupq_n_u32(i)) {}
+
+ // Copy from other vector.
+ HH_INLINE explicit V128(const V128& other) : v_(other.v_) {}
+ template <typename U>
+ HH_INLINE explicit V128(const V128<U>& other) : v_((const uint32x4_t)other.v_) {}
+ HH_INLINE V128& operator=(const V128& other) {
+ v_ = other.v_;
+ return *this;
+ }
+
+ // Convert from/to intrinsics.
+ HH_INLINE V128(const Intrinsic& v) : v_(v) {}
+ HH_INLINE V128& operator=(const Intrinsic& v) {
+ v_ = v;
+ return *this;
+ }
+ HH_INLINE operator Intrinsic() const { return v_; }
+
+ // There are no greater-than comparison instructions for unsigned T.
+ HH_INLINE V128 operator==(const V128& other) const {
+ return V128(vceqq_u32(v_, other.v_));
+ }
+
+ HH_INLINE V128& operator+=(const V128& other) {
+ v_ = vaddq_u32(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator-=(const V128& other) {
+ v_ = vsubq_u32(v_, other.v_);
+ return *this;
+ }
+
+ HH_INLINE V128& operator&=(const V128& other) {
+ v_ = vandq_u32(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator|=(const V128& other) {
+ v_ = vorrq_u32(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator^=(const V128& other) {
+ v_ = veorq_u32(v_, other.v_);
+ return *this;
+ }
+
+ HH_INLINE V128& operator<<=(const int count) {
+ switch (count) {
+ INTRINSIC_EACH_32(vshlq_n_u32, v_)
+ }
+ return *this;
+ }
+ HH_INLINE V128& operator<<=(const Intrinsic& count) {
+ v_ = vshlq_u32(v_, vreinterpretq_s32_u32(count));
+ return *this;
+ }
+
+ HH_INLINE V128& operator>>=(const int count) {
+ switch (count) {
+ INTRINSIC_EACH_32(vshrq_n_u32, v_)
+ }
+ return *this;
+ }
+ HH_INLINE V128& operator>>=(const Intrinsic& count) {
+ v_ = vshlq_u32(v_, vnegq_s32(vreinterpretq_s32_u32(count)));
+ return *this;
+ }
+
+ HH_INLINE V128& ShiftRightInsert(const Intrinsic &value, const int count) {
+ switch (count) {
+ INTRINSIC_EACH_32(vsriq_n_u32, v_, value)
+ }
+ return *this;
+ }
+
+ HH_INLINE V128& AndNot(const Intrinsic &neg_mask) {
+ v_ = vbicq_u32(v_, neg_mask);
+ return *this;
+ }
+ private:
+ Intrinsic v_;
+};
+
+template <>
+class V128<uint64_t> {
+ public:
+ using Intrinsic = uint64x2_t;
+ using T = uint64_t;
+ static constexpr size_t N = 2;
+
+ // Leaves v_ uninitialized - typically used for output parameters.
+ HH_INLINE V128() {}
+
+ // Lane 0 (p_0) is the lowest.
+ HH_INLINE V128(T p_1, T p_0) {
+ alignas(16) const T data[2] = {
+ p_0, p_1
+ };
+ v_ = vld1q_u64(data);
+ }
+
+ // Broadcasts i to all lanes (usually by loading from memory).
+ HH_INLINE explicit V128(T i) : v_(vdupq_n_u64(i)) {}
+
+ // Copy from other vector.
+ HH_INLINE explicit V128(const V128& other) : v_(other.v_) {}
+ template <typename U>
+ HH_INLINE explicit V128(const V128<U>& other) : v_(other) {}
+ HH_INLINE V128& operator=(const V128& other) {
+ v_ = other.v_;
+ return *this;
+ }
+
+ // Convert from/to intrinsics.
+ HH_INLINE V128(const Intrinsic& v) : v_(v) {}
+ HH_INLINE V128& operator=(const Intrinsic& v) {
+ v_ = v;
+ return *this;
+ }
+ HH_INLINE operator Intrinsic() const { return v_; }
+
+ // There are no greater-than comparison instructions for unsigned T.
+ HH_INLINE V128 operator==(const V128& other) const {
+ return V128(vceqq_u64(v_, other.v_));
+ }
+
+ HH_INLINE V128& operator+=(const V128& other) {
+ v_ = vaddq_u64(v_, other.v_);
+ // Prevent Clang from converting to vaddhn when nearby vmovn, which
+ // causes four spills in the main loop on ARMv7a.
+#ifdef __GNUC__
+ __asm__("" : "+w" (v_));
+#endif
+ return *this;
+ }
+ HH_INLINE V128& operator-=(const V128& other) {
+ v_ = vsubq_u64(v_, other.v_);
+ return *this;
+ }
+
+ HH_INLINE V128& operator&=(const V128& other) {
+ v_ = vandq_u64(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator|=(const V128& other) {
+ v_ = vorrq_u64(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator^=(const V128& other) {
+ v_ = veorq_u64(v_, other.v_);
+ return *this;
+ }
+
+ HH_INLINE V128& operator<<=(const int count) {
+ switch (count) {
+ INTRINSIC_EACH_64(vshlq_n_u64, v_)
+ }
+ return *this;
+ }
+ HH_INLINE V128& operator<<=(const Intrinsic& count) {
+ v_ = vshlq_u64(v_, vreinterpretq_s64_u64(count));
+ return *this;
+ }
+
+ HH_INLINE V128& operator>>=(const int count) {
+ switch (count) {
+ INTRINSIC_EACH_64(vshrq_n_u64, v_)
+ }
+ return *this;
+ }
+ HH_INLINE V128& operator>>=(const Intrinsic& count) {
+ v_ = vshlq_u64(v_, vnegq_s64(vreinterpretq_s64_u64(count)));
+ return *this;
+ }
+
+ HH_INLINE V128& AndNot(const Intrinsic &neg_mask) {
+ v_ = vbicq_u64(v_, neg_mask);
+ return *this;
+ }
+
+ HH_INLINE V128& ShiftRightInsert(const Intrinsic &value, const int count) {
+ switch (count) {
+ INTRINSIC_EACH_64(vsriq_n_u64, v_, value)
+ }
+ return *this;
+ }
+
+ private:
+ Intrinsic v_;
+};
+
+// TODO: Enable. For now, this is disabled for the following reasons:
+// 1. ARMv7a lacks float64x2_t.
+// 2. ARMv7a's float32x4_t is not IEE-754 compliant
+// 3. We don't actually use the float vectors right now.
+#if 0
+template <>
+class V128<float> {
+ public:
+ using Intrinsic = float32x4_t;
+ using T = float;
+ static constexpr size_t N = 4;
+
+ // Leaves v_ uninitialized - typically used for output parameters.
+ HH_INLINE V128() {}
+
+ // Lane 0 (p_0) is the lowest.
+ HH_INLINE V128(T p_3, T p_2, T p_1, T p_0) {
+ HH_ALIGNAS(16) float tmp[4] = { p_0, p_1, p_2, p_3 };
+ v_ = vld1q_f32(tmp);
+ }
+
+ // Broadcasts to all lanes.
+ HH_INLINE explicit V128(T f) : v_(vdupq_n_f32(f)) {}
+
+ // Copy from other vector.
+ HH_INLINE explicit V128(const V128& other) : v_(other.v_) {}
+ template <typename U>
+ HH_INLINE explicit V128(const V128<U>& other) : v_(other) {}
+ HH_INLINE V128& operator=(const V128& other) {
+ v_ = other.v_;
+ return *this;
+ }
+
+ // Convert from/to intrinsics.
+ HH_INLINE V128(const Intrinsic& v) : v_(v) {}
+ HH_INLINE V128& operator=(const Intrinsic& v) {
+ v_ = v;
+ return *this;
+ }
+ HH_INLINE operator Intrinsic() const { return v_; }
+
+ HH_INLINE V128 operator==(const V128& other) const {
+ return V128(vceqq_f32(v_, other.v_));
+ }
+ HH_INLINE V128 operator<(const V128& other) const {
+ return V128(vcltq_f32(v_, other.v_));
+ }
+ HH_INLINE V128 operator>(const V128& other) const {
+ return V128(vcltq_f32(other.v_, v_));
+ }
+
+ HH_INLINE V128& operator*=(const V128& other) {
+ v_ = vmulq_f32(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator/=(const V128& other) {
+ v_ = vdivq_f32(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator+=(const V128& other) {
+ v_ = vaddq_f32(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator-=(const V128& other) {
+ v_ = vsubq_f32(v_, other.v_);
+ return *this;
+ }
+
+ HH_INLINE V128& operator&=(const V128& other) {
+ v_ = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(v_), vreinterpretq_u32_f32(other.v_)));
+ return *this;
+ }
+ HH_INLINE V128& operator|=(const V128& other) {
+ v_ = vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(v_), vreinterpretq_u32_f32(other.v_)));
+ return *this;
+ }
+ HH_INLINE V128& operator^=(const V128& other) {
+ v_ = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(v_), vreinterpretq_u32_f32(other.v_)));
+ return *this;
+ }
+
+ private:
+ Intrinsic v_;
+};
+
+template <>
+class V128<double> {
+ public:
+ using Intrinsic = float64x2_t;
+ using T = double;
+ static constexpr size_t N = 2;
+
+ // Leaves v_ uninitialized - typically used for output parameters.
+ HH_INLINE V128() {}
+
+ // Lane 0 (p_0) is the lowest.
+ HH_INLINE V128(T p_1, T p_0) {
+ HH_ALIGNAS(16) double tmp[2] = { p_0, p_1 };
+ v_ = vld1q_f64(tmp);
+ }
+
+ // Broadcasts to all lanes.
+ HH_INLINE explicit V128(T f) : v_(vdupq_n_f64(f)) {}
+
+ // Copy from other vector.
+ HH_INLINE explicit V128(const V128& other) : v_(other.v_) {}
+ template <typename U>
+ HH_INLINE explicit V128(const V128<U>& other) : v_(other) {}
+ HH_INLINE V128& operator=(const V128& other) {
+ v_ = other.v_;
+ return *this;
+ }
+
+ // Convert from/to intrinsics.
+ HH_INLINE V128(const Intrinsic& v) : v_(v) {}
+ HH_INLINE V128& operator=(const Intrinsic& v) {
+ v_ = v;
+ return *this;
+ }
+ HH_INLINE operator Intrinsic() const { return v_; }
+
+ HH_INLINE V128 operator==(const V128& other) const {
+ return V128(vceqq_f64(v_, other.v_));
+ }
+ HH_INLINE V128 operator<(const V128& other) const {
+ return V128(vcltq_f64(v_, other.v_));
+ }
+ HH_INLINE V128 operator>(const V128& other) const {
+ return V128(vcltq_f64(other.v_, v_));
+ }
+
+ HH_INLINE V128& operator*=(const V128& other) {
+ v_ = vmulq_f64(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator/=(const V128& other) {
+ v_ = vdivq_f64(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator+=(const V128& other) {
+ v_ = vaddq_f64(v_, other.v_);
+ return *this;
+ }
+ HH_INLINE V128& operator-=(const V128& other) {
+ v_ = vsubq_f64(v_, other.v_);
+ return *this;
+ }
+
+ HH_INLINE V128& operator&=(const V128& other) {
+ v_ = vreinterpretq_f64_u64(vandq_u64(vreinterpretq_u64_f64(v_), vreinterpretq_u64_f64(other.v_)));
+ return *this;
+ }
+ HH_INLINE V128& operator|=(const V128& other) {
+ v_ = vreinterpretq_f64_u64(vorrq_u64(vreinterpretq_u64_f64(v_), vreinterpretq_u64_f64(other.v_)));
+ return *this;
+ }
+ HH_INLINE V128& operator^=(const V128& other) {
+ v_ = vreinterpretq_f64_u64(veorq_u64(vreinterpretq_u64_f64(v_), vreinterpretq_u64_f64(other.v_)));
+ return *this;
+ }
+
+ private:
+ Intrinsic v_;
+};
+#endif
+
+// Nonmember functions for any V128 via member functions.
+
+template <typename T>
+HH_INLINE V128<T> operator*(const V128<T>& left, const V128<T>& right) {
+ V128<T> t(left);
+ return t *= right;
+}
+
+template <typename T>
+HH_INLINE V128<T> operator/(const V128<T>& left, const V128<T>& right) {
+ V128<T> t(left);
+ return t /= right;
+}
+
+template <typename T>
+HH_INLINE V128<T> operator+(const V128<T>& left, const V128<T>& right) {
+ V128<T> t(left);
+ return t += right;
+}
+
+template <typename T>
+HH_INLINE V128<T> operator-(const V128<T>& left, const V128<T>& right) {
+ V128<T> t(left);
+ return t -= right;
+}
+
+template <typename T>
+HH_INLINE V128<T> operator&(const V128<T>& left, const V128<T>& right) {
+ V128<T> t(left);
+ return t &= right;
+}
+
+template <typename T>
+HH_INLINE V128<T> operator|(const V128<T>& left, const V128<T>& right) {
+ V128<T> t(left);
+ return t |= right;
+}
+
+template <typename T>
+HH_INLINE V128<T> operator^(const V128<T>& left, const V128<T>& right) {
+ V128<T> t(left);
+ return t ^= right;
+}
+
+template <typename T>
+HH_INLINE V128<T> operator<<(const V128<T>& v, const int count) {
+ V128<T> t(v);
+ return t <<= count;
+}
+
+template <typename T>
+HH_INLINE V128<T> operator>>(const V128<T>& v, const int count) {
+ V128<T> t(v);
+ return t >>= count;
+}
+
+using V16x8U = V128<uint8_t>;
+using V8x16U = V128<uint16_t>;
+using V4x32U = V128<uint32_t>;
+using V2x64U = V128<uint64_t>;
+using V4x32F = V128<float>;
+using V2x64F = V128<double>;
+
+// Load/Store for any V128.
+
+// We differentiate between targets' vector types via template specialization.
+// Calling Load<V>(floats) is more natural than Load(V8x32F(), floats) and may
+// generate better code in unoptimized builds. Only declare the primary
+// templates to avoid needing mutual exclusion with vector256.
+
+template <class V>
+HH_INLINE V Load(const typename V::T* const HH_RESTRICT from);
+
+template <class V>
+HH_INLINE V LoadUnaligned(const typename V::T* const HH_RESTRICT from);
+
+// "from" must be vector-aligned.
+#ifdef __GNUC__
+#define HH_ALIGN(x) __builtin_assume_aligned((x), 16)
+#else
+#define HH_ALIGN(x) (x)
+#endif
+template <>
+HH_INLINE V16x8U Load<V16x8U>(const V16x8U::T* const HH_RESTRICT from) {
+ const uint8_t* const HH_RESTRICT p = reinterpret_cast<const uint8_t*>(HH_ALIGN(from));
+ return V16x8U(vld1q_u8(p));
+}
+template <>
+HH_INLINE V8x16U Load<V8x16U>(const V8x16U::T* const HH_RESTRICT from) {
+ const uint16_t* const HH_RESTRICT p = reinterpret_cast<const uint16_t*>(HH_ALIGN(from));
+ return V8x16U(vld1q_u16(p));
+}
+template <>
+HH_INLINE V4x32U Load<V4x32U>(const V4x32U::T* const HH_RESTRICT from) {
+ const uint32_t* const HH_RESTRICT p = reinterpret_cast<const uint32_t*>(HH_ALIGN(from));
+ return V4x32U(vld1q_u32(p));
+}
+template <>
+HH_INLINE V2x64U Load<V2x64U>(const V2x64U::T* const HH_RESTRICT from) {
+ const uint64_t* const HH_RESTRICT p = reinterpret_cast<const uint64_t*>(HH_ALIGN(from));
+ return V2x64U(vld1q_u64(p));
+}
+#if 0
+template <>
+HH_INLINE V4x32F Load<V4x32F>(const V4x32F::T* const HH_RESTRICT from) {
+ return V4x32F(vld1q_f32(from));
+}
+template <>
+HH_INLINE V2x64F Load<V2x64F>(const V2x64F::T* const HH_RESTRICT from) {
+ return V2x64F(vld1q_f64(from));
+}
+#endif
+// GCC for ARM 32-bit flips out on unaligned reads after a cast.
+// Only vld1q_u8 is safe on unaligned pointers.
+template <>
+HH_INLINE V16x8U
+LoadUnaligned<V16x8U>(const V16x8U::T* const HH_RESTRICT from) {
+ const uint8_t* const HH_RESTRICT p = reinterpret_cast<const uint8_t*>(from);
+ return V16x8U(vld1q_u8(p));
+}
+template <>
+HH_INLINE V8x16U
+LoadUnaligned<V8x16U>(const V8x16U::T* const HH_RESTRICT from) {
+ const uint8_t* const HH_RESTRICT p = reinterpret_cast<const uint8_t*>(from);
+ return V8x16U(vreinterpretq_u16_u8(vld1q_u8(p)));
+}
+template <>
+HH_INLINE V4x32U
+LoadUnaligned<V4x32U>(const V4x32U::T* const HH_RESTRICT from) {
+ const uint8_t* const HH_RESTRICT p = reinterpret_cast<const uint8_t*>(from);
+ return V4x32U(vreinterpretq_u32_u8(vld1q_u8(p)));
+}
+template <>
+HH_INLINE V2x64U
+LoadUnaligned<V2x64U>(const V2x64U::T* const HH_RESTRICT from) {
+ const uint8_t* const HH_RESTRICT p = reinterpret_cast<const uint8_t*>(from);
+ return V2x64U(vreinterpretq_u64_u8(vld1q_u8(p)));
+}
+#if 0
+template <>
+HH_INLINE V4x32F
+LoadUnaligned<V4x32F>(const V4x32F::T* const HH_RESTRICT from) {
+ return V4x32F(vld1q_f32(from));
+}
+template <>
+HH_INLINE V2x64F
+LoadUnaligned<V2x64F>(const V2x64F::T* const HH_RESTRICT from) {
+ return V2x64F(vld1q_f64(from));
+}
+#endif
+
+// "to" must be vector-aligned.
+template <typename T>
+HH_INLINE void Store(const V128<T>& v, T* const HH_RESTRICT to);
+
+template<>
+HH_INLINE void Store<uint8_t>(const V128<uint8_t>& v, uint8_t* HH_RESTRICT to) {
+ uint8_t *const HH_RESTRICT p = reinterpret_cast<uint8_t *>(HH_ALIGN(to));
+ vst1q_u8(p, v);
+}
+
+template<>
+HH_INLINE void Store<uint16_t>(const V128<uint16_t>& v, uint16_t* const HH_RESTRICT to) {
+ uint16_t *const HH_RESTRICT p = reinterpret_cast<uint16_t *>(HH_ALIGN(to));
+ vst1q_u16(p, v);
+}
+
+template<>
+HH_INLINE void Store<uint32_t>(const V128<uint32_t>& v, uint32_t* const HH_RESTRICT to) {
+ uint32_t *const HH_RESTRICT p = reinterpret_cast<uint32_t *>(HH_ALIGN(to));
+ vst1q_u32(p, v);
+}
+
+#undef HH_ALIGN
+template<>
+HH_INLINE void Store<uint64_t>(const V128<uint64_t>& v, uint64_t* const HH_RESTRICT to) {
+ uint64_t *const HH_RESTRICT p = reinterpret_cast<uint64_t *>(HH_ALIGN(to));
+ vst1q_u64(p, v);
+}
+#if 0
+HH_INLINE void Store(const V128<float>& v, float* const HH_RESTRICT to) {
+ vst1q_f32(to, v);
+}
+HH_INLINE void Store(const V128<double>& v, double* const HH_RESTRICT to) {
+ vst1q_f64(to, v);
+}
+#endif
+template <typename T>
+HH_INLINE void StoreUnaligned(const V128<T>& v, T* const HH_RESTRICT to);
+
+template<>
+HH_INLINE void StoreUnaligned<uint8_t>(const V128<uint8_t>& v, uint8_t* const HH_RESTRICT to) {
+ vst1q_u8(to, v);
+}
+
+template<>
+HH_INLINE void StoreUnaligned<uint16_t>(const V128<uint16_t>& v, uint16_t* const HH_RESTRICT to) {
+ vst1q_u8(reinterpret_cast<uint8_t *>(to), vreinterpretq_u8_u16(v));
+}
+
+template<>
+HH_INLINE void StoreUnaligned<uint32_t>(const V128<uint32_t>& v, uint32_t* const HH_RESTRICT to) {
+ vst1q_u8(reinterpret_cast<uint8_t *>(to), vreinterpretq_u8_u32(v));
+}
+
+template<>
+HH_INLINE void StoreUnaligned<uint64_t>(const V128<uint64_t>& v, uint64_t* const HH_RESTRICT to) {
+ vst1q_u8(reinterpret_cast<uint8_t *>(to), vreinterpretq_u8_u64(v));
+}
+#if 0
+HH_INLINE void StoreUnaligned(const V128<float>& v,
+ float* const HH_RESTRICT to) {
+ _mm_storeu_ps(to, v);
+}
+HH_INLINE void StoreUnaligned(const V128<double>& v,
+ double* const HH_RESTRICT to) {
+ _mm_storeu_pd(to, v);
+}
+#endif
+
+// TODO: Enable.
+#if 0
+// Writes directly to (aligned) memory, bypassing the cache. This is useful for
+// data that will not be read again in the near future.
+template <typename T>
+HH_INLINE void Stream(const V128<T>& v, T* const HH_RESTRICT to) {
+ _mm_stream_si128(reinterpret_cast<__m128i * HH_RESTRICT>(to), v);
+}
+HH_INLINE void Stream(const V128<float>& v, float* const HH_RESTRICT to) {
+ _mm_stream_ps(to, v);
+}
+HH_INLINE void Stream(const V128<double>& v, double* const HH_RESTRICT to) {
+ _mm_stream_pd(to, v);
+}
+#endif
+
+// Miscellaneous functions.
+
+template <typename T>
+HH_INLINE V128<T> RotateLeft(const V128<T>& v, const int count) {
+ const size_t num_bits = sizeof(T) * 8;
+ const V128<T>& tmp = v << count;
+ return tmp.ShiftRightAccumulate(v, num_bits - count);
+}
+
+template <typename T>
+HH_INLINE V128<T> AndNot(const V128<T>& neg_mask, const V128<T>& values) {
+ V128<T> tmp = values;
+ return tmp.AndNot(neg_mask);
+}
+#if 0
+template <>
+HH_INLINE V128<float> AndNot(const V128<float>& neg_mask,
+ const V128<float>& values) {
+ return V128<float>(_mm_andnot_ps(neg_mask, values));
+}
+template <>
+HH_INLINE V128<double> AndNot(const V128<double>& neg_mask,
+ const V128<double>& values) {
+ return V128<double>(_mm_andnot_pd(neg_mask, values));
+}
+#endif
+HH_INLINE V4x32U Select(const V4x32U& a, const V4x32U& b, const V4x32U& mask) {
+ return V4x32U(vbslq_u32(mask, a, b));
+}
+
+HH_INLINE V2x64U Select(const V2x64U& a, const V2x64U& b, const V2x64U& mask) {
+ return V2x64U(vbslq_u64(mask, a, b));
+}
+
+// Min/Max
+
+HH_INLINE V16x8U Min(const V16x8U& v0, const V16x8U& v1) {
+ return V16x8U(vminq_u8(v0, v1));
+}
+
+HH_INLINE V16x8U Max(const V16x8U& v0, const V16x8U& v1) {
+ return V16x8U(vmaxq_u8(v0, v1));
+}
+
+HH_INLINE V8x16U Min(const V8x16U& v0, const V8x16U& v1) {
+ return V8x16U(vminq_u16(v0, v1));
+}
+
+HH_INLINE V8x16U Max(const V8x16U& v0, const V8x16U& v1) {
+ return V8x16U(vmaxq_u16(v0, v1));
+}
+
+HH_INLINE V4x32U Min(const V4x32U& v0, const V4x32U& v1) {
+ return V4x32U(vminq_u32(v0, v1));
+}
+
+HH_INLINE V4x32U Max(const V4x32U& v0, const V4x32U& v1) {
+ return V4x32U(vmaxq_u32(v0, v1));
+}
+
+
+#if 0
+HH_INLINE V4x32F Min(const V4x32F& v0, const V4x32F& v1) {
+ return V4x32F(vminq_f32(v0, v1));
+}
+
+HH_INLINE V4x32F Max(const V4x32F& v0, const V4x32F& v1) {
+ return V4x32F(vmaxq_f32(v0, v1));
+}
+
+HH_INLINE V2x64F Min(const V2x64F& v0, const V2x64F& v1) {
+ return V2x64F(vminq_f64(v0, v1));
+}
+
+HH_INLINE V2x64F Max(const V2x64F& v0, const V2x64F& v1) {
+ return V2x64F(vmaxq_f64(v0, v1));
+}
+#endif
+
+} // namespace HH_TARGET_NAME
+} // namespace highwayhash
+
+#endif // HH_DISABLE_TARGET_SPECIFIC
+#endif // HIGHWAYHASH_VECTOR128_H_
diff --git a/highwayhash/vector_test_neon.cc b/highwayhash/vector_test_neon.cc
new file mode 100644
index 0000000..4f4d9a5
--- /dev/null
+++ b/highwayhash/vector_test_neon.cc
@@ -0,0 +1,19 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// WARNING: this is a "restricted" source file; avoid including any headers
+// unless they are also restricted. See arch_specific.h for details.
+
+#define HH_TARGET_NAME NEON
+#include "highwayhash/vector_test_target.cc"
diff --git a/highwayhash/vector_test_target.cc b/highwayhash/vector_test_target.cc
index 9b3afdb..90d9741 100644
--- a/highwayhash/vector_test_target.cc
+++ b/highwayhash/vector_test_target.cc
@@ -24,6 +24,8 @@
#include "highwayhash/vector256.h"
#elif HH_TARGET == HH_TARGET_SSE41
#include "highwayhash/vector128.h"
+#elif HH_TARGET == HH_TARGET_NEON
+#include "highwayhash/vector_neon.h"
#elif HH_TARGET == HH_TARGET_Portable
#include "highwayhash/scalar.h"
#else
@@ -37,7 +39,7 @@
#if HH_TARGET == HH_TARGET_AVX2
template <typename T>
using V = V256<T>;
-#elif HH_TARGET == HH_TARGET_SSE41
+#elif HH_TARGET == HH_TARGET_SSE41 || HH_TARGET == HH_TARGET_NEON
template <typename T>
using V = V128<T>;
#elif HH_TARGET == HH_TARGET_Portable
