azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Change how common polyhash function are included

Browse Source 
Recent versions of LLVM complain when caller and callee have
different vector attributes: https://reviews.llvm.org/D82562.

To mitigate this problem this commit changes how we include
polyHashUnroll* and polyHashTail into Traits: we duplicate them
inside each trait with correct attributes by using
```
#pragma clang attribute push(ATTRIBUTE, apply_to = function)
```
This solution is not pretty for sure, but Clang is not complaining
anymore.
This commit is contained in:
Sergey Bogolepov
2021-05-28 12:46:23 +07:00
committed by Space
parent 493fa1c1e3
commit 601465fa81
4 changed files with 191 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/runtime/src/main/cpp/polyhash/arm.cpp
+2
View File
@@ -55,6 +55,8 @@ struct NeonTraits {
#endif #endif
}; };
#include "polyhash/attributeSensitiveFunctions.inc"
static int polyHashUnalignedUnrollUpTo16(int n, uint16_t const* str) { static int polyHashUnalignedUnrollUpTo16(int n, uint16_t const* str) {
Vec128Type res = initVec128(); Vec128Type res = initVec128();
kotlin-native/runtime/src/main/cpp/polyhash/attributeSensitiveFunctions.inc
+152
View File
@@ -0,0 +1,152 @@
// These functions should be included inside Traits with correct attributes (e.g. via #pragma clang attribute push).
template<typename Traits>
ALWAYS_INLINE static void polyHashTail(int& n, uint16_t const*& str, typename Traits::Vec128Type& res, uint32_t const* b, uint32_t const* p) {
using VecType = typename Traits::VecType;
using Vec128Type = typename Traits::Vec128Type;
using U16VecType = typename Traits::U16VecType;
const int vecLength = sizeof(VecType) / 4;
if (n < vecLength / 4) return;
VecType x = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str));
res = Traits::vec128Mul(res, *reinterpret_cast<Vec128Type const*>(b));
VecType z = Traits::vecMul(x, *reinterpret_cast<VecType const*>(p));
res = Traits::vec128Add(res, Traits::squash1(z));
str += vecLength;
n -= vecLength / 4;
}
template<typename Traits>
ALWAYS_INLINE static void polyHashUnroll2(int& n, uint16_t const*& str, typename Traits::Vec128Type& res, uint32_t const* b, uint32_t const* p) {
using VecType = typename Traits::VecType;
using Vec128Type = typename Traits::Vec128Type;
using U16VecType = typename Traits::U16VecType;
const int vecLength = sizeof(VecType) / 4;
if (n < vecLength / 2) return;
res = Traits::vec128Mul(res, *reinterpret_cast<Vec128Type const*>(b));
VecType res0 = Traits::initVec();
VecType res1 = Traits::initVec();
do {
VecType x0 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str));
VecType x1 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength));
res0 = Traits::vecMul(res0, *reinterpret_cast<VecType const*>(b));
res1 = Traits::vecMul(res1, *reinterpret_cast<VecType const*>(b));
VecType z0 = Traits::vecMul(x0, *reinterpret_cast<VecType const*>(p));
VecType z1 = Traits::vecMul(x1, *reinterpret_cast<VecType const*>(p + vecLength));
res0 = Traits::vecAdd(res0, z0);
res1 = Traits::vecAdd(res1, z1);
str += vecLength * 2;
n -= vecLength / 2;
} while (n >= vecLength / 2);
res = Traits::vec128Add(res, Traits::squash2(res0, res1));
}
template<typename Traits>
ALWAYS_INLINE static void polyHashUnroll4(int& n, uint16_t const*& str, typename Traits::Vec128Type& res, uint32_t const* b, uint32_t const* p) {
using VecType = typename Traits::VecType;
using Vec128Type = typename Traits::Vec128Type;
using U16VecType = typename Traits::U16VecType;
const int vecLength = sizeof(VecType) / 4;
if (n < vecLength) return;
res = Traits::vec128Mul(res, *reinterpret_cast<Vec128Type const*>(b));
VecType res0 = Traits::initVec();
VecType res1 = Traits::initVec();
VecType res2 = Traits::initVec();
VecType res3 = Traits::initVec();
do {
VecType x0 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str));
VecType x1 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength));
VecType x2 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 2));
VecType x3 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 3));
res0 = Traits::vecMul(res0, *reinterpret_cast<VecType const*>(b));
res1 = Traits::vecMul(res1, *reinterpret_cast<VecType const*>(b));
res2 = Traits::vecMul(res2, *reinterpret_cast<VecType const*>(b));
res3 = Traits::vecMul(res3, *reinterpret_cast<VecType const*>(b));
VecType z0 = Traits::vecMul(x0, *reinterpret_cast<VecType const*>(p));
VecType z1 = Traits::vecMul(x1, *reinterpret_cast<VecType const*>(p + vecLength));
VecType z2 = Traits::vecMul(x2, *reinterpret_cast<VecType const*>(p + vecLength * 2));
VecType z3 = Traits::vecMul(x3, *reinterpret_cast<VecType const*>(p + vecLength * 3));
res0 = Traits::vecAdd(res0, z0);
res1 = Traits::vecAdd(res1, z1);
res2 = Traits::vecAdd(res2, z2);
res3 = Traits::vecAdd(res3, z3);
str += vecLength * 4;
n -= vecLength;
} while (n >= vecLength);
res = Traits::vec128Add(res, Traits::vec128Add(Traits::squash2(res0, res1), Traits::squash2(res2, res3)));
}
template<typename Traits>
ALWAYS_INLINE static void polyHashUnroll8(int& n, uint16_t const*& str, typename Traits::Vec128Type& res, uint32_t const* b, uint32_t const* p) {
using VecType = typename Traits::VecType;
using Vec128Type = typename Traits::Vec128Type;
using U16VecType = typename Traits::U16VecType;
const int vecLength = sizeof(VecType) / 4;
if (n < vecLength * 2) return;
VecType res0 = Traits::initVec();
VecType res1 = Traits::initVec();
VecType res2 = Traits::initVec();
VecType res3 = Traits::initVec();
VecType res4 = Traits::initVec();
VecType res5 = Traits::initVec();
VecType res6 = Traits::initVec();
VecType res7 = Traits::initVec();
do {
VecType x0 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str));
VecType x1 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength));
VecType x2 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 2));
VecType x3 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 3));
VecType x4 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 4));
VecType x5 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 5));
VecType x6 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 6));
VecType x7 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 7));
res0 = Traits::vecMul(res0, *reinterpret_cast<VecType const*>(b));
res1 = Traits::vecMul(res1, *reinterpret_cast<VecType const*>(b));
res2 = Traits::vecMul(res2, *reinterpret_cast<VecType const*>(b));
res3 = Traits::vecMul(res3, *reinterpret_cast<VecType const*>(b));
res4 = Traits::vecMul(res4, *reinterpret_cast<VecType const*>(b));
res5 = Traits::vecMul(res5, *reinterpret_cast<VecType const*>(b));
res6 = Traits::vecMul(res6, *reinterpret_cast<VecType const*>(b));
res7 = Traits::vecMul(res7, *reinterpret_cast<VecType const*>(b));
VecType z0 = Traits::vecMul(x0, *reinterpret_cast<VecType const*>(p));
VecType z1 = Traits::vecMul(x1, *reinterpret_cast<VecType const*>(p + vecLength));
VecType z2 = Traits::vecMul(x2, *reinterpret_cast<VecType const*>(p + vecLength * 2));
VecType z3 = Traits::vecMul(x3, *reinterpret_cast<VecType const*>(p + vecLength * 3));
VecType z4 = Traits::vecMul(x4, *reinterpret_cast<VecType const*>(p + vecLength * 4));
VecType z5 = Traits::vecMul(x5, *reinterpret_cast<VecType const*>(p + vecLength * 5));
VecType z6 = Traits::vecMul(x6, *reinterpret_cast<VecType const*>(p + vecLength * 6));
VecType z7 = Traits::vecMul(x7, *reinterpret_cast<VecType const*>(p + vecLength * 7));
res0 = Traits::vecAdd(res0, z0);
res1 = Traits::vecAdd(res1, z1);
res2 = Traits::vecAdd(res2, z2);
res3 = Traits::vecAdd(res3, z3);
res4 = Traits::vecAdd(res4, z4);
res5 = Traits::vecAdd(res5, z5);
res6 = Traits::vecAdd(res6, z6);
res7 = Traits::vecAdd(res7, z7);
str += vecLength * 8;
n -= vecLength * 2;
} while (n >= vecLength * 2);
Vec128Type sum1 = Traits::vec128Add(Traits::squash2(res0, res1), Traits::squash2(res2, res3));
Vec128Type sum2 = Traits::vec128Add(Traits::squash2(res4, res5), Traits::squash2(res6, res7));
res = Traits::vec128Add(res, Traits::vec128Add(sum1, sum2));
}
kotlin-native/runtime/src/main/cpp/polyhash/common.h
-151
View File
@@ -39,155 +39,4 @@ constexpr std::array<uint32_t, Count> RepeatingPowers(uint32_t base, uint8_t exp
return result; return result;
} }
template<typename Traits>
ALWAYS_INLINE void polyHashTail(int& n, uint16_t const*& str, typename Traits::Vec128Type& res, uint32_t const* b, uint32_t const* p) {
using VecType = typename Traits::VecType;
using Vec128Type = typename Traits::Vec128Type;
using U16VecType = typename Traits::U16VecType;
const int vecLength = sizeof(VecType) / 4;
if (n < vecLength / 4) return;
VecType x = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str));
res = Traits::vec128Mul(res, *reinterpret_cast<Vec128Type const*>(b));
VecType z = Traits::vecMul(x, *reinterpret_cast<VecType const*>(p));
res = Traits::vec128Add(res, Traits::squash1(z));
str += vecLength;
n -= vecLength / 4;
}
template<typename Traits>
ALWAYS_INLINE void polyHashUnroll2(int& n, uint16_t const*& str, typename Traits::Vec128Type& res, uint32_t const* b, uint32_t const* p) {
using VecType = typename Traits::VecType;
using Vec128Type = typename Traits::Vec128Type;
using U16VecType = typename Traits::U16VecType;
const int vecLength = sizeof(VecType) / 4;
if (n < vecLength / 2) return;
res = Traits::vec128Mul(res, *reinterpret_cast<Vec128Type const*>(b));
VecType res0 = Traits::initVec();
VecType res1 = Traits::initVec();
do {
VecType x0 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str));
VecType x1 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength));
res0 = Traits::vecMul(res0, *reinterpret_cast<VecType const*>(b));
res1 = Traits::vecMul(res1, *reinterpret_cast<VecType const*>(b));
VecType z0 = Traits::vecMul(x0, *reinterpret_cast<VecType const*>(p));
VecType z1 = Traits::vecMul(x1, *reinterpret_cast<VecType const*>(p + vecLength));
res0 = Traits::vecAdd(res0, z0);
res1 = Traits::vecAdd(res1, z1);
str += vecLength * 2;
n -= vecLength / 2;
} while (n >= vecLength / 2);
res = Traits::vec128Add(res, Traits::squash2(res0, res1));
}
template<typename Traits>
ALWAYS_INLINE void polyHashUnroll4(int& n, uint16_t const*& str, typename Traits::Vec128Type& res, uint32_t const* b, uint32_t const* p) {
using VecType = typename Traits::VecType;
using Vec128Type = typename Traits::Vec128Type;
using U16VecType = typename Traits::U16VecType;
const int vecLength = sizeof(VecType) / 4;
if (n < vecLength) return;
res = Traits::vec128Mul(res, *reinterpret_cast<Vec128Type const*>(b));
VecType res0 = Traits::initVec();
VecType res1 = Traits::initVec();
VecType res2 = Traits::initVec();
VecType res3 = Traits::initVec();
do {
VecType x0 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str));
VecType x1 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength));
VecType x2 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 2));
VecType x3 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 3));
res0 = Traits::vecMul(res0, *reinterpret_cast<VecType const*>(b));
res1 = Traits::vecMul(res1, *reinterpret_cast<VecType const*>(b));
res2 = Traits::vecMul(res2, *reinterpret_cast<VecType const*>(b));
res3 = Traits::vecMul(res3, *reinterpret_cast<VecType const*>(b));
VecType z0 = Traits::vecMul(x0, *reinterpret_cast<VecType const*>(p));
VecType z1 = Traits::vecMul(x1, *reinterpret_cast<VecType const*>(p + vecLength));
VecType z2 = Traits::vecMul(x2, *reinterpret_cast<VecType const*>(p + vecLength * 2));
VecType z3 = Traits::vecMul(x3, *reinterpret_cast<VecType const*>(p + vecLength * 3));
res0 = Traits::vecAdd(res0, z0);
res1 = Traits::vecAdd(res1, z1);
res2 = Traits::vecAdd(res2, z2);
res3 = Traits::vecAdd(res3, z3);
str += vecLength * 4;
n -= vecLength;
} while (n >= vecLength);
res = Traits::vec128Add(res, Traits::vec128Add(Traits::squash2(res0, res1), Traits::squash2(res2, res3)));
}
template<typename Traits>
ALWAYS_INLINE void polyHashUnroll8(int& n, uint16_t const*& str, typename Traits::Vec128Type& res, uint32_t const* b, uint32_t const* p) {
using VecType = typename Traits::VecType;
using Vec128Type = typename Traits::Vec128Type;
using U16VecType = typename Traits::U16VecType;
const int vecLength = sizeof(VecType) / 4;
if (n < vecLength * 2) return;
VecType res0 = Traits::initVec();
VecType res1 = Traits::initVec();
VecType res2 = Traits::initVec();
VecType res3 = Traits::initVec();
VecType res4 = Traits::initVec();
VecType res5 = Traits::initVec();
VecType res6 = Traits::initVec();
VecType res7 = Traits::initVec();
do {
VecType x0 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str));
VecType x1 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength));
VecType x2 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 2));
VecType x3 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 3));
VecType x4 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 4));
VecType x5 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 5));
VecType x6 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 6));
VecType x7 = Traits::u16Load(*reinterpret_cast<U16VecType const*>(str + vecLength * 7));
res0 = Traits::vecMul(res0, *reinterpret_cast<VecType const*>(b));
res1 = Traits::vecMul(res1, *reinterpret_cast<VecType const*>(b));
res2 = Traits::vecMul(res2, *reinterpret_cast<VecType const*>(b));
res3 = Traits::vecMul(res3, *reinterpret_cast<VecType const*>(b));
res4 = Traits::vecMul(res4, *reinterpret_cast<VecType const*>(b));
res5 = Traits::vecMul(res5, *reinterpret_cast<VecType const*>(b));
res6 = Traits::vecMul(res6, *reinterpret_cast<VecType const*>(b));
res7 = Traits::vecMul(res7, *reinterpret_cast<VecType const*>(b));
VecType z0 = Traits::vecMul(x0, *reinterpret_cast<VecType const*>(p));
VecType z1 = Traits::vecMul(x1, *reinterpret_cast<VecType const*>(p + vecLength));
VecType z2 = Traits::vecMul(x2, *reinterpret_cast<VecType const*>(p + vecLength * 2));
VecType z3 = Traits::vecMul(x3, *reinterpret_cast<VecType const*>(p + vecLength * 3));
VecType z4 = Traits::vecMul(x4, *reinterpret_cast<VecType const*>(p + vecLength * 4));
VecType z5 = Traits::vecMul(x5, *reinterpret_cast<VecType const*>(p + vecLength * 5));
VecType z6 = Traits::vecMul(x6, *reinterpret_cast<VecType const*>(p + vecLength * 6));
VecType z7 = Traits::vecMul(x7, *reinterpret_cast<VecType const*>(p + vecLength * 7));
res0 = Traits::vecAdd(res0, z0);
res1 = Traits::vecAdd(res1, z1);
res2 = Traits::vecAdd(res2, z2);
res3 = Traits::vecAdd(res3, z3);
res4 = Traits::vecAdd(res4, z4);
res5 = Traits::vecAdd(res5, z5);
res6 = Traits::vecAdd(res6, z6);
res7 = Traits::vecAdd(res7, z7);
str += vecLength * 8;
n -= vecLength * 2;
} while (n >= vecLength * 2);
Vec128Type sum1 = Traits::vec128Add(Traits::squash2(res0, res1), Traits::squash2(res2, res3));
Vec128Type sum2 = Traits::vec128Add(Traits::squash2(res4, res5), Traits::squash2(res6, res7));
res = Traits::vec128Add(res, Traits::vec128Add(sum1, sum2));
}
#endif // RUNTIME_POLYHASH_COMMON_H #endif // RUNTIME_POLYHASH_COMMON_H
kotlin-native/runtime/src/main/cpp/polyhash/x86.cpp
+37 -25
View File
@@ -22,29 +22,33 @@ alignas(32) constexpr auto b16 = RepeatingPowers<8>(31, 16); // [base^16, base^1
alignas(32) constexpr auto b8 = RepeatingPowers<8>(31, 8); // [base^8, base^8, .., base^8 ] (8) alignas(32) constexpr auto b8 = RepeatingPowers<8>(31, 8); // [base^8, base^8, .., base^8 ] (8)
alignas(32) constexpr auto b4 = RepeatingPowers<8>(31, 4); // [base^4, base^4, .., base^4 ] (8) alignas(32) constexpr auto b4 = RepeatingPowers<8>(31, 4); // [base^4, base^4, .., base^4 ] (8)
#pragma clang attribute push(__SSE41__, apply_to = function)
struct SSETraits { struct SSETraits {
using VecType = __m128i; using VecType = __m128i;
using Vec128Type = __m128i; using Vec128Type = __m128i;
using U16VecType = __m128i; using U16VecType = __m128i;
__SSE41__ static VecType initVec() { return _mm_setzero_si128(); } static VecType initVec() { return _mm_setzero_si128(); }
__SSE41__ static Vec128Type initVec128() { return _mm_setzero_si128(); } static Vec128Type initVec128() { return _mm_setzero_si128(); }
__SSE41__ static int vec128toInt(Vec128Type x) { return _mm_cvtsi128_si32(x); } static int vec128toInt(Vec128Type x) { return _mm_cvtsi128_si32(x); }
__SSE41__ static VecType u16Load(U16VecType x) { return _mm_cvtepu16_epi32(x); } static VecType u16Load(U16VecType x) { return _mm_cvtepu16_epi32(x); }
__SSE41__ static Vec128Type vec128Mul(Vec128Type x, Vec128Type y) { return _mm_mullo_epi32(x, y); } static Vec128Type vec128Mul(Vec128Type x, Vec128Type y) { return _mm_mullo_epi32(x, y); }
__SSE41__ static Vec128Type vec128Add(Vec128Type x, Vec128Type y) { return _mm_add_epi32(x, y); } static Vec128Type vec128Add(Vec128Type x, Vec128Type y) { return _mm_add_epi32(x, y); }
__SSE41__ static VecType vecMul(VecType x, VecType y) { return _mm_mullo_epi32(x, y); } static VecType vecMul(VecType x, VecType y) { return _mm_mullo_epi32(x, y); }
__SSE41__ static VecType vecAdd(VecType x, VecType y) { return _mm_add_epi32(x, y); } static VecType vecAdd(VecType x, VecType y) { return _mm_add_epi32(x, y); }
__SSE41__ static Vec128Type squash2(VecType x, VecType y) { static Vec128Type squash2(VecType x, VecType y) {
return squash1(_mm_hadd_epi32(x, y)); // [x0 + x1, x2 + x3, y0 + y1, y2 + y3] return squash1(_mm_hadd_epi32(x, y)); // [x0 + x1, x2 + x3, y0 + y1, y2 + y3]
} }
__SSE41__ static Vec128Type squash1(VecType z) { static Vec128Type squash1(VecType z) {
VecType sum = _mm_hadd_epi32(z, z); // [z0 + z1, z2 + z3, z0 + z1, z2 + z3] VecType sum = _mm_hadd_epi32(z, z); // [z0 + z1, z2 + z3, z0 + z1, z2 + z3]
return _mm_hadd_epi32(sum, sum); // [z0..3, same, same, same] return _mm_hadd_epi32(sum, sum); // [z0..3, same, same, same]
} }
__SSE41__ static int polyHashUnalignedUnrollUpTo8(int n, uint16_t const* str) { #include "polyhash/attributeSensitiveFunctions.inc"
static int polyHashUnalignedUnrollUpTo8(int n, uint16_t const* str) {
Vec128Type res = initVec128(); Vec128Type res = initVec128();
polyHashUnroll2<SSETraits>(n, str, res, &b8[0], &p64[56]); polyHashUnroll2<SSETraits>(n, str, res, &b8[0], &p64[56]);
@@ -53,7 +57,7 @@ struct SSETraits {
return vec128toInt(res); return vec128toInt(res);
} }
__SSE41__ static int polyHashUnalignedUnrollUpTo16(int n, uint16_t const* str) { static int polyHashUnalignedUnrollUpTo16(int n, uint16_t const* str) {
Vec128Type res = initVec128(); Vec128Type res = initVec128();
polyHashUnroll4<SSETraits>(n, str, res, &b16[0], &p64[48]); polyHashUnroll4<SSETraits>(n, str, res, &b16[0], &p64[48]);
@@ -64,24 +68,28 @@ struct SSETraits {
} }
}; };
#pragma clang attribute pop
#pragma clang attribute push(__AVX2__, apply_to = function)
struct AVX2Traits { struct AVX2Traits {
using VecType = __m256i; using VecType = __m256i;
using Vec128Type = __m128i; using Vec128Type = __m128i;
using U16VecType = __m128i; using U16VecType = __m128i;
__AVX2__ static VecType initVec() { return _mm256_setzero_si256(); } static VecType initVec() { return _mm256_setzero_si256(); }
__AVX2__ static Vec128Type initVec128() { return _mm_setzero_si128(); } static Vec128Type initVec128() { return _mm_setzero_si128(); }
__AVX2__ static int vec128toInt(Vec128Type x) { return _mm_cvtsi128_si32(x); } static int vec128toInt(Vec128Type x) { return _mm_cvtsi128_si32(x); }
__AVX2__ static VecType u16Load(U16VecType x) { return _mm256_cvtepu16_epi32(x); } static VecType u16Load(U16VecType x) { return _mm256_cvtepu16_epi32(x); }
__AVX2__ static Vec128Type vec128Mul(Vec128Type x, Vec128Type y) { return _mm_mullo_epi32(x, y); } static Vec128Type vec128Mul(Vec128Type x, Vec128Type y) { return _mm_mullo_epi32(x, y); }
__AVX2__ static Vec128Type vec128Add(Vec128Type x, Vec128Type y) { return _mm_add_epi32(x, y); } static Vec128Type vec128Add(Vec128Type x, Vec128Type y) { return _mm_add_epi32(x, y); }
__AVX2__ static VecType vecMul(VecType x, VecType y) { return _mm256_mullo_epi32(x, y); } static VecType vecMul(VecType x, VecType y) { return _mm256_mullo_epi32(x, y); }
__AVX2__ static VecType vecAdd(VecType x, VecType y) { return _mm256_add_epi32(x, y); } static VecType vecAdd(VecType x, VecType y) { return _mm256_add_epi32(x, y); }
__AVX2__ static Vec128Type squash2(VecType x, VecType y) { static Vec128Type squash2(VecType x, VecType y) {
return squash1(_mm256_hadd_epi32(x, y)); // [x0 + x1, x2 + x3, y0 + y1, y2 + y3, x4 + x5, x6 + x7, y4 + y5, y6 + y7] return squash1(_mm256_hadd_epi32(x, y)); // [x0 + x1, x2 + x3, y0 + y1, y2 + y3, x4 + x5, x6 + x7, y4 + y5, y6 + y7]
} }
__AVX2__ static Vec128Type squash1(VecType z) { static Vec128Type squash1(VecType z) {
VecType sum = _mm256_hadd_epi32(z, z); // [z0 + z1, z2 + z3, z0 + z1, z2 + z3, z4 + z5, z6 + z7, z4 + z5, z6 + z7] VecType sum = _mm256_hadd_epi32(z, z); // [z0 + z1, z2 + z3, z0 + z1, z2 + z3, z4 + z5, z6 + z7, z4 + z5, z6 + z7]
sum = _mm256_hadd_epi32(sum, sum); // [z0..3, z0..3, z0..3, z0..3, z4..7, z4..7, z4..7, z4..7] sum = _mm256_hadd_epi32(sum, sum); // [z0..3, z0..3, z0..3, z0..3, z4..7, z4..7, z4..7, z4..7]
Vec128Type lo = _mm256_extracti128_si256(sum, 0); // [z0..3, same, same, same] Vec128Type lo = _mm256_extracti128_si256(sum, 0); // [z0..3, same, same, same]
@@ -89,7 +97,9 @@ struct AVX2Traits {
return _mm_add_epi32(lo, hi); // [z0..7, same, same, same] return _mm_add_epi32(lo, hi); // [z0..7, same, same, same]
} }
__AVX2__ static int polyHashUnalignedUnrollUpTo16(int n, uint16_t const* str) { #include "polyhash/attributeSensitiveFunctions.inc"
static int polyHashUnalignedUnrollUpTo16(int n, uint16_t const* str) {
Vec128Type res = initVec128(); Vec128Type res = initVec128();
polyHashUnroll2<AVX2Traits>(n, str, res, &b16[0], &p64[48]); polyHashUnroll2<AVX2Traits>(n, str, res, &b16[0], &p64[48]);
@@ -99,7 +109,7 @@ struct AVX2Traits {
return vec128toInt(res); return vec128toInt(res);
} }
__AVX2__ static int polyHashUnalignedUnrollUpTo32(int n, uint16_t const* str) { static int polyHashUnalignedUnrollUpTo32(int n, uint16_t const* str) {
Vec128Type res = initVec128(); Vec128Type res = initVec128();
polyHashUnroll4<AVX2Traits>(n, str, res, &b32[0], &p64[32]); polyHashUnroll4<AVX2Traits>(n, str, res, &b32[0], &p64[32]);
@@ -110,7 +120,7 @@ struct AVX2Traits {
return vec128toInt(res); return vec128toInt(res);
} }
__AVX2__ static int polyHashUnalignedUnrollUpTo64(int n, uint16_t const* str) { static int polyHashUnalignedUnrollUpTo64(int n, uint16_t const* str) {
Vec128Type res = initVec128(); Vec128Type res = initVec128();
polyHashUnroll8<AVX2Traits>(n, str, res, &b64[0], &p64[0]); polyHashUnroll8<AVX2Traits>(n, str, res, &b64[0], &p64[0]);
@@ -123,6 +133,8 @@ struct AVX2Traits {
} }
}; };
#pragma clang attribute pop
#if defined(__x86_64__) #if defined(__x86_64__)
const bool x64 = true; const bool x64 = true;
#else #else