afeica
/
cocos-engine-external
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
cocos-engine-external/sources/enoki/array_sse42.h

2411 lines
92 KiB
C++
Raw Blame History

/*
enoki/array_sse42.h -- Packed SIMD array (SSE4.2 specialization)
Enoki is a C++ template library that enables transparent vectorization
of numerical kernels using SIMD instruction sets available on current
processor architectures.
Copyrighe (c) 2019 Wenzel Jakob <wenzel.jakob@epfl.ch>
All rights reserved. Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file.
*/
#pragma once
NAMESPACE_BEGIN(enoki)
NAMESPACE_BEGIN(detail)
/// Compressed look-up table for the store_compress() operation [256 bytes]
alignas(16) const uint8_t compress_lut_128[16*16] = {
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x00, 0x01, 0x02, 0x03, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x04, 0x05, 0x06, 0x07,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x08, 0x09, 0x0a, 0x0b, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00, 0x01, 0x02, 0x03,
0x08, 0x09, 0x0a, 0x0b, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x80, 0x80, 0x80, 0x80, 0x0c, 0x0d, 0x0e, 0x0f,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x00, 0x01, 0x02, 0x03, 0x0c, 0x0d, 0x0e, 0x0f, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x04, 0x05, 0x06, 0x07, 0x0c, 0x0d, 0x0e, 0x0f,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00, 0x01, 0x02, 0x03,
0x04, 0x05, 0x06, 0x07, 0x0c, 0x0d, 0x0e, 0x0f, 0x80, 0x80, 0x80, 0x80,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b,
0x0c, 0x0d, 0x0e, 0x0f, 0x80, 0x80, 0x80, 0x80, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x80, 0x80, 0x80, 0x80,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b,
0x0c, 0x0d, 0x0e, 0x0f
};
template <> struct is_native<float, 4> : std::true_type { } ;
template <> struct is_native<float, 3> : std::true_type { };
template <> struct is_native<double, 2> : std::true_type { };
template <typename Value> struct is_native<Value, 4, enable_if_int32_t<Value>> : std::true_type { };
template <typename Value> struct is_native<Value, 3, enable_if_int32_t<Value>> : std::true_type { };
template <typename Value> struct is_native<Value, 2, enable_if_int64_t<Value>> : std::true_type { };
NAMESPACE_END(detail)
/// Partial overload of StaticArrayImpl using SSE4.2 intrinsics (single precision)
template <bool IsMask_, typename Derived_> struct alignas(16)
StaticArrayImpl<float, 4, IsMask_, Derived_>
: StaticArrayBase<float, 4, IsMask_, Derived_> {
ENOKI_NATIVE_ARRAY(float, 4, __m128)
// -----------------------------------------------------------------------
//! @{ \name Value constructors
// -----------------------------------------------------------------------
ENOKI_INLINE StaticArrayImpl(Value value) : m(_mm_set1_ps(value)) { }
ENOKI_INLINE StaticArrayImpl(Value v0, Value v1, Value v2, Value v3)
: m(_mm_setr_ps(v0, v1, v2, v3)) { }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Type converting constructors
// -----------------------------------------------------------------------
#if defined(ENOKI_X86_F16C)
ENOKI_CONVERT(half) {
m = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *) a.derived().data()));
}
#endif
ENOKI_CONVERT(float) : m(a.derived().m) { }
ENOKI_CONVERT(int32_t) : m(_mm_cvtepi32_ps(a.derived().m)) { }
ENOKI_CONVERT(uint32_t) {
#if defined(ENOKI_X86_AVX512DQ) && defined(ENOKI_X86_AVX512VL)
m = _mm_cvtepu32_ps(a.derived().m);
#else
int32_array_t<Derived> ai(a);
Derived result =
Derived(ai & 0x7fffffff) +
(Derived(float(1u << 31)) & mask_t<Derived>(sr<31>(ai)));
m = result.m;
#endif
}
#if defined(ENOKI_X86_AVX)
ENOKI_CONVERT(double) : m(_mm256_cvtpd_ps(a.derived().m)) { }
#else
ENOKI_CONVERT(double)
: m(_mm_shuffle_ps(_mm_cvtpd_ps(low(a).m), _mm_cvtpd_ps(high(a).m),
_MM_SHUFFLE(1, 0, 1, 0))) { }
#endif
#if defined(ENOKI_X86_AVX512DQ) && defined(ENOKI_X86_AVX512VL)
ENOKI_CONVERT(int64_t) : m(_mm256_cvtepi64_ps(a.derived().m)) { }
ENOKI_CONVERT(uint64_t) : m(_mm256_cvtepu64_ps(a.derived().m)) { }
#endif
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Reinterpreting constructors, mask converters
// -----------------------------------------------------------------------
ENOKI_REINTERPRET(bool) {
int ival;
memcpy(&ival, a.derived().data(), 4);
m = _mm_castsi128_ps(_mm_cvtepi8_epi32(
_mm_cmpgt_epi8(_mm_cvtsi32_si128(ival), _mm_setzero_si128())));
}
ENOKI_REINTERPRET(float) : m(a.derived().m) { }
ENOKI_REINTERPRET(int32_t) : m(_mm_castsi128_ps(a.derived().m)) { }
ENOKI_REINTERPRET(uint32_t) : m(_mm_castsi128_ps(a.derived().m)) { }
#if defined(ENOKI_X86_AVX)
ENOKI_REINTERPRET(double)
: m(_mm_castsi128_ps(
detail::mm256_cvtepi64_epi32(_mm256_castpd_si256(a.derived().m)))) { }
#else
ENOKI_REINTERPRET(double)
: m(_mm_castsi128_ps(detail::mm256_cvtepi64_epi32(
_mm_castpd_si128(low(a).m), _mm_castpd_si128(high(a).m)))) { }
#endif
#if defined(ENOKI_X86_AVX2)
ENOKI_REINTERPRET(uint64_t)
: m(_mm_castsi128_ps(
detail::mm256_cvtepi64_epi32(a.derived().m))) { }
ENOKI_REINTERPRET(int64_t)
: m(_mm_castsi128_ps(
detail::mm256_cvtepi64_epi32(a.derived().m))) { }
#else
ENOKI_REINTERPRET(uint64_t)
: m(_mm_castsi128_ps(
detail::mm256_cvtepi64_epi32(low(a).m, high(a).m))) { }
ENOKI_REINTERPRET(int64_t)
: m(_mm_castsi128_ps(
detail::mm256_cvtepi64_epi32(low(a).m, high(a).m))) { }
#endif
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Converting from/to half size vectors
// -----------------------------------------------------------------------
StaticArrayImpl(const Array1 &a1, const Array2 &a2)
: m(_mm_setr_ps(a1.coeff(0), a1.coeff(1), a2.coeff(0), a2.coeff(1))) { }
ENOKI_INLINE Array1 low_() const { return Array1(coeff(0), coeff(1)); }
ENOKI_INLINE Array2 high_() const { return Array2(coeff(2), coeff(3)); }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Vertical operations
// -----------------------------------------------------------------------
ENOKI_INLINE Derived add_(Ref a) const { return _mm_add_ps(m, a.m); }
ENOKI_INLINE Derived sub_(Ref a) const { return _mm_sub_ps(m, a.m); }
ENOKI_INLINE Derived mul_(Ref a) const { return _mm_mul_ps(m, a.m); }
ENOKI_INLINE Derived div_(Ref a) const { return _mm_div_ps(m, a.m); }
template <typename T> ENOKI_INLINE Derived or_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_mask_mov_ps(m, a.k, _mm_set1_ps(memcpy_cast<Value>(int32_t(-1))));
else
#endif
return _mm_or_ps(m, a.m);
}
template <typename T> ENOKI_INLINE Derived and_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_maskz_mov_ps(a.k, m);
else
#endif
return _mm_and_ps(m, a.m);
}
template <typename T> ENOKI_INLINE Derived xor_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_mask_xor_ps(m, a.k, m, _mm_set1_ps(memcpy_cast<Value>(int32_t(-1))));
else
#endif
return _mm_xor_ps(m, a.m);
}
template <typename T> ENOKI_INLINE Derived andnot_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_mask_mov_ps(m, a.k, _mm_setzero_ps());
else
#endif
return _mm_andnot_ps(a.m, m);
}
#if defined(ENOKI_X86_AVX512VL)
#define ENOKI_COMP(name, NAME) mask_t<Derived>::from_k(_mm_cmp_ps_mask(m, a.m, _CMP_##NAME))
#elif defined(ENOKI_X86_AVX)
#define ENOKI_COMP(name, NAME) mask_t<Derived>(_mm_cmp_ps(m, a.m, _CMP_##NAME))
#else
#define ENOKI_COMP(name, NAME) mask_t<Derived>(_mm_cmp##name##_ps(m, a.m))
#endif
ENOKI_INLINE auto lt_ (Ref a) const { return ENOKI_COMP(lt, LT_OQ); }
ENOKI_INLINE auto gt_ (Ref a) const { return ENOKI_COMP(gt, GT_OQ); }
ENOKI_INLINE auto le_ (Ref a) const { return ENOKI_COMP(le, LE_OQ); }
ENOKI_INLINE auto ge_ (Ref a) const { return ENOKI_COMP(ge, GE_OQ); }
ENOKI_INLINE auto eq_ (Ref a) const {
using Int = int_array_t<Derived>;
if constexpr (IsMask_)
return mask_t<Derived>(eq(Int(derived()), Int(a)));
else
return ENOKI_COMP(eq, EQ_OQ);
}
ENOKI_INLINE auto neq_(Ref a) const {
using Int = int_array_t<Derived>;
if constexpr (IsMask_)
return mask_t<Derived>(neq(Int(derived()), Int(a)));
else
return ENOKI_COMP(neq, NEQ_UQ);
}
#undef ENOKI_COMP
ENOKI_INLINE Derived abs_() const { return _mm_andnot_ps(_mm_set1_ps(-0.f), m); }
ENOKI_INLINE Derived min_(Ref b) const { return _mm_min_ps(b.m, m); }
ENOKI_INLINE Derived max_(Ref b) const { return _mm_max_ps(b.m, m); }
ENOKI_INLINE Derived ceil_() const { return _mm_ceil_ps(m); }
ENOKI_INLINE Derived floor_() const { return _mm_floor_ps(m); }
ENOKI_INLINE Derived sqrt_() const { return _mm_sqrt_ps(m); }
ENOKI_INLINE Derived round_() const {
return _mm_round_ps(m, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
}
ENOKI_INLINE Derived trunc_() const {
return _mm_round_ps(m, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
}
template <typename Mask>
static ENOKI_INLINE Derived select_(const Mask &m, Ref t, Ref f) {
#if !defined(ENOKI_X86_AVX512VL)
return _mm_blendv_ps(f.m, t.m, m.m);
#else
return _mm_mask_blend_ps(m.k, f.m, t.m);
#endif
}
#if defined(ENOKI_X86_FMA)
ENOKI_INLINE Derived fmadd_ (Ref b, Ref c) const { return _mm_fmadd_ps (m, b.m, c.m); }
ENOKI_INLINE Derived fmsub_ (Ref b, Ref c) const { return _mm_fmsub_ps (m, b.m, c.m); }
ENOKI_INLINE Derived fnmadd_ (Ref b, Ref c) const { return _mm_fnmadd_ps (m, b.m, c.m); }
ENOKI_INLINE Derived fnmsub_ (Ref b, Ref c) const { return _mm_fnmsub_ps (m, b.m, c.m); }
ENOKI_INLINE Derived fmsubadd_(Ref b, Ref c) const { return _mm_fmsubadd_ps(m, b.m, c.m); }
ENOKI_INLINE Derived fmaddsub_(Ref b, Ref c) const { return _mm_fmaddsub_ps(m, b.m, c.m); }
#endif
template <int I0, int I1, int I2, int I3>
ENOKI_INLINE Derived shuffle_() const {
#if defined(ENOKI_X86_AVX)
return _mm_permute_ps(m, _MM_SHUFFLE(I3, I2, I1, I0));
#else
return _mm_shuffle_ps(m, m, _MM_SHUFFLE(I3, I2, I1, I0));
#endif
}
template <typename Index>
ENOKI_INLINE Derived shuffle_(const Index &index) const {
#if defined(ENOKI_X86_AVX)
return _mm_permutevar_ps(m, index.m);
#else
return Base::shuffle_(index);
#endif
}
#if defined(ENOKI_X86_AVX512VL)
ENOKI_INLINE Derived ldexp_(Ref arg) const { return _mm_scalef_ps(m, arg.m); }
ENOKI_INLINE std::pair<Derived, Derived> frexp_() const {
return std::make_pair<Derived, Derived>(
_mm_getmant_ps(m, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src),
_mm_getexp_ps(m));
}
#endif
ENOKI_INLINE Derived rcp_() const {
#if defined(ENOKI_X86_AVX512ER)
/* rel err < 2^28, use as is (even in non-approximate mode) */
return _mm512_castps512_ps128(
_mm512_rcp28_ps(_mm512_castps128_ps512(m)));
#else
/* Use best reciprocal approximation available on the current
hardware and refine */
__m128 r;
#if defined(ENOKI_X86_AVX512VL)
r = _mm_rcp14_ps(m); /* rel error < 2^-14 */
#else
r = _mm_rcp_ps(m); /* rel error < 1.5*2^-12 */
#endif
/* Refine using one Newton-Raphson iteration */
__m128 t0 = _mm_add_ps(r, r),
t1 = _mm_mul_ps(r, m),
ro = r;
(void) ro;
#if defined(ENOKI_X86_FMA)
r = _mm_fnmadd_ps(t1, r, t0);
#else
r = _mm_sub_ps(t0, _mm_mul_ps(r, t1));
#endif
#if defined(ENOKI_X86_AVX512VL)
return _mm_fixupimm_ps(r, m, _mm_set1_epi32(0x0087A622), 0);
#else
return _mm_blendv_ps(r, ro, t1); /* mask bit is '1' iff t1 == nan */
#endif
#endif
}
ENOKI_INLINE Derived rsqrt_() const {
#if defined(ENOKI_X86_AVX512ER)
/* rel err < 2^28, use as is (even in non-approximate mode) */
return _mm512_castps512_ps128(
_mm512_rsqrt28_ps(_mm512_castps128_ps512(m)));
#else
/* Use best reciprocal square root approximation available
on the current hardware and refine */
__m128 r;
#if defined(ENOKI_X86_AVX512VL)
r = _mm_rsqrt14_ps(m); /* rel error < 2^-14 */
#else
r = _mm_rsqrt_ps(m); /* rel error < 1.5*2^-12 */
#endif
/* Refine using one Newton-Raphson iteration */
const __m128 c0 = _mm_set1_ps(.5f),
c1 = _mm_set1_ps(3.f);
__m128 t0 = _mm_mul_ps(r, c0),
t1 = _mm_mul_ps(r, m),
ro = r;
(void) ro;
#if defined(ENOKI_X86_FMA)
r = _mm_mul_ps(_mm_fnmadd_ps(t1, r, c1), t0);
#else
r = _mm_mul_ps(_mm_sub_ps(c1, _mm_mul_ps(t1, r)), t0);
#endif
#if defined(ENOKI_X86_AVX512VL)
return _mm_fixupimm_ps(r, m, _mm_set1_epi32(0x0383A622), 0);
#else
return _mm_blendv_ps(r, ro, t1); /* mask bit is '1' iff t1 == nan */
#endif
#endif
}
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Horizontal operations
// -----------------------------------------------------------------------
#define ENOKI_HORIZONTAL_OP(name, op) \
ENOKI_INLINE Value name##_() const { \
__m128 t1 = _mm_movehdup_ps(m); \
__m128 t2 = _mm_##op##_ps(m, t1); \
t1 = _mm_movehl_ps(t1, t2); \
t2 = _mm_##op##_ss(t2, t1); \
return _mm_cvtss_f32(t2); \
}
ENOKI_HORIZONTAL_OP(hsum, add)
ENOKI_HORIZONTAL_OP(hprod, mul)
ENOKI_HORIZONTAL_OP(hmin, min)
ENOKI_HORIZONTAL_OP(hmax, max)
#undef ENOKI_HORIZONTAL_OP
ENOKI_INLINE bool all_() const { return _mm_movemask_ps(m) == 0xF;}
ENOKI_INLINE bool any_() const { return _mm_movemask_ps(m) != 0x0; }
ENOKI_INLINE uint32_t bitmask_() const { return (uint32_t) _mm_movemask_ps(m); }
ENOKI_INLINE size_t count_() const { return (size_t) _mm_popcnt_u32(bitmask_()); }
ENOKI_INLINE Value dot_(Ref a) const {
return _mm_cvtss_f32(_mm_dp_ps(m, a.m, 0b11110001));
}
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Masked versions of key operations
// -----------------------------------------------------------------------
#if defined(ENOKI_X86_AVX512VL)
template <typename Mask>
ENOKI_INLINE void massign_(const Derived &a, const Mask &mask) { m = _mm_mask_mov_ps(m, mask.k, a.m); }
template <typename Mask>
ENOKI_INLINE void madd_ (const Derived &a, const Mask &mask) { m = _mm_mask_add_ps(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void msub_ (const Derived &a, const Mask &mask) { m = _mm_mask_sub_ps(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mmul_ (const Derived &a, const Mask &mask) { m = _mm_mask_mul_ps(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mdiv_ (const Derived &a, const Mask &mask) { m = _mm_mask_div_ps(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mor_ (const Derived &a, const Mask &mask) { m = _mm_mask_or_ps(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mand_ (const Derived &a, const Mask &mask) { m = _mm_mask_and_ps(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mxor_ (const Derived &a, const Mask &mask) { m = _mm_mask_xor_ps(m, mask.k, m, a.m); }
#endif
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Initialization, loading/writing data
// -----------------------------------------------------------------------
ENOKI_INLINE void store_(void *ptr) const {
assert((uintptr_t) ptr % 16 == 0);
_mm_store_ps((Value *) ENOKI_ASSUME_ALIGNED(ptr, 16), m);
}
template <typename Mask>
ENOKI_INLINE void store_(void *ptr, const Mask &mask) const {
#if defined(ENOKI_X86_AVX512VL)
_mm_mask_store_ps((Value *) ptr, mask.k, m);
#elif defined(ENOKI_X86_AVX)
_mm_maskstore_ps((Value *) ptr, _mm_castps_si128(mask.m), m);
#else
Base::store_(ptr, mask);
#endif
}
ENOKI_INLINE void store_unaligned_(void *ptr) const {
_mm_storeu_ps((Value *) ptr, m);
}
template <typename Mask>
ENOKI_INLINE void store_unaligned_(void *ptr, const Mask &mask) const {
#if defined(ENOKI_X86_AVX512VL)
_mm_mask_storeu_ps((Value *) ptr, mask.k, m);
#elif defined(ENOKI_X86_AVX)
_mm_maskstore_ps((Value *) ptr, _mm_castps_si128(mask.m), m);
#else
Base::store_unaligned_(ptr, mask);
#endif
}
static ENOKI_INLINE Derived load_(const void *ptr) {
assert((uintptr_t) ptr % 16 == 0);
return _mm_load_ps((const Value *) ENOKI_ASSUME_ALIGNED(ptr, 16));
}
template <typename Mask>
static ENOKI_INLINE Derived load_(const void *ptr, const Mask &mask) {
#if defined(ENOKI_X86_AVX512VL)
return _mm_maskz_load_ps(mask.k, ptr);
#elif defined(ENOKI_X86_AVX)
return _mm_maskload_ps((const Value *) ptr, _mm_castps_si128(mask.m));
#else
return Base::load_(ptr, mask);
#endif
}
static ENOKI_INLINE Derived load_unaligned_(const void *ptr) {
return _mm_loadu_ps((const Value *) ptr);
}
template <typename Mask>
static ENOKI_INLINE Derived load_unaligned_(const void *ptr, const Mask &mask) {
#if defined(ENOKI_X86_AVX512VL)
return _mm_maskz_loadu_ps(mask.k, ptr);
#elif defined(ENOKI_X86_AVX)
return _mm_maskload_ps((const Value *) ptr, _mm_castps_si128(mask.m));
#else
return Base::load_unaligned_(ptr, mask);
#endif
}
static ENOKI_INLINE Derived zero_() { return _mm_setzero_ps(); }
#if defined(ENOKI_X86_AVX2)
template <size_t Stride, typename Index, typename Mask>
static ENOKI_INLINE Derived gather_(const void *ptr, const Index &index, const Mask &mask) {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (sizeof(scalar_t<Index>) == 4)
return _mm_mmask_i32gather_ps(_mm_setzero_ps(), mask.k, index.m, (const float *) ptr, Stride);
else
return _mm256_mmask_i64gather_ps(_mm_setzero_ps(), mask.k, index.m, (const float *) ptr, Stride);
#else
if constexpr (sizeof(scalar_t<Index>) == 4)
return _mm_mask_i32gather_ps(_mm_setzero_ps(), (const float *) ptr, index.m, mask.m, Stride);
else
return _mm256_mask_i64gather_ps(_mm_setzero_ps(), (const float *) ptr, index.m, mask.m, Stride);
#endif
}
#endif
#if defined(ENOKI_X86_AVX512VL)
template <size_t Stride, typename Index, typename Mask>
ENOKI_INLINE void scatter_(void *ptr, const Index &index, const Mask &mask) const {
if constexpr (sizeof(scalar_t<Index>) == 4)
_mm_mask_i32scatter_ps(ptr, mask.k, index.m, m, Stride);
else
_mm256_mask_i64scatter_ps(ptr, mask.k, index.m, m, Stride);
}
#endif
template <typename Mask>
ENOKI_INLINE Value extract_(const Mask &mask) const {
#if !defined(ENOKI_X86_AVX512VL)
unsigned int k = (unsigned int) _mm_movemask_ps(mask.m);
return coeff((size_t) (detail::tzcnt_scalar(k) & 3));
#else
return _mm_cvtss_f32(_mm_mask_compress_ps(_mm_setzero_ps(), mask.k, m));
#endif
}
template <typename Mask>
ENOKI_INLINE size_t compress_(float *&ptr, const Mask &mask) const {
#if !defined(ENOKI_X86_AVX512VL)
unsigned int k = (unsigned int) _mm_movemask_ps(mask.m);
/** Fancy LUT-based partitioning algorithm, see
https://deplinenoise.files.wordpress.com/2015/03/gdc2015_afredriksson_simd.pdf */
__m128i shuf = _mm_load_si128(((const __m128i *) detail::compress_lut_128) + k),
perm = _mm_shuffle_epi8(_mm_castps_si128(m), shuf);
_mm_storeu_si128((__m128i *) ptr, perm);
#else
unsigned int k = (unsigned int) mask.k;
_mm_storeu_ps(ptr, _mm_mask_compress_ps(_mm_setzero_ps(), mask.k, m));
#endif
size_t kn = (size_t) _mm_popcnt_u32(k);
ptr += kn;
return kn;
}
//! @}
// -----------------------------------------------------------------------
} ENOKI_MAY_ALIAS;
/// Partial overload of StaticArrayImpl using SSE4.2 intrinsics (double precision)
template <bool IsMask_, typename Derived_> struct alignas(16)
StaticArrayImpl<double, 2, IsMask_, Derived_>
: StaticArrayBase<double, 2, IsMask_, Derived_> {
ENOKI_NATIVE_ARRAY(double, 2, __m128d)
// -----------------------------------------------------------------------
//! @{ \name Value constructors
// -----------------------------------------------------------------------
ENOKI_INLINE StaticArrayImpl(Value value) : m(_mm_set1_pd(value)) { }
ENOKI_INLINE StaticArrayImpl(Value v0, Value v1)
: m(_mm_setr_pd(v0, v1)) { }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Type converting constructors
// -----------------------------------------------------------------------
/* No vectorized conversions from float/[u]int32_t (too small) */
ENOKI_CONVERT(double) : m(a.derived().m) { }
#if defined(ENOKI_X86_AVX512DQ) && defined(ENOKI_X86_AVX512VL)
ENOKI_CONVERT(int64_t) : m(_mm_cvtepi64_pd(a.derived().m)) { }
ENOKI_CONVERT(uint64_t) : m(_mm_cvtepu64_pd(a.derived().m)) { }
#endif
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Reinterpreting constructors, mask converters
// -----------------------------------------------------------------------
ENOKI_REINTERPRET(bool) {
int16_t ival;
memcpy(&ival, a.derived().data(), 2);
m = _mm_castsi128_pd(_mm_cvtepi8_epi64(_mm_cmpgt_epi8(
_mm_cvtsi32_si128((int) ival), _mm_setzero_si128())));
}
ENOKI_REINTERPRET(float) {
ENOKI_TRACK_SCALAR("Constructor (reinterpreting, float32[2] -> double[2])");
auto v0 = a.derived().coeff(0), v1 = a.derived().coeff(1);
m = _mm_castps_pd(_mm_setr_ps(v0, v0, v1, v1));
}
ENOKI_REINTERPRET(int32_t) {
ENOKI_TRACK_SCALAR("Constructor (reinterpreting, int32[2] -> double[2])");
auto v0 = a.derived().coeff(0), v1 = a.derived().coeff(1);
m = _mm_castsi128_pd(_mm_setr_epi32(v0, v0, v1, v1));
}
ENOKI_REINTERPRET(uint32_t) {
ENOKI_TRACK_SCALAR("Constructor (reinterpreting, uint32[2] -> double[2])");
auto v0 = a.derived().coeff(0), v1 = a.derived().coeff(1);
m = _mm_castsi128_pd(_mm_setr_epi32((int32_t) v0, (int32_t) v0,
(int32_t) v1, (int32_t) v1));
}
ENOKI_REINTERPRET(double) : m(a.derived().m) { }
ENOKI_REINTERPRET(int64_t) : m(_mm_castsi128_pd(a.derived().m)) { }
ENOKI_REINTERPRET(uint64_t) : m(_mm_castsi128_pd(a.derived().m)) { }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Converting from/to half size vectors
// -----------------------------------------------------------------------
StaticArrayImpl(const Array1 &a1, const Array2 &a2)
: m(_mm_setr_pd(a1.coeff(0), a2.coeff(0))) { }
ENOKI_INLINE Array1 low_() const { return Array1(coeff(0)); }
ENOKI_INLINE Array2 high_() const { return Array2(coeff(1)); }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Vertical operations
// -----------------------------------------------------------------------
ENOKI_INLINE Derived add_(Ref a) const { return _mm_add_pd(m, a.m); }
ENOKI_INLINE Derived sub_(Ref a) const { return _mm_sub_pd(m, a.m); }
ENOKI_INLINE Derived mul_(Ref a) const { return _mm_mul_pd(m, a.m); }
ENOKI_INLINE Derived div_(Ref a) const { return _mm_div_pd(m, a.m); }
template <typename T> ENOKI_INLINE Derived or_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_mask_mov_pd(m, a.k, _mm_set1_pd(memcpy_cast<Value>(int64_t(-1))));
else
#endif
return _mm_or_pd(m, a.m);
}
template <typename T> ENOKI_INLINE Derived and_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_maskz_mov_pd(a.k, m);
else
#endif
return _mm_and_pd(m, a.m);
}
template <typename T> ENOKI_INLINE Derived xor_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_mask_xor_pd(m, a.k, m, _mm_set1_pd(memcpy_cast<Value>(int64_t(-1))));
else
#endif
return _mm_xor_pd(m, a.m);
}
template <typename T> ENOKI_INLINE Derived andnot_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_mask_mov_pd(m, a.k, _mm_setzero_pd());
else
#endif
return _mm_andnot_pd(a.m, m);
}
#if defined(ENOKI_X86_AVX512VL)
#define ENOKI_COMP(name, NAME) mask_t<Derived>::from_k(_mm_cmp_pd_mask(m, a.m, _CMP_##NAME))
#elif defined(ENOKI_X86_AVX)
#define ENOKI_COMP(name, NAME) mask_t<Derived>(_mm_cmp_pd(m, a.m, _CMP_##NAME))
#else
#define ENOKI_COMP(name, NAME) mask_t<Derived>(_mm_cmp##name##_pd(m, a.m))
#endif
ENOKI_INLINE auto lt_ (Ref a) const { return ENOKI_COMP(lt, LT_OQ); }
ENOKI_INLINE auto gt_ (Ref a) const { return ENOKI_COMP(gt, GT_OQ); }
ENOKI_INLINE auto le_ (Ref a) const { return ENOKI_COMP(le, LE_OQ); }
ENOKI_INLINE auto ge_ (Ref a) const { return ENOKI_COMP(ge, GE_OQ); }
ENOKI_INLINE auto eq_ (Ref a) const {
using Int = int_array_t<Derived>;
if constexpr (IsMask_)
return mask_t<Derived>(eq(Int(derived()), Int(a)));
else
return ENOKI_COMP(eq, EQ_OQ);
}
ENOKI_INLINE auto neq_(Ref a) const {
using Int = int_array_t<Derived>;
if constexpr (IsMask_)
return mask_t<Derived>(neq(Int(derived()), Int(a)));
else
return ENOKI_COMP(neq, NEQ_UQ);
}
#undef ENOKI_COMP
ENOKI_INLINE Derived abs_() const { return _mm_andnot_pd(_mm_set1_pd(-0.), m); }
ENOKI_INLINE Derived min_(Ref b) const { return _mm_min_pd(b.m, m); }
ENOKI_INLINE Derived max_(Ref b) const { return _mm_max_pd(b.m, m); }
ENOKI_INLINE Derived ceil_() const { return _mm_ceil_pd(m); }
ENOKI_INLINE Derived floor_() const { return _mm_floor_pd(m); }
ENOKI_INLINE Derived sqrt_() const { return _mm_sqrt_pd(m); }
ENOKI_INLINE Derived round_() const {
return _mm_round_pd(m, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
}
ENOKI_INLINE Derived trunc_() const {
return _mm_round_pd(m, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
}
template <typename Mask>
static ENOKI_INLINE Derived select_(const Mask &m, Ref t, Ref f) {
#if !defined(ENOKI_X86_AVX512VL)
return _mm_blendv_pd(f.m, t.m, m.m);
#else
return _mm_mask_blend_pd(m.k, f.m, t.m);
#endif
}
#if defined(ENOKI_X86_FMA)
ENOKI_INLINE Derived fmadd_ (Ref b, Ref c) const { return _mm_fmadd_pd (m, b.m, c.m); }
ENOKI_INLINE Derived fmsub_ (Ref b, Ref c) const { return _mm_fmsub_pd (m, b.m, c.m); }
ENOKI_INLINE Derived fnmadd_ (Ref b, Ref c) const { return _mm_fnmadd_pd (m, b.m, c.m); }
ENOKI_INLINE Derived fnmsub_ (Ref b, Ref c) const { return _mm_fnmsub_pd (m, b.m, c.m); }
ENOKI_INLINE Derived fmsubadd_(Ref b, Ref c) const { return _mm_fmsubadd_pd(m, b.m, c.m); }
ENOKI_INLINE Derived fmaddsub_(Ref b, Ref c) const { return _mm_fmaddsub_pd(m, b.m, c.m); }
#endif
#if defined(ENOKI_X86_AVX)
#define ENOKI_SHUFFLE_PD(m, flags) _mm_permute_pd(m, flags)
#else
#define ENOKI_SHUFFLE_PD(m, flags) _mm_shuffle_pd(m, m, flags)
#endif
template <int I0, int I1>
ENOKI_INLINE Derived shuffle_() const {
return ENOKI_SHUFFLE_PD(m, (I1 << 1) | I0);
}
template <typename Index>
ENOKI_INLINE Derived shuffle_(const Index &index) const {
#if defined(ENOKI_X86_AVX)
return _mm_permutevar_pd(m, _mm_slli_epi64(index.m, 1));
#else
return Base::shuffle_(index);
#endif
}
#if defined(ENOKI_X86_AVX512VL)
ENOKI_INLINE Derived ldexp_(Ref arg) const { return _mm_scalef_pd(m, arg.m); }
ENOKI_INLINE std::pair<Derived, Derived> frexp_() const {
return std::make_pair<Derived, Derived>(
_mm_getmant_pd(m, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src),
_mm_getexp_pd(m));
}
#endif
#if defined(ENOKI_X86_AVX512VL) || defined(ENOKI_X86_AVX512ER)
ENOKI_INLINE Derived rcp_() const {
/* Use best reciprocal approximation available on the current
hardware and refine */
__m128d r;
#if defined(ENOKI_X86_AVX512ER)
/* rel err < 2^28 */
r = _mm512_castpd512_pd128(
_mm512_rcp28_pd(_mm512_castpd128_pd512(m)));
#elif defined(ENOKI_X86_AVX512VL)
r = _mm_rcp14_pd(m); /* rel error < 2^-14 */
#endif
__m128d ro = r, t0, t1;
(void) ro;
/* Refine using 1-2 Newton-Raphson iterations */
ENOKI_UNROLL for (int i = 0; i < (has_avx512er ? 1 : 2); ++i) {
t0 = _mm_add_pd(r, r);
t1 = _mm_mul_pd(r, m);
r = _mm_fnmadd_pd(t1, r, t0);
}
#if defined(ENOKI_X86_AVX512VL)
return _mm_fixupimm_pd(r, m, _mm_set1_epi32(0x0087A622), 0);
#else
return _mm_blendv_pd(r, ro, t1); /* mask bit is '1' iff t1 == nan */
#endif
}
ENOKI_INLINE Derived rsqrt_() const {
/* Use best reciprocal square root approximation available
on the current hardware and refine */
__m128d r;
#if defined(ENOKI_X86_AVX512ER)
/* rel err < 2^28 */
r = _mm512_castpd512_pd128(
_mm512_rsqrt28_pd(_mm512_castpd128_pd512(m)));
#elif defined(ENOKI_X86_AVX512VL)
r = _mm_rsqrt14_pd(m); /* rel error < 2^-14 */
#endif
const __m128d c0 = _mm_set1_pd(0.5),
c1 = _mm_set1_pd(3.0);
__m128d ro = r, t0, t1;
(void) ro;
/* Refine using 1-2 Newton-Raphson iterations */
ENOKI_UNROLL for (int i = 0; i < (has_avx512er ? 1 : 2); ++i) {
t0 = _mm_mul_pd(r, c0);
t1 = _mm_mul_pd(r, m);
r = _mm_mul_pd(_mm_fnmadd_pd(t1, r, c1), t0);
}
#if defined(ENOKI_X86_AVX512VL)
return _mm_fixupimm_pd(r, m, _mm_set1_epi32(0x0383A622), 0);
#else
return _mm_blendv_pd(r, ro, t1); /* mask bit is '1' iff t1 == nan */
#endif
}
#endif
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Horizontal operations
// -----------------------------------------------------------------------
#define ENOKI_HORIZONTAL_OP(name, op) \
ENOKI_INLINE Value name##_() const { \
__m128d t0 = ENOKI_SHUFFLE_PD(m, 1); \
__m128d t1 = _mm_##op##_sd(t0, m); \
return _mm_cvtsd_f64(t1); \
}
ENOKI_HORIZONTAL_OP(hsum, add)
ENOKI_HORIZONTAL_OP(hprod, mul)
ENOKI_HORIZONTAL_OP(hmin, min)
ENOKI_HORIZONTAL_OP(hmax, max)
#undef ENOKI_HORIZONTAL_OP
#undef ENOKI_SHUFFLE_PD
ENOKI_INLINE bool all_() const { return _mm_movemask_pd(m) == 0x3;}
ENOKI_INLINE bool any_() const { return _mm_movemask_pd(m) != 0x0; }
ENOKI_INLINE uint32_t bitmask_() const { return (uint32_t) _mm_movemask_pd(m); }
ENOKI_INLINE size_t count_() const { return (size_t) _mm_popcnt_u32(bitmask_()); }
ENOKI_INLINE Value dot_(Ref a) const {
return _mm_cvtsd_f64(_mm_dp_pd(m, a.m, 0b00110001));
}
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Masked versions of key operations
// -----------------------------------------------------------------------
#if defined(ENOKI_X86_AVX512VL)
template <typename Mask>
ENOKI_INLINE void massign_(const Derived &a, const Mask &mask) { m = _mm_mask_mov_pd(m, mask.k, a.m); }
template <typename Mask>
ENOKI_INLINE void madd_ (const Derived &a, const Mask &mask) { m = _mm_mask_add_pd(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void msub_ (const Derived &a, const Mask &mask) { m = _mm_mask_sub_pd(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mmul_ (const Derived &a, const Mask &mask) { m = _mm_mask_mul_pd(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mdiv_ (const Derived &a, const Mask &mask) { m = _mm_mask_div_pd(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mor_ (const Derived &a, const Mask &mask) { m = _mm_mask_or_pd(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mand_ (const Derived &a, const Mask &mask) { m = _mm_mask_and_pd(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mxor_ (const Derived &a, const Mask &mask) { m = _mm_mask_xor_pd(m, mask.k, m, a.m); }
#endif
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Initialization, loading/writing data
// -----------------------------------------------------------------------
ENOKI_INLINE void store_(void *ptr) const {
assert((uintptr_t) ptr % 16 == 0);
_mm_store_pd((Value *) ENOKI_ASSUME_ALIGNED(ptr, 16), m);
}
template <typename Mask>
ENOKI_INLINE void store_(void *ptr, const Mask &mask) const {
#if defined(ENOKI_X86_AVX512VL)
_mm_mask_store_pd((Value *) ptr, mask.k, m);
#elif defined(ENOKI_X86_AVX)
_mm_maskstore_pd((Value *) ptr, _mm_castpd_si128(mask.m), m);
#else
Base::store_(ptr, mask);
#endif
}
ENOKI_INLINE void store_unaligned_(void *ptr) const {
_mm_storeu_pd((Value *) ptr, m);
}
template <typename Mask>
ENOKI_INLINE void store_unaligned_(void *ptr, const Mask &mask) const {
#if defined(ENOKI_X86_AVX512VL)
_mm_mask_storeu_pd((Value *) ptr, mask.k, m);
#elif defined(ENOKI_X86_AVX)
_mm_maskstore_pd((Value *) ptr, _mm_castpd_si128(mask.m), m);
#else
Base::store_unaligned_(ptr, mask);
#endif
}
static ENOKI_INLINE Derived load_(const void *ptr) {
assert((uintptr_t) ptr % 16 == 0);
return _mm_load_pd((const Value *) ENOKI_ASSUME_ALIGNED(ptr, 16));
}
template <typename Mask>
static ENOKI_INLINE Derived load_(const void *ptr, const Mask &mask) {
#if defined(ENOKI_X86_AVX512VL)
return _mm_maskz_load_pd(mask.k, ptr);
#elif defined(ENOKI_X86_AVX)
return _mm_maskload_pd((const Value *) ptr, _mm_castpd_si128(mask.m));
#else
return Base::load_(ptr, mask);
#endif
}
static ENOKI_INLINE Derived load_unaligned_(const void *ptr) {
return _mm_loadu_pd((const Value *) ptr);
}
template <typename Mask>
static ENOKI_INLINE Derived load_unaligned_(const void *ptr, const Mask &mask) {
#if defined(ENOKI_X86_AVX512VL)
return _mm_maskz_loadu_pd(mask.k, ptr);
#elif defined(ENOKI_X86_AVX)
return _mm_maskload_pd((const Value *) ptr, _mm_castpd_si128(mask.m));
#else
return Base::load_unaligned_(ptr, mask);
#endif
}
static ENOKI_INLINE Derived zero_() { return _mm_setzero_pd(); }
#if defined(ENOKI_X86_AVX2)
template <size_t Stride, typename Index, typename Mask>
static ENOKI_INLINE Derived gather_(const void *ptr, const Index &index, const Mask &mask) {
if constexpr (sizeof(scalar_t<Index>) == 4) {
return Base::template gather_<Stride>(ptr, index, mask);
} else {
#if defined(ENOKI_X86_AVX512VL)
return _mm_mmask_i64gather_pd(_mm_setzero_pd(), mask.k, index.m, (const double *) ptr, Stride);
#else
return _mm_mask_i64gather_pd(_mm_setzero_pd(), (const double *) ptr, index.m, mask.m, Stride);
#endif
}
}
#endif
#if defined(ENOKI_X86_AVX512VL)
template <size_t Stride, typename Index, typename Mask>
ENOKI_INLINE void scatter_(void *ptr, const Index &index, const Mask &mask) const {
if constexpr (sizeof(scalar_t<Index>) == 4)
Base::template scatter_<Stride>(ptr, index, mask);
else
_mm_mask_i64scatter_pd(ptr, mask.k, index.m, m, Stride);
}
template <typename Mask>
ENOKI_INLINE Value extract_(const Mask &mask) const {
return _mm_cvtsd_f64(_mm_mask_compress_pd(_mm_setzero_pd(), mask.k, m));
}
template <typename Mask>
ENOKI_INLINE size_t compress_(double *&ptr, const Mask &mask) const {
_mm_storeu_pd(ptr, _mm_mask_compress_pd(_mm_setzero_pd(), mask.k, m));
size_t kn = (size_t) _mm_popcnt_u32(mask.k);
ptr += kn;
return kn;
}
#endif
//! @}
// -----------------------------------------------------------------------
} ENOKI_MAY_ALIAS;
/// Partial overload of StaticArrayImpl using SSE4.2 intrinsics (32 bit integers)
template <typename Value_, bool IsMask_, typename Derived_> struct alignas(16)
StaticArrayImpl<Value_, 4, IsMask_, Derived_, enable_if_int32_t<Value_>>
: StaticArrayBase<Value_, 4, IsMask_, Derived_> {
ENOKI_NATIVE_ARRAY(Value_, 4, __m128i)
// -----------------------------------------------------------------------
//! @{ \name Value constructors
// -----------------------------------------------------------------------
ENOKI_INLINE StaticArrayImpl(Value value) : m(_mm_set1_epi32((int32_t) value)) { }
ENOKI_INLINE StaticArrayImpl(Value v0, Value v1, Value v2, Value v3)
: m(_mm_setr_epi32((int32_t) v0, (int32_t) v1, (int32_t) v2, (int32_t) v3)) { }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Type converting constructors
// -----------------------------------------------------------------------
ENOKI_CONVERT(float) {
if constexpr (std::is_signed_v<Value>) {
m = _mm_cvttps_epi32(a.derived().m);
} else {
#if defined(ENOKI_X86_AVX512DQ) && defined(ENOKI_X86_AVX512VL)
m = _mm_cvttps_epu32(a.derived().m);
#else
constexpr uint32_t limit = 1u << 31;
const __m128 limit_f = _mm_set1_ps((float) limit);
const __m128i limit_i = _mm_set1_epi32((int) limit);
__m128 v = a.derived().m;
__m128i mask =
_mm_castps_si128(_mm_cmpge_ps(v, limit_f));
__m128i b2 = _mm_add_epi32(
_mm_cvttps_epi32(_mm_sub_ps(v, limit_f)), limit_i);
__m128i b1 = _mm_cvttps_epi32(v);
m = _mm_blendv_epi8(b1, b2, mask);
#endif
}
}
ENOKI_CONVERT(int32_t) : m(a.derived().m) { }
ENOKI_CONVERT(uint32_t) : m(a.derived().m) { }
#if defined(ENOKI_X86_AVX)
ENOKI_CONVERT(double) {
if constexpr (std::is_signed_v<Value>) {
m = _mm256_cvttpd_epi32(a.derived().m);
} else {
#if defined(ENOKI_X86_AVX512DQ) && defined(ENOKI_X86_AVX512VL)
m = _mm256_cvttpd_epu32(a.derived().m);
#else
ENOKI_TRACK_SCALAR("Constructor (converting, double[4] -> uint32[4])");
for (size_t i = 0; i < Size; ++i)
coeff(i) = Value(a.derived().coeff(i));
#endif
}
}
#endif
#if defined(ENOKI_X86_AVX512DQ) && defined(ENOKI_X86_AVX512VL)
ENOKI_CONVERT(int64_t) { m = _mm256_cvtepi64_epi32(a.derived().m); }
ENOKI_CONVERT(uint64_t) { m = _mm256_cvtepi64_epi32(a.derived().m); }
#endif
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Reinterpreting constructors, mask converters
// -----------------------------------------------------------------------
ENOKI_REINTERPRET(bool) {
int ival;
memcpy(&ival, a.derived().data(), 4);
m = _mm_cvtepi8_epi32(
_mm_cmpgt_epi8(_mm_cvtsi32_si128(ival), _mm_setzero_si128()));
}
ENOKI_REINTERPRET(float) : m(_mm_castps_si128(a.derived().m)) { }
ENOKI_REINTERPRET(int32_t) : m(a.derived().m) { }
ENOKI_REINTERPRET(uint32_t) : m(a.derived().m) { }
#if defined(ENOKI_X86_AVX)
ENOKI_REINTERPRET(double)
: m(detail::mm256_cvtepi64_epi32(_mm256_castpd_si256(a.derived().m))) { }
#else
ENOKI_REINTERPRET(double)
: m(detail::mm256_cvtepi64_epi32(_mm_castpd_si128(low(a).m),
_mm_castpd_si128(high(a).m))) { }
#endif
#if defined(ENOKI_X86_AVX2)
ENOKI_REINTERPRET(uint64_t)
: m(detail::mm256_cvtepi64_epi32(a.derived().m)) { }
ENOKI_REINTERPRET(int64_t)
: m(detail::mm256_cvtepi64_epi32(a.derived().m)) {}
#else
ENOKI_REINTERPRET(uint64_t)
: m(detail::mm256_cvtepi64_epi32(low(a).m, high(a).m)) { }
ENOKI_REINTERPRET(int64_t)
: m(detail::mm256_cvtepi64_epi32(low(a).m, high(a).m)) { }
#endif
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Converting from/to half size vectors
// -----------------------------------------------------------------------
StaticArrayImpl(const Array1 &a1, const Array2 &a2)
: m(_mm_setr_epi32((int32_t) a1.coeff(0), (int32_t) a1.coeff(1),
(int32_t) a2.coeff(0), (int32_t) a2.coeff(1))) { }
ENOKI_INLINE Array1 low_() const { return Array1(coeff(0), coeff(1)); }
ENOKI_INLINE Array2 high_() const { return Array2(coeff(2), coeff(3)); }
//! @}
// -----------------------------------------------------------------------
ENOKI_INLINE Derived add_(Ref a) const { return _mm_add_epi32(m, a.m); }
ENOKI_INLINE Derived sub_(Ref a) const { return _mm_sub_epi32(m, a.m); }
ENOKI_INLINE Derived mul_(Ref a) const { return _mm_mullo_epi32(m, a.m); }
template <typename T> ENOKI_INLINE Derived or_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_mask_mov_epi32(m, a.k, _mm_set1_epi32(-1));
else
#endif
return _mm_or_si128(m, a.m);
}
template <typename T> ENOKI_INLINE Derived and_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_maskz_mov_epi32(a.k, m);
else
#endif
return _mm_and_si128(m, a.m);
}
template <typename T> ENOKI_INLINE Derived xor_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_mask_xor_epi32(m, a.k, m, _mm_set1_epi32(-1));
else
#endif
return _mm_xor_si128(m, a.m);
}
template <typename T> ENOKI_INLINE Derived andnot_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_mask_mov_epi32(m, a.k, _mm_setzero_si128());
else
#endif
return _mm_andnot_si128(a.m, m);
}
template <size_t Imm> ENOKI_INLINE Derived sl_() const {
return _mm_slli_epi32(m, (int) Imm);
}
template <size_t Imm> ENOKI_INLINE Derived sr_() const {
return std::is_signed_v<Value> ? _mm_srai_epi32(m, (int) Imm)
: _mm_srli_epi32(m, (int) Imm);
}
ENOKI_INLINE Derived sl_(size_t k) const {
return _mm_sll_epi32(m, _mm_set1_epi64x((long long) k));
}
ENOKI_INLINE Derived sr_(size_t k) const {
return std::is_signed_v<Value>
? _mm_sra_epi32(m, _mm_set1_epi64x((long long) k))
: _mm_srl_epi32(m, _mm_set1_epi64x((long long) k));
}
ENOKI_INLINE Derived sl_(Ref k) const {
#if defined(ENOKI_X86_AVX2)
return _mm_sllv_epi32(m, k.m);
#else
Derived out;
ENOKI_TRACK_SCALAR("sl");
for (size_t i = 0; i < Size; ++i)
out.coeff(i) = coeff(i) << (size_t) k.coeff(i);
return out;
#endif
}
ENOKI_INLINE Derived sr_(Ref k) const {
#if defined(ENOKI_X86_AVX2)
return std::is_signed_v<Value> ? _mm_srav_epi32(m, k.m)
: _mm_srlv_epi32(m, k.m);
#else
Derived out;
ENOKI_TRACK_SCALAR("sr");
for (size_t i = 0; i < Size; ++i)
out.coeff(i) = coeff(i) >> (size_t) k.coeff(i);
return out;
#endif
}
#if defined(ENOKI_X86_AVX512VL)
template <size_t Imm> ENOKI_INLINE Derived rol_() const { return _mm_rol_epi32(m, (int) Imm); }
template <size_t Imm> ENOKI_INLINE Derived ror_() const { return _mm_ror_epi32(m, (int) Imm); }
ENOKI_INLINE Derived rol_(Ref k) const { return _mm_rolv_epi32(m, k.m); }
ENOKI_INLINE Derived ror_(Ref k) const { return _mm_rorv_epi32(m, k.m); }
#endif
ENOKI_INLINE auto eq_(Ref a) const {
using Return = mask_t<Derived>;
#if defined(ENOKI_X86_AVX512VL)
return Return::from_k(_mm_cmpeq_epi32_mask(m, a.m));
#else
return Return(_mm_cmpeq_epi32(m, a.m));
#endif
}
ENOKI_INLINE auto neq_(Ref a) const {
#if defined(ENOKI_X86_AVX512VL)
return mask_t<Derived>::from_k(_mm_cmpneq_epi32_mask(m, a.m));
#else
return ~eq_(a);
#endif
}
ENOKI_INLINE auto lt_(Ref a) const {
using Return = mask_t<Derived>;
#if !defined(ENOKI_X86_AVX512VL)
if constexpr (std::is_signed_v<Value>) {
return Return(_mm_cmpgt_epi32(a.m, m));
} else {
const __m128i offset = _mm_set1_epi32((int32_t) 0x80000000ul);
return Return(_mm_cmpgt_epi32(_mm_sub_epi32(a.m, offset),
_mm_sub_epi32(m, offset)));
}
#else
return Return::from_k(std::is_signed_v<Value>
? _mm_cmplt_epi32_mask(m, a.m)
: _mm_cmplt_epu32_mask(m, a.m));
#endif
}
ENOKI_INLINE auto gt_(Ref a) const {
using Return = mask_t<Derived>;
#if !defined(ENOKI_X86_AVX512VL)
if constexpr (std::is_signed_v<Value>) {
return Return(_mm_cmpgt_epi32(m, a.m));
} else {
const __m128i offset = _mm_set1_epi32((int32_t) 0x80000000ul);
return Return(_mm_cmpgt_epi32(_mm_sub_epi32(m, offset),
_mm_sub_epi32(a.m, offset)));
}
#else
return Return::from_k(std::is_signed_v<Value>
? _mm_cmpgt_epi32_mask(m, a.m)
: _mm_cmpgt_epu32_mask(m, a.m));
#endif
}
ENOKI_INLINE auto le_(Ref a) const {
#if defined(ENOKI_X86_AVX512VL)
return mask_t<Derived>::from_k(std::is_signed_v<Value>
? _mm_cmple_epi32_mask(m, a.m)
: _mm_cmple_epu32_mask(m, a.m));
#else
return ~gt_(a);
#endif
}
ENOKI_INLINE auto ge_(Ref a) const {
#if defined(ENOKI_X86_AVX512VL)
return mask_t<Derived>::from_k(std::is_signed_v<Value>
? _mm_cmpge_epi32_mask(m, a.m)
: _mm_cmpge_epu32_mask(m, a.m));
#else
return ~lt_(a);
#endif
}
ENOKI_INLINE Derived min_(Ref a) const {
return std::is_signed_v<Value> ? _mm_min_epi32(a.m, m)
: _mm_min_epu32(a.m, m);
}
ENOKI_INLINE Derived max_(Ref a) const {
return std::is_signed_v<Value> ? _mm_max_epi32(a.m, m)
: _mm_max_epu32(a.m, m);
}
ENOKI_INLINE Derived abs_() const {
return std::is_signed_v<Value> ? _mm_abs_epi32(m) : m;
}
template <typename Mask>
static ENOKI_INLINE Derived select_(const Mask &m, Ref t, Ref f) {
#if !defined(ENOKI_X86_AVX512VL)
return _mm_blendv_epi8(f.m, t.m, m.m);
#else
return _mm_mask_blend_epi32(m.k, f.m, t.m);
#endif
}
template <int I0, int I1, int I2, int I3>
ENOKI_INLINE Derived shuffle_() const {
return _mm_shuffle_epi32(m, _MM_SHUFFLE(I3, I2, I1, I0));
}
template <typename Index>
ENOKI_INLINE Derived shuffle_(const Index &index) const {
#if defined(ENOKI_X86_AVX)
return _mm_castps_si128(_mm_permutevar_ps(_mm_castsi128_ps(m), index.m));
#else
return Base::shuffle_(index);
#endif
}
ENOKI_INLINE Derived mulhi_(Ref a) const {
Derived even, odd;
if constexpr (std::is_signed_v<Value>) {
even.m = _mm_srli_epi64(_mm_mul_epi32(m, a.m), 32);
odd.m = _mm_mul_epi32(_mm_srli_epi64(m, 32), _mm_srli_epi64(a.m, 32));
} else {
even.m = _mm_srli_epi64(_mm_mul_epu32(m, a.m), 32);
odd.m = _mm_mul_epu32(_mm_srli_epi64(m, 32), _mm_srli_epi64(a.m, 32));
}
#if defined(ENOKI_X86_AVX512VL)
const mask_t<Derived> blend = mask_t<Derived>::from_k(0b0101);
#else
const mask_t<Derived> blend(Value(-1), Value(0), Value(-1), Value(0));
#endif
return select(blend, even, odd);
}
#if defined(ENOKI_X86_AVX512CD) && defined(ENOKI_X86_AVX512VL)
ENOKI_INLINE Derived lzcnt_() const { return _mm_lzcnt_epi32(m); }
ENOKI_INLINE Derived tzcnt_() const { return Value(32) - lzcnt(~derived() & (derived() - Value(1))); }
#endif
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Horizontal operations
// -----------------------------------------------------------------------
#define ENOKI_HORIZONTAL_OP(name, op) \
ENOKI_INLINE Value name##_() const { \
__m128i t1 = _mm_shuffle_epi32(m, 0x4e); \
__m128i t2 = _mm_##op##_epi32(m, t1); \
t1 = _mm_shufflelo_epi16(t2, 0x4e); \
t2 = _mm_##op##_epi32(t2, t1); \
return (Value) _mm_cvtsi128_si32(t2); \
}
#define ENOKI_HORIZONTAL_OP_SIGNED(name, op) \
ENOKI_INLINE Value name##_() const { \
__m128i t1 = _mm_shuffle_epi32(m, 0x4e); \
__m128i t2 = std::is_signed_v<Value> ? _mm_##op##_epi32(m, t1) : \
_mm_##op##_epu32(m, t1); \
t1 = _mm_shufflelo_epi16(t2, 0x4e); \
t2 = std::is_signed_v<Value> ? _mm_##op##_epi32(t2, t1) : \
_mm_##op##_epu32(t2, t1); \
return (Value) _mm_cvtsi128_si32(t2); \
}
ENOKI_HORIZONTAL_OP(hsum, add)
ENOKI_HORIZONTAL_OP(hprod, mullo)
ENOKI_HORIZONTAL_OP_SIGNED(hmin, min)
ENOKI_HORIZONTAL_OP_SIGNED(hmax, max)
#undef ENOKI_HORIZONTAL_OP
#undef ENOKI_HORIZONTAL_OP_SIGNED
ENOKI_INLINE bool all_() const { return _mm_movemask_ps(_mm_castsi128_ps(m)) == 0xF;}
ENOKI_INLINE bool any_() const { return _mm_movemask_ps(_mm_castsi128_ps(m)) != 0x0; }
ENOKI_INLINE uint32_t bitmask_() const { return (uint32_t) _mm_movemask_ps(_mm_castsi128_ps(m)); }
ENOKI_INLINE size_t count_() const { return (size_t) _mm_popcnt_u32(bitmask_()); }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Masked versions of key operations
// -----------------------------------------------------------------------
#if defined(ENOKI_X86_AVX512VL)
template <typename Mask>
ENOKI_INLINE void massign_(const Derived &a, const Mask &mask) { m = _mm_mask_mov_epi32(m, mask.k, a.m); }
template <typename Mask>
ENOKI_INLINE void madd_ (const Derived &a, const Mask &mask) { m = _mm_mask_add_epi32(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void msub_ (const Derived &a, const Mask &mask) { m = _mm_mask_sub_epi32(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mmul_ (const Derived &a, const Mask &mask) { m = _mm_mask_mullo_epi32(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mor_ (const Derived &a, const Mask &mask) { m = _mm_mask_or_epi32(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mand_ (const Derived &a, const Mask &mask) { m = _mm_mask_and_epi32(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mxor_ (const Derived &a, const Mask &mask) { m = _mm_mask_xor_epi32(m, mask.k, m, a.m); }
#endif
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Initialization, loading/writing data
// -----------------------------------------------------------------------
ENOKI_INLINE void store_(void *ptr) const {
assert((uintptr_t) ptr % 16 == 0);
_mm_store_si128((__m128i *) ENOKI_ASSUME_ALIGNED(ptr, 16), m);
}
template <typename Mask>
ENOKI_INLINE void store_(void *ptr, const Mask &mask) const {
#if defined(ENOKI_X86_AVX512VL)
_mm_mask_store_epi32(ptr, mask.k, m);
#elif defined(ENOKI_X86_AVX2)
_mm_maskstore_epi32((int *) ptr, mask.m, m);
#else
Base::store_(ptr, mask);
#endif
}
ENOKI_INLINE void store_unaligned_(void *ptr) const {
_mm_storeu_si128((__m128i *) ptr, m);
}
template <typename Mask>
ENOKI_INLINE void store_unaligned_(void *ptr, const Mask &mask) const {
#if defined(ENOKI_X86_AVX512VL)
_mm_mask_storeu_epi32(ptr, mask.k, m);
#elif defined(ENOKI_X86_AVX2)
_mm_maskstore_epi32((int *) ptr, mask.m, m);
#else
Base::store_unaligned_(ptr, mask);
#endif
}
static ENOKI_INLINE Derived load_(const void *ptr) {
assert((uintptr_t) ptr % 16 == 0);
return _mm_load_si128((const __m128i *) ENOKI_ASSUME_ALIGNED(ptr, 16));
}
template <typename Mask>
static ENOKI_INLINE Derived load_(const void *ptr, const Mask &mask) {
#if defined(ENOKI_X86_AVX512VL)
return _mm_maskz_load_epi32(mask.k, ptr);
#elif defined(ENOKI_X86_AVX2)
return _mm_maskload_epi32((const int *) ptr, mask.m);
#else
return Base::load_(ptr, mask);
#endif
}
static ENOKI_INLINE Derived load_unaligned_(const void *ptr) {
return _mm_loadu_si128((const __m128i *) ptr);
}
template <typename Mask>
static ENOKI_INLINE Derived load_unaligned_(const void *ptr, const Mask &mask) {
#if defined(ENOKI_X86_AVX512VL)
return _mm_maskz_loadu_epi32(mask.k, ptr);
#elif defined(ENOKI_X86_AVX2)
return _mm_maskload_epi32((const int *) ptr, mask.m);
#else
return Base::load_unaligned_(ptr, mask);
#endif
}
static ENOKI_INLINE Derived zero_() { return _mm_setzero_si128(); }
#if defined(ENOKI_X86_AVX2)
template <size_t Stride, typename Index, typename Mask>
static ENOKI_INLINE Derived gather_(const void *ptr, const Index &index, const Mask &mask) {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (sizeof(scalar_t<Index>) == 4)
return _mm_mmask_i32gather_epi32(_mm_setzero_si128(), mask.k, index.m, (const int *) ptr, Stride);
else
return _mm256_mmask_i64gather_epi32(_mm_setzero_si128(), mask.k, index.m, (const int *) ptr, Stride);
#else
if constexpr (sizeof(scalar_t<Index>) == 4)
return _mm_mask_i32gather_epi32(_mm_setzero_si128(), (const int *) ptr, index.m, mask.m, Stride);
else
return _mm256_mask_i64gather_epi32(_mm_setzero_si128(), (const int *) ptr, index.m, mask.m, Stride);
#endif
}
#endif
#if defined(ENOKI_X86_AVX512VL)
template <size_t Stride, typename Index, typename Mask>
ENOKI_INLINE void scatter_(void *ptr, const Index &index, const Mask &mask) const {
if constexpr (sizeof(scalar_t<Index>) == 4)
_mm_mask_i32scatter_epi32(ptr, mask.k, index.m, m, Stride);
else
_mm256_mask_i64scatter_epi32(ptr, mask.k, index.m, m, Stride);
}
#endif
template <typename Mask>
ENOKI_INLINE Value extract_(const Mask &mask) const {
#if !defined(ENOKI_X86_AVX512VL)
unsigned int k = (unsigned int) _mm_movemask_ps(_mm_castsi128_ps(mask.m));
return coeff((size_t) (detail::tzcnt_scalar(k) & 3));
#else
return (Value) _mm_cvtsi128_si32(_mm_mask_compress_epi32(_mm_setzero_si128(), mask.k, m));
#endif
}
template <typename T, typename Mask>
ENOKI_INLINE size_t compress_(T *&ptr, const Mask &mask) const {
#if !defined(ENOKI_X86_AVX512VL)
unsigned int k = (unsigned int) _mm_movemask_ps(_mm_castsi128_ps(mask.m));
/** Fancy LUT-based partitioning algorithm, see
https://deplinenoise.files.wordpress.com/2015/03/gdc2015_afredriksson_simd.pdf */
__m128i shuf = _mm_load_si128(((const __m128i *) detail::compress_lut_128) + k),
perm = _mm_shuffle_epi8(m, shuf);
_mm_storeu_si128((__m128i *) ptr, perm);
#else
_mm_storeu_si128((__m128i *) ptr,
_mm_mask_compress_epi32(_mm_setzero_si128(), mask.k, m));
unsigned int k = (unsigned int) mask.k;
#endif
size_t kn = (size_t) _mm_popcnt_u32(k);
ptr += kn;
return kn;
}
//! @}
// -----------------------------------------------------------------------
} ENOKI_MAY_ALIAS;
/// Partial overload of StaticArrayImpl using SSE4.2 intrinsics (64 bit integers)
template <typename Value_, bool IsMask_, typename Derived_> struct alignas(16)
StaticArrayImpl<Value_, 2, IsMask_, Derived_, enable_if_int64_t<Value_>>
: StaticArrayBase<Value_, 2, IsMask_, Derived_> {
ENOKI_NATIVE_ARRAY(Value_, 2, __m128i)
// -----------------------------------------------------------------------
//! @{ \name Value constructors
// -----------------------------------------------------------------------
ENOKI_INLINE StaticArrayImpl(Value value) : m(_mm_set1_epi64x((int64_t) value)) { }
ENOKI_INLINE StaticArrayImpl(Value v0, Value v1) {
alignas(16) Value data[2];
data[0] = (Value) v0;
data[1] = (Value) v1;
m = _mm_load_si128((__m128i *) data);
}
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Type converting constructors
// -----------------------------------------------------------------------
#if defined(ENOKI_X86_AVX512DQ) && defined(ENOKI_X86_AVX512VL)
ENOKI_CONVERT(double) {
if constexpr (std::is_signed_v<Value>)
m = _mm_cvttpd_epi64(a.derived().m);
else
m = _mm_cvttpd_epu64(a.derived().m);
}
#endif
ENOKI_CONVERT(int64_t) : m(a.derived().m) { }
ENOKI_CONVERT(uint64_t) : m(a.derived().m) { }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Reinterpreting constructors, mask converters
// -----------------------------------------------------------------------
ENOKI_REINTERPRET(bool) {
int16_t ival;
memcpy(&ival, a.derived().data(), 2);
m = _mm_cvtepi8_epi64(
_mm_cmpgt_epi8(_mm_cvtsi32_si128((int) ival), _mm_setzero_si128()));
}
ENOKI_REINTERPRET(float) {
ENOKI_TRACK_SCALAR("Constructor (reinterpreting, float32[2] -> int64[2])");
auto v0 = a.derived().coeff(0), v1 = a.derived().coeff(1);
m = _mm_castps_si128(_mm_setr_ps(v0, v0, v1, v1));
}
ENOKI_REINTERPRET(int32_t) {
ENOKI_TRACK_SCALAR("Constructor (reinterpreting, int32[2] -> int64[2])");
auto v0 = a.derived().coeff(0), v1 = a.derived().coeff(1);
m = _mm_setr_epi32(v0, v0, v1, v1);
}
ENOKI_REINTERPRET(uint32_t) {
ENOKI_TRACK_SCALAR("Constructor (reinterpreting, uint32[2] -> int64[2])");
auto v0 = a.derived().coeff(0), v1 = a.derived().coeff(1);
m = _mm_setr_epi32((int32_t) v0, (int32_t) v0, (int32_t) v1,
(int32_t) v1);
}
ENOKI_REINTERPRET(double) : m(_mm_castpd_si128(a.derived().m)) { }
ENOKI_REINTERPRET(int64_t) : m(a.derived().m) { }
ENOKI_REINTERPRET(uint64_t) : m(a.derived().m) { }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Converting from/to half size vectors
// -----------------------------------------------------------------------
StaticArrayImpl(const Array1 &a1, const Array2 &a2) {
alignas(16) Value data[2];
data[0] = (Value) a1.coeff(0);
data[1] = (Value) a2.coeff(0);
m = _mm_load_si128((__m128i *) data);
}
ENOKI_INLINE Array1 low_() const { return Array1(coeff(0)); }
ENOKI_INLINE Array2 high_() const { return Array2(coeff(1)); }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Vertical operations
// -----------------------------------------------------------------------
ENOKI_INLINE Derived add_(Ref a) const { return _mm_add_epi64(m, a.m); }
ENOKI_INLINE Derived sub_(Ref a) const { return _mm_sub_epi64(m, a.m); }
ENOKI_INLINE Derived mul_(Ref a) const {
#if defined(ENOKI_X86_AVX512DQ) && defined(ENOKI_X86_AVX512VL)
return _mm_mullo_epi64(m, a.m);
#else
Derived result;
ENOKI_TRACK_SCALAR("mul");
for (size_t i = 0; i < Size; ++i)
result.coeff(i) = coeff(i) * a.coeff(i);
return result;
#endif
}
template <typename T> ENOKI_INLINE Derived or_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_mask_mov_epi64(m, a.k, _mm_set1_epi64x(-1));
else
#endif
return _mm_or_si128(m, a.m);
}
template <typename T> ENOKI_INLINE Derived and_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_maskz_mov_epi64(a.k, m);
else
#endif
return _mm_and_si128(m, a.m);
}
template <typename T> ENOKI_INLINE Derived xor_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_mask_xor_epi64(m, a.k, m, _mm_set1_epi64x(-1));
else
#endif
return _mm_xor_si128(m, a.m);
}
template <typename T> ENOKI_INLINE Derived andnot_(const T &a) const {
#if defined(ENOKI_X86_AVX512VL)
if constexpr (is_mask_v<T>)
return _mm_mask_mov_epi64(m, a.k, _mm_setzero_si128());
else
#endif
return _mm_andnot_si128(a.m, m);
}
template <size_t k> ENOKI_INLINE Derived sl_() const {
return _mm_slli_epi64(m, (int) k);
}
template <size_t k> ENOKI_INLINE Derived sr_() const {
if constexpr (std::is_signed_v<Value>) {
#if defined(ENOKI_X86_AVX512VL)
return _mm_srai_epi64(m, (int) k);
#else
Derived out;
ENOKI_TRACK_SCALAR("sr");
for (size_t i = 0; i < Size; ++i)
out.coeff(i) = coeff(i) >> k;
return out;
#endif
} else {
return _mm_srli_epi64(m, (int) k);
}
}
ENOKI_INLINE Derived sl_(size_t k) const {
return _mm_sll_epi64(m, _mm_set1_epi64x((long long) k));
}
ENOKI_INLINE Derived sr_(size_t k) const {
if constexpr (std::is_signed_v<Value>) {
#if defined(ENOKI_X86_AVX512VL)
return _mm_sra_epi64(m, _mm_set1_epi64x((long long) k));
#else
Derived out;
ENOKI_TRACK_SCALAR("sr");
for (size_t i = 0; i < Size; ++i)
out.coeff(i) = coeff(i) >> k;
return out;
#endif
} else {
return _mm_srl_epi64(m, _mm_set1_epi64x((long long) k));
}
}
ENOKI_INLINE Derived sl_(Ref k) const {
#if defined(ENOKI_X86_AVX2)
return _mm_sllv_epi64(m, k.m);
#else
Derived out;
ENOKI_TRACK_SCALAR("sl");
for (size_t i = 0; i < Size; ++i)
out.coeff(i) = coeff(i) << (unsigned int) k.coeff(i);
return out;
#endif
}
ENOKI_INLINE Derived sr_(Ref k) const {
if constexpr (std::is_signed_v<Value>) {
#if defined(ENOKI_X86_AVX512VL)
return _mm_srav_epi64(m, k.m);
#endif
} else {
#if defined(ENOKI_X86_AVX2)
return _mm_srlv_epi64(m, k.m);
#endif
}
Derived out;
ENOKI_TRACK_SCALAR("sr");
for (size_t i = 0; i < Size; ++i)
out.coeff(i) = coeff(i) >> (unsigned int) k.coeff(i);
return out;
}
#if defined(ENOKI_X86_AVX512VL)
template <size_t Imm> ENOKI_INLINE Derived rol_() const { return _mm_rol_epi64(m, (int) Imm); }
template <size_t Imm> ENOKI_INLINE Derived ror_() const { return _mm_ror_epi64(m, (int) Imm); }
ENOKI_INLINE Derived rol_(Ref k) const { return _mm_rolv_epi64(m, k.m); }
ENOKI_INLINE Derived ror_(Ref k) const { return _mm_rorv_epi64(m, k.m); }
#endif
ENOKI_INLINE auto eq_(Ref a) const {
using Return = mask_t<Derived>;
#if defined(ENOKI_X86_AVX512VL)
return Return::from_k(_mm_cmpeq_epi64_mask(m, a.m));
#else
return Return(_mm_cmpeq_epi64(m, a.m));
#endif
}
ENOKI_INLINE auto neq_(Ref a) const {
#if defined(ENOKI_X86_AVX512VL)
return mask_t<Derived>::from_k(_mm_cmpneq_epi64_mask(m, a.m));
#else
return ~eq_(a);
#endif
}
ENOKI_INLINE auto lt_(Ref a) const {
using Return = mask_t<Derived>;
#if !defined(ENOKI_X86_AVX512VL)
if constexpr (std::is_signed_v<Value>) {
return Return(_mm_cmpgt_epi64(a.m, m));
} else {
const __m128i offset =
_mm_set1_epi64x((long long) 0x8000000000000000ull);
return Return(_mm_cmpgt_epi64(
_mm_sub_epi64(a.m, offset),
_mm_sub_epi64(m, offset)
));
}
#else
return Return::from_k(std::is_signed_v<Value>
? _mm_cmplt_epi64_mask(m, a.m)
: _mm_cmplt_epu64_mask(m, a.m));
#endif
}
ENOKI_INLINE auto gt_(Ref a) const {
using Return = mask_t<Derived>;
#if !defined(ENOKI_X86_AVX512VL)
if constexpr (std::is_signed_v<Value>) {
return Return(_mm_cmpgt_epi64(m, a.m));
} else {
const __m128i offset =
_mm_set1_epi64x((long long) 0x8000000000000000ull);
return Return(_mm_cmpgt_epi64(
_mm_sub_epi64(m, offset),
_mm_sub_epi64(a.m, offset)
));
}
#else
return Return::from_k(std::is_signed_v<Value>
? _mm_cmpgt_epi64_mask(m, a.m)
: _mm_cmpgt_epu64_mask(m, a.m));
#endif
}
ENOKI_INLINE auto le_(Ref a) const {
#if defined(ENOKI_X86_AVX512VL)
return mask_t<Derived>::from_k(std::is_signed_v<Value>
? _mm_cmple_epi64_mask(m, a.m)
: _mm_cmple_epu64_mask(m, a.m));
#else
return ~gt_(a);
#endif
}
ENOKI_INLINE auto ge_(Ref a) const {
#if defined(ENOKI_X86_AVX512VL)
return mask_t<Derived>::from_k(std::is_signed_v<Value>
? _mm_cmpge_epi64_mask(m, a.m)
: _mm_cmpge_epu64_mask(m, a.m));
#else
return ~lt_(a);
#endif
}
ENOKI_INLINE Derived min_(Ref a) const {
#if defined(ENOKI_X86_AVX512VL)
return std::is_signed_v<Value> ? _mm_min_epi64(a.m, m)
: _mm_min_epu64(a.m, m);
#else
return select(derived() < a, derived(), a);
#endif
}
ENOKI_INLINE Derived max_(Ref a) const {
#if defined(ENOKI_X86_AVX512VL)
return std::is_signed_v<Value> ? _mm_max_epi64(a.m, m)
: _mm_max_epu64(a.m, m);
#else
return select(derived() > a, derived(), a);
#endif
}
ENOKI_INLINE Derived abs_() const {
if constexpr (std::is_signed_v<Value>) {
#if defined(ENOKI_X86_AVX512VL)
return _mm_abs_epi64(m);
#else
return select(derived() < zero<Derived>(),
~derived() + Derived(Value(1)), derived());
#endif
} else {
return m;
}
}
template <typename Mask>
static ENOKI_INLINE Derived select_(const Mask &m, Ref t, Ref f) {
#if !defined(ENOKI_X86_AVX512VL)
return _mm_blendv_epi8(f.m, t.m, m.m);
#else
return _mm_mask_blend_epi64(m.k, f.m, t.m);
#endif
}
ENOKI_INLINE Derived mulhi_(Ref a) const {
ENOKI_TRACK_SCALAR("mulhi");
return Derived(
mulhi(coeff(0), a.coeff(0)),
mulhi(coeff(1), a.coeff(1))
);
}
template <int I0, int I1>
ENOKI_INLINE Derived shuffle_() const {
return _mm_shuffle_epi32(
m, _MM_SHUFFLE(I1 * 2 + 1, I1 * 2, I0 * 2 + 1, I0 * 2));
}
template <typename Index>
ENOKI_INLINE Derived shuffle_(const Index &index) const {
#if defined(ENOKI_X86_AVX)
return _mm_castpd_si128(_mm_permutevar_pd(_mm_castsi128_pd(m), _mm_slli_epi64(index.m, 1)));
#else
return Base::shuffle_(index);
#endif
}
#if defined(ENOKI_X86_AVX512CD) && defined(ENOKI_X86_AVX512VL)
ENOKI_INLINE Derived lzcnt_() const { return _mm_lzcnt_epi64(m); }
ENOKI_INLINE Derived tzcnt_() const { return Value(64) - lzcnt(~derived() & (derived() - Value(1))); }
#endif
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Masked versions of key operations
// -----------------------------------------------------------------------
#if defined(ENOKI_X86_AVX512VL)
template <typename Mask>
ENOKI_INLINE void massign_(const Derived &a, const Mask &mask) { m = _mm_mask_mov_epi64(m, mask.k, a.m); }
template <typename Mask>
ENOKI_INLINE void madd_ (const Derived &a, const Mask &mask) { m = _mm_mask_add_epi64(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void msub_ (const Derived &a, const Mask &mask) { m = _mm_mask_sub_epi64(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mmul_ (const Derived &a, const Mask &mask) { m = _mm_mask_mullo_epi64(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mor_ (const Derived &a, const Mask &mask) { m = _mm_mask_or_epi64(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mand_ (const Derived &a, const Mask &mask) { m = _mm_mask_and_epi64(m, mask.k, m, a.m); }
template <typename Mask>
ENOKI_INLINE void mxor_ (const Derived &a, const Mask &mask) { m = _mm_mask_xor_epi64(m, mask.k, m, a.m); }
#endif
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Horizontal operations
// -----------------------------------------------------------------------
#define ENOKI_HORIZONTAL_OP(name, op) \
ENOKI_INLINE Value name##_() const { \
Value t1 = Value(detail::mm_extract_epi64<1>(m)); \
Value t2 = Value(detail::mm_cvtsi128_si64(m)); \
return op; \
}
ENOKI_HORIZONTAL_OP(hsum, t1 + t2)
ENOKI_HORIZONTAL_OP(hprod, t1 * t2)
ENOKI_HORIZONTAL_OP(hmin, min(t1, t2))
ENOKI_HORIZONTAL_OP(hmax, max(t1, t2))
#undef ENOKI_HORIZONTAL_OP
ENOKI_INLINE bool all_() const { return _mm_movemask_pd(_mm_castsi128_pd(m)) == 0x3;}
ENOKI_INLINE bool any_() const { return _mm_movemask_pd(_mm_castsi128_pd(m)) != 0x0; }
ENOKI_INLINE uint32_t bitmask_() const { return (uint32_t) _mm_movemask_pd(_mm_castsi128_pd(m)); }
ENOKI_INLINE size_t count_() const { return (size_t) _mm_popcnt_u32(bitmask_()); }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Initialization, loading/writing data
// -----------------------------------------------------------------------
ENOKI_INLINE void store_(void *ptr) const {
assert((uintptr_t) ptr % 16 == 0);
_mm_store_si128((__m128i *) ENOKI_ASSUME_ALIGNED(ptr, 16), m);
}
template <typename Mask>
ENOKI_INLINE void store_(void *ptr, const Mask &mask) const {
#if defined(ENOKI_X86_AVX512VL)
_mm_mask_store_epi64(ptr, mask.k, m);
#elif defined(ENOKI_X86_AVX2)
_mm_maskstore_epi64((long long *) ptr, mask.m, m);
#else
Base::store_(ptr, mask);
#endif
}
ENOKI_INLINE void store_unaligned_(void *ptr) const {
_mm_storeu_si128((__m128i *) ptr, m);
}
template <typename Mask>
ENOKI_INLINE void store_unaligned_(void *ptr, const Mask &mask) const {
#if defined(ENOKI_X86_AVX512VL)
_mm_mask_storeu_epi64(ptr, mask.k, m);
#elif defined(ENOKI_X86_AVX2)
_mm_maskstore_epi64((long long *) ptr, mask.m, m);
#else
Base::store_unaligned_(ptr, mask);
#endif
}
static ENOKI_INLINE Derived load_(const void *ptr) {
assert((uintptr_t) ptr % 16 == 0);
return _mm_load_si128((const __m128i *) ENOKI_ASSUME_ALIGNED(ptr, 16));
}
template <typename Mask>
static ENOKI_INLINE Derived load_(const void *ptr, const Mask &mask) {
#if defined(ENOKI_X86_AVX512VL)
return _mm_maskz_load_epi64(mask.k, ptr);
#elif defined(ENOKI_X86_AVX2)
return _mm_maskload_epi64((const long long *) ptr, mask.m);
#else
return Base::load_(ptr, mask);
#endif
}
static ENOKI_INLINE Derived load_unaligned_(const void *ptr) {
return _mm_loadu_si128((const __m128i *) ptr);
}
template <typename Mask>
static ENOKI_INLINE Derived load_unaligned_(const void *ptr, const Mask &mask) {
#if defined(ENOKI_X86_AVX512VL)
return _mm_maskz_loadu_epi64(mask.k, ptr);
#elif defined(ENOKI_X86_AVX2)
return _mm_maskload_epi64((const long long *) ptr, mask.m);
#else
return Base::load_unaligned_(ptr, mask);
#endif
}
static ENOKI_INLINE Derived zero_() { return _mm_setzero_si128(); }
#if defined(ENOKI_X86_AVX2)
template <size_t Stride, typename Index, typename Mask>
static ENOKI_INLINE Derived gather_(const void *ptr, const Index &index, const Mask &mask) {
if constexpr (sizeof(scalar_t<Index>) == 4) {
return Base::template gather_<Stride>(ptr, index, mask);
} else {
#if defined(ENOKI_X86_AVX512VL)
return _mm_mmask_i64gather_epi64(_mm_setzero_si128(), mask.k, index.m, (const long long *) ptr, Stride);
#else
return _mm_mask_i64gather_epi64(_mm_setzero_si128(), (const long long *) ptr, index.m, mask.m, Stride);
#endif
}
}
#endif
#if defined(ENOKI_X86_AVX512VL)
template <size_t Stride, typename Index, typename Mask>
ENOKI_INLINE void scatter_(void *ptr, const Index &index, const Mask &mask) const {
if constexpr (sizeof(scalar_t<Index>) == 4)
Base::template scatter_<Stride>(ptr, index, mask);
else
_mm_mask_i64scatter_epi64(ptr, mask.k, index.m, m, Stride);
}
template <typename Mask>
ENOKI_INLINE Value extract_(const Mask &mask) const {
return (Value) detail::mm_cvtsi128_si64(_mm_mask_compress_epi64(_mm_setzero_si128(), mask.k, m));
}
template <typename Mask>
ENOKI_INLINE size_t compress_(Value_ *&ptr, const Mask &mask) const {
_mm_storeu_si128((__m128i *) ptr, _mm_mask_compress_epi64(_mm_setzero_si128(), mask.k, m));
size_t kn = (size_t) _mm_popcnt_u32(mask.k);
ptr += kn;
return kn;
}
#endif
//! @}
// -----------------------------------------------------------------------
} ENOKI_MAY_ALIAS;
/// Partial overload of StaticArrayImpl for the n=3 case (single precision)
template <bool IsMask_, typename Derived_> struct alignas(16)
StaticArrayImpl<float, 3, IsMask_, Derived_>
: StaticArrayImpl<float, 4, IsMask_, Derived_> {
using Base = StaticArrayImpl<float, 4, IsMask_, Derived_>;
ENOKI_DECLARE_3D_ARRAY(StaticArrayImpl)
#if defined(ENOKI_X86_F16C)
template <typename Derived2>
ENOKI_INLINE StaticArrayImpl(
const StaticArrayBase<half, 3, IsMask_, Derived2> &a) {
uint16_t temp[4];
memcpy(temp, a.derived().data(), sizeof(uint16_t) * 3);
temp[3] = 0;
m = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *) temp));
}
#endif
template <int I0, int I1, int I2>
ENOKI_INLINE Derived shuffle_() const {
return Base::template shuffle_<I0, I1, I2, 3>();
}
template <typename Index>
ENOKI_INLINE Derived shuffle_(const Index &index) const {
return Base::shuffle_(index);
}
// -----------------------------------------------------------------------
//! @{ \name Horizontal operations (adapted for the n=3 case)
// -----------------------------------------------------------------------
#define ENOKI_HORIZONTAL_OP(name, op) \
ENOKI_INLINE Value name##_() const { \
__m128 t1 = _mm_movehl_ps(m, m); \
__m128 t2 = _mm_##op##_ss(m, t1); \
t1 = _mm_movehdup_ps(m); \
t1 = _mm_##op##_ss(t1, t2); \
return _mm_cvtss_f32(t1); \
}
ENOKI_HORIZONTAL_OP(hsum, add)
ENOKI_HORIZONTAL_OP(hprod, mul)
ENOKI_HORIZONTAL_OP(hmin, min)
ENOKI_HORIZONTAL_OP(hmax, max)
#undef ENOKI_HORIZONTAL_OP
ENOKI_INLINE Value dot_(Ref a) const {
return _mm_cvtss_f32(_mm_dp_ps(m, a.m, 0b01110001));
}
ENOKI_INLINE bool all_() const { return (_mm_movemask_ps(m) & 7) == 7; }
ENOKI_INLINE bool any_() const { return (_mm_movemask_ps(m) & 7) != 0; }
ENOKI_INLINE uint32_t bitmask_() const { return (uint32_t) _mm_movemask_ps(m) & 7; }
ENOKI_INLINE size_t count_() const { return (size_t) _mm_popcnt_u32(bitmask_()); }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Loading/writing data (adapted for the n=3 case)
// -----------------------------------------------------------------------
static ENOKI_INLINE auto mask_() {
#if defined(ENOKI_X86_AVX512VL)
return mask_t<Derived>::from_k((__mmask8) 7);
#else
return mask_t<Derived>(_mm_castsi128_ps(_mm_setr_epi32(-1, -1, -1, 0)));
#endif
}
using Base::load_;
using Base::load_unaligned_;
using Base::store_;
using Base::store_unaligned_;
ENOKI_INLINE void store_(void *ptr) const {
memcpy(ptr, &m, sizeof(Value) * 3);
}
ENOKI_INLINE void store_unaligned_(void *ptr) const {
store_(ptr);
}
static ENOKI_INLINE Derived load_(const void *ptr) {
return Base::load_unaligned_(ptr);
}
static ENOKI_INLINE Derived load_unaligned_(const void *ptr) {
Derived result;
memcpy(&result.m, ptr, sizeof(Value) * 3);
return result;
}
#if defined(ENOKI_X86_AVX)
template <typename Mask>
ENOKI_INLINE void store_(void *ptr, const Mask &mask) const {
Base::store_(ptr, mask & mask_());
}
template <typename Mask>
ENOKI_INLINE void store_unaligned_(void *ptr, const Mask &mask) const {
Base::store_unaligned_(ptr, mask & mask_());
}
template <typename Mask>
static ENOKI_INLINE Derived load_(const void *ptr, const Mask &mask) {
return Base::load_(ptr, mask & mask_());
}
template <typename Mask>
static ENOKI_INLINE Derived load_unaligned_(const void *ptr, const Mask &mask) {
return Base::load_unaligned_(ptr, mask & mask_());
}
#endif
#if defined(ENOKI_X86_AVX2)
template <size_t Stride, typename Index, typename Mask>
static ENOKI_INLINE Derived gather_(const void *ptr, const Index &index, const Mask &mask) {
return Base::template gather_<Stride>(ptr, index, mask & mask_());
}
#endif
#if defined(ENOKI_X86_AVX512VL)
template <size_t Stride, typename Index, typename Mask>
ENOKI_INLINE void scatter_(void *ptr, const Index &index, const Mask &mask) const {
Base::template scatter_<Stride>(ptr, index, mask & mask_());
}
#endif
template <typename Mask>
ENOKI_INLINE size_t compress_(float *&ptr, const Mask &mask) const {
return Base::compress_(ptr, mask & mask_());
}
//! @}
// -----------------------------------------------------------------------
} ENOKI_MAY_ALIAS;
/// Partial overload of StaticArrayImpl for the n=3 case (32 bit integers)
template <typename Value_, bool IsMask_, typename Derived_> struct alignas(16)
StaticArrayImpl<Value_, 3, IsMask_, Derived_, enable_if_int32_t<Value_>>
: StaticArrayImpl<Value_, 4, IsMask_, Derived_> {
using Base = StaticArrayImpl<Value_, 4, IsMask_, Derived_>;
ENOKI_DECLARE_3D_ARRAY(StaticArrayImpl)
template <int I0, int I1, int I2>
ENOKI_INLINE Derived shuffle_() const {
return Base::template shuffle_<I0, I1, I2, 3>();
}
template <typename Index>
ENOKI_INLINE Derived shuffle_(const Index &index) const {
return Base::shuffle_(index);
}
// -----------------------------------------------------------------------
//! @{ \name Horizontal operations (adapted for the n=3 case)
// -----------------------------------------------------------------------
#define ENOKI_HORIZONTAL_OP(name, op) \
ENOKI_INLINE Value name##_() const { \
__m128i t1 = _mm_unpackhi_epi32(m, m); \
__m128i t2 = _mm_##op##_epi32(m, t1); \
t1 = _mm_shuffle_epi32(m, 1); \
t1 = _mm_##op##_epi32(t1, t2); \
return (Value) _mm_cvtsi128_si32(t1); \
}
#define ENOKI_HORIZONTAL_OP_SIGNED(name, op) \
ENOKI_INLINE Value name##_() const { \
__m128i t2, t1 = _mm_unpackhi_epi32(m, m); \
if constexpr (std::is_signed<Value>::value) \
t2 = _mm_##op##_epi32(m, t1); \
else \
t2 = _mm_##op##_epu32(m, t1); \
t1 = _mm_shuffle_epi32(m, 1); \
if constexpr (std::is_signed<Value>::value) \
t1 = _mm_##op##_epi32(t1, t2); \
else \
t1 = _mm_##op##_epu32(t1, t2); \
return (Value) _mm_cvtsi128_si32(t1); \
}
ENOKI_HORIZONTAL_OP(hsum, add)
ENOKI_HORIZONTAL_OP(hprod, mullo)
ENOKI_HORIZONTAL_OP_SIGNED(hmin, min)
ENOKI_HORIZONTAL_OP_SIGNED(hmax, max)
#undef ENOKI_HORIZONTAL_OP
#undef ENOKI_HORIZONTAL_OP_SIGNED
ENOKI_INLINE bool all_() const { return (_mm_movemask_ps(_mm_castsi128_ps(m)) & 7) == 7;}
ENOKI_INLINE bool any_() const { return (_mm_movemask_ps(_mm_castsi128_ps(m)) & 7) != 0; }
ENOKI_INLINE uint32_t bitmask_() const { return (uint32_t) _mm_movemask_ps(_mm_castsi128_ps(m)) & 7; }
ENOKI_INLINE size_t count_() const { return (size_t) _mm_popcnt_u32(bitmask_()); }
//! @}
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
//! @{ \name Loading/writing data (adapted for the n=3 case)
// -----------------------------------------------------------------------
static ENOKI_INLINE auto mask_() {
#if defined(ENOKI_X86_AVX512VL)
return mask_t<Derived>::from_k((__mmask8) 7);
#else
return mask_t<Derived>(_mm_setr_epi32(-1, -1, -1, 0));
#endif
}
using Base::load_;
using Base::load_unaligned_;
using Base::store_;
using Base::store_unaligned_;
ENOKI_INLINE void store_(void *ptr) const {
memcpy(ptr, &m, sizeof(Value) * 3);
}
ENOKI_INLINE void store_unaligned_(void *ptr) const {
store_(ptr);
}
static ENOKI_INLINE Derived load_(const void *ptr) {
return Base::load_unaligned_(ptr);
}
static ENOKI_INLINE Derived load_unaligned_(const void *ptr) {
Derived result;
memcpy(&result.m, ptr, sizeof(Value) * 3);
return result;
}
#if defined(ENOKI_X86_AVX2)
template <typename Mask>
ENOKI_INLINE void store_unaligned_(void *ptr, const Mask &mask) const {
return Base::store_unaligned_(ptr, mask & mask_());
}
template <typename Mask>
ENOKI_INLINE void store_(void *ptr, const Mask &mask) const {
return Base::store_(ptr, mask & mask_());
}
template <typename Mask>
static ENOKI_INLINE Derived load_(const void *ptr, const Mask &mask) {
return Base::load_(ptr, mask & mask_());
}
template <typename Mask>
static ENOKI_INLINE Derived load_unaligned_(const void *ptr, const Mask &mask) {
return Base::load_unaligned_(ptr, mask & mask_());
}
#endif
#if defined(ENOKI_X86_AVX2)
template <size_t Stride, typename Index, typename Mask>
static ENOKI_INLINE Derived gather_(const void *ptr, const Index &index, const Mask &mask) {
return Base::template gather_<Stride>(ptr, index, mask & mask_());
}
#endif
#if defined(ENOKI_X86_AVX512VL)
template <size_t Stride, typename Index, typename Mask>
ENOKI_INLINE void scatter_(void *ptr, const Index &index, const Mask &mask) const {
Base::template scatter_<Stride>(ptr, index, mask & mask_());
}
#endif
template <typename T, typename Mask>
ENOKI_INLINE size_t compress_(T *&ptr, const Mask &mask) const {
return Base::compress_(ptr, mask & mask_());
}
//! @}
// -----------------------------------------------------------------------
} ENOKI_MAY_ALIAS;
#if defined(ENOKI_X86_AVX512VL)
template <typename Derived_>
ENOKI_DECLARE_KMASK(float, 4, Derived_, int)
template <typename Derived_>
ENOKI_DECLARE_KMASK(float, 3, Derived_, int)
template <typename Derived_>
ENOKI_DECLARE_KMASK(double, 2, Derived_, int)
template <typename Value_, typename Derived_>
ENOKI_DECLARE_KMASK(Value_, 4, Derived_, enable_if_int32_t<Value_>)
template <typename Value_, typename Derived_>
ENOKI_DECLARE_KMASK(Value_, 3, Derived_, enable_if_int32_t<Value_>)
template <typename Value_, typename Derived_>
ENOKI_DECLARE_KMASK(Value_, 2, Derived_, enable_if_int64_t<Value_>)
#endif
NAMESPACE_END(enoki)
Reference in New Issue View Git Blame Copy Permalink