Dante/Kmake
1
0
Fork 0
forked from LeenkxTeam/Kmake
Code Pull Requests Activity

Upload Kmake

Browse Source
This commit is contained in:
Gorochu
2026-05-26 23:36:42 -07:00
parent ba051b2f74
commit 555ec72358
41615 changed files with 13344630 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
deps/v8/third_party/highway/src/hwy/contrib/algo/copy-inl.h vendored Normal file
View File

@ -0,0 +1,145 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Per-target include guard
#if defined(HIGHWAY_HWY_CONTRIB_ALGO_COPY_INL_H_) == \
defined(HWY_TARGET_TOGGLE) // NOLINT
#ifdef HIGHWAY_HWY_CONTRIB_ALGO_COPY_INL_H_
#undef HIGHWAY_HWY_CONTRIB_ALGO_COPY_INL_H_
#else
#define HIGHWAY_HWY_CONTRIB_ALGO_COPY_INL_H_
#endif
#include <stddef.h>
#include <stdint.h>
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
// These functions avoid having to write a loop plus remainder handling in the
// (unfortunately still common) case where arrays are not aligned/padded. If the
// inputs are known to be aligned/padded, it is more efficient to write a single
// loop using Load(). We do not provide a CopyAlignedPadded because it
// would be more verbose than such a loop.
// Fills `to`[0, `count`) with `value`.
template <class D, typename T = TFromD<D>>
void Fill(D d, T value, size_t count, T* HWY_RESTRICT to) {
const size_t N = Lanes(d);
const Vec<D> v = Set(d, value);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
StoreU(v, d, to + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
SafeFillN(remaining, value, d, to + idx);
}
// Copies `from`[0, `count`) to `to`, which must not overlap `from`.
template <class D, typename T = TFromD<D>>
void Copy(D d, const T* HWY_RESTRICT from, size_t count, T* HWY_RESTRICT to) {
const size_t N = Lanes(d);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
const Vec<D> v = LoadU(d, from + idx);
StoreU(v, d, to + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
SafeCopyN(remaining, d, from + idx, to + idx);
}
// For idx in [0, count) in ascending order, appends `from[idx]` to `to` if the
// corresponding mask element of `func(d, v)` is true. Returns the STL-style end
// of the newly written elements in `to`.
//
// `func` is either a functor with a templated operator()(d, v) returning a
// mask, or a generic lambda if using C++14. Due to apparent limitations of
// Clang on Windows, it is currently necessary to add HWY_ATTR before the
// opening { of the lambda to avoid errors about "function .. requires target".
//
// NOTE: this is only supported for 16-, 32- or 64-bit types.
// NOTE: Func may be called a second time for elements it has already seen, but
// these elements will not be written to `to` again.
template <class D, class Func, typename T = TFromD<D>>
T* CopyIf(D d, const T* HWY_RESTRICT from, size_t count, T* HWY_RESTRICT to,
const Func& func) {
const size_t N = Lanes(d);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
const Vec<D> v = LoadU(d, from + idx);
to += CompressBlendedStore(v, func(d, v), d, to);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return to;
#if HWY_MEM_OPS_MIGHT_FAULT
// Proceed one by one.
const CappedTag<T, 1> d1;
for (; idx < count; ++idx) {
using V1 = Vec<decltype(d1)>;
// Workaround for -Waggressive-loop-optimizations on GCC 8
// (iteration 2305843009213693951 invokes undefined behavior for T=i64)
const uintptr_t addr = reinterpret_cast<uintptr_t>(from);
const T* HWY_RESTRICT from_idx =
reinterpret_cast<const T * HWY_RESTRICT>(addr + (idx * sizeof(T)));
const V1 v = LoadU(d1, from_idx);
// Avoid storing to `to` unless we know it should be kept - otherwise, we
// might overrun the end if it was allocated for the exact count.
if (CountTrue(d1, func(d1, v)) == 0) continue;
StoreU(v, d1, to);
to += 1;
}
#else
// Start index of the last unaligned whole vector, ending at the array end.
const size_t last = count - N;
// Number of elements before `from` or already written.
const size_t invalid = idx - last;
HWY_DASSERT(0 != invalid && invalid < N);
const Mask<D> mask = Not(FirstN(d, invalid));
const Vec<D> v = MaskedLoad(mask, d, from + last);
to += CompressBlendedStore(v, And(mask, func(d, v)), d, to);
#endif
return to;
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_ALGO_COPY_INL_H_

210
deps/v8/third_party/highway/src/hwy/contrib/algo/copy_test.cc vendored Normal file
View File

@ -0,0 +1,210 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stddef.h>
#include "hwy/aligned_allocator.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/algo/copy_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/algo/copy-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
// If your project requires C++14 or later, you can ignore this and pass lambdas
// directly to Transform, without requiring an lvalue as we do here for C++11.
#if __cplusplus < 201402L
#define HWY_GENERIC_LAMBDA 0
#else
#define HWY_GENERIC_LAMBDA 1
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
// Returns random integer in [0, 128), which fits in any lane type.
template <typename T>
T Random7Bit(RandomState& rng) {
return ConvertScalarTo<T>(Random32(&rng) & 127);
}
// In C++14, we can instead define these as generic lambdas next to where they
// are invoked.
#if !HWY_GENERIC_LAMBDA
struct IsOdd {
template <class D, class V>
Mask<D> operator()(D d, V v) const {
return TestBit(v, Set(d, TFromD<D>{1}));
}
};
#endif // !HWY_GENERIC_LAMBDA
// Invokes Test (e.g. TestCopyIf) with all arg combinations. T comes from
// ForFloatTypes.
template <class Test>
struct ForeachCountAndMisalign {
template <typename T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) const {
RandomState rng;
const size_t N = Lanes(d);
const size_t misalignments[3] = {0, N / 4, 3 * N / 5};
for (size_t count = 0; count < 2 * N; ++count) {
for (size_t ma : misalignments) {
for (size_t mb : misalignments) {
Test()(d, count, ma, mb, rng);
}
}
}
}
};
struct TestFill {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
// HWY_MAX prevents error when misalign == count == 0.
AlignedFreeUniquePtr<T[]> pa =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count));
AlignedFreeUniquePtr<T[]> pb = AllocateAligned<T>(misalign_b + count + 1);
HWY_ASSERT(pa && pb);
T* expected = pa.get() + misalign_a;
const T value = Random7Bit<T>(rng);
for (size_t i = 0; i < count; ++i) {
expected[i] = value;
}
T* actual = pb.get() + misalign_b;
actual[count] = ConvertScalarTo<T>(0); // sentinel
Fill(d, value, count, actual);
HWY_ASSERT_EQ(ConvertScalarTo<T>(0), actual[count]); // no write past end
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName(HWY_TARGET);
hwy::detail::AssertArrayEqual(info, expected, actual, count, target_name,
__FILE__, __LINE__);
}
};
void TestAllFill() {
ForAllTypes(ForPartialVectors<ForeachCountAndMisalign<TestFill>>());
}
struct TestCopy {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
// Prevents error if size to allocate is zero.
AlignedFreeUniquePtr<T[]> pa =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count));
AlignedFreeUniquePtr<T[]> pb =
AllocateAligned<T>(HWY_MAX(1, misalign_b + count));
HWY_ASSERT(pa && pb);
T* a = pa.get() + misalign_a;
for (size_t i = 0; i < count; ++i) {
a[i] = Random7Bit<T>(rng);
}
T* b = pb.get() + misalign_b;
Copy(d, a, count, b);
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName(HWY_TARGET);
hwy::detail::AssertArrayEqual(info, a, b, count, target_name, __FILE__,
__LINE__);
}
};
void TestAllCopy() {
ForAllTypes(ForPartialVectors<ForeachCountAndMisalign<TestCopy>>());
}
struct TestCopyIf {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
const size_t padding = Lanes(ScalableTag<T>());
// Prevents error if size to allocate is zero.
AlignedFreeUniquePtr<T[]> pa =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count));
AlignedFreeUniquePtr<T[]> pb =
AllocateAligned<T>(HWY_MAX(1, misalign_b + count + padding));
AlignedFreeUniquePtr<T[]> expected = AllocateAligned<T>(HWY_MAX(1, count));
HWY_ASSERT(pa && pb && expected);
T* a = pa.get() + misalign_a;
for (size_t i = 0; i < count; ++i) {
a[i] = Random7Bit<T>(rng);
}
T* b = pb.get() + misalign_b;
size_t num_odd = 0;
for (size_t i = 0; i < count; ++i) {
if (a[i] & 1) {
expected[num_odd++] = a[i];
}
}
#if HWY_GENERIC_LAMBDA
const auto is_odd = [](const auto d, const auto v) HWY_ATTR {
return TestBit(v, Set(d, TFromD<decltype(d)>{1}));
};
#else
const IsOdd is_odd;
#endif
T* end = CopyIf(d, a, count, b, is_odd);
const size_t num_written = static_cast<size_t>(end - b);
HWY_ASSERT_EQ(num_odd, num_written);
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName(HWY_TARGET);
hwy::detail::AssertArrayEqual(info, expected.get(), b, num_odd, target_name,
__FILE__, __LINE__);
}
};
void TestAllCopyIf() {
ForUI163264(ForPartialVectors<ForeachCountAndMisalign<TestCopyIf>>());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(CopyTest);
HWY_EXPORT_AND_TEST_P(CopyTest, TestAllFill);
HWY_EXPORT_AND_TEST_P(CopyTest, TestAllCopy);
HWY_EXPORT_AND_TEST_P(CopyTest, TestAllCopyIf);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

113
deps/v8/third_party/highway/src/hwy/contrib/algo/find-inl.h vendored Normal file
View File

@ -0,0 +1,113 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Per-target include guard
#if defined(HIGHWAY_HWY_CONTRIB_ALGO_FIND_INL_H_) == \
defined(HWY_TARGET_TOGGLE) // NOLINT
#ifdef HIGHWAY_HWY_CONTRIB_ALGO_FIND_INL_H_
#undef HIGHWAY_HWY_CONTRIB_ALGO_FIND_INL_H_
#else
#define HIGHWAY_HWY_CONTRIB_ALGO_FIND_INL_H_
#endif
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
// Returns index of the first element equal to `value` in `in[0, count)`, or
// `count` if not found.
template <class D, typename T = TFromD<D>>
size_t Find(D d, T value, const T* HWY_RESTRICT in, size_t count) {
const size_t N = Lanes(d);
const Vec<D> broadcasted = Set(d, value);
size_t i = 0;
if (count >= N) {
for (; i <= count - N; i += N) {
const intptr_t pos = FindFirstTrue(d, Eq(broadcasted, LoadU(d, in + i)));
if (pos >= 0) return i + static_cast<size_t>(pos);
}
}
if (i != count) {
#if HWY_MEM_OPS_MIGHT_FAULT
// Scan single elements.
const CappedTag<T, 1> d1;
using V1 = Vec<decltype(d1)>;
const V1 broadcasted1 = Set(d1, GetLane(broadcasted));
for (; i < count; ++i) {
if (AllTrue(d1, Eq(broadcasted1, LoadU(d1, in + i)))) {
return i;
}
}
#else
const size_t remaining = count - i;
HWY_DASSERT(0 != remaining && remaining < N);
const Mask<D> mask = FirstN(d, remaining);
const Vec<D> v = MaskedLoad(mask, d, in + i);
// Apply mask so that we don't 'find' the zero-padding from MaskedLoad.
const intptr_t pos = FindFirstTrue(d, And(Eq(broadcasted, v), mask));
if (pos >= 0) return i + static_cast<size_t>(pos);
#endif // HWY_MEM_OPS_MIGHT_FAULT
}
return count; // not found
}
// Returns index of the first element in `in[0, count)` for which `func(d, vec)`
// returns true, otherwise `count`.
template <class D, class Func, typename T = TFromD<D>>
size_t FindIf(D d, const T* HWY_RESTRICT in, size_t count, const Func& func) {
const size_t N = Lanes(d);
size_t i = 0;
if (count >= N) {
for (; i <= count - N; i += N) {
const intptr_t pos = FindFirstTrue(d, func(d, LoadU(d, in + i)));
if (pos >= 0) return i + static_cast<size_t>(pos);
}
}
if (i != count) {
#if HWY_MEM_OPS_MIGHT_FAULT
// Scan single elements.
const CappedTag<T, 1> d1;
for (; i < count; ++i) {
if (AllTrue(d1, func(d1, LoadU(d1, in + i)))) {
return i;
}
}
#else
const size_t remaining = count - i;
HWY_DASSERT(0 != remaining && remaining < N);
const Mask<D> mask = FirstN(d, remaining);
const Vec<D> v = MaskedLoad(mask, d, in + i);
// Apply mask so that we don't 'find' the zero-padding from MaskedLoad.
const intptr_t pos = FindFirstTrue(d, And(func(d, v), mask));
if (pos >= 0) return i + static_cast<size_t>(pos);
#endif // HWY_MEM_OPS_MIGHT_FAULT
}
return count; // not found
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_ALGO_FIND_INL_H_

230
deps/v8/third_party/highway/src/hwy/contrib/algo/find_test.cc vendored Normal file
View File

@ -0,0 +1,230 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdio.h>
#include <algorithm> // std::find_if
#include <vector>
#include "hwy/aligned_allocator.h"
#include "hwy/base.h"
#include "hwy/print.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/algo/find_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/algo/find-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
// If your project requires C++14 or later, you can ignore this and pass lambdas
// directly to FindIf, without requiring an lvalue as we do here for C++11.
#if __cplusplus < 201402L
#define HWY_GENERIC_LAMBDA 0
#else
#define HWY_GENERIC_LAMBDA 1
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
// Returns random number in [-8, 8] - we use knowledge of the range to Find()
// values we know are not present.
template <typename T>
T Random(RandomState& rng) {
const int32_t bits = static_cast<int32_t>(Random32(&rng)) & 1023;
double val = (bits - 512) / 64.0;
// Clamp negative to zero for unsigned types.
if (!hwy::IsSigned<T>() && val < 0.0) {
val = -val;
}
return ConvertScalarTo<T>(val);
}
// In C++14, we can instead define these as generic lambdas next to where they
// are invoked.
#if !HWY_GENERIC_LAMBDA
class GreaterThan {
public:
GreaterThan(int val) : val_(val) {}
template <class D, class V>
Mask<D> operator()(D d, V v) const {
return Gt(v, Set(d, ConvertScalarTo<TFromD<D>>(val_)));
}
private:
int val_;
};
#endif // !HWY_GENERIC_LAMBDA
// Invokes Test (e.g. TestFind) with all arg combinations.
template <class Test>
struct ForeachCountAndMisalign {
template <typename T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) const {
RandomState rng;
const size_t N = Lanes(d);
const size_t misalignments[3] = {0, N / 4, 3 * N / 5};
// Find() checks 8 vectors at a time, so we want to cover a fairly large
// range without oversampling (checking every possible count).
std::vector<size_t> counts(AdjustedReps(512));
for (size_t& count : counts) {
count = static_cast<size_t>(rng()) % (16 * N + 1);
}
counts[0] = 0; // ensure we test count=0.
for (size_t count : counts) {
for (size_t m : misalignments) {
Test()(d, count, m, rng);
}
}
}
};
struct TestFind {
template <class D>
void operator()(D d, size_t count, size_t misalign, RandomState& rng) {
using T = TFromD<D>;
// Must allocate at least one even if count is zero.
AlignedFreeUniquePtr<T[]> storage =
AllocateAligned<T>(HWY_MAX(1, misalign + count));
HWY_ASSERT(storage);
T* in = storage.get() + misalign;
for (size_t i = 0; i < count; ++i) {
in[i] = Random<T>(rng);
}
// For each position, search for that element (which we know is there)
for (size_t pos = 0; pos < count; ++pos) {
const size_t actual = Find(d, in[pos], in, count);
// We may have found an earlier occurrence of the same value; ensure the
// value is the same, and that it is the first.
if (!IsEqual(in[pos], in[actual])) {
fprintf(stderr, "%s count %d, found %.15f at %d but wanted %.15f\n",
hwy::TypeName(T(), Lanes(d)).c_str(), static_cast<int>(count),
ConvertScalarTo<double>(in[actual]), static_cast<int>(actual),
ConvertScalarTo<double>(in[pos]));
HWY_ASSERT(false);
}
for (size_t i = 0; i < actual; ++i) {
if (IsEqual(in[i], in[pos])) {
fprintf(stderr, "%s count %d, found %f at %d but Find returned %d\n",
hwy::TypeName(T(), Lanes(d)).c_str(), static_cast<int>(count),
ConvertScalarTo<double>(in[i]), static_cast<int>(i),
static_cast<int>(actual));
HWY_ASSERT(false);
}
}
}
// Also search for values we know not to be present (out of range)
HWY_ASSERT_EQ(count, Find(d, ConvertScalarTo<T>(9), in, count));
HWY_ASSERT_EQ(count, Find(d, ConvertScalarTo<T>(-9), in, count));
}
};
void TestAllFind() {
ForAllTypes(ForPartialVectors<ForeachCountAndMisalign<TestFind>>());
}
struct TestFindIf {
template <class D>
void operator()(D d, size_t count, size_t misalign, RandomState& rng) {
using T = TFromD<D>;
using TI = MakeSigned<T>;
// Must allocate at least one even if count is zero.
AlignedFreeUniquePtr<T[]> storage =
AllocateAligned<T>(HWY_MAX(1, misalign + count));
HWY_ASSERT(storage);
T* in = storage.get() + misalign;
for (size_t i = 0; i < count; ++i) {
in[i] = Random<T>(rng);
HWY_ASSERT(ConvertScalarTo<TI>(in[i]) <= 8);
HWY_ASSERT(!hwy::IsSigned<T>() || ConvertScalarTo<TI>(in[i]) >= -8);
}
bool found_any = false;
bool not_found_any = false;
// unsigned T would be promoted to signed and compare greater than any
// negative val, whereas Set() would just cast to an unsigned value and the
// comparison remains unsigned, so avoid negative numbers there.
const int min_val = IsSigned<T>() ? -9 : 0;
// Includes out-of-range value 9 to test the not-found path.
for (int val = min_val; val <= 9; ++val) {
#if HWY_GENERIC_LAMBDA
const auto greater = [val](const auto d, const auto v) HWY_ATTR {
return Gt(v, Set(d, ConvertScalarTo<T>(val)));
};
#else
const GreaterThan greater(val);
#endif
const size_t actual = FindIf(d, in, count, greater);
found_any |= actual < count;
not_found_any |= actual == count;
const auto pos = std::find_if(
in, in + count, [val](T x) { return x > ConvertScalarTo<T>(val); });
// Convert returned iterator to index.
const size_t expected = static_cast<size_t>(pos - in);
if (expected != actual) {
fprintf(stderr, "%s count %d val %d, expected %d actual %d\n",
hwy::TypeName(T(), Lanes(d)).c_str(), static_cast<int>(count),
val, static_cast<int>(expected), static_cast<int>(actual));
hwy::detail::PrintArray(hwy::detail::MakeTypeInfo<T>(), "in", in, count,
0, count);
HWY_ASSERT(false);
}
}
// We will always not-find something due to val=9.
HWY_ASSERT(not_found_any);
// We'll find something unless the input is empty or {0} - because 0 > i
// is false for all i=[0,9].
if (count != 0 && in[0] != ConvertScalarTo<T>(0)) {
HWY_ASSERT(found_any);
}
}
};
void TestAllFindIf() {
ForAllTypes(ForPartialVectors<ForeachCountAndMisalign<TestFindIf>>());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(FindTest);
HWY_EXPORT_AND_TEST_P(FindTest, TestAllFind);
HWY_EXPORT_AND_TEST_P(FindTest, TestAllFindIf);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

228
deps/v8/third_party/highway/src/hwy/contrib/algo/transform-inl.h vendored Normal file
View File

@ -0,0 +1,228 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Per-target include guard
#if defined(HIGHWAY_HWY_CONTRIB_ALGO_TRANSFORM_INL_H_) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_ALGO_TRANSFORM_INL_H_
#undef HIGHWAY_HWY_CONTRIB_ALGO_TRANSFORM_INL_H_
#else
#define HIGHWAY_HWY_CONTRIB_ALGO_TRANSFORM_INL_H_
#endif
#include <stddef.h>
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
// These functions avoid having to write a loop plus remainder handling in the
// (unfortunately still common) case where arrays are not aligned/padded. If the
// inputs are known to be aligned/padded, it is more efficient to write a single
// loop using Load(). We do not provide a TransformAlignedPadded because it
// would be more verbose than such a loop.
//
// Func is either a functor with a templated operator()(d, v[, v1[, v2]]), or a
// generic lambda if using C++14. The d argument is the same as was passed to
// the Generate etc. functions. Due to apparent limitations of Clang, it is
// currently necessary to add HWY_ATTR before the opening { of the lambda to
// avoid errors about "always_inline function .. requires target".
//
// We do not check HWY_MEM_OPS_MIGHT_FAULT because LoadN/StoreN do not fault.
// Fills `out[0, count)` with the vectors returned by `func(d, index_vec)`,
// where `index_vec` is `Vec<RebindToUnsigned<D>>`. On the first call to `func`,
// the value of its lane i is i, and increases by `Lanes(d)` after every call.
// Note that some of these indices may be `>= count`, but the elements that
// `func` returns in those lanes will not be written to `out`.
template <class D, class Func, typename T = TFromD<D>>
void Generate(D d, T* HWY_RESTRICT out, size_t count, const Func& func) {
const RebindToUnsigned<D> du;
using TU = TFromD<decltype(du)>;
const size_t N = Lanes(d);
size_t idx = 0;
Vec<decltype(du)> vidx = Iota(du, 0);
if (count >= N) {
for (; idx <= count - N; idx += N) {
StoreU(func(d, vidx), d, out + idx);
vidx = Add(vidx, Set(du, static_cast<TU>(N)));
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
StoreN(func(d, vidx), d, out + idx, remaining);
}
// Calls `func(d, v)` for each input vector; out of bound lanes with index i >=
// `count` are instead taken from `no[i % Lanes(d)]`.
template <class D, class Func, typename T = TFromD<D>>
void Foreach(D d, const T* HWY_RESTRICT in, const size_t count, const Vec<D> no,
const Func& func) {
const size_t N = Lanes(d);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
const Vec<D> v = LoadU(d, in + idx);
func(d, v);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
const Vec<D> v = LoadNOr(no, d, in + idx, remaining);
func(d, v);
}
// Replaces `inout[idx]` with `func(d, inout[idx])`. Example usage: multiplying
// array elements by a constant.
template <class D, class Func, typename T = TFromD<D>>
void Transform(D d, T* HWY_RESTRICT inout, size_t count, const Func& func) {
const size_t N = Lanes(d);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
const Vec<D> v = LoadU(d, inout + idx);
StoreU(func(d, v), d, inout + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
const Vec<D> v = LoadN(d, inout + idx, remaining);
StoreN(func(d, v), d, inout + idx, remaining);
}
// Replaces `inout[idx]` with `func(d, inout[idx], in1[idx])`. Example usage:
// multiplying array elements by those of another array.
template <class D, class Func, typename T = TFromD<D>>
void Transform1(D d, T* HWY_RESTRICT inout, size_t count,
const T* HWY_RESTRICT in1, const Func& func) {
const size_t N = Lanes(d);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
const Vec<D> v = LoadU(d, inout + idx);
const Vec<D> v1 = LoadU(d, in1 + idx);
StoreU(func(d, v, v1), d, inout + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
const Vec<D> v = LoadN(d, inout + idx, remaining);
const Vec<D> v1 = LoadN(d, in1 + idx, remaining);
StoreN(func(d, v, v1), d, inout + idx, remaining);
}
// Replaces `inout[idx]` with `func(d, inout[idx], in1[idx], in2[idx])`. Example
// usage: FMA of elements from three arrays, stored into the first array.
template <class D, class Func, typename T = TFromD<D>>
void Transform2(D d, T* HWY_RESTRICT inout, size_t count,
const T* HWY_RESTRICT in1, const T* HWY_RESTRICT in2,
const Func& func) {
const size_t N = Lanes(d);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
const Vec<D> v = LoadU(d, inout + idx);
const Vec<D> v1 = LoadU(d, in1 + idx);
const Vec<D> v2 = LoadU(d, in2 + idx);
StoreU(func(d, v, v1, v2), d, inout + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
const Vec<D> v = LoadN(d, inout + idx, remaining);
const Vec<D> v1 = LoadN(d, in1 + idx, remaining);
const Vec<D> v2 = LoadN(d, in2 + idx, remaining);
StoreN(func(d, v, v1, v2), d, inout + idx, remaining);
}
template <class D, typename T = TFromD<D>>
void Replace(D d, T* HWY_RESTRICT inout, size_t count, T new_t, T old_t) {
const size_t N = Lanes(d);
const Vec<D> old_v = Set(d, old_t);
const Vec<D> new_v = Set(d, new_t);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
Vec<D> v = LoadU(d, inout + idx);
StoreU(IfThenElse(Eq(v, old_v), new_v, v), d, inout + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
const Vec<D> v = LoadN(d, inout + idx, remaining);
StoreN(IfThenElse(Eq(v, old_v), new_v, v), d, inout + idx, remaining);
}
template <class D, class Func, typename T = TFromD<D>>
void ReplaceIf(D d, T* HWY_RESTRICT inout, size_t count, T new_t,
const Func& func) {
const size_t N = Lanes(d);
const Vec<D> new_v = Set(d, new_t);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
Vec<D> v = LoadU(d, inout + idx);
StoreU(IfThenElse(func(d, v), new_v, v), d, inout + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
const Vec<D> v = LoadN(d, inout + idx, remaining);
StoreN(IfThenElse(func(d, v), new_v, v), d, inout + idx, remaining);
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_ALGO_TRANSFORM_INL_H_

464
deps/v8/third_party/highway/src/hwy/contrib/algo/transform_test.cc vendored Normal file
View File

@ -0,0 +1,464 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <string.h> // memcpy
#include <vector>
#include "hwy/aligned_allocator.h"
#include "hwy/base.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/algo/transform_test.cc" //NOLINT
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/algo/transform-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
// If your project requires C++14 or later, you can ignore this and pass lambdas
// directly to Transform, without requiring an lvalue as we do here for C++11.
#if __cplusplus < 201402L
#define HWY_GENERIC_LAMBDA 0
#else
#define HWY_GENERIC_LAMBDA 1
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
constexpr double kAlpha = 1.5; // arbitrary scalar
// Returns random floating-point number in [-8, 8) to ensure computations do
// not exceed float32 precision.
template <typename T>
T Random(RandomState& rng) {
const int32_t bits = static_cast<int32_t>(Random32(&rng)) & 1023;
const double val = (bits - 512) / 64.0;
// Clamp negative to zero for unsigned types.
return ConvertScalarTo<T>(
HWY_MAX(ConvertScalarTo<double>(hwy::LowestValue<T>()), val));
}
// SCAL, AXPY names are from BLAS.
template <typename T>
HWY_NOINLINE void SimpleSCAL(const T* x, T* out, size_t count) {
for (size_t i = 0; i < count; ++i) {
out[i] = ConvertScalarTo<T>(ConvertScalarTo<T>(kAlpha) * x[i]);
}
}
template <typename T>
HWY_NOINLINE void SimpleAXPY(const T* x, const T* y, T* out, size_t count) {
for (size_t i = 0; i < count; ++i) {
out[i] = ConvertScalarTo<T>(
ConvertScalarTo<T>(ConvertScalarTo<T>(kAlpha) * x[i]) + y[i]);
}
}
template <typename T>
HWY_NOINLINE void SimpleFMA4(const T* x, const T* y, const T* z, T* out,
size_t count) {
for (size_t i = 0; i < count; ++i) {
out[i] = ConvertScalarTo<T>(x[i] * y[i] + z[i]);
}
}
// In C++14, we can instead define these as generic lambdas next to where they
// are invoked.
#if !HWY_GENERIC_LAMBDA
// Generator that returns even numbers by doubling the output indices.
struct Gen2 {
template <class D, class VU>
Vec<D> operator()(D d, VU vidx) const {
return BitCast(d, Add(vidx, vidx));
}
};
struct SCAL {
template <class D, class V>
Vec<D> operator()(D d, V v) const {
using T = TFromD<D>;
return Mul(Set(d, ConvertScalarTo<T>(kAlpha)), v);
}
};
struct AXPY {
template <class D, class V>
Vec<D> operator()(D d, V v, V v1) const {
using T = TFromD<D>;
return MulAdd(Set(d, ConvertScalarTo<T>(kAlpha)), v, v1);
}
};
struct FMA4 {
template <class D, class V>
Vec<D> operator()(D /*d*/, V v, V v1, V v2) const {
return MulAdd(v, v1, v2);
}
};
#endif // !HWY_GENERIC_LAMBDA
// Invokes Test (e.g. TestTransform1) with all arg combinations. T comes from
// ForFloatTypes.
template <class Test>
struct ForeachCountAndMisalign {
template <typename T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) const {
RandomState rng;
const size_t N = Lanes(d);
const size_t misalignments[3] = {0, N / 4, 3 * N / 5};
for (size_t count = 0; count < 2 * N; ++count) {
for (size_t ma : misalignments) {
for (size_t mb : misalignments) {
Test()(d, count, ma, mb, rng);
}
}
}
}
};
// Fills an array with random values, placing a given sentinel value both before
// (when misalignment space is available) and after. Requires an allocation of
// at least count + misalign + 1 elements.
template <typename T>
T* FillRandom(AlignedFreeUniquePtr<T[]>& pa, size_t count, size_t misalign,
T sentinel, RandomState& rng) {
for (size_t i = 0; i < misalign; ++i) {
pa[i] = sentinel;
}
T* a = pa.get() + misalign;
for (size_t i = 0; i < count; ++i) {
a[i] = Random<T>(rng);
}
a[count] = sentinel;
return a;
}
// Output-only, no loads
struct TestGenerate {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t /*misalign_b*/,
RandomState& /*rng*/) {
using T = TFromD<D>;
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(misalign_a + count + 1);
AlignedFreeUniquePtr<T[]> expected = AllocateAligned<T>(HWY_MAX(1, count));
HWY_ASSERT(pa && expected);
T* actual = pa.get() + misalign_a;
for (size_t i = 0; i < count; ++i) {
expected[i] = ConvertScalarTo<T>(2 * i);
}
// TODO(janwas): can we update the apply_to in HWY_PUSH_ATTRIBUTES so that
// the attribute also applies to lambdas? If so, remove HWY_ATTR.
#if HWY_GENERIC_LAMBDA
const auto gen2 = [](const auto d, const auto vidx)
HWY_ATTR { return BitCast(d, Add(vidx, vidx)); };
#else
const Gen2 gen2;
#endif
actual[count] = ConvertScalarTo<T>(0); // sentinel
Generate(d, actual, count, gen2);
HWY_ASSERT_EQ(ConvertScalarTo<T>(0), actual[count]); // no write past end
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName(HWY_TARGET);
hwy::detail::AssertArrayEqual(info, expected.get(), actual, count,
target_name, __FILE__, __LINE__);
}
};
// Input-only, no stores
struct TestForeach {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& /*rng*/) {
if (misalign_b != 0) return;
using T = TFromD<D>;
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(misalign_a + count + 1);
HWY_ASSERT(pa);
T* actual = pa.get() + misalign_a;
T max = hwy::LowestValue<T>();
for (size_t i = 0; i < count; ++i) {
actual[i] = hwy::ConvertScalarTo<T>(i <= count / 2 ? 2 * i : i);
max = HWY_MAX(max, actual[i]);
}
// Place sentinel values in the misalignment area and at the input's end.
for (size_t i = 0; i < misalign_a; ++i) {
pa[i] = ConvertScalarTo<T>(2 * count);
}
actual[count] = ConvertScalarTo<T>(2 * count);
const Vec<D> vmin = Set(d, hwy::LowestValue<T>());
// TODO(janwas): can we update the apply_to in HWY_PUSH_ATTRIBUTES so that
// the attribute also applies to lambdas? If so, remove HWY_ATTR.
Vec<D> vmax = vmin;
const auto func = [&vmax](const D, const Vec<D> v)
HWY_ATTR { vmax = Max(vmax, v); };
Foreach(d, actual, count, vmin, func);
const char* target_name = hwy::TargetName(HWY_TARGET);
AssertEqual(max, ReduceMax(d, vmax), target_name, __FILE__, __LINE__);
}
};
// Zero extra input arrays
struct TestTransform {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
if (misalign_b != 0) return;
using T = TFromD<D>;
// Prevents error if size to allocate is zero.
AlignedFreeUniquePtr<T[]> pa =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count + 1));
AlignedFreeUniquePtr<T[]> expected = AllocateAligned<T>(HWY_MAX(1, count));
HWY_ASSERT(pa && expected);
const T sentinel = ConvertScalarTo<T>(-42);
T* a = FillRandom(pa, count, misalign_a, sentinel, rng);
SimpleSCAL(a, expected.get(), count);
// TODO(janwas): can we update the apply_to in HWY_PUSH_ATTRIBUTES so that
// the attribute also applies to lambdas? If so, remove HWY_ATTR.
#if HWY_GENERIC_LAMBDA
const auto scal = [](const auto d, const auto v) HWY_ATTR {
return Mul(Set(d, ConvertScalarTo<T>(kAlpha)), v);
};
#else
const SCAL scal;
#endif
Transform(d, a, count, scal);
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName(HWY_TARGET);
hwy::detail::AssertArrayEqual(info, expected.get(), a, count, target_name,
__FILE__, __LINE__);
// Ensure no out-of-bound writes.
for (size_t i = 0; i < misalign_a; ++i) {
HWY_ASSERT_EQ(sentinel, pa[i]);
}
HWY_ASSERT_EQ(sentinel, a[count]);
}
};
// One extra input array
struct TestTransform1 {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
// Prevents error if size to allocate is zero.
AlignedFreeUniquePtr<T[]> pa =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count + 1));
AlignedFreeUniquePtr<T[]> pb =
AllocateAligned<T>(HWY_MAX(1, misalign_b + count));
AlignedFreeUniquePtr<T[]> expected = AllocateAligned<T>(HWY_MAX(1, count));
HWY_ASSERT(pa && pb && expected);
const T sentinel = ConvertScalarTo<T>(-42);
T* a = FillRandom(pa, count, misalign_a, sentinel, rng);
T* b = pb.get() + misalign_b;
for (size_t i = 0; i < count; ++i) {
b[i] = Random<T>(rng);
}
SimpleAXPY(a, b, expected.get(), count);
#if HWY_GENERIC_LAMBDA
const auto axpy = [](const auto d, const auto v, const auto v1) HWY_ATTR {
return MulAdd(Set(d, ConvertScalarTo<T>(kAlpha)), v, v1);
};
#else
const AXPY axpy;
#endif
Transform1(d, a, count, b, axpy);
AssertArraySimilar(expected.get(), a, count, hwy::TargetName(HWY_TARGET),
__FILE__, __LINE__);
// Ensure no out-of-bound writes.
for (size_t i = 0; i < misalign_a; ++i) {
HWY_ASSERT_EQ(sentinel, pa[i]);
}
HWY_ASSERT_EQ(sentinel, a[count]);
}
};
// Two extra input arrays
struct TestTransform2 {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
// Prevents error if size to allocate is zero.
AlignedFreeUniquePtr<T[]> pa =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count + 1));
AlignedFreeUniquePtr<T[]> pb =
AllocateAligned<T>(HWY_MAX(1, misalign_b + count));
AlignedFreeUniquePtr<T[]> pc =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count));
AlignedFreeUniquePtr<T[]> expected = AllocateAligned<T>(HWY_MAX(1, count));
HWY_ASSERT(pa && pb && pc && expected);
const T sentinel = ConvertScalarTo<T>(-42);
T* a = FillRandom(pa, count, misalign_a, sentinel, rng);
T* b = pb.get() + misalign_b;
T* c = pc.get() + misalign_a;
for (size_t i = 0; i < count; ++i) {
b[i] = Random<T>(rng);
c[i] = Random<T>(rng);
}
SimpleFMA4(a, b, c, expected.get(), count);
#if HWY_GENERIC_LAMBDA
const auto fma4 = [](auto /*d*/, auto v, auto v1, auto v2)
HWY_ATTR { return MulAdd(v, v1, v2); };
#else
const FMA4 fma4;
#endif
Transform2(d, a, count, b, c, fma4);
AssertArraySimilar(expected.get(), a, count, hwy::TargetName(HWY_TARGET),
__FILE__, __LINE__);
// Ensure no out-of-bound writes.
for (size_t i = 0; i < misalign_a; ++i) {
HWY_ASSERT_EQ(sentinel, pa[i]);
}
HWY_ASSERT_EQ(sentinel, a[count]);
}
};
template <typename T>
class IfEq {
public:
IfEq(T val) : val_(val) {}
template <class D, class V>
Mask<D> operator()(D d, V v) const {
return Eq(v, Set(d, val_));
}
private:
T val_;
};
struct TestReplace {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
if (misalign_b != 0) return;
if (count == 0) return;
using T = TFromD<D>;
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(misalign_a + count + 1);
AlignedFreeUniquePtr<T[]> pb = AllocateAligned<T>(count);
AlignedFreeUniquePtr<T[]> expected = AllocateAligned<T>(count);
HWY_ASSERT(pa && pb && expected);
const T sentinel = ConvertScalarTo<T>(-42);
T* a = FillRandom(pa, count, misalign_a, sentinel, rng);
std::vector<size_t> positions(AdjustedReps(count));
for (size_t& pos : positions) {
pos = static_cast<size_t>(rng()) % count;
}
for (size_t pos = 0; pos < count; ++pos) {
const T old_t = a[pos];
const T new_t = Random<T>(rng);
for (size_t i = 0; i < count; ++i) {
expected[i] = IsEqual(a[i], old_t) ? new_t : a[i];
}
// Copy so ReplaceIf gets the same input (and thus also outputs expected)
memcpy(pb.get(), a, count * sizeof(T));
Replace(d, a, count, new_t, old_t);
HWY_ASSERT_ARRAY_EQ(expected.get(), a, count);
// Ensure no out-of-bound writes.
for (size_t i = 0; i < misalign_a; ++i) {
HWY_ASSERT_EQ(sentinel, pa[i]);
}
HWY_ASSERT_EQ(sentinel, a[count]);
ReplaceIf(d, pb.get(), count, new_t, IfEq<T>(old_t));
HWY_ASSERT_ARRAY_EQ(expected.get(), pb.get(), count);
// Ensure no out-of-bound writes.
for (size_t i = 0; i < misalign_a; ++i) {
HWY_ASSERT_EQ(sentinel, pa[i]);
}
HWY_ASSERT_EQ(sentinel, a[count]);
}
}
};
void TestAllGenerate() {
// The test BitCast-s the indices, which does not work for floats.
ForIntegerTypes(ForPartialVectors<ForeachCountAndMisalign<TestGenerate>>());
}
void TestAllForeach() {
ForAllTypes(ForPartialVectors<ForeachCountAndMisalign<TestForeach>>());
}
void TestAllTransform() {
ForFloatTypes(ForPartialVectors<ForeachCountAndMisalign<TestTransform>>());
}
void TestAllTransform1() {
ForFloatTypes(ForPartialVectors<ForeachCountAndMisalign<TestTransform1>>());
}
void TestAllTransform2() {
ForFloatTypes(ForPartialVectors<ForeachCountAndMisalign<TestTransform2>>());
}
void TestAllReplace() {
ForFloatTypes(ForPartialVectors<ForeachCountAndMisalign<TestReplace>>());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(TransformTest);
HWY_EXPORT_AND_TEST_P(TransformTest, TestAllGenerate);
HWY_EXPORT_AND_TEST_P(TransformTest, TestAllForeach);
HWY_EXPORT_AND_TEST_P(TransformTest, TestAllTransform);
HWY_EXPORT_AND_TEST_P(TransformTest, TestAllTransform1);
HWY_EXPORT_AND_TEST_P(TransformTest, TestAllTransform2);
HWY_EXPORT_AND_TEST_P(TransformTest, TestAllReplace);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

2851
deps/v8/third_party/highway/src/hwy/contrib/bit_pack/bit_pack-inl.h vendored Normal file
View File

File diff suppressed because it is too large Load Diff

244
deps/v8/third_party/highway/src/hwy/contrib/bit_pack/bit_pack_test.cc vendored Normal file
View File

@ -0,0 +1,244 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdio.h>
#include <vector>
#include "hwy/aligned_allocator.h"
#include "hwy/base.h"
#include "hwy/nanobenchmark.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/bit_pack/bit_pack_test.cc" // NOLINT
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/timer.h"
#include "hwy/contrib/bit_pack/bit_pack-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
#ifndef HWY_BIT_PACK_BENCHMARK
#define HWY_BIT_PACK_BENCHMARK 0
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
// Used to prevent running benchmark (slow) for partial vectors and targets
// except the best available. Global, not per-target, hence must be outside
// HWY_NAMESPACE. Declare first because HWY_ONCE is only true after some code
// has been re-included.
extern size_t last_bits;
extern uint64_t best_target;
#if HWY_ONCE
size_t last_bits = 0;
uint64_t best_target = ~0ull;
#endif
namespace HWY_NAMESPACE {
namespace {
template <size_t kBits, typename T>
T Random(RandomState& rng) {
return ConvertScalarTo<T>(Random32(&rng) & kBits);
}
template <typename T>
class Checker {
public:
explicit Checker(size_t num) { raw_.reserve(num); }
void NotifyRaw(T raw) { raw_.push_back(raw); }
void NotifyRawOutput(size_t bits, T raw) {
if (raw_[num_verified_] != raw) {
HWY_ABORT("%zu bits: pos %zu of %zu, expected %.0f actual %.0f\n", bits,
num_verified_, raw_.size(),
ConvertScalarTo<double>(raw_[num_verified_]),
ConvertScalarTo<double>(raw));
}
++num_verified_;
}
private:
std::vector<T> raw_;
size_t num_verified_ = 0;
};
template <template <size_t> class PackT, size_t kVectors, size_t kBits>
struct TestPack {
template <typename T, class D>
void operator()(T /* t */, D d) {
constexpr size_t kLoops = 16; // working set slightly larger than L1
const size_t N = Lanes(d);
RandomState rng(N * 129);
static_assert(kBits <= kVectors, "");
const size_t num_per_loop = N * kVectors;
const size_t num = num_per_loop * kLoops;
const size_t num_packed_per_loop = N * kBits;
const size_t num_packed = num_packed_per_loop * kLoops;
Checker<T> checker(num);
AlignedFreeUniquePtr<T[]> raw = hwy::AllocateAligned<T>(num);
AlignedFreeUniquePtr<T[]> raw2 = hwy::AllocateAligned<T>(num);
AlignedFreeUniquePtr<T[]> packed = hwy::AllocateAligned<T>(num_packed);
HWY_ASSERT(raw && raw2 && packed);
for (size_t i = 0; i < num; ++i) {
raw[i] = Random<kBits, T>(rng);
checker.NotifyRaw(raw[i]);
}
best_target = HWY_MIN(best_target, HWY_TARGET);
const bool run_bench = HWY_BIT_PACK_BENCHMARK && (kBits != last_bits) &&
(HWY_TARGET == best_target);
last_bits = kBits;
const PackT<kBits> func;
if (run_bench) {
const size_t kNumInputs = 1;
const size_t num_items = num * size_t(Unpredictable1());
const FuncInput inputs[kNumInputs] = {num_items};
Result results[kNumInputs];
Params p;
p.verbose = false;
p.max_evals = 7;
p.target_rel_mad = 0.002;
const size_t num_results = MeasureClosure(
[&](FuncInput) HWY_ATTR {
for (size_t i = 0, pi = 0; i < num;
i += num_per_loop, pi += num_packed_per_loop) {
func.Pack(d, raw.get() + i, packed.get() + pi);
}
T& val = packed.get()[Random32(&rng) % num_packed];
T zero = static_cast<T>(Unpredictable1() - 1);
val = static_cast<T>(val + zero);
for (size_t i = 0, pi = 0; i < num;
i += num_per_loop, pi += num_packed_per_loop) {
func.Unpack(d, packed.get() + pi, raw2.get() + i);
}
return raw2[Random32(&rng) % num];
},
inputs, kNumInputs, results, p);
if (num_results != kNumInputs) {
fprintf(stderr, "MeasureClosure failed.\n");
return;
}
// Print throughput for pack+unpack round trip
for (size_t i = 0; i < num_results; ++i) {
const size_t bytes_per_element = (kBits + 7) / 8;
const double bytes =
static_cast<double>(results[i].input * bytes_per_element);
const double seconds =
results[i].ticks / platform::InvariantTicksPerSecond();
printf("Bits:%2d elements:%3d GB/s:%4.1f (+/-%3.1f%%)\n",
static_cast<int>(kBits), static_cast<int>(results[i].input),
1E-9 * bytes / seconds, results[i].variability * 100.0);
}
} else {
for (size_t i = 0, pi = 0; i < num;
i += num_per_loop, pi += num_packed_per_loop) {
func.Pack(d, raw.get() + i, packed.get() + pi);
}
T& val = packed.get()[Random32(&rng) % num_packed];
T zero = static_cast<T>(Unpredictable1() - 1);
val = static_cast<T>(val + zero);
for (size_t i = 0, pi = 0; i < num;
i += num_per_loop, pi += num_packed_per_loop) {
func.Unpack(d, packed.get() + pi, raw2.get() + i);
}
}
for (size_t i = 0; i < num; ++i) {
checker.NotifyRawOutput(kBits, raw2[i]);
}
}
};
void TestAllPack8() {
ForShrinkableVectors<TestPack<Pack8, 8, 1>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 2>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 3>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 4>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 5>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 6>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 7>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 8>>()(uint8_t());
}
void TestAllPack16() {
ForShrinkableVectors<TestPack<Pack16, 16, 1>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 2>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 3>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 4>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 5>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 6>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 7>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 8>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 9>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 10>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 11>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 12>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 13>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 14>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 15>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 16>>()(uint16_t());
}
void TestAllPack32() {
ForShrinkableVectors<TestPack<Pack32, 32, 1>>()(uint32_t());
ForShrinkableVectors<TestPack<Pack32, 32, 2>>()(uint32_t());
ForShrinkableVectors<TestPack<Pack32, 32, 6>>()(uint32_t());
ForShrinkableVectors<TestPack<Pack32, 32, 11>>()(uint32_t());
ForShrinkableVectors<TestPack<Pack32, 32, 16>>()(uint32_t());
ForShrinkableVectors<TestPack<Pack32, 32, 31>>()(uint32_t());
ForShrinkableVectors<TestPack<Pack32, 32, 32>>()(uint32_t());
}
void TestAllPack64() {
// Fails, but only on GCC 13.
#if !(HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1400 && \
HWY_TARGET == HWY_RVV)
ForShrinkableVectors<TestPack<Pack64, 64, 1>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 5>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 12>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 16>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 27>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 31>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 33>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 41>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 61>>()(uint64_t());
#endif
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(BitPackTest);
HWY_EXPORT_AND_TEST_P(BitPackTest, TestAllPack8);
HWY_EXPORT_AND_TEST_P(BitPackTest, TestAllPack16);
HWY_EXPORT_AND_TEST_P(BitPackTest, TestAllPack32);
HWY_EXPORT_AND_TEST_P(BitPackTest, TestAllPack64);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

361
deps/v8/third_party/highway/src/hwy/contrib/dot/dot-inl.h vendored Normal file
View File

@ -0,0 +1,361 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// clang-format off
#if defined(HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_) == defined(HWY_TARGET_TOGGLE) // NOLINT
// clang-format on
#ifdef HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_
#undef HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_
#else
#define HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_
#endif
#include <stddef.h>
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
struct Dot {
// Specify zero or more of these, ORed together, as the kAssumptions template
// argument to Compute. Each one may improve performance or reduce code size,
// at the cost of additional requirements on the arguments.
enum Assumptions {
// num_elements is at least N, which may be up to HWY_MAX_BYTES / sizeof(T).
kAtLeastOneVector = 1,
// num_elements is divisible by N (a power of two, so this can be used if
// the problem size is known to be a power of two >= HWY_MAX_BYTES /
// sizeof(T)).
kMultipleOfVector = 2,
// RoundUpTo(num_elements, N) elements are accessible; their value does not
// matter (will be treated as if they were zero).
kPaddedToVector = 4,
};
// Returns sum{pa[i] * pb[i]} for floating-point inputs, including float16_t
// and double if HWY_HAVE_FLOAT16/64. Aligning the
// pointers to a multiple of N elements is helpful but not required.
template <int kAssumptions, class D, typename T = TFromD<D>>
static HWY_INLINE T Compute(const D d, const T* const HWY_RESTRICT pa,
const T* const HWY_RESTRICT pb,
const size_t num_elements) {
static_assert(IsFloat<T>(), "MulAdd requires float type");
using V = decltype(Zero(d));
const size_t N = Lanes(d);
size_t i = 0;
constexpr bool kIsAtLeastOneVector =
(kAssumptions & kAtLeastOneVector) != 0;
constexpr bool kIsMultipleOfVector =
(kAssumptions & kMultipleOfVector) != 0;
constexpr bool kIsPaddedToVector = (kAssumptions & kPaddedToVector) != 0;
// Won't be able to do a full vector load without padding => scalar loop.
if (!kIsAtLeastOneVector && !kIsMultipleOfVector && !kIsPaddedToVector &&
HWY_UNLIKELY(num_elements < N)) {
// Only 2x unroll to avoid excessive code size.
T sum0 = ConvertScalarTo<T>(0);
T sum1 = ConvertScalarTo<T>(0);
for (; i + 2 <= num_elements; i += 2) {
// For reasons unknown, fp16 += does not compile on clang (Arm).
sum0 = ConvertScalarTo<T>(sum0 + pa[i + 0] * pb[i + 0]);
sum1 = ConvertScalarTo<T>(sum1 + pa[i + 1] * pb[i + 1]);
}
if (i < num_elements) {
sum1 = ConvertScalarTo<T>(sum1 + pa[i] * pb[i]);
}
return ConvertScalarTo<T>(sum0 + sum1);
}
// Compiler doesn't make independent sum* accumulators, so unroll manually.
// 2 FMA ports * 4 cycle latency = up to 8 in-flight, but that is excessive
// for unaligned inputs (each unaligned pointer halves the throughput
// because it occupies both L1 load ports for a cycle). We cannot have
// arrays of vectors on RVV/SVE, so always unroll 4x.
V sum0 = Zero(d);
V sum1 = Zero(d);
V sum2 = Zero(d);
V sum3 = Zero(d);
// Main loop: unrolled
for (; i + 4 * N <= num_elements; /* i += 4 * N */) { // incr in loop
const auto a0 = LoadU(d, pa + i);
const auto b0 = LoadU(d, pb + i);
i += N;
sum0 = MulAdd(a0, b0, sum0);
const auto a1 = LoadU(d, pa + i);
const auto b1 = LoadU(d, pb + i);
i += N;
sum1 = MulAdd(a1, b1, sum1);
const auto a2 = LoadU(d, pa + i);
const auto b2 = LoadU(d, pb + i);
i += N;
sum2 = MulAdd(a2, b2, sum2);
const auto a3 = LoadU(d, pa + i);
const auto b3 = LoadU(d, pb + i);
i += N;
sum3 = MulAdd(a3, b3, sum3);
}
// Up to 3 iterations of whole vectors
for (; i + N <= num_elements; i += N) {
const auto a = LoadU(d, pa + i);
const auto b = LoadU(d, pb + i);
sum0 = MulAdd(a, b, sum0);
}
if (!kIsMultipleOfVector) {
const size_t remaining = num_elements - i;
if (remaining != 0) {
if (kIsPaddedToVector) {
const auto mask = FirstN(d, remaining);
const auto a = LoadU(d, pa + i);
const auto b = LoadU(d, pb + i);
sum1 = MulAdd(IfThenElseZero(mask, a), IfThenElseZero(mask, b), sum1);
} else {
// Unaligned load such that the last element is in the highest lane -
// ensures we do not touch any elements outside the valid range.
// If we get here, then num_elements >= N.
HWY_DASSERT(i >= N);
i += remaining - N;
const auto skip = FirstN(d, N - remaining);
const auto a = LoadU(d, pa + i); // always unaligned
const auto b = LoadU(d, pb + i);
sum1 = MulAdd(IfThenZeroElse(skip, a), IfThenZeroElse(skip, b), sum1);
}
}
} // kMultipleOfVector
// Reduction tree: sum of all accumulators by pairs, then across lanes.
sum0 = Add(sum0, sum1);
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum2);
return ReduceSum(d, sum0);
}
// f32 * bf16
template <int kAssumptions, class DF, HWY_IF_F32_D(DF)>
static HWY_INLINE float Compute(const DF df,
const float* const HWY_RESTRICT pa,
const hwy::bfloat16_t* const HWY_RESTRICT pb,
const size_t num_elements) {
#if HWY_TARGET == HWY_SCALAR
const Rebind<hwy::bfloat16_t, DF> dbf;
#else
const Repartition<hwy::bfloat16_t, DF> dbf;
using VBF = decltype(Zero(dbf));
#endif
const Half<decltype(dbf)> dbfh;
using VF = decltype(Zero(df));
const size_t NF = Lanes(df);
constexpr bool kIsAtLeastOneVector =
(kAssumptions & kAtLeastOneVector) != 0;
constexpr bool kIsMultipleOfVector =
(kAssumptions & kMultipleOfVector) != 0;
constexpr bool kIsPaddedToVector = (kAssumptions & kPaddedToVector) != 0;
// Won't be able to do a full vector load without padding => scalar loop.
if (!kIsAtLeastOneVector && !kIsMultipleOfVector && !kIsPaddedToVector &&
HWY_UNLIKELY(num_elements < NF)) {
// Only 2x unroll to avoid excessive code size.
float sum0 = 0.0f;
float sum1 = 0.0f;
size_t i = 0;
for (; i + 2 <= num_elements; i += 2) {
sum0 += pa[i + 0] * ConvertScalarTo<float>(pb[i + 0]);
sum1 += pa[i + 1] * ConvertScalarTo<float>(pb[i + 1]);
}
for (; i < num_elements; ++i) {
sum1 += pa[i] * ConvertScalarTo<float>(pb[i]);
}
return sum0 + sum1;
}
// Compiler doesn't make independent sum* accumulators, so unroll manually.
// 2 FMA ports * 4 cycle latency = up to 8 in-flight, but that is excessive
// for unaligned inputs (each unaligned pointer halves the throughput
// because it occupies both L1 load ports for a cycle). We cannot have
// arrays of vectors on RVV/SVE, so always unroll 4x.
VF sum0 = Zero(df);
VF sum1 = Zero(df);
VF sum2 = Zero(df);
VF sum3 = Zero(df);
size_t i = 0;
#if HWY_TARGET != HWY_SCALAR // PromoteUpperTo supported
// Main loop: unrolled
for (; i + 4 * NF <= num_elements; /* i += 4 * N */) { // incr in loop
const VF a0 = LoadU(df, pa + i);
const VBF b0 = LoadU(dbf, pb + i);
i += NF;
sum0 = MulAdd(a0, PromoteLowerTo(df, b0), sum0);
const VF a1 = LoadU(df, pa + i);
i += NF;
sum1 = MulAdd(a1, PromoteUpperTo(df, b0), sum1);
const VF a2 = LoadU(df, pa + i);
const VBF b2 = LoadU(dbf, pb + i);
i += NF;
sum2 = MulAdd(a2, PromoteLowerTo(df, b2), sum2);
const VF a3 = LoadU(df, pa + i);
i += NF;
sum3 = MulAdd(a3, PromoteUpperTo(df, b2), sum3);
}
#endif // HWY_TARGET == HWY_SCALAR
// Up to 3 iterations of whole vectors
for (; i + NF <= num_elements; i += NF) {
const VF a = LoadU(df, pa + i);
const VF b = PromoteTo(df, LoadU(dbfh, pb + i));
sum0 = MulAdd(a, b, sum0);
}
if (!kIsMultipleOfVector) {
const size_t remaining = num_elements - i;
if (remaining != 0) {
if (kIsPaddedToVector) {
const auto mask = FirstN(df, remaining);
const VF a = LoadU(df, pa + i);
const VF b = PromoteTo(df, LoadU(dbfh, pb + i));
sum1 = MulAdd(IfThenElseZero(mask, a), IfThenElseZero(mask, b), sum1);
} else {
// Unaligned load such that the last element is in the highest lane -
// ensures we do not touch any elements outside the valid range.
// If we get here, then num_elements >= N.
HWY_DASSERT(i >= NF);
i += remaining - NF;
const auto skip = FirstN(df, NF - remaining);
const VF a = LoadU(df, pa + i); // always unaligned
const VF b = PromoteTo(df, LoadU(dbfh, pb + i));
sum1 = MulAdd(IfThenZeroElse(skip, a), IfThenZeroElse(skip, b), sum1);
}
}
} // kMultipleOfVector
// Reduction tree: sum of all accumulators by pairs, then across lanes.
sum0 = Add(sum0, sum1);
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum2);
return ReduceSum(df, sum0);
}
// Returns sum{pa[i] * pb[i]} for bfloat16 inputs. Aligning the pointers to a
// multiple of N elements is helpful but not required.
template <int kAssumptions, class D, HWY_IF_BF16_D(D)>
static HWY_INLINE float Compute(const D d,
const bfloat16_t* const HWY_RESTRICT pa,
const bfloat16_t* const HWY_RESTRICT pb,
const size_t num_elements) {
const RebindToUnsigned<D> du16;
const Repartition<float, D> df32;
using V = decltype(Zero(df32));
const size_t N = Lanes(d);
size_t i = 0;
constexpr bool kIsAtLeastOneVector =
(kAssumptions & kAtLeastOneVector) != 0;
constexpr bool kIsMultipleOfVector =
(kAssumptions & kMultipleOfVector) != 0;
constexpr bool kIsPaddedToVector = (kAssumptions & kPaddedToVector) != 0;
// Won't be able to do a full vector load without padding => scalar loop.
if (!kIsAtLeastOneVector && !kIsMultipleOfVector && !kIsPaddedToVector &&
HWY_UNLIKELY(num_elements < N)) {
float sum0 = 0.0f; // Only 2x unroll to avoid excessive code size for..
float sum1 = 0.0f; // this unlikely(?) case.
for (; i + 2 <= num_elements; i += 2) {
sum0 += F32FromBF16(pa[i + 0]) * F32FromBF16(pb[i + 0]);
sum1 += F32FromBF16(pa[i + 1]) * F32FromBF16(pb[i + 1]);
}
if (i < num_elements) {
sum1 += F32FromBF16(pa[i]) * F32FromBF16(pb[i]);
}
return sum0 + sum1;
}
// See comment in the other Compute() overload. Unroll 2x, but we need
// twice as many sums for ReorderWidenMulAccumulate.
V sum0 = Zero(df32);
V sum1 = Zero(df32);
V sum2 = Zero(df32);
V sum3 = Zero(df32);
// Main loop: unrolled
for (; i + 2 * N <= num_elements; /* i += 2 * N */) { // incr in loop
const auto a0 = LoadU(d, pa + i);
const auto b0 = LoadU(d, pb + i);
i += N;
sum0 = ReorderWidenMulAccumulate(df32, a0, b0, sum0, sum1);
const auto a1 = LoadU(d, pa + i);
const auto b1 = LoadU(d, pb + i);
i += N;
sum2 = ReorderWidenMulAccumulate(df32, a1, b1, sum2, sum3);
}
// Possibly one more iteration of whole vectors
if (i + N <= num_elements) {
const auto a0 = LoadU(d, pa + i);
const auto b0 = LoadU(d, pb + i);
i += N;
sum0 = ReorderWidenMulAccumulate(df32, a0, b0, sum0, sum1);
}
if (!kIsMultipleOfVector) {
const size_t remaining = num_elements - i;
if (remaining != 0) {
if (kIsPaddedToVector) {
const auto mask = FirstN(du16, remaining);
const auto va = LoadU(d, pa + i);
const auto vb = LoadU(d, pb + i);
const auto a16 = BitCast(d, IfThenElseZero(mask, BitCast(du16, va)));
const auto b16 = BitCast(d, IfThenElseZero(mask, BitCast(du16, vb)));
sum2 = ReorderWidenMulAccumulate(df32, a16, b16, sum2, sum3);
} else {
// Unaligned load such that the last element is in the highest lane -
// ensures we do not touch any elements outside the valid range.
// If we get here, then num_elements >= N.
HWY_DASSERT(i >= N);
i += remaining - N;
const auto skip = FirstN(du16, N - remaining);
const auto va = LoadU(d, pa + i); // always unaligned
const auto vb = LoadU(d, pb + i);
const auto a16 = BitCast(d, IfThenZeroElse(skip, BitCast(du16, va)));
const auto b16 = BitCast(d, IfThenZeroElse(skip, BitCast(du16, vb)));
sum2 = ReorderWidenMulAccumulate(df32, a16, b16, sum2, sum3);
}
}
} // kMultipleOfVector
// Reduction tree: sum of all accumulators by pairs, then across lanes.
sum0 = Add(sum0, sum1);
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum2);
return ReduceSum(df32, sum0);
}
};
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_

292
deps/v8/third_party/highway/src/hwy/contrib/dot/dot_test.cc vendored Normal file
View File

@ -0,0 +1,292 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "hwy/aligned_allocator.h"
#include "hwy/base.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/dot/dot_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/dot/dot-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
template <typename T1, typename T2>
HWY_NOINLINE T1 SimpleDot(const T1* pa, const T2* pb, size_t num) {
float sum = 0.0f;
for (size_t i = 0; i < num; ++i) {
sum += ConvertScalarTo<float>(pa[i]) * ConvertScalarTo<float>(pb[i]);
}
return ConvertScalarTo<T1>(sum);
}
HWY_MAYBE_UNUSED HWY_NOINLINE float SimpleDot(const float* pa,
const hwy::bfloat16_t* pb,
size_t num) {
float sum = 0.0f;
for (size_t i = 0; i < num; ++i) {
sum += pa[i] * F32FromBF16(pb[i]);
}
return sum;
}
// Overload is required because the generic template hits an internal compiler
// error on aarch64 clang.
HWY_MAYBE_UNUSED HWY_NOINLINE float SimpleDot(const bfloat16_t* pa,
const bfloat16_t* pb,
size_t num) {
float sum = 0.0f;
for (size_t i = 0; i < num; ++i) {
sum += F32FromBF16(pa[i]) * F32FromBF16(pb[i]);
}
return sum;
}
class TestDot {
// Computes/verifies one dot product.
template <int kAssumptions, class D>
void Test(D d, size_t num, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
const size_t N = Lanes(d);
const auto random_t = [&rng]() {
const int32_t bits = static_cast<int32_t>(Random32(&rng)) & 1023;
return static_cast<float>(bits - 512) * (1.0f / 64);
};
const size_t padded =
(kAssumptions & Dot::kPaddedToVector) ? RoundUpTo(num, N) : num;
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(misalign_a + padded);
AlignedFreeUniquePtr<T[]> pb = AllocateAligned<T>(misalign_b + padded);
HWY_ASSERT(pa && pb);
T* a = pa.get() + misalign_a;
T* b = pb.get() + misalign_b;
size_t i = 0;
for (; i < num; ++i) {
a[i] = ConvertScalarTo<T>(random_t());
b[i] = ConvertScalarTo<T>(random_t());
}
// Fill padding with NaN - the values are not used, but avoids MSAN errors.
for (; i < padded; ++i) {
ScalableTag<float> df1;
a[i] = ConvertScalarTo<T>(GetLane(NaN(df1)));
b[i] = ConvertScalarTo<T>(GetLane(NaN(df1)));
}
const double expected = SimpleDot(a, b, num);
const double magnitude = expected > 0.0 ? expected : -expected;
const double actual =
ConvertScalarTo<double>(Dot::Compute<kAssumptions>(d, a, b, num));
const double max = static_cast<double>(8 * 8 * num);
HWY_ASSERT(-max <= actual && actual <= max);
const double tolerance =
96.0 * ConvertScalarTo<double>(Epsilon<T>()) * HWY_MAX(magnitude, 1.0);
HWY_ASSERT(expected - tolerance <= actual &&
actual <= expected + tolerance);
}
// Runs tests with various alignments.
template <int kAssumptions, class D>
void ForeachMisalign(D d, size_t num, RandomState& rng) {
const size_t N = Lanes(d);
const size_t misalignments[3] = {0, N / 4, 3 * N / 5};
for (size_t ma : misalignments) {
for (size_t mb : misalignments) {
Test<kAssumptions>(d, num, ma, mb, rng);
}
}
}
// Runs tests with various lengths compatible with the given assumptions.
template <int kAssumptions, class D>
void ForeachCount(D d, RandomState& rng) {
const size_t N = Lanes(d);
const size_t counts[] = {1,
3,
7,
16,
HWY_MAX(N / 2, 1),
HWY_MAX(2 * N / 3, 1),
N,
N + 1,
4 * N / 3,
3 * N,
8 * N,
8 * N + 2};
for (size_t num : counts) {
if ((kAssumptions & Dot::kAtLeastOneVector) && num < N) continue;
if ((kAssumptions & Dot::kMultipleOfVector) && (num % N) != 0) continue;
ForeachMisalign<kAssumptions>(d, num, rng);
}
}
public:
// Must be inlined on aarch64 for bf16, else clang crashes.
template <class T, class D>
HWY_INLINE void operator()(T /*unused*/, D d) {
RandomState rng;
// All 8 combinations of the three length-related flags:
ForeachCount<0>(d, rng);
ForeachCount<Dot::kAtLeastOneVector>(d, rng);
ForeachCount<Dot::kMultipleOfVector>(d, rng);
ForeachCount<Dot::kMultipleOfVector | Dot::kAtLeastOneVector>(d, rng);
ForeachCount<Dot::kPaddedToVector>(d, rng);
ForeachCount<Dot::kPaddedToVector | Dot::kAtLeastOneVector>(d, rng);
ForeachCount<Dot::kPaddedToVector | Dot::kMultipleOfVector>(d, rng);
ForeachCount<Dot::kPaddedToVector | Dot::kMultipleOfVector |
Dot::kAtLeastOneVector>(d, rng);
}
};
class TestDotF32BF16 {
// Computes/verifies one dot product.
template <int kAssumptions, class D>
void Test(D d, size_t num, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
using T2 = hwy::bfloat16_t;
const size_t N = Lanes(d);
const auto random_t = [&rng]() {
const int32_t bits = static_cast<int32_t>(Random32(&rng)) & 1023;
return static_cast<float>(bits - 512) * (1.0f / 64);
};
const size_t padded =
(kAssumptions & Dot::kPaddedToVector) ? RoundUpTo(num, N) : num;
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(misalign_a + padded);
AlignedFreeUniquePtr<T2[]> pb = AllocateAligned<T2>(misalign_b + padded);
HWY_ASSERT(pa && pb);
T* a = pa.get() + misalign_a;
T2* b = pb.get() + misalign_b;
size_t i = 0;
for (; i < num; ++i) {
a[i] = ConvertScalarTo<T>(random_t());
b[i] = ConvertScalarTo<T2>(random_t());
}
// Fill padding with NaN - the values are not used, but avoids MSAN errors.
for (; i < padded; ++i) {
ScalableTag<float> df1;
a[i] = ConvertScalarTo<T>(GetLane(NaN(df1)));
b[i] = ConvertScalarTo<T2>(GetLane(NaN(df1)));
}
const double expected = SimpleDot(a, b, num);
const double magnitude = expected > 0.0 ? expected : -expected;
const double actual =
ConvertScalarTo<double>(Dot::Compute<kAssumptions>(d, a, b, num));
const double max = static_cast<double>(8 * 8 * num);
HWY_ASSERT(-max <= actual && actual <= max);
const double tolerance =
64.0 * ConvertScalarTo<double>(Epsilon<T2>()) * HWY_MAX(magnitude, 1.0);
HWY_ASSERT(expected - tolerance <= actual &&
actual <= expected + tolerance);
}
// Runs tests with various alignments.
template <int kAssumptions, class D>
void ForeachMisalign(D d, size_t num, RandomState& rng) {
const size_t N = Lanes(d);
const size_t misalignments[3] = {0, N / 4, 3 * N / 5};
for (size_t ma : misalignments) {
for (size_t mb : misalignments) {
Test<kAssumptions>(d, num, ma, mb, rng);
}
}
}
// Runs tests with various lengths compatible with the given assumptions.
template <int kAssumptions, class D>
void ForeachCount(D d, RandomState& rng) {
const size_t N = Lanes(d);
const size_t counts[] = {1,
3,
7,
16,
HWY_MAX(N / 2, 1),
HWY_MAX(2 * N / 3, 1),
N,
N + 1,
4 * N / 3,
3 * N,
8 * N,
8 * N + 2};
for (size_t num : counts) {
if ((kAssumptions & Dot::kAtLeastOneVector) && num < N) continue;
if ((kAssumptions & Dot::kMultipleOfVector) && (num % N) != 0) continue;
ForeachMisalign<kAssumptions>(d, num, rng);
}
}
public:
// Must be inlined on aarch64 for bf16, else clang crashes.
template <class T, class D>
HWY_INLINE void operator()(T /*unused*/, D d) {
RandomState rng;
// All 8 combinations of the three length-related flags:
ForeachCount<0>(d, rng);
ForeachCount<Dot::kAtLeastOneVector>(d, rng);
ForeachCount<Dot::kMultipleOfVector>(d, rng);
ForeachCount<Dot::kMultipleOfVector | Dot::kAtLeastOneVector>(d, rng);
ForeachCount<Dot::kPaddedToVector>(d, rng);
ForeachCount<Dot::kPaddedToVector | Dot::kAtLeastOneVector>(d, rng);
ForeachCount<Dot::kPaddedToVector | Dot::kMultipleOfVector>(d, rng);
ForeachCount<Dot::kPaddedToVector | Dot::kMultipleOfVector |
Dot::kAtLeastOneVector>(d, rng);
}
};
// All floating-point types, both arguments same.
void TestAllDot() { ForFloatTypes(ForPartialVectors<TestDot>()); }
// Mixed f32 and bf16.
void TestAllDotF32BF16() {
ForPartialVectors<TestDotF32BF16> test;
test(float());
}
// Both bf16.
void TestAllDotBF16() { ForShrinkableVectors<TestDot>()(bfloat16_t()); }
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(DotTest);
HWY_EXPORT_AND_TEST_P(DotTest, TestAllDot);
HWY_EXPORT_AND_TEST_P(DotTest, TestAllDotF32BF16);
HWY_EXPORT_AND_TEST_P(DotTest, TestAllDotBF16);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

145
deps/v8/third_party/highway/src/hwy/contrib/image/image.cc vendored Normal file
View File

@ -0,0 +1,145 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/image/image.h"
#include <algorithm> // std::swap
#include <cstddef>
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/image/image.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
size_t GetVectorSize() { return Lanes(ScalableTag<uint8_t>()); }
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(GetVectorSize); // Local function.
} // namespace
size_t ImageBase::VectorSize() {
// Do not cache result - must return the current value, which may be greater
// than the first call if it was subject to DisableTargets!
return HWY_DYNAMIC_DISPATCH(GetVectorSize)();
}
size_t ImageBase::BytesPerRow(const size_t xsize, const size_t sizeof_t) {
const size_t vec_size = VectorSize();
size_t valid_bytes = xsize * sizeof_t;
// Allow unaligned accesses starting at the last valid value - this may raise
// msan errors unless the user calls InitializePaddingForUnalignedAccesses.
// Skip for the scalar case because no extra lanes will be loaded.
if (vec_size != 1) {
HWY_DASSERT(vec_size >= sizeof_t);
valid_bytes += vec_size - sizeof_t;
}
// Round up to vector and cache line size.
const size_t align = HWY_MAX(vec_size, HWY_ALIGNMENT);
size_t bytes_per_row = RoundUpTo(valid_bytes, align);
// During the lengthy window before writes are committed to memory, CPUs
// guard against read after write hazards by checking the address, but
// only the lower 11 bits. We avoid a false dependency between writes to
// consecutive rows by ensuring their sizes are not multiples of 2 KiB.
// Avoid2K prevents the same problem for the planes of an Image3.
if (bytes_per_row % HWY_ALIGNMENT == 0) {
bytes_per_row += align;
}
HWY_DASSERT(bytes_per_row % align == 0);
return bytes_per_row;
}
ImageBase::ImageBase(const size_t xsize, const size_t ysize,
const size_t sizeof_t)
: xsize_(static_cast<uint32_t>(xsize)),
ysize_(static_cast<uint32_t>(ysize)),
bytes_(nullptr, AlignedFreer(&AlignedFreer::DoNothing, nullptr)) {
HWY_ASSERT(sizeof_t == 1 || sizeof_t == 2 || sizeof_t == 4 || sizeof_t == 8);
bytes_per_row_ = 0;
// Dimensions can be zero, e.g. for lazily-allocated images. Only allocate
// if nonzero, because "zero" bytes still have padding/bookkeeping overhead.
if (xsize != 0 && ysize != 0) {
bytes_per_row_ = BytesPerRow(xsize, sizeof_t);
bytes_ = AllocateAligned<uint8_t>(bytes_per_row_ * ysize);
HWY_ASSERT(bytes_.get() != nullptr);
InitializePadding(sizeof_t, Padding::kRoundUp);
}
}
ImageBase::ImageBase(const size_t xsize, const size_t ysize,
const size_t bytes_per_row, void* const aligned)
: xsize_(static_cast<uint32_t>(xsize)),
ysize_(static_cast<uint32_t>(ysize)),
bytes_per_row_(bytes_per_row),
bytes_(static_cast<uint8_t*>(aligned),
AlignedFreer(&AlignedFreer::DoNothing, nullptr)) {
const size_t vec_size = VectorSize();
HWY_ASSERT(bytes_per_row % vec_size == 0);
HWY_ASSERT(reinterpret_cast<uintptr_t>(aligned) % vec_size == 0);
}
void ImageBase::InitializePadding(const size_t sizeof_t, Padding padding) {
#if HWY_IS_MSAN || HWY_IDE
if (xsize_ == 0 || ysize_ == 0) return;
const size_t vec_size = VectorSize(); // Bytes, independent of sizeof_t!
if (vec_size == 1) return; // Scalar mode: no padding needed
const size_t valid_size = xsize_ * sizeof_t;
const size_t initialize_size = padding == Padding::kRoundUp
? RoundUpTo(valid_size, vec_size)
: valid_size + vec_size - sizeof_t;
if (valid_size == initialize_size) return;
for (size_t y = 0; y < ysize_; ++y) {
uint8_t* HWY_RESTRICT row = static_cast<uint8_t*>(VoidRow(y));
#if defined(__clang__) && (__clang_major__ <= 6)
// There's a bug in msan in clang-6 when handling AVX2 operations. This
// workaround allows tests to pass on msan, although it is slower and
// prevents msan warnings from uninitialized images.
memset(row, 0, initialize_size);
#else
memset(row + valid_size, 0, initialize_size - valid_size);
#endif // clang6
}
#else
(void)sizeof_t;
(void)padding;
#endif // HWY_IS_MSAN
}
void ImageBase::Swap(ImageBase& other) {
std::swap(xsize_, other.xsize_);
std::swap(ysize_, other.ysize_);
std::swap(bytes_per_row_, other.bytes_per_row_);
std::swap(bytes_, other.bytes_);
}
} // namespace hwy
#endif // HWY_ONCE

467
deps/v8/third_party/highway/src/hwy/contrib/image/image.h vendored Normal file
View File

@ -0,0 +1,467 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_CONTRIB_IMAGE_IMAGE_H_
#define HIGHWAY_HWY_CONTRIB_IMAGE_IMAGE_H_
// SIMD/multicore-friendly planar image representation with row accessors.
#include <string.h>
#include <utility> // std::move
#include "hwy/aligned_allocator.h"
#include "hwy/base.h"
namespace hwy {
// Type-independent parts of Image<> - reduces code duplication and facilitates
// moving member function implementations to cc file.
struct HWY_CONTRIB_DLLEXPORT ImageBase {
// Returns required alignment in bytes for externally allocated memory.
static size_t VectorSize();
// Returns distance [bytes] between the start of two consecutive rows, a
// multiple of VectorSize but NOT kAlias (see implementation).
static size_t BytesPerRow(size_t xsize, size_t sizeof_t);
// No allocation (for output params or unused images)
ImageBase()
: xsize_(0),
ysize_(0),
bytes_per_row_(0),
bytes_(nullptr, AlignedFreer(&AlignedFreer::DoNothing, nullptr)) {}
// Allocates memory (this is the common case)
ImageBase(size_t xsize, size_t ysize, size_t sizeof_t);
// References but does not take ownership of external memory. Useful for
// interoperability with other libraries. `aligned` must be aligned to a
// multiple of VectorSize() and `bytes_per_row` must also be a multiple of
// VectorSize() or preferably equal to BytesPerRow().
ImageBase(size_t xsize, size_t ysize, size_t bytes_per_row, void* aligned);
// Copy construction/assignment is forbidden to avoid inadvertent copies,
// which can be very expensive. Use CopyImageTo() instead.
ImageBase(const ImageBase& other) = delete;
ImageBase& operator=(const ImageBase& other) = delete;
// Move constructor (required for returning Image from function)
ImageBase(ImageBase&& other) noexcept = default;
// Move assignment (required for std::vector)
ImageBase& operator=(ImageBase&& other) noexcept = default;
void Swap(ImageBase& other);
// Useful for pre-allocating image with some padding for alignment purposes
// and later reporting the actual valid dimensions. Caller is responsible
// for ensuring xsize/ysize are <= the original dimensions.
void ShrinkTo(const size_t xsize, const size_t ysize) {
xsize_ = static_cast<uint32_t>(xsize);
ysize_ = static_cast<uint32_t>(ysize);
// NOTE: we can't recompute bytes_per_row for more compact storage and
// better locality because that would invalidate the image contents.
}
// How many pixels.
HWY_INLINE size_t xsize() const { return xsize_; }
HWY_INLINE size_t ysize() const { return ysize_; }
// NOTE: do not use this for copying rows - the valid xsize may be much less.
HWY_INLINE size_t bytes_per_row() const { return bytes_per_row_; }
// Raw access to byte contents, for interfacing with other libraries.
// Unsigned char instead of char to avoid surprises (sign extension).
HWY_INLINE uint8_t* bytes() {
void* p = bytes_.get();
return static_cast<uint8_t * HWY_RESTRICT>(HWY_ASSUME_ALIGNED(p, 64));
}
HWY_INLINE const uint8_t* bytes() const {
const void* p = bytes_.get();
return static_cast<const uint8_t * HWY_RESTRICT>(HWY_ASSUME_ALIGNED(p, 64));
}
protected:
// Returns pointer to the start of a row.
HWY_INLINE void* VoidRow(const size_t y) const {
#if HWY_IS_ASAN || HWY_IS_MSAN || HWY_IS_TSAN
if (y >= ysize_) {
HWY_ABORT("Row(%d) >= %u\n", static_cast<int>(y), ysize_);
}
#endif
void* row = bytes_.get() + y * bytes_per_row_;
return HWY_ASSUME_ALIGNED(row, 64);
}
enum class Padding {
// Allow Load(d, row + x) for x = 0; x < xsize(); x += Lanes(d). Default.
kRoundUp,
// Allow LoadU(d, row + x) for x <= xsize() - 1. This requires an extra
// vector to be initialized. If done by default, this would suppress
// legitimate msan warnings. We therefore require users to explicitly call
// InitializePadding before using unaligned loads (e.g. convolution).
kUnaligned
};
// Initializes the minimum bytes required to suppress msan warnings from
// legitimate (according to Padding mode) vector loads/stores on the right
// border, where some lanes are uninitialized and assumed to be unused.
void InitializePadding(size_t sizeof_t, Padding padding);
// (Members are non-const to enable assignment during move-assignment.)
uint32_t xsize_; // In valid pixels, not including any padding.
uint32_t ysize_;
size_t bytes_per_row_; // Includes padding.
AlignedFreeUniquePtr<uint8_t[]> bytes_;
};
// Single channel, aligned rows separated by padding. T must be POD.
//
// 'Single channel' (one 2D array per channel) simplifies vectorization
// (repeating the same operation on multiple adjacent components) without the
// complexity of a hybrid layout (8 R, 8 G, 8 B, ...). In particular, clients
// can easily iterate over all components in a row and Image requires no
// knowledge of the pixel format beyond the component type "T".
//
// 'Aligned' means each row is aligned to the L1 cache line size. This prevents
// false sharing between two threads operating on adjacent rows.
//
// 'Padding' is still relevant because vectors could potentially be larger than
// a cache line. By rounding up row sizes to the vector size, we allow
// reading/writing ALIGNED vectors whose first lane is a valid sample. This
// avoids needing a separate loop to handle remaining unaligned lanes.
//
// This image layout could also be achieved with a vector and a row accessor
// function, but a class wrapper with support for "deleter" allows wrapping
// existing memory allocated by clients without copying the pixels. It also
// provides convenient accessors for xsize/ysize, which shortens function
// argument lists. Supports move-construction so it can be stored in containers.
template <typename ComponentType>
class Image : public ImageBase {
public:
using T = ComponentType;
Image() = default;
Image(const size_t xsize, const size_t ysize)
: ImageBase(xsize, ysize, sizeof(T)) {}
Image(const size_t xsize, const size_t ysize, size_t bytes_per_row,
void* aligned)
: ImageBase(xsize, ysize, bytes_per_row, aligned) {}
void InitializePaddingForUnalignedAccesses() {
InitializePadding(sizeof(T), Padding::kUnaligned);
}
HWY_INLINE const T* ConstRow(const size_t y) const {
return static_cast<const T*>(VoidRow(y));
}
HWY_INLINE const T* ConstRow(const size_t y) {
return static_cast<const T*>(VoidRow(y));
}
// Returns pointer to non-const. This allows passing const Image* parameters
// when the callee is only supposed to fill the pixels, as opposed to
// allocating or resizing the image.
HWY_INLINE T* MutableRow(const size_t y) const {
return static_cast<T*>(VoidRow(y));
}
HWY_INLINE T* MutableRow(const size_t y) {
return static_cast<T*>(VoidRow(y));
}
// Returns number of pixels (some of which are padding) per row. Useful for
// computing other rows via pointer arithmetic. WARNING: this must
// NOT be used to determine xsize.
HWY_INLINE intptr_t PixelsPerRow() const {
return static_cast<intptr_t>(bytes_per_row_ / sizeof(T));
}
};
using ImageF = Image<float>;
// A bundle of 3 same-sized images. To fill an existing Image3 using
// single-channel producers, we also need access to each const Image*. Const
// prevents breaking the same-size invariant, while still allowing pixels to be
// changed via MutableRow.
template <typename ComponentType>
class Image3 {
public:
using T = ComponentType;
using ImageT = Image<T>;
static constexpr size_t kNumPlanes = 3;
Image3() : planes_{ImageT(), ImageT(), ImageT()} {}
Image3(const size_t xsize, const size_t ysize)
: planes_{ImageT(xsize, ysize), ImageT(xsize, ysize),
ImageT(xsize, ysize)} {}
Image3(Image3&& other) noexcept {
for (size_t i = 0; i < kNumPlanes; i++) {
planes_[i] = std::move(other.planes_[i]);
}
}
Image3(ImageT&& plane0, ImageT&& plane1, ImageT&& plane2) {
if (!SameSize(plane0, plane1) || !SameSize(plane0, plane2)) {
HWY_ABORT(
"Not same size: %d x %d, %d x %d, %d x %d\n",
static_cast<int>(plane0.xsize()), static_cast<int>(plane0.ysize()),
static_cast<int>(plane1.xsize()), static_cast<int>(plane1.ysize()),
static_cast<int>(plane2.xsize()), static_cast<int>(plane2.ysize()));
}
planes_[0] = std::move(plane0);
planes_[1] = std::move(plane1);
planes_[2] = std::move(plane2);
}
// Copy construction/assignment is forbidden to avoid inadvertent copies,
// which can be very expensive. Use CopyImageTo instead.
Image3(const Image3& other) = delete;
Image3& operator=(const Image3& other) = delete;
Image3& operator=(Image3&& other) noexcept {
for (size_t i = 0; i < kNumPlanes; i++) {
planes_[i] = std::move(other.planes_[i]);
}
return *this;
}
HWY_INLINE const T* ConstPlaneRow(const size_t c, const size_t y) const {
return static_cast<const T*>(VoidPlaneRow(c, y));
}
HWY_INLINE const T* ConstPlaneRow(const size_t c, const size_t y) {
return static_cast<const T*>(VoidPlaneRow(c, y));
}
HWY_INLINE T* MutablePlaneRow(const size_t c, const size_t y) const {
return static_cast<T*>(VoidPlaneRow(c, y));
}
HWY_INLINE T* MutablePlaneRow(const size_t c, const size_t y) {
return static_cast<T*>(VoidPlaneRow(c, y));
}
HWY_INLINE const ImageT& Plane(size_t idx) const { return planes_[idx]; }
void Swap(Image3& other) {
for (size_t c = 0; c < 3; ++c) {
other.planes_[c].Swap(planes_[c]);
}
}
void ShrinkTo(const size_t xsize, const size_t ysize) {
for (ImageT& plane : planes_) {
plane.ShrinkTo(xsize, ysize);
}
}
// Sizes of all three images are guaranteed to be equal.
HWY_INLINE size_t xsize() const { return planes_[0].xsize(); }
HWY_INLINE size_t ysize() const { return planes_[0].ysize(); }
// Returns offset [bytes] from one row to the next row of the same plane.
// WARNING: this must NOT be used to determine xsize, nor for copying rows -
// the valid xsize may be much less.
HWY_INLINE size_t bytes_per_row() const { return planes_[0].bytes_per_row(); }
// Returns number of pixels (some of which are padding) per row. Useful for
// computing other rows via pointer arithmetic. WARNING: this must NOT be used
// to determine xsize.
HWY_INLINE intptr_t PixelsPerRow() const { return planes_[0].PixelsPerRow(); }
private:
// Returns pointer to the start of a row.
HWY_INLINE void* VoidPlaneRow(const size_t c, const size_t y) const {
#if HWY_IS_ASAN || HWY_IS_MSAN || HWY_IS_TSAN
if (c >= kNumPlanes || y >= ysize()) {
HWY_ABORT("PlaneRow(%d, %d) >= %d\n", static_cast<int>(c),
static_cast<int>(y), static_cast<int>(ysize()));
}
#endif
// Use the first plane's stride because the compiler might not realize they
// are all equal. Thus we only need a single multiplication for all planes.
const size_t row_offset = y * planes_[0].bytes_per_row();
const void* row = planes_[c].bytes() + row_offset;
return static_cast<const T * HWY_RESTRICT>(
HWY_ASSUME_ALIGNED(row, HWY_ALIGNMENT));
}
private:
ImageT planes_[kNumPlanes];
};
using Image3F = Image3<float>;
// Rectangular region in image(s). Factoring this out of Image instead of
// shifting the pointer by x0/y0 allows this to apply to multiple images with
// different resolutions. Can compare size via SameSize(rect1, rect2).
class Rect {
public:
// Most windows are xsize_max * ysize_max, except those on the borders where
// begin + size_max > end.
constexpr Rect(size_t xbegin, size_t ybegin, size_t xsize_max,
size_t ysize_max, size_t xend, size_t yend)
: x0_(xbegin),
y0_(ybegin),
xsize_(ClampedSize(xbegin, xsize_max, xend)),
ysize_(ClampedSize(ybegin, ysize_max, yend)) {}
// Construct with origin and known size (typically from another Rect).
constexpr Rect(size_t xbegin, size_t ybegin, size_t xsize, size_t ysize)
: x0_(xbegin), y0_(ybegin), xsize_(xsize), ysize_(ysize) {}
// Construct a rect that covers a whole image.
template <typename Image>
explicit Rect(const Image& image)
: Rect(0, 0, image.xsize(), image.ysize()) {}
Rect() : Rect(0, 0, 0, 0) {}
Rect(const Rect&) = default;
Rect& operator=(const Rect&) = default;
Rect Subrect(size_t xbegin, size_t ybegin, size_t xsize_max,
size_t ysize_max) {
return Rect(x0_ + xbegin, y0_ + ybegin, xsize_max, ysize_max, x0_ + xsize_,
y0_ + ysize_);
}
template <typename T>
const T* ConstRow(const Image<T>* image, size_t y) const {
return image->ConstRow(y + y0_) + x0_;
}
template <typename T>
T* MutableRow(const Image<T>* image, size_t y) const {
return image->MutableRow(y + y0_) + x0_;
}
template <typename T>
const T* ConstPlaneRow(const Image3<T>& image, size_t c, size_t y) const {
return image.ConstPlaneRow(c, y + y0_) + x0_;
}
template <typename T>
T* MutablePlaneRow(Image3<T>* image, const size_t c, size_t y) const {
return image->MutablePlaneRow(c, y + y0_) + x0_;
}
// Returns true if this Rect fully resides in the given image. ImageT could be
// Image<T> or Image3<T>; however if ImageT is Rect, results are nonsensical.
template <class ImageT>
bool IsInside(const ImageT& image) const {
return (x0_ + xsize_ <= image.xsize()) && (y0_ + ysize_ <= image.ysize());
}
size_t x0() const { return x0_; }
size_t y0() const { return y0_; }
size_t xsize() const { return xsize_; }
size_t ysize() const { return ysize_; }
private:
// Returns size_max, or whatever is left in [begin, end).
static constexpr size_t ClampedSize(size_t begin, size_t size_max,
size_t end) {
return (begin + size_max <= end) ? size_max
: (end > begin ? end - begin : 0);
}
size_t x0_;
size_t y0_;
size_t xsize_;
size_t ysize_;
};
// Works for any image-like input type(s).
template <class Image1, class Image2>
HWY_MAYBE_UNUSED bool SameSize(const Image1& image1, const Image2& image2) {
return image1.xsize() == image2.xsize() && image1.ysize() == image2.ysize();
}
// Mirrors out of bounds coordinates and returns valid coordinates unchanged.
// We assume the radius (distance outside the image) is small compared to the
// image size, otherwise this might not terminate.
// The mirror is outside the last column (border pixel is also replicated).
static HWY_INLINE HWY_MAYBE_UNUSED size_t Mirror(int64_t x,
const int64_t xsize) {
HWY_DASSERT(xsize != 0);
// TODO(janwas): replace with branchless version
while (x < 0 || x >= xsize) {
if (x < 0) {
x = -x - 1;
} else {
x = 2 * xsize - 1 - x;
}
}
return static_cast<size_t>(x);
}
// Wrap modes for ensuring X/Y coordinates are in the valid range [0, size):
// Mirrors (repeating the edge pixel once). Useful for convolutions.
struct WrapMirror {
HWY_INLINE size_t operator()(const int64_t coord, const size_t size) const {
return Mirror(coord, static_cast<int64_t>(size));
}
};
// Returns the same coordinate, for when we know "coord" is already valid (e.g.
// interior of an image).
struct WrapUnchanged {
HWY_INLINE size_t operator()(const int64_t coord, size_t /*size*/) const {
return static_cast<size_t>(coord);
}
};
// Similar to Wrap* but for row pointers (reduces Row() multiplications).
class WrapRowMirror {
public:
template <class View>
WrapRowMirror(const View& image, size_t ysize)
: first_row_(image.ConstRow(0)), last_row_(image.ConstRow(ysize - 1)) {}
const float* operator()(const float* const HWY_RESTRICT row,
const int64_t stride) const {
if (row < first_row_) {
const int64_t num_before = first_row_ - row;
// Mirrored; one row before => row 0, two before = row 1, ...
return first_row_ + num_before - stride;
}
if (row > last_row_) {
const int64_t num_after = row - last_row_;
// Mirrored; one row after => last row, two after = last - 1, ...
return last_row_ - num_after + stride;
}
return row;
}
private:
const float* const HWY_RESTRICT first_row_;
const float* const HWY_RESTRICT last_row_;
};
struct WrapRowUnchanged {
HWY_INLINE const float* operator()(const float* const HWY_RESTRICT row,
int64_t /*stride*/) const {
return row;
}
};
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_IMAGE_IMAGE_H_

153
deps/v8/third_party/highway/src/hwy/contrib/image/image_test.cc vendored Normal file
View File

@ -0,0 +1,153 @@
// Copyright (c) the JPEG XL Project
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/image/image.h"
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <random>
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/image/image_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/tests/test_util-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
// Ensure we can always write full aligned vectors.
struct TestAlignedT {
template <typename T>
void operator()(T /*unused*/) const {
std::mt19937 rng(129);
std::uniform_int_distribution<int> dist(0, 16);
const ScalableTag<T> d;
for (size_t ysize = 1; ysize < 4; ++ysize) {
for (size_t xsize = 1; xsize < 64; ++xsize) {
Image<T> img(xsize, ysize);
for (size_t y = 0; y < ysize; ++y) {
T* HWY_RESTRICT row = img.MutableRow(y);
for (size_t x = 0; x < xsize; x += Lanes(d)) {
const auto values = Iota(d, dist(rng));
Store(values, d, row + x);
}
}
// Sanity check to prevent optimizing out the writes
const auto x = std::uniform_int_distribution<size_t>(0, xsize - 1)(rng);
const auto y = std::uniform_int_distribution<size_t>(0, ysize - 1)(rng);
HWY_ASSERT(img.ConstRow(y)[x] < 16 + Lanes(d));
}
}
}
};
void TestAligned() { ForUnsignedTypes(TestAlignedT()); }
// Ensure we can write an unaligned vector starting at the last valid value.
struct TestUnalignedT {
template <typename T>
void operator()(T /*unused*/) const {
std::mt19937 rng(129);
std::uniform_int_distribution<int> dist(0, 3);
const ScalableTag<T> d;
for (size_t ysize = 1; ysize < 4; ++ysize) {
for (size_t xsize = 1; xsize < 128; ++xsize) {
Image<T> img(xsize, ysize);
img.InitializePaddingForUnalignedAccesses();
// This test reads padding, which only works if it was initialized,
// which only happens in MSAN builds.
#if HWY_IS_MSAN || HWY_IDE
// Initialize only the valid samples
for (size_t y = 0; y < ysize; ++y) {
T* HWY_RESTRICT row = img.MutableRow(y);
for (size_t x = 0; x < xsize; ++x) {
row[x] = ConvertScalarTo<T>(1u << dist(rng));
}
}
// Read padding bits
auto accum = Zero(d);
for (size_t y = 0; y < ysize; ++y) {
T* HWY_RESTRICT row = img.MutableRow(y);
for (size_t x = 0; x < xsize; ++x) {
accum = Or(accum, LoadU(d, row + x));
}
}
// Ensure padding was zero
const size_t N = Lanes(d);
auto lanes = AllocateAligned<T>(N);
HWY_ASSERT(lanes);
Store(accum, d, lanes.get());
for (size_t i = 0; i < N; ++i) {
HWY_ASSERT(lanes[i] < 16);
}
#else // Check that writing padding does not overwrite valid samples
// Initialize only the valid samples
for (size_t y = 0; y < ysize; ++y) {
T* HWY_RESTRICT row = img.MutableRow(y);
for (size_t x = 0; x < xsize; ++x) {
row[x] = ConvertScalarTo<T>(x);
}
}
// Zero padding and rightmost sample
for (size_t y = 0; y < ysize; ++y) {
T* HWY_RESTRICT row = img.MutableRow(y);
StoreU(Zero(d), d, row + xsize - 1);
}
// Ensure no samples except the rightmost were overwritten
for (size_t y = 0; y < ysize; ++y) {
T* HWY_RESTRICT row = img.MutableRow(y);
for (size_t x = 0; x < xsize - 1; ++x) {
HWY_ASSERT_EQ(ConvertScalarTo<T>(x), row[x]);
}
}
#endif
}
}
}
};
void TestUnaligned() { ForUnsignedTypes(TestUnalignedT()); }
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(ImageTest);
HWY_EXPORT_AND_TEST_P(ImageTest, TestAligned);
HWY_EXPORT_AND_TEST_P(ImageTest, TestUnaligned);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

1752
deps/v8/third_party/highway/src/hwy/contrib/math/math-inl.h vendored Normal file
View File

File diff suppressed because it is too large Load Diff

664
deps/v8/third_party/highway/src/hwy/contrib/math/math_test.cc vendored Normal file
View File

@ -0,0 +1,664 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <cfloat> // FLT_MAX
#include <cmath> // std::abs
#include "hwy/base.h"
#include "hwy/nanobenchmark.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/math/math_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/math/math-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
// We have had test failures caused by excess precision due to keeping
// intermediate results in 80-bit x87 registers. One such failure mode is that
// Log1p computes a 1.0 which is not exactly equal to 1.0f, causing is_pole to
// incorrectly evaluate to false.
#undef HWY_MATH_TEST_EXCESS_PRECISION
#if HWY_ARCH_X86_32 && HWY_COMPILER_GCC_ACTUAL && \
(HWY_TARGET == HWY_SCALAR || HWY_TARGET == HWY_EMU128)
// GCC 13+: because CMAKE_CXX_EXTENSIONS is OFF, we build with -std= and hence
// also -fexcess-precision=standard, so there is no problem. See #1708 and
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=323.
#if HWY_COMPILER_GCC_ACTUAL >= 1300
#define HWY_MATH_TEST_EXCESS_PRECISION 0
#else // HWY_COMPILER_GCC_ACTUAL < 1300
// The build system must enable SSE2, e.g. via HWY_CMAKE_SSE2 - see
// https://stackoverflow.com/questions/20869904/c-handling-of-excess-precision .
#if defined(__SSE2__) // correct flag given, no problem
#define HWY_MATH_TEST_EXCESS_PRECISION 0
#else
#define HWY_MATH_TEST_EXCESS_PRECISION 1
#pragma message( \
"Skipping scalar math_test on 32-bit x86 GCC <13 without HWY_CMAKE_SSE2")
#endif // defined(__SSE2__)
#endif // HWY_COMPILER_GCC_ACTUAL
#else // not (x86-32, GCC, scalar target): running math_test normally
#define HWY_MATH_TEST_EXCESS_PRECISION 0
#endif // HWY_ARCH_X86_32 etc
template <class T, class D>
HWY_NOINLINE void TestMath(const char* name, T (*fx1)(T),
Vec<D> (*fxN)(D, VecArg<Vec<D>>), D d, T min, T max,
uint64_t max_error_ulp) {
if (HWY_MATH_TEST_EXCESS_PRECISION) {
static bool once = true;
if (once) {
once = false;
fprintf(stderr,
"Skipping math_test due to GCC issue with excess precision.\n");
}
return;
}
using UintT = MakeUnsigned<T>;
const UintT min_bits = BitCastScalar<UintT>(min);
const UintT max_bits = BitCastScalar<UintT>(max);
// If min is negative and max is positive, the range needs to be broken into
// two pieces, [+0, max] and [-0, min], otherwise [min, max].
int range_count = 1;
UintT ranges[2][2] = {{min_bits, max_bits}, {0, 0}};
if ((min < 0.0) && (max > 0.0)) {
ranges[0][0] = BitCastScalar<UintT>(ConvertScalarTo<T>(+0.0));
ranges[0][1] = max_bits;
ranges[1][0] = BitCastScalar<UintT>(ConvertScalarTo<T>(-0.0));
ranges[1][1] = min_bits;
range_count = 2;
}
uint64_t max_ulp = 0;
// Emulation is slower, so cannot afford as many.
constexpr UintT kSamplesPerRange = static_cast<UintT>(AdjustedReps(4000));
for (int range_index = 0; range_index < range_count; ++range_index) {
const UintT start = ranges[range_index][0];
const UintT stop = ranges[range_index][1];
const UintT step = HWY_MAX(1, ((stop - start) / kSamplesPerRange));
for (UintT value_bits = start; value_bits <= stop; value_bits += step) {
// For reasons unknown, the HWY_MAX is necessary on RVV, otherwise
// value_bits can be less than start, and thus possibly NaN.
const T value =
BitCastScalar<T>(HWY_MIN(HWY_MAX(start, value_bits), stop));
const T actual = GetLane(fxN(d, Set(d, value)));
const T expected = fx1(value);
// Skip small inputs and outputs on armv7, it flushes subnormals to zero.
#if HWY_TARGET <= HWY_NEON_WITHOUT_AES && HWY_ARCH_ARM_V7
if ((std::abs(value) < 1e-37f) || (std::abs(expected) < 1e-37f)) {
continue;
}
#endif
const auto ulp = hwy::detail::ComputeUlpDelta(actual, expected);
max_ulp = HWY_MAX(max_ulp, ulp);
if (ulp > max_error_ulp) {
fprintf(stderr, "%s: %s(%f) expected %E actual %E ulp %g max ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), name, value, expected,
actual, static_cast<double>(ulp),
static_cast<uint32_t>(max_error_ulp));
}
}
}
fprintf(stderr, "%s: %s max_ulp %g\n", hwy::TypeName(T(), Lanes(d)).c_str(),
name, static_cast<double>(max_ulp));
HWY_ASSERT(max_ulp <= max_error_ulp);
}
#define DEFINE_MATH_TEST_FUNC(NAME) \
HWY_NOINLINE void TestAll##NAME() { \
ForFloat3264Types(ForPartialVectors<Test##NAME>()); \
}
#undef DEFINE_MATH_TEST
#define DEFINE_MATH_TEST(NAME, F32x1, F32xN, F32_MIN, F32_MAX, F32_ERROR, \
F64x1, F64xN, F64_MIN, F64_MAX, F64_ERROR) \
struct Test##NAME { \
template <class T, class D> \
HWY_NOINLINE void operator()(T, D d) { \
if (sizeof(T) == 4) { \
TestMath<T, D>(HWY_STR(NAME), F32x1, F32xN, d, F32_MIN, F32_MAX, \
F32_ERROR); \
} else { \
TestMath<T, D>(HWY_STR(NAME), F64x1, F64xN, d, \
static_cast<T>(F64_MIN), static_cast<T>(F64_MAX), \
F64_ERROR); \
} \
} \
}; \
DEFINE_MATH_TEST_FUNC(NAME)
// Floating point values closest to but less than 1.0. Avoid variables with
// static initializers inside HWY_BEFORE_NAMESPACE/HWY_AFTER_NAMESPACE to
// ensure target-specific code does not leak into startup code.
float kNearOneF() { return BitCastScalar<float>(0x3F7FFFFF); }
double kNearOneD() { return BitCastScalar<double>(0x3FEFFFFFFFFFFFFFULL); }
// The discrepancy is unacceptably large for MSYS2 (less accurate libm?), so
// only increase the error tolerance there.
constexpr uint64_t Cos64ULP() {
#if defined(__MINGW32__)
return 23;
#else
return 3;
#endif
}
constexpr uint64_t ACosh32ULP() {
#if defined(__MINGW32__)
return 8;
#else
return 3;
#endif
}
template <class D>
static Vec<D> SinCosSin(const D d, VecArg<Vec<D>> x) {
Vec<D> s, c;
CallSinCos(d, x, s, c);
return s;
}
template <class D>
static Vec<D> SinCosCos(const D d, VecArg<Vec<D>> x) {
Vec<D> s, c;
CallSinCos(d, x, s, c);
return c;
}
// on targets without FMA the result is less inaccurate
constexpr uint64_t SinCosSin32ULP() {
#if !(HWY_NATIVE_FMA)
return 256;
#else
return 3;
#endif
}
constexpr uint64_t SinCosCos32ULP() {
#if !(HWY_NATIVE_FMA)
return 64;
#else
return 3;
#endif
}
// clang-format off
DEFINE_MATH_TEST(Acos,
std::acos, CallAcos, -1.0f, +1.0f, 3, // NEON is 3 instead of 2
std::acos, CallAcos, -1.0, +1.0, 2)
DEFINE_MATH_TEST(Acosh,
std::acosh, CallAcosh, +1.0f, +FLT_MAX, ACosh32ULP(),
std::acosh, CallAcosh, +1.0, +DBL_MAX, 3)
DEFINE_MATH_TEST(Asin,
std::asin, CallAsin, -1.0f, +1.0f, 4, // 4 ulp on Armv7, not 2
std::asin, CallAsin, -1.0, +1.0, 2)
DEFINE_MATH_TEST(Asinh,
std::asinh, CallAsinh, -FLT_MAX, +FLT_MAX, 3,
std::asinh, CallAsinh, -DBL_MAX, +DBL_MAX, 3)
DEFINE_MATH_TEST(Atan,
std::atan, CallAtan, -FLT_MAX, +FLT_MAX, 3,
std::atan, CallAtan, -DBL_MAX, +DBL_MAX, 3)
// NEON has ULP 4 instead of 3
DEFINE_MATH_TEST(Atanh,
std::atanh, CallAtanh, -kNearOneF(), +kNearOneF(), 4,
std::atanh, CallAtanh, -kNearOneD(), +kNearOneD(), 3)
DEFINE_MATH_TEST(Cos,
std::cos, CallCos, -39000.0f, +39000.0f, 3,
std::cos, CallCos, -39000.0, +39000.0, Cos64ULP())
DEFINE_MATH_TEST(Exp,
std::exp, CallExp, -FLT_MAX, +104.0f, 1,
std::exp, CallExp, -DBL_MAX, +104.0, 1)
DEFINE_MATH_TEST(Exp2,
std::exp2, CallExp2, -FLT_MAX, +128.0f, 2,
std::exp2, CallExp2, -DBL_MAX, +128.0, 2)
DEFINE_MATH_TEST(Expm1,
std::expm1, CallExpm1, -FLT_MAX, +104.0f, 4,
std::expm1, CallExpm1, -DBL_MAX, +104.0, 4)
DEFINE_MATH_TEST(Log,
std::log, CallLog, +FLT_MIN, +FLT_MAX, 1,
std::log, CallLog, +DBL_MIN, +DBL_MAX, 1)
DEFINE_MATH_TEST(Log10,
std::log10, CallLog10, +FLT_MIN, +FLT_MAX, 2,
std::log10, CallLog10, +DBL_MIN, +DBL_MAX, 2)
DEFINE_MATH_TEST(Log1p,
std::log1p, CallLog1p, +0.0f, +1e37f, 3, // NEON is 3 instead of 2
std::log1p, CallLog1p, +0.0, +DBL_MAX, 2)
DEFINE_MATH_TEST(Log2,
std::log2, CallLog2, +FLT_MIN, +FLT_MAX, 2,
std::log2, CallLog2, +DBL_MIN, +DBL_MAX, 2)
DEFINE_MATH_TEST(Sin,
std::sin, CallSin, -39000.0f, +39000.0f, 3,
std::sin, CallSin, -39000.0, +39000.0, 4) // MSYS is 4 instead of 3
DEFINE_MATH_TEST(Sinh,
std::sinh, CallSinh, -80.0f, +80.0f, 4,
std::sinh, CallSinh, -709.0, +709.0, 4)
DEFINE_MATH_TEST(Tanh,
std::tanh, CallTanh, -FLT_MAX, +FLT_MAX, 4,
std::tanh, CallTanh, -DBL_MAX, +DBL_MAX, 4)
DEFINE_MATH_TEST(SinCosSin,
std::sin, SinCosSin, -39000.0f, +39000.0f, SinCosSin32ULP(),
std::sin, SinCosSin, -39000.0, +39000.0, 1)
DEFINE_MATH_TEST(SinCosCos,
std::cos, SinCosCos, -39000.0f, +39000.0f, SinCosCos32ULP(),
std::cos, SinCosCos, -39000.0, +39000.0, 1)
// clang-format on
template <typename T, class D>
void Atan2TestCases(T /*unused*/, D d, size_t& padded,
AlignedFreeUniquePtr<T[]>& out_y,
AlignedFreeUniquePtr<T[]>& out_x,
AlignedFreeUniquePtr<T[]>& out_expected) {
struct YX {
T y;
T x;
T expected;
};
const T pos = ConvertScalarTo<T>(1E5);
const T neg = ConvertScalarTo<T>(-1E7);
const T p0 = ConvertScalarTo<T>(0);
// -0 is not enough to get an actual negative zero.
const T n0 = ConvertScalarTo<T>(-0.0);
const T p1 = ConvertScalarTo<T>(1);
const T n1 = ConvertScalarTo<T>(-1);
const T p2 = ConvertScalarTo<T>(2);
const T n2 = ConvertScalarTo<T>(-2);
const T inf = GetLane(Inf(d));
const T nan = GetLane(NaN(d));
const T pi = ConvertScalarTo<T>(3.141592653589793238);
const YX test_cases[] = { // 45 degree steps:
{p0, p1, p0}, // E
{n1, p1, -pi / 4}, // SE
{n1, p0, -pi / 2}, // S
{n1, n1, -3 * pi / 4}, // SW
{p0, n1, pi}, // W
{p1, n1, 3 * pi / 4}, // NW
{p1, p0, pi / 2}, // N
{p1, p1, pi / 4}, // NE
// y = ±0, x < 0 or -0
{p0, n1, pi},
{n0, n2, -pi},
// y = ±0, x > 0 or +0
{p0, p2, p0},
{n0, p2, n0},
// y = ±∞, x finite
{inf, p2, pi / 2},
{-inf, p2, -pi / 2},
// y = ±∞, x = -∞
{inf, -inf, 3 * pi / 4},
{-inf, -inf, -3 * pi / 4},
// y = ±∞, x = +∞
{inf, inf, pi / 4},
{-inf, inf, -pi / 4},
// y < 0, x = ±0
{n2, p0, -pi / 2},
{n1, n0, -pi / 2},
// y > 0, x = ±0
{pos, p0, pi / 2},
{p2, n0, pi / 2},
// finite y > 0, x = -∞
{pos, -inf, pi},
// finite y < 0, x = -∞
{neg, -inf, -pi},
// finite y > 0, x = +∞
{pos, inf, p0},
// finite y < 0, x = +∞
{neg, inf, n0},
// y NaN xor x NaN
{nan, p0, nan},
{pos, nan, nan}};
const size_t kNumTestCases = sizeof(test_cases) / sizeof(test_cases[0]);
const size_t N = Lanes(d);
padded = RoundUpTo(kNumTestCases, N); // allow loading whole vectors
out_y = AllocateAligned<T>(padded);
out_x = AllocateAligned<T>(padded);
out_expected = AllocateAligned<T>(padded);
HWY_ASSERT(out_y && out_x && out_expected);
size_t i = 0;
for (; i < kNumTestCases; ++i) {
out_y[i] = test_cases[i].y;
out_x[i] = test_cases[i].x;
out_expected[i] = test_cases[i].expected;
}
for (; i < padded; ++i) {
out_y[i] = p0;
out_x[i] = p0;
out_expected[i] = p0;
}
}
struct TestAtan2 {
template <typename T, class D>
HWY_NOINLINE void operator()(T t, D d) {
const size_t N = Lanes(d);
size_t padded;
AlignedFreeUniquePtr<T[]> in_y, in_x, expected;
Atan2TestCases(t, d, padded, in_y, in_x, expected);
const Vec<D> tolerance = Set(d, ConvertScalarTo<T>(1E-5));
for (size_t i = 0; i < padded; ++i) {
const T actual = ConvertScalarTo<T>(atan2(in_y[i], in_x[i]));
// fprintf(stderr, "%zu: table %f atan2 %f\n", i, expected[i], actual);
HWY_ASSERT_EQ(expected[i], actual);
}
for (size_t i = 0; i < padded; i += N) {
const Vec<D> y = Load(d, &in_y[i]);
const Vec<D> x = Load(d, &in_x[i]);
#if HWY_ARCH_ARM_A64
// TODO(b/287462770): inline to work around incorrect SVE codegen
const Vec<D> actual = Atan2(d, y, x);
#else
const Vec<D> actual = CallAtan2(d, y, x);
#endif
const Vec<D> vexpected = Load(d, &expected[i]);
const Mask<D> exp_nan = IsNaN(vexpected);
const Mask<D> act_nan = IsNaN(actual);
HWY_ASSERT_MASK_EQ(d, exp_nan, act_nan);
// If not NaN, then compare with tolerance
const Mask<D> ge = Ge(actual, Sub(vexpected, tolerance));
const Mask<D> le = Le(actual, Add(vexpected, tolerance));
const Mask<D> ok = Or(act_nan, And(le, ge));
if (!AllTrue(d, ok)) {
const size_t mismatch =
static_cast<size_t>(FindKnownFirstTrue(d, Not(ok)));
fprintf(stderr, "Mismatch for i=%d expected %E actual %E\n",
static_cast<int>(i + mismatch), expected[i + mismatch],
ExtractLane(actual, mismatch));
HWY_ASSERT(0);
}
}
}
};
HWY_NOINLINE void TestAllAtan2() {
if (HWY_MATH_TEST_EXCESS_PRECISION) return;
ForFloat3264Types(ForPartialVectors<TestAtan2>());
}
template <typename T, class D>
void HypotTestCases(T /*unused*/, D d, size_t& padded,
AlignedFreeUniquePtr<T[]>& out_a,
AlignedFreeUniquePtr<T[]>& out_b,
AlignedFreeUniquePtr<T[]>& out_expected) {
using TU = MakeUnsigned<T>;
struct AB {
T a;
T b;
};
constexpr int kNumOfMantBits = MantissaBits<T>();
static_assert(kNumOfMantBits > 0, "kNumOfMantBits > 0 must be true");
// Ensures inputs are not constexpr.
const TU u1 = static_cast<TU>(hwy::Unpredictable1());
const double k1 = static_cast<double>(u1);
const T pos = ConvertScalarTo<T>(1E5 * k1);
const T neg = ConvertScalarTo<T>(-1E7 * k1);
const T p0 = ConvertScalarTo<T>(k1 - 1.0);
// -0 is not enough to get an actual negative zero.
const T n0 = ScalarCopySign<T>(p0, neg);
const T p1 = ConvertScalarTo<T>(k1);
const T n1 = ConvertScalarTo<T>(-k1);
const T p2 = ConvertScalarTo<T>(2 * k1);
const T n2 = ConvertScalarTo<T>(-2 * k1);
const T inf = BitCastScalar<T>(ExponentMask<T>() * u1);
const T neg_inf = ScalarCopySign(inf, n1);
const T nan = BitCastScalar<T>(
static_cast<TU>(ExponentMask<T>() | (u1 << (kNumOfMantBits - 1))));
const double max_as_f64 = ConvertScalarTo<double>(HighestValue<T>()) * k1;
const T max = ConvertScalarTo<T>(max_as_f64);
const T huge = ConvertScalarTo<T>(max_as_f64 * 0.25);
const T neg_huge = ScalarCopySign(huge, n1);
const T huge2 = ConvertScalarTo<T>(max_as_f64 * 0.039415044328304796);
const T large = ConvertScalarTo<T>(3.512227595593985E18 * k1);
const T neg_large = ScalarCopySign(large, n1);
const T large2 = ConvertScalarTo<T>(2.1190576943127544E16 * k1);
const T small = ConvertScalarTo<T>(1.067033284841808E-11 * k1);
const T neg_small = ScalarCopySign(small, n1);
const T small2 = ConvertScalarTo<T>(1.9401409532292856E-12 * k1);
const T tiny = BitCastScalar<T>(static_cast<TU>(u1 << kNumOfMantBits));
const T neg_tiny = ScalarCopySign(tiny, n1);
const T tiny2 =
ConvertScalarTo<T>(78.68466968859765 * ConvertScalarTo<double>(tiny));
const AB test_cases[] = {{p0, p0}, {p0, n0},
{n0, n0}, {p1, p1},
{p1, n1}, {n1, n1},
{p2, p2}, {p2, n2},
{p2, pos}, {p2, neg},
{n2, pos}, {n2, neg},
{n2, n2}, {p0, tiny},
{p0, neg_tiny}, {n0, tiny},
{n0, neg_tiny}, {p1, tiny},
{p1, neg_tiny}, {n1, tiny},
{n1, neg_tiny}, {tiny, p0},
{tiny2, p0}, {tiny, tiny2},
{neg_tiny, tiny2}, {huge, huge2},
{neg_huge, huge2}, {huge, p0},
{huge, tiny}, {huge2, tiny2},
{large, p0}, {large, large2},
{neg_large, p0}, {neg_large, large2},
{small, p0}, {small, small2},
{neg_small, p0}, {neg_small, small2},
{max, p0}, {max, huge},
{max, max}, {p0, inf},
{n0, inf}, {p1, inf},
{n1, inf}, {p2, inf},
{n2, inf}, {p0, neg_inf},
{n0, neg_inf}, {p1, neg_inf},
{n1, neg_inf}, {p2, neg_inf},
{n2, neg_inf}, {p0, nan},
{n0, nan}, {p1, nan},
{n1, nan}, {p2, nan},
{n2, nan}, {huge, inf},
{inf, nan}, {neg_inf, nan},
{nan, nan}};
const size_t kNumTestCases = sizeof(test_cases) / sizeof(test_cases[0]);
const size_t N = Lanes(d);
padded = RoundUpTo(kNumTestCases, N); // allow loading whole vectors
out_a = AllocateAligned<T>(padded);
out_b = AllocateAligned<T>(padded);
out_expected = AllocateAligned<T>(padded);
HWY_ASSERT(out_a && out_b && out_expected);
size_t i = 0;
for (; i < kNumTestCases; ++i) {
const T a =
test_cases[i].a * hwy::ConvertScalarTo<T>(hwy::Unpredictable1());
const T b = test_cases[i].b;
#if HWY_TARGET <= HWY_NEON_WITHOUT_AES && HWY_ARCH_ARM_V7
// Ignore test cases that have infinite or NaN inputs on Armv7 NEON
if (!ScalarIsFinite(a) || !ScalarIsFinite(b)) {
out_a[i] = p0;
out_b[i] = p0;
out_expected[i] = p0;
continue;
}
#endif
out_a[i] = a;
out_b[i] = b;
if (ScalarIsInf(a) || ScalarIsInf(b)) {
out_expected[i] = inf;
} else if (ScalarIsNaN(a) || ScalarIsNaN(b)) {
out_expected[i] = nan;
} else {
out_expected[i] = std::hypot(a, b);
}
}
for (; i < padded; ++i) {
out_a[i] = p0;
out_b[i] = p0;
out_expected[i] = p0;
}
}
struct TestHypot {
template <typename T, class D>
HWY_NOINLINE void operator()(T t, D d) {
if (HWY_MATH_TEST_EXCESS_PRECISION) {
return;
}
const size_t N = Lanes(d);
constexpr uint64_t kMaxErrorUlp = 4;
size_t padded;
AlignedFreeUniquePtr<T[]> in_a, in_b, expected;
HypotTestCases(t, d, padded, in_a, in_b, expected);
auto actual1_lanes = AllocateAligned<T>(N);
auto actual2_lanes = AllocateAligned<T>(N);
HWY_ASSERT(actual1_lanes && actual2_lanes);
uint64_t max_ulp = 0;
for (size_t i = 0; i < padded; i += N) {
const auto a = Load(d, in_a.get() + i);
const auto b = Load(d, in_b.get() + i);
#if HWY_ARCH_ARM_A64
// TODO(b/287462770): inline to work around incorrect SVE codegen
const auto actual1 = Hypot(d, a, b);
const auto actual2 = Hypot(d, b, a);
#else
const auto actual1 = CallHypot(d, a, b);
const auto actual2 = CallHypot(d, b, a);
#endif
Store(actual1, d, actual1_lanes.get());
Store(actual2, d, actual2_lanes.get());
for (size_t j = 0; j < N; j++) {
const T val_a = in_a[i + j];
const T val_b = in_b[i + j];
const T expected_val = expected[i + j];
const T actual1_val = actual1_lanes[j];
const T actual2_val = actual2_lanes[j];
const auto ulp1 =
hwy::detail::ComputeUlpDelta(actual1_val, expected_val);
if (ulp1 > kMaxErrorUlp) {
fprintf(stderr,
"%s: Hypot(%e, %e) lane %d expected %E actual %E ulp %g max "
"ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), val_a, val_b,
static_cast<int>(j), expected_val, actual1_val,
static_cast<double>(ulp1),
static_cast<uint32_t>(kMaxErrorUlp));
}
const auto ulp2 =
hwy::detail::ComputeUlpDelta(actual2_val, expected_val);
if (ulp2 > kMaxErrorUlp) {
fprintf(stderr,
"%s: Hypot(%e, %e) expected %E actual %E ulp %g max ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), val_b, val_a,
expected_val, actual2_val, static_cast<double>(ulp2),
static_cast<uint32_t>(kMaxErrorUlp));
}
max_ulp = HWY_MAX(max_ulp, HWY_MAX(ulp1, ulp2));
}
}
if (max_ulp != 0) {
fprintf(stderr, "%s: Hypot max_ulp %g\n",
hwy::TypeName(T(), Lanes(d)).c_str(),
static_cast<double>(max_ulp));
HWY_ASSERT(max_ulp <= kMaxErrorUlp);
}
}
};
HWY_NOINLINE void TestAllHypot() {
if (HWY_MATH_TEST_EXCESS_PRECISION) return;
ForFloat3264Types(ForPartialVectors<TestHypot>());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(HwyMathTest);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAcos);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAcosh);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAsin);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAsinh);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAtan);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAtanh);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllCos);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllExp);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllExp2);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllExpm1);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllLog);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllLog10);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllLog1p);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllLog2);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllSin);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllSinh);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllTanh);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAtan2);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllSinCosSin);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllSinCosCos);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllHypot);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

449
deps/v8/third_party/highway/src/hwy/contrib/matvec/matvec-inl.h vendored Normal file
View File

@ -0,0 +1,449 @@
// Copyright 2023 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Include guard (still compiled once per target)
#if defined(HIGHWAY_HWY_CONTRIB_MATVEC_MATVEC_INL_H_) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_MATVEC_MATVEC_INL_H_
#undef HIGHWAY_HWY_CONTRIB_MATVEC_MATVEC_INL_H_
#else
#define HIGHWAY_HWY_CONTRIB_MATVEC_MATVEC_INL_H_
#endif
#include "hwy/cache_control.h"
#include "hwy/contrib/thread_pool/thread_pool.h"
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
template <typename TA, typename TB>
TA AddScalar(TA a, TB b) {
return ConvertScalarTo<TA>(ConvertScalarTo<float>(a) +
ConvertScalarTo<float>(b));
}
template <size_t kOuter, size_t kInner, typename T, bool kAdd>
HWY_NOINLINE void MatVecAddImpl(const T* HWY_RESTRICT mat,
const T* HWY_RESTRICT vec,
const T* HWY_RESTRICT add, T* HWY_RESTRICT out,
hwy::ThreadPool& pool) {
(void)add;
// Process multiple rows at a time so that we write multiples of a cache line
// to avoid false sharing (>= 64). 128 is better than 256. 512 has too little
// parallelization potential.
constexpr size_t kChunkSize = 64 / sizeof(T);
const uint64_t num_chunks = static_cast<uint64_t>(kOuter / kChunkSize);
const ScalableTag<T> d;
const size_t N = Lanes(d);
// Required for Stream loop, otherwise we might have partial vectors.
HWY_DASSERT(kChunkSize >= N);
pool.Run(0, num_chunks,
[&](const uint64_t chunk, size_t /*thread*/) HWY_ATTR {
// MSVC workaround: duplicate to ensure constexpr.
constexpr size_t kChunkSize = 64 / sizeof(T);
// Software write-combining to avoid cache pollution from out.
// Although `out` may be used later, keeping it out of the cache
// now and avoiding RFOs is a consistent 5% overall win.
HWY_ALIGN T buf[kChunkSize];
// Only handle entire chunks here because the Stream is not masked.
// Remaining rows are handled after the pool.Run.
const size_t begin = static_cast<size_t>(chunk * kChunkSize);
for (size_t idx_row = 0; idx_row < kChunkSize; ++idx_row) {
auto sum0 = Zero(d);
auto sum1 = Zero(d);
// 4x unrolling barely helps SKX but likely helps Arm V2.
auto sum2 = Zero(d);
auto sum3 = Zero(d);
const T* HWY_RESTRICT row = &mat[(begin + idx_row) * kInner];
size_t i = 0;
// No clear win from prefetching from the next 1..3 rows.
// clflush &row[i] is slow, clflushopt less so but not helping.
HWY_UNROLL(1)
for (; i + 4 * N <= kInner; i += 4 * N) {
const auto a0 = LoadU(d, row + i + 0 * N);
const auto v0 = LoadU(d, vec + i + 0 * N);
sum0 = MulAdd(a0, v0, sum0);
const auto a1 = LoadU(d, row + i + 1 * N);
const auto v1 = LoadU(d, vec + i + 1 * N);
sum1 = MulAdd(a1, v1, sum1);
const auto a2 = LoadU(d, row + i + 2 * N);
const auto v2 = LoadU(d, vec + i + 2 * N);
sum2 = MulAdd(a2, v2, sum2);
const auto a3 = LoadU(d, row + i + 3 * N);
const auto v3 = LoadU(d, vec + i + 3 * N);
sum3 = MulAdd(a3, v3, sum3);
}
// Last entire vectors
for (; i + N <= kInner; i += N) {
const auto a0 = LoadU(d, row + i);
const auto v0 = LoadU(d, vec + i);
sum0 = MulAdd(a0, v0, sum0);
}
const size_t remainder = kInner - i;
if (remainder != 0) {
const auto a0 = LoadN(d, row + i, remainder);
const auto v0 = LoadN(d, vec + i, remainder);
sum1 = MulAdd(a0, v0, sum1);
}
// Reduction tree: sum of all accumulators, then their lanes
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum1);
sum0 = Add(sum0, sum2);
buf[idx_row] = ReduceSum(d, sum0);
HWY_IF_CONSTEXPR(kAdd) {
buf[idx_row] = AddScalar(buf[idx_row], add[begin + idx_row]);
}
} // idx_row
HWY_UNROLL(4) // 1..4 iterations
for (size_t i = 0; i != kChunkSize; i += N) {
Stream(Load(d, buf + i), d, out + begin + i);
}
});
hwy::FlushStream();
// Handle remainder rows which are not a multiple of the chunk size.
for (size_t r = num_chunks * kChunkSize; r < kOuter; ++r) {
auto sum0 = Zero(d);
const T* HWY_RESTRICT row = &mat[r * kInner];
size_t i = 0;
HWY_UNROLL(1)
for (; i + N <= kInner; i += N) {
const auto a0 = LoadU(d, row + i);
const auto v0 = LoadU(d, vec + i);
sum0 = MulAdd(a0, v0, sum0);
}
const size_t remainder = kInner - i;
if (remainder != 0) {
const auto a0 = LoadN(d, row + i, remainder);
const auto v0 = LoadN(d, vec + i, remainder);
sum0 = MulAdd(a0, v0, sum0);
}
out[r] = ReduceSum(d, sum0);
HWY_IF_CONSTEXPR(kAdd) { out[r] = AddScalar(out[r], add[r]); }
} // r
}
// Multiplies mat with vec, adds add and puts the result in out.
//
// mat is a (kOuter, kInner)-shaped array, where element [i,j] is located at
// index i * kInner + j.
//
// vec is a (kInner,)-shaped array.
//
// add is a (kOuter,)-shaped array.
//
// out is a (kOuter,)-shaped array that will set to mat @ vec + add.
template <size_t kOuter, size_t kInner, typename T>
HWY_NOINLINE void MatVecAdd(const T* HWY_RESTRICT mat,
const T* HWY_RESTRICT vec,
const T* HWY_RESTRICT add, T* HWY_RESTRICT out,
hwy::ThreadPool& pool) {
MatVecAddImpl<kOuter, kInner, T, true>(mat, vec, add, out, pool);
}
// Multiplies mat with vec and puts the result in out.
//
// mat is a (kOuter, kInner)-shaped array, where element [i,j] is located at
// index i * kInner + j.
//
// vec is a (kInner,)-shaped array.
//
// out is a (kOuter,)-shaped array that will set to mat @ vec.
template <size_t kOuter, size_t kInner, typename T>
HWY_NOINLINE void MatVec(const T* HWY_RESTRICT mat, const T* HWY_RESTRICT vec,
T* HWY_RESTRICT out, hwy::ThreadPool& pool) {
MatVecAddImpl<kOuter, kInner, T, false>(mat, vec, /*add=*/nullptr, out, pool);
}
// This target lacks too many ops required in our implementation, use
// HWY_EMU128 instead.
#if HWY_TARGET != HWY_SCALAR
// Specialization for bf16 matrix, which halves memory bandwidth requirements.
template <size_t kOuter, size_t kInner, bool kAdd>
HWY_NOINLINE void MatVecAddImpl(const hwy::bfloat16_t* HWY_RESTRICT mat,
const float* HWY_RESTRICT vec,
const float* HWY_RESTRICT add,
float* HWY_RESTRICT out,
hwy::ThreadPool& pool) {
// Process multiple rows at a time so that we write multiples of a cache line
// to avoid false sharing (>= 64). 128 is better than 256. 512 has too little
// parallelization potential.
constexpr size_t kChunkSize = 64 / sizeof(float);
const uint64_t num_chunks = static_cast<uint64_t>(kOuter / kChunkSize);
const ScalableTag<float> d;
const Repartition<hwy::bfloat16_t, decltype(d)> d16;
// In the remainder loop, we only process a single f32 vector, so load half
// vectors of bf16 to avoid overrun.
const Half<decltype(d16)> d16h;
using V = Vec<decltype(d)>;
using V16 = Vec<decltype(d16)>;
using V16H = Vec<decltype(d16h)>;
const size_t N = Lanes(d);
// Required for Stream loop, otherwise we might have partial vectors.
HWY_DASSERT(kChunkSize >= N);
pool.Run(0, num_chunks,
[&](const uint64_t chunk, size_t /*thread*/) HWY_ATTR {
// MSVC workaround: duplicate to ensure constexpr.
constexpr size_t kChunkSize = 64 / sizeof(float);
// Software write-combining to avoid cache pollution from out.
// Although `out` may be used later, keeping it out of the cache
// now and avoiding RFOs is a consistent 5% overall win.
HWY_ALIGN float buf[kChunkSize];
// Only handle entire chunks here because the Stream is not masked.
// Remaining rows are handled after the pool.Run.
const size_t begin = static_cast<size_t>(chunk * kChunkSize);
for (size_t idx_row = 0; idx_row < kChunkSize; ++idx_row) {
auto sum0 = Zero(d);
auto sum1 = Zero(d);
// 4x unrolling barely helps SKX but likely helps Arm V2.
auto sum2 = Zero(d);
auto sum3 = Zero(d);
const hwy::bfloat16_t* HWY_RESTRICT row =
&mat[(begin + idx_row) * kInner];
size_t i = 0;
// No clear win from prefetching from the next 1..3 rows.
// clflush &row[i] is slow, clflushopt less so but not helping.
HWY_UNROLL(1)
for (; i + 4 * N <= kInner; i += 4 * N) {
const V16 b0 = LoadU(d16, row + i + 0 * N);
const V a0 = PromoteLowerTo(d, b0);
const V a1 = PromoteUpperTo(d, b0);
const V16 b1 = LoadU(d16, row + i + 2 * N);
const V a2 = PromoteLowerTo(d, b1);
const V a3 = PromoteUpperTo(d, b1);
const V v0 = LoadU(d, vec + i + 0 * N);
sum0 = MulAdd(a0, v0, sum0);
const V v1 = LoadU(d, vec + i + 1 * N);
sum1 = MulAdd(a1, v1, sum1);
const V v2 = LoadU(d, vec + i + 2 * N);
sum2 = MulAdd(a2, v2, sum2);
const V v3 = LoadU(d, vec + i + 3 * N);
sum3 = MulAdd(a3, v3, sum3);
}
// Last entire vectors
for (; i + N <= kInner; i += N) {
const V16H b0 = LoadU(d16h, row + i);
const V a0 = PromoteTo(d, b0);
const V v0 = LoadU(d, vec + i);
sum0 = MulAdd(a0, v0, sum0);
}
const size_t remainder = kInner - i;
if (remainder != 0) {
const V16H b0 = LoadN(d16h, row + i, remainder);
const V a0 = PromoteTo(d, b0);
const V v0 = LoadN(d, vec + i, remainder);
sum1 = MulAdd(a0, v0, sum1);
}
// Reduction tree: sum of all accumulators, then their lanes
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum1);
sum0 = Add(sum0, sum2);
buf[idx_row] = ReduceSum(d, sum0);
HWY_IF_CONSTEXPR(kAdd) {
buf[idx_row] = AddScalar(buf[idx_row], add[begin + idx_row]);
}
} // idx_row
HWY_UNROLL(4) // 1..4 iterations
for (size_t i = 0; i != kChunkSize; i += N) {
Stream(Load(d, buf + i), d, out + begin + i);
}
});
hwy::FlushStream();
// Handle remainder rows which are not a multiple of the chunk size.
for (size_t r = num_chunks * kChunkSize; r < kOuter; ++r) {
auto sum0 = Zero(d);
const hwy::bfloat16_t* HWY_RESTRICT row = &mat[r * kInner];
size_t i = 0;
HWY_UNROLL(1)
for (; i + N <= kInner; i += N) {
const V16H b0 = LoadU(d16h, row + i);
const V a0 = PromoteTo(d, b0);
const V v0 = LoadU(d, vec + i);
sum0 = MulAdd(a0, v0, sum0);
}
const size_t remainder = kInner - i;
if (remainder != 0) {
const V16H b0 = LoadN(d16h, row + i, remainder);
const V a0 = PromoteTo(d, b0);
const V v0 = LoadN(d, vec + i, remainder);
sum0 = MulAdd(a0, v0, sum0);
}
out[r] = ReduceSum(d, sum0);
HWY_IF_CONSTEXPR(kAdd) { out[r] = AddScalar(out[r], add[r]); }
} // r
}
template <size_t kOuter, size_t kInner>
HWY_NOINLINE void MatVecAdd(const hwy::bfloat16_t* HWY_RESTRICT mat,
const float* HWY_RESTRICT vec,
const float* HWY_RESTRICT add,
float* HWY_RESTRICT out, hwy::ThreadPool& pool) {
MatVecAddImpl<kOuter, kInner, true>(mat, vec, add, out, pool);
}
template <size_t kOuter, size_t kInner>
HWY_NOINLINE void MatVec(const hwy::bfloat16_t* HWY_RESTRICT mat,
const float* HWY_RESTRICT vec, float* HWY_RESTRICT out,
hwy::ThreadPool& pool) {
MatVecAddImpl<kOuter, kInner, false>(mat, vec, /*add=*/nullptr, out, pool);
}
// Both mat and vec are bf16.
template <size_t kOuter, size_t kInner, bool kAdd>
HWY_NOINLINE void MatVecAddImpl(const hwy::bfloat16_t* HWY_RESTRICT mat,
const hwy::bfloat16_t* HWY_RESTRICT vec,
const hwy::bfloat16_t* HWY_RESTRICT add,
float* HWY_RESTRICT out,
hwy::ThreadPool& pool) {
// Process multiple rows at a time so that we write multiples of a cache line
// to avoid false sharing (>= 64). 128 is better than 256. 512 has too little
// parallelization potential.
constexpr size_t kChunkSize = 64 / sizeof(bfloat16_t);
const uint64_t num_chunks = static_cast<uint64_t>(kOuter / kChunkSize);
const ScalableTag<float> df;
const Repartition<hwy::bfloat16_t, decltype(df)> d16;
using V16 = Vec<decltype(d16)>;
const size_t N = Lanes(d16);
// Required for Stream loop, otherwise we might have partial vectors.
HWY_DASSERT(kChunkSize >= N);
pool.Run(0, num_chunks,
[&](const uint64_t chunk, size_t /*thread*/) HWY_ATTR {
// MSVC workaround: duplicate to ensure constexpr.
constexpr size_t kChunkSize = 64 / sizeof(bfloat16_t);
// Software write-combining to avoid cache pollution from out.
// Although `out` may be used later, keeping it out of the cache
// now and avoiding RFOs is a consistent 5% overall win.
HWY_ALIGN float buf[kChunkSize];
// Only handle entire chunks here because the Stream is not masked.
// Remaining rows are handled after the pool.Run.
const size_t begin = static_cast<size_t>(chunk * kChunkSize);
for (size_t idx_row = 0; idx_row < kChunkSize; ++idx_row) {
auto sum0 = Zero(df);
auto sum1 = Zero(df);
auto sum2 = Zero(df);
auto sum3 = Zero(df);
const hwy::bfloat16_t* HWY_RESTRICT row =
&mat[(begin + idx_row) * kInner];
size_t i = 0;
// No clear win from prefetching from the next 1..3 rows.
// clflush &row[i] is slow, clflushopt less so but not helping.
HWY_UNROLL(1)
for (; i + 2 * N <= kInner; i += 2 * N) {
const V16 b0 = LoadU(d16, row + i + 0 * N);
const V16 b1 = LoadU(d16, row + i + 1 * N);
const V16 v0 = LoadU(d16, vec + i + 0 * N);
const V16 v1 = LoadU(d16, vec + i + 1 * N);
sum0 = ReorderWidenMulAccumulate(df, b0, v0, sum0, sum1);
sum2 = ReorderWidenMulAccumulate(df, b1, v1, sum2, sum3);
}
// Last entire vector
for (; i + N <= kInner; i += N) {
const V16 b0 = LoadU(d16, row + i);
const V16 v0 = LoadU(d16, vec + i);
sum0 = ReorderWidenMulAccumulate(df, b0, v0, sum0, sum1);
}
const size_t remainder = kInner - i;
if (remainder != 0) {
const V16 b0 = LoadN(d16, row + i, remainder);
const V16 v0 = LoadN(d16, vec + i, remainder);
sum2 = ReorderWidenMulAccumulate(df, b0, v0, sum2, sum3);
}
// Reduction tree: sum of all accumulators, then their lanes
sum0 = Add(sum0, sum1);
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum2);
buf[idx_row] = ReduceSum(df, sum0);
HWY_IF_CONSTEXPR(kAdd) {
buf[idx_row] = AddScalar(buf[idx_row], add[begin + idx_row]);
}
} // idx_row
HWY_UNROLL(4) // 1..4 iterations
for (size_t i = 0; i != kChunkSize; i += N / 2) {
Stream(Load(df, buf + i), df, out + begin + i);
}
});
hwy::FlushStream();
// Handle remainder rows which are not a multiple of the chunk size.
for (size_t r = num_chunks * kChunkSize; r < kOuter; ++r) {
auto sum0 = Zero(df);
auto sum1 = Zero(df);
const hwy::bfloat16_t* HWY_RESTRICT row = &mat[r * kInner];
size_t i = 0;
HWY_UNROLL(1)
for (; i + N <= kInner; i += N) {
const V16 b0 = LoadU(d16, row + i);
const V16 v0 = LoadU(d16, vec + i);
sum0 = ReorderWidenMulAccumulate(df, b0, v0, sum0, sum1);
}
const size_t remainder = kInner - i;
if (remainder != 0) {
const V16 b0 = LoadN(d16, row + i, remainder);
const V16 v0 = LoadN(d16, vec + i, remainder);
sum0 = ReorderWidenMulAccumulate(df, b0, v0, sum0, sum1);
}
out[r] = ReduceSum(df, Add(sum0, sum1));
HWY_IF_CONSTEXPR(kAdd) { out[r] = AddScalar(out[r], add[r]); }
} // r
}
template <size_t kOuter, size_t kInner>
HWY_NOINLINE void MatVecAdd(const hwy::bfloat16_t* HWY_RESTRICT mat,
const hwy::bfloat16_t* HWY_RESTRICT vec,
const hwy::bfloat16_t* HWY_RESTRICT add,
float* HWY_RESTRICT out, hwy::ThreadPool& pool) {
MatVecAddImpl<kOuter, kInner, true>(mat, vec, add, out, pool);
}
template <size_t kOuter, size_t kInner>
HWY_NOINLINE void MatVec(const hwy::bfloat16_t* HWY_RESTRICT mat,
const hwy::bfloat16_t* HWY_RESTRICT vec,
float* HWY_RESTRICT out, hwy::ThreadPool& pool) {
MatVecAddImpl<kOuter, kInner, false>(mat, vec, /*add=*/nullptr, out, pool);
}
#endif // HWY_TARGET != HWY_SCALAR
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_MATVEC_MATVEC_INL_H_

293
deps/v8/third_party/highway/src/hwy/contrib/matvec/matvec_test.cc vendored Normal file
View File

@ -0,0 +1,293 @@
// Copyright 2023 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/base.h"
// Reduce targets to avoid timeout under emulation.
#ifndef HWY_DISABLED_TARGETS
#define HWY_DISABLED_TARGETS \
(HWY_SVE2_128 | HWY_SVE2 | HWY_SVE_256 | HWY_NEON_WITHOUT_AES)
#endif
#include <stddef.h>
#include <stdint.h>
#include "hwy/aligned_allocator.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/matvec/matvec_test.cc" // NOLINT
#include "hwy/foreach_target.h" // IWYU pragma: keep
// Must come after foreach_target.h
#include "hwy/contrib/algo/transform-inl.h"
#include "hwy/contrib/matvec/matvec-inl.h"
#include "hwy/highway.h"
#include "hwy/contrib/thread_pool/thread_pool.h"
#include "hwy/contrib/thread_pool/topology.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
template <typename MatT, typename T>
HWY_NOINLINE void SimpleMatVecAdd(const MatT* HWY_RESTRICT mat,
const T* HWY_RESTRICT vec,
const T* HWY_RESTRICT add, size_t rows,
size_t cols, T* HWY_RESTRICT out,
ThreadPool& pool) {
if (add) {
pool.Run(0, rows, [=](uint64_t r, size_t /*thread*/) {
T dot = ConvertScalarTo<T>(0);
for (size_t c = 0; c < cols; c++) {
// For reasons unknown, fp16 += does not compile on clang (Arm).
dot = ConvertScalarTo<T>(dot + mat[r * cols + c] * vec[c]);
}
out[r] = dot + add[r];
});
} else {
pool.Run(0, rows, [=](uint64_t r, size_t /*thread*/) {
T dot = ConvertScalarTo<T>(0);
for (size_t c = 0; c < cols; c++) {
// For reasons unknown, fp16 += does not compile on clang (Arm).
dot = ConvertScalarTo<T>(dot + mat[r * cols + c] * vec[c]);
}
out[r] = dot;
});
}
}
HWY_MAYBE_UNUSED HWY_NOINLINE void SimpleMatVecAdd(
const hwy::bfloat16_t* HWY_RESTRICT mat, const float* HWY_RESTRICT vec,
const float* add, size_t rows, size_t cols, float* HWY_RESTRICT out,
ThreadPool& pool) {
if (add) {
pool.Run(0, rows, [=](uint64_t r, size_t /*thread*/) {
float dot = 0.0f;
for (size_t c = 0; c < cols; c++) {
dot += F32FromBF16(mat[r * cols + c]) * vec[c];
}
out[r] = dot + add[r];
});
} else {
pool.Run(0, rows, [=](uint64_t r, size_t /*thread*/) {
float dot = 0.0f;
for (size_t c = 0; c < cols; c++) {
dot += F32FromBF16(mat[r * cols + c]) * vec[c];
}
out[r] = dot;
});
}
}
HWY_MAYBE_UNUSED HWY_NOINLINE void SimpleMatVecAdd(
const hwy::bfloat16_t* HWY_RESTRICT mat,
const hwy::bfloat16_t* HWY_RESTRICT vec,
const hwy::bfloat16_t* HWY_RESTRICT add, size_t rows, size_t cols,
float* HWY_RESTRICT out, ThreadPool& pool) {
if (add) {
pool.Run(0, rows, [=](uint64_t r, size_t /*thread*/) {
float dot = 0.0f;
for (size_t c = 0; c < cols; c++) {
dot += F32FromBF16(mat[r * cols + c]) * F32FromBF16(vec[c]);
}
out[r] = dot + F32FromBF16(add[r]);
});
} else {
pool.Run(0, rows, [=](uint64_t r, size_t /*thread*/) {
float dot = 0.0f;
for (size_t c = 0; c < cols; c++) {
dot += F32FromBF16(mat[r * cols + c]) * F32FromBF16(vec[c]);
}
out[r] = dot;
});
}
}
struct GenerateMod {
template <class D, HWY_IF_NOT_BF16_D(D), HWY_IF_LANES_GT_D(D, 1)>
Vec<D> operator()(D d, Vec<RebindToUnsigned<D>> indices) const {
const RebindToUnsigned<D> du;
return Reverse2(d, ConvertTo(d, And(indices, Set(du, 0xF))));
}
template <class D, HWY_IF_NOT_BF16_D(D), HWY_IF_LANES_LE_D(D, 1)>
Vec<D> operator()(D d, Vec<RebindToUnsigned<D>> indices) const {
const RebindToUnsigned<D> du;
return ConvertTo(d, And(indices, Set(du, 0xF)));
}
// Requires >= 4 bf16 lanes for float32 Reverse2.
template <class D, HWY_IF_BF16_D(D), HWY_IF_LANES_GT_D(D, 2)>
Vec<D> operator()(D d, Vec<RebindToUnsigned<D>> indices) const {
const RebindToUnsigned<D> du;
const RebindToSigned<D> di;
const RepartitionToWide<decltype(di)> dw;
const RebindToFloat<decltype(dw)> df;
indices = And(indices, Set(du, 0xF));
const Vec<decltype(df)> i0 = ConvertTo(df, PromoteLowerTo(dw, indices));
const Vec<decltype(df)> i1 = ConvertTo(df, PromoteUpperTo(dw, indices));
return OrderedDemote2To(d, Reverse2(df, i0), Reverse2(df, i1));
}
// For one or two lanes, we don't have OrderedDemote2To nor Reverse2.
template <class D, HWY_IF_BF16_D(D), HWY_IF_LANES_LE_D(D, 2)>
Vec<D> operator()(D d, Vec<RebindToUnsigned<D>> indices) const {
const Rebind<float, D> df;
return DemoteTo(d, Set(df, GetLane(indices)));
}
};
// MatT is usually the same as T, but can also be bfloat16_t when T = float.
template <typename MatT, typename VecT>
class TestMatVecAdd {
template <size_t kRows, size_t kCols, class D, typename T = TFromD<D>>
HWY_NOINLINE void Test(D d, ThreadPool& pool) {
// This target lacks too many ops required in our implementation, use
// HWY_EMU128 instead.
#if HWY_TARGET != HWY_SCALAR
const Repartition<MatT, D> dm;
const Repartition<VecT, D> dv;
const size_t misalign = 3 * Lanes(d) / 5;
// Fill matrix and vector with small integer values
const size_t area = kRows * kCols;
AlignedFreeUniquePtr<MatT[]> storage_m =
AllocateAligned<MatT>(misalign + area);
AlignedFreeUniquePtr<VecT[]> storage_v =
AllocateAligned<VecT>(misalign + kCols);
AlignedFreeUniquePtr<VecT[]> storage_a =
AllocateAligned<VecT>(misalign + kRows);
HWY_ASSERT(storage_m && storage_v && storage_a);
MatT* pm = storage_m.get() + misalign;
VecT* pv = storage_v.get() + misalign;
VecT* av = storage_a.get() + misalign;
Generate(dm, pm, area, GenerateMod());
Generate(dv, pv, kCols, GenerateMod());
Generate(dv, av, kRows, GenerateMod());
AlignedFreeUniquePtr<T[]> expected_without_add = AllocateAligned<T>(kRows);
HWY_ASSERT(expected_without_add);
SimpleMatVecAdd(pm, pv, static_cast<VecT*>(nullptr), kRows, kCols,
expected_without_add.get(), pool);
AlignedFreeUniquePtr<T[]> actual_without_add = AllocateAligned<T>(kRows);
HWY_ASSERT(actual_without_add);
MatVec<kRows, kCols>(pm, pv, actual_without_add.get(), pool);
const auto assert_close = [&](const AlignedFreeUniquePtr<T[]>& expected,
const AlignedFreeUniquePtr<T[]>& actual,
bool with_add) {
for (size_t i = 0; i < kRows; ++i) {
const double exp = ConvertScalarTo<double>(expected[i]);
const double act = ConvertScalarTo<double>(actual[i]);
const double tolerance =
exp * 20 * 1.0 /
(1ULL << HWY_MIN(MantissaBits<MatT>(), MantissaBits<VecT>()));
if (!(exp - tolerance <= act && act <= exp + tolerance)) {
fprintf(stderr,
"%s/%s %zu x %zu, %s: mismatch at %zu %f %f; tol %f\n",
TypeName(MatT(), 1).c_str(), TypeName(VecT(), 1).c_str(),
kRows, kCols, (with_add ? "with add" : "without add"), i, exp,
act, tolerance);
HWY_ASSERT(0);
}
}
};
assert_close(expected_without_add, actual_without_add, /*with_add=*/false);
AlignedFreeUniquePtr<T[]> expected_with_add = AllocateAligned<T>(kRows);
SimpleMatVecAdd(pm, pv, av, kRows, kCols, expected_with_add.get(), pool);
AlignedFreeUniquePtr<T[]> actual_with_add = AllocateAligned<T>(kRows);
MatVecAdd<kRows, kCols>(pm, pv, av, actual_with_add.get(), pool);
assert_close(expected_with_add, actual_with_add, /*with_add=*/true);
#else
(void)d;
(void)pool;
#endif // HWY_TARGET != HWY_SCALAR
}
template <class D>
HWY_NOINLINE void CreatePoolAndTest(D d, size_t num_threads) {
// Threads might not work on WASM; run only on main thread.
if (HaveThreadingSupport()) num_threads = 0;
ThreadPool pool(HWY_MIN(num_threads, ThreadPool::MaxThreads()));
Test<AdjustedReps(192), AdjustedReps(256)>(d, pool);
// Fewer tests due to compiler OOM
#if !HWY_ARCH_RISCV
Test<40, AdjustedReps(512)>(d, pool);
Test<AdjustedReps(1024), 50>(d, pool);
// Too large for low-precision vectors/accumulators.
if (sizeof(TFromD<D>) != 2 && sizeof(VecT) != 2) {
Test<AdjustedReps(1536), AdjustedReps(1536)>(d, pool);
}
#endif // !HWY_ARCH_RISCV
}
public:
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
CreatePoolAndTest(d, 13);
// Fewer tests due to compiler OOM
#if !HWY_ARCH_RISCV
CreatePoolAndTest(d, 16);
#endif
}
};
void TestAllMatVecAdd() {
#if HWY_HAVE_FLOAT16
ForPartialVectors<TestMatVecAdd<float16_t, float16_t>>()(float16_t());
#endif
ForPartialVectors<TestMatVecAdd<float, float>>()(float());
#if HWY_HAVE_FLOAT64
ForPartialVectors<TestMatVecAdd<double, double>>()(double());
#endif
}
void TestAllMatVecBF16() {
ForGEVectors<32, TestMatVecAdd<bfloat16_t, float>>()(float());
}
void TestAllMatVecBF16Both() {
ForGEVectors<32, TestMatVecAdd<bfloat16_t, bfloat16_t>>()(float());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(MatVecTest);
HWY_EXPORT_AND_TEST_P(MatVecTest, TestAllMatVecAdd);
HWY_EXPORT_AND_TEST_P(MatVecTest, TestAllMatVecBF16);
HWY_EXPORT_AND_TEST_P(MatVecTest, TestAllMatVecBF16Both);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

384
deps/v8/third_party/highway/src/hwy/contrib/random/random-inl.h vendored Normal file
View File

@ -0,0 +1,384 @@
/*
* Original implementation written in 2019
* by David Blackman and Sebastiano Vigna (vigna@acm.org)
* Available at https://prng.di.unimi.it/ with creative commons license:
* To the extent possible under law, the author has dedicated all copyright
* and related and neighboring rights to this software to the public domain
* worldwide. This software is distributed without any warranty.
* See <http://creativecommons.org/publicdomain/zero/1.0/>.
*
* This implementation is a Vector port of the original implementation
* written by Marco Barbone (m.barbone19@imperial.ac.uk).
* I take no credit for the original implementation.
* The code is provided as is and the original license applies.
*/
#if defined(HIGHWAY_HWY_CONTRIB_RANDOM_RANDOM_H_) == \
defined(HWY_TARGET_TOGGLE) // NOLINT
#ifdef HIGHWAY_HWY_CONTRIB_RANDOM_RANDOM_H_
#undef HIGHWAY_HWY_CONTRIB_RANDOM_RANDOM_H_
#else
#define HIGHWAY_HWY_CONTRIB_RANDOM_RANDOM_H_
#endif
#include <array>
#include <cstdint>
#include <limits>
#include "hwy/aligned_allocator.h"
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE(); // required if not using HWY_ATTR
namespace hwy {
namespace HWY_NAMESPACE { // required: unique per target
namespace internal {
namespace {
#if HWY_HAVE_FLOAT64
// C++ < 17 does not support hexfloat
#if __cpp_hex_float > 201603L
constexpr double kMulConst = 0x1.0p-53;
#else
constexpr double kMulConst =
0.00000000000000011102230246251565404236316680908203125;
#endif // __cpp_hex_float
#endif // HWY_HAVE_FLOAT64
constexpr std::uint64_t kJump[] = {0x180ec6d33cfd0aba, 0xd5a61266f0c9392c,
0xa9582618e03fc9aa, 0x39abdc4529b1661c};
constexpr std::uint64_t kLongJump[] = {0x76e15d3efefdcbbf, 0xc5004e441c522fb3,
0x77710069854ee241, 0x39109bb02acbe635};
} // namespace
class SplitMix64 {
public:
constexpr explicit SplitMix64(const std::uint64_t state) noexcept
: state_(state) {}
HWY_CXX14_CONSTEXPR std::uint64_t operator()() {
std::uint64_t z = (state_ += 0x9e3779b97f4a7c15);
z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9;
z = (z ^ (z >> 27)) * 0x94d049bb133111eb;
return z ^ (z >> 31);
}
private:
std::uint64_t state_;
};
class Xoshiro {
public:
HWY_CXX14_CONSTEXPR explicit Xoshiro(const std::uint64_t seed) noexcept
: state_{} {
SplitMix64 splitMix64{seed};
for (auto &element : state_) {
element = splitMix64();
}
}
HWY_CXX14_CONSTEXPR explicit Xoshiro(const std::uint64_t seed,
const std::uint64_t thread_id) noexcept
: Xoshiro(seed) {
for (auto i = UINT64_C(0); i < thread_id; ++i) {
Jump();
}
}
HWY_CXX14_CONSTEXPR std::uint64_t operator()() noexcept { return Next(); }
#if HWY_HAVE_FLOAT64
HWY_CXX14_CONSTEXPR double Uniform() noexcept {
return static_cast<double>(Next() >> 11) * kMulConst;
}
#endif
HWY_CXX14_CONSTEXPR std::array<std::uint64_t, 4> GetState() const {
return {state_[0], state_[1], state_[2], state_[3]};
}
HWY_CXX17_CONSTEXPR void SetState(
std::array<std::uint64_t, 4> state) noexcept {
state_[0] = state[0];
state_[1] = state[1];
state_[2] = state[2];
state_[3] = state[3];
}
static constexpr std::uint64_t StateSize() noexcept { return 4; }
/* This is the jump function for the generator. It is equivalent to 2^128
* calls to next(); it can be used to generate 2^128 non-overlapping
* subsequences for parallel computations. */
HWY_CXX14_CONSTEXPR void Jump() noexcept { Jump(kJump); }
/* This is the long-jump function for the generator. It is equivalent to 2^192
* calls to next(); it can be used to generate 2^64 starting points, from each
* of which jump() will generate 2^64 non-overlapping subsequences for
* parallel distributed computations. */
HWY_CXX14_CONSTEXPR void LongJump() noexcept { Jump(kLongJump); }
private:
std::uint64_t state_[4];
static constexpr std::uint64_t Rotl(const std::uint64_t x, int k) noexcept {
return (x << k) | (x >> (64 - k));
}
HWY_CXX14_CONSTEXPR std::uint64_t Next() noexcept {
const std::uint64_t result = Rotl(state_[0] + state_[3], 23) + state_[0];
const std::uint64_t t = state_[1] << 17;
state_[2] ^= state_[0];
state_[3] ^= state_[1];
state_[1] ^= state_[2];
state_[0] ^= state_[3];
state_[2] ^= t;
state_[3] = Rotl(state_[3], 45);
return result;
}
HWY_CXX14_CONSTEXPR void Jump(const std::uint64_t (&jumpArray)[4]) noexcept {
std::uint64_t s0 = 0;
std::uint64_t s1 = 0;
std::uint64_t s2 = 0;
std::uint64_t s3 = 0;
for (const std::uint64_t i : jumpArray)
for (std::uint_fast8_t b = 0; b < 64; b++) {
if (i & std::uint64_t{1UL} << b) {
s0 ^= state_[0];
s1 ^= state_[1];
s2 ^= state_[2];
s3 ^= state_[3];
}
Next();
}
state_[0] = s0;
state_[1] = s1;
state_[2] = s2;
state_[3] = s3;
}
};
} // namespace internal
class VectorXoshiro {
private:
using VU64 = Vec<ScalableTag<std::uint64_t>>;
using StateType = AlignedNDArray<std::uint64_t, 2>;
#if HWY_HAVE_FLOAT64
using VF64 = Vec<ScalableTag<double>>;
#endif
public:
explicit VectorXoshiro(const std::uint64_t seed,
const std::uint64_t threadNumber = 0)
: state_{{internal::Xoshiro::StateSize(),
Lanes(ScalableTag<std::uint64_t>{})}},
streams{state_.shape().back()} {
internal::Xoshiro xoshiro{seed};
for (std::uint64_t i = 0; i < threadNumber; ++i) {
xoshiro.LongJump();
}
for (size_t i = 0UL; i < streams; ++i) {
const auto state = xoshiro.GetState();
for (size_t j = 0UL; j < internal::Xoshiro::StateSize(); ++j) {
state_[{j}][i] = state[j];
}
xoshiro.Jump();
}
}
HWY_INLINE VU64 operator()() noexcept { return Next(); }
AlignedVector<std::uint64_t> operator()(const std::size_t n) {
AlignedVector<std::uint64_t> result(n);
const ScalableTag<std::uint64_t> tag{};
auto s0 = Load(tag, state_[{0}].data());
auto s1 = Load(tag, state_[{1}].data());
auto s2 = Load(tag, state_[{2}].data());
auto s3 = Load(tag, state_[{3}].data());
for (std::uint64_t i = 0; i < n; i += Lanes(tag)) {
const auto next = Update(s0, s1, s2, s3);
Store(next, tag, result.data() + i);
}
Store(s0, tag, state_[{0}].data());
Store(s1, tag, state_[{1}].data());
Store(s2, tag, state_[{2}].data());
Store(s3, tag, state_[{3}].data());
return result;
}
template <std::uint64_t N>
std::array<std::uint64_t, N> operator()() noexcept {
alignas(HWY_ALIGNMENT) std::array<std::uint64_t, N> result;
const ScalableTag<std::uint64_t> tag{};
auto s0 = Load(tag, state_[{0}].data());
auto s1 = Load(tag, state_[{1}].data());
auto s2 = Load(tag, state_[{2}].data());
auto s3 = Load(tag, state_[{3}].data());
for (std::uint64_t i = 0; i < N; i += Lanes(tag)) {
const auto next = Update(s0, s1, s2, s3);
Store(next, tag, result.data() + i);
}
Store(s0, tag, state_[{0}].data());
Store(s1, tag, state_[{1}].data());
Store(s2, tag, state_[{2}].data());
Store(s3, tag, state_[{3}].data());
return result;
}
std::uint64_t StateSize() const noexcept {
return streams * internal::Xoshiro::StateSize();
}
const StateType &GetState() const { return state_; }
#if HWY_HAVE_FLOAT64
HWY_INLINE VF64 Uniform() noexcept {
const ScalableTag<double> real_tag{};
const auto MUL_VALUE = Set(real_tag, internal::kMulConst);
const auto bits = ShiftRight<11>(Next());
const auto real = ConvertTo(real_tag, bits);
return Mul(real, MUL_VALUE);
}
AlignedVector<double> Uniform(const std::size_t n) {
AlignedVector<double> result(n);
const ScalableTag<std::uint64_t> tag{};
const ScalableTag<double> real_tag{};
const auto MUL_VALUE = Set(real_tag, internal::kMulConst);
auto s0 = Load(tag, state_[{0}].data());
auto s1 = Load(tag, state_[{1}].data());
auto s2 = Load(tag, state_[{2}].data());
auto s3 = Load(tag, state_[{3}].data());
for (std::uint64_t i = 0; i < n; i += Lanes(real_tag)) {
const auto next = Update(s0, s1, s2, s3);
const auto bits = ShiftRight<11>(next);
const auto real = ConvertTo(real_tag, bits);
const auto uniform = Mul(real, MUL_VALUE);
Store(uniform, real_tag, result.data() + i);
}
Store(s0, tag, state_[{0}].data());
Store(s1, tag, state_[{1}].data());
Store(s2, tag, state_[{2}].data());
Store(s3, tag, state_[{3}].data());
return result;
}
template <std::uint64_t N>
std::array<double, N> Uniform() noexcept {
alignas(HWY_ALIGNMENT) std::array<double, N> result;
const ScalableTag<std::uint64_t> tag{};
const ScalableTag<double> real_tag{};
const auto MUL_VALUE = Set(real_tag, internal::kMulConst);
auto s0 = Load(tag, state_[{0}].data());
auto s1 = Load(tag, state_[{1}].data());
auto s2 = Load(tag, state_[{2}].data());
auto s3 = Load(tag, state_[{3}].data());
for (std::uint64_t i = 0; i < N; i += Lanes(real_tag)) {
const auto next = Update(s0, s1, s2, s3);
const auto bits = ShiftRight<11>(next);
const auto real = ConvertTo(real_tag, bits);
const auto uniform = Mul(real, MUL_VALUE);
Store(uniform, real_tag, result.data() + i);
}
Store(s0, tag, state_[{0}].data());
Store(s1, tag, state_[{1}].data());
Store(s2, tag, state_[{2}].data());
Store(s3, tag, state_[{3}].data());
return result;
}
#endif
private:
StateType state_;
const std::uint64_t streams;
HWY_INLINE static VU64 Update(VU64 &s0, VU64 &s1, VU64 &s2,
VU64 &s3) noexcept {
const auto result = Add(RotateRight<41>(Add(s0, s3)), s0);
const auto t = ShiftLeft<17>(s1);
s2 = Xor(s2, s0);
s3 = Xor(s3, s1);
s1 = Xor(s1, s2);
s0 = Xor(s0, s3);
s2 = Xor(s2, t);
s3 = RotateRight<19>(s3);
return result;
}
HWY_INLINE VU64 Next() noexcept {
const ScalableTag<std::uint64_t> tag{};
auto s0 = Load(tag, state_[{0}].data());
auto s1 = Load(tag, state_[{1}].data());
auto s2 = Load(tag, state_[{2}].data());
auto s3 = Load(tag, state_[{3}].data());
auto result = Update(s0, s1, s2, s3);
Store(s0, tag, state_[{0}].data());
Store(s1, tag, state_[{1}].data());
Store(s2, tag, state_[{2}].data());
Store(s3, tag, state_[{3}].data());
return result;
}
};
template <std::uint64_t size = 1024>
class CachedXoshiro {
public:
using result_type = std::uint64_t;
static constexpr result_type(min)() {
return (std::numeric_limits<result_type>::min)();
}
static constexpr result_type(max)() {
return (std::numeric_limits<result_type>::max)();
}
explicit CachedXoshiro(const result_type seed,
const result_type threadNumber = 0)
: generator_{seed, threadNumber},
cache_{generator_.operator()<size>()},
index_{0} {}
result_type operator()() noexcept {
if (HWY_UNLIKELY(index_ == size)) {
cache_ = std::move(generator_.operator()<size>());
index_ = 0;
}
return cache_[index_++];
}
private:
VectorXoshiro generator_;
alignas(HWY_ALIGNMENT) std::array<result_type, size> cache_;
std::size_t index_;
static_assert((size & (size - 1)) == 0 && size != 0,
"only power of 2 are supported");
};
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_MATH_MATH_INL_H_

318
deps/v8/third_party/highway/src/hwy/contrib/random/random_test.cc vendored Normal file
View File

@ -0,0 +1,318 @@
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cstdint>
#include <cstdio>
#include <ctime>
#include <iostream> // cerr
#include <random>
#include <vector>
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/random/random_test.cc" // NOLINT
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/random/random-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE { // required: unique per target
namespace {
constexpr std::uint64_t tests = 1UL << 10;
std::uint64_t GetSeed() { return static_cast<uint64_t>(std::time(nullptr)); }
void RngLoop(const std::uint64_t seed, std::uint64_t* HWY_RESTRICT result,
const size_t size) {
const ScalableTag<std::uint64_t> d;
VectorXoshiro generator{seed};
for (size_t i = 0; i < size; i += Lanes(d)) {
Store(generator(), d, result + i);
}
}
#if HWY_HAVE_FLOAT64
void UniformLoop(const std::uint64_t seed, double* HWY_RESTRICT result,
const size_t size) {
const ScalableTag<double> d;
VectorXoshiro generator{seed};
for (size_t i = 0; i < size; i += Lanes(d)) {
Store(generator.Uniform(), d, result + i);
}
}
#endif
void TestSeeding() {
const std::uint64_t seed = GetSeed();
VectorXoshiro generator{seed};
internal::Xoshiro reference{seed};
const auto& state = generator.GetState();
const ScalableTag<std::uint64_t> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 0UL; i < lanes; ++i) {
const auto& reference_state = reference.GetState();
for (std::size_t j = 0UL; j < reference_state.size(); ++j) {
if (state[{j}][i] != reference_state[j]) {
std::cerr << "SEED: " << seed << "\n";
std::cerr << "TEST SEEDING ERROR: ";
std::cerr << "state[" << j << "][" << i << "] -> " << state[{j}][i]
<< " != " << reference_state[j] << "\n";
HWY_ASSERT(0);
}
}
reference.Jump();
}
}
void TestMultiThreadSeeding() {
const std::uint64_t seed = GetSeed();
const std::uint64_t threadId = std::random_device()() % 1000;
VectorXoshiro generator{seed, threadId};
internal::Xoshiro reference{seed};
for (std::size_t i = 0UL; i < threadId; ++i) {
reference.LongJump();
}
const auto& state = generator.GetState();
const ScalableTag<std::uint64_t> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 0UL; i < lanes; ++i) {
const auto& reference_state = reference.GetState();
for (std::size_t j = 0UL; j < reference_state.size(); ++j) {
if (state[{j}][i] != reference_state[j]) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST SEEDING ERROR: ";
std::cerr << "state[" << j << "][" << i << "] -> " << state[{j}][i]
<< " != " << reference_state[j] << "\n";
HWY_ASSERT(0);
}
}
reference.Jump();
}
}
void TestRandomUint64() {
const std::uint64_t seed = GetSeed();
const auto result_array = hwy::MakeUniqueAlignedArray<std::uint64_t>(tests);
RngLoop(seed, result_array.get(), tests);
std::vector<internal::Xoshiro> reference;
reference.emplace_back(seed);
const ScalableTag<std::uint64_t> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 1UL; i < lanes; ++i) {
auto rng = reference.back();
rng.Jump();
reference.emplace_back(rng);
}
for (std::size_t i = 0UL; i < tests; i += lanes) {
for (std::size_t lane = 0UL; lane < lanes; ++lane) {
const std::uint64_t result = reference[lane]();
if (result_array[i + lane] != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST UINT64 GENERATOR ERROR: result_array[" << i + lane
<< "] -> " << result_array[i + lane] << " != " << result
<< std::endl;
HWY_ASSERT(0);
}
}
}
}
void TestUniformDist() {
#if HWY_HAVE_FLOAT64
const std::uint64_t seed = GetSeed();
const auto result_array = hwy::MakeUniqueAlignedArray<double>(tests);
UniformLoop(seed, result_array.get(), tests);
internal::Xoshiro reference{seed};
const ScalableTag<double> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 0UL; i < tests; i += lanes) {
const double result = reference.Uniform();
if (result_array[i] != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST UNIFORM GENERATOR ERROR: result_array[" << i << "] -> "
<< result_array[i] << " != " << result << std::endl;
HWY_ASSERT(0);
}
}
#endif // HWY_HAVE_FLOAT64
}
void TestNextNRandomUint64() {
const std::uint64_t seed = GetSeed();
VectorXoshiro generator{seed};
const auto result_array = generator.operator()(tests);
std::vector<internal::Xoshiro> reference;
reference.emplace_back(seed);
const ScalableTag<std::uint64_t> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 1UL; i < lanes; ++i) {
auto rng = reference.back();
rng.Jump();
reference.emplace_back(rng);
}
for (std::size_t i = 0UL; i < tests; i += lanes) {
for (std::size_t lane = 0UL; lane < lanes; ++lane) {
const std::uint64_t result = reference[lane]();
if (result_array[i + lane] != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST UINT64 GENERATOR ERROR: result_array[" << i + lane
<< "] -> " << result_array[i + lane] << " != " << result
<< std::endl;
HWY_ASSERT(0);
}
}
}
}
void TestNextFixedNRandomUint64() {
const std::uint64_t seed = GetSeed();
VectorXoshiro generator{seed};
const auto result_array = generator.operator()<tests>();
std::vector<internal::Xoshiro> reference;
reference.emplace_back(seed);
const ScalableTag<std::uint64_t> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 1UL; i < lanes; ++i) {
auto rng = reference.back();
rng.Jump();
reference.emplace_back(rng);
}
for (std::size_t i = 0UL; i < tests; i += lanes) {
for (std::size_t lane = 0UL; lane < lanes; ++lane) {
const std::uint64_t result = reference[lane]();
if (result_array[i + lane] != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST UINT64 GENERATOR ERROR: result_array[" << i + lane
<< "] -> " << result_array[i + lane] << " != " << result
<< std::endl;
HWY_ASSERT(0);
}
}
}
}
void TestNextNUniformDist() {
#if HWY_HAVE_FLOAT64
const std::uint64_t seed = GetSeed();
VectorXoshiro generator{seed};
const auto result_array = generator.Uniform(tests);
internal::Xoshiro reference{seed};
const ScalableTag<double> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 0UL; i < tests; i += lanes) {
const double result = reference.Uniform();
if (result_array[i] != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST UNIFORM GENERATOR ERROR: result_array[" << i << "] -> "
<< result_array[i] << " != " << result << std::endl;
HWY_ASSERT(0);
}
}
#endif // HWY_HAVE_FLOAT64
}
void TestNextFixedNUniformDist() {
#if HWY_HAVE_FLOAT64
const std::uint64_t seed = GetSeed();
VectorXoshiro generator{seed};
const auto result_array = generator.Uniform<tests>();
internal::Xoshiro reference{seed};
const ScalableTag<double> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 0UL; i < tests; i += lanes) {
const double result = reference.Uniform();
if (result_array[i] != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST UNIFORM GENERATOR ERROR: result_array[" << i << "] -> "
<< result_array[i] << " != " << result << std::endl;
HWY_ASSERT(0);
}
}
#endif // HWY_HAVE_FLOAT64
}
void TestCachedXorshiro() {
const std::uint64_t seed = GetSeed();
CachedXoshiro<> generator{seed};
std::vector<internal::Xoshiro> reference;
reference.emplace_back(seed);
const ScalableTag<std::uint64_t> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 1UL; i < lanes; ++i) {
auto rng = reference.back();
rng.Jump();
reference.emplace_back(rng);
}
for (std::size_t i = 0UL; i < tests; i += lanes) {
for (std::size_t lane = 0UL; lane < lanes; ++lane) {
const std::uint64_t result = reference[lane]();
const std::uint64_t got = generator();
if (got != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST CachedXoshiro GENERATOR ERROR: result_array["
<< i + lane << "] -> " << got << " != " << result
<< std::endl;
HWY_ASSERT(0);
}
}
}
}
void TestUniformCachedXorshiro() {
#if HWY_HAVE_FLOAT64
const std::uint64_t seed = GetSeed();
CachedXoshiro<> generator{seed};
std::uniform_real_distribution<double> distribution{0., 1.};
for (std::size_t i = 0UL; i < tests; ++i) {
const double result = distribution(generator);
if (result < 0. || result >= 1.) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST CachedXoshiro GENERATOR ERROR: result_array[" << i
<< "] -> " << result << " not in interval [0, 1)" << std::endl;
HWY_ASSERT(0);
}
}
#endif // HWY_HAVE_FLOAT64
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE(); // required if not using HWY_ATTR
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(HwyRandomTest);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestSeeding);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestMultiThreadSeeding);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestRandomUint64);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestNextNRandomUint64);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestNextFixedNRandomUint64);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestCachedXorshiro);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestUniformDist);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestNextNUniformDist);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestNextFixedNUniformDist);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestUniformCachedXorshiro);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

264
deps/v8/third_party/highway/src/hwy/contrib/sort/BUILD vendored Normal file
View File

@ -0,0 +1,264 @@
package(
default_applicable_licenses = ["//:license"],
default_visibility = ["//visibility:public"],
)
licenses(["notice"])
# Unused on Bazel builds, where this is not defined/known; Copybara replaces
# usages with an empty list.
COMPAT = [
"//buildenv/target:non_prod", # includes mobile/vendor.
]
cc_library(
name = "intel",
# hdrs = select({
# "//third_party/bazel_platforms/cpu:x86_64": [
# "avx512-16bit-common.h",
# "avx512-16bit-qsort.hpp",
# "avx512-32bit-qsort.hpp",
# "avx512-64bit-common.h",
# "avx512-64bit-qsort.hpp",
# "avx512-common-qsort.h",
# ],
# "//conditions:default": [],
# }),
compatible_with = [],
)
cc_library(
name = "vxsort",
srcs = [
# "vxsort/isa_detection.cpp",
# "vxsort/isa_detection_msvc.cpp",
# "vxsort/isa_detection_sane.cpp",
# "vxsort/machine_traits.avx2.cpp",
# "vxsort/smallsort/avx2_load_mask_tables.cpp",
# "vxsort/smallsort/bitonic_sort.AVX2.double.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX2.float.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX2.int32_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX2.int64_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX2.uint32_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX2.uint64_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX512.double.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX512.float.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX512.int32_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX512.int64_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX512.uint32_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX512.uint64_t.generated.cpp",
# "vxsort/vxsort_stats.cpp",
],
hdrs = [
# "vxsort/alignment.h",
# "vxsort/defs.h",
# "vxsort/isa_detection.h",
# "vxsort/machine_traits.avx2.h",
# "vxsort/machine_traits.avx512.h",
# "vxsort/machine_traits.h",
# "vxsort/packer.h",
# "vxsort/smallsort/bitonic_sort.AVX2.double.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX2.float.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX2.int32_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX2.int64_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX2.uint32_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX2.uint64_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX512.double.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX512.float.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX512.int32_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX512.int64_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX512.uint32_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX512.uint64_t.generated.h",
# "vxsort/smallsort/bitonic_sort.h",
# "vxsort/vxsort.h",
# "vxsort/vxsort_stats.h",
],
compatible_with = [],
textual_hdrs = [
# "vxsort/vxsort_targets_disable.h",
# "vxsort/vxsort_targets_enable_avx2.h",
# "vxsort/vxsort_targets_enable_avx512.h",
],
)
VQSORT_SRCS = [
"vqsort.cc",
# Split into separate files to reduce MSVC build time.
"vqsort_128a.cc",
"vqsort_128d.cc",
"vqsort_f16a.cc",
"vqsort_f16d.cc",
"vqsort_f32a.cc",
"vqsort_f32d.cc",
"vqsort_f64a.cc",
"vqsort_f64d.cc",
"vqsort_i16a.cc",
"vqsort_i16d.cc",
"vqsort_i32a.cc",
"vqsort_i32d.cc",
"vqsort_i64a.cc",
"vqsort_i64d.cc",
"vqsort_kv64a.cc",
"vqsort_kv64d.cc",
"vqsort_kv128a.cc",
"vqsort_kv128d.cc",
"vqsort_u16a.cc",
"vqsort_u16d.cc",
"vqsort_u32a.cc",
"vqsort_u32d.cc",
"vqsort_u64a.cc",
"vqsort_u64d.cc",
]
VQSORT_TEXTUAL_HDRS = [
"shared-inl.h",
"sorting_networks-inl.h",
"traits-inl.h",
"traits128-inl.h",
"vqsort-inl.h",
# Placeholder for internal instrumentation. Do not remove.
]
cc_library(
name = "vqsort",
srcs = VQSORT_SRCS,
hdrs = [
"order.h", # part of public interface, included by vqsort.h
"vqsort.h", # public interface
],
compatible_with = [],
local_defines = ["hwy_contrib_EXPORTS"],
textual_hdrs = VQSORT_TEXTUAL_HDRS,
deps = [
":intel", # required if HAVE_INTEL
":vxsort", # required if HAVE_VXSORT
"//:algo",
"//:hwy",
],
)
# -----------------------------------------------------------------------------
# Internal-only targets
# Same as vqsort, but add HWY_COMPILE_ALL_ATTAINABLE to ensure we cover all
# targets. Do not enable this in the main vqsort because it increases
# compile times.
cc_library(
name = "vqsort_for_test",
srcs = VQSORT_SRCS,
hdrs = [
"order.h", # part of public interface, included by vqsort.h
"vqsort.h", # public interface
],
compatible_with = [],
local_defines = [
"hwy_contrib_EXPORTS",
# Build for all targets because sort_test will dynamic-dispatch to all.
"HWY_COMPILE_ALL_ATTAINABLE",
],
textual_hdrs = VQSORT_TEXTUAL_HDRS,
deps = [
"//:algo",
"//:hwy",
],
)
cc_library(
name = "helpers",
testonly = 1,
textual_hdrs = [
"algo-inl.h",
"result-inl.h",
],
deps = [
":vqsort",
"//:nanobenchmark",
# Required for HAVE_PDQSORT, but that is unused and this is
# unavailable to Bazel builds, hence commented out.
# "//third_party/boost/allowed",
# Avoid ips4o and thus TBB to work around hwloc build failure.
],
)
cc_binary(
name = "print_network",
testonly = 1,
srcs = ["print_network.cc"],
deps = [
":helpers",
":vqsort",
"//:hwy",
],
)
TEST_MAIN = select({
"//:compiler_msvc": [],
"//conditions:default": ["@com_google_googletest//:gtest_main"],
})
cc_test(
name = "sort_unit_test",
size = "small",
srcs = ["sort_unit_test.cc"],
# Do not enable fully_static_link (pthread crash on bazel)
local_defines = ["HWY_IS_TEST"],
# for test_suite.
tags = ["hwy_ops_test"],
deps = [
":helpers",
":vqsort_for_test",
"//:hwy",
"//:hwy_test_util",
] + TEST_MAIN,
)
cc_test(
name = "sort_test",
size = "medium",
timeout = "long",
srcs = ["sort_test.cc"],
# Do not enable fully_static_link (pthread crash on bazel)
local_defines = ["HWY_IS_TEST"],
# for test_suite.
tags = ["hwy_ops_test"],
deps = [
":helpers",
":vqsort_for_test",
"//:hwy",
"//:hwy_test_util",
"//:thread_pool",
"//:topology",
] + TEST_MAIN,
)
cc_test(
name = "bench_sort",
size = "medium",
srcs = ["bench_sort.cc"],
# Do not enable fully_static_link (pthread crash on bazel)
local_defines = ["HWY_IS_TEST"],
# for test_suite.
tags = ["hwy_ops_test"],
deps = [
":helpers",
":vqsort",
"//:hwy",
"//:hwy_test_util",
"//:nanobenchmark",
] + TEST_MAIN,
)
cc_binary(
name = "bench_parallel",
testonly = 1,
srcs = ["bench_parallel.cc"],
# Do not enable fully_static_link (pthread crash on bazel)
local_defines = ["HWY_IS_TEST"],
deps = [
":helpers",
":vqsort",
"//:hwy",
"//:hwy_test_util",
"//:nanobenchmark",
] + TEST_MAIN,
)

361
deps/v8/third_party/highway/src/hwy/contrib/sort/README.md vendored Normal file
View File

@ -0,0 +1,361 @@
# Vectorized and performance-portable Quicksort
## Introduction
As of 2022-06-07 this sorts large arrays of built-in types about ten times as
fast as LLVM's `std::sort`. Note that other algorithms such as pdqsort can be
about twice as fast as LLVM's std::sort as of 2023-06.
See also our
[blog post](https://opensource.googleblog.com/2022/06/Vectorized%20and%20performance%20portable%20Quicksort.html)
and [paper](https://arxiv.org/abs/2205.05982).
## Instructions
Here are instructions for reproducing our results with cross-platform CMake,
Linux, or AWS (SVE, NEON).
### CMake, any platform
Please first ensure that Clang (tested with 13.0.1 and 15.0.6) is installed, and
if it is not the default compiler, point the CC and CXX environment variables to
it, e.g.
```
export CC=clang-15
export CXX=clang++-15
```
Then run the usual CMake workflow, also documented in the Highway README, e.g.:
```
mkdir -p build && cd build && cmake .. && make -j
taskset -c 2 tests/bench_sort
```
The optional `taskset -c 2` part reduces the variability of measurements by
preventing the OS from migrating the benchmark between cores.
### Linux
Please first ensure golang, and Clang (tested with 13.0.1) are installed via
your system's package manager.
```
go install github.com/bazelbuild/bazelisk@latest
git clone https://github.com/google/highway
cd highway
CC=clang CXX=clang++ ~/go/bin/bazelisk build -c opt hwy/contrib/sort:all
bazel-bin/hwy/contrib/sort/sort_test
bazel-bin/hwy/contrib/sort/bench_sort
```
### AWS Graviton3
Instance config: amazon linux 5.10 arm64, c7g.8xlarge (largest allowed config is
32 vCPU). Initial launch will fail. Wait a few minutes for an email saying the
config is verified, then re-launch. See IPv4 hostname in list of instances.
`ssh -i /path/key.pem ec2-user@hostname`
Note that the AWS CMake package is too old for llvm, so we build it first:
```
wget https://cmake.org/files/v3.23/cmake-3.23.2.tar.gz
tar -xvzf cmake-3.23.2.tar.gz && cd cmake-3.23.2/
./bootstrap -- -DCMAKE_USE_OPENSSL=OFF
make -j8 && sudo make install
cd ..
```
AWS clang is at version 11.1, which generates unnecessary `AND` instructions
which slow down the sort by 1.15x. We tested with clang trunk as of June 13
(which reports Git hash 8f6512fea000c3a0d394864bb94e524bee375069). To build:
```
git clone --depth 1 https://github.com/llvm/llvm-project.git
cd llvm-project
mkdir -p build && cd build
/usr/local/bin/cmake ../llvm -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" -DCMAKE_BUILD_TYPE=Release
make -j32 && sudo make install
```
```
sudo yum install go
go install github.com/bazelbuild/bazelisk@latest
git clone https://github.com/google/highway
cd highway
CC=/usr/local/bin/clang CXX=/usr/local/bin/clang++ ~/go/bin/bazelisk build -c opt --copt=-march=armv8.2-a+sve hwy/contrib/sort:all
bazel-bin/hwy/contrib/sort/sort_test
bazel-bin/hwy/contrib/sort/bench_sort
```
The above command line enables SVE, which is currently only available on
Graviton 3. You can also test NEON on the same processor, or other Arm CPUs, by
changing the `-march=` option to `--copt=-march=armv8.2-a+crypto`. Note that
such flags will be unnecessary once Clang supports `#pragma target` for NEON and
SVE intrinsics, as it does for x86.
## Results
`bench_sort` outputs the instruction set (AVX3 refers to AVX-512), the sort
algorithm (std for `std::sort`, vq for our vqsort), the type of keys being
sorted (f32 is float), the distribution of keys (uniform32 for uniform random
with range 0-2^32), the number of keys, then the throughput of sorted keys (i.e.
number of key bytes output per second).
Example excerpt from Xeon 6154 (Skylake-X) CPU clocked at 3 GHz:
```
[ RUN ] BenchSortGroup/BenchSort.BenchAllSort/AVX3
AVX3: std: f32: uniform32: 1.00E+06 54 MB/s ( 1 threads)
AVX3: vq: f32: uniform32: 1.00E+06 1143 MB/s ( 1 threads)
```
## Additional results
Thanks to Lukas Bergdoll, who did a thorough [performance analysis](https://github.com/Voultapher/sort-research-rs/blob/main/writeup/intel_avx512/text.md)
on various sort implementations. This helped us identify a performance bug,
caused by obtaining entropy from the OS on each call. This was fixed in #1334
and we look forward to the updated results.
### Optimizations for small arrays
Our initial focus was on large arrays. Since the VQSort paper was published,
we have improved its performance for small arrays:
- Previously, each call to VQSort obtained entropy from the OS. Unpredictable
seeding does help avoid worst-cases, and the cost is negligible when the
input size is at least 100K elements. However, the overhead is very costly
for arrays of just 100 or 1000, so we now obtain entropy only once per
thread and cache the seeds in TLS. This significantly improves the
performance on subsequent calls. Users can also explicitly initialize the
random generator.
- We also improved the efficiency of our sorting network for inputs shorter
than half its size. Our approach avoids costly transposes by interpreting
inputs as a 2D matrix. Previously, we always used 16 rows, which means only
a single vector lane is active for up to 16 elements. We have added 8x2 and
8x4 networks which use more lanes when available, and also 4x1 and 8x1
networks for very small inputs.
- Previously we also loaded (overlapping) full vectors, with the offsets
determined by the number of columns. Now we use the minimum vector size
sufficient for the number of columns, which enables higher IPC on Skylake
and reduces the cost of unaligned loads.
Unfortunately this decreases code reuse; VQSort now consists of about 1500
instructions (https://gcc.godbolt.org/z/ojYKfjPe6). The size of sorting
networks has nearly doubled to 10.8 KiB, 70% of the total. Although large,
this still fits comfortably within 32 KiB instruction caches, and possibly
even in micro-op caches (DSB, 1500-2300 micro-ops), especially given that
not all instructions are guaranteed to execute.
### Study of AVX-512 downclocking
We study whether AVX-512 downclocking affects performance. Using the GHz
reported by perf, we find an upper bound on the effects of downclocking, and
observe that its effect is negligible when compared to scalar code.
This issue has somehow attracted far more attention than seems warranted. An
attempt by Daniel Lemire to measure the
[worst-case](https://lemire.me/blog/2018/08/15/the-dangers-of-avx-512-throttling-a-3-impact/)
only saw a **3% decrease**, and Intel CPUs since Icelake, as well as AMD Zen4,
are much less impacted by throttling, if at all. By contrast, "Silver" and
"Bronze" Intel Xeons have more severe throttling and would require a large(r)
speedup from AVX-512 to outweigh the downclocking. However, these CPUs are
marketed towards "entry compute, network and storage" and "small business and
storage server solutions", and are thus less suitable for the high-performance
workloads we consider.
Our test workstation runs Linux (6.1.20-2rodete1-amd64) and has the same Xeon
Gold 6154 CPU used in our paper because its Skylake microarchitecture is the
most (potentially) affected. The compiler is a Clang similar to the LLVM trunk.
We added a new 'cold' benchmark that initializes random seeds, fills an array
with a constant except at one random index, calls VQSort, and then prints a
random element to ensure the computations are not elided. To run it, we build
bench_sort with `-DSORT_ONLY_COLD=1` and then invoke
`taskset -c 6 setarch -R x86_64 perf stat -r 15 -d bench_sort`. The taskset and
setarch serve to reduce variability by avoiding thread migration, and disabling
address space randomization. `-r 15` requests 15 runs so that perf can display
the variability of the measurements: < 1% for cycles, instructions, L1 dcache
loads; LLC miss variability is much higher (> 10%) presumably due to the
remaining background activity on this machine.
For our measurements, we use the GHz value reported by `perf`. This does not
include time spent in the kernel, and is thus noisy for short runtimes. Note
that running `perf` under `sudo` is not an option because it results in
"Workload failed: Cannot allocate memory". We see results between 2.6 - 2.9 GHz
when running AVX-512 code. This is relative to 3.0 GHz nominal; we disabled
Turbo Boost via MSR and ran `sudo cpupower frequency-set --governor performance`
to prevent unnecessary frequency reductions. To the best of our knowledge, the
remaining gap is explained by time spent in the kernel (in particular handling
page faults) and downclocking. Thus an *upper-bound* for the latter is
(3 - 2.9)/3 to (3 - 2.6)/3, or **1.03 - 1.13x**. Such a frequency reduction
would already be negligible compared to the 2-4x increase in work per cycle from
512-bit SIMD relative to 256 or 128-bit SIMD, which is typically less or not at
all affected by downclocking.
To further tighten this bound, we compare AVX-512 code vs. non-AVX-512 code, in
the form of `std::sort`. Ensuring the remainder of the binary does not use
AVX-512 is nontrivial. Library functions such as `memset` are known to use
AVX-512, and they would not show up in a disassembly of our binary. Neither
would they raise exceptions if run on a CPU lacking AVX-512 support, because
software typically verifies CPU support before running AVX-512. As a first step,
we take care to avoid calls to such library functions in our test, which is more
feasible with a self-contained small binary. In particular, array
zero-initialization typically compiles to `memset` (verified with clang-16), so
we manually initialize the array to the return value of an `Unpredictable1`
function whose implementation is not visible to the compiler. This indeed
compiles to a scalar loop. To further increase confidence that the binary lacks
AVX-512 instructions before VQSort, we replace the initialization loop with
AVX-512 stores. This indeed raises the measured throughput from a fairly
consistent 9 GB/s to 9-15 GB/s, likely because some of the AVX-512 startup now
occurs outside of our timings. We examine this effect in the next section, but
for now we can conclude that because adding AVX-512 makes a difference, the
binary was otherwise not using it. Now we can revert to scalar initialization
and compare the GHz reported for VQSort vs. `std::sort`. Across three runs, the
ranges are 2.8-2.9 and 2.8-2.8 GHz. Thus we conclude: if there is any
downclocking for a single core running AVX-512 on this Skylake-X CPU, the effect
is **under the noise floor of our measurement**, and certainly far below any
speedup one can reasonably predict from 512-bit SIMD. We expect this result to
generalize to AMD Zen4 and any Gold/Platinum Intel Xeon.
### Study of AVX-512 startup overhead
In the previous section, we saw that downclocking is negligible on our system,
but there is a noticeable benefit to warming up AVX-512 before the sort. To
understand why, we refer to Travis Downs' excellent
[measurements](https://travisdowns.github.io/blog/2020/01/17/avxfreq1.html#summary)
of how Skylake reacts to an AVX-512 instruction: 8-20 us of reduced instruction
throughput, an additional potential halt of 10 us, and then downclocking.
Note that downclocking is negligible on a single core per the previous section.
We choose the array length of 10K unsigned 64-bit keys such that VQSort
completes in 7-10 us. Thus in this benchmark, VQSort (almost) finishes before
AVX-512 is fully warmed up, and the speedup is reduced because the startup costs
are amortized over relatively little data. Across five series of 15 runs, the
average of average throughputs is 9.3 GB/s, implying a runtime of 8.6 us
including startup costs.
Note that the two-valued, almost all-equal input distribution is quite skewed.
The above throughput does not reflect the performance attainable on other
distributions, especially uniform random. However, this choice is deliberate
because Quicksort can terminate early if all values in a partition are equal.
When measuring such a 'best-case' input, we are more likely to observe the cost
of startup overhead in surrounding code. Otherwise, this overhead might be
hidden by the increase in sorting time.
Now let us compare this throughput to the previously mentioned measurement with
AVX-512 warmed up (via slow scatter instructions so that initialization takes
about 100 us, well in excess of the warmup period): 15.2 GB/s, or 5.3 us without
startup cost. It appears the 10 us halt is not happening, possibly because we do
not use SIMD floating-point nor multiplication instructions. Thus we only
experience reduced instruction throughput and/or increased latency. The ratio
between cold and warmed-up time is only 1.6, which is plausible if the Skylake
throttling is actually rounding latencies up to a multiple of four cycles, as
Downs speculates. Indeed a large fraction of the SIMD instructions especially in
the VQSort base case are cross-lane or 64-bit min/max operations with latencies
of 3 cycles on Skylake, so their slowdown might only be 1.3x. The measured 1.6x
could plausibly derive from 7/8 of 1.3x and 1/8 of 4x for single-cycle latency
instructions.
Assuming this understanding of AVX-512 startup cost is valid, how long does it
remain active before the CPU reverts to the previous settings? The CPU cannot
know what future instructions are coming, and to prevent unnecessary
transitions, it has a hysteresis (delay after the last AVX-512 instruction
before shutting down) which Downs measures as 680 us. Thus our benchmark
subsequently sleeps for 100 ms to ensure the next run of the binary sees the
original CPU state. Indeed we find for the five series that the slopes of the
lines of best fit are negative in one case, positive in two, and flat in two,
indicating there is no consistent pattern of benefit for earlier or later runs.
What are the implications for users of VQSort? If the surrounding code executes
an AVX-512 instruction at least every 500 us, then AVX-512 remains active and
**any call to VQSort will benefit from it, no matter how small the input**.
This is a reasonable expectation for modern systems whose designers were aware
of data-oriented programming principles, because many (though not all) domains
and operations can benefit from SIMD. By contrast, consider the case of dropping
VQSort into an existing legacy system that does not yet use SIMD. In the case of
10K input sizes, we still observe a 2.3x speedup vs. `std::sort`. However, the
following code may have to deal with throttling for the remainder of the 20 us
startup period. With VQSort we have 8.6 us runtime plus up to 11.4 us throttled
code (potentially running at quarter speed) plus the remaining 3/4 of 11.4 for a
total of 28.6. With `std::sort` we have 19.5 us runtime plus 20 us of normal
subsequent code, or 39.5 us. Thus the overall speedup for the 20 us region plus
VQSort **shrinks to 1.4x**, and it is possible to imagine an actual slowdown for
sufficiently small inputs, when factoring in the throttling of subsequent code.
This unfortunate 'beggar thy neighbor' effect cannot be solved at the level of
individual building blocks such as a sort, and must instead be addressed at the
system level. For example:
- vectorizing more and more parts of the code to amortize startup cost;
- relying on newer CPUs than Skylake (launched 2015!) which have little or no
AVX-512 startup overhead, such as Intel Icelake (2021) or AMD Zen4 (2022);
- ensuring sorts (or anything else using AVX-512) process at least 100 KiB
of data, such that the expected speedup outweighs any startup cost.
Any of these solutions are sufficient to render AVX-512 startup overhead a
non-issue.
### Comparison with Intel's x86-simd-sort and vxsort
Our May 2022 paper compared performance with `ips4o` and `std::sort`. We now add
results for Intel's [x86-simd-sort](https://github.com/intel/x86-simd-sort),
released as open source around October 2022, and
[vxsort](https://github.com/damageboy/vxsort-cpp/tree/master). We find that
VQSort is generally about 1.4 times as fast as either, and in a few cases equal
or up to 2% slower.
Note that vxsort was open-sourced around May 2020; we were unaware of it at the
time of writing because it had been published in the form of a blog series. We
imported both from Github on 2023-06-06 at about 10:15 UTC. Both are integrated
into our bench_sort, running on the same Linux OS and Xeon 6154 CPU mentioned
above. We use uniform random inputs, because vxsort and x86-simd-sort appear to
have much less robust handling of skewed input distributions. They choose the
pivot as the median of three keys, or of 64 bytes, respectively. By contrast,
VQSort draws a 384 byte sample and analyzes their distribution, which improves
load balance and prevents recursing into all-equal partitions. Lacking this, the
other algorithms are more vulnerable to worst-cases. Choosing uniform random
thus prevents disadvantaging the other algorithms.
We sample performance across a range of input sizes and types:
- To isolate the performance of the sorting networks used by all three
algorithms, we start with powers of two up to 128. VQSort is generally the
fastest for 64-bit keys with the following exceptions: tie with vxsort at
N=2 (537 MB/s), slower than vxsort at N=16 (2114 vs. 2147), tie with
x86-simd-sort at N=32 (2643 MB/s). Note that VQSort is about 1.6 times as
fast as both others for N=128; possibly because its 2D structure enables
larger networks.
- The `kPow10` mode in bench_sort measures power of ten input sizes between
10 and 100K. Note that this covers non-power of two sizes, as well as the
crossover point between sorting networks and Quicksort recursion. The
speedups of VQSort relative to x86-simd-sort range from 1.33 to 1.81
(32-bit keys), and 1.25 to 1.68 (64-bit keys), with geomeans of 1.48 and
1.44. The speedups of VQSort relative to vxsort range from 1.08 to 2.10
(32-bit keys), and 1.00 to 1.47 (64-bit keys), with geomeans of 1.41 and
1.20. Note that vxsort matches VQSort at 10 64-bit elements; in all other
cases, VQSort is strictly faster.
- Finally, we study the effect of key type at a fixed input size of 10K
elements. x86-simd-sort requires AVX512-VBMI2 for int16, which our CPU does
not support. Also, both other algorithms do not support 128-bit keys, thus
we only consider 32/64-bit integer and float types. The results in MB/s are:
|Type|VQSort|x86-simd-sort|vxsort|
|---|---|---|---|
|f32|**1551**| 798| 823|
|f64|**1773**|1147| 745|
|i32|**1509**|1042| 968|
|i64|**1365**|1043|1145|
VQSort is the fastest for each type, in some cases even about twice as fast.
Interestingly, vxsort performs at its best on i64, whereas the others are at
their best for f64. A potential explanation is that this CPU can execute two
f64 min/max per cycle, but only one i64.
In conclusion, VQSort is generally more efficient than vxsort and x86-simd-sort
across a range of input sizes and types. Occasionally, it is up to 2% slower,
but the geomean of its speedup (32-bit keys and power-of-ten sizes) vs. vxsort
is **1.41**, and **1.48** vs. x86-simd-sort.

621
deps/v8/third_party/highway/src/hwy/contrib/sort/algo-inl.h vendored Normal file
View File

@ -0,0 +1,621 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Normal include guard for target-independent parts
#ifndef HIGHWAY_HWY_CONTRIB_SORT_ALGO_INL_H_
#define HIGHWAY_HWY_CONTRIB_SORT_ALGO_INL_H_
#include <stddef.h>
#include <stdint.h>
#include <algorithm> // std::sort, std::min, std::max
#include <functional> // std::less, std::greater
#include <vector>
#include "hwy/contrib/sort/vqsort.h"
#include "hwy/highway.h"
#include "hwy/print.h"
// Third-party algorithms
#define HAVE_AVX2SORT 0
#define HAVE_IPS4O 0
// When enabling, consider changing max_threads (required for Table 1a)
#define HAVE_PARALLEL_IPS4O (HAVE_IPS4O && 1)
#define HAVE_PDQSORT 0
#define HAVE_SORT512 0
#define HAVE_VXSORT 0
#if HWY_ARCH_X86
#define HAVE_INTEL 0
#else
#define HAVE_INTEL 0
#endif
#if HAVE_PARALLEL_IPS4O
#include <thread> // NOLINT
#endif
#if HAVE_AVX2SORT
HWY_PUSH_ATTRIBUTES("avx2,avx")
#include "avx2sort.h" //NOLINT
HWY_POP_ATTRIBUTES
#endif
#if HAVE_IPS4O || HAVE_PARALLEL_IPS4O
#include "third_party/ips4o/include/ips4o.hpp"
#include "third_party/ips4o/include/ips4o/thread_pool.hpp"
#endif
#if HAVE_PDQSORT
#include "third_party/boost/allowed/sort/sort.hpp"
#endif
#if HAVE_SORT512
#include "sort512.h" //NOLINT
#endif
// vxsort is difficult to compile for multiple targets because it also uses
// .cpp files, and we'd also have to #undef its include guards. Instead, compile
// only for AVX2 or AVX3 depending on this macro.
#define VXSORT_AVX3 1
#if HAVE_VXSORT
// inlined from vxsort_targets_enable_avx512 (must close before end of header)
#ifdef __GNUC__
#ifdef __clang__
#if VXSORT_AVX3
#pragma clang attribute push(__attribute__((target("avx512f,avx512dq"))), \
apply_to = any(function))
#else
#pragma clang attribute push(__attribute__((target("avx2"))), \
apply_to = any(function))
#endif // VXSORT_AVX3
#else
#pragma GCC push_options
#if VXSORT_AVX3
#pragma GCC target("avx512f,avx512dq")
#else
#pragma GCC target("avx2")
#endif // VXSORT_AVX3
#endif
#endif
#if VXSORT_AVX3
#include "vxsort/machine_traits.avx512.h"
#else
#include "vxsort/machine_traits.avx2.h"
#endif // VXSORT_AVX3
#include "vxsort/vxsort.h"
#ifdef __GNUC__
#ifdef __clang__
#pragma clang attribute pop
#else
#pragma GCC pop_options
#endif
#endif
#endif // HAVE_VXSORT
namespace hwy {
enum class Dist { kUniform8, kUniform16, kUniform32 };
static inline std::vector<Dist> AllDist() {
// Also include lower-entropy distributions to test MaybePartitionTwoValue.
return {Dist::kUniform8, /*Dist::kUniform16,*/ Dist::kUniform32};
}
static inline const char* DistName(Dist dist) {
switch (dist) {
case Dist::kUniform8:
return "uniform8";
case Dist::kUniform16:
return "uniform16";
case Dist::kUniform32:
return "uniform32";
}
return "unreachable";
}
template <typename T>
class InputStats {
public:
void Notify(T value) {
min_ = std::min(min_, value);
max_ = std::max(max_, value);
// Converting to integer would truncate floats, multiplying to save digits
// risks overflow especially when casting, so instead take the sum of the
// bit representations as the checksum.
uint64_t bits = 0;
static_assert(sizeof(T) <= 8, "Expected a built-in type");
CopyBytes<sizeof(T)>(&value, &bits); // not same size
sum_ += bits;
count_ += 1;
}
bool operator==(const InputStats& other) const {
char type_name[100];
detail::TypeName(hwy::detail::MakeTypeInfo<T>(), 1, type_name);
if (count_ != other.count_) {
HWY_ABORT("Sort %s: count %d vs %d\n", type_name,
static_cast<int>(count_), static_cast<int>(other.count_));
}
if (min_ != other.min_ || max_ != other.max_) {
HWY_ABORT("Sort %s: minmax %f/%f vs %f/%f\n", type_name,
static_cast<double>(min_), static_cast<double>(max_),
static_cast<double>(other.min_),
static_cast<double>(other.max_));
}
// Sum helps detect duplicated/lost values
if (sum_ != other.sum_) {
HWY_ABORT("Sort %s: Sum mismatch %g %g; min %g max %g\n", type_name,
static_cast<double>(sum_), static_cast<double>(other.sum_),
static_cast<double>(min_), static_cast<double>(max_));
}
return true;
}
private:
T min_ = hwy::HighestValue<T>();
T max_ = hwy::LowestValue<T>();
uint64_t sum_ = 0;
size_t count_ = 0;
};
enum class Algo {
#if HAVE_INTEL
kIntel,
#endif
#if HAVE_AVX2SORT
kSEA,
#endif
#if HAVE_IPS4O
kIPS4O,
#endif
#if HAVE_PARALLEL_IPS4O
kParallelIPS4O,
#endif
#if HAVE_PDQSORT
kPDQ,
#endif
#if HAVE_SORT512
kSort512,
#endif
#if HAVE_VXSORT
kVXSort,
#endif
kStdSort,
kStdSelect,
kStdPartialSort,
kVQSort,
kVQPartialSort,
kVQSelect,
kHeapSort,
kHeapPartialSort,
kHeapSelect,
};
static inline bool IsVQ(Algo algo) {
switch (algo) {
case Algo::kVQSort:
case Algo::kVQPartialSort:
case Algo::kVQSelect:
return true;
default:
return false;
}
}
static inline bool IsSelect(Algo algo) {
switch (algo) {
case Algo::kStdSelect:
case Algo::kVQSelect:
case Algo::kHeapSelect:
return true;
default:
return false;
}
}
static inline bool IsPartialSort(Algo algo) {
switch (algo) {
case Algo::kStdPartialSort:
case Algo::kVQPartialSort:
case Algo::kHeapPartialSort:
return true;
default:
return false;
}
}
static inline Algo ReferenceAlgoFor(Algo algo) {
if (IsPartialSort(algo)) return Algo::kStdPartialSort;
#if HAVE_PDQSORT
return Algo::kPDQ;
#else
return Algo::kStdSort;
#endif
}
static inline const char* AlgoName(Algo algo) {
switch (algo) {
#if HAVE_INTEL
case Algo::kIntel:
return "intel";
#endif
#if HAVE_AVX2SORT
case Algo::kSEA:
return "sea";
#endif
#if HAVE_IPS4O
case Algo::kIPS4O:
return "ips4o";
#endif
#if HAVE_PARALLEL_IPS4O
case Algo::kParallelIPS4O:
return "par_ips4o";
#endif
#if HAVE_PDQSORT
case Algo::kPDQ:
return "pdq";
#endif
#if HAVE_SORT512
case Algo::kSort512:
return "sort512";
#endif
#if HAVE_VXSORT
case Algo::kVXSort:
return "vxsort";
#endif
case Algo::kStdSort:
return "std";
case Algo::kStdPartialSort:
return "std_partial";
case Algo::kStdSelect:
return "std_select";
case Algo::kVQSort:
return "vq";
case Algo::kVQPartialSort:
return "vq_partial";
case Algo::kVQSelect:
return "vq_select";
case Algo::kHeapSort:
return "heap";
case Algo::kHeapPartialSort:
return "heap_partial";
case Algo::kHeapSelect:
return "heap_select";
}
return "unreachable";
}
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_SORT_ALGO_INL_H_
// Per-target
// clang-format off
#if defined(HIGHWAY_HWY_CONTRIB_SORT_ALGO_TOGGLE) == defined(HWY_TARGET_TOGGLE) // NOLINT
#ifdef HIGHWAY_HWY_CONTRIB_SORT_ALGO_TOGGLE
#undef HIGHWAY_HWY_CONTRIB_SORT_ALGO_TOGGLE
#else
#define HIGHWAY_HWY_CONTRIB_SORT_ALGO_TOGGLE
#endif
// clang-format on
#include "hwy/aligned_allocator.h"
#include "hwy/contrib/sort/traits-inl.h"
#include "hwy/contrib/sort/traits128-inl.h"
#include "hwy/contrib/sort/vqsort-inl.h" // HeapSort
HWY_BEFORE_NAMESPACE();
// Requires target pragma set by HWY_BEFORE_NAMESPACE
#if HAVE_INTEL && HWY_TARGET <= HWY_AVX3
// #include "avx512-16bit-qsort.hpp" // requires AVX512-VBMI2
#include "avx512-32bit-qsort.hpp"
#include "avx512-64bit-qsort.hpp"
#endif
namespace hwy {
namespace HWY_NAMESPACE {
#if HAVE_INTEL || HAVE_VXSORT // only supports ascending order
template <typename T>
using OtherOrder = detail::OrderAscending<T>;
#else
template <typename T>
using OtherOrder = detail::OrderDescending<T>;
#endif
class Xorshift128Plus {
static HWY_INLINE uint64_t SplitMix64(uint64_t z) {
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
return z ^ (z >> 31);
}
public:
// Generates two vectors of 64-bit seeds via SplitMix64 and stores into
// `seeds`. Generating these afresh in each ChoosePivot is too expensive.
template <class DU64>
static void GenerateSeeds(DU64 du64, TFromD<DU64>* HWY_RESTRICT seeds) {
seeds[0] = SplitMix64(0x9E3779B97F4A7C15ull);
for (size_t i = 1; i < 2 * Lanes(du64); ++i) {
seeds[i] = SplitMix64(seeds[i - 1]);
}
}
// Need to pass in the state because vector cannot be class members.
template <class VU64>
static VU64 RandomBits(VU64& state0, VU64& state1) {
VU64 s1 = state0;
VU64 s0 = state1;
const VU64 bits = Add(s1, s0);
state0 = s0;
s1 = Xor(s1, ShiftLeft<23>(s1));
state1 = Xor(s1, Xor(s0, Xor(ShiftRight<18>(s1), ShiftRight<5>(s0))));
return bits;
}
};
template <class D, class VU64, HWY_IF_NOT_FLOAT_D(D)>
Vec<D> RandomValues(D d, VU64& s0, VU64& s1, const VU64 mask) {
const VU64 bits = Xorshift128Plus::RandomBits(s0, s1);
return BitCast(d, And(bits, mask));
}
// It is important to avoid denormals, which are flushed to zero by SIMD but not
// scalar sorts, and NaN, which may be ordered differently in scalar vs. SIMD.
template <class DF, class VU64, HWY_IF_FLOAT_D(DF)>
Vec<DF> RandomValues(DF df, VU64& s0, VU64& s1, const VU64 mask) {
using TF = TFromD<DF>;
const RebindToUnsigned<decltype(df)> du;
using VU = Vec<decltype(du)>;
const VU64 bits64 = And(Xorshift128Plus::RandomBits(s0, s1), mask);
#if HWY_TARGET == HWY_SCALAR // Cannot repartition u64 to smaller types
using TU = MakeUnsigned<TF>;
const VU bits = Set(du, static_cast<TU>(GetLane(bits64) & LimitsMax<TU>()));
#else
const VU bits = BitCast(du, bits64);
#endif
// Avoid NaN/denormal by only generating values in [1, 2), i.e. random
// mantissas with the exponent taken from the representation of 1.0.
const VU k1 = BitCast(du, Set(df, TF{1.0}));
const VU mantissa_mask = Set(du, MantissaMask<TF>());
const VU representation = OrAnd(k1, bits, mantissa_mask);
return BitCast(df, representation);
}
template <class DU64>
Vec<DU64> MaskForDist(DU64 du64, const Dist dist, size_t sizeof_t) {
switch (sizeof_t) {
case 2:
return Set(du64, (dist == Dist::kUniform8) ? 0x00FF00FF00FF00FFull
: 0xFFFFFFFFFFFFFFFFull);
case 4:
return Set(du64, (dist == Dist::kUniform8) ? 0x000000FF000000FFull
: (dist == Dist::kUniform16) ? 0x0000FFFF0000FFFFull
: 0xFFFFFFFFFFFFFFFFull);
case 8:
return Set(du64, (dist == Dist::kUniform8) ? 0x00000000000000FFull
: (dist == Dist::kUniform16) ? 0x000000000000FFFFull
: 0x00000000FFFFFFFFull);
default:
HWY_ABORT("Logic error");
return Zero(du64);
}
}
template <typename T>
InputStats<T> GenerateInput(const Dist dist, T* v, size_t num_lanes) {
SortTag<uint64_t> du64;
using VU64 = Vec<decltype(du64)>;
const size_t N64 = Lanes(du64);
auto seeds = hwy::AllocateAligned<uint64_t>(2 * N64);
Xorshift128Plus::GenerateSeeds(du64, seeds.get());
VU64 s0 = Load(du64, seeds.get());
VU64 s1 = Load(du64, seeds.get() + N64);
#if HWY_TARGET == HWY_SCALAR
const Sisd<T> d;
#else
const Repartition<T, decltype(du64)> d;
#endif
using V = Vec<decltype(d)>;
const size_t N = Lanes(d);
const VU64 mask = MaskForDist(du64, dist, sizeof(T));
auto buf = hwy::AllocateAligned<T>(N);
size_t i = 0;
for (; i + N <= num_lanes; i += N) {
const V values = RandomValues(d, s0, s1, mask);
StoreU(values, d, v + i);
}
if (i < num_lanes) {
const V values = RandomValues(d, s0, s1, mask);
StoreU(values, d, buf.get());
CopyBytes(buf.get(), v + i, (num_lanes - i) * sizeof(T));
}
InputStats<T> input_stats;
for (size_t i = 0; i < num_lanes; ++i) {
input_stats.Notify(v[i]);
}
return input_stats;
}
struct SharedState {
#if HAVE_PARALLEL_IPS4O
const unsigned max_threads = hwy::LimitsMax<unsigned>(); // 16 for Table 1a
ips4o::StdThreadPool pool{static_cast<int>(
HWY_MIN(max_threads, std::thread::hardware_concurrency() / 2))};
#endif
};
// Adapters from Run's num_keys to vqsort-inl.h num_lanes.
template <typename KeyType, class Order>
void CallHeapSort(KeyType* keys, const size_t num_keys, Order) {
const detail::MakeTraits<KeyType, Order> st;
using LaneType = typename decltype(st)::LaneType;
return detail::HeapSort(st, reinterpret_cast<LaneType*>(keys),
num_keys * st.LanesPerKey());
}
template <typename KeyType, class Order>
void CallHeapPartialSort(KeyType* keys, const size_t num_keys,
const size_t k_keys, Order) {
const detail::MakeTraits<KeyType, Order> st;
using LaneType = typename decltype(st)::LaneType;
detail::HeapPartialSort(st, reinterpret_cast<LaneType*>(keys),
num_keys * st.LanesPerKey(),
k_keys * st.LanesPerKey());
}
template <typename KeyType, class Order>
void CallHeapSelect(KeyType* keys, const size_t num_keys, const size_t k_keys,
Order) {
const detail::MakeTraits<KeyType, Order> st;
using LaneType = typename decltype(st)::LaneType;
detail::HeapSelect(st, reinterpret_cast<LaneType*>(keys),
num_keys * st.LanesPerKey(), k_keys * st.LanesPerKey());
}
template <typename KeyType, class Order>
void Run(Algo algo, KeyType* inout, size_t num_keys, SharedState& shared,
size_t /*thread*/, size_t k_keys, Order) {
const std::less<KeyType> less;
const std::greater<KeyType> greater;
constexpr bool kAscending = Order::IsAscending();
#if !HAVE_PARALLEL_IPS4O
(void)shared;
#endif
switch (algo) {
#if HAVE_INTEL && HWY_TARGET <= HWY_AVX3
case Algo::kIntel:
return avx512_qsort<KeyType>(inout, static_cast<int64_t>(num_keys));
#endif
#if HAVE_AVX2SORT
case Algo::kSEA:
return avx2::quicksort(inout, static_cast<int>(num_keys));
#endif
#if HAVE_IPS4O
case Algo::kIPS4O:
if (kAscending) {
return ips4o::sort(inout, inout + num_keys, less);
} else {
return ips4o::sort(inout, inout + num_keys, greater);
}
#endif
#if HAVE_PARALLEL_IPS4O
case Algo::kParallelIPS4O:
if (kAscending) {
return ips4o::parallel::sort(inout, inout + num_keys, less,
shared.pool);
} else {
return ips4o::parallel::sort(inout, inout + num_keys, greater,
shared.pool);
}
#endif
#if HAVE_SORT512
case Algo::kSort512:
HWY_ABORT("not supported");
// return Sort512::Sort(inout, num_keys);
#endif
#if HAVE_PDQSORT
case Algo::kPDQ:
if (kAscending) {
return boost::sort::pdqsort_branchless(inout, inout + num_keys, less);
} else {
return boost::sort::pdqsort_branchless(inout, inout + num_keys,
greater);
}
#endif
#if HAVE_VXSORT
case Algo::kVXSort: {
#if (VXSORT_AVX3 && HWY_TARGET != HWY_AVX3) || \
(!VXSORT_AVX3 && HWY_TARGET != HWY_AVX2)
fprintf(stderr, "Do not call for target %s\n",
hwy::TargetName(HWY_TARGET));
return;
#else
#if VXSORT_AVX3
vxsort::vxsort<KeyType, vxsort::AVX512> vx;
#else
vxsort::vxsort<KeyType, vxsort::AVX2> vx;
#endif
if (kAscending) {
return vx.sort(inout, inout + num_keys - 1);
} else {
fprintf(stderr, "Skipping VX - does not support descending order\n");
return;
}
#endif // enabled for this target
}
#endif // HAVE_VXSORT
case Algo::kStdSort:
if (kAscending) {
return std::sort(inout, inout + num_keys, less);
} else {
return std::sort(inout, inout + num_keys, greater);
}
case Algo::kStdPartialSort:
if (kAscending) {
return std::partial_sort(inout, inout + k_keys, inout + num_keys, less);
} else {
return std::partial_sort(inout, inout + k_keys, inout + num_keys,
greater);
}
case Algo::kStdSelect:
if (kAscending) {
return std::nth_element(inout, inout + k_keys, inout + num_keys, less);
} else {
return std::nth_element(inout, inout + k_keys, inout + num_keys,
greater);
}
case Algo::kVQSort:
return VQSort(inout, num_keys, Order());
case Algo::kVQPartialSort:
return VQPartialSort(inout, num_keys, k_keys, Order());
case Algo::kVQSelect:
return VQSelect(inout, num_keys, k_keys, Order());
case Algo::kHeapSort:
return CallHeapSort(inout, num_keys, Order());
case Algo::kHeapPartialSort:
return CallHeapPartialSort(inout, num_keys, k_keys, Order());
case Algo::kHeapSelect:
return CallHeapSelect(inout, num_keys, k_keys, Order());
default:
HWY_ABORT("Not implemented");
}
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_SORT_ALGO_TOGGLE

242
deps/v8/third_party/highway/src/hwy/contrib/sort/bench_parallel.cc vendored Normal file
View File

@ -0,0 +1,242 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Concurrent, independent sorts for generating more memory traffic and testing
// scalability when bandwidth-limited. If you want to use multiple threads for
// a single sort, you can use ips4o and integrate vqsort by calling it from
// `baseCaseSort` and increasing `IPS4OML_BASE_CASE_SIZE` to say 8192.
#include <stdint.h>
#include <stdio.h>
#include <condition_variable> //NOLINT
#include <functional>
#include <mutex> //NOLINT
#include <thread> //NOLINT
#include <vector>
#include "hwy/timer.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/bench_parallel.cc" //NOLINT
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/algo-inl.h"
#include "hwy/contrib/sort/result-inl.h"
#include "hwy/aligned_allocator.h"
// Last
#include "hwy/tests/test_util-inl.h"
// clang-format on
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
class ThreadPool {
public:
// Starts the given number of worker threads and blocks until they are ready.
explicit ThreadPool(
const size_t num_threads = std::thread::hardware_concurrency())
: num_threads_(num_threads) {
HWY_ASSERT(num_threads_ > 0);
threads_.reserve(num_threads_);
for (size_t i = 0; i < num_threads_; ++i) {
threads_.emplace_back(ThreadFunc, this, i);
}
WorkersReadyBarrier();
}
ThreadPool(const ThreadPool&) = delete;
ThreadPool& operator&(const ThreadPool&) = delete;
// Waits for all threads to exit.
~ThreadPool() {
StartWorkers(kWorkerExit);
for (std::thread& thread : threads_) {
thread.join();
}
}
size_t NumThreads() const { return threads_.size(); }
template <class Func>
void RunOnThreads(size_t max_threads, const Func& func) {
task_ = &CallClosure<Func>;
data_ = &func;
StartWorkers(max_threads);
WorkersReadyBarrier();
}
private:
// After construction and between calls to Run, workers are "ready", i.e.
// waiting on worker_start_cv_. They are "started" by sending a "command"
// and notifying all worker_start_cv_ waiters. (That is why all workers
// must be ready/waiting - otherwise, the notification will not reach all of
// them and the main thread waits in vain for them to report readiness.)
using WorkerCommand = uint64_t;
static constexpr WorkerCommand kWorkerWait = ~1ULL;
static constexpr WorkerCommand kWorkerExit = ~2ULL;
// Calls a closure (lambda with captures).
template <class Closure>
static void CallClosure(const void* f, size_t thread) {
(*reinterpret_cast<const Closure*>(f))(thread);
}
void WorkersReadyBarrier() {
std::unique_lock<std::mutex> lock(mutex_);
// Typically only a single iteration.
while (workers_ready_ != threads_.size()) {
workers_ready_cv_.wait(lock);
}
workers_ready_ = 0;
// Safely handle spurious worker wakeups.
worker_start_command_ = kWorkerWait;
}
// Precondition: all workers are ready.
void StartWorkers(const WorkerCommand worker_command) {
std::unique_lock<std::mutex> lock(mutex_);
worker_start_command_ = worker_command;
// Workers will need this lock, so release it before they wake up.
lock.unlock();
worker_start_cv_.notify_all();
}
static void ThreadFunc(ThreadPool* self, size_t thread) {
// Until kWorkerExit command received:
for (;;) {
std::unique_lock<std::mutex> lock(self->mutex_);
// Notify main thread that this thread is ready.
if (++self->workers_ready_ == self->num_threads_) {
self->workers_ready_cv_.notify_one();
}
RESUME_WAIT:
// Wait for a command.
self->worker_start_cv_.wait(lock);
const WorkerCommand command = self->worker_start_command_;
switch (command) {
case kWorkerWait: // spurious wakeup:
goto RESUME_WAIT; // lock still held, avoid incrementing ready.
case kWorkerExit:
return; // exits thread
default:
break;
}
lock.unlock();
// Command is the maximum number of threads that should run the task.
HWY_ASSERT(command < self->NumThreads());
if (thread < command) {
self->task_(self->data_, thread);
}
}
}
const size_t num_threads_;
// Unmodified after ctor, but cannot be const because we call thread::join().
std::vector<std::thread> threads_;
std::mutex mutex_; // guards both cv and their variables.
std::condition_variable workers_ready_cv_;
size_t workers_ready_ = 0;
std::condition_variable worker_start_cv_;
WorkerCommand worker_start_command_;
// Written by main thread, read by workers (after mutex lock/unlock).
std::function<void(const void*, size_t)> task_; // points to CallClosure
const void* data_; // points to caller's Func
};
template <class Traits>
void RunWithoutVerify(Traits st, const Dist dist, const size_t num_keys,
const Algo algo, SharedState& shared, size_t thread) {
using LaneType = typename Traits::LaneType;
using KeyType = typename Traits::KeyType;
using Order = typename Traits::Order;
const size_t num_lanes = num_keys * st.LanesPerKey();
auto aligned = hwy::AllocateAligned<LaneType>(num_lanes);
(void)GenerateInput(dist, aligned.get(), num_lanes);
const Timestamp t0;
Run(algo, reinterpret_cast<KeyType*>(aligned.get()), num_keys, shared, thread,
/*k_keys=*/0, Order());
HWY_ASSERT(aligned[0] < aligned[num_lanes - 1]);
}
void BenchParallel() {
// Not interested in benchmark results for other targets on x86
if (HWY_ARCH_X86 &&
(HWY_TARGET != HWY_AVX2 && HWY_TARGET != HWY_AVX3 &&
HWY_TARGET != HWY_AVX3_ZEN4 && HWY_TARGET != HWY_AVX3_SPR)) {
return;
}
ThreadPool pool;
const size_t NT = pool.NumThreads();
detail::SharedTraits<detail::TraitsLane<detail::OrderAscending<int64_t>>> st;
using KeyType = typename decltype(st)::KeyType;
const size_t num_keys = size_t{100} * 1000 * 1000;
#if HAVE_IPS4O
const Algo algo = Algo::kIPS4O;
#else
const Algo algo = Algo::kVQSort;
#endif
const Dist dist = Dist::kUniform32;
SharedState shared;
std::vector<SortResult> results;
for (size_t nt = 1; nt < NT; nt += HWY_MAX(1, NT / 16)) {
Timestamp t0;
// Default capture because MSVC wants algo/dist but clang does not.
pool.RunOnThreads(nt, [=, &shared](size_t thread) {
RunWithoutVerify(st, dist, num_keys, algo, shared, thread);
});
const double sec = SecondsSince(t0);
results.emplace_back(algo, dist, num_keys, nt, sec, sizeof(KeyType),
st.KeyString());
results.back().Print();
}
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(BenchParallel);
HWY_EXPORT_AND_TEST_P(BenchParallel, BenchParallel);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
#endif // HWY_ONCE

480
deps/v8/third_party/highway/src/hwy/contrib/sort/bench_sort.cc vendored Normal file
View File

@ -0,0 +1,480 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <vector>
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/bench_sort.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/algo-inl.h"
#include "hwy/contrib/sort/vqsort.h"
#include "hwy/contrib/sort/result-inl.h"
#include "hwy/contrib/sort/sorting_networks-inl.h" // SharedTraits
#include "hwy/contrib/sort/traits-inl.h"
#include "hwy/contrib/sort/traits128-inl.h"
#include "hwy/tests/test_util-inl.h"
#include "hwy/timer-inl.h"
#include "hwy/nanobenchmark.h"
#include "hwy/timer.h"
#include "hwy/per_target.h"
// clang-format on
#if HWY_OS_LINUX
#include <unistd.h> // usleep
#endif
// Mode for larger sorts because M1 is able to access more than the per-core
// share of L2, so 1M elements might still be in cache.
#define SORT_100M 0
#ifndef SORT_ONLY_COLD
#define SORT_ONLY_COLD 0
#endif
#ifndef SORT_BENCH_BASE_AND_PARTITION
#define SORT_BENCH_BASE_AND_PARTITION (!SORT_ONLY_COLD && 0)
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
// Defined within HWY_ONCE, used by BenchAllSort.
extern int64_t first_sort_target;
extern int64_t first_cold_target; // for BenchAllColdSort
namespace HWY_NAMESPACE {
namespace {
using detail::OrderAscending;
using detail::OrderDescending;
using detail::SharedTraits;
using detail::TraitsLane;
#if HWY_TARGET != HWY_SCALAR
using detail::OrderAscending128;
using detail::OrderAscendingKV128;
using detail::Traits128;
#endif // HWY_TARGET != HWY_SCALAR
HWY_NOINLINE void BenchAllColdSort() {
// Only run the best(first) enabled target
if (first_cold_target == 0) first_cold_target = HWY_TARGET;
if (HWY_TARGET != first_cold_target) {
return;
}
char cpu100[100];
if (!platform::HaveTimerStop(cpu100)) {
fprintf(stderr, "CPU '%s' does not support RDTSCP, skipping benchmark.\n",
cpu100);
return;
}
// Initialize random seeds
#if VQSORT_ENABLED
HWY_ASSERT(GetGeneratorState() != nullptr); // vqsort
#endif
RandomState rng(static_cast<uint64_t>(Unpredictable1() * 129)); // this test
using T = uint64_t;
constexpr size_t kSize = 10 * 1000;
HWY_ALIGN T items[kSize];
// Initialize array
#if 0 // optional: deliberate AVX-512 to verify VQSort performance improves
const ScalableTag<T> d;
const RebindToSigned<decltype(d)> di;
const size_t N = Lanes(d);
size_t i = 0;
for (; i + N <= kSize; i += N) {
// Super-slow scatter so that we spend enough time to warm up SKX.
const Vec<decltype(d)> val = Set(d, static_cast<T>(Unpredictable1()));
const Vec<decltype(di)> idx =
Iota(di, static_cast<T>(Unpredictable1() - 1));
ScatterIndex(val, d, items + i, idx);
}
for (; i < kSize; ++i) {
items[i] = static_cast<T>(Unpredictable1());
}
#else // scalar-only, verified with clang-16
for (size_t i = 0; i < kSize; ++i) {
items[i] = static_cast<T>(Unpredictable1());
}
#endif
items[Random32(&rng) % kSize] = static_cast<T>(Unpredictable1() + 1);
const timer::Ticks t0 = timer::Start();
const SortAscending order;
#if VQSORT_ENABLED && 1 // change to && 0 to switch to std::sort.
VQSort(items, kSize, order);
#else
SharedState shared;
Run(Algo::kStdSort, items, kSize, shared, /*thread=*/0, /*k_keys=*/0, order);
#endif
const timer::Ticks t1 = timer::Stop();
const double ticks = static_cast<double>(t1 - t0);
const double elapsed = ticks / platform::InvariantTicksPerSecond();
const double GBps = kSize * sizeof(T) * 1E-9 / elapsed;
fprintf(stderr, "N=%zu GB/s=%.2f ns=%.1f random output: %g\n", kSize, GBps,
elapsed * 1E9, static_cast<double>(items[Random32(&rng) % kSize]));
#if SORT_ONLY_COLD
#if HWY_OS_LINUX
// Long enough for the CPU to switch off AVX-512 mode before the next run.
usleep(100 * 1000); // NOLINT
#endif
#endif
}
#if (VQSORT_ENABLED && SORT_BENCH_BASE_AND_PARTITION) || HWY_IDE
template <class Traits>
HWY_NOINLINE void BenchPartition() {
using LaneType = typename Traits::LaneType;
using KeyType = typename Traits::KeyType;
const SortTag<LaneType> d;
detail::SharedTraits<Traits> st;
const Dist dist = Dist::kUniform8;
double sum = 0.0;
constexpr size_t kLPK = st.LanesPerKey();
HWY_ALIGN LaneType
buf[SortConstants::BufBytes<LaneType, kLPK>(HWY_MAX_BYTES) /
sizeof(LaneType)];
uint64_t* HWY_RESTRICT state = GetGeneratorState();
const size_t max_log2 = AdjustedLog2Reps(20);
for (size_t log2 = max_log2; log2 < max_log2 + 1; ++log2) {
const size_t num_lanes = 1ull << log2;
const size_t num_keys = num_lanes / kLPK;
auto aligned = hwy::AllocateAligned<LaneType>(num_lanes);
std::vector<double> seconds;
const size_t num_reps = (1ull << (14 - log2 / 2)) * 30;
for (size_t rep = 0; rep < num_reps; ++rep) {
(void)GenerateInput(dist, aligned.get(), num_lanes);
// The pivot value can influence performance. Do exactly what vqsort will
// do so that the performance (influenced by prefetching and branch
// prediction) is likely to predict the actual performance inside vqsort.
detail::DrawSamples(d, st, aligned.get(), num_lanes, buf, state);
detail::SortSamples(d, st, buf);
auto pivot = detail::ChoosePivotByRank(d, st, buf);
const Timestamp t0;
detail::Partition(d, st, aligned.get(), num_lanes - 1, pivot, buf);
seconds.push_back(SecondsSince(t0));
// 'Use' the result to prevent optimizing out the partition.
sum += static_cast<double>(aligned.get()[num_lanes / 2]);
}
SortResult(Algo::kVQSort, dist, num_keys, 1, SummarizeMeasurements(seconds),
sizeof(KeyType), st.KeyString())
.Print();
}
HWY_ASSERT(sum != 999999); // Prevent optimizing out
}
HWY_NOINLINE void BenchAllPartition() {
// Not interested in benchmark results for these targets
if (HWY_TARGET == HWY_SSSE3) {
return;
}
BenchPartition<TraitsLane<OrderDescending<float>>>();
BenchPartition<TraitsLane<OrderDescending<int32_t>>>();
BenchPartition<TraitsLane<OrderDescending<int64_t>>>();
BenchPartition<Traits128<OrderAscending128>>();
// BenchPartition<Traits128<OrderDescending128>>();
BenchPartition<Traits128<OrderAscendingKV128>>();
}
template <class Traits>
HWY_NOINLINE void BenchBase(std::vector<SortResult>& results) {
// Not interested in benchmark results for these targets
if (HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4) {
return;
}
using LaneType = typename Traits::LaneType;
using KeyType = typename Traits::KeyType;
const SortTag<LaneType> d;
detail::SharedTraits<Traits> st;
const Dist dist = Dist::kUniform32;
const Algo algo = Algo::kVQSort;
const size_t N = Lanes(d);
constexpr size_t kLPK = st.LanesPerKey();
const size_t num_lanes = SortConstants::BaseCaseNumLanes<kLPK>(N);
const size_t num_keys = num_lanes / kLPK;
auto keys = hwy::AllocateAligned<LaneType>(num_lanes);
auto buf = hwy::AllocateAligned<LaneType>(num_lanes + N);
std::vector<double> seconds;
double sum = 0; // prevents elision
constexpr size_t kMul = AdjustedReps(600); // ensures long enough to measure
for (size_t rep = 0; rep < 30; ++rep) {
InputStats<LaneType> input_stats =
GenerateInput(dist, keys.get(), num_lanes);
const Timestamp t0;
for (size_t i = 0; i < kMul; ++i) {
detail::BaseCase(d, st, keys.get(), num_lanes, buf.get());
sum += static_cast<double>(keys[0]);
}
seconds.push_back(SecondsSince(t0));
// printf("%f\n", seconds.back());
SortOrderVerifier<Traits>()(algo, input_stats, keys.get(), num_keys,
num_keys);
}
HWY_ASSERT(sum < 1E99);
results.emplace_back(algo, dist, num_keys * kMul, 1,
SummarizeMeasurements(seconds), sizeof(KeyType),
st.KeyString());
}
HWY_NOINLINE void BenchAllBase() {
// Not interested in benchmark results for these targets
if (HWY_TARGET == HWY_SSSE3) {
return;
}
std::vector<SortResult> results;
BenchBase<TraitsLane<OrderAscending<float>>>(results);
BenchBase<TraitsLane<OrderDescending<int64_t>>>(results);
BenchBase<Traits128<OrderAscending128>>(results);
for (const SortResult& r : results) {
r.Print();
}
}
#endif // VQSORT_ENABLED && SORT_BENCH_BASE_AND_PARTITION
std::vector<Algo> AlgoForBench() {
return {
#if HAVE_AVX2SORT
Algo::kSEA,
#endif
#if HAVE_PARALLEL_IPS4O
Algo::kParallelIPS4O,
#elif HAVE_IPS4O
Algo::kIPS4O,
#endif
#if HAVE_PDQSORT
Algo::kPDQ,
#endif
#if HAVE_SORT512
Algo::kSort512,
#endif
// Only include if we're compiling for the target it supports.
#if HAVE_VXSORT && ((VXSORT_AVX3 && HWY_TARGET == HWY_AVX3) || \
(!VXSORT_AVX3 && HWY_TARGET == HWY_AVX2))
Algo::kVXSort,
#endif
// Only include if we're compiling for the target it supports.
#if HAVE_INTEL && HWY_TARGET <= HWY_AVX3
Algo::kIntel,
#endif
#if !HAVE_PARALLEL_IPS4O
#if !SORT_100M
// 10-20x slower, but that's OK for the default size when we are not
// testing the parallel nor 100M modes.
// Algo::kStdSort,
#endif
#if VQSORT_ENABLED
Algo::kVQSort,
#endif
#endif // !HAVE_PARALLEL_IPS4O
};
}
template <class Traits>
HWY_NOINLINE void BenchSort(size_t num_keys) {
if (first_sort_target == 0) first_sort_target = HWY_TARGET;
SharedState shared;
detail::SharedTraits<Traits> st;
using Order = typename Traits::Order;
using LaneType = typename Traits::LaneType;
using KeyType = typename Traits::KeyType;
const size_t num_lanes = num_keys * st.LanesPerKey();
auto aligned = hwy::AllocateAligned<LaneType>(num_lanes);
const size_t reps = num_keys > 1000 * 1000 ? 10 : 30;
for (Algo algo : AlgoForBench()) {
// Other algorithms don't depend on the vector instructions, so only run
// them for the first target.
#if !HAVE_VXSORT
if (algo != Algo::kVQSort && HWY_TARGET != first_sort_target) {
continue;
}
#endif
for (Dist dist : AllDist()) {
std::vector<double> seconds;
for (size_t rep = 0; rep < reps; ++rep) {
InputStats<LaneType> input_stats =
GenerateInput(dist, aligned.get(), num_lanes);
const Timestamp t0;
Run(algo, HWY_RCAST_ALIGNED(KeyType*, aligned.get()), num_keys, shared,
/*thread=*/0, /*k_keys=*/0, Order());
seconds.push_back(SecondsSince(t0));
// printf("%f\n", seconds.back());
SortOrderVerifier<Traits>()(algo, input_stats, aligned.get(), num_keys,
num_keys);
}
SortResult(algo, dist, num_keys, 1, SummarizeMeasurements(seconds),
sizeof(KeyType), st.KeyString())
.Print();
} // dist
} // algo
}
enum class BenchmarkModes {
kDefault,
k1M,
k10K,
kAllSmall,
kSmallPow2,
kSmallPow2Between, // includes padding
kPow4,
kPow10
};
std::vector<size_t> SizesToBenchmark(BenchmarkModes mode) {
std::vector<size_t> sizes;
switch (mode) {
default:
case BenchmarkModes::kDefault:
#if HAVE_PARALLEL_IPS4O || SORT_100M
sizes.push_back(100 * 1000 * size_t{1000});
#else
sizes.push_back(100);
sizes.push_back(100 * 1000);
#endif
break;
case BenchmarkModes::k1M:
sizes.push_back(1000 * 1000);
break;
case BenchmarkModes::k10K:
sizes.push_back(10 * 1000);
break;
case BenchmarkModes::kAllSmall:
sizes.reserve(128);
for (size_t i = 1; i <= 128; ++i) {
sizes.push_back(i);
}
break;
case BenchmarkModes::kSmallPow2:
for (size_t size = 2; size <= 128; size *= 2) {
sizes.push_back(size);
}
break;
case BenchmarkModes::kSmallPow2Between:
for (size_t size = 2; size <= 128; size *= 2) {
sizes.push_back(3 * size / 2);
}
break;
case BenchmarkModes::kPow4:
for (size_t size = 4; size <= 256 * 1024; size *= 4) {
sizes.push_back(size);
}
break;
case BenchmarkModes::kPow10:
for (size_t size = 10; size <= 100 * 1000; size *= 10) {
sizes.push_back(size);
}
break;
}
return sizes;
}
HWY_NOINLINE void BenchAllSort() {
// Not interested in benchmark results for these targets. Note that SSE4 is
// numerically less than SSE2, hence it is the lower bound.
if (HWY_SSE4 <= HWY_TARGET && HWY_TARGET <= HWY_SSE2) {
return;
}
#if HAVE_INTEL
if (HWY_TARGET > HWY_AVX3) return;
#endif
for (size_t num_keys : SizesToBenchmark(BenchmarkModes::kSmallPow2)) {
#if !HAVE_INTEL
#if HWY_HAVE_FLOAT16
if (hwy::HaveFloat16()) {
BenchSort<TraitsLane<OtherOrder<float16_t>>>(num_keys);
}
#endif
BenchSort<TraitsLane<OrderAscending<float>>>(num_keys);
#if HWY_HAVE_FLOAT64
if (hwy::HaveFloat64()) {
// BenchSort<TraitsLane<OtherOrder<double>>>(num_keys);
}
#endif
#endif // !HAVE_INTEL
// BenchSort<TraitsLane<OrderAscending<int16_t>>>(num_keys);
BenchSort<TraitsLane<OtherOrder<int32_t>>>(num_keys);
BenchSort<TraitsLane<OrderAscending<int64_t>>>(num_keys);
// BenchSort<TraitsLane<OtherOrder<uint16_t>>>(num_keys);
// BenchSort<TraitsLane<OtherOrder<uint32_t>>>(num_keys);
// BenchSort<TraitsLane<OrderAscending<uint64_t>>>(num_keys);
#if !HAVE_VXSORT && !HAVE_INTEL && HWY_TARGET != HWY_SCALAR
BenchSort<Traits128<OrderAscending128>>(num_keys);
BenchSort<Traits128<OrderAscendingKV128>>(num_keys);
#endif
}
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
int64_t first_sort_target = 0; // none run yet
int64_t first_cold_target = 0; // none run yet
HWY_BEFORE_TEST(BenchSort);
HWY_EXPORT_AND_TEST_P(BenchSort, BenchAllColdSort);
#if SORT_BENCH_BASE_AND_PARTITION
HWY_EXPORT_AND_TEST_P(BenchSort, BenchAllPartition);
HWY_EXPORT_AND_TEST_P(BenchSort, BenchAllBase);
#endif
#if !SORT_ONLY_COLD // skip (warms up vector unit for next run)
HWY_EXPORT_AND_TEST_P(BenchSort, BenchAllSort);
#endif
HWY_AFTER_TEST();
} // namespace hwy
#endif // HWY_ONCE

34
deps/v8/third_party/highway/src/hwy/contrib/sort/order.h vendored Normal file
View File

@ -0,0 +1,34 @@
// Copyright 2023 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Tag arguments that determine the sort order. Used by both vqsort.h and the
// VQSortStatic in vqsort-inl.h. Moved to a separate header so that the latter
// can be used without pulling in the dllimport statements in vqsort.h.
#ifndef HIGHWAY_HWY_CONTRIB_SORT_ORDER_H_
#define HIGHWAY_HWY_CONTRIB_SORT_ORDER_H_
namespace hwy {
struct SortAscending {
static constexpr bool IsAscending() { return true; }
};
struct SortDescending {
static constexpr bool IsAscending() { return false; }
};
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_SORT_ORDER_H_

90
deps/v8/third_party/highway/src/hwy/contrib/sort/print_network.cc vendored Normal file
View File

@ -0,0 +1,90 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdio.h>
#include <vector>
#include "hwy/base.h"
// Based on A.7 in "Entwurf und Implementierung vektorisierter
// Sortieralgorithmen" and code by Mark Blacher.
void PrintMergeNetwork(int rows, int cols) {
printf("\n%d x %d:\n", rows, cols);
// Powers of two
HWY_ASSERT(rows != 0 && (rows & (rows - 1)) == 0);
HWY_ASSERT(cols != 0 && (cols & (cols - 1)) == 0);
HWY_ASSERT(rows >= 4);
HWY_ASSERT(cols >= 2); // otherwise no cross-column merging required
HWY_ASSERT(cols <= 16); // SortTraits lacks Reverse32
// Log(rows) times: sort half of the vectors with reversed groups of the
// other half. Group size halves until we are sorting adjacent vectors.
int group_size = rows;
int num_groups = 1;
for (; group_size >= 2; group_size /= 2, num_groups *= 2) {
// All vectors except those being reversed. Allows us to group the
// ReverseKeys and Sort2 operations, which is easier to read and may help
// in-order machines with high-latency ReverseKeys.
std::vector<int> all_vi;
for (int group = 0; group < num_groups; ++group) {
for (int i = 0; i < group_size / 2; ++i) {
all_vi.push_back(group * group_size + i);
}
}
for (int vi : all_vi) {
const int vr = vi ^ (group_size - 1);
printf("v%x = st.ReverseKeys%d(d, v%x);\n", vr, cols, vr);
}
for (int vi : all_vi) {
const int vr = vi ^ (group_size - 1);
printf("st.Sort2(d, v%x, v%x);\n", vi, vr);
}
printf("\n");
}
// Now merge across columns in all vectors.
if (cols > 2) {
for (int i = 0; i < rows; ++i) {
printf("v%x = st.SortPairsReverse%d(d, v%x);\n", i, cols, i);
}
printf("\n");
}
if (cols >= 16) {
for (int i = 0; i < rows; ++i) {
printf("v%x = st.SortPairsDistance4(d, v%x);\n", i, i);
}
printf("\n");
}
if (cols >= 8) {
for (int i = 0; i < rows; ++i) {
printf("v%x = st.SortPairsDistance2(d, v%x);\n", i, i);
}
printf("\n");
}
for (int i = 0; i < rows; ++i) {
printf("v%x = st.SortPairsDistance1(d, v%x);\n", i, i);
}
printf("\n");
}
int main(int argc, char** argv) {
PrintMergeNetwork(8, 2);
PrintMergeNetwork(8, 4);
PrintMergeNetwork(16, 4);
PrintMergeNetwork(16, 8);
PrintMergeNetwork(16, 16);
return 0;
}

291
deps/v8/third_party/highway/src/hwy/contrib/sort/result-inl.h vendored Normal file
View File

@ -0,0 +1,291 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/algo-inl.h"
// Normal include guard for non-SIMD parts
#ifndef HIGHWAY_HWY_CONTRIB_SORT_RESULT_INL_H_
#define HIGHWAY_HWY_CONTRIB_SORT_RESULT_INL_H_
#include <stdint.h>
#include <stdio.h>
#include <time.h>
#include <algorithm> // std::sort
#include <string>
#include <vector>
#include "hwy/aligned_allocator.h"
#include "hwy/base.h"
#include "hwy/contrib/sort/order.h"
#include "hwy/per_target.h" // DispatchedTarget
#include "hwy/targets.h" // TargetName
namespace hwy {
// Returns trimmed mean (we don't want to run an out-of-L3-cache sort often
// enough for the mode to be reliable).
static inline double SummarizeMeasurements(std::vector<double>& seconds) {
std::sort(seconds.begin(), seconds.end());
double sum = 0;
int count = 0;
const size_t num = seconds.size();
for (size_t i = num / 4; i < num / 2; ++i) {
sum += seconds[i];
count += 1;
}
return sum / count;
}
struct SortResult {
SortResult() {}
SortResult(const Algo algo, Dist dist, size_t num_keys, size_t num_threads,
double sec, size_t sizeof_key, const char* key_name)
: target(DispatchedTarget()),
algo(algo),
dist(dist),
num_keys(num_keys),
num_threads(num_threads),
sec(sec),
sizeof_key(sizeof_key),
key_name(key_name) {}
void Print() const {
const double bytes = static_cast<double>(num_keys) *
static_cast<double>(num_threads) *
static_cast<double>(sizeof_key);
printf("%10s: %12s: %7s: %9s: %05g %4.0f MB/s (%2zu threads)\n",
hwy::TargetName(target), AlgoName(algo), key_name.c_str(),
DistName(dist), static_cast<double>(num_keys), bytes * 1E-6 / sec,
num_threads);
}
int64_t target;
Algo algo;
Dist dist;
size_t num_keys = 0;
size_t num_threads = 0;
double sec = 0.0;
size_t sizeof_key = 0;
std::string key_name;
};
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_SORT_RESULT_INL_H_
// Per-target
#if defined(HIGHWAY_HWY_CONTRIB_SORT_RESULT_TOGGLE) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_SORT_RESULT_TOGGLE
#undef HIGHWAY_HWY_CONTRIB_SORT_RESULT_TOGGLE
#else
#define HIGHWAY_HWY_CONTRIB_SORT_RESULT_TOGGLE
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
// Copies the input, and compares results to that of a reference algorithm.
template <class Traits>
class ReferenceSortVerifier {
using LaneType = typename Traits::LaneType;
using KeyType = typename Traits::KeyType;
using Order = typename Traits::Order;
static constexpr bool kAscending = Order::IsAscending();
static constexpr size_t kLPK = Traits().LanesPerKey();
public:
ReferenceSortVerifier(const LaneType* in_lanes, size_t num_lanes) {
num_lanes_ = num_lanes;
num_keys_ = num_lanes / kLPK;
in_lanes_ = hwy::AllocateAligned<LaneType>(num_lanes);
HWY_ASSERT(in_lanes_);
CopyBytes(in_lanes, in_lanes_.get(), num_lanes * sizeof(LaneType));
}
// For full sorts, k_keys == num_keys.
void operator()(Algo algo, const LaneType* out_lanes, size_t k_keys) {
SharedState shared;
const Traits st;
const CappedTag<LaneType, kLPK> d;
HWY_ASSERT(hwy::IsAligned(in_lanes_.get(), sizeof(KeyType)));
KeyType* in_keys = HWY_RCAST_ALIGNED(KeyType*, in_lanes_.get());
char caption[10];
const char* algo_type = IsPartialSort(algo) ? "PartialSort" : "Sort";
HWY_ASSERT(k_keys <= num_keys_);
Run(ReferenceAlgoFor(algo), in_keys, num_keys_, shared, /*thread=*/0,
k_keys, Order());
if (IsSelect(algo)) {
// Print lanes centered around k_keys.
if (VQSORT_PRINT >= 3) {
const size_t begin_lane = k_keys < 3 ? 0 : (k_keys - 3) * kLPK;
const size_t end_lane = HWY_MIN(num_lanes_, (k_keys + 3) * kLPK);
fprintf(stderr, "\nExpected:\n");
for (size_t i = begin_lane; i < end_lane; i += kLPK) {
snprintf(caption, sizeof(caption), "%4zu ", i / kLPK);
Print(d, caption, st.SetKey(d, &in_lanes_[i]));
}
fprintf(stderr, "\n\nActual:\n");
for (size_t i = begin_lane; i < end_lane; i += kLPK) {
snprintf(caption, sizeof(caption), "%4zu ", i / kLPK);
Print(d, caption, st.SetKey(d, &out_lanes[i]));
}
fprintf(stderr, "\n\n");
}
// At k_keys: should be equivalent, i.e. neither a < b nor b < a.
// SortOrderVerifier will also check the ordering of the rest of the keys.
const size_t k = k_keys * kLPK;
if (st.Compare1(&in_lanes_[k], &out_lanes[k]) ||
st.Compare1(&out_lanes[k], &in_lanes_[k])) {
Print(d, "Expected", st.SetKey(d, &in_lanes_[k]));
Print(d, " Actual", st.SetKey(d, &out_lanes[k]));
HWY_ABORT("Select %s asc=%d: mismatch at k_keys=%zu, num_keys=%zu\n",
st.KeyString(), kAscending, k_keys, num_keys_);
}
} else {
if (VQSORT_PRINT >= 3) {
const size_t lanes_to_print = HWY_MIN(40, k_keys * kLPK);
fprintf(stderr, "\nExpected:\n");
for (size_t i = 0; i < lanes_to_print; i += kLPK) {
snprintf(caption, sizeof(caption), "%4zu ", i / kLPK);
Print(d, caption, st.SetKey(d, &in_lanes_[i]));
}
fprintf(stderr, "\n\nActual:\n");
for (size_t i = 0; i < lanes_to_print; i += kLPK) {
snprintf(caption, sizeof(caption), "%4zu ", i / kLPK);
Print(d, caption, st.SetKey(d, &out_lanes[i]));
}
fprintf(stderr, "\n\n");
}
// Full or partial sort: all elements up to k_keys are equivalent to the
// reference sort. SortOrderVerifier also checks the output's ordering.
for (size_t i = 0; i < k_keys * kLPK; i += kLPK) {
// All up to k_keys should be equivalent, i.e. neither a < b nor b < a.
if (st.Compare1(&in_lanes_[i], &out_lanes[i]) ||
st.Compare1(&out_lanes[i], &in_lanes_[i])) {
Print(d, "Expected", st.SetKey(d, &in_lanes_[i]));
Print(d, " Actual", st.SetKey(d, &out_lanes[i]));
HWY_ABORT("%s %s asc=%d: mismatch at %zu, k_keys=%zu, num_keys=%zu\n",
algo_type, st.KeyString(), kAscending, i / kLPK, k_keys,
num_keys_);
}
}
}
}
private:
hwy::AlignedFreeUniquePtr<LaneType[]> in_lanes_;
size_t num_lanes_;
size_t num_keys_;
};
// Faster than ReferenceSortVerifier, for use in bench_sort. Only verifies
// order, without running a slow reference sorter. This means it can't verify
// Select places the correct key at `k_keys`, nor that input and output keys are
// the same.
template <class Traits>
class SortOrderVerifier {
using LaneType = typename Traits::LaneType;
using Order = typename Traits::Order;
static constexpr bool kAscending = Order::IsAscending();
static constexpr size_t kLPK = Traits().LanesPerKey();
public:
void operator()(Algo algo, const InputStats<LaneType>& input_stats,
const LaneType* output, size_t num_keys, size_t k_keys) {
if (IsSelect(algo)) {
CheckSelectOrder(input_stats, output, num_keys, k_keys);
} else {
CheckSortedOrder(algo, input_stats, output, num_keys, k_keys);
}
}
private:
// For full or partial sorts: ensures keys are in sorted order.
void CheckSortedOrder(const Algo algo,
const InputStats<LaneType>& input_stats,
const LaneType* output, const size_t num_keys,
const size_t k_keys) {
const Traits st;
const CappedTag<LaneType, kLPK> d;
const size_t num_lanes = num_keys * kLPK;
const size_t k = k_keys * kLPK;
const char* algo_type = IsPartialSort(algo) ? "PartialSort" : "Sort";
InputStats<LaneType> output_stats;
// Even for partial sorts, loop over all keys to verify none disappeared.
for (size_t i = 0; i < num_lanes - kLPK; i += kLPK) {
output_stats.Notify(output[i]);
if (kLPK == 2) output_stats.Notify(output[i + 1]);
// Only check the first k_keys (== num_keys for a full sort).
// Reverse order instead of checking !Compare1 so we accept equal keys.
if (i < k - kLPK && st.Compare1(output + i + kLPK, output + i)) {
Print(d, " cur", st.SetKey(d, &output[i]));
Print(d, "next", st.SetKey(d, &output[i + kLPK]));
HWY_ABORT(
"%s %s asc=%d: wrong order at %zu, k_keys=%zu, num_keys=%zu\n",
algo_type, st.KeyString(), kAscending, i / kLPK, k_keys, num_keys);
}
}
output_stats.Notify(output[num_lanes - kLPK]);
if (kLPK == 2) output_stats.Notify(output[num_lanes - kLPK + 1]);
HWY_ASSERT(input_stats == output_stats);
}
// Ensures keys below index k_keys are less, and all above are greater.
void CheckSelectOrder(const InputStats<LaneType>& input_stats,
const LaneType* output, const size_t num_keys,
const size_t k_keys) {
const Traits st;
const CappedTag<LaneType, kLPK> d;
const size_t num_lanes = num_keys * kLPK;
const size_t k = k_keys * kLPK;
InputStats<LaneType> output_stats;
for (size_t i = 0; i < num_lanes - kLPK; i += kLPK) {
output_stats.Notify(output[i]);
if (kLPK == 2) output_stats.Notify(output[i + 1]);
// Reverse order instead of checking !Compare1 so we accept equal keys.
if (i < k ? st.Compare1(output + k, output + i)
: st.Compare1(output + i, output + k)) {
Print(d, "cur", st.SetKey(d, &output[i]));
Print(d, "kth", st.SetKey(d, &output[k]));
HWY_ABORT(
"Select %s asc=%d: wrong order at %zu, k_keys=%zu, num_keys=%zu\n",
st.KeyString(), kAscending, i / kLPK, k_keys, num_keys);
}
}
output_stats.Notify(output[num_lanes - kLPK]);
if (kLPK == 2) output_stats.Notify(output[num_lanes - kLPK + 1]);
HWY_ASSERT(input_stats == output_stats);
}
};
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_SORT_RESULT_TOGGLE

157
deps/v8/third_party/highway/src/hwy/contrib/sort/shared-inl.h vendored Normal file
View File

@ -0,0 +1,157 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Definitions shared between vqsort-inl and sorting_networks-inl.
// Normal include guard for target-independent parts
#ifndef HIGHWAY_HWY_CONTRIB_SORT_SHARED_INL_H_
#define HIGHWAY_HWY_CONTRIB_SORT_SHARED_INL_H_
#include "hwy/base.h"
namespace hwy {
// Internal constants - these are to avoid magic numbers/literals and cannot be
// changed without also changing the associated code.
struct SortConstants {
// SortingNetwork reshapes its input into a matrix. This is the maximum number
// of *lanes* per vector. Must be at least 8 because SortSamples assumes the
// sorting network can handle 128 bytes with 8 rows, so 16 bytes per vector,
// which means 8 lanes for 16-bit types.
#if HWY_COMPILER_MSVC || HWY_IS_DEBUG_BUILD
static constexpr size_t kMaxCols = 8; // avoid build timeout/stack overflow
#else
static constexpr size_t kMaxCols = 16; // enough for u32 in 512-bit vector
#endif
// 16 rows is a compromise between using the 32 AVX-512/SVE/RVV registers,
// fitting within 16 AVX2 registers with only a few spills, keeping BaseCase
// code size reasonable, and minimizing the extra logN factor for larger
// networks (for which only loose upper bounds on size are known).
static constexpr size_t kMaxRows = 16;
// Template argument ensures there is no actual division instruction.
template <size_t kLPK>
static constexpr HWY_INLINE size_t BaseCaseNumLanes(size_t N) {
// We use 8, 8x2, 8x4, and 16x{4..} networks, in units of keys. For N/kLPK
// < 4, we cannot use the 16-row networks.
return (((N / kLPK) >= 4) ? kMaxRows : 8) * HWY_MIN(N, kMaxCols);
}
// Unrolling is important (pipelining and amortizing branch mispredictions);
// 2x is sufficient to reach full memory bandwidth on SKX in Partition, but
// somewhat slower for sorting than 4x.
//
// To change, must also update left + 3 * N etc. in the loop.
static constexpr size_t kPartitionUnroll = 4;
// Chunk := group of keys loaded for sampling a pivot. Matches the typical
// cache line size of 64 bytes to get maximum benefit per L2 miss. Sort()
// ensures vectors are no larger than that, so this can be independent of the
// vector size and thus constexpr.
static constexpr HWY_INLINE size_t LanesPerChunk(size_t sizeof_t) {
return 64 / sizeof_t;
}
template <typename T>
static constexpr HWY_INLINE size_t SampleLanes() {
return 2 * LanesPerChunk(sizeof(T)); // Stored samples
}
static constexpr HWY_INLINE size_t PartitionBufNum(size_t N) {
// The main loop reads kPartitionUnroll vectors, and first loads from
// both left and right beforehand, so it requires 2 * kPartitionUnroll
// vectors. To handle amounts between that and BaseCaseNumLanes(), we
// partition up 3 * kPartitionUnroll + 1 vectors into a two-part buffer.
return 2 * (3 * kPartitionUnroll + 1) * N;
}
// Max across the three buffer usages.
template <typename T, size_t kLPK>
static constexpr HWY_INLINE size_t BufNum(size_t N) {
// BaseCase may write one padding vector, and SortSamples uses the space
// after samples as the buffer.
return HWY_MAX(SampleLanes<T>() + BaseCaseNumLanes<kLPK>(N) + N,
PartitionBufNum(N));
}
// Translates vector_size to lanes and returns size in bytes.
template <typename T, size_t kLPK>
static constexpr HWY_INLINE size_t BufBytes(size_t vector_size) {
return BufNum<T, kLPK>(vector_size / sizeof(T)) * sizeof(T);
}
// Returns max for any type.
template <size_t kLPK>
static constexpr HWY_INLINE size_t MaxBufBytes(size_t vector_size) {
// If 2 lanes per key, it's a 128-bit key with u64 lanes.
return kLPK == 2 ? BufBytes<uint64_t, 2>(vector_size)
: HWY_MAX((BufBytes<uint16_t, 1>(vector_size)),
HWY_MAX((BufBytes<uint32_t, 1>(vector_size)),
(BufBytes<uint64_t, 1>(vector_size))));
}
};
static_assert(SortConstants::MaxBufBytes<1>(64) <= 1664, "Unexpectedly high");
static_assert(SortConstants::MaxBufBytes<2>(64) <= 1664, "Unexpectedly high");
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_SORT_SHARED_INL_H_
// Per-target
// clang-format off
#if defined(HIGHWAY_HWY_CONTRIB_SORT_SHARED_TOGGLE) == defined(HWY_TARGET_TOGGLE) // NOLINT
// clang-format on
#ifdef HIGHWAY_HWY_CONTRIB_SORT_SHARED_TOGGLE
#undef HIGHWAY_HWY_CONTRIB_SORT_SHARED_TOGGLE
#else
#define HIGHWAY_HWY_CONTRIB_SORT_SHARED_TOGGLE
#endif
#include "hwy/highway.h"
// vqsort isn't available on HWY_SCALAR, and builds time out on MSVC opt and
// Armv7 debug, and Armv8 GCC 11 asan hits an internal compiler error likely
// due to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=97696. Armv8 Clang
// hwasan/msan/tsan/asan also fail to build SVE (b/335157772).
#undef VQSORT_ENABLED
#if (HWY_TARGET == HWY_SCALAR) || \
(HWY_COMPILER_MSVC && !HWY_IS_DEBUG_BUILD) || \
(HWY_ARCH_ARM_V7 && HWY_IS_DEBUG_BUILD) || \
(HWY_ARCH_ARM_A64 && HWY_COMPILER_GCC_ACTUAL && HWY_IS_ASAN)
#define VQSORT_ENABLED 0
#else
#define VQSORT_ENABLED 1
#endif
namespace hwy {
namespace HWY_NAMESPACE {
// Default tag / vector width selector.
#if HWY_TARGET == HWY_RVV
// Use LMUL = 1/2; for SEW=64 this ends up emulated via VSETVLI.
template <typename T>
using SortTag = ScalableTag<T, -1>;
#else
template <typename T>
using SortTag = ScalableTag<T>;
#endif
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_SORT_SHARED_TOGGLE

283
deps/v8/third_party/highway/src/hwy/contrib/sort/sort_test.cc vendored Normal file
View File

@ -0,0 +1,283 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <numeric> // std::iota
#include <random>
#include <vector>
#include "hwy/aligned_allocator.h" // IsAligned
#include "hwy/base.h"
#include "hwy/contrib/sort/vqsort.h"
#include "hwy/contrib/thread_pool/thread_pool.h"
#include "hwy/contrib/thread_pool/topology.h"
#include "hwy/per_target.h"
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/sort_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
// After highway.h
#include "hwy/contrib/sort/algo-inl.h"
#include "hwy/contrib/sort/result-inl.h"
#include "hwy/contrib/sort/vqsort-inl.h" // BaseCase
#include "hwy/print-inl.h"
#include "hwy/tests/test_util-inl.h"
// TODO(b/314758657): Compiler bug causes incorrect results on SSE2/S-SSE3.
#undef VQSORT_SKIP
#if !defined(VQSORT_DO_NOT_SKIP) && HWY_COMPILER_CLANG && HWY_ARCH_X86 && \
HWY_TARGET >= HWY_SSSE3
#define VQSORT_SKIP 1
#else
#define VQSORT_SKIP 0
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
using detail::OrderAscending;
using detail::OrderAscendingKV64;
using detail::OrderDescendingKV64;
using detail::SharedTraits;
using detail::TraitsLane;
#if !HAVE_INTEL && HWY_TARGET != HWY_SCALAR
using detail::OrderAscending128;
using detail::OrderAscendingKV128;
using detail::OrderDescending128;
using detail::OrderDescendingKV128;
using detail::Traits128;
#endif // !HAVE_INTEL && HWY_TARGET != HWY_SCALAR
template <typename Key>
void TestSortIota(hwy::ThreadPool& pool) {
pool.Run(128, 300, [](uint64_t task, size_t /*thread*/) {
const size_t num = static_cast<size_t>(task);
Key keys[300];
std::iota(keys, keys + num, Key{0});
VQSort(keys, num, hwy::SortAscending());
for (size_t i = 0; i < num; ++i) {
if (keys[i] != static_cast<Key>(i)) {
HWY_ABORT("num %zu i %zu: not iota, got %.0f\n", num, i,
static_cast<double>(keys[i]));
}
}
});
}
void TestAllSortIota() {
#if VQSORT_ENABLED
hwy::ThreadPool pool(hwy::HaveThreadingSupport() ? 4 : 0);
TestSortIota<uint32_t>(pool);
TestSortIota<int32_t>(pool);
if (hwy::HaveInteger64()) {
TestSortIota<int64_t>(pool);
TestSortIota<uint64_t>(pool);
}
TestSortIota<float>(pool);
if (hwy::HaveFloat64()) {
TestSortIota<double>(pool);
}
fprintf(stderr, "Iota OK\n");
#endif
}
// Supports full/partial sort and select.
template <class Traits>
void TestAnySort(const std::vector<Algo>& algos, size_t num_lanes) {
// Workaround for stack overflow on clang-cl (/F 8388608 does not help).
#if defined(_MSC_VER)
return;
#endif
using Order = typename Traits::Order;
using LaneType = typename Traits::LaneType;
using KeyType = typename Traits::KeyType;
SharedState shared;
SharedTraits<Traits> st;
constexpr size_t kLPK = st.LanesPerKey();
num_lanes = hwy::RoundUpTo(num_lanes, kLPK);
const size_t num_keys = num_lanes / kLPK;
std::mt19937 rng(42);
std::uniform_int_distribution<size_t> k_dist(1, num_keys - 1);
constexpr size_t kMaxMisalign = 16;
auto aligned =
hwy::AllocateAligned<LaneType>(kMaxMisalign + num_lanes + kMaxMisalign);
HWY_ASSERT(aligned);
for (Algo algo : algos) {
if (IsVQ(algo) && (!VQSORT_ENABLED || VQSORT_SKIP)) continue;
for (Dist dist : AllDist()) {
for (size_t misalign :
{size_t{0}, size_t{kLPK}, size_t{3 * kLPK}, kMaxMisalign / 2}) {
for (size_t k_rep = 0; k_rep < AdjustedReps(10); ++k_rep) {
// Skip reps for full sort because they do not use k.
if (!IsPartialSort(algo) && !IsSelect(algo) && k_rep > 0) break;
LaneType* lanes = aligned.get() + misalign;
HWY_ASSERT(hwy::IsAligned(lanes, sizeof(KeyType)));
KeyType* keys = HWY_RCAST_ALIGNED(KeyType*, lanes);
// Set up red zones before/after the keys to sort
for (size_t i = 0; i < misalign; ++i) {
aligned[i] = hwy::LowestValue<LaneType>();
}
for (size_t i = 0; i < kMaxMisalign; ++i) {
lanes[num_lanes + i] = hwy::HighestValue<LaneType>();
}
detail::MaybePoison(aligned.get(), misalign * sizeof(LaneType));
detail::MaybePoison(lanes + num_lanes,
kMaxMisalign * sizeof(LaneType));
InputStats<LaneType> input_stats =
GenerateInput(dist, lanes, num_lanes);
ReferenceSortVerifier<Traits> reference_verifier(lanes, num_lanes);
const size_t k_keys = k_dist(rng);
Run(algo, keys, num_keys, shared, /*thread=*/0, k_keys, Order());
reference_verifier(algo, lanes, k_keys);
SortOrderVerifier<Traits>()(algo, input_stats, lanes, num_keys,
k_keys);
// Check red zones
detail::MaybeUnpoison(aligned.get(), misalign);
detail::MaybeUnpoison(lanes + num_lanes, kMaxMisalign);
for (size_t i = 0; i < misalign; ++i) {
if (aligned[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun left at %d\n", static_cast<int>(i));
}
for (size_t i = num_lanes; i < num_lanes + kMaxMisalign; ++i) {
if (lanes[i] != hwy::HighestValue<LaneType>())
HWY_ABORT("Overrun right at %d\n", static_cast<int>(i));
}
} // k_rep
} // misalign
} // dist
} // algo
}
// Calls TestAnySort with all traits.
void CallAllSortTraits(const std::vector<Algo>& algos, size_t num_lanes) {
#if !HAVE_INTEL
TestAnySort<TraitsLane<OrderAscending<int16_t>>>(algos, num_lanes);
TestAnySort<TraitsLane<OtherOrder<uint16_t>>>(algos, num_lanes);
#endif
TestAnySort<TraitsLane<OtherOrder<int32_t>>>(algos, num_lanes);
TestAnySort<TraitsLane<OtherOrder<uint32_t>>>(algos, num_lanes);
TestAnySort<TraitsLane<OrderAscending<int64_t>>>(algos, num_lanes);
TestAnySort<TraitsLane<OrderAscending<uint64_t>>>(algos, num_lanes);
// WARNING: for float types, SIMD comparisons will flush denormals to
// zero, causing mismatches with scalar sorts. In this test, we avoid
// generating denormal inputs.
#if HWY_HAVE_FLOAT16 // #if protects algo-inl.h's GenerateRandom
// Must also check whether the dynamic-dispatch target supports float16_t!
if (hwy::HaveFloat16()) {
TestAnySort<TraitsLane<OrderAscending<float16_t>>>(algos, num_lanes);
}
#endif
TestAnySort<TraitsLane<OrderAscending<float>>>(algos, num_lanes);
#if HWY_HAVE_FLOAT64 // #if protects algo-inl.h's GenerateRandom
// Must also check whether the dynamic-dispatch target supports float64!
if (hwy::HaveFloat64()) {
TestAnySort<TraitsLane<OtherOrder<double>>>(algos, num_lanes);
}
#endif
// Other algorithms do not support 128-bit nor KV keys.
#if !HAVE_VXSORT && !HAVE_INTEL
TestAnySort<TraitsLane<OrderAscendingKV64>>(algos, num_lanes);
TestAnySort<TraitsLane<OrderDescendingKV64>>(algos, num_lanes);
// 128-bit keys require 128-bit SIMD.
#if HWY_TARGET != HWY_SCALAR
TestAnySort<Traits128<OrderAscending128>>(algos, num_lanes);
TestAnySort<Traits128<OrderDescending128>>(algos, num_lanes);
TestAnySort<Traits128<OrderAscendingKV128>>(algos, num_lanes);
TestAnySort<Traits128<OrderDescendingKV128>>(algos, num_lanes);
#endif // HWY_TARGET != HWY_SCALAR
#endif // !HAVE_VXSORT && !HAVE_INTEL
}
void TestAllSort() {
const std::vector<Algo> algos{
#if HAVE_AVX2SORT
Algo::kSEA,
#endif
#if HAVE_IPS4O
Algo::kIPS4O,
#endif
#if HAVE_PDQSORT
Algo::kPDQ,
#endif
#if HAVE_SORT512
Algo::kSort512,
#endif
Algo::kVQSort, Algo::kHeapSort,
};
for (int num : {129, 504, 3 * 1000, 34567}) {
const size_t num_lanes = AdjustedReps(static_cast<size_t>(num));
CallAllSortTraits(algos, num_lanes);
}
}
void TestAllPartialSort() {
const std::vector<Algo> algos{Algo::kVQPartialSort, Algo::kHeapPartialSort};
for (int num : {129, 504, 3 * 1000, 34567}) {
const size_t num_lanes = AdjustedReps(static_cast<size_t>(num));
CallAllSortTraits(algos, num_lanes);
}
}
void TestAllSelect() {
const std::vector<Algo> algos{Algo::kVQSelect, Algo::kHeapSelect};
for (int num : {129, 504, 3 * 1000, 34567}) {
const size_t num_lanes = AdjustedReps(static_cast<size_t>(num));
CallAllSortTraits(algos, num_lanes);
}
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(SortTest);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllSortIota);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllSort);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllSelect);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllPartialSort);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

574
deps/v8/third_party/highway/src/hwy/contrib/sort/sort_unit_test.cc vendored Normal file
View File

@ -0,0 +1,574 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdio.h>
#include <unordered_map>
#include <vector>
#include "hwy/aligned_allocator.h" // IsAligned
#include "hwy/base.h"
#include "hwy/contrib/sort/vqsort.h"
#include "hwy/detect_compiler_arch.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/sort_unit_test.cc" // NOLINT
// clang-format on
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
// After highway.h
#include "hwy/contrib/sort/algo-inl.h"
#include "hwy/contrib/sort/result-inl.h"
#include "hwy/contrib/sort/traits128-inl.h"
#include "hwy/contrib/sort/vqsort-inl.h" // BaseCase
#include "hwy/print-inl.h"
#include "hwy/tests/test_util-inl.h"
// TODO(b/314758657): Compiler bug causes incorrect results on SSE2/S-SSE3.
#undef VQSORT_SKIP
#if !defined(VQSORT_DO_NOT_SKIP) && HWY_COMPILER_CLANG && HWY_ARCH_X86 && \
HWY_TARGET >= HWY_SSSE3
#define VQSORT_SKIP 1
#else
#define VQSORT_SKIP 0
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
using detail::OrderAscending;
using detail::SharedTraits;
using detail::TraitsLane;
#if !HAVE_INTEL && HWY_TARGET != HWY_SCALAR
using detail::OrderAscending128;
using detail::OrderDescending128;
using detail::Traits128;
#endif // !HAVE_INTEL && HWY_TARGET != HWY_SCALAR
#if VQSORT_ENABLED || HWY_IDE
// Verify the corner cases of LargerSortValue/SmallerSortValue, used to
// implement PrevValue/NextValue.
struct TestFloatLargerSmaller {
template <typename T, class D>
HWY_NOINLINE void operator()(T, D d) {
const Vec<D> p0 = Zero(d);
const Vec<D> p1 = Set(d, ConvertScalarTo<T>(1));
const Vec<D> pinf = Inf(d);
const Vec<D> peps = Set(d, hwy::Epsilon<T>());
const Vec<D> pmax = Set(d, hwy::HighestValue<T>());
const Vec<D> n0 = Neg(p0);
const Vec<D> n1 = Neg(p1);
const Vec<D> ninf = Neg(pinf);
const Vec<D> neps = Neg(peps);
const Vec<D> nmax = Neg(pmax);
// Larger(0) is the smallest subnormal, typically eps * FLT_MIN.
const RebindToUnsigned<D> du;
const Vec<D> psub = BitCast(d, Set(du, 1));
const Vec<D> nsub = Neg(psub);
HWY_ASSERT(AllTrue(d, Lt(psub, peps)));
HWY_ASSERT(AllTrue(d, Gt(nsub, neps)));
// +/-0 moves to +/- smallest subnormal.
HWY_ASSERT_VEC_EQ(d, psub, detail::LargerSortValue(d, p0));
HWY_ASSERT_VEC_EQ(d, nsub, detail::SmallerSortValue(d, p0));
HWY_ASSERT_VEC_EQ(d, psub, detail::LargerSortValue(d, n0));
HWY_ASSERT_VEC_EQ(d, nsub, detail::SmallerSortValue(d, n0));
// The next magnitude larger than 1 is (1 + eps) by definition.
HWY_ASSERT_VEC_EQ(d, Add(p1, peps), detail::LargerSortValue(d, p1));
HWY_ASSERT_VEC_EQ(d, Add(n1, neps), detail::SmallerSortValue(d, n1));
// 1-eps and -1+eps are slightly different, but we can still ensure the
// next values are less than 1 / greater than -1.
HWY_ASSERT(AllTrue(d, Gt(p1, detail::SmallerSortValue(d, p1))));
HWY_ASSERT(AllTrue(d, Lt(n1, detail::LargerSortValue(d, n1))));
// Even for large (finite) values, we can move toward/away from infinity.
HWY_ASSERT_VEC_EQ(d, pinf, detail::LargerSortValue(d, pmax));
HWY_ASSERT_VEC_EQ(d, ninf, detail::SmallerSortValue(d, nmax));
HWY_ASSERT(AllTrue(d, Gt(pmax, detail::SmallerSortValue(d, pmax))));
HWY_ASSERT(AllTrue(d, Lt(nmax, detail::LargerSortValue(d, nmax))));
// For infinities, results are unchanged or the extremal finite value.
HWY_ASSERT_VEC_EQ(d, pinf, detail::LargerSortValue(d, pinf));
HWY_ASSERT_VEC_EQ(d, pmax, detail::SmallerSortValue(d, pinf));
HWY_ASSERT_VEC_EQ(d, nmax, detail::LargerSortValue(d, ninf));
HWY_ASSERT_VEC_EQ(d, ninf, detail::SmallerSortValue(d, ninf));
}
};
HWY_NOINLINE void TestAllFloatLargerSmaller() {
ForFloatTypesDynamic(ForPartialVectors<TestFloatLargerSmaller>());
}
// Previously, LastValue was the largest normal float, so we injected that
// value into arrays containing only infinities. Ensure that does not happen.
struct TestFloatInf {
template <typename T, class D>
HWY_NOINLINE void operator()(T, D d) {
const size_t N = Lanes(d);
const size_t num = N * 3;
auto in = hwy::AllocateAligned<T>(num);
HWY_ASSERT(in);
Fill(d, GetLane(Inf(d)), num, in.get());
VQSort(in.get(), num, SortAscending());
for (size_t i = 0; i < num; i += N) {
HWY_ASSERT(AllTrue(d, IsInf(LoadU(d, in.get() + i))));
}
}
};
HWY_NOINLINE void TestAllFloatInf() {
// TODO(janwas): bfloat16_t not yet supported.
ForFloatTypesDynamic(ForPartialVectors<TestFloatInf>());
}
template <class Traits>
static HWY_NOINLINE void TestMedian3() {
using LaneType = typename Traits::LaneType;
using D = CappedTag<LaneType, 1>;
SharedTraits<Traits> st;
const D d;
using V = Vec<D>;
for (uint32_t bits = 0; bits < 8; ++bits) {
const V v0 = Set(d, LaneType{(bits & (1u << 0)) ? 1u : 0u});
const V v1 = Set(d, LaneType{(bits & (1u << 1)) ? 1u : 0u});
const V v2 = Set(d, LaneType{(bits & (1u << 2)) ? 1u : 0u});
const LaneType m = GetLane(detail::MedianOf3(st, v0, v1, v2));
// If at least half(rounded up) of bits are 1, so is the median.
const size_t count = PopCount(bits);
HWY_ASSERT_EQ((count >= 2) ? static_cast<LaneType>(1) : 0, m);
}
}
HWY_NOINLINE void TestAllMedian() {
TestMedian3<TraitsLane<OrderAscending<uint64_t> > >();
}
template <class Traits>
static HWY_NOINLINE void TestBaseCaseAscDesc() {
using LaneType = typename Traits::LaneType;
SharedTraits<Traits> st;
const SortTag<LaneType> d;
const size_t N = Lanes(d);
constexpr size_t N1 = st.LanesPerKey();
const size_t base_case_num = SortConstants::BaseCaseNumLanes<N1>(N);
constexpr int kDebug = 0;
auto aligned_lanes = hwy::AllocateAligned<LaneType>(N + base_case_num + N);
auto buf = hwy::AllocateAligned<LaneType>(base_case_num + 2 * N);
HWY_ASSERT(aligned_lanes && buf);
std::vector<size_t> lengths;
lengths.push_back(HWY_MAX(1, N1));
lengths.push_back(3 * N1);
lengths.push_back(base_case_num / 2);
lengths.push_back(base_case_num / 2 + N1);
lengths.push_back(base_case_num - N1);
lengths.push_back(base_case_num);
std::vector<size_t> misalignments;
misalignments.push_back(0);
misalignments.push_back(1);
if (N >= 6) misalignments.push_back(N / 2 - 1);
misalignments.push_back(N / 2);
misalignments.push_back(N / 2 + 1);
misalignments.push_back(HWY_MIN(2 * N / 3 + 3, size_t{N - 1}));
for (bool asc : {false, true}) {
for (size_t len : lengths) {
for (size_t misalign : misalignments) {
LaneType* HWY_RESTRICT lanes = aligned_lanes.get() + misalign;
if (kDebug) {
printf("============%s asc %d N1 %d len %d misalign %d\n",
st.KeyString(), asc, static_cast<int>(N1),
static_cast<int>(len), static_cast<int>(misalign));
}
for (size_t i = 0; i < misalign; ++i) {
aligned_lanes[i] = hwy::LowestValue<LaneType>();
}
InputStats<LaneType> input_stats;
for (size_t i = 0; i < len; ++i) {
lanes[i] = asc ? static_cast<LaneType>(LaneType(i) + 1)
: static_cast<LaneType>(LaneType(len) - LaneType(i));
input_stats.Notify(lanes[i]);
if (kDebug >= 2) {
printf("%3zu: %f\n", i, static_cast<double>(lanes[i]));
}
}
for (size_t i = len; i < base_case_num + N; ++i) {
lanes[i] = hwy::LowestValue<LaneType>();
}
detail::BaseCase(d, st, lanes, len, buf.get());
if (kDebug >= 2) {
printf("out>>>>>>\n");
for (size_t i = 0; i < len; ++i) {
printf("%3zu: %f\n", i, static_cast<double>(lanes[i]));
}
}
SortOrderVerifier<Traits>()(Algo::kVQSort, input_stats, lanes, len / N1,
len / N1);
for (size_t i = 0; i < misalign; ++i) {
if (aligned_lanes[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun misalign at %d\n", static_cast<int>(i));
}
for (size_t i = len; i < base_case_num + N; ++i) {
if (lanes[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun right at %d\n", static_cast<int>(i));
}
} // misalign
} // len
} // asc
}
template <class Traits>
static HWY_NOINLINE void TestBaseCase01() {
using LaneType = typename Traits::LaneType;
SharedTraits<Traits> st;
const SortTag<LaneType> d;
const size_t N = Lanes(d);
constexpr size_t N1 = st.LanesPerKey();
const size_t base_case_num = SortConstants::BaseCaseNumLanes<N1>(N);
constexpr int kDebug = 0;
auto lanes = hwy::AllocateAligned<LaneType>(base_case_num + N);
auto buf = hwy::AllocateAligned<LaneType>(base_case_num + 2 * N);
HWY_ASSERT(lanes && buf);
std::vector<size_t> lengths;
lengths.push_back(HWY_MAX(1, N1));
lengths.push_back(3 * N1);
lengths.push_back(base_case_num / 2);
lengths.push_back(base_case_num / 2 + N1);
lengths.push_back(base_case_num - N1);
lengths.push_back(base_case_num);
for (size_t len : lengths) {
if (kDebug) {
printf("============%s 01 N1 %d len %d\n", st.KeyString(),
static_cast<int>(N1), static_cast<int>(len));
}
const uint64_t kMaxBits = AdjustedLog2Reps(HWY_MIN(len, size_t{14}));
for (uint64_t bits = 0; bits < ((1ull << kMaxBits) - 1); ++bits) {
InputStats<LaneType> input_stats;
for (size_t i = 0; i < len; ++i) {
lanes[i] = (i < 64 && (bits & (1ull << i))) ? 1 : 0;
input_stats.Notify(lanes[i]);
if (kDebug >= 2) {
printf("%3zu: %f\n", i, static_cast<double>(lanes[i]));
}
}
for (size_t i = len; i < base_case_num + N; ++i) {
lanes[i] = hwy::LowestValue<LaneType>();
}
detail::BaseCase(d, st, lanes.get(), len, buf.get());
if (kDebug >= 2) {
printf("out>>>>>>\n");
for (size_t i = 0; i < len; ++i) {
printf("%3zu: %f\n", i, static_cast<double>(lanes[i]));
}
}
SortOrderVerifier<Traits>()(Algo::kVQSort, input_stats, lanes.get(),
len / N1, len / N1);
for (size_t i = len; i < base_case_num + N; ++i) {
if (lanes[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun right at %d\n", static_cast<int>(i));
}
} // bits
} // len
}
template <class Traits>
static HWY_NOINLINE void TestBaseCase() {
TestBaseCaseAscDesc<Traits>();
TestBaseCase01<Traits>();
}
HWY_NOINLINE void TestAllBaseCase() {
// Workaround for stack overflow on MSVC debug.
#if defined(_MSC_VER) || VQSORT_SKIP
return;
#endif
TestBaseCase<TraitsLane<OrderAscending<int32_t> > >();
TestBaseCase<TraitsLane<OtherOrder<int64_t> > >();
#if !HAVE_INTEL
TestBaseCase<Traits128<OrderAscending128> >();
TestBaseCase<Traits128<OrderDescending128> >();
#endif
}
template <class Traits>
static HWY_NOINLINE void VerifyPartition(
Traits st, typename Traits::LaneType* HWY_RESTRICT lanes, size_t left,
size_t border, size_t right, const size_t N1,
const typename Traits::LaneType* pivot) {
/* for (size_t i = left; i < right; ++i) {
if (i == border) printf("--\n");
printf("%4zu: %3d\n", i, lanes[i]);
}*/
HWY_ASSERT(left % N1 == 0);
HWY_ASSERT(border % N1 == 0);
HWY_ASSERT(right % N1 == 0);
constexpr bool kAscending = Traits::Order::IsAscending();
for (size_t i = left; i < border; i += N1) {
if (st.Compare1(pivot, lanes + i)) {
HWY_ABORT(
"%s: asc %d left[%d] piv %.0f %.0f compares before %.0f %.0f "
"border %d",
st.KeyString(), kAscending, static_cast<int>(i),
static_cast<double>(pivot[1]), static_cast<double>(pivot[0]),
static_cast<double>(lanes[i + 1]), static_cast<double>(lanes[i + 0]),
static_cast<int>(border));
}
}
for (size_t i = border; i < right; i += N1) {
if (!st.Compare1(pivot, lanes + i)) {
HWY_ABORT(
"%s: asc %d right[%d] piv %.0f %.0f compares after %.0f %.0f "
"border %d",
st.KeyString(), kAscending, static_cast<int>(i),
static_cast<double>(pivot[1]), static_cast<double>(pivot[0]),
static_cast<double>(lanes[i + 1]), static_cast<double>(lanes[i]),
static_cast<int>(border));
}
}
}
template <class Traits>
static HWY_NOINLINE void TestPartition() {
using LaneType = typename Traits::LaneType;
// See HandleSpecialCases and HWY_ASSERT below.
const CappedTag<LaneType, 64 / sizeof(LaneType)> d;
SharedTraits<Traits> st;
constexpr bool kAscending = Traits::Order::IsAscending();
const size_t N = Lanes(d);
constexpr int kDebug = 0;
constexpr size_t N1 = st.LanesPerKey();
const size_t base_case_num = SortConstants::BaseCaseNumLanes<N1>(N);
HWY_ASSERT(2 * N <= base_case_num); // See HandleSpecialCases
// left + len + align
const size_t total = 32 + (base_case_num + 4 * HWY_MAX(N, 4)) + 2 * N;
auto aligned_lanes = hwy::AllocateAligned<LaneType>(total);
HWY_ASSERT(aligned_lanes);
HWY_ALIGN LaneType buf[SortConstants::BufBytes<LaneType, N1>(HWY_MAX_BYTES) /
sizeof(LaneType)];
for (bool in_asc : {false, true}) {
for (int left_i : {0, 1, 7, 8, 30, 31}) {
const size_t left = static_cast<size_t>(left_i) & ~(N1 - 1);
for (size_t ofs :
{N, N + 3, 2 * N, 2 * N + 2, 2 * N + 3, 3 * N - 1, 4 * N - 2}) {
const size_t len = (base_case_num + ofs) & ~(N1 - 1);
for (LaneType pivot1 : {LaneType(0), LaneType(len / 3),
LaneType(2 * len / 3), LaneType(len)}) {
const LaneType pivot2[2] = {pivot1, 0};
const auto pivot = st.SetKey(d, pivot2);
for (size_t misalign = 0; misalign < N; misalign += N1) {
LaneType* HWY_RESTRICT lanes = aligned_lanes.get() + misalign;
const size_t right = left + len;
if (kDebug) {
printf(
"=========%s asc %d left %d len %d right %d piv %.0f %.0f\n",
st.KeyString(), kAscending, static_cast<int>(left),
static_cast<int>(len), static_cast<int>(right),
static_cast<double>(pivot2[1]),
static_cast<double>(pivot2[0]));
}
for (size_t i = 0; i < misalign; ++i) {
aligned_lanes[i] = hwy::LowestValue<LaneType>();
}
for (size_t i = 0; i < left; ++i) {
lanes[i] = hwy::LowestValue<LaneType>();
}
std::unordered_map<LaneType, int> counts;
for (size_t i = left; i < right; ++i) {
lanes[i] = static_cast<LaneType>(
in_asc ? LaneType(i + 1) - static_cast<LaneType>(left)
: static_cast<LaneType>(right) - LaneType(i));
++counts[lanes[i]];
if (kDebug >= 2) {
printf("%3zu: %f\n", i, static_cast<double>(lanes[i]));
}
}
for (size_t i = right; i < total - misalign; ++i) {
lanes[i] = hwy::LowestValue<LaneType>();
}
size_t border = left + detail::Partition(d, st, lanes + left,
right - left, pivot, buf);
if (kDebug >= 2) {
printf("out>>>>>>\n");
for (size_t i = left; i < right; ++i) {
printf("%3zu: %f\n", i, static_cast<double>(lanes[i]));
}
for (size_t i = right; i < total - misalign; ++i) {
printf("%3zu: sentinel %f\n", i, static_cast<double>(lanes[i]));
}
}
for (size_t i = left; i < right; ++i) {
--counts[lanes[i]];
}
for (auto kv : counts) {
if (kv.second != 0) {
PrintValue(kv.first);
HWY_ABORT("Incorrect count %d\n", kv.second);
}
}
VerifyPartition(st, lanes, left, border, right, N1, pivot2);
for (size_t i = 0; i < misalign; ++i) {
if (aligned_lanes[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun misalign at %d\n", static_cast<int>(i));
}
for (size_t i = 0; i < left; ++i) {
if (lanes[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun left at %d\n", static_cast<int>(i));
}
for (size_t i = right; i < total - misalign; ++i) {
if (lanes[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun right at %d\n", static_cast<int>(i));
}
} // misalign
} // pivot
} // len
} // left
} // asc
}
#undef HWY_BROKEN_U128
#if HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1400 && \
HWY_TARGET == HWY_RVV
#define HWY_BROKEN_U128 1
#else
#define HWY_BROKEN_U128 0
#endif
HWY_NOINLINE void TestAllPartition() {
TestPartition<TraitsLane<OtherOrder<int32_t> > >();
#if !HAVE_INTEL && !HWY_BROKEN_U128
TestPartition<Traits128<OrderAscending128> >();
#endif
#if !HWY_IS_DEBUG_BUILD
TestPartition<TraitsLane<OrderAscending<int16_t> > >();
TestPartition<TraitsLane<OrderAscending<int64_t> > >();
TestPartition<TraitsLane<OtherOrder<float> > >();
// OK to check current target, not using dynamic dispatch here.
#if HWY_HAVE_FLOAT64
TestPartition<TraitsLane<OtherOrder<double> > >();
#endif
#if !HAVE_INTEL && !HWY_BROKEN_U128
TestPartition<Traits128<OrderDescending128> >();
#endif
#endif
}
// (used for sample selection for choosing a pivot)
template <typename TU>
static HWY_NOINLINE void TestRandomGenerator() {
static_assert(!hwy::IsSigned<TU>(), "");
SortTag<TU> du;
const size_t N = Lanes(du);
uint64_t* state = GetGeneratorState();
// Ensure lower and upper 32 bits are uniformly distributed.
uint64_t sum_lo = 0, sum_hi = 0;
for (size_t i = 0; i < 1000; ++i) {
const uint64_t bits = detail::RandomBits(state);
sum_lo += bits & 0xFFFFFFFF;
sum_hi += bits >> 32;
}
const double expected = 1000 * (1ULL << 31);
HWY_ASSERT(0.9 * expected <= static_cast<double>(sum_lo) &&
static_cast<double>(sum_lo) <= 1.1 * expected);
HWY_ASSERT(0.9 * expected <= static_cast<double>(sum_hi) &&
static_cast<double>(sum_hi) <= 1.1 * expected);
const size_t lanes_per_block = HWY_MAX(64 / sizeof(TU), N); // power of two
for (uint32_t num_blocks = 2; num_blocks < 100000;
num_blocks = 3 * num_blocks / 2) {
// Generate some numbers and ensure all are in range
uint64_t sum = 0;
constexpr size_t kReps = 10000;
for (size_t rep = 0; rep < kReps; ++rep) {
const uint32_t bits = detail::RandomBits(state) & 0xFFFFFFFF;
const size_t index = detail::RandomChunkIndex(num_blocks, bits);
HWY_ASSERT(((index + 1) * lanes_per_block) <=
num_blocks * lanes_per_block);
sum += index;
}
// Also ensure the mean is near the middle of the range
const double expected = (num_blocks - 1) / 2.0;
const double actual = static_cast<double>(sum) / kReps;
HWY_ASSERT(0.9 * expected <= actual && actual <= 1.1 * expected);
}
}
HWY_NOINLINE void TestAllGenerator() {
TestRandomGenerator<uint32_t>();
TestRandomGenerator<uint64_t>();
}
#else
static void TestAllFloatLargerSmaller() {}
static void TestAllFloatInf() {}
static void TestAllMedian() {}
static void TestAllBaseCase() {}
static void TestAllPartition() {}
static void TestAllGenerator() {}
#endif // VQSORT_ENABLED
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(SortTest);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllFloatLargerSmaller);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllFloatInf);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllMedian);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllBaseCase);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllPartition);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllGenerator);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

902
deps/v8/third_party/highway/src/hwy/contrib/sort/sorting_networks-inl.h vendored Normal file
View File

@ -0,0 +1,902 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Per-target
#if defined(HIGHWAY_HWY_CONTRIB_SORT_SORTING_NETWORKS_TOGGLE) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_SORT_SORTING_NETWORKS_TOGGLE
#undef HIGHWAY_HWY_CONTRIB_SORT_SORTING_NETWORKS_TOGGLE
#else
#define HIGHWAY_HWY_CONTRIB_SORT_SORTING_NETWORKS_TOGGLE
#endif
#include "hwy/contrib/sort/shared-inl.h" // SortConstants
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace detail {
#if VQSORT_ENABLED
using Constants = hwy::SortConstants;
// ------------------------------ SharedTraits
// Code shared between all traits. It's unclear whether these can profitably be
// specialized for Lane vs Block, or optimized like SortPairsDistance1 using
// Compare/DupOdd.
template <class Base>
struct SharedTraits : public Base {
using SharedTraitsForSortingNetwork =
SharedTraits<typename Base::TraitsForSortingNetwork>;
// Conditionally swaps lane 0 with 2, 1 with 3 etc.
template <class D>
HWY_INLINE Vec<D> SortPairsDistance2(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->SwapAdjacentPairs(d, v);
base->Sort2(d, v, swapped);
return base->OddEvenPairs(d, swapped, v);
}
// Swaps with the vector formed by reversing contiguous groups of 8 keys.
template <class D>
HWY_INLINE Vec<D> SortPairsReverse8(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->ReverseKeys8(d, v);
base->Sort2(d, v, swapped);
return base->OddEvenQuads(d, swapped, v);
}
// Swaps with the vector formed by reversing contiguous groups of 8 keys.
template <class D>
HWY_INLINE Vec<D> SortPairsReverse16(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
static_assert(Constants::kMaxCols <= 16, "Need actual Reverse16");
Vec<D> swapped = base->ReverseKeys(d, v);
base->Sort2(d, v, swapped);
return ConcatUpperLower(d, swapped, v); // 8 = half of the vector
}
};
// ------------------------------ Sorting network
// Sorting networks for independent columns in 2, 4 and 8 vectors from
// https://bertdobbelaere.github.io/sorting_networks.html.
template <class D, class Traits, class V = Vec<D>>
HWY_INLINE void Sort2(D d, Traits st, V& v0, V& v1) {
st.Sort2(d, v0, v1);
}
template <class D, class Traits, class V = Vec<D>>
HWY_INLINE void Sort4(D d, Traits st, V& v0, V& v1, V& v2, V& v3) {
st.Sort2(d, v0, v2);
st.Sort2(d, v1, v3);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v1, v2);
}
template <class D, class Traits, class V = Vec<D>>
HWY_INLINE void Sort8(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4, V& v5,
V& v6, V& v7) {
st.Sort2(d, v0, v2);
st.Sort2(d, v1, v3);
st.Sort2(d, v4, v6);
st.Sort2(d, v5, v7);
st.Sort2(d, v0, v4);
st.Sort2(d, v1, v5);
st.Sort2(d, v2, v6);
st.Sort2(d, v3, v7);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
st.Sort2(d, v2, v4);
st.Sort2(d, v3, v5);
st.Sort2(d, v1, v4);
st.Sort2(d, v3, v6);
st.Sort2(d, v1, v2);
st.Sort2(d, v3, v4);
st.Sort2(d, v5, v6);
}
// (Green's irregular) sorting network for independent columns in 16 vectors.
template <class D, class Traits, class V = Vec<D>>
HWY_INLINE void Sort16(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4, V& v5,
V& v6, V& v7, V& v8, V& v9, V& va, V& vb, V& vc, V& vd,
V& ve, V& vf) {
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
st.Sort2(d, v8, v9);
st.Sort2(d, va, vb);
st.Sort2(d, vc, vd);
st.Sort2(d, ve, vf);
st.Sort2(d, v0, v2);
st.Sort2(d, v1, v3);
st.Sort2(d, v4, v6);
st.Sort2(d, v5, v7);
st.Sort2(d, v8, va);
st.Sort2(d, v9, vb);
st.Sort2(d, vc, ve);
st.Sort2(d, vd, vf);
st.Sort2(d, v0, v4);
st.Sort2(d, v1, v5);
st.Sort2(d, v2, v6);
st.Sort2(d, v3, v7);
st.Sort2(d, v8, vc);
st.Sort2(d, v9, vd);
st.Sort2(d, va, ve);
st.Sort2(d, vb, vf);
st.Sort2(d, v0, v8);
st.Sort2(d, v1, v9);
st.Sort2(d, v2, va);
st.Sort2(d, v3, vb);
st.Sort2(d, v4, vc);
st.Sort2(d, v5, vd);
st.Sort2(d, v6, ve);
st.Sort2(d, v7, vf);
st.Sort2(d, v5, va);
st.Sort2(d, v6, v9);
st.Sort2(d, v3, vc);
st.Sort2(d, v7, vb);
st.Sort2(d, vd, ve);
st.Sort2(d, v4, v8);
st.Sort2(d, v1, v2);
st.Sort2(d, v1, v4);
st.Sort2(d, v7, vd);
st.Sort2(d, v2, v8);
st.Sort2(d, vb, ve);
st.Sort2(d, v2, v4);
st.Sort2(d, v5, v6);
st.Sort2(d, v9, va);
st.Sort2(d, vb, vd);
st.Sort2(d, v3, v8);
st.Sort2(d, v7, vc);
st.Sort2(d, v3, v5);
st.Sort2(d, v6, v8);
st.Sort2(d, v7, v9);
st.Sort2(d, va, vc);
st.Sort2(d, v3, v4);
st.Sort2(d, v5, v6);
st.Sort2(d, v7, v8);
st.Sort2(d, v9, va);
st.Sort2(d, vb, vc);
st.Sort2(d, v6, v7);
st.Sort2(d, v8, v9);
}
// ------------------------------ Merging networks
// Blacher's hybrid bitonic/odd-even networks, generated by print_network.cc.
// For acceptable performance, these must be inlined, otherwise vectors are
// loaded from the stack. The kKeysPerVector allows calling from generic code
// but skipping the functions when vectors have too few lanes for
// st.SortPairsDistance1 to compile. `if constexpr` in the caller would also
// work, but is not available in C++11. We write out the (unused) argument types
// rather than `...` because GCC 9 (but not 10) fails to compile with `...`.
template <size_t kKeysPerVector, class D, class Traits, class V,
HWY_IF_LANES_LE(kKeysPerVector, 1)>
HWY_INLINE void Merge8x2(D, Traits, V, V, V, V, V, V, V, V) {}
template <size_t kKeysPerVector, class D, class Traits, class V,
HWY_IF_LANES_LE(kKeysPerVector, 2)>
HWY_INLINE void Merge8x4(D, Traits, V, V, V, V, V, V, V, V) {}
template <size_t kKeysPerVector, class D, class Traits, class V,
HWY_IF_LANES_LE(kKeysPerVector, 1)>
HWY_INLINE void Merge16x2(D, Traits, V, V, V, V, V, V, V, V, V, V, V, V, V, V,
V, V) {}
template <size_t kKeysPerVector, class D, class Traits, class V,
HWY_IF_LANES_LE(kKeysPerVector, 2)>
HWY_INLINE void Merge16x4(D, Traits, V, V, V, V, V, V, V, V, V, V, V, V, V, V,
V, V) {}
template <size_t kKeysPerVector, class D, class Traits, class V,
HWY_IF_LANES_LE(kKeysPerVector, 4)>
HWY_INLINE void Merge16x8(D, Traits, V, V, V, V, V, V, V, V, V, V, V, V, V, V,
V, V) {}
template <size_t kKeysPerVector, class D, class Traits, class V,
HWY_IF_LANES_LE(kKeysPerVector, 8)>
HWY_INLINE void Merge16x16(D, Traits, V, V, V, V, V, V, V, V, V, V, V, V, V, V,
V, V) {}
template <size_t kKeysPerVector, class D, class Traits, class V = Vec<D>,
HWY_IF_LANES_GT(kKeysPerVector, 1)>
HWY_INLINE void Merge8x2(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4,
V& v5, V& v6, V& v7) {
v7 = st.ReverseKeys2(d, v7);
v6 = st.ReverseKeys2(d, v6);
v5 = st.ReverseKeys2(d, v5);
v4 = st.ReverseKeys2(d, v4);
st.Sort2(d, v0, v7);
st.Sort2(d, v1, v6);
st.Sort2(d, v2, v5);
st.Sort2(d, v3, v4);
v3 = st.ReverseKeys2(d, v3);
v2 = st.ReverseKeys2(d, v2);
v7 = st.ReverseKeys2(d, v7);
v6 = st.ReverseKeys2(d, v6);
st.Sort2(d, v0, v3);
st.Sort2(d, v1, v2);
st.Sort2(d, v4, v7);
st.Sort2(d, v5, v6);
v1 = st.ReverseKeys2(d, v1);
v3 = st.ReverseKeys2(d, v3);
v5 = st.ReverseKeys2(d, v5);
v7 = st.ReverseKeys2(d, v7);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
v0 = st.SortPairsDistance1(d, v0);
v1 = st.SortPairsDistance1(d, v1);
v2 = st.SortPairsDistance1(d, v2);
v3 = st.SortPairsDistance1(d, v3);
v4 = st.SortPairsDistance1(d, v4);
v5 = st.SortPairsDistance1(d, v5);
v6 = st.SortPairsDistance1(d, v6);
v7 = st.SortPairsDistance1(d, v7);
}
template <size_t kKeysPerVector, class D, class Traits, class V = Vec<D>,
HWY_IF_LANES_GT(kKeysPerVector, 2)>
HWY_INLINE void Merge8x4(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4,
V& v5, V& v6, V& v7) {
v7 = st.ReverseKeys4(d, v7);
v6 = st.ReverseKeys4(d, v6);
v5 = st.ReverseKeys4(d, v5);
v4 = st.ReverseKeys4(d, v4);
st.Sort2(d, v0, v7);
st.Sort2(d, v1, v6);
st.Sort2(d, v2, v5);
st.Sort2(d, v3, v4);
v3 = st.ReverseKeys4(d, v3);
v2 = st.ReverseKeys4(d, v2);
v7 = st.ReverseKeys4(d, v7);
v6 = st.ReverseKeys4(d, v6);
st.Sort2(d, v0, v3);
st.Sort2(d, v1, v2);
st.Sort2(d, v4, v7);
st.Sort2(d, v5, v6);
v1 = st.ReverseKeys4(d, v1);
v3 = st.ReverseKeys4(d, v3);
v5 = st.ReverseKeys4(d, v5);
v7 = st.ReverseKeys4(d, v7);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
v0 = st.SortPairsReverse4(d, v0);
v1 = st.SortPairsReverse4(d, v1);
v2 = st.SortPairsReverse4(d, v2);
v3 = st.SortPairsReverse4(d, v3);
v4 = st.SortPairsReverse4(d, v4);
v5 = st.SortPairsReverse4(d, v5);
v6 = st.SortPairsReverse4(d, v6);
v7 = st.SortPairsReverse4(d, v7);
v0 = st.SortPairsDistance1(d, v0);
v1 = st.SortPairsDistance1(d, v1);
v2 = st.SortPairsDistance1(d, v2);
v3 = st.SortPairsDistance1(d, v3);
v4 = st.SortPairsDistance1(d, v4);
v5 = st.SortPairsDistance1(d, v5);
v6 = st.SortPairsDistance1(d, v6);
v7 = st.SortPairsDistance1(d, v7);
}
// Only used by the now-deprecated SortingNetwork().
template <size_t kKeysPerVector, class D, class Traits, class V = Vec<D>,
HWY_IF_LANES_GT(kKeysPerVector, 1)>
HWY_INLINE void Merge16x2(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4,
V& v5, V& v6, V& v7, V& v8, V& v9, V& va, V& vb,
V& vc, V& vd, V& ve, V& vf) {
vf = st.ReverseKeys2(d, vf);
ve = st.ReverseKeys2(d, ve);
vd = st.ReverseKeys2(d, vd);
vc = st.ReverseKeys2(d, vc);
vb = st.ReverseKeys2(d, vb);
va = st.ReverseKeys2(d, va);
v9 = st.ReverseKeys2(d, v9);
v8 = st.ReverseKeys2(d, v8);
st.Sort2(d, v0, vf);
st.Sort2(d, v1, ve);
st.Sort2(d, v2, vd);
st.Sort2(d, v3, vc);
st.Sort2(d, v4, vb);
st.Sort2(d, v5, va);
st.Sort2(d, v6, v9);
st.Sort2(d, v7, v8);
v7 = st.ReverseKeys2(d, v7);
v6 = st.ReverseKeys2(d, v6);
v5 = st.ReverseKeys2(d, v5);
v4 = st.ReverseKeys2(d, v4);
vf = st.ReverseKeys2(d, vf);
ve = st.ReverseKeys2(d, ve);
vd = st.ReverseKeys2(d, vd);
vc = st.ReverseKeys2(d, vc);
st.Sort2(d, v0, v7);
st.Sort2(d, v1, v6);
st.Sort2(d, v2, v5);
st.Sort2(d, v3, v4);
st.Sort2(d, v8, vf);
st.Sort2(d, v9, ve);
st.Sort2(d, va, vd);
st.Sort2(d, vb, vc);
v3 = st.ReverseKeys2(d, v3);
v2 = st.ReverseKeys2(d, v2);
v7 = st.ReverseKeys2(d, v7);
v6 = st.ReverseKeys2(d, v6);
vb = st.ReverseKeys2(d, vb);
va = st.ReverseKeys2(d, va);
vf = st.ReverseKeys2(d, vf);
ve = st.ReverseKeys2(d, ve);
st.Sort2(d, v0, v3);
st.Sort2(d, v1, v2);
st.Sort2(d, v4, v7);
st.Sort2(d, v5, v6);
st.Sort2(d, v8, vb);
st.Sort2(d, v9, va);
st.Sort2(d, vc, vf);
st.Sort2(d, vd, ve);
v1 = st.ReverseKeys2(d, v1);
v3 = st.ReverseKeys2(d, v3);
v5 = st.ReverseKeys2(d, v5);
v7 = st.ReverseKeys2(d, v7);
v9 = st.ReverseKeys2(d, v9);
vb = st.ReverseKeys2(d, vb);
vd = st.ReverseKeys2(d, vd);
vf = st.ReverseKeys2(d, vf);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
st.Sort2(d, v8, v9);
st.Sort2(d, va, vb);
st.Sort2(d, vc, vd);
st.Sort2(d, ve, vf);
v0 = st.SortPairsDistance1(d, v0);
v1 = st.SortPairsDistance1(d, v1);
v2 = st.SortPairsDistance1(d, v2);
v3 = st.SortPairsDistance1(d, v3);
v4 = st.SortPairsDistance1(d, v4);
v5 = st.SortPairsDistance1(d, v5);
v6 = st.SortPairsDistance1(d, v6);
v7 = st.SortPairsDistance1(d, v7);
v8 = st.SortPairsDistance1(d, v8);
v9 = st.SortPairsDistance1(d, v9);
va = st.SortPairsDistance1(d, va);
vb = st.SortPairsDistance1(d, vb);
vc = st.SortPairsDistance1(d, vc);
vd = st.SortPairsDistance1(d, vd);
ve = st.SortPairsDistance1(d, ve);
vf = st.SortPairsDistance1(d, vf);
}
template <size_t kKeysPerVector, class D, class Traits, class V = Vec<D>,
HWY_IF_LANES_GT(kKeysPerVector, 2)>
HWY_INLINE void Merge16x4(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4,
V& v5, V& v6, V& v7, V& v8, V& v9, V& va, V& vb,
V& vc, V& vd, V& ve, V& vf) {
vf = st.ReverseKeys4(d, vf);
ve = st.ReverseKeys4(d, ve);
vd = st.ReverseKeys4(d, vd);
vc = st.ReverseKeys4(d, vc);
vb = st.ReverseKeys4(d, vb);
va = st.ReverseKeys4(d, va);
v9 = st.ReverseKeys4(d, v9);
v8 = st.ReverseKeys4(d, v8);
st.Sort2(d, v0, vf);
st.Sort2(d, v1, ve);
st.Sort2(d, v2, vd);
st.Sort2(d, v3, vc);
st.Sort2(d, v4, vb);
st.Sort2(d, v5, va);
st.Sort2(d, v6, v9);
st.Sort2(d, v7, v8);
v7 = st.ReverseKeys4(d, v7);
v6 = st.ReverseKeys4(d, v6);
v5 = st.ReverseKeys4(d, v5);
v4 = st.ReverseKeys4(d, v4);
vf = st.ReverseKeys4(d, vf);
ve = st.ReverseKeys4(d, ve);
vd = st.ReverseKeys4(d, vd);
vc = st.ReverseKeys4(d, vc);
st.Sort2(d, v0, v7);
st.Sort2(d, v1, v6);
st.Sort2(d, v2, v5);
st.Sort2(d, v3, v4);
st.Sort2(d, v8, vf);
st.Sort2(d, v9, ve);
st.Sort2(d, va, vd);
st.Sort2(d, vb, vc);
v3 = st.ReverseKeys4(d, v3);
v2 = st.ReverseKeys4(d, v2);
v7 = st.ReverseKeys4(d, v7);
v6 = st.ReverseKeys4(d, v6);
vb = st.ReverseKeys4(d, vb);
va = st.ReverseKeys4(d, va);
vf = st.ReverseKeys4(d, vf);
ve = st.ReverseKeys4(d, ve);
st.Sort2(d, v0, v3);
st.Sort2(d, v1, v2);
st.Sort2(d, v4, v7);
st.Sort2(d, v5, v6);
st.Sort2(d, v8, vb);
st.Sort2(d, v9, va);
st.Sort2(d, vc, vf);
st.Sort2(d, vd, ve);
v1 = st.ReverseKeys4(d, v1);
v3 = st.ReverseKeys4(d, v3);
v5 = st.ReverseKeys4(d, v5);
v7 = st.ReverseKeys4(d, v7);
v9 = st.ReverseKeys4(d, v9);
vb = st.ReverseKeys4(d, vb);
vd = st.ReverseKeys4(d, vd);
vf = st.ReverseKeys4(d, vf);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
st.Sort2(d, v8, v9);
st.Sort2(d, va, vb);
st.Sort2(d, vc, vd);
st.Sort2(d, ve, vf);
v0 = st.SortPairsReverse4(d, v0);
v1 = st.SortPairsReverse4(d, v1);
v2 = st.SortPairsReverse4(d, v2);
v3 = st.SortPairsReverse4(d, v3);
v4 = st.SortPairsReverse4(d, v4);
v5 = st.SortPairsReverse4(d, v5);
v6 = st.SortPairsReverse4(d, v6);
v7 = st.SortPairsReverse4(d, v7);
v8 = st.SortPairsReverse4(d, v8);
v9 = st.SortPairsReverse4(d, v9);
va = st.SortPairsReverse4(d, va);
vb = st.SortPairsReverse4(d, vb);
vc = st.SortPairsReverse4(d, vc);
vd = st.SortPairsReverse4(d, vd);
ve = st.SortPairsReverse4(d, ve);
vf = st.SortPairsReverse4(d, vf);
v0 = st.SortPairsDistance1(d, v0);
v1 = st.SortPairsDistance1(d, v1);
v2 = st.SortPairsDistance1(d, v2);
v3 = st.SortPairsDistance1(d, v3);
v4 = st.SortPairsDistance1(d, v4);
v5 = st.SortPairsDistance1(d, v5);
v6 = st.SortPairsDistance1(d, v6);
v7 = st.SortPairsDistance1(d, v7);
v8 = st.SortPairsDistance1(d, v8);
v9 = st.SortPairsDistance1(d, v9);
va = st.SortPairsDistance1(d, va);
vb = st.SortPairsDistance1(d, vb);
vc = st.SortPairsDistance1(d, vc);
vd = st.SortPairsDistance1(d, vd);
ve = st.SortPairsDistance1(d, ve);
vf = st.SortPairsDistance1(d, vf);
}
template <size_t kKeysPerVector, class D, class Traits, class V = Vec<D>,
HWY_IF_LANES_GT(kKeysPerVector, 4)>
HWY_INLINE void Merge16x8(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4,
V& v5, V& v6, V& v7, V& v8, V& v9, V& va, V& vb,
V& vc, V& vd, V& ve, V& vf) {
vf = st.ReverseKeys8(d, vf);
ve = st.ReverseKeys8(d, ve);
vd = st.ReverseKeys8(d, vd);
vc = st.ReverseKeys8(d, vc);
vb = st.ReverseKeys8(d, vb);
va = st.ReverseKeys8(d, va);
v9 = st.ReverseKeys8(d, v9);
v8 = st.ReverseKeys8(d, v8);
st.Sort2(d, v0, vf);
st.Sort2(d, v1, ve);
st.Sort2(d, v2, vd);
st.Sort2(d, v3, vc);
st.Sort2(d, v4, vb);
st.Sort2(d, v5, va);
st.Sort2(d, v6, v9);
st.Sort2(d, v7, v8);
v7 = st.ReverseKeys8(d, v7);
v6 = st.ReverseKeys8(d, v6);
v5 = st.ReverseKeys8(d, v5);
v4 = st.ReverseKeys8(d, v4);
vf = st.ReverseKeys8(d, vf);
ve = st.ReverseKeys8(d, ve);
vd = st.ReverseKeys8(d, vd);
vc = st.ReverseKeys8(d, vc);
st.Sort2(d, v0, v7);
st.Sort2(d, v1, v6);
st.Sort2(d, v2, v5);
st.Sort2(d, v3, v4);
st.Sort2(d, v8, vf);
st.Sort2(d, v9, ve);
st.Sort2(d, va, vd);
st.Sort2(d, vb, vc);
v3 = st.ReverseKeys8(d, v3);
v2 = st.ReverseKeys8(d, v2);
v7 = st.ReverseKeys8(d, v7);
v6 = st.ReverseKeys8(d, v6);
vb = st.ReverseKeys8(d, vb);
va = st.ReverseKeys8(d, va);
vf = st.ReverseKeys8(d, vf);
ve = st.ReverseKeys8(d, ve);
st.Sort2(d, v0, v3);
st.Sort2(d, v1, v2);
st.Sort2(d, v4, v7);
st.Sort2(d, v5, v6);
st.Sort2(d, v8, vb);
st.Sort2(d, v9, va);
st.Sort2(d, vc, vf);
st.Sort2(d, vd, ve);
v1 = st.ReverseKeys8(d, v1);
v3 = st.ReverseKeys8(d, v3);
v5 = st.ReverseKeys8(d, v5);
v7 = st.ReverseKeys8(d, v7);
v9 = st.ReverseKeys8(d, v9);
vb = st.ReverseKeys8(d, vb);
vd = st.ReverseKeys8(d, vd);
vf = st.ReverseKeys8(d, vf);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
st.Sort2(d, v8, v9);
st.Sort2(d, va, vb);
st.Sort2(d, vc, vd);
st.Sort2(d, ve, vf);
v0 = st.SortPairsReverse8(d, v0);
v1 = st.SortPairsReverse8(d, v1);
v2 = st.SortPairsReverse8(d, v2);
v3 = st.SortPairsReverse8(d, v3);
v4 = st.SortPairsReverse8(d, v4);
v5 = st.SortPairsReverse8(d, v5);
v6 = st.SortPairsReverse8(d, v6);
v7 = st.SortPairsReverse8(d, v7);
v8 = st.SortPairsReverse8(d, v8);
v9 = st.SortPairsReverse8(d, v9);
va = st.SortPairsReverse8(d, va);
vb = st.SortPairsReverse8(d, vb);
vc = st.SortPairsReverse8(d, vc);
vd = st.SortPairsReverse8(d, vd);
ve = st.SortPairsReverse8(d, ve);
vf = st.SortPairsReverse8(d, vf);
v0 = st.SortPairsDistance2(d, v0);
v1 = st.SortPairsDistance2(d, v1);
v2 = st.SortPairsDistance2(d, v2);
v3 = st.SortPairsDistance2(d, v3);
v4 = st.SortPairsDistance2(d, v4);
v5 = st.SortPairsDistance2(d, v5);
v6 = st.SortPairsDistance2(d, v6);
v7 = st.SortPairsDistance2(d, v7);
v8 = st.SortPairsDistance2(d, v8);
v9 = st.SortPairsDistance2(d, v9);
va = st.SortPairsDistance2(d, va);
vb = st.SortPairsDistance2(d, vb);
vc = st.SortPairsDistance2(d, vc);
vd = st.SortPairsDistance2(d, vd);
ve = st.SortPairsDistance2(d, ve);
vf = st.SortPairsDistance2(d, vf);
v0 = st.SortPairsDistance1(d, v0);
v1 = st.SortPairsDistance1(d, v1);
v2 = st.SortPairsDistance1(d, v2);
v3 = st.SortPairsDistance1(d, v3);
v4 = st.SortPairsDistance1(d, v4);
v5 = st.SortPairsDistance1(d, v5);
v6 = st.SortPairsDistance1(d, v6);
v7 = st.SortPairsDistance1(d, v7);
v8 = st.SortPairsDistance1(d, v8);
v9 = st.SortPairsDistance1(d, v9);
va = st.SortPairsDistance1(d, va);
vb = st.SortPairsDistance1(d, vb);
vc = st.SortPairsDistance1(d, vc);
vd = st.SortPairsDistance1(d, vd);
ve = st.SortPairsDistance1(d, ve);
vf = st.SortPairsDistance1(d, vf);
}
// Unused on MSVC, see below
#if !HWY_COMPILER_MSVC && !HWY_IS_DEBUG_BUILD
template <size_t kKeysPerVector, class D, class Traits, class V = Vec<D>,
HWY_IF_LANES_GT(kKeysPerVector, 8)>
HWY_INLINE void Merge16x16(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4,
V& v5, V& v6, V& v7, V& v8, V& v9, V& va, V& vb,
V& vc, V& vd, V& ve, V& vf) {
vf = st.ReverseKeys16(d, vf);
ve = st.ReverseKeys16(d, ve);
vd = st.ReverseKeys16(d, vd);
vc = st.ReverseKeys16(d, vc);
vb = st.ReverseKeys16(d, vb);
va = st.ReverseKeys16(d, va);
v9 = st.ReverseKeys16(d, v9);
v8 = st.ReverseKeys16(d, v8);
st.Sort2(d, v0, vf);
st.Sort2(d, v1, ve);
st.Sort2(d, v2, vd);
st.Sort2(d, v3, vc);
st.Sort2(d, v4, vb);
st.Sort2(d, v5, va);
st.Sort2(d, v6, v9);
st.Sort2(d, v7, v8);
v7 = st.ReverseKeys16(d, v7);
v6 = st.ReverseKeys16(d, v6);
v5 = st.ReverseKeys16(d, v5);
v4 = st.ReverseKeys16(d, v4);
vf = st.ReverseKeys16(d, vf);
ve = st.ReverseKeys16(d, ve);
vd = st.ReverseKeys16(d, vd);
vc = st.ReverseKeys16(d, vc);
st.Sort2(d, v0, v7);
st.Sort2(d, v1, v6);
st.Sort2(d, v2, v5);
st.Sort2(d, v3, v4);
st.Sort2(d, v8, vf);
st.Sort2(d, v9, ve);
st.Sort2(d, va, vd);
st.Sort2(d, vb, vc);
v3 = st.ReverseKeys16(d, v3);
v2 = st.ReverseKeys16(d, v2);
v7 = st.ReverseKeys16(d, v7);
v6 = st.ReverseKeys16(d, v6);
vb = st.ReverseKeys16(d, vb);
va = st.ReverseKeys16(d, va);
vf = st.ReverseKeys16(d, vf);
ve = st.ReverseKeys16(d, ve);
st.Sort2(d, v0, v3);
st.Sort2(d, v1, v2);
st.Sort2(d, v4, v7);
st.Sort2(d, v5, v6);
st.Sort2(d, v8, vb);
st.Sort2(d, v9, va);
st.Sort2(d, vc, vf);
st.Sort2(d, vd, ve);
v1 = st.ReverseKeys16(d, v1);
v3 = st.ReverseKeys16(d, v3);
v5 = st.ReverseKeys16(d, v5);
v7 = st.ReverseKeys16(d, v7);
v9 = st.ReverseKeys16(d, v9);
vb = st.ReverseKeys16(d, vb);
vd = st.ReverseKeys16(d, vd);
vf = st.ReverseKeys16(d, vf);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
st.Sort2(d, v8, v9);
st.Sort2(d, va, vb);
st.Sort2(d, vc, vd);
st.Sort2(d, ve, vf);
v0 = st.SortPairsReverse16(d, v0);
v1 = st.SortPairsReverse16(d, v1);
v2 = st.SortPairsReverse16(d, v2);
v3 = st.SortPairsReverse16(d, v3);
v4 = st.SortPairsReverse16(d, v4);
v5 = st.SortPairsReverse16(d, v5);
v6 = st.SortPairsReverse16(d, v6);
v7 = st.SortPairsReverse16(d, v7);
v8 = st.SortPairsReverse16(d, v8);
v9 = st.SortPairsReverse16(d, v9);
va = st.SortPairsReverse16(d, va);
vb = st.SortPairsReverse16(d, vb);
vc = st.SortPairsReverse16(d, vc);
vd = st.SortPairsReverse16(d, vd);
ve = st.SortPairsReverse16(d, ve);
vf = st.SortPairsReverse16(d, vf);
v0 = st.SortPairsDistance4(d, v0);
v1 = st.SortPairsDistance4(d, v1);
v2 = st.SortPairsDistance4(d, v2);
v3 = st.SortPairsDistance4(d, v3);
v4 = st.SortPairsDistance4(d, v4);
v5 = st.SortPairsDistance4(d, v5);
v6 = st.SortPairsDistance4(d, v6);
v7 = st.SortPairsDistance4(d, v7);
v8 = st.SortPairsDistance4(d, v8);
v9 = st.SortPairsDistance4(d, v9);
va = st.SortPairsDistance4(d, va);
vb = st.SortPairsDistance4(d, vb);
vc = st.SortPairsDistance4(d, vc);
vd = st.SortPairsDistance4(d, vd);
ve = st.SortPairsDistance4(d, ve);
vf = st.SortPairsDistance4(d, vf);
v0 = st.SortPairsDistance2(d, v0);
v1 = st.SortPairsDistance2(d, v1);
v2 = st.SortPairsDistance2(d, v2);
v3 = st.SortPairsDistance2(d, v3);
v4 = st.SortPairsDistance2(d, v4);
v5 = st.SortPairsDistance2(d, v5);
v6 = st.SortPairsDistance2(d, v6);
v7 = st.SortPairsDistance2(d, v7);
v8 = st.SortPairsDistance2(d, v8);
v9 = st.SortPairsDistance2(d, v9);
va = st.SortPairsDistance2(d, va);
vb = st.SortPairsDistance2(d, vb);
vc = st.SortPairsDistance2(d, vc);
vd = st.SortPairsDistance2(d, vd);
ve = st.SortPairsDistance2(d, ve);
vf = st.SortPairsDistance2(d, vf);
v0 = st.SortPairsDistance1(d, v0);
v1 = st.SortPairsDistance1(d, v1);
v2 = st.SortPairsDistance1(d, v2);
v3 = st.SortPairsDistance1(d, v3);
v4 = st.SortPairsDistance1(d, v4);
v5 = st.SortPairsDistance1(d, v5);
v6 = st.SortPairsDistance1(d, v6);
v7 = st.SortPairsDistance1(d, v7);
v8 = st.SortPairsDistance1(d, v8);
v9 = st.SortPairsDistance1(d, v9);
va = st.SortPairsDistance1(d, va);
vb = st.SortPairsDistance1(d, vb);
vc = st.SortPairsDistance1(d, vc);
vd = st.SortPairsDistance1(d, vd);
ve = st.SortPairsDistance1(d, ve);
vf = st.SortPairsDistance1(d, vf);
}
#endif // !HWY_COMPILER_MSVC && !HWY_IS_DEBUG_BUILD
// Reshapes `buf` into a matrix, sorts columns independently, and then merges
// into a sorted 1D array without transposing.
//
// DEPRECATED, use BaseCase() instead.
template <class Traits, class V>
HWY_INLINE void SortingNetwork(Traits st, size_t cols, V& v0, V& v1, V& v2,
V& v3, V& v4, V& v5, V& v6, V& v7, V& v8, V& v9,
V& va, V& vb, V& vc, V& vd, V& ve, V& vf) {
// traits*-inl assume 'full' vectors (but still capped to kMaxCols).
const CappedTag<typename Traits::LaneType, Constants::kMaxCols> d;
HWY_DASSERT(cols <= Constants::kMaxCols);
// The network width depends on the number of keys, not lanes.
constexpr size_t kLanesPerKey = st.LanesPerKey();
const size_t keys = cols / kLanesPerKey;
constexpr size_t kMaxKeys = MaxLanes(d) / kLanesPerKey;
Sort16(d, st, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf);
// Checking MaxLanes avoids generating HWY_ASSERT code for the unreachable
// code paths: if MaxLanes < 2, then keys <= cols < 2.
if (HWY_LIKELY(keys >= 2 && kMaxKeys >= 2)) {
Merge16x2<kMaxKeys>(d, st, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb,
vc, vd, ve, vf);
if (HWY_LIKELY(keys >= 4 && kMaxKeys >= 4)) {
Merge16x4<kMaxKeys>(d, st, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb,
vc, vd, ve, vf);
if (HWY_LIKELY(keys >= 8 && kMaxKeys >= 8)) {
Merge16x8<kMaxKeys>(d, st, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va,
vb, vc, vd, ve, vf);
// Avoids build timeout. Must match #if condition in kMaxCols.
#if !HWY_COMPILER_MSVC && !HWY_IS_DEBUG_BUILD
if (HWY_LIKELY(keys >= 16 && kMaxKeys >= 16)) {
Merge16x16<kMaxKeys>(d, st, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9,
va, vb, vc, vd, ve, vf);
static_assert(Constants::kMaxCols <= 16, "Add more branches");
}
#endif
}
}
}
}
// As above, but loads from/stores to `buf`. This ensures full vectors are
// aligned, and enables loads/stores without bounds checks.
//
// DEPRECATED, use BaseCase() instead.
template <class Traits, typename T>
HWY_NOINLINE void SortingNetwork(Traits st, T* HWY_RESTRICT buf, size_t cols) {
// traits*-inl assume 'full' vectors (but still capped to kMaxCols).
// However, for smaller arrays and sub-maximal `cols` we have overlapping
// loads where only the lowest `cols` are valid, and we skip Merge16 etc.
const CappedTag<T, Constants::kMaxCols> d;
using V = decltype(Zero(d));
HWY_DASSERT(cols <= Constants::kMaxCols);
// These are aligned iff cols == Lanes(d). We prefer unaligned/non-constexpr
// offsets to duplicating this code for every value of cols.
static_assert(Constants::kMaxRows == 16, "Update loads/stores/args");
V v0 = LoadU(d, buf + 0x0 * cols);
V v1 = LoadU(d, buf + 0x1 * cols);
V v2 = LoadU(d, buf + 0x2 * cols);
V v3 = LoadU(d, buf + 0x3 * cols);
V v4 = LoadU(d, buf + 0x4 * cols);
V v5 = LoadU(d, buf + 0x5 * cols);
V v6 = LoadU(d, buf + 0x6 * cols);
V v7 = LoadU(d, buf + 0x7 * cols);
V v8 = LoadU(d, buf + 0x8 * cols);
V v9 = LoadU(d, buf + 0x9 * cols);
V va = LoadU(d, buf + 0xa * cols);
V vb = LoadU(d, buf + 0xb * cols);
V vc = LoadU(d, buf + 0xc * cols);
V vd = LoadU(d, buf + 0xd * cols);
V ve = LoadU(d, buf + 0xe * cols);
V vf = LoadU(d, buf + 0xf * cols);
SortingNetwork(st, cols, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc,
vd, ve, vf);
StoreU(v0, d, buf + 0x0 * cols);
StoreU(v1, d, buf + 0x1 * cols);
StoreU(v2, d, buf + 0x2 * cols);
StoreU(v3, d, buf + 0x3 * cols);
StoreU(v4, d, buf + 0x4 * cols);
StoreU(v5, d, buf + 0x5 * cols);
StoreU(v6, d, buf + 0x6 * cols);
StoreU(v7, d, buf + 0x7 * cols);
StoreU(v8, d, buf + 0x8 * cols);
StoreU(v9, d, buf + 0x9 * cols);
StoreU(va, d, buf + 0xa * cols);
StoreU(vb, d, buf + 0xb * cols);
StoreU(vc, d, buf + 0xc * cols);
StoreU(vd, d, buf + 0xd * cols);
StoreU(ve, d, buf + 0xe * cols);
StoreU(vf, d, buf + 0xf * cols);
}
#else
template <class Base>
struct SharedTraits : public Base {};
#endif // VQSORT_ENABLED
} // namespace detail
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_SORT_SORTING_NETWORKS_TOGGLE

618
deps/v8/third_party/highway/src/hwy/contrib/sort/traits-inl.h vendored Normal file
View File

@ -0,0 +1,618 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Per-target
#if defined(HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE
#undef HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE
#else
#define HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE
#endif
#include <stddef.h>
#include <stdint.h>
#include "hwy/contrib/sort/order.h" // SortDescending
#include "hwy/contrib/sort/shared-inl.h" // SortConstants
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace detail {
// Base class of both KeyLane variants
template <typename LaneTypeArg, typename KeyTypeArg>
struct KeyLaneBase {
static constexpr bool Is128() { return false; }
constexpr size_t LanesPerKey() const { return 1; }
// What type bench_sort should allocate for generating inputs.
using LaneType = LaneTypeArg;
// What type to pass to VQSort.
using KeyType = KeyTypeArg;
const char* KeyString() const {
return IsSame<KeyTypeArg, float16_t>() ? "f16"
: IsSame<KeyTypeArg, float>() ? "f32"
: IsSame<KeyTypeArg, double>() ? "f64"
: IsSame<KeyTypeArg, int16_t>() ? "i16"
: IsSame<KeyTypeArg, int32_t>() ? "i32"
: IsSame<KeyTypeArg, int64_t>() ? "i64"
: IsSame<KeyTypeArg, uint16_t>() ? "u32"
: IsSame<KeyTypeArg, uint32_t>() ? "u32"
: IsSame<KeyTypeArg, uint64_t>() ? "u64"
: IsSame<KeyTypeArg, hwy::K32V32>() ? "k+v=64"
: "?";
}
};
// Wrapper functions so we can specialize for floats - infinity trumps
// HighestValue (the normal value with the largest magnitude). Must be outside
// Order* classes to enable SFINAE.
template <class D, HWY_IF_FLOAT_OR_SPECIAL_D(D)>
Vec<D> LargestSortValue(D d) {
return Inf(d);
}
template <class D, HWY_IF_NOT_FLOAT_NOR_SPECIAL_D(D)>
Vec<D> LargestSortValue(D d) {
return Set(d, hwy::HighestValue<TFromD<D>>());
}
template <class D, HWY_IF_FLOAT_OR_SPECIAL_D(D)>
Vec<D> SmallestSortValue(D d) {
return Neg(Inf(d));
}
template <class D, HWY_IF_NOT_FLOAT_NOR_SPECIAL_D(D)>
Vec<D> SmallestSortValue(D d) {
return Set(d, hwy::LowestValue<TFromD<D>>());
}
// Returns the next distinct larger value unless already +inf.
template <class D, HWY_IF_FLOAT_OR_SPECIAL_D(D)>
Vec<D> LargerSortValue(D d, Vec<D> v) {
HWY_DASSERT(AllFalse(d, IsNaN(v))); // we replaced all NaN with LastValue.
using T = TFromD<decltype(d)>;
const RebindToUnsigned<D> du;
using VU = Vec<decltype(du)>;
using TU = TFromD<decltype(du)>;
const VU vu = BitCast(du, Abs(v));
// The direction depends on the original sign. Integer comparison is cheaper
// than float comparison and treats -0 as 0 (so we return +epsilon).
const Mask<decltype(du)> was_pos = Le(BitCast(du, v), SignBit(du));
// If positive, add 1, else -1.
const VU add = IfThenElse(was_pos, Set(du, 1u), Set(du, LimitsMax<TU>()));
// Prev/next integer is the prev/next value, even if mantissa under/overflows.
v = BitCast(d, Add(vu, add));
// But we may have overflowed into inf or NaN; replace with inf if positive,
// but the largest (later negated!) value if the input was -inf.
const Mask<D> was_pos_f = RebindMask(d, was_pos);
v = IfThenElse(IsFinite(v), v,
IfThenElse(was_pos_f, Inf(d), Set(d, HighestValue<T>())));
// Restore the original sign - not via CopySignToAbs because we used a mask.
return IfThenElse(was_pos_f, v, Neg(v));
}
// Returns the next distinct smaller value unless already -inf.
template <class D, HWY_IF_FLOAT_OR_SPECIAL_D(D)>
Vec<D> SmallerSortValue(D d, Vec<D> v) {
HWY_DASSERT(AllFalse(d, IsNaN(v))); // we replaced all NaN with LastValue.
using T = TFromD<decltype(d)>;
const RebindToUnsigned<D> du;
using VU = Vec<decltype(du)>;
using TU = TFromD<decltype(du)>;
const VU vu = BitCast(du, Abs(v));
// The direction depends on the original sign. Float comparison because we
// want to treat 0 as -0 so we return -epsilon.
const Mask<D> was_pos = Gt(v, Zero(d));
// If positive, add -1, else 1.
const VU add =
IfThenElse(RebindMask(du, was_pos), Set(du, LimitsMax<TU>()), Set(du, 1));
// Prev/next integer is the prev/next value, even if mantissa under/overflows.
v = BitCast(d, Add(vu, add));
// But we may have overflowed into inf or NaN; replace with +inf (which will
// later be negated) if negative, but the largest value if the input was +inf.
v = IfThenElse(IsFinite(v), v,
IfThenElse(was_pos, Set(d, HighestValue<T>()), Inf(d)));
// Restore the original sign - not via CopySignToAbs because we used a mask.
return IfThenElse(was_pos, v, Neg(v));
}
template <class D, HWY_IF_NOT_FLOAT_NOR_SPECIAL_D(D)>
Vec<D> LargerSortValue(D d, Vec<D> v) {
return Add(v, Set(d, TFromD<D>{1}));
}
template <class D, HWY_IF_NOT_FLOAT_NOR_SPECIAL_D(D)>
Vec<D> SmallerSortValue(D d, Vec<D> v) {
return Sub(v, Set(d, TFromD<D>{1}));
}
// Highway does not provide a lane type for 128-bit keys, so we use uint64_t
// along with an abstraction layer for single-lane vs. lane-pair, which is
// independent of the order.
template <typename LaneType, typename KeyType>
struct KeyLane : public KeyLaneBase<LaneType, KeyType> {
// For HeapSort
HWY_INLINE void Swap(LaneType* a, LaneType* b) const {
const LaneType temp = *a;
*a = *b;
*b = temp;
}
template <class V, class M>
HWY_INLINE V CompressKeys(V keys, M mask) const {
return CompressNot(keys, mask);
}
// Broadcasts one key into a vector
template <class D>
HWY_INLINE Vec<D> SetKey(D d, const LaneType* key) const {
return Set(d, *key);
}
template <class D>
HWY_INLINE Mask<D> EqualKeys(D /*tag*/, Vec<D> a, Vec<D> b) const {
return Eq(a, b);
}
template <class D>
HWY_INLINE Mask<D> NotEqualKeys(D /*tag*/, Vec<D> a, Vec<D> b) const {
return Ne(a, b);
}
// For keys=lanes, any difference counts.
template <class D>
HWY_INLINE bool NoKeyDifference(D /*tag*/, Vec<D> diff) const {
// Must avoid floating-point comparisons (for -0)
const RebindToUnsigned<D> du;
return AllTrue(du, Eq(BitCast(du, diff), Zero(du)));
}
HWY_INLINE bool Equal1(const LaneType* a, const LaneType* b) const {
return *a == *b;
}
template <class D>
HWY_INLINE Vec<D> ReverseKeys(D d, Vec<D> v) const {
return Reverse(d, v);
}
template <class D>
HWY_INLINE Vec<D> ReverseKeys2(D d, Vec<D> v) const {
return Reverse2(d, v);
}
template <class D>
HWY_INLINE Vec<D> ReverseKeys4(D d, Vec<D> v) const {
return Reverse4(d, v);
}
template <class D>
HWY_INLINE Vec<D> ReverseKeys8(D d, Vec<D> v) const {
return Reverse8(d, v);
}
template <class D>
HWY_INLINE Vec<D> ReverseKeys16(D d, Vec<D> v) const {
static_assert(SortConstants::kMaxCols <= 16, "Assumes u32x16 = 512 bit");
return ReverseKeys(d, v);
}
template <class V>
HWY_INLINE V OddEvenKeys(const V odd, const V even) const {
return OddEven(odd, even);
}
template <class D, HWY_IF_T_SIZE_D(D, 2)>
HWY_INLINE Vec<D> SwapAdjacentPairs(D d, const Vec<D> v) const {
const Repartition<uint32_t, D> du32;
return BitCast(d, Shuffle2301(BitCast(du32, v)));
}
template <class D, HWY_IF_T_SIZE_D(D, 4)>
HWY_INLINE Vec<D> SwapAdjacentPairs(D /* tag */, const Vec<D> v) const {
return Shuffle1032(v);
}
template <class D, HWY_IF_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> SwapAdjacentPairs(D /* tag */, const Vec<D> v) const {
return SwapAdjacentBlocks(v);
}
template <class D, HWY_IF_NOT_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> SwapAdjacentQuads(D d, const Vec<D> v) const {
#if HWY_HAVE_FLOAT64 // in case D is float32
const RepartitionToWide<D> dw;
#else
const RepartitionToWide<RebindToUnsigned<D>> dw;
#endif
return BitCast(d, SwapAdjacentPairs(dw, BitCast(dw, v)));
}
template <class D, HWY_IF_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> SwapAdjacentQuads(D d, const Vec<D> v) const {
// Assumes max vector size = 512
return ConcatLowerUpper(d, v, v);
}
template <class D, HWY_IF_NOT_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> OddEvenPairs(D d, const Vec<D> odd,
const Vec<D> even) const {
#if HWY_HAVE_FLOAT64 // in case D is float32
const RepartitionToWide<D> dw;
#else
const RepartitionToWide<RebindToUnsigned<D>> dw;
#endif
return BitCast(d, OddEven(BitCast(dw, odd), BitCast(dw, even)));
}
template <class D, HWY_IF_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> OddEvenPairs(D /* tag */, Vec<D> odd, Vec<D> even) const {
return OddEvenBlocks(odd, even);
}
template <class D, HWY_IF_NOT_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> OddEvenQuads(D d, Vec<D> odd, Vec<D> even) const {
#if HWY_HAVE_FLOAT64 // in case D is float32
const RepartitionToWide<D> dw;
#else
const RepartitionToWide<RebindToUnsigned<D>> dw;
#endif
return BitCast(d, OddEvenPairs(dw, BitCast(dw, odd), BitCast(dw, even)));
}
template <class D, HWY_IF_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> OddEvenQuads(D d, Vec<D> odd, Vec<D> even) const {
return ConcatUpperLower(d, odd, even);
}
};
// Anything order-related depends on the key traits *and* the order (see
// FirstOfLanes). We cannot implement just one Compare function because Lt128
// only compiles if the lane type is u64. Thus we need either overloaded
// functions with a tag type, class specializations, or separate classes.
// We avoid overloaded functions because we want all functions to be callable
// from a SortTraits without per-function wrappers. Specializing would work, but
// we are anyway going to specialize at a higher level.
template <typename T>
struct OrderAscending : public KeyLane<T, T> {
// False indicates the entire key (i.e. lane) should be compared. KV stands
// for key-value.
static constexpr bool IsKV() { return false; }
using Order = SortAscending;
using OrderForSortingNetwork = OrderAscending<T>;
HWY_INLINE bool Compare1(const T* a, const T* b) const { return *a < *b; }
template <class D>
HWY_INLINE Mask<D> Compare(D /* tag */, Vec<D> a, Vec<D> b) const {
return Lt(a, b);
}
// Two halves of Sort2, used in ScanMinMax.
template <class D>
HWY_INLINE Vec<D> First(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Min(a, b);
}
template <class D>
HWY_INLINE Vec<D> Last(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Max(a, b);
}
template <class D>
HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
T* HWY_RESTRICT /* buf */) const {
return MinOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
T* HWY_RESTRICT /* buf */) const {
return MaxOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> FirstValue(D d) const {
return SmallestSortValue(d);
}
template <class D>
HWY_INLINE Vec<D> LastValue(D d) const {
return LargestSortValue(d);
}
template <class D>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
return SmallerSortValue(d, v);
}
};
template <typename T>
struct OrderDescending : public KeyLane<T, T> {
// False indicates the entire key (i.e. lane) should be compared. KV stands
// for key-value.
static constexpr bool IsKV() { return false; }
using Order = SortDescending;
using OrderForSortingNetwork = OrderDescending<T>;
HWY_INLINE bool Compare1(const T* a, const T* b) const { return *b < *a; }
template <class D>
HWY_INLINE Mask<D> Compare(D /* tag */, Vec<D> a, Vec<D> b) const {
return Lt(b, a);
}
template <class D>
HWY_INLINE Vec<D> First(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Max(a, b);
}
template <class D>
HWY_INLINE Vec<D> Last(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Min(a, b);
}
template <class D>
HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
T* HWY_RESTRICT /* buf */) const {
return MaxOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
T* HWY_RESTRICT /* buf */) const {
return MinOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> FirstValue(D d) const {
return LargestSortValue(d);
}
template <class D>
HWY_INLINE Vec<D> LastValue(D d) const {
return SmallestSortValue(d);
}
template <class D>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
return LargerSortValue(d, v);
}
};
struct KeyValue64 : public KeyLane<uint64_t, hwy::K32V32> {
// True indicates only part of the key (i.e. lane) should be compared. KV
// stands for key-value.
static constexpr bool IsKV() { return true; }
template <class D>
HWY_INLINE Mask<D> EqualKeys(D /*tag*/, Vec<D> a, Vec<D> b) const {
return Eq(ShiftRight<32>(a), ShiftRight<32>(b));
}
template <class D>
HWY_INLINE Mask<D> NotEqualKeys(D /*tag*/, Vec<D> a, Vec<D> b) const {
return Ne(ShiftRight<32>(a), ShiftRight<32>(b));
}
HWY_INLINE bool Equal1(const uint64_t* a, const uint64_t* b) const {
return (*a >> 32) == (*b >> 32);
}
// Only count differences in the actual key, not the value.
template <class D>
HWY_INLINE bool NoKeyDifference(D /*tag*/, Vec<D> diff) const {
// Must avoid floating-point comparisons (for -0)
const RebindToUnsigned<D> du;
const Vec<decltype(du)> zero = Zero(du);
const Vec<decltype(du)> keys = ShiftRight<32>(diff); // clear values
return AllTrue(du, Eq(BitCast(du, keys), zero));
}
};
struct OrderAscendingKV64 : public KeyValue64 {
using Order = SortAscending;
using OrderForSortingNetwork = OrderAscending<LaneType>;
HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) const {
return (*a >> 32) < (*b >> 32);
}
template <class D>
HWY_INLINE Mask<D> Compare(D /* tag */, Vec<D> a, Vec<D> b) const {
return Lt(ShiftRight<32>(a), ShiftRight<32>(b));
}
// Not required to be stable (preserving the order of equivalent keys), so
// we can include the value in the comparison.
template <class D>
HWY_INLINE Vec<D> First(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Min(a, b);
}
template <class D>
HWY_INLINE Vec<D> Last(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Max(a, b);
}
template <class D>
HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
uint64_t* HWY_RESTRICT /* buf */) const {
return MinOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
uint64_t* HWY_RESTRICT /* buf */) const {
return MaxOfLanes(d, v);
}
// Same as for regular lanes.
template <class D>
HWY_INLINE Vec<D> FirstValue(D d) const {
return Set(d, hwy::LowestValue<TFromD<D>>());
}
template <class D>
HWY_INLINE Vec<D> LastValue(D d) const {
return Set(d, hwy::HighestValue<TFromD<D>>());
}
template <class D>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
return Sub(v, Set(d, uint64_t{1} << 32));
}
};
struct OrderDescendingKV64 : public KeyValue64 {
using Order = SortDescending;
using OrderForSortingNetwork = OrderDescending<LaneType>;
HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) const {
return (*b >> 32) < (*a >> 32);
}
template <class D>
HWY_INLINE Mask<D> Compare(D /* tag */, Vec<D> a, Vec<D> b) const {
return Lt(ShiftRight<32>(b), ShiftRight<32>(a));
}
// Not required to be stable (preserving the order of equivalent keys), so
// we can include the value in the comparison.
template <class D>
HWY_INLINE Vec<D> First(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Max(a, b);
}
template <class D>
HWY_INLINE Vec<D> Last(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Min(a, b);
}
template <class D>
HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
uint64_t* HWY_RESTRICT /* buf */) const {
return MaxOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
uint64_t* HWY_RESTRICT /* buf */) const {
return MinOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> FirstValue(D d) const {
return Set(d, hwy::HighestValue<TFromD<D>>());
}
template <class D>
HWY_INLINE Vec<D> LastValue(D d) const {
return Set(d, hwy::LowestValue<TFromD<D>>());
}
template <class D>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
return Add(v, Set(d, uint64_t{1} << 32));
}
};
// Shared code that depends on Order.
template <class Base>
struct TraitsLane : public Base {
using TraitsForSortingNetwork =
TraitsLane<typename Base::OrderForSortingNetwork>;
// For each lane i: replaces a[i] with the first and b[i] with the second
// according to Base.
// Corresponds to a conditional swap, which is one "node" of a sorting
// network. Min/Max are cheaper than compare + blend at least for integers.
template <class D>
HWY_INLINE void Sort2(D d, Vec<D>& a, Vec<D>& b) const {
const Base* base = static_cast<const Base*>(this);
const Vec<D> a_copy = a;
// Prior to AVX3, there is no native 64-bit Min/Max, so they compile to 4
// instructions. We can reduce it to a compare + 2 IfThenElse.
#if HWY_AVX3 < HWY_TARGET && HWY_TARGET <= HWY_SSSE3
if (sizeof(TFromD<D>) == 8) {
const Mask<D> cmp = base->Compare(d, a, b);
a = IfThenElse(cmp, a, b);
b = IfThenElse(cmp, b, a_copy);
return;
}
#endif
a = base->First(d, a, b);
b = base->Last(d, a_copy, b);
}
// Conditionally swaps even-numbered lanes with their odd-numbered neighbor.
template <class D, HWY_IF_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> SortPairsDistance1(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->ReverseKeys2(d, v);
// Further to the above optimization, Sort2+OddEvenKeys compile to four
// instructions; we can save one by combining two blends.
#if HWY_AVX3 < HWY_TARGET && HWY_TARGET <= HWY_SSSE3
const Vec<D> cmp = VecFromMask(d, base->Compare(d, v, swapped));
return IfVecThenElse(DupOdd(cmp), swapped, v);
#else
Sort2(d, v, swapped);
return base->OddEvenKeys(swapped, v);
#endif
}
// (See above - we use Sort2 for non-64-bit types.)
template <class D, HWY_IF_NOT_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> SortPairsDistance1(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->ReverseKeys2(d, v);
Sort2(d, v, swapped);
return base->OddEvenKeys(swapped, v);
}
// Swaps with the vector formed by reversing contiguous groups of 4 keys.
template <class D>
HWY_INLINE Vec<D> SortPairsReverse4(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->ReverseKeys4(d, v);
Sort2(d, v, swapped);
return base->OddEvenPairs(d, swapped, v);
}
// Conditionally swaps lane 0 with 4, 1 with 5 etc.
template <class D>
HWY_INLINE Vec<D> SortPairsDistance4(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->SwapAdjacentQuads(d, v);
// Only used in Merge16, so this will not be used on AVX2 (which only has 4
// u64 lanes), so skip the above optimization for 64-bit AVX2.
Sort2(d, v, swapped);
return base->OddEvenQuads(d, swapped, v);
}
};
} // namespace detail
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE

549
deps/v8/third_party/highway/src/hwy/contrib/sort/traits128-inl.h vendored Normal file
View File

@ -0,0 +1,549 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Per-target
#if defined(HIGHWAY_HWY_CONTRIB_SORT_TRAITS128_TOGGLE) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_SORT_TRAITS128_TOGGLE
#undef HIGHWAY_HWY_CONTRIB_SORT_TRAITS128_TOGGLE
#else
#define HIGHWAY_HWY_CONTRIB_SORT_TRAITS128_TOGGLE
#endif
#include <stddef.h>
#include <stdint.h>
#include "hwy/contrib/sort/order.h" // SortDescending
#include "hwy/contrib/sort/shared-inl.h"
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace detail {
// Also used by HeapSort, so do not require VQSORT_ENABLED.
#if HWY_TARGET != HWY_SCALAR || HWY_IDE
// Highway does not provide a lane type for 128-bit keys, so we use uint64_t
// along with an abstraction layer for single-lane vs. lane-pair, which is
// independent of the order.
struct KeyAny128 {
static constexpr bool Is128() { return true; }
constexpr size_t LanesPerKey() const { return 2; }
// What type bench_sort should allocate for generating inputs.
using LaneType = uint64_t;
// KeyType and KeyString are defined by derived classes.
HWY_INLINE void Swap(LaneType* a, LaneType* b) const {
const FixedTag<LaneType, 2> d;
const auto temp = LoadU(d, a);
StoreU(LoadU(d, b), d, a);
StoreU(temp, d, b);
}
template <class V, class M>
HWY_INLINE V CompressKeys(V keys, M mask) const {
return CompressBlocksNot(keys, mask);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> SetKey(D d, const TFromD<D>* key) const {
return LoadDup128(d, key);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> ReverseKeys(D d, Vec<D> v) const {
return ReverseBlocks(d, v);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> ReverseKeys2(D /* tag */, const Vec<D> v) const {
HWY_DASSERT(Lanes(D()) >= 4); // at least 2 keys
return SwapAdjacentBlocks(v);
}
// Only called for 4 keys because we do not support >512-bit vectors.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> ReverseKeys4(D d, const Vec<D> v) const {
HWY_DASSERT(Lanes(D()) == 8); // exactly 4 keys: the 512-bit limit
return ReverseKeys(d, v);
}
// Only called for 4 keys because we do not support >512-bit vectors.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> OddEvenPairs(D d, const Vec<D> odd,
const Vec<D> even) const {
HWY_DASSERT(Lanes(D()) == 8); // exactly 4 keys: the 512-bit limit
return ConcatUpperLower(d, odd, even);
}
template <class V>
HWY_INLINE V OddEvenKeys(const V odd, const V even) const {
return OddEvenBlocks(odd, even);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> ReverseKeys8(D, Vec<D>) const {
HWY_ASSERT(0); // not supported: would require 1024-bit vectors
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> ReverseKeys16(D, Vec<D>) const {
HWY_ASSERT(0); // not supported: would require 2048-bit vectors
}
// This is only called for 8/16 col networks (not supported).
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> SwapAdjacentPairs(D, Vec<D>) const {
HWY_ASSERT(0);
}
// This is only called for 16 col networks (not supported).
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> SwapAdjacentQuads(D, Vec<D>) const {
HWY_ASSERT(0);
}
// This is only called for 8 col networks (not supported).
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> OddEvenQuads(D, Vec<D>, Vec<D>) const {
HWY_ASSERT(0);
}
};
// Base class shared between OrderAscending128, OrderDescending128.
struct Key128 : public KeyAny128 {
// False indicates the entire key should be compared. KV means key-value.
static constexpr bool IsKV() { return false; }
// What type to pass to VQSort.
using KeyType = hwy::uint128_t;
const char* KeyString() const { return "U128"; }
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> EqualKeys(D d, Vec<D> a, Vec<D> b) const {
return Eq128(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> NotEqualKeys(D d, Vec<D> a, Vec<D> b) const {
return Ne128(d, a, b);
}
// For keys=entire 128 bits, any difference counts.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE bool NoKeyDifference(D /*tag*/, Vec<D> diff) const {
// Must avoid floating-point comparisons (for -0)
const RebindToUnsigned<D> du;
return AllTrue(du, Eq(BitCast(du, diff), Zero(du)));
}
HWY_INLINE bool Equal1(const LaneType* a, const LaneType* b) const {
return a[0] == b[0] && a[1] == b[1];
}
// Returns vector with only the top half of each block valid. This allows
// fusing the "replicate upper to lower half" step with a subsequent permute.
template <class Order, class D>
HWY_INLINE HWY_MAYBE_UNUSED Vec<D> CompareTop(D d, Vec<D> a, Vec<D> b) const {
const Mask<D> eqHL = Eq(a, b);
const Vec<D> ltHL = VecFromMask(d, Order().CompareLanes(a, b));
#if HWY_TARGET <= HWY_AVX2 // slightly faster
const Vec<D> ltLX = ShiftLeftLanes<1>(ltHL);
return OrAnd(ltHL, VecFromMask(d, eqHL), ltLX);
#else
return IfThenElse(eqHL, DupEven(ltHL), ltHL);
#endif
}
};
// Anything order-related depends on the key traits *and* the order (see
// FirstOfLanes). We cannot implement just one Compare function because Lt128
// only compiles if the lane type is u64. Thus we need either overloaded
// functions with a tag type, class specializations, or separate classes.
// We avoid overloaded functions because we want all functions to be callable
// from a SortTraits without per-function wrappers. Specializing would work, but
// we are anyway going to specialize at a higher level.
struct OrderAscending128 : public Key128 {
using Order = SortAscending;
using OrderForSortingNetwork = OrderAscending128;
HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) const {
return (a[1] == b[1]) ? a[0] < b[0] : a[1] < b[1];
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> Compare(D d, Vec<D> a, Vec<D> b) const {
return Lt128(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> First(D d, const Vec<D> a, const Vec<D> b) const {
return Min128(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> Last(D d, const Vec<D> a, const Vec<D> b) const {
return Max128(d, a, b);
}
// FirstOfLanes/LastOfLanes are implemented in Traits128.
// Same as for regular lanes because 128-bit keys are u64.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> FirstValue(D d) const {
return Set(d, hwy::LowestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> LastValue(D d) const {
return Set(d, hwy::HighestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
const Vec<D> k0 = Zero(d);
const Vec<D> k1 = OddEven(k0, Set(d, uint64_t{1}));
const Mask<D> borrow = Eq(v, k0); // don't-care, lo == 0
// lo == 0? 1 : 0, 0
const Vec<D> adjust = ShiftLeftLanes<1>(IfThenElseZero(borrow, k1));
return Sub(Sub(v, k1), adjust);
}
// 'Private', used by base class Key128::CompareTop.
template <class V>
HWY_INLINE Mask<DFromV<V> > CompareLanes(V a, V b) const {
return Lt(a, b);
}
};
struct OrderDescending128 : public Key128 {
using Order = SortDescending;
using OrderForSortingNetwork = OrderDescending128;
HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) const {
return (a[1] == b[1]) ? b[0] < a[0] : b[1] < a[1];
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> Compare(D d, Vec<D> a, Vec<D> b) const {
return Lt128(d, b, a);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> First(D d, const Vec<D> a, const Vec<D> b) const {
return Max128(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> Last(D d, const Vec<D> a, const Vec<D> b) const {
return Min128(d, a, b);
}
// FirstOfLanes/LastOfLanes are implemented in Traits128.
// Same as for regular lanes because 128-bit keys are u64.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> FirstValue(D d) const {
return Set(d, hwy::HighestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> LastValue(D d) const {
return Set(d, hwy::LowestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
const Vec<D> k1 = OddEven(Zero(d), Set(d, uint64_t{1}));
const Vec<D> added = Add(v, k1);
const Mask<D> overflowed = Lt(added, v); // false, overflowed
// overflowed? 1 : 0, 0
const Vec<D> adjust = ShiftLeftLanes<1>(IfThenElseZero(overflowed, k1));
return Add(added, adjust);
}
// 'Private', used by base class Key128::CompareTop.
template <class V>
HWY_INLINE Mask<DFromV<V> > CompareLanes(V a, V b) const {
return Lt(b, a);
}
};
// Base class shared between OrderAscendingKV128, OrderDescendingKV128.
struct KeyValue128 : public KeyAny128 {
// True indicates only part of the key (the more significant lane) should be
// compared. KV stands for key-value.
static constexpr bool IsKV() { return true; }
// What type to pass to VQSort.
using KeyType = K64V64;
const char* KeyString() const { return "k+v=128"; }
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> EqualKeys(D d, Vec<D> a, Vec<D> b) const {
return Eq128Upper(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> NotEqualKeys(D d, Vec<D> a, Vec<D> b) const {
return Ne128Upper(d, a, b);
}
HWY_INLINE bool Equal1(const LaneType* a, const LaneType* b) const {
return a[1] == b[1];
}
// Only count differences in the actual key, not the value.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE bool NoKeyDifference(D /*tag*/, Vec<D> diff) const {
// Must avoid floating-point comparisons (for -0)
const RebindToUnsigned<D> du;
const Vec<decltype(du)> zero = Zero(du);
const Vec<decltype(du)> keys = OddEven(diff, zero); // clear values
return AllTrue(du, Eq(BitCast(du, keys), zero));
}
// Returns vector with only the top half of each block valid. This allows
// fusing the "replicate upper to lower half" step with a subsequent permute.
template <class Order, class D>
HWY_INLINE HWY_MAYBE_UNUSED Vec<D> CompareTop(D d, Vec<D> a, Vec<D> b) const {
// Only the upper lane of each block is a key, and only that lane is
// required to be valid, so comparing all lanes is sufficient.
return VecFromMask(d, Order().CompareLanes(a, b));
}
};
struct OrderAscendingKV128 : public KeyValue128 {
using Order = SortAscending;
using OrderForSortingNetwork = OrderAscending128;
HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) const {
return a[1] < b[1];
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> Compare(D d, Vec<D> a, Vec<D> b) const {
return Lt128Upper(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> First(D d, const Vec<D> a, const Vec<D> b) const {
return Min128Upper(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> Last(D d, const Vec<D> a, const Vec<D> b) const {
return Max128Upper(d, a, b);
}
// FirstOfLanes/LastOfLanes are implemented in Traits128.
// Same as for regular lanes because 128-bit keys are u64.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> FirstValue(D d) const {
return Set(d, hwy::LowestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> LastValue(D d) const {
return Set(d, hwy::HighestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
const Vec<D> k1 = OddEven(Set(d, uint64_t{1}), Zero(d));
return Sub(v, k1);
}
// 'Private', used by base class KeyValue128::CompareTop.
template <class V>
HWY_INLINE Mask<DFromV<V> > CompareLanes(V a, V b) const {
return Lt(a, b);
}
};
struct OrderDescendingKV128 : public KeyValue128 {
using Order = SortDescending;
using OrderForSortingNetwork = OrderDescending128;
HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) const {
return b[1] < a[1];
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> Compare(D d, Vec<D> a, Vec<D> b) const {
return Lt128Upper(d, b, a);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> First(D d, const Vec<D> a, const Vec<D> b) const {
return Max128Upper(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> Last(D d, const Vec<D> a, const Vec<D> b) const {
return Min128Upper(d, a, b);
}
// FirstOfLanes/LastOfLanes are implemented in Traits128.
// Same as for regular lanes because 128-bit keys are u64.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> FirstValue(D d) const {
return Set(d, hwy::HighestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> LastValue(D d) const {
return Set(d, hwy::LowestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
const Vec<D> k1 = OddEven(Set(d, uint64_t{1}), Zero(d));
return Add(v, k1);
}
// 'Private', used by base class KeyValue128::CompareTop.
template <class V>
HWY_INLINE Mask<DFromV<V> > CompareLanes(V a, V b) const {
return Lt(b, a);
}
};
// We want to swap 2 u128, i.e. 4 u64 lanes, based on the 0 or FF..FF mask in
// the most-significant of those lanes (the result of CompareTop), so
// replicate it 4x. Only called for >= 256-bit vectors.
#if HWY_TARGET <= HWY_AVX3
template <class V, HWY_IF_V_SIZE_V(V, 64)>
HWY_INLINE V ReplicateTop4x(V v) {
return V{_mm512_permutex_epi64(v.raw, _MM_SHUFFLE(3, 3, 3, 3))};
}
#endif // HWY_TARGET <= HWY_AVX3
#if HWY_TARGET <= HWY_AVX2
template <class V, HWY_IF_V_SIZE_V(V, 32)>
HWY_INLINE V ReplicateTop4x(V v) {
return V{_mm256_permute4x64_epi64(v.raw, _MM_SHUFFLE(3, 3, 3, 3))};
}
#else // HWY_TARGET > HWY_AVX2
template <class V>
HWY_INLINE V ReplicateTop4x(V v) {
#if HWY_TARGET == HWY_SVE_256
return svdup_lane_u64(v, 3);
#else
const ScalableTag<uint64_t> d;
HWY_DASSERT(Lanes(d) == 4 || Lanes(d) == 8); // for table below
HWY_ALIGN static constexpr uint64_t kIndices[8] = {3, 3, 3, 3, 7, 7, 7, 7};
return TableLookupLanes(v, SetTableIndices(d, kIndices));
#endif
}
#endif // HWY_TARGET <= HWY_AVX2
// Shared code that depends on Order.
template <class Base>
struct Traits128 : public Base {
using TraitsForSortingNetwork =
Traits128<typename Base::OrderForSortingNetwork>;
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
TFromD<D>* HWY_RESTRICT buf) const {
const Base* base = static_cast<const Base*>(this);
const size_t N = Lanes(d);
Store(v, d, buf);
v = base->SetKey(d, buf + 0); // result must be broadcasted
for (size_t i = base->LanesPerKey(); i < N; i += base->LanesPerKey()) {
v = base->First(d, v, base->SetKey(d, buf + i));
}
return v;
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
TFromD<D>* HWY_RESTRICT buf) const {
const Base* base = static_cast<const Base*>(this);
const size_t N = Lanes(d);
Store(v, d, buf);
v = base->SetKey(d, buf + 0); // result must be broadcasted
for (size_t i = base->LanesPerKey(); i < N; i += base->LanesPerKey()) {
v = base->Last(d, v, base->SetKey(d, buf + i));
}
return v;
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE void Sort2(D d, Vec<D>& a, Vec<D>& b) const {
const Base* base = static_cast<const Base*>(this);
const Vec<D> a_copy = a;
const auto lt = base->Compare(d, a, b);
a = IfThenElse(lt, a, b);
b = IfThenElse(lt, b, a_copy);
}
// Conditionally swaps even-numbered keys with their odd-numbered neighbor.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> SortPairsDistance1(D d, Vec<D> v) const {
HWY_DASSERT(Lanes(d) >= 4); // required by ReplicateTop4x
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->ReverseKeys2(d, v);
const Vec<D> cmpHx = base->template CompareTop<Base>(d, v, swapped);
return IfVecThenElse(ReplicateTop4x(cmpHx), swapped, v);
}
// Swaps with the vector formed by reversing contiguous groups of four 128-bit
// keys, which implies 512-bit vectors (we do not support more than that).
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> SortPairsReverse4(D d, Vec<D> v) const {
HWY_DASSERT(Lanes(d) == 8); // For TableLookupLanes below
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->ReverseKeys4(d, v);
const Vec<D> cmpHx = base->template CompareTop<Base>(d, v, swapped);
// Similar to ReplicateTop4x, we want to gang together 2 comparison results
// (4 lanes). They are not contiguous, so use permute to replicate 4x.
HWY_ALIGN uint64_t kIndices[8] = {7, 7, 5, 5, 5, 5, 7, 7};
const Vec<D> select = TableLookupLanes(cmpHx, SetTableIndices(d, kIndices));
return IfVecThenElse(select, swapped, v);
}
// Conditionally swaps lane 0 with 4, 1 with 5 etc.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> SortPairsDistance4(D, Vec<D>) const {
// Only used by Merge16, which would require 2048 bit vectors (unsupported).
HWY_ASSERT(0);
}
};
#endif // HWY_TARGET != HWY_SCALAR
} // namespace detail
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_SORT_TRAITS128_TOGGLE

2243
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort-inl.h vendored Normal file
View File

File diff suppressed because it is too large Load Diff

217
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort.cc vendored Normal file
View File

@ -0,0 +1,217 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h"
#include "hwy/base.h"
#include "hwy/contrib/sort/vqsort-inl.h"
#include "hwy/per_target.h"
// Check if we have getrandom from <sys/random.h>. Because <features.h> is
// unavailable on Android and non-Linux RVV, we assume that those systems lack
// getrandom. Note that the only supported sources of entropy are getrandom or
// Windows, thus VQSORT_SECURE_SEED=0 when this is 0 and we are not on Windows.
#if defined(ANDROID) || defined(__ANDROID__) || \
(HWY_ARCH_RISCV && !HWY_OS_LINUX)
#define VQSORT_GETRANDOM 0
#endif
#if !defined(VQSORT_GETRANDOM) && HWY_OS_LINUX
#include <features.h>
// ---- which libc
#if defined(__UCLIBC__)
#define VQSORT_GETRANDOM 1 // added Mar 2015, before uclibc-ng 1.0
#elif defined(__GLIBC__) && defined(__GLIBC_PREREQ)
#if __GLIBC_PREREQ(2, 25)
#define VQSORT_GETRANDOM 1
#else
#define VQSORT_GETRANDOM 0
#endif
#else
// Assume MUSL, which has getrandom since 2018. There is no macro to test, see
// https://www.openwall.com/lists/musl/2013/03/29/13.
#define VQSORT_GETRANDOM 1
#endif // ---- which libc
#endif // linux
#if !defined(VQSORT_GETRANDOM)
#define VQSORT_GETRANDOM 0
#endif
// Choose a seed source for SFC generator: 1=getrandom, 2=CryptGenRandom.
// Allow user override - not all Android support the getrandom wrapper.
#ifndef VQSORT_SECURE_SEED
#if VQSORT_GETRANDOM
#define VQSORT_SECURE_SEED 1
#elif defined(_WIN32) || defined(_WIN64)
#define VQSORT_SECURE_SEED 2
#else
#define VQSORT_SECURE_SEED 0
#endif
#endif // VQSORT_SECURE_SEED
// Pull in dependencies of the chosen seed source.
#if VQSORT_SECURE_SEED == 1
#include <sys/random.h>
#elif VQSORT_SECURE_SEED == 2
#include <windows.h>
#if HWY_COMPILER_MSVC || HWY_COMPILER_CLANGCL
#pragma comment(lib, "advapi32.lib")
#endif // HWY_COMPILER_MSVC || HWY_COMPILER_CLANGCL
// Must come after windows.h.
#include <wincrypt.h>
#endif // VQSORT_SECURE_SEED
namespace hwy {
// Returns false or performs the equivalent of `memcpy(bytes, r, 16)`, where r
// is high-quality (unpredictable, uniformly distributed) random bits.
bool Fill16BytesSecure(void* bytes) {
#if VQSORT_SECURE_SEED == 1
// May block if urandom is not yet initialized.
const ssize_t ret = getrandom(bytes, 16, /*flags=*/0);
if (ret == 16) return true;
#elif VQSORT_SECURE_SEED == 2
HCRYPTPROV hProvider{};
if (CryptAcquireContextA(&hProvider, nullptr, nullptr, PROV_RSA_FULL,
CRYPT_VERIFYCONTEXT)) {
const BOOL ok =
CryptGenRandom(hProvider, 16, reinterpret_cast<BYTE*>(bytes));
CryptReleaseContext(hProvider, 0);
if (ok) return true;
}
#else
(void)bytes;
#endif
return false;
}
void Sorter::operator()(uint16_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint16_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint32_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint32_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint64_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint64_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(int16_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(int16_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(int32_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(int32_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(int64_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(int64_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(float16_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(float16_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(float* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(float* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(double* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(double* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint128_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint128_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(K64V64* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(K64V64* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(K32V32* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(K32V32* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
// Unused, only for ABI compatibility
void Sorter::Fill24Bytes(const void*, size_t, void*) {}
bool Sorter::HaveFloat64() { return hwy::HaveFloat64(); }
Sorter::Sorter() {}
void Sorter::Delete() {}
uint64_t* GetGeneratorState() { return hwy::detail::GetGeneratorStateStatic(); }
} // namespace hwy

303
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort.h vendored Normal file
View File

@ -0,0 +1,303 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Interface to vectorized quicksort with dynamic dispatch. For static dispatch
// without any DLLEXPORT, avoid including this header and instead define
// VQSORT_ONLY_STATIC, then call VQSortStatic* in vqsort-inl.h.
//
// Blog post: https://tinyurl.com/vqsort-blog
// Paper with measurements: https://arxiv.org/abs/2205.05982
//
// To ensure the overhead of using wide vectors (e.g. AVX2 or AVX-512) is
// worthwhile, we recommend using this code for sorting arrays whose size is at
// least 100 KiB. See the README for details.
#ifndef HIGHWAY_HWY_CONTRIB_SORT_VQSORT_H_
#define HIGHWAY_HWY_CONTRIB_SORT_VQSORT_H_
// IWYU pragma: begin_exports
#include <stddef.h>
#include "hwy/base.h"
#include "hwy/contrib/sort/order.h" // SortAscending
// IWYU pragma: end_exports
namespace hwy {
// Vectorized Quicksort: sorts keys[0, n). Does not preserve the ordering of
// equivalent keys (defined as: neither greater nor less than another).
// Dispatches to the best available instruction set. Does not allocate memory.
// Uses about 1.2 KiB stack plus an internal 3-word TLS cache for random state.
HWY_CONTRIB_DLLEXPORT void VQSort(uint16_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(uint16_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(uint32_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(uint32_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(uint64_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(uint64_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(int16_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(int16_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(int32_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(int32_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(int64_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(int64_t* HWY_RESTRICT keys, size_t n,
SortDescending);
// These two must only be called if hwy::HaveFloat16() is true.
HWY_CONTRIB_DLLEXPORT void VQSort(float16_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(float16_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(float* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(float* HWY_RESTRICT keys, size_t n,
SortDescending);
// These two must only be called if hwy::HaveFloat64() is true.
HWY_CONTRIB_DLLEXPORT void VQSort(double* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(double* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(K32V32* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(K32V32* HWY_RESTRICT keys, size_t n,
SortDescending);
// 128-bit types: `n` is still in units of the 128-bit keys.
HWY_CONTRIB_DLLEXPORT void VQSort(uint128_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(uint128_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(K64V64* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(K64V64* HWY_RESTRICT keys, size_t n,
SortDescending);
// Vectorized partial Quicksort:
// Rearranges elements such that the range [0, k) contains the sorted first k
// elements in the range [0, n). Does not preserve the ordering of equivalent
// keys (defined as: neither greater nor less than another).
// Dispatches to the best available instruction set. Does not allocate memory.
// Uses about 1.2 KiB stack plus an internal 3-word TLS cache for random state.
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(int16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(int16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(int32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(int32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(int64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(int64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// These two must only be called if hwy::HaveFloat16() is true.
HWY_CONTRIB_DLLEXPORT void VQPartialSort(float16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(float16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(float* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(float* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// These two must only be called if hwy::HaveFloat64() is true.
HWY_CONTRIB_DLLEXPORT void VQPartialSort(double* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(double* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(K32V32* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(K32V32* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// 128-bit types: `n` and `k` are still in units of the 128-bit keys.
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint128_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint128_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(K64V64* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(K64V64* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// Vectorized Quickselect:
// rearranges elements in [0, n) such that:
// The element pointed at by kth is changed to whatever element would occur in
// that position if [0, n) were sorted. All of the elements before this new kth
// element are less than or equal to the elements after the new kth element.
HWY_CONTRIB_DLLEXPORT void VQSelect(uint16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(uint16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(uint32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(uint32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(uint64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(uint64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(int16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(int16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(int32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(int32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(int64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(int64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// These two must only be called if hwy::HaveFloat16() is true.
HWY_CONTRIB_DLLEXPORT void VQSelect(float16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(float16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(float* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(float* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// These two must only be called if hwy::HaveFloat64() is true.
HWY_CONTRIB_DLLEXPORT void VQSelect(double* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(double* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(K32V32* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(K32V32* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// 128-bit types: `n` and `k` are still in units of the 128-bit keys.
HWY_CONTRIB_DLLEXPORT void VQSelect(uint128_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(uint128_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(K64V64* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(K64V64* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// User-level caching is no longer required, so this class is no longer
// beneficial. We recommend using the simpler VQSort() interface instead, and
// retain this class only for compatibility. It now just calls VQSort.
class HWY_CONTRIB_DLLEXPORT Sorter {
public:
Sorter();
~Sorter() { Delete(); }
// Move-only
Sorter(const Sorter&) = delete;
Sorter& operator=(const Sorter&) = delete;
Sorter(Sorter&& /*other*/) {}
Sorter& operator=(Sorter&& /*other*/) { return *this; }
void operator()(uint16_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(uint16_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(uint32_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(uint32_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(uint64_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(uint64_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(int16_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(int16_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(int32_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(int32_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(int64_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(int64_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
// These two must only be called if hwy::HaveFloat16() is true.
void operator()(float16_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(float16_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(float* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(float* HWY_RESTRICT keys, size_t n, SortDescending) const;
// These two must only be called if hwy::HaveFloat64() is true.
void operator()(double* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(double* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(uint128_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(uint128_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(K64V64* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(K64V64* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(K32V32* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(K32V32* HWY_RESTRICT keys, size_t n, SortDescending) const;
// Unused
static void Fill24Bytes(const void*, size_t, void*);
static bool HaveFloat64(); // Can also use hwy::HaveFloat64 directly.
private:
void Delete();
template <typename T>
T* Get() const {
return unused_;
}
#if HWY_COMPILER_CLANG
HWY_DIAGNOSTICS(push)
HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wunused-private-field")
#endif
void* unused_ = nullptr;
#if HWY_COMPILER_CLANG
HWY_DIAGNOSTICS(pop)
#endif
};
// Used by vqsort-inl.h unless VQSORT_ONLY_STATIC.
HWY_CONTRIB_DLLEXPORT bool Fill16BytesSecure(void* bytes);
// Unused, only provided for binary compatibility.
HWY_CONTRIB_DLLEXPORT uint64_t* GetGeneratorState();
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_SORT_VQSORT_H_

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_128a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_128a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void Sort128Asc(uint128_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSort128Asc(uint128_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void Select128Asc(uint128_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(Sort128Asc);
HWY_EXPORT(PartialSort128Asc);
HWY_EXPORT(Select128Asc);
} // namespace
void VQSort(uint128_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(Sort128Asc)(keys, n);
}
void VQPartialSort(uint128_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSort128Asc)(keys, n, k);
}
void VQSelect(uint128_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(Select128Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_128d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_128d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void Sort128Desc(uint128_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSort128Desc(uint128_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void Select128Desc(uint128_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(Sort128Desc);
HWY_EXPORT(PartialSort128Desc);
HWY_EXPORT(Select128Desc);
} // namespace
void VQSort(uint128_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(Sort128Desc)(keys, n);
}
void VQPartialSort(uint128_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSort128Desc)(keys, n, k);
}
void VQSelect(uint128_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(Select128Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

91
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_f16a.cc vendored Normal file
View File

@ -0,0 +1,91 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_f16a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortF16Asc(float16_t* HWY_RESTRICT keys, const size_t num) {
#if HWY_HAVE_FLOAT16
return VQSortStatic(keys, num, SortAscending());
#else
(void)keys;
(void)num;
HWY_ASSERT(0);
#endif
}
void PartialSortF16Asc(float16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT16
return VQPartialSortStatic(keys, num, k, SortAscending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
void SelectF16Asc(float16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT16
return VQSelectStatic(keys, num, k, SortAscending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortF16Asc);
HWY_EXPORT(PartialSortF16Asc);
HWY_EXPORT(SelectF16Asc);
} // namespace
void VQSort(float16_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortF16Asc)(keys, n);
}
void VQPartialSort(float16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortF16Asc)(keys, n, k);
}
void VQSelect(float16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectF16Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

91
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_f16d.cc vendored Normal file
View File

@ -0,0 +1,91 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_f16d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortF16Desc(float16_t* HWY_RESTRICT keys, const size_t num) {
#if HWY_HAVE_FLOAT16
return VQSortStatic(keys, num, SortDescending());
#else
(void)keys;
(void)num;
HWY_ASSERT(0);
#endif
}
void PartialSortF16Desc(float16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT16
return VQPartialSortStatic(keys, num, k, SortDescending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
void SelectF16Desc(float16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT16
return VQSelectStatic(keys, num, k, SortDescending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortF16Desc);
HWY_EXPORT(PartialSortF16Desc);
HWY_EXPORT(SelectF16Desc);
} // namespace
void VQSort(float16_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortF16Desc)(keys, n);
}
void VQPartialSort(float16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortF16Desc)(keys, n, k);
}
void VQSelect(float16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectF16Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

70
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_f32a.cc vendored Normal file
View File

@ -0,0 +1,70 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_f32a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortF32Asc(float* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortF32Asc(float* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectF32Asc(float* HWY_RESTRICT keys, const size_t num, const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortF32Asc);
HWY_EXPORT(PartialSortF32Asc);
HWY_EXPORT(SelectF32Asc);
} // namespace
void VQSort(float* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortF32Asc)(keys, n);
}
void VQPartialSort(float* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortF32Asc)(keys, n, k);
}
void VQSelect(float* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectF32Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

70
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_f32d.cc vendored Normal file
View File

@ -0,0 +1,70 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_f32d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortF32Desc(float* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortF32Desc(float* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectF32Desc(float* HWY_RESTRICT keys, const size_t num, const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortF32Desc);
HWY_EXPORT(PartialSortF32Desc);
HWY_EXPORT(SelectF32Desc);
} // namespace
void VQSort(float* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortF32Desc)(keys, n);
}
void VQPartialSort(float* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortF32Desc)(keys, n, k);
}
void VQSelect(float* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectF32Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

90
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_f64a.cc vendored Normal file
View File

@ -0,0 +1,90 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_f64a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortF64Asc(double* HWY_RESTRICT keys, const size_t num) {
#if HWY_HAVE_FLOAT64
return VQSortStatic(keys, num, SortAscending());
#else
(void)keys;
(void)num;
HWY_ASSERT(0);
#endif
}
void PartialSortF64Asc(double* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT64
return VQPartialSortStatic(keys, num, k, SortAscending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
void SelectF64Asc(double* HWY_RESTRICT keys, const size_t num, const size_t k) {
#if HWY_HAVE_FLOAT64
return VQSelectStatic(keys, num, k, SortAscending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortF64Asc);
HWY_EXPORT(PartialSortF64Asc);
HWY_EXPORT(SelectF64Asc);
} // namespace
void VQSort(double* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortF64Asc)(keys, n);
}
void VQPartialSort(double* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortF64Asc)(keys, n, k);
}
void VQSelect(double* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectF64Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

91
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_f64d.cc vendored Normal file
View File

@ -0,0 +1,91 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_f64d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortF64Desc(double* HWY_RESTRICT keys, const size_t num) {
#if HWY_HAVE_FLOAT64
return VQSortStatic(keys, num, SortDescending());
#else
(void)keys;
(void)num;
HWY_ASSERT(0);
#endif
}
void PartialSortF64Desc(double* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT64
return VQPartialSortStatic(keys, num, k, SortDescending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
void SelectF64Desc(double* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT64
return VQSelectStatic(keys, num, k, SortDescending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortF64Desc);
HWY_EXPORT(PartialSortF64Desc);
HWY_EXPORT(SelectF64Desc);
} // namespace
void VQSort(double* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortF64Desc)(keys, n);
}
void VQPartialSort(double* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortF64Desc)(keys, n, k);
}
void VQSelect(double* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectF64Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_i16a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_i16a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortI16Asc(int16_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortI16Asc(int16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectI16Asc(int16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortI16Asc);
HWY_EXPORT(PartialSortI16Asc);
HWY_EXPORT(SelectI16Asc);
} // namespace
void VQSort(int16_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortI16Asc)(keys, n);
}
void VQPartialSort(int16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortI16Asc)(keys, n, k);
}
void VQSelect(int16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectI16Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_i16d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_i16d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortI16Desc(int16_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortI16Desc(int16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectI16Desc(int16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortI16Desc);
HWY_EXPORT(PartialSortI16Desc);
HWY_EXPORT(SelectI16Desc);
} // namespace
void VQSort(int16_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortI16Desc)(keys, n);
}
void VQPartialSort(int16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortI16Desc)(keys, n, k);
}
void VQSelect(int16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectI16Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_i32a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_i32a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortI32Asc(int32_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortI32Asc(int32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectI32Asc(int32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortI32Asc);
HWY_EXPORT(PartialSortI32Asc);
HWY_EXPORT(SelectI32Asc);
} // namespace
void VQSort(int32_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortI32Asc)(keys, n);
}
void VQPartialSort(int32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortI32Asc)(keys, n, k);
}
void VQSelect(int32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectI32Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_i32d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_i32d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortI32Desc(int32_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortI32Desc(int32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectI32Desc(int32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortI32Desc);
HWY_EXPORT(PartialSortI32Desc);
HWY_EXPORT(SelectI32Desc);
} // namespace
void VQSort(int32_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortI32Desc)(keys, n);
}
void VQPartialSort(int32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortI32Desc)(keys, n, k);
}
void VQSelect(int32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectI32Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_i64a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_i64a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortI64Asc(int64_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortI64Asc(int64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectI64Asc(int64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortI64Asc);
HWY_EXPORT(PartialSortI64Asc);
HWY_EXPORT(SelectI64Asc);
} // namespace
void VQSort(int64_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortI64Asc)(keys, n);
}
void VQPartialSort(int64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortI64Asc)(keys, n, k);
}
void VQSelect(int64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectI64Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_i64d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_i64d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortI64Desc(int64_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortI64Desc(int64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectI64Desc(int64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortI64Desc);
HWY_EXPORT(PartialSortI64Desc);
HWY_EXPORT(SelectI64Desc);
} // namespace
void VQSort(int64_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortI64Desc)(keys, n);
}
void VQPartialSort(int64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortI64Desc)(keys, n, k);
}
void VQSelect(int64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectI64Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

74
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_kv128a.cc vendored Normal file
View File

@ -0,0 +1,74 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
// clang-format off
// (avoid line break, which would prevent Copybara rules from matching)
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_kv128a.cc" //NOLINT
// clang-format on
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortKV128Asc(K64V64* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortKV128Asc(K64V64* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectKV128Asc(K64V64* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortKV128Asc);
HWY_EXPORT(PartialSortKV128Asc);
HWY_EXPORT(SelectKV128Asc);
} // namespace
void VQSort(K64V64* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortKV128Asc)(keys, n);
}
void VQPartialSort(K64V64* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortKV128Asc)(keys, n, k);
}
void VQSelect(K64V64* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectKV128Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

74
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_kv128d.cc vendored Normal file
View File

@ -0,0 +1,74 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
// clang-format off
// (avoid line break, which would prevent Copybara rules from matching)
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_kv128d.cc" //NOLINT
// clang-format on
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortKV128Desc(K64V64* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortKV128Desc(K64V64* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectKV128Desc(K64V64* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortKV128Desc);
HWY_EXPORT(PartialSortKV128Desc);
HWY_EXPORT(SelectKV128Desc);
} // namespace
void VQSort(K64V64* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortKV128Desc)(keys, n);
}
void VQPartialSort(K64V64* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortKV128Desc)(keys, n, k);
}
void VQSelect(K64V64* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectKV128Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

74
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_kv64a.cc vendored Normal file
View File

@ -0,0 +1,74 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
// clang-format off
// (avoid line break, which would prevent Copybara rules from matching)
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_kv64a.cc" //NOLINT
// clang-format on
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortKV64Asc(K32V32* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortKV64Asc(K32V32* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectKV64Asc(K32V32* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortKV64Asc);
HWY_EXPORT(PartialSortKV64Asc);
HWY_EXPORT(SelectKV64Asc);
} // namespace
void VQSort(K32V32* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortKV64Asc)(keys, n);
}
void VQPartialSort(K32V32* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortKV64Asc)(keys, n, k);
}
void VQSelect(K32V32* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectKV64Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

74
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_kv64d.cc vendored Normal file
View File

@ -0,0 +1,74 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
// clang-format off
// (avoid line break, which would prevent Copybara rules from matching)
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_kv64d.cc" //NOLINT
// clang-format on
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortKV64Desc(K32V32* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortKV64Desc(K32V32* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectKV64Desc(K32V32* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortKV64Desc);
HWY_EXPORT(PartialSortKV64Desc);
HWY_EXPORT(SelectKV64Desc);
} // namespace
void VQSort(K32V32* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortKV64Desc)(keys, n);
}
void VQPartialSort(K32V32* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortKV64Desc)(keys, n, k);
}
void VQSelect(K32V32* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectKV64Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_u16a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_u16a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortU16Asc(uint16_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortU16Asc(uint16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectU16Asc(uint16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortU16Asc);
HWY_EXPORT(PartialSortU16Asc);
HWY_EXPORT(SelectU16Asc);
} // namespace
void VQSort(uint16_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortU16Asc)(keys, n);
}
void VQPartialSort(uint16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortU16Asc)(keys, n, k);
}
void VQSelect(uint16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectU16Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_u16d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_u16d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortU16Desc(uint16_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortU16Desc(uint16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectU16Desc(uint16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortU16Desc);
HWY_EXPORT(PartialSortU16Desc);
HWY_EXPORT(SelectU16Desc);
} // namespace
void VQSort(uint16_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortU16Desc)(keys, n);
}
void VQPartialSort(uint16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortU16Desc)(keys, n, k);
}
void VQSelect(uint16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectU16Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_u32a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_u32a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortU32Asc(uint32_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortU32Asc(uint32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectU32Asc(uint32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortU32Asc);
HWY_EXPORT(PartialSortU32Asc);
HWY_EXPORT(SelectU32Asc);
} // namespace
void VQSort(uint32_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortU32Asc)(keys, n);
}
void VQPartialSort(uint32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortU32Asc)(keys, n, k);
}
void VQSelect(uint32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectU32Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_u32d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_u32d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortU32Desc(uint32_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortU32Desc(uint32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectU32Desc(uint32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortU32Desc);
HWY_EXPORT(PartialSortU32Desc);
HWY_EXPORT(SelectU32Desc);
} // namespace
void VQSort(uint32_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortU32Desc)(keys, n);
}
void VQPartialSort(uint32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortU32Desc)(keys, n, k);
}
void VQSelect(uint32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectU32Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_u64a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_u64a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortU64Asc(uint64_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortU64Asc(uint64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectU64Asc(uint64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortU64Asc);
HWY_EXPORT(PartialSortU64Asc);
HWY_EXPORT(SelectU64Asc);
} // namespace
void VQSort(uint64_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortU64Asc)(keys, n);
}
void VQPartialSort(uint64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortU64Asc)(keys, n, k);
}
void VQSelect(uint64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectU64Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_u64d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_u64d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortU64Desc(uint64_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortU64Desc(uint64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectU64Desc(uint64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortU64Desc);
HWY_EXPORT(PartialSortU64Desc);
HWY_EXPORT(SelectU64Desc);
} // namespace
void VQSort(uint64_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortU64Desc)(keys, n);
}
void VQPartialSort(uint64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortU64Desc)(keys, n, k);
}
void VQSelect(uint64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectU64Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

199
deps/v8/third_party/highway/src/hwy/contrib/thread_pool/futex.h vendored Normal file
View File

@ -0,0 +1,199 @@
// Copyright 2024 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_CONTRIB_THREAD_POOL_FUTEX_H_
#define HIGHWAY_HWY_CONTRIB_THREAD_POOL_FUTEX_H_
// Keyed event (futex): kernel queue of blocked threads, identified by the
// address of an atomic u32 called `current` within the same process (do NOT
// use with shared-memory mappings).
//
// Futex equivalents: https://outerproduct.net/futex-dictionary.html; we
// support Linux/Emscripten/Apple/Windows and C++20 std::atomic::wait, plus a
// usleep fallback.
#include <atomic>
#include <climits> // INT_MAX
#include "hwy/base.h"
#if HWY_ARCH_WASM
#include <emscripten/threading.h>
#include <math.h> // INFINITY
#elif HWY_OS_LINUX
#include <errno.h> // IWYU pragma: keep
#include <linux/futex.h> // FUTEX_*
#include <pthread.h>
#include <sys/syscall.h> // SYS_*
#include <unistd.h>
// Android may not declare these:
#ifndef SYS_futex
#ifdef SYS_futex_time64 // 32-bit with 64-bit time_t
#define SYS_futex SYS_futex_time64
#else
#define SYS_futex __NR_futex
#endif // SYS_futex_time64
#endif // SYS_futex
#ifndef FUTEX_WAIT_PRIVATE
#define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | 128)
#endif
#ifndef FUTEX_WAKE_PRIVATE
#define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | 128)
#endif
#elif HWY_OS_APPLE && !defined(HWY_DISABLE_FUTEX)
// These are private APIs, so add an opt-out.
extern "C" {
int __ulock_wait(uint32_t op, void* address, uint64_t val, uint32_t max_us);
int __ulock_wake(uint32_t op, void* address, uint64_t zero);
} // extern "C"
#define UL_COMPARE_AND_WAIT 1
#define ULF_WAKE_ALL 0x00000100
#elif HWY_OS_WIN && !defined(HWY_DISABLE_FUTEX)
// WakeByAddressAll requires Windows 8, so add an opt-out.
#include <windows.h>
#pragma comment(lib, "synchronization.lib")
#elif HWY_CXX_LANG < 202002L // NOT C++20, which has native support
#define HWY_FUTEX_SLEEP
#include <chrono> // NOLINT (sleep_for)
#endif
namespace hwy {
// Waits until `current != prev` and returns the new value. May return
// immediately if `current` already changed, or after blocking and waking.
static inline uint32_t BlockUntilDifferent(
const uint32_t prev, const std::atomic<uint32_t>& current) {
const auto acq = std::memory_order_acquire;
#if HWY_ARCH_WASM
// It is always safe to cast to void.
volatile void* address =
const_cast<volatile void*>(static_cast<const volatile void*>(&current));
const double max_ms = INFINITY;
for (;;) {
const uint32_t next = current.load(acq);
if (next != prev) return next;
const int ret = emscripten_futex_wait(address, prev, max_ms);
HWY_DASSERT(ret >= 0);
(void)ret;
}
#elif HWY_OS_LINUX
// Safe to cast because std::atomic is a standard layout type.
const uint32_t* address = reinterpret_cast<const uint32_t*>(&current);
// _PRIVATE requires this only be used in the same process, and avoids
// virtual->physical lookups and atomic reference counting.
const int op = FUTEX_WAIT_PRIVATE;
for (;;) {
const uint32_t next = current.load(acq);
if (next != prev) return next;
// timeout=null may prevent interrupts via signal. No lvalue because
// the timespec type is only standardized since C++17 or C11.
const auto ret = syscall(SYS_futex, address, op, prev, nullptr, nullptr, 0);
if (ret == -1) {
HWY_DASSERT(errno == EAGAIN); // otherwise an actual error
}
}
#elif HWY_OS_WIN && !defined(HWY_DISABLE_FUTEX)
// It is always safe to cast to void.
volatile void* address =
const_cast<volatile void*>(static_cast<const volatile void*>(&current));
// API is not const-correct, but only loads from the pointer.
PVOID pprev = const_cast<void*>(static_cast<const void*>(&prev));
const DWORD max_ms = INFINITE;
for (;;) {
const uint32_t next = current.load(acq);
if (next != prev) return next;
const BOOL ok = WaitOnAddress(address, pprev, sizeof(prev), max_ms);
HWY_DASSERT(ok);
(void)ok;
}
#elif HWY_OS_APPLE && !defined(HWY_DISABLE_FUTEX)
// It is always safe to cast to void.
void* address = const_cast<void*>(static_cast<const void*>(&current));
for (;;) {
const uint32_t next = current.load(acq);
if (next != prev) return next;
__ulock_wait(UL_COMPARE_AND_WAIT, address, prev, 0);
}
#elif defined(HWY_FUTEX_SLEEP)
for (;;) {
const uint32_t next = current.load(acq);
if (next != prev) return next;
std::this_thread::sleep_for(std::chrono::microseconds(2));
}
#elif HWY_CXX_LANG >= 202002L
current.wait(prev, acq); // No spurious wakeup.
const uint32_t next = current.load(acq);
HWY_DASSERT(next != prev);
return next;
#else
#error "Logic error, should have reached HWY_FUTEX_SLEEP"
#endif // HWY_OS_*
} // BlockUntilDifferent
// Wakes all threads, if any, that are waiting because they called
// `BlockUntilDifferent` with the same `current`.
static inline void WakeAll(std::atomic<uint32_t>& current) {
#if HWY_ARCH_WASM
// It is always safe to cast to void.
volatile void* address = static_cast<volatile void*>(&current);
const int max_to_wake = INT_MAX; // actually signed
const int ret = emscripten_futex_wake(address, max_to_wake);
HWY_DASSERT(ret >= 0);
(void)ret;
#elif HWY_OS_LINUX
// Safe to cast because std::atomic is a standard layout type.
uint32_t* address = reinterpret_cast<uint32_t*>(&current);
const int max_to_wake = INT_MAX; // actually signed
const auto ret = syscall(SYS_futex, address, FUTEX_WAKE_PRIVATE, max_to_wake,
nullptr, nullptr, 0);
HWY_DASSERT(ret >= 0); // number woken
(void)ret;
#elif HWY_OS_WIN && !defined(HWY_DISABLE_FUTEX)
// It is always safe to cast to void.
void* address = static_cast<void*>(&current);
WakeByAddressAll(address);
#elif HWY_OS_APPLE && !defined(HWY_DISABLE_FUTEX)
// It is always safe to cast to void.
void* address = static_cast<void*>(&current);
__ulock_wake(UL_COMPARE_AND_WAIT | ULF_WAKE_ALL, address, 0);
#elif defined(HWY_FUTEX_SLEEP)
// Sleep loop does not require wakeup.
(void)current;
#elif HWY_CXX_LANG >= 202002L
current.notify_all();
#else
#error "Logic error, should have reached HWY_FUTEX_SLEEP"
#endif
} // WakeAll
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_THREAD_POOL_FUTEX_H_

693
deps/v8/third_party/highway/src/hwy/contrib/thread_pool/thread_pool.h vendored Normal file
View File

@ -0,0 +1,693 @@
// Copyright 2023 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified from BSD-licensed code
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
// See https://github.com/libjxl/libjxl/blob/main/LICENSE.
#ifndef HIGHWAY_HWY_CONTRIB_THREAD_POOL_THREAD_POOL_H_
#define HIGHWAY_HWY_CONTRIB_THREAD_POOL_THREAD_POOL_H_
// IWYU pragma: begin_exports
#include <stddef.h>
#include <stdint.h>
#include <stdio.h> // snprintf
#include <array>
#include <new>
#include <thread> //NOLINT
// IWYU pragma: end_exports
#include <atomic>
#include <vector>
#include "hwy/aligned_allocator.h" // HWY_ALIGNMENT
#include "hwy/base.h"
#include "hwy/cache_control.h" // Pause
#include "hwy/contrib/thread_pool/futex.h"
#include "hwy/contrib/thread_pool/topology.h"
// Temporary NOINLINE for profiling.
#define HWY_POOL_INLINE HWY_NOINLINE
#ifndef HWY_POOL_SETRANGE_INLINE
#if HWY_ARCH_ARM
// Workaround for invalid codegen on Arm (begin_ is larger than expected).
#define HWY_POOL_SETRANGE_INLINE HWY_NOINLINE
#else
#define HWY_POOL_SETRANGE_INLINE
#endif
#endif // HWY_POOL_SETRANGE_INLINE
namespace hwy {
// Generates a random permutation of [0, size). O(1) storage.
class ShuffledIota {
public:
ShuffledIota() : coprime_(1) {} // for PoolWorker
explicit ShuffledIota(uint32_t coprime) : coprime_(coprime) {}
// Returns the next after `current`, using an LCG-like generator.
uint32_t Next(uint32_t current, const Divisor& divisor) const {
HWY_DASSERT(current < divisor.GetDivisor());
// (coprime * i + current) % size, see https://lemire.me/blog/2017/09/18/.
return divisor.Remainder(current + coprime_);
}
// Returns true if a and b have no common denominator except 1. Based on
// binary GCD. Assumes a and b are nonzero. Also used in tests.
static bool CoprimeNonzero(uint32_t a, uint32_t b) {
const size_t trailing_a = Num0BitsBelowLS1Bit_Nonzero32(a);
const size_t trailing_b = Num0BitsBelowLS1Bit_Nonzero32(b);
// If both have at least one trailing zero, they are both divisible by 2.
if (HWY_MIN(trailing_a, trailing_b) != 0) return false;
// If one of them has a trailing zero, shift it out.
a >>= trailing_a;
b >>= trailing_b;
for (;;) {
// Swap such that a >= b.
const uint32_t tmp_a = a;
a = HWY_MAX(tmp_a, b);
b = HWY_MIN(tmp_a, b);
// When the smaller number is 1, they were coprime.
if (b == 1) return true;
a -= b;
// a == b means there was a common factor, so not coprime.
if (a == 0) return false;
a >>= Num0BitsBelowLS1Bit_Nonzero32(a);
}
}
// Returns another coprime >= `start`, or 1 for small `size`.
// Used to seed independent ShuffledIota instances.
static uint32_t FindAnotherCoprime(uint32_t size, uint32_t start) {
if (size <= 2) {
return 1;
}
// Avoids even x for even sizes, which are sure to be rejected.
const uint32_t inc = (size & 1) ? 1 : 2;
for (uint32_t x = start | 1; x < start + size * 16; x += inc) {
if (CoprimeNonzero(x, static_cast<uint32_t>(size))) {
return x;
}
}
HWY_ABORT("unreachable");
}
uint32_t coprime_;
};
// We want predictable struct/class sizes so we can reason about cache lines.
#pragma pack(push, 1)
enum class PoolWaitMode : uint32_t { kBlock, kSpin };
// Worker's private working set.
class PoolWorker { // HWY_ALIGNMENT bytes
static constexpr size_t kMaxVictims = 4;
public:
PoolWorker(size_t thread, size_t num_workers) {
wait_mode_ = PoolWaitMode::kBlock;
num_victims_ = static_cast<uint32_t>(HWY_MIN(kMaxVictims, num_workers));
const Divisor div_workers(static_cast<uint32_t>(num_workers));
// Increase gap between coprimes to reduce collisions.
const uint32_t coprime = ShuffledIota::FindAnotherCoprime(
static_cast<uint32_t>(num_workers),
static_cast<uint32_t>((thread + 1) * 257 + thread * 13));
const ShuffledIota shuffled_iota(coprime);
// To simplify WorkerRun, our own thread is the first to 'steal' from.
victims_[0] = static_cast<uint32_t>(thread);
for (uint32_t i = 1; i < num_victims_; ++i) {
victims_[i] = shuffled_iota.Next(victims_[i - 1], div_workers);
HWY_DASSERT(victims_[i] != thread);
}
(void)padding_;
}
~PoolWorker() = default;
void SetWaitMode(PoolWaitMode wait_mode) {
wait_mode_.store(wait_mode, std::memory_order_release);
}
PoolWaitMode WorkerGetWaitMode() const {
return wait_mode_.load(std::memory_order_acquire);
}
hwy::Span<const uint32_t> Victims() const {
return hwy::Span<const uint32_t>(victims_.data(),
static_cast<size_t>(num_victims_));
}
// Called from main thread in Plan().
HWY_POOL_SETRANGE_INLINE void SetRange(uint64_t begin, uint64_t end) {
const auto rel = std::memory_order_release;
begin_.store(begin, rel);
end_.store(end, rel);
}
// Returns the STL-style end of this worker's assigned range.
uint64_t WorkerGetEnd() const { return end_.load(std::memory_order_acquire); }
// Returns the next task to execute. If >= WorkerGetEnd(), it must be skipped.
uint64_t WorkerReserveTask() {
return begin_.fetch_add(1, std::memory_order_relaxed);
}
private:
std::atomic<uint64_t> begin_;
std::atomic<uint64_t> end_; // only changes during SetRange
std::atomic<PoolWaitMode> wait_mode_; // (32-bit)
uint32_t num_victims_; // <= kPoolMaxVictims
std::array<uint32_t, kMaxVictims> victims_;
uint8_t padding_[HWY_ALIGNMENT - 16 - 8 - sizeof(victims_)];
};
static_assert(sizeof(PoolWorker) == HWY_ALIGNMENT, "");
// Modified by main thread, shared with all workers.
class PoolTasks { // 32 bytes
// Signature of the (internal) function called from workers(s) for each
// `task` in the [`begin`, `end`) passed to Run(). Closures (lambdas) do not
// receive the first argument, which points to the lambda object.
typedef void (*RunFunc)(const void* opaque, uint64_t task, size_t thread_id);
// Calls closure(task, thread). Signature must match RunFunc.
template <class Closure>
static void CallClosure(const void* opaque, uint64_t task, size_t thread) {
(*reinterpret_cast<const Closure*>(opaque))(task, thread);
}
public:
// Called from main thread in Plan().
template <class Closure>
void Store(const Closure& closure, uint64_t begin, uint64_t end) {
const auto rel = std::memory_order_release;
func_.store(static_cast<RunFunc>(&CallClosure<Closure>), rel);
opaque_.store(reinterpret_cast<const void*>(&closure), rel);
begin_.store(begin, rel);
end_.store(end, rel);
}
RunFunc WorkerGet(uint64_t& begin, uint64_t& end, const void*& opaque) const {
const auto acq = std::memory_order_acquire;
begin = begin_.load(acq);
end = end_.load(acq);
opaque = opaque_.load(acq);
return func_.load(acq);
}
private:
std::atomic<RunFunc> func_;
std::atomic<const void*> opaque_;
std::atomic<uint64_t> begin_;
std::atomic<uint64_t> end_;
};
// Modified by main thread, shared with all workers.
class PoolCommands { // 16 bytes
static constexpr uint32_t kInitial = 0;
static constexpr uint32_t kMask = 0xF; // for command, rest is ABA counter.
static constexpr size_t kShift = hwy::CeilLog2(kMask);
public:
static constexpr uint32_t kTerminate = 1;
static constexpr uint32_t kWork = 2;
static constexpr uint32_t kNop = 3;
// Workers must initialize their copy to this so that they wait for the first
// command as intended.
static uint32_t WorkerInitialSeqCmd() { return kInitial; }
// Sends `cmd` to all workers.
void Broadcast(uint32_t cmd) {
HWY_DASSERT(cmd <= kMask);
const uint32_t epoch = ++epoch_;
const uint32_t seq_cmd = (epoch << kShift) | cmd;
seq_cmd_.store(seq_cmd, std::memory_order_release);
// Wake any worker whose wait_mode_ is or was kBlock.
WakeAll(seq_cmd_);
// Workers are either starting up, or waiting for a command. Either way,
// they will not miss this command, so no need to wait for them here.
}
// Returns the command, i.e., one of the public constants, e.g., kTerminate.
uint32_t WorkerWaitForNewCommand(PoolWaitMode wait_mode,
uint32_t& prev_seq_cmd) {
uint32_t seq_cmd;
if (HWY_LIKELY(wait_mode == PoolWaitMode::kSpin)) {
seq_cmd = SpinUntilDifferent(prev_seq_cmd, seq_cmd_);
} else {
seq_cmd = BlockUntilDifferent(prev_seq_cmd, seq_cmd_);
}
prev_seq_cmd = seq_cmd;
return seq_cmd & kMask;
}
private:
static HWY_INLINE uint32_t SpinUntilDifferent(
const uint32_t prev_seq_cmd, std::atomic<uint32_t>& current) {
for (;;) {
hwy::Pause();
const uint32_t seq_cmd = current.load(std::memory_order_acquire);
if (seq_cmd != prev_seq_cmd) return seq_cmd;
}
}
// Counter for ABA-proofing WorkerWaitForNewCommand. Stored next to seq_cmd_
// because both are written at the same time by the main thread. Sharding this
// 4x (one per cache line) is not helpful.
uint32_t epoch_{0};
std::atomic<uint32_t> seq_cmd_{kInitial};
};
// Modified by main thread AND workers.
// TODO(janwas): more scalable tree
class alignas(HWY_ALIGNMENT) PoolBarrier { // 4 * HWY_ALIGNMENT bytes
static constexpr size_t kU64PerCacheLine = HWY_ALIGNMENT / sizeof(uint64_t);
public:
void Reset() {
for (size_t i = 0; i < 4; ++i) {
num_finished_[i * kU64PerCacheLine].store(0, std::memory_order_release);
}
}
void WorkerArrive(size_t thread) {
const size_t i = (thread & 3);
num_finished_[i * kU64PerCacheLine].fetch_add(1, std::memory_order_release);
}
// Spin until all have called Arrive(). Note that workers spin for a new
// command, not the barrier itself.
HWY_POOL_INLINE void WaitAll(size_t num_workers) {
const auto acq = std::memory_order_acquire;
for (;;) {
hwy::Pause();
const uint64_t sum = num_finished_[0 * kU64PerCacheLine].load(acq) +
num_finished_[1 * kU64PerCacheLine].load(acq) +
num_finished_[2 * kU64PerCacheLine].load(acq) +
num_finished_[3 * kU64PerCacheLine].load(acq);
if (sum == num_workers) break;
}
}
private:
// Sharded to reduce contention. Four counters, each in their own cache line.
std::atomic<uint64_t> num_finished_[4 * kU64PerCacheLine];
};
// All mutable pool and worker state.
struct alignas(HWY_ALIGNMENT) PoolMem {
PoolWorker& Worker(size_t thread) {
return *reinterpret_cast<PoolWorker*>(reinterpret_cast<uint8_t*>(&barrier) +
sizeof(barrier) +
thread * sizeof(PoolWorker));
}
PoolTasks tasks;
PoolCommands commands;
// barrier is more write-heavy, hence keep in another cache line.
uint8_t padding[HWY_ALIGNMENT - sizeof(tasks) - sizeof(commands)];
PoolBarrier barrier;
static_assert(sizeof(barrier) % HWY_ALIGNMENT == 0, "");
// Followed by `num_workers` PoolWorker.
};
// Aligned allocation and initialization of variable-length PoolMem.
class PoolMemOwner {
public:
explicit PoolMemOwner(size_t num_threads)
// The main thread also participates.
: num_workers_(num_threads + 1) {
const size_t size = sizeof(PoolMem) + num_workers_ * sizeof(PoolWorker);
bytes_ = hwy::AllocateAligned<uint8_t>(size);
HWY_ASSERT(bytes_);
mem_ = new (bytes_.get()) PoolMem();
for (size_t thread = 0; thread < num_workers_; ++thread) {
new (&mem_->Worker(thread)) PoolWorker(thread, num_workers_);
}
// Publish non-atomic stores in mem_ - that is the only shared state workers
// access before they call WorkerWaitForNewCommand.
std::atomic_thread_fence(std::memory_order_release);
}
~PoolMemOwner() {
for (size_t thread = 0; thread < num_workers_; ++thread) {
mem_->Worker(thread).~PoolWorker();
}
mem_->~PoolMem();
}
size_t NumWorkers() const { return num_workers_; }
PoolMem* Mem() const { return mem_; }
private:
const size_t num_workers_; // >= 1
// Aligned allocation ensures we do not straddle cache lines.
hwy::AlignedFreeUniquePtr<uint8_t[]> bytes_;
PoolMem* mem_;
};
// Plans and executes parallel-for loops with work-stealing. No synchronization
// because there is no mutable shared state.
class ParallelFor { // 0 bytes
// A prior version of this code attempted to assign only as much work as a
// thread will actually use. As with OpenMP's 'guided' strategy, we assigned
// remaining/(k*num_threads) in each iteration. Although the worst-case
// imbalance is bounded, this required several rounds of work allocation, and
// the atomic counter did not scale to > 30 threads.
//
// We now use work stealing instead, where already-finished threads look for
// and perform work from others, as if they were that thread. This deals with
// imbalances as they arise, but care is required to reduce contention. We
// randomize the order in which threads choose victims to steal from.
//
// Results: across 10K calls Run(), we observe a mean of 5.1 tasks per
// thread, and standard deviation 0.67, indicating good load-balance.
public:
// Make preparations for workers to later run `closure(i)` for all `i` in
// `[begin, end)`. Called from the main thread; workers are initializing or
// spinning for a command. Returns false if there are no tasks or workers.
template <class Closure>
static bool Plan(uint64_t begin, uint64_t end, size_t num_workers,
const Closure& closure, PoolMem& mem) {
// If there are no tasks, we are done.
HWY_DASSERT(begin <= end);
const size_t num_tasks = static_cast<size_t>(end - begin);
if (HWY_UNLIKELY(num_tasks == 0)) return false;
// If there are no workers, run all tasks already on the main thread without
// the overhead of planning.
if (HWY_UNLIKELY(num_workers <= 1)) {
for (uint64_t task = begin; task < end; ++task) {
closure(task, /*thread=*/0);
}
return false;
}
// Store for later retrieval by all workers in WorkerRun. Must happen after
// the loop above because it may be re-entered by concurrent threads.
mem.tasks.Store(closure, begin, end);
// Assigning all remainders to the last thread causes imbalance. We instead
// give one more to each thread whose index is less.
const size_t remainder = num_tasks % num_workers;
const size_t min_tasks = num_tasks / num_workers;
uint64_t task = begin;
for (size_t thread = 0; thread < num_workers; ++thread) {
const uint64_t my_end = task + min_tasks + (thread < remainder);
mem.Worker(thread).SetRange(task, my_end);
task = my_end;
}
HWY_DASSERT(task == end);
return true;
}
// Must be called for each `thread` in [0, num_workers), but only if
// Plan returned true.
static HWY_POOL_INLINE void WorkerRun(const size_t thread, size_t num_workers,
PoolMem& mem) {
// Nonzero, otherwise Plan returned false and this should not be called.
HWY_DASSERT(num_workers != 0);
HWY_DASSERT(thread < num_workers);
const PoolTasks& tasks = mem.tasks;
uint64_t begin, end;
const void* opaque;
const auto func = tasks.WorkerGet(begin, end, opaque);
// Special case for <= 1 task per worker - avoid any shared state.
if (HWY_UNLIKELY(end <= begin + num_workers)) {
const uint64_t task = begin + thread;
if (HWY_LIKELY(task < end)) {
func(opaque, task, thread);
}
return;
}
// For each worker in random order, attempt to do all their work.
for (uint32_t victim : mem.Worker(thread).Victims()) {
PoolWorker* other_worker = &mem.Worker(victim);
// Until all of other_worker's work is done:
const uint64_t other_end = other_worker->WorkerGetEnd();
for (;;) {
// On x86 this generates a LOCK prefix, but that is only expensive if
// there is actually contention, which is unlikely because we shard the
// counters, threads do not quite proceed in lockstep due to memory
// traffic, and stealing happens in semi-random order.
uint64_t task = other_worker->WorkerReserveTask();
// The worker that first sets `task` to `other_end` exits this loop.
// After that, `task` can be incremented up to `num_workers - 1` times,
// once per other worker.
HWY_DASSERT(task < other_end + num_workers);
if (HWY_UNLIKELY(task >= other_end)) {
hwy::Pause(); // Reduce coherency traffic while stealing.
break;
}
// `thread` is the one we are actually running on; this is important
// because it is the TLS index for user code.
func(opaque, task, thread);
}
}
}
};
#pragma pack(pop)
// Sets the name of the current thread to the format string `format`, which must
// include %d for `thread`. Currently only implemented for pthreads (*nix and
// OSX); Windows involves throwing an exception.
static inline void SetThreadName(const char* format, int thread) {
#if HWY_OS_LINUX
char buf[16] = {}; // Linux limit, including \0
const int chars_written = snprintf(buf, sizeof(buf), format, thread);
HWY_ASSERT(0 < chars_written &&
chars_written <= static_cast<int>(sizeof(buf) - 1));
HWY_ASSERT(0 == pthread_setname_np(pthread_self(), buf));
#else
(void)format;
(void)thread;
#endif
}
// Highly efficient parallel-for, intended for workloads with thousands of
// fork-join regions which consist of calling tasks[t](i) for a few hundred i,
// using dozens of threads.
//
// To reduce scheduling overhead, we assume that tasks are statically known and
// that threads do not schedule new work themselves. This allows us to avoid
// queues and only store a counter plus the current task. The latter is a
// pointer to a lambda function, without the allocation/indirection required for
// std::function.
//
// To reduce fork/join latency, we use an efficient barrier, optionally
// support spin-waits via SetWaitMode, and avoid any mutex/lock.
//
// To eliminate false sharing and enable reasoning about cache line traffic, the
// worker state uses a single aligned allocation.
//
// For load-balancing, we use work stealing in random order.
class ThreadPool {
static void ThreadFunc(size_t thread, size_t num_workers, PoolMem* mem) {
HWY_DASSERT(thread < num_workers);
SetThreadName("worker%03zu", static_cast<int>(thread));
// Ensure mem is ready to use (synchronize with PoolMemOwner's fence).
std::atomic_thread_fence(std::memory_order_acquire);
PoolWorker& worker = mem->Worker(thread);
PoolCommands& commands = mem->commands;
uint32_t prev_seq_cmd = PoolCommands::WorkerInitialSeqCmd();
for (;;) {
const PoolWaitMode wait_mode = worker.WorkerGetWaitMode();
const uint32_t command =
commands.WorkerWaitForNewCommand(wait_mode, prev_seq_cmd);
if (HWY_UNLIKELY(command == PoolCommands::kTerminate)) {
return; // exits thread
} else if (HWY_LIKELY(command == PoolCommands::kWork)) {
ParallelFor::WorkerRun(thread, num_workers, *mem);
mem->barrier.WorkerArrive(thread);
} else if (command == PoolCommands::kNop) {
// do nothing - used to change wait mode
} else {
HWY_DASSERT(false); // unknown command
}
}
}
public:
// This typically includes hyperthreads, hence it is a loose upper bound.
// -1 because these are in addition to the main thread.
static size_t MaxThreads() {
LogicalProcessorSet lps;
// This is OS dependent, but more accurate if available because it takes
// into account restrictions set by cgroups or numactl/taskset.
if (GetThreadAffinity(lps)) {
return lps.Count() - 1;
}
return static_cast<size_t>(std::thread::hardware_concurrency() - 1);
}
// `num_threads` should not exceed `MaxThreads()`. If `num_threads` <= 1,
// Run() runs only on the main thread. Otherwise, we launch `num_threads - 1`
// threads because the main thread also participates.
explicit ThreadPool(size_t num_threads) : owner_(num_threads) {
(void)busy_; // unused in non-debug builds, avoid warning
const size_t num_workers = owner_.NumWorkers();
// Launch threads without waiting afterwards: they will receive the next
// PoolCommands once ready.
threads_.reserve(num_workers - 1);
for (size_t thread = 0; thread < num_workers - 1; ++thread) {
threads_.emplace_back(ThreadFunc, thread, num_workers, owner_.Mem());
}
}
// Waits for all threads to exit.
~ThreadPool() {
PoolMem& mem = *owner_.Mem();
mem.commands.Broadcast(PoolCommands::kTerminate); // requests threads exit
for (std::thread& thread : threads_) {
HWY_ASSERT(thread.joinable());
thread.join();
}
}
ThreadPool(const ThreadPool&) = delete;
ThreadPool& operator&(const ThreadPool&) = delete;
// Returns number of PoolWorker, i.e., one more than the largest `thread`
// argument. Useful for callers that want to allocate thread-local storage.
size_t NumWorkers() const { return owner_.NumWorkers(); }
// `mode` is initially `kBlock`, which means futex. Switching to `kSpin`
// reduces fork-join overhead especially when there are many calls to `Run`,
// but wastes power when waiting over long intervals. Inexpensive, OK to call
// multiple times, but not concurrently with any `Run`.
void SetWaitMode(PoolWaitMode mode) {
// Run must not be active, otherwise we may overwrite the previous command
// before it is seen by all workers.
HWY_DASSERT(busy_.fetch_add(1) == 0);
PoolMem& mem = *owner_.Mem();
// For completeness/consistency, set on all workers, including the main
// thread, even though it will never wait for a command.
for (size_t thread = 0; thread < owner_.NumWorkers(); ++thread) {
mem.Worker(thread).SetWaitMode(mode);
}
// Send a no-op command so that workers wake as soon as possible. Skip the
// expensive barrier - workers may miss this command, but it is fine for
// them to wake up later and get the next actual command.
mem.commands.Broadcast(PoolCommands::kNop);
HWY_DASSERT(busy_.fetch_add(-1) == 1);
}
// parallel-for: Runs `closure(task, thread)` on worker thread(s) for every
// `task` in `[begin, end)`. Note that the unit of work should be large
// enough to amortize the function call overhead, but small enough that each
// worker processes a few tasks. Thus each `task` is usually a loop.
//
// Not thread-safe - concurrent calls to `Run` in the same ThreadPool are
// forbidden unless NumWorkers() == 0. We check for that in debug builds.
template <class Closure>
void Run(uint64_t begin, uint64_t end, const Closure& closure) {
const size_t num_workers = NumWorkers();
PoolMem& mem = *owner_.Mem();
if (HWY_LIKELY(ParallelFor::Plan(begin, end, num_workers, closure, mem))) {
// Only check if we are going to fork/join.
HWY_DASSERT(busy_.fetch_add(1) == 0);
mem.barrier.Reset();
mem.commands.Broadcast(PoolCommands::kWork);
// Also perform work on main thread instead of busy-waiting.
const size_t thread = num_workers - 1;
ParallelFor::WorkerRun(thread, num_workers, mem);
mem.barrier.WorkerArrive(thread);
mem.barrier.WaitAll(num_workers);
HWY_DASSERT(busy_.fetch_add(-1) == 1);
}
}
// Can pass this as init_closure when no initialization is needed.
// DEPRECATED, better to call the Run() overload without the init_closure arg.
static bool NoInit(size_t /*num_threads*/) { return true; } // DEPRECATED
// DEPRECATED equivalent of NumWorkers. Note that this is not the same as the
// ctor argument because num_threads = 0 has the same effect as 1.
size_t NumThreads() const { return NumWorkers(); } // DEPRECATED
// DEPRECATED prior interface with 32-bit tasks and first calling
// `init_closure(num_threads)`. Instead, perform any init before this, calling
// NumWorkers() for an upper bound on the thread indices, then call the
// other overload.
template <class InitClosure, class RunClosure>
bool Run(uint64_t begin, uint64_t end, const InitClosure& init_closure,
const RunClosure& run_closure) {
if (!init_closure(NumThreads())) return false;
Run(begin, end, run_closure);
return true;
}
// Only for use in tests.
PoolMem& InternalMem() const { return *owner_.Mem(); }
private:
// Unmodified after ctor, but cannot be const because we call thread::join().
std::vector<std::thread> threads_;
PoolMemOwner owner_;
// In debug builds, detects if functions are re-entered; always present so
// that the memory layout does not change.
std::atomic<int> busy_{0};
};
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_THREAD_POOL_THREAD_POOL_H_

400
deps/v8/third_party/highway/src/hwy/contrib/thread_pool/thread_pool_test.cc vendored Normal file
View File

@ -0,0 +1,400 @@
// Copyright 2023 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified from BSD-licensed code
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
// See https://github.com/libjxl/libjxl/blob/main/LICENSE.
#include "hwy/contrib/thread_pool/thread_pool.h"
#include <math.h> // sqrtf
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <atomic>
#include <vector>
#include "hwy/base.h" // PopCount
#include "hwy/contrib/thread_pool/topology.h"
#include "hwy/tests/hwy_gtest.h"
#include "hwy/tests/test_util-inl.h" // AdjustedReps
namespace hwy {
namespace {
using HWY_NAMESPACE::AdjustedReps;
TEST(ThreadPoolTest, TestCoprime) {
// 1 is coprime with anything
for (uint32_t i = 1; i < 500; ++i) {
HWY_ASSERT(ShuffledIota::CoprimeNonzero(1, i));
HWY_ASSERT(ShuffledIota::CoprimeNonzero(i, 1));
}
// Powers of two >= 2 are not coprime
for (size_t i = 1; i < 20; ++i) {
for (size_t j = 1; j < 20; ++j) {
HWY_ASSERT(!ShuffledIota::CoprimeNonzero(1u << i, 1u << j));
}
}
// 2^x and 2^x +/- 1 are coprime
for (size_t i = 1; i < 30; ++i) {
const uint32_t pow2 = 1u << i;
HWY_ASSERT(ShuffledIota::CoprimeNonzero(pow2, pow2 + 1));
HWY_ASSERT(ShuffledIota::CoprimeNonzero(pow2, pow2 - 1));
HWY_ASSERT(ShuffledIota::CoprimeNonzero(pow2 + 1, pow2));
HWY_ASSERT(ShuffledIota::CoprimeNonzero(pow2 - 1, pow2));
}
// Random number x * random y (both >= 2) is not co-prime with x nor y.
RandomState rng;
for (size_t i = 1; i < 5000; ++i) {
const uint32_t x = (Random32(&rng) & 0xFFF7) + 2;
const uint32_t y = (Random32(&rng) & 0xFFF7) + 2;
HWY_ASSERT(!ShuffledIota::CoprimeNonzero(x * y, x));
HWY_ASSERT(!ShuffledIota::CoprimeNonzero(x * y, y));
HWY_ASSERT(!ShuffledIota::CoprimeNonzero(x, x * y));
HWY_ASSERT(!ShuffledIota::CoprimeNonzero(y, x * y));
}
// Primes are all coprime (list from https://oeis.org/A000040)
static constexpr uint32_t primes[] = {
2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47,
53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113,
127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181, 191, 193, 197,
199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271};
for (size_t i = 0; i < sizeof(primes) / sizeof(primes[0]); ++i) {
for (size_t j = i + 1; j < sizeof(primes) / sizeof(primes[0]); ++j) {
HWY_ASSERT(ShuffledIota::CoprimeNonzero(primes[i], primes[j]));
HWY_ASSERT(ShuffledIota::CoprimeNonzero(primes[j], primes[i]));
}
}
}
// Ensures `shuffled` visits [0, size) exactly once starting from `current`.
void VerifyPermutation(uint32_t size, const Divisor& divisor,
const ShuffledIota& shuffled, uint32_t current,
uint32_t* visited) {
for (size_t i = 0; i < size; i++) {
visited[i] = 0;
}
for (size_t i = 0; i < size; i++) {
++visited[current];
current = shuffled.Next(current, divisor);
}
for (size_t i = 0; i < size; i++) {
HWY_ASSERT(visited[i] == 1);
}
}
// Verifies ShuffledIota generates a permutation of [0, size).
TEST(ThreadPoolTest, TestRandomPermutation) {
constexpr size_t kMaxSize = 40;
uint32_t visited[kMaxSize];
// Exhaustive enumeration of size and starting point.
for (uint32_t size = 1; size < kMaxSize; ++size) {
const Divisor divisor(size);
const uint32_t coprime = ShuffledIota::FindAnotherCoprime(size, 1);
const ShuffledIota shuffled(coprime);
for (uint32_t start = 0; start < size; ++start) {
VerifyPermutation(size, divisor, shuffled, start, visited);
}
}
}
// Verifies multiple ShuffledIota are relatively independent.
TEST(ThreadPoolTest, TestMultiplePermutations) {
constexpr size_t kMaxSize = 40;
uint32_t coprimes[kMaxSize];
// One per ShuffledIota; initially the starting value, then its Next().
uint32_t current[kMaxSize];
for (uint32_t size = 1; size < kMaxSize; ++size) {
const Divisor divisor(size);
// Create `size` ShuffledIota instances with unique coprimes.
std::vector<ShuffledIota> shuffled;
for (size_t i = 0; i < size; ++i) {
coprimes[i] = ShuffledIota::FindAnotherCoprime(
size, static_cast<uint32_t>((i + 1) * 257 + i * 13));
shuffled.emplace_back(coprimes[i]);
}
// ShuffledIota[i] starts at i to match the worker thread use case.
for (uint32_t i = 0; i < size; ++i) {
current[i] = i;
}
size_t num_bad = 0;
uint32_t all_visited[kMaxSize] = {0};
// For each step, ensure there are few non-unique current[].
for (size_t step = 0; step < size; ++step) {
// How many times is each number visited?
uint32_t visited[kMaxSize] = {0};
for (size_t i = 0; i < size; ++i) {
visited[current[i]] += 1;
all_visited[current[i]] = 1; // visited at all across all steps?
}
// How many numbers are visited multiple times?
size_t num_contended = 0;
uint32_t max_contention = 0;
for (size_t i = 0; i < size; ++i) {
num_contended += visited[i] > 1;
max_contention = HWY_MAX(max_contention, visited[i]);
}
// Count/print if excessive collisions.
const size_t expected =
static_cast<size_t>(sqrtf(static_cast<float>(size)) * 2.0f);
if ((num_contended > expected) && (max_contention > 3)) {
++num_bad;
if (true) {
fprintf(stderr, "size %u step %zu contended %zu max contention %u\n",
size, step, num_contended, max_contention);
for (size_t i = 0; i < size; ++i) {
fprintf(stderr, " %u\n", current[i]);
}
fprintf(stderr, "coprimes\n");
for (size_t i = 0; i < size; ++i) {
fprintf(stderr, " %u\n", coprimes[i]);
}
}
}
// Advance all ShuffledIota generators.
for (size_t i = 0; i < size; ++i) {
current[i] = shuffled[i].Next(current[i], divisor);
}
} // step
// Ensure each task was visited during at least one step.
for (size_t i = 0; i < size; ++i) {
HWY_ASSERT(all_visited[i] != 0);
}
if (num_bad != 0) {
fprintf(stderr, "size %u total bad: %zu\n", size, num_bad);
}
HWY_ASSERT(num_bad < kMaxSize / 10);
} // size
}
// Ensures all tasks are run. Similar to TestPool below but without threads.
TEST(ThreadPoolTest, TestTasks) {
for (size_t num_threads = 0; num_threads <= 8; ++num_threads) {
PoolMemOwner owner(num_threads);
PoolMem& mem = *owner.Mem();
const size_t num_workers = owner.NumWorkers();
constexpr uint64_t kMaxTasks = 20;
uint64_t mementos[kMaxTasks];
for (uint64_t num_tasks = 0; num_tasks < 20; ++num_tasks) {
for (uint64_t begin = 0; begin < AdjustedReps(32); ++begin) {
const uint64_t end = begin + num_tasks;
ZeroBytes(mementos, sizeof(mementos));
const auto func = [begin, end, &mementos](uint64_t task,
size_t /*thread*/) {
HWY_ASSERT(begin <= task && task < end);
// Store mementos ensure we visited each task.
mementos[task - begin] = 1000 + task;
};
if (ParallelFor::Plan(begin, end, num_workers, func, mem)) {
// The `tasks < workers` special case requires running by all workers.
for (size_t thread = 0; thread < num_workers; ++thread) {
ParallelFor::WorkerRun(thread, num_workers, mem);
}
}
// Ensure all tasks were run.
for (uint64_t task = begin; task < end; ++task) {
HWY_ASSERT_EQ(1000 + task, mementos[task - begin]);
}
}
}
}
}
// Ensures old code with 32-bit tasks and InitClosure still compiles.
TEST(ThreadPoolTest, TestDeprecated) {
ThreadPool pool(0);
pool.Run(1, 10, &ThreadPool::NoInit,
[&](const uint64_t /*task*/, size_t /*thread*/) {});
}
// Ensures task parameter is in bounds, every parameter is reached,
// pool can be reused (multiple consecutive Run calls), pool can be destroyed
// (joining with its threads), num_threads=0 works (runs on current thread).
TEST(ThreadPoolTest, TestPool) {
if (!HaveThreadingSupport()) return;
ThreadPool inner(0);
for (size_t num_threads = 0; num_threads <= 6; num_threads += 3) {
ThreadPool pool(HWY_MIN(ThreadPool::MaxThreads(), num_threads));
constexpr uint64_t kMaxTasks = 20;
std::atomic<uint64_t> mementos[kMaxTasks];
for (uint64_t num_tasks = 0; num_tasks < kMaxTasks; ++num_tasks) {
for (uint64_t begin = 0; begin < AdjustedReps(32); ++begin) {
const uint64_t end = begin + num_tasks;
std::atomic<uint64_t> a_begin;
std::atomic<uint64_t> a_end;
a_begin.store(begin, std::memory_order_release);
a_end.store(end, std::memory_order_release);
for (size_t i = 0; i < kMaxTasks; ++i) {
mementos[i].store(0);
}
pool.Run(begin, end,
[&a_begin, &a_end, &mementos, &inner](uint64_t task,
size_t /*thread*/) {
const uint64_t begin =
a_begin.load(std::memory_order_acquire);
const uint64_t end = a_end.load(std::memory_order_acquire);
HWY_ASSERT(begin <= task && task < end);
// Store mementos ensure we visited each task.
mementos[task - begin].store(1000 + task);
// Re-entering Run is fine on a 0-worker pool.
inner.Run(begin, end,
[begin, end](uint64_t task, size_t /*thread*/) {
HWY_ASSERT(begin <= task && task < end);
});
});
for (uint64_t task = begin; task < end; ++task) {
HWY_ASSERT_EQ(1000 + task, mementos[task - begin].load());
}
}
}
}
}
// Debug tsan builds seem to generate incorrect codegen for [&] of atomics, so
// use a pointer to a state object instead.
struct SmallAssignmentState {
// (Avoid mutex because it may perturb the worker thread scheduling)
std::atomic<uint64_t> num_tasks{0};
std::atomic<uint64_t> num_workers{0};
std::atomic<uint64_t> id_bits{0};
std::atomic<uint64_t> num_calls{0};
};
// Verify "thread" parameter when processing few tasks.
TEST(ThreadPoolTest, TestSmallAssignments) {
if (!HaveThreadingSupport()) return;
static SmallAssignmentState state;
for (size_t num_threads :
{size_t{0}, size_t{1}, size_t{3}, size_t{5}, size_t{8}}) {
ThreadPool pool(HWY_MIN(ThreadPool::MaxThreads(), num_threads));
state.num_workers.store(pool.NumWorkers());
for (size_t mul = 1; mul <= 2; ++mul) {
const size_t num_tasks = pool.NumWorkers() * mul;
state.num_tasks.store(num_tasks);
state.id_bits.store(0);
state.num_calls.store(0);
pool.Run(0, num_tasks, [](uint64_t task, size_t thread) {
HWY_ASSERT(task < state.num_tasks.load());
HWY_ASSERT(thread < state.num_workers.load());
state.num_calls.fetch_add(1);
uint64_t bits = state.id_bits.load();
while (!state.id_bits.compare_exchange_weak(bits,
bits | (1ULL << thread))) {
}
});
// Correct number of tasks.
const uint64_t actual_calls = state.num_calls.load();
HWY_ASSERT(num_tasks == actual_calls);
const size_t num_participants = PopCount(state.id_bits.load());
// <= because some workers may not manage to run any tasks.
HWY_ASSERT(num_participants <= pool.NumWorkers());
}
}
}
struct Counter {
Counter() {
// Suppress "unused-field" warning.
(void)padding;
}
void Assimilate(const Counter& victim) { counter += victim.counter; }
std::atomic<uint64_t> counter{0};
uint64_t padding[15];
};
// Can switch between any wait mode, and multiple times.
TEST(ThreadPoolTest, TestWaitMode) {
if (!HaveThreadingSupport()) return;
const size_t kNumThreads = 9;
ThreadPool pool(kNumThreads);
RandomState rng;
for (size_t i = 0; i < 10; ++i) {
pool.SetWaitMode(Random32(&rng) ? PoolWaitMode::kSpin
: PoolWaitMode::kBlock);
}
}
TEST(ThreadPoolTest, TestCounter) {
if (!HaveThreadingSupport()) return;
const size_t kNumThreads = 12;
ThreadPool pool(kNumThreads);
for (PoolWaitMode mode : {PoolWaitMode::kSpin, PoolWaitMode::kBlock}) {
pool.SetWaitMode(mode);
alignas(128) Counter counters[1+kNumThreads];
const uint64_t kNumTasks = kNumThreads * 19;
pool.Run(0, kNumTasks,
[&counters](const uint64_t task, const size_t thread) {
counters[thread].counter.fetch_add(task);
});
uint64_t expected = 0;
for (uint64_t i = 0; i < kNumTasks; ++i) {
expected += i;
}
for (size_t i = 1; i < pool.NumWorkers(); ++i) {
counters[0].Assimilate(counters[i]);
}
HWY_ASSERT_EQ(expected, counters[0].counter.load());
}
}
} // namespace
} // namespace hwy
HWY_TEST_MAIN();

530
deps/v8/third_party/highway/src/hwy/contrib/thread_pool/topology.cc vendored Normal file
View File

@ -0,0 +1,530 @@
// Copyright 2024 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/thread_pool/topology.h"
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h> // strchr
#include <map>
#include <vector>
#include "hwy/detect_compiler_arch.h" // HWY_OS_WIN
#if HWY_OS_WIN
#ifndef NOMINMAX
#define NOMINMAX
#endif
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
#endif // HWY_OS_WIN
#if HWY_OS_LINUX || HWY_OS_FREEBSD
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <sched.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h> // sysconf
#endif // HWY_OS_LINUX || HWY_OS_FREEBSD
#if HWY_OS_FREEBSD
// must come after sys/types.h.
#include <sys/cpuset.h> // CPU_SET
#endif // HWY_OS_FREEBSD
#if HWY_ARCH_WASM
#include <emscripten/threading.h>
#endif
#include "hwy/base.h"
namespace hwy {
HWY_CONTRIB_DLLEXPORT bool HaveThreadingSupport() {
#if HWY_ARCH_WASM
return emscripten_has_threading_support() != 0;
#else
return true;
#endif
}
HWY_CONTRIB_DLLEXPORT size_t TotalLogicalProcessors() {
size_t lp = 0;
#if HWY_ARCH_WASM
const int num_cores = emscripten_num_logical_cores();
if (num_cores > 0) lp = static_cast<size_t>(num_cores);
#elif HWY_OS_WIN
SYSTEM_INFO sysinfo;
GetSystemInfo(&sysinfo); // always succeeds
// WARNING: this is only for the current group, hence limited to 64.
lp = static_cast<size_t>(sysinfo.dwNumberOfProcessors);
#elif HWY_OS_LINUX
// Use configured, not "online" (_SC_NPROCESSORS_ONLN), because we want an
// upper bound.
const long ret = sysconf(_SC_NPROCESSORS_CONF); // NOLINT(runtime/int)
if (ret < 0) {
fprintf(stderr, "Unexpected value of _SC_NPROCESSORS_CONF: %d\n",
static_cast<int>(ret));
} else {
lp = static_cast<size_t>(ret);
}
#endif
if (HWY_UNLIKELY(lp == 0)) { // Failed to detect.
HWY_IF_CONSTEXPR(HWY_IS_DEBUG_BUILD) {
fprintf(stderr,
"Unknown TotalLogicalProcessors, assuming 1. "
"HWY_OS_: WIN=%d LINUX=%d APPLE=%d;\n"
"HWY_ARCH_: WASM=%d X86=%d PPC=%d ARM=%d RISCV=%d S390X=%d\n",
HWY_OS_WIN, HWY_OS_LINUX, HWY_OS_APPLE, HWY_ARCH_WASM,
HWY_ARCH_X86, HWY_ARCH_PPC, HWY_ARCH_ARM, HWY_ARCH_RISCV,
HWY_ARCH_S390X);
}
return 1;
}
// Warn that we are clamping.
if (HWY_UNLIKELY(lp > kMaxLogicalProcessors)) {
HWY_IF_CONSTEXPR(HWY_IS_DEBUG_BUILD) {
fprintf(stderr, "OS reports %zu processors but clamping to %zu\n", lp,
kMaxLogicalProcessors);
}
lp = kMaxLogicalProcessors;
}
return lp;
}
#ifdef __ANDROID__
#include <sys/syscall.h>
#endif
HWY_CONTRIB_DLLEXPORT bool GetThreadAffinity(LogicalProcessorSet& lps) {
#if HWY_OS_WIN
// Only support the first 64 because WINE does not support processor groups.
const HANDLE hThread = GetCurrentThread();
const DWORD_PTR prev = SetThreadAffinityMask(hThread, ~DWORD_PTR(0));
if (!prev) return false;
(void)SetThreadAffinityMask(hThread, prev);
lps = LogicalProcessorSet(); // clear all
lps.SetNonzeroBitsFrom64(prev);
return true;
#elif HWY_OS_LINUX
cpu_set_t set;
CPU_ZERO(&set);
const pid_t pid = 0; // current thread
#ifdef __ANDROID__
const int err = syscall(__NR_sched_getaffinity, pid, sizeof(cpu_set_t), &set);
#else
const int err = sched_getaffinity(pid, sizeof(cpu_set_t), &set);
#endif // __ANDROID__
if (err != 0) return false;
for (size_t lp = 0; lp < kMaxLogicalProcessors; ++lp) {
#if HWY_COMPILER_GCC_ACTUAL
// Workaround for GCC compiler warning with CPU_ISSET macro
HWY_DIAGNOSTICS(push)
HWY_DIAGNOSTICS_OFF(disable : 4305 4309, ignored "-Wsign-conversion")
#endif
if (CPU_ISSET(static_cast<int>(lp), &set)) {
lps.Set(lp);
}
#if HWY_COMPILER_GCC_ACTUAL
HWY_DIAGNOSTICS(pop)
#endif
}
return true;
#elif HWY_OS_FREEBSD
cpuset_t set;
CPU_ZERO(&set);
const pid_t pid = getpid(); // current thread
const int err = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid,
sizeof(cpuset_t), &set);
if (err != 0) return false;
for (size_t lp = 0; lp < kMaxLogicalProcessors; ++lp) {
#if HWY_COMPILER_GCC_ACTUAL
// Workaround for GCC compiler warning with CPU_ISSET macro
HWY_DIAGNOSTICS(push)
HWY_DIAGNOSTICS_OFF(disable : 4305 4309, ignored "-Wsign-conversion")
#endif
if (CPU_ISSET(static_cast<int>(lp), &set)) {
lps.Set(lp);
}
#if HWY_COMPILER_GCC_ACTUAL
HWY_DIAGNOSTICS(pop)
#endif
}
return true;
#else
// Do not even set lp=0 to force callers to handle this case.
(void)lps;
return false;
#endif
}
HWY_CONTRIB_DLLEXPORT bool SetThreadAffinity(const LogicalProcessorSet& lps) {
#if HWY_OS_WIN
const HANDLE hThread = GetCurrentThread();
const DWORD_PTR prev = SetThreadAffinityMask(hThread, lps.Get64());
return prev != 0;
#elif HWY_OS_LINUX
cpu_set_t set;
CPU_ZERO(&set);
#if HWY_COMPILER_GCC_ACTUAL
// Workaround for GCC compiler warning with CPU_SET macro
HWY_DIAGNOSTICS(push)
HWY_DIAGNOSTICS_OFF(disable : 4305 4309, ignored "-Wsign-conversion")
#endif
lps.Foreach([&set](size_t lp) { CPU_SET(static_cast<int>(lp), &set); });
#if HWY_COMPILER_GCC_ACTUAL
HWY_DIAGNOSTICS(pop)
#endif
const pid_t pid = 0; // current thread
#ifdef __ANDROID__
const int err = syscall(__NR_sched_setaffinity, pid, sizeof(cpu_set_t), &set);
#else
const int err = sched_setaffinity(pid, sizeof(cpu_set_t), &set);
#endif // __ANDROID__
if (err != 0) return false;
return true;
#elif HWY_OS_FREEBSD
cpuset_t set;
CPU_ZERO(&set);
#if HWY_COMPILER_GCC_ACTUAL
// Workaround for GCC compiler warning with CPU_SET macro
HWY_DIAGNOSTICS(push)
HWY_DIAGNOSTICS_OFF(disable : 4305 4309, ignored "-Wsign-conversion")
#endif
lps.Foreach([&set](size_t lp) { CPU_SET(static_cast<int>(lp), &set); });
#if HWY_COMPILER_GCC_ACTUAL
HWY_DIAGNOSTICS(pop)
#endif
const pid_t pid = getpid(); // current thread
const int err = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid,
sizeof(cpuset_t), &set);
if (err != 0) return false;
return true;
#else
// Apple THREAD_AFFINITY_POLICY is only an (often ignored) hint.
(void)lps;
return false;
#endif
}
#if HWY_OS_LINUX
namespace {
class File {
public:
explicit File(const char* path) {
for (;;) {
fd_ = open(path, O_RDONLY);
if (fd_ > 0) return; // success
if (errno == EINTR) continue; // signal: retry
if (errno == ENOENT) return; // not found, give up
if (HWY_IS_DEBUG_BUILD) {
fprintf(stderr, "Unexpected error opening %s: %d\n", path, errno);
}
return; // unknown error, give up
}
}
~File() {
if (fd_ > 0) {
for (;;) {
const int ret = close(fd_);
if (ret == 0) break; // success
if (errno == EINTR) continue; // signal: retry
if (HWY_IS_DEBUG_BUILD) {
fprintf(stderr, "Unexpected error closing file: %d\n", errno);
}
return; // unknown error, ignore
}
}
}
// Returns number of bytes read or 0 on failure.
size_t Read(char* buf200) const {
if (fd_ < 0) return 0;
size_t pos = 0;
for (;;) {
// read instead of `pread`, which might not work for sysfs.
const auto bytes_read = read(fd_, buf200 + pos, 200 - pos);
if (bytes_read == 0) { // EOF: done
buf200[pos++] = '\0';
return pos;
}
if (bytes_read == -1) {
if (errno == EINTR) continue; // signal: retry
if (HWY_IS_DEBUG_BUILD) {
fprintf(stderr, "Unexpected error reading file: %d\n", errno);
}
return 0;
}
pos += static_cast<size_t>(bytes_read);
HWY_ASSERT(pos <= 200);
}
}
private:
int fd_;
};
// Returns bytes read, or 0 on failure.
size_t ReadSysfs(const char* format, size_t lp, char* buf200) {
char path[200];
const int bytes_written = snprintf(path, sizeof(path), format, lp);
HWY_ASSERT(0 < bytes_written &&
bytes_written < static_cast<int>(sizeof(path) - 1));
const File file(path);
return file.Read(buf200);
}
// Interprets [str + pos, str + end) as base-10 ASCII. Stops when any non-digit
// is found, or at end. Returns false if no digits found.
bool ParseDigits(const char* str, const size_t end, size_t& pos, size_t* out) {
HWY_ASSERT(pos <= end);
// 9 digits cannot overflow even 32-bit size_t.
const size_t stop = pos + 9;
*out = 0;
for (; pos < HWY_MIN(end, stop); ++pos) {
const int c = str[pos];
if (c < '0' || c > '9') break;
*out *= 10;
*out += static_cast<size_t>(c - '0');
}
if (pos == 0) { // No digits found
*out = 0;
return false;
}
return true;
}
// Number, plus optional K or M suffix, plus terminator.
bool ParseNumberWithOptionalSuffix(const char* str, size_t len, size_t* out) {
size_t pos = 0;
if (!ParseDigits(str, len, pos, out)) return false;
if (str[pos] == 'K') {
*out <<= 10;
++pos;
}
if (str[pos] == 'M') {
*out <<= 20;
++pos;
}
if (str[pos] != '\0' && str[pos] != '\n') {
HWY_ABORT("Expected [suffix] terminator at %zu %s\n", pos, str);
}
return true;
}
bool ReadNumberWithOptionalSuffix(const char* format, size_t lp, size_t* out) {
char buf200[200];
const size_t pos = ReadSysfs(format, lp, buf200);
if (pos == 0) return false;
return ParseNumberWithOptionalSuffix(buf200, pos, out);
}
const char* kPackage =
"/sys/devices/system/cpu/cpu%zu/topology/physical_package_id";
const char* kCluster = "/sys/devices/system/cpu/cpu%zu/cache/index3/id";
const char* kCore = "/sys/devices/system/cpu/cpu%zu/topology/core_id";
const char* kL2Size = "/sys/devices/system/cpu/cpu%zu/cache/index2/size";
const char* kL3Size = "/sys/devices/system/cpu/cpu%zu/cache/index3/size";
const char* kNode = "/sys/devices/system/node/node%zu/cpulist";
// sysfs values can be arbitrarily large, so store in a map and replace with
// indices in order of appearance.
class Remapper {
public:
// Returns false on error, or sets `out_index` to the index of the sysfs
// value selected by `format` and `lp`.
template <typename T>
bool operator()(const char* format, size_t lp, T* HWY_RESTRICT out_index) {
size_t opaque;
if (!ReadNumberWithOptionalSuffix(format, lp, &opaque)) return false;
const auto ib = indices_.insert({opaque, num_});
num_ += ib.second; // increment if inserted
const size_t index = ib.first->second; // new or existing
HWY_ASSERT(index < num_);
HWY_ASSERT(index < hwy::LimitsMax<T>());
*out_index = static_cast<T>(index);
return true;
}
size_t Num() const { return num_; }
private:
std::map<size_t, size_t> indices_;
size_t num_ = 0;
};
// Stores the global cluster/core values separately for each package so we can
// return per-package arrays.
struct PerPackage {
Remapper clusters;
Remapper cores;
uint8_t smt_per_core[kMaxLogicalProcessors] = {0};
};
// Initializes `lps` and returns a PerPackage vector (empty on failure).
std::vector<PerPackage> DetectPackages(std::vector<Topology::LP>& lps) {
std::vector<PerPackage> empty;
Remapper packages;
for (size_t lp = 0; lp < lps.size(); ++lp) {
if (!packages(kPackage, lp, &lps[lp].package)) return empty;
}
std::vector<PerPackage> per_package(packages.Num());
for (size_t lp = 0; lp < lps.size(); ++lp) {
PerPackage& pp = per_package[lps[lp].package];
if (!pp.clusters(kCluster, lp, &lps[lp].cluster)) return empty;
if (!pp.cores(kCore, lp, &lps[lp].core)) return empty;
// SMT ID is how many LP we have already seen assigned to the same core.
HWY_ASSERT(lps[lp].core < kMaxLogicalProcessors);
lps[lp].smt = pp.smt_per_core[lps[lp].core]++;
HWY_ASSERT(lps[lp].smt < 16);
}
return per_package;
}
// Sets LP.node for all `lps`.
void SetNodes(std::vector<Topology::LP>& lps) {
// For each NUMA node found via sysfs:
for (size_t node = 0;; node++) {
// Read its cpulist so we can scatter `node` to all its `lps`.
char buf200[200];
const size_t bytes_read = ReadSysfs(kNode, node, buf200);
if (bytes_read == 0) break;
constexpr size_t kNotFound = ~size_t{0};
size_t pos = 0;
// Returns first `found_pos >= pos` where `buf200[found_pos] == c`, or
// `kNotFound`.
const auto find = [buf200, &pos](char c) -> size_t {
const char* found_ptr = strchr(buf200 + pos, c);
if (found_ptr == nullptr) return kNotFound;
HWY_ASSERT(found_ptr >= buf200);
const size_t found_pos = static_cast<size_t>(found_ptr - buf200);
HWY_ASSERT(found_pos >= pos && buf200[found_pos] == c);
return found_pos;
};
// Reads LP number and advances `pos`. `end` is for verifying we did not
// read past a known terminator, or the end of string.
const auto parse_lp = [buf200, bytes_read, &pos,
&lps](size_t end) -> size_t {
end = HWY_MIN(end, bytes_read);
size_t lp;
HWY_ASSERT(ParseDigits(buf200, end, pos, &lp));
HWY_IF_CONSTEXPR(HWY_ARCH_RISCV) {
// On RISC-V, both TotalLogicalProcessors and GetThreadAffinity may
// under-report the count, hence clamp.
lp = HWY_MIN(lp, lps.size() - 1);
}
HWY_ASSERT(lp < lps.size());
HWY_ASSERT(pos <= end);
return lp;
};
// Parse all [first-]last separated by commas.
for (;;) {
// Single number or first of range: ends with dash, comma, or end.
const size_t lp_range_first = parse_lp(HWY_MIN(find('-'), find(',')));
if (buf200[pos] == '-') { // range
++pos; // skip dash
// Last of range ends with comma or end.
const size_t lp_range_last = parse_lp(find(','));
for (size_t lp = lp_range_first; lp <= lp_range_last; ++lp) {
lps[lp].node = static_cast<uint8_t>(node);
}
} else { // single number
lps[lp_range_first].node = static_cast<uint8_t>(node);
}
// Done if reached end of string.
if (pos == bytes_read || buf200[pos] == '\0' || buf200[pos] == '\n') {
break;
}
// Comma means at least one more term is coming.
if (buf200[pos] == ',') {
++pos;
continue;
}
HWY_ABORT("Unexpected character at %zu in %s\n", pos, buf200);
} // for pos
} // for node
}
} // namespace
#endif // HWY_OS_LINUX
HWY_CONTRIB_DLLEXPORT Topology::Topology() {
#if HWY_OS_LINUX
lps.resize(TotalLogicalProcessors());
const std::vector<PerPackage>& per_package = DetectPackages(lps);
if (per_package.empty()) return;
SetNodes(lps);
// Allocate per-package/cluster/core vectors. This indicates to callers that
// detection succeeded.
packages.resize(per_package.size());
for (size_t p = 0; p < packages.size(); ++p) {
packages[p].clusters.resize(per_package[p].clusters.Num());
packages[p].cores.resize(per_package[p].cores.Num());
}
// Populate the per-cluster/core sets of LP.
for (size_t lp = 0; lp < lps.size(); ++lp) {
Package& p = packages[lps[lp].package];
p.clusters[lps[lp].cluster].lps.Set(lp);
p.cores[lps[lp].core].lps.Set(lp);
}
// Detect cache sizes (only once per cluster)
for (size_t ip = 0; ip < packages.size(); ++ip) {
Package& p = packages[ip];
for (size_t ic = 0; ic < p.clusters.size(); ++ic) {
Cluster& c = p.clusters[ic];
const size_t lp = c.lps.First();
size_t bytes;
if (ReadNumberWithOptionalSuffix(kL2Size, lp, &bytes)) {
c.private_kib = bytes >> 10;
}
if (ReadNumberWithOptionalSuffix(kL3Size, lp, &bytes)) {
c.shared_kib = bytes >> 10;
}
}
}
#endif
}
} // namespace hwy

108
deps/v8/third_party/highway/src/hwy/contrib/thread_pool/topology.h vendored Normal file
View File

@ -0,0 +1,108 @@
// Copyright 2024 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_CONTRIB_THREAD_POOL_TOPOLOGY_H_
#define HIGHWAY_HWY_CONTRIB_THREAD_POOL_TOPOLOGY_H_
// OS-specific functions for processor topology and thread affinity.
#include <stddef.h>
#include <vector>
#include "hwy/base.h"
#include "hwy/bit_set.h"
namespace hwy {
// Returns false if std::thread should not be used.
HWY_CONTRIB_DLLEXPORT bool HaveThreadingSupport();
// Upper bound on logical processors, including hyperthreads.
static constexpr size_t kMaxLogicalProcessors = 1024; // matches glibc
// Set used by Get/SetThreadAffinity.
using LogicalProcessorSet = BitSet4096<kMaxLogicalProcessors>;
// Returns false, or sets `lps` to all logical processors which are online and
// available to the current thread.
HWY_CONTRIB_DLLEXPORT bool GetThreadAffinity(LogicalProcessorSet& lps);
// Ensures the current thread can only run on the logical processors in `lps`.
// Returns false if not supported (in particular on Apple), or if the
// intersection between `lps` and `GetThreadAffinity` is the empty set.
HWY_CONTRIB_DLLEXPORT bool SetThreadAffinity(const LogicalProcessorSet& lps);
// Returns false, or ensures the current thread will only run on `lp`, which
// must not exceed `TotalLogicalProcessors`. Note that this merely calls
// `SetThreadAffinity`, see the comment there.
static inline bool PinThreadToLogicalProcessor(size_t lp) {
LogicalProcessorSet lps;
lps.Set(lp);
return SetThreadAffinity(lps);
}
// Returns 1 if unknown, otherwise the total number of logical processors
// provided by the hardware clamped to `kMaxLogicalProcessors`.
// These processors are not necessarily all usable; you can determine which are
// via GetThreadAffinity().
HWY_CONTRIB_DLLEXPORT size_t TotalLogicalProcessors();
struct Topology {
// Caller must check packages.empty(); if so, do not use any fields.
HWY_CONTRIB_DLLEXPORT Topology();
// Clique of cores with lower latency to each other. On Apple M1 these are
// four cores sharing an L2. On Zen4 these 'CCX' are up to eight cores sharing
// an L3 and a memory controller, or for Zen4c up to 16 and half the L3 size.
struct Cluster {
LogicalProcessorSet lps;
uint64_t private_kib = 0; // 0 if unknown
uint64_t shared_kib = 0; // 0 if unknown
uint64_t reserved1 = 0;
uint64_t reserved2 = 0;
uint64_t reserved3 = 0;
};
struct Core {
LogicalProcessorSet lps;
uint64_t reserved = 0;
};
struct Package {
std::vector<Cluster> clusters;
std::vector<Core> cores;
};
std::vector<Package> packages;
// Several hundred instances, so prefer a compact representation.
#pragma pack(push, 1)
struct LP {
uint16_t cluster = 0; // < packages[package].clusters.size()
uint16_t core = 0; // < packages[package].cores.size()
uint8_t package = 0; // < packages.size()
uint8_t smt = 0; // < packages[package].cores[core].lps.Count()
uint8_t node = 0;
uint8_t reserved = 0;
};
#pragma pack(pop)
std::vector<LP> lps; // size() == TotalLogicalProcessors().
};
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_THREAD_POOL_TOPOLOGY_H_

89
deps/v8/third_party/highway/src/hwy/contrib/thread_pool/topology_test.cc vendored Normal file
View File

@ -0,0 +1,89 @@
// Copyright 2024 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/thread_pool/topology.h"
#include <stddef.h>
#include <stdio.h>
#include <vector>
#include "hwy/base.h"
#include "hwy/tests/hwy_gtest.h"
#include "hwy/tests/test_util-inl.h"
#include "hwy/timer.h"
namespace hwy {
namespace {
TEST(TopologyTest, TestNum) {
const size_t total = TotalLogicalProcessors();
fprintf(stderr, "TotalLogical %zu\n", total);
LogicalProcessorSet lps;
if (GetThreadAffinity(lps)) {
fprintf(stderr, "Active %zu\n", lps.Count());
HWY_ASSERT(lps.Count() <= total);
}
}
TEST(TopologyTest, TestTopology) {
char cpu100[100];
if (hwy::platform::GetCpuString(cpu100)) {
fprintf(stderr, "%s\n", cpu100);
}
Topology topology;
if (topology.packages.empty()) return;
HWY_ASSERT(!topology.lps.empty());
LogicalProcessorSet nodes;
for (size_t lp = 0; lp < topology.lps.size(); ++lp) {
const size_t node = static_cast<size_t>(topology.lps[lp].node);
if (!nodes.Get(node)) {
fprintf(stderr, "Found NUMA node %zu, LP %zu\n", node, lp);
nodes.Set(node);
}
}
size_t lps_by_cluster = 0;
size_t lps_by_core = 0;
LogicalProcessorSet all_lps;
for (size_t p = 0; p < topology.packages.size(); ++p) {
const Topology::Package& pkg = topology.packages[p];
HWY_ASSERT(!pkg.clusters.empty());
HWY_ASSERT(!pkg.cores.empty());
HWY_ASSERT(pkg.clusters.size() <= pkg.cores.size());
for (const Topology::Cluster& c : pkg.clusters) {
lps_by_cluster += c.lps.Count();
c.lps.Foreach([&all_lps](size_t lp) { all_lps.Set(lp); });
}
for (const Topology::Core& c : pkg.cores) {
lps_by_core += c.lps.Count();
c.lps.Foreach([&all_lps](size_t lp) { all_lps.Set(lp); });
}
}
// Ensure the per-cluster and per-core sets sum to the total.
HWY_ASSERT(lps_by_cluster == topology.lps.size());
HWY_ASSERT(lps_by_core == topology.lps.size());
// .. and are a partition of unity (all LPs are covered)
HWY_ASSERT(all_lps.Count() == topology.lps.size());
}
} // namespace
} // namespace hwy
HWY_TEST_MAIN();

31
deps/v8/third_party/highway/src/hwy/contrib/unroller/README.md vendored Normal file
View File

@ -0,0 +1,31 @@
# Unroller
All contents of the `unroller` folder are experimental and subject to changes.
`Unroller` is a templated function that automatically implements common optimizations that are usually handled by compilers when writing scalar code. Modern CPUs operate much more efficiently when non-dependent calculations are packed into an instruction pipeline. For scalar code, this often means a compiler will take a one-line loop, and compile it down to hundreds of lines of machine code in order to fully capture these efficiencies.
As of today (2023-07-06), compilers are not nearly as good at implementing these optimizations for code written in SIMD intrinsics. `Unroller` is a templated function that takes in an `UnrollerUnit` of SIMD instructions, and then implements unrolling, reordering, hoisting and tail-handling (URHT optimizations) of arrays of data being processed with SIMD intrinsics.
### `UnrollerUnit`
`UnrollerUnit` and `UnrollerUnit2D` are a base classes of functions that `Unroller` needs implemented in order to properly handle URHT. `UnrollerUnit` has default implementations for all but the `Func` method, which defines the SIMD operation to be applied. Many examples of how to implement these functions are in the tests.
### Doubling values of an array example
```
struct DoubleUnit : UnrollerUnit<DoubleUnit, int, int> {
using TT = ScalableTag<int>;
inline Vec<TT> Func(ptrdiff_t idx, Vec<TT> x, Vec<TT> y) {
TT d;
return Mul(x, Set(d, 2));
}
};
```
Leaving all other methods in their default state, the following code will double all the values in array `a` and place them in `r`
```
DoubleUnit dblunit;
int r[N];
Unroller(dblunit, a, r, N);
```

470
deps/v8/third_party/highway/src/hwy/contrib/unroller/unroller-inl.h vendored Normal file
View File

@ -0,0 +1,470 @@
// Copyright 2023 Matthew Kolbe
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#if defined(HIGHWAY_HWY_CONTRIB_UNROLLER_UNROLLER_INL_H_) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_UNROLLER_UNROLLER_INL_H_
#undef HIGHWAY_HWY_CONTRIB_UNROLLER_UNROLLER_INL_H_
#else
#define HIGHWAY_HWY_CONTRIB_UNROLLER_UNROLLER_INL_H_
#endif
#include <cstdlib> // std::abs
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace hn = hwy::HWY_NAMESPACE;
template <class DERIVED, typename IN_T, typename OUT_T>
struct UnrollerUnit {
static constexpr size_t kMaxTSize = HWY_MAX(sizeof(IN_T), sizeof(OUT_T));
using LargerT = SignedFromSize<kMaxTSize>; // only the size matters.
DERIVED* me() { return static_cast<DERIVED*>(this); }
static constexpr size_t MaxUnitLanes() {
return HWY_MAX_LANES_D(hn::ScalableTag<LargerT>);
}
static size_t ActualLanes() { return Lanes(hn::ScalableTag<LargerT>()); }
using LargerD = hn::CappedTag<LargerT, MaxUnitLanes()>;
using IT = hn::Rebind<IN_T, LargerD>;
using OT = hn::Rebind<OUT_T, LargerD>;
IT d_in;
OT d_out;
using Y_VEC = hn::Vec<OT>;
using X_VEC = hn::Vec<IT>;
Y_VEC Func(const ptrdiff_t idx, const X_VEC x, const Y_VEC y) {
return me()->Func(idx, x, y);
}
X_VEC X0Init() { return me()->X0InitImpl(); }
X_VEC X0InitImpl() { return hn::Zero(d_in); }
Y_VEC YInit() { return me()->YInitImpl(); }
Y_VEC YInitImpl() { return hn::Zero(d_out); }
X_VEC Load(const ptrdiff_t idx, IN_T* from) {
return me()->LoadImpl(idx, from);
}
X_VEC LoadImpl(const ptrdiff_t idx, IN_T* from) {
return hn::LoadU(d_in, from + idx);
}
// MaskLoad can take in either a positive or negative number for `places`. if
// the number is positive, then it loads the top `places` values, and if it's
// negative, it loads the bottom |places| values. example: places = 3
// | o | o | o | x | x | x | x | x |
// example places = -3
// | x | x | x | x | x | o | o | o |
X_VEC MaskLoad(const ptrdiff_t idx, IN_T* from, const ptrdiff_t places) {
return me()->MaskLoadImpl(idx, from, places);
}
X_VEC MaskLoadImpl(const ptrdiff_t idx, IN_T* from, const ptrdiff_t places) {
auto mask = hn::FirstN(d_in, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d_in,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
return hn::MaskedLoad(mask, d_in, from + idx);
}
bool StoreAndShortCircuit(const ptrdiff_t idx, OUT_T* to, const Y_VEC x) {
return me()->StoreAndShortCircuitImpl(idx, to, x);
}
bool StoreAndShortCircuitImpl(const ptrdiff_t idx, OUT_T* to, const Y_VEC x) {
hn::StoreU(x, d_out, to + idx);
return true;
}
ptrdiff_t MaskStore(const ptrdiff_t idx, OUT_T* to, const Y_VEC x,
ptrdiff_t const places) {
return me()->MaskStoreImpl(idx, to, x, places);
}
ptrdiff_t MaskStoreImpl(const ptrdiff_t idx, OUT_T* to, const Y_VEC x,
const ptrdiff_t places) {
auto mask = hn::FirstN(d_out, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d_out,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
hn::BlendedStore(x, mask, d_out, to + idx);
return std::abs(places);
}
ptrdiff_t Reduce(const Y_VEC x, OUT_T* to) { return me()->ReduceImpl(x, to); }
ptrdiff_t ReduceImpl(const Y_VEC x, OUT_T* to) {
// default does nothing
(void)x;
(void)to;
return 0;
}
void Reduce(const Y_VEC x0, const Y_VEC x1, const Y_VEC x2, Y_VEC* y) {
me()->ReduceImpl(x0, x1, x2, y);
}
void ReduceImpl(const Y_VEC x0, const Y_VEC x1, const Y_VEC x2, Y_VEC* y) {
// default does nothing
(void)x0;
(void)x1;
(void)x2;
(void)y;
}
};
template <class DERIVED, typename IN0_T, typename IN1_T, typename OUT_T>
struct UnrollerUnit2D {
DERIVED* me() { return static_cast<DERIVED*>(this); }
static constexpr size_t kMaxTSize =
HWY_MAX(sizeof(IN0_T), HWY_MAX(sizeof(IN1_T), sizeof(OUT_T)));
using LargerT = SignedFromSize<kMaxTSize>; // only the size matters.
static constexpr size_t MaxUnitLanes() {
return HWY_MAX_LANES_D(hn::ScalableTag<LargerT>);
}
static size_t ActualLanes() { return Lanes(hn::ScalableTag<LargerT>()); }
using LargerD = hn::CappedTag<LargerT, MaxUnitLanes()>;
using I0T = hn::Rebind<IN0_T, LargerD>;
using I1T = hn::Rebind<IN1_T, LargerD>;
using OT = hn::Rebind<OUT_T, LargerD>;
I0T d_in0;
I1T d_in1;
OT d_out;
using Y_VEC = hn::Vec<OT>;
using X0_VEC = hn::Vec<I0T>;
using X1_VEC = hn::Vec<I1T>;
hn::Vec<OT> Func(const ptrdiff_t idx, const hn::Vec<I0T> x0,
const hn::Vec<I1T> x1, const Y_VEC y) {
return me()->Func(idx, x0, x1, y);
}
X0_VEC X0Init() { return me()->X0InitImpl(); }
X0_VEC X0InitImpl() { return hn::Zero(d_in0); }
X1_VEC X1Init() { return me()->X1InitImpl(); }
X1_VEC X1InitImpl() { return hn::Zero(d_in1); }
Y_VEC YInit() { return me()->YInitImpl(); }
Y_VEC YInitImpl() { return hn::Zero(d_out); }
X0_VEC Load0(const ptrdiff_t idx, IN0_T* from) {
return me()->Load0Impl(idx, from);
}
X0_VEC Load0Impl(const ptrdiff_t idx, IN0_T* from) {
return hn::LoadU(d_in0, from + idx);
}
X1_VEC Load1(const ptrdiff_t idx, IN1_T* from) {
return me()->Load1Impl(idx, from);
}
X1_VEC Load1Impl(const ptrdiff_t idx, IN1_T* from) {
return hn::LoadU(d_in1, from + idx);
}
// maskload can take in either a positive or negative number for `places`. if
// the number is positive, then it loads the top `places` values, and if it's
// negative, it loads the bottom |places| values. example: places = 3
// | o | o | o | x | x | x | x | x |
// example places = -3
// | x | x | x | x | x | o | o | o |
X0_VEC MaskLoad0(const ptrdiff_t idx, IN0_T* from, const ptrdiff_t places) {
return me()->MaskLoad0Impl(idx, from, places);
}
X0_VEC MaskLoad0Impl(const ptrdiff_t idx, IN0_T* from,
const ptrdiff_t places) {
auto mask = hn::FirstN(d_in0, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d_in0,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
return hn::MaskedLoad(mask, d_in0, from + idx);
}
hn::Vec<I1T> MaskLoad1(const ptrdiff_t idx, IN1_T* from,
const ptrdiff_t places) {
return me()->MaskLoad1Impl(idx, from, places);
}
hn::Vec<I1T> MaskLoad1Impl(const ptrdiff_t idx, IN1_T* from,
const ptrdiff_t places) {
auto mask = hn::FirstN(d_in1, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d_in1,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
return hn::MaskedLoad(mask, d_in1, from + idx);
}
// store returns a bool that is `false` when
bool StoreAndShortCircuit(const ptrdiff_t idx, OUT_T* to, const Y_VEC x) {
return me()->StoreAndShortCircuitImpl(idx, to, x);
}
bool StoreAndShortCircuitImpl(const ptrdiff_t idx, OUT_T* to, const Y_VEC x) {
hn::StoreU(x, d_out, to + idx);
return true;
}
ptrdiff_t MaskStore(const ptrdiff_t idx, OUT_T* to, const Y_VEC x,
const ptrdiff_t places) {
return me()->MaskStoreImpl(idx, to, x, places);
}
ptrdiff_t MaskStoreImpl(const ptrdiff_t idx, OUT_T* to, const Y_VEC x,
const ptrdiff_t places) {
auto mask = hn::FirstN(d_out, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d_out,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
hn::BlendedStore(x, mask, d_out, to + idx);
return std::abs(places);
}
ptrdiff_t Reduce(const Y_VEC x, OUT_T* to) { return me()->ReduceImpl(x, to); }
ptrdiff_t ReduceImpl(const Y_VEC x, OUT_T* to) {
// default does nothing
(void)x;
(void)to;
return 0;
}
void Reduce(const Y_VEC x0, const Y_VEC x1, const Y_VEC x2, Y_VEC* y) {
me()->ReduceImpl(x0, x1, x2, y);
}
void ReduceImpl(const Y_VEC x0, const Y_VEC x1, const Y_VEC x2, Y_VEC* y) {
// default does nothing
(void)x0;
(void)x1;
(void)x2;
(void)y;
}
};
template <class FUNC, typename IN_T, typename OUT_T>
inline void Unroller(FUNC& f, IN_T* HWY_RESTRICT x, OUT_T* HWY_RESTRICT y,
const ptrdiff_t n) {
auto xx = f.X0Init();
auto yy = f.YInit();
ptrdiff_t i = 0;
#if HWY_MEM_OPS_MIGHT_FAULT
constexpr auto lane_sz =
static_cast<ptrdiff_t>(RemoveRef<FUNC>::MaxUnitLanes());
if (n < lane_sz) {
const DFromV<decltype(yy)> d;
// this may not fit on the stack for HWY_RVV, but we do not reach this code
// there
HWY_ALIGN IN_T xtmp[static_cast<size_t>(lane_sz)];
HWY_ALIGN OUT_T ytmp[static_cast<size_t>(lane_sz)];
CopyBytes(x, xtmp, static_cast<size_t>(n) * sizeof(IN_T));
xx = f.MaskLoad(0, xtmp, n);
yy = f.Func(0, xx, yy);
Store(Zero(d), d, ytmp);
i += f.MaskStore(0, ytmp, yy, n);
i += f.Reduce(yy, ytmp);
CopyBytes(ytmp, y, static_cast<size_t>(i) * sizeof(OUT_T));
return;
}
#endif
const ptrdiff_t actual_lanes =
static_cast<ptrdiff_t>(RemoveRef<FUNC>::ActualLanes());
if (n > 4 * actual_lanes) {
auto xx1 = f.X0Init();
auto yy1 = f.YInit();
auto xx2 = f.X0Init();
auto yy2 = f.YInit();
auto xx3 = f.X0Init();
auto yy3 = f.YInit();
while (i + 4 * actual_lanes - 1 < n) {
xx = f.Load(i, x);
i += actual_lanes;
xx1 = f.Load(i, x);
i += actual_lanes;
xx2 = f.Load(i, x);
i += actual_lanes;
xx3 = f.Load(i, x);
i -= 3 * actual_lanes;
yy = f.Func(i, xx, yy);
yy1 = f.Func(i + actual_lanes, xx1, yy1);
yy2 = f.Func(i + 2 * actual_lanes, xx2, yy2);
yy3 = f.Func(i + 3 * actual_lanes, xx3, yy3);
if (!f.StoreAndShortCircuit(i, y, yy)) return;
i += actual_lanes;
if (!f.StoreAndShortCircuit(i, y, yy1)) return;
i += actual_lanes;
if (!f.StoreAndShortCircuit(i, y, yy2)) return;
i += actual_lanes;
if (!f.StoreAndShortCircuit(i, y, yy3)) return;
i += actual_lanes;
}
f.Reduce(yy3, yy2, yy1, &yy);
}
while (i + actual_lanes - 1 < n) {
xx = f.Load(i, x);
yy = f.Func(i, xx, yy);
if (!f.StoreAndShortCircuit(i, y, yy)) return;
i += actual_lanes;
}
if (i != n) {
xx = f.MaskLoad(n - actual_lanes, x, i - n);
yy = f.Func(n - actual_lanes, xx, yy);
f.MaskStore(n - actual_lanes, y, yy, i - n);
}
f.Reduce(yy, y);
}
template <class FUNC, typename IN0_T, typename IN1_T, typename OUT_T>
inline void Unroller(FUNC& HWY_RESTRICT f, IN0_T* HWY_RESTRICT x0,
IN1_T* HWY_RESTRICT x1, OUT_T* HWY_RESTRICT y,
const ptrdiff_t n) {
const ptrdiff_t lane_sz =
static_cast<ptrdiff_t>(RemoveRef<FUNC>::ActualLanes());
auto xx00 = f.X0Init();
auto xx10 = f.X1Init();
auto yy = f.YInit();
ptrdiff_t i = 0;
#if HWY_MEM_OPS_MIGHT_FAULT
if (n < lane_sz) {
const DFromV<decltype(yy)> d;
// this may not fit on the stack for HWY_RVV, but we do not reach this code
// there
constexpr auto max_lane_sz =
static_cast<ptrdiff_t>(RemoveRef<FUNC>::MaxUnitLanes());
HWY_ALIGN IN0_T xtmp0[static_cast<size_t>(max_lane_sz)];
HWY_ALIGN IN1_T xtmp1[static_cast<size_t>(max_lane_sz)];
HWY_ALIGN OUT_T ytmp[static_cast<size_t>(max_lane_sz)];
CopyBytes(x0, xtmp0, static_cast<size_t>(n) * sizeof(IN0_T));
CopyBytes(x1, xtmp1, static_cast<size_t>(n) * sizeof(IN1_T));
xx00 = f.MaskLoad0(0, xtmp0, n);
xx10 = f.MaskLoad1(0, xtmp1, n);
yy = f.Func(0, xx00, xx10, yy);
Store(Zero(d), d, ytmp);
i += f.MaskStore(0, ytmp, yy, n);
i += f.Reduce(yy, ytmp);
CopyBytes(ytmp, y, static_cast<size_t>(i) * sizeof(OUT_T));
return;
}
#endif
if (n > 4 * lane_sz) {
auto xx01 = f.X0Init();
auto xx11 = f.X1Init();
auto yy1 = f.YInit();
auto xx02 = f.X0Init();
auto xx12 = f.X1Init();
auto yy2 = f.YInit();
auto xx03 = f.X0Init();
auto xx13 = f.X1Init();
auto yy3 = f.YInit();
while (i + 4 * lane_sz - 1 < n) {
xx00 = f.Load0(i, x0);
xx10 = f.Load1(i, x1);
i += lane_sz;
xx01 = f.Load0(i, x0);
xx11 = f.Load1(i, x1);
i += lane_sz;
xx02 = f.Load0(i, x0);
xx12 = f.Load1(i, x1);
i += lane_sz;
xx03 = f.Load0(i, x0);
xx13 = f.Load1(i, x1);
i -= 3 * lane_sz;
yy = f.Func(i, xx00, xx10, yy);
yy1 = f.Func(i + lane_sz, xx01, xx11, yy1);
yy2 = f.Func(i + 2 * lane_sz, xx02, xx12, yy2);
yy3 = f.Func(i + 3 * lane_sz, xx03, xx13, yy3);
if (!f.StoreAndShortCircuit(i, y, yy)) return;
i += lane_sz;
if (!f.StoreAndShortCircuit(i, y, yy1)) return;
i += lane_sz;
if (!f.StoreAndShortCircuit(i, y, yy2)) return;
i += lane_sz;
if (!f.StoreAndShortCircuit(i, y, yy3)) return;
i += lane_sz;
}
f.Reduce(yy3, yy2, yy1, &yy);
}
while (i + lane_sz - 1 < n) {
xx00 = f.Load0(i, x0);
xx10 = f.Load1(i, x1);
yy = f.Func(i, xx00, xx10, yy);
if (!f.StoreAndShortCircuit(i, y, yy)) return;
i += lane_sz;
}
if (i != n) {
xx00 = f.MaskLoad0(n - lane_sz, x0, i - n);
xx10 = f.MaskLoad1(n - lane_sz, x1, i - n);
yy = f.Func(n - lane_sz, xx00, xx10, yy);
f.MaskStore(n - lane_sz, y, yy, i - n);
}
f.Reduce(yy, y);
}
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_UNROLLER_UNROLLER_INL_H_

491
deps/v8/third_party/highway/src/hwy/contrib/unroller/unroller_test.cc vendored Normal file
View File

@ -0,0 +1,491 @@
// Copyright Google LLC 2021
// Matthew Kolbe 2023
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <vector>
#include "hwy/base.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/unroller/unroller_test.cc" //NOLINT
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/unroller/unroller-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
template <typename T>
T SimpleDot(const T* pa, const T* pb, size_t num) {
T sum = 0;
for (size_t i = 0; i < num; ++i) {
// For reasons unknown, fp16 += does not compile on clang (Arm).
sum = ConvertScalarTo<T>(sum + pa[i] * pb[i]);
}
return sum;
}
template <typename T>
T SimpleAcc(const T* pa, size_t num) {
T sum = 0;
for (size_t i = 0; i < num; ++i) {
sum += pa[i];
}
return sum;
}
template <typename T>
T SimpleMin(const T* pa, size_t num) {
T min = HighestValue<T>();
for (size_t i = 0; i < num; ++i) {
if (min > pa[i]) min = pa[i];
}
return min;
}
template <typename T>
struct MultiplyUnit : UnrollerUnit2D<MultiplyUnit<T>, T, T, T> {
using TT = hn::ScalableTag<T>;
HWY_INLINE hn::Vec<TT> Func(ptrdiff_t idx, const hn::Vec<TT> x0,
const hn::Vec<TT> x1, const hn::Vec<TT> y) {
(void)idx;
(void)y;
return hn::Mul(x0, x1);
}
};
template <typename FROM_T, typename TO_T>
struct ConvertUnit : UnrollerUnit<ConvertUnit<FROM_T, TO_T>, FROM_T, TO_T> {
using Base = UnrollerUnit<ConvertUnit<FROM_T, TO_T>, FROM_T, TO_T>;
using Base::MaxUnitLanes;
using typename Base::LargerD;
using TT_FROM = hn::Rebind<FROM_T, LargerD>;
using TT_TO = hn::Rebind<TO_T, LargerD>;
template <
class ToD, class FromV,
hwy::EnableIf<(sizeof(TFromV<FromV>) > sizeof(TFromD<ToD>))>* = nullptr>
static HWY_INLINE hn::Vec<ToD> DoConvertVector(ToD d, FromV v) {
return hn::DemoteTo(d, v);
}
template <
class ToD, class FromV,
hwy::EnableIf<(sizeof(TFromV<FromV>) == sizeof(TFromD<ToD>))>* = nullptr>
static HWY_INLINE hn::Vec<ToD> DoConvertVector(ToD d, FromV v) {
return hn::ConvertTo(d, v);
}
template <
class ToD, class FromV,
hwy::EnableIf<(sizeof(TFromV<FromV>) < sizeof(TFromD<ToD>))>* = nullptr>
static HWY_INLINE hn::Vec<ToD> DoConvertVector(ToD d, FromV v) {
return hn::PromoteTo(d, v);
}
hn::Vec<TT_TO> Func(ptrdiff_t idx, const hn::Vec<TT_FROM> x,
const hn::Vec<TT_TO> y) {
(void)idx;
(void)y;
TT_TO d;
return DoConvertVector(d, x);
}
};
// Returns a value that does not compare equal to `value`.
template <class D, HWY_IF_FLOAT_D(D)>
HWY_INLINE Vec<D> OtherValue(D d, TFromD<D> /*value*/) {
return NaN(d);
}
template <class D, HWY_IF_NOT_FLOAT_D(D)>
HWY_INLINE Vec<D> OtherValue(D d, TFromD<D> value) {
return hn::Set(d, hwy::AddWithWraparound(value, 1));
}
// Caveat: stores lane indices as MakeSigned<T>, which may overflow for 8-bit T
// on HWY_RVV.
template <typename T>
struct FindUnit : UnrollerUnit<FindUnit<T>, T, MakeSigned<T>> {
using TI = MakeSigned<T>;
using Base = UnrollerUnit<FindUnit<T>, T, TI>;
using Base::ActualLanes;
using Base::MaxUnitLanes;
using D = hn::CappedTag<T, MaxUnitLanes()>;
T to_find;
D d;
using DI = RebindToSigned<D>;
DI di;
FindUnit(T find) : to_find(find) {}
hn::Vec<DI> Func(ptrdiff_t idx, const hn::Vec<D> x, const hn::Vec<DI> y) {
const Mask<D> msk = hn::Eq(x, hn::Set(d, to_find));
const TI first_idx = static_cast<TI>(hn::FindFirstTrue(d, msk));
if (first_idx > -1)
return hn::Set(di, static_cast<TI>(static_cast<TI>(idx) + first_idx));
else
return y;
}
hn::Vec<D> X0InitImpl() { return OtherValue(D(), to_find); }
hn::Vec<DI> YInitImpl() { return hn::Set(di, TI{-1}); }
hn::Vec<D> MaskLoadImpl(const ptrdiff_t idx, T* from,
const ptrdiff_t places) {
auto mask = hn::FirstN(d, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
return hn::IfThenElse(mask, hn::MaskedLoad(mask, d, from + idx),
X0InitImpl());
}
bool StoreAndShortCircuitImpl(const ptrdiff_t idx, TI* to,
const hn::Vec<DI> x) {
(void)idx;
TI a = hn::GetLane(x);
to[0] = a;
if (a == -1) return true;
return false;
}
ptrdiff_t MaskStoreImpl(const ptrdiff_t idx, TI* to, const hn::Vec<DI> x,
const ptrdiff_t places) {
(void)idx;
(void)places;
TI a = hn::GetLane(x);
to[0] = a;
return 1;
}
};
template <typename T>
struct AccumulateUnit : UnrollerUnit<AccumulateUnit<T>, T, T> {
using TT = hn::ScalableTag<T>;
hn::Vec<TT> Func(ptrdiff_t idx, const hn::Vec<TT> x, const hn::Vec<TT> y) {
(void)idx;
return hn::Add(x, y);
}
bool StoreAndShortCircuitImpl(const ptrdiff_t idx, T* to,
const hn::Vec<TT> x) {
// no stores in a reducer
(void)idx;
(void)to;
(void)x;
return true;
}
ptrdiff_t MaskStoreImpl(const ptrdiff_t idx, T* to, const hn::Vec<TT> x,
const ptrdiff_t places) {
// no stores in a reducer
(void)idx;
(void)to;
(void)x;
(void)places;
return 0;
}
ptrdiff_t ReduceImpl(const hn::Vec<TT> x, T* to) {
const hn::ScalableTag<T> d;
(*to) = hn::ReduceSum(d, x);
return 1;
}
void ReduceImpl(const hn::Vec<TT> x0, const hn::Vec<TT> x1,
const hn::Vec<TT> x2, hn::Vec<TT>* y) {
(*y) = hn::Add(hn::Add(*y, x0), hn::Add(x1, x2));
}
};
template <typename T>
struct MinUnit : UnrollerUnit<MinUnit<T>, T, T> {
using Base = UnrollerUnit<MinUnit<T>, T, T>;
using Base::ActualLanes;
using TT = hn::ScalableTag<T>;
TT d;
hn::Vec<TT> Func(const ptrdiff_t idx, const hn::Vec<TT> x,
const hn::Vec<TT> y) {
(void)idx;
return hn::Min(y, x);
}
hn::Vec<TT> YInitImpl() { return hn::Set(d, HighestValue<T>()); }
hn::Vec<TT> MaskLoadImpl(const ptrdiff_t idx, T* from,
const ptrdiff_t places) {
auto mask = hn::FirstN(d, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
auto def = YInitImpl();
return hn::MaskedLoadOr(def, mask, d, from + idx);
}
bool StoreAndShortCircuitImpl(const ptrdiff_t idx, T* to,
const hn::Vec<TT> x) {
// no stores in a reducer
(void)idx;
(void)to;
(void)x;
return true;
}
ptrdiff_t MaskStoreImpl(const ptrdiff_t idx, T* to, const hn::Vec<TT> x,
const ptrdiff_t places) {
// no stores in a reducer
(void)idx;
(void)to;
(void)x;
(void)places;
return 0;
}
ptrdiff_t ReduceImpl(const hn::Vec<TT> x, T* to) {
auto minvect = hn::MinOfLanes(d, x);
(*to) = hn::ExtractLane(minvect, 0);
return 1;
}
void ReduceImpl(const hn::Vec<TT> x0, const hn::Vec<TT> x1,
const hn::Vec<TT> x2, hn::Vec<TT>* y) {
auto a = hn::Min(x1, x0);
auto b = hn::Min(*y, x2);
(*y) = hn::Min(a, b);
}
};
template <typename T>
struct DotUnit : UnrollerUnit2D<DotUnit<T>, T, T, T> {
using TT = hn::ScalableTag<T>;
hn::Vec<TT> Func(const ptrdiff_t idx, const hn::Vec<TT> x0,
const hn::Vec<TT> x1, const hn::Vec<TT> y) {
(void)idx;
return hn::MulAdd(x0, x1, y);
}
bool StoreAndShortCircuitImpl(const ptrdiff_t idx, T* to,
const hn::Vec<TT> x) {
// no stores in a reducer
(void)idx;
(void)to;
(void)x;
return true;
}
ptrdiff_t MaskStoreImpl(const ptrdiff_t idx, T* to, const hn::Vec<TT> x,
const ptrdiff_t places) {
// no stores in a reducer
(void)idx;
(void)to;
(void)x;
(void)places;
return 0;
}
ptrdiff_t ReduceImpl(const hn::Vec<TT> x, T* to) {
const hn::ScalableTag<T> d;
(*to) = hn::ReduceSum(d, x);
return 1;
}
void ReduceImpl(const hn::Vec<TT> x0, const hn::Vec<TT> x1,
const hn::Vec<TT> x2, hn::Vec<TT>* y) {
(*y) = hn::Add(hn::Add(*y, x0), hn::Add(x1, x2));
}
};
template <class D>
std::vector<size_t> Counts(D d) {
const size_t N = Lanes(d);
return std::vector<size_t>{1,
3,
7,
16,
HWY_MAX(N / 2, 1),
HWY_MAX(2 * N / 3, 1),
N,
N + 1,
4 * N / 3,
3 * N,
8 * N,
8 * N + 2,
256 * N - 1,
256 * N};
}
struct TestDot {
template <typename T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
// TODO(janwas): avoid internal compiler error
#if HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2 || HWY_COMPILER_MSVC
(void)d;
#else
RandomState rng;
const auto random_t = [&rng]() {
const int32_t bits = static_cast<int32_t>(Random32(&rng)) & 1023;
return static_cast<float>(bits - 512) * (1.0f / 64);
};
for (size_t num : Counts(d)) {
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(num);
AlignedFreeUniquePtr<T[]> pb = AllocateAligned<T>(num);
AlignedFreeUniquePtr<T[]> py = AllocateAligned<T>(num);
HWY_ASSERT(pa && pb && py);
T* a = pa.get();
T* b = pb.get();
T* y = py.get();
size_t i = 0;
for (; i < num; ++i) {
a[i] = ConvertScalarTo<T>(random_t());
b[i] = ConvertScalarTo<T>(random_t());
}
const T expected_dot = SimpleDot(a, b, num);
MultiplyUnit<T> multfn;
Unroller(multfn, a, b, y, static_cast<ptrdiff_t>(num));
AccumulateUnit<T> accfn;
T dot_via_mul_acc;
Unroller(accfn, y, &dot_via_mul_acc, static_cast<ptrdiff_t>(num));
const double tolerance = 48.0 *
ConvertScalarTo<double>(hwy::Epsilon<T>()) *
ScalarAbs(expected_dot);
HWY_ASSERT(ScalarAbs(expected_dot - dot_via_mul_acc) < tolerance);
DotUnit<T> dotfn;
T dotr;
Unroller(dotfn, a, b, &dotr, static_cast<ptrdiff_t>(num));
HWY_ASSERT(ConvertScalarTo<double>(ScalarAbs((expected_dot - dotr))) <
tolerance);
auto expected_min = SimpleMin(a, num);
MinUnit<T> minfn;
T minr;
Unroller(minfn, a, &minr, static_cast<ptrdiff_t>(num));
HWY_ASSERT(ConvertScalarTo<double>(ScalarAbs(expected_min - minr)) <
1e-7);
}
#endif
}
};
void TestAllDot() { ForFloatTypes(ForPartialVectors<TestDot>()); }
struct TestConvert {
template <typename T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
// TODO(janwas): avoid internal compiler error
#if HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2 || HWY_COMPILER_MSVC
(void)d;
#else
for (size_t num : Counts(d)) {
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(num);
AlignedFreeUniquePtr<int[]> pto = AllocateAligned<int>(num);
HWY_ASSERT(pa && pto);
T* HWY_RESTRICT a = pa.get();
int* HWY_RESTRICT to = pto.get();
for (size_t i = 0; i < num; ++i) {
a[i] = ConvertScalarTo<T>(static_cast<double>(i) * 0.25);
}
ConvertUnit<T, int> cvtfn;
Unroller(cvtfn, a, to, static_cast<ptrdiff_t>(num));
for (size_t i = 0; i < num; ++i) {
// TODO(janwas): RVV QEMU fcvt_rtz appears to 'truncate' 4.75 to 5.
HWY_ASSERT(
static_cast<int>(a[i]) == to[i] ||
(HWY_TARGET == HWY_RVV && static_cast<int>(a[i]) == to[i] - 1));
}
ConvertUnit<int, T> cvtbackfn;
Unroller(cvtbackfn, to, a, static_cast<ptrdiff_t>(num));
for (size_t i = 0; i < num; ++i) {
HWY_ASSERT_EQ(ConvertScalarTo<T>(to[i]), a[i]);
}
}
#endif
}
};
void TestAllConvert() { ForFloat3264Types(ForPartialVectors<TestConvert>()); }
struct TestFind {
template <typename T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
for (size_t num : Counts(d)) {
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(num);
HWY_ASSERT(pa);
T* a = pa.get();
for (size_t i = 0; i < num; ++i) a[i] = ConvertScalarTo<T>(i);
FindUnit<T> cvtfn(ConvertScalarTo<T>(num - 1));
MakeSigned<T> idx = 0;
Unroller(cvtfn, a, &idx, static_cast<ptrdiff_t>(num));
HWY_ASSERT(static_cast<MakeUnsigned<T>>(idx) < num);
HWY_ASSERT(a[idx] == ConvertScalarTo<T>(num - 1));
FindUnit<T> cvtfnzero((T)(0));
Unroller(cvtfnzero, a, &idx, static_cast<ptrdiff_t>(num));
HWY_ASSERT(static_cast<MakeUnsigned<T>>(idx) < num);
HWY_ASSERT(a[idx] == (T)(0));
// For f16, we cannot search for `num` because it may round to a value
// that is actually in the (large) array.
FindUnit<T> cvtfnnotin(HighestValue<T>());
Unroller(cvtfnnotin, a, &idx, static_cast<ptrdiff_t>(num));
HWY_ASSERT(idx == -1);
}
}
};
void TestAllFind() { ForFloatTypes(ForPartialVectors<TestFind>()); }
} // namespace
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(UnrollerTest);
HWY_EXPORT_AND_TEST_P(UnrollerTest, TestAllDot);
HWY_EXPORT_AND_TEST_P(UnrollerTest, TestAllConvert);
HWY_EXPORT_AND_TEST_P(UnrollerTest, TestAllFind);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE