Upload Kmake

deps/v8/third_party/highway/src/hwy/tests/arithmetic_test.cc

@@ -0,0 +1,329 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/arithmetic_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/nanobenchmark.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestPlusMinus {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v2 = Iota(d, hwy::Unpredictable1() + 1);
+    const auto v3 = Iota(d, hwy::Unpredictable1() + 2);
+    const auto v4 = Iota(d, hwy::Unpredictable1() + 3);
+
+    const size_t N = Lanes(d);
+    auto lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(lanes);
+    for (size_t i = 0; i < N; ++i) {
+      lanes[i] = ConvertScalarTo<T>((2 + i) + (3 + i));
+    }
+    HWY_ASSERT_VEC_EQ(d, lanes.get(), Add(v2, v3));
+    HWY_ASSERT_VEC_EQ(d, Set(d, ConvertScalarTo<T>(2)), Sub(v4, v2));
+
+    for (size_t i = 0; i < N; ++i) {
+      lanes[i] = ConvertScalarTo<T>((2 + i) + (4 + i));
+    }
+    auto sum = v2;
+    sum = Add(sum, v4);  // sum == 6,8..
+    HWY_ASSERT_VEC_EQ(d, Load(d, lanes.get()), sum);
+
+    sum = Sub(sum, v4);
+    HWY_ASSERT_VEC_EQ(d, v2, sum);
+  }
+};
+
+struct TestPlusMinusOverflow {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v1 = Iota(d, 1);
+    const auto vMax = Iota(d, LimitsMax<T>());
+    const auto vMin = Iota(d, LimitsMin<T>());
+
+    // Check that no UB triggered.
+    // "assert" here is formal - to avoid compiler dropping calculations
+    HWY_ASSERT_VEC_EQ(d, Add(v1, vMax), Add(vMax, v1));
+    HWY_ASSERT_VEC_EQ(d, Add(vMax, vMax), Add(vMax, vMax));
+    HWY_ASSERT_VEC_EQ(d, Sub(vMin, v1), Sub(vMin, v1));
+    HWY_ASSERT_VEC_EQ(d, Sub(vMin, vMax), Sub(vMin, vMax));
+  }
+};
+
+HWY_NOINLINE void TestAllPlusMinus() {
+  ForAllTypes(ForPartialVectors<TestPlusMinus>());
+  ForIntegerTypes(ForPartialVectors<TestPlusMinusOverflow>());
+}
+
+struct TestAddSub {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v2 = Iota(d, 2);
+    const auto v4 = Iota(d, 4);
+
+    const size_t N = Lanes(d);
+    auto lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(lanes);
+    for (size_t i = 0; i < N; ++i) {
+      lanes[i] = ConvertScalarTo<T>(((i & 1) == 0) ? 2 : ((4 + i) + (2 + i)));
+    }
+    HWY_ASSERT_VEC_EQ(d, lanes.get(), AddSub(v4, v2));
+
+    for (size_t i = 0; i < N; ++i) {
+      if ((i & 1) == 0) {
+        lanes[i] = ConvertScalarTo<T>(-2);
+      }
+    }
+    HWY_ASSERT_VEC_EQ(d, lanes.get(), AddSub(v2, v4));
+  }
+};
+
+HWY_NOINLINE void TestAllAddSub() {
+  ForAllTypes(ForPartialVectors<TestAddSub>());
+}
+
+struct TestAverage {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TI = MakeSigned<T>;
+
+    const RebindToSigned<decltype(d)> di;
+    const RebindToUnsigned<decltype(d)> du;
+
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> v1 = Set(d, static_cast<T>(1));
+    const Vec<D> v2 = Set(d, static_cast<T>(2));
+
+    const Vec<D> vn1 = Set(d, static_cast<T>(-1));
+    const Vec<D> vn2 = Set(d, static_cast<T>(-2));
+    const Vec<D> vn3 = Set(d, static_cast<T>(-3));
+    const Vec<D> vn4 = Set(d, static_cast<T>(-4));
+
+    HWY_ASSERT_VEC_EQ(d, v0, AverageRound(v0, v0));
+    HWY_ASSERT_VEC_EQ(d, v1, AverageRound(v0, v1));
+    HWY_ASSERT_VEC_EQ(d, v1, AverageRound(v1, v1));
+    HWY_ASSERT_VEC_EQ(d, v2, AverageRound(v1, v2));
+    HWY_ASSERT_VEC_EQ(d, v2, AverageRound(v2, v2));
+
+    HWY_ASSERT_VEC_EQ(d, vn1, AverageRound(vn1, vn1));
+    HWY_ASSERT_VEC_EQ(d, vn1, AverageRound(vn1, vn2));
+    HWY_ASSERT_VEC_EQ(d, vn2, AverageRound(vn1, vn3));
+    HWY_ASSERT_VEC_EQ(d, vn2, AverageRound(vn1, vn4));
+    HWY_ASSERT_VEC_EQ(d, vn2, AverageRound(vn2, vn2));
+    HWY_ASSERT_VEC_EQ(d, vn3, AverageRound(vn2, vn4));
+
+    const T kSignedMax = static_cast<T>(LimitsMax<TI>());
+
+    const Vec<D> v_iota1 = Iota(d, static_cast<T>(1));
+    Vec<D> v_neg_even = BitCast(d, Neg(BitCast(di, Add(v_iota1, v_iota1))));
+    HWY_IF_CONSTEXPR(HWY_MAX_LANES_D(D) > static_cast<size_t>(kSignedMax)) {
+      v_neg_even = Or(v_neg_even, SignBit(d));
+    }
+
+    const Vec<D> v_pos_even = And(v_neg_even, Set(d, kSignedMax));
+    const Vec<D> v_pos_odd = Or(v_pos_even, v1);
+
+    const Vec<D> expected_even =
+        Add(ShiftRight<1>(v_neg_even),
+            BitCast(d, ShiftRight<1>(BitCast(du, v_pos_even))));
+
+    HWY_ASSERT_VEC_EQ(d, expected_even, AverageRound(v_neg_even, v_pos_even));
+    HWY_ASSERT_VEC_EQ(d, Add(expected_even, v1),
+                      AverageRound(v_neg_even, v_pos_odd));
+  }
+};
+
+HWY_NOINLINE void TestAllAverage() {
+  ForIntegerTypes(ForPartialVectors<TestAverage>());
+}
+
+struct TestAbs {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> vp1 = Set(d, static_cast<T>(1));
+    const Vec<D> vn1 = Set(d, static_cast<T>(-1));
+    const Vec<D> vpm = Set(d, LimitsMax<T>());
+    const Vec<D> vnm = Set(d, LimitsMin<T>());
+
+    HWY_ASSERT_VEC_EQ(d, v0, Abs(v0));
+    HWY_ASSERT_VEC_EQ(d, vp1, Abs(vp1));
+    HWY_ASSERT_VEC_EQ(d, vp1, Abs(vn1));
+    HWY_ASSERT_VEC_EQ(d, vpm, Abs(vpm));
+    HWY_ASSERT_VEC_EQ(d, vnm, Abs(vnm));
+  }
+};
+
+struct TestFloatAbs {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> vp1 = Set(d, ConvertScalarTo<T>(1));
+    const Vec<D> vn1 = Set(d, ConvertScalarTo<T>(-1));
+    const Vec<D> vp2 = Set(d, ConvertScalarTo<T>(0.01));
+    const Vec<D> vn2 = Set(d, ConvertScalarTo<T>(-0.01));
+
+    HWY_ASSERT_VEC_EQ(d, v0, Abs(v0));
+    HWY_ASSERT_VEC_EQ(d, vp1, Abs(vp1));
+    HWY_ASSERT_VEC_EQ(d, vp1, Abs(vn1));
+    HWY_ASSERT_VEC_EQ(d, vp2, Abs(vp2));
+    HWY_ASSERT_VEC_EQ(d, vp2, Abs(vn2));
+  }
+};
+
+HWY_NOINLINE void TestAllAbs() {
+  ForSignedTypes(ForPartialVectors<TestAbs>());
+  ForFloatTypes(ForPartialVectors<TestFloatAbs>());
+}
+
+struct TestIntegerNeg {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const RebindToUnsigned<D> du;
+    using TU = TFromD<decltype(du)>;
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> v1 = BitCast(d, Set(du, TU{1}));
+    const Vec<D> vp = BitCast(d, Set(du, TU{3}));
+    const Vec<D> vn = Add(Not(vp), v1);  // 2's complement
+    HWY_ASSERT_VEC_EQ(d, v0, Neg(v0));
+    HWY_ASSERT_VEC_EQ(d, vp, Neg(vn));
+    HWY_ASSERT_VEC_EQ(d, vn, Neg(vp));
+  }
+};
+
+struct TestFloatNeg {
+  // Must be inlined on aarch64 for bf16, else clang crashes.
+  template <typename T, class D>
+  HWY_INLINE void operator()(T /*unused*/, D d) {
+    const RebindToUnsigned<D> du;
+    using TU = TFromD<decltype(du)>;
+    // 1.25 in binary16.
+    const Vec<D> vp =
+        BitCast(d, Set(du, static_cast<TU>(Unpredictable1() * 0x3D00)));
+    // Flip sign bit in MSB
+    const Vec<D> vn = BitCast(d, Xor(BitCast(du, vp), SignBit(du)));
+    // Do not check negative zero - we do not yet have proper bfloat16_t Eq().
+    HWY_ASSERT_VEC_EQ(du, BitCast(du, vp), BitCast(du, Neg(vn)));
+    HWY_ASSERT_VEC_EQ(du, BitCast(du, vn), BitCast(du, Neg(vp)));
+  }
+};
+
+struct TestNegOverflow {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto vn = Set(d, LimitsMin<T>());
+    const auto vp = Set(d, LimitsMax<T>());
+    HWY_ASSERT_VEC_EQ(d, Neg(vn), Neg(vn));
+    HWY_ASSERT_VEC_EQ(d, Neg(vp), Neg(vp));
+  }
+};
+
+HWY_NOINLINE void TestAllNeg() {
+  ForFloatTypes(ForPartialVectors<TestFloatNeg>());
+  // Always supported, even if !HWY_HAVE_FLOAT16.
+  ForPartialVectors<TestFloatNeg>()(float16_t());
+
+  ForSignedTypes(ForPartialVectors<TestIntegerNeg>());
+
+  ForSignedTypes(ForPartialVectors<TestNegOverflow>());
+}
+
+struct TestIntegerAbsDiff {
+  template <typename T, HWY_IF_T_SIZE_ONE_OF(T, (1 << 1) | (1 << 2) | (1 << 4))>
+  static inline T ScalarAbsDiff(T a, T b) {
+    using TW = MakeSigned<MakeWide<T>>;
+    const TW diff = static_cast<TW>(static_cast<TW>(a) - static_cast<TW>(b));
+    return static_cast<T>((diff >= 0) ? diff : -diff);
+  }
+  template <typename T, HWY_IF_T_SIZE(T, 8)>
+  static inline T ScalarAbsDiff(T a, T b) {
+    if (a >= b) {
+      return static_cast<T>(static_cast<uint64_t>(a) -
+                            static_cast<uint64_t>(b));
+    } else {
+      return static_cast<T>(static_cast<uint64_t>(b) -
+                            static_cast<uint64_t>(a));
+    }
+  }
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto in_lanes_a = AllocateAligned<T>(N);
+    auto in_lanes_b = AllocateAligned<T>(N);
+    auto out_lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(in_lanes_a && in_lanes_b && out_lanes);
+    constexpr size_t shift_amt_mask = sizeof(T) * 8 - 1;
+    for (size_t i = 0; i < N; ++i) {
+      // Need to mask out shift_amt as i can be greater than or equal to
+      // the number of bits in T if T is int8_t, uint8_t, int16_t, or uint16_t.
+      const auto shift_amt = i & shift_amt_mask;
+      in_lanes_a[i] =
+          static_cast<T>((static_cast<uint64_t>(i) ^ 1u) << shift_amt);
+      in_lanes_b[i] = static_cast<T>(static_cast<uint64_t>(i) << shift_amt);
+      out_lanes[i] = ScalarAbsDiff(in_lanes_a[i], in_lanes_b[i]);
+    }
+    const auto a = Load(d, in_lanes_a.get());
+    const auto b = Load(d, in_lanes_b.get());
+    const auto expected = Load(d, out_lanes.get());
+    HWY_ASSERT_VEC_EQ(d, expected, AbsDiff(a, b));
+    HWY_ASSERT_VEC_EQ(d, expected, AbsDiff(b, a));
+  }
+};
+
+HWY_NOINLINE void TestAllIntegerAbsDiff() {
+  ForPartialVectors<TestIntegerAbsDiff>()(int8_t());
+  ForPartialVectors<TestIntegerAbsDiff>()(uint8_t());
+  ForPartialVectors<TestIntegerAbsDiff>()(int16_t());
+  ForPartialVectors<TestIntegerAbsDiff>()(uint16_t());
+  ForPartialVectors<TestIntegerAbsDiff>()(int32_t());
+  ForPartialVectors<TestIntegerAbsDiff>()(uint32_t());
+#if HWY_HAVE_INTEGER64
+  ForPartialVectors<TestIntegerAbsDiff>()(int64_t());
+  ForPartialVectors<TestIntegerAbsDiff>()(uint64_t());
+#endif
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyArithmeticTest);
+HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllPlusMinus);
+HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllAddSub);
+HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllAverage);
+HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllAbs);
+HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllNeg);
+HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllIntegerAbsDiff);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/bit_permute_test.cc

@@ -0,0 +1,98 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/bit_permute_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestBitShuffle {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET == HWY_SCALAR
+    (void)d;
+#else   // HWY_TARGET != HWY_SCALAR
+    using TU = MakeUnsigned<T>;
+
+    const size_t N = Lanes(d);
+
+    auto in1_lanes = AllocateAligned<T>(N);
+    auto in2_lanes = AllocateAligned<uint8_t>(N * sizeof(T));
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(in1_lanes && in2_lanes && expected);
+
+    constexpr uint8_t kBitIdxMask = static_cast<uint8_t>((sizeof(T) * 8) - 1);
+
+    const Repartition<uint8_t, decltype(d)> du8;
+    const RebindToSigned<decltype(du8)> di8;
+
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; i++) {
+        TU src_val = static_cast<TU>(rng());
+        TU expected_result = static_cast<TU>(0);
+        for (size_t j = 0; j < sizeof(T); j++) {
+          const uint8_t bit_idx = static_cast<uint8_t>(rng() & kBitIdxMask);
+
+          in2_lanes[i * sizeof(T) + j] = bit_idx;
+          expected_result = static_cast<TU>(expected_result |
+                                            (((src_val >> bit_idx) & 1) << j));
+        }
+
+        in1_lanes[i] = static_cast<T>(src_val);
+        expected[i] = static_cast<T>(expected_result);
+      }
+
+      const auto in1 = Load(d, in1_lanes.get());
+      const auto in2 = Load(du8, in2_lanes.get());
+      HWY_ASSERT_VEC_EQ(d, expected.get(), BitShuffle(in1, in2));
+      HWY_ASSERT_VEC_EQ(d, expected.get(), BitShuffle(in1, BitCast(di8, in2)));
+    }
+#endif  // HWY_TARGET == HWY_SCALAR
+  }
+};
+
+HWY_NOINLINE void TestAllBitShuffle() {
+#if HWY_HAVE_INTEGER64
+  ForPartialFixedOrFullScalableVectors<TestBitShuffle>()(int64_t());
+  ForPartialFixedOrFullScalableVectors<TestBitShuffle>()(uint64_t());
+#endif
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyBitPermuteTest);
+HWY_EXPORT_AND_TEST_P(HwyBitPermuteTest, TestAllBitShuffle);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/blockwise_combine_test.cc

@@ -0,0 +1,152 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/blockwise_combine_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+// Scalar does not define CombineShiftRightBytes.
+#if HWY_TARGET != HWY_SCALAR || HWY_IDE
+
+template <int kBytes>
+struct TestCombineShiftRightBytes {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T, D d) {
+    constexpr size_t kBlockSize = 16;
+    static_assert(kBytes < kBlockSize, "Shift count is per block");
+    const Repartition<uint8_t, D> d8;
+    const size_t N8 = Lanes(d8);
+    if (N8 < 16) return;
+    auto hi_bytes = AllocateAligned<uint8_t>(N8);
+    auto lo_bytes = AllocateAligned<uint8_t>(N8);
+    auto expected_bytes = AllocateAligned<uint8_t>(N8);
+    HWY_ASSERT(hi_bytes && lo_bytes && expected_bytes);
+    uint8_t combined[2 * kBlockSize];
+
+    // Random inputs in each lane
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(100); ++rep) {
+      for (size_t i = 0; i < N8; ++i) {
+        hi_bytes[i] = static_cast<uint8_t>(Random64(&rng) & 0xFF);
+        lo_bytes[i] = static_cast<uint8_t>(Random64(&rng) & 0xFF);
+      }
+      for (size_t i = 0; i < N8; i += kBlockSize) {
+        // Arguments are not the same size.
+        CopyBytes<kBlockSize>(&lo_bytes[i], combined);
+        CopyBytes<kBlockSize>(&hi_bytes[i], combined + kBlockSize);
+        CopyBytes<kBlockSize>(combined + kBytes, &expected_bytes[i]);
+      }
+
+      const auto hi = BitCast(d, Load(d8, hi_bytes.get()));
+      const auto lo = BitCast(d, Load(d8, lo_bytes.get()));
+      const auto expected = BitCast(d, Load(d8, expected_bytes.get()));
+      HWY_ASSERT_VEC_EQ(d, expected, CombineShiftRightBytes<kBytes>(d, hi, lo));
+    }
+  }
+};
+
+template <int kLanes>
+struct TestCombineShiftRightLanes {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T, D d) {
+    const Repartition<uint8_t, D> d8;
+    const size_t N8 = Lanes(d8);
+    if (N8 < 16) return;
+
+    auto hi_bytes = AllocateAligned<uint8_t>(N8);
+    auto lo_bytes = AllocateAligned<uint8_t>(N8);
+    auto expected_bytes = AllocateAligned<uint8_t>(N8);
+    HWY_ASSERT(hi_bytes && lo_bytes && expected_bytes);
+    constexpr size_t kBlockSize = 16;
+    uint8_t combined[2 * kBlockSize];
+
+    // Random inputs in each lane
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(100); ++rep) {
+      for (size_t i = 0; i < N8; ++i) {
+        hi_bytes[i] = static_cast<uint8_t>(Random64(&rng) & 0xFF);
+        lo_bytes[i] = static_cast<uint8_t>(Random64(&rng) & 0xFF);
+      }
+      for (size_t i = 0; i < N8; i += kBlockSize) {
+        // Arguments are not the same size.
+        CopyBytes<kBlockSize>(&lo_bytes[i], combined);
+        CopyBytes<kBlockSize>(&hi_bytes[i], combined + kBlockSize);
+        CopyBytes<kBlockSize>(combined + kLanes * sizeof(T),
+                              &expected_bytes[i]);
+      }
+
+      const auto hi = BitCast(d, Load(d8, hi_bytes.get()));
+      const auto lo = BitCast(d, Load(d8, lo_bytes.get()));
+      const auto expected = BitCast(d, Load(d8, expected_bytes.get()));
+      HWY_ASSERT_VEC_EQ(d, expected, CombineShiftRightLanes<kLanes>(d, hi, lo));
+    }
+  }
+};
+
+#endif  // #if HWY_TARGET != HWY_SCALAR
+
+struct TestCombineShiftRight {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T t, D d) {
+// Scalar does not define CombineShiftRightBytes.
+#if HWY_TARGET != HWY_SCALAR || HWY_IDE
+    constexpr int kMaxBytes =
+        HWY_MIN(16, static_cast<int>(MaxLanes(d) * sizeof(T)));
+    constexpr int kMaxLanes = kMaxBytes / static_cast<int>(sizeof(T));
+    TestCombineShiftRightBytes<kMaxBytes - 1>()(t, d);
+    TestCombineShiftRightBytes<HWY_MAX(kMaxBytes / 2, 1)>()(t, d);
+    TestCombineShiftRightBytes<1>()(t, d);
+
+    TestCombineShiftRightLanes<kMaxLanes - 1>()(t, d);
+    TestCombineShiftRightLanes<HWY_MAX(kMaxLanes / 2, -1)>()(t, d);
+    TestCombineShiftRightLanes<1>()(t, d);
+#else
+    (void)t;
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllCombineShiftRight() {
+  // Need at least 2 lanes.
+  ForAllTypes(ForShrinkableVectors<TestCombineShiftRight>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyBlockwiseCombineTest);
+HWY_EXPORT_AND_TEST_P(HwyBlockwiseCombineTest, TestAllCombineShiftRight);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/blockwise_shift_test.cc

@@ -0,0 +1,170 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/blockwise_shift_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestShiftBytes {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    // Scalar does not define Shift*Bytes.
+#if HWY_TARGET != HWY_SCALAR || HWY_IDE
+    const Repartition<uint8_t, D> du8;
+    const size_t N8 = Lanes(du8);
+    const size_t N = Lanes(d);
+
+    // Zero remains zero
+    const auto v0 = Zero(d);
+    HWY_ASSERT_VEC_EQ(d, v0, ShiftLeftBytes<1>(v0));
+    HWY_ASSERT_VEC_EQ(d, v0, ShiftLeftBytes<1>(d, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, ShiftRightBytes<1>(d, v0));
+
+    auto bytes = AllocateAligned<uint8_t>(N8);
+    auto in = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(bytes && in && expected);
+
+    // Zero after shifting out the high/low byte
+    ZeroBytes(bytes.get(), N8);
+    bytes[N8 - 1] = 0x7F;
+    const auto vhi = BitCast(d, Load(du8, bytes.get()));
+    bytes[N8 - 1] = 0;
+    bytes[0] = 0x7F;
+    const auto vlo = BitCast(d, Load(du8, bytes.get()));
+    HWY_ASSERT_VEC_EQ(d, v0, ShiftLeftBytes<1>(vhi));
+    HWY_ASSERT_VEC_EQ(d, v0, ShiftLeftBytes<1>(d, vhi));
+    HWY_ASSERT_VEC_EQ(d, v0, ShiftRightBytes<1>(d, vlo));
+
+    // Check expected result with Iota
+    const uint8_t* in_bytes = reinterpret_cast<const uint8_t*>(in.get());
+    const auto v = BitCast(d, Iota(du8, 1));
+    Store(v, d, in.get());
+
+    uint8_t* expected_bytes = reinterpret_cast<uint8_t*>(expected.get());
+
+    const size_t block_size = HWY_MIN(N8, 16);
+    for (size_t block = 0; block < N8; block += block_size) {
+      expected_bytes[block] = 0;
+      CopyBytes(in_bytes + block, expected_bytes + block + 1, block_size - 1);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftBytes<1>(v));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftBytes<1>(d, v));
+
+    for (size_t block = 0; block < N8; block += block_size) {
+      CopyBytes(in_bytes + block + 1, expected_bytes + block, block_size - 1);
+      expected_bytes[block + block_size - 1] = 0;
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightBytes<1>(d, v));
+#else
+    (void)d;
+#endif  // #if HWY_TARGET != HWY_SCALAR
+  }
+};
+
+HWY_NOINLINE void TestAllShiftBytes() {
+  ForIntegerTypes(ForPartialVectors<TestShiftBytes>());
+}
+
+struct TestShiftLeftLanes {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    // Scalar does not define Shift*Lanes.
+#if HWY_TARGET != HWY_SCALAR || HWY_IDE
+    const auto v = Iota(d, 1);
+    const size_t N = Lanes(d);
+    if (N == 1) return;
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    HWY_ASSERT_VEC_EQ(d, v, ShiftLeftLanes<0>(v));
+    HWY_ASSERT_VEC_EQ(d, v, ShiftLeftLanes<0>(d, v));
+
+    constexpr size_t kLanesPerBlock = 16 / sizeof(T);
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>((i % kLanesPerBlock) == 0 ? 0 : i);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftLanes<1>(v));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftLanes<1>(d, v));
+#else
+    (void)d;
+#endif  // #if HWY_TARGET != HWY_SCALAR
+  }
+};
+
+struct TestShiftRightLanes {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    // Scalar does not define Shift*Lanes.
+#if HWY_TARGET != HWY_SCALAR || HWY_IDE
+    const auto v = Iota(d, 1);
+    const size_t N = Lanes(d);
+    if (N == 1) return;
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    HWY_ASSERT_VEC_EQ(d, v, ShiftRightLanes<0>(d, v));
+
+    constexpr size_t kLanesPerBlock = 16 / sizeof(T);
+
+    for (size_t i = 0; i < N; ++i) {
+      const size_t mod = i % kLanesPerBlock;
+      expected[i] = ConvertScalarTo<T>(
+          ((mod == kLanesPerBlock - 1) || (i >= N - 1)) ? 0 : (2 + i));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightLanes<1>(d, v));
+#else
+    (void)d;
+#endif  // #if HWY_TARGET != HWY_SCALAR
+  }
+};
+
+HWY_NOINLINE void TestAllShiftLeftLanes() {
+  ForAllTypes(ForPartialVectors<TestShiftLeftLanes>());
+}
+
+HWY_NOINLINE void TestAllShiftRightLanes() {
+  ForAllTypes(ForPartialVectors<TestShiftRightLanes>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyBlockwiseShiftTest);
+HWY_EXPORT_AND_TEST_P(HwyBlockwiseShiftTest, TestAllShiftBytes);
+HWY_EXPORT_AND_TEST_P(HwyBlockwiseShiftTest, TestAllShiftLeftLanes);
+HWY_EXPORT_AND_TEST_P(HwyBlockwiseShiftTest, TestAllShiftRightLanes);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/blockwise_test.cc

@@ -0,0 +1,523 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/blockwise_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+template <typename D, int kLane>
+struct TestBroadcastR {
+  HWY_NOINLINE void operator()() const {
+    using T = typename D::T;
+    const D d;
+    const size_t N = Lanes(d);
+    if (kLane >= N) return;
+    auto in_lanes = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(in_lanes && expected);
+    ZeroBytes(in_lanes.get(), N * sizeof(T));
+    const size_t blockN = HWY_MIN(N * sizeof(T), 16) / sizeof(T);
+    // Need to set within each 128-bit block
+    for (size_t block = 0; block < N; block += blockN) {
+      in_lanes[block + kLane] = ConvertScalarTo<T>(block + 1);
+    }
+    PreventElision(in_lanes[0]);  // workaround for f16x1 failure
+    const auto in = Load(d, in_lanes.get());
+    for (size_t block = 0; block < N; block += blockN) {
+      for (size_t i = 0; i < blockN; ++i) {
+        expected[block + i] = ConvertScalarTo<T>(block + 1);
+      }
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Broadcast<kLane>(in));
+
+    TestBroadcastR<D, kLane - 1>()();
+  }
+};
+
+template <class D>
+struct TestBroadcastR<D, -1> {
+  void operator()() const {}
+};
+
+struct TestBroadcast {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    TestBroadcastR<D, HWY_MIN(MaxLanes(d), 16 / sizeof(T)) - 1>()();
+  }
+};
+
+HWY_NOINLINE void TestAllBroadcast() {
+  ForAllTypes(ForPartialVectors<TestBroadcast>());
+}
+
+template <bool kFull>
+struct ChooseTableSize {
+  template <typename T, typename DIdx>
+  using type = DIdx;
+};
+template <>
+struct ChooseTableSize<true> {
+  template <typename T, typename DIdx>
+  using type = ScalableTag<T>;
+};
+
+template <bool kFull>
+struct TestTableLookupBytes {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    RandomState rng;
+
+    const typename ChooseTableSize<kFull>::template type<T, D> d_tbl;
+    const Repartition<uint8_t, decltype(d_tbl)> d_tbl8;
+    const Repartition<uint8_t, D> d8;
+    const size_t N = Lanes(d);
+    const size_t NT8 = Lanes(d_tbl8);
+    const size_t N8 = Lanes(d8);
+
+    auto in_bytes = AllocateAligned<uint8_t>(NT8);
+    auto indices = AllocateAligned<T>(N8);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(in_bytes && indices && expected);
+
+    // Random input bytes
+    for (size_t i = 0; i < NT8; ++i) {
+      in_bytes[i] = Random32(&rng) & 0xFF;
+    }
+    const auto in = BitCast(d_tbl, Load(d_tbl8, in_bytes.get()));
+
+    // Enough test data; for larger vectors, upper lanes will be zero.
+    const uint8_t index_bytes_source[64] = {
+        // Same index as source, multiple outputs from same input,
+        // unused input (9), ascending/descending and nonconsecutive neighbors.
+        0,  2,  1, 2, 15, 12, 13, 14, 6,  7,  8,  5,  4,  3,  10, 11,
+        11, 10, 3, 4, 5,  8,  7,  6,  14, 13, 12, 15, 2,  1,  2,  0,
+        4,  3,  2, 2, 5,  6,  7,  7,  15, 15, 15, 15, 15, 15, 0,  1};
+    const size_t max_index = HWY_MIN(NT8, 16) - 1;
+    uint8_t* index_bytes = reinterpret_cast<uint8_t*>(indices.get());
+    for (size_t i = 0; i < N8; ++i) {
+      index_bytes[i] = (i < 64) ? index_bytes_source[i] : 0;
+      // Avoid asan error for partial vectors.
+      index_bytes[i] = static_cast<uint8_t>(HWY_MIN(index_bytes[i], max_index));
+    }
+
+    uint8_t* expected_bytes = reinterpret_cast<uint8_t*>(expected.get());
+
+    for (size_t block = 0; block < N8; block += 16) {
+      for (size_t i = 0; i < 16 && (block + i) < N8; ++i) {
+        const uint8_t index = index_bytes[block + i];
+        HWY_ASSERT(index <= max_index);
+        // Note that block + index may exceed NT8 on RVV, which is fine because
+        // the operation uses the larger of the table and index vector size.
+        HWY_ASSERT(block + index < HWY_MAX(N8, NT8));
+        // For large vectors, the lane index may wrap around due to block,
+        // also wrap around after 8-bit overflow.
+        expected_bytes[block + i] =
+            in_bytes[(block + index) % HWY_MIN(NT8, 256)];
+      }
+    }
+    {
+      const Vec<D> indices_v = Load(d, indices.get());
+      HWY_ASSERT_VEC_EQ(d, expected.get(), TableLookupBytes(in, indices_v));
+    }
+
+    // Individually test zeroing each byte position.
+    for (size_t i = 0; i < N8; ++i) {
+      const uint8_t prev_expected = expected_bytes[i];
+      const uint8_t prev_index = index_bytes[i];
+      expected_bytes[i] = 0;
+
+      const int idx = 0x80 + (static_cast<int>(Random32(&rng) & 7) << 4);
+      HWY_ASSERT(0x80 <= idx && idx < 256);
+      index_bytes[i] = static_cast<uint8_t>(idx);
+
+      const Vec<D> indices_v = Load(d, indices.get());
+      HWY_ASSERT_VEC_EQ(d, expected.get(), TableLookupBytesOr0(in, indices_v));
+      expected_bytes[i] = prev_expected;
+      index_bytes[i] = prev_index;
+    }
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllTableLookupBytesSame() {
+  // Partial index, same-sized table.
+  ForIntegerTypes(ForPartialVectors<TestTableLookupBytes<false>>());
+}
+
+HWY_NOINLINE void TestAllTableLookupBytesMixed() {
+  // Partial index, full-size table.
+  ForIntegerTypes(ForPartialVectors<TestTableLookupBytes<true>>());
+}
+
+struct TestInterleaveLower {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TU = MakeUnsigned<T>;
+    const size_t N = Lanes(d);
+    auto even_lanes = AllocateAligned<T>(N);
+    auto odd_lanes = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(even_lanes && odd_lanes && expected);
+    for (size_t i = 0; i < N; ++i) {
+      even_lanes[i] = ConvertScalarTo<T>(2 * i + 0);
+      odd_lanes[i] = ConvertScalarTo<T>(2 * i + 1);
+    }
+    const auto even = Load(d, even_lanes.get());
+    const auto odd = Load(d, odd_lanes.get());
+
+    const size_t blockN = HWY_MIN(16 / sizeof(T), N);
+    for (size_t i = 0; i < Lanes(d); ++i) {
+      const size_t block = i / blockN;
+      const size_t index = (i % blockN) + block * 2 * blockN;
+      expected[i] = ConvertScalarTo<T>(index & LimitsMax<TU>());
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveLower(even, odd));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveLower(d, even, odd));
+  }
+};
+
+struct TestInterleaveUpper {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    if (N == 1) return;
+    auto even_lanes = AllocateAligned<T>(N);
+    auto odd_lanes = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(even_lanes && odd_lanes && expected);
+    for (size_t i = 0; i < N; ++i) {
+      even_lanes[i] = ConvertScalarTo<T>(2 * i + 0);
+      odd_lanes[i] = ConvertScalarTo<T>(2 * i + 1);
+    }
+    const auto even = Load(d, even_lanes.get());
+    const auto odd = Load(d, odd_lanes.get());
+
+    const size_t blockN = HWY_MIN(16 / sizeof(T), N);
+    for (size_t i = 0; i < Lanes(d); ++i) {
+      const size_t block = i / blockN;
+      expected[i] =
+          ConvertScalarTo<T>((i % blockN) + block * 2 * blockN + blockN);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveUpper(d, even, odd));
+  }
+};
+
+struct TestInterleaveEven {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto even_lanes = AllocateAligned<T>(N);
+    auto odd_lanes = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(even_lanes && odd_lanes && expected);
+    for (size_t i = 0; i < N; ++i) {
+      even_lanes[i] = ConvertScalarTo<T>(2 * i + 0);
+      odd_lanes[i] = ConvertScalarTo<T>(2 * i + 1);
+    }
+    const auto even = Load(d, even_lanes.get());
+    const auto odd = Load(d, odd_lanes.get());
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>(2 * i - (i & 1));
+    }
+
+    HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveEven(even, odd));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveEven(d, even, odd));
+  }
+};
+
+struct TestInterleaveOdd {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto even_lanes = AllocateAligned<T>(N);
+    auto odd_lanes = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(even_lanes && odd_lanes && expected);
+    for (size_t i = 0; i < N; ++i) {
+      even_lanes[i] = ConvertScalarTo<T>(2 * i + 0);
+      odd_lanes[i] = ConvertScalarTo<T>(2 * i + 1);
+    }
+    const auto even = Load(d, even_lanes.get());
+    const auto odd = Load(d, odd_lanes.get());
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>((2 * i) - (i & 1) + 2);
+    }
+
+    HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveOdd(d, even, odd));
+  }
+};
+
+HWY_NOINLINE void TestAllInterleave() {
+  // Not DemoteVectors because this cannot be supported by HWY_SCALAR.
+  ForAllTypes(ForShrinkableVectors<TestInterleaveLower>());
+  ForAllTypes(ForShrinkableVectors<TestInterleaveUpper>());
+  ForAllTypes(ForShrinkableVectors<TestInterleaveEven>());
+  ForAllTypes(ForShrinkableVectors<TestInterleaveOdd>());
+}
+
+struct TestZipLower {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using WideT = MakeWide<T>;
+    static_assert(sizeof(T) * 2 == sizeof(WideT), "Must be double-width");
+    static_assert(IsSigned<T>() == IsSigned<WideT>(), "Must have same sign");
+    const size_t N = Lanes(d);
+    auto even_lanes = AllocateAligned<T>(N);
+    auto odd_lanes = AllocateAligned<T>(N);
+    // At least 2 lanes for HWY_SCALAR
+    auto zip_lanes = AllocateAligned<T>(HWY_MAX(N, 2));
+    HWY_ASSERT(even_lanes && odd_lanes && zip_lanes);
+    const T kMaxT = LimitsMax<T>();
+    for (size_t i = 0; i < N; ++i) {
+      even_lanes[i] = ConvertScalarTo<T>((2 * i + 0) & kMaxT);
+      odd_lanes[i] = ConvertScalarTo<T>((2 * i + 1) & kMaxT);
+    }
+    const auto even = Load(d, even_lanes.get());
+    const auto odd = Load(d, odd_lanes.get());
+
+    const Repartition<WideT, D> dw;
+#if HWY_TARGET == HWY_SCALAR
+    // Safely handle big-endian
+    const auto expected = Set(dw, static_cast<WideT>(1ULL << (sizeof(T) * 8)));
+#else
+    const size_t blockN = HWY_MIN(size_t(16) / sizeof(T), N);
+    for (size_t i = 0; i < N; i += 2) {
+      const size_t base = (i / blockN) * blockN;
+      const size_t mod = i % blockN;
+      zip_lanes[i + 0] = even_lanes[mod / 2 + base];
+      zip_lanes[i + 1] = odd_lanes[mod / 2 + base];
+      // Without this, `expected` is incorrect with Clang and 512-bit SVE: the
+      // first byte of the second block is 0x10 instead of 0x20 as it should be.
+      PreventElision(zip_lanes[i + 0]);
+    }
+    const Vec<decltype(dw)> expected = BitCast(dw, Load(d, zip_lanes.get()));
+#endif  // HWY_TARGET == HWY_SCALAR
+    HWY_ASSERT_VEC_EQ(dw, expected, ZipLower(even, odd));
+    HWY_ASSERT_VEC_EQ(dw, expected, ZipLower(dw, even, odd));
+  }
+};
+
+#if HWY_TARGET == HWY_SCALAR
+template <class Test>
+using ForZipToWideVectors = ForPartialVectors<Test>;
+#else
+template <class Test>
+using ForZipToWideVectors = ForShrinkableVectors<Test>;
+#endif
+
+HWY_NOINLINE void TestAllZipLower() {
+  const ForZipToWideVectors<TestZipLower> lower_unsigned;
+  lower_unsigned(uint8_t());
+  lower_unsigned(uint16_t());
+#if HWY_HAVE_INTEGER64
+  lower_unsigned(uint32_t());  // generates u64
+#endif
+
+  const ForZipToWideVectors<TestZipLower> lower_signed;
+  lower_signed(int8_t());
+  lower_signed(int16_t());
+#if HWY_HAVE_INTEGER64
+  lower_signed(int32_t());  // generates i64
+#endif
+
+  // No float - concatenating f32 does not result in a f64
+}
+
+// Remove this test (so it does not show as having run) if the only target is
+// HWY_SCALAR, which does not support this op.
+#if HWY_TARGETS != HWY_SCALAR
+
+struct TestZipUpper {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET == HWY_SCALAR
+    (void)d;
+#else
+    using WideT = MakeWide<T>;
+    static_assert(sizeof(T) * 2 == sizeof(WideT), "Must be double-width");
+    static_assert(IsSigned<T>() == IsSigned<WideT>(), "Must have same sign");
+    const size_t N = Lanes(d);
+    if (N < 16 / sizeof(T)) return;
+    auto even_lanes = AllocateAligned<T>(N);
+    auto odd_lanes = AllocateAligned<T>(N);
+    auto zip_lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(even_lanes && odd_lanes && zip_lanes);
+    const T kMaxT = LimitsMax<T>();
+    for (size_t i = 0; i < N; ++i) {
+      even_lanes[i] = ConvertScalarTo<T>((2 * i + 0) & kMaxT);
+      odd_lanes[i] = ConvertScalarTo<T>((2 * i + 1) & kMaxT);
+    }
+    const auto even = Load(d, even_lanes.get());
+    const auto odd = Load(d, odd_lanes.get());
+
+    const size_t blockN = HWY_MIN(size_t(16) / sizeof(T), N);
+
+    for (size_t i = 0; i < N; i += 2) {
+      const size_t base = (i / blockN) * blockN + blockN / 2;
+      const size_t mod = i % blockN;
+      zip_lanes[i + 0] = even_lanes[mod / 2 + base];
+      zip_lanes[i + 1] = odd_lanes[mod / 2 + base];
+      // See comment at previous call to PreventElision.
+      PreventElision(zip_lanes[i + 0]);
+    }
+    const Repartition<WideT, D> dw;
+    const Vec<decltype(dw)> expected = BitCast(dw, Load(d, zip_lanes.get()));
+    HWY_ASSERT_VEC_EQ(dw, expected, ZipUpper(dw, even, odd));
+#endif  // HWY_TARGET == HWY_SCALAR
+  }
+};
+
+HWY_NOINLINE void TestAllZipUpper() {
+  const ForShrinkableVectors<TestZipUpper> upper_unsigned;
+  upper_unsigned(uint8_t());
+  upper_unsigned(uint16_t());
+#if HWY_HAVE_INTEGER64
+  upper_unsigned(uint32_t());  // generates u64
+#endif
+
+  const ForShrinkableVectors<TestZipUpper> upper_signed;
+  upper_signed(int8_t());
+  upper_signed(int16_t());
+#if HWY_HAVE_INTEGER64
+  upper_signed(int32_t());  // generates i64
+#endif
+
+  // No float - concatenating f32 does not result in a f64
+}
+
+#endif  // HWY_TARGETS != HWY_SCALAR
+
+class TestSpecialShuffle32 {
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v = Iota(d, 0);
+    VerifyLanes32(d, Shuffle2301(v), 2, 3, 0, 1, __FILE__, __LINE__);
+    VerifyLanes32(d, Shuffle1032(v), 1, 0, 3, 2, __FILE__, __LINE__);
+    VerifyLanes32(d, Shuffle0321(v), 0, 3, 2, 1, __FILE__, __LINE__);
+    VerifyLanes32(d, Shuffle2103(v), 2, 1, 0, 3, __FILE__, __LINE__);
+    VerifyLanes32(d, Shuffle0123(v), 0, 1, 2, 3, __FILE__, __LINE__);
+  }
+
+ private:
+  // HWY_INLINE works around a Clang SVE compiler bug where all but the first
+  // 128 bits (the NEON register) of actual are zero.
+  template <class D, class V>
+  HWY_INLINE void VerifyLanes32(D d, VecArg<V> actual, const size_t i3,
+                                const size_t i2, const size_t i1,
+                                const size_t i0, const char* filename,
+                                const int line) {
+    using T = TFromD<D>;
+    constexpr size_t kBlockN = 16 / sizeof(T);
+    const size_t N = Lanes(d);
+    if (N < 4) return;
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+    for (size_t block = 0; block < N; block += kBlockN) {
+      expected[block + 3] = ConvertScalarTo<T>(block + i3);
+      expected[block + 2] = ConvertScalarTo<T>(block + i2);
+      expected[block + 1] = ConvertScalarTo<T>(block + i1);
+      expected[block + 0] = ConvertScalarTo<T>(block + i0);
+    }
+    AssertVecEqual(d, expected.get(), actual, filename, line);
+  }
+};
+
+class TestSpecialShuffle64 {
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v = Iota(d, 0);
+    VerifyLanes64(d, Shuffle01(v), 0, 1, __FILE__, __LINE__);
+  }
+
+ private:
+  // HWY_INLINE works around a Clang SVE compiler bug where all but the first
+  // 128 bits (the NEON register) of actual are zero.
+  template <class D, class V>
+  HWY_INLINE void VerifyLanes64(D d, VecArg<V> actual, const size_t i1,
+                                const size_t i0, const char* filename,
+                                const int line) {
+    using T = TFromD<D>;
+    constexpr size_t kBlockN = 16 / sizeof(T);
+    const size_t N = Lanes(d);
+    if (N < 2) return;
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+    for (size_t block = 0; block < N; block += kBlockN) {
+      expected[block + 1] = ConvertScalarTo<T>(block + i1);
+      expected[block + 0] = ConvertScalarTo<T>(block + i0);
+    }
+    AssertVecEqual(d, expected.get(), actual, filename, line);
+  }
+};
+
+HWY_NOINLINE void TestAllSpecialShuffles() {
+  const ForGEVectors<128, TestSpecialShuffle32> test32;
+  test32(uint32_t());
+  test32(int32_t());
+  test32(float());
+
+#if HWY_HAVE_INTEGER64
+  const ForGEVectors<128, TestSpecialShuffle64> test64;
+  test64(uint64_t());
+  test64(int64_t());
+#endif
+
+#if HWY_HAVE_FLOAT64
+  const ForGEVectors<128, TestSpecialShuffle64> test_d;
+  test_d(double());
+#endif
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyBlockwiseTest);
+HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllBroadcast);
+HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllTableLookupBytesSame);
+HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllTableLookupBytesMixed);
+HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllInterleave);
+HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllZipLower);
+#if HWY_TARGETS != HWY_SCALAR
+HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllZipUpper);
+#endif
+HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllSpecialShuffles);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/cast_test.cc

@@ -0,0 +1,312 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "hwy/base.h"
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/cast_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+// Cast and ensure bytes are the same. Called directly from TestAllBitCast or
+// via TestBitCastFrom.
+template <typename ToT>
+struct TestBitCast {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Repartition<ToT, D> dto;
+    const size_t N = Lanes(d);
+    const size_t Nto = Lanes(dto);
+    if (N == 0 || Nto == 0) return;
+    HWY_ASSERT_EQ(N * sizeof(T), Nto * sizeof(ToT));
+    const auto vf = Iota(d, 1);
+    const auto vt = BitCast(dto, vf);
+    // Must return the same bits
+    auto from_lanes = AllocateAligned<T>(Lanes(d));
+    auto to_lanes = AllocateAligned<ToT>(Lanes(dto));
+    HWY_ASSERT(from_lanes && to_lanes);
+    Store(vf, d, from_lanes.get());
+    Store(vt, dto, to_lanes.get());
+    HWY_ASSERT(
+        BytesEqual(from_lanes.get(), to_lanes.get(), Lanes(d) * sizeof(T)));
+  }
+};
+
+// From D to all types.
+struct TestBitCastFrom {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T t, D d) {
+    TestBitCast<uint8_t>()(t, d);
+    TestBitCast<uint16_t>()(t, d);
+    TestBitCast<uint32_t>()(t, d);
+#if HWY_HAVE_INTEGER64
+    TestBitCast<uint64_t>()(t, d);
+#endif
+    TestBitCast<int8_t>()(t, d);
+    TestBitCast<int16_t>()(t, d);
+    TestBitCast<int32_t>()(t, d);
+#if HWY_HAVE_INTEGER64
+    TestBitCast<int64_t>()(t, d);
+#endif
+    TestBitCast<float>()(t, d);
+#if HWY_HAVE_FLOAT64
+    TestBitCast<double>()(t, d);
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllBitCast() {
+  // For HWY_SCALAR and partial vectors, we can only cast to same-sized types:
+  // the former can't partition its single lane, and the latter can be smaller
+  // than a destination type.
+  const ForPartialVectors<TestBitCast<uint8_t>> to_u8;
+  to_u8(uint8_t());
+  to_u8(int8_t());
+
+  const ForPartialVectors<TestBitCast<int8_t>> to_i8;
+  to_i8(uint8_t());
+  to_i8(int8_t());
+
+  const ForPartialVectors<TestBitCast<uint16_t>> to_u16;
+  to_u16(uint16_t());
+  to_u16(int16_t());
+
+  const ForPartialVectors<TestBitCast<int16_t>> to_i16;
+  to_i16(uint16_t());
+  to_i16(int16_t());
+
+  const ForPartialVectors<TestBitCast<uint32_t>> to_u32;
+  to_u32(uint32_t());
+  to_u32(int32_t());
+  to_u32(float());
+
+  const ForPartialVectors<TestBitCast<int32_t>> to_i32;
+  to_i32(uint32_t());
+  to_i32(int32_t());
+  to_i32(float());
+
+#if HWY_HAVE_INTEGER64
+  const ForPartialVectors<TestBitCast<uint64_t>> to_u64;
+  to_u64(uint64_t());
+  to_u64(int64_t());
+#if HWY_HAVE_FLOAT64
+  to_u64(double());
+#endif
+
+  const ForPartialVectors<TestBitCast<int64_t>> to_i64;
+  to_i64(uint64_t());
+  to_i64(int64_t());
+#if HWY_HAVE_FLOAT64
+  to_i64(double());
+#endif
+#endif  // HWY_HAVE_INTEGER64
+
+  const ForPartialVectors<TestBitCast<float>> to_float;
+  to_float(uint32_t());
+  to_float(int32_t());
+  to_float(float());
+
+#if HWY_HAVE_FLOAT64
+  const ForPartialVectors<TestBitCast<double>> to_double;
+  to_double(double());
+#if HWY_HAVE_INTEGER64
+  to_double(uint64_t());
+  to_double(int64_t());
+#endif  // HWY_HAVE_INTEGER64
+#endif  // HWY_HAVE_FLOAT64
+
+#if HWY_TARGET != HWY_SCALAR
+  // For non-scalar vectors, we can cast all types to all.
+  ForAllTypes(ForGEVectors<64, TestBitCastFrom>());
+#endif
+}
+
+template <class TTo>
+struct TestResizeBitCastToOneLaneVect {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    if (N == 0) {
+      return;
+    }
+
+    auto from_lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(from_lanes);
+    auto v = Iota(d, 1);
+    Store(v, d, from_lanes.get());
+
+    const size_t num_of_bytes_to_copy = HWY_MIN(N * sizeof(T), sizeof(TTo));
+
+    int8_t active_bits_mask_i8_arr[sizeof(TTo)] = {};
+    for (size_t i = 0; i < num_of_bytes_to_copy; i++) {
+      active_bits_mask_i8_arr[i] = int8_t{-1};
+    }
+
+    TTo active_bits_int_mask;
+    CopyBytes<sizeof(TTo)>(active_bits_mask_i8_arr, &active_bits_int_mask);
+
+    const FixedTag<TTo, 1> d_to;
+    TTo expected_bits = 0;
+    CopyBytes(from_lanes.get(), &expected_bits, num_of_bytes_to_copy);
+
+    const auto expected = Set(d_to, expected_bits);
+    const auto v_active_bits_mask = Set(d_to, active_bits_int_mask);
+
+    const auto actual_1 = And(v_active_bits_mask, ResizeBitCast(d_to, v));
+    const auto actual_2 = ZeroExtendResizeBitCast(d_to, d, v);
+
+    HWY_ASSERT_VEC_EQ(d_to, expected, actual_1);
+    HWY_ASSERT_VEC_EQ(d_to, expected, actual_2);
+  }
+};
+
+HWY_NOINLINE void TestAllResizeBitCastToOneLaneVect() {
+  ForAllTypes(ForPartialVectors<TestResizeBitCastToOneLaneVect<uint32_t>>());
+#if HWY_HAVE_INTEGER64
+  ForAllTypes(ForPartialVectors<TestResizeBitCastToOneLaneVect<uint64_t>>());
+#endif
+}
+
+// Cast and ensure bytes are the same. Called directly from
+// TestAllSameSizeResizeBitCast or via TestSameSizeResizeBitCastFrom.
+template <typename ToT>
+struct TestSameSizeResizeBitCast {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Repartition<ToT, D> dto;
+    const auto v = Iota(d, 1);
+    const auto expected = BitCast(dto, v);
+
+    const VFromD<decltype(dto)> actual_1 = ResizeBitCast(dto, v);
+    const VFromD<decltype(dto)> actual_2 = ZeroExtendResizeBitCast(dto, d, v);
+
+    HWY_ASSERT_VEC_EQ(dto, expected, actual_1);
+    HWY_ASSERT_VEC_EQ(dto, expected, actual_2);
+  }
+};
+
+// From D to all types.
+struct TestSameSizeResizeBitCastFrom {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T t, D d) {
+    TestSameSizeResizeBitCast<uint8_t>()(t, d);
+    TestSameSizeResizeBitCast<uint16_t>()(t, d);
+    TestSameSizeResizeBitCast<uint32_t>()(t, d);
+#if HWY_HAVE_INTEGER64
+    TestSameSizeResizeBitCast<uint64_t>()(t, d);
+#endif
+    TestSameSizeResizeBitCast<int8_t>()(t, d);
+    TestSameSizeResizeBitCast<int16_t>()(t, d);
+    TestSameSizeResizeBitCast<int32_t>()(t, d);
+#if HWY_HAVE_INTEGER64
+    TestSameSizeResizeBitCast<int64_t>()(t, d);
+#endif
+    TestSameSizeResizeBitCast<float>()(t, d);
+#if HWY_HAVE_FLOAT64
+    TestSameSizeResizeBitCast<double>()(t, d);
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllSameSizeResizeBitCast() {
+  // For HWY_SCALAR and partial vectors, we can only cast to same-sized types:
+  // the former can't partition its single lane, and the latter can be smaller
+  // than a destination type.
+  const ForPartialVectors<TestSameSizeResizeBitCast<uint8_t>> to_u8;
+  to_u8(uint8_t());
+  to_u8(int8_t());
+
+  const ForPartialVectors<TestSameSizeResizeBitCast<int8_t>> to_i8;
+  to_i8(uint8_t());
+  to_i8(int8_t());
+
+  const ForPartialVectors<TestSameSizeResizeBitCast<uint16_t>> to_u16;
+  to_u16(uint16_t());
+  to_u16(int16_t());
+
+  const ForPartialVectors<TestSameSizeResizeBitCast<int16_t>> to_i16;
+  to_i16(uint16_t());
+  to_i16(int16_t());
+
+  const ForPartialVectors<TestSameSizeResizeBitCast<uint32_t>> to_u32;
+  to_u32(uint32_t());
+  to_u32(int32_t());
+  to_u32(float());
+
+  const ForPartialVectors<TestSameSizeResizeBitCast<int32_t>> to_i32;
+  to_i32(uint32_t());
+  to_i32(int32_t());
+  to_i32(float());
+
+#if HWY_HAVE_INTEGER64
+  const ForPartialVectors<TestSameSizeResizeBitCast<uint64_t>> to_u64;
+  to_u64(uint64_t());
+  to_u64(int64_t());
+#if HWY_HAVE_FLOAT64
+  to_u64(double());
+#endif
+
+  const ForPartialVectors<TestSameSizeResizeBitCast<int64_t>> to_i64;
+  to_i64(uint64_t());
+  to_i64(int64_t());
+#if HWY_HAVE_FLOAT64
+  to_i64(double());
+#endif
+#endif  // HWY_HAVE_INTEGER64
+
+  const ForPartialVectors<TestSameSizeResizeBitCast<float>> to_float;
+  to_float(uint32_t());
+  to_float(int32_t());
+  to_float(float());
+
+#if HWY_HAVE_FLOAT64
+  const ForPartialVectors<TestSameSizeResizeBitCast<double>> to_double;
+  to_double(double());
+#if HWY_HAVE_INTEGER64
+  to_double(uint64_t());
+  to_double(int64_t());
+#endif  // HWY_HAVE_INTEGER64
+#endif  // HWY_HAVE_FLOAT64
+
+#if HWY_TARGET != HWY_SCALAR
+  // For non-scalar vectors, we can cast all types to all.
+  ForAllTypes(ForGEVectors<64, TestSameSizeResizeBitCastFrom>());
+#endif
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyCastTest);
+HWY_EXPORT_AND_TEST_P(HwyCastTest, TestAllBitCast);
+HWY_EXPORT_AND_TEST_P(HwyCastTest, TestAllResizeBitCastToOneLaneVect);
+HWY_EXPORT_AND_TEST_P(HwyCastTest, TestAllSameSizeResizeBitCast);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/combine_test.cc

@@ -0,0 +1,257 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/combine_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestLowerHalf {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Half<D> d2;
+
+    const size_t N = Lanes(d);
+    auto lanes = AllocateAligned<T>(N);
+    auto lanes2 = AllocateAligned<T>(N);
+    HWY_ASSERT(lanes && lanes2);
+    ZeroBytes(lanes.get(), N * sizeof(T));
+    ZeroBytes(lanes2.get(), N * sizeof(T));
+    const auto v = Iota(d, 1);
+    Store(LowerHalf(d2, v), d2, lanes.get());
+    Store(LowerHalf(v), d2, lanes2.get());  // optionally without D
+    size_t i = 0;
+    for (; i < Lanes(d2); ++i) {
+      HWY_ASSERT_EQ(ConvertScalarTo<T>(1 + i), lanes[i]);
+      HWY_ASSERT_EQ(ConvertScalarTo<T>(1 + i), lanes2[i]);
+    }
+    // Other half remains unchanged
+    for (; i < N; ++i) {
+      HWY_ASSERT_EQ(ConvertScalarTo<T>(0), lanes[i]);
+      HWY_ASSERT_EQ(ConvertScalarTo<T>(0), lanes2[i]);
+    }
+  }
+};
+
+struct TestLowerQuarter {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Half<D> d2;
+    const Half<decltype(d2)> d4;
+
+    const size_t N = Lanes(d);
+    auto lanes = AllocateAligned<T>(N);
+    auto lanes2 = AllocateAligned<T>(N);
+    HWY_ASSERT(lanes && lanes2);
+    ZeroBytes(lanes.get(), N * sizeof(T));
+    ZeroBytes(lanes2.get(), N * sizeof(T));
+    const auto v = Iota(d, 1);
+    const auto lo = LowerHalf(d4, LowerHalf(d2, v));
+    const auto lo2 = LowerHalf(LowerHalf(v));  // optionally without D
+    Store(lo, d4, lanes.get());
+    Store(lo2, d4, lanes2.get());
+    size_t i = 0;
+    for (; i < Lanes(d4); ++i) {
+      HWY_ASSERT_EQ(ConvertScalarTo<T>(i + 1), lanes[i]);
+      HWY_ASSERT_EQ(ConvertScalarTo<T>(i + 1), lanes2[i]);
+    }
+    // Upper 3/4 remain unchanged
+    for (; i < N; ++i) {
+      HWY_ASSERT_EQ(ConvertScalarTo<T>(0), lanes[i]);
+      HWY_ASSERT_EQ(ConvertScalarTo<T>(0), lanes2[i]);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllLowerHalf() {
+  ForAllTypes(ForHalfVectors<TestLowerHalf>());
+
+  // The minimum vector size is 128 bits, so there's no guarantee we can have
+  // quarters of 64-bit lanes, hence test 'all' other types.
+  ForHalfVectors<TestLowerQuarter, 2> test_quarter;
+  ForUI8(test_quarter);
+  ForUI16(test_quarter);  // exclude float16_t - cannot compare
+  ForUIF32(test_quarter);
+}
+
+struct TestUpperHalf {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    // Scalar does not define UpperHalf.
+#if HWY_TARGET != HWY_SCALAR
+    const Half<D> d2;
+    const size_t N2 = Lanes(d2);
+    if (N2 < 2) return;
+    HWY_ASSERT_EQ(N2 * 2, Lanes(d));
+    auto expected = AllocateAligned<T>(N2);
+    HWY_ASSERT(expected);
+    size_t i = 0;
+    for (; i < N2; ++i) {
+      expected[i] = ConvertScalarTo<T>(N2 + 1 + i);
+    }
+    HWY_ASSERT_VEC_EQ(d2, expected.get(), UpperHalf(d2, Iota(d, 1)));
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllUpperHalf() {
+  ForAllTypes(ForHalfVectors<TestUpperHalf>());
+}
+
+struct TestZeroExtendVector {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Twice<D> d2;
+
+    const Vec<D> v = IotaForSpecial(d, 1);
+    const size_t N = Lanes(d);
+    const size_t N2 = Lanes(d2);
+    // If equal, then N was already MaxLanes(d) and it's not clear what
+    // Combine or ZeroExtendVector should return.
+    if (N2 == N) return;
+    HWY_ASSERT(N2 == 2 * N);
+    auto lanes = AllocateAligned<T>(N2);
+    HWY_ASSERT(lanes);
+    Store(v, d, &lanes[0]);
+    Store(v, d, &lanes[N]);
+
+    const VFromD<decltype(d2)> ext = ZeroExtendVector(d2, v);
+    Store(ext, d2, lanes.get());
+
+    // Lower half is unchanged
+    HWY_ASSERT_VEC_EQ(d, v, Load(d, &lanes[0]));
+    // Upper half is zero
+    HWY_ASSERT_VEC_EQ(d, Zero(d), Load(d, &lanes[N]));
+  }
+};
+
+HWY_NOINLINE void TestAllZeroExtendVector() {
+  ForAllTypesAndSpecial(ForExtendableVectors<TestZeroExtendVector>());
+}
+
+struct TestCombine {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Twice<D> d2;
+    const size_t N2 = Lanes(d2);
+    if (N2 < 2) return;
+    auto lanes = AllocateAligned<T>(N2);
+    HWY_ASSERT(lanes);
+
+    const Vec<D> lo = Iota(d, 1);
+    const Vec<D> hi = Iota(d, N2 / 2 + 1);
+    const Vec<decltype(d2)> combined = Combine(d2, hi, lo);
+    Store(combined, d2, lanes.get());
+
+    const Vec<decltype(d2)> expected = Iota(d2, 1);
+    HWY_ASSERT_VEC_EQ(d2, expected, combined);
+  }
+};
+
+HWY_NOINLINE void TestAllCombine() {
+  ForAllTypes(ForExtendableVectors<TestCombine>());
+}
+
+struct TestInterleaveWholeHalves {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    const size_t N = Lanes(d);
+    using TU = MakeUnsigned<T>;
+
+    constexpr TU kMsb = SignMask<T>();
+    const TU hi_bit = (!IsFloat<T>() && !IsSpecialFloat<T>() && N < kMsb)
+                          ? static_cast<TU>(N)
+                          : kMsb;
+    const TU lo_mask = static_cast<TU>(hi_bit - 1u);
+
+    const RebindToUnsigned<decltype(d)> du;
+    const auto v0 = And(Iota(d, 0), BitCast(d, Set(du, lo_mask)));
+    const auto v1 = Or(v0, BitCast(d, Set(du, hi_bit)));
+
+    auto v0_lanes = AllocateAligned<T>(N);
+    auto v1_lanes = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(v0_lanes && v1_lanes && expected);
+
+    Store(v0, d, v0_lanes.get());
+    Store(v1, d, v1_lanes.get());
+
+    const size_t half_N = N / 2;
+    for (size_t i = 0; i < half_N; i++) {
+      expected[2 * i] = v0_lanes[i];
+      expected[2 * i + 1] = v1_lanes[i];
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveWholeLower(d, v0, v1));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveWholeLower(v0, v1));
+
+    for (size_t i = 0; i < half_N; i++) {
+      expected[2 * i] = v1_lanes[i];
+      expected[2 * i + 1] = v0_lanes[i];
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveWholeLower(d, v1, v0));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveWholeLower(v1, v0));
+
+    for (size_t i = 0; i < half_N; i++) {
+      expected[2 * i] = v0_lanes[i + half_N];
+      expected[2 * i + 1] = v1_lanes[i + half_N];
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveWholeUpper(d, v0, v1));
+
+    for (size_t i = 0; i < half_N; i++) {
+      expected[2 * i] = v1_lanes[i + half_N];
+      expected[2 * i + 1] = v0_lanes[i + half_N];
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveWholeUpper(d, v1, v0));
+#else
+    (void)d;
+#endif  // HWY_TARGET != HWY_SCALAR
+  }
+};
+
+HWY_NOINLINE void TestAllInterleaveWholeHalves() {
+  ForAllTypes(ForShrinkableVectors<TestInterleaveWholeHalves>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyCombineTest);
+HWY_EXPORT_AND_TEST_P(HwyCombineTest, TestAllLowerHalf);
+HWY_EXPORT_AND_TEST_P(HwyCombineTest, TestAllUpperHalf);
+HWY_EXPORT_AND_TEST_P(HwyCombineTest, TestAllZeroExtendVector);
+HWY_EXPORT_AND_TEST_P(HwyCombineTest, TestAllCombine);
+HWY_EXPORT_AND_TEST_P(HwyCombineTest, TestAllInterleaveWholeHalves);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/compare_test.cc

@@ -0,0 +1,702 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/compare_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+// All types.
+struct TestEquality {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v2 = Iota(d, 2);
+    const auto v2b = Iota(d, 2);
+    const auto v3 = Iota(d, 3);
+
+    const auto mask_false = MaskFalse(d);
+    const auto mask_true = MaskTrue(d);
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq(v2, v3));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq(v3, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq(v2, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq(v2, v2b));
+
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne(v2, v3));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne(v3, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne(v2, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne(v2, v2b));
+  }
+};
+
+HWY_NOINLINE void TestAllEquality() {
+  ForAllTypes(ForPartialVectors<TestEquality>());
+}
+
+// a > b should be true, verify that for Gt/Lt and with swapped args.
+template <class D>
+void EnsureGreater(D d, TFromD<D> a, TFromD<D> b, const char* file, int line) {
+  const auto mask_false = MaskFalse(d);
+  const auto mask_true = MaskTrue(d);
+
+  const auto va = Set(d, a);
+  const auto vb = Set(d, b);
+  AssertMaskEqual(d, mask_true, Gt(va, vb), file, line);
+  AssertMaskEqual(d, mask_false, Lt(va, vb), file, line);
+
+  // Swapped order
+  AssertMaskEqual(d, mask_false, Gt(vb, va), file, line);
+  AssertMaskEqual(d, mask_true, Lt(vb, va), file, line);
+
+  // Also ensure irreflexive
+  AssertMaskEqual(d, mask_false, Gt(va, va), file, line);
+  AssertMaskEqual(d, mask_false, Gt(vb, vb), file, line);
+  AssertMaskEqual(d, mask_false, Lt(va, va), file, line);
+  AssertMaskEqual(d, mask_false, Lt(vb, vb), file, line);
+}
+
+#define HWY_ENSURE_GREATER(d, a, b) EnsureGreater(d, a, b, __FILE__, __LINE__)
+
+// a >= b should be true, verify that for Ge/Le and with swapped args.
+template <class D>
+void EnsureGreaterOrEqual(D d, TFromD<D> a, TFromD<D> b, const char* file,
+                          int line) {
+  const auto mask_true = MaskTrue(d);
+
+  const auto va = Set(d, a);
+  const auto vb = Set(d, b);
+
+  const auto mask_eq = Eq(va, vb);
+
+  AssertMaskEqual(d, mask_true, Ge(va, vb), file, line);
+  AssertMaskEqual(d, mask_eq, Le(va, vb), file, line);
+
+  // Swapped order
+  AssertMaskEqual(d, mask_eq, Ge(vb, va), file, line);
+  AssertMaskEqual(d, mask_true, Le(vb, va), file, line);
+
+  // va >= va, vb >= vb, va <= va, and vb <= vb should all be true if
+  // both a and b are non-NaN values
+  AssertMaskEqual(d, mask_true, Ge(va, va), file, line);
+  AssertMaskEqual(d, mask_true, Ge(vb, vb), file, line);
+  AssertMaskEqual(d, mask_true, Le(va, va), file, line);
+  AssertMaskEqual(d, mask_true, Le(vb, vb), file, line);
+}
+
+#define HWY_ENSURE_GREATER_OR_EQUAL(d, a, b) \
+  EnsureGreaterOrEqual(d, a, b, __FILE__, __LINE__)
+
+struct TestStrictUnsigned {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const T max = LimitsMax<T>();
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> v2 = And(Iota(d, 2), Set(d, 255));  // 0..255
+
+    const Mask<D> mask_false = MaskFalse(d);
+
+    // Individual values of interest
+    HWY_ENSURE_GREATER(d, 2, 1);
+    HWY_ENSURE_GREATER(d, 1, 0);
+    HWY_ENSURE_GREATER(d, 128, 127);
+    HWY_ENSURE_GREATER(d, max, max / 2);
+    HWY_ENSURE_GREATER(d, max, 1);
+    HWY_ENSURE_GREATER(d, max, 0);
+
+    // Also use Iota to ensure lanes are independent
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v2, v0));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v0, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v0, v0));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v0, v0));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v2, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v2, v2));
+  }
+};
+
+HWY_NOINLINE void TestAllStrictUnsigned() {
+  ForUnsignedTypes(ForPartialVectors<TestStrictUnsigned>());
+}
+
+struct TestWeakUnsigned {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const T max = LimitsMax<T>();
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> v1 = Set(d, 1u);
+    const Vec<D> v2 = And(Iota(d, 2), Set(d, 255u));  // 0..255
+
+    const Mask<D> mask_true = MaskTrue(d);
+
+    // Individual values of interest
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 2, 2);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 2, 1);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 1, 1);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 1, 0);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 0, 0);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 128, 127);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 128, 128);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 127, 127);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, max, max);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, max, max / 2);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, max, 1);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, max, 0);
+
+    // Also use Iota to ensure lanes are independent
+    const auto mask_v2_is_eq_to_v0 = Eq(v2, v0);
+    HWY_ASSERT_MASK_EQ(d, mask_v2_is_eq_to_v0, Le(v2, v0));
+    HWY_ASSERT_MASK_EQ(d, mask_v2_is_eq_to_v0, Ge(v0, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Le(v0, v0));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ge(v0, v0));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Le(v2, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ge(v2, v2));
+
+    const auto v2_plus_1 = Add(v2, v1);
+    HWY_ASSERT_MASK_EQ(d, Lt(v2, v2_plus_1), Le(v2, v2_plus_1));
+    HWY_ASSERT_MASK_EQ(d, Gt(v2, v2_plus_1), Ge(v2, v2_plus_1));
+    HWY_ASSERT_MASK_EQ(d, Lt(v2_plus_1, v2), Le(v2_plus_1, v2));
+    HWY_ASSERT_MASK_EQ(d, Gt(v2_plus_1, v2), Ge(v2_plus_1, v2));
+
+    const auto v2_minus_1 = Sub(v2, v1);
+    HWY_ASSERT_MASK_EQ(d, Lt(v2, v2_minus_1), Le(v2, v2_minus_1));
+    HWY_ASSERT_MASK_EQ(d, Gt(v2, v2_minus_1), Ge(v2, v2_minus_1));
+    HWY_ASSERT_MASK_EQ(d, Lt(v2_minus_1, v2), Le(v2_minus_1, v2));
+    HWY_ASSERT_MASK_EQ(d, Gt(v2_minus_1, v2), Ge(v2_minus_1, v2));
+  }
+};
+
+HWY_NOINLINE void TestAllWeakUnsigned() {
+  ForUnsignedTypes(ForPartialVectors<TestStrictUnsigned>());
+}
+
+struct TestStrictInt {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const T min = LimitsMin<T>();
+    const T max = LimitsMax<T>();
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> v2 = And(Iota(d, 2), Set(d, 127));  // 0..127
+    const Vec<D> vn = Sub(Neg(v2), Set(d, 1));       // -1..-128
+
+    const Mask<D> mask_false = MaskFalse(d);
+    const Mask<D> mask_true = MaskTrue(d);
+
+    // Individual values of interest
+    HWY_ENSURE_GREATER(d, 2, 1);
+    HWY_ENSURE_GREATER(d, 1, 0);
+    HWY_ENSURE_GREATER(d, 0, -1);
+    HWY_ENSURE_GREATER(d, -1, -2);
+    HWY_ENSURE_GREATER(d, max, max / 2);
+    HWY_ENSURE_GREATER(d, max, 1);
+    HWY_ENSURE_GREATER(d, max, 0);
+    HWY_ENSURE_GREATER(d, max, -1);
+    HWY_ENSURE_GREATER(d, max, min);
+    HWY_ENSURE_GREATER(d, 0, min);
+    HWY_ENSURE_GREATER(d, min / 2, min);
+
+    // Also use Iota to ensure lanes are independent
+    HWY_ASSERT_MASK_EQ(d, mask_true, Gt(v2, vn));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt(vn, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v2, vn));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Gt(vn, v2));
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v0, v0));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v2, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt(vn, vn));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v0, v0));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v2, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Gt(vn, vn));
+  }
+};
+
+// S-SSE3 bug (#795): same upper, differing MSB in lower
+struct TestStrictInt64 {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto m0 = MaskFalse(d);
+    const auto m1 = MaskTrue(d);
+    HWY_ASSERT_MASK_EQ(d, m0, Lt(Set(d, 0x380000000LL), Set(d, 0x300000001LL)));
+    HWY_ASSERT_MASK_EQ(d, m1, Lt(Set(d, 0xF00000000LL), Set(d, 0xF80000000LL)));
+    HWY_ASSERT_MASK_EQ(d, m1, Lt(Set(d, 0xF00000000LL), Set(d, 0xF80000001LL)));
+  }
+};
+
+HWY_NOINLINE void TestAllStrictInt() {
+  ForSignedTypes(ForPartialVectors<TestStrictInt>());
+  ForPartialVectors<TestStrictInt64>()(int64_t());
+}
+
+struct TestWeakInt {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const T min = LimitsMin<T>();
+    const T max = LimitsMax<T>();
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> v1 = Set(d, 1);
+    const Vec<D> v2 = And(Iota(d, 2), Set(d, 127));  // 0..127
+    const Vec<D> vn = Sub(Neg(v2), Set(d, 1));       // -1..-128
+
+    const auto mask_false = MaskFalse(d);
+    const auto mask_true = MaskTrue(d);
+
+    // Individual values of interest
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 2, 2);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 2, 1);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 1, 1);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 1, 0);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 0, 0);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 0, -1);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, -1, -1);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, -1, -2);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, -2, -2);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, max, max);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, max, max / 2);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, max, 1);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, max, 0);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, max, -1);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, max, min);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, 0, min);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, min / 2, min);
+    HWY_ENSURE_GREATER_OR_EQUAL(d, min, min);
+
+    // Also use Iota to ensure lanes are independent
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ge(v2, vn));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Le(vn, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Le(v2, vn));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ge(vn, v2));
+
+    HWY_ASSERT_MASK_EQ(d, mask_true, Le(v0, v0));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Le(v2, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Le(vn, vn));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ge(v0, v0));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ge(v2, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ge(vn, vn));
+
+    const auto v2_plus_1 = Add(v2, v1);
+    HWY_ASSERT_MASK_EQ(d, Lt(v2, v2_plus_1), Le(v2, v2_plus_1));
+    HWY_ASSERT_MASK_EQ(d, Gt(v2, v2_plus_1), Ge(v2, v2_plus_1));
+    HWY_ASSERT_MASK_EQ(d, Lt(v2_plus_1, v2), Le(v2_plus_1, v2));
+    HWY_ASSERT_MASK_EQ(d, Gt(v2_plus_1, v2), Ge(v2_plus_1, v2));
+
+    const auto v2_minus_1 = Sub(v2, v1);
+    HWY_ASSERT_MASK_EQ(d, Lt(v2, v2_minus_1), Le(v2, v2_minus_1));
+    HWY_ASSERT_MASK_EQ(d, Gt(v2, v2_minus_1), Ge(v2, v2_minus_1));
+    HWY_ASSERT_MASK_EQ(d, Lt(v2_minus_1, v2), Le(v2_minus_1, v2));
+    HWY_ASSERT_MASK_EQ(d, Gt(v2_minus_1, v2), Ge(v2_minus_1, v2));
+  }
+};
+
+HWY_NOINLINE void TestAllWeakInt() {
+  ForSignedTypes(ForPartialVectors<TestWeakInt>());
+}
+
+struct TestStrictFloat {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const T huge_pos = ConvertScalarTo<T>(sizeof(T) >= 4 ? 1E36 : 1E4);
+    const T huge_neg = -huge_pos;
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> v2 = Iota(d, 2);
+    const Vec<D> vn = Neg(v2);
+
+    const Mask<D> mask_false = MaskFalse(d);
+    const Mask<D> mask_true = MaskTrue(d);
+
+    // Individual values of interest
+    HWY_ENSURE_GREATER(d, 2, 1);
+    HWY_ENSURE_GREATER(d, 1, 0);
+    HWY_ENSURE_GREATER(d, 0, -1);
+    HWY_ENSURE_GREATER(d, -1, -2);
+    HWY_ENSURE_GREATER(d, huge_pos, 1);
+    HWY_ENSURE_GREATER(d, huge_pos, 0);
+    HWY_ENSURE_GREATER(d, huge_pos, -1);
+    HWY_ENSURE_GREATER(d, huge_pos, huge_neg);
+    HWY_ENSURE_GREATER(d, 0, huge_neg);
+
+    // Also use Iota to ensure lanes are independent
+    HWY_ASSERT_MASK_EQ(d, mask_true, Gt(v2, vn));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt(vn, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v2, vn));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Gt(vn, v2));
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v0, v0));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v2, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt(vn, vn));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v0, v0));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v2, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Gt(vn, vn));
+  }
+};
+
+HWY_NOINLINE void TestAllStrictFloat() {
+  ForFloatTypes(ForPartialVectors<TestStrictFloat>());
+}
+
+struct TestWeakFloat {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> v2 = Iota(d, 2);
+    const Vec<D> vn = Iota(d, -ConvertScalarTo<T>(Lanes(d)));
+
+    const Mask<D> mask_false = MaskFalse(d);
+    const Mask<D> mask_true = MaskTrue(d);
+
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ge(v2, v2));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Le(vn, vn));
+
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ge(v2, vn));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Le(vn, v2));
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, Le(v2, vn));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ge(vn, v2));
+  }
+};
+
+HWY_NOINLINE void TestAllWeakFloat() {
+  ForFloatTypes(ForPartialVectors<TestWeakFloat>());
+}
+
+struct TestIsNegative {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const RebindToSigned<decltype(d)> di;
+    const RebindToUnsigned<decltype(d)> du;
+    using TU = TFromD<decltype(du)>;
+
+    const auto mask_false = MaskFalse(d);
+    const auto mask_true = MaskTrue(d);
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, IsNegative(Zero(d)));
+    HWY_ASSERT_MASK_EQ(d, mask_true,
+                       IsNegative(Set(d, ConvertScalarTo<T>(-1))));
+
+    const auto vsignbit = SignBit(d);
+    const auto vp = AndNot(vsignbit, Iota(d, 1));
+    const auto vn = Or(vp, vsignbit);
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, IsNegative(vp));
+    HWY_ASSERT_MASK_EQ(d, mask_true, IsNegative(vn));
+
+    const auto s1 = BitCast(d, ShiftLeft<sizeof(TU) * 8 - 1>(Iota(du, 1)));
+
+    const auto x1 = Xor3(vp, s1, BitCast(d, Set(du, TU{0x71})));
+    const auto x2 = Xor(x1, vsignbit);
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, And(IsNegative(x1), IsNegative(x2)));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Or(IsNegative(x1), IsNegative(x2)));
+
+    const auto expected_1 =
+        RebindMask(d, MaskFromVec(BroadcastSignBit(BitCast(di, x1))));
+    const auto expected_2 =
+        RebindMask(d, MaskFromVec(BroadcastSignBit(BitCast(di, x2))));
+
+    HWY_ASSERT_MASK_EQ(d, expected_1, IsNegative(x1));
+    HWY_ASSERT_MASK_EQ(d, expected_2, IsNegative(x2));
+  }
+};
+
+HWY_NOINLINE void TestAllIsNegative() {
+  ForFloatTypes(ForPartialVectors<TestIsNegative>());
+  ForSignedTypes(ForPartialVectors<TestIsNegative>());
+}
+
+template <class D>
+static HWY_NOINLINE Vec<D> Make128(D d, uint64_t hi, uint64_t lo) {
+  alignas(16) uint64_t in[2];
+  in[0] = lo;
+  in[1] = hi;
+  return LoadDup128(d, in);
+}
+
+struct TestLt128 {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using V = Vec<D>;
+    const V v00 = Zero(d);
+    const V v01 = Make128(d, 0, 1);
+    const V v10 = Make128(d, 1, 0);
+    const V v11 = Add(v01, v10);
+
+    const auto mask_false = MaskFalse(d);
+    const auto mask_true = MaskTrue(d);
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, v00, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, v01, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, v10, v10));
+
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128(d, v00, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128(d, v01, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128(d, v01, v11));
+
+    // Reversed order
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, v01, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, v10, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, v11, v01));
+
+    // Also check 128-bit blocks are independent
+    const V iota = Iota(d, 1);
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128(d, iota, Add(iota, v01)));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128(d, iota, Add(iota, v10)));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, Add(iota, v01), iota));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, Add(iota, v10), iota));
+
+    // Max value
+    const V vm = Make128(d, LimitsMax<T>(), LimitsMax<T>());
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, vm, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, vm, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, vm, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, vm, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128(d, vm, v11));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128(d, v00, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128(d, v01, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128(d, v10, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128(d, v11, vm));
+  }
+};
+
+HWY_NOINLINE void TestAllLt128() { ForGEVectors<128, TestLt128>()(uint64_t()); }
+
+struct TestLt128Upper {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using V = Vec<D>;
+    const V v00 = Zero(d);
+    const V v01 = Make128(d, 0, 1);
+    const V v10 = Make128(d, 1, 0);
+    const V v11 = Add(v01, v10);
+
+    const auto mask_false = MaskFalse(d);
+    const auto mask_true = MaskTrue(d);
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, v00, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, v01, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, v10, v10));
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, v00, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128Upper(d, v01, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128Upper(d, v01, v11));
+
+    // Reversed order
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, v01, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, v10, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, v11, v01));
+
+    // Also check 128-bit blocks are independent
+    const V iota = Iota(d, 1);
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, iota, Add(iota, v01)));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128Upper(d, iota, Add(iota, v10)));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, Add(iota, v01), iota));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, Add(iota, v10), iota));
+
+    // Max value
+    const V vm = Make128(d, LimitsMax<T>(), LimitsMax<T>());
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, vm, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, vm, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, vm, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, vm, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Lt128Upper(d, vm, v11));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128Upper(d, v00, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128Upper(d, v01, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128Upper(d, v10, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Lt128Upper(d, v11, vm));
+  }
+};
+
+HWY_NOINLINE void TestAllLt128Upper() {
+  ForGEVectors<128, TestLt128Upper>()(uint64_t());
+}
+
+struct TestEq128 {  // Also Ne128
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using V = Vec<D>;
+    const V v00 = Zero(d);
+    const V v01 = Make128(d, 0, 1);
+    const V v10 = Make128(d, 1, 0);
+    const V v11 = Add(v01, v10);
+
+    const auto mask_false = MaskFalse(d);
+    const auto mask_true = MaskTrue(d);
+
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq128(d, v00, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq128(d, v01, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq128(d, v10, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne128(d, v00, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne128(d, v01, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne128(d, v10, v10));
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, v00, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, v01, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, v01, v11));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, v00, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, v01, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, v01, v11));
+
+    // Reversed order
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, v01, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, v10, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, v11, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, v01, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, v10, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, v11, v01));
+
+    // Also check 128-bit blocks are independent
+    const V iota = Iota(d, 1);
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, iota, Add(iota, v01)));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, iota, Add(iota, v10)));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, Add(iota, v01), iota));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, Add(iota, v10), iota));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, iota, Add(iota, v01)));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, iota, Add(iota, v10)));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, Add(iota, v01), iota));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, Add(iota, v10), iota));
+
+    // Max value
+    const V vm = Make128(d, LimitsMax<T>(), LimitsMax<T>());
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq128(d, vm, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne128(d, vm, vm));
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, vm, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, vm, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, vm, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, vm, v11));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, v00, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, v01, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, v10, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128(d, v11, vm));
+
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, vm, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, vm, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, vm, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, vm, v11));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, v00, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, v01, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, v10, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128(d, v11, vm));
+  }
+};
+
+HWY_NOINLINE void TestAllEq128() { ForGEVectors<128, TestEq128>()(uint64_t()); }
+
+struct TestEq128Upper {  // Also Ne128Upper
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using V = Vec<D>;
+    const V v00 = Zero(d);
+    const V v01 = Make128(d, 0, 1);
+    const V v10 = Make128(d, 1, 0);
+    const V v11 = Add(v01, v10);
+
+    const auto mask_false = MaskFalse(d);
+    const auto mask_true = MaskTrue(d);
+
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq128Upper(d, v00, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq128Upper(d, v01, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq128Upper(d, v10, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne128Upper(d, v00, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne128Upper(d, v01, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne128Upper(d, v10, v10));
+
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq128Upper(d, v00, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne128Upper(d, v00, v01));
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, v01, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, v01, v11));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, v01, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, v01, v11));
+
+    // Reversed order
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq128Upper(d, v01, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne128Upper(d, v01, v00));
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, v10, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, v11, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, v10, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, v11, v01));
+
+    // Also check 128-bit blocks are independent
+    const V iota = Iota(d, 1);
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq128Upper(d, iota, Add(iota, v01)));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne128Upper(d, iota, Add(iota, v01)));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, iota, Add(iota, v10)));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, iota, Add(iota, v10)));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq128Upper(d, Add(iota, v01), iota));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne128Upper(d, Add(iota, v01), iota));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, Add(iota, v10), iota));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, Add(iota, v10), iota));
+
+    // Max value
+    const V vm = Make128(d, LimitsMax<T>(), LimitsMax<T>());
+    HWY_ASSERT_MASK_EQ(d, mask_true, Eq128Upper(d, vm, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Ne128Upper(d, vm, vm));
+
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, vm, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, vm, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, vm, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, vm, v11));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, v00, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, v01, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, v10, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_false, Eq128Upper(d, v11, vm));
+
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, vm, v00));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, vm, v01));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, vm, v10));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, vm, v11));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, v00, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, v01, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, v10, vm));
+    HWY_ASSERT_MASK_EQ(d, mask_true, Ne128Upper(d, v11, vm));
+  }
+};
+
+HWY_NOINLINE void TestAllEq128Upper() {
+  ForGEVectors<128, TestEq128Upper>()(uint64_t());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyCompareTest);
+HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllEquality);
+HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllStrictUnsigned);
+HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllStrictInt);
+HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllStrictFloat);
+HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllWeakUnsigned);
+HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllWeakInt);
+HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllWeakFloat);
+HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllIsNegative);
+HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllLt128);
+HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllLt128Upper);
+HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllEq128);
+HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllEq128Upper);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/compress_test.cc

@@ -0,0 +1,830 @@
+// Copyright 2022 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <array>  // IWYU pragma: keep
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/compress_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+// Regenerate tables used in the implementation, instead of testing.
+#define HWY_PRINT_TABLES 0
+
+#if !HWY_PRINT_TABLES || HWY_IDE
+
+template <class D, class DI, typename T = TFromD<D>, typename TI = TFromD<DI>>
+void CheckStored(D d, DI di, const char* op, size_t expected_pos,
+                 size_t actual_pos, size_t num_to_check,
+                 const AlignedFreeUniquePtr<T[]>& in,
+                 const AlignedFreeUniquePtr<TI[]>& mask_lanes,
+                 const AlignedFreeUniquePtr<T[]>& expected, const T* actual_u,
+                 int line) {
+  if (expected_pos != actual_pos) {
+    hwy::Abort(__FILE__, line,
+               "%s: size mismatch for %s: expected %d, actual %d\n", op,
+               TypeName(T(), Lanes(d)).c_str(), static_cast<int>(expected_pos),
+               static_cast<int>(actual_pos));
+  }
+  // Modified from AssertVecEqual - we may not be checking all lanes.
+  for (size_t i = 0; i < num_to_check; ++i) {
+    if (!IsEqual(expected[i], actual_u[i])) {
+      const size_t N = Lanes(d);
+      fprintf(stderr, "%s: mismatch at i=%d of %d, line %d:\n\n", op,
+              static_cast<int>(i), static_cast<int>(num_to_check), line);
+      Print(di, "mask", Load(di, mask_lanes.get()), 0, N);
+      Print(d, "in", Load(d, in.get()), 0, N);
+      Print(d, "expect", Load(d, expected.get()), 0, num_to_check);
+      Print(d, "actual", Load(d, actual_u), 0, num_to_check);
+      HWY_ASSERT(false);
+    }
+  }
+}
+
+struct TestCompress {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    using TU = MakeUnsigned<T>;
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(d);
+    const size_t bits_size = RoundUpTo((N + 7) / 8, 8);
+
+    for (int frac : {0, 2, 3}) {
+      // For CompressStore
+      const size_t misalign = static_cast<size_t>(frac) * N / 4;
+
+      auto in_lanes = AllocateAligned<T>(N);
+      auto mask_lanes = AllocateAligned<TI>(N);
+      auto garbage = AllocateAligned<TU>(N);
+      auto expected = AllocateAligned<T>(N);
+      auto actual_a = AllocateAligned<T>(misalign + N);
+      auto bits = AllocateAligned<uint8_t>(bits_size);
+      HWY_ASSERT(in_lanes && mask_lanes && garbage && expected && actual_a &&
+                 bits);
+
+      T* actual_u = actual_a.get() + misalign;
+      ZeroBytes(bits.get(), bits_size);  // for MSAN
+
+      // Each lane should have a chance of having mask=true.
+      for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+        size_t expected_pos = 0;
+        for (size_t i = 0; i < N; ++i) {
+          in_lanes[i] = RandomFiniteValue<T>(&rng);
+          mask_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+          if (mask_lanes[i] > 0) {
+            expected[expected_pos++] = in_lanes[i];
+          }
+          garbage[i] = static_cast<TU>(Random64(&rng));
+        }
+        size_t num_to_check;
+        if (CompressIsPartition<T>::value) {
+          // For non-native Compress, also check that mask=false lanes were
+          // moved to the back of the vector (highest indices).
+          size_t extra = expected_pos;
+          for (size_t i = 0; i < N; ++i) {
+            if (mask_lanes[i] == 0) {
+              expected[extra++] = in_lanes[i];
+            }
+          }
+          HWY_ASSERT(extra == N);
+          num_to_check = N;
+        } else {
+          // For native Compress, only the mask=true lanes are defined.
+          num_to_check = expected_pos;
+        }
+
+        const auto in = Load(d, in_lanes.get());
+        const auto mask =
+            RebindMask(d, Gt(Load(di, mask_lanes.get()), Zero(di)));
+        StoreMaskBits(d, mask, bits.get());
+
+        // Compress
+        ZeroBytes(actual_u, N * sizeof(T));
+        StoreU(Compress(in, mask), d, actual_u);
+        CheckStored(d, di, "Compress", expected_pos, expected_pos, num_to_check,
+                    in_lanes, mask_lanes, expected, actual_u, __LINE__);
+
+        // CompressNot
+        ZeroBytes(actual_u, N * sizeof(T));
+        StoreU(CompressNot(in, Not(mask)), d, actual_u);
+        CheckStored(d, di, "CompressNot", expected_pos, expected_pos,
+                    num_to_check, in_lanes, mask_lanes, expected, actual_u,
+                    __LINE__);
+
+        // CompressStore
+        ZeroBytes(actual_u, N * sizeof(T));
+        const size_t size1 = CompressStore(in, mask, d, actual_u);
+        // expected_pos instead of num_to_check because this op is not
+        // affected by CompressIsPartition.
+        CheckStored(d, di, "CompressStore", expected_pos, size1, expected_pos,
+                    in_lanes, mask_lanes, expected, actual_u, __LINE__);
+
+        // CompressBlendedStore
+        memcpy(actual_u, garbage.get(), N * sizeof(T));
+        const size_t size2 = CompressBlendedStore(in, mask, d, actual_u);
+        // expected_pos instead of num_to_check because this op only writes
+        // the mask=true lanes.
+        CheckStored(d, di, "CompressBlendedStore", expected_pos, size2,
+                    expected_pos, in_lanes, mask_lanes, expected, actual_u,
+                    __LINE__);
+        // Subsequent lanes are untouched.
+        for (size_t i = size2; i < N; ++i) {
+#if HWY_COMPILER_MSVC && HWY_TARGET == HWY_AVX2
+          // TODO(eustas): re-enable when compiler is fixed
+#else
+          HWY_ASSERT_EQ(garbage[i], reinterpret_cast<TU*>(actual_u)[i]);
+#endif
+        }
+
+        // CompressBits
+        ZeroBytes(actual_u, N * sizeof(T));
+        StoreU(CompressBits(in, bits.get()), d, actual_u);
+        CheckStored(d, di, "CompressBits", expected_pos, expected_pos,
+                    num_to_check, in_lanes, mask_lanes, expected, actual_u,
+                    __LINE__);
+
+        // CompressBitsStore
+        ZeroBytes(actual_u, N * sizeof(T));
+        const size_t size3 = CompressBitsStore(in, bits.get(), d, actual_u);
+        // expected_pos instead of num_to_check because this op is not
+        // affected by CompressIsPartition.
+        CheckStored(d, di, "CompressBitsStore", expected_pos, size3,
+                    expected_pos, in_lanes, mask_lanes, expected, actual_u,
+                    __LINE__);
+      }  // rep
+    }    // frac
+  }      // operator()
+};
+
+HWY_NOINLINE void TestAllCompress() {
+  ForAllTypes(ForPartialVectors<TestCompress>());
+}
+
+struct TestCompressBlocks {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET == HWY_SCALAR
+    (void)d;
+#else
+    static_assert(sizeof(T) == 8 && !IsSigned<T>(), "Should be u64");
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(d);
+
+    auto in_lanes = AllocateAligned<T>(N);
+    auto mask_lanes = AllocateAligned<TI>(N);
+    auto expected = AllocateAligned<T>(N);
+    auto actual = AllocateAligned<T>(N);
+    HWY_ASSERT(in_lanes && mask_lanes && expected && actual);
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      size_t expected_pos = 0;
+      for (size_t i = 0; i < N; i += 2) {
+        in_lanes[i] = RandomFiniteValue<T>(&rng);
+        in_lanes[i + 1] = RandomFiniteValue<T>(&rng);
+        mask_lanes[i + 1] = mask_lanes[i] = TI{(Random32(&rng) & 8) ? 1 : 0};
+        if (mask_lanes[i] > 0) {
+          expected[expected_pos++] = in_lanes[i];
+          expected[expected_pos++] = in_lanes[i + 1];
+        }
+      }
+      size_t num_to_check;
+      if (CompressIsPartition<T>::value) {
+        // For non-native Compress, also check that mask=false lanes were
+        // moved to the back of the vector (highest indices).
+        size_t extra = expected_pos;
+        for (size_t i = 0; i < N; ++i) {
+          if (mask_lanes[i] == 0) {
+            expected[extra++] = in_lanes[i];
+          }
+        }
+        HWY_ASSERT(extra == N);
+        num_to_check = N;
+      } else {
+        // For native Compress, only the mask=true lanes are defined.
+        num_to_check = expected_pos;
+      }
+
+      const auto in = Load(d, in_lanes.get());
+      const auto mask = RebindMask(d, Gt(Load(di, mask_lanes.get()), Zero(di)));
+
+      // CompressBlocksNot
+      ZeroBytes(actual.get(), N * sizeof(T));
+      StoreU(CompressBlocksNot(in, Not(mask)), d, actual.get());
+      CheckStored(d, di, "CompressBlocksNot", expected_pos, expected_pos,
+                  num_to_check, in_lanes, mask_lanes, expected, actual.get(),
+                  __LINE__);
+    }  // rep
+#endif  // HWY_TARGET == HWY_SCALAR
+  }     // operator()
+};
+
+HWY_NOINLINE void TestAllCompressBlocks() {
+  ForGE128Vectors<TestCompressBlocks>()(uint64_t());
+}
+
+#endif  // !HWY_PRINT_TABLES
+
+#if HWY_PRINT_TABLES || HWY_IDE
+void PrintCompress8x8Tables() {
+  printf("======================================= 8x8\n");
+  constexpr size_t N = 8;
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    std::array<uint8_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    for (size_t i = 0; i < N; ++i) {
+      printf("%d,", indices[i]);
+    }
+    printf(code & 1 ? "//\n" : "/**/");
+  }
+  printf("\n");
+}
+
+void PrintCompress16x8Tables() {
+  printf("======================================= 16x8\n");
+  constexpr size_t N = 8;  // 128-bit SIMD
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    std::array<uint8_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    // Doubled (for converting lane to byte indices)
+    for (size_t i = 0; i < N; ++i) {
+      printf("%d,", 2 * indices[i]);
+    }
+    printf(code & 1 ? "//\n" : "/**/");
+  }
+  printf("\n");
+}
+
+void PrintCompressNot16x8Tables() {
+  printf("======================================= Not 16x8\n");
+  constexpr size_t N = 8;  // 128-bit SIMD
+  for (uint64_t not_code = 0; not_code < (1ull << N); ++not_code) {
+    const uint64_t code = ~not_code;
+    std::array<uint8_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    // Doubled (for converting lane to byte indices)
+    for (size_t i = 0; i < N; ++i) {
+      printf("%d,", 2 * indices[i]);
+    }
+    printf(not_code & 1 ? "//\n" : "/**/");
+  }
+  printf("\n");
+}
+
+// Compressed to nibbles, unpacked via variable right shift. Also includes
+// FirstN bits in the nibble MSB.
+void PrintCompress32x8Tables() {
+  printf("======================================= 32/64x8\n");
+  constexpr size_t N = 8;  // AVX2 or 64-bit AVX3
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    const size_t count = PopCount(code);
+    std::array<uint32_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    // Convert to nibbles
+    uint64_t packed = 0;
+    for (size_t i = 0; i < N; ++i) {
+      HWY_ASSERT(indices[i] < N);
+      if (i < count) {
+        indices[i] |= N;
+        HWY_ASSERT(indices[i] < 0x10);
+      }
+      packed += indices[i] << (i * 4);
+    }
+
+    HWY_ASSERT(packed < (1ull << (N * 4)));
+    printf("0x%08x,", static_cast<uint32_t>(packed));
+  }
+  printf("\n");
+}
+
+void PrintCompressNot32x8Tables() {
+  printf("======================================= Not 32/64x8\n");
+  constexpr size_t N = 8;  // AVX2 or 64-bit AVX3
+  for (uint64_t not_code = 0; not_code < (1ull << N); ++not_code) {
+    const uint64_t code = ~not_code;
+    const size_t count = PopCount(code);
+    std::array<uint32_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    // Convert to nibbles
+    uint64_t packed = 0;
+    for (size_t i = 0; i < N; ++i) {
+      HWY_ASSERT(indices[i] < N);
+      if (i < count) {
+        indices[i] |= N;
+        HWY_ASSERT(indices[i] < 0x10);
+      }
+      packed += indices[i] << (i * 4);
+    }
+
+    HWY_ASSERT(packed < (1ull << (N * 4)));
+    printf("0x%08x,", static_cast<uint32_t>(packed));
+  }
+  printf("\n");
+}
+
+// Compressed to nibbles (for AVX3 64x4)
+void PrintCompress64x4NibbleTables() {
+  printf("======================================= 64x4Nibble\n");
+  constexpr size_t N = 4;  // AVX2
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    std::array<uint32_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    // Convert to nibbles
+    uint64_t packed = 0;
+    for (size_t i = 0; i < N; ++i) {
+      HWY_ASSERT(indices[i] < N);
+      packed += indices[i] << (i * 4);
+    }
+
+    HWY_ASSERT(packed < (1ull << (N * 4)));
+    printf("0x%08x,", static_cast<uint32_t>(packed));
+  }
+  printf("\n");
+}
+
+void PrintCompressNot64x4NibbleTables() {
+  printf("======================================= Not 64x4Nibble\n");
+  constexpr size_t N = 4;  // AVX2
+  for (uint64_t not_code = 0; not_code < (1ull << N); ++not_code) {
+    const uint64_t code = ~not_code;
+    std::array<uint32_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    // Convert to nibbles
+    uint64_t packed = 0;
+    for (size_t i = 0; i < N; ++i) {
+      HWY_ASSERT(indices[i] < N);
+      packed += indices[i] << (i * 4);
+    }
+
+    HWY_ASSERT(packed < (1ull << (N * 4)));
+    printf("0x%08x,", static_cast<uint32_t>(packed));
+  }
+  printf("\n");
+}
+
+void PrintCompressNot64x2NibbleTables() {
+  printf("======================================= Not 64x2Nibble\n");
+  constexpr size_t N = 2;  // 128-bit
+  for (uint64_t not_code = 0; not_code < (1ull << N); ++not_code) {
+    const uint64_t code = ~not_code;
+    std::array<uint32_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    // Convert to nibbles
+    uint64_t packed = 0;
+    for (size_t i = 0; i < N; ++i) {
+      HWY_ASSERT(indices[i] < N);
+      packed += indices[i] << (i * 4);
+    }
+
+    HWY_ASSERT(packed < (1ull << (N * 4)));
+    printf("0x%08x,", static_cast<uint32_t>(packed));
+  }
+  printf("\n");
+}
+
+void PrintCompress64x4Tables() {
+  printf("======================================= 64x4 uncompressed\n");
+  constexpr size_t N = 4;  // SVE_256
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    std::array<size_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    // Store uncompressed indices because SVE TBL returns 0 if an index is out
+    // of bounds. On AVX3 we simply variable-shift because permute indices are
+    // interpreted modulo N. Compression is not worth the extra shift+AND
+    // because the table is anyway only 512 bytes.
+    for (size_t i = 0; i < N; ++i) {
+      printf("%d,", static_cast<int>(indices[i]));
+    }
+  }
+  printf("\n");
+}
+
+void PrintCompressNot64x4Tables() {
+  printf("======================================= Not 64x4 uncompressed\n");
+  constexpr size_t N = 4;  // SVE_256
+  for (uint64_t not_code = 0; not_code < (1ull << N); ++not_code) {
+    const uint64_t code = ~not_code;
+    std::array<size_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    // Store uncompressed indices because SVE TBL returns 0 if an index is out
+    // of bounds. On AVX3 we simply variable-shift because permute indices are
+    // interpreted modulo N. Compression is not worth the extra shift+AND
+    // because the table is anyway only 512 bytes.
+    for (size_t i = 0; i < N; ++i) {
+      printf("%d,", static_cast<int>(indices[i]));
+    }
+  }
+  printf("\n");
+}
+
+// Same as above, but prints pairs of u32 indices (for AVX2). Also includes
+// FirstN bits in the nibble MSB.
+void PrintCompress64x4PairTables() {
+  printf("======================================= 64x4 u32 index\n");
+  constexpr size_t N = 4;  // AVX2
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    const size_t count = PopCount(code);
+    std::array<size_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    // Store uncompressed indices because SVE TBL returns 0 if an index is out
+    // of bounds. On AVX3 we simply variable-shift because permute indices are
+    // interpreted modulo N. Compression is not worth the extra shift+AND
+    // because the table is anyway only 512 bytes.
+    for (size_t i = 0; i < N; ++i) {
+      const int first_n_bit = i < count ? 8 : 0;
+      const int low = static_cast<int>(2 * indices[i]) + first_n_bit;
+      HWY_ASSERT(low < 0x10);
+      printf("%d, %d, ", low, low + 1);
+    }
+  }
+  printf("\n");
+}
+
+void PrintCompressNot64x4PairTables() {
+  printf("======================================= Not 64x4 u32 index\n");
+  constexpr size_t N = 4;  // AVX2
+  for (uint64_t not_code = 0; not_code < (1ull << N); ++not_code) {
+    const uint64_t code = ~not_code;
+    const size_t count = PopCount(code);
+    std::array<size_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    // Store uncompressed indices because SVE TBL returns 0 if an index is out
+    // of bounds. On AVX3 we simply variable-shift because permute indices are
+    // interpreted modulo N. Compression is not worth the extra shift+AND
+    // because the table is anyway only 512 bytes.
+    for (size_t i = 0; i < N; ++i) {
+      const int first_n_bit = i < count ? 8 : 0;
+      const int low = static_cast<int>(2 * indices[i]) + first_n_bit;
+      HWY_ASSERT(low < 0x10);
+      printf("%d, %d, ", low, low + 1);
+    }
+  }
+  printf("\n");
+}
+
+// 4-tuple of byte indices
+void PrintCompress32x4Tables() {
+  printf("======================================= 32x4\n");
+  using T = uint32_t;
+  constexpr size_t N = 4;  // SSE4
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    std::array<uint32_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    for (size_t i = 0; i < N; ++i) {
+      for (size_t idx_byte = 0; idx_byte < sizeof(T); ++idx_byte) {
+        printf("%d,", static_cast<int>(sizeof(T) * indices[i] + idx_byte));
+      }
+    }
+  }
+  printf("\n");
+}
+
+void PrintCompressNot32x4Tables() {
+  printf("======================================= Not 32x4\n");
+  using T = uint32_t;
+  constexpr size_t N = 4;  // SSE4
+  for (uint64_t not_code = 0; not_code < (1ull << N); ++not_code) {
+    const uint64_t code = ~not_code;
+    std::array<uint32_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    for (size_t i = 0; i < N; ++i) {
+      for (size_t idx_byte = 0; idx_byte < sizeof(T); ++idx_byte) {
+        printf("%d,", static_cast<int>(sizeof(T) * indices[i] + idx_byte));
+      }
+    }
+  }
+  printf("\n");
+}
+
+// 8-tuple of byte indices
+void PrintCompress64x2Tables() {
+  printf("======================================= 64x2\n");
+  using T = uint64_t;
+  constexpr size_t N = 2;  // SSE4
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    std::array<uint32_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    for (size_t i = 0; i < N; ++i) {
+      for (size_t idx_byte = 0; idx_byte < sizeof(T); ++idx_byte) {
+        printf("%d,", static_cast<int>(sizeof(T) * indices[i] + idx_byte));
+      }
+    }
+  }
+  printf("\n");
+}
+
+void PrintCompressNot64x2Tables() {
+  printf("======================================= Not 64x2\n");
+  using T = uint64_t;
+  constexpr size_t N = 2;  // SSE4
+  for (uint64_t not_code = 0; not_code < (1ull << N); ++not_code) {
+    const uint64_t code = ~not_code;
+    std::array<uint32_t, N> indices{0};
+    size_t pos = 0;
+    // All lanes where mask = true
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[pos++] = i;
+      }
+    }
+    // All lanes where mask = false
+    for (size_t i = 0; i < N; ++i) {
+      if (!(code & (1ull << i))) {
+        indices[pos++] = i;
+      }
+    }
+    HWY_ASSERT(pos == N);
+
+    for (size_t i = 0; i < N; ++i) {
+      for (size_t idx_byte = 0; idx_byte < sizeof(T); ++idx_byte) {
+        printf("%d,", static_cast<int>(sizeof(T) * indices[i] + idx_byte));
+      }
+    }
+  }
+  printf("\n");
+}
+
+HWY_NOINLINE void PrintTables() {
+  // Only print once.
+#if HWY_TARGET == HWY_STATIC_TARGET
+  PrintCompress32x8Tables();
+  PrintCompressNot32x8Tables();
+  PrintCompress64x4NibbleTables();
+  PrintCompressNot64x4NibbleTables();
+  PrintCompressNot64x2NibbleTables();
+  PrintCompress64x4Tables();
+  PrintCompressNot64x4Tables();
+  PrintCompress32x4Tables();
+  PrintCompressNot32x4Tables();
+  PrintCompress64x2Tables();
+  PrintCompressNot64x2Tables();
+  PrintCompress64x4PairTables();
+  PrintCompressNot64x4PairTables();
+  PrintCompress16x8Tables();
+  PrintCompress8x8Tables();
+  PrintCompressNot16x8Tables();
+#endif
+}
+
+#endif  // HWY_PRINT_TABLES
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyCompressTest);
+#if HWY_PRINT_TABLES
+// Only print instead of running tests; this will be visible in the log.
+HWY_EXPORT_AND_TEST_P(HwyCompressTest, PrintTables);
+#else
+HWY_EXPORT_AND_TEST_P(HwyCompressTest, TestAllCompress);
+HWY_EXPORT_AND_TEST_P(HwyCompressTest, TestAllCompressBlocks);
+#endif
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/concat_test.cc

@@ -0,0 +1,156 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#include "hwy/nanobenchmark.h"
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/concat_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestConcat {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    if (N == 1) return;
+    const size_t half_bytes = N * sizeof(T) / 2;
+
+    auto hi = AllocateAligned<T>(N);
+    auto lo = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(hi && lo && expected);
+    RandomState rng;
+    for (size_t rep = 0; rep < 10; ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        hi[i] = ConvertScalarTo<T>(Random64(&rng) & 0xFF);
+        lo[i] = ConvertScalarTo<T>(Random64(&rng) & 0xFF);
+      }
+
+      {
+        CopyBytes(&hi[N / 2], &expected[N / 2], half_bytes);
+        CopyBytes(&lo[0], &expected[0], half_bytes);
+        const Vec<D> vhi = Load(d, hi.get());
+        const Vec<D> vlo = Load(d, lo.get());
+        HWY_ASSERT_VEC_EQ(d, expected.get(), ConcatUpperLower(d, vhi, vlo));
+      }
+
+      {
+        CopyBytes(&hi[N / 2], &expected[N / 2], half_bytes);
+        CopyBytes(&lo[N / 2], &expected[0], half_bytes);
+        const Vec<D> vhi = Load(d, hi.get());
+        const Vec<D> vlo = Load(d, lo.get());
+        HWY_ASSERT_VEC_EQ(d, expected.get(), ConcatUpperUpper(d, vhi, vlo));
+      }
+
+      {
+        CopyBytes(&hi[0], &expected[N / 2], half_bytes);
+        CopyBytes(&lo[N / 2], &expected[0], half_bytes);
+        const Vec<D> vhi = Load(d, hi.get());
+        const Vec<D> vlo = Load(d, lo.get());
+        HWY_ASSERT_VEC_EQ(d, expected.get(), ConcatLowerUpper(d, vhi, vlo));
+      }
+
+      {
+        CopyBytes(&hi[0], &expected[N / 2], half_bytes);
+        CopyBytes(&lo[0], &expected[0], half_bytes);
+        const Vec<D> vhi = Load(d, hi.get());
+        const Vec<D> vlo = Load(d, lo.get());
+        HWY_ASSERT_VEC_EQ(d, expected.get(), ConcatLowerLower(d, vhi, vlo));
+      }
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllConcat() {
+  ForAllTypes(ForShrinkableVectors<TestConcat>());
+}
+
+struct TestConcatOddEven {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    const size_t N = Lanes(d);
+    const Vec<D> hi = Iota(d, hwy::Unpredictable1() + N - 1);  // N, N+1, ...
+    const Vec<D> lo = Iota(d, hwy::Unpredictable1() - 1);      // 0,1,2,3,...
+    const Vec<D> even = Add(lo, lo);
+    const Vec<D> odd = Add(even, Set(d, 1));
+    HWY_ASSERT_VEC_EQ(d, odd, ConcatOdd(d, hi, lo));
+    HWY_ASSERT_VEC_EQ(d, even, ConcatEven(d, hi, lo));
+
+    const Vec<D> v_1 = Set(d, ConvertScalarTo<T>(1));
+    const Vec<D> v_2 = Set(d, ConvertScalarTo<T>(2));
+    const Vec<D> v_3 = Set(d, ConvertScalarTo<T>(3));
+    const Vec<D> v_4 = Set(d, ConvertScalarTo<T>(4));
+
+    const Half<decltype(d)> dh;
+    const Vec<D> v_12 = InterleaveLower(v_1, v_2); /* {1, 2, 1, 2, ...} */
+    const Vec<D> v_34 = InterleaveLower(v_3, v_4); /* {3, 4, 3, 4, ...} */
+    const Vec<D> v_13 =
+        ConcatLowerLower(d, v_3, v_1); /* {1, 1, ..., 3, 3, ...} */
+    const Vec<D> v_24 =
+        ConcatLowerLower(d, v_4, v_2); /* {2, 2, ..., 4, 4, ...} */
+
+    const Vec<D> concat_even_1234_result = ConcatEven(d, v_34, v_12);
+    const Vec<D> concat_odd_1234_result = ConcatOdd(d, v_34, v_12);
+
+    HWY_ASSERT_VEC_EQ(d, v_13, concat_even_1234_result);
+    HWY_ASSERT_VEC_EQ(d, v_24, concat_odd_1234_result);
+    HWY_ASSERT_VEC_EQ(dh, LowerHalf(dh, v_3),
+                      UpperHalf(dh, concat_even_1234_result));
+    HWY_ASSERT_VEC_EQ(dh, LowerHalf(dh, v_4),
+                      UpperHalf(dh, concat_odd_1234_result));
+
+    // This test catches inadvertent saturation.
+    const Vec<D> min = Set(d, LowestValue<T>());
+    const Vec<D> max = Set(d, HighestValue<T>());
+    HWY_ASSERT_VEC_EQ(d, max, ConcatOdd(d, max, max));
+    HWY_ASSERT_VEC_EQ(d, max, ConcatEven(d, max, max));
+    HWY_ASSERT_VEC_EQ(d, min, ConcatOdd(d, min, min));
+    HWY_ASSERT_VEC_EQ(d, min, ConcatEven(d, min, min));
+#else
+    (void)d;
+#endif  // HWY_TARGET != HWY_SCALAR
+  }
+};
+
+HWY_NOINLINE void TestAllConcatOddEven() {
+  ForAllTypes(ForShrinkableVectors<TestConcatOddEven>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyConcatTest);
+HWY_EXPORT_AND_TEST_P(HwyConcatTest, TestAllConcat);
+HWY_EXPORT_AND_TEST_P(HwyConcatTest, TestAllConcatOddEven);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/count_test.cc

@@ -0,0 +1,312 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/count_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestPopulationCount {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+    size_t N = Lanes(d);
+    auto data = AllocateAligned<T>(N);
+    auto popcnt = AllocateAligned<T>(N);
+    HWY_ASSERT(data && popcnt);
+    for (size_t i = 0; i < AdjustedReps(1 << 18) / N; i++) {
+      for (size_t j = 0; j < N; j++) {
+        data[j] = static_cast<T>(rng());
+        popcnt[j] = static_cast<T>(PopCount(data[j]));
+      }
+      HWY_ASSERT_VEC_EQ(d, popcnt.get(), PopulationCount(Load(d, data.get())));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllPopulationCount() {
+  ForUnsignedTypes(ForPartialVectors<TestPopulationCount>());
+}
+
+template <class T, HWY_IF_NOT_FLOAT_NOR_SPECIAL(T), HWY_IF_T_SIZE(T, 4)>
+static HWY_INLINE T LeadingZeroCountOfValue(T val) {
+  const uint32_t u32_val = static_cast<uint32_t>(val);
+  return static_cast<T>(u32_val ? Num0BitsAboveMS1Bit_Nonzero32(u32_val) : 32);
+}
+template <class T, HWY_IF_NOT_FLOAT_NOR_SPECIAL(T), HWY_IF_T_SIZE(T, 8)>
+static HWY_INLINE T LeadingZeroCountOfValue(T val) {
+  const uint64_t u64_val = static_cast<uint64_t>(val);
+  return static_cast<T>(u64_val ? Num0BitsAboveMS1Bit_Nonzero64(u64_val) : 64);
+}
+template <class T, HWY_IF_NOT_FLOAT_NOR_SPECIAL(T),
+          HWY_IF_T_SIZE_ONE_OF(T, (1 << 1) | (1 << 2))>
+static HWY_INLINE T LeadingZeroCountOfValue(T val) {
+  using TU = MakeUnsigned<T>;
+  constexpr uint32_t kNumOfExtraLeadingZeros{32 - (sizeof(T) * 8)};
+  return static_cast<T>(
+      LeadingZeroCountOfValue(static_cast<uint32_t>(static_cast<TU>(val))) -
+      kNumOfExtraLeadingZeros);
+}
+
+struct TestLeadingZeroCount {
+  template <class T, class D>
+  HWY_ATTR_NO_MSAN HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+    using TU = MakeUnsigned<T>;
+    const RebindToUnsigned<decltype(d)> du;
+    size_t N = Lanes(d);
+    auto data = AllocateAligned<T>(N);
+    auto lzcnt = AllocateAligned<T>(N);
+    HWY_ASSERT(data && lzcnt);
+
+    constexpr T kNumOfBitsInT = static_cast<T>(sizeof(T) * 8);
+    for (size_t j = 0; j < N; j++) {
+      lzcnt[j] = kNumOfBitsInT;
+    }
+    HWY_ASSERT_VEC_EQ(d, lzcnt.get(), LeadingZeroCount(Zero(d)));
+
+    for (size_t j = 0; j < N; j++) {
+      lzcnt[j] = static_cast<T>(kNumOfBitsInT - 1);
+    }
+    HWY_ASSERT_VEC_EQ(d, lzcnt.get(),
+                      LeadingZeroCount(Set(d, static_cast<T>(1))));
+
+    for (size_t j = 0; j < N; j++) {
+      lzcnt[j] = static_cast<T>(kNumOfBitsInT - 2);
+    }
+    HWY_ASSERT_VEC_EQ(d, lzcnt.get(),
+                      LeadingZeroCount(Set(d, static_cast<T>(2))));
+
+    for (size_t j = 0; j < N; j++) {
+      lzcnt[j] = static_cast<T>(0);
+    }
+    HWY_ASSERT_VEC_EQ(
+        d, lzcnt.get(),
+        LeadingZeroCount(BitCast(d, Set(du, TU{1} << (kNumOfBitsInT - 1)))));
+
+    for (size_t j = 0; j < N; j++) {
+      lzcnt[j] = static_cast<T>(1);
+    }
+    HWY_ASSERT_VEC_EQ(
+        d, lzcnt.get(),
+        LeadingZeroCount(Set(d, static_cast<T>(1) << (kNumOfBitsInT - 2))));
+
+    for (size_t j = 0; j < N; j++) {
+      lzcnt[j] = static_cast<T>(kNumOfBitsInT - 5);
+    }
+    HWY_ASSERT_VEC_EQ(d, lzcnt.get(),
+                      LeadingZeroCount(Set(d, static_cast<T>(0x1D))));
+
+    for (size_t i = 0; i < AdjustedReps(1000); i++) {
+      for (size_t j = 0; j < N; j++) {
+        data[j] = static_cast<T>(rng());
+        lzcnt[j] = LeadingZeroCountOfValue(data[j]);
+      }
+      HWY_ASSERT_VEC_EQ(d, lzcnt.get(), LeadingZeroCount(Load(d, data.get())));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllLeadingZeroCount() {
+  ForIntegerTypes(ForPartialVectors<TestLeadingZeroCount>());
+}
+
+template <class T, HWY_IF_NOT_FLOAT_NOR_SPECIAL(T),
+          HWY_IF_T_SIZE_ONE_OF(T, (1 << 1) | (1 << 2) | (1 << 4))>
+static HWY_INLINE T TrailingZeroCountOfValue(T val) {
+  using TU = MakeUnsigned<T>;
+  constexpr size_t kNumOfBitsInT = sizeof(T) * 8;
+  const uint32_t u32_val = static_cast<uint32_t>(static_cast<TU>(val));
+  return static_cast<T>(u32_val ? Num0BitsBelowLS1Bit_Nonzero32(u32_val)
+                                : kNumOfBitsInT);
+}
+template <class T, HWY_IF_NOT_FLOAT_NOR_SPECIAL(T), HWY_IF_T_SIZE(T, 8)>
+static HWY_INLINE T TrailingZeroCountOfValue(T val) {
+  const uint64_t u64_val = static_cast<uint64_t>(val);
+  return static_cast<T>(u64_val ? Num0BitsBelowLS1Bit_Nonzero64(u64_val) : 64);
+}
+
+struct TestTrailingZeroCount {
+  template <class T, class D>
+  HWY_ATTR_NO_MSAN HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+    using TU = MakeUnsigned<T>;
+    const RebindToUnsigned<decltype(d)> du;
+
+    size_t N = Lanes(d);
+    auto data = AllocateAligned<T>(N);
+    auto tzcnt = AllocateAligned<T>(N);
+    HWY_ASSERT(data && tzcnt);
+
+    constexpr T kNumOfBitsInT = static_cast<T>(sizeof(T) * 8);
+    for (size_t j = 0; j < N; j++) {
+      tzcnt[j] = kNumOfBitsInT;
+    }
+    HWY_ASSERT_VEC_EQ(d, tzcnt.get(), TrailingZeroCount(Zero(d)));
+
+    for (size_t j = 0; j < N; j++) {
+      tzcnt[j] = static_cast<T>(0);
+    }
+    HWY_ASSERT_VEC_EQ(d, tzcnt.get(),
+                      TrailingZeroCount(Set(d, static_cast<T>(1))));
+
+    for (size_t j = 0; j < N; j++) {
+      tzcnt[j] = static_cast<T>(1);
+    }
+    HWY_ASSERT_VEC_EQ(d, tzcnt.get(),
+                      TrailingZeroCount(Set(d, static_cast<T>(2))));
+
+    for (size_t j = 0; j < N; j++) {
+      tzcnt[j] = static_cast<T>(kNumOfBitsInT - 1);
+    }
+    HWY_ASSERT_VEC_EQ(
+        d, tzcnt.get(),
+        TrailingZeroCount(BitCast(d, Set(du, TU{1} << (kNumOfBitsInT - 1)))));
+
+    for (size_t j = 0; j < N; j++) {
+      tzcnt[j] = static_cast<T>(kNumOfBitsInT - 2);
+    }
+    HWY_ASSERT_VEC_EQ(
+        d, tzcnt.get(),
+        TrailingZeroCount(Set(d, static_cast<T>(1) << (kNumOfBitsInT - 2))));
+
+    for (size_t j = 0; j < N; j++) {
+      tzcnt[j] = static_cast<T>(3);
+    }
+    HWY_ASSERT_VEC_EQ(d, tzcnt.get(),
+                      TrailingZeroCount(Set(d, static_cast<T>(0x68))));
+
+    for (size_t i = 0; i < AdjustedReps(1000); i++) {
+      for (size_t j = 0; j < N; j++) {
+        data[j] = static_cast<T>(rng());
+        tzcnt[j] = TrailingZeroCountOfValue(data[j]);
+      }
+      HWY_ASSERT_VEC_EQ(d, tzcnt.get(), TrailingZeroCount(Load(d, data.get())));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllTrailingZeroCount() {
+  ForIntegerTypes(ForPartialVectors<TestTrailingZeroCount>());
+}
+
+class TestHighestSetBitIndex {
+ private:
+  template <class V>
+  static HWY_INLINE V NormalizedHighestSetBitIndex(V v) {
+    const DFromV<decltype(v)> d;
+    const RebindToSigned<decltype(d)> di;
+    const auto hsb_idx = BitCast(di, HighestSetBitIndex(v));
+    return BitCast(d, Or(BroadcastSignBit(hsb_idx), hsb_idx));
+  }
+
+ public:
+  template <class T, class D>
+  HWY_ATTR_NO_MSAN HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+    using TU = MakeUnsigned<T>;
+    const RebindToUnsigned<decltype(d)> du;
+
+    size_t N = Lanes(d);
+    auto data = AllocateAligned<T>(N);
+    auto hsb_index = AllocateAligned<T>(N);
+    HWY_ASSERT(data && hsb_index);
+
+    constexpr T kNumOfBitsInT = static_cast<T>(sizeof(T) * 8);
+    constexpr T kMsbIdx = static_cast<T>(kNumOfBitsInT - 1);
+
+    for (size_t j = 0; j < N; j++) {
+      hsb_index[j] = static_cast<T>(-1);
+    }
+    HWY_ASSERT_VEC_EQ(d, hsb_index.get(),
+                      NormalizedHighestSetBitIndex(Zero(d)));
+
+    for (size_t j = 0; j < N; j++) {
+      hsb_index[j] = static_cast<T>(0);
+    }
+    HWY_ASSERT_VEC_EQ(d, hsb_index.get(),
+                      NormalizedHighestSetBitIndex(Set(d, static_cast<T>(1))));
+
+    for (size_t j = 0; j < N; j++) {
+      hsb_index[j] = static_cast<T>(1);
+    }
+    HWY_ASSERT_VEC_EQ(d, hsb_index.get(),
+                      NormalizedHighestSetBitIndex(Set(d, static_cast<T>(3))));
+
+    for (size_t j = 0; j < N; j++) {
+      hsb_index[j] = static_cast<T>(kNumOfBitsInT - 1);
+    }
+    HWY_ASSERT_VEC_EQ(d, hsb_index.get(),
+                      NormalizedHighestSetBitIndex(
+                          BitCast(d, Set(du, TU{1} << (kNumOfBitsInT - 1)))));
+
+    for (size_t j = 0; j < N; j++) {
+      hsb_index[j] = static_cast<T>(kNumOfBitsInT - 2);
+    }
+    HWY_ASSERT_VEC_EQ(d, hsb_index.get(),
+                      NormalizedHighestSetBitIndex(
+                          Set(d, static_cast<T>(1) << (kNumOfBitsInT - 2))));
+
+    for (size_t j = 0; j < N; j++) {
+      hsb_index[j] = static_cast<T>(5);
+    }
+    HWY_ASSERT_VEC_EQ(
+        d, hsb_index.get(),
+        NormalizedHighestSetBitIndex(Set(d, static_cast<T>(0x2B))));
+
+    for (size_t i = 0; i < AdjustedReps(1000); i++) {
+      for (size_t j = 0; j < N; j++) {
+        data[j] = static_cast<T>(rng());
+        hsb_index[j] =
+            static_cast<T>(kMsbIdx - LeadingZeroCountOfValue(data[j]));
+      }
+      HWY_ASSERT_VEC_EQ(d, hsb_index.get(),
+                        NormalizedHighestSetBitIndex(Load(d, data.get())));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllHighestSetBitIndex() {
+  ForIntegerTypes(ForPartialVectors<TestHighestSetBitIndex>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyCountTest);
+HWY_EXPORT_AND_TEST_P(HwyCountTest, TestAllPopulationCount);
+HWY_EXPORT_AND_TEST_P(HwyCountTest, TestAllLeadingZeroCount);
+HWY_EXPORT_AND_TEST_P(HwyCountTest, TestAllTrailingZeroCount);
+HWY_EXPORT_AND_TEST_P(HwyCountTest, TestAllHighestSetBitIndex);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/crypto_test.cc

@@ -0,0 +1,723 @@
+// Copyright 2021 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <string.h>  // memcpy
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/crypto_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+#define HWY_PRINT_CLMUL_GOLDEN 0
+
+#if HWY_TARGET != HWY_SCALAR
+
+class TestAES {
+  template <typename T, class D>
+  HWY_NOINLINE void TestSBox(T /*unused*/, D d) {
+    // The generic implementation of the S-box is difficult to verify by
+    // inspection, so we add a white-box test that verifies it using enumeration
+    // (outputs for 0..255 vs. https://en.wikipedia.org/wiki/Rijndael_S-box).
+    const uint8_t sbox[256] = {
+        0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b,
+        0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
+        0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26,
+        0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
+        0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2,
+        0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0,
+        0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed,
+        0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
+        0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f,
+        0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5,
+        0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec,
+        0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
+        0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14,
+        0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c,
+        0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d,
+        0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
+        0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f,
+        0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e,
+        0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11,
+        0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
+        0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f,
+        0xb0, 0x54, 0xbb, 0x16};
+
+    // Ensure it's safe to load an entire vector by padding.
+    const size_t N = Lanes(d);
+    const size_t padded = RoundUpTo(256, N);
+    auto expected = AllocateAligned<T>(padded);
+    HWY_ASSERT(expected);
+    // Must wrap around to match the input (Iota).
+    for (size_t pos = 0; pos < padded;) {
+      const size_t remaining = HWY_MIN(padded - pos, size_t(256));
+      memcpy(expected.get() + pos, sbox, remaining);
+      pos += remaining;
+    }
+
+    for (size_t i = 0; i < 256; i += N) {
+      const auto in = Iota(d, i);
+      HWY_ASSERT_VEC_EQ(d, expected.get() + i, detail::SubBytes(in));
+    }
+  }
+
+ public:
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T t, D d) {
+    // Test vector (after first KeyAddition) from
+    // https://csrc.nist.gov/CSRC/media/Projects/Cryptographic-Standards-and-Guidelines/documents/examples/AES_Core128.pdf
+    alignas(16) static constexpr uint8_t test_lanes[16] = {
+        0x40, 0xBF, 0xAB, 0xF4, 0x06, 0xEE, 0x4D, 0x30,
+        0x42, 0xCA, 0x6B, 0x99, 0x7A, 0x5C, 0x58, 0x16};
+    const auto test = LoadDup128(d, test_lanes);
+
+    // = ShiftRow result
+    alignas(16) static constexpr uint8_t expected_sr_lanes[16] = {
+        0x09, 0x28, 0x7F, 0x47, 0x6F, 0x74, 0x6A, 0xBF,
+        0x2C, 0x4A, 0x62, 0x04, 0xDA, 0x08, 0xE3, 0xEE};
+    const auto expected_sr = LoadDup128(d, expected_sr_lanes);
+
+    // = MixColumn result
+    alignas(16) static constexpr uint8_t expected_mc_lanes[16] = {
+        0x52, 0x9F, 0x16, 0xC2, 0x97, 0x86, 0x15, 0xCA,
+        0xE0, 0x1A, 0xAE, 0x54, 0xBA, 0x1A, 0x26, 0x59};
+    const auto expected_mc = LoadDup128(d, expected_mc_lanes);
+
+    // = KeyAddition result
+    alignas(16) static constexpr uint8_t expected_lanes[16] = {
+        0xF2, 0x65, 0xE8, 0xD5, 0x1F, 0xD2, 0x39, 0x7B,
+        0xC3, 0xB9, 0x97, 0x6D, 0x90, 0x76, 0x50, 0x5C};
+    const auto expected = LoadDup128(d, expected_lanes);
+
+    alignas(16) uint8_t key_lanes[16];
+    for (size_t i = 0; i < 16; ++i) {
+      key_lanes[i] = expected_mc_lanes[i] ^ expected_lanes[i];
+    }
+    const auto round_key = LoadDup128(d, key_lanes);
+
+    HWY_ASSERT_VEC_EQ(d, expected_mc, AESRound(test, Zero(d)));
+    HWY_ASSERT_VEC_EQ(d, expected, AESRound(test, round_key));
+    HWY_ASSERT_VEC_EQ(d, expected_sr, AESLastRound(test, Zero(d)));
+    HWY_ASSERT_VEC_EQ(d, Xor(expected_sr, round_key),
+                      AESLastRound(test, round_key));
+
+    TestSBox(t, d);
+  }
+};
+HWY_NOINLINE void TestAllAES() { ForGEVectors<128, TestAES>()(uint8_t()); }
+
+class TestAESInverse {
+  template <typename T, class D>
+  HWY_NOINLINE void TestInverseSBox(T /*unused*/, D d) {
+    // The generic implementation of the inverse S-box is difficult to verify by
+    // inspection, so we add a white-box test that verifies it using enumeration
+    // (outputs for 0..255 vs. https://en.wikipedia.org/wiki/Rijndael_S-box).
+    const uint8_t inv_sbox[256] = {
+        0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e,
+        0x81, 0xf3, 0xd7, 0xfb, 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87,
+        0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 0x54, 0x7b, 0x94, 0x32,
+        0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
+        0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49,
+        0x6d, 0x8b, 0xd1, 0x25, 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16,
+        0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 0x6c, 0x70, 0x48, 0x50,
+        0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
+        0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05,
+        0xb8, 0xb3, 0x45, 0x06, 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02,
+        0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 0x3a, 0x91, 0x11, 0x41,
+        0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
+        0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8,
+        0x1c, 0x75, 0xdf, 0x6e, 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89,
+        0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 0xfc, 0x56, 0x3e, 0x4b,
+        0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
+        0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59,
+        0x27, 0x80, 0xec, 0x5f, 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d,
+        0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 0xa0, 0xe0, 0x3b, 0x4d,
+        0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
+        0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63,
+        0x55, 0x21, 0x0c, 0x7d};
+
+    // Ensure it's safe to load an entire vector by padding.
+    const size_t N = Lanes(d);
+    const size_t padded = RoundUpTo(256, N);
+    auto expected = AllocateAligned<T>(padded);
+    HWY_ASSERT(expected);
+    // Must wrap around to match the input (Iota).
+    for (size_t pos = 0; pos < padded;) {
+      const size_t remaining = HWY_MIN(padded - pos, size_t(256));
+      memcpy(expected.get() + pos, inv_sbox, remaining);
+      pos += remaining;
+    }
+
+    for (size_t i = 0; i < 256; i += N) {
+      const auto in = Iota(d, i);
+      HWY_ASSERT_VEC_EQ(d, expected.get() + i, detail::InvSubBytes(in));
+    }
+  }
+
+  template <typename T, class D>
+  HWY_INLINE void TestAESRoundInv(T /*unused*/, D d) {
+    // Test vector (after first KeyAddition) from page 37 of
+    // https://nvlpubs.nist.gov/nistpubs/fips/nist.fips.197.pdf
+    alignas(16) static constexpr uint8_t test_lanes[16] = {
+        0x7A, 0xD5, 0xFD, 0xA7, 0x89, 0xEF, 0x4E, 0x27,
+        0x2B, 0xCA, 0x10, 0x0B, 0x3D, 0x9F, 0xF5, 0x9F};
+    const auto test = LoadDup128(d, test_lanes);
+
+    // = InvShiftRow result
+    alignas(16) static constexpr uint8_t expected_isr_lanes[16] = {
+        0xBD, 0x6E, 0x7C, 0x3D, 0xF2, 0xB5, 0x77, 0x9E,
+        0x0B, 0x61, 0x21, 0x6E, 0x8B, 0x10, 0xB6, 0x89};
+    const auto expected_isr = LoadDup128(d, expected_isr_lanes);
+
+    // = InvMixColumn result
+    alignas(16) static constexpr uint8_t expected_imc_lanes[16] = {
+        0x47, 0x73, 0xB9, 0x1F, 0xF7, 0x2F, 0x35, 0x43,
+        0x61, 0xCB, 0x01, 0x8E, 0xA1, 0xE6, 0xCF, 0x2C};
+    const auto expected_imc = LoadDup128(d, expected_imc_lanes);
+
+    // = KeyAddition result
+    alignas(16) static constexpr uint8_t expected_lanes[16] = {
+        0x13, 0xAA, 0x29, 0xBE, 0x9C, 0x8F, 0xAF, 0xF6,
+        0xF7, 0x70, 0xF5, 0x80, 0x00, 0xF7, 0xBF, 0x03};
+    const auto expected = LoadDup128(d, expected_lanes);
+
+    alignas(16) uint8_t key_lanes[16];
+    for (size_t i = 0; i < 16; ++i) {
+      key_lanes[i] = expected_imc_lanes[i] ^ expected_lanes[i];
+    }
+    const auto round_key = LoadDup128(d, key_lanes);
+
+    HWY_ASSERT_VEC_EQ(d, expected_isr, AESLastRoundInv(test, Zero(d)));
+    HWY_ASSERT_VEC_EQ(d, expected_imc, AESRoundInv(test, Zero(d)));
+    HWY_ASSERT_VEC_EQ(d, expected_imc,
+                      AESInvMixColumns(AESLastRoundInv(test, Zero(d))));
+    HWY_ASSERT_VEC_EQ(d, expected, AESRoundInv(test, round_key));
+  }
+
+  template <typename T, class D>
+  HWY_INLINE void TestAESLastRoundInv(T /*unused*/, D d) {
+    // Test vector (after the KeyAddition operation of round 9) from page 38 of
+    // https://nvlpubs.nist.gov/nistpubs/fips/nist.fips.197.pdf
+    alignas(16) static constexpr uint8_t test_lanes[16] = {
+        0x63, 0x53, 0xE0, 0x8C, 0x09, 0x60, 0xE1, 0x04,
+        0xCD, 0x70, 0xB7, 0x51, 0xBA, 0xCA, 0xD0, 0xE7};
+    const auto test = LoadDup128(d, test_lanes);
+
+    // = InvShiftRow result
+    alignas(16) static constexpr uint8_t expected_isr_lanes[16] = {
+        0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70,
+        0x80, 0x90, 0xA0, 0xB0, 0xC0, 0xD0, 0xE0, 0xF0};
+    const auto expected_isr = LoadDup128(d, expected_isr_lanes);
+
+    // = KeyAddition result
+    alignas(16) static constexpr uint8_t expected_lanes[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF};
+    const auto expected = LoadDup128(d, expected_lanes);
+
+    alignas(16) uint8_t key_lanes[16];
+    for (size_t i = 0; i < 16; ++i) {
+      key_lanes[i] = expected_isr_lanes[i] ^ expected_lanes[i];
+    }
+    const auto round_key = LoadDup128(d, key_lanes);
+
+    HWY_ASSERT_VEC_EQ(d, expected_isr, AESLastRoundInv(test, Zero(d)));
+    HWY_ASSERT_VEC_EQ(d, expected, AESLastRoundInv(test, round_key));
+  }
+
+ public:
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T t, D d) {
+    TestAESRoundInv(t, d);
+    TestAESLastRoundInv(t, d);
+    TestInverseSBox(t, d);
+  }
+};
+HWY_NOINLINE void TestAllAESInverse() {
+  ForGEVectors<128, TestAESInverse>()(uint8_t());
+}
+
+struct TestAESKeyGenAssist {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*t*/, D d) {
+    alignas(16) static constexpr uint8_t kTestVect1[16] = {
+        0x27, 0xCF, 0x73, 0xC3, 0x27, 0xCF, 0x73, 0xC3,
+        0x74, 0x01, 0x90, 0x5A, 0x74, 0x01, 0x90, 0x5A};
+    alignas(16) static constexpr uint8_t kExpectedResult1[16] = {
+        0xCC, 0x8A, 0x8F, 0x2E, 0xAA, 0x8F, 0x2E, 0xCC,
+        0x92, 0x7C, 0x60, 0xBE, 0x5C, 0x60, 0xBE, 0x92};
+
+    const auto expected_1 = LoadDup128(d, kExpectedResult1);
+    const auto actual_1 = AESKeyGenAssist<0x20>(LoadDup128(d, kTestVect1));
+    HWY_ASSERT_VEC_EQ(d, expected_1, actual_1);
+
+    alignas(16) static constexpr uint8_t kTestVect2[16] = {
+        0xD0, 0x14, 0xF9, 0xA8, 0x57, 0x5C, 0x00, 0x6E,
+        0xE1, 0x3F, 0x0C, 0xC8, 0xC9, 0xEE, 0x25, 0x89};
+    alignas(16) static constexpr uint8_t kExpectedResult2[16] = {
+        0x5B, 0x4A, 0x63, 0x9F, 0x7C, 0x63, 0x9F, 0x5B,
+        0xDD, 0x28, 0x3F, 0xA7, 0x1E, 0x3F, 0xA7, 0xDD};
+
+    const auto expected_2 = LoadDup128(d, kExpectedResult2);
+    const auto actual_2 = AESKeyGenAssist<0x36>(LoadDup128(d, kTestVect2));
+    HWY_ASSERT_VEC_EQ(d, expected_2, actual_2);
+  }
+};
+
+HWY_NOINLINE void TestAllAESKeyGenAssist() {
+  ForGEVectors<128, TestAESKeyGenAssist>()(uint8_t());
+}
+
+#else
+HWY_NOINLINE void TestAllAES() {}
+HWY_NOINLINE void TestAllAESInverse() {}
+HWY_NOINLINE void TestAllAESKeyGenAssist() {}
+#endif  // HWY_TARGET != HWY_SCALAR
+
+struct TestCLMul {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    // needs 64 bit lanes and 128-bit result
+#if HWY_TARGET != HWY_SCALAR && HWY_HAVE_INTEGER64
+    const size_t N = Lanes(d);
+    if (N == 1) return;
+
+    auto in1 = AllocateAligned<T>(N);
+    auto in2 = AllocateAligned<T>(N);
+    HWY_ASSERT(in1 && in2);
+
+    constexpr size_t kCLMulNum = 512;
+    // Depends on rng!
+    static constexpr uint64_t kCLMulLower[kCLMulNum] = {
+        0x24511d4ce34d6350ULL, 0x4ca582edde1236bbULL, 0x537e58f72dac25a8ULL,
+        0x4e942d5e130b9225ULL, 0x75a906c519257a68ULL, 0x1df9f85126d96c5eULL,
+        0x464e7c13f4ad286aULL, 0x138535ee35dabc40ULL, 0xb2f7477b892664ecULL,
+        0x01557b077167c25dULL, 0xf32682490ee49624ULL, 0x0025bac603b9e140ULL,
+        0xcaa86aca3e3daf40ULL, 0x1fbcfe4af73eb6c4ULL, 0x8ee8064dd0aae5dcULL,
+        0x1248cb547858c213ULL, 0x37a55ee5b10fb34cULL, 0x6eb5c97b958f86e2ULL,
+        0x4b1ab3eb655ea7cdULL, 0x1d66645a85627520ULL, 0xf8728e96daa36748ULL,
+        0x38621043e6ff5e3bULL, 0xd1d28b5da5ffefb4ULL, 0x0a5cd65931546df7ULL,
+        0x2a0639be3d844150ULL, 0x0e2d0f18c8d6f045ULL, 0xfacc770b963326c1ULL,
+        0x19611b31ca2ef141ULL, 0xabea29510dd87518ULL, 0x18a7dc4b205f2768ULL,
+        0x9d3975ea5612dc86ULL, 0x06319c139e374773ULL, 0x6641710400b4c390ULL,
+        0x356c29b6001c3670ULL, 0xe9e04d851e040a00ULL, 0x21febe561222d79aULL,
+        0xc071eaae6e148090ULL, 0x0eed351a0af94f5bULL, 0x04324eedb3c03688ULL,
+        0x39e89b136e0d6ccdULL, 0x07d0fd2777a31600ULL, 0x44b8573827209822ULL,
+        0x6d690229ea177d78ULL, 0x1b9749d960ba9f18ULL, 0x190945271c0fbb94ULL,
+        0x189aea0e07d2c88eULL, 0xf18eab6b65a6beb2ULL, 0x57744b21c13d0d84ULL,
+        0xf63050a613e95c2eULL, 0x12cd20d25f97102fULL, 0x5a5df0678dbcba60ULL,
+        0x0b08fb80948bfafcULL, 0x44cf1cbe7c6fc3c8ULL, 0x166a470ef25da288ULL,
+        0x2c498a609204e48cULL, 0x261b0a22585697ecULL, 0x737750574af7dde4ULL,
+        0x4079959c60b01e0cULL, 0x06ed8aac13f782d6ULL, 0x019d454ba9b5ef20ULL,
+        0xea1edbf96d49e858ULL, 0x17c2f3ebde9ac469ULL, 0x5cf72706e3d6f5e4ULL,
+        0x16e856aa3c841516ULL, 0x256f7e3cef83368eULL, 0x47e17c8eb2774e77ULL,
+        0x9b48ac150a804821ULL, 0x584523f61ccfdf22ULL, 0xedcb6a2a75d9e7f2ULL,
+        0x1fe3d1838e537aa7ULL, 0x778872e9f64549caULL, 0x2f1cea6f0d3faf92ULL,
+        0x0e8c4b6a9343f326ULL, 0x01902d1ba3048954ULL, 0xc5c1fd5269e91dc0ULL,
+        0x0ef8a4707817eb9cULL, 0x1f696f09a5354ca4ULL, 0x369cd9de808b818cULL,
+        0xf6917d1dd43fd784ULL, 0x7f4b76bf40dc166fULL, 0x4ce67698724ace12ULL,
+        0x02c3bf60e6e9cd92ULL, 0xb8229e45b21458e8ULL, 0x415efd41e91adf49ULL,
+        0x5edfcd516bb921cdULL, 0x5ff2c29429fd187eULL, 0x0af666b17103b3e0ULL,
+        0x1f5e4ff8f54c9a5bULL, 0x429253d8a5544ba6ULL, 0x19de2fdf9f4d9dcaULL,
+        0x29bf3d37ddc19a40ULL, 0x04d4513a879552baULL, 0x5cc7476cf71ee155ULL,
+        0x40011f8c238784a5ULL, 0x1a3ae50b0fd2ee2bULL, 0x7db22f432ba462baULL,
+        0x417290b0bee2284aULL, 0x055a6bd5bb853db2ULL, 0xaa667daeed8c2a34ULL,
+        0x0d6b316bda7f3577ULL, 0x72d35598468e3d5dULL, 0x375b594804bfd33aULL,
+        0x16ed3a319b540ae8ULL, 0x093bace4b4695afdULL, 0xc7118754ec2737ceULL,
+        0x0fff361f0505c81aULL, 0x996e9e7291321af0ULL, 0x496b1d9b0b89ba8cULL,
+        0x65a98b2e9181da9cULL, 0x70759c8dd45575dfULL, 0x3446fe727f5e2cbbULL,
+        0x1121ae609d195e74ULL, 0x5ff5d68ce8a21018ULL, 0x0e27eca3825b60d6ULL,
+        0x82f628bceca3d1daULL, 0x2756a0914e344047ULL, 0xa460406c1c708d50ULL,
+        0x63ce32a0c083e491ULL, 0xc883e5a685c480e0ULL, 0x602c951891e600f9ULL,
+        0x02ecb2e3911ca5f8ULL, 0x0d8675f4bb70781aULL, 0x43545cc3c78ea496ULL,
+        0x04164b01d6b011c2ULL, 0x3acbb323dcab2c9bULL, 0x31c5ba4e22793082ULL,
+        0x5a6484af5f7c2d10ULL, 0x1a929b16194e8078ULL, 0x7a6a75d03b313924ULL,
+        0x0553c73a35b1d525ULL, 0xf18628c51142be34ULL, 0x1b51cf80d7efd8f5ULL,
+        0x52e0ca4df63ee258ULL, 0x0e977099160650c9ULL, 0x6be1524e92024f70ULL,
+        0x0ee2152625438b9dULL, 0xfa32af436f6d8eb4ULL, 0x5ecf49c2154287e5ULL,
+        0x6b72f4ae3590569dULL, 0x086c5ee6e87bfb68ULL, 0x737a4f0dc04b6187ULL,
+        0x08c3439280edea41ULL, 0x9547944f01636c5cULL, 0x6acfbfc2571cd71fULL,
+        0x85d7842972449637ULL, 0x252ea5e5a7fad86aULL, 0x4e41468f99ba1632ULL,
+        0x095e0c3ae63b25a2ULL, 0xb005ce88fd1c9425ULL, 0x748e668abbe09f03ULL,
+        0xb2cfdf466b187d18ULL, 0x60b11e633d8fe845ULL, 0x07144c4d246db604ULL,
+        0x139bcaac55e96125ULL, 0x118679b5a6176327ULL, 0x1cebe90fa4d9f83fULL,
+        0x22244f52f0d312acULL, 0x669d4e17c9bfb713ULL, 0x96390e0b834bb0d0ULL,
+        0x01f7f0e82ba08071ULL, 0x2dffeee31ca6d284ULL, 0x1f4738745ef039feULL,
+        0x4ce0dd2b603b6420ULL, 0x0035fc905910a4d5ULL, 0x07df2b533df6fb04ULL,
+        0x1cee2735c9b910ddULL, 0x2bc4af565f7809eaULL, 0x2f876c1f5cb1076cULL,
+        0x33e079524099d056ULL, 0x169e0405d2f9efbaULL, 0x018643ab548a358cULL,
+        0x1bb6fc4331cffe92ULL, 0x05111d3a04e92faaULL, 0x23c27ecf0d638b73ULL,
+        0x1b79071dc1685d68ULL, 0x0662d20aba8e1e0cULL, 0xe7f6440277144c6fULL,
+        0x4ca38b64c22196c0ULL, 0x43c05f6d1936fbeeULL, 0x0654199d4d1faf0fULL,
+        0xf2014054e71c2d04ULL, 0x0a103e47e96b4c84ULL, 0x7986e691dd35b040ULL,
+        0x4e1ebb53c306a341ULL, 0x2775bb3d75d65ba6ULL, 0x0562ab0adeff0f15ULL,
+        0x3c2746ad5eba3eacULL, 0x1facdb5765680c60ULL, 0xb802a60027d81d00ULL,
+        0x1191d0f6366ae3a9ULL, 0x81a97b5ae0ea5d14ULL, 0x06bee05b6178a770ULL,
+        0xc7baeb2fe1d6aeb3ULL, 0x594cb5b867d04fdfULL, 0xf515a80138a4e350ULL,
+        0x646417ad8073cf38ULL, 0x4a229a43373fb8d4ULL, 0x10fa6eafff1ca453ULL,
+        0x9f060700895cc731ULL, 0x00521133d11d11f4ULL, 0xb940a2bb912a7a5cULL,
+        0x3fab180670ad2a3cULL, 0x45a5f0e5b6fdb95dULL, 0x27c1baad6f946b15ULL,
+        0x336c6bdbe527cf58ULL, 0x3b83aa602a5baea3ULL, 0xdf749153f9bcc376ULL,
+        0x1a05513a6c0b4a90ULL, 0xb81e0b570a075c47ULL, 0x471fabb40bdc27ceULL,
+        0x9dec9472f6853f60ULL, 0x361f71b88114193bULL, 0x3b550a8c4feeff00ULL,
+        0x0f6cde5a68bc9bc0ULL, 0x3f50121a925703e0ULL, 0x6967ff66d6d343a9ULL,
+        0xff6b5bd2ce7bc3ccULL, 0x05474cea08bf6cd8ULL, 0xf76eabbfaf108eb0ULL,
+        0x067529be4fc6d981ULL, 0x4d766b137cf8a988ULL, 0x2f09c7395c5cfbbdULL,
+        0x388793712da06228ULL, 0x02c9ff342c8f339aULL, 0x152c734139a860a3ULL,
+        0x35776eb2b270c04dULL, 0x0f8d8b41f11c4608ULL, 0x0c2071665be6b288ULL,
+        0xc034e212b3f71d88ULL, 0x071d961ef3276f99ULL, 0xf98598ee75b60773ULL,
+        0x062062c58c6724e4ULL, 0xd156438e2125572cULL, 0x38552d59a7f0f7c8ULL,
+        0x1a402178206e413cULL, 0x1f1f996c68293b26ULL, 0x8bce3cafe1730f7eULL,
+        0x2d0480a0828f6bf5ULL, 0x6c99cffa171f92f6ULL, 0x0087f842bb0ac681ULL,
+        0x11d7ed06e1e7fd3eULL, 0x07cb1186f2385dc6ULL, 0x5d7763ebff1e170fULL,
+        0x2dacc870231ac292ULL, 0x8486317a9ffb390cULL, 0x1c3a6dd20c959ac6ULL,
+        0x90dc96e3992e06b8ULL, 0x70d60bfa33e72b67ULL, 0x70c9bddd0985ee63ULL,
+        0x012c9767b3673093ULL, 0xfcd3bc5580f6a88aULL, 0x0ac80017ef6308c3ULL,
+        0xdb67d709ef4bba09ULL, 0x4c63e324f0e247ccULL, 0xa15481d3fe219d60ULL,
+        0x094c4279cdccb501ULL, 0x965a28c72575cb82ULL, 0x022869db25e391ebULL,
+        0x37f528c146023910ULL, 0x0c1290636917deceULL, 0x9aee25e96251ca9cULL,
+        0x728ac5ba853b69c2ULL, 0x9f272c93c4be20c8ULL, 0x06c1aa6319d28124ULL,
+        0x4324496b1ca8a4f7ULL, 0x0096ecfe7dfc0189ULL, 0x9e06131b19ae0020ULL,
+        0x15278b15902f4597ULL, 0x2a9fece8c13842d8ULL, 0x1d4e6781f0e1355eULL,
+        0x6855b712d3dbf7c0ULL, 0x06a07fad99be6f46ULL, 0x3ed9d7957e4d1d7cULL,
+        0x0c326f7cbc248bb2ULL, 0xe6363ad2c537cf51ULL, 0x0e12eb1c40723f13ULL,
+        0xf5c6ac850afba803ULL, 0x0322a79d615fa9f0ULL, 0x6116696ed97bd5f8ULL,
+        0x0d438080fbbdc9f1ULL, 0x2e4dc42c38f1e243ULL, 0x64948e9104f3a5bfULL,
+        0x9fd622371bdb5f00ULL, 0x0f12bf082b2a1b6eULL, 0x4b1f8d867d78031cULL,
+        0x134392ea9f5ef832ULL, 0xf3d70472321bc23eULL, 0x05fcbe5e9eea268eULL,
+        0x136dede7175a22cfULL, 0x1308f8baac2cbcccULL, 0xd691026f0915eb64ULL,
+        0x0e49a668345c3a38ULL, 0x24ddbbe8bc96f331ULL, 0x4d2ec9479b640578ULL,
+        0x450f0697327b359cULL, 0x32b45360f4488ee0ULL, 0x4f6d9ecec46a105aULL,
+        0x5500c63401ae8e80ULL, 0x47dea495cf6f98baULL, 0x13dc9a2dfca80babULL,
+        0xe6f8a93f7b24ca92ULL, 0x073f57a6d900a87fULL, 0x9ddb935fd3aa695aULL,
+        0x101e98d24b39e8aaULL, 0x6b8d0eb95a507ddcULL, 0x45a908b3903d209bULL,
+        0x6c96a3e119e617d4ULL, 0x2442787543d3be48ULL, 0xd3bc055c7544b364ULL,
+        0x7693bb042ca8653eULL, 0xb95e3a4ea5d0101eULL, 0x116f0d459bb94a73ULL,
+        0x841244b72cdc5e90ULL, 0x1271acced6cb34d3ULL, 0x07d289106524d638ULL,
+        0x537c9cf49c01b5bbULL, 0x8a8e16706bb7a5daULL, 0x12e50a9c499dc3a9ULL,
+        0x1cade520db2ba830ULL, 0x1add52f000d7db70ULL, 0x12cf15db2ce78e30ULL,
+        0x0657eaf606bfc866ULL, 0x4026816d3b05b1d0ULL, 0x1ba0ebdf90128e4aULL,
+        0xdfd649375996dd6eULL, 0x0f416e906c23d9aeULL, 0x384273cad0582a24ULL,
+        0x2ff27b0378a46189ULL, 0xc4ecd18a2d7a7616ULL, 0x35cef0b5cd51d640ULL,
+        0x7d582363643f48b7ULL, 0x0984ad746ad0ab7cULL, 0x2990a999835f9688ULL,
+        0x2d4df66a97b19e05ULL, 0x592c79720af99aa2ULL, 0x052863c230602cd3ULL,
+        0x5f5e2b15edcf2840ULL, 0x01dff1b694b978b0ULL, 0x14345a48b622025eULL,
+        0x028fab3b6407f715ULL, 0x3455d188e6feca50ULL, 0x1d0d40288fb1b5fdULL,
+        0x4685c5c2b6a1e5aeULL, 0x3a2077b1e5fe5adeULL, 0x1bc55d611445a0d8ULL,
+        0x05480ae95f3f83feULL, 0xbbb59cfcf7e17fb6ULL, 0x13f7f10970bbb990ULL,
+        0x6d00ac169425a352ULL, 0x7da0db397ef2d5d3ULL, 0x5b512a247f8d2479ULL,
+        0x637eaa6a977c3c32ULL, 0x3720f0ae37cba89cULL, 0x443df6e6aa7f525bULL,
+        0x28664c287dcef321ULL, 0x03c267c00cf35e49ULL, 0x690185572d4021deULL,
+        0x2707ff2596e321c2ULL, 0xd865f5af7722c380ULL, 0x1ea285658e33aafbULL,
+        0xc257c5e88755bef4ULL, 0x066f67275cfcc31eULL, 0xb09931945cc0fed0ULL,
+        0x58c1dc38d6e3a03fULL, 0xf99489678fc94ee8ULL, 0x75045bb99be5758aULL,
+        0x6c163bc34b40feefULL, 0x0420063ce7bdd3b4ULL, 0xf86ef10582bf2e28ULL,
+        0x162c3449ca14858cULL, 0x94106aa61dfe3280ULL, 0x4073ae7a4e7e4941ULL,
+        0x32b13fd179c250b4ULL, 0x0178fbb216a7e744ULL, 0xf840ae2f1cf92669ULL,
+        0x18fc709acc80243dULL, 0x20ac2ebd69f4d558ULL, 0x6e580ad9c73ad46aULL,
+        0x76d2b535b541c19dULL, 0x6c7a3fb9dd0ce0afULL, 0xc3481689b9754f28ULL,
+        0x156e813b6557abdbULL, 0x6ee372e31276eb10ULL, 0x19cf37c038c8d381ULL,
+        0x00d4d906c9ae3072ULL, 0x09f03cbb6dfbfd40ULL, 0x461ba31c4125f3cfULL,
+        0x25b29fc63ad9f05bULL, 0x6808c95c2dddede9ULL, 0x0564224337066d9bULL,
+        0xc87eb5f4a4d966f2ULL, 0x66fc66e1701f5847ULL, 0xc553a3559f74da28ULL,
+        0x1dfd841be574df43ULL, 0x3ee2f100c3ebc082ULL, 0x1a2c4f9517b56e89ULL,
+        0x502f65c4b535c8ffULL, 0x1da5663ab6f96ec0ULL, 0xba1f80b73988152cULL,
+        0x364ff12182ac8dc1ULL, 0xe3457a3c4871db31ULL, 0x6ae9cadf92fd7e84ULL,
+        0x9621ba3d6ca15186ULL, 0x00ff5af878c144ceULL, 0x918464dc130101a4ULL,
+        0x036511e6b187efa6ULL, 0x06667d66550ff260ULL, 0x7fd18913f9b51bc1ULL,
+        0x3740e6b27af77aa8ULL, 0x1f546c2fd358ff8aULL, 0x42f1424e3115c891ULL,
+        0x03767db4e3a1bb33ULL, 0xa171a1c564345060ULL, 0x0afcf632fd7b1324ULL,
+        0xb59508d933ffb7d0ULL, 0x57d766c42071be83ULL, 0x659f0447546114a2ULL,
+        0x4070364481c460aeULL, 0xa2b9752280644d52ULL, 0x04ab884bea5771bdULL,
+        0x87cd135602a232b4ULL, 0x15e54cd9a8155313ULL, 0x1e8005efaa3e1047ULL,
+        0x696b93f4ab15d39fULL, 0x0855a8e540de863aULL, 0x0bb11799e79f9426ULL,
+        0xeffa61e5c1b579baULL, 0x1e060a1d11808219ULL, 0x10e219205667c599ULL,
+        0x2f7b206091c49498ULL, 0xb48854c820064860ULL, 0x21c4aaa3bfbe4a38ULL,
+        0x8f4a032a3fa67e9cULL, 0x3146b3823401e2acULL, 0x3afee26f19d88400ULL,
+        0x167087c485791d38ULL, 0xb67a1ed945b0fb4bULL, 0x02436eb17e27f1c0ULL,
+        0xe05afce2ce2d2790ULL, 0x49c536fc6224cfebULL, 0x178865b3b862b856ULL,
+        0x1ce530de26acde5bULL, 0x87312c0b30a06f38ULL, 0x03e653b578558d76ULL,
+        0x4d3663c21d8b3accULL, 0x038003c23626914aULL, 0xd9d5a2c052a09451ULL,
+        0x39b5acfe08a49384ULL, 0x40f349956d5800e4ULL, 0x0968b6950b1bd8feULL,
+        0xd60b2ca030f3779cULL, 0x7c8bc11a23ce18edULL, 0xcc23374e27630bc2ULL,
+        0x2e38fc2a8bb33210ULL, 0xe421357814ee5c44ULL, 0x315fb65ea71ec671ULL,
+        0xfb1b0223f70ed290ULL, 0x30556c9f983eaf07ULL, 0x8dd438c3d0cd625aULL,
+        0x05a8fd0c7ffde71bULL, 0x764d1313b5aeec7aULL, 0x2036af5de9622f47ULL,
+        0x508a5bfadda292feULL, 0x3f77f04ba2830e90ULL, 0x9047cd9c66ca66d2ULL,
+        0x1168b5318a54eb21ULL, 0xc93462d221da2e15ULL, 0x4c2c7cc54abc066eULL,
+        0x767a56fec478240eULL, 0x095de72546595bd3ULL, 0xc9da535865158558ULL,
+        0x1baccf36f33e73fbULL, 0xf3d7dbe64df77f18ULL, 0x1f8ebbb7be4850b8ULL,
+        0x043c5ed77bce25a1ULL, 0x07d401041b2a178aULL, 0x9181ebb8bd8d5618ULL,
+        0x078b935dc3e4034aULL, 0x7b59c08954214300ULL, 0x03570dc2a4f84421ULL,
+        0xdd8715b82f6b4078ULL, 0x2bb49c8bb544163bULL, 0xc9eb125564d59686ULL,
+        0x5fdc7a38f80b810aULL, 0x3a4a6d8fff686544ULL, 0x28360e2418627d3aULL,
+        0x60874244c95ed992ULL, 0x2115cc1dd9c34ed3ULL, 0xfaa3ef61f55e9efcULL,
+        0x27ac9b1ef1adc7e6ULL, 0x95ea00478fec3f54ULL, 0x5aea808b2d99ab43ULL,
+        0xc8f79e51fe43a580ULL, 0x5dbccd714236ce25ULL, 0x783fa76ed0753458ULL,
+        0x48cb290f19d84655ULL, 0xc86a832f7696099aULL, 0x52f30c6fec0e71d3ULL,
+        0x77d4e91e8cdeb886ULL, 0x7169a703c6a79ccdULL, 0x98208145b9596f74ULL,
+        0x0945695c761c0796ULL, 0x0be897830d17bae0ULL, 0x033ad3924caeeeb4ULL,
+        0xedecb6cfa2d303a8ULL, 0x3f86b074818642e7ULL, 0xeefa7c878a8b03f4ULL,
+        0x093c101b80922551ULL, 0xfb3b4e6c26ac0034ULL, 0x162bf87999b94f5eULL,
+        0xeaedae76e975b17cULL, 0x1852aa090effe18eULL};
+
+    static constexpr uint64_t kCLMulUpper[kCLMulNum] = {
+        0xbb41199b1d587c69ULL, 0x514d94d55894ee29ULL, 0xebc6cd4d2efd5d16ULL,
+        0x042044ad2de477fdULL, 0xb865c8b0fcdf4b15ULL, 0x0724d7e551cc40f3ULL,
+        0xb15a16f39edb0bccULL, 0x37d64419ede7a171ULL, 0x2aa01bb80c753401ULL,
+        0x06ff3f8a95fdaf4dULL, 0x79898cc0838546deULL, 0x776acbd1b237c60aULL,
+        0x4c1753be4f4e0064ULL, 0x0ba9243601206ed3ULL, 0xd567c3b1bf3ec557ULL,
+        0x043fac7bcff61fb3ULL, 0x49356232b159fb2fULL, 0x3910c82038102d4dULL,
+        0x30592fef753eb300ULL, 0x7b2660e0c92a9e9aULL, 0x8246c9248d671ef0ULL,
+        0x5a0dcd95147af5faULL, 0x43fde953909cc0eaULL, 0x06147b972cb96e1bULL,
+        0xd84193a6b2411d80ULL, 0x00cd7711b950196fULL, 0x1088f9f4ade7fa64ULL,
+        0x05a13096ec113cfbULL, 0x958d816d53b00edcULL, 0x3846154a7cdba9cbULL,
+        0x8af516db6b27d1e6ULL, 0x1a1d462ab8a33b13ULL, 0x4040b0ac1b2c754cULL,
+        0x05127fe9af2fe1d6ULL, 0x9f96e79374321fa6ULL, 0x06ff64a4d9c326f3ULL,
+        0x28709566e158ac15ULL, 0x301701d7111ca51cULL, 0x31e0445d1b9d9544ULL,
+        0x0a95aff69bf1d03eULL, 0x7c298c8414ecb879ULL, 0x00801499b4143195ULL,
+        0x91521a00dd676a5cULL, 0x2777526a14c2f723ULL, 0xfa26aac6a6357dddULL,
+        0x1d265889b0187a4bULL, 0xcd6e70fa8ed283e4ULL, 0x18a815aa50ea92caULL,
+        0xc01e082694a263c6ULL, 0x4b40163ba53daf25ULL, 0xbc658caff6501673ULL,
+        0x3ba35359586b9652ULL, 0x74f96acc97a4936cULL, 0x3989dfdb0cf1d2cfULL,
+        0x358a01eaa50dda32ULL, 0x01109a5ed8f0802bULL, 0x55b84922e63c2958ULL,
+        0x55b14843d87551d5ULL, 0x1db8ec61b1b578d8ULL, 0x79a2d49ef8c3658fULL,
+        0xa304516816b3fbe0ULL, 0x163ecc09cc7b82f9ULL, 0xab91e8d22aabef00ULL,
+        0x0ed6b09262de8354ULL, 0xcfd47d34cf73f6f2ULL, 0x7dbd1db2390bc6c3ULL,
+        0x5ae789d3875e7b00ULL, 0x1d60fd0e70fe8fa4ULL, 0x690bc15d5ae4f6f5ULL,
+        0x121ef5565104fb44ULL, 0x6e98e89297353b54ULL, 0x42554949249d62edULL,
+        0xd6d6d16b12df78d2ULL, 0x320b33549b74975dULL, 0xd2a0618763d22e00ULL,
+        0x0808deb93cba2017ULL, 0x01bd3b2302a2cc70ULL, 0x0b7b8dd4d71c8dd6ULL,
+        0x34d60a3382a0756cULL, 0x40984584c8219629ULL, 0xf1152cba10093a66ULL,
+        0x068001c6b2159ccbULL, 0x3d70f13c6cda0800ULL, 0x0e6b6746a322b956ULL,
+        0x83a494319d8c770bULL, 0x0faecf64a8553e9aULL, 0xa34919222c39b1bcULL,
+        0x0c63850d89e71c6fULL, 0x585f0bee92e53dc8ULL, 0x10f222b13b4fa5deULL,
+        0x61573114f94252f2ULL, 0x09d59c311fba6c27ULL, 0x014effa7da49ed4eULL,
+        0x4a400a1bc1c31d26ULL, 0xc9091c047b484972ULL, 0x3989f341ec2230ccULL,
+        0xdcb03a98b3aee41eULL, 0x4a54a676a33a95e1ULL, 0xe499b7753951ef7cULL,
+        0x2f43b1d1061d8b48ULL, 0xc3313bdc68ceb146ULL, 0x5159f6bc0e99227fULL,
+        0x98128e6d9c05efcaULL, 0x15ea32b27f77815bULL, 0xe882c054e2654eecULL,
+        0x003d2cdb8faee8c6ULL, 0xb416dd333a9fe1dfULL, 0x73f6746aefcfc98bULL,
+        0x93dc114c10a38d70ULL, 0x05055941657845eaULL, 0x2ed7351347349334ULL,
+        0x26fb1ee2c69ae690ULL, 0xa4575d10dc5b28e0ULL, 0x3395b11295e485ebULL,
+        0xe840f198a224551cULL, 0x78e6e5a431d941d4ULL, 0xa1fee3ceab27f391ULL,
+        0x07d35b3c5698d0dcULL, 0x983c67fca9174a29ULL, 0x2bb6bbae72b5144aULL,
+        0xa7730b8d13ce58efULL, 0x51b5272883de1998ULL, 0xb334e128bb55e260ULL,
+        0x1cacf5fbbe1b9974ULL, 0x71a9df4bb743de60ULL, 0x5176fe545c2d0d7aULL,
+        0xbe592ecf1a16d672ULL, 0x27aa8a30c3efe460ULL, 0x4c78a32f47991e06ULL,
+        0x383459294312f26aULL, 0x97ba789127f1490cULL, 0x51c9aa8a3abd1ef1ULL,
+        0xcc7355188121e50fULL, 0x0ecb3a178ae334c1ULL, 0x84879a5e574b7160ULL,
+        0x0765298f6389e8f3ULL, 0x5c6750435539bb22ULL, 0x11a05cf056c937b5ULL,
+        0xb5dc2172dbfb7662ULL, 0x3ffc17915d9f40e8ULL, 0xbc7904daf3b431b0ULL,
+        0x71f2088490930a7cULL, 0xa89505fd9efb53c4ULL, 0x02e194afd61c5671ULL,
+        0x99a97f4abf35fcecULL, 0x26830aad30fae96fULL, 0x4b2abc16b25cf0b0ULL,
+        0x07ec6fffa1cafbdbULL, 0xf38188fde97a280cULL, 0x121335701afff64dULL,
+        0xea5ef38b4e672a64ULL, 0x477edbcae3eabf03ULL, 0xa32813cc0e0d244dULL,
+        0x13346d2af4972eefULL, 0xcbc18357af1cfa9aULL, 0x561b630316e73fa6ULL,
+        0xe9dfb53249249305ULL, 0x5d2b9dd1479312eeULL, 0x3458008119b56d04ULL,
+        0x50e6790b49801385ULL, 0x5bb9febe2349492bULL, 0x0c2813954299098fULL,
+        0xf747b0c890a071d5ULL, 0x417e8f82cc028d77ULL, 0xa134fee611d804f8ULL,
+        0x24c99ee9a0408761ULL, 0x3ebb224e727137f3ULL, 0x0686022073ceb846ULL,
+        0xa05e901fb82ad7daULL, 0x0ece7dc43ab470fcULL, 0x2d334ecc58f7d6a3ULL,
+        0x23166fadacc54e40ULL, 0x9c3a4472f839556eULL, 0x071717ab5267a4adULL,
+        0xb6600ac351ba3ea0ULL, 0x30ec748313bb63d4ULL, 0xb5374e39287b23ccULL,
+        0x074d75e784238aebULL, 0x77315879243914a4ULL, 0x3bbb1971490865f1ULL,
+        0xa355c21f4fbe02d3ULL, 0x0027f4bb38c8f402ULL, 0xeef8708e652bc5f0ULL,
+        0x7b9aa56cf9440050ULL, 0x113ac03c16cfc924ULL, 0x395db36d3e4bef9fULL,
+        0x5d826fabcaa597aeULL, 0x2a77d3c58786d7e0ULL, 0x85996859a3ba19d4ULL,
+        0x01e7e3c904c2d97fULL, 0x34f90b9b98d51fd0ULL, 0x243aa97fd2e99bb7ULL,
+        0x40a0cebc4f65c1e8ULL, 0x46d3922ed4a5503eULL, 0x446e7ecaf1f9c0a4ULL,
+        0x49dc11558bc2e6aeULL, 0xe7a9f20881793af8ULL, 0x5771cc4bc98103f1ULL,
+        0x2446ea6e718fce90ULL, 0x25d14aca7f7da198ULL, 0x4347af186f9af964ULL,
+        0x10cb44fc9146363aULL, 0x8a35587afce476b4ULL, 0x575144662fee3d3aULL,
+        0x69f41177a6bc7a05ULL, 0x02ff8c38d6b3c898ULL, 0x57c73589a226ca40ULL,
+        0x732f6b5baae66683ULL, 0x00c008bbedd4bb34ULL, 0x7412ff09524d6cadULL,
+        0xb8fd0b5ad8c145a8ULL, 0x74bd9f94b6cdc7dfULL, 0x68233b317ca6c19cULL,
+        0x314b9c2c08b15c54ULL, 0x5bd1ad72072ebd08ULL, 0x6610e6a6c07030e4ULL,
+        0xa4fc38e885ead7ceULL, 0x36975d1ca439e034ULL, 0xa358f0fe358ffb1aULL,
+        0x38e247ad663acf7dULL, 0x77daed3643b5deb8ULL, 0x5507c2aeae1ec3d0ULL,
+        0xfdec226c73acf775ULL, 0x1b87ff5f5033492dULL, 0xa832dee545d9033fULL,
+        0x1cee43a61e41783bULL, 0xdff82b2e2d822f69ULL, 0x2bbc9a376cb38cf2ULL,
+        0x117b1cdaf765dc02ULL, 0x26a407f5682be270ULL, 0x8eb664cf5634af28ULL,
+        0x17cb4513bec68551ULL, 0xb0df6527900cbfd0ULL, 0x335a2dc79c5afdfcULL,
+        0xa2f0ca4cd38dca88ULL, 0x1c370713b81a2de1ULL, 0x849d5df654d1adfcULL,
+        0x2fd1f7675ae14e44ULL, 0x4ff64dfc02247f7bULL, 0x3a2bcf40e395a48dULL,
+        0x436248c821b187c1ULL, 0x29f4337b1c7104c0ULL, 0xfc317c46e6630ec4ULL,
+        0x2774bccc4e3264c7ULL, 0x2d03218d9d5bee23ULL, 0x36a0ed04d659058aULL,
+        0x452484461573cab6ULL, 0x0708edf87ed6272bULL, 0xf07960a1587446cbULL,
+        0x3660167b067d84e0ULL, 0x65990a6993ddf8c4ULL, 0x0b197cd3d0b40b3fULL,
+        0x1dcec4ab619f3a05ULL, 0x722ab223a84f9182ULL, 0x0822d61a81e7c38fULL,
+        0x3d22ad75da563201ULL, 0x93cef6979fd35e0fULL, 0x05c3c25ae598b14cULL,
+        0x1338df97dd496377ULL, 0x15bc324dc9c20acfULL, 0x96397c6127e6e8cfULL,
+        0x004d01069ef2050fULL, 0x2fcf2e27893fdcbcULL, 0x072f77c3e44f4a5cULL,
+        0x5eb1d80b3fe44918ULL, 0x1f59e7c28cc21f22ULL, 0x3390ce5df055c1f8ULL,
+        0x4c0ef11df92cb6bfULL, 0x50f82f9e0848c900ULL, 0x08d0fde3ffc0ae38ULL,
+        0xbd8d0089a3fbfb73ULL, 0x118ba5b0f311ef59ULL, 0x9be9a8407b926a61ULL,
+        0x4ea04fbb21318f63ULL, 0xa1c8e7bb07b871ffULL, 0x1253a7262d5d3b02ULL,
+        0x13e997a0512e5b29ULL, 0x54318460ce9055baULL, 0x4e1d8a4db0054798ULL,
+        0x0b235226e2cade32ULL, 0x2588732c1476b315ULL, 0x16a378750ba8ac68ULL,
+        0xba0b116c04448731ULL, 0x4dd02bd47694c2f1ULL, 0x16d6797b218b6b25ULL,
+        0x769eb3709cfbf936ULL, 0x197746a0ce396f38ULL, 0x7d17ad8465961d6eULL,
+        0xfe58f4998ae19bb4ULL, 0x36df24305233ce69ULL, 0xb88a4eb008f4ee72ULL,
+        0x302b2eb923334787ULL, 0x15a4e3edbe13d448ULL, 0x39a4bf64dd7730ceULL,
+        0xedf25421b31090c4ULL, 0x4d547fc131be3b69ULL, 0x2b316e120ca3b90eULL,
+        0x0faf2357bf18a169ULL, 0x71f34b54ee2c1d62ULL, 0x18eaf6e5c93a3824ULL,
+        0x7e168ba03c1b4c18ULL, 0x1a534dd586d9e871ULL, 0xa2cccd307f5f8c38ULL,
+        0x2999a6fb4dce30f6ULL, 0x8f6d3b02c1d549a6ULL, 0x5cf7f90d817aac5aULL,
+        0xd2a4ceefe66c8170ULL, 0x11560edc4ca959feULL, 0x89e517e6f0dc464dULL,
+        0x75bb8972dddd2085ULL, 0x13859ed1e459d65aULL, 0x057114653326fa84ULL,
+        0xe2e6f465173cc86cULL, 0x0ada4076497d7de4ULL, 0xa856fa10ec6dbf8aULL,
+        0x41505d9a7c25d875ULL, 0x3091b6278382eccdULL, 0x055737185b2c3f13ULL,
+        0x2f4df8ecd6f9c632ULL, 0x0633e89c33552d98ULL, 0xf7673724d16db440ULL,
+        0x7331bd08e636c391ULL, 0x0252f29672fee426ULL, 0x1fc384946b6b9ddeULL,
+        0x03460c12c901443aULL, 0x003a0792e10abcdaULL, 0x8dbec31f624e37d0ULL,
+        0x667420d5bfe4dcbeULL, 0xfbfa30e874ed7641ULL, 0x46d1ae14db7ecef6ULL,
+        0x216bd7e8f5448768ULL, 0x32bcd40d3d69cc88ULL, 0x2e991dbc39b65abeULL,
+        0x0e8fb123a502f553ULL, 0x3d2d486b2c7560c0ULL, 0x09aba1db3079fe03ULL,
+        0xcb540c59398c9bceULL, 0x363970e5339ed600ULL, 0x2caee457c28af00eULL,
+        0x005e7d7ee47f41a0ULL, 0x69fad3eb10f44100ULL, 0x048109388c75beb3ULL,
+        0x253dddf96c7a6fb8ULL, 0x4c47f705b9d47d09ULL, 0x6cec894228b5e978ULL,
+        0x04044bb9f8ff45c2ULL, 0x079e75704d775caeULL, 0x073bd54d2a9e2c33ULL,
+        0xcec7289270a364fbULL, 0x19e7486f19cd9e4eULL, 0xb50ac15b86b76608ULL,
+        0x0620cf81f165c812ULL, 0x63eaaf13be7b11d4ULL, 0x0e0cf831948248c2ULL,
+        0xf0412df8f46e7957ULL, 0x671c1fe752517e3fULL, 0x8841bfb04dd3f540ULL,
+        0x122de4142249f353ULL, 0x40a4959fb0e76870ULL, 0x25cfd3d4b4bbc459ULL,
+        0x78a07c82930c60d0ULL, 0x12c2de24d4cbc969ULL, 0x85d44866096ad7f4ULL,
+        0x1fd917ca66b2007bULL, 0x01fbbb0751764764ULL, 0x3d2a4953c6fe0fdcULL,
+        0xcc1489c5737afd94ULL, 0x1817c5b6a5346f41ULL, 0xe605a6a7e9985644ULL,
+        0x3c50412328ff1946ULL, 0xd8c7fd65817f1291ULL, 0x0bd66975ab66339bULL,
+        0x2baf8fa1c7d10fa9ULL, 0x24abdf06ddef848dULL, 0x14df0c9b2ea4f6c2ULL,
+        0x2be950edfd2cb1f7ULL, 0x21911e21094178b6ULL, 0x0fa54d518a93b379ULL,
+        0xb52508e0ac01ab42ULL, 0x0e035b5fd8cb79beULL, 0x1c1c6d1a3b3c8648ULL,
+        0x286037b42ea9871cULL, 0xfe67bf311e48a340ULL, 0x02324131e932a472ULL,
+        0x2486dc2dd919e2deULL, 0x008aec7f1da1d2ebULL, 0x63269ba0e8d3eb3aULL,
+        0x23c0f11154adb62fULL, 0xc6052393ecd4c018ULL, 0x523585b7d2f5b9fcULL,
+        0xf7e6f8c1e87564c9ULL, 0x09eb9fe5dd32c1a3ULL, 0x4d4f86886e055472ULL,
+        0x67ea17b58a37966bULL, 0x3d3ce8c23b1ed1a8ULL, 0x0df97c5ac48857ceULL,
+        0x9b6992623759eb12ULL, 0x275aa9551ae091f2ULL, 0x08855e19ac5e62e5ULL,
+        0x1155fffe0ae083ccULL, 0xbc9c78db7c570240ULL, 0x074560c447dd2418ULL,
+        0x3bf78d330bcf1e70ULL, 0x49867cd4b7ed134bULL, 0x8e6eee0cb4470accULL,
+        0x1dabafdf59233dd6ULL, 0xea3a50d844fc3fb8ULL, 0x4f03f4454764cb87ULL,
+        0x1f2f41cc36c9e6ecULL, 0x53cba4df42963441ULL, 0x10883b70a88d91fbULL,
+        0x62b1fc77d4eb9481ULL, 0x893d8f2604b362e1ULL, 0x0933b7855368b440ULL,
+        0x9351b545703b2fceULL, 0x59c1d489b9bdd3b4ULL, 0xe72a9c4311417b18ULL,
+        0x5355df77e88eb226ULL, 0xe802c37aa963d7e1ULL, 0x381c3747bd6c3bc3ULL,
+        0x378565573444258cULL, 0x37848b1e52b43c18ULL, 0x5da2cd32bdce12b6ULL,
+        0x13166c5da615f6fdULL, 0xa51ef95efcc66ac8ULL, 0x640c95e473f1e541ULL,
+        0x6ec68def1f217500ULL, 0x49ce3543c76a4079ULL, 0x5fc6fd3cddc706b5ULL,
+        0x05c3c0f0f6a1fb0dULL, 0xe7820c0996ad1bddULL, 0x21f0d752a088f35cULL,
+        0x755405b51d6fc4a0ULL, 0x7ec7649ca4b0e351ULL, 0x3d2b6a46a251f790ULL,
+        0x23e1176b19f418adULL, 0x06056575efe8ac05ULL, 0x0f75981b6966e477ULL,
+        0x06e87ec41ad437e4ULL, 0x43f6c255d5e1cb84ULL, 0xe4e67d1120ceb580ULL,
+        0x2cd67b9e12c26d7bULL, 0xcd00b5ff7fd187f1ULL, 0x3f6cd40accdc4106ULL,
+        0x3e895c835459b330ULL, 0x0814d53a217c0850ULL, 0xc9111fe78bc3a62dULL,
+        0x719967e351473204ULL, 0xe757707d24282aa4ULL, 0x7226b7f5607f98e6ULL,
+        0x7b268ffae3c08d96ULL, 0x16d3917c8b86020eULL, 0x5128bca51c49ea64ULL,
+        0x345ffea02bb1698dULL, 0x9460f5111fe4fbc8ULL, 0x60dd1aa5762852cbULL,
+        0xbb7440ed3c81667cULL, 0x0a4b12affa7f6f5cULL, 0x95cbcb0ae03861b6ULL,
+        0x07ab3b0591db6070ULL, 0xc6476a4c3de78982ULL, 0x204e82e8623ad725ULL,
+        0x569a5b4e8ac2a5ccULL, 0x425a1d77d72ebae2ULL, 0xcdaad5551ab33830ULL,
+        0x0b7c68fd8422939eULL, 0x46d9a01f53ec3020ULL, 0x102871edbb29e852ULL,
+        0x7a8e8084039075a5ULL, 0x40eaede8615e376aULL, 0x4dc67d757a1c751fULL,
+        0x1176ef33063f9145ULL, 0x4ea230285b1c8156ULL, 0x6b2aa46ce0027392ULL,
+        0x32b13230fba1b068ULL, 0x0e69796851bb984fULL, 0xb749f4542db698c0ULL,
+        0x19ad0241ffffd49cULL, 0x2f41e92ef6caff52ULL, 0x4d0b068576747439ULL,
+        0x14d607aef7463e00ULL, 0x1443d00d85fb440eULL, 0x529b43bf68688780ULL,
+        0x21133a6bc3a3e378ULL, 0x865b6436dae0e7e5ULL, 0x6b4fe83dc1d6defcULL,
+        0x03a5858a0ca0be46ULL, 0x1e841b187e67f312ULL, 0x61ee22ef40a66940ULL,
+        0x0494bd2e9e741ef8ULL, 0x4eb59e323010e72cULL, 0x19f2abcfb749810eULL,
+        0xb30f1e4f994ef9bcULL, 0x53cf6cdd51bd2d96ULL, 0x263943036497a514ULL,
+        0x0d4b52170aa2edbaULL, 0x0c4758a1c7b4f758ULL, 0x178dadb1b502b51aULL,
+        0x1ddbb20a602eb57aULL, 0x1fc2e2564a9f27fdULL, 0xd5f8c50a0e3d6f90ULL,
+        0x0081da3bbe72ac09ULL, 0xcf140d002ccdb200ULL, 0x0ae8389f09b017feULL,
+        0x17cc9ffdc03f4440ULL, 0x04eb921d704bcdddULL, 0x139a0ce4cdc521abULL,
+        0x0bfce00c145cb0f0ULL, 0x99925ff132eff707ULL, 0x063f6e5da50c3d35ULL,
+        0xa0c25dea3f0e6e29ULL, 0x0c7a9048cc8e040fULL,
+    };
+
+    const size_t padded = RoundUpTo(kCLMulNum, N);
+    auto expected_lower = AllocateAligned<T>(padded);
+    auto expected_upper = AllocateAligned<T>(padded);
+    HWY_ASSERT(expected_lower && expected_upper);
+    CopyBytes<kCLMulNum * sizeof(T)>(kCLMulLower, expected_lower.get());
+    CopyBytes<kCLMulNum * sizeof(T)>(kCLMulUpper, expected_upper.get());
+    const size_t padding_size = (padded - kCLMulNum) * sizeof(T);
+    memset(expected_lower.get() + kCLMulNum, 0, padding_size);
+    memset(expected_upper.get() + kCLMulNum, 0, padding_size);
+
+    // Random inputs in each lane
+    RandomState rng;
+    for (size_t rep = 0; rep < kCLMulNum / N; ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        in1[i] = Random64(&rng);
+        in2[i] = Random64(&rng);
+      }
+
+      const auto a = Load(d, in1.get());
+      const auto b = Load(d, in2.get());
+#if HWY_PRINT_CLMUL_GOLDEN
+      Store(CLMulLower(a, b), d, expected_lower.get() + rep * N);
+      Store(CLMulUpper(a, b), d, expected_upper.get() + rep * N);
+#else
+      HWY_ASSERT_VEC_EQ(d, expected_lower.get() + rep * N, CLMulLower(a, b));
+      HWY_ASSERT_VEC_EQ(d, expected_upper.get() + rep * N, CLMulUpper(a, b));
+#endif
+    }
+
+#if HWY_PRINT_CLMUL_GOLDEN
+    // RVV lacks PRIu64, so print 32-bit halves.
+    for (size_t i = 0; i < kCLMulNum; ++i) {
+      printf("0x%08x%08xULL,", static_cast<uint32_t>(expected_lower[i] >> 32),
+             static_cast<uint32_t>(expected_lower[i] & 0xFFFFFFFFU));
+    }
+    printf("\n");
+    for (size_t i = 0; i < kCLMulNum; ++i) {
+      printf("0x%08x%08xULL,", static_cast<uint32_t>(expected_upper[i] >> 32),
+             static_cast<uint32_t>(expected_upper[i] & 0xFFFFFFFFU));
+    }
+#endif  // HWY_PRINT_CLMUL_GOLDEN
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllCLMul() { ForGEVectors<128, TestCLMul>()(uint64_t()); }
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyCryptoTest);
+HWY_EXPORT_AND_TEST_P(HwyCryptoTest, TestAllAES);
+HWY_EXPORT_AND_TEST_P(HwyCryptoTest, TestAllAESInverse);
+HWY_EXPORT_AND_TEST_P(HwyCryptoTest, TestAllCLMul);
+HWY_EXPORT_AND_TEST_P(HwyCryptoTest, TestAllAESKeyGenAssist);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/demote_test.cc

@@ -0,0 +1,849 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <cmath>  // std::isfinite
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/demote_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+// Causes build timeout.
+#if !HWY_IS_MSAN
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+template <typename ToT>
+struct TestDemoteTo {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D from_d) {
+    static_assert(!IsFloat<ToT>(), "Use TestDemoteToFloat for float output");
+    static_assert(sizeof(T) > sizeof(ToT), "Input type must be wider");
+    const Rebind<ToT, D> to_d;
+
+    const size_t N = Lanes(from_d);
+    auto from = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<ToT>(N);
+    HWY_ASSERT(from && expected);
+
+    // Narrower range in the wider type, for clamping before we cast
+    const T min = ConvertScalarTo<T>(IsSigned<T>() ? LimitsMin<ToT>()
+                                                   : static_cast<ToT>(0));
+    const T max = LimitsMax<ToT>();
+
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        const uint64_t bits = rng();
+        CopyBytes<sizeof(T)>(&bits, &from[i]);  // not same size
+        expected[i] = static_cast<ToT>(HWY_MIN(HWY_MAX(min, from[i]), max));
+      }
+      const auto in = Load(from_d, from.get());
+      HWY_ASSERT_VEC_EQ(to_d, expected.get(), DemoteTo(to_d, in));
+    }
+
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        const uint64_t bits = rng();
+        CopyBytes<sizeof(ToT)>(&bits, &expected[i]);  // not same size
+
+        if (!IsSigned<T>() && IsSigned<ToT>()) {
+          expected[i] &= static_cast<ToT>(max);
+        }
+
+        from[i] = ConvertScalarTo<T>(expected[i]);
+      }
+
+      const auto in = Load(from_d, from.get());
+      HWY_ASSERT_VEC_EQ(to_d, expected.get(), DemoteTo(to_d, in));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllDemoteToInt() {
+  const ForDemoteVectors<TestDemoteTo<uint8_t>> from_i16_to_u8;
+  from_i16_to_u8(int16_t());
+  from_i16_to_u8(uint16_t());
+
+  const ForDemoteVectors<TestDemoteTo<int8_t>> from_i16_to_i8;
+  from_i16_to_i8(int16_t());
+  from_i16_to_i8(uint16_t());
+
+  const ForDemoteVectors<TestDemoteTo<uint8_t>, 2> from_i32_to_u8;
+  from_i32_to_u8(int32_t());
+  from_i32_to_u8(uint32_t());
+
+  const ForDemoteVectors<TestDemoteTo<int8_t>, 2> from_i32_to_i8;
+  from_i32_to_i8(int32_t());
+  from_i32_to_i8(uint32_t());
+
+#if HWY_HAVE_INTEGER64
+  const ForDemoteVectors<TestDemoteTo<uint8_t>, 3> from_i64_to_u8;
+  from_i64_to_u8(int64_t());
+  from_i64_to_u8(uint64_t());
+
+  const ForDemoteVectors<TestDemoteTo<int8_t>, 3> from_i64_to_i8;
+  from_i64_to_i8(int64_t());
+  from_i64_to_i8(uint64_t());
+#endif
+
+  const ForDemoteVectors<TestDemoteTo<uint16_t>> from_i32_to_u16;
+  from_i32_to_u16(int32_t());
+  from_i32_to_u16(uint32_t());
+
+  const ForDemoteVectors<TestDemoteTo<int16_t>> from_i32_to_i16;
+  from_i32_to_i16(int32_t());
+  from_i32_to_i16(uint32_t());
+
+#if HWY_HAVE_INTEGER64
+  const ForDemoteVectors<TestDemoteTo<uint16_t>, 2> from_i64_to_u16;
+  from_i64_to_u16(int64_t());
+  from_i64_to_u16(uint64_t());
+
+  const ForDemoteVectors<TestDemoteTo<int16_t>, 2> from_i64_to_i16;
+  from_i64_to_i16(int64_t());
+  from_i64_to_i16(uint64_t());
+
+  const ForDemoteVectors<TestDemoteTo<uint32_t>> from_i64_to_u32;
+  from_i64_to_u32(int64_t());
+  from_i64_to_u32(uint64_t());
+
+  const ForDemoteVectors<TestDemoteTo<int32_t>> from_i64_to_i32;
+  from_i64_to_i32(int64_t());
+  from_i64_to_i32(uint64_t());
+#endif
+}
+
+HWY_NOINLINE void TestAllDemoteToMixed() {
+#if HWY_HAVE_FLOAT64
+  const ForDemoteVectors<TestDemoteTo<int32_t>> to_i32;
+  to_i32(double());
+
+  const ForDemoteVectors<TestDemoteTo<uint32_t>> to_u32;
+  to_u32(double());
+#endif
+}
+
+template <typename ToT>
+struct TestDemoteToFloat {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D from_d) {
+    // For floats, we clamp differently and cannot call LimitsMin.
+    static_assert(IsFloat<ToT>(), "Use TestDemoteTo for integer output");
+    static_assert(sizeof(T) > sizeof(ToT), "Input type must be wider");
+    const Rebind<ToT, D> to_d;
+
+    using TU = MakeUnsigned<T>;
+    using ToTU = MakeUnsigned<ToT>;
+
+    const size_t N = Lanes(from_d);
+    auto from = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<ToT>(N);
+    HWY_ASSERT(from && expected);
+
+    constexpr int kMaxToBiasedExp = static_cast<int>(MaxExponentField<ToT>());
+    static_assert(kMaxToBiasedExp > 0, "kMaxToBiasedExp > 0 must be true");
+
+    constexpr int kMaxFromBiasedExp = static_cast<int>(MaxExponentField<T>());
+    static_assert(kMaxFromBiasedExp >= kMaxToBiasedExp,
+                  "kMaxFromBiasedExp >= kMaxToBiasedExp must be true");
+
+    constexpr int kMaxFromExpBias = (kMaxFromBiasedExp / 2);
+    constexpr int kMaxToExpBias = (kMaxToBiasedExp / 2);
+
+    constexpr int kMinToTNormalFromTBiasedExp =
+        1 - kMaxToExpBias + kMaxFromExpBias;
+    static_assert(kMinToTNormalFromTBiasedExp > 0,
+                  "kMinToTNormalFromTBiasedExp must be greater than 0");
+    static_assert(
+        kMinToTNormalFromTBiasedExp < kMaxFromExpBias,
+        "kMinToTNormalFromTBiasedExp must be less than kMaxFromExpBias");
+
+    const T min_normal = BitCastScalar<T>(static_cast<TU>(
+        static_cast<TU>(kMinToTNormalFromTBiasedExp) << MantissaBits<T>()));
+    HWY_ASSERT(ScalarIsFinite(min_normal));
+    HWY_ASSERT(min_normal > static_cast<T>(0) &&
+               min_normal < static_cast<T>(1));
+
+    const T max_denormal =
+        ConvertScalarTo<T>(BitCastScalar<ToT>(MantissaMask<ToT>()));
+    HWY_ASSERT(ScalarIsFinite(max_denormal));
+    HWY_ASSERT(max_denormal > static_cast<T>(0) && max_denormal < min_normal);
+
+    const T min_denormal = ConvertScalarTo<T>(BitCastScalar<ToT>(ToTU{1}));
+    HWY_ASSERT(ScalarIsFinite(min_denormal));
+    HWY_ASSERT(min_denormal > static_cast<T>(0) && min_denormal < max_denormal);
+
+    const T half_min_denormal =
+        static_cast<T>(min_denormal * static_cast<T>(0.5));
+    HWY_ASSERT(ScalarIsFinite(half_min_denormal));
+    HWY_ASSERT(half_min_denormal > static_cast<T>(0) &&
+               half_min_denormal < min_denormal);
+
+    const T max_abs = ConvertScalarTo<T>(HighestValue<ToT>());
+    HWY_ASSERT(ScalarIsFinite(max_abs));
+    HWY_ASSERT(max_abs > static_cast<T>(1));
+
+    const T min_out_of_range = BitCastScalar<T>(
+        static_cast<TU>((BitCastScalar<TU>(max_abs) & ExponentMask<T>()) +
+                        (TU{1} << MantissaBits<T>())));
+    HWY_ASSERT(!ScalarIsNaN(min_out_of_range));
+    HWY_ASSERT(max_abs < min_out_of_range);
+
+    const ToTU kToTPosInfBits = ExponentMask<ToT>();
+
+    RandomState rng;
+
+    // Check that values that are within the range of a normal finite ToT are
+    // converted to a correctly rounded normal value
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        const T rand_val = RandomFiniteValue<T>(&rng);
+        const T magn = ScalarAbs(rand_val);
+        const T clipped = ScalarCopySign(
+            HWY_MIN(HWY_MAX(magn, min_normal), max_abs), rand_val);
+        from[i] = clipped;
+        expected[i] = ConvertScalarTo<ToT>(clipped);
+      }
+
+      HWY_ASSERT_VEC_EQ(to_d, expected.get(),
+                        DemoteTo(to_d, Load(from_d, from.get())));
+    }
+
+    // Check that values that are between min_denormal and max_denormal are
+    // converted to a correctly rounded denormal value
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        const T rand_val = RandomFiniteValue<T>(&rng);
+        const T magn = ScalarAbs(rand_val);
+        const T clipped = ScalarCopySign(
+            HWY_MIN(HWY_MAX(magn, min_denormal), max_denormal), rand_val);
+        from[i] = clipped;
+        expected[i] = ConvertScalarTo<ToT>(clipped);
+      }
+
+      HWY_ASSERT_VEC_EQ(to_d, expected.get(),
+                        DemoteTo(to_d, Load(from_d, from.get())));
+    }
+
+    // Check that denormal values whose absolute value is less than or equal to
+    // half_min_denormal is converted to zero
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        const T rand_val = RandomFiniteValue<T>(&rng);
+        const T magn = ScalarAbs(rand_val);
+        const T clipped =
+            ScalarCopySign(HWY_MIN(magn, half_min_denormal), rand_val);
+        from[i] = clipped;
+      }
+
+      HWY_ASSERT_VEC_EQ(to_d, Zero(to_d),
+                        DemoteTo(to_d, Load(from_d, from.get())));
+    }
+
+    // Check that finite values that are out of the range of ToT are correctly
+    // converted to positive infinity or negative infinity
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        const T rand_val = RandomFiniteValue<T>(&rng);
+        const T abs_rand_val = ScalarAbs(rand_val);
+        const T rand_out_of_range_val =
+            ScalarCopySign(HWY_MAX(abs_rand_val, min_out_of_range), rand_val);
+
+        from[i] = rand_out_of_range_val;
+
+        expected[i] = BitCastScalar<ToT>(static_cast<ToTU>(
+            kToTPosInfBits | (static_cast<ToTU>(ScalarSignBit(rand_val))
+                              << (sizeof(ToTU) * 8 - 1))));
+      }
+
+      HWY_ASSERT_VEC_EQ(to_d, expected.get(),
+                        DemoteTo(to_d, Load(from_d, from.get())));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllDemoteToFloat() {
+#if HWY_HAVE_FLOAT64
+  const ForDemoteVectors<TestDemoteToFloat<float>, 1> to_float;
+  to_float(double());
+
+  const ForDemoteVectors<TestDemoteToFloat<hwy::float16_t>, 2> f64_to_f16;
+  f64_to_f16(double());
+#endif
+
+  const ForDemoteVectors<TestDemoteToFloat<hwy::float16_t>, 1> f32_to_f16;
+  f32_to_f16(float());
+}
+
+struct TestDemoteUI64ToFloat {
+  // This helper function avoids an internal compiler error on GCC 8 AVX3,
+  // see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111117.
+  template <class D>
+  static HWY_NOINLINE void Verify(D from_d, TFromD<D> from, float expected) {
+    const Rebind<float, D> df32;
+    HWY_ASSERT_VEC_EQ(df32, Set(df32, expected),
+                      DemoteTo(df32, Set(from_d, from)));
+  }
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D from_d) {
+    const Rebind<float, D> df32;
+
+    Verify(from_d, static_cast<T>(0), 0.0f);
+    Verify(from_d, LimitsMax<T>(), static_cast<float>(LimitsMax<T>()));
+    Verify(from_d, static_cast<T>(11808), 11808.0f);
+    Verify(from_d, static_cast<T>(261162016), 261162016.0f);
+    Verify(from_d, static_cast<T>(18665497952256LL), 18665497952256.0f);
+
+    if (IsSigned<T>()) {
+      Verify(from_d, static_cast<T>(-1), -1.0f);
+      Verify(from_d, LimitsMin<T>(), static_cast<float>(LimitsMin<T>()));
+      Verify(from_d, static_cast<T>(-17633), -17633.0f);
+      Verify(from_d, static_cast<T>(-3888877568LL), -3888877568.0f);
+      Verify(from_d, static_cast<T>(-17851503083520LL), -17851503083520.0f);
+    }
+
+    const size_t N = Lanes(from_d);
+    auto from = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<float>(N);
+    HWY_ASSERT(from && expected);
+
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; i++) {
+        const uint64_t bits = rng();
+        CopySameSize(&bits, &from[i]);
+        expected[i] = static_cast<float>(from[i]);
+      }
+
+      HWY_ASSERT_VEC_EQ(df32, expected.get(),
+                        DemoteTo(df32, Load(from_d, from.get())));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllDemoteUI64ToFloat() {
+#if HWY_HAVE_INTEGER64
+  const ForDemoteVectors<TestDemoteUI64ToFloat, 1> to_float;
+  to_float(int64_t());
+  to_float(uint64_t());
+#endif
+}
+
+struct TestDemoteToBF16 {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D from_d) {
+    // For floats, we clamp differently and cannot call LimitsMin.
+    static_assert(IsSame<T, float>(),
+                  "TestDemoteToBF16 can only be called if T is float");
+    const Rebind<bfloat16_t, D> to_d;
+    const Rebind<uint32_t, D> du32;
+    const Rebind<uint16_t, D> du16;
+
+    const size_t N = Lanes(from_d);
+    auto from = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<bfloat16_t>(N);
+    HWY_ASSERT(from && expected);
+
+    const auto u16_zero_vect = Zero(du16);
+    const auto u16_one_vect = Set(du16, 1);
+
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        from[i] = RandomFiniteValue<T>(&rng);
+
+        uint32_t fromBits;
+        CopyBytes<sizeof(uint32_t)>(&from[i], &fromBits);
+
+        uint16_t bf16Bits = static_cast<uint16_t>(fromBits >> 16);
+        CopyBytes<sizeof(uint16_t)>(&bf16Bits, &expected[i]);
+      }
+
+      const auto in = Load(from_d, from.get());
+      const auto actual = DemoteTo(to_d, in);
+
+      // Adjust expected to account for any possible rounding that was
+      // carried out by the DemoteTo operation
+      auto expected_vect = BitCast(du16, Load(to_d, expected.get()));
+
+      const auto low_f32_bits = TruncateTo(du16, BitCast(du32, in));
+
+      // max_diff_from_expected is equal to (low_f32_bits == 0 ? 0 : 1)
+      const auto max_diff_from_expected =
+          Add(VecFromMask(du16, Eq(low_f32_bits, u16_zero_vect)), u16_one_vect);
+
+      // expected_adj is equal to (actual_bits - expected_bits == 1 &&
+      // max_diff_from_expected != 0) ? 1 : 0, where actual_bits is the bits of
+      // actual and expected_bits is the bits of expected.
+      auto expected_adj =
+          And(max_diff_from_expected,
+              VecFromMask(du16, Eq(Sub(BitCast(du16, actual), expected_vect),
+                                   u16_one_vect)));
+
+      // Increment expected_vect by expected_adj
+      expected_vect = Add(expected_vect, expected_adj);
+
+      // Store the adjusted expected_vect back into expected
+      Store(BitCast(to_d, expected_vect), to_d, expected.get());
+
+      HWY_ASSERT_VEC_EQ(to_d, expected.get(), actual);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllDemoteToBF16() {
+  const ForDemoteVectors<TestDemoteToBF16, 1> to_bf16;
+  to_bf16(float());
+}
+
+template <class D>
+AlignedFreeUniquePtr<float[]> ReorderBF16TestCases(D d, size_t& padded) {
+  const float test_cases[] = {
+      // Same as BF16TestCases:
+      // +/- 1
+      1.0f,
+      -1.0f,
+      // +/- 0
+      0.0f,
+      -0.0f,
+      // near 0
+      0.25f,
+      -0.25f,
+      // +/- integer
+      4.0f,
+      -32.0f,
+      // positive +/- delta
+      2.015625f,
+      3.984375f,
+      // negative +/- delta
+      -2.015625f,
+      -3.984375f,
+
+      // No huge values - would interfere with sum. But add more to fill 2 * N:
+      -2.0f,
+      -10.0f,
+      0.03125f,
+      1.03125f,
+      1.5f,
+      2.0f,
+      4.0f,
+      5.0f,
+      6.0f,
+      8.0f,
+      10.0f,
+      256.0f,
+      448.0f,
+      2080.0f,
+  };
+  const size_t kNumTestCases = sizeof(test_cases) / sizeof(test_cases[0]);
+  const size_t N = Lanes(d);
+  padded = RoundUpTo(kNumTestCases, 2 * N);  // allow loading pairs of vectors
+  auto in = AllocateAligned<float>(padded);
+  auto expected = AllocateAligned<float>(padded);
+  HWY_ASSERT(in && expected);
+  CopyBytes(test_cases, in.get(), kNumTestCases * sizeof(float));
+  ZeroBytes(in.get() + kNumTestCases, (padded - kNumTestCases) * sizeof(float));
+  return in;
+}
+
+class TestReorderDemote2To {
+  // In-place N^2 selection sort to avoid dependencies
+  void Sort(float* p, size_t count) {
+    for (size_t i = 0; i < count - 1; ++i) {
+      // Find min_element
+      size_t idx_min = i;
+      for (size_t j = i + 1; j < count; j++) {
+        if (p[j] < p[idx_min]) {
+          idx_min = j;
+        }
+      }
+
+      // Swap with current
+      const float tmp = p[i];
+      p[i] = p[idx_min];
+      p[idx_min] = tmp;
+    }
+  }
+
+ public:
+  template <typename TF32, class DF32>
+  HWY_NOINLINE void operator()(TF32 /*t*/, DF32 d32) {
+#if HWY_TARGET != HWY_SCALAR
+    size_t padded;
+    auto in = ReorderBF16TestCases(d32, padded);
+
+    using TBF16 = bfloat16_t;
+    const Repartition<TBF16, DF32> dbf16;
+    const Half<decltype(dbf16)> dbf16_half;
+    const size_t N = Lanes(d32);
+    auto temp16 = AllocateAligned<TBF16>(2 * N);
+    auto expected = AllocateAligned<float>(2 * N);
+    auto actual = AllocateAligned<float>(2 * N);
+    HWY_ASSERT(temp16 && expected && actual);
+
+    for (size_t i = 0; i < padded; i += 2 * N) {
+      const auto f0 = Load(d32, &in[i + 0]);
+      const auto f1 = Load(d32, &in[i + N]);
+      const auto v16 = ReorderDemote2To(dbf16, f0, f1);
+      Store(v16, dbf16, temp16.get());
+      const auto promoted0 = PromoteTo(d32, Load(dbf16_half, temp16.get() + 0));
+      const auto promoted1 = PromoteTo(d32, Load(dbf16_half, temp16.get() + N));
+
+      // Smoke test: sum should be same (with tolerance for non-associativity)
+      const auto sum_expected = ReduceSum(d32, Add(f0, f1));
+      const auto sum_actual = ReduceSum(d32, Add(promoted0, promoted1));
+
+      HWY_ASSERT(sum_expected - 1E-4 <= sum_actual &&
+                 sum_actual <= sum_expected + 1E-4);
+
+      // Ensure values are the same after sorting to undo the Reorder
+      Store(f0, d32, expected.get() + 0);
+      Store(f1, d32, expected.get() + N);
+      Store(promoted0, d32, actual.get() + 0);
+      Store(promoted1, d32, actual.get() + N);
+      Sort(expected.get(), 2 * N);
+      Sort(actual.get(), 2 * N);
+      HWY_ASSERT_VEC_EQ(d32, expected.get() + 0, Load(d32, actual.get() + 0));
+      HWY_ASSERT_VEC_EQ(d32, expected.get() + N, Load(d32, actual.get() + N));
+    }
+#else  // HWY_SCALAR
+    (void)d32;
+#endif
+  }
+};
+
+class TestIntegerReorderDemote2To {
+#if HWY_TARGET != HWY_SCALAR
+
+ private:
+  // In-place N^2 selection sort to avoid dependencies
+  template <class T>
+  static void Sort(T* p, size_t count) {
+    for (size_t i = 0; i < count - 1; ++i) {
+      // Find min_element
+      size_t idx_min = i;
+      for (size_t j = i + 1; j < count; j++) {
+        if (p[j] < p[idx_min]) {
+          idx_min = j;
+        }
+      }
+
+      // Swap with current
+      const T tmp = p[i];
+      p[i] = p[idx_min];
+      p[idx_min] = tmp;
+    }
+  }
+
+  template <class T, class D, class DN>
+  static void DoIntegerReorderDemote2ToTest(DN dn, T /* t */, D d) {
+    using TN = TFromD<DN>;
+
+    const size_t N = Lanes(d);
+    const size_t twiceN = N * 2;
+    auto from = AllocateAligned<T>(twiceN);
+    auto expected = AllocateAligned<TN>(twiceN);
+    auto actual = AllocateAligned<TN>(twiceN);
+    HWY_ASSERT(from && expected && actual);
+
+    // Narrower range in the wider type, for clamping before we cast
+    const T min = ConvertScalarTo<T>(IsSigned<T>() ? LimitsMin<TN>() : TN{0});
+    const T max = LimitsMax<TN>();
+
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < twiceN; ++i) {
+        const uint64_t bits = rng();
+        CopyBytes<sizeof(T)>(&bits, &from[i]);  // not same size
+        expected[i] = static_cast<TN>(HWY_MIN(HWY_MAX(min, from[i]), max));
+      }
+
+      const auto in_1 = Load(d, from.get());
+      const auto in_2 = Load(d, from.get() + N);
+      const auto demoted_vect = ReorderDemote2To(dn, in_1, in_2);
+      Store(demoted_vect, dn, actual.get());
+      Sort(actual.get(), twiceN);
+      Sort(expected.get(), twiceN);
+      HWY_ASSERT_VEC_EQ(dn, expected.get(), Load(dn, actual.get()));
+    }
+
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < twiceN; ++i) {
+        const uint64_t bits = rng();
+        CopyBytes<sizeof(TN)>(&bits, &expected[i]);  // not same size
+        if (!IsSigned<T>() && IsSigned<TN>()) {
+          expected[i] &= static_cast<TN>(max);
+        }
+
+        from[i] = ConvertScalarTo<T>(expected[i]);
+      }
+
+      const auto in_1 = Load(d, from.get());
+      const auto in_2 = Load(d, from.get() + N);
+      const auto demoted_vect = ReorderDemote2To(dn, in_1, in_2);
+      Store(demoted_vect, dn, actual.get());
+      Sort(actual.get(), twiceN);
+      Sort(expected.get(), twiceN);
+      HWY_ASSERT_VEC_EQ(dn, expected.get(), Load(dn, actual.get()));
+    }
+  }
+#endif
+
+ public:
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*t*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    const RepartitionToNarrow<D> dn;
+    const RebindToSigned<decltype(dn)> dn_i;
+    const RebindToUnsigned<decltype(dn)> dn_u;
+
+    DoIntegerReorderDemote2ToTest(dn_i, T(), d);
+    DoIntegerReorderDemote2ToTest(dn_u, T(), d);
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllReorderDemote2To() {
+  ForUI163264(ForShrinkableVectors<TestIntegerReorderDemote2To>());
+  ForShrinkableVectors<TestReorderDemote2To>()(float());
+}
+
+struct TestFloatOrderedDemote2To {
+  template <typename TN, class DN>
+  HWY_NOINLINE void operator()(TN /*t*/, DN dn) {
+#if HWY_TARGET != HWY_SCALAR
+    const RepartitionToWide<decltype(dn)> df;
+    using TF = TFromD<decltype(df)>;
+    const RebindToUnsigned<decltype(dn)> du16;
+    const RebindToUnsigned<decltype(df)> du32;
+    const Half<decltype(du16)> du16_half;
+    const size_t N = Lanes(df);
+    const size_t twiceN = N * 2;
+    auto from = AllocateAligned<TF>(twiceN);
+    auto expected = AllocateAligned<TN>(twiceN);
+    HWY_ASSERT(from && expected);
+
+    const auto u16_zero_vect = Zero(du16);
+    const auto u16_one_vect = Set(du16, 1);
+
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < twiceN; ++i) {
+        from[i] = RandomFiniteValue<TF>(&rng);
+
+        uint32_t u32Bits;
+        CopyBytes<sizeof(uint32_t)>(&from[i], &u32Bits);
+
+        const uint16_t expected_bf16_bits =
+            static_cast<uint16_t>(u32Bits >> 16);
+
+        CopyBytes<sizeof(TN)>(&expected_bf16_bits, &expected[i]);
+      }
+
+      const auto in_1 = Load(df, from.get());
+      const auto in_2 = Load(df, from.get() + N);
+      const auto actual = OrderedDemote2To(dn, in_1, in_2);
+
+      // Adjust expected to account for any possible rounding that was
+      // carried out by the OrderedDemote2To operation
+      auto expected_vect = BitCast(du16, Load(dn, expected.get()));
+
+      const auto low_f32_bits =
+          Combine(du16, TruncateTo(du16_half, BitCast(du32, in_2)),
+                  TruncateTo(du16_half, BitCast(du32, in_1)));
+      // max_diff_from_expected is equal to (low_f32_bits == 0 ? 0 : 1)
+      const auto max_diff_from_expected =
+          Add(VecFromMask(du16, Eq(low_f32_bits, u16_zero_vect)), u16_one_vect);
+
+      // expected_adj is equal to (actual_bits - expected_bits == 1 &&
+      // max_diff_from_expected != 0) ? 1 : 0, where actual_bits is the bits of
+      // actual and expected_bits is the bits of expected.
+      auto expected_adj =
+          And(max_diff_from_expected,
+              VecFromMask(du16, Eq(Sub(BitCast(du16, actual), expected_vect),
+                                   u16_one_vect)));
+
+      // Increment expected_vect by expected_adj
+      expected_vect = Add(expected_vect, expected_adj);
+
+      // Store the adjusted expected_vect back into expected
+      Store(BitCast(dn, expected_vect), dn, expected.get());
+      HWY_ASSERT_VEC_EQ(dn, expected.get(), actual);
+    }
+#else
+    (void)dn;
+#endif
+  }
+};
+
+class TestIntegerOrderedDemote2To {
+#if HWY_TARGET != HWY_SCALAR
+
+ private:
+  template <class T, class D, class DN>
+  static void DoIntegerOrderedDemote2ToTest(DN dn, T /*t*/, D d) {
+    using TN = TFromD<DN>;
+
+    const size_t N = Lanes(d);
+    const size_t twiceN = N * 2;
+    auto from = AllocateAligned<T>(twiceN);
+    auto expected = AllocateAligned<TN>(twiceN);
+    HWY_ASSERT(from && expected);
+
+    // Narrower range in the wider type, for clamping before we cast
+    const T min = ConvertScalarTo<T>(IsSigned<T>() ? LimitsMin<TN>() : TN{0});
+    const T max = LimitsMax<TN>();
+
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < twiceN; ++i) {
+        const uint64_t bits = rng();
+        CopyBytes<sizeof(T)>(&bits, &from[i]);  // not same size
+        expected[i] = static_cast<TN>(HWY_MIN(HWY_MAX(min, from[i]), max));
+      }
+
+      const auto in_1 = Load(d, from.get());
+      const auto in_2 = Load(d, from.get() + N);
+      const auto actual = OrderedDemote2To(dn, in_1, in_2);
+      HWY_ASSERT_VEC_EQ(dn, expected.get(), actual);
+    }
+
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < twiceN; ++i) {
+        const uint64_t bits = rng();
+        CopyBytes<sizeof(TN)>(&bits, &expected[i]);  // not same size
+        if (!IsSigned<T>() && IsSigned<TN>()) {
+          expected[i] &= static_cast<TN>(max);
+        }
+
+        from[i] = ConvertScalarTo<T>(expected[i]);
+      }
+
+      const auto in_1 = Load(d, from.get());
+      const auto in_2 = Load(d, from.get() + N);
+      const auto actual = OrderedDemote2To(dn, in_1, in_2);
+      HWY_ASSERT_VEC_EQ(dn, expected.get(), actual);
+    }
+  }
+#endif
+
+ public:
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*t*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    const RepartitionToNarrow<D> dn;
+    const RebindToSigned<decltype(dn)> dn_i;
+    const RebindToUnsigned<decltype(dn)> dn_u;
+
+    DoIntegerOrderedDemote2ToTest(dn_i, T(), d);
+    DoIntegerOrderedDemote2ToTest(dn_u, T(), d);
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllOrderedDemote2To() {
+  ForUI163264(ForShrinkableVectors<TestIntegerOrderedDemote2To>());
+  ForShrinkableVectors<TestFloatOrderedDemote2To>()(bfloat16_t());
+  // TODO(janwas): replace previous line with this once supported
+  // ForSpecialTypes(ForShrinkableVectors<TestFloatOrderedDemote2To>());
+}
+
+struct TestI32F64 {
+  template <typename TF, class DF>
+  HWY_NOINLINE void operator()(TF /*unused*/, const DF df) {
+    using TI = int32_t;
+    const Rebind<TI, DF> di;
+    const size_t N = Lanes(df);
+
+    // Integer positive
+    HWY_ASSERT_VEC_EQ(di, Iota(di, 4), DemoteTo(di, Iota(df, 4.0)));
+
+    // Integer negative
+    HWY_ASSERT_VEC_EQ(di, Iota(di, -static_cast<TI>(N)),
+                      DemoteTo(di, Iota(df, -ConvertScalarTo<TF>(N))));
+
+    // Above positive
+    HWY_ASSERT_VEC_EQ(di, Iota(di, 2), DemoteTo(di, Iota(df, 2.001)));
+
+    // Below positive
+    HWY_ASSERT_VEC_EQ(di, Iota(di, 3), DemoteTo(di, Iota(df, 3.9999)));
+
+    const TF eps = static_cast<TF>(0.0001);
+    // Above negative
+    HWY_ASSERT_VEC_EQ(
+        di, Iota(di, -static_cast<TI>(N)),
+        DemoteTo(di, Iota(df, -ConvertScalarTo<TF>(N + 1) + eps)));
+
+    // Below negative
+    HWY_ASSERT_VEC_EQ(
+        di, Iota(di, -static_cast<TI>(N + 1)),
+        DemoteTo(di, Iota(df, -ConvertScalarTo<TF>(N + 1) - eps)));
+
+    // Huge positive float
+    HWY_ASSERT_VEC_EQ(di, Set(di, LimitsMax<TI>()),
+                      DemoteTo(di, Set(df, TF(1E12))));
+
+    // Huge negative float
+    HWY_ASSERT_VEC_EQ(di, Set(di, LimitsMin<TI>()),
+                      DemoteTo(di, Set(df, TF(-1E12))));
+  }
+};
+
+HWY_NOINLINE void TestAllI32F64() {
+#if HWY_HAVE_FLOAT64
+  ForDemoteVectors<TestI32F64>()(double());
+#endif
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#endif  //  !HWY_IS_MSAN
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+#if !HWY_IS_MSAN
+HWY_BEFORE_TEST(HwyDemoteTest);
+HWY_EXPORT_AND_TEST_P(HwyDemoteTest, TestAllDemoteToInt);
+HWY_EXPORT_AND_TEST_P(HwyDemoteTest, TestAllDemoteToMixed);
+HWY_EXPORT_AND_TEST_P(HwyDemoteTest, TestAllDemoteToFloat);
+HWY_EXPORT_AND_TEST_P(HwyDemoteTest, TestAllDemoteUI64ToFloat);
+HWY_EXPORT_AND_TEST_P(HwyDemoteTest, TestAllDemoteToBF16);
+HWY_EXPORT_AND_TEST_P(HwyDemoteTest, TestAllReorderDemote2To);
+HWY_EXPORT_AND_TEST_P(HwyDemoteTest, TestAllOrderedDemote2To);
+HWY_EXPORT_AND_TEST_P(HwyDemoteTest, TestAllI32F64);
+HWY_AFTER_TEST();
+#endif  //  !HWY_IS_MSAN
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/div_test.cc

@@ -0,0 +1,281 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/div_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/nanobenchmark.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestIntegerDiv {
+  template <class D, typename T = TFromD<D>>
+  static HWY_NOINLINE void DoDiv(D d, const T* HWY_RESTRICT a_lanes,
+                                 const T* HWY_RESTRICT b_lanes, bool neg_a,
+                                 bool neg_b) {
+    const RebindToSigned<decltype(d)> di;
+    using TI = TFromD<decltype(di)>;
+    const size_t N = Lanes(d);
+    using V = VFromD<D>;
+    auto expected = AllocateAligned<T>(N);
+    auto expected_even = AllocateAligned<T>(N);
+    auto expected_odd = AllocateAligned<T>(N);
+    HWY_ASSERT(expected && expected_even && expected_odd);
+
+    V a = Load(d, a_lanes);
+    V b = Load(d, b_lanes);
+    if (neg_a) a = BitCast(d, Neg(BitCast(di, a)));
+    if (neg_b) b = BitCast(d, Neg(BitCast(di, b)));
+
+    for (size_t i = 0; i < N; i++) {
+      const T a1 =
+          neg_a ? static_cast<T>(-static_cast<TI>(a_lanes[i])) : a_lanes[i];
+      const T b1 =
+          neg_b ? static_cast<T>(-static_cast<TI>(b_lanes[i])) : b_lanes[i];
+      HWY_ASSERT(b1 != 0);
+      expected[i] = static_cast<T>(a1 / b1);
+      if ((i & 1) == 0) {
+        expected_even[i] = expected[i];
+        expected_odd[i] = static_cast<T>(0);
+      } else {
+        expected_even[i] = static_cast<T>(0);
+        expected_odd[i] = expected[i];
+      }
+    }
+
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Div(a, b));
+
+    const V vmin = Set(d, LimitsMin<T>());
+    const V zero = Zero(d);
+    const V all_ones = Set(d, static_cast<T>(-1));
+
+    HWY_ASSERT_VEC_EQ(d, expected_even.get(),
+                      OddEven(zero, Div(a, OddEven(zero, b))));
+    HWY_ASSERT_VEC_EQ(d, expected_odd.get(),
+                      OddEven(Div(a, OddEven(b, zero)), zero));
+
+    HWY_ASSERT_VEC_EQ(
+        d, expected_even.get(),
+        OddEven(zero, Div(OddEven(vmin, a), OddEven(all_ones, b))));
+    HWY_ASSERT_VEC_EQ(
+        d, expected_odd.get(),
+        OddEven(Div(OddEven(a, vmin), OddEven(b, all_ones)), zero));
+  }
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TU = MakeUnsigned<T>;
+    using TI = MakeSigned<T>;
+    using V = VFromD<D>;
+
+    const size_t N = Lanes(d);
+
+#if HWY_TARGET <= HWY_AVX3 && HWY_IS_MSAN
+    // Workaround for MSAN bug on AVX3
+    if (sizeof(T) <= 2 && N >= 16) return;
+#endif
+
+#if HWY_COMPILER_CLANG && HWY_ARCH_RISCV && HWY_TARGET == HWY_EMU128
+    // Workaround for incorrect codegen. The implementation splits vectors
+    // into halves and then combines them; the upper half is incorrect.
+    if (sizeof(T) == 4 && N == 4) return;
+#endif
+
+    const T k1 = static_cast<T>(Unpredictable1());
+    const T kMin = static_cast<T>(LimitsMin<T>() * k1);
+    const T kMax = static_cast<T>(LimitsMax<T>() * k1);
+    const V vmin = Set(d, kMin);
+    const V vmax = Set(d, kMax);
+    const V v1 = Set(d, static_cast<T>(k1));
+    const V v2 = Set(d, static_cast<T>(k1 + 1));
+    const V v3 = Set(d, static_cast<T>(k1 + 2));
+
+    HWY_ASSERT_VEC_EQ(d, vmin, Div(vmin, v1));
+    HWY_ASSERT_VEC_EQ(d, vmax, Div(vmax, v1));
+    HWY_ASSERT_VEC_EQ(d, Set(d, static_cast<T>(kMin / 2)), Div(vmin, v2));
+    HWY_ASSERT_VEC_EQ(d, Set(d, static_cast<T>(kMin / 3)), Div(vmin, v3));
+    HWY_ASSERT_VEC_EQ(d, Set(d, static_cast<T>(kMax / 2)), Div(vmax, v2));
+    HWY_ASSERT_VEC_EQ(d, Set(d, static_cast<T>(kMax / 3)), Div(vmax, v3));
+
+    auto in1 = AllocateAligned<T>(N);
+    auto in2 = AllocateAligned<T>(N);
+    HWY_ASSERT(in1 && in2);
+
+    // Random inputs in each lane
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        const T rnd_a0 = static_cast<T>(Random64(&rng) &
+                                        static_cast<uint64_t>(LimitsMax<TU>()));
+        const T rnd_b0 = static_cast<T>(Random64(&rng) &
+                                        static_cast<uint64_t>(LimitsMax<TI>()));
+
+        const T rnd_b = static_cast<T>(rnd_b0 | static_cast<T>(rnd_b0 == 0));
+        const T rnd_a = static_cast<T>(
+            rnd_a0 + static_cast<T>(IsSigned<T>() && rnd_a0 == LimitsMin<T>() &&
+                                    ScalarAbs(rnd_b) == static_cast<T>(1)));
+
+        in1[i] = rnd_a;
+        in2[i] = rnd_b;
+      }
+
+      const bool neg_a = true;
+      const bool neg_b = true;
+      DoDiv(d, in1.get(), in2.get(), false, false);
+      DoDiv(d, in1.get(), in2.get(), false, neg_b);
+      DoDiv(d, in1.get(), in2.get(), neg_a, false);
+      DoDiv(d, in1.get(), in2.get(), neg_a, neg_b);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllIntegerDiv() {
+  ForIntegerTypes(ForPartialVectors<TestIntegerDiv>());
+}
+
+struct TestIntegerMod {
+  template <class D>
+  static HWY_NOINLINE void DoTestIntegerMod(D d, const VecArg<VFromD<D>> a,
+                                            const VecArg<VFromD<D>> b) {
+    using T = TFromD<D>;
+
+    const size_t N = Lanes(d);
+
+#if HWY_TARGET <= HWY_AVX3 && HWY_IS_MSAN
+    // Workaround for MSAN bug on AVX3
+    if (sizeof(T) <= 2 && N >= 16) {
+      return;
+    }
+#endif
+
+#if HWY_COMPILER_CLANG && HWY_ARCH_RISCV && HWY_TARGET == HWY_EMU128
+    // Workaround for incorrect codegen. The implementation splits vectors
+    // into halves and then combines them; the lower half is incorrect.
+    if (sizeof(T) == 4 && N == 4) return;
+#endif
+
+    auto a_lanes = AllocateAligned<T>(N);
+    auto b_lanes = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    auto expected_even = AllocateAligned<T>(N);
+    auto expected_odd = AllocateAligned<T>(N);
+    HWY_ASSERT(a_lanes && b_lanes && expected && expected_even && expected_odd);
+
+    Store(a, d, a_lanes.get());
+    Store(b, d, b_lanes.get());
+
+    for (size_t i = 0; i < N; i++) {
+      expected[i] = static_cast<T>(a_lanes[i] % b_lanes[i]);
+      if ((i & 1) == 0) {
+        expected_even[i] = expected[i];
+        expected_odd[i] = static_cast<T>(0);
+      } else {
+        expected_even[i] = static_cast<T>(0);
+        expected_odd[i] = expected[i];
+      }
+    }
+
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Mod(a, b));
+
+    const auto vmin = Set(d, LimitsMin<T>());
+    const auto zero = Zero(d);
+    const auto all_ones = Set(d, static_cast<T>(-1));
+
+    HWY_ASSERT_VEC_EQ(d, expected_even.get(),
+                      OddEven(zero, Mod(a, OddEven(zero, b))));
+    HWY_ASSERT_VEC_EQ(d, expected_odd.get(),
+                      OddEven(Mod(a, OddEven(b, zero)), zero));
+
+    HWY_ASSERT_VEC_EQ(
+        d, expected_even.get(),
+        OddEven(zero, Mod(OddEven(vmin, a), OddEven(all_ones, b))));
+    HWY_ASSERT_VEC_EQ(
+        d, expected_odd.get(),
+        OddEven(Mod(OddEven(a, vmin), OddEven(b, all_ones)), zero));
+  }
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TU = MakeUnsigned<T>;
+    using TI = MakeSigned<T>;
+
+    const size_t N = Lanes(d);
+    auto in1 = AllocateAligned<T>(N);
+    auto in2 = AllocateAligned<T>(N);
+    HWY_ASSERT(in1 && in2);
+
+    const RebindToSigned<decltype(d)> di;
+
+    // Random inputs in each lane
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        const T rnd_a0 = static_cast<T>(Random64(&rng) &
+                                        static_cast<uint64_t>(LimitsMax<TU>()));
+        const T rnd_b0 = static_cast<T>(Random64(&rng) &
+                                        static_cast<uint64_t>(LimitsMax<TI>()));
+
+        const T rnd_b = static_cast<T>(rnd_b0 | static_cast<T>(rnd_b0 == 0));
+        const T rnd_a = static_cast<T>(
+            rnd_a0 + static_cast<T>(IsSigned<T>() && rnd_a0 == LimitsMin<T>() &&
+                                    ScalarAbs(rnd_b) == static_cast<T>(1)));
+
+        in1[i] = rnd_a;
+        in2[i] = rnd_b;
+      }
+
+      const auto a = Load(d, in1.get());
+      const auto b = Load(d, in2.get());
+
+      const auto neg_a = BitCast(d, Neg(BitCast(di, a)));
+      const auto neg_b = BitCast(d, Neg(BitCast(di, b)));
+
+      DoTestIntegerMod(d, a, b);
+      DoTestIntegerMod(d, a, neg_b);
+      DoTestIntegerMod(d, neg_a, b);
+      DoTestIntegerMod(d, neg_a, neg_b);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllIntegerMod() {
+  ForIntegerTypes(ForPartialVectors<TestIntegerMod>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyDivTest);
+HWY_EXPORT_AND_TEST_P(HwyDivTest, TestAllIntegerDiv);
+HWY_EXPORT_AND_TEST_P(HwyDivTest, TestAllIntegerMod);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/dup128_vec_test.cc

@@ -0,0 +1,237 @@
+// Copyright 2023 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/dup128_vec_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestDup128VecFromValues {
+  template <class D, HWY_IF_T_SIZE_D(D, 1)>
+  static HWY_INLINE Vec<D> VecFromValues(
+      D d, TFromD<D> t0, TFromD<D> t1, TFromD<D> t2, TFromD<D> t3, TFromD<D> t4,
+      TFromD<D> t5, TFromD<D> t6, TFromD<D> t7, TFromD<D> t8, TFromD<D> t9,
+      TFromD<D> t10, TFromD<D> t11, TFromD<D> t12, TFromD<D> t13, TFromD<D> t14,
+      TFromD<D> t15) {
+    return Dup128VecFromValues(d, t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10,
+                               t11, t12, t13, t14, t15);
+  }
+  template <class D, HWY_IF_T_SIZE_D(D, 2)>
+  static HWY_INLINE Vec<D> VecFromValues(
+      D d, TFromD<D> t0, TFromD<D> t1, TFromD<D> t2, TFromD<D> t3, TFromD<D> t4,
+      TFromD<D> t5, TFromD<D> t6, TFromD<D> t7, TFromD<D> /*t8*/,
+      TFromD<D> /*t9*/, TFromD<D> /*t10*/, TFromD<D> /*t11*/, TFromD<D> /*t12*/,
+      TFromD<D> /*t13*/, TFromD<D> /*t14*/, TFromD<D> /*t15*/) {
+    return Dup128VecFromValues(d, t0, t1, t2, t3, t4, t5, t6, t7);
+  }
+  template <class D, HWY_IF_T_SIZE_D(D, 4)>
+  static HWY_INLINE Vec<D> VecFromValues(D d, TFromD<D> t0, TFromD<D> t1,
+                                         TFromD<D> t2, TFromD<D> t3,
+                                         TFromD<D> /*t4*/, TFromD<D> /*t5*/,
+                                         TFromD<D> /*t6*/, TFromD<D> /*t7*/,
+                                         TFromD<D> /*t8*/, TFromD<D> /*t9*/,
+                                         TFromD<D> /*t10*/, TFromD<D> /*t11*/,
+                                         TFromD<D> /*t12*/, TFromD<D> /*t13*/,
+                                         TFromD<D> /*t14*/, TFromD<D> /*t15*/) {
+    return Dup128VecFromValues(d, t0, t1, t2, t3);
+  }
+  template <class D, HWY_IF_T_SIZE_D(D, 8)>
+  static HWY_INLINE Vec<D> VecFromValues(D d, TFromD<D> t0, TFromD<D> t1,
+                                         TFromD<D> /*t2*/, TFromD<D> /*t3*/,
+                                         TFromD<D> /*t4*/, TFromD<D> /*t5*/,
+                                         TFromD<D> /*t6*/, TFromD<D> /*t7*/,
+                                         TFromD<D> /*t8*/, TFromD<D> /*t9*/,
+                                         TFromD<D> /*t10*/, TFromD<D> /*t11*/,
+                                         TFromD<D> /*t12*/, TFromD<D> /*t13*/,
+                                         TFromD<D> /*t14*/, TFromD<D> /*t15*/) {
+    return Dup128VecFromValues(d, t0, t1);
+  }
+
+  template <class D, class T, HWY_IF_NOT_SPECIAL_FLOAT_D(D)>
+  static HWY_INLINE TFromD<D> CastValueToLaneType(D /*d*/, T val) {
+    return static_cast<TFromD<D>>(val);
+  }
+
+  template <class D, class T, HWY_IF_BF16_D(D)>
+  static HWY_INLINE hwy::bfloat16_t CastValueToLaneType(D /*d*/, T val) {
+    return BF16FromF32(static_cast<float>(val));
+  }
+
+  template <class D, class T, HWY_IF_F16_D(D)>
+  static HWY_INLINE hwy::float16_t CastValueToLaneType(D /*d*/, T val) {
+    return F16FromF32(static_cast<float>(val));
+  }
+
+  template <class D, typename T2, HWY_IF_NOT_SPECIAL_FLOAT_D(D)>
+  static HWY_INLINE Vec<D> BlockwiseIota(D d, T2 start) {
+    return BroadcastBlock<0>(Iota(d, static_cast<TFromD<D>>(start)));
+  }
+
+  template <class D, typename T2, HWY_IF_BF16_D(D)>
+  static HWY_INLINE Vec<D> BlockwiseIota(D d, T2 start) {
+#if HWY_TARGET == HWY_SCALAR
+    return Set(d, BF16FromF32(static_cast<float>(start)));
+#else  // HWY_TARGET != HWY_SCALAR
+#if HWY_MAX_BYTES >= 32 &&                              \
+    (HWY_TARGET == HWY_RVV || HWY_TARGET <= HWY_AVX2 || \
+     HWY_TARGET == HWY_WASM_EMU256 || HWY_TARGET == HWY_SVE_256)
+
+#if HWY_TARGET == HWY_RVV
+    const ScalableTag<float, 1> df32;
+#else
+    const FixedTag<float, 8> df32;
+#endif
+    const Rebind<hwy::bfloat16_t, decltype(df32)> dbf16;
+
+    const auto vbf16_iota = DemoteTo(dbf16, Iota(df32, start));
+#else
+    const FixedTag<float, 4> df32;
+    const Repartition<hwy::bfloat16_t, decltype(df32)> dbf16;
+
+    const auto vbf16_iota = OrderedDemote2To(
+        dbf16, Iota(df32, start), Iota(df32, static_cast<float>(start) + 4.0f));
+#endif
+
+    return BroadcastBlock<0>(ResizeBitCast(d, vbf16_iota));
+#endif  // HWY_TARGET == HWY_SCALAR
+  }
+
+  template <class D, typename T2, HWY_IF_F16_D(D)>
+  static HWY_INLINE Vec<D> BlockwiseIota(D d, T2 start) {
+#if HWY_HAVE_FLOAT16
+    return BroadcastBlock<0>(Iota(d, start));
+#elif HWY_TARGET == HWY_SCALAR
+    return Set(d, F16FromF32(static_cast<float>(start)));
+#else  // !HWY_HAVE_FLOAT16 && HWY_TARGET != HWY_SCALAR
+#if HWY_MAX_BYTES >= 32 &&                              \
+    (HWY_TARGET == HWY_RVV || HWY_TARGET <= HWY_AVX2 || \
+     HWY_TARGET == HWY_WASM_EMU256 || HWY_TARGET == HWY_SVE_256)
+
+#if HWY_TARGET == HWY_RVV
+    const ScalableTag<float, 1> df32;
+#else
+    const FixedTag<float, 8> df32;
+#endif
+    const Rebind<hwy::float16_t, decltype(df32)> df16;
+
+    const auto vf16_iota = DemoteTo(df16, Iota(df32, start));
+#else
+    const FixedTag<float, 4> df32;
+    const Repartition<hwy::float16_t, decltype(df32)> df16;
+    const Half<decltype(df16)> dh_f16;
+
+    const auto vf16_iota = Combine(
+        df16, DemoteTo(dh_f16, Iota(df32, static_cast<float>(start) + 4.0f)),
+        DemoteTo(dh_f16, Iota(df32, start)));
+#endif
+
+    return BroadcastBlock<0>(ResizeBitCast(d, vf16_iota));
+#endif  // HWY_HAVE_FLOAT16
+  }
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    HWY_ASSERT_VEC_EQ(
+        d, Zero(d),
+        VecFromValues(d, CastValueToLaneType(d, 0), CastValueToLaneType(d, 0),
+                      CastValueToLaneType(d, 0), CastValueToLaneType(d, 0),
+                      CastValueToLaneType(d, 0), CastValueToLaneType(d, 0),
+                      CastValueToLaneType(d, 0), CastValueToLaneType(d, 0),
+                      CastValueToLaneType(d, 0), CastValueToLaneType(d, 0),
+                      CastValueToLaneType(d, 0), CastValueToLaneType(d, 0),
+                      CastValueToLaneType(d, 0), CastValueToLaneType(d, 0),
+                      CastValueToLaneType(d, 0), CastValueToLaneType(d, 0)));
+    HWY_ASSERT_VEC_EQ(
+        d, Set(d, CastValueToLaneType(d, 1)),
+        VecFromValues(d, CastValueToLaneType(d, 1), CastValueToLaneType(d, 1),
+                      CastValueToLaneType(d, 1), CastValueToLaneType(d, 1),
+                      CastValueToLaneType(d, 1), CastValueToLaneType(d, 1),
+                      CastValueToLaneType(d, 1), CastValueToLaneType(d, 1),
+                      CastValueToLaneType(d, 1), CastValueToLaneType(d, 1),
+                      CastValueToLaneType(d, 1), CastValueToLaneType(d, 1),
+                      CastValueToLaneType(d, 1), CastValueToLaneType(d, 1),
+                      CastValueToLaneType(d, 1), CastValueToLaneType(d, 1)));
+    HWY_ASSERT_VEC_EQ(
+        d, BlockwiseIota(d, 1),
+        VecFromValues(d, CastValueToLaneType(d, 1), CastValueToLaneType(d, 2),
+                      CastValueToLaneType(d, 3), CastValueToLaneType(d, 4),
+                      CastValueToLaneType(d, 5), CastValueToLaneType(d, 6),
+                      CastValueToLaneType(d, 7), CastValueToLaneType(d, 8),
+                      CastValueToLaneType(d, 9), CastValueToLaneType(d, 10),
+                      CastValueToLaneType(d, 11), CastValueToLaneType(d, 12),
+                      CastValueToLaneType(d, 13), CastValueToLaneType(d, 14),
+                      CastValueToLaneType(d, 15), CastValueToLaneType(d, 16)));
+    HWY_ASSERT_VEC_EQ(
+        d, BlockwiseIota(d, -16),
+        VecFromValues(d, CastValueToLaneType(d, -16),
+                      CastValueToLaneType(d, -15), CastValueToLaneType(d, -14),
+                      CastValueToLaneType(d, -13), CastValueToLaneType(d, -12),
+                      CastValueToLaneType(d, -11), CastValueToLaneType(d, -10),
+                      CastValueToLaneType(d, -9), CastValueToLaneType(d, -8),
+                      CastValueToLaneType(d, -7), CastValueToLaneType(d, -6),
+                      CastValueToLaneType(d, -5), CastValueToLaneType(d, -4),
+                      CastValueToLaneType(d, -3), CastValueToLaneType(d, -2),
+                      CastValueToLaneType(d, -1)));
+
+    RandomState rng;
+    auto rand_vals = AllocateAligned<T>(16);
+    HWY_ASSERT(rand_vals);
+
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < 16; ++i) {
+        rand_vals[i] = RandomFiniteValue<T>(&rng);
+      }
+
+      const auto expected = LoadDup128(d, rand_vals.get());
+      const auto actual = VecFromValues(
+          d, rand_vals[0], rand_vals[1], rand_vals[2], rand_vals[3],
+          rand_vals[4], rand_vals[5], rand_vals[6], rand_vals[7], rand_vals[8],
+          rand_vals[9], rand_vals[10], rand_vals[11], rand_vals[12],
+          rand_vals[13], rand_vals[14], rand_vals[15]);
+      HWY_ASSERT_VEC_EQ(d, expected, actual);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllDup128VecFromValues() {
+  const ForPartialVectors<TestDup128VecFromValues> func;
+  ForIntegerTypes(func);
+  func(hwy::float16_t());
+  func(hwy::bfloat16_t());
+  ForFloat3264Types(func);
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyDup128VecTest);
+HWY_EXPORT_AND_TEST_P(HwyDup128VecTest, TestAllDup128VecFromValues);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/expand_test.cc

@@ -0,0 +1,291 @@
+// Copyright 2022 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <array>  // IWYU pragma: keep
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/expand_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+// Regenerate tables used in the implementation, instead of testing.
+#define HWY_PRINT_TABLES 0
+
+#if !HWY_PRINT_TABLES || HWY_IDE
+
+template <class D, class DI, typename T = TFromD<D>, typename TI = TFromD<DI>>
+void CheckExpanded(D d, DI di, const char* op,
+                   const AlignedFreeUniquePtr<T[]>& in,
+                   const AlignedFreeUniquePtr<TI[]>& mask_lanes,
+                   const AlignedFreeUniquePtr<T[]>& expected, const T* actual_u,
+                   int line) {
+  const size_t N = Lanes(d);
+  // Modified from AssertVecEqual to also print mask etc.
+  for (size_t i = 0; i < N; ++i) {
+    if (!IsEqual(expected[i], actual_u[i])) {
+      fprintf(stderr, "%s: mismatch at i=%d of %d, line %d:\n\n", op,
+              static_cast<int>(i), static_cast<int>(N), line);
+      Print(di, "mask", Load(di, mask_lanes.get()), 0, N);
+      Print(d, "in", Load(d, in.get()), 0, N);
+      Print(d, "expect", Load(d, expected.get()), 0, N);
+      Print(d, "actual", Load(d, actual_u), 0, N);
+      HWY_ASSERT(false);
+    }
+  }
+}
+
+struct TestExpand {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // Used for mask > 0 comparison.
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(d);
+    const size_t bits_size = RoundUpTo((N + 7) / 8, 8);
+
+    for (int frac : {0, 2, 3}) {
+      // For LoadExpand
+      const size_t misalign = static_cast<size_t>(frac) * N / 4;
+
+      auto in_lanes = AllocateAligned<T>(N);
+      auto mask_lanes = AllocateAligned<TI>(N);
+      auto expected = AllocateAligned<T>(N);
+      auto actual_a = AllocateAligned<T>(misalign + N);
+      auto bits = AllocateAligned<uint8_t>(bits_size);
+      HWY_ASSERT(in_lanes && mask_lanes && expected && actual_a && bits);
+
+      T* actual_u = actual_a.get() + misalign;
+      ZeroBytes(bits.get(), bits_size);  // Prevents MSAN error.
+
+      // Random input vector, used in all iterations.
+      for (size_t i = 0; i < N; ++i) {
+        in_lanes[i] = RandomFiniteValue<T>(&rng);
+      }
+
+      // Each lane should have a chance of having mask=true.
+      for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+        size_t in_pos = 0;
+        for (size_t i = 0; i < N; ++i) {
+          mask_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+          if (mask_lanes[i] > 0) {
+            expected[i] = in_lanes[in_pos++];
+          } else {
+            expected[i] = ConvertScalarTo<T>(0);
+          }
+        }
+
+        const auto in = Load(d, in_lanes.get());
+        const auto mask =
+            RebindMask(d, Gt(Load(di, mask_lanes.get()), Zero(di)));
+        StoreMaskBits(d, mask, bits.get());
+
+        // Expand
+        ZeroBytes(actual_u, N * sizeof(T));
+        StoreU(Expand(in, mask), d, actual_u);
+        CheckExpanded(d, di, "Expand", in_lanes, mask_lanes, expected, actual_u,
+                      __LINE__);
+
+        // LoadExpand
+        ZeroBytes(actual_u, N * sizeof(T));
+        StoreU(LoadExpand(mask, d, in_lanes.get()), d, actual_u);
+        CheckExpanded(d, di, "LoadExpand", in_lanes, mask_lanes, expected,
+                      actual_u, __LINE__);
+      }  // rep
+    }    // frac
+  }      // operator()
+};
+
+HWY_NOINLINE void TestAllExpand() {
+  ForAllTypes(ForPartialVectors<TestExpand>());
+}
+
+#endif  // !HWY_PRINT_TABLES
+
+#if HWY_PRINT_TABLES || HWY_IDE
+void PrintExpand8x8Tables() {
+  printf("// %s\n", __FUNCTION__);
+  constexpr size_t N = 8;
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    std::array<uint8_t, N> indices{0};
+    size_t pos = 0;
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[i] = pos++;
+      } else {
+        indices[i] = 0x80;  // Output of TableLookupBytes will be zero.
+      }
+    }
+    HWY_ASSERT(pos == PopCount(code));
+
+    for (size_t i = 0; i < N; ++i) {
+      printf("%d,", indices[i]);
+    }
+    printf("//\n");
+  }
+  printf("\n");
+}
+
+// For SVE
+void PrintExpand16x8LaneTables() {
+  printf("// %s\n", __FUNCTION__);
+  constexpr size_t N = 8;  // (128-bit SIMD)
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    std::array<uint8_t, N> indices{0};
+    size_t pos = 0;
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[i] = pos++;
+      } else {
+        indices[i] = 0xFF;  // This is out of bounds for SVE.
+      }
+    }
+    HWY_ASSERT(pos == PopCount(code));
+
+    for (size_t i = 0; i < N; ++i) {
+      printf("%d,", indices[i]);
+    }
+    printf("//\n");
+  }
+  printf("\n");
+}
+
+void PrintExpand16x8ByteTables() {
+  printf("// %s\n", __FUNCTION__);
+  constexpr size_t N = 8;  // 128-bit SIMD
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    std::array<uint8_t, N> indices{0};
+    size_t pos = 0;
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[i] = pos++;
+      } else {
+        indices[i] = 64;  // The output of TableLookupBytesOr0 will be zero.
+      }
+    }
+    HWY_ASSERT(pos == PopCount(code));
+
+    // Doubled (for converting lane to byte indices)
+    for (size_t i = 0; i < N; ++i) {
+      printf("%d,", 2 * indices[i]);
+    }
+    printf("//\n");
+  }
+  printf("\n");
+}
+
+// Compressed to nibbles, unpacked via variable right shift. MSB indicates the
+// output should be zero (which AVX2 permutevar8x32 cannot do by itself).
+void PrintExpand32x8NibbleTables() {
+  printf("// %s\n", __FUNCTION__);
+  constexpr size_t N = 8;  // (AVX2 or 64-bit AVX3)
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    std::array<uint32_t, N> indices{0};
+    size_t pos = 0;
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[i] = pos++;
+      } else {
+        indices[i] = 0xF;
+      }
+    }
+    HWY_ASSERT(pos == PopCount(code));
+
+    // Convert to nibbles.
+    uint64_t packed = 0;
+    for (size_t i = 0; i < N; ++i) {
+      HWY_ASSERT(indices[i] < N || indices[i] == 0xF);
+      packed += indices[i] << (i * 4);
+    }
+
+    HWY_ASSERT(packed < (1ull << (N * 4)));
+    printf("0x%08x,", static_cast<uint32_t>(packed));
+  }
+  printf("\n");
+}
+
+// Compressed to nibbles, MSB set if output should be zero.
+void PrintExpand64x4NibbleTables() {
+  printf("// %s\n", __FUNCTION__);
+  constexpr size_t N = 4;  // AVX2
+  for (uint64_t code = 0; code < (1ull << N); ++code) {
+    std::array<uint32_t, N> indices{0};
+    size_t pos = 0;
+    for (size_t i = 0; i < N; ++i) {
+      if (code & (1ull << i)) {
+        indices[i] = pos++;
+      } else {
+        indices[i] = 0xF;
+      }
+    }
+    HWY_ASSERT(pos == PopCount(code));
+
+    // Convert to nibbles
+    uint64_t packed = 0;
+    for (size_t i = 0; i < N; ++i) {
+      HWY_ASSERT(indices[i] < N || indices[i] == 0xF);
+      packed += indices[i] << (i * 4);
+    }
+
+    HWY_ASSERT(packed < (1ull << (N * 4)));
+    printf("0x%08x,", static_cast<uint32_t>(packed));
+  }
+  printf("\n");
+}
+
+HWY_NOINLINE void PrintTables() {
+  // Only print once.
+#if HWY_TARGET == HWY_STATIC_TARGET
+  PrintExpand32x8NibbleTables();
+  PrintExpand64x4NibbleTables();
+  PrintExpand16x8LaneTables();
+  PrintExpand16x8ByteTables();
+  PrintExpand8x8Tables();
+#endif
+}
+
+#endif  // HWY_PRINT_TABLES
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyExpandTest);
+#if HWY_PRINT_TABLES
+// Only print instead of running tests; this will be visible in the log.
+HWY_EXPORT_AND_TEST_P(HwyExpandTest, PrintTables);
+#else
+HWY_EXPORT_AND_TEST_P(HwyExpandTest, TestAllExpand);
+#endif
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/float_test.cc

@@ -0,0 +1,536 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Tests some ops specific to floating-point types (Div, Round etc.)
+
+#include <stdio.h>
+
+#include <cmath>  // std::ceil, std::floor
+
+#include "hwy/base.h"
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/float_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+HWY_NOINLINE void TestAllF16FromF32() {
+  const FixedTag<float, 1> d1;
+
+  // +/- 0
+  HWY_ASSERT_EQ(0, BitCastScalar<uint16_t>(hwy::F16FromF32(0.0f)));
+  HWY_ASSERT_EQ(0x8000, BitCastScalar<uint16_t>(hwy::F16FromF32(-0.0f)));
+  // smallest f32 subnormal
+  HWY_ASSERT_EQ(0,
+                BitCastScalar<uint16_t>(hwy::F16FromF32(5.87747175411E-39f)));
+  HWY_ASSERT_EQ(0x8000,
+                BitCastScalar<uint16_t>(hwy::F16FromF32(-5.87747175411E-39f)));
+  // largest f16 subnormal
+  HWY_ASSERT_EQ(0x3FF, BitCastScalar<uint16_t>(hwy::F16FromF32(6.0975552E-5f)));
+  HWY_ASSERT_EQ(0x83FF,
+                BitCastScalar<uint16_t>(hwy::F16FromF32(-6.0975552E-5f)));
+  // smallest normalized f16
+  HWY_ASSERT_EQ(0x400,
+                BitCastScalar<uint16_t>(hwy::F16FromF32(6.103515625E-5f)));
+  HWY_ASSERT_EQ(0x8400,
+                BitCastScalar<uint16_t>(hwy::F16FromF32(-6.103515625E-5f)));
+
+  // rounding to nearest even
+  HWY_ASSERT_EQ((15 << 10) + 0,  // round down to even: 0[10..0] => 0
+                BitCastScalar<uint16_t>(hwy::F16FromF32(1.00048828125f)));
+  HWY_ASSERT_EQ((15 << 10) + 1,  // round up: 0[1..1] => 1
+                BitCastScalar<uint16_t>(hwy::F16FromF32(1.00097644329f)));
+  HWY_ASSERT_EQ((15 << 10) + 2,  // round up to even: 1[10..0] => 10
+                BitCastScalar<uint16_t>(hwy::F16FromF32(1.00146484375f)));
+
+  // greater than f16 max => inf
+  HWY_ASSERT_EQ(0x7C00, BitCastScalar<uint16_t>(hwy::F16FromF32(7E4f)));
+  HWY_ASSERT_EQ(0xFC00, BitCastScalar<uint16_t>(hwy::F16FromF32(-7E4f)));
+  // infinity
+  HWY_ASSERT_EQ(0x7C00,
+                BitCastScalar<uint16_t>(hwy::F16FromF32(GetLane(Inf(d1)))));
+  HWY_ASSERT_EQ(0xFC00,
+                BitCastScalar<uint16_t>(hwy::F16FromF32(-GetLane(Inf(d1)))));
+  // NaN
+  HWY_ASSERT_EQ(0x7FFF,
+                BitCastScalar<uint16_t>(hwy::F16FromF32(GetLane(NaN(d1)))));
+  HWY_ASSERT_EQ(0xFFFF,
+                BitCastScalar<uint16_t>(hwy::F16FromF32(-GetLane(NaN(d1)))));
+}
+
+HWY_NOINLINE void TestAllF32FromF16() {
+  const FixedTag<float, 1> d1;
+
+  // +/- 0
+  HWY_ASSERT_EQ(0.0f, hwy::F32FromF16(BitCastScalar<float16_t>(uint16_t{0})));
+  HWY_ASSERT_EQ(-0.0f,
+                hwy::F32FromF16(BitCastScalar<float16_t>(uint16_t{0x8000})));
+  // largest f16 subnormal
+  HWY_ASSERT_EQ(6.0975552E-5f,
+                hwy::F32FromF16(BitCastScalar<float16_t>(uint16_t{0x3FF})));
+  HWY_ASSERT_EQ(-6.0975552E-5f,
+                hwy::F32FromF16(BitCastScalar<float16_t>(uint16_t{0x83FF})));
+  // smallest normalized f16
+  HWY_ASSERT_EQ(6.103515625E-5f,
+                hwy::F32FromF16(BitCastScalar<float16_t>(uint16_t{0x400})));
+  HWY_ASSERT_EQ(-6.103515625E-5f,
+                hwy::F32FromF16(BitCastScalar<float16_t>(uint16_t{0x8400})));
+  // infinity
+  HWY_ASSERT_EQ(GetLane(Inf(d1)),
+                hwy::F32FromF16(BitCastScalar<float16_t>(uint16_t{0x7C00})));
+  HWY_ASSERT_EQ(-GetLane(Inf(d1)),
+                hwy::F32FromF16(BitCastScalar<float16_t>(uint16_t{0xFC00})));
+  // NaN
+  HWY_ASSERT_EQ(GetLane(NaN(d1)),
+                hwy::F32FromF16(BitCastScalar<float16_t>(uint16_t{0x7FFF})));
+  HWY_ASSERT_EQ(-GetLane(NaN(d1)),
+                hwy::F32FromF16(BitCastScalar<float16_t>(uint16_t{0xFFFF})));
+}
+
+struct TestDiv {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v = Iota(d, -2);
+    const auto v1 = Set(d, ConvertScalarTo<T>(1));
+
+    // Unchanged after division by 1.
+    HWY_ASSERT_VEC_EQ(d, v, Div(v, v1));
+
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>((static_cast<double>(i) - 2.0) / 2.0);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Div(v, Set(d, ConvertScalarTo<T>(2))));
+  }
+};
+
+HWY_NOINLINE void TestAllDiv() { ForFloatTypes(ForPartialVectors<TestDiv>()); }
+
+struct TestApproximateReciprocal {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v = Iota(d, -2);
+    const auto nonzero =
+        IfThenElse(Eq(v, Zero(d)), Set(d, ConvertScalarTo<T>(1)), v);
+    const size_t N = Lanes(d);
+    auto input = AllocateAligned<T>(N);
+    auto actual = AllocateAligned<T>(N);
+    HWY_ASSERT(input && actual);
+
+    Store(nonzero, d, input.get());
+    Store(ApproximateReciprocal(nonzero), d, actual.get());
+
+    double max_l1 = 0.0;
+    double worst_expected = 0.0;
+    double worst_actual = 0.0;
+    for (size_t i = 0; i < N; ++i) {
+      const double expected = 1.0 / input[i];
+      const double l1 = ScalarAbs(expected - actual[i]);
+      if (l1 > max_l1) {
+        max_l1 = l1;
+        worst_expected = expected;
+        worst_actual = actual[i];
+      }
+    }
+    const double abs_worst_expected = ScalarAbs(worst_expected);
+    if (abs_worst_expected > 1E-5) {
+      const double max_rel = max_l1 / abs_worst_expected;
+      fprintf(stderr, "max l1 %f rel %f (%f vs %f)\n", max_l1, max_rel,
+              worst_expected, worst_actual);
+      HWY_ASSERT(max_rel < 0.004);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllApproximateReciprocal() {
+  ForFloatTypes(ForPartialVectors<TestApproximateReciprocal>());
+}
+
+struct TestSquareRoot {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto vi = Iota(d, 0);
+    HWY_ASSERT_VEC_EQ(d, vi, Sqrt(Mul(vi, vi)));
+  }
+};
+
+HWY_NOINLINE void TestAllSquareRoot() {
+  ForFloatTypes(ForPartialVectors<TestSquareRoot>());
+}
+
+struct TestReciprocalSquareRoot {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> v = Set(d, ConvertScalarTo<T>(123.0f));
+    const size_t N = Lanes(d);
+    auto lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(lanes);
+    Store(ApproximateReciprocalSqrt(v), d, lanes.get());
+    for (size_t i = 0; i < N; ++i) {
+      T err = ConvertScalarTo<T>(ConvertScalarTo<float>(lanes[i]) - 0.090166f);
+      if (err < ConvertScalarTo<T>(0)) err = -err;
+      if (static_cast<double>(err) >= 4E-4) {
+        HWY_ABORT("Lane %d (%d): actual %f err %f\n", static_cast<int>(i),
+                  static_cast<int>(N), static_cast<double>(lanes[i]),
+                  static_cast<double>(err));
+      }
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllReciprocalSquareRoot() {
+  ForFloatTypes(ForPartialVectors<TestReciprocalSquareRoot>());
+}
+
+template <typename T, class D>
+AlignedFreeUniquePtr<T[]> RoundTestCases(T /*unused*/, D d, size_t& padded) {
+  const T eps = Epsilon<T>();
+  const T huge = ConvertScalarTo<T>(sizeof(T) >= 4 ? 1E34 : 3E4);
+  const T test_cases[] = {
+      // +/- 1
+      ConvertScalarTo<T>(1), ConvertScalarTo<T>(-1),
+      // +/- 0
+      ConvertScalarTo<T>(0), ConvertScalarTo<T>(-0),
+      // near 0
+      ConvertScalarTo<T>(0.4), ConvertScalarTo<T>(-0.4),
+      // +/- integer
+      ConvertScalarTo<T>(4), ConvertScalarTo<T>(-32),
+      // positive near limit
+      ConvertScalarTo<T>(MantissaEnd<T>() - ConvertScalarTo<T>(1.5)),
+      ConvertScalarTo<T>(MantissaEnd<T>() + ConvertScalarTo<T>(1.5)),
+      // negative near limit
+      ConvertScalarTo<T>(-MantissaEnd<T>() - ConvertScalarTo<T>(1.5)),
+      ConvertScalarTo<T>(-MantissaEnd<T>() + ConvertScalarTo<T>(1.5)),
+      // positive tiebreak
+      ConvertScalarTo<T>(1.5), ConvertScalarTo<T>(2.5),
+      // negative tiebreak
+      ConvertScalarTo<T>(-1.5), ConvertScalarTo<T>(-2.5),
+      // positive +/- delta
+      ConvertScalarTo<T>(2.0001), ConvertScalarTo<T>(3.9999),
+      // negative +/- delta
+      ConvertScalarTo<T>(-999.9999), ConvertScalarTo<T>(-998.0001),
+      // positive +/- epsilon
+      ConvertScalarTo<T>(ConvertScalarTo<T>(1) + eps),
+      ConvertScalarTo<T>(ConvertScalarTo<T>(1) - eps),
+      // negative +/- epsilon
+      ConvertScalarTo<T>(ConvertScalarTo<T>(-1) + eps),
+      ConvertScalarTo<T>(ConvertScalarTo<T>(-1) - eps),
+      // +/- huge (but still fits in float)
+      huge, -huge,
+      // +/- infinity
+      GetLane(Inf(d)), GetLane(Neg(Inf(d))),
+      // qNaN
+      GetLane(NaN(d))};
+  const size_t kNumTestCases = sizeof(test_cases) / sizeof(test_cases[0]);
+  const size_t N = Lanes(d);
+  padded = RoundUpTo(kNumTestCases, N);  // allow loading whole vectors
+  auto in = AllocateAligned<T>(padded);
+  auto expected = AllocateAligned<T>(padded);
+  HWY_ASSERT(in && expected);
+  CopyBytes(test_cases, in.get(), kNumTestCases * sizeof(T));
+  ZeroBytes(in.get() + kNumTestCases, (padded - kNumTestCases) * sizeof(T));
+  return in;
+}
+
+struct TestRound {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T t, D d) {
+    size_t padded;
+    auto in = RoundTestCases(t, d, padded);
+    auto expected = AllocateAligned<T>(padded);
+    HWY_ASSERT(expected);
+
+    for (size_t i = 0; i < padded; ++i) {
+// Avoid [std::]round, which does not round to nearest *even*.
+// NOTE: std:: version from C++11 cmath is not defined in RVV GCC, see
+// https://lists.freebsd.org/pipermail/freebsd-current/2014-January/048130.html
+// Cast to f32/64 because nearbyint does not support _Float16.
+#if HWY_HAVE_FLOAT64
+      const double f = ConvertScalarTo<double>(in[i]);
+#else
+      const float f = ConvertScalarTo<float>(in[i]);
+#endif
+      expected[i] = ConvertScalarTo<T>(nearbyint(f));
+    }
+    for (size_t i = 0; i < padded; i += Lanes(d)) {
+      HWY_ASSERT_VEC_EQ(d, &expected[i], Round(Load(d, &in[i])));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllRound() {
+  ForFloatTypes(ForPartialVectors<TestRound>());
+}
+
+struct TestNearestInt {
+  static HWY_INLINE int16_t RoundScalarFloatToInt(float16_t f) {
+    return static_cast<int16_t>(std::lrintf(ConvertScalarTo<float>(f)));
+  }
+
+  static HWY_INLINE int32_t RoundScalarFloatToInt(float f) {
+    return static_cast<int32_t>(std::lrintf(f));
+  }
+
+  static HWY_INLINE int64_t RoundScalarFloatToInt(double f) {
+    return static_cast<int64_t>(std::llrint(f));
+  }
+
+  template <typename TF, class DF>
+  HWY_NOINLINE void operator()(TF tf, const DF df) {
+    using TI = MakeSigned<TF>;
+    const RebindToSigned<DF> di;
+
+    size_t padded;
+    auto in = RoundTestCases(tf, df, padded);
+    auto expected = AllocateAligned<TI>(padded);
+    HWY_ASSERT(expected);
+
+    constexpr double kMax = static_cast<double>(LimitsMax<TI>());
+    for (size_t i = 0; i < padded; ++i) {
+      if (ScalarIsNaN(in[i])) {
+        // We replace NaN with 0 below (no_nan)
+        expected[i] = 0;
+      } else if (ScalarIsInf(in[i]) ||
+                 ConvertScalarTo<double>(ScalarAbs(in[i])) >= kMax) {
+        // Avoid undefined result for std::lrintf or std::llrint
+        expected[i] = ScalarSignBit(in[i]) ? LimitsMin<TI>() : LimitsMax<TI>();
+      } else {
+        expected[i] = RoundScalarFloatToInt(in[i]);
+      }
+    }
+    for (size_t i = 0; i < padded; i += Lanes(df)) {
+      const auto v = Load(df, &in[i]);
+      const auto no_nan = IfThenElse(Eq(v, v), v, Zero(df));
+      HWY_ASSERT_VEC_EQ(di, &expected[i], NearestInt(no_nan));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllNearestInt() {
+  ForFloatTypes(ForPartialVectors<TestNearestInt>());
+}
+
+struct TestDemoteToNearestInt {
+  template <typename TF, class DF>
+  HWY_NOINLINE void operator()(TF tf, const DF df) {
+    using TI = MakeNarrow<MakeSigned<TF>>;
+    const Rebind<TI, DF> di;
+
+    size_t padded;
+    auto in = RoundTestCases(tf, df, padded);
+    auto expected = AllocateAligned<TI>(padded);
+    HWY_ASSERT(expected);
+
+    constexpr double kMax = static_cast<double>(LimitsMax<TI>());
+    for (size_t i = 0; i < padded; ++i) {
+      if (ScalarIsNaN(in[i])) {
+        // We replace NaN with 0 below (no_nan)
+        expected[i] = 0;
+      } else if (ScalarIsInf(in[i]) ||
+                 static_cast<double>(ScalarAbs(in[i])) >= kMax) {
+        // Avoid undefined result for std::lrint
+        expected[i] = ScalarSignBit(in[i]) ? LimitsMin<TI>() : LimitsMax<TI>();
+      } else {
+        expected[i] =
+            static_cast<TI>(std::lrint(ConvertScalarTo<double>(in[i])));
+      }
+    }
+    for (size_t i = 0; i < padded; i += Lanes(df)) {
+      const auto v = Load(df, &in[i]);
+      const auto no_nan = IfThenElse(Eq(v, v), v, Zero(df));
+      HWY_ASSERT_VEC_EQ(di, &expected[i], DemoteToNearestInt(di, no_nan));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllDemoteToNearestInt() {
+#if HWY_HAVE_FLOAT64
+  ForDemoteVectors<TestDemoteToNearestInt>()(double());
+#endif
+}
+
+struct TestTrunc {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T t, D d) {
+    size_t padded;
+    auto in = RoundTestCases(t, d, padded);
+    auto expected = AllocateAligned<T>(padded);
+    HWY_ASSERT(expected);
+
+    for (size_t i = 0; i < padded; ++i) {
+      // NOTE: std:: version from C++11 cmath is not defined in RVV GCC, see
+      // https://lists.freebsd.org/pipermail/freebsd-current/2014-January/048130.html
+      // Cast to double because trunc does not support _Float16.
+      expected[i] = ConvertScalarTo<T>(trunc(ConvertScalarTo<double>(in[i])));
+    }
+    for (size_t i = 0; i < padded; i += Lanes(d)) {
+      HWY_ASSERT_VEC_EQ(d, &expected[i], Trunc(Load(d, &in[i])));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllTrunc() {
+  ForFloatTypes(ForPartialVectors<TestTrunc>());
+}
+
+struct TestCeil {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T t, D d) {
+    const RebindToSigned<decltype(d)> di;
+    using TI = MakeSigned<T>;
+
+    size_t padded;
+    auto in = RoundTestCases(t, d, padded);
+    auto expected = AllocateAligned<T>(padded);
+    auto expected_int = AllocateAligned<TI>(padded);
+    HWY_ASSERT(expected && expected_int);
+
+    constexpr double kMinOutOfRangeVal = -static_cast<double>(LimitsMin<TI>());
+    static_assert(kMinOutOfRangeVal > 0.0,
+                  "kMinOutOfRangeVal > 0.0 must be true");
+
+    for (size_t i = 0; i < padded; ++i) {
+      // Cast to double because ceil does not support _Float16.
+      const double ceil_val = std::ceil(ConvertScalarTo<double>(in[i]));
+      expected[i] = ConvertScalarTo<T>(ceil_val);
+      if (ScalarIsNaN(ceil_val)) {
+        expected_int[i] = 0;
+      } else if (ScalarIsInf(ceil_val) || static_cast<double>(ScalarAbs(
+                                              ceil_val)) >= kMinOutOfRangeVal) {
+        expected_int[i] =
+            ScalarSignBit(ceil_val) ? LimitsMin<TI>() : LimitsMax<TI>();
+      } else {
+        expected_int[i] = ConvertScalarTo<TI>(ceil_val);
+      }
+    }
+    for (size_t i = 0; i < padded; i += Lanes(d)) {
+      const auto v = Load(d, &in[i]);
+      HWY_ASSERT_VEC_EQ(d, &expected[i], Ceil(v));
+      HWY_ASSERT_VEC_EQ(di, &expected_int[i],
+                        IfThenZeroElse(RebindMask(di, IsNaN(v)), CeilInt(v)));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllCeil() {
+  ForFloatTypes(ForPartialVectors<TestCeil>());
+}
+
+struct TestFloor {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T t, D d) {
+    const RebindToSigned<decltype(d)> di;
+    using TI = MakeSigned<T>;
+
+    size_t padded;
+    auto in = RoundTestCases(t, d, padded);
+    auto expected = AllocateAligned<T>(padded);
+    auto expected_int = AllocateAligned<TI>(padded);
+    HWY_ASSERT(expected && expected_int);
+
+    constexpr double kMinOutOfRangeVal = -static_cast<double>(LimitsMin<TI>());
+    static_assert(kMinOutOfRangeVal > 0.0,
+                  "kMinOutOfRangeVal > 0.0 must be true");
+
+    for (size_t i = 0; i < padded; ++i) {
+      // Cast to double because floor does not support _Float16.
+      const double floor_val = std::floor(ConvertScalarTo<double>(in[i]));
+      expected[i] = ConvertScalarTo<T>(floor_val);
+      if (ScalarIsNaN(floor_val)) {
+        expected_int[i] = 0;
+      } else if (ScalarIsInf(floor_val) ||
+                 static_cast<double>(ScalarAbs(floor_val)) >=
+                     kMinOutOfRangeVal) {
+        expected_int[i] =
+            ScalarSignBit(floor_val) ? LimitsMin<TI>() : LimitsMax<TI>();
+      } else {
+        expected_int[i] = ConvertScalarTo<TI>(floor_val);
+      }
+    }
+    for (size_t i = 0; i < padded; i += Lanes(d)) {
+      const auto v = Load(d, &in[i]);
+      HWY_ASSERT_VEC_EQ(d, &expected[i], Floor(v));
+      HWY_ASSERT_VEC_EQ(di, &expected_int[i],
+                        IfThenZeroElse(RebindMask(di, IsNaN(v)), FloorInt(v)));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllFloor() {
+  ForFloatTypes(ForPartialVectors<TestFloor>());
+}
+
+struct TestAbsDiff {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto in_lanes_a = AllocateAligned<T>(N);
+    auto in_lanes_b = AllocateAligned<T>(N);
+    auto out_lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(in_lanes_a && in_lanes_b && out_lanes);
+    for (size_t i = 0; i < N; ++i) {
+      in_lanes_a[i] = ConvertScalarTo<T>((i ^ 1u) << i);
+      in_lanes_b[i] = ConvertScalarTo<T>(i << i);
+      out_lanes[i] = ConvertScalarTo<T>(
+          ScalarAbs(ConvertScalarTo<T>(in_lanes_a[i] - in_lanes_b[i])));
+    }
+    const auto a = Load(d, in_lanes_a.get());
+    const auto b = Load(d, in_lanes_b.get());
+    const auto expected = Load(d, out_lanes.get());
+    HWY_ASSERT_VEC_EQ(d, expected, AbsDiff(a, b));
+    HWY_ASSERT_VEC_EQ(d, expected, AbsDiff(b, a));
+  }
+};
+
+HWY_NOINLINE void TestAllAbsDiff() {
+  ForFloatTypes(ForPartialVectors<TestAbsDiff>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyFloatTest);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllF16FromF32);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllF32FromF16);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllDiv);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllApproximateReciprocal);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllSquareRoot);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllReciprocalSquareRoot);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllRound);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllNearestInt);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllDemoteToNearestInt);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllTrunc);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllCeil);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllFloor);
+HWY_EXPORT_AND_TEST_P(HwyFloatTest, TestAllAbsDiff);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/fma_test.cc

@@ -0,0 +1,186 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/fma_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+#ifndef HWY_NATIVE_FMA
+#error "Bug in set_macros-inl.h, did not set HWY_NATIVE_FMA"
+#endif
+
+struct TestMulAdd {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> k0 = Zero(d);
+    const Vec<D> v1 = Iota(d, 1);
+    const Vec<D> v2 = Iota(d, 2);
+
+    // Unlike RebindToSigned, we want to leave floating-point unchanged.
+    // This allows Neg for unsigned types.
+    const Rebind<If<IsFloat<T>(), T, MakeSigned<T>>, D> dif;
+    const Vec<D> neg_v2 = BitCast(d, Neg(BitCast(dif, v2)));
+
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+    HWY_ASSERT_VEC_EQ(d, k0, MulAdd(k0, k0, k0));
+    HWY_ASSERT_VEC_EQ(d, v2, MulAdd(k0, v1, v2));
+    HWY_ASSERT_VEC_EQ(d, v2, MulAdd(v1, k0, v2));
+    HWY_ASSERT_VEC_EQ(d, k0, NegMulAdd(k0, k0, k0));
+    HWY_ASSERT_VEC_EQ(d, v2, NegMulAdd(k0, v1, v2));
+    HWY_ASSERT_VEC_EQ(d, v2, NegMulAdd(v1, k0, v2));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>((i + 1) * (i + 2));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), MulAdd(v2, v1, k0));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), MulAdd(v1, v2, k0));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulAdd(neg_v2, v1, k0));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulAdd(v1, neg_v2, k0));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>((i + 2) * (i + 2) + (i + 1));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), MulAdd(v2, v2, v1));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulAdd(neg_v2, v2, v1));
+
+    for (size_t i = 0; i < N; ++i) {
+      const T nm = ConvertScalarTo<T>(-static_cast<int>(i + 2));
+      const T f = ConvertScalarTo<T>(i + 2);
+      const T a = ConvertScalarTo<T>(i + 1);
+      expected[i] = ConvertScalarTo<T>(nm * f + a);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulAdd(v2, v2, v1));
+  }
+};
+
+HWY_NOINLINE void TestAllMulAdd() {
+  ForAllTypes(ForPartialVectors<TestMulAdd>());
+}
+
+struct TestMulSub {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> k0 = Zero(d);
+    const Vec<D> kNeg0 = Set(d, ConvertScalarTo<T>(-0.0));
+    const Vec<D> v1 = Iota(d, 1);
+    const Vec<D> v2 = Iota(d, 2);
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    // Unlike RebindToSigned, we want to leave floating-point unchanged.
+    // This allows Neg for unsigned types.
+    const Rebind<If<IsFloat<T>(), T, MakeSigned<T>>, D> dif;
+
+    HWY_ASSERT_VEC_EQ(d, k0, MulSub(k0, k0, k0));
+    HWY_ASSERT_VEC_EQ(d, kNeg0, NegMulSub(k0, k0, k0));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>(-static_cast<int>(i + 2));
+    }
+    const auto neg_k0 = BitCast(d, Neg(BitCast(dif, k0)));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), MulSub(k0, v1, v2));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), MulSub(v1, k0, v2));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulSub(neg_k0, v1, v2));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulSub(v1, neg_k0, v2));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>((i + 1) * (i + 2));
+    }
+    const auto neg_v1 = BitCast(d, Neg(BitCast(dif, v1)));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), MulSub(v1, v2, k0));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), MulSub(v2, v1, k0));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulSub(neg_v1, v2, k0));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulSub(v2, neg_v1, k0));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>((i + 2) * (i + 2) - (1 + i));
+    }
+    const auto neg_v2 = BitCast(d, Neg(BitCast(dif, v2)));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), MulSub(v2, v2, v1));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulSub(neg_v2, v2, v1));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulSub(v2, neg_v2, v1));
+  }
+};
+
+HWY_NOINLINE void TestAllMulSub() {
+  ForAllTypes(ForPartialVectors<TestMulSub>());
+}
+
+struct TestMulAddSub {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> k0 = Zero(d);
+    const Vec<D> v1 = Iota(d, 1);
+    const Vec<D> v2 = Iota(d, 2);
+
+    // Unlike RebindToSigned, we want to leave floating-point unchanged.
+    // This allows Neg for unsigned types.
+    const Rebind<If<IsFloat<T>(), T, MakeSigned<T>>, D> dif;
+    const Vec<D> neg_v2 = BitCast(d, Neg(BitCast(dif, v2)));
+
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    HWY_ASSERT_VEC_EQ(d, k0, MulAddSub(k0, k0, k0));
+
+    const auto v2_negated_if_even = OddEven(v2, neg_v2);
+    HWY_ASSERT_VEC_EQ(d, v2_negated_if_even, MulAddSub(k0, v1, v2));
+    HWY_ASSERT_VEC_EQ(d, v2_negated_if_even, MulAddSub(v1, k0, v2));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] =
+          ConvertScalarTo<T>(((i & 1) == 0) ? ((i + 2) * (i + 2) - (i + 1))
+                                            : ((i + 2) * (i + 2) + (i + 1)));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), MulAddSub(v2, v2, v1));
+  }
+};
+
+HWY_NOINLINE void TestAllMulAddSub() {
+  ForAllTypes(ForPartialVectors<TestMulAddSub>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyFmaTest);
+HWY_EXPORT_AND_TEST_P(HwyFmaTest, TestAllMulAdd);
+HWY_EXPORT_AND_TEST_P(HwyFmaTest, TestAllMulSub);
+HWY_EXPORT_AND_TEST_P(HwyFmaTest, TestAllMulAddSub);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/foreach_vec_test.cc

@@ -0,0 +1,654 @@
+// Copyright 2023 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/foreach_vec_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct ForeachVectorTestPerLaneSizeState {
+  size_t num_of_lanes_mask;
+#if HWY_HAVE_SCALABLE
+  int pow2_mask;
+#endif
+};
+
+struct ForeachVectorTestState {
+  ForeachVectorTestPerLaneSizeState per_lane_size_states[16];
+  int lane_sizes_mask;
+};
+
+template <class D>
+static HWY_INLINE void UpdateForeachVectorTestState(
+    ForeachVectorTestState &state, D d) {
+  using T = TFromD<D>;
+  static_assert(sizeof(T) >= 1 && sizeof(T) <= 8,
+                "sizeof(T) must be between 1 and 8");
+
+  state.lane_sizes_mask |= (1 << sizeof(T));
+
+  ForeachVectorTestPerLaneSizeState *per_lane_size_state =
+      &state.per_lane_size_states[sizeof(T)];
+
+  const size_t lanes = Lanes(d);
+  HWY_ASSERT(lanes > 0 && (lanes & (lanes - 1)) == 0);
+
+  per_lane_size_state->num_of_lanes_mask |= lanes;
+
+#if HWY_HAVE_SCALABLE
+  constexpr int kPow2 = D().Pow2();
+#if HWY_TARGET == HWY_RVV
+  static_assert(kPow2 >= detail::MinPow2<T>(),
+                "kPow2 >= detail::MinPow2<T>() must be true");
+#endif
+  static_assert(kPow2 <= detail::MaxPow2(),
+                "kPow2 <= detail::MaxPow2() must be true");
+
+  if (HWY_TARGET == HWY_RVV || kPow2 >= -3) {
+    per_lane_size_state->pow2_mask |= (1 << (kPow2 + 3));
+  }
+#endif
+}
+
+static constexpr int kMaxSupportedLaneSize = HWY_HAVE_INTEGER64 ? 8 : 4;
+static constexpr int kSupportedLaneSizesMask =
+    (1 << 1) | (1 << 2) | (1 << 4) | (HWY_HAVE_INTEGER64 ? (1 << 8) : 0);
+
+#if HWY_HAVE_SCALABLE
+static constexpr int kSupportedU8Pow2Mask =
+    (HWY_TARGET == HWY_RVV) ? 0x7F : 0x0F;
+#endif
+
+static HWY_INLINE size_t LanesPerVectWithLaneSize(size_t lanes_per_u8_vect,
+                                                  int lane_size) {
+#if HWY_TARGET == HWY_SCALAR
+  (void)lanes_per_u8_vect;
+  (void)lane_size;
+  return 1;
+#else
+  return lanes_per_u8_vect / static_cast<size_t>(lane_size);
+#endif
+}
+
+#define HWY_DECLARE_FOREACH_VECTOR_TEST(TestClass)       \
+  static ForeachVectorTestState TestClass##State;        \
+                                                         \
+  struct TestClass {                                     \
+    template <class T, class D>                          \
+    HWY_INLINE void operator()(T, D d) {                 \
+      UpdateForeachVectorTestState(TestClass##State, d); \
+    }                                                    \
+  };
+
+HWY_DECLARE_FOREACH_VECTOR_TEST(TestForMaxPow2)
+
+HWY_NOINLINE void TestAllForMaxPow2() {
+  ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForMaxPow2State);
+  ForUnsignedTypes(ForMaxPow2<TestForMaxPow2>());
+
+  HWY_ASSERT(TestForMaxPow2State.lane_sizes_mask == kSupportedLaneSizesMask);
+
+  const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>());
+
+  for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) {
+    ForeachVectorTestPerLaneSizeState *per_lane_size_state =
+        &TestForMaxPow2State.per_lane_size_states[lane_size];
+
+    const size_t lanes = LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size);
+    HWY_ASSERT(per_lane_size_state->num_of_lanes_mask ==
+               ((lanes << (HWY_TARGET == HWY_RVV ? 2 : 1)) - 1));
+
+#if HWY_HAVE_SCALABLE
+    const int expected_pow2_mask =
+        (kSupportedU8Pow2Mask & 0x1F) & (-((lane_size + 1) / 2));
+    HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask);
+#endif
+  }
+}
+
+HWY_DECLARE_FOREACH_VECTOR_TEST(TestForExtendableVectors)
+
+#if HWY_TARGET == HWY_RVV
+template <int kPow2, class Test, class T,
+          hwy::EnableIf<(-kPow2 < detail::MinPow2<T>())> * = nullptr>
+static HWY_INLINE void ExecuteTestForExtendableVectors(const Test & /*test*/,
+                                                       T /*unused*/) {}
+template <int kPow2, class Test, class T,
+          hwy::EnableIf<(-kPow2 >= detail::MinPow2<T>())> * = nullptr>
+static HWY_INLINE void ExecuteTestForExtendableVectors(const Test &test,
+                                                       T /*unused*/) {
+  test(T());
+}
+#endif
+
+template <int kPow2>
+static HWY_NOINLINE void DoTestAllForExtendableVectors() {
+  static_assert(kPow2 >= 0 && kPow2 <= 3, "kPow2 must be between 0 and 3");
+
+  ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForExtendableVectorsState);
+
+  const ForExtendableVectors<TestForExtendableVectors, kPow2> test;
+#if HWY_TARGET == HWY_RVV
+  test(uint8_t());
+  ExecuteTestForExtendableVectors<kPow2>(test, uint16_t());
+  ExecuteTestForExtendableVectors<kPow2>(test, uint32_t());
+  ExecuteTestForExtendableVectors<kPow2>(test, uint64_t());
+#else
+  ForUnsignedTypes(test);
+#endif
+
+#if HWY_TARGET == HWY_SCALAR
+  HWY_ASSERT(TestForExtendableVectorsState.lane_sizes_mask == 0);
+#else  // HWY_TARGET != HWY_SCALAR
+  const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t, -kPow2>());
+#if HWY_TARGET == HWY_RVV
+  const int expected_lane_sizes_mask =
+      kSupportedLaneSizesMask &
+      ((1 << 1) | (1 << 2) | ((kPow2 <= 2) ? (1 << 4) : 0) |
+       ((kPow2 <= 1) ? (1 << 8) : 0));
+#else
+  const int expected_lane_sizes_mask =
+      kSupportedLaneSizesMask & (((lanes_per_u8_vect >= 1) ? (1 << 1) : 0) |
+                                 ((lanes_per_u8_vect >= 2) ? (1 << 2) : 0) |
+                                 ((lanes_per_u8_vect >= 4) ? (1 << 4) : 0) |
+                                 ((lanes_per_u8_vect >= 8) ? (1 << 8) : 0));
+#endif
+  HWY_ASSERT(TestForExtendableVectorsState.lane_sizes_mask ==
+             expected_lane_sizes_mask);
+#endif  // HWY_TARGET == HWY_SCALAR
+
+  for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) {
+    ForeachVectorTestPerLaneSizeState *per_lane_size_state =
+        &TestForExtendableVectorsState.per_lane_size_states[lane_size];
+
+#if HWY_TARGET == HWY_SCALAR
+    HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0);
+#else
+    if ((expected_lane_sizes_mask & (1 << lane_size)) != 0) {
+      const size_t lanes =
+          LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size);
+      HWY_ASSERT(per_lane_size_state->num_of_lanes_mask ==
+                 ((lanes << (HWY_TARGET == HWY_RVV ? 4 : 1)) - 1));
+
+#if HWY_HAVE_SCALABLE
+      const int expected_pow2_mask =
+          ((kSupportedU8Pow2Mask >> kPow2) & (-((lane_size + 1) / 2))) |
+          ((HWY_TARGET == HWY_RVV) ? 0 : (1 << (3 - kPow2)));
+      HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask);
+#endif
+    } else {
+      HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0);
+#if HWY_HAVE_SCALABLE
+      HWY_ASSERT(per_lane_size_state->pow2_mask == 0);
+#endif
+    }
+#endif  // HWY_TARGET == HWY_SCALAR
+  }
+}
+
+HWY_NOINLINE void TestAllForExtendableVectors() {
+  DoTestAllForExtendableVectors<1>();
+  DoTestAllForExtendableVectors<2>();
+  DoTestAllForExtendableVectors<3>();
+}
+
+HWY_DECLARE_FOREACH_VECTOR_TEST(TestForShrinkableVectors)
+
+HWY_NOINLINE void TestAllForShrinkableVectors() {
+  ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForShrinkableVectorsState);
+  ForUnsignedTypes(ForShrinkableVectors<TestForShrinkableVectors>());
+
+#if HWY_TARGET == HWY_SCALAR
+  HWY_ASSERT(TestForShrinkableVectorsState.lane_sizes_mask == 0);
+#else   // HWY_TARGET != HWY_SCALAR
+  HWY_ASSERT(TestForShrinkableVectorsState.lane_sizes_mask ==
+             kSupportedLaneSizesMask);
+  const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>());
+#endif  // HWY_TARGET == HWY_SCALAR
+
+  for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) {
+    ForeachVectorTestPerLaneSizeState *per_lane_size_state =
+        &TestForShrinkableVectorsState.per_lane_size_states[lane_size];
+
+#if HWY_TARGET == HWY_SCALAR
+    HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0);
+#else  // HWY_TARGET != HWY_SCALAR
+    const size_t lanes = LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size);
+
+#if HWY_HAVE_SCALABLE
+    const int expected_pow2_mask =
+        kSupportedU8Pow2Mask & (-2 * ((lane_size + 1) / 2));
+    const size_t expected_lanes_mask =
+        (lanes * static_cast<size_t>(expected_pow2_mask)) >> 3;
+
+    HWY_ASSERT((per_lane_size_state->num_of_lanes_mask & expected_lanes_mask) ==
+               expected_lanes_mask);
+    HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask);
+#else   // !HWY_HAVE_SCALABLE
+    const size_t expected_lanes_mask =
+        static_cast<size_t>(((lanes << 1) - 1) & (~size_t{1}));
+    HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == expected_lanes_mask);
+#endif  // HWY_HAVE_SCALABLE
+#endif  // HWY_TARGET == HWY_SCALAR
+  }
+}
+
+HWY_DECLARE_FOREACH_VECTOR_TEST(TestForGEVectors)
+
+template <size_t kMinBits, class Test, class T,
+          HWY_IF_LANES_LE(kMinBits, sizeof(T) * 8 - 1)>
+static HWY_INLINE void ExecuteTestForGEVectors(const Test & /*test*/,
+                                               T /*unused*/) {}
+
+template <size_t kMinBits, class Test, class T,
+          HWY_IF_LANES_GT(kMinBits, sizeof(T) * 8 - 1)>
+static HWY_INLINE void ExecuteTestForGEVectors(const Test &test, T /*unused*/) {
+  test(T());
+}
+
+template <size_t kMinBits, class Test>
+static HWY_NOINLINE void DoTestAllForGEVectors(const Test &test) {
+  static_assert(kMinBits >= 16, "kMinBits >= 16 must be true");
+
+  ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForGEVectorsState);
+
+  test(uint8_t());
+  test(uint16_t());
+  ExecuteTestForGEVectors<kMinBits>(test, uint32_t());
+#if HWY_HAVE_INTEGER64
+  ExecuteTestForGEVectors<kMinBits>(test, uint64_t());
+#endif
+
+#if HWY_TARGET == HWY_SCALAR
+  HWY_ASSERT(TestForGEVectorsState.lane_sizes_mask == 0);
+#else  // HWY_TARGET != HWY_SCALAR
+  const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>());
+
+#if HWY_TARGET == HWY_RVV
+  const size_t lanes_per_largest_u8_vect = lanes_per_u8_vect * 8;
+#else
+  const size_t lanes_per_largest_u8_vect = lanes_per_u8_vect;
+#endif  // HWY_TARGET == HWY_RVV
+
+  constexpr int kGEVectSupportedLaneSizesMask =
+      kSupportedLaneSizesMask &
+      ((1 << 1) | (1 << 2) | ((kMinBits >= 32) ? (1 << 4) : 0) |
+       ((kMinBits >= 64) ? (1 << 8) : 0));
+
+  const int expected_lane_sizes_mask =
+      (lanes_per_largest_u8_vect >= (kMinBits / 8))
+          ? kGEVectSupportedLaneSizesMask
+          : 0;
+
+  constexpr size_t kSupportedU8VecSizesMask =
+      static_cast<size_t>(((static_cast<size_t>(HWY_MAX_BYTES) << 1) - 1) &
+                          (~((kMinBits / 8) - 1)));
+
+  HWY_ASSERT(TestForGEVectorsState.lane_sizes_mask == expected_lane_sizes_mask);
+#endif  // HWY_TARGET == HWY_SCALAR
+
+#if HWY_HAVE_SCALABLE
+  constexpr int kMinVecPow2 =
+      static_cast<int>(CeilLog2(HWY_MIN(kMinBits / 16, 8))) - 3;
+  static_assert(kMinVecPow2 >= -3 && kMinVecPow2 <= 0,
+                "kMinVecPow2 must be between -3 and 0");
+
+  constexpr int kGEVectSupportedU8Pow2Mask =
+      kSupportedU8Pow2Mask & (-(1 << (kMinVecPow2 + 3)));
+
+#if HWY_TARGET == HWY_RVV
+  const int ge_vect_supported_u8_pow2_mask =
+      kGEVectSupportedU8Pow2Mask &
+      ((kMinBits <= 128)
+           ? -1
+           : ((lanes_per_u8_vect < (kMinBits / 64))
+                  ? 0
+                  : (0x40 |
+                     ((lanes_per_u8_vect >= (kMinBits / 32)) ? 0x20 : 0) |
+                     ((lanes_per_u8_vect >= (kMinBits / 16)) ? 0x10 : 0) |
+                     ((lanes_per_u8_vect >= (kMinBits / 8)) ? 0x08 : 0))));
+#else
+  const int ge_vect_supported_u8_pow2_mask =
+      (lanes_per_u8_vect >= (kMinBits / 8)) ? kGEVectSupportedU8Pow2Mask : 0;
+#endif
+
+#endif
+
+  for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) {
+    ForeachVectorTestPerLaneSizeState *per_lane_size_state =
+        &TestForGEVectorsState.per_lane_size_states[lane_size];
+
+#if HWY_TARGET == HWY_SCALAR
+    HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0);
+#else  // HWY_TARGET != HWY_SCALAR
+    if (kMinBits >= static_cast<size_t>(lane_size * 8)) {
+      const size_t expected_lanes_mask =
+          (((lanes_per_largest_u8_vect << 1) - 1) & kSupportedU8VecSizesMask) /
+          static_cast<size_t>(lane_size);
+      HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == expected_lanes_mask);
+
+#if HWY_HAVE_SCALABLE
+      const int expected_pow2_mask =
+          ge_vect_supported_u8_pow2_mask & (-((lane_size + 1) / 2));
+      HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask);
+#endif
+    } else {
+      HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0);
+#if HWY_HAVE_SCALABLE
+      HWY_ASSERT(per_lane_size_state->pow2_mask == 0);
+#endif
+    }
+#endif  // HWY_TARGET == HWY_SCALAR
+  }
+}
+
+HWY_NOINLINE void TestAllForGEVectors() {
+  DoTestAllForGEVectors<16>(ForGEVectors<16, TestForGEVectors>());
+  DoTestAllForGEVectors<32>(ForGEVectors<32, TestForGEVectors>());
+  DoTestAllForGEVectors<64>(ForGEVectors<64, TestForGEVectors>());
+  DoTestAllForGEVectors<128>(ForGEVectors<128, TestForGEVectors>());
+  DoTestAllForGEVectors<256>(ForGEVectors<256, TestForGEVectors>());
+  DoTestAllForGEVectors<512>(ForGEVectors<512, TestForGEVectors>());
+}
+
+HWY_DECLARE_FOREACH_VECTOR_TEST(TestForPromoteVectors)
+
+template <int kSrcLaneSizePow2, int kPromotePow2, class Test, class T,
+          hwy::EnableIf<(kSrcLaneSizePow2 + kPromotePow2 <=
+                         (HWY_HAVE_INTEGER64 ? 3 : 2))> * = nullptr>
+static HWY_INLINE void ExecuteTestForPromoteVectors(const Test &test,
+                                                    T /*unused*/) {
+  test(T());
+}
+
+template <int kSrcLaneSizePow2, int kPromotePow2, class Test, class T,
+          hwy::EnableIf<(kSrcLaneSizePow2 + kPromotePow2 >
+                         (HWY_HAVE_INTEGER64 ? 3 : 2))> * = nullptr>
+static HWY_INLINE void ExecuteTestForPromoteVectors(const Test & /*test*/,
+                                                    T /*unused*/) {}
+
+template <int kPow2>
+static HWY_NOINLINE void DoTestAllForPromoteVectors() {
+  ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForPromoteVectorsState);
+
+  const ForPromoteVectors<TestForPromoteVectors, kPow2> test;
+  test(uint8_t());
+  ExecuteTestForPromoteVectors<1, kPow2>(test, uint16_t());
+  ExecuteTestForPromoteVectors<2, kPow2>(test, uint32_t());
+
+  constexpr int kMaxSupportedPromoteLaneSize = kMaxSupportedLaneSize >> kPow2;
+  static_assert(kMaxSupportedPromoteLaneSize > 0,
+                "kMaxSupportedPromoteLaneSize > 0 must be true");
+
+  constexpr int kSupportedPromoteLaneSizesMask =
+      kSupportedLaneSizesMask & ((2 << kMaxSupportedPromoteLaneSize) - 1);
+
+  HWY_ASSERT(TestForPromoteVectorsState.lane_sizes_mask ==
+             kSupportedPromoteLaneSizesMask);
+
+  const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>());
+
+  for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) {
+    ForeachVectorTestPerLaneSizeState *per_lane_size_state =
+        &TestForPromoteVectorsState.per_lane_size_states[lane_size];
+
+    if (lane_size <= kMaxSupportedPromoteLaneSize) {
+      const size_t lanes =
+          LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size) >>
+          (HWY_TARGET == HWY_SCALAR ? 0 : kPow2);
+      HWY_ASSERT(per_lane_size_state->num_of_lanes_mask ==
+                 ((lanes << (HWY_TARGET == HWY_RVV ? 4 : 1)) - 1));
+
+#if HWY_HAVE_SCALABLE
+      const int expected_pow2_mask =
+          ((kSupportedU8Pow2Mask >> kPow2) & (-((lane_size + 1) / 2))) |
+          ((HWY_TARGET == HWY_RVV) ? 0 : (1 << (3 - kPow2)));
+      HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask);
+#endif
+    } else {
+      HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0);
+#if HWY_HAVE_SCALABLE
+      HWY_ASSERT(per_lane_size_state->pow2_mask == 0);
+#endif
+    }
+  }
+}
+
+HWY_NOINLINE void TestAllForPromoteVectors() {
+  DoTestAllForPromoteVectors<1>();
+  DoTestAllForPromoteVectors<2>();
+#if HWY_HAVE_INTEGER64
+  DoTestAllForPromoteVectors<3>();
+#endif
+}
+
+HWY_DECLARE_FOREACH_VECTOR_TEST(TestForDemoteVectors)
+
+template <int kSrcLaneSizePow2, int kDemotePow2, class Test, class T,
+          hwy::EnableIf<(kSrcLaneSizePow2 >= kDemotePow2)> * = nullptr>
+static HWY_INLINE void ExecuteTestForDemoteVectors(const Test &test,
+                                                   T /*unused*/) {
+  test(T());
+}
+
+template <int kSrcLaneSizePow2, int kDemotePow2, class Test, class T,
+          hwy::EnableIf<(kSrcLaneSizePow2 < kDemotePow2)> * = nullptr>
+static HWY_INLINE void ExecuteTestForDemoteVectors(const Test & /*test*/,
+                                                   T /*unused*/) {}
+
+template <int kPow2>
+HWY_NOINLINE void DoTestAllForDemoteVectors() {
+  ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForDemoteVectorsState);
+
+  const ForDemoteVectors<TestForDemoteVectors, kPow2> test;
+  ExecuteTestForDemoteVectors<1, kPow2>(test, uint16_t());
+  ExecuteTestForDemoteVectors<2, kPow2>(test, uint32_t());
+#if HWY_HAVE_INTEGER64
+  ExecuteTestForDemoteVectors<3, kPow2>(test, uint64_t());
+#endif
+
+  constexpr int kMinDemotableLaneSize = 1 << kPow2;
+  constexpr int kSupportedDemoteLaneSizesMask =
+      kSupportedLaneSizesMask & (-(1 << kMinDemotableLaneSize));
+
+  HWY_ASSERT(TestForDemoteVectorsState.lane_sizes_mask ==
+             kSupportedDemoteLaneSizesMask);
+
+  const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>());
+
+  for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) {
+    ForeachVectorTestPerLaneSizeState *per_lane_size_state =
+        &TestForDemoteVectorsState.per_lane_size_states[lane_size];
+
+    if (lane_size >= kMinDemotableLaneSize) {
+      const size_t lanes =
+          LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size);
+      HWY_ASSERT(per_lane_size_state->num_of_lanes_mask ==
+                 ((lanes << (HWY_TARGET == HWY_RVV ? 4 : 1)) - 1));
+
+#if HWY_HAVE_SCALABLE
+      const int expected_pow2_mask =
+          kSupportedU8Pow2Mask &
+          ((-lane_size) | (((lane_size >> kPow2) > 1) ? (lane_size >> 1) : 0));
+      HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask);
+#endif
+    } else {
+      HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == 0);
+#if HWY_HAVE_SCALABLE
+      HWY_ASSERT(per_lane_size_state->pow2_mask == 0);
+#endif
+    }
+  }
+}
+
+HWY_NOINLINE void TestAllForDemoteVectors() {
+  DoTestAllForDemoteVectors<1>();
+  DoTestAllForDemoteVectors<2>();
+#if HWY_HAVE_INTEGER64
+  DoTestAllForDemoteVectors<3>();
+#endif
+}
+
+HWY_DECLARE_FOREACH_VECTOR_TEST(TestForHalfVectors)
+
+template <int kPow2>
+static HWY_NOINLINE void DoTestAllForHalfVectors() {
+  ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForHalfVectorsState);
+  ForUnsignedTypes(ForHalfVectors<TestForHalfVectors, kPow2>());
+
+#if HWY_TARGET == HWY_SCALAR
+  const size_t kMinSrcVectLanes = 1;
+#else
+  const size_t kMinSrcVectLanes = size_t{1} << kPow2;
+#endif
+
+  const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>());
+
+#if HWY_TARGET == HWY_SCALAR || HWY_TARGET == HWY_RVV
+  const int expected_lane_sizes_mask = kSupportedLaneSizesMask;
+#else
+  const int expected_lane_sizes_mask =
+      kSupportedLaneSizesMask &
+      (((lanes_per_u8_vect >= kMinSrcVectLanes) ? (1 << 1) : 0) |
+       ((lanes_per_u8_vect >= 2 * kMinSrcVectLanes) ? (1 << 2) : 0) |
+       ((lanes_per_u8_vect >= 4 * kMinSrcVectLanes) ? (1 << 4) : 0) |
+       ((lanes_per_u8_vect >= 8 * kMinSrcVectLanes) ? (1 << 8) : 0));
+#endif
+
+  HWY_ASSERT(TestForHalfVectorsState.lane_sizes_mask ==
+             expected_lane_sizes_mask);
+
+  for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) {
+    ForeachVectorTestPerLaneSizeState *per_lane_size_state =
+        &TestForHalfVectorsState.per_lane_size_states[lane_size];
+
+    const size_t lanes = LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size);
+    HWY_ASSERT(per_lane_size_state->num_of_lanes_mask ==
+               (((lanes << (HWY_TARGET == HWY_RVV ? 4 : 1)) - 1) &
+                (size_t{0} - kMinSrcVectLanes)));
+
+#if HWY_HAVE_SCALABLE
+    const int expected_pow2_mask =
+        kSupportedU8Pow2Mask & ((-(((lane_size + 1) / 2) << kPow2)) |
+                                (lanes >= kMinSrcVectLanes ? 8 : 0));
+    HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask);
+#endif
+  }
+}
+
+HWY_NOINLINE void TestAllForHalfVectors() {
+  DoTestAllForHalfVectors<1>();
+  DoTestAllForHalfVectors<2>();
+}
+
+HWY_DECLARE_FOREACH_VECTOR_TEST(TestForPartialVectors)
+
+HWY_NOINLINE void TestAllForPartialVectors() {
+  ZeroBytes<sizeof(ForeachVectorTestState)>(&TestForPartialVectorsState);
+  ForUnsignedTypes(ForPartialVectors<TestForPartialVectors>());
+
+  HWY_ASSERT(TestForPartialVectorsState.lane_sizes_mask ==
+             kSupportedLaneSizesMask);
+
+  const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>());
+
+  for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) {
+    ForeachVectorTestPerLaneSizeState *per_lane_size_state =
+        &TestForPartialVectorsState.per_lane_size_states[lane_size];
+
+    const size_t lanes = LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size);
+    HWY_ASSERT(per_lane_size_state->num_of_lanes_mask ==
+               ((lanes << (HWY_TARGET == HWY_RVV ? 4 : 1)) - 1));
+
+#if HWY_HAVE_SCALABLE
+    const int expected_pow2_mask =
+        kSupportedU8Pow2Mask & (-((lane_size + 1) / 2));
+    HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask);
+#endif
+  }
+}
+
+HWY_DECLARE_FOREACH_VECTOR_TEST(TestForPartialFixedOrFullVectors)
+
+HWY_NOINLINE void TestAllForPartialFixedOrFullVectors() {
+  ZeroBytes<sizeof(ForeachVectorTestState)>(
+      &TestForPartialFixedOrFullVectorsState);
+  ForUnsignedTypes(
+      ForPartialFixedOrFullScalableVectors<TestForPartialFixedOrFullVectors>());
+
+  HWY_ASSERT(TestForPartialFixedOrFullVectorsState.lane_sizes_mask ==
+             kSupportedLaneSizesMask);
+
+  const size_t lanes_per_u8_vect = Lanes(ScalableTag<uint8_t>());
+
+  for (int lane_size = 1; lane_size <= kMaxSupportedLaneSize; lane_size <<= 1) {
+    ForeachVectorTestPerLaneSizeState *per_lane_size_state =
+        &TestForPartialFixedOrFullVectorsState.per_lane_size_states[lane_size];
+
+    const size_t lanes = LanesPerVectWithLaneSize(lanes_per_u8_vect, lane_size);
+#if HWY_TARGET == HWY_RVV
+    const size_t expected_lanes_mask =
+        ((lanes * 16) - 1) & (size_t{0} - ((lanes_per_u8_vect + 7) / 8));
+#elif HWY_HAVE_SCALABLE || HWY_TARGET_IS_SVE
+    const size_t expected_lanes_mask = lanes;
+#else
+    const size_t expected_lanes_mask = (lanes << 1) - 1;
+#endif
+
+    HWY_ASSERT(per_lane_size_state->num_of_lanes_mask == expected_lanes_mask);
+
+#if HWY_HAVE_SCALABLE
+#if HWY_TARGET == HWY_RVV
+    const int expected_pow2_mask = kSupportedU8Pow2Mask & (-lane_size);
+#else
+    const int expected_pow2_mask = 8;
+#endif
+    HWY_ASSERT(per_lane_size_state->pow2_mask == expected_pow2_mask);
+#endif  // HWY_HAVE_SCALABLE
+  }
+}
+
+#undef HWY_DECLARE_FOREACH_VECTOR_TEST
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyForeachVecTest);
+HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForMaxPow2);
+HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForExtendableVectors);
+HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForShrinkableVectors);
+HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForGEVectors);
+HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForPromoteVectors);
+HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForDemoteVectors);
+HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForHalfVectors);
+HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForPartialVectors);
+HWY_EXPORT_AND_TEST_P(HwyForeachVecTest, TestAllForPartialFixedOrFullVectors);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/hwy_gtest.h

@@ -0,0 +1,645 @@
+// Copyright 2021 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TESTS_HWY_GTEST_H_
+#define HWY_TESTS_HWY_GTEST_H_
+
+// Adapter/replacement for GUnit to run tests for all targets.
+
+#include "hwy/base.h"
+
+// Allow opting out of GUnit.
+#ifndef HWY_TEST_STANDALONE
+// GUnit and its dependencies no longer support MSVC.
+#if HWY_COMPILER_MSVC
+#define HWY_TEST_STANDALONE 1
+#else
+#define HWY_TEST_STANDALONE 0
+#endif  // HWY_COMPILER_MSVC
+#endif  // HWY_TEST_STANDALONE
+
+#include <stdint.h>
+#if HWY_TEST_STANDALONE
+#include <stdlib.h>
+#include <string.h>
+#endif
+
+#include <string>
+#include <tuple>
+
+#if !HWY_TEST_STANDALONE
+#include "gtest/gtest.h"  // IWYU pragma: export
+#endif
+#include "hwy/detect_targets.h"
+#include "hwy/targets.h"
+
+namespace hwy {
+
+#if !HWY_TEST_STANDALONE
+
+// googletest before 1.10 didn't define INSTANTIATE_TEST_SUITE_P() but instead
+// used INSTANTIATE_TEST_CASE_P which is now deprecated.
+#ifdef INSTANTIATE_TEST_SUITE_P
+#define HWY_GTEST_INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_SUITE_P
+#else
+#define HWY_GTEST_INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_CASE_P
+#endif
+
+// Helper class to run parametric tests using the hwy target as parameter. To
+// use this define the following in your test:
+//   class MyTestSuite : public TestWithParamTarget {
+//    ...
+//   };
+//   HWY_TARGET_INSTANTIATE_TEST_SUITE_P(MyTestSuite);
+//   TEST_P(MyTestSuite, MyTest) { ... }
+class TestWithParamTarget : public testing::TestWithParam<int64_t> {
+ protected:
+  void SetUp() override { SetSupportedTargetsForTest(GetParam()); }
+
+  void TearDown() override {
+    // Check that the parametric test calls SupportedTargets() when the source
+    // was compiled with more than one target. In the single-target case only
+    // static dispatch will be used anyway.
+#if (HWY_TARGETS & (HWY_TARGETS - 1)) != 0
+    EXPECT_TRUE(GetChosenTarget().IsInitialized())
+        << "This hwy target parametric test doesn't use dynamic-dispatch and "
+           "doesn't need to be parametric.";
+#endif
+    SetSupportedTargetsForTest(0);
+  }
+};
+
+// Function to convert the test parameter of a TestWithParamTarget for
+// displaying it in the gtest test name.
+static inline std::string TestParamTargetName(
+    const testing::TestParamInfo<int64_t>& info) {
+  return TargetName(info.param);
+}
+
+#define HWY_TARGET_INSTANTIATE_TEST_SUITE_P(suite)              \
+  HWY_GTEST_INSTANTIATE_TEST_SUITE_P(                           \
+      suite##Group, suite,                                      \
+      testing::ValuesIn(::hwy::SupportedAndGeneratedTargets()), \
+      ::hwy::TestParamTargetName)
+
+// Helper class similar to TestWithParamTarget to run parametric tests that
+// depend on the target and another parametric test. If you need to use multiple
+// extra parameters use a std::tuple<> of them and ::testing::Generate(...) as
+// the generator. To use this class define the following in your test:
+//   class MyTestSuite : public TestWithParamTargetT<int> {
+//    ...
+//   };
+//   HWY_TARGET_INSTANTIATE_TEST_SUITE_P_T(MyTestSuite, ::testing::Range(0, 9));
+//   TEST_P(MyTestSuite, MyTest) { ... GetParam() .... }
+template <typename T>
+class TestWithParamTargetAndT
+    : public ::testing::TestWithParam<std::tuple<int64_t, T>> {
+ public:
+  // Expose the parametric type here so it can be used by the
+  // HWY_TARGET_INSTANTIATE_TEST_SUITE_P_T macro.
+  using HwyParamType = T;
+
+ protected:
+  void SetUp() override {
+    SetSupportedTargetsForTest(std::get<0>(
+        ::testing::TestWithParam<std::tuple<int64_t, T>>::GetParam()));
+  }
+
+  void TearDown() override {
+    // Check that the parametric test calls SupportedTargets() when the source
+    // was compiled with more than one target. In the single-target case only
+    // static dispatch will be used anyway.
+#if (HWY_TARGETS & (HWY_TARGETS - 1)) != 0
+    EXPECT_TRUE(GetChosenTarget().IsInitialized())
+        << "This hwy target parametric test doesn't use dynamic-dispatch and "
+           "doesn't need to be parametric.";
+#endif
+    SetSupportedTargetsForTest(0);
+  }
+
+  T GetParam() {
+    return std::get<1>(
+        ::testing::TestWithParam<std::tuple<int64_t, T>>::GetParam());
+  }
+};
+
+template <typename T>
+std::string TestParamTargetNameAndT(
+    const testing::TestParamInfo<std::tuple<int64_t, T>>& info) {
+  return std::string(TargetName(std::get<0>(info.param))) + "_" +
+         ::testing::PrintToString(std::get<1>(info.param));
+}
+
+#define HWY_TARGET_INSTANTIATE_TEST_SUITE_P_T(suite, generator)     \
+  HWY_GTEST_INSTANTIATE_TEST_SUITE_P(                               \
+      suite##Group, suite,                                          \
+      ::testing::Combine(                                           \
+          testing::ValuesIn(::hwy::SupportedAndGeneratedTargets()), \
+          generator),                                               \
+      ::hwy::TestParamTargetNameAndT<suite::HwyParamType>)
+
+// Helper macro to export a function and define a test that tests it. This is
+// equivalent to do a HWY_EXPORT of a void(void) function and run it in a test:
+//   class MyTestSuite : public TestWithParamTarget {
+//    ...
+//   };
+//   HWY_TARGET_INSTANTIATE_TEST_SUITE_P(MyTestSuite);
+//   HWY_EXPORT_AND_TEST_P(MyTestSuite, MyTest);
+#define HWY_EXPORT_AND_TEST_P(suite, func_name)                   \
+  HWY_EXPORT(func_name);                                          \
+  TEST_P(suite, func_name) { HWY_DYNAMIC_DISPATCH(func_name)(); } \
+  static_assert(true, "For requiring trailing semicolon")
+
+#define HWY_EXPORT_AND_TEST_P_T(suite, func_name)                           \
+  HWY_EXPORT(func_name);                                                    \
+  TEST_P(suite, func_name) { HWY_DYNAMIC_DISPATCH(func_name)(GetParam()); } \
+  static_assert(true, "For requiring trailing semicolon")
+
+#define HWY_BEFORE_TEST(suite)                      \
+  class suite : public hwy::TestWithParamTarget {}; \
+  HWY_TARGET_INSTANTIATE_TEST_SUITE_P(suite);       \
+  static_assert(true, "For requiring trailing semicolon")
+
+#define HWY_AFTER_TEST() static_assert(true, "For requiring trailing semicolon")
+
+#define HWY_TEST_MAIN() static_assert(true, "For requiring trailing semicolon")
+
+#else  // HWY_TEST_STANDALONE
+
+namespace {
+
+class GTestFilterPattern {
+ private:
+  struct FilterPatternComponent {
+    bool has_match_any_string_wildcard;
+    size_t min_num_of_leading_chars_to_match;
+    const char* subpattern_start;
+    size_t subpattern_to_match_len;
+  };
+
+ public:
+  GTestFilterPattern() = default;
+  GTestFilterPattern(const GTestFilterPattern&) = default;
+  GTestFilterPattern(GTestFilterPattern&&) = default;
+  GTestFilterPattern& operator=(const GTestFilterPattern&) = default;
+  GTestFilterPattern& operator=(GTestFilterPattern&&) = default;
+  GTestFilterPattern(const char* gtest_filter_pattern,
+                     size_t gtest_filter_pattern_len);
+
+ public:
+  bool Matches(const char* test_name,
+               size_t remaining_to_match_len) const noexcept;
+
+ private:
+  std::vector<FilterPatternComponent> pattern_components_;
+  size_t min_test_name_len_;
+};
+
+GTestFilterPattern::GTestFilterPattern(const char* gtest_filter_pattern_str,
+                                       size_t gtest_filter_pattern_len) {
+  size_t min_test_name_len = 0;
+  const char* const gtest_filter_pattern_str_end =
+      gtest_filter_pattern_str + gtest_filter_pattern_len;
+
+  for (const char* subpattern_start = gtest_filter_pattern_str;
+       subpattern_start != gtest_filter_pattern_str_end;) {
+    size_t min_num_of_leading_chars_to_match = 0;
+    bool has_match_any_string_wildcard = false;
+
+    // Advance subpattern_start past any '*' or '?' characters
+    for (char first_non_wildcard_ch;
+         subpattern_start != gtest_filter_pattern_str_end &&
+         ((first_non_wildcard_ch = (*subpattern_start)) == '*' ||
+          first_non_wildcard_ch == '?');
+         ++subpattern_start) {
+      if (first_non_wildcard_ch == '*') {
+        has_match_any_string_wildcard = true;
+      } else {
+        ++min_num_of_leading_chars_to_match;
+      }
+    }
+
+    // If subpattern_start != gtest_filter_pattern_str_end is true,
+    // subpattern_start points to a non-wildcard character
+
+    const char* subpattern_end;
+    if ((subpattern_end = subpattern_start) != gtest_filter_pattern_str_end) {
+      // Find the next '*' character past subpattern_start if there are any
+      // '*' characters past subpattern_start in the subpattern
+      while ((++subpattern_end) != gtest_filter_pattern_str_end &&
+             (*subpattern_end) != '*') {
+      }
+
+      // Decrement subpattern_end while subpattern_end != subpattern_start + 1
+      // is true and subpattern_end - 1 points to a '?' character
+      for (;
+           subpattern_end != subpattern_start + 1 && subpattern_end[-1] == '?';
+           --subpattern_end) {
+      }
+
+      // subpattern_end - 1 now points to a non-wildcard character
+    }
+
+    // Add the current filter pattern component to pattern_components_
+    const FilterPatternComponent curr_filter_component{
+        has_match_any_string_wildcard, min_num_of_leading_chars_to_match,
+        subpattern_start,
+        static_cast<size_t>(subpattern_end - subpattern_start)};
+    pattern_components_.push_back(curr_filter_component);
+
+    // Advance to the next subpattern by setting subpattern_start to
+    // subpattern_end
+    subpattern_start = subpattern_end;
+  }
+
+  min_test_name_len_ = min_test_name_len;
+}
+
+bool GTestFilterPattern::Matches(const char* test_name,
+                                 size_t remaining_to_match_len) const noexcept {
+  if (remaining_to_match_len < min_test_name_len_) {
+    return false;
+  }
+
+  const size_t num_of_pattern_components = pattern_components_.size();
+  for (size_t i = num_of_pattern_components; i != 0; i--) {
+    const FilterPatternComponent& curr_pattern_component =
+        pattern_components_[i - 1];
+    const size_t subpattern_to_match_len =
+        curr_pattern_component.subpattern_to_match_len;
+    const size_t min_num_to_match =
+        curr_pattern_component.min_num_of_leading_chars_to_match +
+        subpattern_to_match_len;
+
+    if (remaining_to_match_len < min_num_to_match) {
+      return false;
+    }
+
+    if (subpattern_to_match_len != 0) {
+      const bool is_restartable_subpattern_match =
+          i != num_of_pattern_components &&
+          pattern_components_[i].has_match_any_string_wildcard;
+
+      const char* subpattern_start = curr_pattern_component.subpattern_start;
+
+      bool restart_match;
+      do {
+        restart_match = false;
+        const size_t test_name_match_substr_offset =
+            remaining_to_match_len - subpattern_to_match_len;
+
+        bool matches_subpattern =
+            test_name[test_name_match_substr_offset] == subpattern_start[0] &&
+            test_name[test_name_match_substr_offset + subpattern_to_match_len -
+                      1] == subpattern_start[subpattern_to_match_len - 1];
+        if (matches_subpattern) {
+          for (size_t i = 1; i != subpattern_to_match_len - 1; i++) {
+            char c1 = test_name[test_name_match_substr_offset + i];
+            char c2 = subpattern_start[i];
+
+            if (c1 != c2 && c2 != '?') {
+              matches_subpattern = false;
+              break;
+            }
+          }
+        }
+
+        if (!matches_subpattern) {
+          restart_match = is_restartable_subpattern_match &&
+                          (--remaining_to_match_len) >= min_num_to_match;
+          if (!restart_match) {
+            return false;
+          }
+        }
+      } while (restart_match);
+    }
+
+    remaining_to_match_len -= min_num_to_match;
+  }
+
+  return true;
+}
+
+std::vector<GTestFilterPattern>& PositiveGTestFilterPatterns() {
+  static std::vector<GTestFilterPattern> positive_test_filter_patterns;
+  return positive_test_filter_patterns;
+}
+
+std::vector<GTestFilterPattern>& NegativeGTestFilterPatterns() {
+  static std::vector<GTestFilterPattern> negative_test_filter_patterns;
+  return negative_test_filter_patterns;
+}
+
+// ShouldOnlyListHighwayTestNames() returns true if the names of the unit tests
+// should be outputted without executing the unit tests.
+//
+// Otherwise, if the unit tests should be executed,
+// ShouldOnlyListHighwayTestNames() returns true.
+bool& ShouldOnlyListHighwayTestNames() {
+  static bool should_only_list_test_names = false;
+  return should_only_list_test_names;
+}
+
+// ParseGTestFilterPatterns parses the filter patterns passed into the
+// --gtest_filter= command line argument (or set by the GTEST_FILTER environment
+// variable if there is no --gtest_filter= command line argument present)
+static void ParseGTestFilterPatterns(const char* gtest_filter_str) {
+  std::vector<GTestFilterPattern>* ptr_to_positive_test_filter_patterns =
+      &PositiveGTestFilterPatterns();
+  std::vector<GTestFilterPattern>* ptr_to_negative_test_filter_patterns =
+      &NegativeGTestFilterPatterns();
+
+  std::vector<GTestFilterPattern>* ptr_to_vector_to_append_pattern_to =
+      ptr_to_positive_test_filter_patterns;
+
+  char first_filter_pattern_ch;
+  bool colon_delimiter_encountered = false;
+  while ((first_filter_pattern_ch = (*gtest_filter_str)) != '\0') {
+    if (first_filter_pattern_ch == ':') {
+      colon_delimiter_encountered = true;
+      ++gtest_filter_str;
+      continue;
+    }
+
+    if (first_filter_pattern_ch == '-' &&
+        ptr_to_vector_to_append_pattern_to ==
+            ptr_to_positive_test_filter_patterns) {
+      ptr_to_vector_to_append_pattern_to = ptr_to_negative_test_filter_patterns;
+      if (!colon_delimiter_encountered &&
+          ptr_to_positive_test_filter_patterns->empty()) {
+        ptr_to_positive_test_filter_patterns->emplace_back("*", 1);
+      }
+      ++gtest_filter_str;
+      continue;
+    }
+
+    size_t filter_pattern_len = 1;
+
+    // Find the next filter pattern delimiter character or null terminator
+    for (char filter_pattern_end_ch;
+         (filter_pattern_end_ch = gtest_filter_str[filter_pattern_len]) !=
+             '\0' &&
+         filter_pattern_end_ch != ':' &&
+         (filter_pattern_end_ch != '-' ||
+          ptr_to_vector_to_append_pattern_to ==
+              ptr_to_negative_test_filter_patterns);
+         ++filter_pattern_len) {
+    }
+
+    // Add the current filter pattern to *ptr_to_vector_to_append_pattern_to
+    ptr_to_vector_to_append_pattern_to->emplace_back(gtest_filter_str,
+                                                     filter_pattern_len);
+
+    // Advance gtest_filter_str by filter_pattern_len chars
+    gtest_filter_str += filter_pattern_len;
+  }
+}
+
+// TestNameMatchesGTestFilter(test_name) returns true if any of the following
+// are true:
+// - test_name matches the filter passed in by the last --gtest_filter= command
+//   line argument if any --gtest_filter= arguments are present on the command
+//   line
+// - test_name matches the filter set by the GTEST_FILTER environment variable
+//   if no --gtest_filter= commands were passed into the command line and
+//   the GTEST_FILTER environment variable is set
+// - there were no --gtest_filter= arguments on the command line and the
+//   GTEST_FILTER environment variable is not set
+//
+// Otherwise, TestNameMatchesGTestFilter(test_name) returns false
+static HWY_INLINE HWY_MAYBE_UNUSED bool TestNameMatchesGTestFilter(
+    const char* test_name, size_t test_name_len) {
+  for (const GTestFilterPattern& negative_pattern :
+       NegativeGTestFilterPatterns()) {
+    if (negative_pattern.Matches(test_name, test_name_len)) {
+      return false;
+    }
+  }
+  for (const GTestFilterPattern& positive_pattern :
+       PositiveGTestFilterPatterns()) {
+    if (positive_pattern.Matches(test_name, test_name_len)) {
+      return true;
+    }
+  }
+  return false;
+}
+static HWY_INLINE HWY_MAYBE_UNUSED bool TestNameMatchesGTestFilter(
+    const char* test_name) {
+  return TestNameMatchesGTestFilter(test_name, strlen(test_name));
+}
+static HWY_INLINE HWY_MAYBE_UNUSED bool TestNameMatchesGTestFilter(
+    const std::string& test_name) {
+  return TestNameMatchesGTestFilter(test_name.data(), test_name.length());
+}
+
+// InitTestProgramOptions processes the GTEST_FILTER environment variable, the
+// --gtest_filter= command line argument, and the --gtest_list_tests command
+// line argument
+static HWY_MAYBE_UNUSED void InitTestProgramOptions(
+    const int argc, const char* const* const argv) {
+  // Suppress warning that is normally emitted by MSVC by the getenv call below
+  HWY_DIAGNOSTICS(push)
+#if HWY_COMPILER_MSVC || HWY_COMPILER_CLANGCL
+  HWY_DIAGNOSTICS_OFF(disable : 4996, ignored "-Wdeprecated-declarations")
+#endif
+  const char* gtest_filter = getenv("GTEST_FILTER");
+  HWY_DIAGNOSTICS(pop)
+
+  if (!gtest_filter) {
+    gtest_filter = "*";
+  }
+  for (int i = 1; i < argc; i++) {
+    const char* const curr_arg = argv[i];
+    if (!curr_arg) {
+      break;
+    }
+    if (curr_arg[0] == '-' && curr_arg[1] == '-' && curr_arg[2] == 'g' &&
+        curr_arg[3] == 't' && curr_arg[4] == 'e' && curr_arg[5] == 's' &&
+        curr_arg[6] == 't' && curr_arg[7] == '_') { /* --gtest_ */
+      switch (curr_arg[8]) {
+        case 'f':
+          if (curr_arg[9] == 'i' && curr_arg[10] == 'l' &&
+              curr_arg[11] == 't' && curr_arg[12] == 'e' &&
+              curr_arg[13] == 'r' && curr_arg[14] == '=') {
+            // If the --gtest_filter= command line option is specified, only
+            // execute the tests that match the specified filter
+            gtest_filter = curr_arg + 15;
+          }
+          break;
+        case 'l':
+          if (curr_arg[9] == 'i' && curr_arg[10] == 's' &&
+              curr_arg[11] == 't' && curr_arg[12] == '_' &&
+              curr_arg[13] == 't' && curr_arg[14] == 'e' &&
+              curr_arg[15] == 's' && curr_arg[16] == 't' &&
+              curr_arg[17] == 's' && curr_arg[18] == '\0') {
+            // If the --gtest_list_tests command line option is specified,
+            // output the name of the unit tests but do not execute the unit
+            // tests
+            ShouldOnlyListHighwayTestNames() = true;
+            break;
+          }
+        default:
+          break;
+      }
+    }
+  }
+
+  // Initialize PositiveGTestFilterPatterns() and NegativeGTestFilterPatterns()
+  // from gtest_filter
+  ParseGTestFilterPatterns(gtest_filter);
+}
+
+}  // namespace
+
+// Cannot be a function, otherwise the HWY_EXPORT table defined here will not
+// be visible to HWY_DYNAMIC_DISPATCH.
+#define HWY_EXPORT_AND_TEST_P(suite, func_name)                         \
+  full_test_name = #suite;                                              \
+  full_test_name += "Group/";                                           \
+  full_test_name += #suite;                                             \
+  full_test_name += '.';                                                \
+  full_test_name_suite_prefix_len = full_test_name.length();            \
+  full_test_name += #func_name;                                         \
+  full_test_name += '/';                                                \
+  full_test_name_prefix_len = full_test_name.length();                  \
+  HWY_EXPORT(func_name);                                                \
+  hwy::SetSupportedTargetsForTest(0);                                   \
+  for (int64_t target : hwy::SupportedAndGeneratedTargets()) {          \
+    hwy::SetSupportedTargetsForTest(target);                            \
+    full_test_name.resize(full_test_name_prefix_len);                   \
+    full_test_name += hwy::TargetName(target);                          \
+    if (hwy::TestNameMatchesGTestFilter(full_test_name)) {              \
+      if (hwy::ShouldOnlyListHighwayTestNames()) {                      \
+        const char* full_test_name_c_str = full_test_name.c_str();      \
+        if (need_to_output_suite_name) {                                \
+          need_to_output_suite_name = false;                            \
+          printf("%sGroup/%s.\n", #suite, #suite);                      \
+        }                                                               \
+        printf("  %s\n",                                                \
+               full_test_name_c_str + full_test_name_suite_prefix_len); \
+      } else {                                                          \
+        fprintf(stderr, "=== %s for %s:\n", #func_name,                 \
+                hwy::TargetName(target));                               \
+        HWY_DYNAMIC_DISPATCH(func_name)();                              \
+      }                                                                 \
+    }                                                                   \
+  }                                                                     \
+  /* Disable the mask after the test. */                                \
+  hwy::SetSupportedTargetsForTest(0);                                   \
+  static_assert(true, "For requiring trailing semicolon")
+
+// HWY_BEFORE_TEST may reside inside a namespace, but HWY_AFTER_TEST will define
+// a main() at namespace scope that wants to call into that namespace, so stash
+// the function address in a singleton defined in namespace hwy.
+using VoidFunc = void (*)(void);
+
+VoidFunc& GetRunAll() {
+  static VoidFunc func;
+  return func;
+}
+
+struct RegisterRunAll {
+  RegisterRunAll(VoidFunc func) { hwy::GetRunAll() = func; }
+};
+
+#define HWY_BEFORE_TEST(suite)                                 \
+  void RunAll();                                               \
+  static hwy::RegisterRunAll HWY_CONCAT(reg_, suite)(&RunAll); \
+  void RunAll() {                                              \
+    std::string full_test_name;                                \
+    size_t full_test_name_suite_prefix_len;                    \
+    size_t full_test_name_prefix_len;                          \
+    bool need_to_output_suite_name = true;                     \
+    static_assert(true, "For requiring trailing semicolon")
+
+// Must be followed by semicolon, then a closing brace for ONE namespace.
+#define HWY_AFTER_TEST()                          \
+  } /* RunAll*/                                   \
+  } /* namespace */                               \
+  int main(int argc, char** argv) {               \
+    hwy::InitTestProgramOptions(argc, argv);      \
+    hwy::GetRunAll()();                           \
+    if (!hwy::ShouldOnlyListHighwayTestNames()) { \
+      fprintf(stderr, "Success.\n");              \
+    }                                             \
+    return 0
+
+// -------------------- Non-SIMD test cases:
+
+struct FuncAndName {
+  VoidFunc func;
+  const char* name;
+  const char* suite_name;
+  const char* full_name;
+};
+
+// Singleton of registered tests to be run by HWY_TEST_MAIN
+std::vector<FuncAndName>& GetFuncAndNames() {
+  static std::vector<FuncAndName> vec;
+  return vec;
+}
+
+// For use by TEST; adds to the list.
+struct RegisterTest {
+  RegisterTest(VoidFunc func, const char* name, const char* suite_name,
+               const char* full_name) {
+    hwy::GetFuncAndNames().push_back({func, name, suite_name, full_name});
+  }
+};
+
+// Registers a function to be called by `HWY_TEST_MAIN`.
+#define TEST(suite, func)                                                    \
+  void func();                                                               \
+  static hwy::RegisterTest HWY_CONCAT(                                       \
+      reg_, func)({&func, #func, #suite, #suite "Group/" #suite "." #func}); \
+  void func()
+
+// Expands to a main() that calls all TEST. Must reside at namespace scope.
+#define HWY_TEST_MAIN()                                                    \
+  int main(int argc, char** argv) {                                        \
+    hwy::InitTestProgramOptions(argc, argv);                               \
+    const char* suite_name_of_prev_test = nullptr;                         \
+    for (const auto& func_and_name : hwy::GetFuncAndNames()) {             \
+      if (hwy::TestNameMatchesGTestFilter(func_and_name.full_name)) {      \
+        if (hwy::ShouldOnlyListHighwayTestNames()) {                       \
+          const char* const suite_name_of_curr_test =                      \
+              func_and_name.suite_name;                                    \
+          if (suite_name_of_curr_test != suite_name_of_prev_test &&        \
+              (!suite_name_of_prev_test ||                                 \
+               strcmp(suite_name_of_prev_test, suite_name_of_curr_test) == \
+                   0)) {                                                   \
+            suite_name_of_prev_test = suite_name_of_curr_test;             \
+            printf("%sGroup/%s.\n", suite_name_of_curr_test,               \
+                   suite_name_of_curr_test);                               \
+          }                                                                \
+          printf("  %s\n", func_and_name.name);                            \
+        } else {                                                           \
+          fprintf(stderr, "=== %s:\n", func_and_name.name);                \
+          func_and_name.func();                                            \
+        }                                                                  \
+      }                                                                    \
+    }                                                                      \
+    if (!hwy::ShouldOnlyListHighwayTestNames()) {                          \
+      fprintf(stderr, "Success.\n");                                       \
+    }                                                                      \
+    return 0;                                                              \
+  }                                                                        \
+  static_assert(true, "For requiring trailing semicolon")
+
+#endif  // HWY_TEST_STANDALONE
+
+}  // namespace hwy
+
+#endif  // HWY_TESTS_HWY_GTEST_H_

deps/v8/third_party/highway/src/hwy/tests/if_test.cc

@@ -0,0 +1,372 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/if_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestIfThenElse {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    // TODO(janwas): file compiler bug report
+#if HWY_COMPILER_CLANG && (HWY_COMPILER_CLANG < 1800) && HWY_ARCH_ARM
+    if (IsSpecialFloat<T>()) return;
+#endif
+
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(d);
+    auto in1 = AllocateAligned<T>(N);
+    auto in2 = AllocateAligned<T>(N);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(in1 && in2 && bool_lanes && expected);
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        in1[i] = ConvertScalarTo<T>(Random32(&rng));
+        in2[i] = ConvertScalarTo<T>(Random32(&rng));
+        bool_lanes[i] = (Random32(&rng) & 16) ? TI(1) : TI(0);
+      }
+
+      const auto v1 = Load(d, in1.get());
+      const auto v2 = Load(d, in2.get());
+      const auto mask = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+
+      for (size_t i = 0; i < N; ++i) {
+        expected[i] = bool_lanes[i] ? in1[i] : in2[i];
+      }
+      HWY_ASSERT_VEC_EQ(d, expected.get(), IfThenElse(mask, v1, v2));
+
+      for (size_t i = 0; i < N; ++i) {
+        expected[i] = bool_lanes[i] ? in1[i] : ConvertScalarTo<T>(0);
+      }
+      HWY_ASSERT_VEC_EQ(d, expected.get(), IfThenElseZero(mask, v1));
+
+      for (size_t i = 0; i < N; ++i) {
+        expected[i] = bool_lanes[i] ? ConvertScalarTo<T>(0) : in2[i];
+      }
+      HWY_ASSERT_VEC_EQ(d, expected.get(), IfThenZeroElse(mask, v2));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllIfThenElse() {
+  ForAllTypesAndSpecial(ForPartialVectors<TestIfThenElse>());
+}
+
+struct TestIfVecThenElse {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TU = MakeUnsigned<T>;  // For all-one mask
+    const Rebind<TU, D> du;
+    const size_t N = Lanes(d);
+    auto in1 = AllocateAligned<T>(N);
+    auto in2 = AllocateAligned<T>(N);
+    auto vec_lanes = AllocateAligned<TU>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(in1 && in2 && vec_lanes && expected);
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        in1[i] = ConvertScalarTo<T>(Random32(&rng));
+        in2[i] = ConvertScalarTo<T>(Random32(&rng));
+        vec_lanes[i] = (Random32(&rng) & 16) ? static_cast<TU>(~TU(0)) : TU(0);
+      }
+
+      const auto v1 = Load(d, in1.get());
+      const auto v2 = Load(d, in2.get());
+      const auto vec = BitCast(d, Load(du, vec_lanes.get()));
+
+      for (size_t i = 0; i < N; ++i) {
+        expected[i] = vec_lanes[i] ? in1[i] : in2[i];
+      }
+      HWY_ASSERT_VEC_EQ(d, expected.get(), IfVecThenElse(vec, v1, v2));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllIfVecThenElse() {
+  ForAllTypes(ForPartialVectors<TestIfVecThenElse>());
+}
+
+class TestBitwiseIfThenElse {
+ private:
+  template <class T>
+  static T ValueFromBitPattern(hwy::FloatTag /* type_tag */, T /* unused */,
+                               uint64_t bits) {
+    using TI = MakeSigned<T>;
+    return ConvertScalarTo<T>(
+        ConvertScalarTo<T>(static_cast<TI>(bits & MantissaMask<T>())) +
+        MantissaEnd<T>());
+  }
+  template <class T>
+  static MakeUnsigned<T> ValueFromBitPattern(hwy::NonFloatTag /* type_tag */,
+                                             T /* unused */, uint64_t bits) {
+    return static_cast<MakeUnsigned<T>>(bits);
+  }
+
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TU = MakeUnsigned<T>;
+    using TVal = RemoveConst<decltype(ValueFromBitPattern(IsFloatTag<T>(), T(),
+                                                          uint64_t{0}))>;
+    static_assert(!IsFloat<T>() || IsSame<TVal, T>(),
+                  "TVal should be the same as T if T is a floating-point type");
+    static_assert(IsFloat<T>() || IsSame<TVal, TU>(),
+                  "TVal should be the same as TU if T is a integer type");
+
+    static TVal a0 = ValueFromBitPattern(IsFloatTag<T>(), T(),
+                                         uint64_t{0x0FF00FF00FF00FF0u});
+    static TVal b0 = ValueFromBitPattern(IsFloatTag<T>(), T(),
+                                         uint64_t{0x33CC33CC33CC33CCu});
+    static TVal c0 = ValueFromBitPattern(IsFloatTag<T>(), T(),
+                                         uint64_t{0x55AA55AA55AA55AAu});
+    static TVal a1 = ValueFromBitPattern(IsFloatTag<T>(), T(),
+                                         uint64_t{0xF00FF00FF00FF00Fu});
+    static TVal b1 = ValueFromBitPattern(IsFloatTag<T>(), T(),
+                                         uint64_t{0xCC33CC33CC33CC33u});
+    static TVal c1 = ValueFromBitPattern(IsFloatTag<T>(), T(),
+                                         uint64_t{0xAA55AA55AA55AA55u});
+
+    const RebindToUnsigned<decltype(d)> du;
+    const Rebind<TVal, decltype(d)> d_val;
+    const auto v_a0 = BitCast(d, Set(d_val, a0));
+    const auto v_b0 = BitCast(d, Set(d_val, b0));
+    const auto v_c0 = BitCast(d, Set(d_val, c0));
+
+    const auto v_a1 = BitCast(d, Set(d_val, a1));
+    const auto v_b1 = BitCast(d, Set(d_val, b1));
+    const auto v_c1 = BitCast(d, Set(d_val, c1));
+
+    static TVal expected_1 = ValueFromBitPattern(IsFloatTag<T>(), T(),
+                                                 uint64_t{0x53CA53CA53CA53CAu});
+    HWY_ASSERT_VEC_EQ(d, BitCast(d, Set(d_val, expected_1)),
+                      BitwiseIfThenElse(v_a0, v_b0, v_c0));
+
+    static TVal expected_2 = ValueFromBitPattern(IsFloatTag<T>(), T(),
+                                                 uint64_t{0xCA53CA53CA53CA53u});
+    HWY_ASSERT_VEC_EQ(d, BitCast(d, Set(d_val, expected_2)),
+                      BitwiseIfThenElse(v_a1, v_b1, v_c1));
+
+    static TVal expected_3 = ValueFromBitPattern(IsFloatTag<T>(), T(),
+                                                 uint64_t{0x1DB81DB81DB81DB8u});
+    HWY_ASSERT_VEC_EQ(d, BitCast(d, Set(d_val, expected_3)),
+                      BitwiseIfThenElse(v_b1, v_a0, v_c0));
+
+    const auto v_all_ones = BitCast(d, Set(du, static_cast<TU>(-1)));
+    HWY_ASSERT_VEC_EQ(d, v_a0, BitwiseIfThenElse(v_all_ones, v_a0, v_b0));
+    HWY_ASSERT_VEC_EQ(d, v_b0, BitwiseIfThenElse(Zero(d), v_a0, v_b0));
+  }
+};
+
+HWY_NOINLINE void TestAllBitwiseIfThenElse() {
+  ForAllTypes(ForPartialVectors<TestBitwiseIfThenElse>());
+}
+
+struct TestZeroIfNegative {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v0 = Zero(d);
+    auto vp = Iota(d, 1);
+    auto vn = Iota(d, ConvertScalarTo<T>((sizeof(T) >= 2) ? -10000 : -100));
+
+    if (MaxLanes(d) > (sizeof(T) >= 2 ? 10000 : 100)) {
+      const auto vsignbit = SignBit(d);
+      vp = AndNot(vsignbit, vp);
+      vn = Or(vn, vsignbit);
+    }
+
+    // Zero and positive remain unchanged
+    HWY_ASSERT_VEC_EQ(d, v0, ZeroIfNegative(v0));
+    HWY_ASSERT_VEC_EQ(d, vp, ZeroIfNegative(vp));
+
+    // Negative are all replaced with zero
+    HWY_ASSERT_VEC_EQ(d, v0, ZeroIfNegative(vn));
+  }
+};
+
+HWY_NOINLINE void TestAllZeroIfNegative() {
+  ForFloatTypes(ForPartialVectors<TestZeroIfNegative>());
+  ForSignedTypes(ForPartialVectors<TestZeroIfNegative>());
+}
+
+struct TestIfNegative {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v0 = Zero(d);
+    const auto vp = Iota(d, 1);
+    const auto vsignbit = SignBit(d);
+    const auto vn = Or(vp, vsignbit);
+
+    // Zero and positive remain unchanged
+    HWY_ASSERT_VEC_EQ(d, v0, IfNegativeThenElse(v0, vn, v0));
+    HWY_ASSERT_VEC_EQ(d, vn, IfNegativeThenElse(v0, v0, vn));
+    HWY_ASSERT_VEC_EQ(d, vp, IfNegativeThenElse(vp, vn, vp));
+    HWY_ASSERT_VEC_EQ(d, vn, IfNegativeThenElse(vp, vp, vn));
+
+    // Negative are replaced with 2nd arg
+    HWY_ASSERT_VEC_EQ(d, v0, IfNegativeThenElse(vn, v0, vp));
+    HWY_ASSERT_VEC_EQ(d, vn, IfNegativeThenElse(vn, vn, v0));
+    HWY_ASSERT_VEC_EQ(d, vp, IfNegativeThenElse(vn, vp, vn));
+
+    const RebindToSigned<decltype(d)> di;
+    const RebindToUnsigned<decltype(d)> du;
+    using TU = TFromD<decltype(du)>;
+
+    const auto s1 = BitCast(d, ShiftLeft<sizeof(TU) * 8 - 1>(Iota(du, 1)));
+
+    const auto m1 = Xor3(vp, s1, BitCast(d, Set(du, TU{0x71})));
+    const auto x1 = Xor(vp, BitCast(d, Set(du, TU{0x2B})));
+    const auto x2 = Xor(vp, BitCast(d, Set(du, TU{0xE2})));
+    const auto m2 = Xor(m1, vsignbit);
+
+    const auto m1_s = BitCast(d, BroadcastSignBit(BitCast(di, m1)));
+
+    const auto expected_1 = BitwiseIfThenElse(m1_s, x1, x2);
+    const auto expected_2 = BitwiseIfThenElse(m1_s, x2, x1);
+
+    HWY_ASSERT_VEC_EQ(d, expected_1, IfNegativeThenElse(m1, x1, x2));
+    HWY_ASSERT_VEC_EQ(d, expected_2, IfNegativeThenElse(m2, x1, x2));
+
+    const auto expected_3 = And(m1_s, x1);
+    const auto expected_4 = AndNot(m1_s, x2);
+
+    HWY_ASSERT_VEC_EQ(d, expected_3, IfNegativeThenElseZero(m1, x1));
+    HWY_ASSERT_VEC_EQ(d, expected_3, IfNegativeThenZeroElse(m2, x1));
+
+    HWY_ASSERT_VEC_EQ(d, expected_4, IfNegativeThenZeroElse(m1, x2));
+    HWY_ASSERT_VEC_EQ(d, expected_4, IfNegativeThenElseZero(m2, x2));
+  }
+};
+
+HWY_NOINLINE void TestAllIfNegative() {
+  ForFloatTypes(ForPartialVectors<TestIfNegative>());
+  ForSignedTypes(ForPartialVectors<TestIfNegative>());
+}
+
+struct TestIfNegativeThenNegOrUndefIfZero {
+  template <class D, HWY_IF_LANES_LE_D(D, 1)>
+  static HWY_INLINE void TestMoreThan1LaneIfNegativeThenNegOrUndefIfZero(
+      D /*d*/, Vec<D> /*v1*/, Vec<D> /*v2*/) {}
+#if HWY_TARGET != HWY_SCALAR
+  // NOINLINE works around a clang compiler bug for PPC9 partial vectors.
+  template <class D, HWY_IF_LANES_GT_D(D, 1)>
+  static HWY_NOINLINE void TestMoreThan1LaneIfNegativeThenNegOrUndefIfZero(
+      D d, Vec<D> v1, Vec<D> v2) {
+#if HWY_HAVE_SCALABLE
+    if (Lanes(d) < 2) {
+      return;
+    }
+#endif
+
+    const Vec<D> v3 = InterleaveLower(d, v1, v1);
+    const Vec<D> v4 = InterleaveUpper(d, v1, v1);
+    const Vec<D> v5 = InterleaveLower(d, v1, v2);
+    const Vec<D> v6 = InterleaveUpper(d, v1, v2);
+    const Vec<D> v7 = InterleaveLower(d, v2, v1);
+    const Vec<D> v8 = InterleaveUpper(d, v2, v1);
+
+    HWY_ASSERT_VEC_EQ(d, v3, IfNegativeThenNegOrUndefIfZero(v3, v3));
+    HWY_ASSERT_VEC_EQ(d, v4, IfNegativeThenNegOrUndefIfZero(v4, v4));
+    HWY_ASSERT_VEC_EQ(d, v3, IfNegativeThenNegOrUndefIfZero(v5, v5));
+    HWY_ASSERT_VEC_EQ(d, v4, IfNegativeThenNegOrUndefIfZero(v6, v6));
+    HWY_ASSERT_VEC_EQ(d, v3, IfNegativeThenNegOrUndefIfZero(v7, v7));
+    HWY_ASSERT_VEC_EQ(d, v4, IfNegativeThenNegOrUndefIfZero(v8, v8));
+
+    HWY_ASSERT_VEC_EQ(d, v5, IfNegativeThenNegOrUndefIfZero(v3, v5));
+    HWY_ASSERT_VEC_EQ(d, v6, IfNegativeThenNegOrUndefIfZero(v4, v6));
+    HWY_ASSERT_VEC_EQ(d, v7, IfNegativeThenNegOrUndefIfZero(v3, v7));
+    HWY_ASSERT_VEC_EQ(d, v8, IfNegativeThenNegOrUndefIfZero(v4, v8));
+
+    const Vec<D> zero = Zero(d);
+    HWY_ASSERT_VEC_EQ(d, zero, IfNegativeThenNegOrUndefIfZero(v3, zero));
+    HWY_ASSERT_VEC_EQ(d, zero, IfNegativeThenNegOrUndefIfZero(v4, zero));
+    HWY_ASSERT_VEC_EQ(d, zero, IfNegativeThenNegOrUndefIfZero(v5, zero));
+    HWY_ASSERT_VEC_EQ(d, zero, IfNegativeThenNegOrUndefIfZero(v6, zero));
+    HWY_ASSERT_VEC_EQ(d, zero, IfNegativeThenNegOrUndefIfZero(v7, zero));
+    HWY_ASSERT_VEC_EQ(d, zero, IfNegativeThenNegOrUndefIfZero(v8, zero));
+  }
+#endif
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v1 = PositiveIota(d);
+    const auto v2 = Neg(v1);
+
+    HWY_ASSERT_VEC_EQ(d, v1, IfNegativeThenNegOrUndefIfZero(v1, v1));
+    HWY_ASSERT_VEC_EQ(d, v2, IfNegativeThenNegOrUndefIfZero(v1, v2));
+    HWY_ASSERT_VEC_EQ(d, v2, IfNegativeThenNegOrUndefIfZero(v2, v1));
+    HWY_ASSERT_VEC_EQ(d, v1, IfNegativeThenNegOrUndefIfZero(v2, v2));
+
+    const auto zero = Zero(d);
+    HWY_ASSERT_VEC_EQ(d, zero, IfNegativeThenNegOrUndefIfZero(zero, zero));
+    HWY_ASSERT_VEC_EQ(d, zero, IfNegativeThenNegOrUndefIfZero(v1, zero));
+    HWY_ASSERT_VEC_EQ(d, zero, IfNegativeThenNegOrUndefIfZero(v2, zero));
+
+    const auto vmin = Set(d, LowestValue<T>());
+    const auto vmax = Set(d, HighestValue<T>());
+
+    HWY_ASSERT_VEC_EQ(d, v2, IfNegativeThenNegOrUndefIfZero(vmin, v1));
+    HWY_ASSERT_VEC_EQ(d, v1, IfNegativeThenNegOrUndefIfZero(vmin, v2));
+    HWY_ASSERT_VEC_EQ(d, v1, IfNegativeThenNegOrUndefIfZero(vmax, v1));
+    HWY_ASSERT_VEC_EQ(d, v2, IfNegativeThenNegOrUndefIfZero(vmax, v2));
+
+    TestMoreThan1LaneIfNegativeThenNegOrUndefIfZero(d, v1, v2);
+  }
+};
+
+HWY_NOINLINE void TestAllIfNegativeThenNegOrUndefIfZero() {
+  ForSignedTypes(ForPartialVectors<TestIfNegativeThenNegOrUndefIfZero>());
+  ForFloatTypes(ForPartialVectors<TestIfNegativeThenNegOrUndefIfZero>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyIfTest);
+HWY_EXPORT_AND_TEST_P(HwyIfTest, TestAllIfThenElse);
+HWY_EXPORT_AND_TEST_P(HwyIfTest, TestAllIfVecThenElse);
+HWY_EXPORT_AND_TEST_P(HwyIfTest, TestAllBitwiseIfThenElse);
+HWY_EXPORT_AND_TEST_P(HwyIfTest, TestAllZeroIfNegative);
+HWY_EXPORT_AND_TEST_P(HwyIfTest, TestAllIfNegative);
+HWY_EXPORT_AND_TEST_P(HwyIfTest, TestAllIfNegativeThenNegOrUndefIfZero);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/in_range_float_to_int_conv_test.cc

@@ -0,0 +1,644 @@
+// Copyright 2024 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "hwy/base.h"
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/in_range_float_to_int_conv_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/nanobenchmark.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+// HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(condition, msg) checks that condition
+// is true using static_assert if constexpr BitCastScalar is available and
+// Highway is being compiled in C++11 mode.
+//
+// Otherwise, if constexpr BitCastScalar is not available or Highway is being
+// compiled in C++11 mode, HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(condition,
+// msg) checks that condition is true using a run-time assertion.
+#if (HWY_HAS_BUILTIN(__builtin_bit_cast) || HWY_COMPILER_MSVC >= 1926) && \
+    __cpp_constexpr >= 201304L
+#define HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(condition, msg) \
+  static_assert((condition), msg)
+#else
+#define HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(condition, msg) \
+  do {                                                          \
+    if (HWY_UNLIKELY(!(condition))) {                           \
+      HWY_ABORT("Assert %s failed:\n%s", #condition, msg);      \
+    }                                                           \
+  } while (0)
+#endif
+
+template <class TTo>
+class TestConvertInRangeFloatToInt {
+  static_assert(!IsFloat<TTo>() && !IsSpecialFloat<TTo>(),
+                "TTo must be an integer type");
+
+ private:
+  // LargestLt1FloatVal<T>() returns the largest value of T that is less than 1
+  template <class T, HWY_IF_FLOAT_OR_SPECIAL(T)>
+  static HWY_INLINE HWY_BITCASTSCALAR_CONSTEXPR T LargestLt1FloatVal() {
+    using TU = MakeUnsigned<T>;
+    return BitCastScalar<T>(
+        static_cast<TU>(BitCastScalar<TU>(ConvertScalarTo<T>(1)) - 1u));
+  }
+
+  // RoundedDownFloatSum(hi, lo) returns the rounded down value of hi + lo
+
+  // RoundedDownFloatSum(hi, lo) should only be called if
+  // (ScalarAbs(hi) >= ScalarAbs(lo) || hi == 0) is true
+  template <class T, HWY_IF_FLOAT(T)>
+  static HWY_INLINE HWY_BITCASTSCALAR_CXX14_CONSTEXPR RemoveCvRef<T>
+  RoundedDownFloatSum(T hi, T lo) {
+    using NonCvRefT = RemoveCvRef<T>;
+    using TU = MakeUnsigned<NonCvRefT>;
+
+    const NonCvRefT sum = static_cast<NonCvRefT>(hi + lo);
+    const NonCvRefT carry = static_cast<NonCvRefT>((hi - sum) + lo);
+
+    const TU sum_bits = BitCastScalar<TU>(sum);
+    const TU carry_bits = BitCastScalar<TU>(carry);
+
+    return BitCastScalar<NonCvRefT>(static_cast<TU>(
+        sum_bits - (((sum_bits ^ carry_bits) >> (sizeof(TU) * 8 - 1)) &
+                    static_cast<TU>(carry != 0))));
+  }
+
+  // ConvIntToRoundedDownF64 converts val to a F64, with inexact conversion
+  // rounded down
+  template <class T, HWY_IF_T_SIZE_LE(T, 4)>
+  static HWY_INLINE constexpr double ConvIntToRoundedDownF64(T val) {
+    return static_cast<double>(val);
+  }
+
+  template <class T, HWY_IF_T_SIZE(T, 8)>
+  static HWY_INLINE HWY_BITCASTSCALAR_CXX14_CONSTEXPR double
+  ConvIntToRoundedDownF64(T val) {
+    using NonCvRefT = RemoveCvRef<T>;
+    return RoundedDownFloatSum(
+        static_cast<double>(static_cast<NonCvRefT>(static_cast<uint64_t>(val) &
+                                                   0xFFE0000000000000ULL)),
+        static_cast<double>(static_cast<NonCvRefT>(static_cast<uint64_t>(val) &
+                                                   0x001FFFFFFFFFFFFFULL)));
+  }
+
+  // RoundFloatDownToPrecision rounds val down to a floating-point value with a
+  // mantissa that has at most kBitPrecision bits of precision
+  template <int kBitPrecision, class T, HWY_IF_FLOAT_OR_SPECIAL(T)>
+  static HWY_INLINE HWY_BITCASTSCALAR_CXX14_CONSTEXPR RemoveCvRef<T>
+  RoundFloatDownToPrecision(T val) {
+    static_assert(kBitPrecision > 0, "kBitPrecision > 0 must be true");
+
+    using NonCvRefT = RemoveCvRef<T>;
+    using TU = MakeUnsigned<NonCvRefT>;
+
+    // kTMantBitPrecision is the number of bits in the mantissa of val,
+    // including the implied one bit
+    constexpr int kTMantBitPrecision = MantissaBits<NonCvRefT>() + 1;
+    static_assert(kTMantBitPrecision > 0,
+                  "kTMantBitPrecision > 0 must be true");
+
+    constexpr int kNumOfBitsToZeroOut =
+        HWY_MAX(kTMantBitPrecision - kBitPrecision, 0);
+    constexpr TU kZeroOutMask =
+        static_cast<TU>((1ULL << kNumOfBitsToZeroOut) - 1ULL);
+
+    return BitCastScalar<NonCvRefT>(
+        static_cast<TU>(BitCastScalar<TU>(val) & (~kZeroOutMask)));
+  }
+
+  // LowestInRangeValForF2IConv<TFrom> returns the lowest finite value of TFrom
+  // that is greater than LimitsMin<TTo>() - 1
+  template <class TFrom>
+  static HWY_INLINE HWY_BITCASTSCALAR_CXX14_CONSTEXPR TFrom
+  LowestInRangeValForF2IConv() {
+    using TFArith = If<(sizeof(TFrom) <= sizeof(float)), float, double>;
+
+    // kTFromMantBitPrecision is equal to the number of bits in the mantissa of
+    // TFrom, including the implied one bit
+    constexpr int kTFromMantBitPrecision = MantissaBits<TFrom>() + 1;
+
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TFArith kLowestTFromVal =
+        ConvertScalarTo<TFArith>(LowestValue<TFrom>());
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TFArith kLowestTToVal =
+        RoundFloatDownToPrecision<kTFromMantBitPrecision>(RoundedDownFloatSum(
+            static_cast<TFArith>(LimitsMin<TTo>()),
+            static_cast<TFArith>(-LargestLt1FloatVal<TFArith>())));
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TFArith kLowestInRangeVal =
+        HWY_MAX(kLowestTFromVal, kLowestTToVal);
+
+    HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(
+        ScalarIsFinite(kLowestInRangeVal) &&
+            kLowestInRangeVal >= static_cast<TFArith>(LimitsMin<int64_t>()),
+        "kLowestInRangeVal must be a finite value that is greater than or "
+        "equal to LimitsMin<int64_t>()");
+    HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(
+        kLowestInRangeVal < static_cast<TFArith>(0.0),
+        "kLowestInRangeVal must be less than zero");
+
+    // Disable the C4056 (overflow in floating-point constant arithmetic)
+    // warning that MSVC generates when kLowestInRangeVal is cast to an
+    // int64_t in the HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT statements below
+    // as kLowestInRangeVal is known to be within the range of an int64_t due to
+    // the HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT checks above
+
+#if HWY_COMPILER_MSVC
+    HWY_DIAGNOSTICS(push)
+    HWY_DIAGNOSTICS_OFF(disable : 4056, ignored "-Woverflow")
+#endif
+
+    HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(
+        static_cast<int64_t>(kLowestInRangeVal) <= 0,
+        "static_cast<int64_t>(kLowestInRangeVal) must be less than "
+        "or equal to 0");
+    HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(
+        static_cast<int64_t>(kLowestInRangeVal) >=
+            static_cast<int64_t>(LimitsMin<TTo>()),
+        "static_cast<int64_t>(kLowestInRangeVal) must be greater "
+        "than or equal to LimitsMin<TTo>()");
+
+#if HWY_COMPILER_MSVC
+    HWY_DIAGNOSTICS(pop)
+#endif
+
+    return ConvertScalarTo<TFrom>(kLowestInRangeVal);
+  }
+
+  // HighestInRangeValForF2IConv<TFrom> returns the largest finite value of
+  // TFrom that is less than LimitsMax<TTo>() + 1
+  template <class TFrom>
+  static HWY_INLINE HWY_BITCASTSCALAR_CXX14_CONSTEXPR TFrom
+  HighestInRangeValForF2IConv() {
+    using TFArith = If<(sizeof(TFrom) <= sizeof(float)), float, double>;
+
+    // kTFromMantBitPrecision is equal to the number of bits in the mantissa of
+    // TFrom, including the implied one bit
+    constexpr int kTFromMantBitPrecision = MantissaBits<TFrom>() + 1;
+
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TFArith kHighestTFromVal =
+        ConvertScalarTo<TFArith>(HighestValue<TFrom>());
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TFArith kHighestTToVal =
+        static_cast<TFArith>(RoundFloatDownToPrecision<kTFromMantBitPrecision>(
+            RoundedDownFloatSum(ConvIntToRoundedDownF64(LimitsMax<TTo>()),
+                                LargestLt1FloatVal<double>())));
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TFArith kHighestInRangeVal =
+        HWY_MIN(kHighestTFromVal, kHighestTToVal);
+
+    HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(
+        ScalarIsFinite(kHighestInRangeVal) &&
+            kHighestInRangeVal < static_cast<TFArith>(18446744073709551616.0),
+        "kHighestInRangeVal must be a finite value that is less than or "
+        "equal to LimitsMax<uint64_t>()");
+    HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(
+        kHighestInRangeVal > 0, "kHighestInRangeVal must be greater than 0");
+    HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(
+        static_cast<uint64_t>(kHighestInRangeVal) > 0,
+        "static_cast<uint64_t>(kHighestInRangeVal) must be greater than 0");
+    HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(
+        static_cast<uint64_t>(kHighestInRangeVal) <=
+            static_cast<uint64_t>(LimitsMax<TTo>()),
+        "static_cast<uint64_t>(kHighestInRangeVal) must be less "
+        "than or equal to LimitsMax<TTo>()");
+
+    return ConvertScalarTo<TFrom>(kHighestInRangeVal);
+  }
+
+  template <class DTo, class VFrom,
+            HWY_IF_T_SIZE_LE_D(DTo, sizeof(TFromV<VFrom>) - 1)>
+  static HWY_INLINE Vec<DTo> DoConvVector(DTo d_to, VFrom v_from) {
+    return DemoteTo(d_to, v_from);
+  }
+  template <class DTo, class VFrom, HWY_IF_T_SIZE_D(DTo, sizeof(TFromV<VFrom>)),
+            hwy::EnableIf<IsFloat<TFromD<DTo>>() == IsFloat<TFromV<VFrom>>()>* =
+                nullptr>
+  static HWY_INLINE Vec<DTo> DoConvVector(DTo d_to, VFrom v_from) {
+    return BitCast(d_to, v_from);
+  }
+  template <class DTo, class VFrom, HWY_IF_T_SIZE_D(DTo, sizeof(TFromV<VFrom>)),
+            hwy::EnableIf<IsFloat<TFromD<DTo>>() != IsFloat<TFromV<VFrom>>()>* =
+                nullptr>
+  static HWY_INLINE Vec<DTo> DoConvVector(DTo d_to, VFrom v_from) {
+    return ConvertTo(d_to, v_from);
+  }
+  template <class DTo, class VFrom,
+            HWY_IF_T_SIZE_GT_D(DTo, sizeof(TFromV<VFrom>))>
+  static HWY_INLINE Vec<DTo> DoConvVector(DTo d_to, VFrom v_from) {
+    return PromoteTo(d_to, v_from);
+  }
+  template <class DTo, class VFrom,
+            HWY_IF_T_SIZE_LE_D(DTo, sizeof(TFromV<VFrom>) - 1)>
+  static HWY_INLINE Vec<DTo> DoInRangeF2IConvVector(DTo d_to, VFrom v_from) {
+    return DemoteInRangeTo(d_to, v_from);
+  }
+  template <class DTo, class VFrom, HWY_IF_T_SIZE_D(DTo, sizeof(TFromV<VFrom>))>
+  static HWY_INLINE Vec<DTo> DoInRangeF2IConvVector(DTo d_to, VFrom v_from) {
+    return ConvertInRangeTo(d_to, v_from);
+  }
+  template <class DTo, class VFrom,
+            HWY_IF_T_SIZE_GT_D(DTo, sizeof(TFromV<VFrom>))>
+  static HWY_INLINE Vec<DTo> DoInRangeF2IConvVector(DTo d_to, VFrom v_from) {
+    return PromoteInRangeTo(d_to, v_from);
+  }
+
+ public:
+  template <typename TFrom, class DFrom>
+  HWY_NOINLINE void operator()(TFrom /*unused*/, DFrom d_from) {
+    static_assert(IsFloat<TFrom>(), "TFrom must be a floating-point type");
+
+    using TIFrom = MakeSigned<TFrom>;
+    using TUFrom = MakeUnsigned<TFrom>;
+    const RebindToSigned<decltype(d_from)> di_from;
+    const RebindToUnsigned<decltype(d_from)> du_from;
+
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TFrom kLowestInRangeFltVal =
+        LowestInRangeValForF2IConv<TFrom>();
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TFrom kHighestInRangeFltVal =
+        HighestInRangeValForF2IConv<TFrom>();
+
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TUFrom kLowestInRangeFltValBits =
+        BitCastScalar<TUFrom>(kLowestInRangeFltVal);
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TUFrom kHighestInRangeFltValBits =
+        BitCastScalar<TUFrom>(kHighestInRangeFltVal);
+
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TUFrom kMinOutOfRangeRandFltBits =
+        static_cast<TUFrom>(
+            HWY_MAX(kLowestInRangeFltValBits &
+                        static_cast<TUFrom>(LimitsMax<TIFrom>()),
+                    kHighestInRangeFltValBits) +
+            1u);
+
+    HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(
+        kMinOutOfRangeRandFltBits > kHighestInRangeFltValBits,
+        "kMinOutOfRangeRandFltBits > kHighestInRangeFltValBits must be true");
+    HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(
+        kMinOutOfRangeRandFltBits <= static_cast<TUFrom>(LimitsMax<TIFrom>()),
+        "kMinOutOfRangeRandFltBits <= LimitsMax<TIFrom>() must be true");
+
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TTo kLowestInRangeIntVal =
+        ConvertScalarTo<TTo>(kLowestInRangeFltVal);
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const TTo kHighestInRangeIntVal =
+        ConvertScalarTo<TTo>(kHighestInRangeFltVal);
+
+    const Rebind<TTo, decltype(d_from)> d_to;
+
+    HWY_ASSERT_VEC_EQ(
+        d_to, Set(d_to, static_cast<TTo>(0)),
+        DoInRangeF2IConvVector(d_to, Set(d_from, ConvertScalarTo<TFrom>(0))));
+    HWY_ASSERT_VEC_EQ(
+        d_to, Set(d_to, static_cast<TTo>(1)),
+        DoInRangeF2IConvVector(d_to, Set(d_from, ConvertScalarTo<TFrom>(1))));
+
+    if (IsSigned<TTo>()) {
+      HWY_ASSERT_VEC_EQ(d_to, Set(d_to, static_cast<TTo>(-1)),
+                        DoInRangeF2IConvVector(
+                            d_to, Set(d_from, ConvertScalarTo<TFrom>(-1))));
+    }
+
+    HWY_ASSERT_VEC_EQ(
+        d_to, Set(d_to, kLowestInRangeIntVal),
+        DoInRangeF2IConvVector(d_to, Set(d_from, kLowestInRangeFltVal)));
+    HWY_ASSERT_VEC_EQ(
+        d_to, Set(d_to, kHighestInRangeIntVal),
+        DoInRangeF2IConvVector(d_to, Set(d_from, kHighestInRangeFltVal)));
+
+    constexpr TIFrom kIotaMask =
+        static_cast<TIFrom>(static_cast<uint64_t>(MantissaMask<TFrom>()) &
+                            static_cast<uint64_t>(LimitsMax<TTo>() / 2));
+
+    const auto flt_iota = DoConvVector(
+        d_from, Add(And(Iota(di_from, TIFrom{0}), Set(di_from, kIotaMask)),
+                    Set(di_from, TIFrom{1})));
+    const auto expected_f2i_iota =
+        Add(And(Iota(d_to, TTo{0}), Set(d_to, static_cast<TTo>(kIotaMask))),
+            Set(d_to, TTo{1}));
+    HWY_ASSERT_VEC_EQ(d_to, expected_f2i_iota, DoConvVector(d_to, flt_iota));
+
+    const size_t N = Lanes(d_from);
+    auto from_lanes = AllocateAligned<TFrom>(N);
+    auto expected = AllocateAligned<TTo>(N);
+    HWY_ASSERT(from_lanes && expected);
+
+    constexpr int kMaxBiasedExp = static_cast<int>(MaxExponentField<TFrom>());
+    static_assert(kMaxBiasedExp > 0, "kMaxBiasedExp > 0 must be true");
+
+    constexpr int kExpBias = kMaxBiasedExp >> 1;
+    static_assert(kExpBias > 0, "kExpBias > 0 must be true");
+
+    constexpr int kMinOutOfRangeBiasedExp =
+        static_cast<int>(HWY_MIN(static_cast<unsigned>(kExpBias) +
+                                     static_cast<unsigned>(sizeof(TTo) * 8) -
+                                     static_cast<unsigned>(IsSigned<TTo>()),
+                                 static_cast<unsigned>(kMaxBiasedExp)));
+    static_assert(kMinOutOfRangeBiasedExp > 0,
+                  "kMinOutOfRangeBiasedExp > 0 must be true");
+    static_assert(
+        (kMaxBiasedExp - kMinOutOfRangeBiasedExp + 1) > 0,
+        "kMaxBiasedExp - kMinOutOfRangeBiasedExp + 1 must be greater than 0");
+
+    constexpr int kNumOfMantBits = MantissaBits<TFrom>();
+    static_assert(kNumOfMantBits > 0, "kNumOfMantBits > 0 must be true");
+
+    constexpr TUFrom kExpMask = ExponentMask<TFrom>();
+    constexpr TUFrom kMantAndSignMask = static_cast<TUFrom>(
+        (~kExpMask) &
+        (IsSigned<TTo>() ? LimitsMax<TUFrom>()
+                         : static_cast<TUFrom>(LimitsMax<TIFrom>())));
+
+    const int non_elided_one = Unpredictable1();
+
+    const auto pos_inf = BitCast(
+        d_from,
+        Set(du_from,
+            static_cast<TUFrom>(
+                kExpMask | static_cast<TUFrom>(
+                               static_cast<unsigned>(non_elided_one) - 1u))));
+    const auto neg_inf = BitCast(
+        d_from,
+        Set(du_from, static_cast<TUFrom>(
+                         kExpMask | static_cast<TUFrom>(LimitsMin<TIFrom>()) |
+                         static_cast<TUFrom>(
+                             static_cast<unsigned>(non_elided_one) - 1u))));
+    const auto pos_nan = BitCast(
+        d_from,
+        Set(di_from, static_cast<TIFrom>(static_cast<TIFrom>(-non_elided_one) &
+                                         LimitsMax<TIFrom>())));
+    const auto neg_nan =
+        BitCast(d_from, Set(di_from, static_cast<TIFrom>(-non_elided_one)));
+
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; i++) {
+        uint64_t rand_bits = rng();
+
+        const TUFrom exp_bits =
+            static_cast<TUFrom>(((rand_bits >> kNumOfMantBits) %
+                                 static_cast<uint64_t>(kMinOutOfRangeBiasedExp))
+                                << kNumOfMantBits);
+        const TFrom rand_in_range_val = BitCastScalar<TFrom>(
+            static_cast<TUFrom>((rand_bits & kMantAndSignMask) | exp_bits));
+
+        HWY_ASSERT(ScalarIsFinite(rand_in_range_val));
+        HWY_ASSERT(rand_in_range_val >= kLowestInRangeFltVal);
+        HWY_ASSERT(rand_in_range_val <= kHighestInRangeFltVal);
+
+        from_lanes[i] = rand_in_range_val;
+        expected[i] = ConvertScalarTo<TTo>(rand_in_range_val);
+      }
+
+#if HWY_COMPILER_CLANG && HWY_ARCH_RISCV && HWY_TARGET == HWY_EMU128
+      // Workaround for incorrect codegen. Off by one in the upper lane.
+      if (sizeof(TTo) == 4 && N == 2) return;
+#endif
+
+      const auto from = Load(d_from, from_lanes.get());
+
+      HWY_ASSERT_VEC_EQ(d_to, expected.get(),
+                        DoInRangeF2IConvVector(d_to, from));
+      HWY_ASSERT_VEC_EQ(
+          d_to, expected.get(),
+          OddEven(DoInRangeF2IConvVector(d_to, OddEven(from, pos_nan)),
+                  DoInRangeF2IConvVector(d_to, OddEven(pos_nan, from))));
+      HWY_ASSERT_VEC_EQ(
+          d_to, expected.get(),
+          OddEven(DoInRangeF2IConvVector(d_to, OddEven(from, neg_nan)),
+                  DoInRangeF2IConvVector(d_to, OddEven(neg_nan, from))));
+      HWY_ASSERT_VEC_EQ(
+          d_to, expected.get(),
+          OddEven(DoInRangeF2IConvVector(d_to, OddEven(from, pos_inf)),
+                  DoInRangeF2IConvVector(d_to, OddEven(pos_inf, from))));
+      HWY_ASSERT_VEC_EQ(
+          d_to, expected.get(),
+          OddEven(DoInRangeF2IConvVector(d_to, OddEven(from, neg_inf)),
+                  DoInRangeF2IConvVector(d_to, OddEven(neg_inf, from))));
+    }
+
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const uint64_t
+        kOutOfRangeRandBitsModulus =
+            static_cast<uint64_t>(static_cast<TUFrom>(LimitsMax<TIFrom>()) -
+                                  kMinOutOfRangeRandFltBits + 1);
+    HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT(
+        kOutOfRangeRandBitsModulus != 0,
+        "kOutOfRangeRandBitsModulus != 0 must be true");
+
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      ZeroBytes(expected.get(), sizeof(TTo) * N);
+
+      for (size_t i = 0; i < N; i++) {
+        uint64_t rand_bits = rng();
+
+        const uint64_t rand_mag_bits = static_cast<uint64_t>(
+            (rand_bits % kOutOfRangeRandBitsModulus) +
+            static_cast<uint64_t>(kMinOutOfRangeRandFltBits));
+
+        const TFrom rand_out_of_range_val =
+            BitCastScalar<TFrom>(static_cast<TUFrom>(
+                rand_mag_bits |
+                (rand_bits & (1ULL << (sizeof(TFrom) * 8 - 1)))));
+
+        HWY_ASSERT(!(ScalarIsFinite(rand_out_of_range_val) &&
+                     rand_out_of_range_val >= kLowestInRangeFltVal &&
+                     rand_out_of_range_val <= kHighestInRangeFltVal));
+
+        from_lanes[i] = rand_out_of_range_val;
+      }
+
+      const auto from = Load(d_from, from_lanes.get());
+      const auto actual = DoInRangeF2IConvVector(d_to, from);
+      HWY_ASSERT_VEC_EQ(
+          d_to, expected.get(),
+          And(actual,
+              Set(d_to, static_cast<TTo>(static_cast<unsigned>(non_elided_one) -
+                                         1u))));
+
+      for (size_t i = 0; i < N; i++) {
+        expected[i] = static_cast<TTo>(-1);
+      }
+
+      HWY_ASSERT_VEC_EQ(
+          d_to, expected.get(),
+          Or(actual,
+             Set(d_to,
+                 static_cast<TTo>(TTo{0} - static_cast<TTo>(non_elided_one)))));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllConvertInRangeFloatToInt() {
+  // Fails due to vfcvt_rtz rounding instead of truncating. This is likely a
+  // continuation of the QEMU issue partially fixed by
+  // https://lists.gnu.org/archive/html/qemu-riscv/2022-12/msg00377.html.
+#if HWY_TARGET != HWY_RVV
+#if HWY_HAVE_FLOAT16
+  ForPartialVectors<TestConvertInRangeFloatToInt<int16_t>>()(hwy::float16_t());
+  ForPartialVectors<TestConvertInRangeFloatToInt<uint16_t>>()(hwy::float16_t());
+#endif
+
+  ForPartialVectors<TestConvertInRangeFloatToInt<int32_t>>()(float());
+  ForPartialVectors<TestConvertInRangeFloatToInt<uint32_t>>()(float());
+
+#if HWY_HAVE_INTEGER64
+  ForPromoteVectors<TestConvertInRangeFloatToInt<int64_t>>()(float());
+  ForPromoteVectors<TestConvertInRangeFloatToInt<uint64_t>>()(float());
+#endif
+
+#if HWY_HAVE_FLOAT64
+  ForDemoteVectors<TestConvertInRangeFloatToInt<int32_t>>()(double());
+  ForDemoteVectors<TestConvertInRangeFloatToInt<uint32_t>>()(double());
+  ForPartialVectors<TestConvertInRangeFloatToInt<int64_t>>()(double());
+  ForPartialVectors<TestConvertInRangeFloatToInt<uint64_t>>()(double());
+#endif
+#endif  // HWY_TARGET != HWY_RVV
+}
+
+template <class TTo>
+struct TestPromoteInRangeUpperLowerFloatToInt {
+  template <typename TFrom, class DFrom>
+  HWY_NOINLINE void operator()(TFrom /*unused*/, DFrom d_from) {
+    static_assert(IsFloat<TFrom>(), "TFrom must be a floating-point type");
+    static_assert(!IsFloat<TTo>() && !IsSpecialFloat<TTo>(),
+                  "TTo must be an integer type");
+
+    const size_t N = Lanes(d_from);
+    HWY_ASSERT(N >= 2);
+
+    auto from_lanes = AllocateAligned<TFrom>(N);
+    auto expected = AllocateAligned<TTo>(N / 2);
+    HWY_ASSERT(from_lanes && expected);
+
+    constexpr uint64_t kIotaMask =
+        static_cast<uint64_t>(MantissaMask<TFrom>()) &
+        static_cast<uint64_t>(LimitsMax<TTo>() / 2);
+
+    const Repartition<TTo, decltype(d_from)> d_to;
+
+    for (size_t i = 0; i < N; ++i) {
+      const uint64_t from_val = static_cast<uint64_t>((i & kIotaMask) + 1u);
+      from_lanes[i] = ConvertScalarTo<TFrom>(from_val);
+    }
+
+    for (size_t i = 0; i < N / 2; ++i) {
+      const uint64_t from_val = static_cast<uint64_t>((i & kIotaMask) + 1u);
+      expected[i] = static_cast<TTo>(from_val);
+    }
+
+    const auto from = Load(d_from, from_lanes.get());
+    HWY_ASSERT_VEC_EQ(d_to, expected.get(), PromoteInRangeLowerTo(d_to, from));
+
+#if HWY_TARGET != HWY_SCALAR
+    for (size_t i = 0; i < N / 2; ++i) {
+      const uint64_t from_val =
+          static_cast<uint64_t>(((i + (N / 2)) & kIotaMask) + 1u);
+      expected[i] = static_cast<TTo>(from_val);
+    }
+
+    HWY_ASSERT_VEC_EQ(d_to, expected.get(), PromoteInRangeUpperTo(d_to, from));
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllPromoteInRangeUpperLowerFloatToInt() {
+#if HWY_HAVE_INTEGER64
+  ForShrinkableVectors<TestPromoteInRangeUpperLowerFloatToInt<int64_t>, 1>()(
+      float());
+  ForShrinkableVectors<TestPromoteInRangeUpperLowerFloatToInt<uint64_t>, 1>()(
+      float());
+#endif
+}
+
+template <class TTo>
+struct TestPromoteInRangeOddEvenFloatToInt {
+  template <typename TFrom, class DFrom>
+  HWY_NOINLINE void operator()(TFrom /*unused*/, DFrom d_from) {
+    static_assert(IsFloat<TFrom>(), "TFrom must be a floating-point type");
+    static_assert(!IsFloat<TTo>() && !IsSpecialFloat<TTo>(),
+                  "TTo must be an integer type");
+
+    const size_t N = Lanes(d_from);
+    HWY_ASSERT(N >= 2);
+
+    auto from_lanes = AllocateAligned<TFrom>(N);
+    auto expected = AllocateAligned<TTo>(N / 2);
+    HWY_ASSERT(from_lanes && expected);
+
+    constexpr uint64_t kIotaMask =
+        static_cast<uint64_t>(MantissaMask<TFrom>()) &
+        static_cast<uint64_t>(LimitsMax<TTo>() / 2);
+
+    const Repartition<TTo, decltype(d_from)> d_to;
+
+    for (size_t i = 0; i < N; ++i) {
+      const uint64_t from_val = static_cast<uint64_t>((i & kIotaMask) + 1u);
+      from_lanes[i] = ConvertScalarTo<TFrom>(from_val);
+    }
+
+    for (size_t i = 0; i < N / 2; ++i) {
+      const uint64_t from_val =
+          static_cast<uint64_t>(((2 * i) & kIotaMask) + 1u);
+      expected[i] = static_cast<TTo>(from_val);
+    }
+
+    const auto from = Load(d_from, from_lanes.get());
+    HWY_ASSERT_VEC_EQ(d_to, expected.get(), PromoteInRangeEvenTo(d_to, from));
+
+#if HWY_TARGET != HWY_SCALAR
+    for (size_t i = 0; i < N / 2; ++i) {
+      const uint64_t from_val =
+          static_cast<uint64_t>(((2 * i + 1) & kIotaMask) + 1u);
+      expected[i] = static_cast<TTo>(from_val);
+    }
+
+    HWY_ASSERT_VEC_EQ(d_to, expected.get(), PromoteInRangeOddTo(d_to, from));
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllPromoteInRangeOddEvenFloatToInt() {
+#if HWY_HAVE_INTEGER64
+  ForShrinkableVectors<TestPromoteInRangeOddEvenFloatToInt<int64_t>, 1>()(
+      float());
+  ForShrinkableVectors<TestPromoteInRangeOddEvenFloatToInt<uint64_t>, 1>()(
+      float());
+#endif
+}
+
+#undef HWY_IN_RANGE_F2I_CONV_TEST_CONST_ASSERT
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyInRangeFloatToIntConvTest);
+HWY_EXPORT_AND_TEST_P(HwyInRangeFloatToIntConvTest,
+                      TestAllConvertInRangeFloatToInt);
+HWY_EXPORT_AND_TEST_P(HwyInRangeFloatToIntConvTest,
+                      TestAllPromoteInRangeUpperLowerFloatToInt);
+HWY_EXPORT_AND_TEST_P(HwyInRangeFloatToIntConvTest,
+                      TestAllPromoteInRangeOddEvenFloatToInt);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/interleaved_test.cc

@@ -0,0 +1,189 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/interleaved_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestLoadStoreInterleaved2 {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+
+    RandomState rng;
+
+    constexpr size_t kVectors = 2;
+    // Data to be interleaved
+    auto in = AllocateAligned<T>(kVectors * N);
+    // Ensure unaligned; kVectors plus one zero vector.
+    auto actual_aligned = AllocateAligned<T>((kVectors + 1) * N + 1);
+    HWY_ASSERT(in && actual_aligned);
+
+    for (size_t i = 0; i < kVectors * N; ++i) {
+      in[i] = ConvertScalarTo<T>(Random32(&rng) & 0x7F);
+    }
+    const Vec<D> in0 = Load(d, &in[0 * N]);
+    const Vec<D> in1 = Load(d, &in[1 * N]);
+
+    T* actual = actual_aligned.get() + 1;
+    StoreInterleaved2(in0, in1, d, actual);
+    StoreU(Zero(d), d, actual + kVectors * N);
+
+    Vec<D> out0, out1;
+    LoadInterleaved2(d, actual, out0, out1);
+    HWY_ASSERT_VEC_EQ(d, in0, out0);
+    HWY_ASSERT_VEC_EQ(d, in1, out1);
+    HWY_ASSERT_VEC_EQ(d, Zero(d), LoadU(d, actual + kVectors * N));
+  }
+};
+
+HWY_NOINLINE void TestAllLoadStoreInterleaved2() {
+  ForAllTypes(ForMaxPow2<TestLoadStoreInterleaved2>());
+  // Temporarily disable this test for special floats on arm7.
+#ifndef HWY_ARCH_ARM_V7
+  ForSpecialTypes(ForMaxPow2<TestLoadStoreInterleaved2>());
+#endif
+}
+
+// Workaround for build timeout on GCC 12 aarch64, see #776.
+#undef HWY_BROKEN_LOAD34
+#if HWY_ARCH_ARM_A64 && HWY_COMPILER_GCC_ACTUAL && \
+    HWY_COMPILER_GCC_ACTUAL < 1300
+#define HWY_BROKEN_LOAD34 1
+#else
+#define HWY_BROKEN_LOAD34 0
+#endif
+
+struct TestLoadStoreInterleaved3 {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_BROKEN_LOAD34
+    (void)d;
+#else   // !HWY_BROKEN_LOAD34
+    const size_t N = Lanes(d);
+
+    RandomState rng;
+
+    constexpr size_t kVectors = 3;
+    // Data to be interleaved
+    auto in = AllocateAligned<T>(kVectors * N);
+    // Ensure unaligned; kVectors plus one zero vector.
+    auto actual_aligned = AllocateAligned<T>((kVectors + 1) * N + 1);
+    HWY_ASSERT(in && actual_aligned);
+
+    for (size_t i = 0; i < kVectors * N; ++i) {
+      in[i] = ConvertScalarTo<T>(Random32(&rng) & 0x7F);
+    }
+    const Vec<D> in0 = Load(d, &in[0 * N]);
+    const Vec<D> in1 = Load(d, &in[1 * N]);
+    const Vec<D> in2 = Load(d, &in[2 * N]);
+
+    T* actual = actual_aligned.get() + 1;
+    StoreInterleaved3(in0, in1, in2, d, actual);
+    StoreU(Zero(d), d, actual + kVectors * N);
+
+    Vec<D> out0, out1, out2;
+    LoadInterleaved3(d, actual, out0, out1, out2);
+    HWY_ASSERT_VEC_EQ(d, in0, out0);
+    HWY_ASSERT_VEC_EQ(d, in1, out1);
+    HWY_ASSERT_VEC_EQ(d, in2, out2);
+    HWY_ASSERT_VEC_EQ(d, Zero(d), LoadU(d, actual + kVectors * N));
+#endif  // HWY_BROKEN_LOAD34
+  }
+};
+
+HWY_NOINLINE void TestAllLoadStoreInterleaved3() {
+  ForAllTypes(ForMaxPow2<TestLoadStoreInterleaved3>());
+  // Temporarily disable this test for special floats on arm7.
+#ifndef HWY_ARCH_ARM_V7
+  ForSpecialTypes(ForMaxPow2<TestLoadStoreInterleaved3>());
+#endif
+}
+
+struct TestLoadStoreInterleaved4 {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_BROKEN_LOAD34
+    (void)d;
+#else   // !HWY_BROKEN_LOAD34
+    const size_t N = Lanes(d);
+
+    RandomState rng;
+
+    constexpr size_t kVectors = 4;
+    // Data to be interleaved
+    auto in = AllocateAligned<T>(kVectors * N);
+    // Ensure unaligned; kVectors plus one zero vector.
+    auto actual_aligned = AllocateAligned<T>((kVectors + 1) * N + 1);
+    HWY_ASSERT(in && actual_aligned);
+
+    for (size_t i = 0; i < kVectors * N; ++i) {
+      in[i] = ConvertScalarTo<T>(Random32(&rng) & 0x7F);
+    }
+    const Vec<D> in0 = Load(d, &in[0 * N]);
+    const Vec<D> in1 = Load(d, &in[1 * N]);
+    const Vec<D> in2 = Load(d, &in[2 * N]);
+    const Vec<D> in3 = Load(d, &in[3 * N]);
+
+    T* actual = actual_aligned.get() + 1;
+    StoreInterleaved4(in0, in1, in2, in3, d, actual);
+    StoreU(Zero(d), d, actual + kVectors * N);
+
+    Vec<D> out0, out1, out2, out3;
+    LoadInterleaved4(d, actual, out0, out1, out2, out3);
+    HWY_ASSERT_VEC_EQ(d, in0, out0);
+    HWY_ASSERT_VEC_EQ(d, in1, out1);
+    HWY_ASSERT_VEC_EQ(d, in2, out2);
+    HWY_ASSERT_VEC_EQ(d, in3, out3);
+    HWY_ASSERT_VEC_EQ(d, Zero(d), LoadU(d, actual + kVectors * N));
+#endif  // HWY_BROKEN_LOAD34
+  }
+};
+
+HWY_NOINLINE void TestAllLoadStoreInterleaved4() {
+  ForAllTypes(ForMaxPow2<TestLoadStoreInterleaved4>());
+  // Temporarily disable this test for special floats on arm7.
+#ifndef HWY_ARCH_ARM_V7
+  ForSpecialTypes(ForMaxPow2<TestLoadStoreInterleaved4>());
+#endif
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyInterleavedTest);
+HWY_EXPORT_AND_TEST_P(HwyInterleavedTest, TestAllLoadStoreInterleaved2);
+HWY_EXPORT_AND_TEST_P(HwyInterleavedTest, TestAllLoadStoreInterleaved3);
+HWY_EXPORT_AND_TEST_P(HwyInterleavedTest, TestAllLoadStoreInterleaved4);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/list_targets.cc

@@ -0,0 +1,77 @@
+// Copyright 2020 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Simple tool to print the list of targets that were compiled in when building
+// this tool.
+
+#include <stdint.h>
+#include <stdio.h>
+
+#include "hwy/highway.h"
+
+namespace {
+
+void PrintTargets(const char* msg, int64_t targets) {
+  fprintf(stderr, "%s", msg);
+  // For each bit other than the sign bit:
+  for (int64_t x = targets & hwy::LimitsMax<int64_t>(); x != 0;
+       x = x & (x - 1)) {
+    // Extract value of least-significant bit.
+    fprintf(stderr, " %s", hwy::TargetName(x & (~x + 1)));
+  }
+  fprintf(stderr, "\n");
+}
+
+}  // namespace
+
+int main() {
+#ifdef HWY_COMPILE_ONLY_EMU128
+  const int only_emu128 = 1;
+#else
+  const int only_emu128 = 0;
+#endif
+#ifdef HWY_COMPILE_ONLY_SCALAR
+  const int only_scalar = 1;
+#else
+  const int only_scalar = 0;
+#endif
+#ifdef HWY_COMPILE_ONLY_STATIC
+  const int only_static = 1;
+#else
+  const int only_static = 0;
+#endif
+#ifdef HWY_COMPILE_ALL_ATTAINABLE
+  const int all_attain = 1;
+#else
+  const int all_attain = 0;
+#endif
+#ifdef HWY_IS_TEST
+  const int is_test = 1;
+#else
+  const int is_test = 0;
+#endif
+
+  fprintf(stderr,
+          "Config: emu128:%d scalar:%d static:%d all_attain:%d is_test:%d\n",
+          only_emu128, only_scalar, only_static, all_attain, is_test);
+  PrintTargets("Compiled HWY_TARGETS:  ", HWY_TARGETS);
+  PrintTargets("HWY_ATTAINABLE_TARGETS:", HWY_ATTAINABLE_TARGETS);
+  PrintTargets("HWY_BASELINE_TARGETS:  ", HWY_BASELINE_TARGETS);
+  PrintTargets("HWY_STATIC_TARGET:     ", HWY_STATIC_TARGET);
+  PrintTargets("HWY_BROKEN_TARGETS:    ", HWY_BROKEN_TARGETS);
+  PrintTargets("HWY_DISABLED_TARGETS:  ", HWY_DISABLED_TARGETS);
+  PrintTargets("Current CPU supports:  ", hwy::SupportedTargets());
+  return 0;
+}

deps/v8/third_party/highway/src/hwy/tests/logical_test.cc

@@ -0,0 +1,166 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/logical_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestNot {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> ones = VecFromMask(d, Eq(v0, v0));
+    const Vec<D> v1 = Set(d, 1);
+    const Vec<D> vnot1 = Set(d, static_cast<T>(~static_cast<T>(1)));
+
+    HWY_ASSERT_VEC_EQ(d, v0, Not(ones));
+    HWY_ASSERT_VEC_EQ(d, ones, Not(v0));
+    HWY_ASSERT_VEC_EQ(d, v1, Not(vnot1));
+    HWY_ASSERT_VEC_EQ(d, vnot1, Not(v1));
+  }
+};
+
+HWY_NOINLINE void TestAllNot() {
+  ForIntegerTypes(ForPartialVectors<TestNot>());
+}
+
+struct TestLogical {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v0 = Zero(d);
+    const auto vi = Iota(d, 0);
+
+    auto v = vi;
+    v = And(v, vi);
+    HWY_ASSERT_VEC_EQ(d, vi, v);
+    v = And(v, v0);
+    HWY_ASSERT_VEC_EQ(d, v0, v);
+
+    v = Or(v, vi);
+    HWY_ASSERT_VEC_EQ(d, vi, v);
+    v = Or(v, v0);
+    HWY_ASSERT_VEC_EQ(d, vi, v);
+
+    v = Xor(v, vi);
+    HWY_ASSERT_VEC_EQ(d, v0, v);
+    v = Xor(v, v0);
+    HWY_ASSERT_VEC_EQ(d, v0, v);
+
+    HWY_ASSERT_VEC_EQ(d, v0, And(v0, vi));
+    HWY_ASSERT_VEC_EQ(d, v0, And(vi, v0));
+    HWY_ASSERT_VEC_EQ(d, vi, And(vi, vi));
+
+    HWY_ASSERT_VEC_EQ(d, vi, Or(v0, vi));
+    HWY_ASSERT_VEC_EQ(d, vi, Or(vi, v0));
+    HWY_ASSERT_VEC_EQ(d, vi, Or(vi, vi));
+
+    HWY_ASSERT_VEC_EQ(d, vi, Xor(v0, vi));
+    HWY_ASSERT_VEC_EQ(d, vi, Xor(vi, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, Xor(vi, vi));
+
+    HWY_ASSERT_VEC_EQ(d, vi, AndNot(v0, vi));
+    HWY_ASSERT_VEC_EQ(d, v0, AndNot(vi, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, AndNot(vi, vi));
+
+    HWY_ASSERT_VEC_EQ(d, v0, Or3(v0, v0, v0));
+    HWY_ASSERT_VEC_EQ(d, vi, Or3(v0, vi, v0));
+    HWY_ASSERT_VEC_EQ(d, vi, Or3(v0, v0, vi));
+    HWY_ASSERT_VEC_EQ(d, vi, Or3(v0, vi, vi));
+    HWY_ASSERT_VEC_EQ(d, vi, Or3(vi, v0, v0));
+    HWY_ASSERT_VEC_EQ(d, vi, Or3(vi, vi, v0));
+    HWY_ASSERT_VEC_EQ(d, vi, Or3(vi, v0, vi));
+    HWY_ASSERT_VEC_EQ(d, vi, Or3(vi, vi, vi));
+
+    HWY_ASSERT_VEC_EQ(d, v0, Xor3(v0, v0, v0));
+    HWY_ASSERT_VEC_EQ(d, vi, Xor3(v0, vi, v0));
+    HWY_ASSERT_VEC_EQ(d, vi, Xor3(v0, v0, vi));
+    HWY_ASSERT_VEC_EQ(d, v0, Xor3(v0, vi, vi));
+    HWY_ASSERT_VEC_EQ(d, vi, Xor3(vi, v0, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, Xor3(vi, vi, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, Xor3(vi, v0, vi));
+    HWY_ASSERT_VEC_EQ(d, vi, Xor3(vi, vi, vi));
+
+    HWY_ASSERT_VEC_EQ(d, v0, OrAnd(v0, v0, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, OrAnd(v0, vi, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, OrAnd(v0, v0, vi));
+    HWY_ASSERT_VEC_EQ(d, vi, OrAnd(v0, vi, vi));
+    HWY_ASSERT_VEC_EQ(d, vi, OrAnd(vi, v0, v0));
+    HWY_ASSERT_VEC_EQ(d, vi, OrAnd(vi, vi, v0));
+    HWY_ASSERT_VEC_EQ(d, vi, OrAnd(vi, v0, vi));
+    HWY_ASSERT_VEC_EQ(d, vi, OrAnd(vi, vi, vi));
+  }
+};
+
+HWY_NOINLINE void TestAllLogical() {
+  ForAllTypes(ForPartialVectors<TestLogical>());
+}
+
+struct TestTestBit {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t kNumBits = sizeof(T) * 8;
+    for (size_t i = 0; i < kNumBits; ++i) {
+      const Vec<D> bit1 = Set(d, static_cast<T>(1ull << i));
+      const Vec<D> bit2 = Set(d, static_cast<T>(1ull << ((i + 1) % kNumBits)));
+      const Vec<D> bit3 = Set(d, static_cast<T>(1ull << ((i + 2) % kNumBits)));
+      const Vec<D> bits12 = Or(bit1, bit2);
+      const Vec<D> bits23 = Or(bit2, bit3);
+      HWY_ASSERT(AllTrue(d, TestBit(bit1, bit1)));
+      HWY_ASSERT(AllTrue(d, TestBit(bits12, bit1)));
+      HWY_ASSERT(AllTrue(d, TestBit(bits12, bit2)));
+
+      HWY_ASSERT(AllFalse(d, TestBit(bits12, bit3)));
+      HWY_ASSERT(AllFalse(d, TestBit(bits23, bit1)));
+      HWY_ASSERT(AllFalse(d, TestBit(bit1, bit2)));
+      HWY_ASSERT(AllFalse(d, TestBit(bit2, bit1)));
+      HWY_ASSERT(AllFalse(d, TestBit(bit1, bit3)));
+      HWY_ASSERT(AllFalse(d, TestBit(bit3, bit1)));
+      HWY_ASSERT(AllFalse(d, TestBit(bit2, bit3)));
+      HWY_ASSERT(AllFalse(d, TestBit(bit3, bit2)));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllTestBit() {
+  ForIntegerTypes(ForPartialVectors<TestTestBit>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyLogicalTest);
+HWY_EXPORT_AND_TEST_P(HwyLogicalTest, TestAllNot);
+HWY_EXPORT_AND_TEST_P(HwyLogicalTest, TestAllLogical);
+HWY_EXPORT_AND_TEST_P(HwyLogicalTest, TestAllTestBit);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/mask_combine_test.cc

@@ -0,0 +1,191 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/mask_combine_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestLowerAndUpperHalvesOfMask {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    using TI = MakeSigned<T>;
+
+    const Half<decltype(d)> dh;
+    const RebindToSigned<decltype(d)> di;
+    const RebindToSigned<decltype(dh)> dh_i;
+
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto expected = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes && expected);
+
+    ZeroBytes(bool_lanes.get(), N * sizeof(TI));
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N, size_t(6)));
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      for (size_t i = 0; i < N; ++i) {
+        expected[i] = static_cast<TI>(-static_cast<TI>(bool_lanes[i]));
+      }
+
+      const auto in_mask =
+          RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+
+      const auto expected_vec = Load(di, expected.get());
+      const auto expected_lo_mask =
+          RebindMask(dh, MaskFromVec(LowerHalf(dh_i, expected_vec)));
+      const auto expected_hi_mask =
+          RebindMask(dh, MaskFromVec(UpperHalf(dh_i, expected_vec)));
+
+      HWY_ASSERT_MASK_EQ(dh, expected_lo_mask, LowerHalfOfMask(dh, in_mask));
+      HWY_ASSERT_MASK_EQ(dh, expected_hi_mask, UpperHalfOfMask(dh, in_mask));
+    }
+
+    HWY_ASSERT_MASK_EQ(dh, FirstN(dh, 1), LowerHalfOfMask(dh, FirstN(d, 1)));
+    HWY_ASSERT_MASK_EQ(dh, FirstN(dh, (N / 2) - 1),
+                       LowerHalfOfMask(dh, FirstN(d, (N / 2) - 1)));
+
+    const size_t hi_N = HWY_MAX(HWY_MIN((N / 2) - 1, 5), 1);
+    HWY_ASSERT_MASK_EQ(dh, Not(FirstN(dh, (N / 2) - hi_N)),
+                       UpperHalfOfMask(dh, Not(FirstN(d, N - hi_N))));
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllLowerAndUpperHalvesOfMask() {
+  ForAllTypes(ForShrinkableVectors<TestLowerAndUpperHalvesOfMask>());
+}
+
+struct TestCombineMasks {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    using TI = MakeSigned<T>;
+
+    const Twice<decltype(d)> dt;
+    const RebindToSigned<decltype(d)> di;
+    const RebindToSigned<decltype(dt)> dt_i;
+
+    const size_t N = Lanes(di);
+    auto bool_lanes = AllocateAligned<TI>(N * 2);
+    auto expected = AllocateAligned<TI>(N * 2);
+    HWY_ASSERT(bool_lanes && expected);
+
+    ZeroBytes(bool_lanes.get(), N * 2 * sizeof(TI));
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N * 2, size_t(6)));
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      for (size_t i = 0; i < N * 2; ++i) {
+        expected[i] = static_cast<TI>(-static_cast<TI>(bool_lanes[i]));
+      }
+
+      const auto m0 = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+      const auto m1 =
+          RebindMask(d, Gt(Load(di, bool_lanes.get() + N), Zero(di)));
+
+      const auto combined_mask = CombineMasks(dt, m1, m0);
+      const auto expected_mask =
+          RebindMask(dt, MaskFromVec(Load(dt_i, expected.get())));
+
+      HWY_ASSERT_VEC_EQ(dt_i, expected.get(),
+                        BitCast(dt_i, VecFromMask(dt, combined_mask)));
+      HWY_ASSERT_VEC_EQ(dt_i, expected.get(),
+                        Combine(dt_i, BitCast(di, VecFromMask(d, m1)),
+                                BitCast(di, VecFromMask(d, m0))));
+      HWY_ASSERT_MASK_EQ(dt, expected_mask, combined_mask);
+    }
+
+    const size_t max_hi_lanes = HWY_MIN(max_lanes, N);
+    for (size_t code = 0; code < (1ull << max_hi_lanes); ++code) {
+      for (size_t i = 0; i < max_hi_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      const size_t lo_lane_count = (code + 1) & (N - 1);
+
+      for (size_t i = 0; i < N; ++i) {
+        expected[i] = static_cast<TI>((i < lo_lane_count) ? TI(-1) : TI(0));
+        expected[N + i] = static_cast<TI>(-static_cast<TI>(bool_lanes[i]));
+      }
+
+      const auto m0 = FirstN(d, lo_lane_count);
+      const auto m1 = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+
+      const auto combined_mask = CombineMasks(dt, m1, m0);
+      const auto expected_mask =
+          RebindMask(dt, MaskFromVec(Load(dt_i, expected.get())));
+
+      HWY_ASSERT_VEC_EQ(dt_i, expected.get(),
+                        BitCast(dt_i, VecFromMask(dt, combined_mask)));
+      HWY_ASSERT_VEC_EQ(dt_i, expected.get(),
+                        Combine(dt_i, BitCast(di, VecFromMask(d, m1)),
+                                BitCast(di, VecFromMask(d, m0))));
+      HWY_ASSERT_MASK_EQ(dt, expected_mask, combined_mask);
+    }
+
+    HWY_ASSERT_MASK_EQ(dt, FirstN(dt, N - 1),
+                       CombineMasks(dt, FirstN(d, 0), FirstN(d, N - 1)));
+    HWY_ASSERT_MASK_EQ(dt, FirstN(dt, N),
+                       CombineMasks(dt, FirstN(d, 0), FirstN(d, N)));
+    HWY_ASSERT_MASK_EQ(dt, FirstN(dt, 2 * N - 1),
+                       CombineMasks(dt, FirstN(d, N - 1), FirstN(d, N)));
+    HWY_ASSERT_MASK_EQ(dt, FirstN(dt, 2 * N),
+                       CombineMasks(dt, FirstN(d, N), FirstN(d, N)));
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllCombineMasks() {
+  ForAllTypes(ForExtendableVectors<TestCombineMasks>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyMaskCombineTest);
+HWY_EXPORT_AND_TEST_P(HwyMaskCombineTest, TestAllLowerAndUpperHalvesOfMask);
+HWY_EXPORT_AND_TEST_P(HwyMaskCombineTest, TestAllCombineMasks);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/mask_convert_test.cc

@@ -0,0 +1,351 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/mask_convert_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+template <class TTo>
+struct TestPromoteMaskTo {
+  using TTo_I = MakeSigned<TTo>;
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TI = MakeSigned<T>;
+
+    const Rebind<TTo, decltype(d)> d_to;
+    const RebindToSigned<decltype(d)> di;
+    const RebindToSigned<decltype(d_to)> di_to;
+
+    const size_t N = Lanes(di);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto expected = AllocateAligned<TTo_I>(N);
+    HWY_ASSERT(bool_lanes && expected);
+
+    ZeroBytes(bool_lanes.get(), N * sizeof(TI));
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N, size_t(6)));
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      for (size_t i = 0; i < N; ++i) {
+        expected[i] = static_cast<TTo_I>(-static_cast<TI>(bool_lanes[i]));
+      }
+
+      const auto m = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+      const auto promoted_mask = PromoteMaskTo(d_to, d, m);
+
+      const auto expected_mask =
+          RebindMask(d_to, MaskFromVec(Load(di_to, expected.get())));
+
+      HWY_ASSERT_VEC_EQ(di_to, expected.get(),
+                        BitCast(di_to, VecFromMask(d_to, promoted_mask)));
+      HWY_ASSERT_MASK_EQ(d_to, expected_mask, promoted_mask);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllPromoteMaskTo() {
+  const ForPromoteVectors<TestPromoteMaskTo<int16_t>, 1> to_i16div2;
+  to_i16div2(int8_t());
+  to_i16div2(uint8_t());
+
+  const ForPromoteVectors<TestPromoteMaskTo<uint16_t>, 1> to_u16div2;
+  to_u16div2(int8_t());
+  to_u16div2(uint8_t());
+
+  const ForPromoteVectors<TestPromoteMaskTo<int32_t>, 1> to_i32div2;
+  to_i32div2(int16_t());
+  to_i32div2(uint16_t());
+#if HWY_HAVE_FLOAT16
+  to_i32div2(float16_t());
+#endif
+
+  const ForPromoteVectors<TestPromoteMaskTo<int32_t>, 1> to_u32div2;
+  to_u32div2(int16_t());
+  to_u32div2(uint16_t());
+#if HWY_HAVE_FLOAT16
+  to_u32div2(float16_t());
+#endif
+
+  const ForPromoteVectors<TestPromoteMaskTo<int32_t>, 2> to_i32div4;
+  to_i32div4(int8_t());
+
+#if HWY_HAVE_INTEGER64
+  const ForPromoteVectors<TestPromoteMaskTo<int64_t>, 1> to_i64div2;
+  to_i64div2(int32_t());
+  to_i64div2(uint32_t());
+  to_i64div2(float());
+
+  const ForPromoteVectors<TestPromoteMaskTo<uint64_t>, 1> to_u64div2;
+  to_u64div2(int32_t());
+  to_u64div2(uint32_t());
+  to_u64div2(float());
+
+  const ForPromoteVectors<TestPromoteMaskTo<int64_t>, 2> to_i64div4;
+  to_i64div4(int16_t());
+
+  const ForPromoteVectors<TestPromoteMaskTo<int64_t>, 3> to_i64div8;
+  to_i64div8(int8_t());
+#endif
+
+#if HWY_HAVE_FLOAT64
+  const ForPromoteVectors<TestPromoteMaskTo<double>, 1> to_f64div2;
+  to_f64div2(int32_t());
+  to_f64div2(uint32_t());
+  to_f64div2(float());
+
+#if HWY_HAVE_FLOAT16
+  const ForPromoteVectors<TestPromoteMaskTo<double>, 2> to_f64div4;
+  to_f64div4(float16_t());
+#endif  // HWY_HAVE_FLOAT16
+#endif  // HWY_HAVE_FLOAT64
+}
+
+template <class TTo>
+struct TestDemoteMaskTo {
+  using TTo_I = MakeSigned<TTo>;
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TI = MakeSigned<T>;
+
+    const Rebind<TTo, decltype(d)> d_to;
+    const RebindToSigned<decltype(d)> di;
+    const RebindToSigned<decltype(d_to)> di_to;
+
+    const size_t N = Lanes(di);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto expected = AllocateAligned<TTo_I>(N);
+    HWY_ASSERT(bool_lanes && expected);
+
+    ZeroBytes(bool_lanes.get(), N * sizeof(TI));
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N, size_t(6)));
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      for (size_t i = 0; i < N; ++i) {
+        expected[i] = static_cast<TTo_I>(-static_cast<TI>(bool_lanes[i]));
+      }
+
+      const auto m = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+      const auto demoted_mask = DemoteMaskTo(d_to, d, m);
+
+      const auto expected_mask =
+          RebindMask(d_to, MaskFromVec(Load(di_to, expected.get())));
+
+      HWY_ASSERT_VEC_EQ(di_to, expected.get(),
+                        BitCast(di_to, VecFromMask(d_to, demoted_mask)));
+      HWY_ASSERT_MASK_EQ(d_to, expected_mask, demoted_mask);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllDemoteMaskTo() {
+  const ForDemoteVectors<TestDemoteMaskTo<int8_t>> from_uif16_to_i8;
+  from_uif16_to_i8(int16_t());
+  from_uif16_to_i8(uint16_t());
+#if HWY_HAVE_FLOAT16
+  from_uif16_to_i8(float16_t());
+#endif
+
+  const ForDemoteVectors<TestDemoteMaskTo<uint8_t>> from_uif16_to_u8;
+  from_uif16_to_u8(int16_t());
+  from_uif16_to_u8(uint16_t());
+#if HWY_HAVE_FLOAT16
+  from_uif16_to_u8(float16_t());
+#endif
+
+  const ForDemoteVectors<TestDemoteMaskTo<int16_t>> from_uif32_to_i16;
+  from_uif32_to_i16(int32_t());
+  from_uif32_to_i16(uint32_t());
+#if HWY_HAVE_FLOAT16
+  from_uif32_to_i16(float());
+#endif
+
+  const ForDemoteVectors<TestDemoteMaskTo<uint16_t>> from_uif32_to_u16;
+  from_uif32_to_u16(int32_t());
+  from_uif32_to_u16(uint32_t());
+  from_uif32_to_u16(float());
+
+#if HWY_HAVE_FLOAT16
+  const ForDemoteVectors<TestDemoteMaskTo<float16_t>> from_uif32_to_f16;
+  from_uif32_to_f16(int32_t());
+  from_uif32_to_f16(uint32_t());
+  from_uif32_to_f16(float());
+#endif
+
+  const ForDemoteVectors<TestDemoteMaskTo<int8_t>, 2> from_i32_to_i8;
+  from_i32_to_i8(int32_t());
+
+#if HWY_HAVE_INTEGER64
+  const ForDemoteVectors<TestDemoteMaskTo<int32_t>> from_uif64_to_i32;
+  from_uif64_to_i32(int64_t());
+  from_uif64_to_i32(uint64_t());
+#if HWY_HAVE_FLOAT64
+  from_uif64_to_i32(double());
+#endif
+
+  const ForDemoteVectors<TestDemoteMaskTo<uint32_t>> from_uif64_to_u32;
+  from_uif64_to_u32(int64_t());
+  from_uif64_to_u32(uint64_t());
+#if HWY_HAVE_FLOAT64
+  from_uif64_to_u32(double());
+#endif
+
+  const ForDemoteVectors<TestDemoteMaskTo<float>> from_uif64_to_f32;
+  from_uif64_to_f32(int64_t());
+  from_uif64_to_f32(uint64_t());
+#if HWY_HAVE_FLOAT64
+  from_uif64_to_f32(double());
+#endif
+
+  const ForDemoteVectors<TestDemoteMaskTo<int16_t>, 2> from_i64_to_i16;
+  from_i64_to_i16(int64_t());
+
+#if HWY_HAVE_FLOAT64 && HWY_HAVE_FLOAT16
+  const ForDemoteVectors<TestDemoteMaskTo<float16_t>, 2> from_f64_to_f16;
+  from_f64_to_f16(double());
+#endif
+
+  const ForDemoteVectors<TestDemoteMaskTo<int8_t>, 3> from_i64_to_i8;
+  from_i64_to_i8(int64_t());
+#endif
+}
+
+struct TestOrderedDemote2MasksTo {
+#if HWY_TARGET != HWY_SCALAR
+  template <class DTo, class D>
+  static HWY_NOINLINE void DoTestOrderedDemote2Masks(DTo d_to, D d) {
+    using T = TFromD<D>;
+    using TTo = TFromD<DTo>;
+    using TI = MakeSigned<T>;
+    using TTo_I = MakeSigned<TTo>;
+
+    const RebindToSigned<decltype(d)> di;
+    const RebindToSigned<decltype(d_to)> di_to;
+
+    const size_t N = Lanes(di);
+    auto bool_lanes = AllocateAligned<TI>(N * 2);
+    auto expected = AllocateAligned<TTo_I>(N * 2);
+    HWY_ASSERT(bool_lanes && expected);
+
+    ZeroBytes(bool_lanes.get(), N * 2 * sizeof(TI));
+    ZeroBytes(expected.get(), N * 2 * sizeof(TTo_I));
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N * 2, size_t(6)));
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      const size_t idx2 = N + (code & (N - 1));
+      bool_lanes[idx2] = TI(1);
+
+      for (size_t i = 0; i < N * 2; ++i) {
+        expected[i] = static_cast<TTo_I>(-static_cast<TI>(bool_lanes[i]));
+      }
+
+      const auto m0 = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+      const auto m1 =
+          RebindMask(d, Gt(Load(di, bool_lanes.get() + N), Zero(di)));
+      const auto expected_mask =
+          RebindMask(d_to, MaskFromVec(Load(di_to, expected.get())));
+
+      HWY_ASSERT_MASK_EQ(d_to, expected_mask,
+                         OrderedDemote2MasksTo(d_to, d, m0, m1));
+
+      bool_lanes[idx2] = TI(0);
+    }
+
+    HWY_ASSERT_MASK_EQ(
+        d_to, FirstN(d_to, N - 1),
+        OrderedDemote2MasksTo(d_to, d, FirstN(d, N - 1), FirstN(d, 0)));
+    HWY_ASSERT_MASK_EQ(
+        d_to, FirstN(d_to, N),
+        OrderedDemote2MasksTo(d_to, d, FirstN(d, N), FirstN(d, 0)));
+    HWY_ASSERT_MASK_EQ(
+        d_to, FirstN(d_to, 2 * N - 1),
+        OrderedDemote2MasksTo(d_to, d, FirstN(d, N), FirstN(d, N - 1)));
+    HWY_ASSERT_MASK_EQ(
+        d_to, FirstN(d_to, 2 * N),
+        OrderedDemote2MasksTo(d_to, d, FirstN(d, N), FirstN(d, N)));
+  }
+
+  template <class D, HWY_IF_T_SIZE_ONE_OF_D(
+                         D, (HWY_HAVE_FLOAT16 ? (1 << 4) : 0) | (1 << 8))>
+  static HWY_INLINE void DoTestOrderedDemote2MasksToFloat(D d) {
+    using TF = MakeFloat<MakeNarrow<MakeUnsigned<TFromD<D>>>>;
+    DoTestOrderedDemote2Masks(Repartition<TF, decltype(d)>(), d);
+  }
+
+  template <class D,
+            HWY_IF_T_SIZE_ONE_OF_D(D, (1 << 1) | (1 << 2) |
+                                          (HWY_HAVE_FLOAT16 ? 0 : (1 << 4)))>
+  static HWY_INLINE void DoTestOrderedDemote2MasksToFloat(D /*d*/) {}
+#endif  // HWY_TARGET != HWY_SCALAR
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    const RebindToSigned<decltype(d)> di;
+    const RebindToUnsigned<decltype(d)> du;
+
+    DoTestOrderedDemote2Masks(RepartitionToNarrow<decltype(di)>(), d);
+    DoTestOrderedDemote2Masks(RepartitionToNarrow<decltype(du)>(), d);
+    DoTestOrderedDemote2MasksToFloat(d);
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllOrderedDemote2MasksTo() {
+  ForUIF163264(ForShrinkableVectors<TestOrderedDemote2MasksTo>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyMaskConvertTest);
+HWY_EXPORT_AND_TEST_P(HwyMaskConvertTest, TestAllPromoteMaskTo);
+HWY_EXPORT_AND_TEST_P(HwyMaskConvertTest, TestAllDemoteMaskTo);
+HWY_EXPORT_AND_TEST_P(HwyMaskConvertTest, TestAllOrderedDemote2MasksTo);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/mask_mem_test.cc

@@ -0,0 +1,403 @@
+// Copyright 2019 Google LLC
+// Copyright 2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-License-Identifier: BSD-3-Clause
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdio.h>
+#include <string.h>  // memcmp
+
+#include "hwy/base.h"
+#include "hwy/nanobenchmark.h"
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/mask_mem_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestMaskedLoad {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(bool_lanes && lanes);
+
+    const Vec<D> v = IotaForSpecial(d, 1);
+    const Vec<D> v2 = IotaForSpecial(d, 2);
+    Store(v, d, lanes.get());
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+      }
+
+      const auto mask_i = Load(di, bool_lanes.get());
+      const auto mask = RebindMask(d, Gt(mask_i, Zero(di)));
+      const auto expected = IfThenElseZero(mask, Load(d, lanes.get()));
+      const auto expected2 = IfThenElse(mask, Load(d, lanes.get()), v2);
+      const auto actual = MaskedLoad(mask, d, lanes.get());
+      const auto actual2 = MaskedLoadOr(v2, mask, d, lanes.get());
+      HWY_ASSERT_VEC_EQ(d, expected, actual);
+      HWY_ASSERT_VEC_EQ(d, expected2, actual2);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllMaskedLoad() {
+  ForAllTypesAndSpecial(ForPartialVectors<TestMaskedLoad>());
+}
+
+struct TestMaskedScatter {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    using VI = Vec<decltype(di)>;
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto lanes = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(bool_lanes && lanes && expected);
+
+    const Vec<D> v = Iota(d, hwy::Unpredictable1() - 1);
+    Store(v, d, lanes.get());
+
+    const VI indices = Reverse(di, Iota(di, hwy::Unpredictable1() - 1));
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      ZeroBytes(expected.get(), N * sizeof(T));
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+        if (bool_lanes[i]) {
+          expected[N - 1 - i] = ConvertScalarTo<T>(i);
+        }
+      }
+
+      const VI mask_i = Load(di, bool_lanes.get());
+      const auto mask = RebindMask(d, Gt(mask_i, Zero(di)));
+      ZeroBytes(lanes.get(), N * sizeof(T));
+      MaskedScatterIndex(v, mask, d, lanes.get(), indices);
+      HWY_ASSERT_VEC_EQ(d, expected.get(), Load(d, lanes.get()));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllMaskedScatter() {
+  ForUIF3264(ForPartialVectors<TestMaskedScatter>());
+}
+
+struct TestScatterIndexN {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    using VI = Vec<decltype(di)>;
+    const size_t N = Lanes(d);
+    auto lanes = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(lanes && expected);
+
+    const Vec<D> v = Iota(d, hwy::Unpredictable1() - 1);
+    Store(v, d, lanes.get());
+
+    const VI indices = Reverse(di, Iota(di, hwy::Unpredictable1() - 1));
+
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      // Choose 1 to N lanes to store
+      const size_t max_lanes_to_store = (Random32(&rng) % N) + 1;
+
+      ZeroBytes(expected.get(), N * sizeof(T));
+      for (size_t i = 0; i < max_lanes_to_store; ++i) {
+        expected[N - 1 - i] = ConvertScalarTo<T>(i);
+      }
+
+      ZeroBytes(lanes.get(), N * sizeof(T));
+      ScatterIndexN(v, d, lanes.get(), indices, max_lanes_to_store);
+      HWY_ASSERT_VEC_EQ(d, expected.get(), Load(d, lanes.get()));
+    }
+
+    // Zero store is just zeroes
+    ZeroBytes(expected.get(), N * sizeof(T));
+    ZeroBytes(lanes.get(), N * sizeof(T));
+    ScatterIndexN(v, d, lanes.get(), indices, 0);
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Load(d, lanes.get()));
+
+    // Load is clamped at min(N, max_lanes_to_load)
+    auto larger_memory = AllocateAligned<T>(N * 2);
+    auto larger_expected = AllocateAligned<T>(N * 2);
+    HWY_ASSERT(larger_memory && larger_expected);
+    ZeroBytes(larger_expected.get(), N * sizeof(T) * 2);
+    for (size_t i = 0; i < N; ++i) {
+      larger_expected[N - 1 - i] = ConvertScalarTo<T>(i);
+    }
+    ZeroBytes(larger_memory.get(), N * sizeof(T));
+    ScatterIndexN(v, d, larger_memory.get(), indices, N + 1);
+    HWY_ASSERT_VEC_EQ(d, larger_expected.get(), Load(d, larger_memory.get()));
+  }
+};
+
+HWY_NOINLINE void TestAllScatterIndexN() {
+  ForUIF3264(ForPartialVectors<TestScatterIndexN>());
+}
+
+struct TestMaskedGather {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    using VI = Vec<decltype(di)>;
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(bool_lanes && lanes);
+
+    const Vec<D> v = Iota(d, hwy::Unpredictable1() - 1);
+    Store(v, d, lanes.get());
+
+    const Vec<D> no = Set(d, ConvertScalarTo<T>(2));
+
+    const VI indices = Reverse(di, Iota(di, hwy::Unpredictable1() - 1));
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = static_cast<TI>((Random32(&rng) & 1024) ? 1 : 0);
+      }
+
+      const VI mask_i = Load(di, bool_lanes.get());
+      const auto mask = RebindMask(d, Gt(mask_i, Zero(di)));
+      const Vec<D> expected_z = IfThenElseZero(mask, Reverse(d, v));
+      const Vec<D> expected_or = IfThenElse(mask, Reverse(d, v), no);
+      const Vec<D> actual_z = MaskedGatherIndex(mask, d, lanes.get(), indices);
+      const Vec<D> actual_or =
+          MaskedGatherIndexOr(no, mask, d, lanes.get(), indices);
+      HWY_ASSERT_VEC_EQ(d, expected_z, actual_z);
+      HWY_ASSERT_VEC_EQ(d, expected_or, actual_or);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllMaskedGather() {
+  ForUIF3264(ForPartialVectors<TestMaskedGather>());
+}
+
+struct TestGatherIndexN {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    using VI = Vec<decltype(di)>;
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(bool_lanes && lanes);
+
+    const Vec<D> v = Iota(d, hwy::Unpredictable1() - 1);
+    Store(v, d, lanes.get());
+
+    const VI indices = Reverse(di, Iota(di, hwy::Unpredictable1() - 1));
+
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      // Choose 1 to N lanes to load
+      const size_t max_lanes_to_load = (Random32(&rng) % N) + 1;
+
+      // Convert lane count to mask to compare results
+      const auto mask = FirstN(d, max_lanes_to_load);
+
+      const Vec<D> expected = IfThenElseZero(mask, Reverse(d, v));
+      const Vec<D> actual =
+          GatherIndexN(d, lanes.get(), indices, max_lanes_to_load);
+      HWY_ASSERT_VEC_EQ(d, expected, actual);
+    }
+
+    // Zero load is just zeroes
+    const Vec<D> zeroes = Zero(d);
+    const Vec<D> actual_zero = GatherIndexN(d, lanes.get(), indices, 0);
+    HWY_ASSERT_VEC_EQ(d, zeroes, actual_zero);
+
+    // Load is clamped at min(N, max_lanes_to_load)
+    const auto clamped_mask = FirstN(d, N);
+    const Vec<D> expected_clamped = IfThenElseZero(clamped_mask, Reverse(d, v));
+    const Vec<D> actual_clamped = GatherIndexN(d, lanes.get(), indices, N + 1);
+    HWY_ASSERT_VEC_EQ(d, expected_clamped, actual_clamped);
+  }
+};
+
+HWY_NOINLINE void TestAllGatherIndexN() {
+  ForUIF3264(ForPartialVectors<TestGatherIndexN>());
+}
+
+struct TestBlendedStore {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto actual = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(bool_lanes && actual && expected);
+
+    const Vec<D> v = IotaForSpecial(d, 1);
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+        // Re-initialize to something distinct from v[i].
+        actual[i] = ConvertScalarTo<T>(127 - (i & 127));
+        expected[i] = bool_lanes[i] ? ConvertScalarTo<T>(i + 1) : actual[i];
+      }
+
+      const auto mask = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+      BlendedStore(v, mask, d, actual.get());
+      HWY_ASSERT_VEC_EQ(d, expected.get(), Load(d, actual.get()));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllBlendedStore() {
+  ForAllTypesAndSpecial(ForPartialVectors<TestBlendedStore>());
+}
+
+class TestStoreMaskBits {
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*t*/, D /*d*/) {
+    RandomState rng;
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(di);
+    const size_t expected_num_bytes = (N + 7) / 8;
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto expected = AllocateAligned<uint8_t>(expected_num_bytes);
+    auto actual = AllocateAligned<uint8_t>(HWY_MAX(8, expected_num_bytes));
+    HWY_ASSERT(bool_lanes && actual && expected);
+
+    const ScalableTag<uint8_t, -3> d_bits;
+
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      // Generate random mask pattern.
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = static_cast<TI>((rng() & 1024) ? 1 : 0);
+      }
+      const auto bools = Load(di, bool_lanes.get());
+      const auto mask = Gt(bools, Zero(di));
+
+      // Requires at least 8 bytes, ensured above.
+      const size_t bytes_written = StoreMaskBits(di, mask, actual.get());
+      if (bytes_written != expected_num_bytes) {
+        fprintf(stderr, "%s expected %d bytes, actual %d\n",
+                TypeName(T(), N).c_str(), static_cast<int>(expected_num_bytes),
+                static_cast<int>(bytes_written));
+
+        HWY_ASSERT(false);
+      }
+
+      // Requires at least 8 bytes, ensured above.
+      const auto mask2 = LoadMaskBits(di, actual.get());
+      HWY_ASSERT_MASK_EQ(di, mask, mask2);
+
+      memset(expected.get(), 0, expected_num_bytes);
+      for (size_t i = 0; i < N; ++i) {
+        expected[i / 8] =
+            static_cast<uint8_t>(expected[i / 8] | (bool_lanes[i] << (i % 8)));
+      }
+
+      size_t i = 0;
+      // Stored bits must match original mask
+      for (; i < N; ++i) {
+        const TI is_set = (actual[i / 8] & (1 << (i % 8))) ? 1 : 0;
+        if (is_set != bool_lanes[i]) {
+          fprintf(stderr, "%s lane %d: expected %d, actual %d\n",
+                  TypeName(T(), N).c_str(), static_cast<int>(i),
+                  static_cast<int>(bool_lanes[i]), static_cast<int>(is_set));
+          Print(di, "bools", bools, 0, N);
+          Print(d_bits, "expected bytes", Load(d_bits, expected.get()), 0,
+                expected_num_bytes);
+          Print(d_bits, "actual bytes", Load(d_bits, actual.get()), 0,
+                expected_num_bytes);
+
+          HWY_ASSERT(false);
+        }
+      }
+      // Any partial bits in the last byte must be zero
+      for (; i < 8 * bytes_written; ++i) {
+        const int bit = (actual[i / 8] & (1 << (i % 8)));
+        if (bit != 0) {
+          fprintf(stderr, "%s: bit #%d should be zero\n",
+                  TypeName(T(), N).c_str(), static_cast<int>(i));
+          Print(di, "bools", bools, 0, N);
+          Print(d_bits, "expected bytes", Load(d_bits, expected.get()), 0,
+                expected_num_bytes);
+          Print(d_bits, "actual bytes", Load(d_bits, actual.get()), 0,
+                expected_num_bytes);
+
+          HWY_ASSERT(false);
+        }
+      }
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllStoreMaskBits() {
+  ForAllTypes(ForPartialVectors<TestStoreMaskBits>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyMaskMemTest);
+HWY_EXPORT_AND_TEST_P(HwyMaskMemTest, TestAllMaskedLoad);
+HWY_EXPORT_AND_TEST_P(HwyMaskMemTest, TestAllMaskedScatter);
+HWY_EXPORT_AND_TEST_P(HwyMaskMemTest, TestAllScatterIndexN);
+HWY_EXPORT_AND_TEST_P(HwyMaskMemTest, TestAllMaskedGather);
+HWY_EXPORT_AND_TEST_P(HwyMaskMemTest, TestAllGatherIndexN);
+HWY_EXPORT_AND_TEST_P(HwyMaskMemTest, TestAllBlendedStore);
+HWY_EXPORT_AND_TEST_P(HwyMaskMemTest, TestAllStoreMaskBits);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/mask_slide_test.cc

@@ -0,0 +1,159 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/mask_slide_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestSlideMaskDownLanes {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    using TI = MakeSigned<T>;
+
+    const RebindToSigned<decltype(d)> di;
+
+    const size_t N = Lanes(d);
+    if (N < 2) {
+      return;
+    }
+
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto expected = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes && expected);
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N, size_t(6)));
+
+    ZeroBytes(bool_lanes.get(), max_lanes * sizeof(TI));
+    for (size_t i = max_lanes; i < N; i++) {
+      bool_lanes[i] = TI(-1);
+    }
+
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(-1) : TI(0);
+      }
+
+      for (size_t i = 0; i < max_lanes; i++) {
+        ZeroBytes(expected.get() + N - i, i * sizeof(TI));
+        for (size_t j = 0; j < N - i; j++) {
+          expected[j] = bool_lanes[j + i];
+        }
+
+        const auto src_mask =
+            MaskFromVec(BitCast(d, Load(di, bool_lanes.get())));
+        const auto expected_mask =
+            MaskFromVec(BitCast(d, Load(di, expected.get())));
+        const auto actual_mask = SlideMaskDownLanes(d, src_mask, i);
+        HWY_ASSERT_MASK_EQ(d, expected_mask, actual_mask);
+
+        if (i == 1) {
+          HWY_ASSERT_MASK_EQ(d, expected_mask, SlideMask1Down(d, src_mask));
+        }
+      }
+    }
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllSlideMaskDownLanes() {
+  ForAllTypes(ForPartialVectors<TestSlideMaskDownLanes>());
+}
+
+struct TestSlideMaskUpLanes {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    using TI = MakeSigned<T>;
+
+    const RebindToSigned<decltype(d)> di;
+
+    const size_t N = Lanes(d);
+    if (N < 2) {
+      return;
+    }
+
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto expected = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes && expected);
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N, size_t(6)));
+
+    ZeroBytes(bool_lanes.get(), max_lanes * sizeof(TI));
+    for (size_t i = max_lanes; i < N; i++) {
+      bool_lanes[i] = TI(-1);
+    }
+
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(-1) : TI(0);
+      }
+
+      for (size_t i = 0; i < max_lanes; i++) {
+        ZeroBytes(expected.get(), i * sizeof(TI));
+        for (size_t j = 0; j < N - i; j++) {
+          expected[j + i] = bool_lanes[j];
+        }
+
+        const auto src_mask =
+            MaskFromVec(BitCast(d, Load(di, bool_lanes.get())));
+        const auto expected_mask =
+            MaskFromVec(BitCast(d, Load(di, expected.get())));
+        const auto actual_mask = SlideMaskUpLanes(d, src_mask, i);
+        HWY_ASSERT_MASK_EQ(d, expected_mask, actual_mask);
+
+        if (i == 1) {
+          HWY_ASSERT_MASK_EQ(d, expected_mask, SlideMask1Up(d, src_mask));
+        }
+      }
+    }
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllSlideMaskUpLanes() {
+  ForAllTypes(ForPartialVectors<TestSlideMaskUpLanes>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyMaskSlideTest);
+HWY_EXPORT_AND_TEST_P(HwyMaskSlideTest, TestAllSlideMaskDownLanes);
+HWY_EXPORT_AND_TEST_P(HwyMaskSlideTest, TestAllSlideMaskUpLanes);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/mask_test.cc

@@ -0,0 +1,578 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>  // memcmp
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/mask_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+// All types.
+struct TestMaskFalse {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_HAVE_SCALABLE || HWY_TARGET_IS_SVE || HWY_TARGET == HWY_SCALAR
+    // For RVV, SVE and SCALAR, use the underlying native vector.
+    const DFromV<Vec<D>> d2;
+#else
+    // Other targets are strongly-typed, but we can safely ResizeBitCast to the
+    // native vector. All targets have at least 128-bit vectors, but NEON also
+    // supports 64-bit vectors.
+    constexpr size_t kMinD2Lanes = (HWY_TARGET_IS_NEON ? 8 : 16) / sizeof(T);
+    const FixedTag<T, HWY_MAX(HWY_MAX_LANES_D(D), kMinD2Lanes)> d2;
+#endif
+    static_assert(d2.MaxBytes() >= d.MaxBytes(),
+                  "d2.MaxBytes() >= d.MaxBytes() should be true");
+    using V2 = Vec<decltype(d2)>;
+
+    // Various ways of checking that false masks are false.
+    HWY_ASSERT(AllFalse(d, MaskFalse(d)));
+    HWY_ASSERT_EQ(0, CountTrue(d, MaskFalse(d)));
+    HWY_ASSERT_VEC_EQ(d, Zero(d), VecFromMask(d, MaskFalse(d)));
+
+#if HWY_HAVE_SCALABLE || HWY_TARGET_IS_SVE
+    // For these targets, we can treat the result as if it were a vector of type
+    // `V2`. On SVE, vectors are always full (not fractional) and caps are only
+    // enforced by Highway ops. On RVV, LMUL must match but caps can also be
+    // ignored. For safety, MaskFalse also sets lanes >= `Lanes(d)` to false,
+    // and we verify that here.
+    HWY_ASSERT(AllFalse(d2, MaskFalse(d)));
+    HWY_ASSERT_EQ(0, CountTrue(d2, MaskFalse(d)));
+    HWY_ASSERT_VEC_EQ(d2, Zero(d2), VecFromMask(d2, MaskFalse(d)));
+#endif
+
+    // All targets support, and strongly-typed (non-scalable) targets require,
+    // ResizeBitCast before we compare to the 'native' underlying vector size.
+    const V2 actual2 = ResizeBitCast(d2, VecFromMask(d, MaskFalse(d)));
+    HWY_ASSERT_VEC_EQ(d2, Zero(d2), actual2);
+  }
+};
+
+HWY_NOINLINE void TestAllMaskFalse() {
+  ForAllTypes(ForPartialVectors<TestMaskFalse>());
+}
+
+struct TestFromVec {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(lanes);
+
+    memset(lanes.get(), 0, N * sizeof(T));
+    const auto actual_false = MaskFromVec(Load(d, lanes.get()));
+    HWY_ASSERT_MASK_EQ(d, MaskFalse(d), actual_false);
+
+    memset(lanes.get(), 0xFF, N * sizeof(T));
+    const auto actual_true = MaskFromVec(Load(d, lanes.get()));
+    HWY_ASSERT_MASK_EQ(d, MaskTrue(d), actual_true);
+  }
+};
+
+HWY_NOINLINE void TestAllFromVec() {
+  ForAllTypes(ForPartialVectors<TestFromVec>());
+}
+
+struct TestFirstN {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(bool_lanes);
+
+    using TN = SignedFromSize<HWY_MIN(sizeof(size_t), sizeof(T))>;
+    const size_t max_len = static_cast<size_t>(LimitsMax<TN>());
+
+    const Vec<D> k1 = Set(d, ConvertScalarTo<T>(1));
+
+    const size_t max_lanes = HWY_MIN(2 * N, AdjustedReps(512));
+    for (size_t len = 0; len <= HWY_MIN(max_lanes, max_len); ++len) {
+      // Loop instead of Iota+Lt to avoid wraparound for 8-bit T.
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = ConvertScalarTo<T>(i < len ? 1 : 0);
+      }
+      const Mask<D> expected = Eq(Load(d, bool_lanes.get()), k1);
+      HWY_ASSERT_MASK_EQ(d, expected, FirstN(d, len));
+    }
+
+    // Also ensure huge values yield all-true (unless the vector is actually
+    // larger than max_len).
+    for (size_t i = 0; i < N; ++i) {
+      bool_lanes[i] = ConvertScalarTo<T>(i < max_len ? 1 : 0);
+    }
+    const Mask<D> expected = Eq(Load(d, bool_lanes.get()), k1);
+    HWY_ASSERT_MASK_EQ(d, expected, FirstN(d, max_len));
+  }
+};
+
+HWY_NOINLINE void TestAllFirstN() {
+  ForAllTypes(ForPartialVectors<TestFirstN>());
+}
+
+struct TestMaskVec {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes);
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+      }
+
+      const auto mask = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+      HWY_ASSERT_MASK_EQ(d, mask, MaskFromVec(VecFromMask(d, mask)));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllMaskVec() {
+  const ForPartialVectors<TestMaskVec> test;
+
+  test(uint16_t());
+  test(int16_t());
+  // TODO(janwas): float16_t - cannot compare yet
+
+  ForUIF3264(test);
+}
+
+struct TestAllTrueFalse {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto zero = Zero(d);
+    auto v = zero;
+
+    const size_t N = Lanes(d);
+    auto lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(lanes);
+    ZeroBytes(lanes.get(), N * sizeof(T));
+
+    HWY_ASSERT(AllTrue(d, Eq(v, zero)));
+    HWY_ASSERT(!AllFalse(d, Eq(v, zero)));
+
+    // Single lane implies AllFalse = !AllTrue. Otherwise, there are multiple
+    // lanes and one is nonzero.
+    const bool expected_all_false = (N != 1);
+
+    // Set each lane to nonzero and back to zero
+    for (size_t i = 0; i < N; ++i) {
+      lanes[i] = ConvertScalarTo<T>(1);
+      v = Load(d, lanes.get());
+
+      HWY_ASSERT(!AllTrue(d, Eq(v, zero)));
+
+      HWY_ASSERT(expected_all_false ^ AllFalse(d, Eq(v, zero)));
+
+      lanes[i] = ConvertScalarTo<T>(-1);
+      v = Load(d, lanes.get());
+      HWY_ASSERT(!AllTrue(d, Eq(v, zero)));
+      HWY_ASSERT(expected_all_false ^ AllFalse(d, Eq(v, zero)));
+
+      // Reset to all zero
+      lanes[i] = ConvertScalarTo<T>(0);
+      v = Load(d, lanes.get());
+      HWY_ASSERT(AllTrue(d, Eq(v, zero)));
+      HWY_ASSERT(!AllFalse(d, Eq(v, zero)));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllAllTrueFalse() {
+  ForAllTypes(ForPartialVectors<TestAllTrueFalse>());
+}
+
+struct TestCountTrue {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(di);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes);
+    memset(bool_lanes.get(), 0, N * sizeof(TI));
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = HWY_MIN(N, size_t(10));
+
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      // Number of zeros written = number of mask lanes that are true.
+      size_t expected = 0;
+      for (size_t i = 0; i < max_lanes; ++i) {
+        const bool is_true = (code & (1ull << i)) != 0;
+        bool_lanes[i] = is_true ? TI(1) : TI(0);
+        expected += is_true;
+      }
+
+      const auto mask = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+      const size_t actual = CountTrue(d, mask);
+      HWY_ASSERT_EQ(expected, actual);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllCountTrue() {
+  ForAllTypes(ForPartialVectors<TestCountTrue>());
+}
+
+struct TestFindFirstTrue {  // Also FindKnownFirstTrue
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(di);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes);
+    memset(bool_lanes.get(), 0, N * sizeof(TI));
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N, size_t(9)));
+
+    HWY_ASSERT_EQ(intptr_t(-1), FindFirstTrue(d, MaskFalse(d)));
+    HWY_ASSERT_EQ(intptr_t(0), FindFirstTrue(d, MaskTrue(d)));
+    HWY_ASSERT_EQ(size_t(0), FindKnownFirstTrue(d, MaskTrue(d)));
+
+    for (size_t code = 1; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      const size_t expected =
+          Num0BitsBelowLS1Bit_Nonzero32(static_cast<uint32_t>(code));
+      const auto mask = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+      HWY_ASSERT_EQ(static_cast<intptr_t>(expected), FindFirstTrue(d, mask));
+      HWY_ASSERT_EQ(expected, FindKnownFirstTrue(d, mask));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllFindFirstTrue() {
+  ForAllTypes(ForPartialVectors<TestFindFirstTrue>());
+}
+
+struct TestFindLastTrue {  // Also FindKnownLastTrue
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(di);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes);
+    memset(bool_lanes.get(), 0, N * sizeof(TI));
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N, size_t(9)));
+
+    HWY_ASSERT_EQ(intptr_t(-1), FindLastTrue(d, MaskFalse(d)));
+    HWY_ASSERT_EQ(intptr_t(Lanes(d) - 1), FindLastTrue(d, MaskTrue(d)));
+    HWY_ASSERT_EQ(size_t(Lanes(d) - 1), FindKnownLastTrue(d, MaskTrue(d)));
+
+    for (size_t code = 1; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      const size_t expected =
+          31 - Num0BitsAboveMS1Bit_Nonzero32(static_cast<uint32_t>(code));
+      const auto mask = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+      HWY_ASSERT_EQ(static_cast<intptr_t>(expected), FindLastTrue(d, mask));
+      HWY_ASSERT_EQ(expected, FindKnownLastTrue(d, mask));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllFindLastTrue() {
+  ForAllTypes(ForPartialVectors<TestFindLastTrue>());
+}
+
+struct TestLogicalMask {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto m0 = MaskFalse(d);
+    const auto m_all = MaskTrue(d);
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(di);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes);
+    memset(bool_lanes.get(), 0, N * sizeof(TI));
+
+    HWY_ASSERT_MASK_EQ(d, m0, Not(m_all));
+    HWY_ASSERT_MASK_EQ(d, m_all, Not(m0));
+
+    HWY_ASSERT_MASK_EQ(d, m_all, ExclusiveNeither(m0, m0));
+    HWY_ASSERT_MASK_EQ(d, m0, ExclusiveNeither(m_all, m0));
+    HWY_ASSERT_MASK_EQ(d, m0, ExclusiveNeither(m0, m_all));
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N, size_t(6)));
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      const auto m = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+
+      HWY_ASSERT_MASK_EQ(d, m0, Xor(m, m));
+      HWY_ASSERT_MASK_EQ(d, m0, AndNot(m, m));
+      HWY_ASSERT_MASK_EQ(d, m0, AndNot(m_all, m));
+
+      HWY_ASSERT_MASK_EQ(d, m, Or(m, m));
+      HWY_ASSERT_MASK_EQ(d, m, Or(m0, m));
+      HWY_ASSERT_MASK_EQ(d, m, Or(m, m0));
+      HWY_ASSERT_MASK_EQ(d, m, Xor(m0, m));
+      HWY_ASSERT_MASK_EQ(d, m, Xor(m, m0));
+      HWY_ASSERT_MASK_EQ(d, m, And(m, m));
+      HWY_ASSERT_MASK_EQ(d, m, And(m_all, m));
+      HWY_ASSERT_MASK_EQ(d, m, And(m, m_all));
+      HWY_ASSERT_MASK_EQ(d, m, AndNot(m0, m));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllLogicalMask() {
+  ForAllTypes(ForPartialVectors<TestLogicalMask>());
+}
+
+struct TestSetBeforeFirst {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(di);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes);
+    memset(bool_lanes.get(), 0, N * sizeof(TI));
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N, size_t(6)));
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      const auto m = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+
+      const size_t first_set_lane_idx =
+          (code != 0)
+              ? Num0BitsBelowLS1Bit_Nonzero64(static_cast<uint64_t>(code))
+              : N;
+      const auto expected_mask = FirstN(d, first_set_lane_idx);
+
+      HWY_ASSERT_MASK_EQ(d, expected_mask, SetBeforeFirst(m));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllSetBeforeFirst() {
+  ForAllTypes(ForPartialVectors<TestSetBeforeFirst>());
+}
+
+struct TestSetAtOrBeforeFirst {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(di);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes);
+    memset(bool_lanes.get(), 0, N * sizeof(TI));
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N, size_t(6)));
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      const auto m = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+
+      const size_t idx_after_first_set_lane =
+          (code != 0)
+              ? (Num0BitsBelowLS1Bit_Nonzero64(static_cast<uint64_t>(code)) + 1)
+              : N;
+      const auto expected_mask = FirstN(d, idx_after_first_set_lane);
+
+      HWY_ASSERT_MASK_EQ(d, expected_mask, SetAtOrBeforeFirst(m));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllSetAtOrBeforeFirst() {
+  ForAllTypes(ForPartialVectors<TestSetAtOrBeforeFirst>());
+}
+
+struct TestSetOnlyFirst {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(di);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes);
+    memset(bool_lanes.get(), 0, N * sizeof(TI));
+    auto expected_lanes = AllocateAligned<TI>(N);
+    HWY_ASSERT(expected_lanes);
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N, size_t(6)));
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      memset(expected_lanes.get(), 0, N * sizeof(TI));
+      if (code != 0) {
+        const size_t idx_of_first_lane =
+            Num0BitsBelowLS1Bit_Nonzero64(static_cast<uint64_t>(code));
+        expected_lanes[idx_of_first_lane] = TI(1);
+      }
+
+      const auto m = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+      const auto expected_mask =
+          RebindMask(d, Gt(Load(di, expected_lanes.get()), Zero(di)));
+
+      HWY_ASSERT_MASK_EQ(d, expected_mask, SetOnlyFirst(m));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllSetOnlyFirst() {
+  ForAllTypes(ForPartialVectors<TestSetOnlyFirst>());
+}
+
+struct TestSetAtOrAfterFirst {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(di);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes);
+    memset(bool_lanes.get(), 0, N * sizeof(TI));
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    const size_t max_lanes = AdjustedLog2Reps(HWY_MIN(N, size_t(6)));
+    for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+      for (size_t i = 0; i < max_lanes; ++i) {
+        bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0);
+      }
+
+      const auto m = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di)));
+
+      const size_t first_set_lane_idx =
+          (code != 0)
+              ? Num0BitsBelowLS1Bit_Nonzero64(static_cast<uint64_t>(code))
+              : N;
+      const auto expected_at_or_after_first_mask =
+          Not(FirstN(d, first_set_lane_idx));
+      const auto actual_at_or_after_first_mask = SetAtOrAfterFirst(m);
+
+      HWY_ASSERT_MASK_EQ(d, expected_at_or_after_first_mask,
+                         actual_at_or_after_first_mask);
+      HWY_ASSERT_MASK_EQ(
+          d, SetOnlyFirst(m),
+          And(actual_at_or_after_first_mask, SetAtOrBeforeFirst(m)));
+      HWY_ASSERT_MASK_EQ(d, m, And(m, actual_at_or_after_first_mask));
+      HWY_ASSERT(
+          AllTrue(d, Xor(actual_at_or_after_first_mask, SetBeforeFirst(m))));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllSetAtOrAfterFirst() {
+  ForAllTypes(ForPartialVectors<TestSetAtOrAfterFirst>());
+}
+
+struct TestDup128MaskFromMaskBits {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    const size_t N = Lanes(di);
+    constexpr size_t kLanesPer16ByteBlock = 16 / sizeof(T);
+
+    auto expected = AllocateAligned<TI>(N);
+    HWY_ASSERT(expected);
+
+    // For all combinations of zero/nonzero state of subset of lanes:
+    constexpr size_t kMaxLanesToCheckPerBlk =
+        HWY_MIN(HWY_MAX_LANES_D(D), HWY_MIN(kLanesPer16ByteBlock, 10));
+    const size_t max_lanes = HWY_MIN(N, kMaxLanesToCheckPerBlk);
+
+    for (unsigned code = 0; code < (1u << max_lanes); ++code) {
+      for (size_t i = 0; i < N; i++) {
+        expected[i] = static_cast<TI>(
+            -static_cast<TI>((code >> (i & (kLanesPer16ByteBlock - 1))) & 1));
+      }
+
+      const auto expected_mask =
+          MaskFromVec(BitCast(d, LoadDup128(di, expected.get())));
+
+      const auto m = Dup128MaskFromMaskBits(d, code);
+      HWY_ASSERT_VEC_EQ(di, expected.get(), VecFromMask(di, RebindMask(di, m)));
+      HWY_ASSERT_MASK_EQ(d, expected_mask, m);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllDup128MaskFromMaskBits() {
+  ForAllTypes(ForPartialVectors<TestDup128MaskFromMaskBits>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyMaskTest);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllMaskFalse);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllFromVec);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllFirstN);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllMaskVec);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllAllTrueFalse);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllCountTrue);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllFindFirstTrue);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllFindLastTrue);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllLogicalMask);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllSetBeforeFirst);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllSetAtOrBeforeFirst);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllSetOnlyFirst);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllSetAtOrAfterFirst);
+HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllDup128MaskFromMaskBits);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/masked_arithmetic_test.cc

@@ -0,0 +1,401 @@
+// Copyright 2023 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "hwy/base.h"
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/masked_arithmetic_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/nanobenchmark.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestAddSubMul {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+    const Vec<D> v2 = Iota(d, hwy::Unpredictable1() + 1);
+    const Vec<D> v3 = Iota(d, hwy::Unpredictable1() + 2);
+    const Vec<D> v4 = Iota(d, hwy::Unpredictable1() + 3);
+    // So that we can subtract two iotas without resulting in a constant.
+    const Vec<D> tv4 = Add(v4, v4);
+    // For range-limited (so mul does not overflow), non-constant inputs.
+    // We cannot just And() because T might be floating-point.
+    alignas(16) static const T mod_lanes[16] = {
+        ConvertScalarTo<T>(0), ConvertScalarTo<T>(1), ConvertScalarTo<T>(2),
+        ConvertScalarTo<T>(hwy::Unpredictable1() + 2)};
+    const Vec<D> in_mul = LoadDup128(d, mod_lanes);
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    using VI = Vec<decltype(di)>;
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto expected_add = AllocateAligned<T>(N);
+    auto expected_sub = AllocateAligned<T>(N);
+    auto expected_mul = AllocateAligned<T>(N);
+    HWY_ASSERT(bool_lanes && expected_add && expected_sub && expected_mul);
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+        if (bool_lanes[i]) {
+          expected_add[i] = ConvertScalarTo<T>(2 * i + 7);
+          expected_sub[i] = ConvertScalarTo<T>(i + 5);
+          const size_t mod_i = i & ((16 / sizeof(T)) - 1);
+          expected_mul[i] =
+              ConvertScalarTo<T>(mod_lanes[mod_i] * mod_lanes[mod_i]);
+        } else {
+          expected_add[i] = ConvertScalarTo<T>(i + 2);
+          expected_sub[i] = ConvertScalarTo<T>(i + 2);
+          expected_mul[i] = ConvertScalarTo<T>(i + 2);
+        }
+      }
+
+      const VI mask_i = Load(di, bool_lanes.get());
+      const Mask<D> mask = RebindMask(d, Gt(mask_i, Zero(di)));
+
+      HWY_ASSERT_VEC_EQ(d, expected_add.get(), MaskedAddOr(v2, mask, v3, v4));
+      HWY_ASSERT_VEC_EQ(d, expected_sub.get(), MaskedSubOr(v2, mask, tv4, v3));
+      HWY_ASSERT_VEC_EQ(d, expected_mul.get(),
+                        MaskedMulOr(v2, mask, in_mul, in_mul));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllAddSubMul() {
+  ForAllTypes(ForPartialVectors<TestAddSubMul>());
+}
+
+struct TestUnsignedSatAddSub {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    using VI = Vec<decltype(di)>;
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes);
+
+    const Vec<D> v2 = Iota(d, hwy::Unpredictable1() + 1);
+
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> vi = Iota(d, 1);
+    const Vec<D> vm = Set(d, LimitsMax<T>());
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+      }
+      const VI mask_i = Load(di, bool_lanes.get());
+      const Mask<D> mask = RebindMask(d, Gt(mask_i, Zero(di)));
+
+      const Vec<D> disabled_lane_val = Iota(d, 2);
+
+      Vec<D> expected_add =
+          IfThenElse(mask, Set(d, static_cast<T>(0)), disabled_lane_val);
+      HWY_ASSERT_VEC_EQ(d, expected_add, MaskedSatAddOr(v2, mask, v0, v0));
+      expected_add = IfThenElse(mask, vi, disabled_lane_val);
+      HWY_ASSERT_VEC_EQ(d, expected_add, MaskedSatAddOr(v2, mask, v0, vi));
+      expected_add = IfThenElse(mask, Set(d, static_cast<T>(LimitsMax<T>())),
+                                disabled_lane_val);
+      HWY_ASSERT_VEC_EQ(d, expected_add, MaskedSatAddOr(v2, mask, v0, vm));
+      HWY_ASSERT_VEC_EQ(d, expected_add, MaskedSatAddOr(v2, mask, vi, vm));
+      HWY_ASSERT_VEC_EQ(d, expected_add, MaskedSatAddOr(v2, mask, vm, vm));
+
+      Vec<D> expected_sub =
+          IfThenElse(mask, Set(d, static_cast<T>(0)), disabled_lane_val);
+      HWY_ASSERT_VEC_EQ(d, expected_sub, MaskedSatSubOr(v2, mask, v0, v0));
+      HWY_ASSERT_VEC_EQ(d, expected_sub, MaskedSatSubOr(v2, mask, v0, vi));
+      HWY_ASSERT_VEC_EQ(d, expected_sub, MaskedSatSubOr(v2, mask, vi, vi));
+      HWY_ASSERT_VEC_EQ(d, expected_sub, MaskedSatSubOr(v2, mask, vi, vm));
+      expected_sub = IfThenElse(mask, Sub(vm, vi), disabled_lane_val);
+      HWY_ASSERT_VEC_EQ(d, expected_sub, MaskedSatSubOr(v2, mask, vm, vi));
+    }
+  }
+};
+
+struct TestSignedSatAddSub {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    using VI = Vec<decltype(di)>;
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    HWY_ASSERT(bool_lanes);
+
+    const Vec<D> v2 = Iota(d, hwy::Unpredictable1() + 1);
+
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> vpm = Set(d, LimitsMax<T>());
+    const Vec<D> vi = PositiveIota(d);
+    const Vec<D> vn = Sub(v0, vi);
+    const Vec<D> vnm = Set(d, LimitsMin<T>());
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+      }
+      const VI mask_i = Load(di, bool_lanes.get());
+      const Mask<D> mask = RebindMask(d, Gt(mask_i, Zero(di)));
+
+      const Vec<D> disabled_lane_val = Iota(d, 2);
+
+      Vec<D> expected_add = IfThenElse(mask, v0, disabled_lane_val);
+      HWY_ASSERT_VEC_EQ(d, expected_add, MaskedSatAddOr(v2, mask, v0, v0));
+      expected_add = IfThenElse(mask, vi, disabled_lane_val);
+      HWY_ASSERT_VEC_EQ(d, expected_add, MaskedSatAddOr(v2, mask, v0, vi));
+      expected_add = IfThenElse(mask, vpm, disabled_lane_val);
+      HWY_ASSERT_VEC_EQ(d, expected_add, MaskedSatAddOr(v2, mask, v0, vpm));
+      HWY_ASSERT_VEC_EQ(d, expected_add, MaskedSatAddOr(v2, mask, vi, vpm));
+      HWY_ASSERT_VEC_EQ(d, expected_add, MaskedSatAddOr(v2, mask, vpm, vpm));
+
+      Vec<D> expected_sub = IfThenElse(mask, v0, disabled_lane_val);
+      HWY_ASSERT_VEC_EQ(d, expected_sub, MaskedSatSubOr(v2, mask, v0, v0));
+      expected_sub = IfThenElse(mask, Sub(v0, vi), disabled_lane_val);
+      HWY_ASSERT_VEC_EQ(d, expected_sub, MaskedSatSubOr(v2, mask, v0, vi));
+      expected_sub = IfThenElse(mask, vn, disabled_lane_val);
+      HWY_ASSERT_VEC_EQ(d, expected_sub, MaskedSatSubOr(v2, mask, vn, v0));
+      expected_sub = IfThenElse(mask, vnm, disabled_lane_val);
+      HWY_ASSERT_VEC_EQ(d, expected_sub, MaskedSatSubOr(v2, mask, vnm, vi));
+      HWY_ASSERT_VEC_EQ(d, expected_sub, MaskedSatSubOr(v2, mask, vnm, vpm));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllSatAddSub() {
+  ForU816(ForPartialVectors<TestUnsignedSatAddSub>());
+  ForI816(ForPartialVectors<TestSignedSatAddSub>());
+}
+
+struct TestDiv {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    using VI = Vec<decltype(di)>;
+
+    // Wrap after 7 so that even float16_t can represent 1 << iota1.
+    const VI viota1 = And(Iota(di, hwy::Unpredictable1()), Set(di, 7));
+    const Vec<D> pows = ConvertTo(d, Shl(Set(di, 1), viota1));
+    const Vec<D> no = ConvertTo(d, viota1);
+
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(bool_lanes && expected);
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      ZeroBytes(expected.get(), N * sizeof(T));
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+        const size_t iota1 = (i + 1) & 7;
+        expected[i] = ConvertScalarTo<T>(iota1);
+        if (bool_lanes[i]) {
+          expected[i] =
+              ConvertScalarTo<T>(static_cast<double>(1 << iota1) / 2.0);
+        }
+      }
+
+      const VI mask_i = Load(di, bool_lanes.get());
+      const Mask<D> mask = RebindMask(d, Gt(mask_i, Zero(di)));
+
+      const Vec<D> div = Set(d, ConvertScalarTo<T>(2));
+      HWY_ASSERT_VEC_EQ(d, expected.get(), MaskedDivOr(no, mask, pows, div));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllDiv() { ForFloatTypes(ForPartialVectors<TestDiv>()); }
+
+struct TestIntegerDivMod {
+  template <class D, HWY_IF_SIGNED_D(D)>
+  static HWY_INLINE void DoSignedDivModTests(
+      D d, const TFromD<D>* HWY_RESTRICT expected_quot,
+      const TFromD<D>* HWY_RESTRICT expected_mod,
+      const TFromD<D>* HWY_RESTRICT neg_expected_quot,
+      const TFromD<D>* HWY_RESTRICT neg_expected_mod, Mask<D> mask, Vec<D> va,
+      Vec<D> vb) {
+    using T = TFromD<D>;
+
+    const auto v1 = Set(d, static_cast<T>(1));
+    const auto vneg1 = Set(d, static_cast<T>(-1));
+
+    const auto neg_a = Neg(va);
+    const auto neg_b = Neg(vb);
+
+    HWY_ASSERT_VEC_EQ(d, neg_expected_quot,
+                      MaskedDivOr(vneg1, mask, neg_a, vb));
+    HWY_ASSERT_VEC_EQ(d, neg_expected_quot,
+                      MaskedDivOr(vneg1, mask, va, neg_b));
+    HWY_ASSERT_VEC_EQ(d, expected_quot, MaskedDivOr(v1, mask, neg_a, neg_b));
+
+    HWY_ASSERT_VEC_EQ(d, neg_expected_mod, MaskedModOr(neg_b, mask, neg_a, vb));
+    HWY_ASSERT_VEC_EQ(d, expected_mod, MaskedModOr(vb, mask, va, neg_b));
+    HWY_ASSERT_VEC_EQ(d, neg_expected_mod,
+                      MaskedModOr(neg_b, mask, neg_a, neg_b));
+  }
+
+  template <class D, HWY_IF_UNSIGNED_D(D)>
+  static HWY_INLINE void DoSignedDivModTests(
+      D /*d*/, const TFromD<D>* HWY_RESTRICT /*expected_quot*/,
+      const TFromD<D>* HWY_RESTRICT /*expected_mod*/,
+      const TFromD<D>* HWY_RESTRICT /*neg_expected_quot*/,
+      const TFromD<D>* HWY_RESTRICT /*neg_expected_mod*/, Mask<D> /*mask*/,
+      Vec<D> /*va*/, Vec<D> /*vb*/) {}
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+
+    const auto v1 = Set(d, static_cast<T>(1));
+    const auto vmax = Set(d, LimitsMax<T>());
+
+    const auto vb = Max(And(Iota(d, hwy::Unpredictable1() + 1),
+                            Set(d, static_cast<T>(LimitsMax<T>() >> 1))),
+                        Set(d, static_cast<T>(2)));
+    const auto va =
+        Max(And(Sub(vmax, Iota(d, static_cast<T>(hwy::Unpredictable1() - 1))),
+                vmax),
+            Add(vb, vb));
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    using TU = MakeUnsigned<T>;
+    const Rebind<TI, D> di;
+    using VI = Vec<decltype(di)>;
+    const size_t N = Lanes(d);
+
+#if HWY_TARGET <= HWY_AVX3 && HWY_IS_MSAN
+    // Workaround for MSAN bug on AVX3
+    if (sizeof(T) <= 2 && N >= 16) {
+      return;
+    }
+#endif
+#if HWY_COMPILER_CLANG && HWY_ARCH_RISCV && HWY_TARGET == HWY_EMU128
+    // Workaround for incorrect codegen. Off by one in the lowest lane.
+    if (sizeof(T) == 4) return;
+#endif
+
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto expected_quot = AllocateAligned<T>(N);
+    auto expected_mod = AllocateAligned<T>(N);
+    auto neg_expected_quot = AllocateAligned<T>(N);
+    auto neg_expected_mod = AllocateAligned<T>(N);
+    HWY_ASSERT(bool_lanes && expected_quot && expected_mod &&
+               neg_expected_quot && neg_expected_mod);
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        const auto a0 = static_cast<T>((static_cast<TU>(LimitsMax<T>()) - i) &
+                                       LimitsMax<T>());
+        const auto b0 =
+            static_cast<T>((i + 2u) & static_cast<TU>(LimitsMax<T>() >> 1));
+
+        const auto b = static_cast<T>(HWY_MAX(b0, 2));
+        const auto a = static_cast<T>(HWY_MAX(a0, b + b));
+
+        bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+        if (bool_lanes[i]) {
+          expected_quot[i] = static_cast<T>(a / b);
+          expected_mod[i] = static_cast<T>(a % b);
+        } else {
+          expected_quot[i] = static_cast<T>(1);
+          expected_mod[i] = b;
+        }
+
+        neg_expected_quot[i] =
+            static_cast<T>(static_cast<T>(0) - expected_quot[i]);
+        neg_expected_mod[i] =
+            static_cast<T>(static_cast<T>(0) - expected_mod[i]);
+      }
+
+      const VI mask_i = Load(di, bool_lanes.get());
+      const Mask<D> mask = RebindMask(d, Gt(mask_i, Zero(di)));
+
+      HWY_ASSERT_VEC_EQ(d, expected_quot.get(), MaskedDivOr(v1, mask, va, vb));
+      HWY_ASSERT_VEC_EQ(d, expected_mod.get(), MaskedModOr(vb, mask, va, vb));
+      DoSignedDivModTests(d, expected_quot.get(), expected_mod.get(),
+                          neg_expected_quot.get(), neg_expected_mod.get(), mask,
+                          va, vb);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllIntegerDivMod() {
+  ForIntegerTypes(ForPartialVectors<TestIntegerDivMod>());
+}
+
+struct TestFloatExceptions {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> v4 = Iota(d, hwy::Unpredictable1() + 3);
+    const Mask<D> m0 = MaskFalse(d);
+
+    // No overflow
+    const Vec<D> inf = Inf(d);
+    HWY_ASSERT_VEC_EQ(d, v4, MaskedAddOr(v4, m0, inf, inf));
+    HWY_ASSERT_VEC_EQ(d, v4, MaskedSubOr(v4, m0, Neg(inf), Neg(inf)));
+
+    // No underflow
+    const Vec<D> eps = Set(d, Epsilon<T>());
+    const Vec<D> half = Set(d, static_cast<T>(0.5f));
+    HWY_ASSERT_VEC_EQ(d, v4, MaskedMulOr(v4, m0, eps, half));
+
+    // Division by zero
+    const Vec<D> v0 = Set(d, ConvertScalarTo<T>(0));
+    HWY_ASSERT_VEC_EQ(d, v4, MaskedDivOr(v4, m0, v4, v0));
+  }
+};
+
+HWY_NOINLINE void TestAllFloatExceptions() {
+  ForFloatTypes(ForPartialVectors<TestFloatExceptions>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyMaskedArithmeticTest);
+HWY_EXPORT_AND_TEST_P(HwyMaskedArithmeticTest, TestAllAddSubMul);
+HWY_EXPORT_AND_TEST_P(HwyMaskedArithmeticTest, TestAllSatAddSub);
+HWY_EXPORT_AND_TEST_P(HwyMaskedArithmeticTest, TestAllDiv);
+HWY_EXPORT_AND_TEST_P(HwyMaskedArithmeticTest, TestAllIntegerDivMod);
+HWY_EXPORT_AND_TEST_P(HwyMaskedArithmeticTest, TestAllFloatExceptions);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/masked_minmax_test.cc

@@ -0,0 +1,156 @@
+// Copyright 2023 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "hwy/base.h"
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/masked_minmax_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/nanobenchmark.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestUnsignedMinMax {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+    const Vec<D> v2 = Iota(d, hwy::Unpredictable1() + 1);
+    const Vec<D> v3 = Iota(d, hwy::Unpredictable1() + 2);
+    const Vec<D> v4 = Iota(d, hwy::Unpredictable1() + 3);
+    const Vec<D> k0 = Zero(d);
+    const Vec<D> vm = Set(d, LimitsMax<T>());
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    using VI = Vec<decltype(di)>;
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto expected_min = AllocateAligned<T>(N);
+    auto expected_max = AllocateAligned<T>(N);
+    HWY_ASSERT(bool_lanes && expected_min && expected_max);
+
+    // Ensure unsigned 0 < max.
+    HWY_ASSERT_VEC_EQ(d, k0, MaskedMinOr(v2, MaskTrue(d), k0, vm));
+    HWY_ASSERT_VEC_EQ(d, k0, MaskedMinOr(v2, MaskTrue(d), vm, k0));
+    HWY_ASSERT_VEC_EQ(d, vm, MaskedMaxOr(v2, MaskTrue(d), k0, vm));
+    HWY_ASSERT_VEC_EQ(d, vm, MaskedMaxOr(v2, MaskTrue(d), vm, k0));
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+        const T t2 = static_cast<T>(AddWithWraparound(static_cast<T>(i), 2));
+        const T t3 = static_cast<T>(AddWithWraparound(static_cast<T>(i), 3));
+        const T t4 = static_cast<T>(AddWithWraparound(static_cast<T>(i), 4));
+        if (bool_lanes[i]) {
+          expected_min[i] = HWY_MIN(t3, t4);
+          expected_max[i] = HWY_MAX(t3, t4);
+        } else {
+          expected_min[i] = expected_max[i] = t2;
+        }
+      }
+
+      const VI mask_i = Load(di, bool_lanes.get());
+      const Mask<D> mask = RebindMask(d, Gt(mask_i, Zero(di)));
+
+      HWY_ASSERT_VEC_EQ(d, expected_min.get(), MaskedMinOr(v2, mask, v3, v4));
+      HWY_ASSERT_VEC_EQ(d, expected_min.get(), MaskedMinOr(v2, mask, v4, v3));
+      HWY_ASSERT_VEC_EQ(d, expected_max.get(), MaskedMaxOr(v2, mask, v3, v4));
+      HWY_ASSERT_VEC_EQ(d, expected_max.get(), MaskedMaxOr(v2, mask, v4, v3));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllUnsignedMinMax() {
+  ForUnsignedTypes(ForPartialVectors<TestUnsignedMinMax>());
+}
+
+struct TestSignedMinMax {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+    const Vec<D> v2 = Iota(d, hwy::Unpredictable1() + 1);
+    const Vec<D> v3 = Iota(d, hwy::Unpredictable1() + 2);
+    const Vec<D> v4 = Iota(d, hwy::Unpredictable1() + 3);
+    const Vec<D> k0 = Zero(d);
+    const Vec<D> vm = Set(d, LowestValue<T>());
+
+    using TI = MakeSigned<T>;  // For mask > 0 comparison
+    const Rebind<TI, D> di;
+    using VI = Vec<decltype(di)>;
+    const size_t N = Lanes(d);
+    auto bool_lanes = AllocateAligned<TI>(N);
+    auto expected_min = AllocateAligned<T>(N);
+    auto expected_max = AllocateAligned<T>(N);
+    HWY_ASSERT(bool_lanes && expected_min && expected_max);
+
+    // Ensure signed min < 0.
+    HWY_ASSERT_VEC_EQ(d, vm, MaskedMinOr(v2, MaskTrue(d), k0, vm));
+    HWY_ASSERT_VEC_EQ(d, vm, MaskedMinOr(v2, MaskTrue(d), vm, k0));
+    HWY_ASSERT_VEC_EQ(d, k0, MaskedMaxOr(v2, MaskTrue(d), k0, vm));
+    HWY_ASSERT_VEC_EQ(d, k0, MaskedMaxOr(v2, MaskTrue(d), vm, k0));
+
+    // Each lane should have a chance of having mask=true.
+    for (size_t rep = 0; rep < AdjustedReps(200); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0);
+        const T t2 = AddWithWraparound(ConvertScalarTo<T>(i), 2);
+        const T t3 = AddWithWraparound(ConvertScalarTo<T>(i), 3);
+        const T t4 = AddWithWraparound(ConvertScalarTo<T>(i), 4);
+        if (bool_lanes[i]) {
+          expected_min[i] = HWY_MIN(t3, t4);
+          expected_max[i] = HWY_MAX(t3, t4);
+        } else {
+          expected_min[i] = expected_max[i] = t2;
+        }
+      }
+
+      const VI mask_i = Load(di, bool_lanes.get());
+      const Mask<D> mask = RebindMask(d, Gt(mask_i, Zero(di)));
+
+      HWY_ASSERT_VEC_EQ(d, expected_min.get(), MaskedMinOr(v2, mask, v3, v4));
+      HWY_ASSERT_VEC_EQ(d, expected_min.get(), MaskedMinOr(v2, mask, v4, v3));
+      HWY_ASSERT_VEC_EQ(d, expected_max.get(), MaskedMaxOr(v2, mask, v3, v4));
+      HWY_ASSERT_VEC_EQ(d, expected_max.get(), MaskedMaxOr(v2, mask, v4, v3));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllSignedMinMax() {
+  ForSignedTypes(ForPartialVectors<TestSignedMinMax>());
+  ForFloatTypes(ForPartialVectors<TestSignedMinMax>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyMaskedMinMaxTest);
+HWY_EXPORT_AND_TEST_P(HwyMaskedMinMaxTest, TestAllUnsignedMinMax);
+HWY_EXPORT_AND_TEST_P(HwyMaskedMinMaxTest, TestAllSignedMinMax);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/memory_test.cc

@@ -0,0 +1,586 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+// Ensure incompatibilities with Windows macros (e.g. #define StoreFence) are
+// detected. Must come before Highway headers.
+#include "hwy/base.h"
+#include "hwy/tests/test_util.h"
+#if defined(_WIN32) || defined(_WIN64)
+#include <windows.h>
+#endif
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/memory_test.cc"
+#include "hwy/cache_control.h"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestLoadStore {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    const VFromD<D> hi = IotaForSpecial(d, 1 + N);
+    const VFromD<D> lo = IotaForSpecial(d, 1);
+    auto lanes = AllocateAligned<T>(2 * N);
+    auto lanes2 = AllocateAligned<T>(2 * N);
+    auto lanes3 = AllocateAligned<T>(N);
+    HWY_ASSERT(lanes && lanes2 && lanes3);
+
+    Store(hi, d, &lanes[N]);
+    Store(lo, d, &lanes[0]);
+
+    // Aligned load
+    const VFromD<D> lo2 = Load(d, &lanes[0]);
+    HWY_ASSERT_VEC_EQ(d, lo2, lo);
+
+    // Aligned store
+    Store(lo2, d, &lanes2[0]);
+    Store(hi, d, &lanes2[N]);
+    for (size_t i = 0; i < 2 * N; ++i) {
+      HWY_ASSERT_EQ(lanes[i], lanes2[i]);
+    }
+
+    // Unaligned load
+    const VFromD<D> vu = LoadU(d, &lanes[1]);
+    Store(vu, d, lanes3.get());
+    for (size_t i = 0; i < N; ++i) {
+      HWY_ASSERT_EQ(i + 2, lanes3[i]);
+    }
+
+    // Unaligned store
+    StoreU(lo2, d, &lanes2[N / 2]);
+    size_t i = 0;
+    for (; i < N / 2; ++i) {
+      HWY_ASSERT_EQ(lanes[i], lanes2[i]);
+    }
+    for (; i < 3 * N / 2; ++i) {
+      HWY_ASSERT_EQ(i - N / 2 + 1, lanes2[i]);
+    }
+    // Subsequent values remain unchanged.
+    for (; i < 2 * N; ++i) {
+      HWY_ASSERT_EQ(i + 1, lanes2[i]);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllLoadStore() {
+  ForAllTypesAndSpecial(ForPartialVectors<TestLoadStore>());
+}
+
+struct TestSafeCopyN {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    const auto v = Iota(d, 1);
+    auto from = AllocateAligned<T>(N + 2);
+    auto to = AllocateAligned<T>(N + 2);
+    HWY_ASSERT(from && to);
+    Store(v, d, from.get());
+
+    // 0: nothing changes
+    to[0] = ConvertScalarTo<T>(0);
+    SafeCopyN(0, d, from.get(), to.get());
+    HWY_ASSERT_EQ(T(), to[0]);
+
+    // 1: only first changes
+    to[1] = ConvertScalarTo<T>(0);
+    SafeCopyN(1, d, from.get(), to.get());
+    HWY_ASSERT_EQ(ConvertScalarTo<T>(1), to[0]);
+    HWY_ASSERT_EQ(T(), to[1]);
+
+    // N-1: last does not change
+    to[N - 1] = ConvertScalarTo<T>(0);
+    SafeCopyN(N - 1, d, from.get(), to.get());
+    HWY_ASSERT_EQ(T(), to[N - 1]);
+    // Also check preceding lanes
+    to[N - 1] = ConvertScalarTo<T>(N);
+    HWY_ASSERT_VEC_EQ(d, to.get(), v);
+
+    // N: all change
+    to[N] = ConvertScalarTo<T>(0);
+    SafeCopyN(N, d, from.get(), to.get());
+    HWY_ASSERT_VEC_EQ(d, to.get(), v);
+    HWY_ASSERT_EQ(T(), to[N]);
+
+    // N+1: subsequent lane does not change if using masked store
+    to[N + 1] = ConvertScalarTo<T>(0);
+    SafeCopyN(N + 1, d, from.get(), to.get());
+    HWY_ASSERT_VEC_EQ(d, to.get(), v);
+#if !HWY_MEM_OPS_MIGHT_FAULT
+    HWY_ASSERT_EQ(T(), to[N + 1]);
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllSafeCopyN() {
+  ForAllTypes(ForPartialVectors<TestSafeCopyN>());
+}
+
+struct TestLoadDup128 {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    // Scalar does not define LoadDup128.
+#if HWY_TARGET != HWY_SCALAR || HWY_IDE
+    constexpr size_t N128 = 16 / sizeof(T);
+    alignas(16) T lanes[N128];
+    for (size_t i = 0; i < N128; ++i) {
+      lanes[i] = ConvertScalarTo<T>(1 + i);
+    }
+
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>(i % N128 + 1);
+    }
+
+    HWY_ASSERT_VEC_EQ(d, expected.get(), LoadDup128(d, lanes));
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllLoadDup128() {
+  ForAllTypes(ForGEVectors<128, TestLoadDup128>());
+}
+
+struct TestStream {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> v = Iota(d, 1);
+    const size_t affected_bytes =
+        (Lanes(d) * sizeof(T) + HWY_STREAM_MULTIPLE - 1) &
+        ~size_t(HWY_STREAM_MULTIPLE - 1);
+    const size_t affected_lanes = affected_bytes / sizeof(T);
+    auto out = AllocateAligned<T>(2 * affected_lanes);
+    HWY_ASSERT(out);
+    ZeroBytes(out.get(), 2 * affected_lanes * sizeof(T));
+
+    Stream(v, d, out.get());
+    FlushStream();
+    const Vec<D> actual = Load(d, out.get());
+    HWY_ASSERT_VEC_EQ(d, v, actual);
+    // Ensure Stream didn't modify more memory than expected
+    for (size_t i = affected_lanes; i < 2 * affected_lanes; ++i) {
+      HWY_ASSERT_EQ(ConvertScalarTo<T>(0), out[i]);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllStream() {
+  const ForPartialVectors<TestStream> test;
+  // No u8,u16.
+  test(uint32_t());
+  test(uint64_t());
+  // No i8,i16.
+  test(int32_t());
+  test(int64_t());
+  ForFloatTypes(test);
+}
+
+// Assumes little-endian byte order!
+struct TestScatter {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using Offset = MakeSigned<T>;
+    const Rebind<Offset, D> d_offsets;
+
+    const size_t N = Lanes(d);
+    const size_t range = 4 * N;                  // number of items to scatter
+    const size_t max_bytes = range * sizeof(T);  // upper bound on offset
+
+    RandomState rng;
+
+    auto values = AllocateAligned<T>(range);
+    auto offsets = AllocateAligned<Offset>(N);  // or indices
+    // Scatter into these regions, ensure vector results match scalar
+    auto expected = AllocateAligned<T>(range);
+    auto actual = AllocateAligned<T>(range);
+    HWY_ASSERT(values && offsets && expected && actual);
+
+    // Data to be scattered
+    uint8_t* bytes = reinterpret_cast<uint8_t*>(values.get());
+    for (size_t i = 0; i < max_bytes; ++i) {
+      bytes[i] = static_cast<uint8_t>(Random32(&rng) & 0xFF);
+    }
+    const Vec<D> data = Load(d, values.get());
+
+    for (size_t rep = 0; rep < 100; ++rep) {
+      // Byte offsets
+      ZeroBytes(expected.get(), range * sizeof(T));
+      ZeroBytes(actual.get(), range * sizeof(T));
+      for (size_t i = 0; i < N; ++i) {
+        // Must be aligned
+        offsets[i] = static_cast<Offset>((Random32(&rng) % range) * sizeof(T));
+        CopyBytes<sizeof(T)>(
+            values.get() + i,
+            reinterpret_cast<uint8_t*>(expected.get()) + offsets[i]);
+      }
+      const auto voffsets = Load(d_offsets, offsets.get());
+      ScatterOffset(data, d, actual.get(), voffsets);
+      if (!BytesEqual(expected.get(), actual.get(), max_bytes)) {
+        Print(d, "Data", data);
+        Print(d_offsets, "Offsets", voffsets);
+        HWY_ASSERT(false);
+      }
+
+      // Indices
+      ZeroBytes(expected.get(), range * sizeof(T));
+      ZeroBytes(actual.get(), range * sizeof(T));
+      for (size_t i = 0; i < N; ++i) {
+        offsets[i] = static_cast<Offset>(Random32(&rng) % range);
+        CopyBytes<sizeof(T)>(values.get() + i, &expected[size_t(offsets[i])]);
+      }
+      const auto vindices = Load(d_offsets, offsets.get());
+      ScatterIndex(data, d, actual.get(), vindices);
+      if (!BytesEqual(expected.get(), actual.get(), max_bytes)) {
+        Print(d, "Data", data);
+        Print(d_offsets, "Indices", vindices);
+        HWY_ASSERT(false);
+      }
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllScatter() {
+  ForUIF3264(ForPartialVectors<TestScatter>());
+}
+
+struct TestGather {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using Offset = MakeSigned<T>;
+
+    const size_t N = Lanes(d);
+    const size_t range = 4 * N;                  // number of items to gather
+    const size_t max_bytes = range * sizeof(T);  // upper bound on offset
+
+    RandomState rng;
+    auto values = AllocateAligned<T>(range);
+    auto expected = AllocateAligned<T>(N);
+    auto offsets = AllocateAligned<Offset>(N);
+    auto indices = AllocateAligned<Offset>(N);
+    HWY_ASSERT(values && expected && offsets && indices);
+
+    // Data to be gathered from
+    uint8_t* bytes = reinterpret_cast<uint8_t*>(values.get());
+    for (size_t i = 0; i < max_bytes; ++i) {
+      bytes[i] = static_cast<uint8_t>(Random32(&rng) & 0xFF);
+    }
+
+    for (size_t rep = 0; rep < 100; ++rep) {
+      // Offsets
+      for (size_t i = 0; i < N; ++i) {
+        // Must be aligned
+        offsets[i] = static_cast<Offset>((Random32(&rng) % range) * sizeof(T));
+        CopyBytes<sizeof(T)>(bytes + offsets[i], &expected[i]);
+      }
+
+      const Rebind<Offset, D> d_offset;
+      const T* base = values.get();
+      auto actual = GatherOffset(d, base, Load(d_offset, offsets.get()));
+      HWY_ASSERT_VEC_EQ(d, expected.get(), actual);
+
+      // Indices
+      for (size_t i = 0; i < N; ++i) {
+        indices[i] =
+            static_cast<Offset>(Random32(&rng) % (max_bytes / sizeof(T)));
+        CopyBytes<sizeof(T)>(base + indices[i], &expected[i]);
+      }
+      actual = GatherIndex(d, base, Load(d_offset, indices.get()));
+      HWY_ASSERT_VEC_EQ(d, expected.get(), actual);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllGather() {
+  ForUIF3264(ForPartialVectors<TestGather>());
+}
+
+HWY_NOINLINE void TestAllCache() {
+  LoadFence();
+  FlushStream();
+  int test = 0;
+  Prefetch(&test);
+  FlushCacheline(&test);
+  Pause();
+}
+
+template <int kNo, class T, HWY_IF_NOT_FLOAT_NOR_SPECIAL(T)>
+HWY_INLINE T GenerateOtherValue(size_t val) {
+  const T conv_val = static_cast<T>(val);
+  return (conv_val == static_cast<T>(kNo)) ? static_cast<T>(-17) : conv_val;
+}
+template <int kNo, class T, HWY_IF_FLOAT3264(T)>
+HWY_INLINE T GenerateOtherValue(size_t val) {
+  const T flt_val = static_cast<T>(val);
+  return (flt_val == static_cast<T>(kNo) ? static_cast<T>(0.5426808228865735)
+                                         : flt_val);
+}
+template <int kNo, class T, HWY_IF_BF16(T)>
+HWY_INLINE T GenerateOtherValue(size_t val) {
+  return BF16FromF32(GenerateOtherValue<kNo, float>(val));
+}
+template <int kNo, class T, HWY_IF_F16(T)>
+HWY_INLINE T GenerateOtherValue(size_t val) {
+  return F16FromF32(GenerateOtherValue<kNo, float>(val));
+}
+
+struct TestLoadN {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    constexpr size_t kMaxLanesPerBlock = 16 / sizeof(T);
+    const size_t lpb = HWY_MIN(N, kMaxLanesPerBlock);
+    HWY_ASSERT(lpb >= 1);
+    HWY_ASSERT(N <= (static_cast<size_t>(~size_t(0)) / 4));
+
+    const size_t load_buf_len = (3 * N) + 4;
+
+    auto load_buf = AllocateAligned<T>(load_buf_len);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(load_buf && expected);
+
+    for (size_t i = 0; i < load_buf_len; i++) {
+      load_buf[i] = GenerateOtherValue<0, T>(i + 1);
+    }
+
+    ZeroBytes(expected.get(), N * sizeof(T));
+    // Without Load(), the vector type for special floats might not match.
+    HWY_ASSERT_VEC_EQ(d, Load(d, expected.get()), LoadN(d, load_buf.get(), 0));
+
+    for (size_t i = 0; i <= lpb; i++) {
+      CopyBytes(load_buf.get(), expected.get(), i * sizeof(T));
+      const VFromD<D> actual_1 = LoadN(d, load_buf.get(), i);
+      HWY_ASSERT_VEC_EQ(d, Load(d, expected.get()), actual_1);
+
+      CopyBytes(load_buf.get() + 3, expected.get(), i * sizeof(T));
+      const VFromD<D> actual_2 = LoadN(d, load_buf.get() + 3, i);
+      HWY_ASSERT_VEC_EQ(d, Load(d, expected.get()), actual_2);
+    }
+
+    const size_t lplb = HWY_MAX(N / 4, lpb);
+    for (size_t i = HWY_MAX(lpb * 2, lplb); i <= N * 2; i += lplb) {
+      const size_t max_num_of_lanes_to_load = i + (11 & (lpb - 1));
+      const size_t expected_num_of_lanes_loaded =
+          HWY_MIN(max_num_of_lanes_to_load, N);
+
+      CopyBytes(load_buf.get(), expected.get(),
+                expected_num_of_lanes_loaded * sizeof(T));
+      const VFromD<D> actual_1 =
+          LoadN(d, load_buf.get(), max_num_of_lanes_to_load);
+      HWY_ASSERT_VEC_EQ(d, Load(d, expected.get()), actual_1);
+
+      CopyBytes(load_buf.get() + 3, expected.get(),
+                expected_num_of_lanes_loaded * sizeof(T));
+      const VFromD<D> actual_2 =
+          LoadN(d, load_buf.get() + 3, max_num_of_lanes_to_load);
+      HWY_ASSERT_VEC_EQ(d, Load(d, expected.get()), actual_2);
+    }
+
+    load_buf[0] = GenerateOtherValue<0, T>(0);
+    CopyBytes(load_buf.get(), expected.get(), N * sizeof(T));
+    HWY_ASSERT_VEC_EQ(d, Load(d, expected.get()), LoadN(d, load_buf.get(), N));
+  }
+};
+
+HWY_NOINLINE void TestAllLoadN() {
+  ForAllTypesAndSpecial(ForPartialVectors<TestLoadN>());
+}
+
+struct TestLoadNOr {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    constexpr int kNo = 2;
+    const size_t N = Lanes(d);
+    constexpr size_t kMaxLanesPerBlock = 16 / sizeof(T);
+    const size_t lpb = HWY_MIN(N, kMaxLanesPerBlock);
+    HWY_ASSERT(lpb >= 1);
+    HWY_ASSERT(N <= (static_cast<size_t>(~size_t(0)) / 4));
+
+    const size_t load_buf_len = (3 * N) + 4;
+
+    auto load_buf = AllocateAligned<T>(load_buf_len);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(load_buf && expected);
+
+    for (size_t i = 0; i < load_buf_len; i++) {
+      load_buf[i] = GenerateOtherValue<kNo, T>(i + 1);
+    }
+    const Vec<D> no = Set(d, ConvertScalarTo<T>(kNo));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>(kNo);
+    }
+    // Without Load(), the vector type for special floats might not match.
+    HWY_ASSERT_VEC_EQ(d, Load(d, expected.get()),
+                      LoadNOr(no, d, load_buf.get(), 0));
+
+    for (size_t i = 0; i <= lpb; i++) {
+      CopyBytes(load_buf.get(), expected.get(), i * sizeof(T));
+      const VFromD<D> actual_1 = LoadNOr(no, d, load_buf.get(), i);
+      HWY_ASSERT_VEC_EQ(d, Load(d, expected.get()), actual_1);
+
+      CopyBytes(load_buf.get() + 3, expected.get(), i * sizeof(T));
+      const VFromD<D> actual_2 = LoadNOr(no, d, load_buf.get() + 3, i);
+      HWY_ASSERT_VEC_EQ(d, Load(d, expected.get()), actual_2);
+    }
+
+    const size_t lplb = HWY_MAX(N / 4, lpb);
+    for (size_t i = HWY_MAX(lpb * 2, lplb); i <= N * 2; i += lplb) {
+      const size_t max_num_of_lanes_to_load = i + (11 & (lpb - 1));
+      const size_t expected_num_of_lanes_loaded =
+          HWY_MIN(max_num_of_lanes_to_load, N);
+
+      CopyBytes(load_buf.get(), expected.get(),
+                expected_num_of_lanes_loaded * sizeof(T));
+      const VFromD<D> actual_1 =
+          LoadNOr(no, d, load_buf.get(), max_num_of_lanes_to_load);
+      HWY_ASSERT_VEC_EQ(d, Load(d, expected.get()), actual_1);
+
+      CopyBytes(load_buf.get() + 3, expected.get(),
+                expected_num_of_lanes_loaded * sizeof(T));
+      const VFromD<D> actual_2 =
+          LoadNOr(no, d, load_buf.get() + 3, max_num_of_lanes_to_load);
+      HWY_ASSERT_VEC_EQ(d, Load(d, expected.get()), actual_2);
+    }
+
+    load_buf[0] = GenerateOtherValue<kNo, T>(kNo);
+    CopyBytes(load_buf.get(), expected.get(), N * sizeof(T));
+    HWY_ASSERT_VEC_EQ(d, Load(d, expected.get()),
+                      LoadNOr(no, d, load_buf.get(), N));
+  }
+};
+
+HWY_NOINLINE void TestAllLoadNOr() {
+  ForAllTypesAndSpecial(ForPartialVectors<TestLoadNOr>());
+}
+
+class TestStoreN {
+ private:
+  template <class T, HWY_IF_FLOAT_OR_SPECIAL(T)>
+  static HWY_INLINE T NegativeFillValue() {
+    return LowestValue<T>();
+  }
+
+  template <class T, HWY_IF_NOT_FLOAT_NOR_SPECIAL(T)>
+  static HWY_INLINE T NegativeFillValue() {
+    return static_cast<T>(-1);
+  }
+
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    constexpr size_t kMaxLanesPerBlock = 16 / sizeof(T);
+    const size_t lpb = HWY_MIN(N, kMaxLanesPerBlock);
+    HWY_ASSERT(lpb >= 1);
+
+    const size_t full_dvec_N = Lanes(DFromV<Vec<D>>());
+    HWY_ASSERT(N <= full_dvec_N);
+    HWY_ASSERT(full_dvec_N <= (static_cast<size_t>(~size_t(0)) / 8));
+
+    const size_t buf_offset = HWY_MAX(kMaxLanesPerBlock, full_dvec_N);
+    const size_t buf_size = buf_offset + 3 * full_dvec_N + 4;
+    auto expected = AllocateAligned<T>(buf_size);
+    auto actual = AllocateAligned<T>(buf_size);
+    HWY_ASSERT(expected && actual);
+
+    const T neg_fill_val = NegativeFillValue<T>();
+    for (size_t i = 0; i < buf_size; i++) {
+      expected[i] = neg_fill_val;
+      actual[i] = neg_fill_val;
+    }
+
+    const Vec<D> v_neg_fill_val = Set(d, neg_fill_val);
+
+    for (size_t i = 0; i <= lpb; i++) {
+      const Vec<D> v = IotaForSpecial(d, i + 1);
+      const Vec<D> v_expected = IfThenElse(FirstN(d, i), v, v_neg_fill_val);
+
+      Store(v_expected, d, expected.get() + buf_offset);
+      Store(v_neg_fill_val, d, actual.get() + buf_offset);
+      StoreN(v, d, actual.get() + buf_offset, i);
+
+      HWY_ASSERT_ARRAY_EQ(expected.get(), actual.get(), buf_size);
+
+      StoreU(v_expected, d, expected.get() + buf_offset + 3);
+      StoreU(v_neg_fill_val, d, actual.get() + buf_offset + 3);
+      StoreN(v, d, actual.get() + buf_offset + 3, i);
+      HWY_ASSERT_ARRAY_EQ(expected.get(), actual.get(), buf_size);
+    }
+
+    const size_t lplb = HWY_MAX(N / 4, lpb);
+    for (size_t i = HWY_MAX(lpb * 2, lplb); i <= N * 2; i += lplb) {
+      const size_t max_num_of_lanes_to_store = i + (11 & (lpb - 1));
+      const size_t expected_num_of_lanes_written =
+          HWY_MIN(max_num_of_lanes_to_store, N);
+
+      const Vec<D> v = IotaForSpecial(d, max_num_of_lanes_to_store + 1);
+      const Vec<D> v_expected = IfThenElse(
+          FirstN(d, expected_num_of_lanes_written), v, v_neg_fill_val);
+
+      Store(v_expected, d, expected.get() + buf_offset);
+      Store(v_neg_fill_val, d, actual.get() + buf_offset);
+      StoreN(v, d, actual.get() + buf_offset, max_num_of_lanes_to_store);
+
+      HWY_ASSERT_ARRAY_EQ(expected.get(), actual.get(), buf_size);
+
+      StoreU(v_expected, d, expected.get() + buf_offset + 3);
+      StoreU(v_neg_fill_val, d, actual.get() + buf_offset + 3);
+      StoreN(v, d, actual.get() + buf_offset + 3, max_num_of_lanes_to_store);
+      HWY_ASSERT_ARRAY_EQ(expected.get(), actual.get(), buf_size);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllStoreN() {
+  ForAllTypesAndSpecial(ForPartialVectors<TestStoreN>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyMemoryTest);
+HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllLoadStore);
+HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllSafeCopyN);
+HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllLoadDup128);
+HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllStream);
+HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllScatter);
+HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllGather);
+HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllCache);
+HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllLoadN);
+HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllLoadNOr);
+HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllStoreN);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/minmax_test.cc

@@ -0,0 +1,281 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/minmax_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestUnsignedMinMax {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v0 = Zero(d);
+    // Leave headroom such that v1 < v2 even after wraparound.
+    const auto mod = And(Iota(d, 0), Set(d, LimitsMax<T>() >> 1));
+    const auto v1 = Add(mod, Set(d, static_cast<T>(1)));
+    const auto v2 = Add(mod, Set(d, static_cast<T>(2)));
+    HWY_ASSERT_VEC_EQ(d, v1, Min(v1, v2));
+    HWY_ASSERT_VEC_EQ(d, v2, Max(v1, v2));
+    HWY_ASSERT_VEC_EQ(d, v0, Min(v1, v0));
+    HWY_ASSERT_VEC_EQ(d, v1, Max(v1, v0));
+
+    const auto vmin = Set(d, LimitsMin<T>());
+    const auto vmax = Set(d, LimitsMax<T>());
+
+    HWY_ASSERT_VEC_EQ(d, vmin, Min(vmin, vmax));
+    HWY_ASSERT_VEC_EQ(d, vmin, Min(vmax, vmin));
+
+    HWY_ASSERT_VEC_EQ(d, vmax, Max(vmin, vmax));
+    HWY_ASSERT_VEC_EQ(d, vmax, Max(vmax, vmin));
+  }
+};
+
+struct TestSignedMinMax {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    // Leave headroom such that v1 < v2 even after wraparound.
+    const auto mod =
+        And(Iota(d, 0), Set(d, ConvertScalarTo<T>(LimitsMax<T>() >> 1)));
+    const auto v1 = Add(mod, Set(d, ConvertScalarTo<T>(1)));
+    const auto v2 = Add(mod, Set(d, ConvertScalarTo<T>(2)));
+    const auto v_neg = Sub(Zero(d), v1);
+    HWY_ASSERT_VEC_EQ(d, v1, Min(v1, v2));
+    HWY_ASSERT_VEC_EQ(d, v2, Max(v1, v2));
+    HWY_ASSERT_VEC_EQ(d, v_neg, Min(v1, v_neg));
+    HWY_ASSERT_VEC_EQ(d, v1, Max(v1, v_neg));
+
+    const auto v0 = Zero(d);
+    const auto vmin = Set(d, LimitsMin<T>());
+    const auto vmax = Set(d, LimitsMax<T>());
+    HWY_ASSERT_VEC_EQ(d, vmin, Min(v0, vmin));
+    HWY_ASSERT_VEC_EQ(d, vmin, Min(vmin, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, Max(v0, vmin));
+    HWY_ASSERT_VEC_EQ(d, v0, Max(vmin, v0));
+
+    HWY_ASSERT_VEC_EQ(d, vmin, Min(vmin, vmax));
+    HWY_ASSERT_VEC_EQ(d, vmin, Min(vmax, vmin));
+
+    HWY_ASSERT_VEC_EQ(d, vmax, Max(vmin, vmax));
+    HWY_ASSERT_VEC_EQ(d, vmax, Max(vmax, vmin));
+  }
+};
+
+struct TestFloatMinMax {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v1 = Iota(d, 1);
+    const auto v2 = Iota(d, 2);
+    const auto v_neg = Iota(d, -ConvertScalarTo<T>(Lanes(d)));
+    HWY_ASSERT_VEC_EQ(d, v1, Min(v1, v2));
+    HWY_ASSERT_VEC_EQ(d, v2, Max(v1, v2));
+    HWY_ASSERT_VEC_EQ(d, v_neg, Min(v1, v_neg));
+    HWY_ASSERT_VEC_EQ(d, v1, Max(v1, v_neg));
+
+    const auto v0 = Zero(d);
+    const auto vmin = Set(d, ConvertScalarTo<T>(-1E30));
+    const auto vmax = Set(d, ConvertScalarTo<T>(1E30));
+    HWY_ASSERT_VEC_EQ(d, vmin, Min(v0, vmin));
+    HWY_ASSERT_VEC_EQ(d, vmin, Min(vmin, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, Max(v0, vmin));
+    HWY_ASSERT_VEC_EQ(d, v0, Max(vmin, v0));
+
+    HWY_ASSERT_VEC_EQ(d, vmin, Min(vmin, vmax));
+    HWY_ASSERT_VEC_EQ(d, vmin, Min(vmax, vmin));
+
+    HWY_ASSERT_VEC_EQ(d, vmax, Max(vmin, vmax));
+    HWY_ASSERT_VEC_EQ(d, vmax, Max(vmax, vmin));
+  }
+};
+
+HWY_NOINLINE void TestAllMinMax() {
+  ForUnsignedTypes(ForPartialVectors<TestUnsignedMinMax>());
+  ForSignedTypes(ForPartialVectors<TestSignedMinMax>());
+  ForFloatTypes(ForPartialVectors<TestFloatMinMax>());
+}
+
+template <class D>
+static HWY_NOINLINE Vec<D> Make128(D d, uint64_t hi, uint64_t lo) {
+  alignas(16) uint64_t in[2];
+  in[0] = lo;
+  in[1] = hi;
+  return LoadDup128(d, in);
+}
+
+struct TestMinMax128 {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using V = Vec<D>;
+    const size_t N = Lanes(d);
+    auto a_lanes = AllocateAligned<T>(N);
+    auto b_lanes = AllocateAligned<T>(N);
+    auto min_lanes = AllocateAligned<T>(N);
+    auto max_lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(a_lanes && b_lanes && min_lanes && max_lanes);
+    RandomState rng;
+
+    const V v00 = Zero(d);
+    const V v01 = Make128(d, 0, 1);
+    const V v10 = Make128(d, 1, 0);
+    const V v11 = Add(v01, v10);
+
+    // Same arg
+    HWY_ASSERT_VEC_EQ(d, v00, Min128(d, v00, v00));
+    HWY_ASSERT_VEC_EQ(d, v01, Min128(d, v01, v01));
+    HWY_ASSERT_VEC_EQ(d, v10, Min128(d, v10, v10));
+    HWY_ASSERT_VEC_EQ(d, v11, Min128(d, v11, v11));
+    HWY_ASSERT_VEC_EQ(d, v00, Max128(d, v00, v00));
+    HWY_ASSERT_VEC_EQ(d, v01, Max128(d, v01, v01));
+    HWY_ASSERT_VEC_EQ(d, v10, Max128(d, v10, v10));
+    HWY_ASSERT_VEC_EQ(d, v11, Max128(d, v11, v11));
+
+    // First arg less
+    HWY_ASSERT_VEC_EQ(d, v00, Min128(d, v00, v01));
+    HWY_ASSERT_VEC_EQ(d, v01, Min128(d, v01, v10));
+    HWY_ASSERT_VEC_EQ(d, v10, Min128(d, v10, v11));
+    HWY_ASSERT_VEC_EQ(d, v01, Max128(d, v00, v01));
+    HWY_ASSERT_VEC_EQ(d, v10, Max128(d, v01, v10));
+    HWY_ASSERT_VEC_EQ(d, v11, Max128(d, v10, v11));
+
+    // Second arg less
+    HWY_ASSERT_VEC_EQ(d, v00, Min128(d, v01, v00));
+    HWY_ASSERT_VEC_EQ(d, v01, Min128(d, v10, v01));
+    HWY_ASSERT_VEC_EQ(d, v10, Min128(d, v11, v10));
+    HWY_ASSERT_VEC_EQ(d, v01, Max128(d, v01, v00));
+    HWY_ASSERT_VEC_EQ(d, v10, Max128(d, v10, v01));
+    HWY_ASSERT_VEC_EQ(d, v11, Max128(d, v11, v10));
+
+    // Also check 128-bit blocks are independent
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        a_lanes[i] = Random64(&rng);
+        b_lanes[i] = Random64(&rng);
+      }
+      const V a = Load(d, a_lanes.get());
+      const V b = Load(d, b_lanes.get());
+      for (size_t i = 0; i < N; i += 2) {
+        const bool lt = a_lanes[i + 1] == b_lanes[i + 1]
+                            ? (a_lanes[i] < b_lanes[i])
+                            : (a_lanes[i + 1] < b_lanes[i + 1]);
+        min_lanes[i + 0] = lt ? a_lanes[i + 0] : b_lanes[i + 0];
+        min_lanes[i + 1] = lt ? a_lanes[i + 1] : b_lanes[i + 1];
+        max_lanes[i + 0] = lt ? b_lanes[i + 0] : a_lanes[i + 0];
+        max_lanes[i + 1] = lt ? b_lanes[i + 1] : a_lanes[i + 1];
+      }
+      HWY_ASSERT_VEC_EQ(d, min_lanes.get(), Min128(d, a, b));
+      HWY_ASSERT_VEC_EQ(d, max_lanes.get(), Max128(d, a, b));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllMinMax128() {
+  ForGEVectors<128, TestMinMax128>()(uint64_t());
+}
+
+struct TestMinMax128Upper {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using V = Vec<D>;
+    const size_t N = Lanes(d);
+    auto a_lanes = AllocateAligned<T>(N);
+    auto b_lanes = AllocateAligned<T>(N);
+    auto min_lanes = AllocateAligned<T>(N);
+    auto max_lanes = AllocateAligned<T>(N);
+    RandomState rng;
+
+    const V v00 = Zero(d);
+    const V v01 = Make128(d, 0, 1);
+    const V v10 = Make128(d, 1, 0);
+    const V v11 = Add(v01, v10);
+
+    // Same arg
+    HWY_ASSERT_VEC_EQ(d, v00, Min128Upper(d, v00, v00));
+    HWY_ASSERT_VEC_EQ(d, v01, Min128Upper(d, v01, v01));
+    HWY_ASSERT_VEC_EQ(d, v10, Min128Upper(d, v10, v10));
+    HWY_ASSERT_VEC_EQ(d, v11, Min128Upper(d, v11, v11));
+    HWY_ASSERT_VEC_EQ(d, v00, Max128Upper(d, v00, v00));
+    HWY_ASSERT_VEC_EQ(d, v01, Max128Upper(d, v01, v01));
+    HWY_ASSERT_VEC_EQ(d, v10, Max128Upper(d, v10, v10));
+    HWY_ASSERT_VEC_EQ(d, v11, Max128Upper(d, v11, v11));
+
+    // Equivalent but not equal (chooses second arg)
+    HWY_ASSERT_VEC_EQ(d, v01, Min128Upper(d, v00, v01));
+    HWY_ASSERT_VEC_EQ(d, v11, Min128Upper(d, v10, v11));
+    HWY_ASSERT_VEC_EQ(d, v00, Min128Upper(d, v01, v00));
+    HWY_ASSERT_VEC_EQ(d, v10, Min128Upper(d, v11, v10));
+    HWY_ASSERT_VEC_EQ(d, v00, Max128Upper(d, v01, v00));
+    HWY_ASSERT_VEC_EQ(d, v10, Max128Upper(d, v11, v10));
+    HWY_ASSERT_VEC_EQ(d, v01, Max128Upper(d, v00, v01));
+    HWY_ASSERT_VEC_EQ(d, v11, Max128Upper(d, v10, v11));
+
+    // First arg less
+    HWY_ASSERT_VEC_EQ(d, v01, Min128Upper(d, v01, v10));
+    HWY_ASSERT_VEC_EQ(d, v10, Max128Upper(d, v01, v10));
+
+    // Second arg less
+    HWY_ASSERT_VEC_EQ(d, v01, Min128Upper(d, v10, v01));
+    HWY_ASSERT_VEC_EQ(d, v10, Max128Upper(d, v10, v01));
+
+    // Also check 128-bit blocks are independent
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        a_lanes[i] = Random64(&rng);
+        b_lanes[i] = Random64(&rng);
+      }
+      const V a = Load(d, a_lanes.get());
+      const V b = Load(d, b_lanes.get());
+      for (size_t i = 0; i < N; i += 2) {
+        const bool lt = a_lanes[i + 1] < b_lanes[i + 1];
+        min_lanes[i + 0] = lt ? a_lanes[i + 0] : b_lanes[i + 0];
+        min_lanes[i + 1] = lt ? a_lanes[i + 1] : b_lanes[i + 1];
+        max_lanes[i + 0] = lt ? b_lanes[i + 0] : a_lanes[i + 0];
+        max_lanes[i + 1] = lt ? b_lanes[i + 1] : a_lanes[i + 1];
+      }
+      HWY_ASSERT_VEC_EQ(d, min_lanes.get(), Min128Upper(d, a, b));
+      HWY_ASSERT_VEC_EQ(d, max_lanes.get(), Max128Upper(d, a, b));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllMinMax128Upper() {
+  ForGEVectors<128, TestMinMax128Upper>()(uint64_t());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyMinMaxTest);
+HWY_EXPORT_AND_TEST_P(HwyMinMaxTest, TestAllMinMax);
+HWY_EXPORT_AND_TEST_P(HwyMinMaxTest, TestAllMinMax128);
+HWY_EXPORT_AND_TEST_P(HwyMinMaxTest, TestAllMinMax128Upper);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/mul_by_pow2_test.cc

@@ -0,0 +1,618 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/mul_by_pow2_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+template <class D>
+static void MulByPow2TestCases(
+    D d, size_t& padded, AlignedFreeUniquePtr<TFromD<D>[]>& out_val,
+    AlignedFreeUniquePtr<MakeSigned<TFromD<D>>[]>& out_exp,
+    AlignedFreeUniquePtr<TFromD<D>[]>& out_expected) {
+  using T = TFromD<D>;
+  using TI = MakeSigned<T>;
+  using TU = MakeUnsigned<T>;
+
+  struct TestCaseVals {
+    T val;
+    TI exp;
+    T expected;
+  };
+
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T k0 = ConvertScalarTo<T>(0.0);
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T k1 = ConvertScalarTo<T>(1.0);
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kNeg1 = ConvertScalarTo<T>(-1.0);
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kPosInf =
+      BitCastScalar<T>(ExponentMask<T>());
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kNegInf = BitCastScalar<T>(
+      static_cast<TU>(ExponentMask<T>() | (TU{1} << (sizeof(TU) * 8 - 1))));
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kNaN = BitCastScalar<T>(
+      static_cast<TU>(ExponentMask<T>() | (ExponentMask<T>() >> 1)));
+
+  constexpr TI kMinInt = LimitsMin<TI>();
+  constexpr TI kMaxInt = LimitsMax<TI>();
+
+  constexpr int kNumOfMantBits = MantissaBits<T>();
+  constexpr TI kExpBias = static_cast<TI>(MaxExponentField<T>() >> 1);
+  static_assert(kExpBias > 0, "kExpBias > 0 must be true");
+
+  constexpr TI kHugeExp = static_cast<TI>(kExpBias + 2);
+  constexpr TI kTinyExp = static_cast<TI>(-kExpBias - kNumOfMantBits - 2);
+
+  const TestCaseVals test_cases[] = {
+      {k0, static_cast<TI>(0), k0},
+      {k0, kTinyExp, k0},
+      {k0, kMinInt, k0},
+      {k0, kHugeExp, k0},
+      {k0, kMaxInt, k0},
+      {kNaN, static_cast<TI>(0), kNaN},
+      {kNaN, kTinyExp, kNaN},
+      {kNaN, kMinInt, kNaN},
+      {kNaN, kHugeExp, kNaN},
+      {kNaN, kMaxInt, kNaN},
+      {k1, static_cast<TI>(0), k1},
+      {k1, kTinyExp, k0},
+      {k1, kMinInt, k0},
+      {k1, kHugeExp, kPosInf},
+      {k1, kMaxInt, kPosInf},
+      {kNeg1, static_cast<TI>(0), kNeg1},
+      {kNeg1, kTinyExp, k0},
+      {kNeg1, kMinInt, k0},
+      {kNeg1, kHugeExp, kNegInf},
+      {kNeg1, kMaxInt, kNegInf},
+      {kPosInf, static_cast<TI>(0), kPosInf},
+      {kPosInf, kTinyExp, kPosInf},
+      {kPosInf, kMinInt, kPosInf},
+      {kPosInf, kHugeExp, kPosInf},
+      {kPosInf, kMaxInt, kPosInf},
+      {kNegInf, static_cast<TI>(0), kNegInf},
+      {kNegInf, kTinyExp, kNegInf},
+      {kNegInf, kMinInt, kNegInf},
+      {kNegInf, kHugeExp, kNegInf},
+      {kNegInf, kMaxInt, kNegInf},
+      {ConvertScalarTo<T>(0.04222115867157571), kMaxInt, kPosInf},
+      {ConvertScalarTo<T>(20.486009923543207), kMaxInt, kPosInf},
+      {ConvertScalarTo<T>(-2.100807236091572), kMaxInt, kNegInf},
+      {ConvertScalarTo<T>(-0.02786731762944917), kMinInt, k0},
+      {ConvertScalarTo<T>(-87.33609795949775), kMinInt, k0},
+      {ConvertScalarTo<T>(-0.01091789786912318), kMaxInt, kNegInf},
+      {ConvertScalarTo<T>(13.598303329265983), kMinInt, k0},
+      {ConvertScalarTo<T>(4.461800938646495), kMinInt, k0},
+      {ConvertScalarTo<T>(13.8359375), static_cast<TI>(-4),
+       ConvertScalarTo<T>(0.86474609375)},
+      {ConvertScalarTo<T>(27.671875), static_cast<TI>(-3),
+       ConvertScalarTo<T>(3.458984375)},
+      {ConvertScalarTo<T>(41.5078125), static_cast<TI>(-2),
+       ConvertScalarTo<T>(10.376953125)},
+      {ConvertScalarTo<T>(55.34375), static_cast<TI>(-1),
+       ConvertScalarTo<T>(27.671875)},
+      {ConvertScalarTo<T>(69.1796875), static_cast<TI>(0),
+       ConvertScalarTo<T>(69.1796875)},
+      {ConvertScalarTo<T>(83.015625), static_cast<TI>(1),
+       ConvertScalarTo<T>(166.03125)},
+      {ConvertScalarTo<T>(96.8515625), static_cast<TI>(2),
+       ConvertScalarTo<T>(387.40625)},
+      {ConvertScalarTo<T>(110.6875), static_cast<TI>(3),
+       ConvertScalarTo<T>(885.5)},
+      {ConvertScalarTo<T>(124.5234375), static_cast<TI>(4),
+       ConvertScalarTo<T>(1992.375)},
+      {ConvertScalarTo<T>(138.359375), static_cast<TI>(-4),
+       ConvertScalarTo<T>(8.6474609375)},
+      {ConvertScalarTo<T>(152.1953125), static_cast<TI>(-3),
+       ConvertScalarTo<T>(19.0244140625)},
+      {ConvertScalarTo<T>(166.03125), static_cast<TI>(-2),
+       ConvertScalarTo<T>(41.5078125)},
+      {ConvertScalarTo<T>(179.8671875), static_cast<TI>(-1),
+       ConvertScalarTo<T>(89.93359375)},
+      {ConvertScalarTo<T>(193.703125), static_cast<TI>(0),
+       ConvertScalarTo<T>(193.703125)},
+      {ConvertScalarTo<T>(207.5390625), static_cast<TI>(1),
+       ConvertScalarTo<T>(415.078125)},
+      {ConvertScalarTo<T>(221.375), static_cast<TI>(2),
+       ConvertScalarTo<T>(885.5)},
+      {ConvertScalarTo<T>(235.2109375), static_cast<TI>(3),
+       ConvertScalarTo<T>(1881.6875)},
+      {ConvertScalarTo<T>(249.046875), static_cast<TI>(4),
+       ConvertScalarTo<T>(3984.75)},
+      {ConvertScalarTo<T>(262.8828125), static_cast<TI>(-4),
+       ConvertScalarTo<T>(16.43017578125)},
+      {ConvertScalarTo<T>(276.71875), static_cast<TI>(-3),
+       ConvertScalarTo<T>(34.58984375)},
+      {ConvertScalarTo<T>(290.5546875), static_cast<TI>(-2),
+       ConvertScalarTo<T>(72.638671875)},
+      {ConvertScalarTo<T>(304.390625), static_cast<TI>(-1),
+       ConvertScalarTo<T>(152.1953125)},
+      {ConvertScalarTo<T>(318.2265625), static_cast<TI>(0),
+       ConvertScalarTo<T>(318.2265625)},
+      {ConvertScalarTo<T>(332.0625), static_cast<TI>(1),
+       ConvertScalarTo<T>(664.125)},
+      {ConvertScalarTo<T>(345.8984375), static_cast<TI>(2),
+       ConvertScalarTo<T>(1383.59375)},
+      {ConvertScalarTo<T>(359.734375), static_cast<TI>(3),
+       ConvertScalarTo<T>(2877.875)},
+      {ConvertScalarTo<T>(373.5703125), static_cast<TI>(4),
+       ConvertScalarTo<T>(5977.125)},
+      {ConvertScalarTo<T>(387.40625), static_cast<TI>(-4),
+       ConvertScalarTo<T>(24.212890625)},
+      {ConvertScalarTo<T>(401.2421875), static_cast<TI>(-3),
+       ConvertScalarTo<T>(50.1552734375)},
+      {ConvertScalarTo<T>(415.078125), static_cast<TI>(-2),
+       ConvertScalarTo<T>(103.76953125)},
+      {ConvertScalarTo<T>(428.9140625), static_cast<TI>(-1),
+       ConvertScalarTo<T>(214.45703125)},
+      {ConvertScalarTo<T>(442.75), static_cast<TI>(0),
+       ConvertScalarTo<T>(442.75)}};
+
+  const size_t kNumTestCases = sizeof(test_cases) / sizeof(test_cases[0]);
+  const size_t N = Lanes(d);
+  padded = RoundUpTo(kNumTestCases, N);  // allow loading whole vectors
+
+  out_val = AllocateAligned<T>(padded);
+  out_exp = AllocateAligned<TI>(padded);
+  out_expected = AllocateAligned<T>(padded);
+  HWY_ASSERT(out_val && out_exp && out_expected);
+
+  size_t i = 0;
+  for (; i < kNumTestCases; ++i) {
+    out_val[i] = test_cases[i].val;
+    out_exp[i] = test_cases[i].exp;
+    out_expected[i] = test_cases[i].expected;
+  }
+  for (; i < padded; ++i) {
+    out_val[i] = k0;
+    out_exp[i] = static_cast<TI>(0);
+    out_expected[i] = k0;
+  }
+}
+
+struct TestMulByPow2 {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TI = MakeSigned<T>;
+
+    const RebindToSigned<decltype(d)> di;
+
+    size_t padded;
+    AlignedFreeUniquePtr<T[]> val_lanes;
+    AlignedFreeUniquePtr<TI[]> exp_lanes;
+    AlignedFreeUniquePtr<T[]> expected;
+
+    MulByPow2TestCases(d, padded, val_lanes, exp_lanes, expected);
+
+    const size_t N = Lanes(d);
+    for (size_t i = 0; i < padded; i += N) {
+      const auto val = Load(d, val_lanes.get() + i);
+      const auto exp = Load(di, exp_lanes.get() + i);
+      HWY_ASSERT_VEC_EQ(d, expected.get() + i, MulByPow2(val, exp));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllMulByPow2() {
+  ForFloatTypes(ForPartialVectors<TestMulByPow2>());
+}
+
+template <class D>
+static void MulByFloorPow2TestCases(
+    D d, size_t& padded, AlignedFreeUniquePtr<TFromD<D>[]>& out_val,
+    AlignedFreeUniquePtr<TFromD<D>[]>& out_exp,
+    AlignedFreeUniquePtr<TFromD<D>[]>& out_expected) {
+  using T = TFromD<D>;
+  using TI = MakeSigned<T>;
+  using TU = MakeUnsigned<T>;
+
+  struct TestCaseVals {
+    T val;
+    T exp;
+    T expected;
+  };
+
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T k0 = ConvertScalarTo<T>(0.0);
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T k1 = ConvertScalarTo<T>(1.0);
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kNeg1 = ConvertScalarTo<T>(-1.0);
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kPosInf =
+      BitCastScalar<T>(ExponentMask<T>());
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kNegInf = BitCastScalar<T>(
+      static_cast<TU>(ExponentMask<T>() | (TU{1} << (sizeof(TU) * 8 - 1))));
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kNaN = BitCastScalar<T>(
+      static_cast<TU>(ExponentMask<T>() | (ExponentMask<T>() >> 1)));
+
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kMinInt =
+      ConvertScalarTo<T>(LimitsMin<TI>());
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kMaxInt =
+      ConvertScalarTo<T>(LimitsMax<TI>());
+
+  constexpr int kNumOfMantBits = MantissaBits<T>();
+  constexpr TI kExpBias = static_cast<TI>(MaxExponentField<T>() >> 1);
+  static_assert(kExpBias > 0, "kExpBias > 0 must be true");
+
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kHugeExp =
+      ConvertScalarTo<T>(static_cast<TI>(kExpBias + 2));
+  HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kTinyExp =
+      ConvertScalarTo<T>(static_cast<TI>(-kExpBias - kNumOfMantBits - 2));
+
+  const TestCaseVals test_cases[] = {
+      {k0, k0, k0},
+      {k0, kNegInf, k0},
+      {k0, kTinyExp, k0},
+      {k0, kMinInt, k0},
+      {k0, kHugeExp, k0},
+      {k0, kMaxInt, k0},
+      {k0, kPosInf, kNaN},
+      {k0, kNaN, kNaN},
+      {kNaN, k0, kNaN},
+      {kNaN, kTinyExp, kNaN},
+      {kNaN, kMinInt, kNaN},
+      {kNaN, kHugeExp, kNaN},
+      {kNaN, kMaxInt, kNaN},
+      {kNaN, kNaN, kNaN},
+      {k1, k0, k1},
+      {k1, kNegInf, k0},
+      {k1, kTinyExp, k0},
+      {k1, kMinInt, k0},
+      {k1, kHugeExp, kPosInf},
+      {k1, kMaxInt, kPosInf},
+      {k1, kPosInf, kPosInf},
+      {k1, kNaN, kNaN},
+      {kNeg1, k0, kNeg1},
+      {kNeg1, kNegInf, k0},
+      {kNeg1, kTinyExp, k0},
+      {kNeg1, kMinInt, k0},
+      {kNeg1, kHugeExp, kNegInf},
+      {kNeg1, kMaxInt, kNegInf},
+      {kNeg1, kPosInf, kNegInf},
+      {kNeg1, kNaN, kNaN},
+      {kPosInf, k0, kPosInf},
+      {kPosInf, kNegInf, kNaN},
+      {kPosInf, kTinyExp, kPosInf},
+      {kPosInf, kMinInt, kPosInf},
+      {kPosInf, kHugeExp, kPosInf},
+      {kPosInf, kMaxInt, kPosInf},
+      {kPosInf, kPosInf, kPosInf},
+      {kPosInf, kNaN, kNaN},
+      {kNegInf, k0, kNegInf},
+      {kNegInf, kNegInf, kNaN},
+      {kNegInf, kTinyExp, kNegInf},
+      {kNegInf, kMinInt, kNegInf},
+      {kNegInf, kHugeExp, kNegInf},
+      {kNegInf, kMaxInt, kNegInf},
+      {kNegInf, kPosInf, kNegInf},
+      {kNegInf, kNaN, kNaN},
+      {ConvertScalarTo<T>(1059.5310687496399), kNegInf, k0},
+      {ConvertScalarTo<T>(97.37421519431503), kHugeExp, kPosInf},
+      {ConvertScalarTo<T>(-219.77349288592427), kHugeExp, kNegInf},
+      {ConvertScalarTo<T>(-0.07908260899094358), kMinInt, k0},
+      {ConvertScalarTo<T>(-0.37461255269075006), kPosInf, kNegInf},
+      {ConvertScalarTo<T>(-1.4206473428158297), kHugeExp, kNegInf},
+      {ConvertScalarTo<T>(-0.05931912296693983), kTinyExp, k0},
+      {ConvertScalarTo<T>(6.775641488836299E-4), kMinInt, k0},
+      {ConvertScalarTo<T>(0.17859422400264996), kNegInf, k0},
+      {ConvertScalarTo<T>(-3.2925797046308722), kNaN, kNaN},
+      {ConvertScalarTo<T>(63.76025467272075), kMaxInt, kPosInf},
+      {ConvertScalarTo<T>(-949.496801885732), kPosInf, kNegInf},
+      {ConvertScalarTo<T>(0.1711619674802602), kTinyExp, k0},
+      {ConvertScalarTo<T>(307.00132360897254), kMaxInt, kPosInf},
+      {ConvertScalarTo<T>(427.2906513922638), kNegInf, k0},
+      {ConvertScalarTo<T>(0.0022915367711067672), kPosInf, kPosInf},
+      {ConvertScalarTo<T>(0.049692878940840575), kTinyExp, k0},
+      {ConvertScalarTo<T>(0.010399031566202213), kPosInf, kPosInf},
+      {ConvertScalarTo<T>(-3.5924694052830457), kMaxInt, kNegInf},
+      {ConvertScalarTo<T>(-35.6155724065998), kNegInf, k0},
+      {ConvertScalarTo<T>(-626.2755450079259), kNaN, kNaN},
+      {ConvertScalarTo<T>(1510.5106491633483), kNaN, kNaN},
+      {ConvertScalarTo<T>(6.473376273141844E-4), kMinInt, k0},
+      {ConvertScalarTo<T>(31.725913740728213), kHugeExp, kPosInf},
+      {ConvertScalarTo<T>(-0.13049751588470118), kMinInt, k0},
+      {ConvertScalarTo<T>(0.3124908582707691), kTinyExp, k0},
+      {ConvertScalarTo<T>(-54.267473800450226), kMaxInt, kNegInf},
+      {ConvertScalarTo<T>(0.019324406584034517),
+       ConvertScalarTo<T>(-0.7529789930294555),
+       ConvertScalarTo<T>(0.009662203292017259)},
+      {ConvertScalarTo<T>(-23.589356466919032),
+       ConvertScalarTo<T>(-2.658473701120114),
+       ConvertScalarTo<T>(-2.948669558364879)},
+      {ConvertScalarTo<T>(0.001358052526188437),
+       ConvertScalarTo<T>(-1.668405066102655),
+       ConvertScalarTo<T>(3.3951313154710925E-4)},
+      {ConvertScalarTo<T>(12.387671870605264),
+       ConvertScalarTo<T>(0.4676324281911401),
+       ConvertScalarTo<T>(12.387671870605264)},
+      {ConvertScalarTo<T>(-816.5904074657877),
+       ConvertScalarTo<T>(-3.161282649113014),
+       ConvertScalarTo<T>(-51.03690046661173)},
+      {ConvertScalarTo<T>(1.9061852134515243),
+       ConvertScalarTo<T>(3.7447600028715566),
+       ConvertScalarTo<T>(15.249481707612194)},
+      {ConvertScalarTo<T>(-0.27333495359539833),
+       ConvertScalarTo<T>(-2.3750517462565393),
+       ConvertScalarTo<T>(-0.03416686919942479)},
+      {ConvertScalarTo<T>(-0.0014594113131094902),
+       ConvertScalarTo<T>(3.085720424782481),
+       ConvertScalarTo<T>(-0.011675290504875922)},
+      {ConvertScalarTo<T>(637.3241672064872),
+       ConvertScalarTo<T>(4.671463920871573),
+       ConvertScalarTo<T>(10197.186675303796)},
+      {ConvertScalarTo<T>(16.944732879565052),
+       ConvertScalarTo<T>(-4.369178496185725),
+       ConvertScalarTo<T>(0.5295229024864079)},
+      {ConvertScalarTo<T>(-0.012238932661175737),
+       ConvertScalarTo<T>(-2.3194292578861413),
+       ConvertScalarTo<T>(-0.001529866582646967)},
+      {ConvertScalarTo<T>(-1.9210108199587403),
+       ConvertScalarTo<T>(0.14282162575576896),
+       ConvertScalarTo<T>(-1.9210108199587403)},
+      {ConvertScalarTo<T>(0.11043467327357612),
+       ConvertScalarTo<T>(-2.415250859773567),
+       ConvertScalarTo<T>(0.013804334159197015)},
+      {ConvertScalarTo<T>(0.002707227717617229),
+       ConvertScalarTo<T>(-2.6905028085982394),
+       ConvertScalarTo<T>(3.3840346470215363E-4)},
+      {ConvertScalarTo<T>(0.04979758601051541),
+       ConvertScalarTo<T>(-0.9179775209533863),
+       ConvertScalarTo<T>(0.024898793005257706)},
+      {ConvertScalarTo<T>(3.309059475799939),
+       ConvertScalarTo<T>(-3.6346379131430786),
+       ConvertScalarTo<T>(0.20681621723749619)},
+      {ConvertScalarTo<T>(-816.1508196321369),
+       ConvertScalarTo<T>(1.2488081335071424),
+       ConvertScalarTo<T>(-1632.3016392642737)},
+      {ConvertScalarTo<T>(-0.6125198372033248),
+       ConvertScalarTo<T>(-1.8407754804187284),
+       ConvertScalarTo<T>(-0.1531299593008312)},
+      {ConvertScalarTo<T>(0.09615218136362058),
+       ConvertScalarTo<T>(-0.39747719382682956),
+       ConvertScalarTo<T>(0.04807609068181029)},
+      {ConvertScalarTo<T>(-0.0021822722598431347),
+       ConvertScalarTo<T>(3.4238838799239812),
+       ConvertScalarTo<T>(-0.017458178078745078)},
+      {ConvertScalarTo<T>(-0.001069341172497859),
+       ConvertScalarTo<T>(2.4555415282769784),
+       ConvertScalarTo<T>(-0.004277364689991436)},
+      {ConvertScalarTo<T>(0.7280398964478823),
+       ConvertScalarTo<T>(-2.0683513150597843),
+       ConvertScalarTo<T>(0.09100498705598528)},
+      {ConvertScalarTo<T>(0.02852633429447464),
+       ConvertScalarTo<T>(-1.1775178332450202),
+       ConvertScalarTo<T>(0.00713158357361866)},
+      {ConvertScalarTo<T>(-0.0018815213571262807),
+       ConvertScalarTo<T>(-3.550897133834961),
+       ConvertScalarTo<T>(-1.1759508482039255E-4)},
+      {ConvertScalarTo<T>(0.0926207425083316),
+       ConvertScalarTo<T>(0.366218741424757),
+       ConvertScalarTo<T>(0.0926207425083316)},
+      {ConvertScalarTo<T>(-64.3799309390691),
+       ConvertScalarTo<T>(4.344404407280892),
+       ConvertScalarTo<T>(-1030.0788950251056)},
+      {ConvertScalarTo<T>(-0.02400946981744935),
+       ConvertScalarTo<T>(-2.341821021170456),
+       ConvertScalarTo<T>(-0.0030011837271811687)},
+      {ConvertScalarTo<T>(-0.08822585458970057),
+       ConvertScalarTo<T>(-0.945924667731356),
+       ConvertScalarTo<T>(-0.044112927294850286)},
+      {ConvertScalarTo<T>(122.34449441358599),
+       ConvertScalarTo<T>(0.5100210866180717),
+       ConvertScalarTo<T>(122.34449441358599)},
+      {ConvertScalarTo<T>(3.2559200224126434),
+       ConvertScalarTo<T>(1.5437558991072338),
+       ConvertScalarTo<T>(6.511840044825287)},
+      {ConvertScalarTo<T>(0.2933688026098801),
+       ConvertScalarTo<T>(3.077238809133668),
+       ConvertScalarTo<T>(2.3469504208790406)},
+      {ConvertScalarTo<T>(21.628049007866707),
+       ConvertScalarTo<T>(-2.8991237865697235),
+       ConvertScalarTo<T>(2.7035061259833384)},
+      {ConvertScalarTo<T>(63.435740493259665),
+       ConvertScalarTo<T>(-4.41713462765055),
+       ConvertScalarTo<T>(1.9823668904143645)},
+      {ConvertScalarTo<T>(-1.079270457351184),
+       ConvertScalarTo<T>(0.4150174591000348),
+       ConvertScalarTo<T>(-1.079270457351184)},
+      {ConvertScalarTo<T>(-0.05268637133114153),
+       ConvertScalarTo<T>(2.6980126913247116),
+       ConvertScalarTo<T>(-0.2107454853245661)},
+      {ConvertScalarTo<T>(-0.22694789316698213),
+       ConvertScalarTo<T>(1.7745238901054419),
+       ConvertScalarTo<T>(-0.45389578633396427)},
+      {ConvertScalarTo<T>(2.9243076223545907),
+       ConvertScalarTo<T>(3.106526706578818),
+       ConvertScalarTo<T>(23.394460978836726)},
+      {ConvertScalarTo<T>(-0.023896337084267257),
+       ConvertScalarTo<T>(4.237862481909235),
+       ConvertScalarTo<T>(-0.3823413933482761)},
+      {ConvertScalarTo<T>(8.839593057654776),
+       ConvertScalarTo<T>(1.3978500181677522),
+       ConvertScalarTo<T>(17.67918611530955)},
+      {ConvertScalarTo<T>(0.229610115587917),
+       ConvertScalarTo<T>(-4.211477622810463),
+       ConvertScalarTo<T>(0.007175316112122406)},
+      {ConvertScalarTo<T>(-1304.7253453052683),
+       ConvertScalarTo<T>(-4.783018570610186),
+       ConvertScalarTo<T>(-40.772667040789635)},
+      {ConvertScalarTo<T>(-0.025447754903889074),
+       ConvertScalarTo<T>(-1.710158670881026),
+       ConvertScalarTo<T>(-0.0063619387259722686)},
+      {ConvertScalarTo<T>(0.21914082057419035),
+       ConvertScalarTo<T>(2.356083011091819),
+       ConvertScalarTo<T>(0.8765632822967614)},
+      {ConvertScalarTo<T>(5.42862647542766),
+       ConvertScalarTo<T>(-3.5603790618102806),
+       ConvertScalarTo<T>(0.33928915471422877)},
+      {ConvertScalarTo<T>(-13.37913117518154),
+       ConvertScalarTo<T>(0.8910026966707747),
+       ConvertScalarTo<T>(-13.37913117518154)},
+      {ConvertScalarTo<T>(0.0016923959034403932),
+       ConvertScalarTo<T>(-0.0395086178195927),
+       ConvertScalarTo<T>(8.461979517201966E-4)},
+      {ConvertScalarTo<T>(-9.92565371535828),
+       ConvertScalarTo<T>(1.863550485706607),
+       ConvertScalarTo<T>(-19.85130743071656)},
+      {ConvertScalarTo<T>(-0.20406817303532074),
+       ConvertScalarTo<T>(0.4044778224004972),
+       ConvertScalarTo<T>(-0.20406817303532074)},
+      {ConvertScalarTo<T>(0.001427935411153914),
+       ConvertScalarTo<T>(-3.623007282999475),
+       ConvertScalarTo<T>(8.924596319711963E-5)},
+      {ConvertScalarTo<T>(-0.001792980244032676),
+       ConvertScalarTo<T>(-2.877489018179081),
+       ConvertScalarTo<T>(-2.241225305040845E-4)},
+      {ConvertScalarTo<T>(10.724293893361738),
+       ConvertScalarTo<T>(-0.11727937890947453),
+       ConvertScalarTo<T>(5.362146946680869)},
+      {ConvertScalarTo<T>(-0.003683378523927509),
+       ConvertScalarTo<T>(2.9771453279729068),
+       ConvertScalarTo<T>(-0.014733514095710037)},
+      {ConvertScalarTo<T>(-0.19933257881305555),
+       ConvertScalarTo<T>(-4.935752394433358),
+       ConvertScalarTo<T>(-0.006229143087907986)},
+      {ConvertScalarTo<T>(-0.05446912403204286),
+       ConvertScalarTo<T>(4.696207616307656),
+       ConvertScalarTo<T>(-0.8715059845126858)},
+      {ConvertScalarTo<T>(-0.9314238362892934),
+       ConvertScalarTo<T>(0.8598717343606272),
+       ConvertScalarTo<T>(-0.9314238362892934)},
+      {ConvertScalarTo<T>(37.9580113757031),
+       ConvertScalarTo<T>(0.16132223888265382),
+       ConvertScalarTo<T>(37.9580113757031)},
+      {ConvertScalarTo<T>(-0.0011390770646514143),
+       ConvertScalarTo<T>(0.7120556932780127),
+       ConvertScalarTo<T>(-0.0011390770646514143)},
+      {ConvertScalarTo<T>(-0.0029396730491874634),
+       ConvertScalarTo<T>(-0.5212230786233737),
+       ConvertScalarTo<T>(-0.0014698365245937317)},
+      {ConvertScalarTo<T>(-0.0028807746370910136),
+       ConvertScalarTo<T>(1.8728808720779935),
+       ConvertScalarTo<T>(-0.005761549274182027)},
+      {ConvertScalarTo<T>(0.013988891550764875),
+       ConvertScalarTo<T>(-3.3438559116920574),
+       ConvertScalarTo<T>(8.743057219228047E-4)},
+      {ConvertScalarTo<T>(745.7501203846241),
+       ConvertScalarTo<T>(-1.0139927506606408),
+       ConvertScalarTo<T>(186.43753009615602)},
+      {ConvertScalarTo<T>(45.347287390908605),
+       ConvertScalarTo<T>(-3.9706807059278515),
+       ConvertScalarTo<T>(2.834205461931788)},
+      {ConvertScalarTo<T>(-2.3894976687908356),
+       ConvertScalarTo<T>(4.7410857372930515),
+       ConvertScalarTo<T>(-38.23196270065337)},
+      {ConvertScalarTo<T>(-0.001029165846088247),
+       ConvertScalarTo<T>(3.8187838912978687),
+       ConvertScalarTo<T>(-0.008233326768705976)},
+      {ConvertScalarTo<T>(0.017570690697900453),
+       ConvertScalarTo<T>(1.9090470977828633),
+       ConvertScalarTo<T>(0.035141381395800905)}};
+
+  const size_t kNumTestCases = sizeof(test_cases) / sizeof(test_cases[0]);
+  const size_t N = Lanes(d);
+  padded = RoundUpTo(kNumTestCases, N);  // allow loading whole vectors
+
+  out_val = AllocateAligned<T>(padded);
+  out_exp = AllocateAligned<T>(padded);
+  out_expected = AllocateAligned<T>(padded);
+  HWY_ASSERT(out_val && out_exp && out_expected);
+
+  size_t i = 0;
+  for (; i < kNumTestCases; ++i) {
+    out_val[i] = test_cases[i].val;
+    out_exp[i] = test_cases[i].exp;
+    out_expected[i] = test_cases[i].expected;
+  }
+  for (; i < padded; ++i) {
+    out_val[i] = k0;
+    out_exp[i] = k0;
+    out_expected[i] = k0;
+  }
+}
+
+struct TestMulByFloorPow2 {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TU = MakeUnsigned<T>;
+
+    size_t padded;
+    AlignedFreeUniquePtr<T[]> val_lanes;
+    AlignedFreeUniquePtr<T[]> exp_lanes;
+    AlignedFreeUniquePtr<T[]> expected;
+
+    MulByFloorPow2TestCases(d, padded, val_lanes, exp_lanes, expected);
+    HWY_ASSERT(padded >= 2);
+
+    const size_t N = Lanes(d);
+    for (size_t i = 0; i < padded; i += N) {
+      const auto val = Load(d, val_lanes.get() + i);
+      const auto exp = Load(d, exp_lanes.get() + i);
+      HWY_ASSERT_VEC_EQ(d, expected.get() + i, MulByFloorPow2(val, exp));
+      HWY_ASSERT_VEC_EQ(d, expected.get() + i, MulByFloorPow2(val, Floor(exp)));
+    }
+
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kNaN = BitCastScalar<T>(
+        static_cast<TU>(ExponentMask<T>() | (ExponentMask<T>() >> 1)));
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kNegInf = BitCastScalar<T>(
+        static_cast<TU>(ExponentMask<T>() | (TU{1} << (sizeof(TU) * 8 - 1))));
+    HWY_BITCASTSCALAR_CXX14_CONSTEXPR const T kPosInf =
+        BitCastScalar<T>(ExponentMask<T>());
+
+    val_lanes[0] = kNaN;
+    exp_lanes[0] = kNegInf;
+    expected[0] = kNaN;
+
+    val_lanes[1] = kNaN;
+    exp_lanes[1] = kPosInf;
+    expected[1] = kNaN;
+
+    for (size_t i = 0; i < 2; i += N) {
+      const auto val = Load(d, val_lanes.get());
+      const auto exp = Load(d, exp_lanes.get());
+      const auto mul_result = MulByFloorPow2(val, exp);
+      const auto actual = Or(
+          mul_result, VecFromMask(d, And(And(IsNaN(val), IsInf(exp)),
+                                         Eq(mul_result, ZeroIfNegative(exp)))));
+
+      HWY_ASSERT_VEC_EQ(d, expected.get(), actual);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllMulByFloorPow2() {
+  ForFloatTypes(ForPartialVectors<TestMulByFloorPow2>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyMulByPow2Test);
+HWY_EXPORT_AND_TEST_P(HwyMulByPow2Test, TestAllMulByPow2);
+HWY_EXPORT_AND_TEST_P(HwyMulByPow2Test, TestAllMulByFloorPow2);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/mul_pairwise_test.cc

@@ -0,0 +1,361 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/mul_pairwise_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestWidenMulPairwiseAdd {
+  // Must be inlined on aarch64 for bf16, else clang crashes.
+  template <typename TN, class DN>
+  HWY_INLINE void operator()(TN /*unused*/, DN dn) {
+    using TW = MakeWide<TN>;
+    const RepartitionToWide<DN> dw;
+    using VW = Vec<decltype(dw)>;
+    using VN = Vec<decltype(dn)>;
+    const size_t NN = Lanes(dn);
+
+    const VW f0 = Zero(dw);
+    const VW f1 = Set(dw, ConvertScalarTo<TW>(1));
+    const VN bf0 = Zero(dn);
+    // Cannot Set() bfloat16_t directly.
+    const VN bf1 = ReorderDemote2To(dn, f1, f1);
+
+    // Any input zero => both outputs zero
+    HWY_ASSERT_VEC_EQ(dw, f0, WidenMulPairwiseAdd(dw, bf0, bf0));
+    HWY_ASSERT_VEC_EQ(dw, f0, WidenMulPairwiseAdd(dw, bf0, bf1));
+    HWY_ASSERT_VEC_EQ(dw, f0, WidenMulPairwiseAdd(dw, bf1, bf0));
+
+    // delta[p] := p all others zero.
+    auto delta_w = AllocateAligned<TW>(NN);
+    HWY_ASSERT(delta_w);
+    for (size_t p = 0; p < NN; ++p) {
+      // Workaround for incorrect Clang wasm codegen: re-initialize the entire
+      // array rather than zero-initialize once and then set lane p to p.
+      for (size_t i = 0; i < NN; ++i) {
+        delta_w[i] = static_cast<TW>((i == p) ? p : 0);
+      }
+      const VW delta0 = Load(dw, delta_w.get() + 0);
+      const VW delta1 = Load(dw, delta_w.get() + NN / 2);
+      const VN delta = OrderedDemote2To(dn, delta0, delta1);
+
+      const VW expected = InsertLane(f0, p / 2, static_cast<TW>(p));
+      {
+        const VW actual = WidenMulPairwiseAdd(dw, delta, bf1);
+        HWY_ASSERT_VEC_EQ(dw, expected, actual);
+      }
+      // Swapped arg order
+      {
+        const VW actual = WidenMulPairwiseAdd(dw, bf1, delta);
+        HWY_ASSERT_VEC_EQ(dw, expected, actual);
+      }
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllWidenMulPairwiseAdd() {
+  ForShrinkableVectors<TestWidenMulPairwiseAdd>()(bfloat16_t());
+  ForShrinkableVectors<TestWidenMulPairwiseAdd>()(int16_t());
+  ForShrinkableVectors<TestWidenMulPairwiseAdd>()(uint16_t());
+}
+
+struct TestSatWidenMulPairwiseAdd {
+  template <typename TN, class DN>
+  HWY_NOINLINE void operator()(TN /*unused*/, DN dn) {
+    static_assert(IsSame<TN, int8_t>(), "TN should be int8_t");
+
+    using TN_U = MakeUnsigned<TN>;
+    using TW = MakeWide<TN>;
+    const RepartitionToWide<DN> dw;
+    using VW = Vec<decltype(dw)>;
+    using VN = Vec<decltype(dn)>;
+    const size_t NN = Lanes(dn);
+    const size_t NW = Lanes(dw);
+    HWY_ASSERT(NN == NW * 2);
+
+    const RebindToUnsigned<decltype(dn)> dn_u;
+
+    const VW f0 = Zero(dw);
+    const VN nf0 = Zero(dn);
+    const VN nf1 = Set(dn, TN{1});
+
+    // Any input zero => both outputs zero
+    HWY_ASSERT_VEC_EQ(dw, f0,
+                      SatWidenMulPairwiseAdd(dw, BitCast(dn_u, nf0), nf0));
+    HWY_ASSERT_VEC_EQ(dw, f0,
+                      SatWidenMulPairwiseAdd(dw, BitCast(dn_u, nf0), nf1));
+    HWY_ASSERT_VEC_EQ(dw, f0,
+                      SatWidenMulPairwiseAdd(dw, BitCast(dn_u, nf1), nf0));
+
+    // delta[p] := p all others zero.
+    auto delta_w = AllocateAligned<TW>(NN);
+    HWY_ASSERT(delta_w);
+
+    auto expected = AllocateAligned<TW>(NW);
+    HWY_ASSERT(expected);
+    Store(f0, dw, expected.get());
+
+    for (size_t p = 0; p < NN; ++p) {
+      // Workaround for incorrect Clang wasm codegen: re-initialize the entire
+      // array rather than zero-initialize once and then set lane p to p.
+
+      const TN pn = static_cast<TN>(p);
+      const TN_U pn_u = static_cast<TN_U>(pn);
+      for (size_t i = 0; i < NN; ++i) {
+        delta_w[i] = static_cast<TW>((i == p) ? pn : 0);
+      }
+      const VW delta0 = Load(dw, delta_w.get() + 0);
+      const VW delta1 = Load(dw, delta_w.get() + NN / 2);
+      const VN delta = OrderedDemote2To(dn, delta0, delta1);
+
+      expected[p / 2] = static_cast<TW>(pn_u);
+      const VW actual_1 = SatWidenMulPairwiseAdd(dw, BitCast(dn_u, delta), nf1);
+      HWY_ASSERT_VEC_EQ(dw, expected.get(), actual_1);
+
+      // Swapped arg order
+      expected[p / 2] = static_cast<TW>(pn);
+      const VW actual_2 = SatWidenMulPairwiseAdd(dw, BitCast(dn_u, nf1), delta);
+      HWY_ASSERT_VEC_EQ(dw, expected.get(), actual_2);
+
+      expected[p / 2] = TW{0};
+    }
+
+    const auto vn_signed_min = Set(dn, LimitsMin<TN>());
+    const auto vn_signed_max = Set(dn, LimitsMax<TN>());
+    const auto vn_unsigned_max = Set(dn_u, LimitsMax<TN_U>());
+    const auto vw_signed_min = Set(dw, LimitsMin<TW>());
+    const auto vw_signed_max = Set(dw, LimitsMax<TW>());
+    const auto vw_neg_tn_unsigned_max =
+        Set(dw, static_cast<TW>(-static_cast<TW>(LimitsMax<TN_U>())));
+
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_signed_max,
+        SatWidenMulPairwiseAdd(dw, vn_unsigned_max, vn_signed_max));
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_signed_min,
+        SatWidenMulPairwiseAdd(dw, vn_unsigned_max, vn_signed_min));
+    HWY_ASSERT_VEC_EQ(dw, vw_neg_tn_unsigned_max,
+                      SatWidenMulPairwiseAdd(
+                          dw, vn_unsigned_max,
+                          InterleaveLower(dn, vn_signed_max, vn_signed_min)));
+    HWY_ASSERT_VEC_EQ(dw, vw_neg_tn_unsigned_max,
+                      SatWidenMulPairwiseAdd(
+                          dw, vn_unsigned_max,
+                          InterleaveLower(dn, vn_signed_min, vn_signed_max)));
+
+    constexpr TN kSignedMax = LimitsMax<TN>();
+    constexpr TN kZeroIotaRepl = static_cast<TN>(LimitsMax<TN>() - 16);
+
+    auto in_a = AllocateAligned<TN>(NN);
+    auto in_b = AllocateAligned<TN>(NN);
+    auto in_neg_b = AllocateAligned<TN>(NN);
+    HWY_ASSERT(in_a && in_b && in_neg_b);
+
+    for (size_t i = 0; i < NN; i++) {
+      const auto val = ((i + 1) & kSignedMax);
+      const auto a_val = static_cast<TN>((val != 0) ? val : kZeroIotaRepl);
+      const auto b_val = static_cast<TN>((a_val & 63) + 20);
+      in_a[i] = a_val;
+      in_b[i] = static_cast<TN>(b_val);
+      in_neg_b[i] = static_cast<TN>(-b_val);
+    }
+
+    for (size_t i = 0; i < NW; i++) {
+      const TW a0 = static_cast<TW>(in_a[2 * i]);
+      const TW a1 = static_cast<TW>(in_a[2 * i + 1]);
+      expected[i] = static_cast<TW>(a0 * a0 + a1 * a1);
+    }
+
+    auto vn_a = Load(dn, in_a.get());
+    HWY_ASSERT_VEC_EQ(dw, expected.get(),
+                      SatWidenMulPairwiseAdd(dw, BitCast(dn_u, vn_a), vn_a));
+
+    for (size_t i = 0; i < NW; i++) {
+      expected[i] = static_cast<TW>(-expected[i]);
+    }
+
+    HWY_ASSERT_VEC_EQ(
+        dw, expected.get(),
+        SatWidenMulPairwiseAdd(dw, BitCast(dn_u, vn_a), Neg(vn_a)));
+
+    auto vn_b = Load(dn, in_b.get());
+
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_signed_max,
+        SatWidenMulPairwiseAdd(
+            dw, InterleaveLower(dn_u, BitCast(dn_u, vn_b), vn_unsigned_max),
+            InterleaveLower(dn, vn_b, vn_signed_max)));
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_signed_max,
+        SatWidenMulPairwiseAdd(
+            dw, InterleaveUpper(dn_u, BitCast(dn_u, vn_b), vn_unsigned_max),
+            InterleaveUpper(dn, vn_b, vn_signed_max)));
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_signed_max,
+        SatWidenMulPairwiseAdd(
+            dw, InterleaveLower(dn_u, vn_unsigned_max, BitCast(dn_u, vn_b)),
+            InterleaveLower(dn, vn_signed_max, vn_b)));
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_signed_max,
+        SatWidenMulPairwiseAdd(
+            dw, InterleaveUpper(dn_u, vn_unsigned_max, BitCast(dn_u, vn_b)),
+            InterleaveUpper(dn, vn_signed_max, vn_b)));
+
+    const auto vn_neg_b = Load(dn, in_neg_b.get());
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_signed_min,
+        SatWidenMulPairwiseAdd(
+            dw, InterleaveLower(dn_u, BitCast(dn_u, vn_b), vn_unsigned_max),
+            InterleaveLower(dn, vn_neg_b, vn_signed_min)));
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_signed_min,
+        SatWidenMulPairwiseAdd(
+            dw, InterleaveUpper(dn_u, BitCast(dn_u, vn_b), vn_unsigned_max),
+            InterleaveUpper(dn, vn_neg_b, vn_signed_min)));
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_signed_min,
+        SatWidenMulPairwiseAdd(
+            dw, InterleaveLower(dn_u, vn_unsigned_max, BitCast(dn_u, vn_b)),
+            InterleaveLower(dn, vn_signed_min, vn_neg_b)));
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_signed_min,
+        SatWidenMulPairwiseAdd(
+            dw, InterleaveUpper(dn_u, vn_unsigned_max, BitCast(dn_u, vn_b)),
+            InterleaveUpper(dn, vn_signed_min, vn_neg_b)));
+
+    constexpr size_t kMaxLanesPerNBlock = 16 / sizeof(TN);
+    constexpr size_t kMaxLanesPerWBlock = 16 / sizeof(TW);
+
+    for (size_t i = 0; i < NW; i++) {
+      const size_t blk_idx = i / kMaxLanesPerWBlock;
+      const TW b = static_cast<TW>(
+          in_b[blk_idx * kMaxLanesPerNBlock + (i & (kMaxLanesPerWBlock - 1))]);
+      expected[i] =
+          static_cast<TW>(b * b + static_cast<TW>(LimitsMax<TN_U>()) *
+                                      static_cast<TW>(LimitsMin<TN>()));
+    }
+    HWY_ASSERT_VEC_EQ(
+        dw, expected.get(),
+        SatWidenMulPairwiseAdd(
+            dw, InterleaveLower(dn_u, vn_unsigned_max, BitCast(dn_u, vn_b)),
+            InterleaveLower(dn, vn_signed_min, vn_b)));
+    HWY_ASSERT_VEC_EQ(
+        dw, expected.get(),
+        SatWidenMulPairwiseAdd(
+            dw, InterleaveLower(dn_u, BitCast(dn_u, vn_b), vn_unsigned_max),
+            InterleaveLower(dn, vn_b, vn_signed_min)));
+  }
+};
+
+HWY_NOINLINE void TestAllSatWidenMulPairwiseAdd() {
+  ForShrinkableVectors<TestSatWidenMulPairwiseAdd>()(int8_t());
+}
+
+struct TestSatWidenMulPairwiseAccumulate {
+  template <class TN, class DN>
+  HWY_INLINE void operator()(TN /*unused*/, DN dn) {
+    static_assert(IsSigned<TN>() && !IsFloat<TN>() && !IsSpecialFloat<TN>(),
+                  "TN must be a signed integer type");
+
+    using TW = MakeWide<TN>;
+    const RepartitionToWide<DN> dw;
+    using VW = Vec<decltype(dw)>;
+    using VN = Vec<decltype(dn)>;
+    const size_t NN = Lanes(dn);
+    const size_t NW = Lanes(dw);
+    HWY_ASSERT(NN == NW * 2);
+
+    const VN vn_min = Set(dn, LimitsMin<TN>());
+    const VN vn_kneg1 = Set(dn, static_cast<TN>(-1));
+    const VN vn_k1 = Set(dn, static_cast<TN>(1));
+    const VN vn_max = Set(dn, LimitsMax<TN>());
+
+    const VW vw_min = Set(dw, LimitsMin<TW>());
+    const VW vw_kneg7 = Set(dw, static_cast<TW>(-7));
+    const VW vw_kneg1 = Set(dw, static_cast<TW>(-1));
+    const VW vw_k0 = Zero(dw);
+    const VW vw_k1 = Set(dw, static_cast<TW>(1));
+    const VW vw_k5 = Set(dw, static_cast<TW>(5));
+    const VW vw_max = Set(dw, LimitsMax<TW>());
+
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_min, SatWidenMulPairwiseAccumulate(dw, vn_max, vn_min, vw_min));
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_max, SatWidenMulPairwiseAccumulate(dw, vn_max, vn_max, vw_max));
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_max,
+        SatWidenMulPairwiseAccumulate(dw, vn_min, vn_min, vw_kneg1));
+
+    const VN vn_p = PositiveIota(dn);
+    const VN vn_n = Neg(vn_p);
+
+    const VW vw_sum2_p = SumsOf2(vn_p);
+    const VW vw_sum2_n = SumsOf2(vn_n);
+
+    const VW vw_p2 = Add(MulEven(vn_p, vn_p), MulOdd(vn_p, vn_p));
+    const VW vw_n2 = Add(MulEven(vn_p, vn_n), MulOdd(vn_p, vn_n));
+
+    HWY_ASSERT_VEC_EQ(dw, vw_p2,
+                      SatWidenMulPairwiseAccumulate(dw, vn_p, vn_p, vw_k0));
+    HWY_ASSERT_VEC_EQ(dw, vw_n2,
+                      SatWidenMulPairwiseAccumulate(dw, vn_p, vn_n, vw_k0));
+    HWY_ASSERT_VEC_EQ(dw, Add(vw_p2, vw_k1),
+                      SatWidenMulPairwiseAccumulate(dw, vn_p, vn_p, vw_k1));
+    HWY_ASSERT_VEC_EQ(dw, Add(vw_n2, vw_k1),
+                      SatWidenMulPairwiseAccumulate(dw, vn_n, vn_p, vw_k1));
+    HWY_ASSERT_VEC_EQ(dw, Add(vw_sum2_p, vw_k5),
+                      SatWidenMulPairwiseAccumulate(dw, vn_p, vn_k1, vw_k5));
+    HWY_ASSERT_VEC_EQ(dw, Add(vw_sum2_n, vw_kneg7),
+                      SatWidenMulPairwiseAccumulate(dw, vn_n, vn_k1, vw_kneg7));
+    HWY_ASSERT_VEC_EQ(
+        dw, Add(vw_sum2_p, vw_kneg7),
+        SatWidenMulPairwiseAccumulate(dw, vn_kneg1, vn_n, vw_kneg7));
+    HWY_ASSERT_VEC_EQ(dw, Add(vw_sum2_n, vw_kneg7),
+                      SatWidenMulPairwiseAccumulate(dw, vn_k1, vn_n, vw_kneg7));
+  }
+};
+
+HWY_NOINLINE void TestAllSatWidenMulPairwiseAccumulate() {
+  ForShrinkableVectors<TestSatWidenMulPairwiseAccumulate>()(int16_t());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyMulPairwiseTest);
+HWY_EXPORT_AND_TEST_P(HwyMulPairwiseTest, TestAllWidenMulPairwiseAdd);
+HWY_EXPORT_AND_TEST_P(HwyMulPairwiseTest, TestAllSatWidenMulPairwiseAdd);
+HWY_EXPORT_AND_TEST_P(HwyMulPairwiseTest, TestAllSatWidenMulPairwiseAccumulate);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/mul_test.cc

@@ -0,0 +1,446 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/mul_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+template <size_t kBits>
+constexpr uint64_t FirstBits() {
+  return (1ull << kBits) - 1;
+}
+template <>
+constexpr uint64_t FirstBits<64>() {
+  return ~uint64_t{0};
+}
+
+struct TestUnsignedMul {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> v1 = Set(d, static_cast<T>(1));
+    const Vec<D> vi = Iota(d, 1);
+    const Vec<D> vj = Iota(d, 3);
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    HWY_ASSERT_VEC_EQ(d, v0, Mul(v0, v0));
+    HWY_ASSERT_VEC_EQ(d, v1, Mul(v1, v1));
+    HWY_ASSERT_VEC_EQ(d, vi, Mul(v1, vi));
+    HWY_ASSERT_VEC_EQ(d, vi, Mul(vi, v1));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>((1 + i) * (1 + i));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Mul(vi, vi));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>((1 + i) * (3 + i));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Mul(vi, vj));
+
+    const T max = LimitsMax<T>();
+    const auto vmax = Set(d, max);
+    HWY_ASSERT_VEC_EQ(d, vmax, Mul(vmax, v1));
+    HWY_ASSERT_VEC_EQ(d, vmax, Mul(v1, vmax));
+
+    constexpr uint64_t kMask = FirstBits<sizeof(T) * 8>();
+    const T max2 = (static_cast<uint64_t>(max) * max) & kMask;
+    HWY_ASSERT_VEC_EQ(d, Set(d, max2), Mul(vmax, vmax));
+  }
+};
+
+struct TestSignedMul {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> v1 = Set(d, static_cast<T>(1));
+    const Vec<D> vi = Iota(d, 1);
+    // i8 is not supported, so T is large enough to avoid wraparound.
+    const Vec<D> vn = Iota(d, -static_cast<T>(N));
+    HWY_ASSERT_VEC_EQ(d, v0, Mul(v0, v0));
+    HWY_ASSERT_VEC_EQ(d, v1, Mul(v1, v1));
+    HWY_ASSERT_VEC_EQ(d, vi, Mul(v1, vi));
+    HWY_ASSERT_VEC_EQ(d, vi, Mul(vi, v1));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>((1 + i) * (1 + i));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Mul(vi, vi));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = static_cast<T>((-static_cast<T>(N) + static_cast<T>(i)) *
+                                   static_cast<T>(1 + i));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Mul(vn, vi));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Mul(vi, vn));
+  }
+};
+
+struct TestMulOverflow {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto vMax = Set(d, LimitsMax<T>());
+    HWY_ASSERT_VEC_EQ(d, Mul(vMax, vMax), Mul(vMax, vMax));
+  }
+};
+
+struct TestDivOverflow {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> vZero = Set(d, ConvertScalarTo<T>(0));
+    const Vec<D> v1 = Set(d, ConvertScalarTo<T>(1));
+    HWY_ASSERT_VEC_EQ(d, Div(v1, vZero), Div(v1, vZero));
+  }
+};
+
+HWY_NOINLINE void TestAllMul() {
+  ForUnsignedTypes(ForPartialVectors<TestUnsignedMul>());
+  ForSignedTypes(ForPartialVectors<TestSignedMul>());
+
+  ForSignedTypes(ForPartialVectors<TestMulOverflow>());
+
+  ForFloatTypes(ForPartialVectors<TestDivOverflow>());
+}
+
+struct TestMulHigh {
+  template <size_t kSize, class T, hwy::EnableIf<(kSize != 8)>* = nullptr>
+  static HWY_INLINE RemoveCvRef<T> ScalarMulHigh(hwy::SizeTag<kSize>, T a,
+                                                 T b) {
+    using TW = MakeWide<RemoveCvRef<T>>;
+    return static_cast<T>((static_cast<TW>(a) * static_cast<TW>(b)) >>
+                          (sizeof(RemoveCvRef<T>) * 8));
+  }
+
+  template <class T>
+  static HWY_INLINE RemoveCvRef<T> ScalarMulHigh(hwy::SizeTag<8>, T a, T b) {
+    RemoveCvRef<T> hi;
+    Mul128(a, b, &hi);
+    return hi;
+  }
+
+  template <class T>
+  static HWY_INLINE RemoveCvRef<T> ScalarMulHigh(T a, T b) {
+    using NonCvRefT = RemoveCvRef<T>;
+    return ScalarMulHigh(hwy::SizeTag<sizeof(NonCvRefT)>(),
+                         static_cast<NonCvRefT>(a), static_cast<NonCvRefT>(b));
+  }
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TU = MakeUnsigned<T>;
+    const size_t N = Lanes(d);
+    auto in_lanes = AllocateAligned<T>(N);
+    auto expected_lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(in_lanes && expected_lanes);
+
+    const Vec<D> vi = Iota(d, 1);
+    const Vec<D> vni = Iota(d, static_cast<T>(0ULL - static_cast<uint64_t>(N)));
+
+    const Vec<D> v0 = Zero(d);
+    HWY_ASSERT_VEC_EQ(d, v0, MulHigh(v0, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, MulHigh(v0, vi));
+    HWY_ASSERT_VEC_EQ(d, v0, MulHigh(vi, v0));
+
+    // Large positive squared
+    for (size_t i = 0; i < N; ++i) {
+      in_lanes[i] = static_cast<T>(LimitsMax<T>() >> (i & (sizeof(T) * 8 - 1)));
+      expected_lanes[i] = ScalarMulHigh(in_lanes[i], in_lanes[i]);
+    }
+    Vec<D> v = Load(d, in_lanes.get());
+    HWY_ASSERT_VEC_EQ(d, expected_lanes.get(), MulHigh(v, v));
+
+    // Large positive * small positive
+    for (size_t i = 0; i < N; ++i) {
+      expected_lanes[i] = ScalarMulHigh(in_lanes[i], static_cast<T>(1 + i));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected_lanes.get(), MulHigh(v, vi));
+    HWY_ASSERT_VEC_EQ(d, expected_lanes.get(), MulHigh(vi, v));
+
+    // Large positive * small negative
+    for (size_t i = 0; i < N; ++i) {
+      const T neg = static_cast<T>(static_cast<TU>(i) - static_cast<TU>(N));
+      expected_lanes[i] = ScalarMulHigh(in_lanes[i], neg);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected_lanes.get(), MulHigh(v, vni));
+    HWY_ASSERT_VEC_EQ(d, expected_lanes.get(), MulHigh(vni, v));
+  }
+};
+
+HWY_NOINLINE void TestAllMulHigh() {
+  ForPartialVectors<TestMulHigh> test;
+  ForIntegerTypes(test);
+}
+
+struct TestMulFixedPoint15 {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v0 = Zero(d);
+    HWY_ASSERT_VEC_EQ(d, v0, MulFixedPoint15(v0, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, MulFixedPoint15(v0, v0));
+
+    const size_t N = Lanes(d);
+    auto in1 = AllocateAligned<T>(N);
+    auto in2 = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(in1 && in2 && expected);
+
+    // Random inputs in each lane
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(10000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        in1[i] = ConvertScalarTo<T>(Random64(&rng) & 0xFFFF);
+        in2[i] = ConvertScalarTo<T>(Random64(&rng) & 0xFFFF);
+      }
+
+      for (size_t i = 0; i < N; ++i) {
+        // There are three ways to compute the results. x86 and Arm are defined
+        // using 32-bit multiplication results:
+        const int arm =
+            static_cast<int32_t>(2u * static_cast<uint32_t>(in1[i] * in2[i]) +
+                                 0x8000u) >>
+            16;
+        const int x86 = (((in1[i] * in2[i]) >> 14) + 1) >> 1;
+        // On other platforms, split the result into upper and lower 16 bits.
+        const auto v1 = Set(d, in1[i]);
+        const auto v2 = Set(d, in2[i]);
+        const int hi = GetLane(MulHigh(v1, v2));
+        const int lo = GetLane(Mul(v1, v2)) & 0xFFFF;
+        const int split = 2 * hi + ((lo + 0x4000) >> 15);
+        expected[i] = ConvertScalarTo<T>(arm);
+        if (in1[i] != -32768 || in2[i] != -32768) {
+          HWY_ASSERT_EQ(arm, x86);
+          HWY_ASSERT_EQ(arm, split);
+        }
+      }
+
+      const auto a = Load(d, in1.get());
+      const auto b = Load(d, in2.get());
+      HWY_ASSERT_VEC_EQ(d, expected.get(), MulFixedPoint15(a, b));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllMulFixedPoint15() {
+  ForPartialVectors<TestMulFixedPoint15>()(int16_t());
+}
+
+struct TestMulEven {
+  template <class D, HWY_IF_SIGNED_D(D)>
+  HWY_INLINE void DoTestNegMulEven(D /*d*/, Vec<D> v) {
+    using T = TFromD<D>;
+    using Wide = MakeWide<T>;
+    const Repartition<Wide, D> d2;
+
+    const auto v_squared = MulEven(v, v);
+    const auto neg_v_squared = Neg(v_squared);
+    const auto neg_v = Neg(v);
+    HWY_ASSERT_VEC_EQ(d2, v_squared, MulEven(neg_v, neg_v));
+    HWY_ASSERT_VEC_EQ(d2, neg_v_squared, MulEven(neg_v, v));
+    HWY_ASSERT_VEC_EQ(d2, neg_v_squared, MulEven(v, neg_v));
+  }
+  template <class D, HWY_IF_UNSIGNED_D(D)>
+  HWY_INLINE void DoTestNegMulEven(D /*d*/, Vec<D> /*v*/) {}
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using Wide = MakeWide<T>;
+    const Repartition<Wide, D> d2;
+    const auto v0 = Zero(d);
+    HWY_ASSERT_VEC_EQ(d2, Zero(d2), MulEven(v0, v0));
+
+    constexpr size_t kShiftAmtMask = sizeof(T) * 8 - 1;
+    const size_t N = Lanes(d);
+    auto in_lanes = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<Wide>(Lanes(d2));
+    HWY_ASSERT(in_lanes && expected);
+    for (size_t i = 0; i < N; i += 2) {
+      in_lanes[i + 0] =
+          ConvertScalarTo<T>(LimitsMax<T>() >> (i & kShiftAmtMask));
+      if (N != 1) {
+        in_lanes[i + 1] = 1;  // unused
+      }
+      expected[i / 2] =
+          static_cast<Wide>(Wide(in_lanes[i + 0]) * in_lanes[i + 0]);
+    }
+
+    const auto v = Load(d, in_lanes.get());
+    HWY_ASSERT_VEC_EQ(d2, expected.get(), MulEven(v, v));
+
+    DoTestNegMulEven(d, v);
+  }
+};
+
+struct TestMulOdd {
+  template <class D, HWY_IF_SIGNED_D(D)>
+  HWY_INLINE void DoTestNegMulOdd(D d, Vec<D> v) {
+    using T = TFromD<D>;
+    using Wide = MakeWide<T>;
+    const Repartition<Wide, D> d2;
+
+    const auto v_squared = MulOdd(v, v);
+    const auto neg_v_squared = Neg(v_squared);
+    const auto neg_v = Neg(v);
+    HWY_ASSERT_VEC_EQ(d2, v_squared, MulOdd(neg_v, neg_v));
+    HWY_ASSERT_VEC_EQ(d2, neg_v_squared, MulOdd(neg_v, v));
+    HWY_ASSERT_VEC_EQ(d2, neg_v_squared, MulOdd(v, neg_v));
+    HWY_ASSERT_VEC_EQ(d2, neg_v_squared, MulEven(DupOdd(v), DupOdd(neg_v)));
+    HWY_ASSERT_VEC_EQ(d2, neg_v_squared,
+                      MulEven(Reverse2(d, v), Reverse2(d, neg_v)));
+  }
+  template <class D, HWY_IF_UNSIGNED_D(D)>
+  HWY_INLINE void DoTestNegMulOdd(D /*d*/, Vec<D> /*v*/) {}
+
+  template <typename T, class D, HWY_IF_LANES_GT_D(D, 1)>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    const size_t N = Lanes(d);
+    if (N < 2) return;
+
+    using Wide = MakeWide<T>;
+    const Repartition<Wide, D> d2;
+    const auto v0 = Zero(d);
+    HWY_ASSERT_VEC_EQ(d2, Zero(d2), MulOdd(v0, v0));
+
+    constexpr size_t kShiftAmtMask = sizeof(T) * 8 - 1;
+    auto in_lanes = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<Wide>(Lanes(d2));
+    HWY_ASSERT(in_lanes && expected);
+    for (size_t i = 0; i < N; i += 2) {
+      in_lanes[i + 0] = 1;  // unused
+      in_lanes[i + 1] =
+          ConvertScalarTo<T>(LimitsMax<T>() >> (i & kShiftAmtMask));
+      expected[i / 2] =
+          static_cast<Wide>(Wide(in_lanes[i + 1]) * in_lanes[i + 1]);
+    }
+
+    const auto v = Load(d, in_lanes.get());
+    HWY_ASSERT_VEC_EQ(d2, expected.get(), MulOdd(v, v));
+
+    const auto v_dupodd = DupOdd(v);
+    HWY_ASSERT_VEC_EQ(d2, expected.get(), MulEven(v_dupodd, v_dupodd));
+    HWY_ASSERT_VEC_EQ(d2, expected.get(), MulOdd(v_dupodd, v_dupodd));
+    HWY_ASSERT_VEC_EQ(d2, expected.get(), MulOdd(v_dupodd, v));
+    HWY_ASSERT_VEC_EQ(d2, expected.get(), MulOdd(v, v_dupodd));
+
+    const auto v_reverse2 = Reverse2(d, v);
+    HWY_ASSERT_VEC_EQ(d2, expected.get(), MulEven(v_reverse2, v_reverse2));
+
+    DoTestNegMulOdd(d, v);
+#else
+    (void)d;
+#endif
+  }
+  template <typename T, class D, HWY_IF_LANES_LE_D(D, 1)>
+  HWY_INLINE void operator()(T /*unused*/, D /*d*/) {}
+};
+
+#if HWY_HAVE_INTEGER64 && HWY_TARGET != HWY_SCALAR
+struct TestMulEvenOdd64 {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v0 = Zero(d);
+    HWY_ASSERT_VEC_EQ(d, Zero(d), MulEven(v0, v0));
+    HWY_ASSERT_VEC_EQ(d, Zero(d), MulOdd(v0, v0));
+
+    const size_t N = Lanes(d);
+    if (N == 1) return;
+
+    auto in1 = AllocateAligned<T>(N);
+    auto in2 = AllocateAligned<T>(N);
+    auto expected_even = AllocateAligned<T>(N);
+    auto expected_odd = AllocateAligned<T>(N);
+    HWY_ASSERT(in1 && in2 && expected_even && expected_odd);
+
+    // Random inputs in each lane
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        in1[i] = static_cast<T>(Random64(&rng));
+        in2[i] = static_cast<T>(Random64(&rng));
+      }
+
+      for (size_t i = 0; i < N; i += 2) {
+        expected_even[i] = Mul128(in1[i], in2[i], &expected_even[i + 1]);
+        expected_odd[i] = Mul128(in1[i + 1], in2[i + 1], &expected_odd[i + 1]);
+      }
+
+      const auto a = Load(d, in1.get());
+      const auto b = Load(d, in2.get());
+      HWY_ASSERT_VEC_EQ(d, expected_even.get(), MulEven(a, b));
+      HWY_ASSERT_VEC_EQ(d, expected_odd.get(), MulOdd(a, b));
+    }
+  }
+};
+#endif  // HWY_HAVE_INTEGER64 && HWY_TARGET != HWY_SCALAR
+
+HWY_NOINLINE void TestAllMulEven() {
+  ForUI8(ForGEVectors<16, TestMulEven>());
+  ForUI16(ForGEVectors<32, TestMulEven>());
+
+#if HWY_HAVE_INTEGER64
+  ForUI32(ForGEVectors<64, TestMulEven>());
+#if HWY_TARGET != HWY_SCALAR
+  ForGEVectors<128, TestMulEvenOdd64>()(int64_t());
+  ForGEVectors<128, TestMulEvenOdd64>()(uint64_t());
+#endif  // HWY_TARGET != HWY_SCALAR
+#endif  // HWY_HAVE_INTEGER64
+}
+
+HWY_NOINLINE void TestAllMulOdd() {
+  ForUI8(ForGEVectors<16, TestMulOdd>());
+  ForUI16(ForGEVectors<32, TestMulOdd>());
+#if HWY_HAVE_INTEGER64
+  ForUI32(ForGEVectors<64, TestMulOdd>());
+#endif
+
+  // uint64_t MulOdd is already tested in TestMulEvenOdd64
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyMulTest);
+HWY_EXPORT_AND_TEST_P(HwyMulTest, TestAllMul);
+HWY_EXPORT_AND_TEST_P(HwyMulTest, TestAllMulHigh);
+HWY_EXPORT_AND_TEST_P(HwyMulTest, TestAllMulFixedPoint15);
+HWY_EXPORT_AND_TEST_P(HwyMulTest, TestAllMulEven);
+HWY_EXPORT_AND_TEST_P(HwyMulTest, TestAllMulOdd);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/reduction_test.cc

@@ -0,0 +1,374 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/reduction_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestSumOfLanes {
+  template <typename D,
+            hwy::EnableIf<!IsSigned<TFromD<D>>() ||
+                          ((HWY_MAX_LANES_D(D) & 1) != 0)>* = nullptr>
+  HWY_NOINLINE void SignedEvenLengthVectorTests(D /*d*/) {
+    // do nothing
+  }
+  template <typename D,
+            hwy::EnableIf<IsSigned<TFromD<D>>() &&
+                          ((HWY_MAX_LANES_D(D) & 1) == 0)>* = nullptr>
+  HWY_NOINLINE void SignedEvenLengthVectorTests(D d) {
+    using T = TFromD<D>;
+
+    const size_t lanes = Lanes(d);
+
+#if HWY_HAVE_SCALABLE
+    // On platforms that use scalable vectors, it is possible for Lanes(d) to be
+    // odd but for MaxLanes(d) to be even if Lanes(d) < 2 is true.
+    if (lanes < 2) return;
+#endif
+
+    const T pairs = ConvertScalarTo<T>(lanes / 2);
+
+    // Lanes are the repeated sequence -2, 1, [...]; each pair sums to -1,
+    // so the eventual total is just -(N/2).
+    Vec<decltype(d)> v = InterleaveLower(Set(d, ConvertScalarTo<T>(-2)),
+                                         Set(d, ConvertScalarTo<T>(1)));
+    HWY_ASSERT_VEC_EQ(d, Set(d, ConvertScalarTo<T>(-pairs)), SumOfLanes(d, v));
+    HWY_ASSERT_EQ(ConvertScalarTo<T>(-pairs), ReduceSum(d, v));
+
+    // Similar test with a positive result.
+    v = InterleaveLower(Set(d, ConvertScalarTo<T>(-2)),
+                        Set(d, ConvertScalarTo<T>(4)));
+    HWY_ASSERT_VEC_EQ(d,
+                      Set(d, ConvertScalarTo<T>(pairs * ConvertScalarTo<T>(2))),
+                      SumOfLanes(d, v));
+    HWY_ASSERT_EQ(ConvertScalarTo<T>(pairs * ConvertScalarTo<T>(2)),
+                  ReduceSum(d, v));
+  }
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto in_lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(in_lanes);
+
+    // Lane i = bit i, higher lanes 0
+    T sum = ConvertScalarTo<T>(0);
+    // Avoid setting sign bit and cap so that f16 precision is not exceeded.
+    constexpr size_t kBits = HWY_MIN(sizeof(T) * 8 - 1, 9);
+    for (size_t i = 0; i < N; ++i) {
+      in_lanes[i] = ConvertScalarTo<T>(i < kBits ? 1ull << i : 0ull);
+      sum = AddWithWraparound(sum, in_lanes[i]);
+    }
+    HWY_ASSERT_VEC_EQ(d, Set(d, sum), SumOfLanes(d, Load(d, in_lanes.get())));
+    HWY_ASSERT_EQ(T(sum), ReduceSum(d, Load(d, in_lanes.get())));
+    // Lane i = i (iota) to include upper lanes
+    sum = ConvertScalarTo<T>(0);
+    for (size_t i = 0; i < N; ++i) {
+      sum = AddWithWraparound(sum, ConvertScalarTo<T>(i));
+    }
+    HWY_ASSERT_VEC_EQ(d, Set(d, sum), SumOfLanes(d, Iota(d, 0)));
+    HWY_ASSERT_EQ(T(sum), ReduceSum(d, Iota(d, 0)));
+
+    // Run more tests only for signed types with even vector lengths. Some of
+    // this code may not otherwise compile, so put it in a templated function.
+    SignedEvenLengthVectorTests(d);
+  }
+};
+
+HWY_NOINLINE void TestAllSumOfLanes() {
+  ForAllTypes(ForPartialVectors<TestSumOfLanes>());
+}
+
+struct TestMinOfLanes {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto in_lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(in_lanes);
+
+    // Lane i = bit i, higher lanes = 2 (not the minimum)
+    T min = HighestValue<T>();
+    // Avoid setting sign bit and cap at double precision
+    constexpr size_t kBits = HWY_MIN(sizeof(T) * 8 - 1, 51);
+    for (size_t i = 0; i < N; ++i) {
+      in_lanes[i] = ConvertScalarTo<T>(i < kBits ? 1ull << i : 2ull);
+      min = HWY_MIN(min, in_lanes[i]);
+    }
+    HWY_ASSERT_VEC_EQ(d, Set(d, min), MinOfLanes(d, Load(d, in_lanes.get())));
+
+    // Lane i = N - i to include upper lanes
+    min = HighestValue<T>();
+    for (size_t i = 0; i < N; ++i) {
+      in_lanes[i] = ConvertScalarTo<T>(N - i);  // no 8-bit T so no wraparound
+      min = HWY_MIN(min, in_lanes[i]);
+    }
+    HWY_ASSERT_VEC_EQ(d, Set(d, min), MinOfLanes(d, Load(d, in_lanes.get())));
+
+    // Bug #910: also check negative values
+    min = HighestValue<T>();
+    const T input_copy[] = {ConvertScalarTo<T>(-1),
+                            ConvertScalarTo<T>(-2),
+                            1,
+                            2,
+                            3,
+                            4,
+                            5,
+                            6,
+                            7,
+                            8,
+                            9,
+                            10,
+                            11,
+                            12,
+                            13,
+                            14};
+    size_t i = 0;
+    for (; i < HWY_MIN(N, sizeof(input_copy) / sizeof(T)); ++i) {
+      in_lanes[i] = input_copy[i];
+      min = HWY_MIN(min, input_copy[i]);
+    }
+    // Pad with neutral element to full vector (so we can load)
+    for (; i < N; ++i) {
+      in_lanes[i] = min;
+    }
+    HWY_ASSERT_VEC_EQ(d, Set(d, min), MinOfLanes(d, Load(d, in_lanes.get())));
+    HWY_ASSERT_EQ(min, ReduceMin(d, Load(d, in_lanes.get())));
+  }
+};
+
+struct TestMaxOfLanes {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto in_lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(in_lanes);
+
+    T max = LowestValue<T>();
+    // Avoid setting sign bit and cap at double precision
+    constexpr size_t kBits = HWY_MIN(sizeof(T) * 8 - 1, 51);
+    for (size_t i = 0; i < N; ++i) {
+      in_lanes[i] = ConvertScalarTo<T>(i < kBits ? 1ull << i : 0ull);
+      max = HWY_MAX(max, in_lanes[i]);
+    }
+    HWY_ASSERT_VEC_EQ(d, Set(d, max), MaxOfLanes(d, Load(d, in_lanes.get())));
+
+    // Lane i = i to include upper lanes
+    max = LowestValue<T>();
+    for (size_t i = 0; i < N; ++i) {
+      in_lanes[i] = ConvertScalarTo<T>(i);  // no 8-bit T so no wraparound
+      max = HWY_MAX(max, in_lanes[i]);
+    }
+    HWY_ASSERT_VEC_EQ(d, Set(d, max), MaxOfLanes(d, Load(d, in_lanes.get())));
+
+    // Bug #910: also check negative values
+    max = LowestValue<T>();
+    const T input_copy[] = {ConvertScalarTo<T>(-1),
+                            ConvertScalarTo<T>(-2),
+                            1,
+                            2,
+                            3,
+                            4,
+                            5,
+                            6,
+                            7,
+                            8,
+                            9,
+                            10,
+                            11,
+                            12,
+                            13,
+                            14};
+    size_t i = 0;
+    for (; i < HWY_MIN(N, sizeof(input_copy) / sizeof(T)); ++i) {
+      in_lanes[i] = input_copy[i];
+      max = HWY_MAX(max, in_lanes[i]);
+    }
+    // Pad with neutral element to full vector (so we can load)
+    for (; i < N; ++i) {
+      in_lanes[i] = max;
+    }
+    HWY_ASSERT_VEC_EQ(d, Set(d, max), MaxOfLanes(d, Load(d, in_lanes.get())));
+    HWY_ASSERT_EQ(max, ReduceMax(d, Load(d, in_lanes.get())));
+  }
+};
+
+HWY_NOINLINE void TestAllMinMaxOfLanes() {
+  ForAllTypes(ForPartialVectors<TestMinOfLanes>());
+  ForAllTypes(ForPartialVectors<TestMaxOfLanes>());
+}
+
+struct TestSumsOf2 {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+    using TW = MakeWide<T>;
+
+    const size_t N = Lanes(d);
+    if (N < 2) return;
+    const RepartitionToWide<D> dw;
+
+    auto in_lanes = AllocateAligned<T>(N);
+    auto sum_lanes = AllocateAligned<TW>(N / 2);
+    HWY_ASSERT(in_lanes && sum_lanes);
+
+    for (size_t rep = 0; rep < 100; ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        in_lanes[i] = RandomFiniteValue<T>(&rng);
+      }
+
+      for (size_t idx_sum = 0; idx_sum < N / 2; ++idx_sum) {
+        TW sum = static_cast<TW>(static_cast<TW>(in_lanes[idx_sum * 2]) +
+                                 static_cast<TW>(in_lanes[idx_sum * 2 + 1]));
+        sum_lanes[idx_sum] = sum;
+      }
+
+      const Vec<D> in = Load(d, in_lanes.get());
+      HWY_ASSERT_VEC_EQ(dw, sum_lanes.get(), SumsOf2(in));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllSumsOf2() {
+  ForGEVectors<16, TestSumsOf2>()(int8_t());
+  ForGEVectors<16, TestSumsOf2>()(uint8_t());
+
+  ForGEVectors<32, TestSumsOf2>()(int16_t());
+  ForGEVectors<32, TestSumsOf2>()(uint16_t());
+#if HWY_HAVE_FLOAT16
+  ForGEVectors<32, TestSumsOf2>()(float16_t());
+#endif
+
+#if HWY_HAVE_INTEGER64
+  ForGEVectors<64, TestSumsOf2>()(int32_t());
+  ForGEVectors<64, TestSumsOf2>()(uint32_t());
+#endif
+
+#if HWY_HAVE_FLOAT64
+  ForGEVectors<64, TestSumsOf2>()(float());
+#endif
+}
+
+struct TestSumsOf4 {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+    using TW = MakeWide<T>;
+    using TW2 = MakeWide<TW>;
+
+    const size_t N = Lanes(d);
+    if (N < 4) return;
+    const Repartition<TW2, D> dw2;
+
+    auto in_lanes = AllocateAligned<T>(N);
+    auto sum_lanes = AllocateAligned<TW2>(N / 4);
+    HWY_ASSERT(in_lanes && sum_lanes);
+
+    for (size_t rep = 0; rep < 100; ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        in_lanes[i] = RandomFiniteValue<T>(&rng);
+      }
+
+      for (size_t idx_sum = 0; idx_sum < N / 4; ++idx_sum) {
+        TW2 sum = static_cast<TW2>(static_cast<TW>(in_lanes[idx_sum * 4]) +
+                                   static_cast<TW>(in_lanes[idx_sum * 4 + 1]) +
+                                   static_cast<TW>(in_lanes[idx_sum * 4 + 2]) +
+                                   static_cast<TW>(in_lanes[idx_sum * 4 + 3]));
+        sum_lanes[idx_sum] = sum;
+      }
+
+      const Vec<D> in = Load(d, in_lanes.get());
+      HWY_ASSERT_VEC_EQ(dw2, sum_lanes.get(), SumsOf4(in));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllSumsOf4() {
+  ForGEVectors<32, TestSumsOf4>()(int8_t());
+  ForGEVectors<32, TestSumsOf4>()(uint8_t());
+
+#if HWY_HAVE_INTEGER64
+  ForGEVectors<64, TestSumsOf4>()(int16_t());
+  ForGEVectors<64, TestSumsOf4>()(uint16_t());
+#endif
+}
+
+struct TestSumsOf8 {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+    using TW = MakeWide<MakeWide<MakeWide<T>>>;
+
+    const size_t N = Lanes(d);
+    if (N < 8) return;
+    const Repartition<TW, D> d64;
+
+    auto in_lanes = AllocateAligned<T>(N);
+    auto sum_lanes = AllocateAligned<TW>(N / 8);
+    HWY_ASSERT(in_lanes && sum_lanes);
+
+    for (size_t rep = 0; rep < 100; ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        in_lanes[i] = ConvertScalarTo<T>(Random64(&rng) & 0xFF);
+      }
+
+      for (size_t idx_sum = 0; idx_sum < N / 8; ++idx_sum) {
+        TW sum = 0;
+        for (size_t i = 0; i < 8; ++i) {
+          sum += in_lanes[idx_sum * 8 + i];
+        }
+        sum_lanes[idx_sum] = sum;
+      }
+
+      const Vec<D> in = Load(d, in_lanes.get());
+      HWY_ASSERT_VEC_EQ(d64, sum_lanes.get(), SumsOf8(in));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllSumsOf8() {
+  ForGEVectors<64, TestSumsOf8>()(int8_t());
+  ForGEVectors<64, TestSumsOf8>()(uint8_t());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyReductionTest);
+HWY_EXPORT_AND_TEST_P(HwyReductionTest, TestAllSumOfLanes);
+HWY_EXPORT_AND_TEST_P(HwyReductionTest, TestAllMinMaxOfLanes);
+HWY_EXPORT_AND_TEST_P(HwyReductionTest, TestAllSumsOf2);
+HWY_EXPORT_AND_TEST_P(HwyReductionTest, TestAllSumsOf4);
+HWY_EXPORT_AND_TEST_P(HwyReductionTest, TestAllSumsOf8);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/resize_test.cc

@@ -0,0 +1,188 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/resize_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+#if HWY_TARGET != HWY_SCALAR
+
+template <class DTo, class DFrom>
+HWY_INLINE void DoTruncResizeBitCastTest(DTo d_to, DFrom d_from) {
+  if (Lanes(d_to) == 0) return;
+
+  const VFromD<DFrom> v = Iota(d_from, 1);
+  const VFromD<DTo> expected = Iota(d_to, 1);
+
+  const VFromD<DTo> actual_1 = ResizeBitCast(d_to, v);
+  HWY_ASSERT_VEC_EQ(d_to, expected, actual_1);
+
+  const VFromD<DTo> actual_2 = ZeroExtendResizeBitCast(d_to, d_from, v);
+  HWY_ASSERT_VEC_EQ(d_to, expected, actual_2);
+}
+
+struct TestTruncatingResizeBitCastHalf {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Half<D> dh;
+    DoTruncResizeBitCastTest(dh, d);
+
+    const auto v_full = Iota(d, 1);
+    const VFromD<decltype(dh)> expected_full_to_half = LowerHalf(dh, v_full);
+    HWY_ASSERT_VEC_EQ(dh, expected_full_to_half, ResizeBitCast(dh, v_full));
+    HWY_ASSERT_VEC_EQ(dh, expected_full_to_half,
+                      ZeroExtendResizeBitCast(dh, d, v_full));
+  }
+};
+
+struct TestTruncatingResizeBitCastQuarter {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Half<Half<decltype(d)>> d_quarter;
+    if (MaxLanes(d_quarter) == MaxLanes(d) / 4) {
+      DoTruncResizeBitCastTest(d_quarter, d);
+    }
+  }
+};
+
+struct TestTruncatingResizeBitCastEighth {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Half<Half<Half<decltype(d)>>> d_eighth;
+    if (MaxLanes(d_eighth) == MaxLanes(d) / 8) {
+      DoTruncResizeBitCastTest(d_eighth, d);
+    }
+  }
+};
+
+#endif  // HWY_TARGET != HWY_SCALAR
+
+HWY_NOINLINE void TestAllTruncatingResizeBitCast() {
+#if HWY_TARGET != HWY_SCALAR
+  ForAllTypes(ForShrinkableVectors<TestTruncatingResizeBitCastHalf, 1>());
+  ForAllTypes(ForShrinkableVectors<TestTruncatingResizeBitCastQuarter, 2>());
+  ForAllTypes(ForShrinkableVectors<TestTruncatingResizeBitCastEighth, 3>());
+#endif
+}
+
+class TestExtendingResizeBitCast {
+#if HWY_TARGET != HWY_SCALAR
+
+ private:
+  template <class DTo, class DFrom>
+  static HWY_INLINE void DoExtResizeBitCastTest(DTo d_to, DFrom d_from) {
+    const size_t N = Lanes(d_from);
+    const auto active_elements_mask = FirstN(d_to, N);
+
+    const VFromD<DTo> expected =
+        IfThenElseZero(active_elements_mask, Iota(d_to, 1));
+    const VFromD<DFrom> v = Iota(d_from, 1);
+
+    const VFromD<DTo> actual_1 = ResizeBitCast(d_to, v);
+    const VFromD<DTo> actual_2 = ZeroExtendResizeBitCast(d_to, d_from, v);
+
+    HWY_ASSERT_VEC_EQ(d_to, expected,
+                      IfThenElseZero(active_elements_mask, actual_1));
+    HWY_ASSERT_VEC_EQ(d_to, expected, actual_2);
+  }
+
+  template <class DFrom>
+  static HWY_INLINE void DoExtResizeBitCastToTwiceDTest(DFrom d_from) {
+    using DTo = Twice<DFrom>;
+    const DTo d_to;
+    DoExtResizeBitCastTest(d_to, d_from);
+
+    const VFromD<DFrom> v = Iota(d_from, 1);
+    const VFromD<DTo> expected = ZeroExtendVector(d_to, v);
+
+    const VFromD<DTo> actual_1 = ResizeBitCast(d_to, v);
+    const VFromD<DTo> actual_2 = ZeroExtendResizeBitCast(d_to, d_from, v);
+
+    HWY_ASSERT_VEC_EQ(d_from, v, LowerHalf(d_from, actual_1));
+    HWY_ASSERT_VEC_EQ(d_from, v, LowerHalf(d_from, actual_2));
+    HWY_ASSERT_VEC_EQ(d_to, expected, actual_2);
+  }
+#endif  // HWY_TARGET != HWY_SCALAR
+
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    DoExtResizeBitCastToTwiceDTest(d);
+
+    constexpr size_t kMaxLanes = MaxLanes(d);
+#if HWY_TARGET == HWY_RVV
+    constexpr int kFromVectPow2 = DFromV<VFromD<D>>().Pow2();
+    static_assert(kFromVectPow2 >= -3 && kFromVectPow2 <= 3,
+                  "kFromVectPow2 must be between -3 and 3");
+
+    constexpr size_t kScaledMaxLanes =
+        HWY_MAX((kMaxLanes << 3) >> (kFromVectPow2 + 3), 1);
+    constexpr size_t kQuadrupleScaledLimit = kScaledMaxLanes;
+    constexpr size_t kOctupleScaledLimit = kScaledMaxLanes;
+    constexpr int kQuadruplePow2 = HWY_MIN(kFromVectPow2 + 2, 3);
+    constexpr int kOctuplePow2 = HWY_MIN(kFromVectPow2 + 3, 3);
+#else
+    constexpr size_t kQuadrupleScaledLimit = kMaxLanes * 4;
+    constexpr size_t kOctupleScaledLimit = kMaxLanes * 8;
+    constexpr int kQuadruplePow2 = 0;
+    constexpr int kOctuplePow2 = 0;
+#endif
+
+    const CappedTag<T, kQuadrupleScaledLimit, kQuadruplePow2> d_quadruple;
+    const CappedTag<T, kOctupleScaledLimit, kOctuplePow2> d_octuple;
+
+    if (MaxLanes(d_quadruple) == kMaxLanes * 4) {
+      DoExtResizeBitCastTest(d_quadruple, d);
+      if (MaxLanes(d_octuple) == kMaxLanes * 8) {
+        DoExtResizeBitCastTest(d_octuple, d);
+      }
+    }
+#else
+    (void)d;
+#endif  // HWY_TARGET != HWY_SCALAR
+  }
+};
+
+HWY_NOINLINE void TestAllExtendingResizeBitCast() {
+  ForAllTypes(ForExtendableVectors<TestExtendingResizeBitCast>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyResizeTest);
+HWY_EXPORT_AND_TEST_P(HwyResizeTest, TestAllTruncatingResizeBitCast);
+HWY_EXPORT_AND_TEST_P(HwyResizeTest, TestAllExtendingResizeBitCast);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/reverse_test.cc

@@ -0,0 +1,309 @@
+// Copyright 2022 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/reverse_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestReverse {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    const RebindToUnsigned<D> du;  // Iota does not support float16_t.
+    const auto v = BitCast(d, Iota(du, 1));
+    auto expected = AllocateAligned<T>(N);
+    auto copy = AllocateAligned<T>(N);
+    HWY_ASSERT(expected && copy);
+
+    // Can't set float16_t value directly, need to permute in memory.
+    Store(v, d, copy.get());
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = copy[N - 1 - i];
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Reverse(d, v));
+  }
+};
+
+struct TestReverse2 {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    const RebindToUnsigned<D> du;  // Iota does not support float16_t.
+    const auto v = BitCast(d, Iota(du, 1));
+    auto expected = AllocateAligned<T>(N);
+    auto copy = AllocateAligned<T>(N);
+    HWY_ASSERT(expected && copy);
+    if (N == 1) {
+      Store(v, d, expected.get());
+      HWY_ASSERT_VEC_EQ(d, expected.get(), Reverse2(d, v));
+      return;
+    }
+
+    // Can't set float16_t value directly, need to permute in memory.
+    Store(v, d, copy.get());
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = copy[i ^ 1];
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Reverse2(d, v));
+  }
+};
+
+struct TestReverse4 {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    const RebindToUnsigned<D> du;  // Iota does not support float16_t.
+    const auto v = BitCast(d, Iota(du, 1));
+    auto expected = AllocateAligned<T>(N);
+    auto copy = AllocateAligned<T>(N);
+    HWY_ASSERT(expected && copy);
+
+    // Can't set float16_t value directly, need to permute in memory.
+    Store(v, d, copy.get());
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = copy[i ^ 3];
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Reverse4(d, v));
+  }
+};
+
+struct TestReverse8 {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    const RebindToUnsigned<D> du;  // Iota does not support float16_t.
+    const auto v = BitCast(d, Iota(du, 1));
+    auto expected = AllocateAligned<T>(N);
+    auto copy = AllocateAligned<T>(N);
+    HWY_ASSERT(expected && copy);
+
+    // Can't set float16_t value directly, need to permute in memory.
+    Store(v, d, copy.get());
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = copy[i ^ 7];
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Reverse8(d, v));
+  }
+};
+
+static HWY_INLINE uint8_t ReverseBytesOfValue(uint8_t val) { return val; }
+
+static HWY_INLINE uint16_t ReverseBytesOfValue(uint16_t val) {
+  const uint32_t u32_val = val;
+  return static_cast<uint16_t>(((u32_val << 8) & 0xFF00u) |
+                               ((u32_val >> 8) & 0x00FFu));
+}
+
+static HWY_INLINE uint32_t ReverseBytesOfValue(uint32_t val) {
+  return static_cast<uint32_t>(
+      ((val << 24) & 0xFF000000u) | ((val << 8) & 0x00FF0000u) |
+      ((val >> 8) & 0x0000FF00u) | ((val >> 24) & 0x000000FFu));
+}
+
+static HWY_INLINE uint64_t ReverseBytesOfValue(uint64_t val) {
+  return static_cast<uint64_t>(
+      ((val << 56) & 0xFF00000000000000u) |
+      ((val << 40) & 0x00FF000000000000u) |
+      ((val << 24) & 0x0000FF0000000000u) | ((val << 8) & 0x000000FF00000000u) |
+      ((val >> 8) & 0x00000000FF000000u) | ((val >> 24) & 0x0000000000FF0000u) |
+      ((val >> 40) & 0x000000000000FF00u) |
+      ((val >> 56) & 0x00000000000000FFu));
+}
+
+template <class T, HWY_IF_SIGNED(T)>
+static HWY_INLINE T ReverseBytesOfValue(T val) {
+  using TU = MakeUnsigned<T>;
+  return static_cast<T>(ReverseBytesOfValue(static_cast<TU>(val)));
+}
+
+struct TestReverseLaneBytes {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto in = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(in && expected);
+
+    const auto v_iota = Iota(d, 0);
+    for (size_t i = 0; i < N; i++) {
+      expected[i] = ReverseBytesOfValue(ConvertScalarTo<T>(i));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ReverseLaneBytes(v_iota));
+
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(10000); ++rep) {
+      for (size_t i = 0; i < N; i++) {
+        in[i] = ConvertScalarTo<T>(Random64(&rng));
+        expected[i] = ReverseBytesOfValue(in[i]);
+      }
+
+      const auto v = Load(d, in.get());
+      HWY_ASSERT_VEC_EQ(d, expected.get(), ReverseLaneBytes(v));
+    }
+  }
+};
+
+class TestReverseBits {
+ private:
+  template <class T>
+  static HWY_INLINE T ReverseBitsOfEachByte(T val) {
+    using TU = MakeUnsigned<T>;
+    constexpr TU kMaxUnsignedVal{LimitsMax<TU>()};
+    constexpr TU kShrMask1 =
+        static_cast<TU>(0x5555555555555555u & kMaxUnsignedVal);
+    constexpr TU kShrMask2 =
+        static_cast<TU>(0x3333333333333333u & kMaxUnsignedVal);
+    constexpr TU kShrMask3 =
+        static_cast<TU>(0x0F0F0F0F0F0F0F0Fu & kMaxUnsignedVal);
+
+    constexpr TU kShlMask1 = static_cast<TU>(~kShrMask1);
+    constexpr TU kShlMask2 = static_cast<TU>(~kShrMask2);
+    constexpr TU kShlMask3 = static_cast<TU>(~kShrMask3);
+
+    TU result = static_cast<TU>(val);
+    result = static_cast<TU>(((result << 1) & kShlMask1) |
+                             ((result >> 1) & kShrMask1));
+    result = static_cast<TU>(((result << 2) & kShlMask2) |
+                             ((result >> 2) & kShrMask2));
+    result = static_cast<TU>(((result << 4) & kShlMask3) |
+                             ((result >> 4) & kShrMask3));
+    return static_cast<T>(result);
+  }
+
+  template <class T>
+  static HWY_INLINE T ReverseBitsOfValue(T val) {
+    return ReverseBytesOfValue(ReverseBitsOfEachByte(val));
+  }
+
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto in = AllocateAligned<T>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(in && expected);
+
+    const auto v_iota = Iota(d, 0);
+    for (size_t i = 0; i < N; i++) {
+      expected[i] = ReverseBitsOfValue(ConvertScalarTo<T>(i));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ReverseBits(v_iota));
+
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(10000); ++rep) {
+      for (size_t i = 0; i < N; i++) {
+        in[i] = ConvertScalarTo<T>(Random64(&rng));
+        expected[i] = ReverseBitsOfValue(in[i]);
+      }
+
+      const auto v = Load(d, in.get());
+      HWY_ASSERT_VEC_EQ(d, expected.get(), ReverseBits(v));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllReverse() {
+  ForAllTypes(ForPartialVectors<TestReverse>());
+}
+
+HWY_NOINLINE void TestAllReverse2() {
+  ForUIF64(ForGEVectors<128, TestReverse2>());
+  ForUIF32(ForGEVectors<64, TestReverse2>());
+  ForUIF16(ForGEVectors<32, TestReverse2>());
+  ForUI8(ForGEVectors<16, TestReverse2>());
+}
+
+HWY_NOINLINE void TestAllReverse4() {
+  ForUIF64(ForGEVectors<256, TestReverse4>());
+  ForUIF32(ForGEVectors<128, TestReverse4>());
+  ForUIF16(ForGEVectors<64, TestReverse4>());
+  ForUI8(ForGEVectors<32, TestReverse4>());
+}
+
+HWY_NOINLINE void TestAllReverse8() {
+  ForUIF64(ForGEVectors<512, TestReverse8>());
+  ForUIF32(ForGEVectors<256, TestReverse8>());
+  ForUIF16(ForGEVectors<128, TestReverse8>());
+  ForUI8(ForGEVectors<64, TestReverse8>());
+}
+
+HWY_NOINLINE void TestAllReverseLaneBytes() {
+  ForUI163264(ForPartialVectors<TestReverseLaneBytes>());
+}
+
+HWY_NOINLINE void TestAllReverseBits() {
+  ForIntegerTypes(ForPartialVectors<TestReverseBits>());
+}
+
+struct TestReverseBlocks {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    const RebindToUnsigned<D> du;  // Iota does not support float16_t.
+    const auto v = BitCast(d, Iota(du, 1));
+    auto expected = AllocateAligned<T>(N);
+    auto copy = AllocateAligned<T>(N);
+    HWY_ASSERT(expected && copy);
+
+    constexpr size_t kLanesPerBlock = 16 / sizeof(T);
+    const size_t num_blocks = N / kLanesPerBlock;
+    HWY_ASSERT(num_blocks != 0);
+
+    // Can't set float16_t value directly, need to permute in memory.
+    Store(v, d, copy.get());
+    for (size_t i = 0; i < N; ++i) {
+      const size_t idx_block = i / kLanesPerBlock;
+      const size_t base = (num_blocks - 1 - idx_block) * kLanesPerBlock;
+      expected[i] = copy[base + (i % kLanesPerBlock)];
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ReverseBlocks(d, v));
+  }
+};
+
+HWY_NOINLINE void TestAllReverseBlocks() {
+  ForAllTypes(ForGEVectors<128, TestReverseBlocks>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyReverseTest);
+HWY_EXPORT_AND_TEST_P(HwyReverseTest, TestAllReverse);
+HWY_EXPORT_AND_TEST_P(HwyReverseTest, TestAllReverse2);
+HWY_EXPORT_AND_TEST_P(HwyReverseTest, TestAllReverse4);
+HWY_EXPORT_AND_TEST_P(HwyReverseTest, TestAllReverse8);
+HWY_EXPORT_AND_TEST_P(HwyReverseTest, TestAllReverseLaneBytes);
+HWY_EXPORT_AND_TEST_P(HwyReverseTest, TestAllReverseBits);
+HWY_EXPORT_AND_TEST_P(HwyReverseTest, TestAllReverseBlocks);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/rotate_test.cc

@@ -0,0 +1,298 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/rotate_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestRotateLeft {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TU = MakeUnsigned<T>;
+
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    constexpr size_t kBits = sizeof(T) * 8;
+    const Vec<D> mask_shift = Set(d, static_cast<T>(kBits - 1));
+    // Cover as many bit positions as possible to test shifting out
+    const Vec<D> values =
+        Shl(Set(d, static_cast<T>(1)), And(Iota(d, 0), mask_shift));
+    const Vec<D> values2 = Xor(values, SignBit(d));
+
+    // Rotate by 0
+    HWY_ASSERT_VEC_EQ(d, values, RotateLeft<0>(values));
+    HWY_ASSERT_VEC_EQ(d, values2, RotateLeft<0>(values2));
+
+    // Rotate by 1
+    Store(values, d, expected.get());
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] =
+          ConvertScalarTo<T>((static_cast<TU>(expected[i]) << 1) |
+                             (static_cast<TU>(expected[i]) >> (kBits - 1)));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), RotateLeft<1>(values));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>(expected[i] ^ static_cast<T>(1));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), RotateLeft<1>(values2));
+
+    // Rotate by half
+    Store(values, d, expected.get());
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] =
+          ConvertScalarTo<T>((static_cast<TU>(expected[i]) << (kBits / 2)) |
+                             (static_cast<TU>(expected[i]) >> (kBits / 2)));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), RotateLeft<kBits / 2>(values));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>(
+          expected[i] ^ (static_cast<T>(1) << ((kBits / 2) - 1)));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), RotateLeft<kBits / 2>(values2));
+
+    // Rotate by max
+    Store(values, d, expected.get());
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] =
+          ConvertScalarTo<T>((static_cast<TU>(expected[i]) << (kBits - 1)) |
+                             (static_cast<TU>(expected[i]) >> 1));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), RotateLeft<kBits - 1>(values));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] =
+          ConvertScalarTo<T>(expected[i] ^ (static_cast<T>(1) << (kBits - 2)));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), RotateLeft<kBits - 1>(values2));
+  }
+};
+
+HWY_NOINLINE void TestAllRotateLeft() {
+  ForIntegerTypes(ForPartialVectors<TestRotateLeft>());
+}
+
+struct TestRotateRight {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TU = MakeUnsigned<T>;
+
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    constexpr size_t kBits = sizeof(T) * 8;
+    const Vec<D> mask_shift = Set(d, static_cast<T>(kBits - 1));
+    // Cover as many bit positions as possible to test shifting out
+    const Vec<D> values =
+        Shl(Set(d, static_cast<T>(1)), And(Iota(d, 0), mask_shift));
+    const Vec<D> values2 = Xor(values, SignBit(d));
+
+    // Rotate by 0
+    HWY_ASSERT_VEC_EQ(d, values, RotateRight<0>(values));
+    HWY_ASSERT_VEC_EQ(d, values2, RotateRight<0>(values2));
+
+    // Rotate by 1
+    Store(values, d, expected.get());
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] =
+          ConvertScalarTo<T>((static_cast<TU>(expected[i]) >> 1) |
+                             (static_cast<TU>(expected[i]) << (kBits - 1)));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), RotateRight<1>(values));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] =
+          ConvertScalarTo<T>(expected[i] ^ (static_cast<T>(1) << (kBits - 2)));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), RotateRight<1>(values2));
+
+    // Rotate by half
+    Store(values, d, expected.get());
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] =
+          ConvertScalarTo<T>((static_cast<TU>(expected[i]) >> (kBits / 2)) |
+                             (static_cast<TU>(expected[i]) << (kBits / 2)));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), RotateRight<kBits / 2>(values));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>(
+          expected[i] ^ (static_cast<T>(1) << ((kBits / 2) - 1)));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), RotateRight<kBits / 2>(values2));
+
+    // Rotate by max
+    Store(values, d, expected.get());
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] =
+          ConvertScalarTo<T>((static_cast<TU>(expected[i]) >> (kBits - 1)) |
+                             (static_cast<TU>(expected[i]) << 1));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), RotateRight<kBits - 1>(values));
+
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>(expected[i] ^ static_cast<T>(1));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), RotateRight<kBits - 1>(values2));
+  }
+};
+
+HWY_NOINLINE void TestAllRotateRight() {
+  ForIntegerTypes(ForPartialVectors<TestRotateRight>());
+}
+
+struct TestVariableRotations {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TU = MakeUnsigned<T>;
+
+    constexpr TU kBits1 = static_cast<TU>(0x7C29085C41482973ULL);
+    constexpr TU kBits2 = static_cast<TU>(0xD3C8835FBD1A89BAULL);
+
+    const auto viota0 = Iota(d, 0);
+    const auto va = Xor(Set(d, static_cast<T>(kBits1)), viota0);
+    const auto vb = Xor(Set(d, static_cast<T>(kBits2)), viota0);
+
+    const size_t N = Lanes(d);
+    auto expected1 = AllocateAligned<T>(N);
+    auto expected2 = AllocateAligned<T>(N);
+    auto expected3 = AllocateAligned<T>(N);
+    auto expected4 = AllocateAligned<T>(N);
+    HWY_ASSERT(expected1 && expected2 && expected3 && expected4);
+
+    constexpr size_t kBits = sizeof(T) * 8;
+
+    auto vrotate_amt1 = viota0;
+    auto vrotate_amt2 = Sub(Set(d, static_cast<T>(kBits)), viota0);
+    auto vrotate_amt_incr = Set(d, static_cast<T>(N));
+
+    const RebindToSigned<decltype(d)> di;
+
+    for (size_t i = 0; i < kBits; i += N) {
+      for (size_t j = 0; j < N; j++) {
+        const size_t shift_amt_1 = (i + j) & (kBits - 1);
+        const size_t shift_amt_2 = (size_t{0} - shift_amt_1) & (kBits - 1);
+
+        const TU val_a = static_cast<TU>(kBits1 ^ j);
+        const TU val_b = static_cast<TU>(kBits2 ^ j);
+
+        expected1[j] =
+            static_cast<T>((val_a << shift_amt_1) | (val_a >> shift_amt_2));
+        expected2[j] =
+            static_cast<T>((val_a >> shift_amt_1) | (val_a << shift_amt_2));
+        expected3[j] =
+            static_cast<T>((val_b << shift_amt_1) | (val_b >> shift_amt_2));
+        expected4[j] =
+            static_cast<T>((val_b >> shift_amt_1) | (val_b << shift_amt_2));
+      }
+
+      const auto vrotate_amt3 = BitCast(d, Neg(BitCast(di, vrotate_amt1)));
+      const auto vrotate_amt4 = BitCast(d, Neg(BitCast(di, vrotate_amt2)));
+
+      HWY_ASSERT_VEC_EQ(d, expected1.get(), Rol(va, vrotate_amt1));
+      HWY_ASSERT_VEC_EQ(d, expected2.get(), Ror(va, vrotate_amt1));
+      HWY_ASSERT_VEC_EQ(d, expected3.get(), Rol(vb, vrotate_amt1));
+      HWY_ASSERT_VEC_EQ(d, expected4.get(), Ror(vb, vrotate_amt1));
+
+      HWY_ASSERT_VEC_EQ(d, expected1.get(), Ror(va, vrotate_amt2));
+      HWY_ASSERT_VEC_EQ(d, expected2.get(), Rol(va, vrotate_amt2));
+      HWY_ASSERT_VEC_EQ(d, expected3.get(), Ror(vb, vrotate_amt2));
+      HWY_ASSERT_VEC_EQ(d, expected4.get(), Rol(vb, vrotate_amt2));
+
+      HWY_ASSERT_VEC_EQ(d, expected1.get(), Ror(va, vrotate_amt3));
+      HWY_ASSERT_VEC_EQ(d, expected2.get(), Rol(va, vrotate_amt3));
+      HWY_ASSERT_VEC_EQ(d, expected3.get(), Ror(vb, vrotate_amt3));
+      HWY_ASSERT_VEC_EQ(d, expected4.get(), Rol(vb, vrotate_amt3));
+
+      HWY_ASSERT_VEC_EQ(d, expected1.get(), Rol(va, vrotate_amt4));
+      HWY_ASSERT_VEC_EQ(d, expected2.get(), Ror(va, vrotate_amt4));
+      HWY_ASSERT_VEC_EQ(d, expected3.get(), Rol(vb, vrotate_amt4));
+      HWY_ASSERT_VEC_EQ(d, expected4.get(), Ror(vb, vrotate_amt4));
+
+      vrotate_amt1 = Add(vrotate_amt1, vrotate_amt_incr);
+      vrotate_amt2 = Sub(vrotate_amt2, vrotate_amt_incr);
+    }
+
+    for (int i = 0; i < static_cast<int>(kBits); ++i) {
+      for (size_t j = 0; j < N; j++) {
+        const int shift_amt_2 =
+            static_cast<int>(static_cast<size_t>(-i) & (kBits - 1));
+
+        const TU val_a = static_cast<TU>(kBits1 ^ j);
+        const TU val_b = static_cast<TU>(kBits2 ^ j);
+
+        expected1[j] = static_cast<T>((val_a << i) | (val_a >> shift_amt_2));
+        expected2[j] = static_cast<T>((val_a >> i) | (val_a << shift_amt_2));
+        expected3[j] = static_cast<T>((val_b << i) | (val_b >> shift_amt_2));
+        expected4[j] = static_cast<T>((val_b >> i) | (val_b << shift_amt_2));
+      }
+
+      HWY_ASSERT_VEC_EQ(d, expected1.get(), RotateLeftSame(va, i));
+      HWY_ASSERT_VEC_EQ(d, expected2.get(), RotateRightSame(va, i));
+      HWY_ASSERT_VEC_EQ(d, expected3.get(), RotateLeftSame(vb, i));
+      HWY_ASSERT_VEC_EQ(d, expected4.get(), RotateRightSame(vb, i));
+
+      HWY_ASSERT_VEC_EQ(d, expected1.get(), RotateRightSame(va, -i));
+      HWY_ASSERT_VEC_EQ(d, expected2.get(), RotateLeftSame(va, -i));
+      HWY_ASSERT_VEC_EQ(d, expected3.get(), RotateRightSame(vb, -i));
+      HWY_ASSERT_VEC_EQ(d, expected4.get(), RotateLeftSame(vb, -i));
+
+      HWY_ASSERT_VEC_EQ(d, expected1.get(),
+                        RotateRightSame(va, static_cast<int>(kBits) - i));
+      HWY_ASSERT_VEC_EQ(d, expected2.get(),
+                        RotateLeftSame(va, static_cast<int>(kBits) - i));
+      HWY_ASSERT_VEC_EQ(d, expected3.get(),
+                        RotateRightSame(vb, static_cast<int>(kBits) - i));
+      HWY_ASSERT_VEC_EQ(d, expected4.get(),
+                        RotateLeftSame(vb, static_cast<int>(kBits) - i));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllVariableRotations() {
+  ForIntegerTypes(ForPartialVectors<TestVariableRotations>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyRotateTest);
+HWY_EXPORT_AND_TEST_P(HwyRotateTest, TestAllRotateLeft);
+HWY_EXPORT_AND_TEST_P(HwyRotateTest, TestAllRotateRight);
+HWY_EXPORT_AND_TEST_P(HwyRotateTest, TestAllVariableRotations);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/saturated_test.cc

@@ -0,0 +1,168 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/saturated_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestUnsignedSaturatedAddSub {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v0 = Zero(d);
+    const auto vi = Iota(d, 1);
+    const auto vm = Set(d, LimitsMax<T>());
+
+    HWY_ASSERT_VEC_EQ(d, Add(v0, v0), SaturatedAdd(v0, v0));
+    HWY_ASSERT_VEC_EQ(d, Add(v0, vi), SaturatedAdd(v0, vi));
+    HWY_ASSERT_VEC_EQ(d, Add(v0, vm), SaturatedAdd(v0, vm));
+    HWY_ASSERT_VEC_EQ(d, vm, SaturatedAdd(vi, vm));
+    HWY_ASSERT_VEC_EQ(d, vm, SaturatedAdd(vm, vm));
+
+    HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(v0, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(v0, vi));
+    HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(vi, vi));
+    HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(vi, vm));
+    HWY_ASSERT_VEC_EQ(d, Sub(vm, vi), SaturatedSub(vm, vi));
+  }
+};
+
+struct TestSignedSaturatedAddSub {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> vpm = Set(d, LimitsMax<T>());
+    const Vec<D> vi = PositiveIota(d);
+    const Vec<D> vn = Sub(v0, vi);
+    const Vec<D> vnm = Set(d, LimitsMin<T>());
+    HWY_ASSERT_MASK_EQ(d, MaskTrue(d), Gt(vi, v0));
+    HWY_ASSERT_MASK_EQ(d, MaskTrue(d), Lt(vn, v0));
+
+    HWY_ASSERT_VEC_EQ(d, v0, SaturatedAdd(v0, v0));
+    HWY_ASSERT_VEC_EQ(d, vi, SaturatedAdd(v0, vi));
+    HWY_ASSERT_VEC_EQ(d, vpm, SaturatedAdd(v0, vpm));
+    HWY_ASSERT_VEC_EQ(d, vpm, SaturatedAdd(vi, vpm));
+    HWY_ASSERT_VEC_EQ(d, vpm, SaturatedAdd(vpm, vpm));
+
+    HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(v0, v0));
+    HWY_ASSERT_VEC_EQ(d, Sub(v0, vi), SaturatedSub(v0, vi));
+    HWY_ASSERT_VEC_EQ(d, vn, SaturatedSub(vn, v0));
+    HWY_ASSERT_VEC_EQ(d, vnm, SaturatedSub(vnm, vi));
+    HWY_ASSERT_VEC_EQ(d, vnm, SaturatedSub(vnm, vpm));
+  }
+};
+
+struct TestSaturatedAddSubOverflow {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v1 = Iota(d, 1);
+    const auto vMax = Iota(d, LimitsMax<T>());
+    const auto vMin = Iota(d, LimitsMin<T>());
+
+    // Check that no UB triggered.
+    // "assert" here is formal - to avoid compiler dropping calculations
+    HWY_ASSERT_VEC_EQ(d, SaturatedAdd(v1, vMax), SaturatedAdd(vMax, v1));
+    HWY_ASSERT_VEC_EQ(d, SaturatedAdd(vMax, vMax), SaturatedAdd(vMax, vMax));
+    HWY_ASSERT_VEC_EQ(d, SaturatedAdd(vMin, vMax), SaturatedAdd(vMin, vMax));
+    HWY_ASSERT_VEC_EQ(d, SaturatedAdd(vMin, vMin), SaturatedAdd(vMin, vMin));
+    HWY_ASSERT_VEC_EQ(d, SaturatedSub(vMin, v1), SaturatedSub(vMin, v1));
+    HWY_ASSERT_VEC_EQ(d, SaturatedSub(vMin, vMax), SaturatedSub(vMin, vMax));
+    HWY_ASSERT_VEC_EQ(d, SaturatedSub(vMax, vMin), SaturatedSub(vMax, vMin));
+    HWY_ASSERT_VEC_EQ(d, SaturatedSub(vMin, vMin), SaturatedSub(vMin, vMin));
+  }
+};
+
+HWY_NOINLINE void TestAllSaturatedAddSub() {
+  ForUnsignedTypes(ForPartialVectors<TestUnsignedSaturatedAddSub>());
+  ForSignedTypes(ForPartialVectors<TestSignedSaturatedAddSub>());
+  ForIntegerTypes(ForPartialVectors<TestSaturatedAddSubOverflow>());
+}
+
+struct TestSaturatedAbs {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> vp1 = Set(d, static_cast<T>(1));
+    const Vec<D> vn1 = Set(d, static_cast<T>(-1));
+    const Vec<D> vpm = Set(d, LimitsMax<T>());
+    const Vec<D> vnm = Set(d, LimitsMin<T>());
+
+    HWY_ASSERT_VEC_EQ(d, v0, SaturatedAbs(v0));
+    HWY_ASSERT_VEC_EQ(d, vp1, SaturatedAbs(vp1));
+    HWY_ASSERT_VEC_EQ(d, vp1, SaturatedAbs(vn1));
+    HWY_ASSERT_VEC_EQ(d, vpm, SaturatedAbs(vpm));
+    HWY_ASSERT_VEC_EQ(d, vpm, SaturatedAbs(vnm));
+  }
+};
+
+HWY_NOINLINE void TestAllSaturatedAbs() {
+  ForSignedTypes(ForPartialVectors<TestSaturatedAbs>());
+}
+
+struct TestSaturatedNeg {
+  template <class D>
+  static HWY_NOINLINE void VerifySatNegOverflow(D d) {
+    using T = TFromD<D>;
+    HWY_ASSERT_VEC_EQ(d, Set(d, LimitsMax<T>()),
+                      SaturatedNeg(Set(d, LimitsMin<T>())));
+  }
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    VerifySatNegOverflow(d);
+
+    const RebindToUnsigned<D> du;
+    using TU = TFromD<decltype(du)>;
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> v1 = BitCast(d, Set(du, TU{1}));
+    const Vec<D> vp = BitCast(d, Set(du, TU{3}));
+    const Vec<D> vn = Add(Not(vp), v1);  // 2's complement
+    HWY_ASSERT_VEC_EQ(d, v0, SaturatedNeg(v0));
+    HWY_ASSERT_VEC_EQ(d, vp, SaturatedNeg(vn));
+    HWY_ASSERT_VEC_EQ(d, vn, SaturatedNeg(vp));
+  }
+};
+
+HWY_NOINLINE void TestAllSaturatedNeg() {
+  ForSignedTypes(ForPartialVectors<TestSaturatedNeg>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwySaturatedTest);
+HWY_EXPORT_AND_TEST_P(HwySaturatedTest, TestAllSaturatedAddSub);
+HWY_EXPORT_AND_TEST_P(HwySaturatedTest, TestAllSaturatedAbs);
+HWY_EXPORT_AND_TEST_P(HwySaturatedTest, TestAllSaturatedNeg);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/shift_test.cc

@@ -0,0 +1,523 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/shift_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+template <bool kSigned>
+struct TestLeftShifts {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T t, D d) {
+    if (kSigned) {
+      // Also test positive values
+      TestLeftShifts</*kSigned=*/false>()(t, d);
+    }
+
+    using TI = MakeSigned<T>;
+    using TU = MakeUnsigned<T>;
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    // Values to shift
+    const auto values = Iota(d, kSigned ? -TI(N) : TI(0));
+    constexpr size_t kMaxShift = (sizeof(T) * 8) - 1;
+
+    // 0
+    HWY_ASSERT_VEC_EQ(d, values, ShiftLeft<0>(values));
+    HWY_ASSERT_VEC_EQ(d, values, ShiftLeftSame(values, 0));
+
+    // 1
+    for (size_t i = 0; i < N; ++i) {
+      const T value =
+          kSigned ? static_cast<T>(static_cast<T>(i) - static_cast<T>(N))
+                  : static_cast<T>(i);
+      expected[i] = static_cast<T>(static_cast<TU>(value) << 1);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeft<1>(values));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftSame(values, 1));
+
+    // max
+    for (size_t i = 0; i < N; ++i) {
+      const T value =
+          kSigned ? static_cast<T>(static_cast<T>(i) - static_cast<T>(N))
+                  : static_cast<T>(i);
+      expected[i] = static_cast<T>(static_cast<TU>(value) << kMaxShift);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeft<kMaxShift>(values));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftSame(values, kMaxShift));
+  }
+};
+
+template <bool kSigned>
+struct TestVariableLeftShifts {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T t, D d) {
+    if (kSigned) {
+      // Also test positive values
+      TestVariableLeftShifts</*kSigned=*/false>()(t, d);
+    }
+
+    using TI = MakeSigned<T>;
+    using TU = MakeUnsigned<T>;
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    const auto v0 = Zero(d);
+    const auto v1 = Set(d, 1);
+    const auto values = Iota(d, kSigned ? -TI(N) : TI(0));  // value to shift
+
+    constexpr size_t kMaxShift = (sizeof(T) * 8) - 1;
+    const auto max_shift = Set(d, kMaxShift);
+    const auto small_shifts = And(Iota(d, 0), max_shift);
+    const auto large_shifts = Sub(max_shift, small_shifts);
+
+    // Same: 0
+    HWY_ASSERT_VEC_EQ(d, values, Shl(values, v0));
+
+    // Same: 1
+    for (size_t i = 0; i < N; ++i) {
+      const T value =
+          kSigned ? static_cast<T>(static_cast<T>(i) - static_cast<T>(N))
+                  : static_cast<T>(i);
+      expected[i] = static_cast<T>(static_cast<TU>(value) << 1);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Shl(values, v1));
+
+    // Same: max
+    for (size_t i = 0; i < N; ++i) {
+      const T value =
+          kSigned ? static_cast<T>(static_cast<T>(i) - static_cast<T>(N))
+                  : static_cast<T>(i);
+      expected[i] = static_cast<T>(static_cast<TU>(value) << kMaxShift);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Shl(values, max_shift));
+
+    // Variable: small
+    for (size_t i = 0; i < N; ++i) {
+      const T value =
+          kSigned ? static_cast<T>(static_cast<T>(i) - static_cast<T>(N))
+                  : static_cast<T>(i);
+      expected[i] = static_cast<T>(static_cast<TU>(value) << (i & kMaxShift));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Shl(values, small_shifts));
+
+    // Variable: large
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] =
+          static_cast<T>(static_cast<TU>(1) << (kMaxShift - (i & kMaxShift)));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Shl(v1, large_shifts));
+  }
+};
+
+struct TestUnsignedRightShifts {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    const auto values = Iota(d, 0);
+
+    const T kMax = LimitsMax<T>();
+    constexpr size_t kMaxShift = (sizeof(T) * 8) - 1;
+
+    // Shift by 0
+    HWY_ASSERT_VEC_EQ(d, values, ShiftRight<0>(values));
+    HWY_ASSERT_VEC_EQ(d, values, ShiftRightSame(values, 0));
+
+    // Shift by 1
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = static_cast<T>(static_cast<T>(i & kMax) >> 1);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRight<1>(values));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightSame(values, 1));
+
+    // max
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = static_cast<T>(static_cast<T>(i & kMax) >> kMaxShift);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRight<kMaxShift>(values));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightSame(values, kMaxShift));
+  }
+};
+
+struct TestVariableUnsignedRightShifts {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    const auto v0 = Zero(d);
+    const auto v1 = Set(d, 1);
+    const auto values = Iota(d, 0);
+
+    const T kMax = LimitsMax<T>();
+    const auto max = Set(d, kMax);
+
+    constexpr size_t kMaxShift = (sizeof(T) * 8) - 1;
+    const auto max_shift = Set(d, kMaxShift);
+    const auto small_shifts = And(Iota(d, 0), max_shift);
+    const auto large_shifts = Sub(max_shift, small_shifts);
+
+    // Same: 0
+    HWY_ASSERT_VEC_EQ(d, values, Shr(values, v0));
+
+    // Same: 1
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = static_cast<T>(static_cast<T>(i & kMax) >> 1);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(values, v1));
+
+    // Same: max
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = static_cast<T>(static_cast<T>(i & kMax) >> kMaxShift);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(values, max_shift));
+
+    // Variable: small
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>(T(i) >> (i & kMaxShift));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(values, small_shifts));
+
+    // Variable: Large
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>(kMax >> (kMaxShift - (i & kMaxShift)));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(max, large_shifts));
+  }
+};
+
+template <int kAmount, typename T>
+T RightShiftNegative(T val) {
+  // C++ shifts are implementation-defined for negative numbers, and we have
+  // seen divisions replaced with shifts, so resort to bit operations.
+  using TU = hwy::MakeUnsigned<T>;
+  TU bits;
+  CopySameSize(&val, &bits);
+
+  const TU shifted = TU(bits >> kAmount);
+
+  const TU all = TU(~TU(0));
+  const size_t num_zero = sizeof(TU) * 8 - 1 - kAmount;
+  const TU sign_extended = static_cast<TU>((all << num_zero) & LimitsMax<TU>());
+
+  bits = shifted | sign_extended;
+  CopySameSize(&bits, &val);
+  return val;
+}
+
+class TestSignedRightShifts {
+ public:
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+    constexpr T kMin = LimitsMin<T>();
+    constexpr T kMax = LimitsMax<T>();
+    constexpr size_t kMaxShift = (sizeof(T) * 8) - 1;
+
+    // First test positive values, negative are checked below.
+    const auto v0 = Zero(d);
+    const auto values = And(Iota(d, 0), Set(d, kMax));
+
+    // Shift by 0
+    HWY_ASSERT_VEC_EQ(d, values, ShiftRight<0>(values));
+    HWY_ASSERT_VEC_EQ(d, values, ShiftRightSame(values, 0));
+
+    // Shift by 1
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = static_cast<T>(static_cast<T>(i & kMax) >> 1);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRight<1>(values));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightSame(values, 1));
+
+    // max
+    HWY_ASSERT_VEC_EQ(d, v0, ShiftRight<kMaxShift>(values));
+    HWY_ASSERT_VEC_EQ(d, v0, ShiftRightSame(values, kMaxShift));
+
+    // Even negative value
+    Test<0>(kMin, d, __LINE__);
+    Test<1>(kMin, d, __LINE__);
+    Test<2>(kMin, d, __LINE__);
+    Test<kMaxShift>(kMin, d, __LINE__);
+
+    const T odd = ConvertScalarTo<T>(kMin + 1);
+    Test<0>(odd, d, __LINE__);
+    Test<1>(odd, d, __LINE__);
+    Test<2>(odd, d, __LINE__);
+    Test<kMaxShift>(odd, d, __LINE__);
+  }
+
+ private:
+  template <int kAmount, typename T, class D>
+  void Test(T val, D d, int line) {
+    const auto expected = Set(d, RightShiftNegative<kAmount>(val));
+    const auto in = Set(d, val);
+    const char* file = __FILE__;
+    AssertVecEqual(d, expected, ShiftRight<kAmount>(in), file, line);
+    AssertVecEqual(d, expected, ShiftRightSame(in, kAmount), file, line);
+  }
+};
+
+struct TestVariableSignedRightShifts {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TU = MakeUnsigned<T>;
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    constexpr T kMin = LimitsMin<T>();
+    constexpr T kMax = LimitsMax<T>();
+
+    constexpr size_t kMaxShift = (sizeof(T) * 8) - 1;
+
+    // First test positive values, negative are checked below.
+    const auto v0 = Zero(d);
+    const auto positive = And(Iota(d, 0), Set(d, kMax));
+
+    // Shift by 0
+    HWY_ASSERT_VEC_EQ(d, positive, ShiftRight<0>(positive));
+    HWY_ASSERT_VEC_EQ(d, positive, ShiftRightSame(positive, 0));
+
+    // Shift by 1
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = static_cast<T>(static_cast<T>(i & kMax) >> 1);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRight<1>(positive));
+    HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightSame(positive, 1));
+
+    // max
+    HWY_ASSERT_VEC_EQ(d, v0, ShiftRight<kMaxShift>(positive));
+    HWY_ASSERT_VEC_EQ(d, v0, ShiftRightSame(positive, kMaxShift));
+
+    const auto max_shift = Set(d, kMaxShift);
+    const auto small_shifts = And(Iota(d, 0), max_shift);
+    const auto large_shifts = Sub(max_shift, small_shifts);
+
+    const auto negative = Iota(d, kMin);
+
+    // Test varying negative to shift
+    for (size_t i = 0; i < N; ++i) {
+      const T val = ConvertScalarTo<T>(static_cast<TU>(kMin) + i);
+      expected[i] =
+          (val < 0) ? RightShiftNegative<1>(val) : ConvertScalarTo<T>(val >> 1);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(negative, Set(d, 1)));
+
+    // Shift MSB right by small amounts
+    for (size_t i = 0; i < N; ++i) {
+      const size_t amount = i & kMaxShift;
+      const TU shifted = static_cast<TU>(~((1ull << (kMaxShift - amount)) - 1));
+      CopySameSize(&shifted, &expected[i]);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(Set(d, kMin), small_shifts));
+
+    // Shift MSB right by large amounts
+    for (size_t i = 0; i < N; ++i) {
+      const size_t amount = kMaxShift - (i & kMaxShift);
+      const TU shifted = static_cast<TU>(~((1ull << (kMaxShift - amount)) - 1));
+      CopySameSize(&shifted, &expected[i]);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(Set(d, kMin), large_shifts));
+  }
+};
+
+HWY_NOINLINE void TestAllShifts() {
+  ForUnsignedTypes(ForPartialVectors<TestLeftShifts</*kSigned=*/false>>());
+  ForSignedTypes(ForPartialVectors<TestLeftShifts</*kSigned=*/true>>());
+  ForUnsignedTypes(ForPartialVectors<TestUnsignedRightShifts>());
+  ForSignedTypes(ForPartialVectors<TestSignedRightShifts>());
+}
+
+HWY_NOINLINE void TestAllVariableShifts() {
+  ForUnsignedTypes(
+      ForPartialVectors<TestVariableLeftShifts</*kSigned=*/false>>());
+  ForSignedTypes(ForPartialVectors<TestVariableLeftShifts</*kSigned=*/true>>());
+  ForUnsignedTypes(ForPartialVectors<TestVariableUnsignedRightShifts>());
+  ForSignedTypes(ForPartialVectors<TestVariableSignedRightShifts>());
+}
+
+struct TestRoundingShiftRight {
+  template <int kShiftAmt, class D>
+  static HWY_INLINE void VerifyRoundingShiftRight(
+      D d, const TFromD<D>* HWY_RESTRICT expected, Vec<D> v,
+      const char* filename, const int line) {
+    AssertVecEqual(d, expected, RoundingShiftRight<kShiftAmt>(v), filename,
+                   line);
+    AssertVecEqual(d, expected, RoundingShiftRightSame(v, kShiftAmt), filename,
+                   line);
+  }
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using TU = MakeUnsigned<T>;
+    const auto iota0 = Iota(d, T{0});
+
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    Store(iota0, d, expected.get());
+    VerifyRoundingShiftRight<0>(d, expected.get(), iota0, __FILE__, __LINE__);
+
+    const auto v1 = Set(d, T{1});
+    const auto iota1 = Add(iota0, v1);
+    Store(iota1, d, expected.get());
+    VerifyRoundingShiftRight<0>(d, expected.get(), iota1, __FILE__, __LINE__);
+
+    const auto v2 = Set(d, T{2});
+    const auto iota2 = Add(iota0, v2);
+    Store(iota2, d, expected.get());
+    VerifyRoundingShiftRight<0>(d, expected.get(), iota2, __FILE__, __LINE__);
+
+    const auto iota3 = Add(iota0, Set(d, T{3}));
+    Store(iota3, d, expected.get());
+    VerifyRoundingShiftRight<0>(d, expected.get(), iota3, __FILE__, __LINE__);
+
+    const auto seq4 = Add(iota0, SignBit(d));
+    Store(seq4, d, expected.get());
+    VerifyRoundingShiftRight<0>(d, expected.get(), seq4, __FILE__, __LINE__);
+
+    const auto seq5 = Add(seq4, v1);
+    Store(seq5, d, expected.get());
+    VerifyRoundingShiftRight<0>(d, expected.get(), seq5, __FILE__, __LINE__);
+
+    const auto v0 = Zero(d);
+    Store(AverageRound(iota1, v0), d, expected.get());
+    VerifyRoundingShiftRight<1>(d, expected.get(), iota1, __FILE__, __LINE__);
+
+    Store(AverageRound(iota2, v0), d, expected.get());
+    VerifyRoundingShiftRight<1>(d, expected.get(), iota2, __FILE__, __LINE__);
+
+    Store(AverageRound(seq4, v0), d, expected.get());
+    VerifyRoundingShiftRight<1>(d, expected.get(), seq4, __FILE__, __LINE__);
+
+    Store(AverageRound(seq5, v0), d, expected.get());
+    VerifyRoundingShiftRight<1>(d, expected.get(), seq5, __FILE__, __LINE__);
+
+    const auto seq6 = And(
+        Xor(iota1,
+            Set(d, static_cast<T>(0x70FB991A05AC6B24ULL & LimitsMax<TU>()))),
+        Set(d, static_cast<T>(~static_cast<T>(0x10))));
+    Store(ShiftRight<5>(seq6), d, expected.get());
+    VerifyRoundingShiftRight<5>(d, expected.get(), seq6, __FILE__, __LINE__);
+
+    const auto seq7 =
+        Or(Xor(iota2,
+               Set(d, static_cast<T>(0x6ED498B16EC87C63ULL & LimitsMax<TU>()))),
+           Set(d, static_cast<T>(0x04)));
+    Store(Add(ShiftRight<3>(seq7), v1), d, expected.get());
+    VerifyRoundingShiftRight<3>(d, expected.get(), seq7, __FILE__, __LINE__);
+
+    const auto seq8 = And(
+        Xor(iota1,
+            Set(d, static_cast<T>(0x186958FE04C94D77ULL & LimitsMax<TU>()))),
+        Set(d, static_cast<T>(~static_cast<T>(0x08))));
+    Store(ShiftRight<4>(seq8), d, expected.get());
+    VerifyRoundingShiftRight<4>(d, expected.get(), seq8, __FILE__, __LINE__);
+
+    const auto seq9 =
+        Or(Xor(iota2,
+               Set(d, static_cast<T>(0x7FC4E62077CC7655ULL & LimitsMax<TU>()))),
+           v2);
+    Store(Add(ShiftRight<2>(seq9), v1), d, expected.get());
+    VerifyRoundingShiftRight<2>(d, expected.get(), seq9, __FILE__, __LINE__);
+  }
+};
+
+HWY_NOINLINE void TestAllRoundingShiftRight() {
+  ForIntegerTypes(ForPartialVectors<TestRoundingShiftRight>());
+}
+
+struct TestVariableRoundingShr {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    constexpr size_t kNumOfBits = sizeof(T) * 8;
+    const auto iota1 = Iota(d, T{1});
+
+    const auto v0 = Zero(d);
+    const auto v1 = Set(d, T{1});
+    const auto sign_bit = SignBit(d);
+
+    for (size_t i = 0; i < kNumOfBits; i += N) {
+      auto shift_amt = Iota(d, static_cast<T>(i & (kNumOfBits - 1)));
+
+      HWY_IF_CONSTEXPR(HWY_MAX_LANES_D(D) > kNumOfBits) {
+        shift_amt = And(shift_amt, Set(d, static_cast<T>(kNumOfBits - 1)));
+      }
+
+      const auto half_bit = ShiftRight<1>(Shl(v1, shift_amt));
+
+      const auto in_0 = AndNot(half_bit, Or(Shl(iota1, shift_amt), v1));
+      const auto in_1 = Or(in_0, half_bit);
+      const auto in_2 = Xor(in_0, sign_bit);
+      const auto in_3 = Xor(in_1, sign_bit);
+
+      const auto round_decr = VecFromMask(d, Ne(half_bit, v0));
+
+      const auto expected_0 = Shr(in_0, shift_amt);
+      const auto expected_1 = Sub(Shr(in_1, shift_amt), round_decr);
+      const auto expected_2 = Shr(in_2, shift_amt);
+      const auto expected_3 = Sub(Shr(in_3, shift_amt), round_decr);
+
+      HWY_ASSERT_VEC_EQ(d, expected_0, RoundingShr(in_0, shift_amt));
+      HWY_ASSERT_VEC_EQ(d, expected_1, RoundingShr(in_1, shift_amt));
+      HWY_ASSERT_VEC_EQ(d, expected_2, RoundingShr(in_2, shift_amt));
+      HWY_ASSERT_VEC_EQ(d, expected_3, RoundingShr(in_3, shift_amt));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllVariableRoundingShr() {
+  ForIntegerTypes(ForPartialVectors<TestVariableRoundingShr>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyShiftTest);
+HWY_EXPORT_AND_TEST_P(HwyShiftTest, TestAllShifts);
+HWY_EXPORT_AND_TEST_P(HwyShiftTest, TestAllVariableShifts);
+HWY_EXPORT_AND_TEST_P(HwyShiftTest, TestAllRoundingShiftRight);
+HWY_EXPORT_AND_TEST_P(HwyShiftTest, TestAllVariableRoundingShr);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/shuffle4_test.cc

@@ -0,0 +1,232 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/shuffle4_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+class TestPer4LaneBlockShuffle {
+ private:
+  template <class D, HWY_IF_LANES_LE_D(D, 1)>
+  static HWY_INLINE VFromD<D> InterleaveMaskVectors(D /*d*/, VFromD<D> a,
+                                                    VFromD<D> /*b*/) {
+    return a;
+  }
+#if HWY_TARGET != HWY_SCALAR
+  template <class D, HWY_IF_LANES_GT_D(D, 1)>
+  static HWY_INLINE VFromD<D> InterleaveMaskVectors(D d, VFromD<D> a,
+                                                    VFromD<D> b) {
+    return InterleaveLower(d, a, b);
+  }
+#endif
+  template <class D>
+  static HWY_INLINE Mask<D> Per4LaneBlockShufValidMask(D d, const size_t N,
+                                                       const size_t idx1,
+                                                       const size_t idx0) {
+    if (N < 4) {
+      const RebindToSigned<decltype(d)> di;
+      using TI = TFromD<decltype(di)>;
+      const auto lane_0_valid =
+          Set(di, static_cast<TI>(-static_cast<int>(idx0 < N)));
+      if (N > 1) {
+        const auto lane_1_valid =
+            Set(di, static_cast<TI>(-static_cast<int>(idx1 < N)));
+        return RebindMask(d, MaskFromVec(InterleaveMaskVectors(di, lane_0_valid,
+                                                               lane_1_valid)));
+      }
+      return RebindMask(d, MaskFromVec(lane_0_valid));
+    }
+
+    return FirstN(d, N);
+  }
+
+  // TODO(b/287462770): inline to work around incorrect SVE codegen
+  template <class D>
+  static HWY_INLINE void DoCheckPer4LaneBlkShufResult(
+      D d, const size_t N, VFromD<D> actual,
+      const TFromD<D>* HWY_RESTRICT src_lanes, TFromD<D>* HWY_RESTRICT expected,
+      size_t idx3, size_t idx2, size_t idx1, size_t idx0) {
+    for (size_t i = 0; i < N; i += 4) {
+      expected[i] = src_lanes[i + idx0];
+      expected[i + 1] = src_lanes[i + idx1];
+      expected[i + 2] = src_lanes[i + idx2];
+      expected[i + 3] = src_lanes[i + idx3];
+    }
+
+    if (N < 4) {
+      if (idx0 >= N) expected[0] = TFromD<D>{0};
+      if (idx1 >= N) expected[1] = TFromD<D>{0};
+    }
+
+    const auto valid_lanes_mask = Per4LaneBlockShufValidMask(d, N, idx1, idx0);
+    HWY_ASSERT_VEC_EQ(d, expected, IfThenElseZero(valid_lanes_mask, actual));
+  }
+
+#if HWY_TARGET != HWY_SCALAR
+  template <class D>
+  static HWY_NOINLINE void TestTblLookupPer4LaneBlkShuf(
+      D d, const size_t N, const TFromD<D>* HWY_RESTRICT src_lanes,
+      TFromD<D>* HWY_RESTRICT expected) {
+    const auto v = Load(d, src_lanes);
+    for (size_t idx3210 = 0; idx3210 <= 0xFF; idx3210++) {
+      const size_t idx3 = (idx3210 >> 6) & 3;
+      const size_t idx2 = (idx3210 >> 4) & 3;
+      const size_t idx1 = (idx3210 >> 2) & 3;
+      const size_t idx0 = idx3210 & 3;
+
+      const auto actual = detail::TblLookupPer4LaneBlkShuf(v, idx3210);
+      DoCheckPer4LaneBlkShufResult(d, N, actual, src_lanes, expected, idx3,
+                                   idx2, idx1, idx0);
+    }
+  }
+#endif
+
+  template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class D>
+  static HWY_INLINE void DoTestPer4LaneBlkShuffle(
+      D d, const size_t N, const VFromD<D> v,
+      const TFromD<D>* HWY_RESTRICT src_lanes,
+      TFromD<D>* HWY_RESTRICT expected) {
+    const auto actual = Per4LaneBlockShuffle<kIdx3, kIdx2, kIdx1, kIdx0>(v);
+    DoCheckPer4LaneBlkShufResult(d, N, actual, src_lanes, expected, kIdx3,
+                                 kIdx2, kIdx1, kIdx0);
+  }
+
+  template <class D>
+  static HWY_NOINLINE void DoTestPer4LaneBlkShuffles(
+      D d, const size_t N, const VecArg<VFromD<D>> v,
+      TFromD<D>* HWY_RESTRICT src_lanes, TFromD<D>* HWY_RESTRICT expected) {
+    Store(v, d, src_lanes);
+#if HWY_TARGET != HWY_SCALAR
+    TestTblLookupPer4LaneBlkShuf(d, N, src_lanes, expected);
+#endif
+    DoTestPer4LaneBlkShuffle<0, 1, 2, 3>(d, N, v, src_lanes, expected);
+#if !HWY_COMPILER_MSVC  // speed up MSVC builds
+    DoTestPer4LaneBlkShuffle<0, 1, 3, 2>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<0, 2, 3, 1>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<0, 3, 0, 2>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<1, 0, 1, 0>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<1, 0, 3, 1>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<1, 0, 3, 2>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<1, 2, 0, 3>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<1, 2, 1, 3>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<1, 1, 0, 0>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<2, 0, 1, 3>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<2, 0, 2, 0>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<2, 1, 2, 0>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<2, 2, 0, 0>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<2, 3, 0, 1>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<2, 3, 3, 0>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<3, 0, 2, 1>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<3, 1, 0, 3>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<3, 1, 3, 1>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<3, 2, 1, 0>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<3, 2, 3, 2>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<3, 3, 0, 1>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<3, 3, 1, 1>(d, N, v, src_lanes, expected);
+    DoTestPer4LaneBlkShuffle<3, 3, 2, 2>(d, N, v, src_lanes, expected);
+#endif
+  }
+
+  template <class D>
+  static HWY_INLINE Vec<D> GenerateTestVect(hwy::NonFloatTag /*tag*/, D d) {
+    const RebindToUnsigned<decltype(d)> du;
+    using TU = TFromD<decltype(du)>;
+    constexpr TU kIotaStart =
+        static_cast<TU>(0x0706050403020101u & LimitsMax<TU>());
+    return BitCast(d, Iota(du, kIotaStart));
+  }
+
+  template <class D>
+  static HWY_INLINE Vec<D> GenerateTestVect(hwy::FloatTag /*tag*/, D d) {
+    const RebindToUnsigned<decltype(d)> du;
+    using T = TFromD<decltype(d)>;
+    using TU = TFromD<decltype(du)>;
+
+    constexpr size_t kNumOfBitsInT = sizeof(T) * 8;
+    constexpr TU kIntBitsMask =
+        (kNumOfBitsInT > 16) ? static_cast<TU>(static_cast<TU>(~TU{0}) >> 16)
+                             : TU{0};
+
+    const auto flt_iota = Set(d, 1);
+    if (kIntBitsMask == 0) return flt_iota;
+
+    const auto int_iota =
+        And(GenerateTestVect(hwy::NonFloatTag(), du), Set(du, kIntBitsMask));
+    return Or(flt_iota, BitCast(d, int_iota));
+  }
+
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    const size_t alloc_len = static_cast<size_t>((N + 3) & (~size_t{3}));
+    HWY_ASSERT(alloc_len >= 4);
+
+    auto expected = AllocateAligned<T>(alloc_len);
+    auto src_lanes = AllocateAligned<T>(alloc_len);
+    HWY_ASSERT(expected && src_lanes);
+
+    const T k0 = ConvertScalarTo<T>(0);
+    expected[alloc_len - 4] = k0;
+    expected[alloc_len - 3] = k0;
+    expected[alloc_len - 2] = k0;
+    expected[alloc_len - 1] = k0;
+    src_lanes[alloc_len - 4] = k0;
+    src_lanes[alloc_len - 3] = k0;
+    src_lanes[alloc_len - 2] = k0;
+    src_lanes[alloc_len - 1] = k0;
+
+    const auto v = GenerateTestVect(hwy::IsFloatTag<T>(), d);
+    DoTestPer4LaneBlkShuffles(d, N, v, src_lanes.get(), expected.get());
+
+    const RebindToUnsigned<decltype(d)> du;
+    using TU = TFromD<decltype(du)>;
+    const auto msb_mask =
+        BitCast(d, Set(du, static_cast<TU>(TU{1} << (sizeof(TU) * 8 - 1))));
+
+    DoTestPer4LaneBlkShuffles(d, N, Xor(v, msb_mask), src_lanes.get(),
+                              expected.get());
+  }
+};
+
+HWY_NOINLINE void TestAllPer4LaneBlockShuffle() {
+  ForAllTypes(ForPartialFixedOrFullScalableVectors<TestPer4LaneBlockShuffle>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyShuffle4Test);
+HWY_EXPORT_AND_TEST_P(HwyShuffle4Test, TestAllPer4LaneBlockShuffle);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/sign_test.cc

@@ -0,0 +1,101 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/sign_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestCopySign {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v0 = Zero(d);
+    const auto vp = Iota(d, 1);
+    const auto vn = Iota(d, -1E5);  // assumes N < 10^5
+
+    // Zero remains zero regardless of sign
+    HWY_ASSERT_VEC_EQ(d, v0, CopySign(v0, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, CopySign(v0, vp));
+    HWY_ASSERT_VEC_EQ(d, v0, CopySign(v0, vn));
+    HWY_ASSERT_VEC_EQ(d, v0, CopySignToAbs(v0, v0));
+    HWY_ASSERT_VEC_EQ(d, v0, CopySignToAbs(v0, vp));
+    HWY_ASSERT_VEC_EQ(d, v0, CopySignToAbs(v0, vn));
+
+    // Positive input, positive sign => unchanged
+    HWY_ASSERT_VEC_EQ(d, vp, CopySign(vp, vp));
+    HWY_ASSERT_VEC_EQ(d, vp, CopySignToAbs(vp, vp));
+
+    // Positive input, negative sign => negated
+    HWY_ASSERT_VEC_EQ(d, Neg(vp), CopySign(vp, vn));
+    HWY_ASSERT_VEC_EQ(d, Neg(vp), CopySignToAbs(vp, vn));
+
+    // Negative input, negative sign => unchanged
+    HWY_ASSERT_VEC_EQ(d, vn, CopySign(vn, vn));
+
+    // Negative input, positive sign => negated
+    HWY_ASSERT_VEC_EQ(d, Neg(vn), CopySign(vn, vp));
+  }
+};
+
+HWY_NOINLINE void TestAllCopySign() {
+  ForFloatTypes(ForPartialVectors<TestCopySign>());
+}
+
+struct TestBroadcastSignBit {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto s0 = Zero(d);
+    const auto s1 = Set(d, -1);  // all bit set
+    const auto vpos = And(Iota(d, 0), Set(d, LimitsMax<T>()));
+    const auto vneg = Sub(s1, vpos);
+
+    HWY_ASSERT_VEC_EQ(d, s0, BroadcastSignBit(vpos));
+    HWY_ASSERT_VEC_EQ(d, s0, BroadcastSignBit(Set(d, LimitsMax<T>())));
+
+    HWY_ASSERT_VEC_EQ(d, s1, BroadcastSignBit(vneg));
+    HWY_ASSERT_VEC_EQ(d, s1, BroadcastSignBit(Set(d, LimitsMin<T>())));
+    HWY_ASSERT_VEC_EQ(d, s1, BroadcastSignBit(Set(d, LimitsMin<T>() / 2)));
+  }
+};
+
+HWY_NOINLINE void TestAllBroadcastSignBit() {
+  ForSignedTypes(ForPartialVectors<TestBroadcastSignBit>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwySignTest);
+HWY_EXPORT_AND_TEST_P(HwySignTest, TestAllCopySign);
+HWY_EXPORT_AND_TEST_P(HwySignTest, TestAllBroadcastSignBit);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/slide_up_down_test.cc

@@ -0,0 +1,466 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string.h>  // memset
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/slide_up_down_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+class TestSlideUpLanes {
+ private:
+  template <class D>
+  static HWY_INLINE void DoTestSlideUpLanes(D d,
+                                            TFromD<D>* HWY_RESTRICT expected,
+                                            const size_t N,
+                                            const size_t slide_amt) {
+    for (size_t i = 0; i < N; i++) {
+      expected[i] = ConvertScalarTo<TFromD<D>>(
+          (i >= slide_amt) ? (i - slide_amt + 1) : 0);
+    }
+
+    const auto v = Iota(d, 1);
+    HWY_ASSERT_VEC_EQ(d, expected, SlideUpLanes(d, v, slide_amt));
+    if (slide_amt == 1) {
+      HWY_ASSERT_VEC_EQ(d, expected, Slide1Up(d, v));
+    }
+  }
+#if !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+  template <class D>
+  static HWY_NOINLINE void DoTestSlideUpLanesWithConstAmt_0_7(
+      D d, TFromD<D>* HWY_RESTRICT expected, const size_t N) {
+    DoTestSlideUpLanes(d, expected, N, 0);
+    if (N <= 1) return;
+    DoTestSlideUpLanes(d, expected, N, 1);
+    if (N <= 2) return;
+    DoTestSlideUpLanes(d, expected, N, 2);
+    DoTestSlideUpLanes(d, expected, N, 3);
+    if (N <= 4) return;
+    DoTestSlideUpLanes(d, expected, N, 4);
+    DoTestSlideUpLanes(d, expected, N, 5);
+    DoTestSlideUpLanes(d, expected, N, 6);
+    DoTestSlideUpLanes(d, expected, N, 7);
+  }
+  template <class D, HWY_IF_LANES_LE_D(D, 8)>
+  static HWY_INLINE void DoTestSlideUpLanesWithConstAmt_8_15(
+      D /*d*/, TFromD<D>* HWY_RESTRICT /*expected*/, const size_t /*N*/) {}
+  template <class D, HWY_IF_LANES_GT_D(D, 8)>
+  static HWY_NOINLINE void DoTestSlideUpLanesWithConstAmt_8_15(
+      D d, TFromD<D>* HWY_RESTRICT expected, const size_t N) {
+    if (N <= 8) return;
+    DoTestSlideUpLanes(d, expected, N, 8);
+    DoTestSlideUpLanes(d, expected, N, 9);
+    DoTestSlideUpLanes(d, expected, N, 10);
+    DoTestSlideUpLanes(d, expected, N, 11);
+    DoTestSlideUpLanes(d, expected, N, 12);
+    DoTestSlideUpLanes(d, expected, N, 13);
+    DoTestSlideUpLanes(d, expected, N, 14);
+    DoTestSlideUpLanes(d, expected, N, 15);
+  }
+#if HWY_TARGET <= HWY_AVX2 || HWY_TARGET == HWY_WASM_EMU256
+  template <class D, HWY_IF_LANES_LE_D(D, 16)>
+  static HWY_INLINE void DoTestSlideUpLanesWithConstAmt_16_31(
+      D /*d*/, TFromD<D>* HWY_RESTRICT /*expected*/, const size_t /*N*/) {}
+  template <class D, HWY_IF_LANES_GT_D(D, 16)>
+  static HWY_NOINLINE void DoTestSlideUpLanesWithConstAmt_16_31(
+      D d, TFromD<D>* HWY_RESTRICT expected, const size_t N) {
+    if (N <= 16) return;
+    DoTestSlideUpLanes(d, expected, N, 16);
+    DoTestSlideUpLanes(d, expected, N, 17);
+    DoTestSlideUpLanes(d, expected, N, 18);
+    DoTestSlideUpLanes(d, expected, N, 19);
+    DoTestSlideUpLanes(d, expected, N, 20);
+    DoTestSlideUpLanes(d, expected, N, 21);
+    DoTestSlideUpLanes(d, expected, N, 22);
+    DoTestSlideUpLanes(d, expected, N, 23);
+    DoTestSlideUpLanes(d, expected, N, 24);
+    DoTestSlideUpLanes(d, expected, N, 25);
+    DoTestSlideUpLanes(d, expected, N, 26);
+    DoTestSlideUpLanes(d, expected, N, 27);
+    DoTestSlideUpLanes(d, expected, N, 28);
+    DoTestSlideUpLanes(d, expected, N, 29);
+    DoTestSlideUpLanes(d, expected, N, 30);
+    DoTestSlideUpLanes(d, expected, N, 31);
+  }
+#if HWY_TARGET <= HWY_AVX3
+  template <class D, HWY_IF_LANES_LE_D(D, 32)>
+  static HWY_INLINE void DoTestSlideUpLanesWithConstAmt_32_63(
+      D /*d*/, TFromD<D>* HWY_RESTRICT /*expected*/, const size_t /*N*/) {}
+  template <class D, HWY_IF_LANES_GT_D(D, 32)>
+  static HWY_NOINLINE void DoTestSlideUpLanesWithConstAmt_32_63(
+      D d, TFromD<D>* HWY_RESTRICT expected, const size_t N) {
+    if (N <= 32) return;
+    DoTestSlideUpLanes(d, expected, N, 32);
+    DoTestSlideUpLanes(d, expected, N, 33);
+    DoTestSlideUpLanes(d, expected, N, 34);
+    DoTestSlideUpLanes(d, expected, N, 35);
+    DoTestSlideUpLanes(d, expected, N, 36);
+    DoTestSlideUpLanes(d, expected, N, 37);
+    DoTestSlideUpLanes(d, expected, N, 38);
+    DoTestSlideUpLanes(d, expected, N, 39);
+    DoTestSlideUpLanes(d, expected, N, 40);
+    DoTestSlideUpLanes(d, expected, N, 41);
+    DoTestSlideUpLanes(d, expected, N, 42);
+    DoTestSlideUpLanes(d, expected, N, 43);
+    DoTestSlideUpLanes(d, expected, N, 44);
+    DoTestSlideUpLanes(d, expected, N, 45);
+    DoTestSlideUpLanes(d, expected, N, 46);
+    DoTestSlideUpLanes(d, expected, N, 47);
+    DoTestSlideUpLanes(d, expected, N, 48);
+    DoTestSlideUpLanes(d, expected, N, 49);
+    DoTestSlideUpLanes(d, expected, N, 50);
+    DoTestSlideUpLanes(d, expected, N, 51);
+    DoTestSlideUpLanes(d, expected, N, 52);
+    DoTestSlideUpLanes(d, expected, N, 53);
+    DoTestSlideUpLanes(d, expected, N, 54);
+    DoTestSlideUpLanes(d, expected, N, 55);
+    DoTestSlideUpLanes(d, expected, N, 56);
+    DoTestSlideUpLanes(d, expected, N, 57);
+    DoTestSlideUpLanes(d, expected, N, 58);
+    DoTestSlideUpLanes(d, expected, N, 59);
+    DoTestSlideUpLanes(d, expected, N, 60);
+    DoTestSlideUpLanes(d, expected, N, 61);
+    DoTestSlideUpLanes(d, expected, N, 62);
+    DoTestSlideUpLanes(d, expected, N, 63);
+  }
+#endif  // HWY_TARGET <= HWY_AVX3
+#endif  // HWY_TARGET <= HWY_AVX2 || HWY_TARGET == HWY_WASM_EMU256
+#endif  // !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    for (size_t i = 0; i < N; i++) {
+      size_t slide_amt = i;
+#if !HWY_COMPILER_MSVC
+      PreventElision(slide_amt);
+#endif
+      DoTestSlideUpLanes(d, expected.get(), N, slide_amt);
+    }
+
+#if !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+    DoTestSlideUpLanesWithConstAmt_0_7(d, expected.get(), N);
+    DoTestSlideUpLanesWithConstAmt_8_15(d, expected.get(), N);
+#if HWY_TARGET <= HWY_AVX2 || HWY_TARGET == HWY_WASM_EMU256
+    DoTestSlideUpLanesWithConstAmt_16_31(d, expected.get(), N);
+#if HWY_TARGET <= HWY_AVX3
+    DoTestSlideUpLanesWithConstAmt_32_63(d, expected.get(), N);
+#endif  // HWY_TARGET <= HWY_AVX3
+#endif  // HWY_TARGET <= HWY_AVX2 || HWY_TARGET == HWY_WASM_EMU256
+#endif  // !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+  }
+};
+
+HWY_NOINLINE void TestAllSlideUpLanes() {
+  ForAllTypes(ForPartialVectors<TestSlideUpLanes>());
+}
+
+#if !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+// DoTestSlideDownLanes needs to be inlined on targets where
+// DoTestSlideDownLanesWithConstAmt_0_7, DoTestSlideDownLanesWithConstAmt_8_15,
+// DoTestSlideDownLanesWithConstAmt_16_31, and
+// DoTestSlideDownLanesWithConstAmt_32_63 are called since the implementation
+// of SlideDownLanes(d, v, N) for the SSE2/SSSE3/SSE4/AVX2/AVX3/NEON/WASM
+// targets has an optimized path for the case where __builtin_constant_p(N) is
+// true (or in other words, when N is known to be a constant) when compiled with
+// GCC or Clang and optimizations are enabled.
+
+// If DoTestSlideDownLanes is not inlined on the
+// SSE2/SSSE3/SSE4/AVX2/AVX3/NEON/WASM targets,
+// DoTestSlideDownLanesWithConstAmt_0_7, DoTestSlideDownLanesWithConstAmt_8_15,
+// DoTestSlideDownLanesWithConstAmt_16_31, and
+// DoTestSlideDownLanesWithConstAmt_32_63 will fail to throughly test the
+// implementations of SlideDownLanes(d, v, N) in optimized builds compiled with
+// GCC or Clang for the case where N is known to be a constant.
+#define HWY_SLIDE_DOWN_TEST_INLINE HWY_INLINE
+#else
+// DoTestSlideDownLanes should not be inlined on RVV targets to work around RVV
+// miscompilation.
+#define HWY_SLIDE_DOWN_TEST_INLINE HWY_NOINLINE
+#endif
+
+class TestSlideDownLanes {
+ private:
+  // HWY_SLIDE_DOWN_TEST_INLINE is required here to work around RVV
+  // miscompilation.
+  template <class D>
+  static HWY_SLIDE_DOWN_TEST_INLINE void DoTestSlideDownLanes(
+      D d, TFromD<D>* HWY_RESTRICT expected, const size_t N,
+      const size_t slide_amt) {
+    for (size_t i = 0; i < N; i++) {
+      const size_t src_idx = slide_amt + i;
+      expected[i] = ConvertScalarTo<TFromD<D>>((src_idx < N) ? src_idx : 0);
+    }
+
+    const Vec<D> v = Iota(d, 0);
+    HWY_ASSERT_VEC_EQ(d, expected, SlideDownLanes(d, v, slide_amt));
+    if (slide_amt == 1) {
+      HWY_ASSERT_VEC_EQ(d, expected, Slide1Down(d, v));
+    }
+  }
+#if !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+  template <class D>
+  static HWY_NOINLINE void DoTestSlideDownLanesWithConstAmt_0_7(
+      D d, TFromD<D>* HWY_RESTRICT expected, const size_t N) {
+    DoTestSlideDownLanes(d, expected, N, 0);
+    if (N <= 1) return;
+    DoTestSlideDownLanes(d, expected, N, 1);
+    if (N <= 2) return;
+    DoTestSlideDownLanes(d, expected, N, 2);
+    DoTestSlideDownLanes(d, expected, N, 3);
+    if (N <= 4) return;
+    DoTestSlideDownLanes(d, expected, N, 4);
+    DoTestSlideDownLanes(d, expected, N, 5);
+    DoTestSlideDownLanes(d, expected, N, 6);
+    DoTestSlideDownLanes(d, expected, N, 7);
+  }
+  template <class D, HWY_IF_LANES_LE_D(D, 8)>
+  static HWY_INLINE void DoTestSlideDownLanesWithConstAmt_8_15(
+      D /*d*/, TFromD<D>* HWY_RESTRICT /*expected*/, const size_t /*N*/) {}
+  template <class D, HWY_IF_LANES_GT_D(D, 8)>
+  static HWY_NOINLINE void DoTestSlideDownLanesWithConstAmt_8_15(
+      D d, TFromD<D>* HWY_RESTRICT expected, const size_t N) {
+    if (N <= 8) return;
+    DoTestSlideDownLanes(d, expected, N, 8);
+    DoTestSlideDownLanes(d, expected, N, 9);
+    DoTestSlideDownLanes(d, expected, N, 10);
+    DoTestSlideDownLanes(d, expected, N, 11);
+    DoTestSlideDownLanes(d, expected, N, 12);
+    DoTestSlideDownLanes(d, expected, N, 13);
+    DoTestSlideDownLanes(d, expected, N, 14);
+    DoTestSlideDownLanes(d, expected, N, 15);
+  }
+#if HWY_TARGET <= HWY_AVX2 || HWY_TARGET == HWY_WASM_EMU256
+  template <class D, HWY_IF_LANES_LE_D(D, 16)>
+  static HWY_INLINE void DoTestSlideDownLanesWithConstAmt_16_31(
+      D /*d*/, TFromD<D>* HWY_RESTRICT /*expected*/, const size_t /*N*/) {}
+  template <class D, HWY_IF_LANES_GT_D(D, 16)>
+  static HWY_NOINLINE void DoTestSlideDownLanesWithConstAmt_16_31(
+      D d, TFromD<D>* HWY_RESTRICT expected, const size_t N) {
+    if (N <= 16) return;
+    DoTestSlideDownLanes(d, expected, N, 16);
+    DoTestSlideDownLanes(d, expected, N, 17);
+    DoTestSlideDownLanes(d, expected, N, 18);
+    DoTestSlideDownLanes(d, expected, N, 19);
+    DoTestSlideDownLanes(d, expected, N, 20);
+    DoTestSlideDownLanes(d, expected, N, 21);
+    DoTestSlideDownLanes(d, expected, N, 22);
+    DoTestSlideDownLanes(d, expected, N, 23);
+    DoTestSlideDownLanes(d, expected, N, 24);
+    DoTestSlideDownLanes(d, expected, N, 25);
+    DoTestSlideDownLanes(d, expected, N, 26);
+    DoTestSlideDownLanes(d, expected, N, 27);
+    DoTestSlideDownLanes(d, expected, N, 28);
+    DoTestSlideDownLanes(d, expected, N, 29);
+    DoTestSlideDownLanes(d, expected, N, 30);
+    DoTestSlideDownLanes(d, expected, N, 31);
+  }
+#if HWY_TARGET <= HWY_AVX3
+  template <class D, HWY_IF_LANES_LE_D(D, 32)>
+  static HWY_INLINE void DoTestSlideDownLanesWithConstAmt_32_63(
+      D /*d*/, TFromD<D>* HWY_RESTRICT /*expected*/, const size_t /*N*/) {}
+  template <class D, HWY_IF_LANES_GT_D(D, 32)>
+  static HWY_NOINLINE void DoTestSlideDownLanesWithConstAmt_32_63(
+      D d, TFromD<D>* HWY_RESTRICT expected, const size_t N) {
+    if (N <= 32) return;
+    DoTestSlideDownLanes(d, expected, N, 32);
+    DoTestSlideDownLanes(d, expected, N, 33);
+    DoTestSlideDownLanes(d, expected, N, 34);
+    DoTestSlideDownLanes(d, expected, N, 35);
+    DoTestSlideDownLanes(d, expected, N, 36);
+    DoTestSlideDownLanes(d, expected, N, 37);
+    DoTestSlideDownLanes(d, expected, N, 38);
+    DoTestSlideDownLanes(d, expected, N, 39);
+    DoTestSlideDownLanes(d, expected, N, 40);
+    DoTestSlideDownLanes(d, expected, N, 41);
+    DoTestSlideDownLanes(d, expected, N, 42);
+    DoTestSlideDownLanes(d, expected, N, 43);
+    DoTestSlideDownLanes(d, expected, N, 44);
+    DoTestSlideDownLanes(d, expected, N, 45);
+    DoTestSlideDownLanes(d, expected, N, 46);
+    DoTestSlideDownLanes(d, expected, N, 47);
+    DoTestSlideDownLanes(d, expected, N, 48);
+    DoTestSlideDownLanes(d, expected, N, 49);
+    DoTestSlideDownLanes(d, expected, N, 50);
+    DoTestSlideDownLanes(d, expected, N, 51);
+    DoTestSlideDownLanes(d, expected, N, 52);
+    DoTestSlideDownLanes(d, expected, N, 53);
+    DoTestSlideDownLanes(d, expected, N, 54);
+    DoTestSlideDownLanes(d, expected, N, 55);
+    DoTestSlideDownLanes(d, expected, N, 56);
+    DoTestSlideDownLanes(d, expected, N, 57);
+    DoTestSlideDownLanes(d, expected, N, 58);
+    DoTestSlideDownLanes(d, expected, N, 59);
+    DoTestSlideDownLanes(d, expected, N, 60);
+    DoTestSlideDownLanes(d, expected, N, 61);
+    DoTestSlideDownLanes(d, expected, N, 62);
+    DoTestSlideDownLanes(d, expected, N, 63);
+  }
+#endif  // HWY_TARGET <= HWY_AVX3
+#endif  // HWY_TARGET <= HWY_AVX2 || HWY_TARGET == HWY_WASM_EMU256
+#endif  // !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    for (size_t i = 0; i < N; i++) {
+      size_t slide_amt = i;
+#if !HWY_COMPILER_MSVC
+      PreventElision(slide_amt);
+#endif
+      DoTestSlideDownLanes(d, expected.get(), N, slide_amt);
+    }
+
+#if !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+    DoTestSlideDownLanesWithConstAmt_0_7(d, expected.get(), N);
+    DoTestSlideDownLanesWithConstAmt_8_15(d, expected.get(), N);
+#if HWY_TARGET <= HWY_AVX2 || HWY_TARGET == HWY_WASM_EMU256
+    DoTestSlideDownLanesWithConstAmt_16_31(d, expected.get(), N);
+#if HWY_TARGET <= HWY_AVX3
+    DoTestSlideDownLanesWithConstAmt_32_63(d, expected.get(), N);
+#endif  // HWY_TARGET <= HWY_AVX3
+#endif  // HWY_TARGET <= HWY_AVX2 || HWY_TARGET == HWY_WASM_EMU256
+#endif  // !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+  }
+};
+
+#undef HWY_SLIDE_DOWN_TEST_INLINE
+
+HWY_NOINLINE void TestAllSlideDownLanes() {
+  ForAllTypes(ForPartialVectors<TestSlideDownLanes>());
+}
+
+struct TestSlide1 {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto iota0 = Iota(d, 0);
+    const auto iota1 = Iota(d, 1);
+
+    const auto expected_slide_down_result =
+        IfThenElseZero(FirstN(d, Lanes(d) - 1), iota1);
+
+    HWY_ASSERT_VEC_EQ(d, iota0, Slide1Up(d, iota1));
+    HWY_ASSERT_VEC_EQ(d, expected_slide_down_result, Slide1Down(d, iota0));
+  }
+};
+
+HWY_NOINLINE void TestAllSlide1() {
+  ForAllTypes(ForPartialVectors<TestSlide1>());
+}
+
+class TestSlideBlocks {
+ private:
+  template <int kBlocks, class D>
+  static HWY_INLINE void DoTestSlideByKBlocks(D d) {
+    using T = TFromD<D>;
+    constexpr size_t kLanesPerBlock = 16 / sizeof(T);
+    constexpr size_t kLanesToSlide =
+        static_cast<size_t>(kBlocks) * kLanesPerBlock;
+
+    const auto iota_0 = Iota(d, 0);
+    const auto iota_k = Iota(d, kLanesToSlide);
+
+    const auto first_k_lanes_mask = FirstN(d, kLanesToSlide);
+    const auto expected_slide_up_result =
+        IfThenZeroElse(first_k_lanes_mask, iota_0);
+    HWY_ASSERT_VEC_EQ(d, expected_slide_up_result,
+                      SlideUpBlocks<kBlocks>(d, iota_k));
+
+    const RebindToUnsigned<decltype(d)> du;
+    using TU = TFromD<decltype(du)>;
+    const auto slide_down_result_mask = BitCast(
+        d, Reverse(du, IfThenZeroElse(RebindMask(du, first_k_lanes_mask),
+                                      Set(du, hwy::LimitsMax<TU>()))));
+
+    const auto expected_slide_down_result = And(slide_down_result_mask, iota_k);
+    HWY_ASSERT_VEC_EQ(d, expected_slide_down_result,
+                      SlideDownBlocks<kBlocks>(d, iota_0));
+  }
+
+#if HWY_MAX_BYTES >= 32
+  template <class D, HWY_IF_V_SIZE_LE_D(D, 16)>
+  static HWY_INLINE void DoTestSlideBy1Block(D /*d*/, size_t /*N*/) {}
+
+  template <class D, HWY_IF_V_SIZE_GT_D(D, 16)>
+  static HWY_INLINE void DoTestSlideBy1Block(D d, size_t N) {
+    if (N < (32 / sizeof(TFromD<D>))) return;
+    DoTestSlideByKBlocks<1>(d);
+  }
+
+#if HWY_MAX_BYTES >= 64
+  template <class D, HWY_IF_V_SIZE_LE_D(D, 32)>
+  static HWY_INLINE void DoTestSlideBy2And3Blocks(D /*d*/, size_t /*N*/) {}
+
+  template <class D, HWY_IF_V_SIZE_GT_D(D, 32)>
+  static HWY_INLINE void DoTestSlideBy2And3Blocks(D d, size_t N) {
+    if (N < (64 / sizeof(TFromD<D>))) return;
+    DoTestSlideByKBlocks<2>(d);
+    DoTestSlideByKBlocks<3>(d);
+  }
+#endif  // HWY_MAX_BYTES >= 64
+#endif  // HWY_MAX_BYTES >= 32
+
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    DoTestSlideByKBlocks<0>(d);
+#if HWY_MAX_BYTES >= 32
+    const size_t N = Lanes(d);
+    DoTestSlideBy1Block(d, N);
+#if HWY_MAX_BYTES >= 64
+    DoTestSlideBy2And3Blocks(d, N);
+#endif  // HWY_MAX_BYTES >= 64
+#endif  // HWY_MAX_BYTES >= 32
+  }
+};
+
+HWY_NOINLINE void TestAllSlideBlocks() {
+  ForAllTypes(ForPartialVectors<TestSlideBlocks>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwySlideUpDownTest);
+HWY_EXPORT_AND_TEST_P(HwySlideUpDownTest, TestAllSlideUpLanes);
+HWY_EXPORT_AND_TEST_P(HwySlideUpDownTest, TestAllSlideDownLanes);
+HWY_EXPORT_AND_TEST_P(HwySlideUpDownTest, TestAllSlide1);
+HWY_EXPORT_AND_TEST_P(HwySlideUpDownTest, TestAllSlideBlocks);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/sums_abs_diff_test.cc

@@ -0,0 +1,353 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/sums_abs_diff_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestSumsOf8AbsDiff {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    RandomState rng;
+    using TW = MakeWide<MakeWide<MakeWide<T>>>;
+
+    const size_t N = Lanes(d);
+    if (N < 8) return;
+    const Repartition<TW, D> d64;
+
+    auto in_lanes_a = AllocateAligned<T>(N);
+    auto in_lanes_b = AllocateAligned<T>(N);
+    auto sum_lanes = AllocateAligned<TW>(N / 8);
+    HWY_ASSERT(in_lanes_a && in_lanes_b && sum_lanes);
+
+    for (size_t rep = 0; rep < 100; ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        uint64_t rand64_val = Random64(&rng);
+        in_lanes_a[i] = ConvertScalarTo<T>(rand64_val & 0xFF);
+        in_lanes_b[i] = ConvertScalarTo<T>((rand64_val >> 8) & 0xFF);
+      }
+
+      for (size_t idx_sum = 0; idx_sum < N / 8; ++idx_sum) {
+        uint64_t sum = 0;
+        for (size_t i = 0; i < 8; ++i) {
+          const auto lane_diff =
+              static_cast<int16_t>(in_lanes_a[idx_sum * 8 + i]) -
+              static_cast<int16_t>(in_lanes_b[idx_sum * 8 + i]);
+          sum +=
+              static_cast<uint64_t>((lane_diff >= 0) ? lane_diff : -lane_diff);
+        }
+        sum_lanes[idx_sum] = static_cast<TW>(sum);
+      }
+
+      const Vec<D> a = Load(d, in_lanes_a.get());
+      const Vec<D> b = Load(d, in_lanes_b.get());
+      HWY_ASSERT_VEC_EQ(d64, sum_lanes.get(), SumsOf8AbsDiff(a, b));
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllSumsOf8AbsDiff() {
+  ForGEVectors<64, TestSumsOf8AbsDiff>()(int8_t());
+  ForGEVectors<64, TestSumsOf8AbsDiff>()(uint8_t());
+}
+
+struct TestSumsOfAdjQuadAbsDiff {
+#if HWY_TARGET != HWY_SCALAR
+  template <size_t kAOffset, size_t kBOffset, class D,
+            HWY_IF_LANES_LE_D(D, kAOffset * 4 + 3)>
+  static HWY_INLINE void DoTestSumsOfAdjQuadAbsDiff(D /*d*/,
+                                                    RandomState& /*rng*/) {}
+
+  template <size_t kAOffset, size_t kBOffset, class D,
+            HWY_IF_LANES_GT_D(D, kAOffset * 4 + 3),
+            HWY_IF_LANES_LE_D(D, kBOffset * 4 + 3)>
+  static HWY_INLINE void DoTestSumsOfAdjQuadAbsDiff(D /*d*/,
+                                                    RandomState& /*rng*/) {}
+
+  template <size_t kAOffset, size_t kBOffset, class D,
+            HWY_IF_LANES_GT_D(D, kAOffset * 4 + 3),
+            HWY_IF_LANES_GT_D(D, kBOffset * 4 + 3)>
+  static HWY_NOINLINE void DoTestSumsOfAdjQuadAbsDiff(D d, RandomState& rng) {
+    static_assert(kAOffset <= 1, "kAOffset <= 1 must be true");
+    static_assert(kBOffset <= 3, "kBOffset <= 3 must be true");
+
+    using T = TFromD<D>;
+    using TW = MakeWide<T>;
+    using TW_I = MakeSigned<TW>;
+    static_assert(sizeof(T) == 1, "sizeof(T) == 1 must be true");
+
+    const RepartitionToWide<decltype(d)> dw;
+
+    const size_t N = Lanes(d);
+    if (N <= (kAOffset * 4 + 3) || N <= (kBOffset * 4 + 3)) {
+      return;
+    }
+
+    const size_t num_valid_sum_lanes =
+        (N < (kAOffset * 4 + 3 + (N / 2))) ? 1 : (N / 2);
+
+    auto in_lanes_a = AllocateAligned<T>(N);
+    auto in_lanes_b = AllocateAligned<T>(N);
+    auto sum_lanes = AllocateAligned<TW>(N / 2);
+    HWY_ASSERT(in_lanes_a && in_lanes_b && sum_lanes);
+
+    ZeroBytes(sum_lanes.get(), (N / 2) * sizeof(TW));
+
+    for (size_t rep = 0; rep < 100; ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        uint64_t rand64_val = Random64(&rng);
+        in_lanes_a[i] = ConvertScalarTo<T>(rand64_val & 0xFF);
+        in_lanes_b[i] = ConvertScalarTo<T>((rand64_val >> 8) & 0xFF);
+      }
+
+      for (size_t i = 0; i < num_valid_sum_lanes; ++i) {
+        size_t blk_idx = i / 8;
+        size_t idx_in_blk = i & 7;
+
+        const TW_I a0 = static_cast<TW_I>(
+            in_lanes_a[blk_idx * 16 + kAOffset * 4 + idx_in_blk]);
+        const TW_I a1 = static_cast<TW_I>(
+            in_lanes_a[blk_idx * 16 + kAOffset * 4 + idx_in_blk + 1]);
+        const TW_I a2 = static_cast<TW_I>(
+            in_lanes_a[blk_idx * 16 + kAOffset * 4 + idx_in_blk + 2]);
+        const TW_I a3 = static_cast<TW_I>(
+            in_lanes_a[blk_idx * 16 + kAOffset * 4 + idx_in_blk + 3]);
+
+        const TW_I b0 =
+            static_cast<TW_I>(in_lanes_b[blk_idx * 16 + kBOffset * 4]);
+        const TW_I b1 =
+            static_cast<TW_I>(in_lanes_b[blk_idx * 16 + kBOffset * 4 + 1]);
+        const TW_I b2 =
+            static_cast<TW_I>(in_lanes_b[blk_idx * 16 + kBOffset * 4 + 2]);
+        const TW_I b3 =
+            static_cast<TW_I>(in_lanes_b[blk_idx * 16 + kBOffset * 4 + 3]);
+
+        const TW_I diff0 = static_cast<TW_I>(ScalarAbs(a0 - b0));
+        const TW_I diff1 = static_cast<TW_I>(ScalarAbs(a1 - b1));
+        const TW_I diff2 = static_cast<TW_I>(ScalarAbs(a2 - b2));
+        const TW_I diff3 = static_cast<TW_I>(ScalarAbs(a3 - b3));
+        sum_lanes[i] = static_cast<TW>(diff0 + diff1 + diff2 + diff3);
+      }
+
+      const Vec<decltype(dw)> actual = IfThenElseZero(
+          FirstN(dw, num_valid_sum_lanes),
+          SumsOfAdjQuadAbsDiff<kAOffset, kBOffset>(Load(d, in_lanes_a.get()),
+                                                   Load(d, in_lanes_b.get())));
+      HWY_ASSERT_VEC_EQ(dw, sum_lanes.get(), actual);
+    }
+  }
+
+  template <class D, class D2 = DFromV<Vec<D>>,
+            HWY_IF_LANES_LE_D(D, HWY_MAX_LANES_D(D2) - 1)>
+  static HWY_INLINE void FullOrFixedVecQuadSumTests(D /*d*/,
+                                                    RandomState& /*rng*/) {}
+
+  template <class D, class D2 = DFromV<Vec<D>>,
+            HWY_IF_LANES_GT_D(D, HWY_MAX_LANES_D(D2) - 1)>
+  static HWY_INLINE void FullOrFixedVecQuadSumTests(D d, RandomState& rng) {
+    DoTestSumsOfAdjQuadAbsDiff<0, 1>(d, rng);
+    DoTestSumsOfAdjQuadAbsDiff<0, 2>(d, rng);
+    DoTestSumsOfAdjQuadAbsDiff<0, 3>(d, rng);
+    DoTestSumsOfAdjQuadAbsDiff<1, 0>(d, rng);
+    DoTestSumsOfAdjQuadAbsDiff<1, 1>(d, rng);
+    DoTestSumsOfAdjQuadAbsDiff<1, 2>(d, rng);
+    DoTestSumsOfAdjQuadAbsDiff<1, 3>(d, rng);
+  }
+#endif  // HWY_TARGET != HWY_SCALAR
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    RandomState rng;
+    DoTestSumsOfAdjQuadAbsDiff<0, 0>(d, rng);
+    FullOrFixedVecQuadSumTests(d, rng);
+#else
+    (void)d;
+#endif  // HWY_TARGET != HWY_SCALAR
+  }
+};
+
+HWY_NOINLINE void TestAllSumsOfAdjQuadAbsDiff() {
+  ForGEVectors<32, TestSumsOfAdjQuadAbsDiff>()(int8_t());
+  ForGEVectors<32, TestSumsOfAdjQuadAbsDiff>()(uint8_t());
+}
+
+struct TestSumsOfShuffledQuadAbsDiff {
+#if HWY_TARGET != HWY_SCALAR
+  template <size_t kIdx3, size_t kIdx2, size_t kIdx1, size_t kIdx0, class D>
+  static HWY_NOINLINE void DoTestSumsOfShuffledQuadAbsDiff(D d,
+                                                           RandomState& rng) {
+    static_assert(kIdx0 <= 3, "kIdx0 <= 3 must be true");
+    static_assert(kIdx1 <= 3, "kIdx1 <= 3 must be true");
+    static_assert(kIdx2 <= 3, "kIdx2 <= 3 must be true");
+    static_assert(kIdx3 <= 3, "kIdx3 <= 3 must be true");
+
+    using T = TFromD<D>;
+    using TW = MakeWide<T>;
+    using TW_I = MakeSigned<TW>;
+    static_assert(sizeof(T) == 1, "sizeof(T) == 1 must be true");
+
+    const RepartitionToWide<decltype(d)> dw;
+    const RepartitionToWide<decltype(dw)> dw2;
+
+    const size_t N = Lanes(d);
+    const size_t num_valid_sum_lanes = (N < 8) ? 1 : (N / 2);
+
+    const size_t in_lanes_a_alloc_len = HWY_MAX(N, 16);
+    auto in_lanes_a = AllocateAligned<T>(in_lanes_a_alloc_len);
+    auto in_lanes_b = AllocateAligned<T>(N);
+    auto a_shuf_lanes = AllocateAligned<T>(in_lanes_a_alloc_len);
+    auto sum_lanes = AllocateAligned<TW>(N / 2);
+    HWY_ASSERT(in_lanes_a && in_lanes_b && a_shuf_lanes && sum_lanes);
+
+    ZeroBytes(in_lanes_a.get(), sizeof(T) * in_lanes_a_alloc_len);
+    ZeroBytes(a_shuf_lanes.get(), sizeof(T) * in_lanes_a_alloc_len);
+    ZeroBytes(sum_lanes.get(), (N / 2) * sizeof(TW));
+
+    for (size_t rep = 0; rep < 100; ++rep) {
+      for (size_t i = 0; i < N; ++i) {
+        uint64_t rand64_val = Random64(&rng);
+        in_lanes_a[i] = ConvertScalarTo<T>(rand64_val & 0xFF);
+        in_lanes_b[i] = ConvertScalarTo<T>((rand64_val >> 8) & 0xFF);
+      }
+
+      const auto a = Load(d, in_lanes_a.get());
+      const auto a_shuf = BitCast(
+          d, Per4LaneBlockShuffle<kIdx3, kIdx2, kIdx1, kIdx0>(BitCast(dw2, a)));
+      Store(a_shuf, d, a_shuf_lanes.get());
+
+      for (size_t i = 0; i < num_valid_sum_lanes; ++i) {
+        size_t blk_idx = i / 8;
+        size_t idx_in_blk = i & 7;
+
+        const auto a0 =
+            static_cast<TW_I>(a_shuf_lanes[blk_idx * 16 + (idx_in_blk / 4) * 8 +
+                                           (idx_in_blk & 3)]);
+        const auto a1 =
+            static_cast<TW_I>(a_shuf_lanes[blk_idx * 16 + (idx_in_blk / 4) * 8 +
+                                           (idx_in_blk & 3) + 1]);
+        const auto a2 =
+            static_cast<TW_I>(a_shuf_lanes[blk_idx * 16 + (idx_in_blk / 4) * 8 +
+                                           (idx_in_blk & 3) + 2]);
+        const auto a3 =
+            static_cast<TW_I>(a_shuf_lanes[blk_idx * 16 + (idx_in_blk / 4) * 8 +
+                                           (idx_in_blk & 3) + 3]);
+
+        const auto b0 = static_cast<TW_I>(in_lanes_b[(i / 2) * 4]);
+        const auto b1 = static_cast<TW_I>(in_lanes_b[(i / 2) * 4 + 1]);
+        const auto b2 = static_cast<TW_I>(in_lanes_b[(i / 2) * 4 + 2]);
+        const auto b3 = static_cast<TW_I>(in_lanes_b[(i / 2) * 4 + 3]);
+
+        const auto diff0 = a0 - b0;
+        const auto diff1 = a1 - b1;
+        const auto diff2 = a2 - b2;
+        const auto diff3 = a3 - b3;
+
+        sum_lanes[i] = static_cast<TW>(((diff0 < 0) ? (-diff0) : diff0) +
+                                       ((diff1 < 0) ? (-diff1) : diff1) +
+                                       ((diff2 < 0) ? (-diff2) : diff2) +
+                                       ((diff3 < 0) ? (-diff3) : diff3));
+      }
+
+      const auto actual =
+          IfThenElseZero(FirstN(dw, num_valid_sum_lanes),
+                         SumsOfShuffledQuadAbsDiff<kIdx3, kIdx2, kIdx1, kIdx0>(
+                             a, Load(d, in_lanes_b.get())));
+      HWY_ASSERT_VEC_EQ(dw, sum_lanes.get(), actual);
+    }
+  }
+
+  template <class D, HWY_IF_LANES_LE_D(D, 4)>
+  static HWY_INLINE void AtLeast8LanesShufQuadSumTests(D /*d*/,
+                                                       RandomState& /*rng*/) {}
+
+  template <class D, HWY_IF_LANES_GT_D(D, 4)>
+  static HWY_INLINE void AtLeast8LanesShufQuadSumTests(D d, RandomState& rng) {
+    if (Lanes(d) >= 8) {
+      DoTestSumsOfShuffledQuadAbsDiff<0, 0, 0, 1>(d, rng);
+    }
+  }
+
+  template <class D, HWY_IF_LANES_LE_D(D, 8)>
+  static HWY_INLINE void AtLeast16LanesShufQuadSumTests(D /*d*/,
+                                                        RandomState& /*rng*/) {}
+
+  template <class D, HWY_IF_LANES_GT_D(D, 8)>
+  static HWY_INLINE void AtLeast16LanesShufQuadSumTests(D d, RandomState& rng) {
+    if (Lanes(d) >= 16) {
+      DoTestSumsOfShuffledQuadAbsDiff<3, 2, 1, 0>(d, rng);
+      DoTestSumsOfShuffledQuadAbsDiff<0, 3, 1, 2>(d, rng);
+      DoTestSumsOfShuffledQuadAbsDiff<2, 3, 0, 1>(d, rng);
+    }
+  }
+
+  template <class D, class D2 = DFromV<Vec<D>>,
+            HWY_IF_LANES_LE_D(D, HWY_MAX_LANES_D(D2) - 1)>
+  static HWY_INLINE void FullOrFixedVecShufQuadSumTests(D /*d*/,
+                                                        RandomState& /*rng*/) {}
+
+  template <class D, class D2 = DFromV<Vec<D>>,
+            HWY_IF_LANES_GT_D(D, HWY_MAX_LANES_D(D2) - 1)>
+  static HWY_INLINE void FullOrFixedVecShufQuadSumTests(D d, RandomState& rng) {
+    AtLeast8LanesShufQuadSumTests(d, rng);
+    AtLeast16LanesShufQuadSumTests(d, rng);
+  }
+#endif  // HWY_TARGET != HWY_SCALAR
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    RandomState rng;
+    DoTestSumsOfShuffledQuadAbsDiff<0, 0, 0, 0>(d, rng);
+    FullOrFixedVecShufQuadSumTests(d, rng);
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllSumsOfShuffledQuadAbsDiff() {
+  ForGEVectors<32, TestSumsOfShuffledQuadAbsDiff>()(int8_t());
+  ForGEVectors<32, TestSumsOfShuffledQuadAbsDiff>()(uint8_t());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwySumsAbsDiffTest);
+HWY_EXPORT_AND_TEST_P(HwySumsAbsDiffTest, TestAllSumsOf8AbsDiff);
+HWY_EXPORT_AND_TEST_P(HwySumsAbsDiffTest, TestAllSumsOfAdjQuadAbsDiff);
+HWY_EXPORT_AND_TEST_P(HwySumsAbsDiffTest, TestAllSumsOfShuffledQuadAbsDiff);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/swizzle_block_test.cc

@@ -0,0 +1,266 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/swizzle_block_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestOddEvenBlocks {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    const auto even = Iota(d, 1);
+    const auto odd = Iota(d, 1 + N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+    for (size_t i = 0; i < N; ++i) {
+      const size_t idx_block = i / (16 / sizeof(T));
+      expected[i] = ConvertScalarTo<T>(1 + i + ((idx_block & 1) ? N : 0));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), OddEvenBlocks(odd, even));
+  }
+};
+
+HWY_NOINLINE void TestAllOddEvenBlocks() {
+  ForAllTypes(ForGEVectors<128, TestOddEvenBlocks>());
+}
+
+struct TestSwapAdjacentBlocks {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    constexpr size_t kLanesPerBlock = 16 / sizeof(T);
+    if (N < 2 * kLanesPerBlock) return;
+    const auto vi = Iota(d, 1);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+    for (size_t i = 0; i < N; ++i) {
+      const size_t idx_block = i / kLanesPerBlock;
+      const size_t base = (idx_block ^ 1) * kLanesPerBlock;
+      const size_t mod = i % kLanesPerBlock;
+      expected[i] = ConvertScalarTo<T>(1 + base + mod);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), SwapAdjacentBlocks(vi));
+  }
+};
+
+HWY_NOINLINE void TestAllSwapAdjacentBlocks() {
+  ForAllTypes(ForGEVectors<128, TestSwapAdjacentBlocks>());
+}
+
+class TestInsertBlock {
+ private:
+  template <int kBlock, class D,
+            HWY_IF_V_SIZE_GT_D(D, static_cast<size_t>(kBlock) * 16)>
+  static HWY_INLINE void DoTestInsertBlock(D d, const size_t N,
+                                           TFromD<D>* HWY_RESTRICT expected) {
+    // kBlock * 16 < D.MaxBytes() is true
+    using T = TFromD<D>;
+    using TI = MakeSigned<T>;
+    using TU = MakeUnsigned<T>;
+
+    const RebindToUnsigned<decltype(d)> du;
+    const BlockDFromD<decltype(d)> d_block;
+    const RebindToUnsigned<decltype(d_block)> du_block;
+    using V = Vec<D>;
+    using VB = Vec<decltype(d_block)>;
+    constexpr TU kPositiveMask = static_cast<TU>(LimitsMax<TI>());
+    constexpr TU kSignBit = static_cast<TU>(~kPositiveMask);
+
+    for (size_t i = 0; i < N; i++) {
+      const T val = ConvertScalarTo<T>(i);
+      TU val_bits;
+      CopySameSize(&val, &val_bits);
+      val_bits &= kPositiveMask;
+      CopySameSize(&val_bits, &expected[i]);
+    }
+
+    constexpr size_t kLanesPer16ByteBlk = 16 / sizeof(T);
+    constexpr size_t kBlkLaneOffset =
+        static_cast<size_t>(kBlock) * kLanesPer16ByteBlk;
+    if (kBlkLaneOffset < N) {
+      const size_t num_of_lanes_in_blk =
+          HWY_MIN(N - kBlkLaneOffset, kLanesPer16ByteBlk);
+      for (size_t i = 0; i < num_of_lanes_in_blk; i++) {
+        const T val =
+            ConvertScalarTo<T>(static_cast<TU>(i) + static_cast<TU>(kBlock));
+        TU val_bits;
+        CopySameSize(&val, &val_bits);
+        val_bits |= kSignBit;
+        CopySameSize(&val_bits, &expected[kBlkLaneOffset + i]);
+      }
+    }
+
+    const V v = And(Iota(d, 0), BitCast(d, Set(du, kPositiveMask)));
+    const VB blk_to_insert =
+        Or(Iota(d_block, kBlock), BitCast(d_block, Set(du_block, kSignBit)));
+    const V actual = InsertBlock<kBlock>(v, blk_to_insert);
+    HWY_ASSERT_VEC_EQ(d, expected, actual);
+  }
+  template <int kBlock, class D,
+            HWY_IF_V_SIZE_LE_D(D, static_cast<size_t>(kBlock) * 16)>
+  static HWY_INLINE void DoTestInsertBlock(
+      D /*d*/, const size_t /*N*/, TFromD<D>* HWY_RESTRICT /*expected*/) {
+    // If kBlock * 16 >= D.MaxBytes() is true, do nothing
+  }
+
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    DoTestInsertBlock<0>(d, N, expected.get());
+    DoTestInsertBlock<1>(d, N, expected.get());
+    DoTestInsertBlock<2>(d, N, expected.get());
+    DoTestInsertBlock<3>(d, N, expected.get());
+  }
+};
+
+HWY_NOINLINE void TestAllInsertBlock() {
+  ForAllTypes(ForPartialFixedOrFullScalableVectors<TestInsertBlock>());
+}
+
+class TestExtractBlock {
+ private:
+  template <int kBlock, class D,
+            HWY_IF_V_SIZE_GT_D(D, static_cast<size_t>(kBlock) * 16)>
+  static HWY_INLINE void DoTestExtractBlock(D d, const size_t N,
+                                            TFromD<D>* HWY_RESTRICT expected) {
+    // kBlock * 16 < D.MaxBytes() is true
+    using T = TFromD<D>;
+
+    constexpr size_t kLanesPer16ByteBlk = 16 / sizeof(T);
+    constexpr size_t kBlkLaneOffset =
+        static_cast<size_t>(kBlock) * kLanesPer16ByteBlk;
+    if (kBlkLaneOffset >= N) return;
+
+    const BlockDFromD<decltype(d)> d_block;
+    static_assert(d_block.MaxLanes() <= kLanesPer16ByteBlk,
+                  "d_block.MaxLanes() <= kLanesPer16ByteBlk must be true");
+
+    for (size_t i = 0; i < kLanesPer16ByteBlk; i++) {
+      expected[i] = ConvertScalarTo<T>(kBlkLaneOffset + i);
+    }
+
+    const auto v = Iota(d, 0);
+    const Vec<BlockDFromD<decltype(d_block)>> actual = ExtractBlock<kBlock>(v);
+    HWY_ASSERT_VEC_EQ(d_block, expected, actual);
+  }
+  template <int kBlock, class D,
+            HWY_IF_V_SIZE_LE_D(D, static_cast<size_t>(kBlock) * 16)>
+  static HWY_INLINE void DoTestExtractBlock(
+      D /*d*/, const size_t /*N*/, TFromD<D>* HWY_RESTRICT /*expected*/) {
+    // If kBlock * 16 >= D.MaxBytes() is true, do nothing
+  }
+
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    constexpr size_t kLanesPer16ByteBlk = 16 / sizeof(T);
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(kLanesPer16ByteBlk);
+    HWY_ASSERT(expected);
+
+    DoTestExtractBlock<0>(d, N, expected.get());
+    DoTestExtractBlock<1>(d, N, expected.get());
+    DoTestExtractBlock<2>(d, N, expected.get());
+    DoTestExtractBlock<3>(d, N, expected.get());
+  }
+};
+
+HWY_NOINLINE void TestAllExtractBlock() {
+  ForAllTypes(ForPartialFixedOrFullScalableVectors<TestExtractBlock>());
+}
+
+class TestBroadcastBlock {
+ private:
+  template <int kBlock, class D,
+            HWY_IF_V_SIZE_GT_D(D, static_cast<size_t>(kBlock) * 16)>
+  static HWY_INLINE void DoTestBroadcastBlock(
+      D d, const size_t N, TFromD<D>* HWY_RESTRICT expected) {
+    // kBlock * 16 < D.MaxBytes() is true
+    using T = TFromD<D>;
+
+    constexpr size_t kLanesPer16ByteBlk = 16 / sizeof(T);
+    constexpr size_t kBlkLaneOffset =
+        static_cast<size_t>(kBlock) * kLanesPer16ByteBlk;
+    if (kBlkLaneOffset >= N) return;
+
+    for (size_t i = 0; i < N; i++) {
+      const size_t idx_in_blk = i & (kLanesPer16ByteBlk - 1);
+      expected[i] =
+          ConvertScalarTo<T>(kBlkLaneOffset + kLanesPer16ByteBlk + idx_in_blk);
+    }
+
+    const auto v = Iota(d, kLanesPer16ByteBlk);
+    const auto actual = BroadcastBlock<kBlock>(v);
+    HWY_ASSERT_VEC_EQ(d, expected, actual);
+  }
+  template <int kBlock, class D,
+            HWY_IF_V_SIZE_LE_D(D, static_cast<size_t>(kBlock) * 16)>
+  static HWY_INLINE void DoTestBroadcastBlock(
+      D /*d*/, const size_t /*N*/, TFromD<D>* HWY_RESTRICT /*expected*/) {
+    // If kBlock * 16 >= D.MaxBytes() is true, do nothing
+  }
+
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+
+    DoTestBroadcastBlock<0>(d, N, expected.get());
+    DoTestBroadcastBlock<1>(d, N, expected.get());
+    DoTestBroadcastBlock<2>(d, N, expected.get());
+    DoTestBroadcastBlock<3>(d, N, expected.get());
+  }
+};
+
+HWY_NOINLINE void TestAllBroadcastBlock() {
+  ForAllTypes(ForPartialFixedOrFullScalableVectors<TestBroadcastBlock>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwySwizzleBlockTest);
+HWY_EXPORT_AND_TEST_P(HwySwizzleBlockTest, TestAllOddEvenBlocks);
+HWY_EXPORT_AND_TEST_P(HwySwizzleBlockTest, TestAllSwapAdjacentBlocks);
+HWY_EXPORT_AND_TEST_P(HwySwizzleBlockTest, TestAllInsertBlock);
+HWY_EXPORT_AND_TEST_P(HwySwizzleBlockTest, TestAllExtractBlock);
+HWY_EXPORT_AND_TEST_P(HwySwizzleBlockTest, TestAllBroadcastBlock);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/swizzle_test.cc

@@ -0,0 +1,422 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#include "hwy/base.h"
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/swizzle_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestGetLane {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v = Iota(d, 1);
+    HWY_ASSERT_EQ(T(1), GetLane(v));
+  }
+};
+
+HWY_NOINLINE void TestAllGetLane() {
+  ForAllTypes(ForPartialVectors<TestGetLane>());
+}
+
+struct TestExtractLane {
+#if !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+  template <class D, HWY_IF_LANES_D(BlockDFromD<D>, 1)>
+  static HWY_INLINE void DoTestExtractLaneWithConstAmt_0_7(D /*d*/, Vec<D> v) {
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(1), ExtractLane(v, 0));
+  }
+
+  template <class D, HWY_IF_LANES_D(BlockDFromD<D>, 2)>
+  static HWY_INLINE void DoTestExtractLaneWithConstAmt_0_7(D /*d*/, Vec<D> v) {
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(1), ExtractLane(v, 0));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(2), ExtractLane(v, 1));
+  }
+
+  template <class D, HWY_IF_LANES_D(BlockDFromD<D>, 4)>
+  static HWY_INLINE void DoTestExtractLaneWithConstAmt_0_7(D /*d*/, Vec<D> v) {
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(1), ExtractLane(v, 0));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(2), ExtractLane(v, 1));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(3), ExtractLane(v, 2));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(4), ExtractLane(v, 3));
+  }
+
+  template <class D, HWY_IF_LANES_GT_D(BlockDFromD<D>, 4)>
+  static HWY_INLINE void DoTestExtractLaneWithConstAmt_0_7(D /*d*/, Vec<D> v) {
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(1), ExtractLane(v, 0));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(2), ExtractLane(v, 1));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(3), ExtractLane(v, 2));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(4), ExtractLane(v, 3));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(5), ExtractLane(v, 4));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(6), ExtractLane(v, 5));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(7), ExtractLane(v, 6));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(8), ExtractLane(v, 7));
+  }
+
+  template <class D, HWY_IF_LANES_LE_D(BlockDFromD<D>, 8)>
+  static HWY_INLINE void DoTestExtractLaneWithConstAmt_8_15(D /*d*/,
+                                                            Vec<D> /*v*/) {}
+
+  template <class D, HWY_IF_LANES_GT_D(BlockDFromD<D>, 8)>
+  static HWY_INLINE void DoTestExtractLaneWithConstAmt_8_15(D /*d*/, Vec<D> v) {
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(9), ExtractLane(v, 8));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(10), ExtractLane(v, 9));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(11), ExtractLane(v, 10));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(12), ExtractLane(v, 11));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(13), ExtractLane(v, 12));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(14), ExtractLane(v, 13));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(15), ExtractLane(v, 14));
+    HWY_ASSERT_EQ(ConvertScalarTo<TFromD<D>>(16), ExtractLane(v, 15));
+  }
+#endif  // !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto v = Iota(d, 1);
+
+#if !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+    DoTestExtractLaneWithConstAmt_0_7(d, v);
+    DoTestExtractLaneWithConstAmt_8_15(d, v);
+#endif  // !HWY_HAVE_SCALABLE && HWY_TARGET < HWY_EMU128 && !HWY_TARGET_IS_SVE
+
+    for (size_t i = 0; i < Lanes(d); ++i) {
+      const T actual = ExtractLane(v, i);
+      HWY_ASSERT_EQ(ConvertScalarTo<T>(i + 1), actual);
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllExtractLane() {
+  ForAllTypes(ForPartialVectors<TestExtractLane>());
+}
+
+struct TestInsertLane {
+#if !HWY_HAVE_SCALABLE
+  template <class D, HWY_IF_LANES_D(BlockDFromD<D>, 1)>
+  static HWY_INLINE void DoTestInsertLaneWithConstAmt_0_7(
+      D d, TFromD<D>* HWY_RESTRICT lanes) {
+    using T = TFromD<D>;
+
+    lanes[0] = ConvertScalarTo<T>(1);
+    Vec<D> v = InsertLane(Zero(d), 0, ConvertScalarTo<T>(1));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+  }
+  template <class D, HWY_IF_LANES_D(BlockDFromD<D>, 2)>
+  static HWY_INLINE void DoTestInsertLaneWithConstAmt_0_7(
+      D d, TFromD<D>* HWY_RESTRICT lanes) {
+    using T = TFromD<D>;
+
+    lanes[0] = ConvertScalarTo<T>(1);
+    Vec<D> v = InsertLane(Zero(d), 0, ConvertScalarTo<T>(1));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[1] = ConvertScalarTo<T>(2);
+    v = InsertLane(v, 1, ConvertScalarTo<T>(2));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+  }
+  template <class D, HWY_IF_LANES_D(BlockDFromD<D>, 4)>
+  static HWY_INLINE void DoTestInsertLaneWithConstAmt_0_7(
+      D d, TFromD<D>* HWY_RESTRICT lanes) {
+    using T = TFromD<D>;
+
+    lanes[0] = ConvertScalarTo<T>(1);
+    Vec<D> v = InsertLane(Zero(d), 0, ConvertScalarTo<T>(1));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[1] = ConvertScalarTo<T>(2);
+    v = InsertLane(v, 1, ConvertScalarTo<T>(2));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[2] = ConvertScalarTo<T>(3);
+    v = InsertLane(v, 2, ConvertScalarTo<T>(3));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[3] = ConvertScalarTo<T>(4);
+    v = InsertLane(v, 3, ConvertScalarTo<T>(4));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+  }
+  template <class D, HWY_IF_LANES_GT_D(BlockDFromD<D>, 4)>
+  static HWY_INLINE void DoTestInsertLaneWithConstAmt_0_7(
+      D d, TFromD<D>* HWY_RESTRICT lanes) {
+    using T = TFromD<D>;
+
+    lanes[0] = ConvertScalarTo<T>(1);
+    Vec<D> v = InsertLane(Zero(d), 0, ConvertScalarTo<T>(1));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[1] = ConvertScalarTo<T>(2);
+    v = InsertLane(v, 1, ConvertScalarTo<T>(2));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[2] = ConvertScalarTo<T>(3);
+    v = InsertLane(v, 2, ConvertScalarTo<T>(3));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[3] = ConvertScalarTo<T>(4);
+    v = InsertLane(v, 3, ConvertScalarTo<T>(4));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[4] = ConvertScalarTo<T>(5);
+    v = InsertLane(v, 4, ConvertScalarTo<T>(5));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[5] = ConvertScalarTo<T>(6);
+    v = InsertLane(v, 5, ConvertScalarTo<T>(6));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[6] = ConvertScalarTo<T>(7);
+    v = InsertLane(v, 6, ConvertScalarTo<T>(7));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[7] = ConvertScalarTo<T>(8);
+    v = InsertLane(v, 7, ConvertScalarTo<T>(8));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+  }
+  template <class D, HWY_IF_LANES_LE_D(BlockDFromD<D>, 8)>
+  static HWY_INLINE void DoTestInsertLaneWithConstAmt_8_15(
+      D, TFromD<D>* HWY_RESTRICT) {}
+  template <class D, HWY_IF_LANES_GT_D(BlockDFromD<D>, 8)>
+  static HWY_INLINE void DoTestInsertLaneWithConstAmt_8_15(
+      D d, TFromD<D>* HWY_RESTRICT lanes) {
+    using T = TFromD<D>;
+    Vec<D> v = Load(d, lanes);
+
+    lanes[8] = ConvertScalarTo<T>(9);
+    v = InsertLane(v, 8, ConvertScalarTo<T>(9));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[9] = ConvertScalarTo<T>(10);
+    v = InsertLane(v, 9, ConvertScalarTo<T>(10));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[10] = ConvertScalarTo<T>(11);
+    v = InsertLane(v, 10, ConvertScalarTo<T>(11));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[11] = ConvertScalarTo<T>(12);
+    v = InsertLane(v, 11, ConvertScalarTo<T>(12));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[12] = ConvertScalarTo<T>(13);
+    v = InsertLane(v, 12, ConvertScalarTo<T>(13));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[13] = ConvertScalarTo<T>(14);
+    v = InsertLane(v, 13, ConvertScalarTo<T>(14));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[14] = ConvertScalarTo<T>(15);
+    v = InsertLane(v, 14, ConvertScalarTo<T>(15));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+
+    lanes[15] = ConvertScalarTo<T>(16);
+    v = InsertLane(v, 15, ConvertScalarTo<T>(16));
+    HWY_ASSERT_VEC_EQ(d, lanes, v);
+  }
+  template <class D, HWY_IF_V_SIZE_LE_D(D, 16)>
+  static HWY_INLINE void DoTestInsertLaneWithConstAmt(
+      D d, TFromD<D>* HWY_RESTRICT lanes) {
+    DoTestInsertLaneWithConstAmt_0_7(d, lanes);
+    DoTestInsertLaneWithConstAmt_8_15(d, lanes);
+    Store(Zero(d), d, lanes);
+  }
+  template <class D, HWY_IF_V_SIZE_GT_D(D, 16)>
+  static HWY_INLINE void DoTestInsertLaneWithConstAmt(D,
+                                                      TFromD<D>* HWY_RESTRICT) {
+  }
+#endif  // !HWY_HAVE_SCALABLE
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    using V = Vec<D>;
+    const V v = IotaForSpecial(d, 1);
+    const size_t N = Lanes(d);
+    auto lanes = AllocateAligned<T>(N);
+    HWY_ASSERT(lanes);
+    Store(Zero(d), d, lanes.get());
+
+#if !HWY_HAVE_SCALABLE
+    DoTestInsertLaneWithConstAmt(d, lanes.get());
+#endif
+
+// TODO(janwas): file compiler bug report
+#if HWY_COMPILER_CLANG && (HWY_COMPILER_CLANG < 1900) && HWY_ARCH_ARM
+    if (IsSpecialFloat<T>()) return;
+#endif
+
+    V v2 = Zero(d);
+    for (size_t i = 0; i < N; ++i) {
+      lanes[i] = ConvertScalarTo<T>(i + 1);
+      v2 = InsertLane(v2, i, ConvertScalarTo<T>(i + 1));
+      HWY_ASSERT_VEC_EQ(d, lanes.get(), v2);
+    }
+    HWY_ASSERT_VEC_EQ(d, v, v2);
+
+    for (size_t i = 0; i < N; ++i) {
+      lanes[i] = ConvertScalarTo<T>(0);
+      const V v3 = Load(d, lanes.get());
+      const V actual = InsertLane(v3, i, ConvertScalarTo<T>(i + 1));
+      HWY_ASSERT_VEC_EQ(d, v, actual);
+      lanes[i] = ConvertScalarTo<T>(i + 1);  // restore lane i
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllInsertLane() {
+  ForAllTypesAndSpecial(ForPartialVectors<TestInsertLane>());
+}
+
+struct TestDupEven {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>((i & ~size_t{1}) + 1);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), DupEven(Iota(d, 1)));
+  }
+};
+
+HWY_NOINLINE void TestAllDupEven() {
+  ForAllTypes(ForShrinkableVectors<TestDupEven>());
+}
+
+struct TestDupOdd {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    const size_t N = Lanes(d);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>((i & ~size_t{1}) + 2);
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), DupOdd(Iota(d, 1)));
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllDupOdd() {
+  ForAllTypes(ForShrinkableVectors<TestDupOdd>());
+}
+
+struct TestOddEven {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const size_t N = Lanes(d);
+    const auto even = Iota(d, 1);
+    const auto odd = Iota(d, 1 + N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(expected);
+    for (size_t i = 0; i < N; ++i) {
+      expected[i] = ConvertScalarTo<T>(1 + i + ((i & 1) ? N : 0));
+    }
+    HWY_ASSERT_VEC_EQ(d, expected.get(), OddEven(odd, even));
+  }
+};
+
+HWY_NOINLINE void TestAllOddEven() {
+  ForAllTypes(ForShrinkableVectors<TestOddEven>());
+}
+
+class TestBroadcastLane {
+ private:
+  template <int kLane, class D,
+            HWY_IF_LANES_GT_D(D, static_cast<size_t>(kLane))>
+  static HWY_INLINE void DoTestBroadcastLane(D d, const size_t N) {
+    using T = TFromD<D>;
+    // kLane < HWY_MAX_LANES_D(D) is true
+    if (kLane >= N) return;
+
+    const Vec<D> expected = Set(d, ConvertScalarTo<T>(kLane + 1));
+
+    const BlockDFromD<decltype(d)> d_block;
+    static_assert(d_block.MaxLanes() <= d.MaxLanes(),
+                  "d_block.MaxLanes() <= d.MaxLanes() must be true");
+    constexpr size_t kLanesPer16ByteBlk = 16 / sizeof(T);
+    constexpr int kBlockIdx = kLane / static_cast<int>(kLanesPer16ByteBlk);
+    constexpr int kLaneInBlkIdx =
+        kLane & static_cast<int>(kLanesPer16ByteBlk - 1);
+
+    const Vec<D> v = Iota(d, 1);
+    const Vec<D> actual = BroadcastLane<kLane>(v);
+    const Vec<decltype(d_block)> actual_block =
+        ExtractBlock<kBlockIdx>(Broadcast<kLaneInBlkIdx>(v));
+
+    HWY_ASSERT_VEC_EQ(d, expected, actual);
+    HWY_ASSERT_VEC_EQ(d_block, ResizeBitCast(d_block, expected), actual_block);
+  }
+  template <int kLane, class D,
+            HWY_IF_LANES_LE_D(D, static_cast<size_t>(kLane))>
+  static HWY_INLINE void DoTestBroadcastLane(D /*d*/, const size_t /*N*/) {
+    // If kLane >= HWY_MAX_LANES_D(D) is true, do nothing
+  }
+
+ public:
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const auto N = Lanes(d);
+
+    DoTestBroadcastLane<0>(d, N);
+    DoTestBroadcastLane<1>(d, N);
+    DoTestBroadcastLane<2>(d, N);
+    DoTestBroadcastLane<3>(d, N);
+    DoTestBroadcastLane<6>(d, N);
+    DoTestBroadcastLane<14>(d, N);
+    DoTestBroadcastLane<29>(d, N);
+    DoTestBroadcastLane<53>(d, N);
+    DoTestBroadcastLane<115>(d, N);
+    DoTestBroadcastLane<251>(d, N);
+    DoTestBroadcastLane<257>(d, N);
+  }
+};
+
+HWY_NOINLINE void TestAllBroadcastLane() {
+  ForAllTypes(ForPartialFixedOrFullScalableVectors<TestBroadcastLane>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwySwizzleTest);
+HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllGetLane);
+HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllExtractLane);
+HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllInsertLane);
+HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllDupEven);
+HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllDupOdd);
+HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllOddEven);
+HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllBroadcastLane);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/table_test.cc

@@ -0,0 +1,213 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/table_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestTableLookupLanes {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const RebindToSigned<D> di;
+    using TI = TFromD<decltype(di)>;
+#if HWY_TARGET != HWY_SCALAR
+    const size_t N = Lanes(d);
+    auto idx = AllocateAligned<TI>(N);
+    auto expected = AllocateAligned<T>(N);
+    HWY_ASSERT(idx && expected);
+    ZeroBytes(idx.get(), N * sizeof(TI));
+    const auto v = Iota(d, 1);
+
+    if (N <= 8) {  // Test all permutations
+      for (size_t i0 = 0; i0 < N; ++i0) {
+        idx[0] = static_cast<TI>(i0);
+
+        for (size_t i1 = 0; i1 < N; ++i1) {
+          if (N >= 2) idx[1] = static_cast<TI>(i1);
+          for (size_t i2 = 0; i2 < N; ++i2) {
+            if (N >= 4) idx[2] = static_cast<TI>(i2);
+            for (size_t i3 = 0; i3 < N; ++i3) {
+              if (N >= 4) idx[3] = static_cast<TI>(i3);
+
+              for (size_t i = 0; i < N; ++i) {
+                expected[i] = ConvertScalarTo<T>(idx[i] + 1);  // == v[idx[i]]
+              }
+
+              const auto opaque1 = IndicesFromVec(d, Load(di, idx.get()));
+              const auto actual1 = TableLookupLanes(v, opaque1);
+              HWY_ASSERT_VEC_EQ(d, expected.get(), actual1);
+
+              const auto opaque2 = SetTableIndices(d, idx.get());
+              const auto actual2 = TableLookupLanes(v, opaque2);
+              HWY_ASSERT_VEC_EQ(d, expected.get(), actual2);
+            }
+          }
+        }
+      }
+    } else {
+      // Too many permutations to test exhaustively; choose one with repeated
+      // and cross-block indices and ensure indices do not exceed #lanes.
+      // For larger vectors, upper lanes will be zero.
+      HWY_ALIGN TI idx_source[16] = {1,  3,  2,  2,  8, 1, 7, 6,
+                                     15, 14, 14, 15, 4, 9, 8, 5};
+      for (size_t i = 0; i < N; ++i) {
+        idx[i] = (i < 16) ? idx_source[i] : 0;
+        // Avoid undefined results / asan error for scalar by capping indices.
+        if (idx[i] >= static_cast<TI>(N)) {
+          idx[i] = static_cast<TI>(N - 1);
+        }
+        expected[i] = ConvertScalarTo<T>(idx[i] + 1);  // == v[idx[i]]
+      }
+
+      const auto opaque1 = IndicesFromVec(d, Load(di, idx.get()));
+      const auto actual1 = TableLookupLanes(v, opaque1);
+      HWY_ASSERT_VEC_EQ(d, expected.get(), actual1);
+
+      const auto opaque2 = SetTableIndices(d, idx.get());
+      const auto actual2 = TableLookupLanes(v, opaque2);
+      HWY_ASSERT_VEC_EQ(d, expected.get(), actual2);
+    }
+#else
+    const TI index = 0;
+    const auto v = Set(d, 1);
+    const auto opaque1 = SetTableIndices(d, &index);
+    HWY_ASSERT_VEC_EQ(d, v, TableLookupLanes(v, opaque1));
+    const auto opaque2 = IndicesFromVec(d, Zero(di));
+    HWY_ASSERT_VEC_EQ(d, v, TableLookupLanes(v, opaque2));
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllTableLookupLanes() {
+  ForAllTypes(ForPartialVectors<TestTableLookupLanes>());
+}
+
+struct TestTwoTablesLookupLanes {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+    const RebindToUnsigned<D> du;
+    using TU = TFromD<decltype(du)>;
+
+    const size_t N = Lanes(d);
+    const size_t twiceN = N * 2;
+    auto idx = AllocateAligned<TU>(twiceN);
+    auto expected = AllocateAligned<T>(twiceN);
+    HWY_ASSERT(idx && expected);
+    ZeroBytes(idx.get(), twiceN * sizeof(TU));
+    const auto a = Iota(d, 1);
+    const auto b = Add(a, Set(d, ConvertScalarTo<T>(N)));
+
+    if (twiceN <= 8) {  // Test all permutations
+      for (size_t i0 = 0; i0 < twiceN; ++i0) {
+        idx[0] = static_cast<TU>(i0);
+
+        for (size_t i1 = 0; i1 < twiceN; ++i1) {
+          if (twiceN >= 2) idx[1] = static_cast<TU>(i1);
+          for (size_t i2 = 0; i2 < twiceN; ++i2) {
+            if (twiceN >= 4) idx[2] = static_cast<TU>(i2);
+            for (size_t i3 = 0; i3 < twiceN; ++i3) {
+              if (twiceN >= 4) idx[3] = static_cast<TU>(i3);
+
+              for (size_t i = 0; i < twiceN; ++i) {
+                expected[i] = ConvertScalarTo<T>(idx[i] + 1);  // == v[idx[i]]
+              }
+
+              const auto opaque1_a = IndicesFromVec(d, Load(du, idx.get()));
+              const auto opaque1_b = IndicesFromVec(d, Load(du, idx.get() + N));
+              const auto actual1_a = TwoTablesLookupLanes(d, a, b, opaque1_a);
+              const auto actual1_b = TwoTablesLookupLanes(d, a, b, opaque1_b);
+              HWY_ASSERT_VEC_EQ(d, expected.get(), actual1_a);
+              HWY_ASSERT_VEC_EQ(d, expected.get() + N, actual1_b);
+
+              const auto opaque2_a = SetTableIndices(d, idx.get());
+              const auto opaque2_b = SetTableIndices(d, idx.get() + N);
+              const auto actual2_a = TwoTablesLookupLanes(d, a, b, opaque2_a);
+              const auto actual2_b = TwoTablesLookupLanes(d, a, b, opaque2_b);
+              HWY_ASSERT_VEC_EQ(d, expected.get(), actual2_a);
+              HWY_ASSERT_VEC_EQ(d, expected.get() + N, actual2_b);
+            }
+          }
+        }
+      }
+    } else {
+      constexpr size_t kLanesPerBlock = 16 / sizeof(T);
+      constexpr size_t kMaxBlockIdx = static_cast<size_t>(LimitsMax<TU>()) >> 1;
+      static_assert(kMaxBlockIdx > 0, "kMaxBlockIdx > 0 must be true");
+
+      const size_t num_of_blocks_per_vect = HWY_MAX(N / kLanesPerBlock, 1);
+      const size_t num_of_blocks_to_check =
+          HWY_MIN(num_of_blocks_per_vect * 2, kMaxBlockIdx);
+
+      for (size_t i = 0; i < num_of_blocks_to_check; i++) {
+        // Too many permutations to test exhaustively; choose one with repeated
+        // and cross-block indices and ensure indices do not exceed #lanes.
+        // For larger vectors, upper lanes will be zero.
+        HWY_ALIGN TU idx_source[16] = {1,  3,  2,  2,  8, 1, 7, 6,
+                                       15, 14, 14, 15, 4, 9, 8, 5};
+        for (size_t j = 0; j < twiceN; ++j) {
+          idx[j] = static_cast<TU>((i * kLanesPerBlock + idx_source[j & 15] +
+                                    (j & static_cast<size_t>(-16))) &
+                                   (twiceN - 1));
+          expected[j] = ConvertScalarTo<T>(idx[j] + 1);  // == v[idx[j]]
+        }
+
+        const auto opaque1_a = IndicesFromVec(d, Load(du, idx.get()));
+        const auto opaque1_b = IndicesFromVec(d, Load(du, idx.get() + N));
+        const auto actual1_a = TwoTablesLookupLanes(d, a, b, opaque1_a);
+        const auto actual1_b = TwoTablesLookupLanes(d, a, b, opaque1_b);
+        HWY_ASSERT_VEC_EQ(d, expected.get(), actual1_a);
+        HWY_ASSERT_VEC_EQ(d, expected.get() + N, actual1_b);
+
+        const auto opaque2_a = SetTableIndices(d, idx.get());
+        const auto opaque2_b = SetTableIndices(d, idx.get() + N);
+        const auto actual2_a = TwoTablesLookupLanes(d, a, b, opaque2_a);
+        const auto actual2_b = TwoTablesLookupLanes(d, a, b, opaque2_b);
+        HWY_ASSERT_VEC_EQ(d, expected.get(), actual2_a);
+        HWY_ASSERT_VEC_EQ(d, expected.get() + N, actual2_b);
+      }
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllTwoTablesLookupLanes() {
+  ForAllTypes(ForPartialVectors<TestTwoTablesLookupLanes>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyTableTest);
+HWY_EXPORT_AND_TEST_P(HwyTableTest, TestAllTableLookupLanes);
+HWY_EXPORT_AND_TEST_P(HwyTableTest, TestAllTwoTablesLookupLanes);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/test_util-inl.h

@@ -0,0 +1,902 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Target-specific helper functions for use by *_test.cc.
+
+#include <stdio.h>
+#include <string.h>  // memset
+
+// IWYU pragma: begin_exports
+#include "hwy/aligned_allocator.h"
+#include "hwy/base.h"
+#include "hwy/detect_targets.h"
+#include "hwy/per_target.h"
+#include "hwy/targets.h"
+#include "hwy/tests/hwy_gtest.h"
+#include "hwy/tests/test_util.h"
+// IWYU pragma: end_exports
+
+// After test_util (also includes highway.h)
+#include "hwy/print-inl.h"
+
+// Per-target include guard
+// clang-format off
+#if defined(HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_) == defined(HWY_TARGET_TOGGLE)  // NOLINT
+// clang-format on
+#ifdef HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
+#undef HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
+#else
+#define HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
+#endif
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+
+// Like Iota, but avoids wrapping around to negative integers.
+template <class D, HWY_IF_FLOAT_D(D)>
+HWY_INLINE Vec<D> PositiveIota(D d) {
+  return Iota(d, 1);
+}
+template <class D, HWY_IF_NOT_FLOAT_NOR_SPECIAL_D(D)>
+HWY_INLINE Vec<D> PositiveIota(D d) {
+  const auto vi = Iota(d, 1);
+  return Max(And(vi, Set(d, LimitsMax<TFromD<D>>())),
+             Set(d, static_cast<TFromD<D>>(1)));
+}
+
+// Same as Iota, but supports bf16. This is possibly too expensive for general
+// use, but fine for tests.
+template <class D, typename First, HWY_IF_NOT_SPECIAL_FLOAT_D(D)>
+VFromD<D> IotaForSpecial(D d, First first) {
+  return Iota(d, first);
+}
+#if HWY_HAVE_FLOAT16
+template <class D, typename First, HWY_IF_F16_D(D), HWY_IF_LANES_GT_D(D, 1)>
+VFromD<D> IotaForSpecial(D d, First first) {
+  return Iota(d, first);
+}
+#else   // !HWY_HAVE_FLOAT16
+template <class D, typename First, HWY_IF_F16_D(D), HWY_IF_LANES_GT_D(D, 1),
+          HWY_IF_POW2_GT_D(D, -1)>
+VFromD<D> IotaForSpecial(D d, First first) {
+  const Repartition<float, D> df;
+  const size_t NW = Lanes(d) / 2;
+  const Half<D> dh;
+  const float first2 = static_cast<float>(first) + static_cast<float>(NW);
+  return Combine(d, DemoteTo(dh, Iota(df, first2)),
+                 DemoteTo(dh, Iota(df, first)));
+  // TODO(janwas): enable when supported for f16
+  // return OrderedDemote2To(d, Iota(df, first), Iota(df, first + NW));
+}
+// For partial vectors, a single f32 vector is enough, and the prior overload
+// might not be able to Repartition.
+template <class D, typename First, HWY_IF_F16_D(D), HWY_IF_LANES_GT_D(D, 1),
+          HWY_IF_POW2_LE_D(D, -1)>
+VFromD<D> IotaForSpecial(D d, First first) {
+  const Rebind<float, D> df;
+  return DemoteTo(d, Iota(df, first));
+}
+#endif  // HWY_HAVE_FLOAT16
+template <class D, typename First, HWY_IF_BF16_D(D), HWY_IF_LANES_GT_D(D, 1),
+          HWY_IF_POW2_GT_D(D, -1)>
+VFromD<D> IotaForSpecial(D d, First first) {
+  const Repartition<float, D> df;
+  const float first1 = ConvertScalarTo<float>(first);
+  const float first2 = first1 + static_cast<float>(Lanes(d) / 2);
+  return OrderedDemote2To(d, Iota(df, first1), Iota(df, first2));
+}
+// For partial vectors, a single f32 vector is enough, and the prior overload
+// might not be able to Repartition.
+template <class D, typename First, HWY_IF_BF16_D(D), HWY_IF_LANES_GT_D(D, 1),
+          HWY_IF_POW2_LE_D(D, -1)>
+VFromD<D> IotaForSpecial(D d, First first) {
+  const Rebind<float, D> df;
+  return DemoteTo(d, Iota(df, first));
+}
+// OrderedDemote2To does not work for single lanes, so special-case that.
+template <class D, typename First, HWY_IF_SPECIAL_FLOAT_D(D),
+          HWY_IF_LANES_D(D, 1)>
+VFromD<D> IotaForSpecial(D d, First first) {
+  const Rebind<float, D> df;
+  return DemoteTo(d, Set(df, static_cast<float>(first)));
+}
+
+// Compare expected array to vector.
+// TODO(b/287462770): inline to work around incorrect SVE codegen.
+template <class D, typename T = TFromD<D>>
+HWY_INLINE void AssertVecEqual(D d, const T* expected, Vec<D> actual,
+                               const char* filename, const int line) {
+  const size_t N = Lanes(d);
+  auto actual_lanes = AllocateAligned<T>(N);
+  HWY_ASSERT(actual_lanes);
+  Store(actual, d, actual_lanes.get());
+
+  const auto info = hwy::detail::MakeTypeInfo<T>();
+  const char* target_name = hwy::TargetName(HWY_TARGET);
+  hwy::detail::AssertArrayEqual(info, expected, actual_lanes.get(), N,
+                                target_name, filename, line);
+}
+
+// Compare expected vector to vector.
+// TODO(b/287462770): inline to work around incorrect SVE codegen.
+template <class D, typename T = TFromD<D>>
+HWY_INLINE void AssertVecEqual(D d, Vec<D> expected, Vec<D> actual,
+                               const char* filename, int line) {
+  const size_t N = Lanes(d);
+  auto expected_lanes = AllocateAligned<T>(N);
+  auto actual_lanes = AllocateAligned<T>(N);
+  HWY_ASSERT(expected_lanes && actual_lanes);
+  Store(expected, d, expected_lanes.get());
+  Store(actual, d, actual_lanes.get());
+
+  const auto info = hwy::detail::MakeTypeInfo<T>();
+  const char* target_name = hwy::TargetName(HWY_TARGET);
+  hwy::detail::AssertArrayEqual(info, expected_lanes.get(), actual_lanes.get(),
+                                N, target_name, filename, line);
+}
+
+// Only checks the valid mask elements (those whose index < Lanes(d)).
+template <class D>
+HWY_NOINLINE void AssertMaskEqual(D d, VecArg<Mask<D>> a, VecArg<Mask<D>> b,
+                                  const char* filename, int line) {
+  // lvalues prevented MSAN failure in farm_sve.
+  const Vec<D> va = VecFromMask(d, a);
+  const Vec<D> vb = VecFromMask(d, b);
+  AssertVecEqual(d, va, vb, filename, line);
+
+  const char* target_name = hwy::TargetName(HWY_TARGET);
+  AssertEqual(CountTrue(d, a), CountTrue(d, b), target_name, filename, line);
+  AssertEqual(AllTrue(d, a), AllTrue(d, b), target_name, filename, line);
+  AssertEqual(AllFalse(d, a), AllFalse(d, b), target_name, filename, line);
+
+  const size_t N = Lanes(d);
+#if HWY_TARGET == HWY_SCALAR
+  const Rebind<uint8_t, D> d8;
+#else
+  const Repartition<uint8_t, D> d8;
+#endif
+  const size_t N8 = Lanes(d8);
+  auto bits_a = AllocateAligned<uint8_t>(HWY_MAX(size_t{8}, N8));
+  auto bits_b = AllocateAligned<uint8_t>(size_t{HWY_MAX(8, N8)});
+  HWY_ASSERT(bits_a && bits_b);
+  memset(bits_a.get(), 0, N8);
+  memset(bits_b.get(), 0, N8);
+  const size_t num_bytes_a = StoreMaskBits(d, a, bits_a.get());
+  const size_t num_bytes_b = StoreMaskBits(d, b, bits_b.get());
+  AssertEqual(num_bytes_a, num_bytes_b, target_name, filename, line);
+  size_t i = 0;
+  // First check whole bytes (if that many elements are still valid)
+  for (; i < N / 8; ++i) {
+    if (bits_a[i] != bits_b[i]) {
+      fprintf(stderr, "Mismatch in byte %d: %d != %d\n", static_cast<int>(i),
+              bits_a[i], bits_b[i]);
+      Print(d8, "expect", Load(d8, bits_a.get()), 0, N8);
+      Print(d8, "actual", Load(d8, bits_b.get()), 0, N8);
+      hwy::Abort(filename, line, "Masks not equal");
+    }
+  }
+  // Then the valid bit(s) in the last byte.
+  const size_t remainder = N % 8;
+  if (remainder != 0) {
+    const int mask = (1 << remainder) - 1;
+    const int valid_a = bits_a[i] & mask;
+    const int valid_b = bits_b[i] & mask;
+    if (valid_a != valid_b) {
+      fprintf(stderr, "Mismatch in last byte %d: %d != %d\n",
+              static_cast<int>(i), valid_a, valid_b);
+      Print(d8, "expect", Load(d8, bits_a.get()), 0, N8);
+      Print(d8, "actual", Load(d8, bits_b.get()), 0, N8);
+      hwy::Abort(filename, line, "Masks not equal");
+    }
+  }
+}
+
+// Only sets valid elements (those whose index < Lanes(d)). This helps catch
+// tests that are not masking off the (undefined) upper mask elements.
+//
+// TODO(janwas): with HWY_NOINLINE GCC zeros the upper half of AVX2 masks.
+template <class D>
+HWY_INLINE Mask<D> MaskTrue(const D d) {
+  return FirstN(d, Lanes(d));
+}
+
+// MaskFalse is now implemented in x86_128-inl.h on AVX3, arm_sve-inl.h on SVE,
+// rvv-inl.h on RVV, and generic_ops-inl.h on all other targets
+
+#ifndef HWY_ASSERT_EQ
+
+#define HWY_ASSERT_EQ(expected, actual)                                     \
+  hwy::AssertEqual(expected, actual, hwy::TargetName(HWY_TARGET), __FILE__, \
+                   __LINE__)
+
+#define HWY_ASSERT_ARRAY_EQ(expected, actual, count)                          \
+  hwy::AssertArrayEqual(expected, actual, count, hwy::TargetName(HWY_TARGET), \
+                        __FILE__, __LINE__)
+
+#define HWY_ASSERT_STRING_EQ(expected, actual)                          \
+  hwy::AssertStringEqual(expected, actual, hwy::TargetName(HWY_TARGET), \
+                         __FILE__, __LINE__)
+
+#define HWY_ASSERT_VEC_EQ(d, expected, actual) \
+  AssertVecEqual(d, expected, actual, __FILE__, __LINE__)
+
+#define HWY_ASSERT_MASK_EQ(d, expected, actual) \
+  AssertMaskEqual(d, expected, actual, __FILE__, __LINE__)
+
+#endif  // HWY_ASSERT_EQ
+
+namespace detail {
+
+// Helpers for instantiating tests with combinations of lane types / counts.
+
+// Calls Test for each CappedTag<T, N> where N is in [kMinLanes, kMul * kMinArg]
+// and the resulting Lanes() is in [min_lanes, max_lanes]. The upper bound
+// is required to ensure capped vectors remain extendable. Implemented by
+// recursively halving kMul until it is zero.
+template <typename T, size_t kMul, size_t kMinArg, class Test, int kPow2 = 0>
+struct ForeachCappedR {
+  static void Do(size_t min_lanes, size_t max_lanes) {
+    const CappedTag<T, kMul * kMinArg, kPow2> d;
+
+    // If we already don't have enough lanes, stop.
+    const size_t lanes = Lanes(d);
+    if (lanes < min_lanes) return;
+
+    if (lanes <= max_lanes) {
+      Test()(T(), d);
+    }
+    ForeachCappedR<T, kMul / 2, kMinArg, Test, kPow2>::Do(min_lanes, max_lanes);
+  }
+};
+
+// Base case to stop the recursion.
+template <typename T, size_t kMinArg, class Test, int kPow2>
+struct ForeachCappedR<T, 0, kMinArg, Test, kPow2> {
+  static void Do(size_t, size_t) {}
+};
+
+#if HWY_HAVE_SCALABLE
+
+template <typename T>
+constexpr int MinPow2() {
+  // Highway follows RVV LMUL in that the smallest fraction is 1/8th (encoded
+  // as kPow2 == -3). The fraction also must not result in zero lanes for the
+  // smallest possible vector size, which is 128 bits even on RISC-V (with the
+  // application processor profile).
+  return HWY_MAX(-3, -static_cast<int>(CeilLog2(16 / sizeof(T))));
+}
+
+constexpr int MaxPow2() {
+#if HWY_TARGET == HWY_RVV
+  // Only RVV allows multiple vector registers.
+  return 3;  // LMUL=8
+#else
+  // For all other platforms, we cannot exceed a full vector.
+  return 0;
+#endif
+}
+
+// Iterates kPow2 up to and including kMaxPow2. Below we specialize for
+// valid=false to stop the iteration. The ForeachPow2Trim enables shorter
+// argument lists, but use ForeachPow2 when you want to specify the actual min.
+template <typename T, int kPow2, int kMaxPow2, bool valid, class Test>
+struct ForeachPow2 {
+  static void Do(size_t min_lanes) {
+    const ScalableTag<T, kPow2> d;
+
+    static_assert(MinPow2<T>() <= kPow2 && kPow2 <= MaxPow2(), "");
+    if (Lanes(d) >= min_lanes) {
+      Test()(T(), d);
+    } else {
+      fprintf(stderr, "%d lanes < %d: T=%d pow=%d\n",
+              static_cast<int>(Lanes(d)), static_cast<int>(min_lanes),
+              static_cast<int>(sizeof(T)), kPow2);
+      HWY_ASSERT(min_lanes != 1);
+    }
+
+    ForeachPow2<T, kPow2 + 1, kMaxPow2, (kPow2 + 1) <= kMaxPow2, Test>::Do(
+        min_lanes);
+  }
+};
+
+// Base case to stop the iteration.
+template <typename T, int kPow2, int kMaxPow2, class Test>
+struct ForeachPow2<T, kPow2, kMaxPow2, /*valid=*/false, Test> {
+  static void Do(size_t) {}
+};
+
+// Iterates kPow2 over [MinPow2<T>() + kAddMin, MaxPow2() - kSubMax].
+// This is a wrapper that shortens argument lists, allowing users to skip the
+// MinPow2 and MaxPow2. Nonzero kAddMin implies a minimum LMUL, and nonzero
+// kSubMax reduces the maximum LMUL (e.g. for type promotions, where the result
+// is larger, thus the input cannot already use the maximum LMUL).
+template <typename T, int kAddMin, int kSubMax, class Test>
+using ForeachPow2Trim =
+    ForeachPow2<T, MinPow2<T>() + kAddMin, MaxPow2() - kSubMax,
+                MinPow2<T>() + kAddMin <= MaxPow2() - kSubMax, Test>;
+
+#else
+// ForeachCappedR already handled all possible sizes.
+#endif  // HWY_HAVE_SCALABLE
+
+}  // namespace detail
+
+// These 'adapters' call a test for all possible N or kPow2 subject to
+// constraints such as "vectors must be extendable" or "vectors >= 128 bits".
+// They may be called directly, or via For*Types. Note that for an adapter C,
+// `C<Test>(T())` does not call the test - the correct invocation is
+// `C<Test>()(T())`, or preferably `ForAllTypes(C<Test>())`. We check at runtime
+// that operator() is called to prevent such bugs. Note that this is not
+// thread-safe, but that is fine because C are typically local variables.
+
+// Calls Test for all powers of two in [1, Lanes(d) * (RVV? 2 : 1) ]. For
+// interleaved_test; RVV segments are limited to 8 registers, so we can only go
+// up to LMUL=2.
+template <class Test>
+class ForMaxPow2 {
+  mutable bool called_ = false;
+
+ public:
+  ~ForMaxPow2() {
+    if (!called_) {
+      HWY_ABORT("Test is incorrect, ensure operator() is called");
+    }
+  }
+
+  template <typename T>
+  void operator()(T /*unused*/) const {
+    called_ = true;
+
+#if HWY_TARGET == HWY_SCALAR
+    detail::ForeachCappedR<T, 1, 1, Test>::Do(1, 1);
+#else
+    detail::ForeachCappedR<T, HWY_LANES(T), 1, Test>::Do(
+        1, Lanes(ScalableTag<T>()));
+
+#if HWY_TARGET == HWY_RVV
+    // To get LMUL=2 (kPow2=1), 2 is what we subtract from MaxPow2()=3.
+    detail::ForeachPow2Trim<T, 0, 2, Test>::Do(1);
+#elif HWY_HAVE_SCALABLE
+    detail::ForeachPow2Trim<T, 0, 0, Test>::Do(1);
+#endif
+#endif  // HWY_TARGET == HWY_SCALAR
+  }
+};
+
+// Calls Test for all powers of two in [1, Lanes(d) >> kPow2]. This is for
+// ops that widen their input, e.g. Combine (not supported by HWY_SCALAR).
+template <class Test, int kPow2 = 1>
+class ForExtendableVectors {
+  mutable bool called_ = false;
+
+ public:
+  ~ForExtendableVectors() {
+    if (!called_) {
+      HWY_ABORT("Test is incorrect, ensure operator() is called");
+    }
+  }
+
+  template <typename T>
+  void operator()(T /*unused*/) const {
+    called_ = true;
+    constexpr size_t kMaxCapped = HWY_LANES(T);
+    // Skip CappedTag that are already full vectors.
+    const size_t max_lanes = Lanes(ScalableTag<T>()) >> kPow2;
+    (void)kMaxCapped;
+    (void)max_lanes;
+#if HWY_TARGET == HWY_SCALAR
+    // not supported
+#else
+    constexpr size_t kMul = kMaxCapped >> kPow2;
+    constexpr size_t kMinArg = size_t{1} << kPow2;
+    detail::ForeachCappedR<T, kMul, kMinArg, Test, -kPow2>::Do(1, max_lanes);
+#if HWY_HAVE_SCALABLE
+    detail::ForeachPow2Trim<T, 0, kPow2, Test>::Do(1);
+#endif
+#endif  // HWY_SCALAR
+  }
+};
+
+// Calls Test for all power of two N in [1 << kPow2, Lanes(d)]. This is for ops
+// that narrow their input, e.g. UpperHalf.
+template <class Test, int kPow2 = 1>
+class ForShrinkableVectors {
+  mutable bool called_ = false;
+
+ public:
+  ~ForShrinkableVectors() {
+    if (!called_) {
+      HWY_ABORT("Test is incorrect, ensure operator() is called");
+    }
+  }
+
+  template <typename T>
+  void operator()(T /*unused*/) const {
+    called_ = true;
+    constexpr size_t kMinLanes = size_t{1} << kPow2;
+    constexpr size_t kMaxCapped = HWY_LANES(T);
+    // For shrinking, an upper limit is unnecessary.
+    constexpr size_t max_lanes = kMaxCapped;
+
+    (void)kMinLanes;
+    (void)max_lanes;
+    (void)max_lanes;
+#if HWY_TARGET == HWY_SCALAR
+    // not supported
+#elif HWY_HAVE_SCALABLE
+    detail::ForeachPow2Trim<T, kPow2, 0, Test>::Do(kMinLanes);
+#else
+    detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
+        kMinLanes, max_lanes);
+#endif  // HWY_TARGET == HWY_SCALAR
+  }
+};
+
+// Calls Test for all supported power of two vectors of at least kMinBits.
+// Examples: AES or 64x64 require 128 bits, casts may require 64 bits.
+template <size_t kMinBits, class Test>
+class ForGEVectors {
+  mutable bool called_ = false;
+
+ public:
+  ~ForGEVectors() {
+    if (!called_) {
+      HWY_ABORT("Test is incorrect, ensure operator() is called");
+    }
+  }
+
+  template <typename T>
+  void operator()(T /*unused*/) const {
+    called_ = true;
+    constexpr size_t kMaxCapped = HWY_LANES(T);
+    constexpr size_t kMinLanes = kMinBits / 8 / sizeof(T);
+    // An upper limit is unnecessary.
+    constexpr size_t max_lanes = kMaxCapped;
+    (void)max_lanes;
+#if HWY_TARGET == HWY_SCALAR
+    (void)kMinLanes;  // not supported
+#else
+    detail::ForeachCappedR<T, HWY_LANES(T) / kMinLanes, kMinLanes, Test>::Do(
+        kMinLanes, max_lanes);
+#if HWY_HAVE_SCALABLE
+    // Can be 0 (handled below) if kMinBits > 128.
+    constexpr size_t kRatio = 128 / kMinBits;
+    constexpr int kMinPow2 =
+        kRatio == 0 ? 0 : -static_cast<int>(CeilLog2(kRatio));
+    constexpr bool kValid = kMinPow2 <= detail::MaxPow2();
+    detail::ForeachPow2<T, kMinPow2, detail::MaxPow2(), kValid, Test>::Do(
+        kMinLanes);
+#endif
+#endif  // HWY_TARGET == HWY_SCALAR
+  }
+};
+
+template <class Test>
+using ForGE128Vectors = ForGEVectors<128, Test>;
+
+// Calls Test for all N that can be promoted (not the same as Extendable because
+// HWY_SCALAR has one lane). Also used for ZipLower, but not ZipUpper.
+template <class Test, int kPow2 = 1>
+class ForPromoteVectors {
+  mutable bool called_ = false;
+
+ public:
+  ~ForPromoteVectors() {
+    if (!called_) {
+      HWY_ABORT("Test is incorrect, ensure operator() is called");
+    }
+  }
+
+  template <typename T>
+  void operator()(T /*unused*/) const {
+    called_ = true;
+    constexpr size_t kFactor = size_t{1} << kPow2;
+    static_assert(kFactor >= 2 && kFactor * sizeof(T) <= sizeof(uint64_t), "");
+    constexpr size_t kMaxCapped = HWY_LANES(T);
+    // Skip CappedTag that are already full vectors.
+    const size_t max_lanes = Lanes(ScalableTag<T>()) >> kPow2;
+    (void)kMaxCapped;
+    (void)max_lanes;
+#if HWY_TARGET == HWY_SCALAR
+    detail::ForeachCappedR<T, 1, 1, Test>::Do(1, 1);
+#else
+    using DLargestFrom = CappedTag<T, (kMaxCapped >> kPow2) * kFactor, -kPow2>;
+    static_assert(HWY_MAX_LANES_D(DLargestFrom) <= (kMaxCapped >> kPow2),
+                  "HWY_MAX_LANES_D(DLargestFrom) must be less than or equal to "
+                  "(kMaxCapped >> kPow2)");
+    detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kFactor, Test, -kPow2>::Do(
+        1, max_lanes);
+#if HWY_HAVE_SCALABLE
+    detail::ForeachPow2Trim<T, 0, kPow2, Test>::Do(1);
+#endif
+#endif  // HWY_SCALAR
+  }
+};
+
+// Calls Test for all N than can be demoted (not the same as Shrinkable because
+// HWY_SCALAR has one lane and as a one-lane vector with a lane size of at least
+// 2 bytes can always be demoted to a vector with a smaller lane type).
+template <class Test, int kPow2 = 1>
+class ForDemoteVectors {
+  mutable bool called_ = false;
+
+ public:
+  ~ForDemoteVectors() {
+    if (!called_) {
+      HWY_ABORT("Test is incorrect, ensure operator() is called");
+    }
+  }
+
+  template <typename T>
+  void operator()(T /*unused*/) const {
+    called_ = true;
+
+#if HWY_HAVE_SCALABLE
+    // kMinTVecPow2 is the smallest Pow2 for a vector with lane type T that is
+    // supported by detail::ForeachPow2Trim
+    constexpr int kMinTVecPow2 = detail::MinPow2<T>();
+
+    // detail::MinPow2<T>() + kMinPow2Adj is the smallest Pow2 for a vector with
+    // lane type T that can be demoted to a vector with a lane size of
+    // (sizeof(T) >> kPow2)
+    constexpr int kMinPow2Adj = HWY_MAX(-3 - kMinTVecPow2 + kPow2, 0);
+
+    detail::ForeachPow2Trim<T, kMinPow2Adj, 0, Test>::Do(1);
+
+    // On targets with scalable vectors, detail::ForeachCappedR below only
+    // needs to be executed for vectors that have less than
+    // Lanes(ScalableTag<T>()) as full vectors were already checked by the
+    // detail::ForeachPow2Trim above.
+    constexpr size_t kMaxCapped = HWY_LANES(T) >> 1;
+    const size_t max_lanes = Lanes(ScalableTag<T>()) >> 1;
+#else
+    // On targets where HWY_HAVE_SCALABLE is 0, any vector with HWY_LANES(T)
+    // or fewer lanes can always be demoted to a vector with a smaller lane
+    // type.
+    constexpr size_t kMaxCapped = HWY_LANES(T);
+    const size_t max_lanes = kMaxCapped;
+#endif
+
+    detail::ForeachCappedR<T, kMaxCapped, 1, Test>::Do(1, max_lanes);
+  }
+};
+
+// For LowerHalf/Quarter.
+template <class Test, int kPow2 = 1>
+class ForHalfVectors {
+  mutable bool called_ = false;
+
+ public:
+  ~ForHalfVectors() {
+    if (!called_) {
+      HWY_ABORT("Test is incorrect, ensure operator() is called");
+    }
+  }
+
+  template <typename T>
+  void operator()(T /*unused*/) const {
+    called_ = true;
+#if HWY_TARGET == HWY_SCALAR
+    detail::ForeachCappedR<T, 1, 1, Test>::Do(1, 1);
+#else
+    constexpr size_t kMinLanes = size_t{1} << kPow2;
+    // For shrinking, an upper limit is unnecessary.
+    constexpr size_t kMaxCapped = HWY_LANES(T);
+    detail::ForeachCappedR<T, (kMaxCapped >> kPow2), kMinLanes, Test>::Do(
+        kMinLanes, kMaxCapped);
+
+// TODO(janwas): call Extendable if kMinLanes check not required?
+#if HWY_HAVE_SCALABLE
+    detail::ForeachPow2Trim<T, kPow2, 0, Test>::Do(kMinLanes);
+#endif
+#endif  // HWY_TARGET == HWY_SCALAR
+  }
+};
+
+// Calls Test for all power of two N in [1, Lanes(d)]. This is the default
+// for ops that do not narrow nor widen their input, nor require 128 bits.
+template <class Test>
+class ForPartialVectors {
+  mutable bool called_ = false;
+
+ public:
+  ~ForPartialVectors() {
+    if (!called_) {
+      HWY_ABORT("Test is incorrect, ensure operator() is called");
+    }
+  }
+
+  template <typename T>
+  void operator()(T t) const {
+    called_ = true;
+#if HWY_TARGET == HWY_SCALAR
+    (void)t;
+    detail::ForeachCappedR<T, 1, 1, Test>::Do(1, 1);
+#else
+    ForExtendableVectors<Test, 0>()(t);
+#endif
+  }
+};
+
+// ForPartialFixedOrFullScalableVectors calls Test for each D where
+// MaxLanes(D()) == MaxLanes(DFromV<VFromD<D>>())
+#if HWY_HAVE_SCALABLE
+template <class Test>
+class ForPartialFixedOrFullScalableVectors {
+  mutable bool called_ = false;
+
+ public:
+  ~ForPartialFixedOrFullScalableVectors() {
+    if (!called_) {
+      HWY_ABORT("Test is incorrect, ensure operator() is called");
+    }
+  }
+
+  template <typename T>
+  void operator()(T /*t*/) const {
+    called_ = true;
+#if HWY_TARGET == HWY_RVV
+    constexpr int kMinPow2 = -3 + static_cast<int>(CeilLog2(sizeof(T)));
+    constexpr int kMaxPow2 = 3;
+#else
+    constexpr int kMinPow2 = 0;
+    constexpr int kMaxPow2 = 0;
+#endif
+    detail::ForeachPow2<T, kMinPow2, kMaxPow2, true, Test>::Do(1);
+  }
+};
+#elif HWY_TARGET_IS_SVE
+template <class Test>
+using ForPartialFixedOrFullScalableVectors =
+    ForGEVectors<HWY_MAX_BYTES * 8, Test>;
+#else
+template <class Test>
+using ForPartialFixedOrFullScalableVectors = ForPartialVectors<Test>;
+#endif
+
+// Type lists to shorten call sites:
+
+template <class Func>
+void ForSignedTypes(const Func& func) {
+  func(int8_t());
+  func(int16_t());
+  func(int32_t());
+#if HWY_HAVE_INTEGER64
+  func(int64_t());
+#endif
+}
+
+template <class Func>
+void ForUnsignedTypes(const Func& func) {
+  func(uint8_t());
+  func(uint16_t());
+  func(uint32_t());
+#if HWY_HAVE_INTEGER64
+  func(uint64_t());
+#endif
+}
+
+template <class Func>
+void ForIntegerTypes(const Func& func) {
+  ForSignedTypes(func);
+  ForUnsignedTypes(func);
+}
+
+template <class Func>
+void ForFloat16Types(const Func& func) {
+#if HWY_HAVE_FLOAT16
+  func(float16_t());
+#else
+  (void)func;
+#endif
+}
+
+template <class Func>
+void ForFloat64Types(const Func& func) {
+#if HWY_HAVE_FLOAT64
+  func(double());
+#else
+  (void)func;
+#endif
+}
+
+// `#if HWY_HAVE_FLOAT*` is sufficient for tests using static dispatch. In
+// sort_test we also use dynamic dispatch, so there we call the For*Dynamic
+// functions which also check hwy::HaveFloat*.
+template <class Func>
+void ForFloat16TypesDynamic(const Func& func) {
+#if HWY_HAVE_FLOAT16
+  if (hwy::HaveFloat16()) {
+    func(float16_t());
+  }
+#else
+  (void)func;
+#endif
+}
+
+template <class Func>
+void ForFloat64TypesDynamic(const Func& func) {
+#if HWY_HAVE_FLOAT64
+  if (hwy::HaveFloat64()) {
+    func(double());
+  }
+#else
+  (void)func;
+#endif
+}
+
+template <class Func>
+void ForFloat3264Types(const Func& func) {
+  func(float());
+  ForFloat64Types(func);
+}
+
+template <class Func>
+void ForFloatTypes(const Func& func) {
+  ForFloat16Types(func);
+  ForFloat3264Types(func);
+}
+
+template <class Func>
+void ForFloatTypesDynamic(const Func& func) {
+  ForFloat16TypesDynamic(func);
+  func(float());
+  ForFloat64TypesDynamic(func);
+}
+
+template <class Func>
+void ForAllTypes(const Func& func) {
+  ForIntegerTypes(func);
+  ForFloatTypes(func);
+}
+
+// For ops that are also unconditionally available for bfloat16_t/float16_t.
+template <class Func>
+void ForSpecialTypes(const Func& func) {
+  func(float16_t());
+  func(bfloat16_t());
+}
+template <class Func>
+void ForAllTypesAndSpecial(const Func& func) {
+  ForAllTypes(func);
+  ForSpecialTypes(func);
+}
+
+template <class Func>
+void ForUI8(const Func& func) {
+  func(uint8_t());
+  func(int8_t());
+}
+
+template <class Func>
+void ForUI16(const Func& func) {
+  func(uint16_t());
+  func(int16_t());
+}
+
+template <class Func>
+void ForUIF16(const Func& func) {
+  ForUI16(func);
+  ForFloat16Types(func);
+}
+
+template <class Func>
+void ForUI32(const Func& func) {
+  func(uint32_t());
+  func(int32_t());
+}
+
+template <class Func>
+void ForUIF32(const Func& func) {
+  ForUI32(func);
+  func(float());
+}
+
+template <class Func>
+void ForUI64(const Func& func) {
+#if HWY_HAVE_INTEGER64
+  func(uint64_t());
+  func(int64_t());
+#endif
+}
+
+template <class Func>
+void ForUIF64(const Func& func) {
+  ForUI64(func);
+  ForFloat64Types(func);
+}
+
+template <class Func>
+void ForUI3264(const Func& func) {
+  ForUI32(func);
+  ForUI64(func);
+}
+
+template <class Func>
+void ForUIF3264(const Func& func) {
+  ForUIF32(func);
+  ForUIF64(func);
+}
+
+template <class Func>
+void ForU816(const Func& func) {
+  func(uint8_t());
+  func(uint16_t());
+}
+
+template <class Func>
+void ForI816(const Func& func) {
+  func(int8_t());
+  func(int16_t());
+}
+
+template <class Func>
+void ForU163264(const Func& func) {
+  func(uint16_t());
+  func(uint32_t());
+#if HWY_HAVE_INTEGER64
+  func(uint64_t());
+#endif
+}
+
+template <class Func>
+void ForUI163264(const Func& func) {
+  ForUI16(func);
+  ForUI3264(func);
+}
+
+template <class Func>
+void ForUIF163264(const Func& func) {
+  ForUIF16(func);
+  ForUIF3264(func);
+}
+
+// For tests that involve loops, adjust the trip count so that emulated tests
+// finish quickly (but always at least 2 iterations to ensure some diversity).
+constexpr size_t AdjustedReps(size_t max_reps) {
+#if HWY_ARCH_RISCV
+  return HWY_MAX(max_reps / 32, 2);
+#elif HWY_IS_DEBUG_BUILD
+  return HWY_MAX(max_reps / 8, 2);
+#elif HWY_ARCH_ARM
+  return HWY_MAX(max_reps / 4, 2);
+#elif HWY_COMPILER_MSVC
+  return HWY_MAX(max_reps / 2, 2);
+#else
+  return HWY_MAX(max_reps, 2);
+#endif
+}
+
+// Same as above, but the loop trip count will be 1 << max_pow2.
+constexpr size_t AdjustedLog2Reps(size_t max_pow2) {
+  // If "negative" (unsigned wraparound), use original.
+#if HWY_ARCH_RISCV
+  return HWY_MIN(max_pow2 - 4, max_pow2);
+#elif HWY_IS_DEBUG_BUILD
+  return HWY_MIN(max_pow2 - 1, max_pow2);
+#elif HWY_ARCH_ARM
+  return HWY_MIN(max_pow2 - 1, max_pow2);
+#else
+  return max_pow2;
+#endif
+}
+
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#endif  // per-target include guard

deps/v8/third_party/highway/src/hwy/tests/test_util.cc

@@ -0,0 +1,118 @@
+// Copyright 2021 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "hwy/tests/test_util.h"
+
+#include <stdio.h>
+
+#include "hwy/base.h"
+#include "hwy/print.h"
+
+namespace hwy {
+
+HWY_TEST_DLLEXPORT bool BytesEqual(const void* p1, const void* p2,
+                                   const size_t size, size_t* pos) {
+  const uint8_t* bytes1 = reinterpret_cast<const uint8_t*>(p1);
+  const uint8_t* bytes2 = reinterpret_cast<const uint8_t*>(p2);
+  for (size_t i = 0; i < size; ++i) {
+    if (bytes1[i] != bytes2[i]) {
+      if (pos != nullptr) {
+        *pos = i;
+      }
+      return false;
+    }
+  }
+  return true;
+}
+
+void AssertStringEqual(const char* expected, const char* actual,
+                       const char* target_name, const char* filename,
+                       int line) {
+  while (*expected == *actual++) {
+    if (*expected++ == '\0') return;
+  }
+
+  Abort(filename, line, "%s string mismatch: expected '%s', got '%s'.\n",
+        target_name, expected, actual);
+}
+
+namespace detail {
+
+HWY_TEST_DLLEXPORT bool IsEqual(const TypeInfo& info, const void* expected_ptr,
+                                const void* actual_ptr) {
+  if (!info.is_float) {
+    return BytesEqual(expected_ptr, actual_ptr, info.sizeof_t);
+  }
+  if (info.sizeof_t == 2) {
+    const float expected = info.is_bf16 ? F32FromBF16Mem(expected_ptr)
+                                        : F32FromF16Mem(expected_ptr);
+    const float actual =
+        info.is_bf16 ? F32FromBF16Mem(actual_ptr) : F32FromF16Mem(actual_ptr);
+    return ComputeUlpDelta(expected, actual) <= 1;
+  } else if (info.sizeof_t == 4) {
+    float expected, actual;
+    CopyBytes<4>(expected_ptr, &expected);
+    CopyBytes<4>(actual_ptr, &actual);
+    return ComputeUlpDelta(expected, actual) <= 1;
+  } else if (info.sizeof_t == 8) {
+    double expected, actual;
+    CopyBytes<8>(expected_ptr, &expected);
+    CopyBytes<8>(actual_ptr, &actual);
+    return ComputeUlpDelta(expected, actual) <= 1;
+  } else {
+    HWY_ABORT("Unexpected float size %d\n", static_cast<int>(info.sizeof_t));
+  }
+}
+
+HWY_TEST_DLLEXPORT HWY_NORETURN void PrintMismatchAndAbort(
+    const TypeInfo& info, const void* expected_ptr, const void* actual_ptr,
+    const char* target_name, const char* filename, int line, size_t lane,
+    size_t num_lanes) {
+  char type_name[100];
+  TypeName(info, 1, type_name);
+  char expected_str[100];
+  ToString(info, expected_ptr, expected_str);
+  char actual_str[100];
+  ToString(info, actual_ptr, actual_str);
+  Abort(filename, line,
+        "%s, %sx%d lane %d mismatch: expected '%s', got '%s'.\n", target_name,
+        type_name, static_cast<int>(num_lanes), static_cast<int>(lane),
+        expected_str, actual_str);
+}
+
+HWY_TEST_DLLEXPORT void AssertArrayEqual(const TypeInfo& info,
+                                         const void* expected_void,
+                                         const void* actual_void, size_t N,
+                                         const char* target_name,
+                                         const char* filename, int line) {
+  const uint8_t* expected_array =
+      reinterpret_cast<const uint8_t*>(expected_void);
+  const uint8_t* actual_array = reinterpret_cast<const uint8_t*>(actual_void);
+  for (size_t i = 0; i < N; ++i) {
+    const void* expected_ptr = expected_array + i * info.sizeof_t;
+    const void* actual_ptr = actual_array + i * info.sizeof_t;
+    if (!IsEqual(info, expected_ptr, actual_ptr)) {
+      fprintf(stderr, "\n\n");
+      PrintArray(info, "expect", expected_array, N, i);
+      PrintArray(info, "actual", actual_array, N, i);
+
+      PrintMismatchAndAbort(info, expected_ptr, actual_ptr, target_name,
+                            filename, line, i, N);
+    }
+  }
+}
+
+}  // namespace detail
+}  // namespace hwy

deps/v8/third_party/highway/src/hwy/tests/test_util.h

@@ -0,0 +1,249 @@
+// Copyright 2021 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TESTS_TEST_UTIL_H_
+#define HWY_TESTS_TEST_UTIL_H_
+
+// Target-independent helper functions for use by *_test.cc.
+
+#include <string.h>
+
+#include <cmath>  // std::isnan
+#include <string>
+
+#include "hwy/base.h"
+#include "hwy/nanobenchmark.h"
+#include "hwy/print.h"
+
+namespace hwy {
+
+// The maximum vector size used in tests when defining test data. DEPRECATED.
+HWY_MAYBE_UNUSED constexpr size_t kTestMaxVectorSize = 64;
+
+// 64-bit random generator (Xorshift128+). Much smaller state than std::mt19937,
+// which triggers a compiler bug.
+class RandomState {
+ public:
+  explicit RandomState(
+      const uint64_t seed = uint64_t{0x123456789} *
+                            static_cast<uint64_t>(hwy::Unpredictable1())) {
+    s0_ = SplitMix64(seed + 0x9E3779B97F4A7C15ull);
+    s1_ = SplitMix64(s0_);
+  }
+
+  HWY_INLINE uint64_t operator()() {
+    uint64_t s1 = s0_;
+    const uint64_t s0 = s1_;
+    const uint64_t bits = s1 + s0;
+    s0_ = s0;
+    s1 ^= s1 << 23;
+    s1 ^= s0 ^ (s1 >> 18) ^ (s0 >> 5);
+    s1_ = s1;
+    return bits;
+  }
+
+ private:
+  static uint64_t SplitMix64(uint64_t z) {
+    z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
+    z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
+    return z ^ (z >> 31);
+  }
+
+  uint64_t s0_;
+  uint64_t s1_;
+};
+
+static HWY_INLINE uint32_t Random32(RandomState* rng) {
+  return static_cast<uint32_t>((*rng)());
+}
+
+static HWY_INLINE uint64_t Random64(RandomState* rng) { return (*rng)(); }
+
+template <class T, HWY_IF_FLOAT_OR_SPECIAL(T)>
+static HWY_INLINE T RandomFiniteValue(RandomState* rng) {
+  const uint64_t rand_bits = Random64(rng);
+
+  using TU = MakeUnsigned<T>;
+  constexpr TU kExponentMask = ExponentMask<T>();
+  constexpr TU kSignMantMask = static_cast<TU>(~kExponentMask);
+  constexpr TU kMaxExpField = static_cast<TU>(MaxExponentField<T>());
+  constexpr int kNumOfMantBits = MantissaBits<T>();
+
+  const TU orig_exp_field_val =
+      static_cast<TU>((rand_bits >> kNumOfMantBits) & kMaxExpField);
+
+  const TU sign_mant_bits = static_cast<TU>(rand_bits & kSignMantMask);
+  const TU exp_bits =
+      static_cast<TU>(HWY_MIN(HWY_MAX(orig_exp_field_val, 1), kMaxExpField - 1)
+                      << kNumOfMantBits);
+
+  return BitCastScalar<T>(static_cast<TU>(sign_mant_bits | exp_bits));
+}
+
+template <class T, HWY_IF_NOT_FLOAT_NOR_SPECIAL(T)>
+static HWY_INLINE T RandomFiniteValue(RandomState* rng) {
+  using TU = MakeUnsigned<T>;
+  return static_cast<T>(Random64(rng) & LimitsMax<TU>());
+}
+
+HWY_TEST_DLLEXPORT bool BytesEqual(const void* p1, const void* p2, size_t size,
+                                   size_t* pos = nullptr);
+
+void AssertStringEqual(const char* expected, const char* actual,
+                       const char* target_name, const char* filename, int line);
+
+namespace detail {
+
+template <typename T, typename TU = MakeUnsigned<T>>
+TU ComputeUlpDelta(const T expected, const T actual) {
+  // Handle -0 == 0 and infinities.
+  if (expected == actual) return 0;
+
+  // Consider "equal" if both are NaN, so we can verify an expected NaN.
+  // Needs a special case because there are many possible NaN representations.
+  if (std::isnan(expected) && std::isnan(actual)) return 0;
+
+  // Compute the difference in units of last place. We do not need to check for
+  // differing signs; they will result in large differences, which is fine.
+  TU ux, uy;
+  CopySameSize(&expected, &ux);
+  CopySameSize(&actual, &uy);
+
+  // Avoid unsigned->signed cast: 2's complement is only guaranteed by C++20.
+  const TU ulp = HWY_MAX(ux, uy) - HWY_MIN(ux, uy);
+  return ulp;
+}
+
+HWY_TEST_DLLEXPORT bool IsEqual(const TypeInfo& info, const void* expected_ptr,
+                                const void* actual_ptr);
+
+HWY_TEST_DLLEXPORT HWY_NORETURN void PrintMismatchAndAbort(
+    const TypeInfo& info, const void* expected_ptr, const void* actual_ptr,
+    const char* target_name, const char* filename, int line, size_t lane = 0,
+    size_t num_lanes = 1);
+
+HWY_TEST_DLLEXPORT void AssertArrayEqual(const TypeInfo& info,
+                                         const void* expected_void,
+                                         const void* actual_void, size_t N,
+                                         const char* target_name,
+                                         const char* filename, int line);
+
+}  // namespace detail
+
+// Returns a name for the vector/part/scalar. The type prefix is u/i/f for
+// unsigned/signed/floating point, followed by the number of bits per lane;
+// then 'x' followed by the number of lanes. Example: u8x16. This is useful for
+// understanding which instantiation of a generic test failed.
+template <typename T>
+std::string TypeName(T /*unused*/, size_t N) {
+  char string100[100];
+  detail::TypeName(detail::MakeTypeInfo<T>(), N, string100);
+  return string100;
+}
+
+// Type large enough to hold either value, to which we cast for comparison.
+template <typename T1, typename T2>
+using LargestType = If<IsFloat<T1>() || IsFloat<T2>(),
+                       FloatFromSize<HWY_MAX(sizeof(T1), sizeof(T2))>,
+                       If<IsSigned<T1>() || IsSigned<T2>(),
+                          SignedFromSize<HWY_MAX(sizeof(T1), sizeof(T2))>,
+                          UnsignedFromSize<HWY_MAX(sizeof(T1), sizeof(T2))>>>;
+
+// TTo is the lane type of the actual value and T is an often but not
+// necessarily larger type of the expected value. Especially for 8-bit lanes
+// initialized via Iota, the actual value often wraps around. To ensure it still
+// compares equal to the expected value, wrap integers.
+// 1) < 64-bit integer: mask
+template <typename TTo, typename T, HWY_IF_NOT_FLOAT_NOR_SPECIAL(TTo),
+          HWY_IF_T_SIZE_LE(TTo, 4)>
+T WrapTo(T value) {
+  return static_cast<T>(static_cast<uint64_t>(value) &
+                        ((uint64_t{1} << (sizeof(TTo) * 8)) - 1));
+}
+// 2) 64-bit integer: no mask (shift would overflow)
+template <typename TTo, typename T, HWY_IF_NOT_FLOAT_NOR_SPECIAL(TTo),
+          HWY_IF_T_SIZE_GT(TTo, 4)>
+T WrapTo(T value) {
+  return value;
+}
+// 3) float or special: do nothing because their value range is sufficient for
+// Iota for any vector length.
+template <typename TTo, typename T, HWY_IF_FLOAT_OR_SPECIAL(TTo)>
+T WrapTo(T value) {
+  return value;
+}
+
+// Compare non-vector, non-string T, after promoting to the largest type.
+template <typename TExpected, typename TActual>
+HWY_INLINE bool IsEqual(const TExpected texpected, const TActual actual) {
+  const TActual expected = ConvertScalarTo<TActual>(WrapTo<TActual>(texpected));
+  const auto info = detail::MakeTypeInfo<TActual>();
+  return detail::IsEqual(info, &expected, &actual);
+}
+
+template <typename TExpected, typename TActual>
+HWY_INLINE void AssertEqual(const TExpected texpected, const TActual actual,
+                            const char* target_name, const char* filename,
+                            int line, size_t lane = 0) {
+  const TActual expected = ConvertScalarTo<TActual>(WrapTo<TActual>(texpected));
+  const auto info = detail::MakeTypeInfo<TActual>();
+  if (!detail::IsEqual(info, &expected, &actual)) {
+    detail::PrintMismatchAndAbort(info, &expected, &actual, target_name,
+                                  filename, line, lane);
+  }
+}
+
+template <typename T>
+HWY_INLINE void AssertArrayEqual(const T* expected, const T* actual,
+                                 size_t count, const char* target_name,
+                                 const char* filename, int line) {
+  const auto info = hwy::detail::MakeTypeInfo<T>();
+  detail::AssertArrayEqual(info, expected, actual, count, target_name, filename,
+                           line);
+}
+
+// Compare with tolerance due to FMA and f16 precision.
+template <typename T>
+HWY_INLINE void AssertArraySimilar(const T* expected, const T* actual,
+                                   size_t count, const char* target_name,
+                                   const char* filename, int line) {
+  const double tolerance =
+      (hwy::IsSame<RemoveCvRef<T>, float16_t>() ? 128.0 : 1.0) /
+      (uint64_t{1} << MantissaBits<T>());
+  for (size_t i = 0; i < count; ++i) {
+    const double exp = ConvertScalarTo<double>(expected[i]);
+    const double act = ConvertScalarTo<double>(actual[i]);
+    const double l1 = ScalarAbs(act - exp);
+    // Cannot divide, so check absolute error.
+    if (exp == 0.0) {
+      if (l1 > tolerance) {
+        HWY_ABORT("%s %s:%d %s mismatch %zu of %zu: %E %E l1 %E tol %E\n",
+                  target_name, filename, line, TypeName(T(), 1).c_str(), i,
+                  count, exp, act, l1, tolerance);
+      }
+    } else {  // relative
+      const double rel = l1 / exp;
+      if (rel > tolerance) {
+        HWY_ABORT("%s %s:%d %s mismatch %zu of %zu: %E %E rel %E tol %E\n",
+                  target_name, filename, line, TypeName(T(), 1).c_str(), i,
+                  count, exp, act, rel, tolerance);
+      }
+    }
+  }
+}
+
+}  // namespace hwy
+
+#endif  // HWY_TESTS_TEST_UTIL_H_

deps/v8/third_party/highway/src/hwy/tests/test_util_test.cc

@@ -0,0 +1,116 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdio.h>
+
+#include <string>
+
+#include "hwy/base.h"
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/test_util_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestName {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T t, D d) {
+    char num[10];
+    std::string expected = IsFloat<T>() ? "f" : (IsSigned<T>() ? "i" : "u");
+    snprintf(num, sizeof(num), "%u", static_cast<unsigned>(sizeof(T) * 8));
+    expected += num;
+
+    const size_t N = Lanes(d);
+    if (N != 1) {
+      expected += 'x';
+      snprintf(num, sizeof(num), "%u", static_cast<unsigned>(N));
+      expected += num;
+    }
+    const std::string actual = TypeName(t, N);
+    if (expected != actual) {
+      HWY_ABORT("%s mismatch: expected '%s', got '%s'.\n",
+                hwy::TargetName(HWY_TARGET), expected.c_str(), actual.c_str());
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllName() { ForAllTypes(ForPartialVectors<TestName>()); }
+
+struct TestEqualInteger {
+  template <class T>
+  HWY_NOINLINE void operator()(T /*t*/) const {
+    HWY_ASSERT_EQ(0, 0);
+    HWY_ASSERT_EQ(1, 1);
+    HWY_ASSERT_EQ(-1, -1);
+    HWY_ASSERT_EQ(LimitsMin<T>(), LimitsMin<T>());
+
+    HWY_ASSERT(!IsEqual(0, 1));
+    HWY_ASSERT(!IsEqual(1, 0));
+    HWY_ASSERT(!IsEqual(1, -1));
+    HWY_ASSERT(!IsEqual(-1, 1));
+    HWY_ASSERT(!IsEqual(LimitsMin<T>(), LimitsMax<T>()));
+    HWY_ASSERT(!IsEqual(LimitsMax<T>(), LimitsMin<T>()));
+  }
+};
+
+struct TestEqualFloat {
+  template <class T>
+  HWY_NOINLINE void operator()(T /*t*/) const {
+    const T k0 = ConvertScalarTo<T>(0);
+    const T p1 = ConvertScalarTo<T>(1);
+    const T n1 = ConvertScalarTo<T>(-1);
+    HWY_ASSERT(IsEqual(k0, k0));
+    HWY_ASSERT(IsEqual(p1, p1));
+    HWY_ASSERT(IsEqual(n1, n1));
+    HWY_ASSERT(IsEqual(MantissaEnd<T>(), MantissaEnd<T>()));
+
+    HWY_ASSERT(!IsEqual(k0, p1));
+    HWY_ASSERT(!IsEqual(p1, k0));
+    HWY_ASSERT(!IsEqual(p1, n1));
+    HWY_ASSERT(!IsEqual(n1, p1));
+    HWY_ASSERT(!IsEqual(LowestValue<T>(), HighestValue<T>()));
+    HWY_ASSERT(!IsEqual(HighestValue<T>(), LowestValue<T>()));
+  }
+};
+
+HWY_NOINLINE void TestAllEqual() {
+  ForIntegerTypes(TestEqualInteger());
+  ForFloatTypes(TestEqualFloat());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(TestUtilTest);
+HWY_EXPORT_AND_TEST_P(TestUtilTest, TestAllName);
+HWY_EXPORT_AND_TEST_P(TestUtilTest, TestAllEqual);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/truncate_test.cc

@@ -0,0 +1,127 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/truncate_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+template <typename From, typename To, class D>
+constexpr bool IsSupportedTruncation() {
+  return (sizeof(To) < sizeof(From) && Rebind<To, D>().Pow2() >= -3 &&
+          Rebind<To, D>().Pow2() + 4 >= static_cast<int>(CeilLog2(sizeof(To))));
+}
+
+struct TestTruncateTo {
+  template <typename From, typename To, class D,
+            hwy::EnableIf<!IsSupportedTruncation<From, To, D>()>* = nullptr>
+  HWY_NOINLINE void testTo(From, To, const D) {
+    // do nothing
+  }
+
+  template <typename From, typename To, class D,
+            hwy::EnableIf<IsSupportedTruncation<From, To, D>()>* = nullptr>
+  HWY_NOINLINE void testTo(From, To, const D d) {
+    constexpr uint32_t base = 0xFA578D00;
+    const Rebind<To, D> dTo;
+    const Vec<D> src = Iota(d, base & hwy::LimitsMax<From>());
+    const Vec<decltype(dTo)> expected = Iota(dTo, base & hwy::LimitsMax<To>());
+    const VFromD<decltype(dTo)> actual = TruncateTo(dTo, src);
+    HWY_ASSERT_VEC_EQ(dTo, expected, actual);
+  }
+
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T from, const D d) {
+    testTo<T, uint8_t, D>(from, uint8_t(), d);
+    testTo<T, uint16_t, D>(from, uint16_t(), d);
+    testTo<T, uint32_t, D>(from, uint32_t(), d);
+  }
+};
+
+HWY_NOINLINE void TestAllTruncate() {
+  ForU163264(ForDemoteVectors<TestTruncateTo>());
+}
+
+struct TestOrderedTruncate2To {
+  template <typename T, class D>
+  HWY_NOINLINE void operator()(T /*t*/, D d) {
+#if HWY_TARGET != HWY_SCALAR
+    const Repartition<MakeNarrow<T>, decltype(d)> dn;
+    using TN = TFromD<decltype(dn)>;
+
+    const size_t N = Lanes(d);
+    const size_t twiceN = N * 2;
+    auto from = AllocateAligned<T>(twiceN);
+    auto expected = AllocateAligned<TN>(twiceN);
+    HWY_ASSERT(from && expected);
+
+    const T max = LimitsMax<TN>();
+
+    constexpr uint32_t iota_base = 0xFA578D00;
+    const auto src_iota_a = Iota(d, iota_base);
+    const auto src_iota_b = Iota(d, iota_base + N);
+    const auto expected_iota_trunc_result = Iota(dn, iota_base);
+    const auto actual_iota_trunc_result =
+        OrderedTruncate2To(dn, src_iota_a, src_iota_b);
+    HWY_ASSERT_VEC_EQ(dn, expected_iota_trunc_result, actual_iota_trunc_result);
+
+    RandomState rng;
+    for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) {
+      for (size_t i = 0; i < twiceN; ++i) {
+        const uint64_t bits = rng();
+        CopyBytes<sizeof(T)>(&bits, &from[i]);  // not same size
+        expected[i] = static_cast<TN>(from[i] & max);
+      }
+
+      const auto in_1 = Load(d, from.get());
+      const auto in_2 = Load(d, from.get() + N);
+      const auto actual = OrderedTruncate2To(dn, in_1, in_2);
+      HWY_ASSERT_VEC_EQ(dn, expected.get(), actual);
+    }
+#else
+    (void)d;
+#endif
+  }
+};
+
+HWY_NOINLINE void TestAllOrderedTruncate2To() {
+  ForU163264(ForShrinkableVectors<TestOrderedTruncate2To>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyTruncateTest);
+HWY_EXPORT_AND_TEST_P(HwyTruncateTest, TestAllTruncate);
+HWY_EXPORT_AND_TEST_P(HwyTruncateTest, TestAllOrderedTruncate2To);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/tuple_test.cc

@@ -0,0 +1,101 @@
+// Copyright 2023 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/tuple_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestCreateAndSet {
+  template <class T, class D>
+  HWY_NOINLINE void operator()(T /*unused*/, D d) {
+#if HWY_HAVE_TUPLE
+    const Vec<D> v0 = Zero(d);
+    const Vec<D> vi = Iota(d, 1);
+    const Vec<D> v2 = Set(d, ConvertScalarTo<T>(2));
+    const Vec<D> v3 = Set(d, ConvertScalarTo<T>(3));
+
+    Vec2<D> t2 = Create2(d, v0, vi);
+    HWY_ASSERT_VEC_EQ(d, v0, Get2<0>(t2));
+    HWY_ASSERT_VEC_EQ(d, vi, Get2<1>(t2));
+
+    t2 = Set2<0>(t2, vi);
+    t2 = Set2<1>(t2, v0);
+    HWY_ASSERT_VEC_EQ(d, vi, Get2<0>(t2));
+    HWY_ASSERT_VEC_EQ(d, v0, Get2<1>(t2));
+
+    Vec3<D> t3 = Create3(d, v0, vi, v2);
+    HWY_ASSERT_VEC_EQ(d, v0, Get3<0>(t3));
+    HWY_ASSERT_VEC_EQ(d, vi, Get3<1>(t3));
+    HWY_ASSERT_VEC_EQ(d, v2, Get3<2>(t3));
+
+    t3 = Set3<0>(t3, v2);
+    t3 = Set3<1>(t3, vi);
+    t3 = Set3<2>(t3, v0);
+    HWY_ASSERT_VEC_EQ(d, v2, Get3<0>(t3));
+    HWY_ASSERT_VEC_EQ(d, vi, Get3<1>(t3));
+    HWY_ASSERT_VEC_EQ(d, v0, Get3<2>(t3));
+
+    Vec4<D> t4 = Create4(d, v0, vi, v2, v3);
+    HWY_ASSERT_VEC_EQ(d, v0, Get4<0>(t4));
+    HWY_ASSERT_VEC_EQ(d, vi, Get4<1>(t4));
+    HWY_ASSERT_VEC_EQ(d, v2, Get4<2>(t4));
+    HWY_ASSERT_VEC_EQ(d, v3, Get4<3>(t4));
+
+    t4 = Set4<0>(t4, v3);
+    t4 = Set4<1>(t4, v2);
+    t4 = Set4<2>(t4, vi);
+    t4 = Set4<3>(t4, v0);
+    HWY_ASSERT_VEC_EQ(d, v3, Get4<0>(t4));
+    HWY_ASSERT_VEC_EQ(d, v2, Get4<1>(t4));
+    HWY_ASSERT_VEC_EQ(d, vi, Get4<2>(t4));
+    HWY_ASSERT_VEC_EQ(d, v0, Get4<3>(t4));
+#else
+    (void)d;
+    fprintf(stderr, "Warning: tuples are disabled for target %s\n",
+            hwy::TargetName(HWY_TARGET));
+#endif  // HWY_HAVE_TUPLE
+  }
+};
+
+HWY_NOINLINE void TestAllCreate() {
+  // RVV can only do tuples up to LMUL=2.
+  ForAllTypes(ForMaxPow2<TestCreateAndSet>());
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(TupleTest);
+HWY_EXPORT_AND_TEST_P(TupleTest, TestAllCreate);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE

deps/v8/third_party/highway/src/hwy/tests/widen_mul_test.cc

@@ -0,0 +1,529 @@
+// Copyright 2019 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#undef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/widen_mul_test.cc"
+#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/highway.h"
+#include "hwy/nanobenchmark.h"  // Unpredictable1
+#include "hwy/tests/test_util-inl.h"
+
+HWY_BEFORE_NAMESPACE();
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+struct TestSatWidenMulAccumFixedPoint {
+  template <class TN, class DN>
+  HWY_INLINE void operator()(TN /*unused*/, DN dn) {
+    static_assert(IsSigned<TN>() && !IsFloat<TN>() && !IsSpecialFloat<TN>(),
+                  "TN must be a signed integer type");
+
+    using TW = MakeWide<TN>;
+    const Rebind<TW, DN> dw;
+    using VW = Vec<decltype(dw)>;
+    using VN = Vec<decltype(dn)>;
+
+    const VN vn_min = Set(dn, LimitsMin<TN>());
+
+    const VW vw_min = Set(dw, LimitsMin<TW>());
+    const VW vw_kneg7 = Set(dw, static_cast<TW>(-7));
+    const VW vw_k0 = Zero(dw);
+    const VW vw_k1 = Set(dw, static_cast<TW>(1));
+    const VW vw_k19 = Set(dw, static_cast<TW>(19));
+    const VW vw_max = Set(dw, LimitsMax<TW>());
+
+    const VN vn_p = Add(
+        And(Iota(dn, TN{0}), Set(dn, static_cast<TN>(LimitsMax<TN>() >> 3))),
+        Set(dn, TN{1}));
+    const VW vw_p = PromoteTo(dw, vn_p);
+    HWY_ASSERT(AllTrue(dw, Gt(vw_p, vw_k0)));
+
+    const auto vw_n_minus7 = Add(Neg(vw_p), vw_kneg7);
+
+    // As it is implementation-defined if vn_min[i] * vn_min[i] * 2 is first
+    // saturated to TW before adding to vw_n_minus7[i], check that
+    // actual_minsqr_sum[i] is equal to either LimitsMax<TW>() + vn_n_minus7[i]
+    // or LimitsMax<TW>() + vn_n_minus7[i] + 1
+    const VW actual_minsqr_sum =
+        SatWidenMulAccumFixedPoint(dw, vn_min, vn_min, vw_n_minus7);
+    const VW min_expected_minsqr_sum = Add(vw_max, vw_n_minus7);
+    const VW max_expected_minsqr_sum = Add(min_expected_minsqr_sum, vw_k1);
+    HWY_ASSERT(
+        AllTrue(dw, And(Ge(actual_minsqr_sum, min_expected_minsqr_sum),
+                        Le(actual_minsqr_sum, max_expected_minsqr_sum))));
+
+    const VN vn_p_plus2 = Add(vn_p, Set(dn, TN{2}));
+    const VN vn_p_plus3 = Add(vn_p, Set(dn, TN{3}));
+
+    const VW vw_pp2_pp3 =
+        Mul(PromoteTo(dw, vn_p_plus2), PromoteTo(dw, vn_p_plus3));
+    const VW vw_pp2_pp3_2 = Add(vw_pp2_pp3, vw_pp2_pp3);
+
+    const VW expected_p_sum = Add(vw_pp2_pp3_2, vw_kneg7);
+    const VW actual_p_sum =
+        SatWidenMulAccumFixedPoint(dw, vn_p_plus2, vn_p_plus3, vw_kneg7);
+    HWY_ASSERT_VEC_EQ(dw, expected_p_sum, actual_p_sum);
+
+    const VW expected_p_sum2 = Add(vw_pp2_pp3_2, vw_k19);
+    const VW actual_p_sum2 =
+        SatWidenMulAccumFixedPoint(dw, vn_p_plus2, vn_p_plus3, vw_k19);
+    HWY_ASSERT_VEC_EQ(dw, expected_p_sum2, actual_p_sum2);
+
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_max,
+        SatWidenMulAccumFixedPoint(dw, vn_p_plus2, vn_p_plus3, vw_max));
+
+    const VN vn_n_minus3 = Neg(vn_p_plus3);
+
+    const VW expected_n_sum = Sub(vw_kneg7, vw_pp2_pp3_2);
+    const VW actual_n_sum =
+        SatWidenMulAccumFixedPoint(dw, vn_p_plus2, vn_n_minus3, vw_kneg7);
+    HWY_ASSERT_VEC_EQ(dw, expected_n_sum, actual_n_sum);
+
+    HWY_ASSERT_VEC_EQ(
+        dw, vw_min,
+        SatWidenMulAccumFixedPoint(dw, vn_p_plus2, vn_n_minus3, vw_min));
+  }
+};
+
+HWY_NOINLINE void TestAllSatWidenMulAccumFixedPoint() {
+  ForPromoteVectors<TestSatWidenMulAccumFixedPoint>()(int16_t());
+}
+
+#ifndef HWY_NATIVE_DOT_BF16
+#error "Update set_macros-inl.h to set this required macro"
+#endif
+
+struct TestMulEvenAdd {
+  // Must be inlined on aarch64 for bf16, else clang crashes.
+  template <typename TN, class DN>
+  HWY_INLINE void operator()(TN /*unused*/, DN dn) {
+    using TW = MakeWide<TN>;
+    const RepartitionToWide<DN> dw;
+    using VW = Vec<decltype(dw)>;
+    using VN = Vec<decltype(dn)>;
+    const size_t NN = Lanes(dn);
+
+    const VW f0 = Zero(dw);
+    const VW f1 = Set(dw, TW{1});
+    const VN bf0 = Zero(dn);
+    // Cannot Set() bfloat16_t directly.
+    const VN bf1 = ReorderDemote2To(dn, f1, f1);
+
+    // Any input zero => both outputs zero
+    HWY_ASSERT_VEC_EQ(dw, f0, MulEvenAdd(dw, bf0, bf0, f0));
+    HWY_ASSERT_VEC_EQ(dw, f0, MulEvenAdd(dw, bf0, bf1, f0));
+    HWY_ASSERT_VEC_EQ(dw, f0, MulEvenAdd(dw, bf1, bf0, f0));
+    HWY_ASSERT_VEC_EQ(dw, f0, MulOddAdd(dw, bf0, bf0, f0));
+    HWY_ASSERT_VEC_EQ(dw, f0, MulOddAdd(dw, bf0, bf1, f0));
+    HWY_ASSERT_VEC_EQ(dw, f0, MulOddAdd(dw, bf1, bf0, f0));
+
+    // delta[p] := 1, all others zero. For each p: Mul(delta, 1, 0) == 1.
+    auto delta_w = AllocateAligned<TW>(NN);
+    HWY_ASSERT(delta_w);
+    for (size_t p = 0; p < NN; ++p) {
+      // Workaround for incorrect Clang wasm codegen: re-initialize the entire
+      // array rather than zero-initialize once and then toggle lane p.
+      for (size_t i = 0; i < NN; ++i) {
+        delta_w[i] = static_cast<TW>(i == p ? Unpredictable1() : 0);
+      }
+      const VW delta0 = Load(dw, delta_w.get());
+      const VW delta1 = Load(dw, delta_w.get() + NN / 2);
+      const VN delta = OrderedDemote2To(dn, delta0, delta1);
+
+      {
+        const VW sum_e = MulEvenAdd(dw, delta, bf1, f0);
+        const VW sum_o = MulOddAdd(dw, delta, bf1, f0);
+        HWY_ASSERT_VEC_EQ(dw, p & 1 ? f0 : f1, SumOfLanes(dw, sum_e));
+        HWY_ASSERT_VEC_EQ(dw, p & 1 ? f1 : f0, SumOfLanes(dw, sum_o));
+      }
+      // Swapped arg order
+      {
+        const VW sum_e = MulEvenAdd(dw, bf1, delta, f0);
+        const VW sum_o = MulOddAdd(dw, bf1, delta, f0);
+        HWY_ASSERT_VEC_EQ(dw, p & 1 ? f0 : f1, SumOfLanes(dw, sum_e));
+        HWY_ASSERT_VEC_EQ(dw, p & 1 ? f1 : f0, SumOfLanes(dw, sum_o));
+      }
+      // Start with nonzero sum
+      {
+        const VW sum_e = MulEvenAdd(dw, delta, bf1, Add(delta0, delta1));
+        const VW sum_o = MulOddAdd(dw, delta, bf1, Add(delta0, delta1));
+        HWY_ASSERT_EQ(TW{3}, ReduceSum(dw, Add(sum_e, sum_o)));
+      }
+
+      // Start with nonzero sum and swap arg order
+      {
+        const VW sum_e = MulEvenAdd(dw, bf1, delta, Add(delta0, delta1));
+        const VW sum_o = MulOddAdd(dw, bf1, delta, Add(delta0, delta1));
+        HWY_ASSERT_EQ(TW{3}, ReduceSum(dw, Add(sum_e, sum_o)));
+      }
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllMulEvenAdd() {
+  ForShrinkableVectors<TestMulEvenAdd>()(bfloat16_t());
+}
+
+struct TestReorderWidenMulAccumulate {
+  // Must be inlined on aarch64 for bf16, else clang crashes.
+  template <typename TN, class DN>
+  HWY_INLINE void operator()(TN /*unused*/, DN dn) {
+    using TW = MakeWide<TN>;
+    const RepartitionToWide<DN> dw;
+    using VW = Vec<decltype(dw)>;
+    using VN = Vec<decltype(dn)>;
+    const size_t NN = Lanes(dn);
+
+    const VW f0 = Zero(dw);
+    const VW f1 = Set(dw, TW{1});
+    const VN bf0 = Zero(dn);
+    // Cannot Set() bfloat16_t directly.
+    const VN bf1 = ReorderDemote2To(dn, f1, f1);
+
+    // Any input zero => both outputs zero
+    VW sum1 = f0;
+    HWY_ASSERT_VEC_EQ(dw, f0,
+                      ReorderWidenMulAccumulate(dw, bf0, bf0, f0, sum1));
+    HWY_ASSERT_VEC_EQ(dw, f0, sum1);
+    HWY_ASSERT_VEC_EQ(dw, f0,
+                      ReorderWidenMulAccumulate(dw, bf0, bf1, f0, sum1));
+    HWY_ASSERT_VEC_EQ(dw, f0, sum1);
+    HWY_ASSERT_VEC_EQ(dw, f0,
+                      ReorderWidenMulAccumulate(dw, bf1, bf0, f0, sum1));
+    HWY_ASSERT_VEC_EQ(dw, f0, sum1);
+
+    // delta[p] := 1, all others zero. For each p: Dot(delta, all-ones) == 1.
+    auto delta_w = AllocateAligned<TW>(NN);
+    HWY_ASSERT(delta_w);
+    for (size_t p = 0; p < NN; ++p) {
+      // Workaround for incorrect Clang wasm codegen: re-initialize the entire
+      // array rather than zero-initialize once and then toggle lane p.
+      for (size_t i = 0; i < NN; ++i) {
+        delta_w[i] = static_cast<TW>(i == p);
+      }
+      const VW delta0 = Load(dw, delta_w.get());
+      const VW delta1 = Load(dw, delta_w.get() + NN / 2);
+      const VN delta = ReorderDemote2To(dn, delta0, delta1);
+
+      {
+        sum1 = f0;
+        const VW sum0 = ReorderWidenMulAccumulate(dw, delta, bf1, f0, sum1);
+        HWY_ASSERT_EQ(TW{1}, ReduceSum(dw, Add(sum0, sum1)));
+      }
+      // Swapped arg order
+      {
+        sum1 = f0;
+        const VW sum0 = ReorderWidenMulAccumulate(dw, bf1, delta, f0, sum1);
+        HWY_ASSERT_EQ(TW{1}, ReduceSum(dw, Add(sum0, sum1)));
+      }
+      // Start with nonzero sum0 or sum1
+      {
+        VW sum0 = delta0;
+        sum1 = delta1;
+        sum0 = ReorderWidenMulAccumulate(dw, delta, bf1, sum0, sum1);
+        HWY_ASSERT_EQ(TW{2}, ReduceSum(dw, Add(sum0, sum1)));
+      }
+      // Start with nonzero sum0 or sum1, and swap arg order
+      {
+        VW sum0 = delta0;
+        sum1 = delta1;
+        sum0 = ReorderWidenMulAccumulate(dw, bf1, delta, sum0, sum1);
+        HWY_ASSERT_EQ(TW{2}, ReduceSum(dw, Add(sum0, sum1)));
+      }
+    }
+  }
+};
+
+
+struct TestWidenMulAccumulate {
+  template <typename TN, class DN>
+  HWY_INLINE void operator()(TN /*unused*/, DN dn) {
+    using TW = MakeWide<TN>;
+    const RepartitionToWide<DN> dw;
+    const Half<DN> dnh;
+    using VW = Vec<decltype(dw)>;
+    using VN = Vec<decltype(dn)>;
+    const size_t NN = Lanes(dn);
+
+    VW f0 = Zero(dw);
+    const VN bf0 = Zero(dn);
+    const VN bf1 = Set(dn, TW{5});
+
+    // Any input zero => both outputs zero
+    VW sum1 = f0;
+    HWY_ASSERT_VEC_EQ(dw, f0,
+                      WidenMulAccumulate(dw, bf0, bf0, f0, sum1));
+    HWY_ASSERT_VEC_EQ(dw, f0, sum1);
+    HWY_ASSERT_VEC_EQ(dw, f0,
+                      WidenMulAccumulate(dw, bf0, bf1, f0, sum1));
+    HWY_ASSERT_VEC_EQ(dw, f0, sum1);
+    HWY_ASSERT_VEC_EQ(dw, f0, WidenMulAccumulate(dw, bf1, bf0, f0, sum1));
+    HWY_ASSERT_VEC_EQ(dw, f0, sum1);
+
+    // delta[p] := 1, all others zero.
+    auto delta_w = AllocateAligned<TW>(NN);
+    HWY_ASSERT(delta_w);
+    for (size_t p = 0; p < NN; ++p) {
+      // Workaround for incorrect Clang wasm codegen: re-initialize the entire
+      // array rather than zero-initialize once and then toggle lane p.
+      for (size_t i = 0; i < NN; ++i) {
+        delta_w[i] = static_cast<TW>(i == p);
+      }
+      const VW delta0 = Load(dw, delta_w.get());
+      const VW delta1 = Load(dw, delta_w.get() + NN / 2);
+      const VN delta = Combine(dn, DemoteTo(dnh, Add(delta0, delta1)),
+                               DemoteTo(dnh, Sub(delta0, delta1)));
+      VW highSumBefore;
+
+      f0 = Set(dw, 6);
+
+      {
+        sum1 = f0;
+        highSumBefore = sum1;
+        const VW sum0 = WidenMulAccumulate(dw, delta, bf1, f0, sum1);
+        const VW expectedLow =
+            MulAdd(PromoteLowerTo(dw, delta), PromoteLowerTo(dw, bf1), f0);
+        HWY_ASSERT_VEC_EQ(dw, expectedLow, sum0);
+        const VW expectedHigh = MulAdd(PromoteUpperTo(dw, delta),
+                                       PromoteUpperTo(dw, bf1), highSumBefore);
+        HWY_ASSERT_VEC_EQ(dw, expectedHigh, sum1);
+      }
+      // Swapped arg order
+      {
+        sum1 = f0;
+        highSumBefore = sum1;
+        const VW sum0 = WidenMulAccumulate(dw, bf1, delta, f0, sum1);
+        const VW expectedLow =
+            MulAdd(PromoteLowerTo(dw, bf1), PromoteLowerTo(dw, delta), f0);
+        HWY_ASSERT_VEC_EQ(dw, expectedLow, sum0);
+        const VW expectedHigh = MulAdd(
+            PromoteUpperTo(dw, bf1), PromoteUpperTo(dw, delta), highSumBefore);
+        HWY_ASSERT_VEC_EQ(dw, expectedHigh, sum1);
+      }
+      // Start with nonzero sum0 or sum1
+      {
+        VW sum0 = PromoteTo(dw, LowerHalf(dnh, delta));
+        VW lowSumBefore = sum0;
+        sum1 = PromoteTo(dw, UpperHalf(dnh, delta));
+        highSumBefore = sum1;
+        sum0 = WidenMulAccumulate(dw, delta, bf1, sum0, sum1);
+        const VW expectedLow = MulAdd(PromoteLowerTo(dw, delta),
+                                      PromoteLowerTo(dw, bf1), lowSumBefore);
+        HWY_ASSERT_VEC_EQ(dw, expectedLow, sum0);
+        const VW expectedHigh = MulAdd(PromoteUpperTo(dw, delta),
+                                       PromoteUpperTo(dw, bf1), highSumBefore);
+        HWY_ASSERT_VEC_EQ(dw, expectedHigh, sum1);
+      }
+      // Start with nonzero sum0 or sum1, and swap arg order
+      {
+        VW sum0 = PromoteTo(dw, LowerHalf(dnh, delta));
+        VW lowSumBefore = sum0;
+        sum1 = PromoteTo(dw, UpperHalf(dnh, delta));
+        highSumBefore = sum1;
+        sum0 = WidenMulAccumulate(dw, bf1, delta, sum0, sum1);
+        const VW expectedLow = MulAdd(PromoteLowerTo(dw, bf1),
+                                      PromoteLowerTo(dw, delta), lowSumBefore);
+        HWY_ASSERT_VEC_EQ(dw, expectedLow, sum0);
+        const VW expectedHigh = MulAdd(
+            PromoteUpperTo(dw, bf1), PromoteUpperTo(dw, delta), highSumBefore);
+        HWY_ASSERT_VEC_EQ(dw, expectedHigh, sum1);
+      }
+    }
+  }
+};
+
+HWY_NOINLINE void TestAllWidenMulAccumulate() {
+  ForShrinkableVectors<TestWidenMulAccumulate>()(int32_t());
+  ForShrinkableVectors<TestWidenMulAccumulate>()(uint32_t());
+  ForShrinkableVectors<TestWidenMulAccumulate>()(int16_t());
+  ForShrinkableVectors<TestWidenMulAccumulate>()(uint16_t());
+  ForShrinkableVectors<TestWidenMulAccumulate>()(int8_t());
+  ForShrinkableVectors<TestWidenMulAccumulate>()(uint8_t());
+#if 0  // not yet implemented
+#if HWY_HAVE_FLOAT16 && HWY_TARGET_IS_NEON
+  ForShrinkableVectors<TestWidenMulAccumulate>()(float16_t());
+#endif
+#endif
+}
+
+HWY_NOINLINE void TestAllReorderWidenMulAccumulate() {
+  ForShrinkableVectors<TestReorderWidenMulAccumulate>()(bfloat16_t());
+  ForShrinkableVectors<TestReorderWidenMulAccumulate>()(int16_t());
+  ForShrinkableVectors<TestReorderWidenMulAccumulate>()(uint16_t());
+}
+
+struct TestRearrangeToOddPlusEven {
+  // Must be inlined on aarch64 for bf16, else clang crashes.
+  template <typename TN, class DN>
+  HWY_INLINE void operator()(TN /*unused*/, DN dn) {
+    using TW = MakeWide<TN>;
+    const RepartitionToWide<DN> dw;
+    using VW = Vec<decltype(dw)>;
+    using VN = Vec<decltype(dn)>;
+    const size_t NW = Lanes(dw);
+
+    const auto expected = AllocateAligned<TW>(NW);
+    HWY_ASSERT(expected);
+    for (size_t iw = 0; iw < NW; ++iw) {
+      const size_t in = iw * 2;  // even, odd is +1
+      const size_t a0 = 1 + in;
+      const size_t b0 = 1 + 2 * NW - a0;
+      const size_t a1 = a0 + 1;
+      const size_t b1 = b0 - 1;
+      expected[iw] = static_cast<TW>(a0 * b0 + a1 * b1);
+    }
+
+    const VW up0 = Iota(dw, 1);
+    const VW up1 = Iota(dw, 1 + NW);
+    // We will compute i * (N-i) to avoid per-lane overflow.
+    const VW down0 = Reverse(dw, up1);
+    const VW down1 = Reverse(dw, up0);
+
+    const VN a = OrderedDemote2To(dn, up0, up1);
+    const VN b = OrderedDemote2To(dn, down0, down1);
+
+    VW sum0 = Zero(dw);
+    VW sum1 = Zero(dw);
+    sum0 = ReorderWidenMulAccumulate(dw, a, b, sum0, sum1);
+    const VW sum_odd_even = RearrangeToOddPlusEven(sum0, sum1);
+    HWY_ASSERT_VEC_EQ(dw, expected.get(), sum_odd_even);
+  }
+};
+
+HWY_NOINLINE void TestAllRearrangeToOddPlusEven() {
+// For reasons unknown, <128 bit crashes aarch64 clang.
+#if HWY_ARCH_ARM_A64 && HWY_COMPILER_CLANG
+  ForGEVectors<128, TestRearrangeToOddPlusEven>()(bfloat16_t());
+#else
+  ForShrinkableVectors<TestRearrangeToOddPlusEven>()(bfloat16_t());
+#endif
+  ForShrinkableVectors<TestRearrangeToOddPlusEven>()(int16_t());
+  ForShrinkableVectors<TestRearrangeToOddPlusEven>()(uint16_t());
+}
+
+template <bool MixedSignedness>
+struct TestSumOfMulQuadAccumulate {
+  template <class DW2, class TN1, class TN2>
+  static HWY_INLINE void TestConsecutiveSeqMulQuadAccum(DW2 dw2, TN1 a0,
+                                                        TN2 b0) {
+    using TW2 = TFromD<DW2>;
+    const Repartition<TN1, DW2> dn1;
+    const Repartition<TN2, DW2> dn2;
+
+    const auto vn_iota0_mod4 = And(Iota(dn1, 0), Set(dn1, TN1{3}));
+
+    const auto va = Add(vn_iota0_mod4, Set(dn1, a0));
+    const auto vb = Add(BitCast(dn2, vn_iota0_mod4), Set(dn2, b0));
+    const auto expected =
+        Set(dw2,
+            static_cast<TW2>((TW2{4} * static_cast<TW2>(a0) * b0) +
+                             (TW2{6} * (static_cast<TW2>(a0) + b0)) + TW2{17}));
+
+    HWY_ASSERT_VEC_EQ(dw2, expected,
+                      SumOfMulQuadAccumulate(dw2, va, vb, Set(dw2, TW2{3})));
+  }
+
+  template <typename TN2, class DN2>
+  HWY_INLINE void operator()(TN2 /*unused*/, DN2 dn2) {
+    static_assert(!MixedSignedness || IsSigned<TN2>(),
+                  "TN2 must be signed if MixedSignedness is true");
+    using TN1 = If<MixedSignedness, MakeUnsigned<TN2>, TN2>;
+    using TW2 = MakeWide<MakeWide<TN2>>;
+
+    const Rebind<TN1, DN2> dn1;
+    const Repartition<TW2, DN2> dw2;
+
+    const auto vn1_k1 = Set(dn1, TN1{1});
+    const auto vn2_k1 = BitCast(dn2, vn1_k1);
+    const auto vn1_k4 = Set(dn1, TN1{4});
+    const auto vn2_k4 = BitCast(dn2, vn1_k4);
+
+    const auto vw2_k0 = Zero(dw2);
+    const auto vw2_k1 = Set(dw2, TW2{1});
+    const auto vw2_k4 = Set(dw2, TW2{4});
+    const auto vw2_k5 = Set(dw2, TW2{5});
+    const auto vw2_k21 = Set(dw2, TW2{21});
+
+    HWY_ASSERT_VEC_EQ(dw2, vw2_k4,
+                      SumOfMulQuadAccumulate(dw2, vn1_k1, vn2_k1, vw2_k0));
+    HWY_ASSERT_VEC_EQ(dw2, vw2_k5,
+                      SumOfMulQuadAccumulate(dw2, vn1_k1, vn2_k1, vw2_k1));
+    HWY_ASSERT_VEC_EQ(dw2, vw2_k21,
+                      SumOfMulQuadAccumulate(dw2, vn1_k1, vn2_k4, vw2_k5));
+    HWY_ASSERT_VEC_EQ(dw2, vw2_k21,
+                      SumOfMulQuadAccumulate(dw2, vn1_k4, vn2_k1, vw2_k5));
+
+    constexpr TN1 kTN1ValWithMaxMag =
+        static_cast<TN1>(IsSigned<TN1>() ? LimitsMin<TN1>() : LimitsMax<TN1>());
+    constexpr TN2 kTN2ValWithMaxMag =
+        static_cast<TN2>(IsSigned<TN2>() ? LimitsMin<TN2>() : LimitsMax<TN2>());
+    HWY_ASSERT_VEC_EQ(
+        dw2,
+        Set(dw2, static_cast<TW2>(static_cast<TW2>(kTN1ValWithMaxMag) *
+                                  kTN2ValWithMaxMag * TW2{4})),
+        SumOfMulQuadAccumulate(dw2, Set(dn1, kTN1ValWithMaxMag),
+                               Set(dn2, kTN2ValWithMaxMag), vw2_k0));
+
+    TestConsecutiveSeqMulQuadAccum(dw2, static_cast<TN1>(27),
+                                   static_cast<TN2>(34));
+    TestConsecutiveSeqMulQuadAccum(dw2, static_cast<TN1>(13),
+                                   static_cast<TN2>(-5));
+    TestConsecutiveSeqMulQuadAccum(dw2, static_cast<TN1>(-29),
+                                   static_cast<TN2>(2));
+    TestConsecutiveSeqMulQuadAccum(dw2, static_cast<TN1>(-14),
+                                   static_cast<TN2>(-35));
+    TestConsecutiveSeqMulQuadAccum(dw2, static_cast<TN1>(LimitsMin<TN1>() + 5),
+                                   static_cast<TN2>(LimitsMax<TN2>() - 4));
+    TestConsecutiveSeqMulQuadAccum(dw2, static_cast<TN1>(LimitsMax<TN1>() - 4),
+                                   static_cast<TN2>(LimitsMin<TN2>() + 11));
+  }
+};
+
+HWY_NOINLINE void TestAllSumOfMulQuadAccumulate() {
+  ForShrinkableVectors<TestSumOfMulQuadAccumulate<false>, 2>()(int8_t());
+  ForShrinkableVectors<TestSumOfMulQuadAccumulate<false>, 2>()(uint8_t());
+  ForShrinkableVectors<TestSumOfMulQuadAccumulate<true>, 2>()(int8_t());
+#if HWY_HAVE_INTEGER64
+  ForShrinkableVectors<TestSumOfMulQuadAccumulate<false>, 2>()(int16_t());
+  ForShrinkableVectors<TestSumOfMulQuadAccumulate<false>, 2>()(uint16_t());
+#endif
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+HWY_AFTER_NAMESPACE();
+
+#if HWY_ONCE
+namespace hwy {
+namespace {
+HWY_BEFORE_TEST(HwyWidenMulTest);
+HWY_EXPORT_AND_TEST_P(HwyWidenMulTest, TestAllSatWidenMulAccumFixedPoint);
+HWY_EXPORT_AND_TEST_P(HwyWidenMulTest, TestAllMulEvenAdd);
+HWY_EXPORT_AND_TEST_P(HwyWidenMulTest, TestAllWidenMulAccumulate);
+HWY_EXPORT_AND_TEST_P(HwyWidenMulTest, TestAllReorderWidenMulAccumulate);
+HWY_EXPORT_AND_TEST_P(HwyWidenMulTest, TestAllRearrangeToOddPlusEven);
+HWY_EXPORT_AND_TEST_P(HwyWidenMulTest, TestAllSumOfMulQuadAccumulate);
+HWY_AFTER_TEST();
+}  // namespace
+}  // namespace hwy
+HWY_TEST_MAIN();
+#endif  // HWY_ONCE