Dante/Kmake
1
0
Fork 0
forked from LeenkxTeam/Kmake
Code Pull Requests Activity

Upload Kmake

Browse Source
This commit is contained in:
Gorochu
2026-05-26 23:36:42 -07:00
parent ba051b2f74
commit 555ec72358
41615 changed files with 13344630 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
deps/v8/third_party/highway/BUILD.gn vendored Normal file
View File

@ -0,0 +1,50 @@
# Copyright 2020 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# This config is applied to targets that depend on libhwy
config("libhwy_external_config") {
include_dirs = [ "src" ]
defines = []
if (target_cpu == "x86") {
# Highway automatically disables AVX2 and AVX3 targets when compiling in
# 32-bit mode due to bugs in the generated code. However, when doing so it
# prints a warning (#pragma message). This flag silences that warning by
# explicitly disabling AVX2 and AVX3 targets.
defines += [ "HWY_BROKEN_TARGETS=(HWY_AVX2|HWY_AVX3)" ]
}
if (target_cpu == "arm64") {
# There are Chrome targets that use HWY_ALIGN on absl::uint128. This breaks
# with SVE, since HWY_ALIGN == 8 for it and the minimal alignment requirement
# for absl::uint128 is 16.
defines += [ "HWY_BROKEN_TARGETS=HWY_ALL_SVE" ]
}
if (target_cpu == "ppc64" || target_cpu == "s390x") {
# Not using <asm/hwcap.h> on ppc64/s390x:
# https://github.com/google/highway/issues/2406
defines += [ "TOOLCHAIN_MISS_ASM_HWCAP_H" ]
}
if (target_cpu == "s390x") {
# z13 relies only on emulation.
# Start using `-march=z14 -mzvector` once ready.
defines += [ "HWY_BROKEN_EMU128=0" ]
}
if (current_os == "aix") {
# enable emulation until highway aix support is ready.
defines += [ "HWY_BROKEN_EMU128=0" ]
}
}
source_set("libhwy") {
sources = [
"src/hwy/abort.cc",
"src/hwy/aligned_allocator.cc",
"src/hwy/per_target.cc",
"src/hwy/print.cc",
"src/hwy/stats.cc",
"src/hwy/targets.cc",
"src/hwy/timer.cc",
]
public_configs = [ ":libhwy_external_config" ]
}

6
deps/v8/third_party/highway/DIR_METADATA vendored Normal file
View File

@ -0,0 +1,6 @@
monorail: {
component: "Internals>Images>Codecs"
}
buganizer_public: {
component_id: 1456316
}

201
deps/v8/third_party/highway/LICENSE vendored Normal file
View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

2
deps/v8/third_party/highway/OWNERS vendored Normal file
View File

@ -0,0 +1,2 @@
bikineev@chromium.org
file:../../INFRA_OWNERS

17
deps/v8/third_party/highway/README.chromium vendored Normal file
View File

@ -0,0 +1,17 @@
Name: Highway: C++ library for SIMD
Short Name: highway
URL: https://github.com/google/highway
Version: N/A
Date: 2024-05-16
Revision: 8295336dd70f1201d42c22ab5b0861de38cf8fbf
License: Apache-2.0
License File: LICENSE
Security Critical: yes
Shipped: yes
CPEPrefix: unknown
Description:
Highway is a C++ library for SIMD (Single Instruction, Multiple Data), i.e. applying the same operation to 'lanes'. This library is a dependency from libjxl to use SIMD instructions across multiple target platforms and runtime CPUs.
Local Modifications:
None.

75
deps/v8/third_party/highway/src/hwy/abort.cc vendored Normal file
View File

@ -0,0 +1,75 @@
// Copyright 2019 Google LLC
// Copyright 2024 Arm Limited and/or its affiliates <open-source-office@arm.com>
// SPDX-License-Identifier: Apache-2.0
// SPDX-License-Identifier: BSD-3-Clause
#include "hwy/abort.h"
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include "hwy/base.h"
#if HWY_IS_ASAN || HWY_IS_MSAN || HWY_IS_TSAN
#include "sanitizer/common_interface_defs.h" // __sanitizer_print_stack_trace
#endif
namespace hwy {
namespace {
std::string GetBaseName(std::string const& file_name) {
auto last_slash = file_name.find_last_of("/\\");
return file_name.substr(last_slash + 1);
}
} // namespace
HWY_DLLEXPORT AbortFunc& GetAbortFunc() {
static AbortFunc func;
return func;
}
HWY_DLLEXPORT AbortFunc SetAbortFunc(AbortFunc func) {
const AbortFunc prev = GetAbortFunc();
GetAbortFunc() = func;
return prev;
}
HWY_DLLEXPORT HWY_NORETURN void HWY_FORMAT(3, 4)
Abort(const char* file, int line, const char* format, ...) {
char buf[800];
va_list args;
va_start(args, format);
vsnprintf(buf, sizeof(buf), format, args);
va_end(args);
AbortFunc handler = GetAbortFunc();
if (handler != nullptr) {
handler(file, line, buf);
} else {
fprintf(stderr, "Abort at %s:%d: %s\n", GetBaseName(file).data(), line,
buf);
}
// If compiled with any sanitizer, they can also print a stack trace.
#if HWY_IS_ASAN || HWY_IS_MSAN || HWY_IS_TSAN
__sanitizer_print_stack_trace();
#endif // HWY_IS_*
fflush(stderr);
// Now terminate the program:
#if HWY_ARCH_RISCV
exit(1); // trap/abort just freeze Spike.
#elif HWY_IS_DEBUG_BUILD && !HWY_COMPILER_MSVC && !HWY_ARCH_ARM
// Facilitates breaking into a debugger, but don't use this in non-debug
// builds because it looks like "illegal instruction", which is misleading.
// Also does not work on Arm.
__builtin_trap();
#else
abort(); // Compile error without this due to HWY_NORETURN.
#endif
}
} // namespace hwy

28
deps/v8/third_party/highway/src/hwy/abort.h vendored Normal file
View File

@ -0,0 +1,28 @@
// Copyright 2024 Arm Limited and/or its affiliates <open-source-office@arm.com>
// SPDX-License-Identifier: Apache-2.0
// SPDX-License-Identifier: BSD-3-Clause
#ifndef HIGHWAY_HWY_ABORT_H_
#define HIGHWAY_HWY_ABORT_H_
#include "hwy/highway_export.h"
namespace hwy {
// Interface for custom abort handler
typedef void (*AbortFunc)(const char* file, int line,
const char* formatted_err);
// Retrieve current abort handler
// Returns null if no abort handler registered, indicating Highway should print and abort
HWY_DLLEXPORT AbortFunc& GetAbortFunc();
// Sets a new abort handler and returns the previous abort handler
// If this handler does not do the aborting itself Highway will use its own abort mechanism
// which allows this to be used to customize the handling of the error itself.
// Returns null if no previous abort handler registered
HWY_DLLEXPORT AbortFunc SetAbortFunc(AbortFunc func);
} // namespace hwy
#endif // HIGHWAY_HWY_ABORT_H_

73
deps/v8/third_party/highway/src/hwy/abort_test.cc vendored Normal file
View File

@ -0,0 +1,73 @@
// Copyright 2024 Arm Limited and/or its affiliates <open-source-office@arm.com>
// SPDX-License-Identifier: Apache-2.0
// SPDX-License-Identifier: BSD-3-Clause
#include "hwy/abort.h"
#include <stdio.h>
#include <string>
#include "hwy/base.h"
#include "hwy/tests/hwy_gtest.h"
#include "hwy/tests/test_util-inl.h" // HWY_ASSERT_EQ
namespace hwy {
namespace {
#ifdef GTEST_HAS_DEATH_TEST
std::string GetBaseName(std::string const& file_name) {
auto last_slash = file_name.find_last_of("/\\");
return file_name.substr(last_slash + 1);
}
TEST(AbortDeathTest, AbortDefault) {
std::string expected = std::string("Abort at ") + GetBaseName(__FILE__) +
":" + std::to_string(__LINE__ + 1) + ": Test Abort";
ASSERT_DEATH(HWY_ABORT("Test %s", "Abort"), expected);
}
TEST(AbortDeathTest, AbortOverride) {
const AbortFunc CustomAbortHandler = [](const char* file, int line,
const char* formatted_err) -> void {
fprintf(stderr, "%s from %02d of %s", formatted_err, line,
GetBaseName(file).data());
};
SetAbortFunc(CustomAbortHandler);
// googletest regex does not support `+` for digits on Windows?!
// https://google.github.io/googletest/advanced.html#regular-expression-syntax
// Hence we insert the expected line number manually.
char buf[100];
const std::string file = GetBaseName(__FILE__);
const int line = __LINE__ + 2; // from which HWY_ABORT is called
snprintf(buf, sizeof(buf), "Test Abort from %02d of %s", line, file.c_str());
ASSERT_DEATH({ HWY_ABORT("Test %s", "Abort"); }, buf);
}
#endif // GTEST_HAS_DEATH_TEST
TEST(AbortTest, AbortOverrideChain) {
AbortFunc FirstHandler = [](const char* file, int line,
const char* formatted_err) -> void {
fprintf(stderr, "%s from %d of %s", formatted_err, line, file);
};
AbortFunc SecondHandler = [](const char* file, int line,
const char* formatted_err) -> void {
fprintf(stderr, "%s from %d of %s", formatted_err, line, file);
};
// Do not check that the first SetAbortFunc returns nullptr, because it is
// not guaranteed to be the first call - other TEST may come first.
(void)SetAbortFunc(FirstHandler);
HWY_ASSERT(GetAbortFunc() == FirstHandler);
HWY_ASSERT(SetAbortFunc(SecondHandler) == FirstHandler);
HWY_ASSERT(GetAbortFunc() == SecondHandler);
HWY_ASSERT(SetAbortFunc(nullptr) == SecondHandler);
HWY_ASSERT(GetAbortFunc() == nullptr);
}
} // namespace
} // namespace hwy
HWY_TEST_MAIN();

156
deps/v8/third_party/highway/src/hwy/aligned_allocator.cc vendored Normal file
View File

@ -0,0 +1,156 @@
// Copyright 2019 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/aligned_allocator.h"
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h> // malloc
#include <atomic>
#include <limits>
#include "hwy/base.h"
namespace hwy {
namespace {
#if HWY_ARCH_RISCV && defined(__riscv_v_intrinsic) && \
__riscv_v_intrinsic >= 11000
// Not actually an upper bound on the size, but this value prevents crossing a
// 4K boundary (relevant on Andes).
constexpr size_t kAlignment = HWY_MAX(HWY_ALIGNMENT, 4096);
#else
constexpr size_t kAlignment = HWY_ALIGNMENT;
#endif
#if HWY_ARCH_X86
// On x86, aliasing can only occur at multiples of 2K. To reduce the chance of
// allocations being equal mod 2K, we round up to kAlias and add a cyclic
// offset which is a multiple of kAlignment. Rounding up to only 1K decreases
// the number of alias-free allocations, but also wastes less memory.
constexpr size_t kAlias = HWY_MAX(kAlignment, 1024);
#else
constexpr size_t kAlias = kAlignment;
#endif
#pragma pack(push, 1)
struct AllocationHeader {
void* allocated;
size_t payload_size;
};
#pragma pack(pop)
// Returns a 'random' (cyclical) offset for AllocateAlignedBytes.
size_t NextAlignedOffset() {
static std::atomic<size_t> next{0};
static_assert(kAlias % kAlignment == 0, "kAlias must be a multiple");
constexpr size_t kGroups = kAlias / kAlignment;
const size_t group = next.fetch_add(1, std::memory_order_relaxed) % kGroups;
const size_t offset = kAlignment * group;
HWY_DASSERT((offset % kAlignment == 0) && offset <= kAlias);
return offset;
}
} // namespace
HWY_DLLEXPORT void* AllocateAlignedBytes(const size_t payload_size,
AllocPtr alloc_ptr, void* opaque_ptr) {
HWY_ASSERT(payload_size != 0); // likely a bug in caller
if (payload_size >= std::numeric_limits<size_t>::max() / 2) {
HWY_DASSERT(false && "payload_size too large");
return nullptr;
}
size_t offset = NextAlignedOffset();
// What: | misalign | unused | AllocationHeader |payload
// Size: |<= kAlias | offset |payload_size
// ^allocated.^aligned.^header............^payload
// The header must immediately precede payload, which must remain aligned.
// To avoid wasting space, the header resides at the end of `unused`,
// which therefore cannot be empty (offset == 0).
if (offset == 0) {
offset = RoundUpTo(sizeof(AllocationHeader), kAlignment);
}
const size_t allocated_size = kAlias + offset + payload_size;
void* allocated;
if (alloc_ptr == nullptr) {
allocated = malloc(allocated_size);
} else {
allocated = (*alloc_ptr)(opaque_ptr, allocated_size);
}
if (allocated == nullptr) return nullptr;
// Always round up even if already aligned - we already asked for kAlias
// extra bytes and there's no way to give them back.
uintptr_t aligned = reinterpret_cast<uintptr_t>(allocated) + kAlias;
static_assert((kAlias & (kAlias - 1)) == 0, "kAlias must be a power of 2");
static_assert(kAlias >= kAlignment, "Cannot align to more than kAlias");
aligned &= ~(kAlias - 1);
const uintptr_t payload = aligned + offset; // still aligned
HWY_DASSERT(payload % kAlignment == 0);
// Stash `allocated` and payload_size inside header for FreeAlignedBytes().
// The allocated_size can be reconstructed from the payload_size.
AllocationHeader* header = reinterpret_cast<AllocationHeader*>(payload) - 1;
HWY_DASSERT(reinterpret_cast<uintptr_t>(header) >= aligned);
header->allocated = allocated;
header->payload_size = payload_size;
return HWY_ASSUME_ALIGNED(reinterpret_cast<void*>(payload), kAlignment);
}
HWY_DLLEXPORT void FreeAlignedBytes(const void* aligned_pointer,
FreePtr free_ptr, void* opaque_ptr) {
if (aligned_pointer == nullptr) return;
const uintptr_t payload = reinterpret_cast<uintptr_t>(aligned_pointer);
HWY_DASSERT(payload % kAlignment == 0);
const AllocationHeader* header =
reinterpret_cast<const AllocationHeader*>(payload) - 1;
if (free_ptr == nullptr) {
free(header->allocated);
} else {
(*free_ptr)(opaque_ptr, header->allocated);
}
}
// static
HWY_DLLEXPORT void AlignedDeleter::DeleteAlignedArray(void* aligned_pointer,
FreePtr free_ptr,
void* opaque_ptr,
ArrayDeleter deleter) {
if (aligned_pointer == nullptr) return;
const uintptr_t payload = reinterpret_cast<uintptr_t>(aligned_pointer);
HWY_DASSERT(payload % kAlignment == 0);
const AllocationHeader* header =
reinterpret_cast<const AllocationHeader*>(payload) - 1;
if (deleter) {
(*deleter)(aligned_pointer, header->payload_size);
}
if (free_ptr == nullptr) {
free(header->allocated);
} else {
(*free_ptr)(opaque_ptr, header->allocated);
}
}
} // namespace hwy

423
deps/v8/third_party/highway/src/hwy/aligned_allocator.h vendored Normal file
View File

@ -0,0 +1,423 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_ALIGNED_ALLOCATOR_H_
#define HIGHWAY_HWY_ALIGNED_ALLOCATOR_H_
// Memory allocator with support for alignment and offsets.
#include <algorithm>
#include <array>
#include <cassert>
#include <cstdint>
#include <cstring>
#include <initializer_list>
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>
#include "hwy/base.h"
#include "hwy/per_target.h"
namespace hwy {
// Minimum alignment of allocated memory for use in HWY_ASSUME_ALIGNED, which
// requires a literal. To prevent false sharing, this should be at least the
// L1 cache line size, usually 64 bytes. However, Intel's L2 prefetchers may
// access pairs of lines, and M1 L2 and POWER8 lines are also 128 bytes.
#define HWY_ALIGNMENT 128
template <typename T>
HWY_API constexpr bool IsAligned(T* ptr, size_t align = HWY_ALIGNMENT) {
return reinterpret_cast<uintptr_t>(ptr) % align == 0;
}
// Pointers to functions equivalent to malloc/free with an opaque void* passed
// to them.
using AllocPtr = void* (*)(void* opaque, size_t bytes);
using FreePtr = void (*)(void* opaque, void* memory);
// Returns null or a pointer to at least `payload_size` (which can be zero)
// bytes of newly allocated memory, aligned to the larger of HWY_ALIGNMENT and
// the vector size. Calls `alloc` with the passed `opaque` pointer to obtain
// memory or malloc() if it is null.
HWY_DLLEXPORT void* AllocateAlignedBytes(size_t payload_size,
AllocPtr alloc_ptr = nullptr,
void* opaque_ptr = nullptr);
// Frees all memory. No effect if `aligned_pointer` == nullptr, otherwise it
// must have been returned from a previous call to `AllocateAlignedBytes`.
// Calls `free_ptr` with the passed `opaque_ptr` pointer to free the memory; if
// `free_ptr` function is null, uses the default free().
HWY_DLLEXPORT void FreeAlignedBytes(const void* aligned_pointer,
FreePtr free_ptr, void* opaque_ptr);
// Class that deletes the aligned pointer passed to operator() calling the
// destructor before freeing the pointer. This is equivalent to the
// std::default_delete but for aligned objects. For a similar deleter equivalent
// to free() for aligned memory see AlignedFreer().
class AlignedDeleter {
public:
AlignedDeleter() : free_(nullptr), opaque_ptr_(nullptr) {}
AlignedDeleter(FreePtr free_ptr, void* opaque_ptr)
: free_(free_ptr), opaque_ptr_(opaque_ptr) {}
template <typename T>
void operator()(T* aligned_pointer) const {
return DeleteAlignedArray(aligned_pointer, free_, opaque_ptr_,
TypedArrayDeleter<T>);
}
private:
template <typename T>
static void TypedArrayDeleter(void* ptr, size_t size_in_bytes) {
size_t elems = size_in_bytes / sizeof(T);
for (size_t i = 0; i < elems; i++) {
// Explicitly call the destructor on each element.
(static_cast<T*>(ptr) + i)->~T();
}
}
// Function prototype that calls the destructor for each element in a typed
// array. TypeArrayDeleter<T> would match this prototype.
using ArrayDeleter = void (*)(void* t_ptr, size_t t_size);
HWY_DLLEXPORT static void DeleteAlignedArray(void* aligned_pointer,
FreePtr free_ptr,
void* opaque_ptr,
ArrayDeleter deleter);
FreePtr free_;
void* opaque_ptr_;
};
// Unique pointer to T with custom aligned deleter. This can be a single
// element U or an array of element if T is a U[]. The custom aligned deleter
// will call the destructor on U or each element of a U[] in the array case.
template <typename T>
using AlignedUniquePtr = std::unique_ptr<T, AlignedDeleter>;
// Aligned memory equivalent of make_unique<T> using the custom allocators
// alloc/free with the passed `opaque` pointer. This function calls the
// constructor with the passed Args... and calls the destructor of the object
// when the AlignedUniquePtr is destroyed.
template <typename T, typename... Args>
AlignedUniquePtr<T> MakeUniqueAlignedWithAlloc(AllocPtr alloc, FreePtr free,
void* opaque, Args&&... args) {
T* ptr = static_cast<T*>(AllocateAlignedBytes(sizeof(T), alloc, opaque));
return AlignedUniquePtr<T>(new (ptr) T(std::forward<Args>(args)...),
AlignedDeleter(free, opaque));
}
// Similar to MakeUniqueAlignedWithAlloc but using the default alloc/free
// functions.
template <typename T, typename... Args>
AlignedUniquePtr<T> MakeUniqueAligned(Args&&... args) {
T* ptr = static_cast<T*>(AllocateAlignedBytes(sizeof(T)));
return AlignedUniquePtr<T>(new (ptr) T(std::forward<Args>(args)...),
AlignedDeleter());
}
template <class T>
struct AlignedAllocator {
using value_type = T;
AlignedAllocator() = default;
template <class V>
explicit AlignedAllocator(const AlignedAllocator<V>&) noexcept {}
template <class V>
value_type* allocate(V n) {
static_assert(std::is_integral<V>::value,
"AlignedAllocator only supports integer types");
static_assert(sizeof(V) <= sizeof(std::size_t),
"V n must be smaller or equal size_t to avoid overflow");
return static_cast<value_type*>(
AllocateAlignedBytes(static_cast<std::size_t>(n) * sizeof(value_type)));
}
template <class V>
void deallocate(value_type* p, HWY_MAYBE_UNUSED V n) {
return FreeAlignedBytes(p, nullptr, nullptr);
}
};
template <class T, class V>
constexpr bool operator==(const AlignedAllocator<T>&,
const AlignedAllocator<V>&) noexcept {
return true;
}
template <class T, class V>
constexpr bool operator!=(const AlignedAllocator<T>&,
const AlignedAllocator<V>&) noexcept {
return false;
}
template <class T>
using AlignedVector = std::vector<T, AlignedAllocator<T>>;
// Helpers for array allocators (avoids overflow)
namespace detail {
// Returns x such that 1u << x == n (if n is a power of two).
static inline constexpr size_t ShiftCount(size_t n) {
return (n <= 1) ? 0 : 1 + ShiftCount(n / 2);
}
template <typename T>
T* AllocateAlignedItems(size_t items, AllocPtr alloc_ptr, void* opaque_ptr) {
constexpr size_t kSize = sizeof(T);
constexpr bool kIsPow2 = (kSize & (kSize - 1)) == 0;
constexpr size_t kBits = ShiftCount(kSize);
static_assert(!kIsPow2 || (1ull << kBits) == kSize, "ShiftCount has a bug");
const size_t bytes = kIsPow2 ? items << kBits : items * kSize;
const size_t check = kIsPow2 ? bytes >> kBits : bytes / kSize;
if (check != items) {
return nullptr; // overflowed
}
return static_cast<T*>(AllocateAlignedBytes(bytes, alloc_ptr, opaque_ptr));
}
} // namespace detail
// Aligned memory equivalent of make_unique<T[]> for array types using the
// custom allocators alloc/free. This function calls the constructor with the
// passed Args... on every created item. The destructor of each element will be
// called when the AlignedUniquePtr is destroyed.
template <typename T, typename... Args>
AlignedUniquePtr<T[]> MakeUniqueAlignedArrayWithAlloc(
size_t items, AllocPtr alloc, FreePtr free, void* opaque, Args&&... args) {
T* ptr = detail::AllocateAlignedItems<T>(items, alloc, opaque);
if (ptr != nullptr) {
for (size_t i = 0; i < items; i++) {
new (ptr + i) T(std::forward<Args>(args)...);
}
}
return AlignedUniquePtr<T[]>(ptr, AlignedDeleter(free, opaque));
}
template <typename T, typename... Args>
AlignedUniquePtr<T[]> MakeUniqueAlignedArray(size_t items, Args&&... args) {
return MakeUniqueAlignedArrayWithAlloc<T, Args...>(
items, nullptr, nullptr, nullptr, std::forward<Args>(args)...);
}
// Custom deleter for std::unique_ptr equivalent to using free() as a deleter
// but for aligned memory.
class AlignedFreer {
public:
// Pass address of this to ctor to skip deleting externally-owned memory.
static void DoNothing(void* /*opaque*/, void* /*aligned_pointer*/) {}
AlignedFreer() : free_(nullptr), opaque_ptr_(nullptr) {}
AlignedFreer(FreePtr free_ptr, void* opaque_ptr)
: free_(free_ptr), opaque_ptr_(opaque_ptr) {}
template <typename T>
void operator()(T* aligned_pointer) const {
// TODO(deymo): assert that we are using a POD type T.
FreeAlignedBytes(aligned_pointer, free_, opaque_ptr_);
}
private:
FreePtr free_;
void* opaque_ptr_;
};
// Unique pointer to single POD, or (if T is U[]) an array of POD. For non POD
// data use AlignedUniquePtr.
template <typename T>
using AlignedFreeUniquePtr = std::unique_ptr<T, AlignedFreer>;
// Allocate an aligned and uninitialized array of POD values as a unique_ptr.
// Upon destruction of the unique_ptr the aligned array will be freed.
template <typename T>
AlignedFreeUniquePtr<T[]> AllocateAligned(const size_t items, AllocPtr alloc,
FreePtr free, void* opaque) {
return AlignedFreeUniquePtr<T[]>(
detail::AllocateAlignedItems<T>(items, alloc, opaque),
AlignedFreer(free, opaque));
}
// Same as previous AllocateAligned(), using default allocate/free functions.
template <typename T>
AlignedFreeUniquePtr<T[]> AllocateAligned(const size_t items) {
return AllocateAligned<T>(items, nullptr, nullptr, nullptr);
}
// A simple span containing data and size of data.
template <typename T>
class Span {
public:
Span() = default;
Span(T* data, size_t size) : size_(size), data_(data) {}
template <typename U>
Span(U u) : Span(u.data(), u.size()) {}
Span(std::initializer_list<const T> v) : Span(v.begin(), v.size()) {}
// Copies the contents of the initializer list to the span.
Span<T>& operator=(std::initializer_list<const T> v) {
HWY_DASSERT(size_ == v.size());
CopyBytes(v.begin(), data_, sizeof(T) * std::min(size_, v.size()));
return *this;
}
// Returns the size of the contained data.
size_t size() const { return size_; }
// Returns a pointer to the contained data.
T* data() { return data_; }
T* data() const { return data_; }
// Returns the element at index.
T& operator[](size_t index) const { return data_[index]; }
// Returns an iterator pointing to the first element of this span.
T* begin() { return data_; }
// Returns a const iterator pointing to the first element of this span.
constexpr const T* cbegin() const { return data_; }
// Returns an iterator pointing just beyond the last element at the
// end of this span.
T* end() { return data_ + size_; }
// Returns a const iterator pointing just beyond the last element at the
// end of this span.
constexpr const T* cend() const { return data_ + size_; }
private:
size_t size_ = 0;
T* data_ = nullptr;
};
// A multi dimensional array containing an aligned buffer.
//
// To maintain alignment, the innermost dimension will be padded to ensure all
// innermost arrays are aligned.
template <typename T, size_t axes>
class AlignedNDArray {
static_assert(std::is_trivial<T>::value,
"AlignedNDArray can only contain trivial types");
public:
AlignedNDArray(AlignedNDArray&& other) = default;
AlignedNDArray& operator=(AlignedNDArray&& other) = default;
// Constructs an array of the provided shape and fills it with zeros.
explicit AlignedNDArray(std::array<size_t, axes> shape) : shape_(shape) {
sizes_ = ComputeSizes(shape_);
memory_shape_ = shape_;
// Round the innermost dimension up to the number of bytes available for
// SIMD operations on this architecture to make sure that each innermost
// array is aligned from the first element.
memory_shape_[axes - 1] = RoundUpTo(memory_shape_[axes - 1], VectorBytes());
memory_sizes_ = ComputeSizes(memory_shape_);
buffer_ = hwy::AllocateAligned<T>(memory_size());
hwy::ZeroBytes(buffer_.get(), memory_size() * sizeof(T));
}
// Returns a span containing the innermost array at the provided indices.
Span<T> operator[](std::array<const size_t, axes - 1> indices) {
return Span<T>(buffer_.get() + Offset(indices), sizes_[indices.size()]);
}
// Returns a const span containing the innermost array at the provided
// indices.
Span<const T> operator[](std::array<const size_t, axes - 1> indices) const {
return Span<const T>(buffer_.get() + Offset(indices),
sizes_[indices.size()]);
}
// Returns the shape of the array, which might be smaller than the allocated
// buffer after padding the last axis to alignment.
const std::array<size_t, axes>& shape() const { return shape_; }
// Returns the shape of the allocated buffer, which might be larger than the
// used size of the array after padding to alignment.
const std::array<size_t, axes>& memory_shape() const { return memory_shape_; }
// Returns the size of the array, which might be smaller than the allocated
// buffer after padding the last axis to alignment.
size_t size() const { return sizes_[0]; }
// Returns the size of the allocated buffer, which might be larger than the
// used size of the array after padding to alignment.
size_t memory_size() const { return memory_sizes_[0]; }
// Returns a pointer to the allocated buffer.
T* data() { return buffer_.get(); }
// Returns a const pointer to the buffer.
const T* data() const { return buffer_.get(); }
// Truncates the array by updating its shape.
//
// The new shape must be equal to or less than the old shape in all axes.
//
// Doesn't modify underlying memory.
void truncate(const std::array<size_t, axes>& new_shape) {
#if HWY_IS_DEBUG_BUILD
for (size_t axis_index = 0; axis_index < axes; ++axis_index) {
HWY_ASSERT(new_shape[axis_index] <= shape_[axis_index]);
}
#endif
shape_ = new_shape;
sizes_ = ComputeSizes(shape_);
}
private:
std::array<size_t, axes> shape_;
std::array<size_t, axes> memory_shape_;
std::array<size_t, axes + 1> sizes_;
std::array<size_t, axes + 1> memory_sizes_;
hwy::AlignedFreeUniquePtr<T[]> buffer_;
// Computes offset in the buffer based on the provided indices.
size_t Offset(std::array<const size_t, axes - 1> indices) const {
size_t offset = 0;
size_t shape_index = 0;
for (const size_t axis_index : indices) {
offset += memory_sizes_[shape_index + 1] * axis_index;
shape_index++;
}
return offset;
}
// Computes the sizes of all sub arrays based on the sizes of each axis.
//
// Does this by multiplying the size of each axis with the previous one in
// reverse order, starting with the conceptual axis of size 1 containing the
// actual elements in the array.
static std::array<size_t, axes + 1> ComputeSizes(
std::array<size_t, axes> shape) {
std::array<size_t, axes + 1> sizes;
size_t axis = shape.size();
sizes[axis] = 1;
while (axis > 0) {
--axis;
sizes[axis] = sizes[axis + 1] * shape[axis];
}
return sizes;
}
};
} // namespace hwy
#endif // HIGHWAY_HWY_ALIGNED_ALLOCATOR_H_

468
deps/v8/third_party/highway/src/hwy/aligned_allocator_test.cc vendored Normal file
View File

@ -0,0 +1,468 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/aligned_allocator.h"
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h> // malloc
#include <array>
#include <random>
#include <set>
#include <vector>
#include "hwy/base.h"
#include "hwy/per_target.h"
#include "hwy/tests/hwy_gtest.h"
#include "hwy/tests/test_util-inl.h" // HWY_ASSERT_EQ
namespace {
// Sample object that keeps track on an external counter of how many times was
// the explicit constructor and destructor called.
template <size_t N>
class SampleObject {
public:
SampleObject() { data_[0] = 'a'; }
explicit SampleObject(int* counter) : counter_(counter) {
if (counter) (*counter)++;
data_[0] = 'b';
}
~SampleObject() {
if (counter_) (*counter_)--;
}
static_assert(N > sizeof(int*), "SampleObject size too small.");
int* counter_ = nullptr;
char data_[N - sizeof(int*)];
};
class FakeAllocator {
public:
// static AllocPtr and FreePtr member to be used with the aligned
// allocator. These functions calls the private non-static members.
static void* StaticAlloc(void* opaque, size_t bytes) {
return reinterpret_cast<FakeAllocator*>(opaque)->Alloc(bytes);
}
static void StaticFree(void* opaque, void* memory) {
return reinterpret_cast<FakeAllocator*>(opaque)->Free(memory);
}
// Returns the number of pending allocations to be freed.
size_t PendingAllocs() { return allocs_.size(); }
private:
void* Alloc(size_t bytes) {
void* ret = malloc(bytes);
allocs_.insert(ret);
return ret;
}
void Free(void* memory) {
if (!memory) return;
HWY_ASSERT(allocs_.end() != allocs_.find(memory));
allocs_.erase(memory);
free(memory);
}
std::set<void*> allocs_;
};
} // namespace
namespace hwy {
namespace {
#if !HWY_TEST_STANDALONE
class AlignedAllocatorTest : public testing::Test {};
#endif
TEST(AlignedAllocatorTest, TestFreeNullptr) {
// Calling free with a nullptr is always ok.
FreeAlignedBytes(/*aligned_pointer=*/nullptr, /*free_ptr=*/nullptr,
/*opaque_ptr=*/nullptr);
}
TEST(AlignedAllocatorTest, TestLog2) {
HWY_ASSERT_EQ(0u, detail::ShiftCount(1));
HWY_ASSERT_EQ(1u, detail::ShiftCount(2));
HWY_ASSERT_EQ(3u, detail::ShiftCount(8));
}
// Allocator returns null when it detects overflow of items * sizeof(T).
TEST(AlignedAllocatorTest, TestOverflow) {
constexpr size_t max = ~size_t(0);
constexpr size_t msb = (max >> 1) + 1;
using Size5 = std::array<uint8_t, 5>;
using Size10 = std::array<uint8_t, 10>;
HWY_ASSERT(nullptr ==
detail::AllocateAlignedItems<uint32_t>(max / 2, nullptr, nullptr));
HWY_ASSERT(nullptr ==
detail::AllocateAlignedItems<uint32_t>(max / 3, nullptr, nullptr));
HWY_ASSERT(nullptr ==
detail::AllocateAlignedItems<Size5>(max / 4, nullptr, nullptr));
HWY_ASSERT(nullptr ==
detail::AllocateAlignedItems<uint16_t>(msb, nullptr, nullptr));
HWY_ASSERT(nullptr ==
detail::AllocateAlignedItems<double>(msb + 1, nullptr, nullptr));
HWY_ASSERT(nullptr ==
detail::AllocateAlignedItems<Size10>(msb / 4, nullptr, nullptr));
}
TEST(AlignedAllocatorTest, TestAllocDefaultPointers) {
const size_t kSize = 7777;
void* ptr = AllocateAlignedBytes(kSize, /*alloc_ptr=*/nullptr,
/*opaque_ptr=*/nullptr);
HWY_ASSERT(ptr != nullptr);
// Make sure the pointer is actually aligned.
HWY_ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr) % HWY_ALIGNMENT);
char* p = static_cast<char*>(ptr);
size_t ret = 0;
for (size_t i = 0; i < kSize; i++) {
// Performs a computation using p[] to prevent it being optimized away.
p[i] = static_cast<char>(i & 0x7F);
if (i) ret += static_cast<size_t>(p[i] * p[i - 1]);
}
HWY_ASSERT(ret != size_t{0});
FreeAlignedBytes(ptr, /*free_ptr=*/nullptr, /*opaque_ptr=*/nullptr);
}
TEST(AlignedAllocatorTest, TestEmptyAlignedUniquePtr) {
AlignedUniquePtr<SampleObject<32>> ptr(nullptr, AlignedDeleter());
AlignedUniquePtr<SampleObject<32>[]> arr(nullptr, AlignedDeleter());
}
TEST(AlignedAllocatorTest, TestEmptyAlignedFreeUniquePtr) {
AlignedFreeUniquePtr<SampleObject<32>> ptr(nullptr, AlignedFreer());
AlignedFreeUniquePtr<SampleObject<32>[]> arr(nullptr, AlignedFreer());
}
TEST(AlignedAllocatorTest, TestCustomAlloc) {
FakeAllocator fake_alloc;
const size_t kSize = 7777;
void* ptr =
AllocateAlignedBytes(kSize, &FakeAllocator::StaticAlloc, &fake_alloc);
HWY_ASSERT(ptr != nullptr);
// We should have only requested one alloc from the allocator.
HWY_ASSERT_EQ(1U, fake_alloc.PendingAllocs());
// Make sure the pointer is actually aligned.
HWY_ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr) % HWY_ALIGNMENT);
FreeAlignedBytes(ptr, &FakeAllocator::StaticFree, &fake_alloc);
HWY_ASSERT_EQ(0U, fake_alloc.PendingAllocs());
}
TEST(AlignedAllocatorTest, TestMakeUniqueAlignedDefaultConstructor) {
{
auto ptr = MakeUniqueAligned<SampleObject<24>>();
// Default constructor sets the data_[0] to 'a'.
HWY_ASSERT_EQ('a', ptr->data_[0]);
HWY_ASSERT(nullptr == ptr->counter_);
}
}
TEST(AlignedAllocatorTest, TestMakeUniqueAligned) {
int counter = 0;
{
// Creates the object, initializes it with the explicit constructor and
// returns an unique_ptr to it.
auto ptr = MakeUniqueAligned<SampleObject<24>>(&counter);
HWY_ASSERT_EQ(1, counter);
// Custom constructor sets the data_[0] to 'b'.
HWY_ASSERT_EQ('b', ptr->data_[0]);
}
HWY_ASSERT_EQ(0, counter);
}
TEST(AlignedAllocatorTest, TestMakeUniqueAlignedArray) {
int counter = 0;
{
// Creates the array of objects and initializes them with the explicit
// constructor.
auto arr = MakeUniqueAlignedArray<SampleObject<24>>(7, &counter);
HWY_ASSERT_EQ(7, counter);
for (size_t i = 0; i < 7; i++) {
// Custom constructor sets the data_[0] to 'b'.
HWY_ASSERT_EQ('b', arr[i].data_[0]);
}
}
HWY_ASSERT_EQ(0, counter);
}
TEST(AlignedAllocatorTest, TestAllocSingleInt) {
auto ptr = AllocateAligned<uint32_t>(1);
HWY_ASSERT(ptr.get() != nullptr);
HWY_ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr.get()) % HWY_ALIGNMENT);
// Force delete of the unique_ptr now to check that it doesn't crash.
ptr.reset(nullptr);
HWY_ASSERT(nullptr == ptr.get());
}
TEST(AlignedAllocatorTest, TestAllocMultipleInt) {
const size_t kSize = 7777;
auto ptr = AllocateAligned<uint32_t>(kSize);
HWY_ASSERT(ptr.get() != nullptr);
HWY_ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr.get()) % HWY_ALIGNMENT);
// ptr[i] is actually (*ptr.get())[i] which will use the operator[] of the
// underlying type chosen by AllocateAligned() for the std::unique_ptr.
HWY_ASSERT(&(ptr[0]) + 1 == &(ptr[1]));
size_t ret = 0;
for (size_t i = 0; i < kSize; i++) {
// Performs a computation using ptr[] to prevent it being optimized away.
ptr[i] = static_cast<uint32_t>(i);
if (i) ret += static_cast<size_t>(ptr[i]) * ptr[i - 1];
}
HWY_ASSERT(ret != size_t{0});
}
TEST(AlignedAllocatorTest, TestAllocateAlignedObjectWithoutDestructor) {
int counter = 0;
{
// This doesn't call the constructor.
auto obj = AllocateAligned<SampleObject<24>>(1);
HWY_ASSERT(obj);
obj[0].counter_ = &counter;
}
// Destroying the unique_ptr shouldn't have called the destructor of the
// SampleObject<24>.
HWY_ASSERT_EQ(0, counter);
}
TEST(AlignedAllocatorTest, TestMakeUniqueAlignedArrayWithCustomAlloc) {
FakeAllocator fake_alloc;
int counter = 0;
{
// Creates the array of objects and initializes them with the explicit
// constructor.
auto arr = MakeUniqueAlignedArrayWithAlloc<SampleObject<24>>(
7, FakeAllocator::StaticAlloc, FakeAllocator::StaticFree, &fake_alloc,
&counter);
HWY_ASSERT(arr.get() != nullptr);
// An array should still only call a single allocation.
HWY_ASSERT_EQ(1u, fake_alloc.PendingAllocs());
HWY_ASSERT_EQ(7, counter);
for (size_t i = 0; i < 7; i++) {
// Custom constructor sets the data_[0] to 'b'.
HWY_ASSERT_EQ('b', arr[i].data_[0]);
}
}
HWY_ASSERT_EQ(0, counter);
HWY_ASSERT_EQ(0u, fake_alloc.PendingAllocs());
}
TEST(AlignedAllocatorTest, TestDefaultInit) {
// The test is whether this compiles. Default-init is useful for output params
// and per-thread storage.
std::vector<AlignedUniquePtr<int[]>> ptrs;
std::vector<AlignedFreeUniquePtr<double[]>> free_ptrs;
ptrs.resize(128);
free_ptrs.resize(128);
// The following is to prevent elision of the pointers.
std::mt19937 rng(129); // Emscripten lacks random_device.
std::uniform_int_distribution<size_t> dist(0, 127);
ptrs[dist(rng)] = MakeUniqueAlignedArray<int>(123);
free_ptrs[dist(rng)] = AllocateAligned<double>(456);
// "Use" pointer without resorting to printf. 0 == 0. Can't shift by 64.
const auto addr1 = reinterpret_cast<uintptr_t>(ptrs[dist(rng)].get());
const auto addr2 = reinterpret_cast<uintptr_t>(free_ptrs[dist(rng)].get());
constexpr size_t kBits = sizeof(uintptr_t) * 8;
HWY_ASSERT_EQ((addr1 >> (kBits - 1)) >> (kBits - 1),
(addr2 >> (kBits - 1)) >> (kBits - 1));
}
using std::array;
using std::vector;
template <typename T>
void CheckEqual(const T& t1, const T& t2) {
HWY_ASSERT_EQ(t1.size(), t2.size());
for (size_t i = 0; i < t1.size(); i++) {
HWY_ASSERT_EQ(t1[i], t2[i]);
}
}
template <typename T>
void CheckEqual(const AlignedNDArray<T, 1>& a, const vector<T>& v) {
const array<size_t, 1> want_shape({v.size()});
const array<size_t, 1> got_shape = a.shape();
CheckEqual(got_shape, want_shape);
Span<const T> a_span = a[{}];
HWY_ASSERT_EQ(a_span.size(), v.size());
for (size_t i = 0; i < a_span.size(); i++) {
HWY_ASSERT_EQ(a_span[i], v[i]);
HWY_ASSERT_EQ(*(a_span.data() + i), v[i]);
}
}
template <typename T>
void CheckEqual(const AlignedNDArray<T, 2>& a, const vector<vector<T>>& v) {
const array<size_t, 2> want_shape({v.size(), v[1].size()});
for (const vector<T>& row : v) {
HWY_ASSERT_EQ(row.size(), want_shape[1]);
}
const std::array<size_t, 2> got_shape = a.shape();
CheckEqual(got_shape, want_shape);
HWY_ASSERT_EQ(a.size(), want_shape[0] * want_shape[1]);
for (size_t row_index = 0; row_index < v.size(); ++row_index) {
vector<T> want_row = v[row_index];
Span<const T> got_row = a[{row_index}];
HWY_ASSERT_EQ(got_row.size(), want_row.size());
for (size_t column_index = 0; column_index < got_row.size();
column_index++) {
HWY_ASSERT_EQ(a[{row_index}][column_index], want_row[column_index]);
HWY_ASSERT_EQ(got_row[column_index], want_row[column_index]);
HWY_ASSERT_EQ(*(a[{row_index}].data() + column_index),
want_row[column_index]);
}
}
}
TEST(AlignedAllocatorTest, TestAlignedNDArray) {
AlignedNDArray<float, 1> a1({4});
CheckEqual(a1, {0, 0, 0, 0});
a1[{}][2] = 3.4f;
CheckEqual(a1, {0, 0, 3.4f, 0});
AlignedNDArray<float, 2> a2({2, 3});
CheckEqual(a2, {{0, 0, 0}, {0, 0, 0}});
a2[{1}][1] = 5.1f;
CheckEqual(a2, {{0, 0, 0}, {0, 5.1f, 0}});
float f0[] = {1.0f, 2.0f, 3.0f};
float f1[] = {4.0f, 5.0f, 6.0f};
hwy::CopyBytes(f0, a2[{0}].data(), 3 * sizeof(float));
hwy::CopyBytes(f1, a2[{1}].data(), 3 * sizeof(float));
CheckEqual(a2, {{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}});
}
// Tests that each innermost row in an AlignedNDArray is aligned to the max
// bytes available for SIMD operations on this architecture.
TEST(AlignedAllocatorTest, TestAlignedNDArrayAlignment) {
AlignedNDArray<float, 4> a({3, 3, 3, 3});
for (size_t d0 = 0; d0 < a.shape()[0]; d0++) {
for (size_t d1 = 0; d1 < a.shape()[1]; d1++) {
for (size_t d2 = 0; d2 < a.shape()[2]; d2++) {
// Check that the address this innermost array starts at is an even
// number of VectorBytes(), which is the max bytes available for SIMD
// operations.
HWY_ASSERT_EQ(
reinterpret_cast<uintptr_t>(a[{d0, d1, d2}].data()) % VectorBytes(),
0);
}
}
}
}
TEST(AlignedAllocatorTest, TestSpanCopyAssignment) {
AlignedNDArray<float, 2> a({2, 2});
CheckEqual(a, {{0.0f, 0.0f}, {0.0f, 0.0f}});
a[{0}] = {1.0f, 2.0f};
a[{1}] = {3.0f, 4.0f};
CheckEqual(a, {{1.0f, 2.0f}, {3.0f, 4.0f}});
}
TEST(AlignedAllocatorTest, TestAlignedNDArrayTruncate) {
AlignedNDArray<size_t, 4> a({8, 8, 8, 8});
const size_t last_axis_memory_shape = a.memory_shape()[3];
const auto compute_value = [&](const std::array<size_t, 4>& index) {
return index[0] * 8 * 8 * 8 + index[1] * 8 * 8 + index[2] * 8 * 8 +
index[3];
};
for (size_t axis0 = 0; axis0 < a.shape()[0]; ++axis0) {
for (size_t axis1 = 0; axis1 < a.shape()[1]; ++axis1) {
for (size_t axis2 = 0; axis2 < a.shape()[2]; ++axis2) {
for (size_t axis3 = 0; axis3 < a.shape()[3]; ++axis3) {
a[{axis0, axis1, axis2}][axis3] =
compute_value({axis0, axis1, axis2, axis3});
}
}
}
}
const auto verify_values = [&](const AlignedNDArray<size_t, 4>& array) {
for (size_t axis0 = 0; axis0 < array.shape()[0]; ++axis0) {
for (size_t axis1 = 0; axis1 < array.shape()[1]; ++axis1) {
for (size_t axis2 = 0; axis2 < array.shape()[2]; ++axis2) {
for (size_t axis3 = 0; axis3 < array.shape()[3]; ++axis3) {
HWY_ASSERT_EQ((array[{axis0, axis1, axis2}][axis3]),
(compute_value({axis0, axis1, axis2, axis3})));
}
}
}
}
};
a.truncate({7, 7, 7, 7});
HWY_ASSERT_EQ(a.shape()[0], 7);
HWY_ASSERT_EQ(a.shape()[1], 7);
HWY_ASSERT_EQ(a.shape()[2], 7);
HWY_ASSERT_EQ(a.shape()[3], 7);
HWY_ASSERT_EQ(a.memory_shape()[0], 8);
HWY_ASSERT_EQ(a.memory_shape()[1], 8);
HWY_ASSERT_EQ(a.memory_shape()[2], 8);
HWY_ASSERT_EQ(a.memory_shape()[3], last_axis_memory_shape);
verify_values(a);
a.truncate({6, 5, 4, 3});
HWY_ASSERT_EQ(a.shape()[0], 6);
HWY_ASSERT_EQ(a.shape()[1], 5);
HWY_ASSERT_EQ(a.shape()[2], 4);
HWY_ASSERT_EQ(a.shape()[3], 3);
HWY_ASSERT_EQ(a.memory_shape()[0], 8);
HWY_ASSERT_EQ(a.memory_shape()[1], 8);
HWY_ASSERT_EQ(a.memory_shape()[2], 8);
HWY_ASSERT_EQ(a.memory_shape()[3], last_axis_memory_shape);
verify_values(a);
}
TEST(AlignedAllocatorTest, TestAlignedVector) {
std::vector<int> vec{0, 1, 2, 3, 4};
HWY_ASSERT_EQ(5, vec.size());
HWY_ASSERT_EQ(0, vec[0]);
HWY_ASSERT_EQ(2, vec.at(2));
HWY_ASSERT_EQ(0, vec.front());
HWY_ASSERT_EQ(4, vec.back());
vec.pop_back();
HWY_ASSERT_EQ(3, vec.back());
HWY_ASSERT_EQ(4, vec.size());
vec.push_back(4);
vec.push_back(5);
HWY_ASSERT_EQ(5, vec.back());
HWY_ASSERT_EQ(6, vec.size());
const size_t initialCapacity = vec.capacity();
// Add elements to exceed initial capacity
for (auto i = vec.size(); i < initialCapacity + 10; ++i) {
vec.push_back(static_cast<int>(i));
}
// Check if the capacity increased and elements are intact
HWY_ASSERT(vec.capacity() > initialCapacity);
for (size_t i = 0; i < vec.size(); ++i) {
HWY_ASSERT_EQ(i, vec[i]);
}
vec.clear();
HWY_ASSERT(vec.empty());
}
} // namespace
} // namespace hwy
HWY_TEST_MAIN();

3004
deps/v8/third_party/highway/src/hwy/base.h vendored Normal file
View File

File diff suppressed because it is too large Load Diff

866
deps/v8/third_party/highway/src/hwy/base_test.cc vendored Normal file
View File

@ -0,0 +1,866 @@
// Copyright 2019 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/base.h"
#include <limits>
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "base_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/tests/test_util-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
HWY_NOINLINE void TestAllLimits() {
HWY_ASSERT_EQ(uint8_t{0}, LimitsMin<uint8_t>());
HWY_ASSERT_EQ(uint16_t{0}, LimitsMin<uint16_t>());
HWY_ASSERT_EQ(uint32_t{0}, LimitsMin<uint32_t>());
HWY_ASSERT_EQ(uint64_t{0}, LimitsMin<uint64_t>());
HWY_ASSERT_EQ(int8_t{-128}, LimitsMin<int8_t>());
HWY_ASSERT_EQ(int16_t{-32768}, LimitsMin<int16_t>());
HWY_ASSERT_EQ(static_cast<int32_t>(0x80000000u), LimitsMin<int32_t>());
HWY_ASSERT_EQ(static_cast<int64_t>(0x8000000000000000ull),
LimitsMin<int64_t>());
HWY_ASSERT_EQ(uint8_t{0xFF}, LimitsMax<uint8_t>());
HWY_ASSERT_EQ(uint16_t{0xFFFF}, LimitsMax<uint16_t>());
HWY_ASSERT_EQ(uint32_t{0xFFFFFFFFu}, LimitsMax<uint32_t>());
HWY_ASSERT_EQ(uint64_t{0xFFFFFFFFFFFFFFFFull}, LimitsMax<uint64_t>());
HWY_ASSERT_EQ(int8_t{0x7F}, LimitsMax<int8_t>());
HWY_ASSERT_EQ(int16_t{0x7FFF}, LimitsMax<int16_t>());
HWY_ASSERT_EQ(int32_t{0x7FFFFFFFu}, LimitsMax<int32_t>());
HWY_ASSERT_EQ(int64_t{0x7FFFFFFFFFFFFFFFull}, LimitsMax<int64_t>());
HWY_ASSERT(LimitsMin<signed char>() == LimitsMin<int8_t>());
HWY_ASSERT(LimitsMin<short>() <= LimitsMin<int16_t>()); // NOLINT
HWY_ASSERT(LimitsMin<int>() <= LimitsMin<int16_t>());
HWY_ASSERT(LimitsMin<long>() <= LimitsMin<int32_t>()); // NOLINT
HWY_ASSERT(LimitsMin<long long>() <= LimitsMin<int64_t>()); // NOLINT
HWY_ASSERT(LimitsMax<signed char>() == LimitsMax<int8_t>());
HWY_ASSERT(LimitsMax<short>() >= LimitsMax<int16_t>()); // NOLINT
HWY_ASSERT(LimitsMax<int>() >= LimitsMax<int16_t>());
HWY_ASSERT(LimitsMax<long>() >= LimitsMax<int32_t>()); // NOLINT
HWY_ASSERT(LimitsMax<long long>() >= LimitsMax<int64_t>()); // NOLINT
HWY_ASSERT_EQ(static_cast<unsigned char>(0), LimitsMin<unsigned char>());
HWY_ASSERT_EQ(static_cast<unsigned short>(0), LimitsMin<unsigned short>());
HWY_ASSERT_EQ(0u, LimitsMin<unsigned>());
HWY_ASSERT_EQ(0ul, LimitsMin<unsigned long>()); // NOLINT
HWY_ASSERT_EQ(0ull, LimitsMin<unsigned long long>()); // NOLINT
HWY_ASSERT(LimitsMax<unsigned char>() == LimitsMax<uint8_t>());
HWY_ASSERT(LimitsMax<unsigned short>() >= LimitsMax<uint16_t>()); // NOLINT
HWY_ASSERT(LimitsMax<unsigned>() >= LimitsMax<uint16_t>());
HWY_ASSERT(LimitsMax<unsigned long>() >= LimitsMax<uint32_t>()); // NOLINT
// NOLINTNEXTLINE
HWY_ASSERT(LimitsMax<unsigned long long>() >= LimitsMax<uint64_t>());
HWY_ASSERT(LimitsMin<char>() == 0 ||
LimitsMin<char>() == LimitsMin<int8_t>());
HWY_ASSERT(LimitsMax<char>() == LimitsMax<int8_t>() ||
LimitsMax<char>() == LimitsMax<uint8_t>());
HWY_ASSERT_EQ(size_t{0}, LimitsMin<size_t>());
HWY_ASSERT(LimitsMin<ptrdiff_t>() < ptrdiff_t{0});
HWY_ASSERT(LimitsMin<intptr_t>() < intptr_t{0});
HWY_ASSERT_EQ(uintptr_t{0}, LimitsMin<uintptr_t>());
HWY_ASSERT(LimitsMin<wchar_t>() <= wchar_t{0});
HWY_ASSERT(LimitsMax<size_t>() > size_t{0});
HWY_ASSERT(LimitsMax<ptrdiff_t>() > ptrdiff_t{0});
HWY_ASSERT(LimitsMax<intptr_t>() > intptr_t{0});
HWY_ASSERT(LimitsMax<uintptr_t>() > uintptr_t{0});
HWY_ASSERT(LimitsMax<wchar_t>() > wchar_t{0});
}
struct TestLowestHighest {
template <class T>
HWY_NOINLINE void operator()(T /*unused*/) const {
// numeric_limits<T>::lowest is only guaranteed to be what we expect (-max)
// for built-in floating-point types.
if (!IsSpecialFloat<T>()) {
HWY_ASSERT_EQ(std::numeric_limits<T>::lowest(), LowestValue<T>());
HWY_ASSERT_EQ(std::numeric_limits<T>::max(), HighestValue<T>());
}
}
};
HWY_NOINLINE void TestAllLowestHighest() { ForAllTypes(TestLowestHighest()); }
struct TestIsUnsigned {
template <class T>
HWY_NOINLINE void operator()(T /*unused*/) const {
static_assert(!IsFloat<T>(), "Expected !IsFloat");
static_assert(!IsSigned<T>(), "Expected !IsSigned");
static_assert(IsInteger<T>(), "Expected IsInteger");
}
};
struct TestIsSigned {
template <class T>
HWY_NOINLINE void operator()(T /*unused*/) const {
static_assert(!IsFloat<T>(), "Expected !IsFloat");
static_assert(IsSigned<T>(), "Expected IsSigned");
static_assert(IsInteger<T>(), "Expected IsInteger");
}
};
struct TestIsFloat {
template <class T>
HWY_NOINLINE void operator()(T /*unused*/) const {
static_assert(IsFloat<T>(), "Expected IsFloat");
static_assert(!IsInteger<T>(), "Expected !IsInteger");
static_assert(IsSigned<T>(), "Floats are also considered signed");
}
};
HWY_NOINLINE void TestAllType() {
const TestIsUnsigned is_unsigned_test;
const TestIsSigned is_signed_test;
ForUnsignedTypes(is_unsigned_test);
ForSignedTypes(is_signed_test);
ForFloatTypes(TestIsFloat());
is_unsigned_test(static_cast<unsigned char>(0));
is_unsigned_test(static_cast<unsigned short>(0));
is_unsigned_test(0u);
is_unsigned_test(0ul);
is_unsigned_test(0ull);
is_unsigned_test(size_t{0});
is_unsigned_test(uintptr_t{0});
is_signed_test(static_cast<signed char>(0));
is_signed_test(static_cast<signed short>(0));
is_signed_test(0);
is_signed_test(0L);
is_signed_test(0LL);
is_signed_test(ptrdiff_t{0});
is_signed_test(intptr_t{0});
static_assert(!IsFloat<char>(), "Expected !IsFloat<char>()");
static_assert(!IsFloat<wchar_t>(), "Expected !IsFloat<wchar_t>()");
static_assert(IsInteger<char>(), "Expected IsInteger<char>()");
static_assert(IsInteger<wchar_t>(), "Expected IsInteger<wchar_t>()");
static_assert(sizeof(MakeUnsigned<hwy::uint128_t>) == 16, "");
static_assert(sizeof(MakeWide<uint64_t>) == 16, "Expected uint128_t");
static_assert(sizeof(MakeNarrow<hwy::uint128_t>) == 8, "Expected uint64_t");
}
struct TestIsSame {
template <class T>
HWY_NOINLINE void operator()(T /*unused*/) const {
static_assert(IsSame<T, T>(), "T == T");
static_assert(!IsSame<MakeSigned<T>, MakeUnsigned<T>>(), "S != U");
static_assert(!IsSame<MakeUnsigned<T>, MakeSigned<T>>(), "U != S");
}
};
HWY_NOINLINE void TestAllIsSame() { ForAllTypes(TestIsSame()); }
HWY_NOINLINE void TestAllBitScan() {
HWY_ASSERT_EQ(size_t{0}, Num0BitsAboveMS1Bit_Nonzero32(0x80000000u));
HWY_ASSERT_EQ(size_t{0}, Num0BitsAboveMS1Bit_Nonzero32(0xFFFFFFFFu));
HWY_ASSERT_EQ(size_t{1}, Num0BitsAboveMS1Bit_Nonzero32(0x40000000u));
HWY_ASSERT_EQ(size_t{1}, Num0BitsAboveMS1Bit_Nonzero32(0x40108210u));
HWY_ASSERT_EQ(size_t{30}, Num0BitsAboveMS1Bit_Nonzero32(2u));
HWY_ASSERT_EQ(size_t{30}, Num0BitsAboveMS1Bit_Nonzero32(3u));
HWY_ASSERT_EQ(size_t{31}, Num0BitsAboveMS1Bit_Nonzero32(1u));
HWY_ASSERT_EQ(size_t{0},
Num0BitsAboveMS1Bit_Nonzero64(0x8000000000000000ull));
HWY_ASSERT_EQ(size_t{0},
Num0BitsAboveMS1Bit_Nonzero64(0xFFFFFFFFFFFFFFFFull));
HWY_ASSERT_EQ(size_t{1},
Num0BitsAboveMS1Bit_Nonzero64(0x4000000000000000ull));
HWY_ASSERT_EQ(size_t{1},
Num0BitsAboveMS1Bit_Nonzero64(0x4010821004200011ull));
HWY_ASSERT_EQ(size_t{62}, Num0BitsAboveMS1Bit_Nonzero64(2ull));
HWY_ASSERT_EQ(size_t{62}, Num0BitsAboveMS1Bit_Nonzero64(3ull));
HWY_ASSERT_EQ(size_t{63}, Num0BitsAboveMS1Bit_Nonzero64(1ull));
HWY_ASSERT_EQ(size_t{0}, Num0BitsBelowLS1Bit_Nonzero32(1u));
HWY_ASSERT_EQ(size_t{1}, Num0BitsBelowLS1Bit_Nonzero32(2u));
HWY_ASSERT_EQ(size_t{30}, Num0BitsBelowLS1Bit_Nonzero32(0xC0000000u));
HWY_ASSERT_EQ(size_t{31}, Num0BitsBelowLS1Bit_Nonzero32(0x80000000u));
HWY_ASSERT_EQ(size_t{0}, Num0BitsBelowLS1Bit_Nonzero64(1ull));
HWY_ASSERT_EQ(size_t{1}, Num0BitsBelowLS1Bit_Nonzero64(2ull));
HWY_ASSERT_EQ(size_t{62},
Num0BitsBelowLS1Bit_Nonzero64(0xC000000000000000ull));
HWY_ASSERT_EQ(size_t{63},
Num0BitsBelowLS1Bit_Nonzero64(0x8000000000000000ull));
}
HWY_NOINLINE void TestAllPopCount() {
HWY_ASSERT_EQ(size_t{0}, PopCount(0u));
HWY_ASSERT_EQ(size_t{1}, PopCount(1u));
HWY_ASSERT_EQ(size_t{1}, PopCount(2u));
HWY_ASSERT_EQ(size_t{2}, PopCount(3u));
HWY_ASSERT_EQ(size_t{1}, PopCount(0x80000000u));
HWY_ASSERT_EQ(size_t{31}, PopCount(0x7FFFFFFFu));
HWY_ASSERT_EQ(size_t{32}, PopCount(0xFFFFFFFFu));
HWY_ASSERT_EQ(size_t{1}, PopCount(0x80000000ull));
HWY_ASSERT_EQ(size_t{31}, PopCount(0x7FFFFFFFull));
HWY_ASSERT_EQ(size_t{32}, PopCount(0xFFFFFFFFull));
HWY_ASSERT_EQ(size_t{33}, PopCount(0x10FFFFFFFFull));
HWY_ASSERT_EQ(size_t{63}, PopCount(0xFFFEFFFFFFFFFFFFull));
HWY_ASSERT_EQ(size_t{64}, PopCount(0xFFFFFFFFFFFFFFFFull));
}
// Exhaustive test for small/large dividends and divisors
HWY_NOINLINE void TestAllDivisor() {
// Small d, small n
for (uint32_t d = 1; d < 256; ++d) {
const Divisor divisor(d);
for (uint32_t n = 0; n < 256; ++n) {
HWY_ASSERT(divisor.Divide(n) == n / d);
HWY_ASSERT(divisor.Remainder(n) == n % d);
}
}
// Large d, small n
for (uint32_t d = 0xFFFFFF00u; d != 0; ++d) {
const Divisor divisor(d);
for (uint32_t n = 0; n < 256; ++n) {
HWY_ASSERT(divisor.Divide(n) == n / d);
HWY_ASSERT(divisor.Remainder(n) == n % d);
}
}
// Small d, large n
for (uint32_t d = 1; d < 256; ++d) {
const Divisor divisor(d);
for (uint32_t n = 0xFFFFFF00u; n != 0; ++n) {
HWY_ASSERT(divisor.Divide(n) == n / d);
HWY_ASSERT(divisor.Remainder(n) == n % d);
}
}
// Large d, large n
for (uint32_t d = 0xFFFFFF00u; d != 0; ++d) {
const Divisor divisor(d);
for (uint32_t n = 0xFFFFFF00u; n != 0; ++n) {
HWY_ASSERT(divisor.Divide(n) == n / d);
HWY_ASSERT(divisor.Remainder(n) == n % d);
}
}
}
struct TestScalarShr {
template <class T>
HWY_NOINLINE void operator()(T /*unused*/) const {
using TU = MakeUnsigned<T>;
constexpr T kMsb = static_cast<T>(1ULL << (sizeof(T) * 8 - 1));
constexpr int kSizeInBits = static_cast<int>(sizeof(T) * 8);
constexpr T kVal1 = static_cast<T>(0x776B0405296C183BULL & LimitsMax<TU>());
constexpr T kVal2 = static_cast<T>(kVal1 | kMsb);
for (int i = 0; i < kSizeInBits; i++) {
T expected1;
T expected2;
const TU expected1_bits = static_cast<TU>(static_cast<TU>(kVal1) >> i);
const TU expected2_bits = static_cast<TU>(
(static_cast<TU>(kVal2) >> i) |
((IsSigned<T>() && i > 0)
? (~((static_cast<TU>(1) << (kSizeInBits - i)) - 1))
: 0));
CopySameSize(&expected1_bits, &expected1);
CopySameSize(&expected2_bits, &expected2);
HWY_ASSERT_EQ(expected1, ScalarShr(kVal1, i));
HWY_ASSERT_EQ(expected2, ScalarShr(kVal2, i));
}
}
};
HWY_NOINLINE void TestAllScalarShr() { ForIntegerTypes(TestScalarShr()); }
template <class T>
static HWY_INLINE void AssertMul128Result(T expected_hi, T expected_lo, T a,
T b, const char* file,
const int line) {
RemoveCvRef<T> actual_hi;
const RemoveCvRef<T> actual_lo = Mul128(a, b, &actual_hi);
hwy::AssertEqual(expected_lo, actual_lo, hwy::TargetName(HWY_TARGET), file,
line);
hwy::AssertEqual(expected_hi, actual_hi, hwy::TargetName(HWY_TARGET), file,
line);
}
HWY_NOINLINE void TestAllMul128() {
AssertMul128Result(static_cast<int64_t>(0), static_cast<int64_t>(0),
static_cast<int64_t>(0), static_cast<int64_t>(0), __FILE__,
__LINE__);
AssertMul128Result(static_cast<int64_t>(0), static_cast<int64_t>(0),
static_cast<int64_t>(0), static_cast<int64_t>(1), __FILE__,
__LINE__);
AssertMul128Result(static_cast<int64_t>(0), static_cast<int64_t>(0),
static_cast<int64_t>(0), static_cast<int64_t>(-1),
__FILE__, __LINE__);
AssertMul128Result(static_cast<int64_t>(0), static_cast<int64_t>(0),
static_cast<int64_t>(1), static_cast<int64_t>(0), __FILE__,
__LINE__);
AssertMul128Result(static_cast<int64_t>(0), static_cast<int64_t>(0),
static_cast<int64_t>(-1), static_cast<int64_t>(0),
__FILE__, __LINE__);
AssertMul128Result(static_cast<int64_t>(0), static_cast<int64_t>(1),
static_cast<int64_t>(1), static_cast<int64_t>(1), __FILE__,
__LINE__);
AssertMul128Result(static_cast<int64_t>(-1), static_cast<int64_t>(-1),
static_cast<int64_t>(-1), static_cast<int64_t>(1),
__FILE__, __LINE__);
AssertMul128Result(static_cast<int64_t>(-1), static_cast<int64_t>(-1),
static_cast<int64_t>(1), static_cast<int64_t>(-1),
__FILE__, __LINE__);
AssertMul128Result(static_cast<int64_t>(0), static_cast<int64_t>(1),
static_cast<int64_t>(-1), static_cast<int64_t>(-1),
__FILE__, __LINE__);
AssertMul128Result(static_cast<uint64_t>(0), static_cast<uint64_t>(0),
static_cast<uint64_t>(0), static_cast<uint64_t>(0),
__FILE__, __LINE__);
AssertMul128Result(static_cast<uint64_t>(0), static_cast<uint64_t>(0),
static_cast<uint64_t>(0), static_cast<uint64_t>(1),
__FILE__, __LINE__);
AssertMul128Result(static_cast<uint64_t>(0), static_cast<uint64_t>(0),
static_cast<uint64_t>(1), static_cast<uint64_t>(0),
__FILE__, __LINE__);
AssertMul128Result(static_cast<uint64_t>(0), static_cast<uint64_t>(1),
static_cast<uint64_t>(1), static_cast<uint64_t>(1),
__FILE__, __LINE__);
AssertMul128Result(static_cast<int64_t>(0x24E331A77C96011DULL),
static_cast<int64_t>(0x3C5385F8E294E438ULL),
static_cast<int64_t>(0x4F87AE233A08DD18ULL),
static_cast<int64_t>(0x76BCCD32975A49CDULL), __FILE__,
__LINE__);
AssertMul128Result(static_cast<int64_t>(0xFD1F5A95DF919291ULL),
static_cast<int64_t>(0x3C5385F8E294E438ULL),
static_cast<int64_t>(0x4F87AE233A08DD18ULL),
static_cast<int64_t>(0xF6BCCD32975A49CDULL), __FILE__,
__LINE__);
AssertMul128Result(static_cast<int64_t>(0xE984CB0E30E8DC36ULL),
static_cast<int64_t>(0xBC5385F8E294E438ULL),
static_cast<int64_t>(0xCF87AE233A08DD18ULL),
static_cast<int64_t>(0x76BCCD32975A49CDULL), __FILE__,
__LINE__);
AssertMul128Result(static_cast<int64_t>(0x01C0F3FC93E46DAAULL),
static_cast<int64_t>(0xBC5385F8E294E438ULL),
static_cast<int64_t>(0xCF87AE233A08DD18ULL),
static_cast<int64_t>(0xF6BCCD32975A49CDULL), __FILE__,
__LINE__);
AssertMul128Result(static_cast<uint64_t>(0x24E331A77C96011DULL),
static_cast<uint64_t>(0x3C5385F8E294E438ULL),
static_cast<uint64_t>(0x4F87AE233A08DD18ULL),
static_cast<uint64_t>(0x76BCCD32975A49CDULL), __FILE__,
__LINE__);
AssertMul128Result(static_cast<uint64_t>(0x4CA708B9199A6FA9ULL),
static_cast<uint64_t>(0x3C5385F8E294E438ULL),
static_cast<uint64_t>(0x4F87AE233A08DD18ULL),
static_cast<uint64_t>(0xF6BCCD32975A49CDULL), __FILE__,
__LINE__);
AssertMul128Result(static_cast<uint64_t>(0x60419840C8432603ULL),
static_cast<uint64_t>(0xBC5385F8E294E438ULL),
static_cast<uint64_t>(0xCF87AE233A08DD18ULL),
static_cast<uint64_t>(0x76BCCD32975A49CDULL), __FILE__,
__LINE__);
AssertMul128Result(static_cast<uint64_t>(0xC8056F526547948FULL),
static_cast<uint64_t>(0xBC5385F8E294E438ULL),
static_cast<uint64_t>(0xCF87AE233A08DD18ULL),
static_cast<uint64_t>(0xF6BCCD32975A49CDULL), __FILE__,
__LINE__);
}
template <class T>
static HWY_INLINE T TestEndianGetIntegerVal(T val) {
static_assert(!IsFloat<T>() && !IsSpecialFloat<T>(),
"T must not be a floating-point type");
using TU = MakeUnsigned<T>;
static_assert(sizeof(T) == sizeof(TU),
"sizeof(T) == sizeof(TU) must be true");
uint8_t result_bytes[sizeof(T)];
const TU val_u = static_cast<TU>(val);
for (size_t i = 0; i < sizeof(T); i++) {
#if HWY_IS_BIG_ENDIAN
const size_t shift_amt = (sizeof(T) - 1 - i) * 8;
#else
const size_t shift_amt = i * 8;
#endif
result_bytes[i] = static_cast<uint8_t>((val_u >> shift_amt) & 0xFF);
}
T result;
CopyBytes<sizeof(T)>(result_bytes, &result);
return result;
}
template <class T, class... Bytes>
static HWY_INLINE T TestEndianCreateValueFromBytes(Bytes&&... bytes) {
static_assert(sizeof(T) > 0, "sizeof(T) > 0 must be true");
static_assert(sizeof...(Bytes) == sizeof(T),
"sizeof...(Bytes) == sizeof(T) must be true");
const uint8_t src_bytes[sizeof(T)]{static_cast<uint8_t>(bytes)...};
T result;
CopyBytes<sizeof(T)>(src_bytes, &result);
return result;
}
#define HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(val) \
HWY_ASSERT_EQ(val, TestEndianGetIntegerVal(val))
HWY_NOINLINE void TestAllEndian() {
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(int8_t{0x01});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(uint8_t{0x01});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(int16_t{0x0102});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(uint16_t{0x0102});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(int32_t{0x01020304});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(uint32_t{0x01020304});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(int64_t{0x0102030405060708});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(uint64_t{0x0102030405060708});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(int16_t{0x0201});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(uint16_t{0x0201});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(int32_t{0x04030201});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(uint32_t{0x04030201});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(int64_t{0x0807060504030201});
HWY_TEST_ENDIAN_CHECK_INTEGER_VAL(uint64_t{0x0807060504030201});
HWY_ASSERT_EQ(HWY_IS_BIG_ENDIAN ? int16_t{0x0102} : int16_t{0x0201},
TestEndianCreateValueFromBytes<int16_t>(0x01, 0x02));
HWY_ASSERT_EQ(HWY_IS_BIG_ENDIAN ? uint16_t{0x0102} : uint16_t{0x0201},
TestEndianCreateValueFromBytes<uint16_t>(0x01, 0x02));
HWY_ASSERT_EQ(
HWY_IS_BIG_ENDIAN ? int32_t{0x01020304} : int32_t{0x04030201},
TestEndianCreateValueFromBytes<int32_t>(0x01, 0x02, 0x03, 0x04));
HWY_ASSERT_EQ(
HWY_IS_BIG_ENDIAN ? uint32_t{0x01020304} : uint32_t{0x04030201},
TestEndianCreateValueFromBytes<uint32_t>(0x01, 0x02, 0x03, 0x04));
HWY_ASSERT_EQ(HWY_IS_BIG_ENDIAN ? int64_t{0x0102030405060708}
: int64_t{0x0807060504030201},
TestEndianCreateValueFromBytes<int64_t>(
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08));
HWY_ASSERT_EQ(HWY_IS_BIG_ENDIAN ? uint64_t{0x0102030405060708}
: uint64_t{0x0807060504030201},
TestEndianCreateValueFromBytes<uint64_t>(
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08));
HWY_ASSERT_EQ(HWY_IS_BIG_ENDIAN ? int16_t{-0x5EFE} : int16_t{0x02A1},
TestEndianCreateValueFromBytes<int16_t>(0xA1, 0x02));
HWY_ASSERT_EQ(
HWY_IS_BIG_ENDIAN ? int32_t{-0x5E4D3CFC} : int32_t{0x04C3B2A1},
TestEndianCreateValueFromBytes<int32_t>(0xA1, 0xB2, 0xC3, 0x04));
HWY_ASSERT_EQ(HWY_IS_BIG_ENDIAN ? int64_t{-0x6E5D4C3B2A1908F8}
: int64_t{0x08F7E6D5C4B3A291},
TestEndianCreateValueFromBytes<int64_t>(
0x91, 0xA2, 0xB3, 0xC4, 0xD5, 0xE6, 0xF7, 0x08));
HWY_ASSERT_EQ(HWY_IS_LITTLE_ENDIAN ? int16_t{-0x5DFF} : int16_t{0x01A2},
TestEndianCreateValueFromBytes<int16_t>(0x01, 0xA2));
HWY_ASSERT_EQ(
HWY_IS_LITTLE_ENDIAN ? int32_t{-0x3B4C5DFF} : int32_t{0x01A2B3C4},
TestEndianCreateValueFromBytes<int32_t>(0x01, 0xA2, 0xB3, 0xC4));
HWY_ASSERT_EQ(HWY_IS_LITTLE_ENDIAN ? int64_t{-0x0718293A4B5C6DFF}
: int64_t{0x0192A3B4C5D6E7F8},
TestEndianCreateValueFromBytes<int64_t>(
0x01, 0x92, 0xA3, 0xB4, 0xC5, 0xD6, 0xE7, 0xF8));
#if HWY_IS_BIG_ENDIAN
HWY_ASSERT_EQ(1.0f,
TestEndianCreateValueFromBytes<float>(0x3F, 0x80, 0x00, 0x00));
HWY_ASSERT_EQ(15922433.0f,
TestEndianCreateValueFromBytes<float>(0x4B, 0x72, 0xF5, 0x01));
HWY_ASSERT_EQ(-12357485.0f,
TestEndianCreateValueFromBytes<float>(0xCB, 0x3C, 0x8F, 0x6D));
#else
HWY_ASSERT_EQ(1.0f,
TestEndianCreateValueFromBytes<float>(0x00, 0x00, 0x80, 0x3F));
HWY_ASSERT_EQ(15922433.0f,
TestEndianCreateValueFromBytes<float>(0x01, 0xF5, 0x72, 0x4B));
HWY_ASSERT_EQ(-12357485.0f,
TestEndianCreateValueFromBytes<float>(0x6D, 0x8F, 0x3C, 0xCB));
#endif
#if HWY_HAVE_FLOAT64
#if HWY_IS_BIG_ENDIAN
HWY_ASSERT_EQ(1.0, TestEndianCreateValueFromBytes<double>(
0x3F, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00));
HWY_ASSERT_EQ(8707235690688195.0,
TestEndianCreateValueFromBytes<double>(0x43, 0x3E, 0xEF, 0x2F,
0x4A, 0x51, 0xAE, 0xC3));
HWY_ASSERT_EQ(-6815854340348452.0,
TestEndianCreateValueFromBytes<double>(0xC3, 0x38, 0x36, 0xFB,
0xC0, 0xCC, 0x1A, 0x24));
#else
HWY_ASSERT_EQ(1.0, TestEndianCreateValueFromBytes<double>(
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F));
HWY_ASSERT_EQ(8707235690688195.0,
TestEndianCreateValueFromBytes<double>(0xC3, 0xAE, 0x51, 0x4A,
0x2F, 0xEF, 0x3E, 0x43));
HWY_ASSERT_EQ(-6815854340348452.0,
TestEndianCreateValueFromBytes<double>(0x24, 0x1A, 0xCC, 0xC0,
0xFB, 0x36, 0x38, 0xC3));
#endif // HWY_IS_BIG_ENDIAN
#endif // HWY_HAVE_FLOAT64
#if HWY_IS_BIG_ENDIAN
HWY_ASSERT_EQ(ConvertScalarTo<bfloat16_t>(1.0f),
BitCastScalar<bfloat16_t>(
TestEndianCreateValueFromBytes<uint16_t>(0x3F, 0x80)));
HWY_ASSERT_EQ(ConvertScalarTo<bfloat16_t>(0.333984375f),
BitCastScalar<bfloat16_t>(
TestEndianCreateValueFromBytes<uint16_t>(0x3E, 0xAB)));
HWY_ASSERT_EQ(ConvertScalarTo<bfloat16_t>(167121905303526337111381770240.0f),
BitCastScalar<bfloat16_t>(
TestEndianCreateValueFromBytes<uint16_t>(0x70, 0x07)));
#else
HWY_ASSERT_EQ(ConvertScalarTo<bfloat16_t>(1.0f),
BitCastScalar<bfloat16_t>(
TestEndianCreateValueFromBytes<uint16_t>(0x80, 0x3F)));
HWY_ASSERT_EQ(ConvertScalarTo<bfloat16_t>(0.333984375f),
BitCastScalar<bfloat16_t>(
TestEndianCreateValueFromBytes<uint16_t>(0xAB, 0x3E)));
HWY_ASSERT_EQ(ConvertScalarTo<bfloat16_t>(167121905303526337111381770240.0f),
BitCastScalar<bfloat16_t>(
TestEndianCreateValueFromBytes<uint16_t>(0x07, 0x70)));
#endif
}
struct TestSpecialFloat {
template <class T>
static constexpr bool EnableSpecialFloatArithOpTest() {
return (hwy::IsSame<T, float16_t>() && HWY_HAVE_SCALAR_F16_OPERATORS) ||
(hwy::IsSame<T, bfloat16_t>() && HWY_HAVE_SCALAR_BF16_OPERATORS);
}
template <class T>
static constexpr HWY_INLINE T EnsureNotNativeSpecialFloat(T&& val) {
#if HWY_HAVE_SCALAR_F16_TYPE
static_assert(!hwy::IsSame<RemoveCvRef<T>, float16_t::Native>(),
"The operator must not return a float16_t::Native");
#endif
#if HWY_HAVE_SCALAR_BF16_TYPE
static_assert(!hwy::IsSame<RemoveCvRef<T>, bfloat16_t::Native>(),
"The operator must not return a bfloat16_t::Native");
#endif
return static_cast<T&&>(val);
}
template <class T>
static HWY_INLINE void AssertSpecialFloatOpResultInRange(float min_expected,
float max_expected,
T actual,
const char* filename,
const int line) {
if (!(actual >= min_expected && actual <= max_expected)) {
hwy::Abort(
filename, line,
"mismatch: value was expected to be between %g and %g, got %g\n",
static_cast<double>(min_expected), static_cast<double>(max_expected),
ConvertScalarTo<double>(actual));
}
}
template <class T,
hwy::EnableIf<EnableSpecialFloatArithOpTest<T>()>* = nullptr>
static HWY_NOINLINE void TestSpecialFloatArithOperators(T /*unused*/) {
#if HWY_HAVE_SCALAR_F16_OPERATORS || HWY_HAVE_SCALAR_BF16_OPERATORS
AssertSpecialFloatOpResultInRange(
-0.008422852f, -0.008361816f,
EnsureNotNativeSpecialFloat(static_cast<T>(-0.008911133f) +
static_cast<T>(5.264282E-4f)),
__FILE__, __LINE__);
AssertSpecialFloatOpResultInRange(
0.44335937f, 0.4453125f,
EnsureNotNativeSpecialFloat(static_cast<T>(-0.0014266968f) +
0.4453125f),
__FILE__, __LINE__);
AssertSpecialFloatOpResultInRange(
39.25f, 39.5f,
EnsureNotNativeSpecialFloat(34.25f + static_cast<T>(5.0625f)), __FILE__,
__LINE__);
AssertSpecialFloatOpResultInRange(
7456.0f, 7488.0f,
EnsureNotNativeSpecialFloat(static_cast<T>(0.29101562f) -
static_cast<T>(-7456.0f)),
__FILE__, __LINE__);
AssertSpecialFloatOpResultInRange(
-2.21875f, -2.203125f,
EnsureNotNativeSpecialFloat(static_cast<T>(1.66893E-4f) - 2.21875f),
__FILE__, __LINE__);
AssertSpecialFloatOpResultInRange(
0.32421875f, 0.32617188f,
EnsureNotNativeSpecialFloat(0.35351562f - static_cast<T>(0.028198242f)),
__FILE__, __LINE__);
AssertSpecialFloatOpResultInRange(
-0.01135254f, -0.011291503f,
EnsureNotNativeSpecialFloat(static_cast<T>(2.109375f) *
static_cast<T>(-0.0053710938f)),
__FILE__, __LINE__);
AssertSpecialFloatOpResultInRange(
2.359375f, 2.375f,
EnsureNotNativeSpecialFloat(static_cast<T>(0.0019454956f) * 1216.0f),
__FILE__, __LINE__);
AssertSpecialFloatOpResultInRange(
-1.1014938E-4f, 3.453125f,
EnsureNotNativeSpecialFloat(-0.00038146973f *
static_cast<T>(-0.00037956237f)),
__FILE__, __LINE__);
AssertSpecialFloatOpResultInRange(
-27.875f, -27.75f,
EnsureNotNativeSpecialFloat(static_cast<T>(-56.5f) /
static_cast<T>(2.03125f)),
__FILE__, __LINE__);
AssertSpecialFloatOpResultInRange(
0.033203125f, 0.033447266f,
EnsureNotNativeSpecialFloat(static_cast<T>(470.0f) / 14080.0f),
__FILE__, __LINE__);
AssertSpecialFloatOpResultInRange(
0.51953125f, 0.5234375f,
EnsureNotNativeSpecialFloat(0.26367188f / static_cast<T>(0.50390625f)),
__FILE__, __LINE__);
T incr_assign_result_1 = static_cast<T>(1.373291E-4f);
EnsureNotNativeSpecialFloat(incr_assign_result_1 +=
static_cast<T>(-20.375f));
AssertSpecialFloatOpResultInRange(-20.375f, -20.25f, incr_assign_result_1,
__FILE__, __LINE__);
T incr_assign_result_2 = static_cast<T>(2.1457672E-4f);
EnsureNotNativeSpecialFloat(incr_assign_result_2 += static_cast<int8_t>(7));
AssertSpecialFloatOpResultInRange(7.0f, 7.03125f, incr_assign_result_2,
__FILE__, __LINE__);
float incr_assign_result3 = -6.747985f;
incr_assign_result3 += static_cast<T>(4.15625f);
AssertSpecialFloatOpResultInRange(-2.59375f, -2.578125f,
incr_assign_result3, __FILE__, __LINE__);
float incr_assign_result4 = 6.71875;
incr_assign_result4 += static_cast<T>(2.359375);
AssertSpecialFloatOpResultInRange(9.0625, 9.125, incr_assign_result4,
__FILE__, __LINE__);
T decr_assign_result_1 = static_cast<T>(4.4059753E-4f);
EnsureNotNativeSpecialFloat(decr_assign_result_1 -=
static_cast<T>(6880.0f));
AssertSpecialFloatOpResultInRange(-6880, -6848, decr_assign_result_1,
__FILE__, __LINE__);
T decr_assign_result_2 = static_cast<T>(85.5f);
EnsureNotNativeSpecialFloat(decr_assign_result_2 -= static_cast<int8_t>(5));
AssertSpecialFloatOpResultInRange(80.5f, 80.5f, decr_assign_result_2,
__FILE__, __LINE__);
float decr_assign_result3 = 9.875f;
decr_assign_result3 -= static_cast<T>(1.5234375f);
AssertSpecialFloatOpResultInRange(8.3125f, 8.375f, decr_assign_result3,
__FILE__, __LINE__);
double decr_assign_result4 = 0.337890625;
decr_assign_result4 -= static_cast<T>(2.328125);
AssertSpecialFloatOpResultInRange(-1.9921875, -1.984375,
decr_assign_result4, __FILE__, __LINE__);
T mul_assign_result_1 = static_cast<T>(15680.0f);
EnsureNotNativeSpecialFloat(mul_assign_result_1 *=
static_cast<T>(0.001373291f));
AssertSpecialFloatOpResultInRange(21.5f, 21.625f, mul_assign_result_1,
__FILE__, __LINE__);
T mul_assign_result_2 = static_cast<T>(2.609375f);
EnsureNotNativeSpecialFloat(mul_assign_result_2 *= static_cast<int8_t>(7));
AssertSpecialFloatOpResultInRange(18.25, 18.375, mul_assign_result_2,
__FILE__, __LINE__);
float mul_assign_result3 = 4.125f;
mul_assign_result3 *= static_cast<T>(3.375f);
AssertSpecialFloatOpResultInRange(13.875f, 13.9375f, mul_assign_result3,
__FILE__, __LINE__);
double mul_assign_result4 = 7.9375;
mul_assign_result4 *= static_cast<T>(0.79296875);
AssertSpecialFloatOpResultInRange(6.28125, 6.3125, mul_assign_result4,
__FILE__, __LINE__);
T div_assign_result_1 = static_cast<T>(11584.0f);
EnsureNotNativeSpecialFloat(div_assign_result_1 /= static_cast<T>(9.5625f));
AssertSpecialFloatOpResultInRange(1208.0f, 1216.0f, div_assign_result_1,
__FILE__, __LINE__);
T div_assign_result_2 = static_cast<T>(0.12109375f);
EnsureNotNativeSpecialFloat(div_assign_result_2 /= static_cast<int8_t>(3));
AssertSpecialFloatOpResultInRange(0.040283203f, 0.040527344f,
div_assign_result_2, __FILE__, __LINE__);
float div_assign_result_3 = 0.21679688f;
div_assign_result_3 /= static_cast<T>(3.421875f);
AssertSpecialFloatOpResultInRange(0.06298828125f, 0.0634765625f,
div_assign_result_3, __FILE__, __LINE__);
double div_assign_result_4 = 5.34375;
div_assign_result_4 /= static_cast<T>(0.337890625);
AssertSpecialFloatOpResultInRange(15.8125, 15.875, div_assign_result_4,
__FILE__, __LINE__);
HWY_ASSERT_EQ(static_cast<T>(-1.0f),
EnsureNotNativeSpecialFloat(-static_cast<T>(1.0f)));
HWY_ASSERT_EQ(static_cast<T>(1.0f),
EnsureNotNativeSpecialFloat(+static_cast<T>(1.0f)));
T pre_incr_result_1 = static_cast<T>(1.0f);
T pre_incr_result_2 = EnsureNotNativeSpecialFloat(++pre_incr_result_1);
HWY_ASSERT_EQ(static_cast<T>(2.0f), pre_incr_result_1);
HWY_ASSERT_EQ(static_cast<T>(2.0f), pre_incr_result_2);
T post_incr_result_1 = static_cast<T>(5.0f);
T post_incr_result_2 = EnsureNotNativeSpecialFloat(post_incr_result_1++);
HWY_ASSERT_EQ(static_cast<T>(6.0f), post_incr_result_1);
HWY_ASSERT_EQ(static_cast<T>(5.0f), post_incr_result_2);
T pre_decr_result_1 = static_cast<T>(-2.0f);
T pre_decr_result_2 = EnsureNotNativeSpecialFloat(--pre_decr_result_1);
HWY_ASSERT_EQ(static_cast<T>(-3.0f), pre_decr_result_1);
HWY_ASSERT_EQ(static_cast<T>(-3.0f), pre_decr_result_2);
T post_decr_result_1 = static_cast<T>(-7.0f);
T post_decr_result_2 = EnsureNotNativeSpecialFloat(post_decr_result_1--);
HWY_ASSERT_EQ(static_cast<T>(-8.0f), post_decr_result_1);
HWY_ASSERT_EQ(static_cast<T>(-7.0f), post_decr_result_2);
HWY_ASSERT(static_cast<T>(1.0f) == 1.0f);
HWY_ASSERT(static_cast<T>(-2.40625f) != 0.0033416748f);
HWY_ASSERT(static_cast<T>(-3248.0f) < 0.0018997193f);
HWY_ASSERT(static_cast<T>(-27904.0f) <= -3.859375f);
HWY_ASSERT(static_cast<T>(1.078125f) > 0.010009765f);
HWY_ASSERT(static_cast<T>(45312.0f) >= 0.00024318695f);
HWY_ASSERT(2.0f == static_cast<T>(2.0f));
HWY_ASSERT(-5.78125f != static_cast<T>(-15168.0f));
HWY_ASSERT(-0.056884766f < static_cast<T>(0.000088214875f));
HWY_ASSERT(0.00008392333f <= static_cast<T>(1384.0f));
HWY_ASSERT(21888.0f > static_cast<T>(-2.578125f));
HWY_ASSERT(0.087402344 >= static_cast<T>(-0.65625f));
#endif // HWY_HAVE_SCALAR_F16_OPERATORS || HWY_HAVE_SCALAR_BF16_OPERATORS
}
template <class T,
hwy::EnableIf<!EnableSpecialFloatArithOpTest<T>()>* = nullptr>
static HWY_INLINE void TestSpecialFloatArithOperators(T /*unused*/) {}
template <class T>
HWY_NOINLINE void operator()(T /*unused*/) const {
static_assert(IsSpecialFloat<T>(), "IsSpecialFloat<T>() must be true");
HWY_ASSERT_EQ(static_cast<uint32_t>(0x436B0000u),
BitCastScalar<uint32_t>(ConvertScalarTo<float>(
BitCastScalar<T>(static_cast<uint16_t>(
IsSame<T, hwy::float16_t>() ? 0x5B58u : 0x436Bu)))));
HWY_ASSERT_EQ(static_cast<uint32_t>(0xBB790000u),
BitCastScalar<uint32_t>(ConvertScalarTo<float>(
BitCastScalar<T>(static_cast<uint16_t>(
IsSame<T, hwy::float16_t>() ? 0x9BC8u : 0xBB79u)))));
HWY_ASSERT_EQ(static_cast<uint32_t>(
IsSame<T, hwy::float16_t>() ? 0xC0B86000u : 0xC5C30000u),
BitCastScalar<uint32_t>(ConvertScalarTo<float>(
BitCastScalar<T>(static_cast<uint16_t>(0xC5C3u)))));
HWY_ASSERT_EQ(static_cast<uint32_t>(
IsSame<T, hwy::float16_t>() ? 0x41D20000u : 0x4E900000u),
BitCastScalar<uint32_t>(ConvertScalarTo<float>(
BitCastScalar<T>(static_cast<uint16_t>(0x4E90u)))));
HWY_ASSERT_EQ(1696.0, ConvertScalarTo<double>(ConvertScalarTo<T>(1696.0f)));
HWY_ASSERT_EQ(
-0.00177001953125f,
ConvertScalarTo<float>(ConvertScalarTo<T>(-0.00177001953125f)));
HWY_ASSERT_EQ(0.49609375f,
ConvertScalarTo<float>(ConvertScalarTo<T>(0.49609375)));
HWY_ASSERT_EQ(
0.000553131103515625,
ConvertScalarTo<double>(ConvertScalarTo<T>(0.000553131103515625)));
HWY_ASSERT_EQ(ConvertScalarTo<T>(3.0f), ConvertScalarTo<T>(3));
HWY_ASSERT_EQ(ConvertScalarTo<T>(-5.5f), ConvertScalarTo<T>(-5.5));
HWY_ASSERT_EQ(ConvertScalarTo<T>(0.82421875f),
ConvertScalarTo<T>(BF16FromF32(0.82421875f)));
HWY_ASSERT_EQ(ConvertScalarTo<T>(-6.375f),
ConvertScalarTo<T>(F16FromF32(-6.375f)));
HWY_ASSERT(ConvertScalarTo<T>(-3.671875f) <
ConvertScalarTo<T>(0.0218505859375f));
HWY_ASSERT(ConvertScalarTo<T>(-0.033447265625f) <=
ConvertScalarTo<T>(8.249282836914062E-5f));
HWY_ASSERT(ConvertScalarTo<T>(23296.0f) > ConvertScalarTo<T>(192.0f));
HWY_ASSERT(ConvertScalarTo<T>(41984.0f) >= ConvertScalarTo<T>(370.0f));
TestSpecialFloatArithOperators(T());
}
};
HWY_NOINLINE void TestAllSpecialFloat() {
TestSpecialFloat test;
test(float16_t());
test(bfloat16_t());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(BaseTest);
HWY_EXPORT_AND_TEST_P(BaseTest, TestAllLimits);
HWY_EXPORT_AND_TEST_P(BaseTest, TestAllLowestHighest);
HWY_EXPORT_AND_TEST_P(BaseTest, TestAllType);
HWY_EXPORT_AND_TEST_P(BaseTest, TestAllIsSame);
HWY_EXPORT_AND_TEST_P(BaseTest, TestAllBitScan);
HWY_EXPORT_AND_TEST_P(BaseTest, TestAllPopCount);
HWY_EXPORT_AND_TEST_P(BaseTest, TestAllDivisor);
HWY_EXPORT_AND_TEST_P(BaseTest, TestAllScalarShr);
HWY_EXPORT_AND_TEST_P(BaseTest, TestAllMul128);
HWY_EXPORT_AND_TEST_P(BaseTest, TestAllEndian);
HWY_EXPORT_AND_TEST_P(BaseTest, TestAllSpecialFloat);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

158
deps/v8/third_party/highway/src/hwy/bit_set.h vendored Normal file
View File

@ -0,0 +1,158 @@
// Copyright 2024 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_BIT_SET_H_
#define HIGHWAY_HWY_BIT_SET_H_
// BitSet with fast Foreach for up to 64 and 4096 members.
#include <stddef.h>
#include "hwy/base.h"
namespace hwy {
// 64-bit specialization of std::bitset, which lacks Foreach.
class BitSet64 {
public:
// No harm if `i` is already set.
void Set(size_t i) {
HWY_DASSERT(i < 64);
bits_ |= (1ULL << i);
HWY_DASSERT(Get(i));
}
// Equivalent to Set(i) for i in [0, 64) where (bits >> i) & 1. This does
// not clear any existing bits.
void SetNonzeroBitsFrom64(uint64_t bits) { bits_ |= bits; }
void Clear(size_t i) {
HWY_DASSERT(i < 64);
bits_ &= ~(1ULL << i);
}
bool Get(size_t i) const {
HWY_DASSERT(i < 64);
return (bits_ & (1ULL << i)) != 0;
}
// Returns true if any Get(i) would return true for i in [0, 64).
bool Any() const { return bits_ != 0; }
// Returns lowest i such that Get(i). Caller must ensure Any() beforehand!
size_t First() const {
HWY_DASSERT(Any());
return Num0BitsBelowLS1Bit_Nonzero64(bits_);
}
// Returns uint64_t(Get(i)) << i for i in [0, 64).
uint64_t Get64() const { return bits_; }
// Calls `func(i)` for each `i` in the set. It is safe for `func` to modify
// the set, but the current Foreach call is unaffected.
template <class Func>
void Foreach(const Func& func) const {
uint64_t remaining_bits = bits_;
while (remaining_bits != 0) {
const size_t i = Num0BitsBelowLS1Bit_Nonzero64(remaining_bits);
remaining_bits &= remaining_bits - 1; // clear LSB
func(i);
}
}
size_t Count() const { return PopCount(bits_); }
private:
uint64_t bits_ = 0;
};
// Two-level bitset for up to kMaxSize <= 4096 values.
template <size_t kMaxSize = 4096>
class BitSet4096 {
public:
// No harm if `i` is already set.
void Set(size_t i) {
HWY_DASSERT(i < kMaxSize);
const size_t idx = i / 64;
const size_t mod = i % 64;
bits_[idx].Set(mod);
nonzero_.Set(idx);
HWY_DASSERT(Get(i));
}
// Equivalent to Set(i) for i in [0, 64) where (bits >> i) & 1. This does
// not clear any existing bits.
void SetNonzeroBitsFrom64(uint64_t bits) {
bits_[0].SetNonzeroBitsFrom64(bits);
if (bits) nonzero_.Set(0);
}
void Clear(size_t i) {
HWY_DASSERT(i < kMaxSize);
const size_t idx = i / 64;
const size_t mod = i % 64;
bits_[idx].Clear(mod);
if (!bits_[idx].Any()) {
nonzero_.Clear(idx);
}
HWY_DASSERT(!Get(i));
}
bool Get(size_t i) const {
HWY_DASSERT(i < kMaxSize);
const size_t idx = i / 64;
const size_t mod = i % 64;
return bits_[idx].Get(mod);
}
// Returns true if any Get(i) would return true for i in [0, 64).
bool Any() const { return nonzero_.Any(); }
// Returns lowest i such that Get(i). Caller must ensure Any() beforehand!
size_t First() const {
HWY_DASSERT(Any());
const size_t idx = nonzero_.First();
return idx * 64 + bits_[idx].First();
}
// Returns uint64_t(Get(i)) << i for i in [0, 64).
uint64_t Get64() const { return bits_[0].Get64(); }
// Calls `func(i)` for each `i` in the set. It is safe for `func` to modify
// the set, but the current Foreach call is only affected if changing one of
// the not yet visited BitSet64 for which Any() is true.
template <class Func>
void Foreach(const Func& func) const {
nonzero_.Foreach([&func, this](size_t idx) {
bits_[idx].Foreach([idx, &func](size_t mod) { func(idx * 64 + mod); });
});
}
size_t Count() const {
size_t total = 0;
nonzero_.Foreach(
[&total, this](size_t idx) { total += bits_[idx].Count(); });
return total;
}
private:
static_assert(kMaxSize <= 64 * 64, "One BitSet64 insufficient");
BitSet64 nonzero_;
BitSet64 bits_[kMaxSize / 64];
};
} // namespace hwy
#endif // HIGHWAY_HWY_BIT_SET_H_

205
deps/v8/third_party/highway/src/hwy/bit_set_test.cc vendored Normal file
View File

@ -0,0 +1,205 @@
// Copyright 2024 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/bit_set.h"
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <algorithm> // std::find
#include <map>
#include <utility> // std::make_pair
#include <vector>
#include "hwy/base.h"
#include "hwy/tests/hwy_gtest.h"
#include "hwy/tests/test_util-inl.h"
#include "hwy/tests/test_util.h"
namespace hwy {
namespace {
using HWY_NAMESPACE::AdjustedReps;
// Template arg for kMin avoids compiler behavior mismatch for lambda capture.
template <class Set, size_t kMax, size_t kMin = 0>
void TestSet() {
Set set;
// Defaults to empty.
HWY_ASSERT(!set.Any());
HWY_ASSERT(set.Count() == 0);
set.Foreach(
[](size_t i) { HWY_ABORT("Set should be empty but got %zu\n", i); });
HWY_ASSERT(!set.Get(0));
HWY_ASSERT(!set.Get(kMax));
// After setting, we can retrieve it.
set.Set(kMax);
HWY_ASSERT(set.Get(kMax));
HWY_ASSERT(set.Any());
HWY_ASSERT(set.First() == kMax);
HWY_ASSERT(set.Count() == 1);
set.Foreach([](size_t i) { HWY_ASSERT(i == kMax); });
// SetNonzeroBitsFrom64 does not clear old bits.
set.SetNonzeroBitsFrom64(1ull << kMin);
HWY_ASSERT(set.Any());
HWY_ASSERT(set.First() == kMin);
HWY_ASSERT(set.Get(kMin));
HWY_ASSERT(set.Get(kMax));
HWY_ASSERT(set.Count() == 2);
set.Foreach([](size_t i) { HWY_ASSERT(i == kMin || i == kMax); });
// After clearing, it is empty again.
set.Clear(kMin);
set.Clear(kMax);
HWY_ASSERT(!set.Any());
HWY_ASSERT(set.Count() == 0);
set.Foreach(
[](size_t i) { HWY_ABORT("Set should be empty but got %zu\n", i); });
HWY_ASSERT(!set.Get(0));
HWY_ASSERT(!set.Get(kMax));
}
TEST(BitSetTest, TestSet64) { TestSet<BitSet64, 63>(); }
TEST(BitSetTest, TestSet4096) { TestSet<BitSet4096<>, 4095>(); }
// Supports membership and random choice, for testing BitSet4096.
class SlowSet {
public:
// Inserting multiple times is a no-op.
void Set(size_t i) {
const auto ib = idx_for_i_.insert(std::make_pair(i, vec_.size()));
if (ib.second) { // inserted
vec_.push_back(i);
HWY_ASSERT(idx_for_i_.size() == vec_.size());
} else {
// Already have `i` and it can be found at the stored index.
HWY_ASSERT(ib.first->first == i);
const size_t idx = ib.first->second;
HWY_ASSERT(vec_[idx] == i);
}
HWY_ASSERT(Get(i));
}
bool Get(size_t i) const {
const auto it = idx_for_i_.find(i);
if (it == idx_for_i_.end()) {
HWY_ASSERT(std::find(vec_.begin(), vec_.end(), i) == vec_.end());
return false;
}
HWY_ASSERT(vec_[it->second] == i);
return true;
}
void Clear(size_t i) {
if (!Get(i)) return;
const size_t idx = idx_for_i_[i];
idx_for_i_.erase(i);
// Move last into gap, unless it was equal to `i`.
const size_t last = vec_.back();
vec_.pop_back();
if (last == i) {
HWY_ASSERT(idx == vec_.size()); // was the last item
} else {
HWY_ASSERT(vec_[idx] == i);
vec_[idx] = last;
idx_for_i_[last] = idx;
HWY_ASSERT(Get(last)); // can still find `last`
}
HWY_ASSERT(!Get(i));
}
size_t Count() const {
HWY_ASSERT(idx_for_i_.size() == vec_.size());
return vec_.size();
}
// Must not call if Count() == 0.
size_t RandomChoice(RandomState& rng) const {
HWY_ASSERT(Count() != 0);
const size_t idx = static_cast<size_t>(hwy::Random32(&rng)) % vec_.size();
return vec_[idx];
}
template <class Set>
void CheckSame(const Set& set) {
HWY_ASSERT(set.Any() == (set.Count() != 0));
HWY_ASSERT(Count() == set.Count());
// Everything set has, we also have.
set.Foreach([this](size_t i) { HWY_ASSERT(Get(i)); });
// Everything we have, set also has.
std::for_each(vec_.begin(), vec_.end(),
[&set](size_t i) { HWY_ASSERT(set.Get(i)); });
// First matches first in the map
if (set.Any()) {
HWY_ASSERT(set.First() == idx_for_i_.begin()->first);
}
}
private:
std::vector<size_t> vec_;
std::map<size_t, size_t> idx_for_i_;
};
void TestSetRandom(uint64_t grow_prob) {
const uint32_t mod = 4096;
RandomState rng;
// Multiple independent random tests:
for (size_t rep = 0; rep < AdjustedReps(100); ++rep) {
BitSet4096<> set;
SlowSet slow_set;
// Mutate sets via random walk and ensure they are the same afterwards.
for (size_t iter = 0; iter < 200; ++iter) {
const uint64_t bits = (Random64(&rng) >> 10) & 0x3FF;
if (bits > 980 && slow_set.Count() != 0) {
// Small chance of reinsertion: already present, unchanged after.
const size_t i = slow_set.RandomChoice(rng);
const size_t count = set.Count();
HWY_ASSERT(set.Get(i));
slow_set.Set(i);
set.Set(i);
HWY_ASSERT(set.Get(i));
HWY_ASSERT(count == set.Count());
} else if (bits < grow_prob) {
// Set random value; no harm if already set.
const size_t i = static_cast<size_t>(Random32(&rng) % mod);
slow_set.Set(i);
set.Set(i);
HWY_ASSERT(set.Get(i));
} else if (slow_set.Count() != 0) {
// Remove existing item.
const size_t i = slow_set.RandomChoice(rng);
const size_t count = set.Count();
HWY_ASSERT(set.Get(i));
slow_set.Clear(i);
set.Clear(i);
HWY_ASSERT(!set.Get(i));
HWY_ASSERT(count == set.Count() + 1);
}
}
slow_set.CheckSame(set);
}
}
// Lower probability of growth so that the set is often nearly empty.
TEST(BitSetTest, TestSetRandomShrink) { TestSetRandom(400); }
TEST(BitSetTest, TestSetRandomGrow) { TestSetRandom(600); }
} // namespace
} // namespace hwy
HWY_TEST_MAIN();

126
deps/v8/third_party/highway/src/hwy/cache_control.h vendored Normal file
View File

@ -0,0 +1,126 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_CACHE_CONTROL_H_
#define HIGHWAY_HWY_CACHE_CONTROL_H_
#include "hwy/base.h"
// Requires SSE2; fails to compile on 32-bit Clang 7 (see
// https://github.com/gperftools/gperftools/issues/946).
#if !defined(__SSE2__) || (HWY_COMPILER_CLANG && HWY_ARCH_X86_32)
#undef HWY_DISABLE_CACHE_CONTROL
#define HWY_DISABLE_CACHE_CONTROL
#endif
#ifndef HWY_DISABLE_CACHE_CONTROL
// intrin.h is sufficient on MSVC and already included by base.h.
#if HWY_ARCH_X86 && !HWY_COMPILER_MSVC
#include <emmintrin.h> // SSE2
#include <xmmintrin.h> // _mm_prefetch
#elif HWY_ARCH_ARM_A64
#include <arm_acle.h>
#endif
#endif // HWY_DISABLE_CACHE_CONTROL
namespace hwy {
// Even if N*sizeof(T) is smaller, Stream may write a multiple of this size.
#define HWY_STREAM_MULTIPLE 16
// The following functions may also require an attribute.
#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL) && !HWY_COMPILER_MSVC
#define HWY_ATTR_CACHE __attribute__((target("sse2")))
#else
#define HWY_ATTR_CACHE
#endif
// Windows.h #defines this, which causes infinite recursion. Temporarily
// undefine to avoid conflict with our function.
// TODO(janwas): remove when this function is removed.
#pragma push_macro("LoadFence")
#undef LoadFence
// Delays subsequent loads until prior loads are visible. Beware of potentially
// differing behavior across architectures and vendors: on Intel but not
// AMD CPUs, also serves as a full fence (waits for all prior instructions to
// complete).
HWY_INLINE HWY_ATTR_CACHE void LoadFence() {
#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL)
_mm_lfence();
#endif
}
// TODO(janwas): remove when this function is removed. (See above.)
#pragma pop_macro("LoadFence")
// Ensures values written by previous `Stream` calls are visible on the current
// core. This is NOT sufficient for synchronizing across cores; when `Stream`
// outputs are to be consumed by other core(s), the producer must publish
// availability (e.g. via mutex or atomic_flag) after `FlushStream`.
HWY_INLINE HWY_ATTR_CACHE void FlushStream() {
#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL)
_mm_sfence();
#endif
}
// Optionally begins loading the cache line containing "p" to reduce latency of
// subsequent actual loads.
template <typename T>
HWY_INLINE HWY_ATTR_CACHE void Prefetch(const T* p) {
(void)p;
#ifndef HWY_DISABLE_CACHE_CONTROL
#if HWY_ARCH_X86
_mm_prefetch(reinterpret_cast<const char*>(p), _MM_HINT_T0);
#elif HWY_COMPILER_GCC // includes clang
// Hint=0 (NTA) behavior differs, but skipping outer caches is probably not
// desirable, so use the default 3 (keep in caches).
__builtin_prefetch(p, /*write=*/0, /*hint=*/3);
#endif
#endif // HWY_DISABLE_CACHE_CONTROL
}
// Invalidates and flushes the cache line containing "p", if possible.
HWY_INLINE HWY_ATTR_CACHE void FlushCacheline(const void* p) {
#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL)
_mm_clflush(p);
#else
(void)p;
#endif
}
// Hints that we are inside a spin loop and potentially reduces power
// consumption and coherency traffic. For example, x86 avoids multiple
// outstanding load requests, which reduces the memory order violation penalty
// when exiting the loop.
HWY_INLINE HWY_ATTR_CACHE void Pause() {
#ifndef HWY_DISABLE_CACHE_CONTROL
#if HWY_ARCH_X86
_mm_pause();
#elif HWY_ARCH_ARM_A64 && HWY_COMPILER_CLANG
// This is documented in ACLE and the YIELD instruction is also available in
// Armv7, but the intrinsic is broken for Armv7 clang, hence A64 only.
__yield();
#elif HWY_ARCH_ARM && HWY_COMPILER_GCC // includes clang
__asm__ volatile("yield" ::: "memory");
#elif HWY_ARCH_PPC && HWY_COMPILER_GCC // includes clang
__asm__ volatile("or 27,27,27" ::: "memory");
#endif
#endif // HWY_DISABLE_CACHE_CONTROL
}
} // namespace hwy
#endif // HIGHWAY_HWY_CACHE_CONTROL_H_

145
deps/v8/third_party/highway/src/hwy/contrib/algo/copy-inl.h vendored Normal file
View File

@ -0,0 +1,145 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Per-target include guard
#if defined(HIGHWAY_HWY_CONTRIB_ALGO_COPY_INL_H_) == \
defined(HWY_TARGET_TOGGLE) // NOLINT
#ifdef HIGHWAY_HWY_CONTRIB_ALGO_COPY_INL_H_
#undef HIGHWAY_HWY_CONTRIB_ALGO_COPY_INL_H_
#else
#define HIGHWAY_HWY_CONTRIB_ALGO_COPY_INL_H_
#endif
#include <stddef.h>
#include <stdint.h>
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
// These functions avoid having to write a loop plus remainder handling in the
// (unfortunately still common) case where arrays are not aligned/padded. If the
// inputs are known to be aligned/padded, it is more efficient to write a single
// loop using Load(). We do not provide a CopyAlignedPadded because it
// would be more verbose than such a loop.
// Fills `to`[0, `count`) with `value`.
template <class D, typename T = TFromD<D>>
void Fill(D d, T value, size_t count, T* HWY_RESTRICT to) {
const size_t N = Lanes(d);
const Vec<D> v = Set(d, value);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
StoreU(v, d, to + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
SafeFillN(remaining, value, d, to + idx);
}
// Copies `from`[0, `count`) to `to`, which must not overlap `from`.
template <class D, typename T = TFromD<D>>
void Copy(D d, const T* HWY_RESTRICT from, size_t count, T* HWY_RESTRICT to) {
const size_t N = Lanes(d);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
const Vec<D> v = LoadU(d, from + idx);
StoreU(v, d, to + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
SafeCopyN(remaining, d, from + idx, to + idx);
}
// For idx in [0, count) in ascending order, appends `from[idx]` to `to` if the
// corresponding mask element of `func(d, v)` is true. Returns the STL-style end
// of the newly written elements in `to`.
//
// `func` is either a functor with a templated operator()(d, v) returning a
// mask, or a generic lambda if using C++14. Due to apparent limitations of
// Clang on Windows, it is currently necessary to add HWY_ATTR before the
// opening { of the lambda to avoid errors about "function .. requires target".
//
// NOTE: this is only supported for 16-, 32- or 64-bit types.
// NOTE: Func may be called a second time for elements it has already seen, but
// these elements will not be written to `to` again.
template <class D, class Func, typename T = TFromD<D>>
T* CopyIf(D d, const T* HWY_RESTRICT from, size_t count, T* HWY_RESTRICT to,
const Func& func) {
const size_t N = Lanes(d);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
const Vec<D> v = LoadU(d, from + idx);
to += CompressBlendedStore(v, func(d, v), d, to);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return to;
#if HWY_MEM_OPS_MIGHT_FAULT
// Proceed one by one.
const CappedTag<T, 1> d1;
for (; idx < count; ++idx) {
using V1 = Vec<decltype(d1)>;
// Workaround for -Waggressive-loop-optimizations on GCC 8
// (iteration 2305843009213693951 invokes undefined behavior for T=i64)
const uintptr_t addr = reinterpret_cast<uintptr_t>(from);
const T* HWY_RESTRICT from_idx =
reinterpret_cast<const T * HWY_RESTRICT>(addr + (idx * sizeof(T)));
const V1 v = LoadU(d1, from_idx);
// Avoid storing to `to` unless we know it should be kept - otherwise, we
// might overrun the end if it was allocated for the exact count.
if (CountTrue(d1, func(d1, v)) == 0) continue;
StoreU(v, d1, to);
to += 1;
}
#else
// Start index of the last unaligned whole vector, ending at the array end.
const size_t last = count - N;
// Number of elements before `from` or already written.
const size_t invalid = idx - last;
HWY_DASSERT(0 != invalid && invalid < N);
const Mask<D> mask = Not(FirstN(d, invalid));
const Vec<D> v = MaskedLoad(mask, d, from + last);
to += CompressBlendedStore(v, And(mask, func(d, v)), d, to);
#endif
return to;
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_ALGO_COPY_INL_H_

210
deps/v8/third_party/highway/src/hwy/contrib/algo/copy_test.cc vendored Normal file
View File

@ -0,0 +1,210 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stddef.h>
#include "hwy/aligned_allocator.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/algo/copy_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/algo/copy-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
// If your project requires C++14 or later, you can ignore this and pass lambdas
// directly to Transform, without requiring an lvalue as we do here for C++11.
#if __cplusplus < 201402L
#define HWY_GENERIC_LAMBDA 0
#else
#define HWY_GENERIC_LAMBDA 1
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
// Returns random integer in [0, 128), which fits in any lane type.
template <typename T>
T Random7Bit(RandomState& rng) {
return ConvertScalarTo<T>(Random32(&rng) & 127);
}
// In C++14, we can instead define these as generic lambdas next to where they
// are invoked.
#if !HWY_GENERIC_LAMBDA
struct IsOdd {
template <class D, class V>
Mask<D> operator()(D d, V v) const {
return TestBit(v, Set(d, TFromD<D>{1}));
}
};
#endif // !HWY_GENERIC_LAMBDA
// Invokes Test (e.g. TestCopyIf) with all arg combinations. T comes from
// ForFloatTypes.
template <class Test>
struct ForeachCountAndMisalign {
template <typename T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) const {
RandomState rng;
const size_t N = Lanes(d);
const size_t misalignments[3] = {0, N / 4, 3 * N / 5};
for (size_t count = 0; count < 2 * N; ++count) {
for (size_t ma : misalignments) {
for (size_t mb : misalignments) {
Test()(d, count, ma, mb, rng);
}
}
}
}
};
struct TestFill {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
// HWY_MAX prevents error when misalign == count == 0.
AlignedFreeUniquePtr<T[]> pa =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count));
AlignedFreeUniquePtr<T[]> pb = AllocateAligned<T>(misalign_b + count + 1);
HWY_ASSERT(pa && pb);
T* expected = pa.get() + misalign_a;
const T value = Random7Bit<T>(rng);
for (size_t i = 0; i < count; ++i) {
expected[i] = value;
}
T* actual = pb.get() + misalign_b;
actual[count] = ConvertScalarTo<T>(0); // sentinel
Fill(d, value, count, actual);
HWY_ASSERT_EQ(ConvertScalarTo<T>(0), actual[count]); // no write past end
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName(HWY_TARGET);
hwy::detail::AssertArrayEqual(info, expected, actual, count, target_name,
__FILE__, __LINE__);
}
};
void TestAllFill() {
ForAllTypes(ForPartialVectors<ForeachCountAndMisalign<TestFill>>());
}
struct TestCopy {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
// Prevents error if size to allocate is zero.
AlignedFreeUniquePtr<T[]> pa =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count));
AlignedFreeUniquePtr<T[]> pb =
AllocateAligned<T>(HWY_MAX(1, misalign_b + count));
HWY_ASSERT(pa && pb);
T* a = pa.get() + misalign_a;
for (size_t i = 0; i < count; ++i) {
a[i] = Random7Bit<T>(rng);
}
T* b = pb.get() + misalign_b;
Copy(d, a, count, b);
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName(HWY_TARGET);
hwy::detail::AssertArrayEqual(info, a, b, count, target_name, __FILE__,
__LINE__);
}
};
void TestAllCopy() {
ForAllTypes(ForPartialVectors<ForeachCountAndMisalign<TestCopy>>());
}
struct TestCopyIf {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
const size_t padding = Lanes(ScalableTag<T>());
// Prevents error if size to allocate is zero.
AlignedFreeUniquePtr<T[]> pa =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count));
AlignedFreeUniquePtr<T[]> pb =
AllocateAligned<T>(HWY_MAX(1, misalign_b + count + padding));
AlignedFreeUniquePtr<T[]> expected = AllocateAligned<T>(HWY_MAX(1, count));
HWY_ASSERT(pa && pb && expected);
T* a = pa.get() + misalign_a;
for (size_t i = 0; i < count; ++i) {
a[i] = Random7Bit<T>(rng);
}
T* b = pb.get() + misalign_b;
size_t num_odd = 0;
for (size_t i = 0; i < count; ++i) {
if (a[i] & 1) {
expected[num_odd++] = a[i];
}
}
#if HWY_GENERIC_LAMBDA
const auto is_odd = [](const auto d, const auto v) HWY_ATTR {
return TestBit(v, Set(d, TFromD<decltype(d)>{1}));
};
#else
const IsOdd is_odd;
#endif
T* end = CopyIf(d, a, count, b, is_odd);
const size_t num_written = static_cast<size_t>(end - b);
HWY_ASSERT_EQ(num_odd, num_written);
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName(HWY_TARGET);
hwy::detail::AssertArrayEqual(info, expected.get(), b, num_odd, target_name,
__FILE__, __LINE__);
}
};
void TestAllCopyIf() {
ForUI163264(ForPartialVectors<ForeachCountAndMisalign<TestCopyIf>>());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(CopyTest);
HWY_EXPORT_AND_TEST_P(CopyTest, TestAllFill);
HWY_EXPORT_AND_TEST_P(CopyTest, TestAllCopy);
HWY_EXPORT_AND_TEST_P(CopyTest, TestAllCopyIf);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

113
deps/v8/third_party/highway/src/hwy/contrib/algo/find-inl.h vendored Normal file
View File

@ -0,0 +1,113 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Per-target include guard
#if defined(HIGHWAY_HWY_CONTRIB_ALGO_FIND_INL_H_) == \
defined(HWY_TARGET_TOGGLE) // NOLINT
#ifdef HIGHWAY_HWY_CONTRIB_ALGO_FIND_INL_H_
#undef HIGHWAY_HWY_CONTRIB_ALGO_FIND_INL_H_
#else
#define HIGHWAY_HWY_CONTRIB_ALGO_FIND_INL_H_
#endif
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
// Returns index of the first element equal to `value` in `in[0, count)`, or
// `count` if not found.
template <class D, typename T = TFromD<D>>
size_t Find(D d, T value, const T* HWY_RESTRICT in, size_t count) {
const size_t N = Lanes(d);
const Vec<D> broadcasted = Set(d, value);
size_t i = 0;
if (count >= N) {
for (; i <= count - N; i += N) {
const intptr_t pos = FindFirstTrue(d, Eq(broadcasted, LoadU(d, in + i)));
if (pos >= 0) return i + static_cast<size_t>(pos);
}
}
if (i != count) {
#if HWY_MEM_OPS_MIGHT_FAULT
// Scan single elements.
const CappedTag<T, 1> d1;
using V1 = Vec<decltype(d1)>;
const V1 broadcasted1 = Set(d1, GetLane(broadcasted));
for (; i < count; ++i) {
if (AllTrue(d1, Eq(broadcasted1, LoadU(d1, in + i)))) {
return i;
}
}
#else
const size_t remaining = count - i;
HWY_DASSERT(0 != remaining && remaining < N);
const Mask<D> mask = FirstN(d, remaining);
const Vec<D> v = MaskedLoad(mask, d, in + i);
// Apply mask so that we don't 'find' the zero-padding from MaskedLoad.
const intptr_t pos = FindFirstTrue(d, And(Eq(broadcasted, v), mask));
if (pos >= 0) return i + static_cast<size_t>(pos);
#endif // HWY_MEM_OPS_MIGHT_FAULT
}
return count; // not found
}
// Returns index of the first element in `in[0, count)` for which `func(d, vec)`
// returns true, otherwise `count`.
template <class D, class Func, typename T = TFromD<D>>
size_t FindIf(D d, const T* HWY_RESTRICT in, size_t count, const Func& func) {
const size_t N = Lanes(d);
size_t i = 0;
if (count >= N) {
for (; i <= count - N; i += N) {
const intptr_t pos = FindFirstTrue(d, func(d, LoadU(d, in + i)));
if (pos >= 0) return i + static_cast<size_t>(pos);
}
}
if (i != count) {
#if HWY_MEM_OPS_MIGHT_FAULT
// Scan single elements.
const CappedTag<T, 1> d1;
for (; i < count; ++i) {
if (AllTrue(d1, func(d1, LoadU(d1, in + i)))) {
return i;
}
}
#else
const size_t remaining = count - i;
HWY_DASSERT(0 != remaining && remaining < N);
const Mask<D> mask = FirstN(d, remaining);
const Vec<D> v = MaskedLoad(mask, d, in + i);
// Apply mask so that we don't 'find' the zero-padding from MaskedLoad.
const intptr_t pos = FindFirstTrue(d, And(func(d, v), mask));
if (pos >= 0) return i + static_cast<size_t>(pos);
#endif // HWY_MEM_OPS_MIGHT_FAULT
}
return count; // not found
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_ALGO_FIND_INL_H_

230
deps/v8/third_party/highway/src/hwy/contrib/algo/find_test.cc vendored Normal file
View File

@ -0,0 +1,230 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdio.h>
#include <algorithm> // std::find_if
#include <vector>
#include "hwy/aligned_allocator.h"
#include "hwy/base.h"
#include "hwy/print.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/algo/find_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/algo/find-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
// If your project requires C++14 or later, you can ignore this and pass lambdas
// directly to FindIf, without requiring an lvalue as we do here for C++11.
#if __cplusplus < 201402L
#define HWY_GENERIC_LAMBDA 0
#else
#define HWY_GENERIC_LAMBDA 1
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
// Returns random number in [-8, 8] - we use knowledge of the range to Find()
// values we know are not present.
template <typename T>
T Random(RandomState& rng) {
const int32_t bits = static_cast<int32_t>(Random32(&rng)) & 1023;
double val = (bits - 512) / 64.0;
// Clamp negative to zero for unsigned types.
if (!hwy::IsSigned<T>() && val < 0.0) {
val = -val;
}
return ConvertScalarTo<T>(val);
}
// In C++14, we can instead define these as generic lambdas next to where they
// are invoked.
#if !HWY_GENERIC_LAMBDA
class GreaterThan {
public:
GreaterThan(int val) : val_(val) {}
template <class D, class V>
Mask<D> operator()(D d, V v) const {
return Gt(v, Set(d, ConvertScalarTo<TFromD<D>>(val_)));
}
private:
int val_;
};
#endif // !HWY_GENERIC_LAMBDA
// Invokes Test (e.g. TestFind) with all arg combinations.
template <class Test>
struct ForeachCountAndMisalign {
template <typename T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) const {
RandomState rng;
const size_t N = Lanes(d);
const size_t misalignments[3] = {0, N / 4, 3 * N / 5};
// Find() checks 8 vectors at a time, so we want to cover a fairly large
// range without oversampling (checking every possible count).
std::vector<size_t> counts(AdjustedReps(512));
for (size_t& count : counts) {
count = static_cast<size_t>(rng()) % (16 * N + 1);
}
counts[0] = 0; // ensure we test count=0.
for (size_t count : counts) {
for (size_t m : misalignments) {
Test()(d, count, m, rng);
}
}
}
};
struct TestFind {
template <class D>
void operator()(D d, size_t count, size_t misalign, RandomState& rng) {
using T = TFromD<D>;
// Must allocate at least one even if count is zero.
AlignedFreeUniquePtr<T[]> storage =
AllocateAligned<T>(HWY_MAX(1, misalign + count));
HWY_ASSERT(storage);
T* in = storage.get() + misalign;
for (size_t i = 0; i < count; ++i) {
in[i] = Random<T>(rng);
}
// For each position, search for that element (which we know is there)
for (size_t pos = 0; pos < count; ++pos) {
const size_t actual = Find(d, in[pos], in, count);
// We may have found an earlier occurrence of the same value; ensure the
// value is the same, and that it is the first.
if (!IsEqual(in[pos], in[actual])) {
fprintf(stderr, "%s count %d, found %.15f at %d but wanted %.15f\n",
hwy::TypeName(T(), Lanes(d)).c_str(), static_cast<int>(count),
ConvertScalarTo<double>(in[actual]), static_cast<int>(actual),
ConvertScalarTo<double>(in[pos]));
HWY_ASSERT(false);
}
for (size_t i = 0; i < actual; ++i) {
if (IsEqual(in[i], in[pos])) {
fprintf(stderr, "%s count %d, found %f at %d but Find returned %d\n",
hwy::TypeName(T(), Lanes(d)).c_str(), static_cast<int>(count),
ConvertScalarTo<double>(in[i]), static_cast<int>(i),
static_cast<int>(actual));
HWY_ASSERT(false);
}
}
}
// Also search for values we know not to be present (out of range)
HWY_ASSERT_EQ(count, Find(d, ConvertScalarTo<T>(9), in, count));
HWY_ASSERT_EQ(count, Find(d, ConvertScalarTo<T>(-9), in, count));
}
};
void TestAllFind() {
ForAllTypes(ForPartialVectors<ForeachCountAndMisalign<TestFind>>());
}
struct TestFindIf {
template <class D>
void operator()(D d, size_t count, size_t misalign, RandomState& rng) {
using T = TFromD<D>;
using TI = MakeSigned<T>;
// Must allocate at least one even if count is zero.
AlignedFreeUniquePtr<T[]> storage =
AllocateAligned<T>(HWY_MAX(1, misalign + count));
HWY_ASSERT(storage);
T* in = storage.get() + misalign;
for (size_t i = 0; i < count; ++i) {
in[i] = Random<T>(rng);
HWY_ASSERT(ConvertScalarTo<TI>(in[i]) <= 8);
HWY_ASSERT(!hwy::IsSigned<T>() || ConvertScalarTo<TI>(in[i]) >= -8);
}
bool found_any = false;
bool not_found_any = false;
// unsigned T would be promoted to signed and compare greater than any
// negative val, whereas Set() would just cast to an unsigned value and the
// comparison remains unsigned, so avoid negative numbers there.
const int min_val = IsSigned<T>() ? -9 : 0;
// Includes out-of-range value 9 to test the not-found path.
for (int val = min_val; val <= 9; ++val) {
#if HWY_GENERIC_LAMBDA
const auto greater = [val](const auto d, const auto v) HWY_ATTR {
return Gt(v, Set(d, ConvertScalarTo<T>(val)));
};
#else
const GreaterThan greater(val);
#endif
const size_t actual = FindIf(d, in, count, greater);
found_any |= actual < count;
not_found_any |= actual == count;
const auto pos = std::find_if(
in, in + count, [val](T x) { return x > ConvertScalarTo<T>(val); });
// Convert returned iterator to index.
const size_t expected = static_cast<size_t>(pos - in);
if (expected != actual) {
fprintf(stderr, "%s count %d val %d, expected %d actual %d\n",
hwy::TypeName(T(), Lanes(d)).c_str(), static_cast<int>(count),
val, static_cast<int>(expected), static_cast<int>(actual));
hwy::detail::PrintArray(hwy::detail::MakeTypeInfo<T>(), "in", in, count,
0, count);
HWY_ASSERT(false);
}
}
// We will always not-find something due to val=9.
HWY_ASSERT(not_found_any);
// We'll find something unless the input is empty or {0} - because 0 > i
// is false for all i=[0,9].
if (count != 0 && in[0] != ConvertScalarTo<T>(0)) {
HWY_ASSERT(found_any);
}
}
};
void TestAllFindIf() {
ForAllTypes(ForPartialVectors<ForeachCountAndMisalign<TestFindIf>>());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(FindTest);
HWY_EXPORT_AND_TEST_P(FindTest, TestAllFind);
HWY_EXPORT_AND_TEST_P(FindTest, TestAllFindIf);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

228
deps/v8/third_party/highway/src/hwy/contrib/algo/transform-inl.h vendored Normal file
View File

@ -0,0 +1,228 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Per-target include guard
#if defined(HIGHWAY_HWY_CONTRIB_ALGO_TRANSFORM_INL_H_) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_ALGO_TRANSFORM_INL_H_
#undef HIGHWAY_HWY_CONTRIB_ALGO_TRANSFORM_INL_H_
#else
#define HIGHWAY_HWY_CONTRIB_ALGO_TRANSFORM_INL_H_
#endif
#include <stddef.h>
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
// These functions avoid having to write a loop plus remainder handling in the
// (unfortunately still common) case where arrays are not aligned/padded. If the
// inputs are known to be aligned/padded, it is more efficient to write a single
// loop using Load(). We do not provide a TransformAlignedPadded because it
// would be more verbose than such a loop.
//
// Func is either a functor with a templated operator()(d, v[, v1[, v2]]), or a
// generic lambda if using C++14. The d argument is the same as was passed to
// the Generate etc. functions. Due to apparent limitations of Clang, it is
// currently necessary to add HWY_ATTR before the opening { of the lambda to
// avoid errors about "always_inline function .. requires target".
//
// We do not check HWY_MEM_OPS_MIGHT_FAULT because LoadN/StoreN do not fault.
// Fills `out[0, count)` with the vectors returned by `func(d, index_vec)`,
// where `index_vec` is `Vec<RebindToUnsigned<D>>`. On the first call to `func`,
// the value of its lane i is i, and increases by `Lanes(d)` after every call.
// Note that some of these indices may be `>= count`, but the elements that
// `func` returns in those lanes will not be written to `out`.
template <class D, class Func, typename T = TFromD<D>>
void Generate(D d, T* HWY_RESTRICT out, size_t count, const Func& func) {
const RebindToUnsigned<D> du;
using TU = TFromD<decltype(du)>;
const size_t N = Lanes(d);
size_t idx = 0;
Vec<decltype(du)> vidx = Iota(du, 0);
if (count >= N) {
for (; idx <= count - N; idx += N) {
StoreU(func(d, vidx), d, out + idx);
vidx = Add(vidx, Set(du, static_cast<TU>(N)));
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
StoreN(func(d, vidx), d, out + idx, remaining);
}
// Calls `func(d, v)` for each input vector; out of bound lanes with index i >=
// `count` are instead taken from `no[i % Lanes(d)]`.
template <class D, class Func, typename T = TFromD<D>>
void Foreach(D d, const T* HWY_RESTRICT in, const size_t count, const Vec<D> no,
const Func& func) {
const size_t N = Lanes(d);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
const Vec<D> v = LoadU(d, in + idx);
func(d, v);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
const Vec<D> v = LoadNOr(no, d, in + idx, remaining);
func(d, v);
}
// Replaces `inout[idx]` with `func(d, inout[idx])`. Example usage: multiplying
// array elements by a constant.
template <class D, class Func, typename T = TFromD<D>>
void Transform(D d, T* HWY_RESTRICT inout, size_t count, const Func& func) {
const size_t N = Lanes(d);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
const Vec<D> v = LoadU(d, inout + idx);
StoreU(func(d, v), d, inout + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
const Vec<D> v = LoadN(d, inout + idx, remaining);
StoreN(func(d, v), d, inout + idx, remaining);
}
// Replaces `inout[idx]` with `func(d, inout[idx], in1[idx])`. Example usage:
// multiplying array elements by those of another array.
template <class D, class Func, typename T = TFromD<D>>
void Transform1(D d, T* HWY_RESTRICT inout, size_t count,
const T* HWY_RESTRICT in1, const Func& func) {
const size_t N = Lanes(d);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
const Vec<D> v = LoadU(d, inout + idx);
const Vec<D> v1 = LoadU(d, in1 + idx);
StoreU(func(d, v, v1), d, inout + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
const Vec<D> v = LoadN(d, inout + idx, remaining);
const Vec<D> v1 = LoadN(d, in1 + idx, remaining);
StoreN(func(d, v, v1), d, inout + idx, remaining);
}
// Replaces `inout[idx]` with `func(d, inout[idx], in1[idx], in2[idx])`. Example
// usage: FMA of elements from three arrays, stored into the first array.
template <class D, class Func, typename T = TFromD<D>>
void Transform2(D d, T* HWY_RESTRICT inout, size_t count,
const T* HWY_RESTRICT in1, const T* HWY_RESTRICT in2,
const Func& func) {
const size_t N = Lanes(d);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
const Vec<D> v = LoadU(d, inout + idx);
const Vec<D> v1 = LoadU(d, in1 + idx);
const Vec<D> v2 = LoadU(d, in2 + idx);
StoreU(func(d, v, v1, v2), d, inout + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
const Vec<D> v = LoadN(d, inout + idx, remaining);
const Vec<D> v1 = LoadN(d, in1 + idx, remaining);
const Vec<D> v2 = LoadN(d, in2 + idx, remaining);
StoreN(func(d, v, v1, v2), d, inout + idx, remaining);
}
template <class D, typename T = TFromD<D>>
void Replace(D d, T* HWY_RESTRICT inout, size_t count, T new_t, T old_t) {
const size_t N = Lanes(d);
const Vec<D> old_v = Set(d, old_t);
const Vec<D> new_v = Set(d, new_t);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
Vec<D> v = LoadU(d, inout + idx);
StoreU(IfThenElse(Eq(v, old_v), new_v, v), d, inout + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
const Vec<D> v = LoadN(d, inout + idx, remaining);
StoreN(IfThenElse(Eq(v, old_v), new_v, v), d, inout + idx, remaining);
}
template <class D, class Func, typename T = TFromD<D>>
void ReplaceIf(D d, T* HWY_RESTRICT inout, size_t count, T new_t,
const Func& func) {
const size_t N = Lanes(d);
const Vec<D> new_v = Set(d, new_t);
size_t idx = 0;
if (count >= N) {
for (; idx <= count - N; idx += N) {
Vec<D> v = LoadU(d, inout + idx);
StoreU(IfThenElse(func(d, v), new_v, v), d, inout + idx);
}
}
// `count` was a multiple of the vector length `N`: already done.
if (HWY_UNLIKELY(idx == count)) return;
const size_t remaining = count - idx;
HWY_DASSERT(0 != remaining && remaining < N);
const Vec<D> v = LoadN(d, inout + idx, remaining);
StoreN(IfThenElse(func(d, v), new_v, v), d, inout + idx, remaining);
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_ALGO_TRANSFORM_INL_H_

464
deps/v8/third_party/highway/src/hwy/contrib/algo/transform_test.cc vendored Normal file
View File

@ -0,0 +1,464 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <string.h> // memcpy
#include <vector>
#include "hwy/aligned_allocator.h"
#include "hwy/base.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/algo/transform_test.cc" //NOLINT
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/algo/transform-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
// If your project requires C++14 or later, you can ignore this and pass lambdas
// directly to Transform, without requiring an lvalue as we do here for C++11.
#if __cplusplus < 201402L
#define HWY_GENERIC_LAMBDA 0
#else
#define HWY_GENERIC_LAMBDA 1
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
constexpr double kAlpha = 1.5; // arbitrary scalar
// Returns random floating-point number in [-8, 8) to ensure computations do
// not exceed float32 precision.
template <typename T>
T Random(RandomState& rng) {
const int32_t bits = static_cast<int32_t>(Random32(&rng)) & 1023;
const double val = (bits - 512) / 64.0;
// Clamp negative to zero for unsigned types.
return ConvertScalarTo<T>(
HWY_MAX(ConvertScalarTo<double>(hwy::LowestValue<T>()), val));
}
// SCAL, AXPY names are from BLAS.
template <typename T>
HWY_NOINLINE void SimpleSCAL(const T* x, T* out, size_t count) {
for (size_t i = 0; i < count; ++i) {
out[i] = ConvertScalarTo<T>(ConvertScalarTo<T>(kAlpha) * x[i]);
}
}
template <typename T>
HWY_NOINLINE void SimpleAXPY(const T* x, const T* y, T* out, size_t count) {
for (size_t i = 0; i < count; ++i) {
out[i] = ConvertScalarTo<T>(
ConvertScalarTo<T>(ConvertScalarTo<T>(kAlpha) * x[i]) + y[i]);
}
}
template <typename T>
HWY_NOINLINE void SimpleFMA4(const T* x, const T* y, const T* z, T* out,
size_t count) {
for (size_t i = 0; i < count; ++i) {
out[i] = ConvertScalarTo<T>(x[i] * y[i] + z[i]);
}
}
// In C++14, we can instead define these as generic lambdas next to where they
// are invoked.
#if !HWY_GENERIC_LAMBDA
// Generator that returns even numbers by doubling the output indices.
struct Gen2 {
template <class D, class VU>
Vec<D> operator()(D d, VU vidx) const {
return BitCast(d, Add(vidx, vidx));
}
};
struct SCAL {
template <class D, class V>
Vec<D> operator()(D d, V v) const {
using T = TFromD<D>;
return Mul(Set(d, ConvertScalarTo<T>(kAlpha)), v);
}
};
struct AXPY {
template <class D, class V>
Vec<D> operator()(D d, V v, V v1) const {
using T = TFromD<D>;
return MulAdd(Set(d, ConvertScalarTo<T>(kAlpha)), v, v1);
}
};
struct FMA4 {
template <class D, class V>
Vec<D> operator()(D /*d*/, V v, V v1, V v2) const {
return MulAdd(v, v1, v2);
}
};
#endif // !HWY_GENERIC_LAMBDA
// Invokes Test (e.g. TestTransform1) with all arg combinations. T comes from
// ForFloatTypes.
template <class Test>
struct ForeachCountAndMisalign {
template <typename T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) const {
RandomState rng;
const size_t N = Lanes(d);
const size_t misalignments[3] = {0, N / 4, 3 * N / 5};
for (size_t count = 0; count < 2 * N; ++count) {
for (size_t ma : misalignments) {
for (size_t mb : misalignments) {
Test()(d, count, ma, mb, rng);
}
}
}
}
};
// Fills an array with random values, placing a given sentinel value both before
// (when misalignment space is available) and after. Requires an allocation of
// at least count + misalign + 1 elements.
template <typename T>
T* FillRandom(AlignedFreeUniquePtr<T[]>& pa, size_t count, size_t misalign,
T sentinel, RandomState& rng) {
for (size_t i = 0; i < misalign; ++i) {
pa[i] = sentinel;
}
T* a = pa.get() + misalign;
for (size_t i = 0; i < count; ++i) {
a[i] = Random<T>(rng);
}
a[count] = sentinel;
return a;
}
// Output-only, no loads
struct TestGenerate {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t /*misalign_b*/,
RandomState& /*rng*/) {
using T = TFromD<D>;
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(misalign_a + count + 1);
AlignedFreeUniquePtr<T[]> expected = AllocateAligned<T>(HWY_MAX(1, count));
HWY_ASSERT(pa && expected);
T* actual = pa.get() + misalign_a;
for (size_t i = 0; i < count; ++i) {
expected[i] = ConvertScalarTo<T>(2 * i);
}
// TODO(janwas): can we update the apply_to in HWY_PUSH_ATTRIBUTES so that
// the attribute also applies to lambdas? If so, remove HWY_ATTR.
#if HWY_GENERIC_LAMBDA
const auto gen2 = [](const auto d, const auto vidx)
HWY_ATTR { return BitCast(d, Add(vidx, vidx)); };
#else
const Gen2 gen2;
#endif
actual[count] = ConvertScalarTo<T>(0); // sentinel
Generate(d, actual, count, gen2);
HWY_ASSERT_EQ(ConvertScalarTo<T>(0), actual[count]); // no write past end
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName(HWY_TARGET);
hwy::detail::AssertArrayEqual(info, expected.get(), actual, count,
target_name, __FILE__, __LINE__);
}
};
// Input-only, no stores
struct TestForeach {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& /*rng*/) {
if (misalign_b != 0) return;
using T = TFromD<D>;
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(misalign_a + count + 1);
HWY_ASSERT(pa);
T* actual = pa.get() + misalign_a;
T max = hwy::LowestValue<T>();
for (size_t i = 0; i < count; ++i) {
actual[i] = hwy::ConvertScalarTo<T>(i <= count / 2 ? 2 * i : i);
max = HWY_MAX(max, actual[i]);
}
// Place sentinel values in the misalignment area and at the input's end.
for (size_t i = 0; i < misalign_a; ++i) {
pa[i] = ConvertScalarTo<T>(2 * count);
}
actual[count] = ConvertScalarTo<T>(2 * count);
const Vec<D> vmin = Set(d, hwy::LowestValue<T>());
// TODO(janwas): can we update the apply_to in HWY_PUSH_ATTRIBUTES so that
// the attribute also applies to lambdas? If so, remove HWY_ATTR.
Vec<D> vmax = vmin;
const auto func = [&vmax](const D, const Vec<D> v)
HWY_ATTR { vmax = Max(vmax, v); };
Foreach(d, actual, count, vmin, func);
const char* target_name = hwy::TargetName(HWY_TARGET);
AssertEqual(max, ReduceMax(d, vmax), target_name, __FILE__, __LINE__);
}
};
// Zero extra input arrays
struct TestTransform {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
if (misalign_b != 0) return;
using T = TFromD<D>;
// Prevents error if size to allocate is zero.
AlignedFreeUniquePtr<T[]> pa =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count + 1));
AlignedFreeUniquePtr<T[]> expected = AllocateAligned<T>(HWY_MAX(1, count));
HWY_ASSERT(pa && expected);
const T sentinel = ConvertScalarTo<T>(-42);
T* a = FillRandom(pa, count, misalign_a, sentinel, rng);
SimpleSCAL(a, expected.get(), count);
// TODO(janwas): can we update the apply_to in HWY_PUSH_ATTRIBUTES so that
// the attribute also applies to lambdas? If so, remove HWY_ATTR.
#if HWY_GENERIC_LAMBDA
const auto scal = [](const auto d, const auto v) HWY_ATTR {
return Mul(Set(d, ConvertScalarTo<T>(kAlpha)), v);
};
#else
const SCAL scal;
#endif
Transform(d, a, count, scal);
const auto info = hwy::detail::MakeTypeInfo<T>();
const char* target_name = hwy::TargetName(HWY_TARGET);
hwy::detail::AssertArrayEqual(info, expected.get(), a, count, target_name,
__FILE__, __LINE__);
// Ensure no out-of-bound writes.
for (size_t i = 0; i < misalign_a; ++i) {
HWY_ASSERT_EQ(sentinel, pa[i]);
}
HWY_ASSERT_EQ(sentinel, a[count]);
}
};
// One extra input array
struct TestTransform1 {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
// Prevents error if size to allocate is zero.
AlignedFreeUniquePtr<T[]> pa =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count + 1));
AlignedFreeUniquePtr<T[]> pb =
AllocateAligned<T>(HWY_MAX(1, misalign_b + count));
AlignedFreeUniquePtr<T[]> expected = AllocateAligned<T>(HWY_MAX(1, count));
HWY_ASSERT(pa && pb && expected);
const T sentinel = ConvertScalarTo<T>(-42);
T* a = FillRandom(pa, count, misalign_a, sentinel, rng);
T* b = pb.get() + misalign_b;
for (size_t i = 0; i < count; ++i) {
b[i] = Random<T>(rng);
}
SimpleAXPY(a, b, expected.get(), count);
#if HWY_GENERIC_LAMBDA
const auto axpy = [](const auto d, const auto v, const auto v1) HWY_ATTR {
return MulAdd(Set(d, ConvertScalarTo<T>(kAlpha)), v, v1);
};
#else
const AXPY axpy;
#endif
Transform1(d, a, count, b, axpy);
AssertArraySimilar(expected.get(), a, count, hwy::TargetName(HWY_TARGET),
__FILE__, __LINE__);
// Ensure no out-of-bound writes.
for (size_t i = 0; i < misalign_a; ++i) {
HWY_ASSERT_EQ(sentinel, pa[i]);
}
HWY_ASSERT_EQ(sentinel, a[count]);
}
};
// Two extra input arrays
struct TestTransform2 {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
// Prevents error if size to allocate is zero.
AlignedFreeUniquePtr<T[]> pa =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count + 1));
AlignedFreeUniquePtr<T[]> pb =
AllocateAligned<T>(HWY_MAX(1, misalign_b + count));
AlignedFreeUniquePtr<T[]> pc =
AllocateAligned<T>(HWY_MAX(1, misalign_a + count));
AlignedFreeUniquePtr<T[]> expected = AllocateAligned<T>(HWY_MAX(1, count));
HWY_ASSERT(pa && pb && pc && expected);
const T sentinel = ConvertScalarTo<T>(-42);
T* a = FillRandom(pa, count, misalign_a, sentinel, rng);
T* b = pb.get() + misalign_b;
T* c = pc.get() + misalign_a;
for (size_t i = 0; i < count; ++i) {
b[i] = Random<T>(rng);
c[i] = Random<T>(rng);
}
SimpleFMA4(a, b, c, expected.get(), count);
#if HWY_GENERIC_LAMBDA
const auto fma4 = [](auto /*d*/, auto v, auto v1, auto v2)
HWY_ATTR { return MulAdd(v, v1, v2); };
#else
const FMA4 fma4;
#endif
Transform2(d, a, count, b, c, fma4);
AssertArraySimilar(expected.get(), a, count, hwy::TargetName(HWY_TARGET),
__FILE__, __LINE__);
// Ensure no out-of-bound writes.
for (size_t i = 0; i < misalign_a; ++i) {
HWY_ASSERT_EQ(sentinel, pa[i]);
}
HWY_ASSERT_EQ(sentinel, a[count]);
}
};
template <typename T>
class IfEq {
public:
IfEq(T val) : val_(val) {}
template <class D, class V>
Mask<D> operator()(D d, V v) const {
return Eq(v, Set(d, val_));
}
private:
T val_;
};
struct TestReplace {
template <class D>
void operator()(D d, size_t count, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
if (misalign_b != 0) return;
if (count == 0) return;
using T = TFromD<D>;
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(misalign_a + count + 1);
AlignedFreeUniquePtr<T[]> pb = AllocateAligned<T>(count);
AlignedFreeUniquePtr<T[]> expected = AllocateAligned<T>(count);
HWY_ASSERT(pa && pb && expected);
const T sentinel = ConvertScalarTo<T>(-42);
T* a = FillRandom(pa, count, misalign_a, sentinel, rng);
std::vector<size_t> positions(AdjustedReps(count));
for (size_t& pos : positions) {
pos = static_cast<size_t>(rng()) % count;
}
for (size_t pos = 0; pos < count; ++pos) {
const T old_t = a[pos];
const T new_t = Random<T>(rng);
for (size_t i = 0; i < count; ++i) {
expected[i] = IsEqual(a[i], old_t) ? new_t : a[i];
}
// Copy so ReplaceIf gets the same input (and thus also outputs expected)
memcpy(pb.get(), a, count * sizeof(T));
Replace(d, a, count, new_t, old_t);
HWY_ASSERT_ARRAY_EQ(expected.get(), a, count);
// Ensure no out-of-bound writes.
for (size_t i = 0; i < misalign_a; ++i) {
HWY_ASSERT_EQ(sentinel, pa[i]);
}
HWY_ASSERT_EQ(sentinel, a[count]);
ReplaceIf(d, pb.get(), count, new_t, IfEq<T>(old_t));
HWY_ASSERT_ARRAY_EQ(expected.get(), pb.get(), count);
// Ensure no out-of-bound writes.
for (size_t i = 0; i < misalign_a; ++i) {
HWY_ASSERT_EQ(sentinel, pa[i]);
}
HWY_ASSERT_EQ(sentinel, a[count]);
}
}
};
void TestAllGenerate() {
// The test BitCast-s the indices, which does not work for floats.
ForIntegerTypes(ForPartialVectors<ForeachCountAndMisalign<TestGenerate>>());
}
void TestAllForeach() {
ForAllTypes(ForPartialVectors<ForeachCountAndMisalign<TestForeach>>());
}
void TestAllTransform() {
ForFloatTypes(ForPartialVectors<ForeachCountAndMisalign<TestTransform>>());
}
void TestAllTransform1() {
ForFloatTypes(ForPartialVectors<ForeachCountAndMisalign<TestTransform1>>());
}
void TestAllTransform2() {
ForFloatTypes(ForPartialVectors<ForeachCountAndMisalign<TestTransform2>>());
}
void TestAllReplace() {
ForFloatTypes(ForPartialVectors<ForeachCountAndMisalign<TestReplace>>());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(TransformTest);
HWY_EXPORT_AND_TEST_P(TransformTest, TestAllGenerate);
HWY_EXPORT_AND_TEST_P(TransformTest, TestAllForeach);
HWY_EXPORT_AND_TEST_P(TransformTest, TestAllTransform);
HWY_EXPORT_AND_TEST_P(TransformTest, TestAllTransform1);
HWY_EXPORT_AND_TEST_P(TransformTest, TestAllTransform2);
HWY_EXPORT_AND_TEST_P(TransformTest, TestAllReplace);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

2851
deps/v8/third_party/highway/src/hwy/contrib/bit_pack/bit_pack-inl.h vendored Normal file
View File

File diff suppressed because it is too large Load Diff

244
deps/v8/third_party/highway/src/hwy/contrib/bit_pack/bit_pack_test.cc vendored Normal file
View File

@ -0,0 +1,244 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdio.h>
#include <vector>
#include "hwy/aligned_allocator.h"
#include "hwy/base.h"
#include "hwy/nanobenchmark.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/bit_pack/bit_pack_test.cc" // NOLINT
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/timer.h"
#include "hwy/contrib/bit_pack/bit_pack-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
#ifndef HWY_BIT_PACK_BENCHMARK
#define HWY_BIT_PACK_BENCHMARK 0
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
// Used to prevent running benchmark (slow) for partial vectors and targets
// except the best available. Global, not per-target, hence must be outside
// HWY_NAMESPACE. Declare first because HWY_ONCE is only true after some code
// has been re-included.
extern size_t last_bits;
extern uint64_t best_target;
#if HWY_ONCE
size_t last_bits = 0;
uint64_t best_target = ~0ull;
#endif
namespace HWY_NAMESPACE {
namespace {
template <size_t kBits, typename T>
T Random(RandomState& rng) {
return ConvertScalarTo<T>(Random32(&rng) & kBits);
}
template <typename T>
class Checker {
public:
explicit Checker(size_t num) { raw_.reserve(num); }
void NotifyRaw(T raw) { raw_.push_back(raw); }
void NotifyRawOutput(size_t bits, T raw) {
if (raw_[num_verified_] != raw) {
HWY_ABORT("%zu bits: pos %zu of %zu, expected %.0f actual %.0f\n", bits,
num_verified_, raw_.size(),
ConvertScalarTo<double>(raw_[num_verified_]),
ConvertScalarTo<double>(raw));
}
++num_verified_;
}
private:
std::vector<T> raw_;
size_t num_verified_ = 0;
};
template <template <size_t> class PackT, size_t kVectors, size_t kBits>
struct TestPack {
template <typename T, class D>
void operator()(T /* t */, D d) {
constexpr size_t kLoops = 16; // working set slightly larger than L1
const size_t N = Lanes(d);
RandomState rng(N * 129);
static_assert(kBits <= kVectors, "");
const size_t num_per_loop = N * kVectors;
const size_t num = num_per_loop * kLoops;
const size_t num_packed_per_loop = N * kBits;
const size_t num_packed = num_packed_per_loop * kLoops;
Checker<T> checker(num);
AlignedFreeUniquePtr<T[]> raw = hwy::AllocateAligned<T>(num);
AlignedFreeUniquePtr<T[]> raw2 = hwy::AllocateAligned<T>(num);
AlignedFreeUniquePtr<T[]> packed = hwy::AllocateAligned<T>(num_packed);
HWY_ASSERT(raw && raw2 && packed);
for (size_t i = 0; i < num; ++i) {
raw[i] = Random<kBits, T>(rng);
checker.NotifyRaw(raw[i]);
}
best_target = HWY_MIN(best_target, HWY_TARGET);
const bool run_bench = HWY_BIT_PACK_BENCHMARK && (kBits != last_bits) &&
(HWY_TARGET == best_target);
last_bits = kBits;
const PackT<kBits> func;
if (run_bench) {
const size_t kNumInputs = 1;
const size_t num_items = num * size_t(Unpredictable1());
const FuncInput inputs[kNumInputs] = {num_items};
Result results[kNumInputs];
Params p;
p.verbose = false;
p.max_evals = 7;
p.target_rel_mad = 0.002;
const size_t num_results = MeasureClosure(
[&](FuncInput) HWY_ATTR {
for (size_t i = 0, pi = 0; i < num;
i += num_per_loop, pi += num_packed_per_loop) {
func.Pack(d, raw.get() + i, packed.get() + pi);
}
T& val = packed.get()[Random32(&rng) % num_packed];
T zero = static_cast<T>(Unpredictable1() - 1);
val = static_cast<T>(val + zero);
for (size_t i = 0, pi = 0; i < num;
i += num_per_loop, pi += num_packed_per_loop) {
func.Unpack(d, packed.get() + pi, raw2.get() + i);
}
return raw2[Random32(&rng) % num];
},
inputs, kNumInputs, results, p);
if (num_results != kNumInputs) {
fprintf(stderr, "MeasureClosure failed.\n");
return;
}
// Print throughput for pack+unpack round trip
for (size_t i = 0; i < num_results; ++i) {
const size_t bytes_per_element = (kBits + 7) / 8;
const double bytes =
static_cast<double>(results[i].input * bytes_per_element);
const double seconds =
results[i].ticks / platform::InvariantTicksPerSecond();
printf("Bits:%2d elements:%3d GB/s:%4.1f (+/-%3.1f%%)\n",
static_cast<int>(kBits), static_cast<int>(results[i].input),
1E-9 * bytes / seconds, results[i].variability * 100.0);
}
} else {
for (size_t i = 0, pi = 0; i < num;
i += num_per_loop, pi += num_packed_per_loop) {
func.Pack(d, raw.get() + i, packed.get() + pi);
}
T& val = packed.get()[Random32(&rng) % num_packed];
T zero = static_cast<T>(Unpredictable1() - 1);
val = static_cast<T>(val + zero);
for (size_t i = 0, pi = 0; i < num;
i += num_per_loop, pi += num_packed_per_loop) {
func.Unpack(d, packed.get() + pi, raw2.get() + i);
}
}
for (size_t i = 0; i < num; ++i) {
checker.NotifyRawOutput(kBits, raw2[i]);
}
}
};
void TestAllPack8() {
ForShrinkableVectors<TestPack<Pack8, 8, 1>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 2>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 3>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 4>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 5>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 6>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 7>>()(uint8_t());
ForShrinkableVectors<TestPack<Pack8, 8, 8>>()(uint8_t());
}
void TestAllPack16() {
ForShrinkableVectors<TestPack<Pack16, 16, 1>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 2>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 3>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 4>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 5>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 6>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 7>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 8>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 9>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 10>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 11>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 12>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 13>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 14>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 15>>()(uint16_t());
ForShrinkableVectors<TestPack<Pack16, 16, 16>>()(uint16_t());
}
void TestAllPack32() {
ForShrinkableVectors<TestPack<Pack32, 32, 1>>()(uint32_t());
ForShrinkableVectors<TestPack<Pack32, 32, 2>>()(uint32_t());
ForShrinkableVectors<TestPack<Pack32, 32, 6>>()(uint32_t());
ForShrinkableVectors<TestPack<Pack32, 32, 11>>()(uint32_t());
ForShrinkableVectors<TestPack<Pack32, 32, 16>>()(uint32_t());
ForShrinkableVectors<TestPack<Pack32, 32, 31>>()(uint32_t());
ForShrinkableVectors<TestPack<Pack32, 32, 32>>()(uint32_t());
}
void TestAllPack64() {
// Fails, but only on GCC 13.
#if !(HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1400 && \
HWY_TARGET == HWY_RVV)
ForShrinkableVectors<TestPack<Pack64, 64, 1>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 5>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 12>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 16>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 27>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 31>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 33>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 41>>()(uint64_t());
ForShrinkableVectors<TestPack<Pack64, 64, 61>>()(uint64_t());
#endif
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(BitPackTest);
HWY_EXPORT_AND_TEST_P(BitPackTest, TestAllPack8);
HWY_EXPORT_AND_TEST_P(BitPackTest, TestAllPack16);
HWY_EXPORT_AND_TEST_P(BitPackTest, TestAllPack32);
HWY_EXPORT_AND_TEST_P(BitPackTest, TestAllPack64);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

361
deps/v8/third_party/highway/src/hwy/contrib/dot/dot-inl.h vendored Normal file
View File

@ -0,0 +1,361 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// clang-format off
#if defined(HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_) == defined(HWY_TARGET_TOGGLE) // NOLINT
// clang-format on
#ifdef HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_
#undef HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_
#else
#define HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_
#endif
#include <stddef.h>
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
struct Dot {
// Specify zero or more of these, ORed together, as the kAssumptions template
// argument to Compute. Each one may improve performance or reduce code size,
// at the cost of additional requirements on the arguments.
enum Assumptions {
// num_elements is at least N, which may be up to HWY_MAX_BYTES / sizeof(T).
kAtLeastOneVector = 1,
// num_elements is divisible by N (a power of two, so this can be used if
// the problem size is known to be a power of two >= HWY_MAX_BYTES /
// sizeof(T)).
kMultipleOfVector = 2,
// RoundUpTo(num_elements, N) elements are accessible; their value does not
// matter (will be treated as if they were zero).
kPaddedToVector = 4,
};
// Returns sum{pa[i] * pb[i]} for floating-point inputs, including float16_t
// and double if HWY_HAVE_FLOAT16/64. Aligning the
// pointers to a multiple of N elements is helpful but not required.
template <int kAssumptions, class D, typename T = TFromD<D>>
static HWY_INLINE T Compute(const D d, const T* const HWY_RESTRICT pa,
const T* const HWY_RESTRICT pb,
const size_t num_elements) {
static_assert(IsFloat<T>(), "MulAdd requires float type");
using V = decltype(Zero(d));
const size_t N = Lanes(d);
size_t i = 0;
constexpr bool kIsAtLeastOneVector =
(kAssumptions & kAtLeastOneVector) != 0;
constexpr bool kIsMultipleOfVector =
(kAssumptions & kMultipleOfVector) != 0;
constexpr bool kIsPaddedToVector = (kAssumptions & kPaddedToVector) != 0;
// Won't be able to do a full vector load without padding => scalar loop.
if (!kIsAtLeastOneVector && !kIsMultipleOfVector && !kIsPaddedToVector &&
HWY_UNLIKELY(num_elements < N)) {
// Only 2x unroll to avoid excessive code size.
T sum0 = ConvertScalarTo<T>(0);
T sum1 = ConvertScalarTo<T>(0);
for (; i + 2 <= num_elements; i += 2) {
// For reasons unknown, fp16 += does not compile on clang (Arm).
sum0 = ConvertScalarTo<T>(sum0 + pa[i + 0] * pb[i + 0]);
sum1 = ConvertScalarTo<T>(sum1 + pa[i + 1] * pb[i + 1]);
}
if (i < num_elements) {
sum1 = ConvertScalarTo<T>(sum1 + pa[i] * pb[i]);
}
return ConvertScalarTo<T>(sum0 + sum1);
}
// Compiler doesn't make independent sum* accumulators, so unroll manually.
// 2 FMA ports * 4 cycle latency = up to 8 in-flight, but that is excessive
// for unaligned inputs (each unaligned pointer halves the throughput
// because it occupies both L1 load ports for a cycle). We cannot have
// arrays of vectors on RVV/SVE, so always unroll 4x.
V sum0 = Zero(d);
V sum1 = Zero(d);
V sum2 = Zero(d);
V sum3 = Zero(d);
// Main loop: unrolled
for (; i + 4 * N <= num_elements; /* i += 4 * N */) { // incr in loop
const auto a0 = LoadU(d, pa + i);
const auto b0 = LoadU(d, pb + i);
i += N;
sum0 = MulAdd(a0, b0, sum0);
const auto a1 = LoadU(d, pa + i);
const auto b1 = LoadU(d, pb + i);
i += N;
sum1 = MulAdd(a1, b1, sum1);
const auto a2 = LoadU(d, pa + i);
const auto b2 = LoadU(d, pb + i);
i += N;
sum2 = MulAdd(a2, b2, sum2);
const auto a3 = LoadU(d, pa + i);
const auto b3 = LoadU(d, pb + i);
i += N;
sum3 = MulAdd(a3, b3, sum3);
}
// Up to 3 iterations of whole vectors
for (; i + N <= num_elements; i += N) {
const auto a = LoadU(d, pa + i);
const auto b = LoadU(d, pb + i);
sum0 = MulAdd(a, b, sum0);
}
if (!kIsMultipleOfVector) {
const size_t remaining = num_elements - i;
if (remaining != 0) {
if (kIsPaddedToVector) {
const auto mask = FirstN(d, remaining);
const auto a = LoadU(d, pa + i);
const auto b = LoadU(d, pb + i);
sum1 = MulAdd(IfThenElseZero(mask, a), IfThenElseZero(mask, b), sum1);
} else {
// Unaligned load such that the last element is in the highest lane -
// ensures we do not touch any elements outside the valid range.
// If we get here, then num_elements >= N.
HWY_DASSERT(i >= N);
i += remaining - N;
const auto skip = FirstN(d, N - remaining);
const auto a = LoadU(d, pa + i); // always unaligned
const auto b = LoadU(d, pb + i);
sum1 = MulAdd(IfThenZeroElse(skip, a), IfThenZeroElse(skip, b), sum1);
}
}
} // kMultipleOfVector
// Reduction tree: sum of all accumulators by pairs, then across lanes.
sum0 = Add(sum0, sum1);
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum2);
return ReduceSum(d, sum0);
}
// f32 * bf16
template <int kAssumptions, class DF, HWY_IF_F32_D(DF)>
static HWY_INLINE float Compute(const DF df,
const float* const HWY_RESTRICT pa,
const hwy::bfloat16_t* const HWY_RESTRICT pb,
const size_t num_elements) {
#if HWY_TARGET == HWY_SCALAR
const Rebind<hwy::bfloat16_t, DF> dbf;
#else
const Repartition<hwy::bfloat16_t, DF> dbf;
using VBF = decltype(Zero(dbf));
#endif
const Half<decltype(dbf)> dbfh;
using VF = decltype(Zero(df));
const size_t NF = Lanes(df);
constexpr bool kIsAtLeastOneVector =
(kAssumptions & kAtLeastOneVector) != 0;
constexpr bool kIsMultipleOfVector =
(kAssumptions & kMultipleOfVector) != 0;
constexpr bool kIsPaddedToVector = (kAssumptions & kPaddedToVector) != 0;
// Won't be able to do a full vector load without padding => scalar loop.
if (!kIsAtLeastOneVector && !kIsMultipleOfVector && !kIsPaddedToVector &&
HWY_UNLIKELY(num_elements < NF)) {
// Only 2x unroll to avoid excessive code size.
float sum0 = 0.0f;
float sum1 = 0.0f;
size_t i = 0;
for (; i + 2 <= num_elements; i += 2) {
sum0 += pa[i + 0] * ConvertScalarTo<float>(pb[i + 0]);
sum1 += pa[i + 1] * ConvertScalarTo<float>(pb[i + 1]);
}
for (; i < num_elements; ++i) {
sum1 += pa[i] * ConvertScalarTo<float>(pb[i]);
}
return sum0 + sum1;
}
// Compiler doesn't make independent sum* accumulators, so unroll manually.
// 2 FMA ports * 4 cycle latency = up to 8 in-flight, but that is excessive
// for unaligned inputs (each unaligned pointer halves the throughput
// because it occupies both L1 load ports for a cycle). We cannot have
// arrays of vectors on RVV/SVE, so always unroll 4x.
VF sum0 = Zero(df);
VF sum1 = Zero(df);
VF sum2 = Zero(df);
VF sum3 = Zero(df);
size_t i = 0;
#if HWY_TARGET != HWY_SCALAR // PromoteUpperTo supported
// Main loop: unrolled
for (; i + 4 * NF <= num_elements; /* i += 4 * N */) { // incr in loop
const VF a0 = LoadU(df, pa + i);
const VBF b0 = LoadU(dbf, pb + i);
i += NF;
sum0 = MulAdd(a0, PromoteLowerTo(df, b0), sum0);
const VF a1 = LoadU(df, pa + i);
i += NF;
sum1 = MulAdd(a1, PromoteUpperTo(df, b0), sum1);
const VF a2 = LoadU(df, pa + i);
const VBF b2 = LoadU(dbf, pb + i);
i += NF;
sum2 = MulAdd(a2, PromoteLowerTo(df, b2), sum2);
const VF a3 = LoadU(df, pa + i);
i += NF;
sum3 = MulAdd(a3, PromoteUpperTo(df, b2), sum3);
}
#endif // HWY_TARGET == HWY_SCALAR
// Up to 3 iterations of whole vectors
for (; i + NF <= num_elements; i += NF) {
const VF a = LoadU(df, pa + i);
const VF b = PromoteTo(df, LoadU(dbfh, pb + i));
sum0 = MulAdd(a, b, sum0);
}
if (!kIsMultipleOfVector) {
const size_t remaining = num_elements - i;
if (remaining != 0) {
if (kIsPaddedToVector) {
const auto mask = FirstN(df, remaining);
const VF a = LoadU(df, pa + i);
const VF b = PromoteTo(df, LoadU(dbfh, pb + i));
sum1 = MulAdd(IfThenElseZero(mask, a), IfThenElseZero(mask, b), sum1);
} else {
// Unaligned load such that the last element is in the highest lane -
// ensures we do not touch any elements outside the valid range.
// If we get here, then num_elements >= N.
HWY_DASSERT(i >= NF);
i += remaining - NF;
const auto skip = FirstN(df, NF - remaining);
const VF a = LoadU(df, pa + i); // always unaligned
const VF b = PromoteTo(df, LoadU(dbfh, pb + i));
sum1 = MulAdd(IfThenZeroElse(skip, a), IfThenZeroElse(skip, b), sum1);
}
}
} // kMultipleOfVector
// Reduction tree: sum of all accumulators by pairs, then across lanes.
sum0 = Add(sum0, sum1);
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum2);
return ReduceSum(df, sum0);
}
// Returns sum{pa[i] * pb[i]} for bfloat16 inputs. Aligning the pointers to a
// multiple of N elements is helpful but not required.
template <int kAssumptions, class D, HWY_IF_BF16_D(D)>
static HWY_INLINE float Compute(const D d,
const bfloat16_t* const HWY_RESTRICT pa,
const bfloat16_t* const HWY_RESTRICT pb,
const size_t num_elements) {
const RebindToUnsigned<D> du16;
const Repartition<float, D> df32;
using V = decltype(Zero(df32));
const size_t N = Lanes(d);
size_t i = 0;
constexpr bool kIsAtLeastOneVector =
(kAssumptions & kAtLeastOneVector) != 0;
constexpr bool kIsMultipleOfVector =
(kAssumptions & kMultipleOfVector) != 0;
constexpr bool kIsPaddedToVector = (kAssumptions & kPaddedToVector) != 0;
// Won't be able to do a full vector load without padding => scalar loop.
if (!kIsAtLeastOneVector && !kIsMultipleOfVector && !kIsPaddedToVector &&
HWY_UNLIKELY(num_elements < N)) {
float sum0 = 0.0f; // Only 2x unroll to avoid excessive code size for..
float sum1 = 0.0f; // this unlikely(?) case.
for (; i + 2 <= num_elements; i += 2) {
sum0 += F32FromBF16(pa[i + 0]) * F32FromBF16(pb[i + 0]);
sum1 += F32FromBF16(pa[i + 1]) * F32FromBF16(pb[i + 1]);
}
if (i < num_elements) {
sum1 += F32FromBF16(pa[i]) * F32FromBF16(pb[i]);
}
return sum0 + sum1;
}
// See comment in the other Compute() overload. Unroll 2x, but we need
// twice as many sums for ReorderWidenMulAccumulate.
V sum0 = Zero(df32);
V sum1 = Zero(df32);
V sum2 = Zero(df32);
V sum3 = Zero(df32);
// Main loop: unrolled
for (; i + 2 * N <= num_elements; /* i += 2 * N */) { // incr in loop
const auto a0 = LoadU(d, pa + i);
const auto b0 = LoadU(d, pb + i);
i += N;
sum0 = ReorderWidenMulAccumulate(df32, a0, b0, sum0, sum1);
const auto a1 = LoadU(d, pa + i);
const auto b1 = LoadU(d, pb + i);
i += N;
sum2 = ReorderWidenMulAccumulate(df32, a1, b1, sum2, sum3);
}
// Possibly one more iteration of whole vectors
if (i + N <= num_elements) {
const auto a0 = LoadU(d, pa + i);
const auto b0 = LoadU(d, pb + i);
i += N;
sum0 = ReorderWidenMulAccumulate(df32, a0, b0, sum0, sum1);
}
if (!kIsMultipleOfVector) {
const size_t remaining = num_elements - i;
if (remaining != 0) {
if (kIsPaddedToVector) {
const auto mask = FirstN(du16, remaining);
const auto va = LoadU(d, pa + i);
const auto vb = LoadU(d, pb + i);
const auto a16 = BitCast(d, IfThenElseZero(mask, BitCast(du16, va)));
const auto b16 = BitCast(d, IfThenElseZero(mask, BitCast(du16, vb)));
sum2 = ReorderWidenMulAccumulate(df32, a16, b16, sum2, sum3);
} else {
// Unaligned load such that the last element is in the highest lane -
// ensures we do not touch any elements outside the valid range.
// If we get here, then num_elements >= N.
HWY_DASSERT(i >= N);
i += remaining - N;
const auto skip = FirstN(du16, N - remaining);
const auto va = LoadU(d, pa + i); // always unaligned
const auto vb = LoadU(d, pb + i);
const auto a16 = BitCast(d, IfThenZeroElse(skip, BitCast(du16, va)));
const auto b16 = BitCast(d, IfThenZeroElse(skip, BitCast(du16, vb)));
sum2 = ReorderWidenMulAccumulate(df32, a16, b16, sum2, sum3);
}
}
} // kMultipleOfVector
// Reduction tree: sum of all accumulators by pairs, then across lanes.
sum0 = Add(sum0, sum1);
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum2);
return ReduceSum(df32, sum0);
}
};
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_

292
deps/v8/third_party/highway/src/hwy/contrib/dot/dot_test.cc vendored Normal file
View File

@ -0,0 +1,292 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "hwy/aligned_allocator.h"
#include "hwy/base.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/dot/dot_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/dot/dot-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
template <typename T1, typename T2>
HWY_NOINLINE T1 SimpleDot(const T1* pa, const T2* pb, size_t num) {
float sum = 0.0f;
for (size_t i = 0; i < num; ++i) {
sum += ConvertScalarTo<float>(pa[i]) * ConvertScalarTo<float>(pb[i]);
}
return ConvertScalarTo<T1>(sum);
}
HWY_MAYBE_UNUSED HWY_NOINLINE float SimpleDot(const float* pa,
const hwy::bfloat16_t* pb,
size_t num) {
float sum = 0.0f;
for (size_t i = 0; i < num; ++i) {
sum += pa[i] * F32FromBF16(pb[i]);
}
return sum;
}
// Overload is required because the generic template hits an internal compiler
// error on aarch64 clang.
HWY_MAYBE_UNUSED HWY_NOINLINE float SimpleDot(const bfloat16_t* pa,
const bfloat16_t* pb,
size_t num) {
float sum = 0.0f;
for (size_t i = 0; i < num; ++i) {
sum += F32FromBF16(pa[i]) * F32FromBF16(pb[i]);
}
return sum;
}
class TestDot {
// Computes/verifies one dot product.
template <int kAssumptions, class D>
void Test(D d, size_t num, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
const size_t N = Lanes(d);
const auto random_t = [&rng]() {
const int32_t bits = static_cast<int32_t>(Random32(&rng)) & 1023;
return static_cast<float>(bits - 512) * (1.0f / 64);
};
const size_t padded =
(kAssumptions & Dot::kPaddedToVector) ? RoundUpTo(num, N) : num;
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(misalign_a + padded);
AlignedFreeUniquePtr<T[]> pb = AllocateAligned<T>(misalign_b + padded);
HWY_ASSERT(pa && pb);
T* a = pa.get() + misalign_a;
T* b = pb.get() + misalign_b;
size_t i = 0;
for (; i < num; ++i) {
a[i] = ConvertScalarTo<T>(random_t());
b[i] = ConvertScalarTo<T>(random_t());
}
// Fill padding with NaN - the values are not used, but avoids MSAN errors.
for (; i < padded; ++i) {
ScalableTag<float> df1;
a[i] = ConvertScalarTo<T>(GetLane(NaN(df1)));
b[i] = ConvertScalarTo<T>(GetLane(NaN(df1)));
}
const double expected = SimpleDot(a, b, num);
const double magnitude = expected > 0.0 ? expected : -expected;
const double actual =
ConvertScalarTo<double>(Dot::Compute<kAssumptions>(d, a, b, num));
const double max = static_cast<double>(8 * 8 * num);
HWY_ASSERT(-max <= actual && actual <= max);
const double tolerance =
96.0 * ConvertScalarTo<double>(Epsilon<T>()) * HWY_MAX(magnitude, 1.0);
HWY_ASSERT(expected - tolerance <= actual &&
actual <= expected + tolerance);
}
// Runs tests with various alignments.
template <int kAssumptions, class D>
void ForeachMisalign(D d, size_t num, RandomState& rng) {
const size_t N = Lanes(d);
const size_t misalignments[3] = {0, N / 4, 3 * N / 5};
for (size_t ma : misalignments) {
for (size_t mb : misalignments) {
Test<kAssumptions>(d, num, ma, mb, rng);
}
}
}
// Runs tests with various lengths compatible with the given assumptions.
template <int kAssumptions, class D>
void ForeachCount(D d, RandomState& rng) {
const size_t N = Lanes(d);
const size_t counts[] = {1,
3,
7,
16,
HWY_MAX(N / 2, 1),
HWY_MAX(2 * N / 3, 1),
N,
N + 1,
4 * N / 3,
3 * N,
8 * N,
8 * N + 2};
for (size_t num : counts) {
if ((kAssumptions & Dot::kAtLeastOneVector) && num < N) continue;
if ((kAssumptions & Dot::kMultipleOfVector) && (num % N) != 0) continue;
ForeachMisalign<kAssumptions>(d, num, rng);
}
}
public:
// Must be inlined on aarch64 for bf16, else clang crashes.
template <class T, class D>
HWY_INLINE void operator()(T /*unused*/, D d) {
RandomState rng;
// All 8 combinations of the three length-related flags:
ForeachCount<0>(d, rng);
ForeachCount<Dot::kAtLeastOneVector>(d, rng);
ForeachCount<Dot::kMultipleOfVector>(d, rng);
ForeachCount<Dot::kMultipleOfVector | Dot::kAtLeastOneVector>(d, rng);
ForeachCount<Dot::kPaddedToVector>(d, rng);
ForeachCount<Dot::kPaddedToVector | Dot::kAtLeastOneVector>(d, rng);
ForeachCount<Dot::kPaddedToVector | Dot::kMultipleOfVector>(d, rng);
ForeachCount<Dot::kPaddedToVector | Dot::kMultipleOfVector |
Dot::kAtLeastOneVector>(d, rng);
}
};
class TestDotF32BF16 {
// Computes/verifies one dot product.
template <int kAssumptions, class D>
void Test(D d, size_t num, size_t misalign_a, size_t misalign_b,
RandomState& rng) {
using T = TFromD<D>;
using T2 = hwy::bfloat16_t;
const size_t N = Lanes(d);
const auto random_t = [&rng]() {
const int32_t bits = static_cast<int32_t>(Random32(&rng)) & 1023;
return static_cast<float>(bits - 512) * (1.0f / 64);
};
const size_t padded =
(kAssumptions & Dot::kPaddedToVector) ? RoundUpTo(num, N) : num;
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(misalign_a + padded);
AlignedFreeUniquePtr<T2[]> pb = AllocateAligned<T2>(misalign_b + padded);
HWY_ASSERT(pa && pb);
T* a = pa.get() + misalign_a;
T2* b = pb.get() + misalign_b;
size_t i = 0;
for (; i < num; ++i) {
a[i] = ConvertScalarTo<T>(random_t());
b[i] = ConvertScalarTo<T2>(random_t());
}
// Fill padding with NaN - the values are not used, but avoids MSAN errors.
for (; i < padded; ++i) {
ScalableTag<float> df1;
a[i] = ConvertScalarTo<T>(GetLane(NaN(df1)));
b[i] = ConvertScalarTo<T2>(GetLane(NaN(df1)));
}
const double expected = SimpleDot(a, b, num);
const double magnitude = expected > 0.0 ? expected : -expected;
const double actual =
ConvertScalarTo<double>(Dot::Compute<kAssumptions>(d, a, b, num));
const double max = static_cast<double>(8 * 8 * num);
HWY_ASSERT(-max <= actual && actual <= max);
const double tolerance =
64.0 * ConvertScalarTo<double>(Epsilon<T2>()) * HWY_MAX(magnitude, 1.0);
HWY_ASSERT(expected - tolerance <= actual &&
actual <= expected + tolerance);
}
// Runs tests with various alignments.
template <int kAssumptions, class D>
void ForeachMisalign(D d, size_t num, RandomState& rng) {
const size_t N = Lanes(d);
const size_t misalignments[3] = {0, N / 4, 3 * N / 5};
for (size_t ma : misalignments) {
for (size_t mb : misalignments) {
Test<kAssumptions>(d, num, ma, mb, rng);
}
}
}
// Runs tests with various lengths compatible with the given assumptions.
template <int kAssumptions, class D>
void ForeachCount(D d, RandomState& rng) {
const size_t N = Lanes(d);
const size_t counts[] = {1,
3,
7,
16,
HWY_MAX(N / 2, 1),
HWY_MAX(2 * N / 3, 1),
N,
N + 1,
4 * N / 3,
3 * N,
8 * N,
8 * N + 2};
for (size_t num : counts) {
if ((kAssumptions & Dot::kAtLeastOneVector) && num < N) continue;
if ((kAssumptions & Dot::kMultipleOfVector) && (num % N) != 0) continue;
ForeachMisalign<kAssumptions>(d, num, rng);
}
}
public:
// Must be inlined on aarch64 for bf16, else clang crashes.
template <class T, class D>
HWY_INLINE void operator()(T /*unused*/, D d) {
RandomState rng;
// All 8 combinations of the three length-related flags:
ForeachCount<0>(d, rng);
ForeachCount<Dot::kAtLeastOneVector>(d, rng);
ForeachCount<Dot::kMultipleOfVector>(d, rng);
ForeachCount<Dot::kMultipleOfVector | Dot::kAtLeastOneVector>(d, rng);
ForeachCount<Dot::kPaddedToVector>(d, rng);
ForeachCount<Dot::kPaddedToVector | Dot::kAtLeastOneVector>(d, rng);
ForeachCount<Dot::kPaddedToVector | Dot::kMultipleOfVector>(d, rng);
ForeachCount<Dot::kPaddedToVector | Dot::kMultipleOfVector |
Dot::kAtLeastOneVector>(d, rng);
}
};
// All floating-point types, both arguments same.
void TestAllDot() { ForFloatTypes(ForPartialVectors<TestDot>()); }
// Mixed f32 and bf16.
void TestAllDotF32BF16() {
ForPartialVectors<TestDotF32BF16> test;
test(float());
}
// Both bf16.
void TestAllDotBF16() { ForShrinkableVectors<TestDot>()(bfloat16_t()); }
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(DotTest);
HWY_EXPORT_AND_TEST_P(DotTest, TestAllDot);
HWY_EXPORT_AND_TEST_P(DotTest, TestAllDotF32BF16);
HWY_EXPORT_AND_TEST_P(DotTest, TestAllDotBF16);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

145
deps/v8/third_party/highway/src/hwy/contrib/image/image.cc vendored Normal file
View File

@ -0,0 +1,145 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/image/image.h"
#include <algorithm> // std::swap
#include <cstddef>
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/image/image.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
size_t GetVectorSize() { return Lanes(ScalableTag<uint8_t>()); }
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(GetVectorSize); // Local function.
} // namespace
size_t ImageBase::VectorSize() {
// Do not cache result - must return the current value, which may be greater
// than the first call if it was subject to DisableTargets!
return HWY_DYNAMIC_DISPATCH(GetVectorSize)();
}
size_t ImageBase::BytesPerRow(const size_t xsize, const size_t sizeof_t) {
const size_t vec_size = VectorSize();
size_t valid_bytes = xsize * sizeof_t;
// Allow unaligned accesses starting at the last valid value - this may raise
// msan errors unless the user calls InitializePaddingForUnalignedAccesses.
// Skip for the scalar case because no extra lanes will be loaded.
if (vec_size != 1) {
HWY_DASSERT(vec_size >= sizeof_t);
valid_bytes += vec_size - sizeof_t;
}
// Round up to vector and cache line size.
const size_t align = HWY_MAX(vec_size, HWY_ALIGNMENT);
size_t bytes_per_row = RoundUpTo(valid_bytes, align);
// During the lengthy window before writes are committed to memory, CPUs
// guard against read after write hazards by checking the address, but
// only the lower 11 bits. We avoid a false dependency between writes to
// consecutive rows by ensuring their sizes are not multiples of 2 KiB.
// Avoid2K prevents the same problem for the planes of an Image3.
if (bytes_per_row % HWY_ALIGNMENT == 0) {
bytes_per_row += align;
}
HWY_DASSERT(bytes_per_row % align == 0);
return bytes_per_row;
}
ImageBase::ImageBase(const size_t xsize, const size_t ysize,
const size_t sizeof_t)
: xsize_(static_cast<uint32_t>(xsize)),
ysize_(static_cast<uint32_t>(ysize)),
bytes_(nullptr, AlignedFreer(&AlignedFreer::DoNothing, nullptr)) {
HWY_ASSERT(sizeof_t == 1 || sizeof_t == 2 || sizeof_t == 4 || sizeof_t == 8);
bytes_per_row_ = 0;
// Dimensions can be zero, e.g. for lazily-allocated images. Only allocate
// if nonzero, because "zero" bytes still have padding/bookkeeping overhead.
if (xsize != 0 && ysize != 0) {
bytes_per_row_ = BytesPerRow(xsize, sizeof_t);
bytes_ = AllocateAligned<uint8_t>(bytes_per_row_ * ysize);
HWY_ASSERT(bytes_.get() != nullptr);
InitializePadding(sizeof_t, Padding::kRoundUp);
}
}
ImageBase::ImageBase(const size_t xsize, const size_t ysize,
const size_t bytes_per_row, void* const aligned)
: xsize_(static_cast<uint32_t>(xsize)),
ysize_(static_cast<uint32_t>(ysize)),
bytes_per_row_(bytes_per_row),
bytes_(static_cast<uint8_t*>(aligned),
AlignedFreer(&AlignedFreer::DoNothing, nullptr)) {
const size_t vec_size = VectorSize();
HWY_ASSERT(bytes_per_row % vec_size == 0);
HWY_ASSERT(reinterpret_cast<uintptr_t>(aligned) % vec_size == 0);
}
void ImageBase::InitializePadding(const size_t sizeof_t, Padding padding) {
#if HWY_IS_MSAN || HWY_IDE
if (xsize_ == 0 || ysize_ == 0) return;
const size_t vec_size = VectorSize(); // Bytes, independent of sizeof_t!
if (vec_size == 1) return; // Scalar mode: no padding needed
const size_t valid_size = xsize_ * sizeof_t;
const size_t initialize_size = padding == Padding::kRoundUp
? RoundUpTo(valid_size, vec_size)
: valid_size + vec_size - sizeof_t;
if (valid_size == initialize_size) return;
for (size_t y = 0; y < ysize_; ++y) {
uint8_t* HWY_RESTRICT row = static_cast<uint8_t*>(VoidRow(y));
#if defined(__clang__) && (__clang_major__ <= 6)
// There's a bug in msan in clang-6 when handling AVX2 operations. This
// workaround allows tests to pass on msan, although it is slower and
// prevents msan warnings from uninitialized images.
memset(row, 0, initialize_size);
#else
memset(row + valid_size, 0, initialize_size - valid_size);
#endif // clang6
}
#else
(void)sizeof_t;
(void)padding;
#endif // HWY_IS_MSAN
}
void ImageBase::Swap(ImageBase& other) {
std::swap(xsize_, other.xsize_);
std::swap(ysize_, other.ysize_);
std::swap(bytes_per_row_, other.bytes_per_row_);
std::swap(bytes_, other.bytes_);
}
} // namespace hwy
#endif // HWY_ONCE

467
deps/v8/third_party/highway/src/hwy/contrib/image/image.h vendored Normal file
View File

@ -0,0 +1,467 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_CONTRIB_IMAGE_IMAGE_H_
#define HIGHWAY_HWY_CONTRIB_IMAGE_IMAGE_H_
// SIMD/multicore-friendly planar image representation with row accessors.
#include <string.h>
#include <utility> // std::move
#include "hwy/aligned_allocator.h"
#include "hwy/base.h"
namespace hwy {
// Type-independent parts of Image<> - reduces code duplication and facilitates
// moving member function implementations to cc file.
struct HWY_CONTRIB_DLLEXPORT ImageBase {
// Returns required alignment in bytes for externally allocated memory.
static size_t VectorSize();
// Returns distance [bytes] between the start of two consecutive rows, a
// multiple of VectorSize but NOT kAlias (see implementation).
static size_t BytesPerRow(size_t xsize, size_t sizeof_t);
// No allocation (for output params or unused images)
ImageBase()
: xsize_(0),
ysize_(0),
bytes_per_row_(0),
bytes_(nullptr, AlignedFreer(&AlignedFreer::DoNothing, nullptr)) {}
// Allocates memory (this is the common case)
ImageBase(size_t xsize, size_t ysize, size_t sizeof_t);
// References but does not take ownership of external memory. Useful for
// interoperability with other libraries. `aligned` must be aligned to a
// multiple of VectorSize() and `bytes_per_row` must also be a multiple of
// VectorSize() or preferably equal to BytesPerRow().
ImageBase(size_t xsize, size_t ysize, size_t bytes_per_row, void* aligned);
// Copy construction/assignment is forbidden to avoid inadvertent copies,
// which can be very expensive. Use CopyImageTo() instead.
ImageBase(const ImageBase& other) = delete;
ImageBase& operator=(const ImageBase& other) = delete;
// Move constructor (required for returning Image from function)
ImageBase(ImageBase&& other) noexcept = default;
// Move assignment (required for std::vector)
ImageBase& operator=(ImageBase&& other) noexcept = default;
void Swap(ImageBase& other);
// Useful for pre-allocating image with some padding for alignment purposes
// and later reporting the actual valid dimensions. Caller is responsible
// for ensuring xsize/ysize are <= the original dimensions.
void ShrinkTo(const size_t xsize, const size_t ysize) {
xsize_ = static_cast<uint32_t>(xsize);
ysize_ = static_cast<uint32_t>(ysize);
// NOTE: we can't recompute bytes_per_row for more compact storage and
// better locality because that would invalidate the image contents.
}
// How many pixels.
HWY_INLINE size_t xsize() const { return xsize_; }
HWY_INLINE size_t ysize() const { return ysize_; }
// NOTE: do not use this for copying rows - the valid xsize may be much less.
HWY_INLINE size_t bytes_per_row() const { return bytes_per_row_; }
// Raw access to byte contents, for interfacing with other libraries.
// Unsigned char instead of char to avoid surprises (sign extension).
HWY_INLINE uint8_t* bytes() {
void* p = bytes_.get();
return static_cast<uint8_t * HWY_RESTRICT>(HWY_ASSUME_ALIGNED(p, 64));
}
HWY_INLINE const uint8_t* bytes() const {
const void* p = bytes_.get();
return static_cast<const uint8_t * HWY_RESTRICT>(HWY_ASSUME_ALIGNED(p, 64));
}
protected:
// Returns pointer to the start of a row.
HWY_INLINE void* VoidRow(const size_t y) const {
#if HWY_IS_ASAN || HWY_IS_MSAN || HWY_IS_TSAN
if (y >= ysize_) {
HWY_ABORT("Row(%d) >= %u\n", static_cast<int>(y), ysize_);
}
#endif
void* row = bytes_.get() + y * bytes_per_row_;
return HWY_ASSUME_ALIGNED(row, 64);
}
enum class Padding {
// Allow Load(d, row + x) for x = 0; x < xsize(); x += Lanes(d). Default.
kRoundUp,
// Allow LoadU(d, row + x) for x <= xsize() - 1. This requires an extra
// vector to be initialized. If done by default, this would suppress
// legitimate msan warnings. We therefore require users to explicitly call
// InitializePadding before using unaligned loads (e.g. convolution).
kUnaligned
};
// Initializes the minimum bytes required to suppress msan warnings from
// legitimate (according to Padding mode) vector loads/stores on the right
// border, where some lanes are uninitialized and assumed to be unused.
void InitializePadding(size_t sizeof_t, Padding padding);
// (Members are non-const to enable assignment during move-assignment.)
uint32_t xsize_; // In valid pixels, not including any padding.
uint32_t ysize_;
size_t bytes_per_row_; // Includes padding.
AlignedFreeUniquePtr<uint8_t[]> bytes_;
};
// Single channel, aligned rows separated by padding. T must be POD.
//
// 'Single channel' (one 2D array per channel) simplifies vectorization
// (repeating the same operation on multiple adjacent components) without the
// complexity of a hybrid layout (8 R, 8 G, 8 B, ...). In particular, clients
// can easily iterate over all components in a row and Image requires no
// knowledge of the pixel format beyond the component type "T".
//
// 'Aligned' means each row is aligned to the L1 cache line size. This prevents
// false sharing between two threads operating on adjacent rows.
//
// 'Padding' is still relevant because vectors could potentially be larger than
// a cache line. By rounding up row sizes to the vector size, we allow
// reading/writing ALIGNED vectors whose first lane is a valid sample. This
// avoids needing a separate loop to handle remaining unaligned lanes.
//
// This image layout could also be achieved with a vector and a row accessor
// function, but a class wrapper with support for "deleter" allows wrapping
// existing memory allocated by clients without copying the pixels. It also
// provides convenient accessors for xsize/ysize, which shortens function
// argument lists. Supports move-construction so it can be stored in containers.
template <typename ComponentType>
class Image : public ImageBase {
public:
using T = ComponentType;
Image() = default;
Image(const size_t xsize, const size_t ysize)
: ImageBase(xsize, ysize, sizeof(T)) {}
Image(const size_t xsize, const size_t ysize, size_t bytes_per_row,
void* aligned)
: ImageBase(xsize, ysize, bytes_per_row, aligned) {}
void InitializePaddingForUnalignedAccesses() {
InitializePadding(sizeof(T), Padding::kUnaligned);
}
HWY_INLINE const T* ConstRow(const size_t y) const {
return static_cast<const T*>(VoidRow(y));
}
HWY_INLINE const T* ConstRow(const size_t y) {
return static_cast<const T*>(VoidRow(y));
}
// Returns pointer to non-const. This allows passing const Image* parameters
// when the callee is only supposed to fill the pixels, as opposed to
// allocating or resizing the image.
HWY_INLINE T* MutableRow(const size_t y) const {
return static_cast<T*>(VoidRow(y));
}
HWY_INLINE T* MutableRow(const size_t y) {
return static_cast<T*>(VoidRow(y));
}
// Returns number of pixels (some of which are padding) per row. Useful for
// computing other rows via pointer arithmetic. WARNING: this must
// NOT be used to determine xsize.
HWY_INLINE intptr_t PixelsPerRow() const {
return static_cast<intptr_t>(bytes_per_row_ / sizeof(T));
}
};
using ImageF = Image<float>;
// A bundle of 3 same-sized images. To fill an existing Image3 using
// single-channel producers, we also need access to each const Image*. Const
// prevents breaking the same-size invariant, while still allowing pixels to be
// changed via MutableRow.
template <typename ComponentType>
class Image3 {
public:
using T = ComponentType;
using ImageT = Image<T>;
static constexpr size_t kNumPlanes = 3;
Image3() : planes_{ImageT(), ImageT(), ImageT()} {}
Image3(const size_t xsize, const size_t ysize)
: planes_{ImageT(xsize, ysize), ImageT(xsize, ysize),
ImageT(xsize, ysize)} {}
Image3(Image3&& other) noexcept {
for (size_t i = 0; i < kNumPlanes; i++) {
planes_[i] = std::move(other.planes_[i]);
}
}
Image3(ImageT&& plane0, ImageT&& plane1, ImageT&& plane2) {
if (!SameSize(plane0, plane1) || !SameSize(plane0, plane2)) {
HWY_ABORT(
"Not same size: %d x %d, %d x %d, %d x %d\n",
static_cast<int>(plane0.xsize()), static_cast<int>(plane0.ysize()),
static_cast<int>(plane1.xsize()), static_cast<int>(plane1.ysize()),
static_cast<int>(plane2.xsize()), static_cast<int>(plane2.ysize()));
}
planes_[0] = std::move(plane0);
planes_[1] = std::move(plane1);
planes_[2] = std::move(plane2);
}
// Copy construction/assignment is forbidden to avoid inadvertent copies,
// which can be very expensive. Use CopyImageTo instead.
Image3(const Image3& other) = delete;
Image3& operator=(const Image3& other) = delete;
Image3& operator=(Image3&& other) noexcept {
for (size_t i = 0; i < kNumPlanes; i++) {
planes_[i] = std::move(other.planes_[i]);
}
return *this;
}
HWY_INLINE const T* ConstPlaneRow(const size_t c, const size_t y) const {
return static_cast<const T*>(VoidPlaneRow(c, y));
}
HWY_INLINE const T* ConstPlaneRow(const size_t c, const size_t y) {
return static_cast<const T*>(VoidPlaneRow(c, y));
}
HWY_INLINE T* MutablePlaneRow(const size_t c, const size_t y) const {
return static_cast<T*>(VoidPlaneRow(c, y));
}
HWY_INLINE T* MutablePlaneRow(const size_t c, const size_t y) {
return static_cast<T*>(VoidPlaneRow(c, y));
}
HWY_INLINE const ImageT& Plane(size_t idx) const { return planes_[idx]; }
void Swap(Image3& other) {
for (size_t c = 0; c < 3; ++c) {
other.planes_[c].Swap(planes_[c]);
}
}
void ShrinkTo(const size_t xsize, const size_t ysize) {
for (ImageT& plane : planes_) {
plane.ShrinkTo(xsize, ysize);
}
}
// Sizes of all three images are guaranteed to be equal.
HWY_INLINE size_t xsize() const { return planes_[0].xsize(); }
HWY_INLINE size_t ysize() const { return planes_[0].ysize(); }
// Returns offset [bytes] from one row to the next row of the same plane.
// WARNING: this must NOT be used to determine xsize, nor for copying rows -
// the valid xsize may be much less.
HWY_INLINE size_t bytes_per_row() const { return planes_[0].bytes_per_row(); }
// Returns number of pixels (some of which are padding) per row. Useful for
// computing other rows via pointer arithmetic. WARNING: this must NOT be used
// to determine xsize.
HWY_INLINE intptr_t PixelsPerRow() const { return planes_[0].PixelsPerRow(); }
private:
// Returns pointer to the start of a row.
HWY_INLINE void* VoidPlaneRow(const size_t c, const size_t y) const {
#if HWY_IS_ASAN || HWY_IS_MSAN || HWY_IS_TSAN
if (c >= kNumPlanes || y >= ysize()) {
HWY_ABORT("PlaneRow(%d, %d) >= %d\n", static_cast<int>(c),
static_cast<int>(y), static_cast<int>(ysize()));
}
#endif
// Use the first plane's stride because the compiler might not realize they
// are all equal. Thus we only need a single multiplication for all planes.
const size_t row_offset = y * planes_[0].bytes_per_row();
const void* row = planes_[c].bytes() + row_offset;
return static_cast<const T * HWY_RESTRICT>(
HWY_ASSUME_ALIGNED(row, HWY_ALIGNMENT));
}
private:
ImageT planes_[kNumPlanes];
};
using Image3F = Image3<float>;
// Rectangular region in image(s). Factoring this out of Image instead of
// shifting the pointer by x0/y0 allows this to apply to multiple images with
// different resolutions. Can compare size via SameSize(rect1, rect2).
class Rect {
public:
// Most windows are xsize_max * ysize_max, except those on the borders where
// begin + size_max > end.
constexpr Rect(size_t xbegin, size_t ybegin, size_t xsize_max,
size_t ysize_max, size_t xend, size_t yend)
: x0_(xbegin),
y0_(ybegin),
xsize_(ClampedSize(xbegin, xsize_max, xend)),
ysize_(ClampedSize(ybegin, ysize_max, yend)) {}
// Construct with origin and known size (typically from another Rect).
constexpr Rect(size_t xbegin, size_t ybegin, size_t xsize, size_t ysize)
: x0_(xbegin), y0_(ybegin), xsize_(xsize), ysize_(ysize) {}
// Construct a rect that covers a whole image.
template <typename Image>
explicit Rect(const Image& image)
: Rect(0, 0, image.xsize(), image.ysize()) {}
Rect() : Rect(0, 0, 0, 0) {}
Rect(const Rect&) = default;
Rect& operator=(const Rect&) = default;
Rect Subrect(size_t xbegin, size_t ybegin, size_t xsize_max,
size_t ysize_max) {
return Rect(x0_ + xbegin, y0_ + ybegin, xsize_max, ysize_max, x0_ + xsize_,
y0_ + ysize_);
}
template <typename T>
const T* ConstRow(const Image<T>* image, size_t y) const {
return image->ConstRow(y + y0_) + x0_;
}
template <typename T>
T* MutableRow(const Image<T>* image, size_t y) const {
return image->MutableRow(y + y0_) + x0_;
}
template <typename T>
const T* ConstPlaneRow(const Image3<T>& image, size_t c, size_t y) const {
return image.ConstPlaneRow(c, y + y0_) + x0_;
}
template <typename T>
T* MutablePlaneRow(Image3<T>* image, const size_t c, size_t y) const {
return image->MutablePlaneRow(c, y + y0_) + x0_;
}
// Returns true if this Rect fully resides in the given image. ImageT could be
// Image<T> or Image3<T>; however if ImageT is Rect, results are nonsensical.
template <class ImageT>
bool IsInside(const ImageT& image) const {
return (x0_ + xsize_ <= image.xsize()) && (y0_ + ysize_ <= image.ysize());
}
size_t x0() const { return x0_; }
size_t y0() const { return y0_; }
size_t xsize() const { return xsize_; }
size_t ysize() const { return ysize_; }
private:
// Returns size_max, or whatever is left in [begin, end).
static constexpr size_t ClampedSize(size_t begin, size_t size_max,
size_t end) {
return (begin + size_max <= end) ? size_max
: (end > begin ? end - begin : 0);
}
size_t x0_;
size_t y0_;
size_t xsize_;
size_t ysize_;
};
// Works for any image-like input type(s).
template <class Image1, class Image2>
HWY_MAYBE_UNUSED bool SameSize(const Image1& image1, const Image2& image2) {
return image1.xsize() == image2.xsize() && image1.ysize() == image2.ysize();
}
// Mirrors out of bounds coordinates and returns valid coordinates unchanged.
// We assume the radius (distance outside the image) is small compared to the
// image size, otherwise this might not terminate.
// The mirror is outside the last column (border pixel is also replicated).
static HWY_INLINE HWY_MAYBE_UNUSED size_t Mirror(int64_t x,
const int64_t xsize) {
HWY_DASSERT(xsize != 0);
// TODO(janwas): replace with branchless version
while (x < 0 || x >= xsize) {
if (x < 0) {
x = -x - 1;
} else {
x = 2 * xsize - 1 - x;
}
}
return static_cast<size_t>(x);
}
// Wrap modes for ensuring X/Y coordinates are in the valid range [0, size):
// Mirrors (repeating the edge pixel once). Useful for convolutions.
struct WrapMirror {
HWY_INLINE size_t operator()(const int64_t coord, const size_t size) const {
return Mirror(coord, static_cast<int64_t>(size));
}
};
// Returns the same coordinate, for when we know "coord" is already valid (e.g.
// interior of an image).
struct WrapUnchanged {
HWY_INLINE size_t operator()(const int64_t coord, size_t /*size*/) const {
return static_cast<size_t>(coord);
}
};
// Similar to Wrap* but for row pointers (reduces Row() multiplications).
class WrapRowMirror {
public:
template <class View>
WrapRowMirror(const View& image, size_t ysize)
: first_row_(image.ConstRow(0)), last_row_(image.ConstRow(ysize - 1)) {}
const float* operator()(const float* const HWY_RESTRICT row,
const int64_t stride) const {
if (row < first_row_) {
const int64_t num_before = first_row_ - row;
// Mirrored; one row before => row 0, two before = row 1, ...
return first_row_ + num_before - stride;
}
if (row > last_row_) {
const int64_t num_after = row - last_row_;
// Mirrored; one row after => last row, two after = last - 1, ...
return last_row_ - num_after + stride;
}
return row;
}
private:
const float* const HWY_RESTRICT first_row_;
const float* const HWY_RESTRICT last_row_;
};
struct WrapRowUnchanged {
HWY_INLINE const float* operator()(const float* const HWY_RESTRICT row,
int64_t /*stride*/) const {
return row;
}
};
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_IMAGE_IMAGE_H_

153
deps/v8/third_party/highway/src/hwy/contrib/image/image_test.cc vendored Normal file
View File

@ -0,0 +1,153 @@
// Copyright (c) the JPEG XL Project
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/image/image.h"
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <random>
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/image/image_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/tests/test_util-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
// Ensure we can always write full aligned vectors.
struct TestAlignedT {
template <typename T>
void operator()(T /*unused*/) const {
std::mt19937 rng(129);
std::uniform_int_distribution<int> dist(0, 16);
const ScalableTag<T> d;
for (size_t ysize = 1; ysize < 4; ++ysize) {
for (size_t xsize = 1; xsize < 64; ++xsize) {
Image<T> img(xsize, ysize);
for (size_t y = 0; y < ysize; ++y) {
T* HWY_RESTRICT row = img.MutableRow(y);
for (size_t x = 0; x < xsize; x += Lanes(d)) {
const auto values = Iota(d, dist(rng));
Store(values, d, row + x);
}
}
// Sanity check to prevent optimizing out the writes
const auto x = std::uniform_int_distribution<size_t>(0, xsize - 1)(rng);
const auto y = std::uniform_int_distribution<size_t>(0, ysize - 1)(rng);
HWY_ASSERT(img.ConstRow(y)[x] < 16 + Lanes(d));
}
}
}
};
void TestAligned() { ForUnsignedTypes(TestAlignedT()); }
// Ensure we can write an unaligned vector starting at the last valid value.
struct TestUnalignedT {
template <typename T>
void operator()(T /*unused*/) const {
std::mt19937 rng(129);
std::uniform_int_distribution<int> dist(0, 3);
const ScalableTag<T> d;
for (size_t ysize = 1; ysize < 4; ++ysize) {
for (size_t xsize = 1; xsize < 128; ++xsize) {
Image<T> img(xsize, ysize);
img.InitializePaddingForUnalignedAccesses();
// This test reads padding, which only works if it was initialized,
// which only happens in MSAN builds.
#if HWY_IS_MSAN || HWY_IDE
// Initialize only the valid samples
for (size_t y = 0; y < ysize; ++y) {
T* HWY_RESTRICT row = img.MutableRow(y);
for (size_t x = 0; x < xsize; ++x) {
row[x] = ConvertScalarTo<T>(1u << dist(rng));
}
}
// Read padding bits
auto accum = Zero(d);
for (size_t y = 0; y < ysize; ++y) {
T* HWY_RESTRICT row = img.MutableRow(y);
for (size_t x = 0; x < xsize; ++x) {
accum = Or(accum, LoadU(d, row + x));
}
}
// Ensure padding was zero
const size_t N = Lanes(d);
auto lanes = AllocateAligned<T>(N);
HWY_ASSERT(lanes);
Store(accum, d, lanes.get());
for (size_t i = 0; i < N; ++i) {
HWY_ASSERT(lanes[i] < 16);
}
#else // Check that writing padding does not overwrite valid samples
// Initialize only the valid samples
for (size_t y = 0; y < ysize; ++y) {
T* HWY_RESTRICT row = img.MutableRow(y);
for (size_t x = 0; x < xsize; ++x) {
row[x] = ConvertScalarTo<T>(x);
}
}
// Zero padding and rightmost sample
for (size_t y = 0; y < ysize; ++y) {
T* HWY_RESTRICT row = img.MutableRow(y);
StoreU(Zero(d), d, row + xsize - 1);
}
// Ensure no samples except the rightmost were overwritten
for (size_t y = 0; y < ysize; ++y) {
T* HWY_RESTRICT row = img.MutableRow(y);
for (size_t x = 0; x < xsize - 1; ++x) {
HWY_ASSERT_EQ(ConvertScalarTo<T>(x), row[x]);
}
}
#endif
}
}
}
};
void TestUnaligned() { ForUnsignedTypes(TestUnalignedT()); }
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(ImageTest);
HWY_EXPORT_AND_TEST_P(ImageTest, TestAligned);
HWY_EXPORT_AND_TEST_P(ImageTest, TestUnaligned);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

1752
deps/v8/third_party/highway/src/hwy/contrib/math/math-inl.h vendored Normal file
View File

File diff suppressed because it is too large Load Diff

664
deps/v8/third_party/highway/src/hwy/contrib/math/math_test.cc vendored Normal file
View File

@ -0,0 +1,664 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <cfloat> // FLT_MAX
#include <cmath> // std::abs
#include "hwy/base.h"
#include "hwy/nanobenchmark.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/math/math_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/math/math-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
// We have had test failures caused by excess precision due to keeping
// intermediate results in 80-bit x87 registers. One such failure mode is that
// Log1p computes a 1.0 which is not exactly equal to 1.0f, causing is_pole to
// incorrectly evaluate to false.
#undef HWY_MATH_TEST_EXCESS_PRECISION
#if HWY_ARCH_X86_32 && HWY_COMPILER_GCC_ACTUAL && \
(HWY_TARGET == HWY_SCALAR || HWY_TARGET == HWY_EMU128)
// GCC 13+: because CMAKE_CXX_EXTENSIONS is OFF, we build with -std= and hence
// also -fexcess-precision=standard, so there is no problem. See #1708 and
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=323.
#if HWY_COMPILER_GCC_ACTUAL >= 1300
#define HWY_MATH_TEST_EXCESS_PRECISION 0
#else // HWY_COMPILER_GCC_ACTUAL < 1300
// The build system must enable SSE2, e.g. via HWY_CMAKE_SSE2 - see
// https://stackoverflow.com/questions/20869904/c-handling-of-excess-precision .
#if defined(__SSE2__) // correct flag given, no problem
#define HWY_MATH_TEST_EXCESS_PRECISION 0
#else
#define HWY_MATH_TEST_EXCESS_PRECISION 1
#pragma message( \
"Skipping scalar math_test on 32-bit x86 GCC <13 without HWY_CMAKE_SSE2")
#endif // defined(__SSE2__)
#endif // HWY_COMPILER_GCC_ACTUAL
#else // not (x86-32, GCC, scalar target): running math_test normally
#define HWY_MATH_TEST_EXCESS_PRECISION 0
#endif // HWY_ARCH_X86_32 etc
template <class T, class D>
HWY_NOINLINE void TestMath(const char* name, T (*fx1)(T),
Vec<D> (*fxN)(D, VecArg<Vec<D>>), D d, T min, T max,
uint64_t max_error_ulp) {
if (HWY_MATH_TEST_EXCESS_PRECISION) {
static bool once = true;
if (once) {
once = false;
fprintf(stderr,
"Skipping math_test due to GCC issue with excess precision.\n");
}
return;
}
using UintT = MakeUnsigned<T>;
const UintT min_bits = BitCastScalar<UintT>(min);
const UintT max_bits = BitCastScalar<UintT>(max);
// If min is negative and max is positive, the range needs to be broken into
// two pieces, [+0, max] and [-0, min], otherwise [min, max].
int range_count = 1;
UintT ranges[2][2] = {{min_bits, max_bits}, {0, 0}};
if ((min < 0.0) && (max > 0.0)) {
ranges[0][0] = BitCastScalar<UintT>(ConvertScalarTo<T>(+0.0));
ranges[0][1] = max_bits;
ranges[1][0] = BitCastScalar<UintT>(ConvertScalarTo<T>(-0.0));
ranges[1][1] = min_bits;
range_count = 2;
}
uint64_t max_ulp = 0;
// Emulation is slower, so cannot afford as many.
constexpr UintT kSamplesPerRange = static_cast<UintT>(AdjustedReps(4000));
for (int range_index = 0; range_index < range_count; ++range_index) {
const UintT start = ranges[range_index][0];
const UintT stop = ranges[range_index][1];
const UintT step = HWY_MAX(1, ((stop - start) / kSamplesPerRange));
for (UintT value_bits = start; value_bits <= stop; value_bits += step) {
// For reasons unknown, the HWY_MAX is necessary on RVV, otherwise
// value_bits can be less than start, and thus possibly NaN.
const T value =
BitCastScalar<T>(HWY_MIN(HWY_MAX(start, value_bits), stop));
const T actual = GetLane(fxN(d, Set(d, value)));
const T expected = fx1(value);
// Skip small inputs and outputs on armv7, it flushes subnormals to zero.
#if HWY_TARGET <= HWY_NEON_WITHOUT_AES && HWY_ARCH_ARM_V7
if ((std::abs(value) < 1e-37f) || (std::abs(expected) < 1e-37f)) {
continue;
}
#endif
const auto ulp = hwy::detail::ComputeUlpDelta(actual, expected);
max_ulp = HWY_MAX(max_ulp, ulp);
if (ulp > max_error_ulp) {
fprintf(stderr, "%s: %s(%f) expected %E actual %E ulp %g max ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), name, value, expected,
actual, static_cast<double>(ulp),
static_cast<uint32_t>(max_error_ulp));
}
}
}
fprintf(stderr, "%s: %s max_ulp %g\n", hwy::TypeName(T(), Lanes(d)).c_str(),
name, static_cast<double>(max_ulp));
HWY_ASSERT(max_ulp <= max_error_ulp);
}
#define DEFINE_MATH_TEST_FUNC(NAME) \
HWY_NOINLINE void TestAll##NAME() { \
ForFloat3264Types(ForPartialVectors<Test##NAME>()); \
}
#undef DEFINE_MATH_TEST
#define DEFINE_MATH_TEST(NAME, F32x1, F32xN, F32_MIN, F32_MAX, F32_ERROR, \
F64x1, F64xN, F64_MIN, F64_MAX, F64_ERROR) \
struct Test##NAME { \
template <class T, class D> \
HWY_NOINLINE void operator()(T, D d) { \
if (sizeof(T) == 4) { \
TestMath<T, D>(HWY_STR(NAME), F32x1, F32xN, d, F32_MIN, F32_MAX, \
F32_ERROR); \
} else { \
TestMath<T, D>(HWY_STR(NAME), F64x1, F64xN, d, \
static_cast<T>(F64_MIN), static_cast<T>(F64_MAX), \
F64_ERROR); \
} \
} \
}; \
DEFINE_MATH_TEST_FUNC(NAME)
// Floating point values closest to but less than 1.0. Avoid variables with
// static initializers inside HWY_BEFORE_NAMESPACE/HWY_AFTER_NAMESPACE to
// ensure target-specific code does not leak into startup code.
float kNearOneF() { return BitCastScalar<float>(0x3F7FFFFF); }
double kNearOneD() { return BitCastScalar<double>(0x3FEFFFFFFFFFFFFFULL); }
// The discrepancy is unacceptably large for MSYS2 (less accurate libm?), so
// only increase the error tolerance there.
constexpr uint64_t Cos64ULP() {
#if defined(__MINGW32__)
return 23;
#else
return 3;
#endif
}
constexpr uint64_t ACosh32ULP() {
#if defined(__MINGW32__)
return 8;
#else
return 3;
#endif
}
template <class D>
static Vec<D> SinCosSin(const D d, VecArg<Vec<D>> x) {
Vec<D> s, c;
CallSinCos(d, x, s, c);
return s;
}
template <class D>
static Vec<D> SinCosCos(const D d, VecArg<Vec<D>> x) {
Vec<D> s, c;
CallSinCos(d, x, s, c);
return c;
}
// on targets without FMA the result is less inaccurate
constexpr uint64_t SinCosSin32ULP() {
#if !(HWY_NATIVE_FMA)
return 256;
#else
return 3;
#endif
}
constexpr uint64_t SinCosCos32ULP() {
#if !(HWY_NATIVE_FMA)
return 64;
#else
return 3;
#endif
}
// clang-format off
DEFINE_MATH_TEST(Acos,
std::acos, CallAcos, -1.0f, +1.0f, 3, // NEON is 3 instead of 2
std::acos, CallAcos, -1.0, +1.0, 2)
DEFINE_MATH_TEST(Acosh,
std::acosh, CallAcosh, +1.0f, +FLT_MAX, ACosh32ULP(),
std::acosh, CallAcosh, +1.0, +DBL_MAX, 3)
DEFINE_MATH_TEST(Asin,
std::asin, CallAsin, -1.0f, +1.0f, 4, // 4 ulp on Armv7, not 2
std::asin, CallAsin, -1.0, +1.0, 2)
DEFINE_MATH_TEST(Asinh,
std::asinh, CallAsinh, -FLT_MAX, +FLT_MAX, 3,
std::asinh, CallAsinh, -DBL_MAX, +DBL_MAX, 3)
DEFINE_MATH_TEST(Atan,
std::atan, CallAtan, -FLT_MAX, +FLT_MAX, 3,
std::atan, CallAtan, -DBL_MAX, +DBL_MAX, 3)
// NEON has ULP 4 instead of 3
DEFINE_MATH_TEST(Atanh,
std::atanh, CallAtanh, -kNearOneF(), +kNearOneF(), 4,
std::atanh, CallAtanh, -kNearOneD(), +kNearOneD(), 3)
DEFINE_MATH_TEST(Cos,
std::cos, CallCos, -39000.0f, +39000.0f, 3,
std::cos, CallCos, -39000.0, +39000.0, Cos64ULP())
DEFINE_MATH_TEST(Exp,
std::exp, CallExp, -FLT_MAX, +104.0f, 1,
std::exp, CallExp, -DBL_MAX, +104.0, 1)
DEFINE_MATH_TEST(Exp2,
std::exp2, CallExp2, -FLT_MAX, +128.0f, 2,
std::exp2, CallExp2, -DBL_MAX, +128.0, 2)
DEFINE_MATH_TEST(Expm1,
std::expm1, CallExpm1, -FLT_MAX, +104.0f, 4,
std::expm1, CallExpm1, -DBL_MAX, +104.0, 4)
DEFINE_MATH_TEST(Log,
std::log, CallLog, +FLT_MIN, +FLT_MAX, 1,
std::log, CallLog, +DBL_MIN, +DBL_MAX, 1)
DEFINE_MATH_TEST(Log10,
std::log10, CallLog10, +FLT_MIN, +FLT_MAX, 2,
std::log10, CallLog10, +DBL_MIN, +DBL_MAX, 2)
DEFINE_MATH_TEST(Log1p,
std::log1p, CallLog1p, +0.0f, +1e37f, 3, // NEON is 3 instead of 2
std::log1p, CallLog1p, +0.0, +DBL_MAX, 2)
DEFINE_MATH_TEST(Log2,
std::log2, CallLog2, +FLT_MIN, +FLT_MAX, 2,
std::log2, CallLog2, +DBL_MIN, +DBL_MAX, 2)
DEFINE_MATH_TEST(Sin,
std::sin, CallSin, -39000.0f, +39000.0f, 3,
std::sin, CallSin, -39000.0, +39000.0, 4) // MSYS is 4 instead of 3
DEFINE_MATH_TEST(Sinh,
std::sinh, CallSinh, -80.0f, +80.0f, 4,
std::sinh, CallSinh, -709.0, +709.0, 4)
DEFINE_MATH_TEST(Tanh,
std::tanh, CallTanh, -FLT_MAX, +FLT_MAX, 4,
std::tanh, CallTanh, -DBL_MAX, +DBL_MAX, 4)
DEFINE_MATH_TEST(SinCosSin,
std::sin, SinCosSin, -39000.0f, +39000.0f, SinCosSin32ULP(),
std::sin, SinCosSin, -39000.0, +39000.0, 1)
DEFINE_MATH_TEST(SinCosCos,
std::cos, SinCosCos, -39000.0f, +39000.0f, SinCosCos32ULP(),
std::cos, SinCosCos, -39000.0, +39000.0, 1)
// clang-format on
template <typename T, class D>
void Atan2TestCases(T /*unused*/, D d, size_t& padded,
AlignedFreeUniquePtr<T[]>& out_y,
AlignedFreeUniquePtr<T[]>& out_x,
AlignedFreeUniquePtr<T[]>& out_expected) {
struct YX {
T y;
T x;
T expected;
};
const T pos = ConvertScalarTo<T>(1E5);
const T neg = ConvertScalarTo<T>(-1E7);
const T p0 = ConvertScalarTo<T>(0);
// -0 is not enough to get an actual negative zero.
const T n0 = ConvertScalarTo<T>(-0.0);
const T p1 = ConvertScalarTo<T>(1);
const T n1 = ConvertScalarTo<T>(-1);
const T p2 = ConvertScalarTo<T>(2);
const T n2 = ConvertScalarTo<T>(-2);
const T inf = GetLane(Inf(d));
const T nan = GetLane(NaN(d));
const T pi = ConvertScalarTo<T>(3.141592653589793238);
const YX test_cases[] = { // 45 degree steps:
{p0, p1, p0}, // E
{n1, p1, -pi / 4}, // SE
{n1, p0, -pi / 2}, // S
{n1, n1, -3 * pi / 4}, // SW
{p0, n1, pi}, // W
{p1, n1, 3 * pi / 4}, // NW
{p1, p0, pi / 2}, // N
{p1, p1, pi / 4}, // NE
// y = ±0, x < 0 or -0
{p0, n1, pi},
{n0, n2, -pi},
// y = ±0, x > 0 or +0
{p0, p2, p0},
{n0, p2, n0},
// y = ±∞, x finite
{inf, p2, pi / 2},
{-inf, p2, -pi / 2},
// y = ±∞, x = -∞
{inf, -inf, 3 * pi / 4},
{-inf, -inf, -3 * pi / 4},
// y = ±∞, x = +∞
{inf, inf, pi / 4},
{-inf, inf, -pi / 4},
// y < 0, x = ±0
{n2, p0, -pi / 2},
{n1, n0, -pi / 2},
// y > 0, x = ±0
{pos, p0, pi / 2},
{p2, n0, pi / 2},
// finite y > 0, x = -∞
{pos, -inf, pi},
// finite y < 0, x = -∞
{neg, -inf, -pi},
// finite y > 0, x = +∞
{pos, inf, p0},
// finite y < 0, x = +∞
{neg, inf, n0},
// y NaN xor x NaN
{nan, p0, nan},
{pos, nan, nan}};
const size_t kNumTestCases = sizeof(test_cases) / sizeof(test_cases[0]);
const size_t N = Lanes(d);
padded = RoundUpTo(kNumTestCases, N); // allow loading whole vectors
out_y = AllocateAligned<T>(padded);
out_x = AllocateAligned<T>(padded);
out_expected = AllocateAligned<T>(padded);
HWY_ASSERT(out_y && out_x && out_expected);
size_t i = 0;
for (; i < kNumTestCases; ++i) {
out_y[i] = test_cases[i].y;
out_x[i] = test_cases[i].x;
out_expected[i] = test_cases[i].expected;
}
for (; i < padded; ++i) {
out_y[i] = p0;
out_x[i] = p0;
out_expected[i] = p0;
}
}
struct TestAtan2 {
template <typename T, class D>
HWY_NOINLINE void operator()(T t, D d) {
const size_t N = Lanes(d);
size_t padded;
AlignedFreeUniquePtr<T[]> in_y, in_x, expected;
Atan2TestCases(t, d, padded, in_y, in_x, expected);
const Vec<D> tolerance = Set(d, ConvertScalarTo<T>(1E-5));
for (size_t i = 0; i < padded; ++i) {
const T actual = ConvertScalarTo<T>(atan2(in_y[i], in_x[i]));
// fprintf(stderr, "%zu: table %f atan2 %f\n", i, expected[i], actual);
HWY_ASSERT_EQ(expected[i], actual);
}
for (size_t i = 0; i < padded; i += N) {
const Vec<D> y = Load(d, &in_y[i]);
const Vec<D> x = Load(d, &in_x[i]);
#if HWY_ARCH_ARM_A64
// TODO(b/287462770): inline to work around incorrect SVE codegen
const Vec<D> actual = Atan2(d, y, x);
#else
const Vec<D> actual = CallAtan2(d, y, x);
#endif
const Vec<D> vexpected = Load(d, &expected[i]);
const Mask<D> exp_nan = IsNaN(vexpected);
const Mask<D> act_nan = IsNaN(actual);
HWY_ASSERT_MASK_EQ(d, exp_nan, act_nan);
// If not NaN, then compare with tolerance
const Mask<D> ge = Ge(actual, Sub(vexpected, tolerance));
const Mask<D> le = Le(actual, Add(vexpected, tolerance));
const Mask<D> ok = Or(act_nan, And(le, ge));
if (!AllTrue(d, ok)) {
const size_t mismatch =
static_cast<size_t>(FindKnownFirstTrue(d, Not(ok)));
fprintf(stderr, "Mismatch for i=%d expected %E actual %E\n",
static_cast<int>(i + mismatch), expected[i + mismatch],
ExtractLane(actual, mismatch));
HWY_ASSERT(0);
}
}
}
};
HWY_NOINLINE void TestAllAtan2() {
if (HWY_MATH_TEST_EXCESS_PRECISION) return;
ForFloat3264Types(ForPartialVectors<TestAtan2>());
}
template <typename T, class D>
void HypotTestCases(T /*unused*/, D d, size_t& padded,
AlignedFreeUniquePtr<T[]>& out_a,
AlignedFreeUniquePtr<T[]>& out_b,
AlignedFreeUniquePtr<T[]>& out_expected) {
using TU = MakeUnsigned<T>;
struct AB {
T a;
T b;
};
constexpr int kNumOfMantBits = MantissaBits<T>();
static_assert(kNumOfMantBits > 0, "kNumOfMantBits > 0 must be true");
// Ensures inputs are not constexpr.
const TU u1 = static_cast<TU>(hwy::Unpredictable1());
const double k1 = static_cast<double>(u1);
const T pos = ConvertScalarTo<T>(1E5 * k1);
const T neg = ConvertScalarTo<T>(-1E7 * k1);
const T p0 = ConvertScalarTo<T>(k1 - 1.0);
// -0 is not enough to get an actual negative zero.
const T n0 = ScalarCopySign<T>(p0, neg);
const T p1 = ConvertScalarTo<T>(k1);
const T n1 = ConvertScalarTo<T>(-k1);
const T p2 = ConvertScalarTo<T>(2 * k1);
const T n2 = ConvertScalarTo<T>(-2 * k1);
const T inf = BitCastScalar<T>(ExponentMask<T>() * u1);
const T neg_inf = ScalarCopySign(inf, n1);
const T nan = BitCastScalar<T>(
static_cast<TU>(ExponentMask<T>() | (u1 << (kNumOfMantBits - 1))));
const double max_as_f64 = ConvertScalarTo<double>(HighestValue<T>()) * k1;
const T max = ConvertScalarTo<T>(max_as_f64);
const T huge = ConvertScalarTo<T>(max_as_f64 * 0.25);
const T neg_huge = ScalarCopySign(huge, n1);
const T huge2 = ConvertScalarTo<T>(max_as_f64 * 0.039415044328304796);
const T large = ConvertScalarTo<T>(3.512227595593985E18 * k1);
const T neg_large = ScalarCopySign(large, n1);
const T large2 = ConvertScalarTo<T>(2.1190576943127544E16 * k1);
const T small = ConvertScalarTo<T>(1.067033284841808E-11 * k1);
const T neg_small = ScalarCopySign(small, n1);
const T small2 = ConvertScalarTo<T>(1.9401409532292856E-12 * k1);
const T tiny = BitCastScalar<T>(static_cast<TU>(u1 << kNumOfMantBits));
const T neg_tiny = ScalarCopySign(tiny, n1);
const T tiny2 =
ConvertScalarTo<T>(78.68466968859765 * ConvertScalarTo<double>(tiny));
const AB test_cases[] = {{p0, p0}, {p0, n0},
{n0, n0}, {p1, p1},
{p1, n1}, {n1, n1},
{p2, p2}, {p2, n2},
{p2, pos}, {p2, neg},
{n2, pos}, {n2, neg},
{n2, n2}, {p0, tiny},
{p0, neg_tiny}, {n0, tiny},
{n0, neg_tiny}, {p1, tiny},
{p1, neg_tiny}, {n1, tiny},
{n1, neg_tiny}, {tiny, p0},
{tiny2, p0}, {tiny, tiny2},
{neg_tiny, tiny2}, {huge, huge2},
{neg_huge, huge2}, {huge, p0},
{huge, tiny}, {huge2, tiny2},
{large, p0}, {large, large2},
{neg_large, p0}, {neg_large, large2},
{small, p0}, {small, small2},
{neg_small, p0}, {neg_small, small2},
{max, p0}, {max, huge},
{max, max}, {p0, inf},
{n0, inf}, {p1, inf},
{n1, inf}, {p2, inf},
{n2, inf}, {p0, neg_inf},
{n0, neg_inf}, {p1, neg_inf},
{n1, neg_inf}, {p2, neg_inf},
{n2, neg_inf}, {p0, nan},
{n0, nan}, {p1, nan},
{n1, nan}, {p2, nan},
{n2, nan}, {huge, inf},
{inf, nan}, {neg_inf, nan},
{nan, nan}};
const size_t kNumTestCases = sizeof(test_cases) / sizeof(test_cases[0]);
const size_t N = Lanes(d);
padded = RoundUpTo(kNumTestCases, N); // allow loading whole vectors
out_a = AllocateAligned<T>(padded);
out_b = AllocateAligned<T>(padded);
out_expected = AllocateAligned<T>(padded);
HWY_ASSERT(out_a && out_b && out_expected);
size_t i = 0;
for (; i < kNumTestCases; ++i) {
const T a =
test_cases[i].a * hwy::ConvertScalarTo<T>(hwy::Unpredictable1());
const T b = test_cases[i].b;
#if HWY_TARGET <= HWY_NEON_WITHOUT_AES && HWY_ARCH_ARM_V7
// Ignore test cases that have infinite or NaN inputs on Armv7 NEON
if (!ScalarIsFinite(a) || !ScalarIsFinite(b)) {
out_a[i] = p0;
out_b[i] = p0;
out_expected[i] = p0;
continue;
}
#endif
out_a[i] = a;
out_b[i] = b;
if (ScalarIsInf(a) || ScalarIsInf(b)) {
out_expected[i] = inf;
} else if (ScalarIsNaN(a) || ScalarIsNaN(b)) {
out_expected[i] = nan;
} else {
out_expected[i] = std::hypot(a, b);
}
}
for (; i < padded; ++i) {
out_a[i] = p0;
out_b[i] = p0;
out_expected[i] = p0;
}
}
struct TestHypot {
template <typename T, class D>
HWY_NOINLINE void operator()(T t, D d) {
if (HWY_MATH_TEST_EXCESS_PRECISION) {
return;
}
const size_t N = Lanes(d);
constexpr uint64_t kMaxErrorUlp = 4;
size_t padded;
AlignedFreeUniquePtr<T[]> in_a, in_b, expected;
HypotTestCases(t, d, padded, in_a, in_b, expected);
auto actual1_lanes = AllocateAligned<T>(N);
auto actual2_lanes = AllocateAligned<T>(N);
HWY_ASSERT(actual1_lanes && actual2_lanes);
uint64_t max_ulp = 0;
for (size_t i = 0; i < padded; i += N) {
const auto a = Load(d, in_a.get() + i);
const auto b = Load(d, in_b.get() + i);
#if HWY_ARCH_ARM_A64
// TODO(b/287462770): inline to work around incorrect SVE codegen
const auto actual1 = Hypot(d, a, b);
const auto actual2 = Hypot(d, b, a);
#else
const auto actual1 = CallHypot(d, a, b);
const auto actual2 = CallHypot(d, b, a);
#endif
Store(actual1, d, actual1_lanes.get());
Store(actual2, d, actual2_lanes.get());
for (size_t j = 0; j < N; j++) {
const T val_a = in_a[i + j];
const T val_b = in_b[i + j];
const T expected_val = expected[i + j];
const T actual1_val = actual1_lanes[j];
const T actual2_val = actual2_lanes[j];
const auto ulp1 =
hwy::detail::ComputeUlpDelta(actual1_val, expected_val);
if (ulp1 > kMaxErrorUlp) {
fprintf(stderr,
"%s: Hypot(%e, %e) lane %d expected %E actual %E ulp %g max "
"ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), val_a, val_b,
static_cast<int>(j), expected_val, actual1_val,
static_cast<double>(ulp1),
static_cast<uint32_t>(kMaxErrorUlp));
}
const auto ulp2 =
hwy::detail::ComputeUlpDelta(actual2_val, expected_val);
if (ulp2 > kMaxErrorUlp) {
fprintf(stderr,
"%s: Hypot(%e, %e) expected %E actual %E ulp %g max ulp %u\n",
hwy::TypeName(T(), Lanes(d)).c_str(), val_b, val_a,
expected_val, actual2_val, static_cast<double>(ulp2),
static_cast<uint32_t>(kMaxErrorUlp));
}
max_ulp = HWY_MAX(max_ulp, HWY_MAX(ulp1, ulp2));
}
}
if (max_ulp != 0) {
fprintf(stderr, "%s: Hypot max_ulp %g\n",
hwy::TypeName(T(), Lanes(d)).c_str(),
static_cast<double>(max_ulp));
HWY_ASSERT(max_ulp <= kMaxErrorUlp);
}
}
};
HWY_NOINLINE void TestAllHypot() {
if (HWY_MATH_TEST_EXCESS_PRECISION) return;
ForFloat3264Types(ForPartialVectors<TestHypot>());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(HwyMathTest);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAcos);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAcosh);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAsin);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAsinh);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAtan);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAtanh);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllCos);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllExp);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllExp2);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllExpm1);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllLog);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllLog10);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllLog1p);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllLog2);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllSin);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllSinh);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllTanh);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAtan2);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllSinCosSin);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllSinCosCos);
HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllHypot);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

449
deps/v8/third_party/highway/src/hwy/contrib/matvec/matvec-inl.h vendored Normal file
View File

@ -0,0 +1,449 @@
// Copyright 2023 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Include guard (still compiled once per target)
#if defined(HIGHWAY_HWY_CONTRIB_MATVEC_MATVEC_INL_H_) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_MATVEC_MATVEC_INL_H_
#undef HIGHWAY_HWY_CONTRIB_MATVEC_MATVEC_INL_H_
#else
#define HIGHWAY_HWY_CONTRIB_MATVEC_MATVEC_INL_H_
#endif
#include "hwy/cache_control.h"
#include "hwy/contrib/thread_pool/thread_pool.h"
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
template <typename TA, typename TB>
TA AddScalar(TA a, TB b) {
return ConvertScalarTo<TA>(ConvertScalarTo<float>(a) +
ConvertScalarTo<float>(b));
}
template <size_t kOuter, size_t kInner, typename T, bool kAdd>
HWY_NOINLINE void MatVecAddImpl(const T* HWY_RESTRICT mat,
const T* HWY_RESTRICT vec,
const T* HWY_RESTRICT add, T* HWY_RESTRICT out,
hwy::ThreadPool& pool) {
(void)add;
// Process multiple rows at a time so that we write multiples of a cache line
// to avoid false sharing (>= 64). 128 is better than 256. 512 has too little
// parallelization potential.
constexpr size_t kChunkSize = 64 / sizeof(T);
const uint64_t num_chunks = static_cast<uint64_t>(kOuter / kChunkSize);
const ScalableTag<T> d;
const size_t N = Lanes(d);
// Required for Stream loop, otherwise we might have partial vectors.
HWY_DASSERT(kChunkSize >= N);
pool.Run(0, num_chunks,
[&](const uint64_t chunk, size_t /*thread*/) HWY_ATTR {
// MSVC workaround: duplicate to ensure constexpr.
constexpr size_t kChunkSize = 64 / sizeof(T);
// Software write-combining to avoid cache pollution from out.
// Although `out` may be used later, keeping it out of the cache
// now and avoiding RFOs is a consistent 5% overall win.
HWY_ALIGN T buf[kChunkSize];
// Only handle entire chunks here because the Stream is not masked.
// Remaining rows are handled after the pool.Run.
const size_t begin = static_cast<size_t>(chunk * kChunkSize);
for (size_t idx_row = 0; idx_row < kChunkSize; ++idx_row) {
auto sum0 = Zero(d);
auto sum1 = Zero(d);
// 4x unrolling barely helps SKX but likely helps Arm V2.
auto sum2 = Zero(d);
auto sum3 = Zero(d);
const T* HWY_RESTRICT row = &mat[(begin + idx_row) * kInner];
size_t i = 0;
// No clear win from prefetching from the next 1..3 rows.
// clflush &row[i] is slow, clflushopt less so but not helping.
HWY_UNROLL(1)
for (; i + 4 * N <= kInner; i += 4 * N) {
const auto a0 = LoadU(d, row + i + 0 * N);
const auto v0 = LoadU(d, vec + i + 0 * N);
sum0 = MulAdd(a0, v0, sum0);
const auto a1 = LoadU(d, row + i + 1 * N);
const auto v1 = LoadU(d, vec + i + 1 * N);
sum1 = MulAdd(a1, v1, sum1);
const auto a2 = LoadU(d, row + i + 2 * N);
const auto v2 = LoadU(d, vec + i + 2 * N);
sum2 = MulAdd(a2, v2, sum2);
const auto a3 = LoadU(d, row + i + 3 * N);
const auto v3 = LoadU(d, vec + i + 3 * N);
sum3 = MulAdd(a3, v3, sum3);
}
// Last entire vectors
for (; i + N <= kInner; i += N) {
const auto a0 = LoadU(d, row + i);
const auto v0 = LoadU(d, vec + i);
sum0 = MulAdd(a0, v0, sum0);
}
const size_t remainder = kInner - i;
if (remainder != 0) {
const auto a0 = LoadN(d, row + i, remainder);
const auto v0 = LoadN(d, vec + i, remainder);
sum1 = MulAdd(a0, v0, sum1);
}
// Reduction tree: sum of all accumulators, then their lanes
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum1);
sum0 = Add(sum0, sum2);
buf[idx_row] = ReduceSum(d, sum0);
HWY_IF_CONSTEXPR(kAdd) {
buf[idx_row] = AddScalar(buf[idx_row], add[begin + idx_row]);
}
} // idx_row
HWY_UNROLL(4) // 1..4 iterations
for (size_t i = 0; i != kChunkSize; i += N) {
Stream(Load(d, buf + i), d, out + begin + i);
}
});
hwy::FlushStream();
// Handle remainder rows which are not a multiple of the chunk size.
for (size_t r = num_chunks * kChunkSize; r < kOuter; ++r) {
auto sum0 = Zero(d);
const T* HWY_RESTRICT row = &mat[r * kInner];
size_t i = 0;
HWY_UNROLL(1)
for (; i + N <= kInner; i += N) {
const auto a0 = LoadU(d, row + i);
const auto v0 = LoadU(d, vec + i);
sum0 = MulAdd(a0, v0, sum0);
}
const size_t remainder = kInner - i;
if (remainder != 0) {
const auto a0 = LoadN(d, row + i, remainder);
const auto v0 = LoadN(d, vec + i, remainder);
sum0 = MulAdd(a0, v0, sum0);
}
out[r] = ReduceSum(d, sum0);
HWY_IF_CONSTEXPR(kAdd) { out[r] = AddScalar(out[r], add[r]); }
} // r
}
// Multiplies mat with vec, adds add and puts the result in out.
//
// mat is a (kOuter, kInner)-shaped array, where element [i,j] is located at
// index i * kInner + j.
//
// vec is a (kInner,)-shaped array.
//
// add is a (kOuter,)-shaped array.
//
// out is a (kOuter,)-shaped array that will set to mat @ vec + add.
template <size_t kOuter, size_t kInner, typename T>
HWY_NOINLINE void MatVecAdd(const T* HWY_RESTRICT mat,
const T* HWY_RESTRICT vec,
const T* HWY_RESTRICT add, T* HWY_RESTRICT out,
hwy::ThreadPool& pool) {
MatVecAddImpl<kOuter, kInner, T, true>(mat, vec, add, out, pool);
}
// Multiplies mat with vec and puts the result in out.
//
// mat is a (kOuter, kInner)-shaped array, where element [i,j] is located at
// index i * kInner + j.
//
// vec is a (kInner,)-shaped array.
//
// out is a (kOuter,)-shaped array that will set to mat @ vec.
template <size_t kOuter, size_t kInner, typename T>
HWY_NOINLINE void MatVec(const T* HWY_RESTRICT mat, const T* HWY_RESTRICT vec,
T* HWY_RESTRICT out, hwy::ThreadPool& pool) {
MatVecAddImpl<kOuter, kInner, T, false>(mat, vec, /*add=*/nullptr, out, pool);
}
// This target lacks too many ops required in our implementation, use
// HWY_EMU128 instead.
#if HWY_TARGET != HWY_SCALAR
// Specialization for bf16 matrix, which halves memory bandwidth requirements.
template <size_t kOuter, size_t kInner, bool kAdd>
HWY_NOINLINE void MatVecAddImpl(const hwy::bfloat16_t* HWY_RESTRICT mat,
const float* HWY_RESTRICT vec,
const float* HWY_RESTRICT add,
float* HWY_RESTRICT out,
hwy::ThreadPool& pool) {
// Process multiple rows at a time so that we write multiples of a cache line
// to avoid false sharing (>= 64). 128 is better than 256. 512 has too little
// parallelization potential.
constexpr size_t kChunkSize = 64 / sizeof(float);
const uint64_t num_chunks = static_cast<uint64_t>(kOuter / kChunkSize);
const ScalableTag<float> d;
const Repartition<hwy::bfloat16_t, decltype(d)> d16;
// In the remainder loop, we only process a single f32 vector, so load half
// vectors of bf16 to avoid overrun.
const Half<decltype(d16)> d16h;
using V = Vec<decltype(d)>;
using V16 = Vec<decltype(d16)>;
using V16H = Vec<decltype(d16h)>;
const size_t N = Lanes(d);
// Required for Stream loop, otherwise we might have partial vectors.
HWY_DASSERT(kChunkSize >= N);
pool.Run(0, num_chunks,
[&](const uint64_t chunk, size_t /*thread*/) HWY_ATTR {
// MSVC workaround: duplicate to ensure constexpr.
constexpr size_t kChunkSize = 64 / sizeof(float);
// Software write-combining to avoid cache pollution from out.
// Although `out` may be used later, keeping it out of the cache
// now and avoiding RFOs is a consistent 5% overall win.
HWY_ALIGN float buf[kChunkSize];
// Only handle entire chunks here because the Stream is not masked.
// Remaining rows are handled after the pool.Run.
const size_t begin = static_cast<size_t>(chunk * kChunkSize);
for (size_t idx_row = 0; idx_row < kChunkSize; ++idx_row) {
auto sum0 = Zero(d);
auto sum1 = Zero(d);
// 4x unrolling barely helps SKX but likely helps Arm V2.
auto sum2 = Zero(d);
auto sum3 = Zero(d);
const hwy::bfloat16_t* HWY_RESTRICT row =
&mat[(begin + idx_row) * kInner];
size_t i = 0;
// No clear win from prefetching from the next 1..3 rows.
// clflush &row[i] is slow, clflushopt less so but not helping.
HWY_UNROLL(1)
for (; i + 4 * N <= kInner; i += 4 * N) {
const V16 b0 = LoadU(d16, row + i + 0 * N);
const V a0 = PromoteLowerTo(d, b0);
const V a1 = PromoteUpperTo(d, b0);
const V16 b1 = LoadU(d16, row + i + 2 * N);
const V a2 = PromoteLowerTo(d, b1);
const V a3 = PromoteUpperTo(d, b1);
const V v0 = LoadU(d, vec + i + 0 * N);
sum0 = MulAdd(a0, v0, sum0);
const V v1 = LoadU(d, vec + i + 1 * N);
sum1 = MulAdd(a1, v1, sum1);
const V v2 = LoadU(d, vec + i + 2 * N);
sum2 = MulAdd(a2, v2, sum2);
const V v3 = LoadU(d, vec + i + 3 * N);
sum3 = MulAdd(a3, v3, sum3);
}
// Last entire vectors
for (; i + N <= kInner; i += N) {
const V16H b0 = LoadU(d16h, row + i);
const V a0 = PromoteTo(d, b0);
const V v0 = LoadU(d, vec + i);
sum0 = MulAdd(a0, v0, sum0);
}
const size_t remainder = kInner - i;
if (remainder != 0) {
const V16H b0 = LoadN(d16h, row + i, remainder);
const V a0 = PromoteTo(d, b0);
const V v0 = LoadN(d, vec + i, remainder);
sum1 = MulAdd(a0, v0, sum1);
}
// Reduction tree: sum of all accumulators, then their lanes
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum1);
sum0 = Add(sum0, sum2);
buf[idx_row] = ReduceSum(d, sum0);
HWY_IF_CONSTEXPR(kAdd) {
buf[idx_row] = AddScalar(buf[idx_row], add[begin + idx_row]);
}
} // idx_row
HWY_UNROLL(4) // 1..4 iterations
for (size_t i = 0; i != kChunkSize; i += N) {
Stream(Load(d, buf + i), d, out + begin + i);
}
});
hwy::FlushStream();
// Handle remainder rows which are not a multiple of the chunk size.
for (size_t r = num_chunks * kChunkSize; r < kOuter; ++r) {
auto sum0 = Zero(d);
const hwy::bfloat16_t* HWY_RESTRICT row = &mat[r * kInner];
size_t i = 0;
HWY_UNROLL(1)
for (; i + N <= kInner; i += N) {
const V16H b0 = LoadU(d16h, row + i);
const V a0 = PromoteTo(d, b0);
const V v0 = LoadU(d, vec + i);
sum0 = MulAdd(a0, v0, sum0);
}
const size_t remainder = kInner - i;
if (remainder != 0) {
const V16H b0 = LoadN(d16h, row + i, remainder);
const V a0 = PromoteTo(d, b0);
const V v0 = LoadN(d, vec + i, remainder);
sum0 = MulAdd(a0, v0, sum0);
}
out[r] = ReduceSum(d, sum0);
HWY_IF_CONSTEXPR(kAdd) { out[r] = AddScalar(out[r], add[r]); }
} // r
}
template <size_t kOuter, size_t kInner>
HWY_NOINLINE void MatVecAdd(const hwy::bfloat16_t* HWY_RESTRICT mat,
const float* HWY_RESTRICT vec,
const float* HWY_RESTRICT add,
float* HWY_RESTRICT out, hwy::ThreadPool& pool) {
MatVecAddImpl<kOuter, kInner, true>(mat, vec, add, out, pool);
}
template <size_t kOuter, size_t kInner>
HWY_NOINLINE void MatVec(const hwy::bfloat16_t* HWY_RESTRICT mat,
const float* HWY_RESTRICT vec, float* HWY_RESTRICT out,
hwy::ThreadPool& pool) {
MatVecAddImpl<kOuter, kInner, false>(mat, vec, /*add=*/nullptr, out, pool);
}
// Both mat and vec are bf16.
template <size_t kOuter, size_t kInner, bool kAdd>
HWY_NOINLINE void MatVecAddImpl(const hwy::bfloat16_t* HWY_RESTRICT mat,
const hwy::bfloat16_t* HWY_RESTRICT vec,
const hwy::bfloat16_t* HWY_RESTRICT add,
float* HWY_RESTRICT out,
hwy::ThreadPool& pool) {
// Process multiple rows at a time so that we write multiples of a cache line
// to avoid false sharing (>= 64). 128 is better than 256. 512 has too little
// parallelization potential.
constexpr size_t kChunkSize = 64 / sizeof(bfloat16_t);
const uint64_t num_chunks = static_cast<uint64_t>(kOuter / kChunkSize);
const ScalableTag<float> df;
const Repartition<hwy::bfloat16_t, decltype(df)> d16;
using V16 = Vec<decltype(d16)>;
const size_t N = Lanes(d16);
// Required for Stream loop, otherwise we might have partial vectors.
HWY_DASSERT(kChunkSize >= N);
pool.Run(0, num_chunks,
[&](const uint64_t chunk, size_t /*thread*/) HWY_ATTR {
// MSVC workaround: duplicate to ensure constexpr.
constexpr size_t kChunkSize = 64 / sizeof(bfloat16_t);
// Software write-combining to avoid cache pollution from out.
// Although `out` may be used later, keeping it out of the cache
// now and avoiding RFOs is a consistent 5% overall win.
HWY_ALIGN float buf[kChunkSize];
// Only handle entire chunks here because the Stream is not masked.
// Remaining rows are handled after the pool.Run.
const size_t begin = static_cast<size_t>(chunk * kChunkSize);
for (size_t idx_row = 0; idx_row < kChunkSize; ++idx_row) {
auto sum0 = Zero(df);
auto sum1 = Zero(df);
auto sum2 = Zero(df);
auto sum3 = Zero(df);
const hwy::bfloat16_t* HWY_RESTRICT row =
&mat[(begin + idx_row) * kInner];
size_t i = 0;
// No clear win from prefetching from the next 1..3 rows.
// clflush &row[i] is slow, clflushopt less so but not helping.
HWY_UNROLL(1)
for (; i + 2 * N <= kInner; i += 2 * N) {
const V16 b0 = LoadU(d16, row + i + 0 * N);
const V16 b1 = LoadU(d16, row + i + 1 * N);
const V16 v0 = LoadU(d16, vec + i + 0 * N);
const V16 v1 = LoadU(d16, vec + i + 1 * N);
sum0 = ReorderWidenMulAccumulate(df, b0, v0, sum0, sum1);
sum2 = ReorderWidenMulAccumulate(df, b1, v1, sum2, sum3);
}
// Last entire vector
for (; i + N <= kInner; i += N) {
const V16 b0 = LoadU(d16, row + i);
const V16 v0 = LoadU(d16, vec + i);
sum0 = ReorderWidenMulAccumulate(df, b0, v0, sum0, sum1);
}
const size_t remainder = kInner - i;
if (remainder != 0) {
const V16 b0 = LoadN(d16, row + i, remainder);
const V16 v0 = LoadN(d16, vec + i, remainder);
sum2 = ReorderWidenMulAccumulate(df, b0, v0, sum2, sum3);
}
// Reduction tree: sum of all accumulators, then their lanes
sum0 = Add(sum0, sum1);
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum2);
buf[idx_row] = ReduceSum(df, sum0);
HWY_IF_CONSTEXPR(kAdd) {
buf[idx_row] = AddScalar(buf[idx_row], add[begin + idx_row]);
}
} // idx_row
HWY_UNROLL(4) // 1..4 iterations
for (size_t i = 0; i != kChunkSize; i += N / 2) {
Stream(Load(df, buf + i), df, out + begin + i);
}
});
hwy::FlushStream();
// Handle remainder rows which are not a multiple of the chunk size.
for (size_t r = num_chunks * kChunkSize; r < kOuter; ++r) {
auto sum0 = Zero(df);
auto sum1 = Zero(df);
const hwy::bfloat16_t* HWY_RESTRICT row = &mat[r * kInner];
size_t i = 0;
HWY_UNROLL(1)
for (; i + N <= kInner; i += N) {
const V16 b0 = LoadU(d16, row + i);
const V16 v0 = LoadU(d16, vec + i);
sum0 = ReorderWidenMulAccumulate(df, b0, v0, sum0, sum1);
}
const size_t remainder = kInner - i;
if (remainder != 0) {
const V16 b0 = LoadN(d16, row + i, remainder);
const V16 v0 = LoadN(d16, vec + i, remainder);
sum0 = ReorderWidenMulAccumulate(df, b0, v0, sum0, sum1);
}
out[r] = ReduceSum(df, Add(sum0, sum1));
HWY_IF_CONSTEXPR(kAdd) { out[r] = AddScalar(out[r], add[r]); }
} // r
}
template <size_t kOuter, size_t kInner>
HWY_NOINLINE void MatVecAdd(const hwy::bfloat16_t* HWY_RESTRICT mat,
const hwy::bfloat16_t* HWY_RESTRICT vec,
const hwy::bfloat16_t* HWY_RESTRICT add,
float* HWY_RESTRICT out, hwy::ThreadPool& pool) {
MatVecAddImpl<kOuter, kInner, true>(mat, vec, add, out, pool);
}
template <size_t kOuter, size_t kInner>
HWY_NOINLINE void MatVec(const hwy::bfloat16_t* HWY_RESTRICT mat,
const hwy::bfloat16_t* HWY_RESTRICT vec,
float* HWY_RESTRICT out, hwy::ThreadPool& pool) {
MatVecAddImpl<kOuter, kInner, false>(mat, vec, /*add=*/nullptr, out, pool);
}
#endif // HWY_TARGET != HWY_SCALAR
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_MATVEC_MATVEC_INL_H_

293
deps/v8/third_party/highway/src/hwy/contrib/matvec/matvec_test.cc vendored Normal file
View File

@ -0,0 +1,293 @@
// Copyright 2023 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/base.h"
// Reduce targets to avoid timeout under emulation.
#ifndef HWY_DISABLED_TARGETS
#define HWY_DISABLED_TARGETS \
(HWY_SVE2_128 | HWY_SVE2 | HWY_SVE_256 | HWY_NEON_WITHOUT_AES)
#endif
#include <stddef.h>
#include <stdint.h>
#include "hwy/aligned_allocator.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/matvec/matvec_test.cc" // NOLINT
#include "hwy/foreach_target.h" // IWYU pragma: keep
// Must come after foreach_target.h
#include "hwy/contrib/algo/transform-inl.h"
#include "hwy/contrib/matvec/matvec-inl.h"
#include "hwy/highway.h"
#include "hwy/contrib/thread_pool/thread_pool.h"
#include "hwy/contrib/thread_pool/topology.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
template <typename MatT, typename T>
HWY_NOINLINE void SimpleMatVecAdd(const MatT* HWY_RESTRICT mat,
const T* HWY_RESTRICT vec,
const T* HWY_RESTRICT add, size_t rows,
size_t cols, T* HWY_RESTRICT out,
ThreadPool& pool) {
if (add) {
pool.Run(0, rows, [=](uint64_t r, size_t /*thread*/) {
T dot = ConvertScalarTo<T>(0);
for (size_t c = 0; c < cols; c++) {
// For reasons unknown, fp16 += does not compile on clang (Arm).
dot = ConvertScalarTo<T>(dot + mat[r * cols + c] * vec[c]);
}
out[r] = dot + add[r];
});
} else {
pool.Run(0, rows, [=](uint64_t r, size_t /*thread*/) {
T dot = ConvertScalarTo<T>(0);
for (size_t c = 0; c < cols; c++) {
// For reasons unknown, fp16 += does not compile on clang (Arm).
dot = ConvertScalarTo<T>(dot + mat[r * cols + c] * vec[c]);
}
out[r] = dot;
});
}
}
HWY_MAYBE_UNUSED HWY_NOINLINE void SimpleMatVecAdd(
const hwy::bfloat16_t* HWY_RESTRICT mat, const float* HWY_RESTRICT vec,
const float* add, size_t rows, size_t cols, float* HWY_RESTRICT out,
ThreadPool& pool) {
if (add) {
pool.Run(0, rows, [=](uint64_t r, size_t /*thread*/) {
float dot = 0.0f;
for (size_t c = 0; c < cols; c++) {
dot += F32FromBF16(mat[r * cols + c]) * vec[c];
}
out[r] = dot + add[r];
});
} else {
pool.Run(0, rows, [=](uint64_t r, size_t /*thread*/) {
float dot = 0.0f;
for (size_t c = 0; c < cols; c++) {
dot += F32FromBF16(mat[r * cols + c]) * vec[c];
}
out[r] = dot;
});
}
}
HWY_MAYBE_UNUSED HWY_NOINLINE void SimpleMatVecAdd(
const hwy::bfloat16_t* HWY_RESTRICT mat,
const hwy::bfloat16_t* HWY_RESTRICT vec,
const hwy::bfloat16_t* HWY_RESTRICT add, size_t rows, size_t cols,
float* HWY_RESTRICT out, ThreadPool& pool) {
if (add) {
pool.Run(0, rows, [=](uint64_t r, size_t /*thread*/) {
float dot = 0.0f;
for (size_t c = 0; c < cols; c++) {
dot += F32FromBF16(mat[r * cols + c]) * F32FromBF16(vec[c]);
}
out[r] = dot + F32FromBF16(add[r]);
});
} else {
pool.Run(0, rows, [=](uint64_t r, size_t /*thread*/) {
float dot = 0.0f;
for (size_t c = 0; c < cols; c++) {
dot += F32FromBF16(mat[r * cols + c]) * F32FromBF16(vec[c]);
}
out[r] = dot;
});
}
}
struct GenerateMod {
template <class D, HWY_IF_NOT_BF16_D(D), HWY_IF_LANES_GT_D(D, 1)>
Vec<D> operator()(D d, Vec<RebindToUnsigned<D>> indices) const {
const RebindToUnsigned<D> du;
return Reverse2(d, ConvertTo(d, And(indices, Set(du, 0xF))));
}
template <class D, HWY_IF_NOT_BF16_D(D), HWY_IF_LANES_LE_D(D, 1)>
Vec<D> operator()(D d, Vec<RebindToUnsigned<D>> indices) const {
const RebindToUnsigned<D> du;
return ConvertTo(d, And(indices, Set(du, 0xF)));
}
// Requires >= 4 bf16 lanes for float32 Reverse2.
template <class D, HWY_IF_BF16_D(D), HWY_IF_LANES_GT_D(D, 2)>
Vec<D> operator()(D d, Vec<RebindToUnsigned<D>> indices) const {
const RebindToUnsigned<D> du;
const RebindToSigned<D> di;
const RepartitionToWide<decltype(di)> dw;
const RebindToFloat<decltype(dw)> df;
indices = And(indices, Set(du, 0xF));
const Vec<decltype(df)> i0 = ConvertTo(df, PromoteLowerTo(dw, indices));
const Vec<decltype(df)> i1 = ConvertTo(df, PromoteUpperTo(dw, indices));
return OrderedDemote2To(d, Reverse2(df, i0), Reverse2(df, i1));
}
// For one or two lanes, we don't have OrderedDemote2To nor Reverse2.
template <class D, HWY_IF_BF16_D(D), HWY_IF_LANES_LE_D(D, 2)>
Vec<D> operator()(D d, Vec<RebindToUnsigned<D>> indices) const {
const Rebind<float, D> df;
return DemoteTo(d, Set(df, GetLane(indices)));
}
};
// MatT is usually the same as T, but can also be bfloat16_t when T = float.
template <typename MatT, typename VecT>
class TestMatVecAdd {
template <size_t kRows, size_t kCols, class D, typename T = TFromD<D>>
HWY_NOINLINE void Test(D d, ThreadPool& pool) {
// This target lacks too many ops required in our implementation, use
// HWY_EMU128 instead.
#if HWY_TARGET != HWY_SCALAR
const Repartition<MatT, D> dm;
const Repartition<VecT, D> dv;
const size_t misalign = 3 * Lanes(d) / 5;
// Fill matrix and vector with small integer values
const size_t area = kRows * kCols;
AlignedFreeUniquePtr<MatT[]> storage_m =
AllocateAligned<MatT>(misalign + area);
AlignedFreeUniquePtr<VecT[]> storage_v =
AllocateAligned<VecT>(misalign + kCols);
AlignedFreeUniquePtr<VecT[]> storage_a =
AllocateAligned<VecT>(misalign + kRows);
HWY_ASSERT(storage_m && storage_v && storage_a);
MatT* pm = storage_m.get() + misalign;
VecT* pv = storage_v.get() + misalign;
VecT* av = storage_a.get() + misalign;
Generate(dm, pm, area, GenerateMod());
Generate(dv, pv, kCols, GenerateMod());
Generate(dv, av, kRows, GenerateMod());
AlignedFreeUniquePtr<T[]> expected_without_add = AllocateAligned<T>(kRows);
HWY_ASSERT(expected_without_add);
SimpleMatVecAdd(pm, pv, static_cast<VecT*>(nullptr), kRows, kCols,
expected_without_add.get(), pool);
AlignedFreeUniquePtr<T[]> actual_without_add = AllocateAligned<T>(kRows);
HWY_ASSERT(actual_without_add);
MatVec<kRows, kCols>(pm, pv, actual_without_add.get(), pool);
const auto assert_close = [&](const AlignedFreeUniquePtr<T[]>& expected,
const AlignedFreeUniquePtr<T[]>& actual,
bool with_add) {
for (size_t i = 0; i < kRows; ++i) {
const double exp = ConvertScalarTo<double>(expected[i]);
const double act = ConvertScalarTo<double>(actual[i]);
const double tolerance =
exp * 20 * 1.0 /
(1ULL << HWY_MIN(MantissaBits<MatT>(), MantissaBits<VecT>()));
if (!(exp - tolerance <= act && act <= exp + tolerance)) {
fprintf(stderr,
"%s/%s %zu x %zu, %s: mismatch at %zu %f %f; tol %f\n",
TypeName(MatT(), 1).c_str(), TypeName(VecT(), 1).c_str(),
kRows, kCols, (with_add ? "with add" : "without add"), i, exp,
act, tolerance);
HWY_ASSERT(0);
}
}
};
assert_close(expected_without_add, actual_without_add, /*with_add=*/false);
AlignedFreeUniquePtr<T[]> expected_with_add = AllocateAligned<T>(kRows);
SimpleMatVecAdd(pm, pv, av, kRows, kCols, expected_with_add.get(), pool);
AlignedFreeUniquePtr<T[]> actual_with_add = AllocateAligned<T>(kRows);
MatVecAdd<kRows, kCols>(pm, pv, av, actual_with_add.get(), pool);
assert_close(expected_with_add, actual_with_add, /*with_add=*/true);
#else
(void)d;
(void)pool;
#endif // HWY_TARGET != HWY_SCALAR
}
template <class D>
HWY_NOINLINE void CreatePoolAndTest(D d, size_t num_threads) {
// Threads might not work on WASM; run only on main thread.
if (HaveThreadingSupport()) num_threads = 0;
ThreadPool pool(HWY_MIN(num_threads, ThreadPool::MaxThreads()));
Test<AdjustedReps(192), AdjustedReps(256)>(d, pool);
// Fewer tests due to compiler OOM
#if !HWY_ARCH_RISCV
Test<40, AdjustedReps(512)>(d, pool);
Test<AdjustedReps(1024), 50>(d, pool);
// Too large for low-precision vectors/accumulators.
if (sizeof(TFromD<D>) != 2 && sizeof(VecT) != 2) {
Test<AdjustedReps(1536), AdjustedReps(1536)>(d, pool);
}
#endif // !HWY_ARCH_RISCV
}
public:
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
CreatePoolAndTest(d, 13);
// Fewer tests due to compiler OOM
#if !HWY_ARCH_RISCV
CreatePoolAndTest(d, 16);
#endif
}
};
void TestAllMatVecAdd() {
#if HWY_HAVE_FLOAT16
ForPartialVectors<TestMatVecAdd<float16_t, float16_t>>()(float16_t());
#endif
ForPartialVectors<TestMatVecAdd<float, float>>()(float());
#if HWY_HAVE_FLOAT64
ForPartialVectors<TestMatVecAdd<double, double>>()(double());
#endif
}
void TestAllMatVecBF16() {
ForGEVectors<32, TestMatVecAdd<bfloat16_t, float>>()(float());
}
void TestAllMatVecBF16Both() {
ForGEVectors<32, TestMatVecAdd<bfloat16_t, bfloat16_t>>()(float());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(MatVecTest);
HWY_EXPORT_AND_TEST_P(MatVecTest, TestAllMatVecAdd);
HWY_EXPORT_AND_TEST_P(MatVecTest, TestAllMatVecBF16);
HWY_EXPORT_AND_TEST_P(MatVecTest, TestAllMatVecBF16Both);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

384
deps/v8/third_party/highway/src/hwy/contrib/random/random-inl.h vendored Normal file
View File

@ -0,0 +1,384 @@
/*
* Original implementation written in 2019
* by David Blackman and Sebastiano Vigna (vigna@acm.org)
* Available at https://prng.di.unimi.it/ with creative commons license:
* To the extent possible under law, the author has dedicated all copyright
* and related and neighboring rights to this software to the public domain
* worldwide. This software is distributed without any warranty.
* See <http://creativecommons.org/publicdomain/zero/1.0/>.
*
* This implementation is a Vector port of the original implementation
* written by Marco Barbone (m.barbone19@imperial.ac.uk).
* I take no credit for the original implementation.
* The code is provided as is and the original license applies.
*/
#if defined(HIGHWAY_HWY_CONTRIB_RANDOM_RANDOM_H_) == \
defined(HWY_TARGET_TOGGLE) // NOLINT
#ifdef HIGHWAY_HWY_CONTRIB_RANDOM_RANDOM_H_
#undef HIGHWAY_HWY_CONTRIB_RANDOM_RANDOM_H_
#else
#define HIGHWAY_HWY_CONTRIB_RANDOM_RANDOM_H_
#endif
#include <array>
#include <cstdint>
#include <limits>
#include "hwy/aligned_allocator.h"
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE(); // required if not using HWY_ATTR
namespace hwy {
namespace HWY_NAMESPACE { // required: unique per target
namespace internal {
namespace {
#if HWY_HAVE_FLOAT64
// C++ < 17 does not support hexfloat
#if __cpp_hex_float > 201603L
constexpr double kMulConst = 0x1.0p-53;
#else
constexpr double kMulConst =
0.00000000000000011102230246251565404236316680908203125;
#endif // __cpp_hex_float
#endif // HWY_HAVE_FLOAT64
constexpr std::uint64_t kJump[] = {0x180ec6d33cfd0aba, 0xd5a61266f0c9392c,
0xa9582618e03fc9aa, 0x39abdc4529b1661c};
constexpr std::uint64_t kLongJump[] = {0x76e15d3efefdcbbf, 0xc5004e441c522fb3,
0x77710069854ee241, 0x39109bb02acbe635};
} // namespace
class SplitMix64 {
public:
constexpr explicit SplitMix64(const std::uint64_t state) noexcept
: state_(state) {}
HWY_CXX14_CONSTEXPR std::uint64_t operator()() {
std::uint64_t z = (state_ += 0x9e3779b97f4a7c15);
z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9;
z = (z ^ (z >> 27)) * 0x94d049bb133111eb;
return z ^ (z >> 31);
}
private:
std::uint64_t state_;
};
class Xoshiro {
public:
HWY_CXX14_CONSTEXPR explicit Xoshiro(const std::uint64_t seed) noexcept
: state_{} {
SplitMix64 splitMix64{seed};
for (auto &element : state_) {
element = splitMix64();
}
}
HWY_CXX14_CONSTEXPR explicit Xoshiro(const std::uint64_t seed,
const std::uint64_t thread_id) noexcept
: Xoshiro(seed) {
for (auto i = UINT64_C(0); i < thread_id; ++i) {
Jump();
}
}
HWY_CXX14_CONSTEXPR std::uint64_t operator()() noexcept { return Next(); }
#if HWY_HAVE_FLOAT64
HWY_CXX14_CONSTEXPR double Uniform() noexcept {
return static_cast<double>(Next() >> 11) * kMulConst;
}
#endif
HWY_CXX14_CONSTEXPR std::array<std::uint64_t, 4> GetState() const {
return {state_[0], state_[1], state_[2], state_[3]};
}
HWY_CXX17_CONSTEXPR void SetState(
std::array<std::uint64_t, 4> state) noexcept {
state_[0] = state[0];
state_[1] = state[1];
state_[2] = state[2];
state_[3] = state[3];
}
static constexpr std::uint64_t StateSize() noexcept { return 4; }
/* This is the jump function for the generator. It is equivalent to 2^128
* calls to next(); it can be used to generate 2^128 non-overlapping
* subsequences for parallel computations. */
HWY_CXX14_CONSTEXPR void Jump() noexcept { Jump(kJump); }
/* This is the long-jump function for the generator. It is equivalent to 2^192
* calls to next(); it can be used to generate 2^64 starting points, from each
* of which jump() will generate 2^64 non-overlapping subsequences for
* parallel distributed computations. */
HWY_CXX14_CONSTEXPR void LongJump() noexcept { Jump(kLongJump); }
private:
std::uint64_t state_[4];
static constexpr std::uint64_t Rotl(const std::uint64_t x, int k) noexcept {
return (x << k) | (x >> (64 - k));
}
HWY_CXX14_CONSTEXPR std::uint64_t Next() noexcept {
const std::uint64_t result = Rotl(state_[0] + state_[3], 23) + state_[0];
const std::uint64_t t = state_[1] << 17;
state_[2] ^= state_[0];
state_[3] ^= state_[1];
state_[1] ^= state_[2];
state_[0] ^= state_[3];
state_[2] ^= t;
state_[3] = Rotl(state_[3], 45);
return result;
}
HWY_CXX14_CONSTEXPR void Jump(const std::uint64_t (&jumpArray)[4]) noexcept {
std::uint64_t s0 = 0;
std::uint64_t s1 = 0;
std::uint64_t s2 = 0;
std::uint64_t s3 = 0;
for (const std::uint64_t i : jumpArray)
for (std::uint_fast8_t b = 0; b < 64; b++) {
if (i & std::uint64_t{1UL} << b) {
s0 ^= state_[0];
s1 ^= state_[1];
s2 ^= state_[2];
s3 ^= state_[3];
}
Next();
}
state_[0] = s0;
state_[1] = s1;
state_[2] = s2;
state_[3] = s3;
}
};
} // namespace internal
class VectorXoshiro {
private:
using VU64 = Vec<ScalableTag<std::uint64_t>>;
using StateType = AlignedNDArray<std::uint64_t, 2>;
#if HWY_HAVE_FLOAT64
using VF64 = Vec<ScalableTag<double>>;
#endif
public:
explicit VectorXoshiro(const std::uint64_t seed,
const std::uint64_t threadNumber = 0)
: state_{{internal::Xoshiro::StateSize(),
Lanes(ScalableTag<std::uint64_t>{})}},
streams{state_.shape().back()} {
internal::Xoshiro xoshiro{seed};
for (std::uint64_t i = 0; i < threadNumber; ++i) {
xoshiro.LongJump();
}
for (size_t i = 0UL; i < streams; ++i) {
const auto state = xoshiro.GetState();
for (size_t j = 0UL; j < internal::Xoshiro::StateSize(); ++j) {
state_[{j}][i] = state[j];
}
xoshiro.Jump();
}
}
HWY_INLINE VU64 operator()() noexcept { return Next(); }
AlignedVector<std::uint64_t> operator()(const std::size_t n) {
AlignedVector<std::uint64_t> result(n);
const ScalableTag<std::uint64_t> tag{};
auto s0 = Load(tag, state_[{0}].data());
auto s1 = Load(tag, state_[{1}].data());
auto s2 = Load(tag, state_[{2}].data());
auto s3 = Load(tag, state_[{3}].data());
for (std::uint64_t i = 0; i < n; i += Lanes(tag)) {
const auto next = Update(s0, s1, s2, s3);
Store(next, tag, result.data() + i);
}
Store(s0, tag, state_[{0}].data());
Store(s1, tag, state_[{1}].data());
Store(s2, tag, state_[{2}].data());
Store(s3, tag, state_[{3}].data());
return result;
}
template <std::uint64_t N>
std::array<std::uint64_t, N> operator()() noexcept {
alignas(HWY_ALIGNMENT) std::array<std::uint64_t, N> result;
const ScalableTag<std::uint64_t> tag{};
auto s0 = Load(tag, state_[{0}].data());
auto s1 = Load(tag, state_[{1}].data());
auto s2 = Load(tag, state_[{2}].data());
auto s3 = Load(tag, state_[{3}].data());
for (std::uint64_t i = 0; i < N; i += Lanes(tag)) {
const auto next = Update(s0, s1, s2, s3);
Store(next, tag, result.data() + i);
}
Store(s0, tag, state_[{0}].data());
Store(s1, tag, state_[{1}].data());
Store(s2, tag, state_[{2}].data());
Store(s3, tag, state_[{3}].data());
return result;
}
std::uint64_t StateSize() const noexcept {
return streams * internal::Xoshiro::StateSize();
}
const StateType &GetState() const { return state_; }
#if HWY_HAVE_FLOAT64
HWY_INLINE VF64 Uniform() noexcept {
const ScalableTag<double> real_tag{};
const auto MUL_VALUE = Set(real_tag, internal::kMulConst);
const auto bits = ShiftRight<11>(Next());
const auto real = ConvertTo(real_tag, bits);
return Mul(real, MUL_VALUE);
}
AlignedVector<double> Uniform(const std::size_t n) {
AlignedVector<double> result(n);
const ScalableTag<std::uint64_t> tag{};
const ScalableTag<double> real_tag{};
const auto MUL_VALUE = Set(real_tag, internal::kMulConst);
auto s0 = Load(tag, state_[{0}].data());
auto s1 = Load(tag, state_[{1}].data());
auto s2 = Load(tag, state_[{2}].data());
auto s3 = Load(tag, state_[{3}].data());
for (std::uint64_t i = 0; i < n; i += Lanes(real_tag)) {
const auto next = Update(s0, s1, s2, s3);
const auto bits = ShiftRight<11>(next);
const auto real = ConvertTo(real_tag, bits);
const auto uniform = Mul(real, MUL_VALUE);
Store(uniform, real_tag, result.data() + i);
}
Store(s0, tag, state_[{0}].data());
Store(s1, tag, state_[{1}].data());
Store(s2, tag, state_[{2}].data());
Store(s3, tag, state_[{3}].data());
return result;
}
template <std::uint64_t N>
std::array<double, N> Uniform() noexcept {
alignas(HWY_ALIGNMENT) std::array<double, N> result;
const ScalableTag<std::uint64_t> tag{};
const ScalableTag<double> real_tag{};
const auto MUL_VALUE = Set(real_tag, internal::kMulConst);
auto s0 = Load(tag, state_[{0}].data());
auto s1 = Load(tag, state_[{1}].data());
auto s2 = Load(tag, state_[{2}].data());
auto s3 = Load(tag, state_[{3}].data());
for (std::uint64_t i = 0; i < N; i += Lanes(real_tag)) {
const auto next = Update(s0, s1, s2, s3);
const auto bits = ShiftRight<11>(next);
const auto real = ConvertTo(real_tag, bits);
const auto uniform = Mul(real, MUL_VALUE);
Store(uniform, real_tag, result.data() + i);
}
Store(s0, tag, state_[{0}].data());
Store(s1, tag, state_[{1}].data());
Store(s2, tag, state_[{2}].data());
Store(s3, tag, state_[{3}].data());
return result;
}
#endif
private:
StateType state_;
const std::uint64_t streams;
HWY_INLINE static VU64 Update(VU64 &s0, VU64 &s1, VU64 &s2,
VU64 &s3) noexcept {
const auto result = Add(RotateRight<41>(Add(s0, s3)), s0);
const auto t = ShiftLeft<17>(s1);
s2 = Xor(s2, s0);
s3 = Xor(s3, s1);
s1 = Xor(s1, s2);
s0 = Xor(s0, s3);
s2 = Xor(s2, t);
s3 = RotateRight<19>(s3);
return result;
}
HWY_INLINE VU64 Next() noexcept {
const ScalableTag<std::uint64_t> tag{};
auto s0 = Load(tag, state_[{0}].data());
auto s1 = Load(tag, state_[{1}].data());
auto s2 = Load(tag, state_[{2}].data());
auto s3 = Load(tag, state_[{3}].data());
auto result = Update(s0, s1, s2, s3);
Store(s0, tag, state_[{0}].data());
Store(s1, tag, state_[{1}].data());
Store(s2, tag, state_[{2}].data());
Store(s3, tag, state_[{3}].data());
return result;
}
};
template <std::uint64_t size = 1024>
class CachedXoshiro {
public:
using result_type = std::uint64_t;
static constexpr result_type(min)() {
return (std::numeric_limits<result_type>::min)();
}
static constexpr result_type(max)() {
return (std::numeric_limits<result_type>::max)();
}
explicit CachedXoshiro(const result_type seed,
const result_type threadNumber = 0)
: generator_{seed, threadNumber},
cache_{generator_.operator()<size>()},
index_{0} {}
result_type operator()() noexcept {
if (HWY_UNLIKELY(index_ == size)) {
cache_ = std::move(generator_.operator()<size>());
index_ = 0;
}
return cache_[index_++];
}
private:
VectorXoshiro generator_;
alignas(HWY_ALIGNMENT) std::array<result_type, size> cache_;
std::size_t index_;
static_assert((size & (size - 1)) == 0 && size != 0,
"only power of 2 are supported");
};
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_MATH_MATH_INL_H_

318
deps/v8/third_party/highway/src/hwy/contrib/random/random_test.cc vendored Normal file
View File

@ -0,0 +1,318 @@
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cstdint>
#include <cstdio>
#include <ctime>
#include <iostream> // cerr
#include <random>
#include <vector>
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/random/random_test.cc" // NOLINT
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/random/random-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE { // required: unique per target
namespace {
constexpr std::uint64_t tests = 1UL << 10;
std::uint64_t GetSeed() { return static_cast<uint64_t>(std::time(nullptr)); }
void RngLoop(const std::uint64_t seed, std::uint64_t* HWY_RESTRICT result,
const size_t size) {
const ScalableTag<std::uint64_t> d;
VectorXoshiro generator{seed};
for (size_t i = 0; i < size; i += Lanes(d)) {
Store(generator(), d, result + i);
}
}
#if HWY_HAVE_FLOAT64
void UniformLoop(const std::uint64_t seed, double* HWY_RESTRICT result,
const size_t size) {
const ScalableTag<double> d;
VectorXoshiro generator{seed};
for (size_t i = 0; i < size; i += Lanes(d)) {
Store(generator.Uniform(), d, result + i);
}
}
#endif
void TestSeeding() {
const std::uint64_t seed = GetSeed();
VectorXoshiro generator{seed};
internal::Xoshiro reference{seed};
const auto& state = generator.GetState();
const ScalableTag<std::uint64_t> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 0UL; i < lanes; ++i) {
const auto& reference_state = reference.GetState();
for (std::size_t j = 0UL; j < reference_state.size(); ++j) {
if (state[{j}][i] != reference_state[j]) {
std::cerr << "SEED: " << seed << "\n";
std::cerr << "TEST SEEDING ERROR: ";
std::cerr << "state[" << j << "][" << i << "] -> " << state[{j}][i]
<< " != " << reference_state[j] << "\n";
HWY_ASSERT(0);
}
}
reference.Jump();
}
}
void TestMultiThreadSeeding() {
const std::uint64_t seed = GetSeed();
const std::uint64_t threadId = std::random_device()() % 1000;
VectorXoshiro generator{seed, threadId};
internal::Xoshiro reference{seed};
for (std::size_t i = 0UL; i < threadId; ++i) {
reference.LongJump();
}
const auto& state = generator.GetState();
const ScalableTag<std::uint64_t> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 0UL; i < lanes; ++i) {
const auto& reference_state = reference.GetState();
for (std::size_t j = 0UL; j < reference_state.size(); ++j) {
if (state[{j}][i] != reference_state[j]) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST SEEDING ERROR: ";
std::cerr << "state[" << j << "][" << i << "] -> " << state[{j}][i]
<< " != " << reference_state[j] << "\n";
HWY_ASSERT(0);
}
}
reference.Jump();
}
}
void TestRandomUint64() {
const std::uint64_t seed = GetSeed();
const auto result_array = hwy::MakeUniqueAlignedArray<std::uint64_t>(tests);
RngLoop(seed, result_array.get(), tests);
std::vector<internal::Xoshiro> reference;
reference.emplace_back(seed);
const ScalableTag<std::uint64_t> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 1UL; i < lanes; ++i) {
auto rng = reference.back();
rng.Jump();
reference.emplace_back(rng);
}
for (std::size_t i = 0UL; i < tests; i += lanes) {
for (std::size_t lane = 0UL; lane < lanes; ++lane) {
const std::uint64_t result = reference[lane]();
if (result_array[i + lane] != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST UINT64 GENERATOR ERROR: result_array[" << i + lane
<< "] -> " << result_array[i + lane] << " != " << result
<< std::endl;
HWY_ASSERT(0);
}
}
}
}
void TestUniformDist() {
#if HWY_HAVE_FLOAT64
const std::uint64_t seed = GetSeed();
const auto result_array = hwy::MakeUniqueAlignedArray<double>(tests);
UniformLoop(seed, result_array.get(), tests);
internal::Xoshiro reference{seed};
const ScalableTag<double> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 0UL; i < tests; i += lanes) {
const double result = reference.Uniform();
if (result_array[i] != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST UNIFORM GENERATOR ERROR: result_array[" << i << "] -> "
<< result_array[i] << " != " << result << std::endl;
HWY_ASSERT(0);
}
}
#endif // HWY_HAVE_FLOAT64
}
void TestNextNRandomUint64() {
const std::uint64_t seed = GetSeed();
VectorXoshiro generator{seed};
const auto result_array = generator.operator()(tests);
std::vector<internal::Xoshiro> reference;
reference.emplace_back(seed);
const ScalableTag<std::uint64_t> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 1UL; i < lanes; ++i) {
auto rng = reference.back();
rng.Jump();
reference.emplace_back(rng);
}
for (std::size_t i = 0UL; i < tests; i += lanes) {
for (std::size_t lane = 0UL; lane < lanes; ++lane) {
const std::uint64_t result = reference[lane]();
if (result_array[i + lane] != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST UINT64 GENERATOR ERROR: result_array[" << i + lane
<< "] -> " << result_array[i + lane] << " != " << result
<< std::endl;
HWY_ASSERT(0);
}
}
}
}
void TestNextFixedNRandomUint64() {
const std::uint64_t seed = GetSeed();
VectorXoshiro generator{seed};
const auto result_array = generator.operator()<tests>();
std::vector<internal::Xoshiro> reference;
reference.emplace_back(seed);
const ScalableTag<std::uint64_t> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 1UL; i < lanes; ++i) {
auto rng = reference.back();
rng.Jump();
reference.emplace_back(rng);
}
for (std::size_t i = 0UL; i < tests; i += lanes) {
for (std::size_t lane = 0UL; lane < lanes; ++lane) {
const std::uint64_t result = reference[lane]();
if (result_array[i + lane] != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST UINT64 GENERATOR ERROR: result_array[" << i + lane
<< "] -> " << result_array[i + lane] << " != " << result
<< std::endl;
HWY_ASSERT(0);
}
}
}
}
void TestNextNUniformDist() {
#if HWY_HAVE_FLOAT64
const std::uint64_t seed = GetSeed();
VectorXoshiro generator{seed};
const auto result_array = generator.Uniform(tests);
internal::Xoshiro reference{seed};
const ScalableTag<double> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 0UL; i < tests; i += lanes) {
const double result = reference.Uniform();
if (result_array[i] != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST UNIFORM GENERATOR ERROR: result_array[" << i << "] -> "
<< result_array[i] << " != " << result << std::endl;
HWY_ASSERT(0);
}
}
#endif // HWY_HAVE_FLOAT64
}
void TestNextFixedNUniformDist() {
#if HWY_HAVE_FLOAT64
const std::uint64_t seed = GetSeed();
VectorXoshiro generator{seed};
const auto result_array = generator.Uniform<tests>();
internal::Xoshiro reference{seed};
const ScalableTag<double> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 0UL; i < tests; i += lanes) {
const double result = reference.Uniform();
if (result_array[i] != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST UNIFORM GENERATOR ERROR: result_array[" << i << "] -> "
<< result_array[i] << " != " << result << std::endl;
HWY_ASSERT(0);
}
}
#endif // HWY_HAVE_FLOAT64
}
void TestCachedXorshiro() {
const std::uint64_t seed = GetSeed();
CachedXoshiro<> generator{seed};
std::vector<internal::Xoshiro> reference;
reference.emplace_back(seed);
const ScalableTag<std::uint64_t> d;
const std::size_t lanes = Lanes(d);
for (std::size_t i = 1UL; i < lanes; ++i) {
auto rng = reference.back();
rng.Jump();
reference.emplace_back(rng);
}
for (std::size_t i = 0UL; i < tests; i += lanes) {
for (std::size_t lane = 0UL; lane < lanes; ++lane) {
const std::uint64_t result = reference[lane]();
const std::uint64_t got = generator();
if (got != result) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST CachedXoshiro GENERATOR ERROR: result_array["
<< i + lane << "] -> " << got << " != " << result
<< std::endl;
HWY_ASSERT(0);
}
}
}
}
void TestUniformCachedXorshiro() {
#if HWY_HAVE_FLOAT64
const std::uint64_t seed = GetSeed();
CachedXoshiro<> generator{seed};
std::uniform_real_distribution<double> distribution{0., 1.};
for (std::size_t i = 0UL; i < tests; ++i) {
const double result = distribution(generator);
if (result < 0. || result >= 1.) {
std::cerr << "SEED: " << seed << std::endl;
std::cerr << "TEST CachedXoshiro GENERATOR ERROR: result_array[" << i
<< "] -> " << result << " not in interval [0, 1)" << std::endl;
HWY_ASSERT(0);
}
}
#endif // HWY_HAVE_FLOAT64
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE(); // required if not using HWY_ATTR
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(HwyRandomTest);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestSeeding);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestMultiThreadSeeding);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestRandomUint64);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestNextNRandomUint64);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestNextFixedNRandomUint64);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestCachedXorshiro);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestUniformDist);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestNextNUniformDist);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestNextFixedNUniformDist);
HWY_EXPORT_AND_TEST_P(HwyRandomTest, TestUniformCachedXorshiro);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

264
deps/v8/third_party/highway/src/hwy/contrib/sort/BUILD vendored Normal file
View File

@ -0,0 +1,264 @@
package(
default_applicable_licenses = ["//:license"],
default_visibility = ["//visibility:public"],
)
licenses(["notice"])
# Unused on Bazel builds, where this is not defined/known; Copybara replaces
# usages with an empty list.
COMPAT = [
"//buildenv/target:non_prod", # includes mobile/vendor.
]
cc_library(
name = "intel",
# hdrs = select({
# "//third_party/bazel_platforms/cpu:x86_64": [
# "avx512-16bit-common.h",
# "avx512-16bit-qsort.hpp",
# "avx512-32bit-qsort.hpp",
# "avx512-64bit-common.h",
# "avx512-64bit-qsort.hpp",
# "avx512-common-qsort.h",
# ],
# "//conditions:default": [],
# }),
compatible_with = [],
)
cc_library(
name = "vxsort",
srcs = [
# "vxsort/isa_detection.cpp",
# "vxsort/isa_detection_msvc.cpp",
# "vxsort/isa_detection_sane.cpp",
# "vxsort/machine_traits.avx2.cpp",
# "vxsort/smallsort/avx2_load_mask_tables.cpp",
# "vxsort/smallsort/bitonic_sort.AVX2.double.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX2.float.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX2.int32_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX2.int64_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX2.uint32_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX2.uint64_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX512.double.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX512.float.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX512.int32_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX512.int64_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX512.uint32_t.generated.cpp",
# "vxsort/smallsort/bitonic_sort.AVX512.uint64_t.generated.cpp",
# "vxsort/vxsort_stats.cpp",
],
hdrs = [
# "vxsort/alignment.h",
# "vxsort/defs.h",
# "vxsort/isa_detection.h",
# "vxsort/machine_traits.avx2.h",
# "vxsort/machine_traits.avx512.h",
# "vxsort/machine_traits.h",
# "vxsort/packer.h",
# "vxsort/smallsort/bitonic_sort.AVX2.double.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX2.float.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX2.int32_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX2.int64_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX2.uint32_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX2.uint64_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX512.double.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX512.float.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX512.int32_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX512.int64_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX512.uint32_t.generated.h",
# "vxsort/smallsort/bitonic_sort.AVX512.uint64_t.generated.h",
# "vxsort/smallsort/bitonic_sort.h",
# "vxsort/vxsort.h",
# "vxsort/vxsort_stats.h",
],
compatible_with = [],
textual_hdrs = [
# "vxsort/vxsort_targets_disable.h",
# "vxsort/vxsort_targets_enable_avx2.h",
# "vxsort/vxsort_targets_enable_avx512.h",
],
)
VQSORT_SRCS = [
"vqsort.cc",
# Split into separate files to reduce MSVC build time.
"vqsort_128a.cc",
"vqsort_128d.cc",
"vqsort_f16a.cc",
"vqsort_f16d.cc",
"vqsort_f32a.cc",
"vqsort_f32d.cc",
"vqsort_f64a.cc",
"vqsort_f64d.cc",
"vqsort_i16a.cc",
"vqsort_i16d.cc",
"vqsort_i32a.cc",
"vqsort_i32d.cc",
"vqsort_i64a.cc",
"vqsort_i64d.cc",
"vqsort_kv64a.cc",
"vqsort_kv64d.cc",
"vqsort_kv128a.cc",
"vqsort_kv128d.cc",
"vqsort_u16a.cc",
"vqsort_u16d.cc",
"vqsort_u32a.cc",
"vqsort_u32d.cc",
"vqsort_u64a.cc",
"vqsort_u64d.cc",
]
VQSORT_TEXTUAL_HDRS = [
"shared-inl.h",
"sorting_networks-inl.h",
"traits-inl.h",
"traits128-inl.h",
"vqsort-inl.h",
# Placeholder for internal instrumentation. Do not remove.
]
cc_library(
name = "vqsort",
srcs = VQSORT_SRCS,
hdrs = [
"order.h", # part of public interface, included by vqsort.h
"vqsort.h", # public interface
],
compatible_with = [],
local_defines = ["hwy_contrib_EXPORTS"],
textual_hdrs = VQSORT_TEXTUAL_HDRS,
deps = [
":intel", # required if HAVE_INTEL
":vxsort", # required if HAVE_VXSORT
"//:algo",
"//:hwy",
],
)
# -----------------------------------------------------------------------------
# Internal-only targets
# Same as vqsort, but add HWY_COMPILE_ALL_ATTAINABLE to ensure we cover all
# targets. Do not enable this in the main vqsort because it increases
# compile times.
cc_library(
name = "vqsort_for_test",
srcs = VQSORT_SRCS,
hdrs = [
"order.h", # part of public interface, included by vqsort.h
"vqsort.h", # public interface
],
compatible_with = [],
local_defines = [
"hwy_contrib_EXPORTS",
# Build for all targets because sort_test will dynamic-dispatch to all.
"HWY_COMPILE_ALL_ATTAINABLE",
],
textual_hdrs = VQSORT_TEXTUAL_HDRS,
deps = [
"//:algo",
"//:hwy",
],
)
cc_library(
name = "helpers",
testonly = 1,
textual_hdrs = [
"algo-inl.h",
"result-inl.h",
],
deps = [
":vqsort",
"//:nanobenchmark",
# Required for HAVE_PDQSORT, but that is unused and this is
# unavailable to Bazel builds, hence commented out.
# "//third_party/boost/allowed",
# Avoid ips4o and thus TBB to work around hwloc build failure.
],
)
cc_binary(
name = "print_network",
testonly = 1,
srcs = ["print_network.cc"],
deps = [
":helpers",
":vqsort",
"//:hwy",
],
)
TEST_MAIN = select({
"//:compiler_msvc": [],
"//conditions:default": ["@com_google_googletest//:gtest_main"],
})
cc_test(
name = "sort_unit_test",
size = "small",
srcs = ["sort_unit_test.cc"],
# Do not enable fully_static_link (pthread crash on bazel)
local_defines = ["HWY_IS_TEST"],
# for test_suite.
tags = ["hwy_ops_test"],
deps = [
":helpers",
":vqsort_for_test",
"//:hwy",
"//:hwy_test_util",
] + TEST_MAIN,
)
cc_test(
name = "sort_test",
size = "medium",
timeout = "long",
srcs = ["sort_test.cc"],
# Do not enable fully_static_link (pthread crash on bazel)
local_defines = ["HWY_IS_TEST"],
# for test_suite.
tags = ["hwy_ops_test"],
deps = [
":helpers",
":vqsort_for_test",
"//:hwy",
"//:hwy_test_util",
"//:thread_pool",
"//:topology",
] + TEST_MAIN,
)
cc_test(
name = "bench_sort",
size = "medium",
srcs = ["bench_sort.cc"],
# Do not enable fully_static_link (pthread crash on bazel)
local_defines = ["HWY_IS_TEST"],
# for test_suite.
tags = ["hwy_ops_test"],
deps = [
":helpers",
":vqsort",
"//:hwy",
"//:hwy_test_util",
"//:nanobenchmark",
] + TEST_MAIN,
)
cc_binary(
name = "bench_parallel",
testonly = 1,
srcs = ["bench_parallel.cc"],
# Do not enable fully_static_link (pthread crash on bazel)
local_defines = ["HWY_IS_TEST"],
deps = [
":helpers",
":vqsort",
"//:hwy",
"//:hwy_test_util",
"//:nanobenchmark",
] + TEST_MAIN,
)

361
deps/v8/third_party/highway/src/hwy/contrib/sort/README.md vendored Normal file
View File

@ -0,0 +1,361 @@
# Vectorized and performance-portable Quicksort
## Introduction
As of 2022-06-07 this sorts large arrays of built-in types about ten times as
fast as LLVM's `std::sort`. Note that other algorithms such as pdqsort can be
about twice as fast as LLVM's std::sort as of 2023-06.
See also our
[blog post](https://opensource.googleblog.com/2022/06/Vectorized%20and%20performance%20portable%20Quicksort.html)
and [paper](https://arxiv.org/abs/2205.05982).
## Instructions
Here are instructions for reproducing our results with cross-platform CMake,
Linux, or AWS (SVE, NEON).
### CMake, any platform
Please first ensure that Clang (tested with 13.0.1 and 15.0.6) is installed, and
if it is not the default compiler, point the CC and CXX environment variables to
it, e.g.
```
export CC=clang-15
export CXX=clang++-15
```
Then run the usual CMake workflow, also documented in the Highway README, e.g.:
```
mkdir -p build && cd build && cmake .. && make -j
taskset -c 2 tests/bench_sort
```
The optional `taskset -c 2` part reduces the variability of measurements by
preventing the OS from migrating the benchmark between cores.
### Linux
Please first ensure golang, and Clang (tested with 13.0.1) are installed via
your system's package manager.
```
go install github.com/bazelbuild/bazelisk@latest
git clone https://github.com/google/highway
cd highway
CC=clang CXX=clang++ ~/go/bin/bazelisk build -c opt hwy/contrib/sort:all
bazel-bin/hwy/contrib/sort/sort_test
bazel-bin/hwy/contrib/sort/bench_sort
```
### AWS Graviton3
Instance config: amazon linux 5.10 arm64, c7g.8xlarge (largest allowed config is
32 vCPU). Initial launch will fail. Wait a few minutes for an email saying the
config is verified, then re-launch. See IPv4 hostname in list of instances.
`ssh -i /path/key.pem ec2-user@hostname`
Note that the AWS CMake package is too old for llvm, so we build it first:
```
wget https://cmake.org/files/v3.23/cmake-3.23.2.tar.gz
tar -xvzf cmake-3.23.2.tar.gz && cd cmake-3.23.2/
./bootstrap -- -DCMAKE_USE_OPENSSL=OFF
make -j8 && sudo make install
cd ..
```
AWS clang is at version 11.1, which generates unnecessary `AND` instructions
which slow down the sort by 1.15x. We tested with clang trunk as of June 13
(which reports Git hash 8f6512fea000c3a0d394864bb94e524bee375069). To build:
```
git clone --depth 1 https://github.com/llvm/llvm-project.git
cd llvm-project
mkdir -p build && cd build
/usr/local/bin/cmake ../llvm -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" -DCMAKE_BUILD_TYPE=Release
make -j32 && sudo make install
```
```
sudo yum install go
go install github.com/bazelbuild/bazelisk@latest
git clone https://github.com/google/highway
cd highway
CC=/usr/local/bin/clang CXX=/usr/local/bin/clang++ ~/go/bin/bazelisk build -c opt --copt=-march=armv8.2-a+sve hwy/contrib/sort:all
bazel-bin/hwy/contrib/sort/sort_test
bazel-bin/hwy/contrib/sort/bench_sort
```
The above command line enables SVE, which is currently only available on
Graviton 3. You can also test NEON on the same processor, or other Arm CPUs, by
changing the `-march=` option to `--copt=-march=armv8.2-a+crypto`. Note that
such flags will be unnecessary once Clang supports `#pragma target` for NEON and
SVE intrinsics, as it does for x86.
## Results
`bench_sort` outputs the instruction set (AVX3 refers to AVX-512), the sort
algorithm (std for `std::sort`, vq for our vqsort), the type of keys being
sorted (f32 is float), the distribution of keys (uniform32 for uniform random
with range 0-2^32), the number of keys, then the throughput of sorted keys (i.e.
number of key bytes output per second).
Example excerpt from Xeon 6154 (Skylake-X) CPU clocked at 3 GHz:
```
[ RUN ] BenchSortGroup/BenchSort.BenchAllSort/AVX3
AVX3: std: f32: uniform32: 1.00E+06 54 MB/s ( 1 threads)
AVX3: vq: f32: uniform32: 1.00E+06 1143 MB/s ( 1 threads)
```
## Additional results
Thanks to Lukas Bergdoll, who did a thorough [performance analysis](https://github.com/Voultapher/sort-research-rs/blob/main/writeup/intel_avx512/text.md)
on various sort implementations. This helped us identify a performance bug,
caused by obtaining entropy from the OS on each call. This was fixed in #1334
and we look forward to the updated results.
### Optimizations for small arrays
Our initial focus was on large arrays. Since the VQSort paper was published,
we have improved its performance for small arrays:
- Previously, each call to VQSort obtained entropy from the OS. Unpredictable
seeding does help avoid worst-cases, and the cost is negligible when the
input size is at least 100K elements. However, the overhead is very costly
for arrays of just 100 or 1000, so we now obtain entropy only once per
thread and cache the seeds in TLS. This significantly improves the
performance on subsequent calls. Users can also explicitly initialize the
random generator.
- We also improved the efficiency of our sorting network for inputs shorter
than half its size. Our approach avoids costly transposes by interpreting
inputs as a 2D matrix. Previously, we always used 16 rows, which means only
a single vector lane is active for up to 16 elements. We have added 8x2 and
8x4 networks which use more lanes when available, and also 4x1 and 8x1
networks for very small inputs.
- Previously we also loaded (overlapping) full vectors, with the offsets
determined by the number of columns. Now we use the minimum vector size
sufficient for the number of columns, which enables higher IPC on Skylake
and reduces the cost of unaligned loads.
Unfortunately this decreases code reuse; VQSort now consists of about 1500
instructions (https://gcc.godbolt.org/z/ojYKfjPe6). The size of sorting
networks has nearly doubled to 10.8 KiB, 70% of the total. Although large,
this still fits comfortably within 32 KiB instruction caches, and possibly
even in micro-op caches (DSB, 1500-2300 micro-ops), especially given that
not all instructions are guaranteed to execute.
### Study of AVX-512 downclocking
We study whether AVX-512 downclocking affects performance. Using the GHz
reported by perf, we find an upper bound on the effects of downclocking, and
observe that its effect is negligible when compared to scalar code.
This issue has somehow attracted far more attention than seems warranted. An
attempt by Daniel Lemire to measure the
[worst-case](https://lemire.me/blog/2018/08/15/the-dangers-of-avx-512-throttling-a-3-impact/)
only saw a **3% decrease**, and Intel CPUs since Icelake, as well as AMD Zen4,
are much less impacted by throttling, if at all. By contrast, "Silver" and
"Bronze" Intel Xeons have more severe throttling and would require a large(r)
speedup from AVX-512 to outweigh the downclocking. However, these CPUs are
marketed towards "entry compute, network and storage" and "small business and
storage server solutions", and are thus less suitable for the high-performance
workloads we consider.
Our test workstation runs Linux (6.1.20-2rodete1-amd64) and has the same Xeon
Gold 6154 CPU used in our paper because its Skylake microarchitecture is the
most (potentially) affected. The compiler is a Clang similar to the LLVM trunk.
We added a new 'cold' benchmark that initializes random seeds, fills an array
with a constant except at one random index, calls VQSort, and then prints a
random element to ensure the computations are not elided. To run it, we build
bench_sort with `-DSORT_ONLY_COLD=1` and then invoke
`taskset -c 6 setarch -R x86_64 perf stat -r 15 -d bench_sort`. The taskset and
setarch serve to reduce variability by avoiding thread migration, and disabling
address space randomization. `-r 15` requests 15 runs so that perf can display
the variability of the measurements: < 1% for cycles, instructions, L1 dcache
loads; LLC miss variability is much higher (> 10%) presumably due to the
remaining background activity on this machine.
For our measurements, we use the GHz value reported by `perf`. This does not
include time spent in the kernel, and is thus noisy for short runtimes. Note
that running `perf` under `sudo` is not an option because it results in
"Workload failed: Cannot allocate memory". We see results between 2.6 - 2.9 GHz
when running AVX-512 code. This is relative to 3.0 GHz nominal; we disabled
Turbo Boost via MSR and ran `sudo cpupower frequency-set --governor performance`
to prevent unnecessary frequency reductions. To the best of our knowledge, the
remaining gap is explained by time spent in the kernel (in particular handling
page faults) and downclocking. Thus an *upper-bound* for the latter is
(3 - 2.9)/3 to (3 - 2.6)/3, or **1.03 - 1.13x**. Such a frequency reduction
would already be negligible compared to the 2-4x increase in work per cycle from
512-bit SIMD relative to 256 or 128-bit SIMD, which is typically less or not at
all affected by downclocking.
To further tighten this bound, we compare AVX-512 code vs. non-AVX-512 code, in
the form of `std::sort`. Ensuring the remainder of the binary does not use
AVX-512 is nontrivial. Library functions such as `memset` are known to use
AVX-512, and they would not show up in a disassembly of our binary. Neither
would they raise exceptions if run on a CPU lacking AVX-512 support, because
software typically verifies CPU support before running AVX-512. As a first step,
we take care to avoid calls to such library functions in our test, which is more
feasible with a self-contained small binary. In particular, array
zero-initialization typically compiles to `memset` (verified with clang-16), so
we manually initialize the array to the return value of an `Unpredictable1`
function whose implementation is not visible to the compiler. This indeed
compiles to a scalar loop. To further increase confidence that the binary lacks
AVX-512 instructions before VQSort, we replace the initialization loop with
AVX-512 stores. This indeed raises the measured throughput from a fairly
consistent 9 GB/s to 9-15 GB/s, likely because some of the AVX-512 startup now
occurs outside of our timings. We examine this effect in the next section, but
for now we can conclude that because adding AVX-512 makes a difference, the
binary was otherwise not using it. Now we can revert to scalar initialization
and compare the GHz reported for VQSort vs. `std::sort`. Across three runs, the
ranges are 2.8-2.9 and 2.8-2.8 GHz. Thus we conclude: if there is any
downclocking for a single core running AVX-512 on this Skylake-X CPU, the effect
is **under the noise floor of our measurement**, and certainly far below any
speedup one can reasonably predict from 512-bit SIMD. We expect this result to
generalize to AMD Zen4 and any Gold/Platinum Intel Xeon.
### Study of AVX-512 startup overhead
In the previous section, we saw that downclocking is negligible on our system,
but there is a noticeable benefit to warming up AVX-512 before the sort. To
understand why, we refer to Travis Downs' excellent
[measurements](https://travisdowns.github.io/blog/2020/01/17/avxfreq1.html#summary)
of how Skylake reacts to an AVX-512 instruction: 8-20 us of reduced instruction
throughput, an additional potential halt of 10 us, and then downclocking.
Note that downclocking is negligible on a single core per the previous section.
We choose the array length of 10K unsigned 64-bit keys such that VQSort
completes in 7-10 us. Thus in this benchmark, VQSort (almost) finishes before
AVX-512 is fully warmed up, and the speedup is reduced because the startup costs
are amortized over relatively little data. Across five series of 15 runs, the
average of average throughputs is 9.3 GB/s, implying a runtime of 8.6 us
including startup costs.
Note that the two-valued, almost all-equal input distribution is quite skewed.
The above throughput does not reflect the performance attainable on other
distributions, especially uniform random. However, this choice is deliberate
because Quicksort can terminate early if all values in a partition are equal.
When measuring such a 'best-case' input, we are more likely to observe the cost
of startup overhead in surrounding code. Otherwise, this overhead might be
hidden by the increase in sorting time.
Now let us compare this throughput to the previously mentioned measurement with
AVX-512 warmed up (via slow scatter instructions so that initialization takes
about 100 us, well in excess of the warmup period): 15.2 GB/s, or 5.3 us without
startup cost. It appears the 10 us halt is not happening, possibly because we do
not use SIMD floating-point nor multiplication instructions. Thus we only
experience reduced instruction throughput and/or increased latency. The ratio
between cold and warmed-up time is only 1.6, which is plausible if the Skylake
throttling is actually rounding latencies up to a multiple of four cycles, as
Downs speculates. Indeed a large fraction of the SIMD instructions especially in
the VQSort base case are cross-lane or 64-bit min/max operations with latencies
of 3 cycles on Skylake, so their slowdown might only be 1.3x. The measured 1.6x
could plausibly derive from 7/8 of 1.3x and 1/8 of 4x for single-cycle latency
instructions.
Assuming this understanding of AVX-512 startup cost is valid, how long does it
remain active before the CPU reverts to the previous settings? The CPU cannot
know what future instructions are coming, and to prevent unnecessary
transitions, it has a hysteresis (delay after the last AVX-512 instruction
before shutting down) which Downs measures as 680 us. Thus our benchmark
subsequently sleeps for 100 ms to ensure the next run of the binary sees the
original CPU state. Indeed we find for the five series that the slopes of the
lines of best fit are negative in one case, positive in two, and flat in two,
indicating there is no consistent pattern of benefit for earlier or later runs.
What are the implications for users of VQSort? If the surrounding code executes
an AVX-512 instruction at least every 500 us, then AVX-512 remains active and
**any call to VQSort will benefit from it, no matter how small the input**.
This is a reasonable expectation for modern systems whose designers were aware
of data-oriented programming principles, because many (though not all) domains
and operations can benefit from SIMD. By contrast, consider the case of dropping
VQSort into an existing legacy system that does not yet use SIMD. In the case of
10K input sizes, we still observe a 2.3x speedup vs. `std::sort`. However, the
following code may have to deal with throttling for the remainder of the 20 us
startup period. With VQSort we have 8.6 us runtime plus up to 11.4 us throttled
code (potentially running at quarter speed) plus the remaining 3/4 of 11.4 for a
total of 28.6. With `std::sort` we have 19.5 us runtime plus 20 us of normal
subsequent code, or 39.5 us. Thus the overall speedup for the 20 us region plus
VQSort **shrinks to 1.4x**, and it is possible to imagine an actual slowdown for
sufficiently small inputs, when factoring in the throttling of subsequent code.
This unfortunate 'beggar thy neighbor' effect cannot be solved at the level of
individual building blocks such as a sort, and must instead be addressed at the
system level. For example:
- vectorizing more and more parts of the code to amortize startup cost;
- relying on newer CPUs than Skylake (launched 2015!) which have little or no
AVX-512 startup overhead, such as Intel Icelake (2021) or AMD Zen4 (2022);
- ensuring sorts (or anything else using AVX-512) process at least 100 KiB
of data, such that the expected speedup outweighs any startup cost.
Any of these solutions are sufficient to render AVX-512 startup overhead a
non-issue.
### Comparison with Intel's x86-simd-sort and vxsort
Our May 2022 paper compared performance with `ips4o` and `std::sort`. We now add
results for Intel's [x86-simd-sort](https://github.com/intel/x86-simd-sort),
released as open source around October 2022, and
[vxsort](https://github.com/damageboy/vxsort-cpp/tree/master). We find that
VQSort is generally about 1.4 times as fast as either, and in a few cases equal
or up to 2% slower.
Note that vxsort was open-sourced around May 2020; we were unaware of it at the
time of writing because it had been published in the form of a blog series. We
imported both from Github on 2023-06-06 at about 10:15 UTC. Both are integrated
into our bench_sort, running on the same Linux OS and Xeon 6154 CPU mentioned
above. We use uniform random inputs, because vxsort and x86-simd-sort appear to
have much less robust handling of skewed input distributions. They choose the
pivot as the median of three keys, or of 64 bytes, respectively. By contrast,
VQSort draws a 384 byte sample and analyzes their distribution, which improves
load balance and prevents recursing into all-equal partitions. Lacking this, the
other algorithms are more vulnerable to worst-cases. Choosing uniform random
thus prevents disadvantaging the other algorithms.
We sample performance across a range of input sizes and types:
- To isolate the performance of the sorting networks used by all three
algorithms, we start with powers of two up to 128. VQSort is generally the
fastest for 64-bit keys with the following exceptions: tie with vxsort at
N=2 (537 MB/s), slower than vxsort at N=16 (2114 vs. 2147), tie with
x86-simd-sort at N=32 (2643 MB/s). Note that VQSort is about 1.6 times as
fast as both others for N=128; possibly because its 2D structure enables
larger networks.
- The `kPow10` mode in bench_sort measures power of ten input sizes between
10 and 100K. Note that this covers non-power of two sizes, as well as the
crossover point between sorting networks and Quicksort recursion. The
speedups of VQSort relative to x86-simd-sort range from 1.33 to 1.81
(32-bit keys), and 1.25 to 1.68 (64-bit keys), with geomeans of 1.48 and
1.44. The speedups of VQSort relative to vxsort range from 1.08 to 2.10
(32-bit keys), and 1.00 to 1.47 (64-bit keys), with geomeans of 1.41 and
1.20. Note that vxsort matches VQSort at 10 64-bit elements; in all other
cases, VQSort is strictly faster.
- Finally, we study the effect of key type at a fixed input size of 10K
elements. x86-simd-sort requires AVX512-VBMI2 for int16, which our CPU does
not support. Also, both other algorithms do not support 128-bit keys, thus
we only consider 32/64-bit integer and float types. The results in MB/s are:
|Type|VQSort|x86-simd-sort|vxsort|
|---|---|---|---|
|f32|**1551**| 798| 823|
|f64|**1773**|1147| 745|
|i32|**1509**|1042| 968|
|i64|**1365**|1043|1145|
VQSort is the fastest for each type, in some cases even about twice as fast.
Interestingly, vxsort performs at its best on i64, whereas the others are at
their best for f64. A potential explanation is that this CPU can execute two
f64 min/max per cycle, but only one i64.
In conclusion, VQSort is generally more efficient than vxsort and x86-simd-sort
across a range of input sizes and types. Occasionally, it is up to 2% slower,
but the geomean of its speedup (32-bit keys and power-of-ten sizes) vs. vxsort
is **1.41**, and **1.48** vs. x86-simd-sort.

621
deps/v8/third_party/highway/src/hwy/contrib/sort/algo-inl.h vendored Normal file
View File

@ -0,0 +1,621 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Normal include guard for target-independent parts
#ifndef HIGHWAY_HWY_CONTRIB_SORT_ALGO_INL_H_
#define HIGHWAY_HWY_CONTRIB_SORT_ALGO_INL_H_
#include <stddef.h>
#include <stdint.h>
#include <algorithm> // std::sort, std::min, std::max
#include <functional> // std::less, std::greater
#include <vector>
#include "hwy/contrib/sort/vqsort.h"
#include "hwy/highway.h"
#include "hwy/print.h"
// Third-party algorithms
#define HAVE_AVX2SORT 0
#define HAVE_IPS4O 0
// When enabling, consider changing max_threads (required for Table 1a)
#define HAVE_PARALLEL_IPS4O (HAVE_IPS4O && 1)
#define HAVE_PDQSORT 0
#define HAVE_SORT512 0
#define HAVE_VXSORT 0
#if HWY_ARCH_X86
#define HAVE_INTEL 0
#else
#define HAVE_INTEL 0
#endif
#if HAVE_PARALLEL_IPS4O
#include <thread> // NOLINT
#endif
#if HAVE_AVX2SORT
HWY_PUSH_ATTRIBUTES("avx2,avx")
#include "avx2sort.h" //NOLINT
HWY_POP_ATTRIBUTES
#endif
#if HAVE_IPS4O || HAVE_PARALLEL_IPS4O
#include "third_party/ips4o/include/ips4o.hpp"
#include "third_party/ips4o/include/ips4o/thread_pool.hpp"
#endif
#if HAVE_PDQSORT
#include "third_party/boost/allowed/sort/sort.hpp"
#endif
#if HAVE_SORT512
#include "sort512.h" //NOLINT
#endif
// vxsort is difficult to compile for multiple targets because it also uses
// .cpp files, and we'd also have to #undef its include guards. Instead, compile
// only for AVX2 or AVX3 depending on this macro.
#define VXSORT_AVX3 1
#if HAVE_VXSORT
// inlined from vxsort_targets_enable_avx512 (must close before end of header)
#ifdef __GNUC__
#ifdef __clang__
#if VXSORT_AVX3
#pragma clang attribute push(__attribute__((target("avx512f,avx512dq"))), \
apply_to = any(function))
#else
#pragma clang attribute push(__attribute__((target("avx2"))), \
apply_to = any(function))
#endif // VXSORT_AVX3
#else
#pragma GCC push_options
#if VXSORT_AVX3
#pragma GCC target("avx512f,avx512dq")
#else
#pragma GCC target("avx2")
#endif // VXSORT_AVX3
#endif
#endif
#if VXSORT_AVX3
#include "vxsort/machine_traits.avx512.h"
#else
#include "vxsort/machine_traits.avx2.h"
#endif // VXSORT_AVX3
#include "vxsort/vxsort.h"
#ifdef __GNUC__
#ifdef __clang__
#pragma clang attribute pop
#else
#pragma GCC pop_options
#endif
#endif
#endif // HAVE_VXSORT
namespace hwy {
enum class Dist { kUniform8, kUniform16, kUniform32 };
static inline std::vector<Dist> AllDist() {
// Also include lower-entropy distributions to test MaybePartitionTwoValue.
return {Dist::kUniform8, /*Dist::kUniform16,*/ Dist::kUniform32};
}
static inline const char* DistName(Dist dist) {
switch (dist) {
case Dist::kUniform8:
return "uniform8";
case Dist::kUniform16:
return "uniform16";
case Dist::kUniform32:
return "uniform32";
}
return "unreachable";
}
template <typename T>
class InputStats {
public:
void Notify(T value) {
min_ = std::min(min_, value);
max_ = std::max(max_, value);
// Converting to integer would truncate floats, multiplying to save digits
// risks overflow especially when casting, so instead take the sum of the
// bit representations as the checksum.
uint64_t bits = 0;
static_assert(sizeof(T) <= 8, "Expected a built-in type");
CopyBytes<sizeof(T)>(&value, &bits); // not same size
sum_ += bits;
count_ += 1;
}
bool operator==(const InputStats& other) const {
char type_name[100];
detail::TypeName(hwy::detail::MakeTypeInfo<T>(), 1, type_name);
if (count_ != other.count_) {
HWY_ABORT("Sort %s: count %d vs %d\n", type_name,
static_cast<int>(count_), static_cast<int>(other.count_));
}
if (min_ != other.min_ || max_ != other.max_) {
HWY_ABORT("Sort %s: minmax %f/%f vs %f/%f\n", type_name,
static_cast<double>(min_), static_cast<double>(max_),
static_cast<double>(other.min_),
static_cast<double>(other.max_));
}
// Sum helps detect duplicated/lost values
if (sum_ != other.sum_) {
HWY_ABORT("Sort %s: Sum mismatch %g %g; min %g max %g\n", type_name,
static_cast<double>(sum_), static_cast<double>(other.sum_),
static_cast<double>(min_), static_cast<double>(max_));
}
return true;
}
private:
T min_ = hwy::HighestValue<T>();
T max_ = hwy::LowestValue<T>();
uint64_t sum_ = 0;
size_t count_ = 0;
};
enum class Algo {
#if HAVE_INTEL
kIntel,
#endif
#if HAVE_AVX2SORT
kSEA,
#endif
#if HAVE_IPS4O
kIPS4O,
#endif
#if HAVE_PARALLEL_IPS4O
kParallelIPS4O,
#endif
#if HAVE_PDQSORT
kPDQ,
#endif
#if HAVE_SORT512
kSort512,
#endif
#if HAVE_VXSORT
kVXSort,
#endif
kStdSort,
kStdSelect,
kStdPartialSort,
kVQSort,
kVQPartialSort,
kVQSelect,
kHeapSort,
kHeapPartialSort,
kHeapSelect,
};
static inline bool IsVQ(Algo algo) {
switch (algo) {
case Algo::kVQSort:
case Algo::kVQPartialSort:
case Algo::kVQSelect:
return true;
default:
return false;
}
}
static inline bool IsSelect(Algo algo) {
switch (algo) {
case Algo::kStdSelect:
case Algo::kVQSelect:
case Algo::kHeapSelect:
return true;
default:
return false;
}
}
static inline bool IsPartialSort(Algo algo) {
switch (algo) {
case Algo::kStdPartialSort:
case Algo::kVQPartialSort:
case Algo::kHeapPartialSort:
return true;
default:
return false;
}
}
static inline Algo ReferenceAlgoFor(Algo algo) {
if (IsPartialSort(algo)) return Algo::kStdPartialSort;
#if HAVE_PDQSORT
return Algo::kPDQ;
#else
return Algo::kStdSort;
#endif
}
static inline const char* AlgoName(Algo algo) {
switch (algo) {
#if HAVE_INTEL
case Algo::kIntel:
return "intel";
#endif
#if HAVE_AVX2SORT
case Algo::kSEA:
return "sea";
#endif
#if HAVE_IPS4O
case Algo::kIPS4O:
return "ips4o";
#endif
#if HAVE_PARALLEL_IPS4O
case Algo::kParallelIPS4O:
return "par_ips4o";
#endif
#if HAVE_PDQSORT
case Algo::kPDQ:
return "pdq";
#endif
#if HAVE_SORT512
case Algo::kSort512:
return "sort512";
#endif
#if HAVE_VXSORT
case Algo::kVXSort:
return "vxsort";
#endif
case Algo::kStdSort:
return "std";
case Algo::kStdPartialSort:
return "std_partial";
case Algo::kStdSelect:
return "std_select";
case Algo::kVQSort:
return "vq";
case Algo::kVQPartialSort:
return "vq_partial";
case Algo::kVQSelect:
return "vq_select";
case Algo::kHeapSort:
return "heap";
case Algo::kHeapPartialSort:
return "heap_partial";
case Algo::kHeapSelect:
return "heap_select";
}
return "unreachable";
}
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_SORT_ALGO_INL_H_
// Per-target
// clang-format off
#if defined(HIGHWAY_HWY_CONTRIB_SORT_ALGO_TOGGLE) == defined(HWY_TARGET_TOGGLE) // NOLINT
#ifdef HIGHWAY_HWY_CONTRIB_SORT_ALGO_TOGGLE
#undef HIGHWAY_HWY_CONTRIB_SORT_ALGO_TOGGLE
#else
#define HIGHWAY_HWY_CONTRIB_SORT_ALGO_TOGGLE
#endif
// clang-format on
#include "hwy/aligned_allocator.h"
#include "hwy/contrib/sort/traits-inl.h"
#include "hwy/contrib/sort/traits128-inl.h"
#include "hwy/contrib/sort/vqsort-inl.h" // HeapSort
HWY_BEFORE_NAMESPACE();
// Requires target pragma set by HWY_BEFORE_NAMESPACE
#if HAVE_INTEL && HWY_TARGET <= HWY_AVX3
// #include "avx512-16bit-qsort.hpp" // requires AVX512-VBMI2
#include "avx512-32bit-qsort.hpp"
#include "avx512-64bit-qsort.hpp"
#endif
namespace hwy {
namespace HWY_NAMESPACE {
#if HAVE_INTEL || HAVE_VXSORT // only supports ascending order
template <typename T>
using OtherOrder = detail::OrderAscending<T>;
#else
template <typename T>
using OtherOrder = detail::OrderDescending<T>;
#endif
class Xorshift128Plus {
static HWY_INLINE uint64_t SplitMix64(uint64_t z) {
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
return z ^ (z >> 31);
}
public:
// Generates two vectors of 64-bit seeds via SplitMix64 and stores into
// `seeds`. Generating these afresh in each ChoosePivot is too expensive.
template <class DU64>
static void GenerateSeeds(DU64 du64, TFromD<DU64>* HWY_RESTRICT seeds) {
seeds[0] = SplitMix64(0x9E3779B97F4A7C15ull);
for (size_t i = 1; i < 2 * Lanes(du64); ++i) {
seeds[i] = SplitMix64(seeds[i - 1]);
}
}
// Need to pass in the state because vector cannot be class members.
template <class VU64>
static VU64 RandomBits(VU64& state0, VU64& state1) {
VU64 s1 = state0;
VU64 s0 = state1;
const VU64 bits = Add(s1, s0);
state0 = s0;
s1 = Xor(s1, ShiftLeft<23>(s1));
state1 = Xor(s1, Xor(s0, Xor(ShiftRight<18>(s1), ShiftRight<5>(s0))));
return bits;
}
};
template <class D, class VU64, HWY_IF_NOT_FLOAT_D(D)>
Vec<D> RandomValues(D d, VU64& s0, VU64& s1, const VU64 mask) {
const VU64 bits = Xorshift128Plus::RandomBits(s0, s1);
return BitCast(d, And(bits, mask));
}
// It is important to avoid denormals, which are flushed to zero by SIMD but not
// scalar sorts, and NaN, which may be ordered differently in scalar vs. SIMD.
template <class DF, class VU64, HWY_IF_FLOAT_D(DF)>
Vec<DF> RandomValues(DF df, VU64& s0, VU64& s1, const VU64 mask) {
using TF = TFromD<DF>;
const RebindToUnsigned<decltype(df)> du;
using VU = Vec<decltype(du)>;
const VU64 bits64 = And(Xorshift128Plus::RandomBits(s0, s1), mask);
#if HWY_TARGET == HWY_SCALAR // Cannot repartition u64 to smaller types
using TU = MakeUnsigned<TF>;
const VU bits = Set(du, static_cast<TU>(GetLane(bits64) & LimitsMax<TU>()));
#else
const VU bits = BitCast(du, bits64);
#endif
// Avoid NaN/denormal by only generating values in [1, 2), i.e. random
// mantissas with the exponent taken from the representation of 1.0.
const VU k1 = BitCast(du, Set(df, TF{1.0}));
const VU mantissa_mask = Set(du, MantissaMask<TF>());
const VU representation = OrAnd(k1, bits, mantissa_mask);
return BitCast(df, representation);
}
template <class DU64>
Vec<DU64> MaskForDist(DU64 du64, const Dist dist, size_t sizeof_t) {
switch (sizeof_t) {
case 2:
return Set(du64, (dist == Dist::kUniform8) ? 0x00FF00FF00FF00FFull
: 0xFFFFFFFFFFFFFFFFull);
case 4:
return Set(du64, (dist == Dist::kUniform8) ? 0x000000FF000000FFull
: (dist == Dist::kUniform16) ? 0x0000FFFF0000FFFFull
: 0xFFFFFFFFFFFFFFFFull);
case 8:
return Set(du64, (dist == Dist::kUniform8) ? 0x00000000000000FFull
: (dist == Dist::kUniform16) ? 0x000000000000FFFFull
: 0x00000000FFFFFFFFull);
default:
HWY_ABORT("Logic error");
return Zero(du64);
}
}
template <typename T>
InputStats<T> GenerateInput(const Dist dist, T* v, size_t num_lanes) {
SortTag<uint64_t> du64;
using VU64 = Vec<decltype(du64)>;
const size_t N64 = Lanes(du64);
auto seeds = hwy::AllocateAligned<uint64_t>(2 * N64);
Xorshift128Plus::GenerateSeeds(du64, seeds.get());
VU64 s0 = Load(du64, seeds.get());
VU64 s1 = Load(du64, seeds.get() + N64);
#if HWY_TARGET == HWY_SCALAR
const Sisd<T> d;
#else
const Repartition<T, decltype(du64)> d;
#endif
using V = Vec<decltype(d)>;
const size_t N = Lanes(d);
const VU64 mask = MaskForDist(du64, dist, sizeof(T));
auto buf = hwy::AllocateAligned<T>(N);
size_t i = 0;
for (; i + N <= num_lanes; i += N) {
const V values = RandomValues(d, s0, s1, mask);
StoreU(values, d, v + i);
}
if (i < num_lanes) {
const V values = RandomValues(d, s0, s1, mask);
StoreU(values, d, buf.get());
CopyBytes(buf.get(), v + i, (num_lanes - i) * sizeof(T));
}
InputStats<T> input_stats;
for (size_t i = 0; i < num_lanes; ++i) {
input_stats.Notify(v[i]);
}
return input_stats;
}
struct SharedState {
#if HAVE_PARALLEL_IPS4O
const unsigned max_threads = hwy::LimitsMax<unsigned>(); // 16 for Table 1a
ips4o::StdThreadPool pool{static_cast<int>(
HWY_MIN(max_threads, std::thread::hardware_concurrency() / 2))};
#endif
};
// Adapters from Run's num_keys to vqsort-inl.h num_lanes.
template <typename KeyType, class Order>
void CallHeapSort(KeyType* keys, const size_t num_keys, Order) {
const detail::MakeTraits<KeyType, Order> st;
using LaneType = typename decltype(st)::LaneType;
return detail::HeapSort(st, reinterpret_cast<LaneType*>(keys),
num_keys * st.LanesPerKey());
}
template <typename KeyType, class Order>
void CallHeapPartialSort(KeyType* keys, const size_t num_keys,
const size_t k_keys, Order) {
const detail::MakeTraits<KeyType, Order> st;
using LaneType = typename decltype(st)::LaneType;
detail::HeapPartialSort(st, reinterpret_cast<LaneType*>(keys),
num_keys * st.LanesPerKey(),
k_keys * st.LanesPerKey());
}
template <typename KeyType, class Order>
void CallHeapSelect(KeyType* keys, const size_t num_keys, const size_t k_keys,
Order) {
const detail::MakeTraits<KeyType, Order> st;
using LaneType = typename decltype(st)::LaneType;
detail::HeapSelect(st, reinterpret_cast<LaneType*>(keys),
num_keys * st.LanesPerKey(), k_keys * st.LanesPerKey());
}
template <typename KeyType, class Order>
void Run(Algo algo, KeyType* inout, size_t num_keys, SharedState& shared,
size_t /*thread*/, size_t k_keys, Order) {
const std::less<KeyType> less;
const std::greater<KeyType> greater;
constexpr bool kAscending = Order::IsAscending();
#if !HAVE_PARALLEL_IPS4O
(void)shared;
#endif
switch (algo) {
#if HAVE_INTEL && HWY_TARGET <= HWY_AVX3
case Algo::kIntel:
return avx512_qsort<KeyType>(inout, static_cast<int64_t>(num_keys));
#endif
#if HAVE_AVX2SORT
case Algo::kSEA:
return avx2::quicksort(inout, static_cast<int>(num_keys));
#endif
#if HAVE_IPS4O
case Algo::kIPS4O:
if (kAscending) {
return ips4o::sort(inout, inout + num_keys, less);
} else {
return ips4o::sort(inout, inout + num_keys, greater);
}
#endif
#if HAVE_PARALLEL_IPS4O
case Algo::kParallelIPS4O:
if (kAscending) {
return ips4o::parallel::sort(inout, inout + num_keys, less,
shared.pool);
} else {
return ips4o::parallel::sort(inout, inout + num_keys, greater,
shared.pool);
}
#endif
#if HAVE_SORT512
case Algo::kSort512:
HWY_ABORT("not supported");
// return Sort512::Sort(inout, num_keys);
#endif
#if HAVE_PDQSORT
case Algo::kPDQ:
if (kAscending) {
return boost::sort::pdqsort_branchless(inout, inout + num_keys, less);
} else {
return boost::sort::pdqsort_branchless(inout, inout + num_keys,
greater);
}
#endif
#if HAVE_VXSORT
case Algo::kVXSort: {
#if (VXSORT_AVX3 && HWY_TARGET != HWY_AVX3) || \
(!VXSORT_AVX3 && HWY_TARGET != HWY_AVX2)
fprintf(stderr, "Do not call for target %s\n",
hwy::TargetName(HWY_TARGET));
return;
#else
#if VXSORT_AVX3
vxsort::vxsort<KeyType, vxsort::AVX512> vx;
#else
vxsort::vxsort<KeyType, vxsort::AVX2> vx;
#endif
if (kAscending) {
return vx.sort(inout, inout + num_keys - 1);
} else {
fprintf(stderr, "Skipping VX - does not support descending order\n");
return;
}
#endif // enabled for this target
}
#endif // HAVE_VXSORT
case Algo::kStdSort:
if (kAscending) {
return std::sort(inout, inout + num_keys, less);
} else {
return std::sort(inout, inout + num_keys, greater);
}
case Algo::kStdPartialSort:
if (kAscending) {
return std::partial_sort(inout, inout + k_keys, inout + num_keys, less);
} else {
return std::partial_sort(inout, inout + k_keys, inout + num_keys,
greater);
}
case Algo::kStdSelect:
if (kAscending) {
return std::nth_element(inout, inout + k_keys, inout + num_keys, less);
} else {
return std::nth_element(inout, inout + k_keys, inout + num_keys,
greater);
}
case Algo::kVQSort:
return VQSort(inout, num_keys, Order());
case Algo::kVQPartialSort:
return VQPartialSort(inout, num_keys, k_keys, Order());
case Algo::kVQSelect:
return VQSelect(inout, num_keys, k_keys, Order());
case Algo::kHeapSort:
return CallHeapSort(inout, num_keys, Order());
case Algo::kHeapPartialSort:
return CallHeapPartialSort(inout, num_keys, k_keys, Order());
case Algo::kHeapSelect:
return CallHeapSelect(inout, num_keys, k_keys, Order());
default:
HWY_ABORT("Not implemented");
}
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_SORT_ALGO_TOGGLE

242
deps/v8/third_party/highway/src/hwy/contrib/sort/bench_parallel.cc vendored Normal file
View File

@ -0,0 +1,242 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Concurrent, independent sorts for generating more memory traffic and testing
// scalability when bandwidth-limited. If you want to use multiple threads for
// a single sort, you can use ips4o and integrate vqsort by calling it from
// `baseCaseSort` and increasing `IPS4OML_BASE_CASE_SIZE` to say 8192.
#include <stdint.h>
#include <stdio.h>
#include <condition_variable> //NOLINT
#include <functional>
#include <mutex> //NOLINT
#include <thread> //NOLINT
#include <vector>
#include "hwy/timer.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/bench_parallel.cc" //NOLINT
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/algo-inl.h"
#include "hwy/contrib/sort/result-inl.h"
#include "hwy/aligned_allocator.h"
// Last
#include "hwy/tests/test_util-inl.h"
// clang-format on
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
class ThreadPool {
public:
// Starts the given number of worker threads and blocks until they are ready.
explicit ThreadPool(
const size_t num_threads = std::thread::hardware_concurrency())
: num_threads_(num_threads) {
HWY_ASSERT(num_threads_ > 0);
threads_.reserve(num_threads_);
for (size_t i = 0; i < num_threads_; ++i) {
threads_.emplace_back(ThreadFunc, this, i);
}
WorkersReadyBarrier();
}
ThreadPool(const ThreadPool&) = delete;
ThreadPool& operator&(const ThreadPool&) = delete;
// Waits for all threads to exit.
~ThreadPool() {
StartWorkers(kWorkerExit);
for (std::thread& thread : threads_) {
thread.join();
}
}
size_t NumThreads() const { return threads_.size(); }
template <class Func>
void RunOnThreads(size_t max_threads, const Func& func) {
task_ = &CallClosure<Func>;
data_ = &func;
StartWorkers(max_threads);
WorkersReadyBarrier();
}
private:
// After construction and between calls to Run, workers are "ready", i.e.
// waiting on worker_start_cv_. They are "started" by sending a "command"
// and notifying all worker_start_cv_ waiters. (That is why all workers
// must be ready/waiting - otherwise, the notification will not reach all of
// them and the main thread waits in vain for them to report readiness.)
using WorkerCommand = uint64_t;
static constexpr WorkerCommand kWorkerWait = ~1ULL;
static constexpr WorkerCommand kWorkerExit = ~2ULL;
// Calls a closure (lambda with captures).
template <class Closure>
static void CallClosure(const void* f, size_t thread) {
(*reinterpret_cast<const Closure*>(f))(thread);
}
void WorkersReadyBarrier() {
std::unique_lock<std::mutex> lock(mutex_);
// Typically only a single iteration.
while (workers_ready_ != threads_.size()) {
workers_ready_cv_.wait(lock);
}
workers_ready_ = 0;
// Safely handle spurious worker wakeups.
worker_start_command_ = kWorkerWait;
}
// Precondition: all workers are ready.
void StartWorkers(const WorkerCommand worker_command) {
std::unique_lock<std::mutex> lock(mutex_);
worker_start_command_ = worker_command;
// Workers will need this lock, so release it before they wake up.
lock.unlock();
worker_start_cv_.notify_all();
}
static void ThreadFunc(ThreadPool* self, size_t thread) {
// Until kWorkerExit command received:
for (;;) {
std::unique_lock<std::mutex> lock(self->mutex_);
// Notify main thread that this thread is ready.
if (++self->workers_ready_ == self->num_threads_) {
self->workers_ready_cv_.notify_one();
}
RESUME_WAIT:
// Wait for a command.
self->worker_start_cv_.wait(lock);
const WorkerCommand command = self->worker_start_command_;
switch (command) {
case kWorkerWait: // spurious wakeup:
goto RESUME_WAIT; // lock still held, avoid incrementing ready.
case kWorkerExit:
return; // exits thread
default:
break;
}
lock.unlock();
// Command is the maximum number of threads that should run the task.
HWY_ASSERT(command < self->NumThreads());
if (thread < command) {
self->task_(self->data_, thread);
}
}
}
const size_t num_threads_;
// Unmodified after ctor, but cannot be const because we call thread::join().
std::vector<std::thread> threads_;
std::mutex mutex_; // guards both cv and their variables.
std::condition_variable workers_ready_cv_;
size_t workers_ready_ = 0;
std::condition_variable worker_start_cv_;
WorkerCommand worker_start_command_;
// Written by main thread, read by workers (after mutex lock/unlock).
std::function<void(const void*, size_t)> task_; // points to CallClosure
const void* data_; // points to caller's Func
};
template <class Traits>
void RunWithoutVerify(Traits st, const Dist dist, const size_t num_keys,
const Algo algo, SharedState& shared, size_t thread) {
using LaneType = typename Traits::LaneType;
using KeyType = typename Traits::KeyType;
using Order = typename Traits::Order;
const size_t num_lanes = num_keys * st.LanesPerKey();
auto aligned = hwy::AllocateAligned<LaneType>(num_lanes);
(void)GenerateInput(dist, aligned.get(), num_lanes);
const Timestamp t0;
Run(algo, reinterpret_cast<KeyType*>(aligned.get()), num_keys, shared, thread,
/*k_keys=*/0, Order());
HWY_ASSERT(aligned[0] < aligned[num_lanes - 1]);
}
void BenchParallel() {
// Not interested in benchmark results for other targets on x86
if (HWY_ARCH_X86 &&
(HWY_TARGET != HWY_AVX2 && HWY_TARGET != HWY_AVX3 &&
HWY_TARGET != HWY_AVX3_ZEN4 && HWY_TARGET != HWY_AVX3_SPR)) {
return;
}
ThreadPool pool;
const size_t NT = pool.NumThreads();
detail::SharedTraits<detail::TraitsLane<detail::OrderAscending<int64_t>>> st;
using KeyType = typename decltype(st)::KeyType;
const size_t num_keys = size_t{100} * 1000 * 1000;
#if HAVE_IPS4O
const Algo algo = Algo::kIPS4O;
#else
const Algo algo = Algo::kVQSort;
#endif
const Dist dist = Dist::kUniform32;
SharedState shared;
std::vector<SortResult> results;
for (size_t nt = 1; nt < NT; nt += HWY_MAX(1, NT / 16)) {
Timestamp t0;
// Default capture because MSVC wants algo/dist but clang does not.
pool.RunOnThreads(nt, [=, &shared](size_t thread) {
RunWithoutVerify(st, dist, num_keys, algo, shared, thread);
});
const double sec = SecondsSince(t0);
results.emplace_back(algo, dist, num_keys, nt, sec, sizeof(KeyType),
st.KeyString());
results.back().Print();
}
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(BenchParallel);
HWY_EXPORT_AND_TEST_P(BenchParallel, BenchParallel);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
#endif // HWY_ONCE

480
deps/v8/third_party/highway/src/hwy/contrib/sort/bench_sort.cc vendored Normal file
View File

@ -0,0 +1,480 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <vector>
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/bench_sort.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/algo-inl.h"
#include "hwy/contrib/sort/vqsort.h"
#include "hwy/contrib/sort/result-inl.h"
#include "hwy/contrib/sort/sorting_networks-inl.h" // SharedTraits
#include "hwy/contrib/sort/traits-inl.h"
#include "hwy/contrib/sort/traits128-inl.h"
#include "hwy/tests/test_util-inl.h"
#include "hwy/timer-inl.h"
#include "hwy/nanobenchmark.h"
#include "hwy/timer.h"
#include "hwy/per_target.h"
// clang-format on
#if HWY_OS_LINUX
#include <unistd.h> // usleep
#endif
// Mode for larger sorts because M1 is able to access more than the per-core
// share of L2, so 1M elements might still be in cache.
#define SORT_100M 0
#ifndef SORT_ONLY_COLD
#define SORT_ONLY_COLD 0
#endif
#ifndef SORT_BENCH_BASE_AND_PARTITION
#define SORT_BENCH_BASE_AND_PARTITION (!SORT_ONLY_COLD && 0)
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
// Defined within HWY_ONCE, used by BenchAllSort.
extern int64_t first_sort_target;
extern int64_t first_cold_target; // for BenchAllColdSort
namespace HWY_NAMESPACE {
namespace {
using detail::OrderAscending;
using detail::OrderDescending;
using detail::SharedTraits;
using detail::TraitsLane;
#if HWY_TARGET != HWY_SCALAR
using detail::OrderAscending128;
using detail::OrderAscendingKV128;
using detail::Traits128;
#endif // HWY_TARGET != HWY_SCALAR
HWY_NOINLINE void BenchAllColdSort() {
// Only run the best(first) enabled target
if (first_cold_target == 0) first_cold_target = HWY_TARGET;
if (HWY_TARGET != first_cold_target) {
return;
}
char cpu100[100];
if (!platform::HaveTimerStop(cpu100)) {
fprintf(stderr, "CPU '%s' does not support RDTSCP, skipping benchmark.\n",
cpu100);
return;
}
// Initialize random seeds
#if VQSORT_ENABLED
HWY_ASSERT(GetGeneratorState() != nullptr); // vqsort
#endif
RandomState rng(static_cast<uint64_t>(Unpredictable1() * 129)); // this test
using T = uint64_t;
constexpr size_t kSize = 10 * 1000;
HWY_ALIGN T items[kSize];
// Initialize array
#if 0 // optional: deliberate AVX-512 to verify VQSort performance improves
const ScalableTag<T> d;
const RebindToSigned<decltype(d)> di;
const size_t N = Lanes(d);
size_t i = 0;
for (; i + N <= kSize; i += N) {
// Super-slow scatter so that we spend enough time to warm up SKX.
const Vec<decltype(d)> val = Set(d, static_cast<T>(Unpredictable1()));
const Vec<decltype(di)> idx =
Iota(di, static_cast<T>(Unpredictable1() - 1));
ScatterIndex(val, d, items + i, idx);
}
for (; i < kSize; ++i) {
items[i] = static_cast<T>(Unpredictable1());
}
#else // scalar-only, verified with clang-16
for (size_t i = 0; i < kSize; ++i) {
items[i] = static_cast<T>(Unpredictable1());
}
#endif
items[Random32(&rng) % kSize] = static_cast<T>(Unpredictable1() + 1);
const timer::Ticks t0 = timer::Start();
const SortAscending order;
#if VQSORT_ENABLED && 1 // change to && 0 to switch to std::sort.
VQSort(items, kSize, order);
#else
SharedState shared;
Run(Algo::kStdSort, items, kSize, shared, /*thread=*/0, /*k_keys=*/0, order);
#endif
const timer::Ticks t1 = timer::Stop();
const double ticks = static_cast<double>(t1 - t0);
const double elapsed = ticks / platform::InvariantTicksPerSecond();
const double GBps = kSize * sizeof(T) * 1E-9 / elapsed;
fprintf(stderr, "N=%zu GB/s=%.2f ns=%.1f random output: %g\n", kSize, GBps,
elapsed * 1E9, static_cast<double>(items[Random32(&rng) % kSize]));
#if SORT_ONLY_COLD
#if HWY_OS_LINUX
// Long enough for the CPU to switch off AVX-512 mode before the next run.
usleep(100 * 1000); // NOLINT
#endif
#endif
}
#if (VQSORT_ENABLED && SORT_BENCH_BASE_AND_PARTITION) || HWY_IDE
template <class Traits>
HWY_NOINLINE void BenchPartition() {
using LaneType = typename Traits::LaneType;
using KeyType = typename Traits::KeyType;
const SortTag<LaneType> d;
detail::SharedTraits<Traits> st;
const Dist dist = Dist::kUniform8;
double sum = 0.0;
constexpr size_t kLPK = st.LanesPerKey();
HWY_ALIGN LaneType
buf[SortConstants::BufBytes<LaneType, kLPK>(HWY_MAX_BYTES) /
sizeof(LaneType)];
uint64_t* HWY_RESTRICT state = GetGeneratorState();
const size_t max_log2 = AdjustedLog2Reps(20);
for (size_t log2 = max_log2; log2 < max_log2 + 1; ++log2) {
const size_t num_lanes = 1ull << log2;
const size_t num_keys = num_lanes / kLPK;
auto aligned = hwy::AllocateAligned<LaneType>(num_lanes);
std::vector<double> seconds;
const size_t num_reps = (1ull << (14 - log2 / 2)) * 30;
for (size_t rep = 0; rep < num_reps; ++rep) {
(void)GenerateInput(dist, aligned.get(), num_lanes);
// The pivot value can influence performance. Do exactly what vqsort will
// do so that the performance (influenced by prefetching and branch
// prediction) is likely to predict the actual performance inside vqsort.
detail::DrawSamples(d, st, aligned.get(), num_lanes, buf, state);
detail::SortSamples(d, st, buf);
auto pivot = detail::ChoosePivotByRank(d, st, buf);
const Timestamp t0;
detail::Partition(d, st, aligned.get(), num_lanes - 1, pivot, buf);
seconds.push_back(SecondsSince(t0));
// 'Use' the result to prevent optimizing out the partition.
sum += static_cast<double>(aligned.get()[num_lanes / 2]);
}
SortResult(Algo::kVQSort, dist, num_keys, 1, SummarizeMeasurements(seconds),
sizeof(KeyType), st.KeyString())
.Print();
}
HWY_ASSERT(sum != 999999); // Prevent optimizing out
}
HWY_NOINLINE void BenchAllPartition() {
// Not interested in benchmark results for these targets
if (HWY_TARGET == HWY_SSSE3) {
return;
}
BenchPartition<TraitsLane<OrderDescending<float>>>();
BenchPartition<TraitsLane<OrderDescending<int32_t>>>();
BenchPartition<TraitsLane<OrderDescending<int64_t>>>();
BenchPartition<Traits128<OrderAscending128>>();
// BenchPartition<Traits128<OrderDescending128>>();
BenchPartition<Traits128<OrderAscendingKV128>>();
}
template <class Traits>
HWY_NOINLINE void BenchBase(std::vector<SortResult>& results) {
// Not interested in benchmark results for these targets
if (HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4) {
return;
}
using LaneType = typename Traits::LaneType;
using KeyType = typename Traits::KeyType;
const SortTag<LaneType> d;
detail::SharedTraits<Traits> st;
const Dist dist = Dist::kUniform32;
const Algo algo = Algo::kVQSort;
const size_t N = Lanes(d);
constexpr size_t kLPK = st.LanesPerKey();
const size_t num_lanes = SortConstants::BaseCaseNumLanes<kLPK>(N);
const size_t num_keys = num_lanes / kLPK;
auto keys = hwy::AllocateAligned<LaneType>(num_lanes);
auto buf = hwy::AllocateAligned<LaneType>(num_lanes + N);
std::vector<double> seconds;
double sum = 0; // prevents elision
constexpr size_t kMul = AdjustedReps(600); // ensures long enough to measure
for (size_t rep = 0; rep < 30; ++rep) {
InputStats<LaneType> input_stats =
GenerateInput(dist, keys.get(), num_lanes);
const Timestamp t0;
for (size_t i = 0; i < kMul; ++i) {
detail::BaseCase(d, st, keys.get(), num_lanes, buf.get());
sum += static_cast<double>(keys[0]);
}
seconds.push_back(SecondsSince(t0));
// printf("%f\n", seconds.back());
SortOrderVerifier<Traits>()(algo, input_stats, keys.get(), num_keys,
num_keys);
}
HWY_ASSERT(sum < 1E99);
results.emplace_back(algo, dist, num_keys * kMul, 1,
SummarizeMeasurements(seconds), sizeof(KeyType),
st.KeyString());
}
HWY_NOINLINE void BenchAllBase() {
// Not interested in benchmark results for these targets
if (HWY_TARGET == HWY_SSSE3) {
return;
}
std::vector<SortResult> results;
BenchBase<TraitsLane<OrderAscending<float>>>(results);
BenchBase<TraitsLane<OrderDescending<int64_t>>>(results);
BenchBase<Traits128<OrderAscending128>>(results);
for (const SortResult& r : results) {
r.Print();
}
}
#endif // VQSORT_ENABLED && SORT_BENCH_BASE_AND_PARTITION
std::vector<Algo> AlgoForBench() {
return {
#if HAVE_AVX2SORT
Algo::kSEA,
#endif
#if HAVE_PARALLEL_IPS4O
Algo::kParallelIPS4O,
#elif HAVE_IPS4O
Algo::kIPS4O,
#endif
#if HAVE_PDQSORT
Algo::kPDQ,
#endif
#if HAVE_SORT512
Algo::kSort512,
#endif
// Only include if we're compiling for the target it supports.
#if HAVE_VXSORT && ((VXSORT_AVX3 && HWY_TARGET == HWY_AVX3) || \
(!VXSORT_AVX3 && HWY_TARGET == HWY_AVX2))
Algo::kVXSort,
#endif
// Only include if we're compiling for the target it supports.
#if HAVE_INTEL && HWY_TARGET <= HWY_AVX3
Algo::kIntel,
#endif
#if !HAVE_PARALLEL_IPS4O
#if !SORT_100M
// 10-20x slower, but that's OK for the default size when we are not
// testing the parallel nor 100M modes.
// Algo::kStdSort,
#endif
#if VQSORT_ENABLED
Algo::kVQSort,
#endif
#endif // !HAVE_PARALLEL_IPS4O
};
}
template <class Traits>
HWY_NOINLINE void BenchSort(size_t num_keys) {
if (first_sort_target == 0) first_sort_target = HWY_TARGET;
SharedState shared;
detail::SharedTraits<Traits> st;
using Order = typename Traits::Order;
using LaneType = typename Traits::LaneType;
using KeyType = typename Traits::KeyType;
const size_t num_lanes = num_keys * st.LanesPerKey();
auto aligned = hwy::AllocateAligned<LaneType>(num_lanes);
const size_t reps = num_keys > 1000 * 1000 ? 10 : 30;
for (Algo algo : AlgoForBench()) {
// Other algorithms don't depend on the vector instructions, so only run
// them for the first target.
#if !HAVE_VXSORT
if (algo != Algo::kVQSort && HWY_TARGET != first_sort_target) {
continue;
}
#endif
for (Dist dist : AllDist()) {
std::vector<double> seconds;
for (size_t rep = 0; rep < reps; ++rep) {
InputStats<LaneType> input_stats =
GenerateInput(dist, aligned.get(), num_lanes);
const Timestamp t0;
Run(algo, HWY_RCAST_ALIGNED(KeyType*, aligned.get()), num_keys, shared,
/*thread=*/0, /*k_keys=*/0, Order());
seconds.push_back(SecondsSince(t0));
// printf("%f\n", seconds.back());
SortOrderVerifier<Traits>()(algo, input_stats, aligned.get(), num_keys,
num_keys);
}
SortResult(algo, dist, num_keys, 1, SummarizeMeasurements(seconds),
sizeof(KeyType), st.KeyString())
.Print();
} // dist
} // algo
}
enum class BenchmarkModes {
kDefault,
k1M,
k10K,
kAllSmall,
kSmallPow2,
kSmallPow2Between, // includes padding
kPow4,
kPow10
};
std::vector<size_t> SizesToBenchmark(BenchmarkModes mode) {
std::vector<size_t> sizes;
switch (mode) {
default:
case BenchmarkModes::kDefault:
#if HAVE_PARALLEL_IPS4O || SORT_100M
sizes.push_back(100 * 1000 * size_t{1000});
#else
sizes.push_back(100);
sizes.push_back(100 * 1000);
#endif
break;
case BenchmarkModes::k1M:
sizes.push_back(1000 * 1000);
break;
case BenchmarkModes::k10K:
sizes.push_back(10 * 1000);
break;
case BenchmarkModes::kAllSmall:
sizes.reserve(128);
for (size_t i = 1; i <= 128; ++i) {
sizes.push_back(i);
}
break;
case BenchmarkModes::kSmallPow2:
for (size_t size = 2; size <= 128; size *= 2) {
sizes.push_back(size);
}
break;
case BenchmarkModes::kSmallPow2Between:
for (size_t size = 2; size <= 128; size *= 2) {
sizes.push_back(3 * size / 2);
}
break;
case BenchmarkModes::kPow4:
for (size_t size = 4; size <= 256 * 1024; size *= 4) {
sizes.push_back(size);
}
break;
case BenchmarkModes::kPow10:
for (size_t size = 10; size <= 100 * 1000; size *= 10) {
sizes.push_back(size);
}
break;
}
return sizes;
}
HWY_NOINLINE void BenchAllSort() {
// Not interested in benchmark results for these targets. Note that SSE4 is
// numerically less than SSE2, hence it is the lower bound.
if (HWY_SSE4 <= HWY_TARGET && HWY_TARGET <= HWY_SSE2) {
return;
}
#if HAVE_INTEL
if (HWY_TARGET > HWY_AVX3) return;
#endif
for (size_t num_keys : SizesToBenchmark(BenchmarkModes::kSmallPow2)) {
#if !HAVE_INTEL
#if HWY_HAVE_FLOAT16
if (hwy::HaveFloat16()) {
BenchSort<TraitsLane<OtherOrder<float16_t>>>(num_keys);
}
#endif
BenchSort<TraitsLane<OrderAscending<float>>>(num_keys);
#if HWY_HAVE_FLOAT64
if (hwy::HaveFloat64()) {
// BenchSort<TraitsLane<OtherOrder<double>>>(num_keys);
}
#endif
#endif // !HAVE_INTEL
// BenchSort<TraitsLane<OrderAscending<int16_t>>>(num_keys);
BenchSort<TraitsLane<OtherOrder<int32_t>>>(num_keys);
BenchSort<TraitsLane<OrderAscending<int64_t>>>(num_keys);
// BenchSort<TraitsLane<OtherOrder<uint16_t>>>(num_keys);
// BenchSort<TraitsLane<OtherOrder<uint32_t>>>(num_keys);
// BenchSort<TraitsLane<OrderAscending<uint64_t>>>(num_keys);
#if !HAVE_VXSORT && !HAVE_INTEL && HWY_TARGET != HWY_SCALAR
BenchSort<Traits128<OrderAscending128>>(num_keys);
BenchSort<Traits128<OrderAscendingKV128>>(num_keys);
#endif
}
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
int64_t first_sort_target = 0; // none run yet
int64_t first_cold_target = 0; // none run yet
HWY_BEFORE_TEST(BenchSort);
HWY_EXPORT_AND_TEST_P(BenchSort, BenchAllColdSort);
#if SORT_BENCH_BASE_AND_PARTITION
HWY_EXPORT_AND_TEST_P(BenchSort, BenchAllPartition);
HWY_EXPORT_AND_TEST_P(BenchSort, BenchAllBase);
#endif
#if !SORT_ONLY_COLD // skip (warms up vector unit for next run)
HWY_EXPORT_AND_TEST_P(BenchSort, BenchAllSort);
#endif
HWY_AFTER_TEST();
} // namespace hwy
#endif // HWY_ONCE

34
deps/v8/third_party/highway/src/hwy/contrib/sort/order.h vendored Normal file
View File

@ -0,0 +1,34 @@
// Copyright 2023 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Tag arguments that determine the sort order. Used by both vqsort.h and the
// VQSortStatic in vqsort-inl.h. Moved to a separate header so that the latter
// can be used without pulling in the dllimport statements in vqsort.h.
#ifndef HIGHWAY_HWY_CONTRIB_SORT_ORDER_H_
#define HIGHWAY_HWY_CONTRIB_SORT_ORDER_H_
namespace hwy {
struct SortAscending {
static constexpr bool IsAscending() { return true; }
};
struct SortDescending {
static constexpr bool IsAscending() { return false; }
};
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_SORT_ORDER_H_

90
deps/v8/third_party/highway/src/hwy/contrib/sort/print_network.cc vendored Normal file
View File

@ -0,0 +1,90 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdio.h>
#include <vector>
#include "hwy/base.h"
// Based on A.7 in "Entwurf und Implementierung vektorisierter
// Sortieralgorithmen" and code by Mark Blacher.
void PrintMergeNetwork(int rows, int cols) {
printf("\n%d x %d:\n", rows, cols);
// Powers of two
HWY_ASSERT(rows != 0 && (rows & (rows - 1)) == 0);
HWY_ASSERT(cols != 0 && (cols & (cols - 1)) == 0);
HWY_ASSERT(rows >= 4);
HWY_ASSERT(cols >= 2); // otherwise no cross-column merging required
HWY_ASSERT(cols <= 16); // SortTraits lacks Reverse32
// Log(rows) times: sort half of the vectors with reversed groups of the
// other half. Group size halves until we are sorting adjacent vectors.
int group_size = rows;
int num_groups = 1;
for (; group_size >= 2; group_size /= 2, num_groups *= 2) {
// All vectors except those being reversed. Allows us to group the
// ReverseKeys and Sort2 operations, which is easier to read and may help
// in-order machines with high-latency ReverseKeys.
std::vector<int> all_vi;
for (int group = 0; group < num_groups; ++group) {
for (int i = 0; i < group_size / 2; ++i) {
all_vi.push_back(group * group_size + i);
}
}
for (int vi : all_vi) {
const int vr = vi ^ (group_size - 1);
printf("v%x = st.ReverseKeys%d(d, v%x);\n", vr, cols, vr);
}
for (int vi : all_vi) {
const int vr = vi ^ (group_size - 1);
printf("st.Sort2(d, v%x, v%x);\n", vi, vr);
}
printf("\n");
}
// Now merge across columns in all vectors.
if (cols > 2) {
for (int i = 0; i < rows; ++i) {
printf("v%x = st.SortPairsReverse%d(d, v%x);\n", i, cols, i);
}
printf("\n");
}
if (cols >= 16) {
for (int i = 0; i < rows; ++i) {
printf("v%x = st.SortPairsDistance4(d, v%x);\n", i, i);
}
printf("\n");
}
if (cols >= 8) {
for (int i = 0; i < rows; ++i) {
printf("v%x = st.SortPairsDistance2(d, v%x);\n", i, i);
}
printf("\n");
}
for (int i = 0; i < rows; ++i) {
printf("v%x = st.SortPairsDistance1(d, v%x);\n", i, i);
}
printf("\n");
}
int main(int argc, char** argv) {
PrintMergeNetwork(8, 2);
PrintMergeNetwork(8, 4);
PrintMergeNetwork(16, 4);
PrintMergeNetwork(16, 8);
PrintMergeNetwork(16, 16);
return 0;
}

291
deps/v8/third_party/highway/src/hwy/contrib/sort/result-inl.h vendored Normal file
View File

@ -0,0 +1,291 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/algo-inl.h"
// Normal include guard for non-SIMD parts
#ifndef HIGHWAY_HWY_CONTRIB_SORT_RESULT_INL_H_
#define HIGHWAY_HWY_CONTRIB_SORT_RESULT_INL_H_
#include <stdint.h>
#include <stdio.h>
#include <time.h>
#include <algorithm> // std::sort
#include <string>
#include <vector>
#include "hwy/aligned_allocator.h"
#include "hwy/base.h"
#include "hwy/contrib/sort/order.h"
#include "hwy/per_target.h" // DispatchedTarget
#include "hwy/targets.h" // TargetName
namespace hwy {
// Returns trimmed mean (we don't want to run an out-of-L3-cache sort often
// enough for the mode to be reliable).
static inline double SummarizeMeasurements(std::vector<double>& seconds) {
std::sort(seconds.begin(), seconds.end());
double sum = 0;
int count = 0;
const size_t num = seconds.size();
for (size_t i = num / 4; i < num / 2; ++i) {
sum += seconds[i];
count += 1;
}
return sum / count;
}
struct SortResult {
SortResult() {}
SortResult(const Algo algo, Dist dist, size_t num_keys, size_t num_threads,
double sec, size_t sizeof_key, const char* key_name)
: target(DispatchedTarget()),
algo(algo),
dist(dist),
num_keys(num_keys),
num_threads(num_threads),
sec(sec),
sizeof_key(sizeof_key),
key_name(key_name) {}
void Print() const {
const double bytes = static_cast<double>(num_keys) *
static_cast<double>(num_threads) *
static_cast<double>(sizeof_key);
printf("%10s: %12s: %7s: %9s: %05g %4.0f MB/s (%2zu threads)\n",
hwy::TargetName(target), AlgoName(algo), key_name.c_str(),
DistName(dist), static_cast<double>(num_keys), bytes * 1E-6 / sec,
num_threads);
}
int64_t target;
Algo algo;
Dist dist;
size_t num_keys = 0;
size_t num_threads = 0;
double sec = 0.0;
size_t sizeof_key = 0;
std::string key_name;
};
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_SORT_RESULT_INL_H_
// Per-target
#if defined(HIGHWAY_HWY_CONTRIB_SORT_RESULT_TOGGLE) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_SORT_RESULT_TOGGLE
#undef HIGHWAY_HWY_CONTRIB_SORT_RESULT_TOGGLE
#else
#define HIGHWAY_HWY_CONTRIB_SORT_RESULT_TOGGLE
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
// Copies the input, and compares results to that of a reference algorithm.
template <class Traits>
class ReferenceSortVerifier {
using LaneType = typename Traits::LaneType;
using KeyType = typename Traits::KeyType;
using Order = typename Traits::Order;
static constexpr bool kAscending = Order::IsAscending();
static constexpr size_t kLPK = Traits().LanesPerKey();
public:
ReferenceSortVerifier(const LaneType* in_lanes, size_t num_lanes) {
num_lanes_ = num_lanes;
num_keys_ = num_lanes / kLPK;
in_lanes_ = hwy::AllocateAligned<LaneType>(num_lanes);
HWY_ASSERT(in_lanes_);
CopyBytes(in_lanes, in_lanes_.get(), num_lanes * sizeof(LaneType));
}
// For full sorts, k_keys == num_keys.
void operator()(Algo algo, const LaneType* out_lanes, size_t k_keys) {
SharedState shared;
const Traits st;
const CappedTag<LaneType, kLPK> d;
HWY_ASSERT(hwy::IsAligned(in_lanes_.get(), sizeof(KeyType)));
KeyType* in_keys = HWY_RCAST_ALIGNED(KeyType*, in_lanes_.get());
char caption[10];
const char* algo_type = IsPartialSort(algo) ? "PartialSort" : "Sort";
HWY_ASSERT(k_keys <= num_keys_);
Run(ReferenceAlgoFor(algo), in_keys, num_keys_, shared, /*thread=*/0,
k_keys, Order());
if (IsSelect(algo)) {
// Print lanes centered around k_keys.
if (VQSORT_PRINT >= 3) {
const size_t begin_lane = k_keys < 3 ? 0 : (k_keys - 3) * kLPK;
const size_t end_lane = HWY_MIN(num_lanes_, (k_keys + 3) * kLPK);
fprintf(stderr, "\nExpected:\n");
for (size_t i = begin_lane; i < end_lane; i += kLPK) {
snprintf(caption, sizeof(caption), "%4zu ", i / kLPK);
Print(d, caption, st.SetKey(d, &in_lanes_[i]));
}
fprintf(stderr, "\n\nActual:\n");
for (size_t i = begin_lane; i < end_lane; i += kLPK) {
snprintf(caption, sizeof(caption), "%4zu ", i / kLPK);
Print(d, caption, st.SetKey(d, &out_lanes[i]));
}
fprintf(stderr, "\n\n");
}
// At k_keys: should be equivalent, i.e. neither a < b nor b < a.
// SortOrderVerifier will also check the ordering of the rest of the keys.
const size_t k = k_keys * kLPK;
if (st.Compare1(&in_lanes_[k], &out_lanes[k]) ||
st.Compare1(&out_lanes[k], &in_lanes_[k])) {
Print(d, "Expected", st.SetKey(d, &in_lanes_[k]));
Print(d, " Actual", st.SetKey(d, &out_lanes[k]));
HWY_ABORT("Select %s asc=%d: mismatch at k_keys=%zu, num_keys=%zu\n",
st.KeyString(), kAscending, k_keys, num_keys_);
}
} else {
if (VQSORT_PRINT >= 3) {
const size_t lanes_to_print = HWY_MIN(40, k_keys * kLPK);
fprintf(stderr, "\nExpected:\n");
for (size_t i = 0; i < lanes_to_print; i += kLPK) {
snprintf(caption, sizeof(caption), "%4zu ", i / kLPK);
Print(d, caption, st.SetKey(d, &in_lanes_[i]));
}
fprintf(stderr, "\n\nActual:\n");
for (size_t i = 0; i < lanes_to_print; i += kLPK) {
snprintf(caption, sizeof(caption), "%4zu ", i / kLPK);
Print(d, caption, st.SetKey(d, &out_lanes[i]));
}
fprintf(stderr, "\n\n");
}
// Full or partial sort: all elements up to k_keys are equivalent to the
// reference sort. SortOrderVerifier also checks the output's ordering.
for (size_t i = 0; i < k_keys * kLPK; i += kLPK) {
// All up to k_keys should be equivalent, i.e. neither a < b nor b < a.
if (st.Compare1(&in_lanes_[i], &out_lanes[i]) ||
st.Compare1(&out_lanes[i], &in_lanes_[i])) {
Print(d, "Expected", st.SetKey(d, &in_lanes_[i]));
Print(d, " Actual", st.SetKey(d, &out_lanes[i]));
HWY_ABORT("%s %s asc=%d: mismatch at %zu, k_keys=%zu, num_keys=%zu\n",
algo_type, st.KeyString(), kAscending, i / kLPK, k_keys,
num_keys_);
}
}
}
}
private:
hwy::AlignedFreeUniquePtr<LaneType[]> in_lanes_;
size_t num_lanes_;
size_t num_keys_;
};
// Faster than ReferenceSortVerifier, for use in bench_sort. Only verifies
// order, without running a slow reference sorter. This means it can't verify
// Select places the correct key at `k_keys`, nor that input and output keys are
// the same.
template <class Traits>
class SortOrderVerifier {
using LaneType = typename Traits::LaneType;
using Order = typename Traits::Order;
static constexpr bool kAscending = Order::IsAscending();
static constexpr size_t kLPK = Traits().LanesPerKey();
public:
void operator()(Algo algo, const InputStats<LaneType>& input_stats,
const LaneType* output, size_t num_keys, size_t k_keys) {
if (IsSelect(algo)) {
CheckSelectOrder(input_stats, output, num_keys, k_keys);
} else {
CheckSortedOrder(algo, input_stats, output, num_keys, k_keys);
}
}
private:
// For full or partial sorts: ensures keys are in sorted order.
void CheckSortedOrder(const Algo algo,
const InputStats<LaneType>& input_stats,
const LaneType* output, const size_t num_keys,
const size_t k_keys) {
const Traits st;
const CappedTag<LaneType, kLPK> d;
const size_t num_lanes = num_keys * kLPK;
const size_t k = k_keys * kLPK;
const char* algo_type = IsPartialSort(algo) ? "PartialSort" : "Sort";
InputStats<LaneType> output_stats;
// Even for partial sorts, loop over all keys to verify none disappeared.
for (size_t i = 0; i < num_lanes - kLPK; i += kLPK) {
output_stats.Notify(output[i]);
if (kLPK == 2) output_stats.Notify(output[i + 1]);
// Only check the first k_keys (== num_keys for a full sort).
// Reverse order instead of checking !Compare1 so we accept equal keys.
if (i < k - kLPK && st.Compare1(output + i + kLPK, output + i)) {
Print(d, " cur", st.SetKey(d, &output[i]));
Print(d, "next", st.SetKey(d, &output[i + kLPK]));
HWY_ABORT(
"%s %s asc=%d: wrong order at %zu, k_keys=%zu, num_keys=%zu\n",
algo_type, st.KeyString(), kAscending, i / kLPK, k_keys, num_keys);
}
}
output_stats.Notify(output[num_lanes - kLPK]);
if (kLPK == 2) output_stats.Notify(output[num_lanes - kLPK + 1]);
HWY_ASSERT(input_stats == output_stats);
}
// Ensures keys below index k_keys are less, and all above are greater.
void CheckSelectOrder(const InputStats<LaneType>& input_stats,
const LaneType* output, const size_t num_keys,
const size_t k_keys) {
const Traits st;
const CappedTag<LaneType, kLPK> d;
const size_t num_lanes = num_keys * kLPK;
const size_t k = k_keys * kLPK;
InputStats<LaneType> output_stats;
for (size_t i = 0; i < num_lanes - kLPK; i += kLPK) {
output_stats.Notify(output[i]);
if (kLPK == 2) output_stats.Notify(output[i + 1]);
// Reverse order instead of checking !Compare1 so we accept equal keys.
if (i < k ? st.Compare1(output + k, output + i)
: st.Compare1(output + i, output + k)) {
Print(d, "cur", st.SetKey(d, &output[i]));
Print(d, "kth", st.SetKey(d, &output[k]));
HWY_ABORT(
"Select %s asc=%d: wrong order at %zu, k_keys=%zu, num_keys=%zu\n",
st.KeyString(), kAscending, i / kLPK, k_keys, num_keys);
}
}
output_stats.Notify(output[num_lanes - kLPK]);
if (kLPK == 2) output_stats.Notify(output[num_lanes - kLPK + 1]);
HWY_ASSERT(input_stats == output_stats);
}
};
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_SORT_RESULT_TOGGLE

157
deps/v8/third_party/highway/src/hwy/contrib/sort/shared-inl.h vendored Normal file
View File

@ -0,0 +1,157 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Definitions shared between vqsort-inl and sorting_networks-inl.
// Normal include guard for target-independent parts
#ifndef HIGHWAY_HWY_CONTRIB_SORT_SHARED_INL_H_
#define HIGHWAY_HWY_CONTRIB_SORT_SHARED_INL_H_
#include "hwy/base.h"
namespace hwy {
// Internal constants - these are to avoid magic numbers/literals and cannot be
// changed without also changing the associated code.
struct SortConstants {
// SortingNetwork reshapes its input into a matrix. This is the maximum number
// of *lanes* per vector. Must be at least 8 because SortSamples assumes the
// sorting network can handle 128 bytes with 8 rows, so 16 bytes per vector,
// which means 8 lanes for 16-bit types.
#if HWY_COMPILER_MSVC || HWY_IS_DEBUG_BUILD
static constexpr size_t kMaxCols = 8; // avoid build timeout/stack overflow
#else
static constexpr size_t kMaxCols = 16; // enough for u32 in 512-bit vector
#endif
// 16 rows is a compromise between using the 32 AVX-512/SVE/RVV registers,
// fitting within 16 AVX2 registers with only a few spills, keeping BaseCase
// code size reasonable, and minimizing the extra logN factor for larger
// networks (for which only loose upper bounds on size are known).
static constexpr size_t kMaxRows = 16;
// Template argument ensures there is no actual division instruction.
template <size_t kLPK>
static constexpr HWY_INLINE size_t BaseCaseNumLanes(size_t N) {
// We use 8, 8x2, 8x4, and 16x{4..} networks, in units of keys. For N/kLPK
// < 4, we cannot use the 16-row networks.
return (((N / kLPK) >= 4) ? kMaxRows : 8) * HWY_MIN(N, kMaxCols);
}
// Unrolling is important (pipelining and amortizing branch mispredictions);
// 2x is sufficient to reach full memory bandwidth on SKX in Partition, but
// somewhat slower for sorting than 4x.
//
// To change, must also update left + 3 * N etc. in the loop.
static constexpr size_t kPartitionUnroll = 4;
// Chunk := group of keys loaded for sampling a pivot. Matches the typical
// cache line size of 64 bytes to get maximum benefit per L2 miss. Sort()
// ensures vectors are no larger than that, so this can be independent of the
// vector size and thus constexpr.
static constexpr HWY_INLINE size_t LanesPerChunk(size_t sizeof_t) {
return 64 / sizeof_t;
}
template <typename T>
static constexpr HWY_INLINE size_t SampleLanes() {
return 2 * LanesPerChunk(sizeof(T)); // Stored samples
}
static constexpr HWY_INLINE size_t PartitionBufNum(size_t N) {
// The main loop reads kPartitionUnroll vectors, and first loads from
// both left and right beforehand, so it requires 2 * kPartitionUnroll
// vectors. To handle amounts between that and BaseCaseNumLanes(), we
// partition up 3 * kPartitionUnroll + 1 vectors into a two-part buffer.
return 2 * (3 * kPartitionUnroll + 1) * N;
}
// Max across the three buffer usages.
template <typename T, size_t kLPK>
static constexpr HWY_INLINE size_t BufNum(size_t N) {
// BaseCase may write one padding vector, and SortSamples uses the space
// after samples as the buffer.
return HWY_MAX(SampleLanes<T>() + BaseCaseNumLanes<kLPK>(N) + N,
PartitionBufNum(N));
}
// Translates vector_size to lanes and returns size in bytes.
template <typename T, size_t kLPK>
static constexpr HWY_INLINE size_t BufBytes(size_t vector_size) {
return BufNum<T, kLPK>(vector_size / sizeof(T)) * sizeof(T);
}
// Returns max for any type.
template <size_t kLPK>
static constexpr HWY_INLINE size_t MaxBufBytes(size_t vector_size) {
// If 2 lanes per key, it's a 128-bit key with u64 lanes.
return kLPK == 2 ? BufBytes<uint64_t, 2>(vector_size)
: HWY_MAX((BufBytes<uint16_t, 1>(vector_size)),
HWY_MAX((BufBytes<uint32_t, 1>(vector_size)),
(BufBytes<uint64_t, 1>(vector_size))));
}
};
static_assert(SortConstants::MaxBufBytes<1>(64) <= 1664, "Unexpectedly high");
static_assert(SortConstants::MaxBufBytes<2>(64) <= 1664, "Unexpectedly high");
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_SORT_SHARED_INL_H_
// Per-target
// clang-format off
#if defined(HIGHWAY_HWY_CONTRIB_SORT_SHARED_TOGGLE) == defined(HWY_TARGET_TOGGLE) // NOLINT
// clang-format on
#ifdef HIGHWAY_HWY_CONTRIB_SORT_SHARED_TOGGLE
#undef HIGHWAY_HWY_CONTRIB_SORT_SHARED_TOGGLE
#else
#define HIGHWAY_HWY_CONTRIB_SORT_SHARED_TOGGLE
#endif
#include "hwy/highway.h"
// vqsort isn't available on HWY_SCALAR, and builds time out on MSVC opt and
// Armv7 debug, and Armv8 GCC 11 asan hits an internal compiler error likely
// due to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=97696. Armv8 Clang
// hwasan/msan/tsan/asan also fail to build SVE (b/335157772).
#undef VQSORT_ENABLED
#if (HWY_TARGET == HWY_SCALAR) || \
(HWY_COMPILER_MSVC && !HWY_IS_DEBUG_BUILD) || \
(HWY_ARCH_ARM_V7 && HWY_IS_DEBUG_BUILD) || \
(HWY_ARCH_ARM_A64 && HWY_COMPILER_GCC_ACTUAL && HWY_IS_ASAN)
#define VQSORT_ENABLED 0
#else
#define VQSORT_ENABLED 1
#endif
namespace hwy {
namespace HWY_NAMESPACE {
// Default tag / vector width selector.
#if HWY_TARGET == HWY_RVV
// Use LMUL = 1/2; for SEW=64 this ends up emulated via VSETVLI.
template <typename T>
using SortTag = ScalableTag<T, -1>;
#else
template <typename T>
using SortTag = ScalableTag<T>;
#endif
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_SORT_SHARED_TOGGLE

283
deps/v8/third_party/highway/src/hwy/contrib/sort/sort_test.cc vendored Normal file
View File

@ -0,0 +1,283 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <numeric> // std::iota
#include <random>
#include <vector>
#include "hwy/aligned_allocator.h" // IsAligned
#include "hwy/base.h"
#include "hwy/contrib/sort/vqsort.h"
#include "hwy/contrib/thread_pool/thread_pool.h"
#include "hwy/contrib/thread_pool/topology.h"
#include "hwy/per_target.h"
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/sort_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
// After highway.h
#include "hwy/contrib/sort/algo-inl.h"
#include "hwy/contrib/sort/result-inl.h"
#include "hwy/contrib/sort/vqsort-inl.h" // BaseCase
#include "hwy/print-inl.h"
#include "hwy/tests/test_util-inl.h"
// TODO(b/314758657): Compiler bug causes incorrect results on SSE2/S-SSE3.
#undef VQSORT_SKIP
#if !defined(VQSORT_DO_NOT_SKIP) && HWY_COMPILER_CLANG && HWY_ARCH_X86 && \
HWY_TARGET >= HWY_SSSE3
#define VQSORT_SKIP 1
#else
#define VQSORT_SKIP 0
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
using detail::OrderAscending;
using detail::OrderAscendingKV64;
using detail::OrderDescendingKV64;
using detail::SharedTraits;
using detail::TraitsLane;
#if !HAVE_INTEL && HWY_TARGET != HWY_SCALAR
using detail::OrderAscending128;
using detail::OrderAscendingKV128;
using detail::OrderDescending128;
using detail::OrderDescendingKV128;
using detail::Traits128;
#endif // !HAVE_INTEL && HWY_TARGET != HWY_SCALAR
template <typename Key>
void TestSortIota(hwy::ThreadPool& pool) {
pool.Run(128, 300, [](uint64_t task, size_t /*thread*/) {
const size_t num = static_cast<size_t>(task);
Key keys[300];
std::iota(keys, keys + num, Key{0});
VQSort(keys, num, hwy::SortAscending());
for (size_t i = 0; i < num; ++i) {
if (keys[i] != static_cast<Key>(i)) {
HWY_ABORT("num %zu i %zu: not iota, got %.0f\n", num, i,
static_cast<double>(keys[i]));
}
}
});
}
void TestAllSortIota() {
#if VQSORT_ENABLED
hwy::ThreadPool pool(hwy::HaveThreadingSupport() ? 4 : 0);
TestSortIota<uint32_t>(pool);
TestSortIota<int32_t>(pool);
if (hwy::HaveInteger64()) {
TestSortIota<int64_t>(pool);
TestSortIota<uint64_t>(pool);
}
TestSortIota<float>(pool);
if (hwy::HaveFloat64()) {
TestSortIota<double>(pool);
}
fprintf(stderr, "Iota OK\n");
#endif
}
// Supports full/partial sort and select.
template <class Traits>
void TestAnySort(const std::vector<Algo>& algos, size_t num_lanes) {
// Workaround for stack overflow on clang-cl (/F 8388608 does not help).
#if defined(_MSC_VER)
return;
#endif
using Order = typename Traits::Order;
using LaneType = typename Traits::LaneType;
using KeyType = typename Traits::KeyType;
SharedState shared;
SharedTraits<Traits> st;
constexpr size_t kLPK = st.LanesPerKey();
num_lanes = hwy::RoundUpTo(num_lanes, kLPK);
const size_t num_keys = num_lanes / kLPK;
std::mt19937 rng(42);
std::uniform_int_distribution<size_t> k_dist(1, num_keys - 1);
constexpr size_t kMaxMisalign = 16;
auto aligned =
hwy::AllocateAligned<LaneType>(kMaxMisalign + num_lanes + kMaxMisalign);
HWY_ASSERT(aligned);
for (Algo algo : algos) {
if (IsVQ(algo) && (!VQSORT_ENABLED || VQSORT_SKIP)) continue;
for (Dist dist : AllDist()) {
for (size_t misalign :
{size_t{0}, size_t{kLPK}, size_t{3 * kLPK}, kMaxMisalign / 2}) {
for (size_t k_rep = 0; k_rep < AdjustedReps(10); ++k_rep) {
// Skip reps for full sort because they do not use k.
if (!IsPartialSort(algo) && !IsSelect(algo) && k_rep > 0) break;
LaneType* lanes = aligned.get() + misalign;
HWY_ASSERT(hwy::IsAligned(lanes, sizeof(KeyType)));
KeyType* keys = HWY_RCAST_ALIGNED(KeyType*, lanes);
// Set up red zones before/after the keys to sort
for (size_t i = 0; i < misalign; ++i) {
aligned[i] = hwy::LowestValue<LaneType>();
}
for (size_t i = 0; i < kMaxMisalign; ++i) {
lanes[num_lanes + i] = hwy::HighestValue<LaneType>();
}
detail::MaybePoison(aligned.get(), misalign * sizeof(LaneType));
detail::MaybePoison(lanes + num_lanes,
kMaxMisalign * sizeof(LaneType));
InputStats<LaneType> input_stats =
GenerateInput(dist, lanes, num_lanes);
ReferenceSortVerifier<Traits> reference_verifier(lanes, num_lanes);
const size_t k_keys = k_dist(rng);
Run(algo, keys, num_keys, shared, /*thread=*/0, k_keys, Order());
reference_verifier(algo, lanes, k_keys);
SortOrderVerifier<Traits>()(algo, input_stats, lanes, num_keys,
k_keys);
// Check red zones
detail::MaybeUnpoison(aligned.get(), misalign);
detail::MaybeUnpoison(lanes + num_lanes, kMaxMisalign);
for (size_t i = 0; i < misalign; ++i) {
if (aligned[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun left at %d\n", static_cast<int>(i));
}
for (size_t i = num_lanes; i < num_lanes + kMaxMisalign; ++i) {
if (lanes[i] != hwy::HighestValue<LaneType>())
HWY_ABORT("Overrun right at %d\n", static_cast<int>(i));
}
} // k_rep
} // misalign
} // dist
} // algo
}
// Calls TestAnySort with all traits.
void CallAllSortTraits(const std::vector<Algo>& algos, size_t num_lanes) {
#if !HAVE_INTEL
TestAnySort<TraitsLane<OrderAscending<int16_t>>>(algos, num_lanes);
TestAnySort<TraitsLane<OtherOrder<uint16_t>>>(algos, num_lanes);
#endif
TestAnySort<TraitsLane<OtherOrder<int32_t>>>(algos, num_lanes);
TestAnySort<TraitsLane<OtherOrder<uint32_t>>>(algos, num_lanes);
TestAnySort<TraitsLane<OrderAscending<int64_t>>>(algos, num_lanes);
TestAnySort<TraitsLane<OrderAscending<uint64_t>>>(algos, num_lanes);
// WARNING: for float types, SIMD comparisons will flush denormals to
// zero, causing mismatches with scalar sorts. In this test, we avoid
// generating denormal inputs.
#if HWY_HAVE_FLOAT16 // #if protects algo-inl.h's GenerateRandom
// Must also check whether the dynamic-dispatch target supports float16_t!
if (hwy::HaveFloat16()) {
TestAnySort<TraitsLane<OrderAscending<float16_t>>>(algos, num_lanes);
}
#endif
TestAnySort<TraitsLane<OrderAscending<float>>>(algos, num_lanes);
#if HWY_HAVE_FLOAT64 // #if protects algo-inl.h's GenerateRandom
// Must also check whether the dynamic-dispatch target supports float64!
if (hwy::HaveFloat64()) {
TestAnySort<TraitsLane<OtherOrder<double>>>(algos, num_lanes);
}
#endif
// Other algorithms do not support 128-bit nor KV keys.
#if !HAVE_VXSORT && !HAVE_INTEL
TestAnySort<TraitsLane<OrderAscendingKV64>>(algos, num_lanes);
TestAnySort<TraitsLane<OrderDescendingKV64>>(algos, num_lanes);
// 128-bit keys require 128-bit SIMD.
#if HWY_TARGET != HWY_SCALAR
TestAnySort<Traits128<OrderAscending128>>(algos, num_lanes);
TestAnySort<Traits128<OrderDescending128>>(algos, num_lanes);
TestAnySort<Traits128<OrderAscendingKV128>>(algos, num_lanes);
TestAnySort<Traits128<OrderDescendingKV128>>(algos, num_lanes);
#endif // HWY_TARGET != HWY_SCALAR
#endif // !HAVE_VXSORT && !HAVE_INTEL
}
void TestAllSort() {
const std::vector<Algo> algos{
#if HAVE_AVX2SORT
Algo::kSEA,
#endif
#if HAVE_IPS4O
Algo::kIPS4O,
#endif
#if HAVE_PDQSORT
Algo::kPDQ,
#endif
#if HAVE_SORT512
Algo::kSort512,
#endif
Algo::kVQSort, Algo::kHeapSort,
};
for (int num : {129, 504, 3 * 1000, 34567}) {
const size_t num_lanes = AdjustedReps(static_cast<size_t>(num));
CallAllSortTraits(algos, num_lanes);
}
}
void TestAllPartialSort() {
const std::vector<Algo> algos{Algo::kVQPartialSort, Algo::kHeapPartialSort};
for (int num : {129, 504, 3 * 1000, 34567}) {
const size_t num_lanes = AdjustedReps(static_cast<size_t>(num));
CallAllSortTraits(algos, num_lanes);
}
}
void TestAllSelect() {
const std::vector<Algo> algos{Algo::kVQSelect, Algo::kHeapSelect};
for (int num : {129, 504, 3 * 1000, 34567}) {
const size_t num_lanes = AdjustedReps(static_cast<size_t>(num));
CallAllSortTraits(algos, num_lanes);
}
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(SortTest);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllSortIota);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllSort);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllSelect);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllPartialSort);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

574
deps/v8/third_party/highway/src/hwy/contrib/sort/sort_unit_test.cc vendored Normal file
View File

@ -0,0 +1,574 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdio.h>
#include <unordered_map>
#include <vector>
#include "hwy/aligned_allocator.h" // IsAligned
#include "hwy/base.h"
#include "hwy/contrib/sort/vqsort.h"
#include "hwy/detect_compiler_arch.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/sort_unit_test.cc" // NOLINT
// clang-format on
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
// After highway.h
#include "hwy/contrib/sort/algo-inl.h"
#include "hwy/contrib/sort/result-inl.h"
#include "hwy/contrib/sort/traits128-inl.h"
#include "hwy/contrib/sort/vqsort-inl.h" // BaseCase
#include "hwy/print-inl.h"
#include "hwy/tests/test_util-inl.h"
// TODO(b/314758657): Compiler bug causes incorrect results on SSE2/S-SSE3.
#undef VQSORT_SKIP
#if !defined(VQSORT_DO_NOT_SKIP) && HWY_COMPILER_CLANG && HWY_ARCH_X86 && \
HWY_TARGET >= HWY_SSSE3
#define VQSORT_SKIP 1
#else
#define VQSORT_SKIP 0
#endif
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
using detail::OrderAscending;
using detail::SharedTraits;
using detail::TraitsLane;
#if !HAVE_INTEL && HWY_TARGET != HWY_SCALAR
using detail::OrderAscending128;
using detail::OrderDescending128;
using detail::Traits128;
#endif // !HAVE_INTEL && HWY_TARGET != HWY_SCALAR
#if VQSORT_ENABLED || HWY_IDE
// Verify the corner cases of LargerSortValue/SmallerSortValue, used to
// implement PrevValue/NextValue.
struct TestFloatLargerSmaller {
template <typename T, class D>
HWY_NOINLINE void operator()(T, D d) {
const Vec<D> p0 = Zero(d);
const Vec<D> p1 = Set(d, ConvertScalarTo<T>(1));
const Vec<D> pinf = Inf(d);
const Vec<D> peps = Set(d, hwy::Epsilon<T>());
const Vec<D> pmax = Set(d, hwy::HighestValue<T>());
const Vec<D> n0 = Neg(p0);
const Vec<D> n1 = Neg(p1);
const Vec<D> ninf = Neg(pinf);
const Vec<D> neps = Neg(peps);
const Vec<D> nmax = Neg(pmax);
// Larger(0) is the smallest subnormal, typically eps * FLT_MIN.
const RebindToUnsigned<D> du;
const Vec<D> psub = BitCast(d, Set(du, 1));
const Vec<D> nsub = Neg(psub);
HWY_ASSERT(AllTrue(d, Lt(psub, peps)));
HWY_ASSERT(AllTrue(d, Gt(nsub, neps)));
// +/-0 moves to +/- smallest subnormal.
HWY_ASSERT_VEC_EQ(d, psub, detail::LargerSortValue(d, p0));
HWY_ASSERT_VEC_EQ(d, nsub, detail::SmallerSortValue(d, p0));
HWY_ASSERT_VEC_EQ(d, psub, detail::LargerSortValue(d, n0));
HWY_ASSERT_VEC_EQ(d, nsub, detail::SmallerSortValue(d, n0));
// The next magnitude larger than 1 is (1 + eps) by definition.
HWY_ASSERT_VEC_EQ(d, Add(p1, peps), detail::LargerSortValue(d, p1));
HWY_ASSERT_VEC_EQ(d, Add(n1, neps), detail::SmallerSortValue(d, n1));
// 1-eps and -1+eps are slightly different, but we can still ensure the
// next values are less than 1 / greater than -1.
HWY_ASSERT(AllTrue(d, Gt(p1, detail::SmallerSortValue(d, p1))));
HWY_ASSERT(AllTrue(d, Lt(n1, detail::LargerSortValue(d, n1))));
// Even for large (finite) values, we can move toward/away from infinity.
HWY_ASSERT_VEC_EQ(d, pinf, detail::LargerSortValue(d, pmax));
HWY_ASSERT_VEC_EQ(d, ninf, detail::SmallerSortValue(d, nmax));
HWY_ASSERT(AllTrue(d, Gt(pmax, detail::SmallerSortValue(d, pmax))));
HWY_ASSERT(AllTrue(d, Lt(nmax, detail::LargerSortValue(d, nmax))));
// For infinities, results are unchanged or the extremal finite value.
HWY_ASSERT_VEC_EQ(d, pinf, detail::LargerSortValue(d, pinf));
HWY_ASSERT_VEC_EQ(d, pmax, detail::SmallerSortValue(d, pinf));
HWY_ASSERT_VEC_EQ(d, nmax, detail::LargerSortValue(d, ninf));
HWY_ASSERT_VEC_EQ(d, ninf, detail::SmallerSortValue(d, ninf));
}
};
HWY_NOINLINE void TestAllFloatLargerSmaller() {
ForFloatTypesDynamic(ForPartialVectors<TestFloatLargerSmaller>());
}
// Previously, LastValue was the largest normal float, so we injected that
// value into arrays containing only infinities. Ensure that does not happen.
struct TestFloatInf {
template <typename T, class D>
HWY_NOINLINE void operator()(T, D d) {
const size_t N = Lanes(d);
const size_t num = N * 3;
auto in = hwy::AllocateAligned<T>(num);
HWY_ASSERT(in);
Fill(d, GetLane(Inf(d)), num, in.get());
VQSort(in.get(), num, SortAscending());
for (size_t i = 0; i < num; i += N) {
HWY_ASSERT(AllTrue(d, IsInf(LoadU(d, in.get() + i))));
}
}
};
HWY_NOINLINE void TestAllFloatInf() {
// TODO(janwas): bfloat16_t not yet supported.
ForFloatTypesDynamic(ForPartialVectors<TestFloatInf>());
}
template <class Traits>
static HWY_NOINLINE void TestMedian3() {
using LaneType = typename Traits::LaneType;
using D = CappedTag<LaneType, 1>;
SharedTraits<Traits> st;
const D d;
using V = Vec<D>;
for (uint32_t bits = 0; bits < 8; ++bits) {
const V v0 = Set(d, LaneType{(bits & (1u << 0)) ? 1u : 0u});
const V v1 = Set(d, LaneType{(bits & (1u << 1)) ? 1u : 0u});
const V v2 = Set(d, LaneType{(bits & (1u << 2)) ? 1u : 0u});
const LaneType m = GetLane(detail::MedianOf3(st, v0, v1, v2));
// If at least half(rounded up) of bits are 1, so is the median.
const size_t count = PopCount(bits);
HWY_ASSERT_EQ((count >= 2) ? static_cast<LaneType>(1) : 0, m);
}
}
HWY_NOINLINE void TestAllMedian() {
TestMedian3<TraitsLane<OrderAscending<uint64_t> > >();
}
template <class Traits>
static HWY_NOINLINE void TestBaseCaseAscDesc() {
using LaneType = typename Traits::LaneType;
SharedTraits<Traits> st;
const SortTag<LaneType> d;
const size_t N = Lanes(d);
constexpr size_t N1 = st.LanesPerKey();
const size_t base_case_num = SortConstants::BaseCaseNumLanes<N1>(N);
constexpr int kDebug = 0;
auto aligned_lanes = hwy::AllocateAligned<LaneType>(N + base_case_num + N);
auto buf = hwy::AllocateAligned<LaneType>(base_case_num + 2 * N);
HWY_ASSERT(aligned_lanes && buf);
std::vector<size_t> lengths;
lengths.push_back(HWY_MAX(1, N1));
lengths.push_back(3 * N1);
lengths.push_back(base_case_num / 2);
lengths.push_back(base_case_num / 2 + N1);
lengths.push_back(base_case_num - N1);
lengths.push_back(base_case_num);
std::vector<size_t> misalignments;
misalignments.push_back(0);
misalignments.push_back(1);
if (N >= 6) misalignments.push_back(N / 2 - 1);
misalignments.push_back(N / 2);
misalignments.push_back(N / 2 + 1);
misalignments.push_back(HWY_MIN(2 * N / 3 + 3, size_t{N - 1}));
for (bool asc : {false, true}) {
for (size_t len : lengths) {
for (size_t misalign : misalignments) {
LaneType* HWY_RESTRICT lanes = aligned_lanes.get() + misalign;
if (kDebug) {
printf("============%s asc %d N1 %d len %d misalign %d\n",
st.KeyString(), asc, static_cast<int>(N1),
static_cast<int>(len), static_cast<int>(misalign));
}
for (size_t i = 0; i < misalign; ++i) {
aligned_lanes[i] = hwy::LowestValue<LaneType>();
}
InputStats<LaneType> input_stats;
for (size_t i = 0; i < len; ++i) {
lanes[i] = asc ? static_cast<LaneType>(LaneType(i) + 1)
: static_cast<LaneType>(LaneType(len) - LaneType(i));
input_stats.Notify(lanes[i]);
if (kDebug >= 2) {
printf("%3zu: %f\n", i, static_cast<double>(lanes[i]));
}
}
for (size_t i = len; i < base_case_num + N; ++i) {
lanes[i] = hwy::LowestValue<LaneType>();
}
detail::BaseCase(d, st, lanes, len, buf.get());
if (kDebug >= 2) {
printf("out>>>>>>\n");
for (size_t i = 0; i < len; ++i) {
printf("%3zu: %f\n", i, static_cast<double>(lanes[i]));
}
}
SortOrderVerifier<Traits>()(Algo::kVQSort, input_stats, lanes, len / N1,
len / N1);
for (size_t i = 0; i < misalign; ++i) {
if (aligned_lanes[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun misalign at %d\n", static_cast<int>(i));
}
for (size_t i = len; i < base_case_num + N; ++i) {
if (lanes[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun right at %d\n", static_cast<int>(i));
}
} // misalign
} // len
} // asc
}
template <class Traits>
static HWY_NOINLINE void TestBaseCase01() {
using LaneType = typename Traits::LaneType;
SharedTraits<Traits> st;
const SortTag<LaneType> d;
const size_t N = Lanes(d);
constexpr size_t N1 = st.LanesPerKey();
const size_t base_case_num = SortConstants::BaseCaseNumLanes<N1>(N);
constexpr int kDebug = 0;
auto lanes = hwy::AllocateAligned<LaneType>(base_case_num + N);
auto buf = hwy::AllocateAligned<LaneType>(base_case_num + 2 * N);
HWY_ASSERT(lanes && buf);
std::vector<size_t> lengths;
lengths.push_back(HWY_MAX(1, N1));
lengths.push_back(3 * N1);
lengths.push_back(base_case_num / 2);
lengths.push_back(base_case_num / 2 + N1);
lengths.push_back(base_case_num - N1);
lengths.push_back(base_case_num);
for (size_t len : lengths) {
if (kDebug) {
printf("============%s 01 N1 %d len %d\n", st.KeyString(),
static_cast<int>(N1), static_cast<int>(len));
}
const uint64_t kMaxBits = AdjustedLog2Reps(HWY_MIN(len, size_t{14}));
for (uint64_t bits = 0; bits < ((1ull << kMaxBits) - 1); ++bits) {
InputStats<LaneType> input_stats;
for (size_t i = 0; i < len; ++i) {
lanes[i] = (i < 64 && (bits & (1ull << i))) ? 1 : 0;
input_stats.Notify(lanes[i]);
if (kDebug >= 2) {
printf("%3zu: %f\n", i, static_cast<double>(lanes[i]));
}
}
for (size_t i = len; i < base_case_num + N; ++i) {
lanes[i] = hwy::LowestValue<LaneType>();
}
detail::BaseCase(d, st, lanes.get(), len, buf.get());
if (kDebug >= 2) {
printf("out>>>>>>\n");
for (size_t i = 0; i < len; ++i) {
printf("%3zu: %f\n", i, static_cast<double>(lanes[i]));
}
}
SortOrderVerifier<Traits>()(Algo::kVQSort, input_stats, lanes.get(),
len / N1, len / N1);
for (size_t i = len; i < base_case_num + N; ++i) {
if (lanes[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun right at %d\n", static_cast<int>(i));
}
} // bits
} // len
}
template <class Traits>
static HWY_NOINLINE void TestBaseCase() {
TestBaseCaseAscDesc<Traits>();
TestBaseCase01<Traits>();
}
HWY_NOINLINE void TestAllBaseCase() {
// Workaround for stack overflow on MSVC debug.
#if defined(_MSC_VER) || VQSORT_SKIP
return;
#endif
TestBaseCase<TraitsLane<OrderAscending<int32_t> > >();
TestBaseCase<TraitsLane<OtherOrder<int64_t> > >();
#if !HAVE_INTEL
TestBaseCase<Traits128<OrderAscending128> >();
TestBaseCase<Traits128<OrderDescending128> >();
#endif
}
template <class Traits>
static HWY_NOINLINE void VerifyPartition(
Traits st, typename Traits::LaneType* HWY_RESTRICT lanes, size_t left,
size_t border, size_t right, const size_t N1,
const typename Traits::LaneType* pivot) {
/* for (size_t i = left; i < right; ++i) {
if (i == border) printf("--\n");
printf("%4zu: %3d\n", i, lanes[i]);
}*/
HWY_ASSERT(left % N1 == 0);
HWY_ASSERT(border % N1 == 0);
HWY_ASSERT(right % N1 == 0);
constexpr bool kAscending = Traits::Order::IsAscending();
for (size_t i = left; i < border; i += N1) {
if (st.Compare1(pivot, lanes + i)) {
HWY_ABORT(
"%s: asc %d left[%d] piv %.0f %.0f compares before %.0f %.0f "
"border %d",
st.KeyString(), kAscending, static_cast<int>(i),
static_cast<double>(pivot[1]), static_cast<double>(pivot[0]),
static_cast<double>(lanes[i + 1]), static_cast<double>(lanes[i + 0]),
static_cast<int>(border));
}
}
for (size_t i = border; i < right; i += N1) {
if (!st.Compare1(pivot, lanes + i)) {
HWY_ABORT(
"%s: asc %d right[%d] piv %.0f %.0f compares after %.0f %.0f "
"border %d",
st.KeyString(), kAscending, static_cast<int>(i),
static_cast<double>(pivot[1]), static_cast<double>(pivot[0]),
static_cast<double>(lanes[i + 1]), static_cast<double>(lanes[i]),
static_cast<int>(border));
}
}
}
template <class Traits>
static HWY_NOINLINE void TestPartition() {
using LaneType = typename Traits::LaneType;
// See HandleSpecialCases and HWY_ASSERT below.
const CappedTag<LaneType, 64 / sizeof(LaneType)> d;
SharedTraits<Traits> st;
constexpr bool kAscending = Traits::Order::IsAscending();
const size_t N = Lanes(d);
constexpr int kDebug = 0;
constexpr size_t N1 = st.LanesPerKey();
const size_t base_case_num = SortConstants::BaseCaseNumLanes<N1>(N);
HWY_ASSERT(2 * N <= base_case_num); // See HandleSpecialCases
// left + len + align
const size_t total = 32 + (base_case_num + 4 * HWY_MAX(N, 4)) + 2 * N;
auto aligned_lanes = hwy::AllocateAligned<LaneType>(total);
HWY_ASSERT(aligned_lanes);
HWY_ALIGN LaneType buf[SortConstants::BufBytes<LaneType, N1>(HWY_MAX_BYTES) /
sizeof(LaneType)];
for (bool in_asc : {false, true}) {
for (int left_i : {0, 1, 7, 8, 30, 31}) {
const size_t left = static_cast<size_t>(left_i) & ~(N1 - 1);
for (size_t ofs :
{N, N + 3, 2 * N, 2 * N + 2, 2 * N + 3, 3 * N - 1, 4 * N - 2}) {
const size_t len = (base_case_num + ofs) & ~(N1 - 1);
for (LaneType pivot1 : {LaneType(0), LaneType(len / 3),
LaneType(2 * len / 3), LaneType(len)}) {
const LaneType pivot2[2] = {pivot1, 0};
const auto pivot = st.SetKey(d, pivot2);
for (size_t misalign = 0; misalign < N; misalign += N1) {
LaneType* HWY_RESTRICT lanes = aligned_lanes.get() + misalign;
const size_t right = left + len;
if (kDebug) {
printf(
"=========%s asc %d left %d len %d right %d piv %.0f %.0f\n",
st.KeyString(), kAscending, static_cast<int>(left),
static_cast<int>(len), static_cast<int>(right),
static_cast<double>(pivot2[1]),
static_cast<double>(pivot2[0]));
}
for (size_t i = 0; i < misalign; ++i) {
aligned_lanes[i] = hwy::LowestValue<LaneType>();
}
for (size_t i = 0; i < left; ++i) {
lanes[i] = hwy::LowestValue<LaneType>();
}
std::unordered_map<LaneType, int> counts;
for (size_t i = left; i < right; ++i) {
lanes[i] = static_cast<LaneType>(
in_asc ? LaneType(i + 1) - static_cast<LaneType>(left)
: static_cast<LaneType>(right) - LaneType(i));
++counts[lanes[i]];
if (kDebug >= 2) {
printf("%3zu: %f\n", i, static_cast<double>(lanes[i]));
}
}
for (size_t i = right; i < total - misalign; ++i) {
lanes[i] = hwy::LowestValue<LaneType>();
}
size_t border = left + detail::Partition(d, st, lanes + left,
right - left, pivot, buf);
if (kDebug >= 2) {
printf("out>>>>>>\n");
for (size_t i = left; i < right; ++i) {
printf("%3zu: %f\n", i, static_cast<double>(lanes[i]));
}
for (size_t i = right; i < total - misalign; ++i) {
printf("%3zu: sentinel %f\n", i, static_cast<double>(lanes[i]));
}
}
for (size_t i = left; i < right; ++i) {
--counts[lanes[i]];
}
for (auto kv : counts) {
if (kv.second != 0) {
PrintValue(kv.first);
HWY_ABORT("Incorrect count %d\n", kv.second);
}
}
VerifyPartition(st, lanes, left, border, right, N1, pivot2);
for (size_t i = 0; i < misalign; ++i) {
if (aligned_lanes[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun misalign at %d\n", static_cast<int>(i));
}
for (size_t i = 0; i < left; ++i) {
if (lanes[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun left at %d\n", static_cast<int>(i));
}
for (size_t i = right; i < total - misalign; ++i) {
if (lanes[i] != hwy::LowestValue<LaneType>())
HWY_ABORT("Overrun right at %d\n", static_cast<int>(i));
}
} // misalign
} // pivot
} // len
} // left
} // asc
}
#undef HWY_BROKEN_U128
#if HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1400 && \
HWY_TARGET == HWY_RVV
#define HWY_BROKEN_U128 1
#else
#define HWY_BROKEN_U128 0
#endif
HWY_NOINLINE void TestAllPartition() {
TestPartition<TraitsLane<OtherOrder<int32_t> > >();
#if !HAVE_INTEL && !HWY_BROKEN_U128
TestPartition<Traits128<OrderAscending128> >();
#endif
#if !HWY_IS_DEBUG_BUILD
TestPartition<TraitsLane<OrderAscending<int16_t> > >();
TestPartition<TraitsLane<OrderAscending<int64_t> > >();
TestPartition<TraitsLane<OtherOrder<float> > >();
// OK to check current target, not using dynamic dispatch here.
#if HWY_HAVE_FLOAT64
TestPartition<TraitsLane<OtherOrder<double> > >();
#endif
#if !HAVE_INTEL && !HWY_BROKEN_U128
TestPartition<Traits128<OrderDescending128> >();
#endif
#endif
}
// (used for sample selection for choosing a pivot)
template <typename TU>
static HWY_NOINLINE void TestRandomGenerator() {
static_assert(!hwy::IsSigned<TU>(), "");
SortTag<TU> du;
const size_t N = Lanes(du);
uint64_t* state = GetGeneratorState();
// Ensure lower and upper 32 bits are uniformly distributed.
uint64_t sum_lo = 0, sum_hi = 0;
for (size_t i = 0; i < 1000; ++i) {
const uint64_t bits = detail::RandomBits(state);
sum_lo += bits & 0xFFFFFFFF;
sum_hi += bits >> 32;
}
const double expected = 1000 * (1ULL << 31);
HWY_ASSERT(0.9 * expected <= static_cast<double>(sum_lo) &&
static_cast<double>(sum_lo) <= 1.1 * expected);
HWY_ASSERT(0.9 * expected <= static_cast<double>(sum_hi) &&
static_cast<double>(sum_hi) <= 1.1 * expected);
const size_t lanes_per_block = HWY_MAX(64 / sizeof(TU), N); // power of two
for (uint32_t num_blocks = 2; num_blocks < 100000;
num_blocks = 3 * num_blocks / 2) {
// Generate some numbers and ensure all are in range
uint64_t sum = 0;
constexpr size_t kReps = 10000;
for (size_t rep = 0; rep < kReps; ++rep) {
const uint32_t bits = detail::RandomBits(state) & 0xFFFFFFFF;
const size_t index = detail::RandomChunkIndex(num_blocks, bits);
HWY_ASSERT(((index + 1) * lanes_per_block) <=
num_blocks * lanes_per_block);
sum += index;
}
// Also ensure the mean is near the middle of the range
const double expected = (num_blocks - 1) / 2.0;
const double actual = static_cast<double>(sum) / kReps;
HWY_ASSERT(0.9 * expected <= actual && actual <= 1.1 * expected);
}
}
HWY_NOINLINE void TestAllGenerator() {
TestRandomGenerator<uint32_t>();
TestRandomGenerator<uint64_t>();
}
#else
static void TestAllFloatLargerSmaller() {}
static void TestAllFloatInf() {}
static void TestAllMedian() {}
static void TestAllBaseCase() {}
static void TestAllPartition() {}
static void TestAllGenerator() {}
#endif // VQSORT_ENABLED
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(SortTest);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllFloatLargerSmaller);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllFloatInf);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllMedian);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllBaseCase);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllPartition);
HWY_EXPORT_AND_TEST_P(SortTest, TestAllGenerator);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

902
deps/v8/third_party/highway/src/hwy/contrib/sort/sorting_networks-inl.h vendored Normal file
View File

@ -0,0 +1,902 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Per-target
#if defined(HIGHWAY_HWY_CONTRIB_SORT_SORTING_NETWORKS_TOGGLE) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_SORT_SORTING_NETWORKS_TOGGLE
#undef HIGHWAY_HWY_CONTRIB_SORT_SORTING_NETWORKS_TOGGLE
#else
#define HIGHWAY_HWY_CONTRIB_SORT_SORTING_NETWORKS_TOGGLE
#endif
#include "hwy/contrib/sort/shared-inl.h" // SortConstants
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace detail {
#if VQSORT_ENABLED
using Constants = hwy::SortConstants;
// ------------------------------ SharedTraits
// Code shared between all traits. It's unclear whether these can profitably be
// specialized for Lane vs Block, or optimized like SortPairsDistance1 using
// Compare/DupOdd.
template <class Base>
struct SharedTraits : public Base {
using SharedTraitsForSortingNetwork =
SharedTraits<typename Base::TraitsForSortingNetwork>;
// Conditionally swaps lane 0 with 2, 1 with 3 etc.
template <class D>
HWY_INLINE Vec<D> SortPairsDistance2(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->SwapAdjacentPairs(d, v);
base->Sort2(d, v, swapped);
return base->OddEvenPairs(d, swapped, v);
}
// Swaps with the vector formed by reversing contiguous groups of 8 keys.
template <class D>
HWY_INLINE Vec<D> SortPairsReverse8(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->ReverseKeys8(d, v);
base->Sort2(d, v, swapped);
return base->OddEvenQuads(d, swapped, v);
}
// Swaps with the vector formed by reversing contiguous groups of 8 keys.
template <class D>
HWY_INLINE Vec<D> SortPairsReverse16(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
static_assert(Constants::kMaxCols <= 16, "Need actual Reverse16");
Vec<D> swapped = base->ReverseKeys(d, v);
base->Sort2(d, v, swapped);
return ConcatUpperLower(d, swapped, v); // 8 = half of the vector
}
};
// ------------------------------ Sorting network
// Sorting networks for independent columns in 2, 4 and 8 vectors from
// https://bertdobbelaere.github.io/sorting_networks.html.
template <class D, class Traits, class V = Vec<D>>
HWY_INLINE void Sort2(D d, Traits st, V& v0, V& v1) {
st.Sort2(d, v0, v1);
}
template <class D, class Traits, class V = Vec<D>>
HWY_INLINE void Sort4(D d, Traits st, V& v0, V& v1, V& v2, V& v3) {
st.Sort2(d, v0, v2);
st.Sort2(d, v1, v3);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v1, v2);
}
template <class D, class Traits, class V = Vec<D>>
HWY_INLINE void Sort8(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4, V& v5,
V& v6, V& v7) {
st.Sort2(d, v0, v2);
st.Sort2(d, v1, v3);
st.Sort2(d, v4, v6);
st.Sort2(d, v5, v7);
st.Sort2(d, v0, v4);
st.Sort2(d, v1, v5);
st.Sort2(d, v2, v6);
st.Sort2(d, v3, v7);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
st.Sort2(d, v2, v4);
st.Sort2(d, v3, v5);
st.Sort2(d, v1, v4);
st.Sort2(d, v3, v6);
st.Sort2(d, v1, v2);
st.Sort2(d, v3, v4);
st.Sort2(d, v5, v6);
}
// (Green's irregular) sorting network for independent columns in 16 vectors.
template <class D, class Traits, class V = Vec<D>>
HWY_INLINE void Sort16(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4, V& v5,
V& v6, V& v7, V& v8, V& v9, V& va, V& vb, V& vc, V& vd,
V& ve, V& vf) {
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
st.Sort2(d, v8, v9);
st.Sort2(d, va, vb);
st.Sort2(d, vc, vd);
st.Sort2(d, ve, vf);
st.Sort2(d, v0, v2);
st.Sort2(d, v1, v3);
st.Sort2(d, v4, v6);
st.Sort2(d, v5, v7);
st.Sort2(d, v8, va);
st.Sort2(d, v9, vb);
st.Sort2(d, vc, ve);
st.Sort2(d, vd, vf);
st.Sort2(d, v0, v4);
st.Sort2(d, v1, v5);
st.Sort2(d, v2, v6);
st.Sort2(d, v3, v7);
st.Sort2(d, v8, vc);
st.Sort2(d, v9, vd);
st.Sort2(d, va, ve);
st.Sort2(d, vb, vf);
st.Sort2(d, v0, v8);
st.Sort2(d, v1, v9);
st.Sort2(d, v2, va);
st.Sort2(d, v3, vb);
st.Sort2(d, v4, vc);
st.Sort2(d, v5, vd);
st.Sort2(d, v6, ve);
st.Sort2(d, v7, vf);
st.Sort2(d, v5, va);
st.Sort2(d, v6, v9);
st.Sort2(d, v3, vc);
st.Sort2(d, v7, vb);
st.Sort2(d, vd, ve);
st.Sort2(d, v4, v8);
st.Sort2(d, v1, v2);
st.Sort2(d, v1, v4);
st.Sort2(d, v7, vd);
st.Sort2(d, v2, v8);
st.Sort2(d, vb, ve);
st.Sort2(d, v2, v4);
st.Sort2(d, v5, v6);
st.Sort2(d, v9, va);
st.Sort2(d, vb, vd);
st.Sort2(d, v3, v8);
st.Sort2(d, v7, vc);
st.Sort2(d, v3, v5);
st.Sort2(d, v6, v8);
st.Sort2(d, v7, v9);
st.Sort2(d, va, vc);
st.Sort2(d, v3, v4);
st.Sort2(d, v5, v6);
st.Sort2(d, v7, v8);
st.Sort2(d, v9, va);
st.Sort2(d, vb, vc);
st.Sort2(d, v6, v7);
st.Sort2(d, v8, v9);
}
// ------------------------------ Merging networks
// Blacher's hybrid bitonic/odd-even networks, generated by print_network.cc.
// For acceptable performance, these must be inlined, otherwise vectors are
// loaded from the stack. The kKeysPerVector allows calling from generic code
// but skipping the functions when vectors have too few lanes for
// st.SortPairsDistance1 to compile. `if constexpr` in the caller would also
// work, but is not available in C++11. We write out the (unused) argument types
// rather than `...` because GCC 9 (but not 10) fails to compile with `...`.
template <size_t kKeysPerVector, class D, class Traits, class V,
HWY_IF_LANES_LE(kKeysPerVector, 1)>
HWY_INLINE void Merge8x2(D, Traits, V, V, V, V, V, V, V, V) {}
template <size_t kKeysPerVector, class D, class Traits, class V,
HWY_IF_LANES_LE(kKeysPerVector, 2)>
HWY_INLINE void Merge8x4(D, Traits, V, V, V, V, V, V, V, V) {}
template <size_t kKeysPerVector, class D, class Traits, class V,
HWY_IF_LANES_LE(kKeysPerVector, 1)>
HWY_INLINE void Merge16x2(D, Traits, V, V, V, V, V, V, V, V, V, V, V, V, V, V,
V, V) {}
template <size_t kKeysPerVector, class D, class Traits, class V,
HWY_IF_LANES_LE(kKeysPerVector, 2)>
HWY_INLINE void Merge16x4(D, Traits, V, V, V, V, V, V, V, V, V, V, V, V, V, V,
V, V) {}
template <size_t kKeysPerVector, class D, class Traits, class V,
HWY_IF_LANES_LE(kKeysPerVector, 4)>
HWY_INLINE void Merge16x8(D, Traits, V, V, V, V, V, V, V, V, V, V, V, V, V, V,
V, V) {}
template <size_t kKeysPerVector, class D, class Traits, class V,
HWY_IF_LANES_LE(kKeysPerVector, 8)>
HWY_INLINE void Merge16x16(D, Traits, V, V, V, V, V, V, V, V, V, V, V, V, V, V,
V, V) {}
template <size_t kKeysPerVector, class D, class Traits, class V = Vec<D>,
HWY_IF_LANES_GT(kKeysPerVector, 1)>
HWY_INLINE void Merge8x2(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4,
V& v5, V& v6, V& v7) {
v7 = st.ReverseKeys2(d, v7);
v6 = st.ReverseKeys2(d, v6);
v5 = st.ReverseKeys2(d, v5);
v4 = st.ReverseKeys2(d, v4);
st.Sort2(d, v0, v7);
st.Sort2(d, v1, v6);
st.Sort2(d, v2, v5);
st.Sort2(d, v3, v4);
v3 = st.ReverseKeys2(d, v3);
v2 = st.ReverseKeys2(d, v2);
v7 = st.ReverseKeys2(d, v7);
v6 = st.ReverseKeys2(d, v6);
st.Sort2(d, v0, v3);
st.Sort2(d, v1, v2);
st.Sort2(d, v4, v7);
st.Sort2(d, v5, v6);
v1 = st.ReverseKeys2(d, v1);
v3 = st.ReverseKeys2(d, v3);
v5 = st.ReverseKeys2(d, v5);
v7 = st.ReverseKeys2(d, v7);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
v0 = st.SortPairsDistance1(d, v0);
v1 = st.SortPairsDistance1(d, v1);
v2 = st.SortPairsDistance1(d, v2);
v3 = st.SortPairsDistance1(d, v3);
v4 = st.SortPairsDistance1(d, v4);
v5 = st.SortPairsDistance1(d, v5);
v6 = st.SortPairsDistance1(d, v6);
v7 = st.SortPairsDistance1(d, v7);
}
template <size_t kKeysPerVector, class D, class Traits, class V = Vec<D>,
HWY_IF_LANES_GT(kKeysPerVector, 2)>
HWY_INLINE void Merge8x4(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4,
V& v5, V& v6, V& v7) {
v7 = st.ReverseKeys4(d, v7);
v6 = st.ReverseKeys4(d, v6);
v5 = st.ReverseKeys4(d, v5);
v4 = st.ReverseKeys4(d, v4);
st.Sort2(d, v0, v7);
st.Sort2(d, v1, v6);
st.Sort2(d, v2, v5);
st.Sort2(d, v3, v4);
v3 = st.ReverseKeys4(d, v3);
v2 = st.ReverseKeys4(d, v2);
v7 = st.ReverseKeys4(d, v7);
v6 = st.ReverseKeys4(d, v6);
st.Sort2(d, v0, v3);
st.Sort2(d, v1, v2);
st.Sort2(d, v4, v7);
st.Sort2(d, v5, v6);
v1 = st.ReverseKeys4(d, v1);
v3 = st.ReverseKeys4(d, v3);
v5 = st.ReverseKeys4(d, v5);
v7 = st.ReverseKeys4(d, v7);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
v0 = st.SortPairsReverse4(d, v0);
v1 = st.SortPairsReverse4(d, v1);
v2 = st.SortPairsReverse4(d, v2);
v3 = st.SortPairsReverse4(d, v3);
v4 = st.SortPairsReverse4(d, v4);
v5 = st.SortPairsReverse4(d, v5);
v6 = st.SortPairsReverse4(d, v6);
v7 = st.SortPairsReverse4(d, v7);
v0 = st.SortPairsDistance1(d, v0);
v1 = st.SortPairsDistance1(d, v1);
v2 = st.SortPairsDistance1(d, v2);
v3 = st.SortPairsDistance1(d, v3);
v4 = st.SortPairsDistance1(d, v4);
v5 = st.SortPairsDistance1(d, v5);
v6 = st.SortPairsDistance1(d, v6);
v7 = st.SortPairsDistance1(d, v7);
}
// Only used by the now-deprecated SortingNetwork().
template <size_t kKeysPerVector, class D, class Traits, class V = Vec<D>,
HWY_IF_LANES_GT(kKeysPerVector, 1)>
HWY_INLINE void Merge16x2(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4,
V& v5, V& v6, V& v7, V& v8, V& v9, V& va, V& vb,
V& vc, V& vd, V& ve, V& vf) {
vf = st.ReverseKeys2(d, vf);
ve = st.ReverseKeys2(d, ve);
vd = st.ReverseKeys2(d, vd);
vc = st.ReverseKeys2(d, vc);
vb = st.ReverseKeys2(d, vb);
va = st.ReverseKeys2(d, va);
v9 = st.ReverseKeys2(d, v9);
v8 = st.ReverseKeys2(d, v8);
st.Sort2(d, v0, vf);
st.Sort2(d, v1, ve);
st.Sort2(d, v2, vd);
st.Sort2(d, v3, vc);
st.Sort2(d, v4, vb);
st.Sort2(d, v5, va);
st.Sort2(d, v6, v9);
st.Sort2(d, v7, v8);
v7 = st.ReverseKeys2(d, v7);
v6 = st.ReverseKeys2(d, v6);
v5 = st.ReverseKeys2(d, v5);
v4 = st.ReverseKeys2(d, v4);
vf = st.ReverseKeys2(d, vf);
ve = st.ReverseKeys2(d, ve);
vd = st.ReverseKeys2(d, vd);
vc = st.ReverseKeys2(d, vc);
st.Sort2(d, v0, v7);
st.Sort2(d, v1, v6);
st.Sort2(d, v2, v5);
st.Sort2(d, v3, v4);
st.Sort2(d, v8, vf);
st.Sort2(d, v9, ve);
st.Sort2(d, va, vd);
st.Sort2(d, vb, vc);
v3 = st.ReverseKeys2(d, v3);
v2 = st.ReverseKeys2(d, v2);
v7 = st.ReverseKeys2(d, v7);
v6 = st.ReverseKeys2(d, v6);
vb = st.ReverseKeys2(d, vb);
va = st.ReverseKeys2(d, va);
vf = st.ReverseKeys2(d, vf);
ve = st.ReverseKeys2(d, ve);
st.Sort2(d, v0, v3);
st.Sort2(d, v1, v2);
st.Sort2(d, v4, v7);
st.Sort2(d, v5, v6);
st.Sort2(d, v8, vb);
st.Sort2(d, v9, va);
st.Sort2(d, vc, vf);
st.Sort2(d, vd, ve);
v1 = st.ReverseKeys2(d, v1);
v3 = st.ReverseKeys2(d, v3);
v5 = st.ReverseKeys2(d, v5);
v7 = st.ReverseKeys2(d, v7);
v9 = st.ReverseKeys2(d, v9);
vb = st.ReverseKeys2(d, vb);
vd = st.ReverseKeys2(d, vd);
vf = st.ReverseKeys2(d, vf);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
st.Sort2(d, v8, v9);
st.Sort2(d, va, vb);
st.Sort2(d, vc, vd);
st.Sort2(d, ve, vf);
v0 = st.SortPairsDistance1(d, v0);
v1 = st.SortPairsDistance1(d, v1);
v2 = st.SortPairsDistance1(d, v2);
v3 = st.SortPairsDistance1(d, v3);
v4 = st.SortPairsDistance1(d, v4);
v5 = st.SortPairsDistance1(d, v5);
v6 = st.SortPairsDistance1(d, v6);
v7 = st.SortPairsDistance1(d, v7);
v8 = st.SortPairsDistance1(d, v8);
v9 = st.SortPairsDistance1(d, v9);
va = st.SortPairsDistance1(d, va);
vb = st.SortPairsDistance1(d, vb);
vc = st.SortPairsDistance1(d, vc);
vd = st.SortPairsDistance1(d, vd);
ve = st.SortPairsDistance1(d, ve);
vf = st.SortPairsDistance1(d, vf);
}
template <size_t kKeysPerVector, class D, class Traits, class V = Vec<D>,
HWY_IF_LANES_GT(kKeysPerVector, 2)>
HWY_INLINE void Merge16x4(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4,
V& v5, V& v6, V& v7, V& v8, V& v9, V& va, V& vb,
V& vc, V& vd, V& ve, V& vf) {
vf = st.ReverseKeys4(d, vf);
ve = st.ReverseKeys4(d, ve);
vd = st.ReverseKeys4(d, vd);
vc = st.ReverseKeys4(d, vc);
vb = st.ReverseKeys4(d, vb);
va = st.ReverseKeys4(d, va);
v9 = st.ReverseKeys4(d, v9);
v8 = st.ReverseKeys4(d, v8);
st.Sort2(d, v0, vf);
st.Sort2(d, v1, ve);
st.Sort2(d, v2, vd);
st.Sort2(d, v3, vc);
st.Sort2(d, v4, vb);
st.Sort2(d, v5, va);
st.Sort2(d, v6, v9);
st.Sort2(d, v7, v8);
v7 = st.ReverseKeys4(d, v7);
v6 = st.ReverseKeys4(d, v6);
v5 = st.ReverseKeys4(d, v5);
v4 = st.ReverseKeys4(d, v4);
vf = st.ReverseKeys4(d, vf);
ve = st.ReverseKeys4(d, ve);
vd = st.ReverseKeys4(d, vd);
vc = st.ReverseKeys4(d, vc);
st.Sort2(d, v0, v7);
st.Sort2(d, v1, v6);
st.Sort2(d, v2, v5);
st.Sort2(d, v3, v4);
st.Sort2(d, v8, vf);
st.Sort2(d, v9, ve);
st.Sort2(d, va, vd);
st.Sort2(d, vb, vc);
v3 = st.ReverseKeys4(d, v3);
v2 = st.ReverseKeys4(d, v2);
v7 = st.ReverseKeys4(d, v7);
v6 = st.ReverseKeys4(d, v6);
vb = st.ReverseKeys4(d, vb);
va = st.ReverseKeys4(d, va);
vf = st.ReverseKeys4(d, vf);
ve = st.ReverseKeys4(d, ve);
st.Sort2(d, v0, v3);
st.Sort2(d, v1, v2);
st.Sort2(d, v4, v7);
st.Sort2(d, v5, v6);
st.Sort2(d, v8, vb);
st.Sort2(d, v9, va);
st.Sort2(d, vc, vf);
st.Sort2(d, vd, ve);
v1 = st.ReverseKeys4(d, v1);
v3 = st.ReverseKeys4(d, v3);
v5 = st.ReverseKeys4(d, v5);
v7 = st.ReverseKeys4(d, v7);
v9 = st.ReverseKeys4(d, v9);
vb = st.ReverseKeys4(d, vb);
vd = st.ReverseKeys4(d, vd);
vf = st.ReverseKeys4(d, vf);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
st.Sort2(d, v8, v9);
st.Sort2(d, va, vb);
st.Sort2(d, vc, vd);
st.Sort2(d, ve, vf);
v0 = st.SortPairsReverse4(d, v0);
v1 = st.SortPairsReverse4(d, v1);
v2 = st.SortPairsReverse4(d, v2);
v3 = st.SortPairsReverse4(d, v3);
v4 = st.SortPairsReverse4(d, v4);
v5 = st.SortPairsReverse4(d, v5);
v6 = st.SortPairsReverse4(d, v6);
v7 = st.SortPairsReverse4(d, v7);
v8 = st.SortPairsReverse4(d, v8);
v9 = st.SortPairsReverse4(d, v9);
va = st.SortPairsReverse4(d, va);
vb = st.SortPairsReverse4(d, vb);
vc = st.SortPairsReverse4(d, vc);
vd = st.SortPairsReverse4(d, vd);
ve = st.SortPairsReverse4(d, ve);
vf = st.SortPairsReverse4(d, vf);
v0 = st.SortPairsDistance1(d, v0);
v1 = st.SortPairsDistance1(d, v1);
v2 = st.SortPairsDistance1(d, v2);
v3 = st.SortPairsDistance1(d, v3);
v4 = st.SortPairsDistance1(d, v4);
v5 = st.SortPairsDistance1(d, v5);
v6 = st.SortPairsDistance1(d, v6);
v7 = st.SortPairsDistance1(d, v7);
v8 = st.SortPairsDistance1(d, v8);
v9 = st.SortPairsDistance1(d, v9);
va = st.SortPairsDistance1(d, va);
vb = st.SortPairsDistance1(d, vb);
vc = st.SortPairsDistance1(d, vc);
vd = st.SortPairsDistance1(d, vd);
ve = st.SortPairsDistance1(d, ve);
vf = st.SortPairsDistance1(d, vf);
}
template <size_t kKeysPerVector, class D, class Traits, class V = Vec<D>,
HWY_IF_LANES_GT(kKeysPerVector, 4)>
HWY_INLINE void Merge16x8(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4,
V& v5, V& v6, V& v7, V& v8, V& v9, V& va, V& vb,
V& vc, V& vd, V& ve, V& vf) {
vf = st.ReverseKeys8(d, vf);
ve = st.ReverseKeys8(d, ve);
vd = st.ReverseKeys8(d, vd);
vc = st.ReverseKeys8(d, vc);
vb = st.ReverseKeys8(d, vb);
va = st.ReverseKeys8(d, va);
v9 = st.ReverseKeys8(d, v9);
v8 = st.ReverseKeys8(d, v8);
st.Sort2(d, v0, vf);
st.Sort2(d, v1, ve);
st.Sort2(d, v2, vd);
st.Sort2(d, v3, vc);
st.Sort2(d, v4, vb);
st.Sort2(d, v5, va);
st.Sort2(d, v6, v9);
st.Sort2(d, v7, v8);
v7 = st.ReverseKeys8(d, v7);
v6 = st.ReverseKeys8(d, v6);
v5 = st.ReverseKeys8(d, v5);
v4 = st.ReverseKeys8(d, v4);
vf = st.ReverseKeys8(d, vf);
ve = st.ReverseKeys8(d, ve);
vd = st.ReverseKeys8(d, vd);
vc = st.ReverseKeys8(d, vc);
st.Sort2(d, v0, v7);
st.Sort2(d, v1, v6);
st.Sort2(d, v2, v5);
st.Sort2(d, v3, v4);
st.Sort2(d, v8, vf);
st.Sort2(d, v9, ve);
st.Sort2(d, va, vd);
st.Sort2(d, vb, vc);
v3 = st.ReverseKeys8(d, v3);
v2 = st.ReverseKeys8(d, v2);
v7 = st.ReverseKeys8(d, v7);
v6 = st.ReverseKeys8(d, v6);
vb = st.ReverseKeys8(d, vb);
va = st.ReverseKeys8(d, va);
vf = st.ReverseKeys8(d, vf);
ve = st.ReverseKeys8(d, ve);
st.Sort2(d, v0, v3);
st.Sort2(d, v1, v2);
st.Sort2(d, v4, v7);
st.Sort2(d, v5, v6);
st.Sort2(d, v8, vb);
st.Sort2(d, v9, va);
st.Sort2(d, vc, vf);
st.Sort2(d, vd, ve);
v1 = st.ReverseKeys8(d, v1);
v3 = st.ReverseKeys8(d, v3);
v5 = st.ReverseKeys8(d, v5);
v7 = st.ReverseKeys8(d, v7);
v9 = st.ReverseKeys8(d, v9);
vb = st.ReverseKeys8(d, vb);
vd = st.ReverseKeys8(d, vd);
vf = st.ReverseKeys8(d, vf);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
st.Sort2(d, v8, v9);
st.Sort2(d, va, vb);
st.Sort2(d, vc, vd);
st.Sort2(d, ve, vf);
v0 = st.SortPairsReverse8(d, v0);
v1 = st.SortPairsReverse8(d, v1);
v2 = st.SortPairsReverse8(d, v2);
v3 = st.SortPairsReverse8(d, v3);
v4 = st.SortPairsReverse8(d, v4);
v5 = st.SortPairsReverse8(d, v5);
v6 = st.SortPairsReverse8(d, v6);
v7 = st.SortPairsReverse8(d, v7);
v8 = st.SortPairsReverse8(d, v8);
v9 = st.SortPairsReverse8(d, v9);
va = st.SortPairsReverse8(d, va);
vb = st.SortPairsReverse8(d, vb);
vc = st.SortPairsReverse8(d, vc);
vd = st.SortPairsReverse8(d, vd);
ve = st.SortPairsReverse8(d, ve);
vf = st.SortPairsReverse8(d, vf);
v0 = st.SortPairsDistance2(d, v0);
v1 = st.SortPairsDistance2(d, v1);
v2 = st.SortPairsDistance2(d, v2);
v3 = st.SortPairsDistance2(d, v3);
v4 = st.SortPairsDistance2(d, v4);
v5 = st.SortPairsDistance2(d, v5);
v6 = st.SortPairsDistance2(d, v6);
v7 = st.SortPairsDistance2(d, v7);
v8 = st.SortPairsDistance2(d, v8);
v9 = st.SortPairsDistance2(d, v9);
va = st.SortPairsDistance2(d, va);
vb = st.SortPairsDistance2(d, vb);
vc = st.SortPairsDistance2(d, vc);
vd = st.SortPairsDistance2(d, vd);
ve = st.SortPairsDistance2(d, ve);
vf = st.SortPairsDistance2(d, vf);
v0 = st.SortPairsDistance1(d, v0);
v1 = st.SortPairsDistance1(d, v1);
v2 = st.SortPairsDistance1(d, v2);
v3 = st.SortPairsDistance1(d, v3);
v4 = st.SortPairsDistance1(d, v4);
v5 = st.SortPairsDistance1(d, v5);
v6 = st.SortPairsDistance1(d, v6);
v7 = st.SortPairsDistance1(d, v7);
v8 = st.SortPairsDistance1(d, v8);
v9 = st.SortPairsDistance1(d, v9);
va = st.SortPairsDistance1(d, va);
vb = st.SortPairsDistance1(d, vb);
vc = st.SortPairsDistance1(d, vc);
vd = st.SortPairsDistance1(d, vd);
ve = st.SortPairsDistance1(d, ve);
vf = st.SortPairsDistance1(d, vf);
}
// Unused on MSVC, see below
#if !HWY_COMPILER_MSVC && !HWY_IS_DEBUG_BUILD
template <size_t kKeysPerVector, class D, class Traits, class V = Vec<D>,
HWY_IF_LANES_GT(kKeysPerVector, 8)>
HWY_INLINE void Merge16x16(D d, Traits st, V& v0, V& v1, V& v2, V& v3, V& v4,
V& v5, V& v6, V& v7, V& v8, V& v9, V& va, V& vb,
V& vc, V& vd, V& ve, V& vf) {
vf = st.ReverseKeys16(d, vf);
ve = st.ReverseKeys16(d, ve);
vd = st.ReverseKeys16(d, vd);
vc = st.ReverseKeys16(d, vc);
vb = st.ReverseKeys16(d, vb);
va = st.ReverseKeys16(d, va);
v9 = st.ReverseKeys16(d, v9);
v8 = st.ReverseKeys16(d, v8);
st.Sort2(d, v0, vf);
st.Sort2(d, v1, ve);
st.Sort2(d, v2, vd);
st.Sort2(d, v3, vc);
st.Sort2(d, v4, vb);
st.Sort2(d, v5, va);
st.Sort2(d, v6, v9);
st.Sort2(d, v7, v8);
v7 = st.ReverseKeys16(d, v7);
v6 = st.ReverseKeys16(d, v6);
v5 = st.ReverseKeys16(d, v5);
v4 = st.ReverseKeys16(d, v4);
vf = st.ReverseKeys16(d, vf);
ve = st.ReverseKeys16(d, ve);
vd = st.ReverseKeys16(d, vd);
vc = st.ReverseKeys16(d, vc);
st.Sort2(d, v0, v7);
st.Sort2(d, v1, v6);
st.Sort2(d, v2, v5);
st.Sort2(d, v3, v4);
st.Sort2(d, v8, vf);
st.Sort2(d, v9, ve);
st.Sort2(d, va, vd);
st.Sort2(d, vb, vc);
v3 = st.ReverseKeys16(d, v3);
v2 = st.ReverseKeys16(d, v2);
v7 = st.ReverseKeys16(d, v7);
v6 = st.ReverseKeys16(d, v6);
vb = st.ReverseKeys16(d, vb);
va = st.ReverseKeys16(d, va);
vf = st.ReverseKeys16(d, vf);
ve = st.ReverseKeys16(d, ve);
st.Sort2(d, v0, v3);
st.Sort2(d, v1, v2);
st.Sort2(d, v4, v7);
st.Sort2(d, v5, v6);
st.Sort2(d, v8, vb);
st.Sort2(d, v9, va);
st.Sort2(d, vc, vf);
st.Sort2(d, vd, ve);
v1 = st.ReverseKeys16(d, v1);
v3 = st.ReverseKeys16(d, v3);
v5 = st.ReverseKeys16(d, v5);
v7 = st.ReverseKeys16(d, v7);
v9 = st.ReverseKeys16(d, v9);
vb = st.ReverseKeys16(d, vb);
vd = st.ReverseKeys16(d, vd);
vf = st.ReverseKeys16(d, vf);
st.Sort2(d, v0, v1);
st.Sort2(d, v2, v3);
st.Sort2(d, v4, v5);
st.Sort2(d, v6, v7);
st.Sort2(d, v8, v9);
st.Sort2(d, va, vb);
st.Sort2(d, vc, vd);
st.Sort2(d, ve, vf);
v0 = st.SortPairsReverse16(d, v0);
v1 = st.SortPairsReverse16(d, v1);
v2 = st.SortPairsReverse16(d, v2);
v3 = st.SortPairsReverse16(d, v3);
v4 = st.SortPairsReverse16(d, v4);
v5 = st.SortPairsReverse16(d, v5);
v6 = st.SortPairsReverse16(d, v6);
v7 = st.SortPairsReverse16(d, v7);
v8 = st.SortPairsReverse16(d, v8);
v9 = st.SortPairsReverse16(d, v9);
va = st.SortPairsReverse16(d, va);
vb = st.SortPairsReverse16(d, vb);
vc = st.SortPairsReverse16(d, vc);
vd = st.SortPairsReverse16(d, vd);
ve = st.SortPairsReverse16(d, ve);
vf = st.SortPairsReverse16(d, vf);
v0 = st.SortPairsDistance4(d, v0);
v1 = st.SortPairsDistance4(d, v1);
v2 = st.SortPairsDistance4(d, v2);
v3 = st.SortPairsDistance4(d, v3);
v4 = st.SortPairsDistance4(d, v4);
v5 = st.SortPairsDistance4(d, v5);
v6 = st.SortPairsDistance4(d, v6);
v7 = st.SortPairsDistance4(d, v7);
v8 = st.SortPairsDistance4(d, v8);
v9 = st.SortPairsDistance4(d, v9);
va = st.SortPairsDistance4(d, va);
vb = st.SortPairsDistance4(d, vb);
vc = st.SortPairsDistance4(d, vc);
vd = st.SortPairsDistance4(d, vd);
ve = st.SortPairsDistance4(d, ve);
vf = st.SortPairsDistance4(d, vf);
v0 = st.SortPairsDistance2(d, v0);
v1 = st.SortPairsDistance2(d, v1);
v2 = st.SortPairsDistance2(d, v2);
v3 = st.SortPairsDistance2(d, v3);
v4 = st.SortPairsDistance2(d, v4);
v5 = st.SortPairsDistance2(d, v5);
v6 = st.SortPairsDistance2(d, v6);
v7 = st.SortPairsDistance2(d, v7);
v8 = st.SortPairsDistance2(d, v8);
v9 = st.SortPairsDistance2(d, v9);
va = st.SortPairsDistance2(d, va);
vb = st.SortPairsDistance2(d, vb);
vc = st.SortPairsDistance2(d, vc);
vd = st.SortPairsDistance2(d, vd);
ve = st.SortPairsDistance2(d, ve);
vf = st.SortPairsDistance2(d, vf);
v0 = st.SortPairsDistance1(d, v0);
v1 = st.SortPairsDistance1(d, v1);
v2 = st.SortPairsDistance1(d, v2);
v3 = st.SortPairsDistance1(d, v3);
v4 = st.SortPairsDistance1(d, v4);
v5 = st.SortPairsDistance1(d, v5);
v6 = st.SortPairsDistance1(d, v6);
v7 = st.SortPairsDistance1(d, v7);
v8 = st.SortPairsDistance1(d, v8);
v9 = st.SortPairsDistance1(d, v9);
va = st.SortPairsDistance1(d, va);
vb = st.SortPairsDistance1(d, vb);
vc = st.SortPairsDistance1(d, vc);
vd = st.SortPairsDistance1(d, vd);
ve = st.SortPairsDistance1(d, ve);
vf = st.SortPairsDistance1(d, vf);
}
#endif // !HWY_COMPILER_MSVC && !HWY_IS_DEBUG_BUILD
// Reshapes `buf` into a matrix, sorts columns independently, and then merges
// into a sorted 1D array without transposing.
//
// DEPRECATED, use BaseCase() instead.
template <class Traits, class V>
HWY_INLINE void SortingNetwork(Traits st, size_t cols, V& v0, V& v1, V& v2,
V& v3, V& v4, V& v5, V& v6, V& v7, V& v8, V& v9,
V& va, V& vb, V& vc, V& vd, V& ve, V& vf) {
// traits*-inl assume 'full' vectors (but still capped to kMaxCols).
const CappedTag<typename Traits::LaneType, Constants::kMaxCols> d;
HWY_DASSERT(cols <= Constants::kMaxCols);
// The network width depends on the number of keys, not lanes.
constexpr size_t kLanesPerKey = st.LanesPerKey();
const size_t keys = cols / kLanesPerKey;
constexpr size_t kMaxKeys = MaxLanes(d) / kLanesPerKey;
Sort16(d, st, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf);
// Checking MaxLanes avoids generating HWY_ASSERT code for the unreachable
// code paths: if MaxLanes < 2, then keys <= cols < 2.
if (HWY_LIKELY(keys >= 2 && kMaxKeys >= 2)) {
Merge16x2<kMaxKeys>(d, st, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb,
vc, vd, ve, vf);
if (HWY_LIKELY(keys >= 4 && kMaxKeys >= 4)) {
Merge16x4<kMaxKeys>(d, st, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb,
vc, vd, ve, vf);
if (HWY_LIKELY(keys >= 8 && kMaxKeys >= 8)) {
Merge16x8<kMaxKeys>(d, st, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va,
vb, vc, vd, ve, vf);
// Avoids build timeout. Must match #if condition in kMaxCols.
#if !HWY_COMPILER_MSVC && !HWY_IS_DEBUG_BUILD
if (HWY_LIKELY(keys >= 16 && kMaxKeys >= 16)) {
Merge16x16<kMaxKeys>(d, st, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9,
va, vb, vc, vd, ve, vf);
static_assert(Constants::kMaxCols <= 16, "Add more branches");
}
#endif
}
}
}
}
// As above, but loads from/stores to `buf`. This ensures full vectors are
// aligned, and enables loads/stores without bounds checks.
//
// DEPRECATED, use BaseCase() instead.
template <class Traits, typename T>
HWY_NOINLINE void SortingNetwork(Traits st, T* HWY_RESTRICT buf, size_t cols) {
// traits*-inl assume 'full' vectors (but still capped to kMaxCols).
// However, for smaller arrays and sub-maximal `cols` we have overlapping
// loads where only the lowest `cols` are valid, and we skip Merge16 etc.
const CappedTag<T, Constants::kMaxCols> d;
using V = decltype(Zero(d));
HWY_DASSERT(cols <= Constants::kMaxCols);
// These are aligned iff cols == Lanes(d). We prefer unaligned/non-constexpr
// offsets to duplicating this code for every value of cols.
static_assert(Constants::kMaxRows == 16, "Update loads/stores/args");
V v0 = LoadU(d, buf + 0x0 * cols);
V v1 = LoadU(d, buf + 0x1 * cols);
V v2 = LoadU(d, buf + 0x2 * cols);
V v3 = LoadU(d, buf + 0x3 * cols);
V v4 = LoadU(d, buf + 0x4 * cols);
V v5 = LoadU(d, buf + 0x5 * cols);
V v6 = LoadU(d, buf + 0x6 * cols);
V v7 = LoadU(d, buf + 0x7 * cols);
V v8 = LoadU(d, buf + 0x8 * cols);
V v9 = LoadU(d, buf + 0x9 * cols);
V va = LoadU(d, buf + 0xa * cols);
V vb = LoadU(d, buf + 0xb * cols);
V vc = LoadU(d, buf + 0xc * cols);
V vd = LoadU(d, buf + 0xd * cols);
V ve = LoadU(d, buf + 0xe * cols);
V vf = LoadU(d, buf + 0xf * cols);
SortingNetwork(st, cols, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc,
vd, ve, vf);
StoreU(v0, d, buf + 0x0 * cols);
StoreU(v1, d, buf + 0x1 * cols);
StoreU(v2, d, buf + 0x2 * cols);
StoreU(v3, d, buf + 0x3 * cols);
StoreU(v4, d, buf + 0x4 * cols);
StoreU(v5, d, buf + 0x5 * cols);
StoreU(v6, d, buf + 0x6 * cols);
StoreU(v7, d, buf + 0x7 * cols);
StoreU(v8, d, buf + 0x8 * cols);
StoreU(v9, d, buf + 0x9 * cols);
StoreU(va, d, buf + 0xa * cols);
StoreU(vb, d, buf + 0xb * cols);
StoreU(vc, d, buf + 0xc * cols);
StoreU(vd, d, buf + 0xd * cols);
StoreU(ve, d, buf + 0xe * cols);
StoreU(vf, d, buf + 0xf * cols);
}
#else
template <class Base>
struct SharedTraits : public Base {};
#endif // VQSORT_ENABLED
} // namespace detail
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_SORT_SORTING_NETWORKS_TOGGLE

618
deps/v8/third_party/highway/src/hwy/contrib/sort/traits-inl.h vendored Normal file
View File

@ -0,0 +1,618 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Per-target
#if defined(HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE
#undef HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE
#else
#define HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE
#endif
#include <stddef.h>
#include <stdint.h>
#include "hwy/contrib/sort/order.h" // SortDescending
#include "hwy/contrib/sort/shared-inl.h" // SortConstants
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace detail {
// Base class of both KeyLane variants
template <typename LaneTypeArg, typename KeyTypeArg>
struct KeyLaneBase {
static constexpr bool Is128() { return false; }
constexpr size_t LanesPerKey() const { return 1; }
// What type bench_sort should allocate for generating inputs.
using LaneType = LaneTypeArg;
// What type to pass to VQSort.
using KeyType = KeyTypeArg;
const char* KeyString() const {
return IsSame<KeyTypeArg, float16_t>() ? "f16"
: IsSame<KeyTypeArg, float>() ? "f32"
: IsSame<KeyTypeArg, double>() ? "f64"
: IsSame<KeyTypeArg, int16_t>() ? "i16"
: IsSame<KeyTypeArg, int32_t>() ? "i32"
: IsSame<KeyTypeArg, int64_t>() ? "i64"
: IsSame<KeyTypeArg, uint16_t>() ? "u32"
: IsSame<KeyTypeArg, uint32_t>() ? "u32"
: IsSame<KeyTypeArg, uint64_t>() ? "u64"
: IsSame<KeyTypeArg, hwy::K32V32>() ? "k+v=64"
: "?";
}
};
// Wrapper functions so we can specialize for floats - infinity trumps
// HighestValue (the normal value with the largest magnitude). Must be outside
// Order* classes to enable SFINAE.
template <class D, HWY_IF_FLOAT_OR_SPECIAL_D(D)>
Vec<D> LargestSortValue(D d) {
return Inf(d);
}
template <class D, HWY_IF_NOT_FLOAT_NOR_SPECIAL_D(D)>
Vec<D> LargestSortValue(D d) {
return Set(d, hwy::HighestValue<TFromD<D>>());
}
template <class D, HWY_IF_FLOAT_OR_SPECIAL_D(D)>
Vec<D> SmallestSortValue(D d) {
return Neg(Inf(d));
}
template <class D, HWY_IF_NOT_FLOAT_NOR_SPECIAL_D(D)>
Vec<D> SmallestSortValue(D d) {
return Set(d, hwy::LowestValue<TFromD<D>>());
}
// Returns the next distinct larger value unless already +inf.
template <class D, HWY_IF_FLOAT_OR_SPECIAL_D(D)>
Vec<D> LargerSortValue(D d, Vec<D> v) {
HWY_DASSERT(AllFalse(d, IsNaN(v))); // we replaced all NaN with LastValue.
using T = TFromD<decltype(d)>;
const RebindToUnsigned<D> du;
using VU = Vec<decltype(du)>;
using TU = TFromD<decltype(du)>;
const VU vu = BitCast(du, Abs(v));
// The direction depends on the original sign. Integer comparison is cheaper
// than float comparison and treats -0 as 0 (so we return +epsilon).
const Mask<decltype(du)> was_pos = Le(BitCast(du, v), SignBit(du));
// If positive, add 1, else -1.
const VU add = IfThenElse(was_pos, Set(du, 1u), Set(du, LimitsMax<TU>()));
// Prev/next integer is the prev/next value, even if mantissa under/overflows.
v = BitCast(d, Add(vu, add));
// But we may have overflowed into inf or NaN; replace with inf if positive,
// but the largest (later negated!) value if the input was -inf.
const Mask<D> was_pos_f = RebindMask(d, was_pos);
v = IfThenElse(IsFinite(v), v,
IfThenElse(was_pos_f, Inf(d), Set(d, HighestValue<T>())));
// Restore the original sign - not via CopySignToAbs because we used a mask.
return IfThenElse(was_pos_f, v, Neg(v));
}
// Returns the next distinct smaller value unless already -inf.
template <class D, HWY_IF_FLOAT_OR_SPECIAL_D(D)>
Vec<D> SmallerSortValue(D d, Vec<D> v) {
HWY_DASSERT(AllFalse(d, IsNaN(v))); // we replaced all NaN with LastValue.
using T = TFromD<decltype(d)>;
const RebindToUnsigned<D> du;
using VU = Vec<decltype(du)>;
using TU = TFromD<decltype(du)>;
const VU vu = BitCast(du, Abs(v));
// The direction depends on the original sign. Float comparison because we
// want to treat 0 as -0 so we return -epsilon.
const Mask<D> was_pos = Gt(v, Zero(d));
// If positive, add -1, else 1.
const VU add =
IfThenElse(RebindMask(du, was_pos), Set(du, LimitsMax<TU>()), Set(du, 1));
// Prev/next integer is the prev/next value, even if mantissa under/overflows.
v = BitCast(d, Add(vu, add));
// But we may have overflowed into inf or NaN; replace with +inf (which will
// later be negated) if negative, but the largest value if the input was +inf.
v = IfThenElse(IsFinite(v), v,
IfThenElse(was_pos, Set(d, HighestValue<T>()), Inf(d)));
// Restore the original sign - not via CopySignToAbs because we used a mask.
return IfThenElse(was_pos, v, Neg(v));
}
template <class D, HWY_IF_NOT_FLOAT_NOR_SPECIAL_D(D)>
Vec<D> LargerSortValue(D d, Vec<D> v) {
return Add(v, Set(d, TFromD<D>{1}));
}
template <class D, HWY_IF_NOT_FLOAT_NOR_SPECIAL_D(D)>
Vec<D> SmallerSortValue(D d, Vec<D> v) {
return Sub(v, Set(d, TFromD<D>{1}));
}
// Highway does not provide a lane type for 128-bit keys, so we use uint64_t
// along with an abstraction layer for single-lane vs. lane-pair, which is
// independent of the order.
template <typename LaneType, typename KeyType>
struct KeyLane : public KeyLaneBase<LaneType, KeyType> {
// For HeapSort
HWY_INLINE void Swap(LaneType* a, LaneType* b) const {
const LaneType temp = *a;
*a = *b;
*b = temp;
}
template <class V, class M>
HWY_INLINE V CompressKeys(V keys, M mask) const {
return CompressNot(keys, mask);
}
// Broadcasts one key into a vector
template <class D>
HWY_INLINE Vec<D> SetKey(D d, const LaneType* key) const {
return Set(d, *key);
}
template <class D>
HWY_INLINE Mask<D> EqualKeys(D /*tag*/, Vec<D> a, Vec<D> b) const {
return Eq(a, b);
}
template <class D>
HWY_INLINE Mask<D> NotEqualKeys(D /*tag*/, Vec<D> a, Vec<D> b) const {
return Ne(a, b);
}
// For keys=lanes, any difference counts.
template <class D>
HWY_INLINE bool NoKeyDifference(D /*tag*/, Vec<D> diff) const {
// Must avoid floating-point comparisons (for -0)
const RebindToUnsigned<D> du;
return AllTrue(du, Eq(BitCast(du, diff), Zero(du)));
}
HWY_INLINE bool Equal1(const LaneType* a, const LaneType* b) const {
return *a == *b;
}
template <class D>
HWY_INLINE Vec<D> ReverseKeys(D d, Vec<D> v) const {
return Reverse(d, v);
}
template <class D>
HWY_INLINE Vec<D> ReverseKeys2(D d, Vec<D> v) const {
return Reverse2(d, v);
}
template <class D>
HWY_INLINE Vec<D> ReverseKeys4(D d, Vec<D> v) const {
return Reverse4(d, v);
}
template <class D>
HWY_INLINE Vec<D> ReverseKeys8(D d, Vec<D> v) const {
return Reverse8(d, v);
}
template <class D>
HWY_INLINE Vec<D> ReverseKeys16(D d, Vec<D> v) const {
static_assert(SortConstants::kMaxCols <= 16, "Assumes u32x16 = 512 bit");
return ReverseKeys(d, v);
}
template <class V>
HWY_INLINE V OddEvenKeys(const V odd, const V even) const {
return OddEven(odd, even);
}
template <class D, HWY_IF_T_SIZE_D(D, 2)>
HWY_INLINE Vec<D> SwapAdjacentPairs(D d, const Vec<D> v) const {
const Repartition<uint32_t, D> du32;
return BitCast(d, Shuffle2301(BitCast(du32, v)));
}
template <class D, HWY_IF_T_SIZE_D(D, 4)>
HWY_INLINE Vec<D> SwapAdjacentPairs(D /* tag */, const Vec<D> v) const {
return Shuffle1032(v);
}
template <class D, HWY_IF_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> SwapAdjacentPairs(D /* tag */, const Vec<D> v) const {
return SwapAdjacentBlocks(v);
}
template <class D, HWY_IF_NOT_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> SwapAdjacentQuads(D d, const Vec<D> v) const {
#if HWY_HAVE_FLOAT64 // in case D is float32
const RepartitionToWide<D> dw;
#else
const RepartitionToWide<RebindToUnsigned<D>> dw;
#endif
return BitCast(d, SwapAdjacentPairs(dw, BitCast(dw, v)));
}
template <class D, HWY_IF_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> SwapAdjacentQuads(D d, const Vec<D> v) const {
// Assumes max vector size = 512
return ConcatLowerUpper(d, v, v);
}
template <class D, HWY_IF_NOT_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> OddEvenPairs(D d, const Vec<D> odd,
const Vec<D> even) const {
#if HWY_HAVE_FLOAT64 // in case D is float32
const RepartitionToWide<D> dw;
#else
const RepartitionToWide<RebindToUnsigned<D>> dw;
#endif
return BitCast(d, OddEven(BitCast(dw, odd), BitCast(dw, even)));
}
template <class D, HWY_IF_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> OddEvenPairs(D /* tag */, Vec<D> odd, Vec<D> even) const {
return OddEvenBlocks(odd, even);
}
template <class D, HWY_IF_NOT_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> OddEvenQuads(D d, Vec<D> odd, Vec<D> even) const {
#if HWY_HAVE_FLOAT64 // in case D is float32
const RepartitionToWide<D> dw;
#else
const RepartitionToWide<RebindToUnsigned<D>> dw;
#endif
return BitCast(d, OddEvenPairs(dw, BitCast(dw, odd), BitCast(dw, even)));
}
template <class D, HWY_IF_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> OddEvenQuads(D d, Vec<D> odd, Vec<D> even) const {
return ConcatUpperLower(d, odd, even);
}
};
// Anything order-related depends on the key traits *and* the order (see
// FirstOfLanes). We cannot implement just one Compare function because Lt128
// only compiles if the lane type is u64. Thus we need either overloaded
// functions with a tag type, class specializations, or separate classes.
// We avoid overloaded functions because we want all functions to be callable
// from a SortTraits without per-function wrappers. Specializing would work, but
// we are anyway going to specialize at a higher level.
template <typename T>
struct OrderAscending : public KeyLane<T, T> {
// False indicates the entire key (i.e. lane) should be compared. KV stands
// for key-value.
static constexpr bool IsKV() { return false; }
using Order = SortAscending;
using OrderForSortingNetwork = OrderAscending<T>;
HWY_INLINE bool Compare1(const T* a, const T* b) const { return *a < *b; }
template <class D>
HWY_INLINE Mask<D> Compare(D /* tag */, Vec<D> a, Vec<D> b) const {
return Lt(a, b);
}
// Two halves of Sort2, used in ScanMinMax.
template <class D>
HWY_INLINE Vec<D> First(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Min(a, b);
}
template <class D>
HWY_INLINE Vec<D> Last(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Max(a, b);
}
template <class D>
HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
T* HWY_RESTRICT /* buf */) const {
return MinOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
T* HWY_RESTRICT /* buf */) const {
return MaxOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> FirstValue(D d) const {
return SmallestSortValue(d);
}
template <class D>
HWY_INLINE Vec<D> LastValue(D d) const {
return LargestSortValue(d);
}
template <class D>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
return SmallerSortValue(d, v);
}
};
template <typename T>
struct OrderDescending : public KeyLane<T, T> {
// False indicates the entire key (i.e. lane) should be compared. KV stands
// for key-value.
static constexpr bool IsKV() { return false; }
using Order = SortDescending;
using OrderForSortingNetwork = OrderDescending<T>;
HWY_INLINE bool Compare1(const T* a, const T* b) const { return *b < *a; }
template <class D>
HWY_INLINE Mask<D> Compare(D /* tag */, Vec<D> a, Vec<D> b) const {
return Lt(b, a);
}
template <class D>
HWY_INLINE Vec<D> First(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Max(a, b);
}
template <class D>
HWY_INLINE Vec<D> Last(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Min(a, b);
}
template <class D>
HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
T* HWY_RESTRICT /* buf */) const {
return MaxOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
T* HWY_RESTRICT /* buf */) const {
return MinOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> FirstValue(D d) const {
return LargestSortValue(d);
}
template <class D>
HWY_INLINE Vec<D> LastValue(D d) const {
return SmallestSortValue(d);
}
template <class D>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
return LargerSortValue(d, v);
}
};
struct KeyValue64 : public KeyLane<uint64_t, hwy::K32V32> {
// True indicates only part of the key (i.e. lane) should be compared. KV
// stands for key-value.
static constexpr bool IsKV() { return true; }
template <class D>
HWY_INLINE Mask<D> EqualKeys(D /*tag*/, Vec<D> a, Vec<D> b) const {
return Eq(ShiftRight<32>(a), ShiftRight<32>(b));
}
template <class D>
HWY_INLINE Mask<D> NotEqualKeys(D /*tag*/, Vec<D> a, Vec<D> b) const {
return Ne(ShiftRight<32>(a), ShiftRight<32>(b));
}
HWY_INLINE bool Equal1(const uint64_t* a, const uint64_t* b) const {
return (*a >> 32) == (*b >> 32);
}
// Only count differences in the actual key, not the value.
template <class D>
HWY_INLINE bool NoKeyDifference(D /*tag*/, Vec<D> diff) const {
// Must avoid floating-point comparisons (for -0)
const RebindToUnsigned<D> du;
const Vec<decltype(du)> zero = Zero(du);
const Vec<decltype(du)> keys = ShiftRight<32>(diff); // clear values
return AllTrue(du, Eq(BitCast(du, keys), zero));
}
};
struct OrderAscendingKV64 : public KeyValue64 {
using Order = SortAscending;
using OrderForSortingNetwork = OrderAscending<LaneType>;
HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) const {
return (*a >> 32) < (*b >> 32);
}
template <class D>
HWY_INLINE Mask<D> Compare(D /* tag */, Vec<D> a, Vec<D> b) const {
return Lt(ShiftRight<32>(a), ShiftRight<32>(b));
}
// Not required to be stable (preserving the order of equivalent keys), so
// we can include the value in the comparison.
template <class D>
HWY_INLINE Vec<D> First(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Min(a, b);
}
template <class D>
HWY_INLINE Vec<D> Last(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Max(a, b);
}
template <class D>
HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
uint64_t* HWY_RESTRICT /* buf */) const {
return MinOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
uint64_t* HWY_RESTRICT /* buf */) const {
return MaxOfLanes(d, v);
}
// Same as for regular lanes.
template <class D>
HWY_INLINE Vec<D> FirstValue(D d) const {
return Set(d, hwy::LowestValue<TFromD<D>>());
}
template <class D>
HWY_INLINE Vec<D> LastValue(D d) const {
return Set(d, hwy::HighestValue<TFromD<D>>());
}
template <class D>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
return Sub(v, Set(d, uint64_t{1} << 32));
}
};
struct OrderDescendingKV64 : public KeyValue64 {
using Order = SortDescending;
using OrderForSortingNetwork = OrderDescending<LaneType>;
HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) const {
return (*b >> 32) < (*a >> 32);
}
template <class D>
HWY_INLINE Mask<D> Compare(D /* tag */, Vec<D> a, Vec<D> b) const {
return Lt(ShiftRight<32>(b), ShiftRight<32>(a));
}
// Not required to be stable (preserving the order of equivalent keys), so
// we can include the value in the comparison.
template <class D>
HWY_INLINE Vec<D> First(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Max(a, b);
}
template <class D>
HWY_INLINE Vec<D> Last(D /* tag */, const Vec<D> a, const Vec<D> b) const {
return Min(a, b);
}
template <class D>
HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
uint64_t* HWY_RESTRICT /* buf */) const {
return MaxOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
uint64_t* HWY_RESTRICT /* buf */) const {
return MinOfLanes(d, v);
}
template <class D>
HWY_INLINE Vec<D> FirstValue(D d) const {
return Set(d, hwy::HighestValue<TFromD<D>>());
}
template <class D>
HWY_INLINE Vec<D> LastValue(D d) const {
return Set(d, hwy::LowestValue<TFromD<D>>());
}
template <class D>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
return Add(v, Set(d, uint64_t{1} << 32));
}
};
// Shared code that depends on Order.
template <class Base>
struct TraitsLane : public Base {
using TraitsForSortingNetwork =
TraitsLane<typename Base::OrderForSortingNetwork>;
// For each lane i: replaces a[i] with the first and b[i] with the second
// according to Base.
// Corresponds to a conditional swap, which is one "node" of a sorting
// network. Min/Max are cheaper than compare + blend at least for integers.
template <class D>
HWY_INLINE void Sort2(D d, Vec<D>& a, Vec<D>& b) const {
const Base* base = static_cast<const Base*>(this);
const Vec<D> a_copy = a;
// Prior to AVX3, there is no native 64-bit Min/Max, so they compile to 4
// instructions. We can reduce it to a compare + 2 IfThenElse.
#if HWY_AVX3 < HWY_TARGET && HWY_TARGET <= HWY_SSSE3
if (sizeof(TFromD<D>) == 8) {
const Mask<D> cmp = base->Compare(d, a, b);
a = IfThenElse(cmp, a, b);
b = IfThenElse(cmp, b, a_copy);
return;
}
#endif
a = base->First(d, a, b);
b = base->Last(d, a_copy, b);
}
// Conditionally swaps even-numbered lanes with their odd-numbered neighbor.
template <class D, HWY_IF_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> SortPairsDistance1(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->ReverseKeys2(d, v);
// Further to the above optimization, Sort2+OddEvenKeys compile to four
// instructions; we can save one by combining two blends.
#if HWY_AVX3 < HWY_TARGET && HWY_TARGET <= HWY_SSSE3
const Vec<D> cmp = VecFromMask(d, base->Compare(d, v, swapped));
return IfVecThenElse(DupOdd(cmp), swapped, v);
#else
Sort2(d, v, swapped);
return base->OddEvenKeys(swapped, v);
#endif
}
// (See above - we use Sort2 for non-64-bit types.)
template <class D, HWY_IF_NOT_T_SIZE_D(D, 8)>
HWY_INLINE Vec<D> SortPairsDistance1(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->ReverseKeys2(d, v);
Sort2(d, v, swapped);
return base->OddEvenKeys(swapped, v);
}
// Swaps with the vector formed by reversing contiguous groups of 4 keys.
template <class D>
HWY_INLINE Vec<D> SortPairsReverse4(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->ReverseKeys4(d, v);
Sort2(d, v, swapped);
return base->OddEvenPairs(d, swapped, v);
}
// Conditionally swaps lane 0 with 4, 1 with 5 etc.
template <class D>
HWY_INLINE Vec<D> SortPairsDistance4(D d, Vec<D> v) const {
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->SwapAdjacentQuads(d, v);
// Only used in Merge16, so this will not be used on AVX2 (which only has 4
// u64 lanes), so skip the above optimization for 64-bit AVX2.
Sort2(d, v, swapped);
return base->OddEvenQuads(d, swapped, v);
}
};
} // namespace detail
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE

549
deps/v8/third_party/highway/src/hwy/contrib/sort/traits128-inl.h vendored Normal file
View File

@ -0,0 +1,549 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Per-target
#if defined(HIGHWAY_HWY_CONTRIB_SORT_TRAITS128_TOGGLE) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_SORT_TRAITS128_TOGGLE
#undef HIGHWAY_HWY_CONTRIB_SORT_TRAITS128_TOGGLE
#else
#define HIGHWAY_HWY_CONTRIB_SORT_TRAITS128_TOGGLE
#endif
#include <stddef.h>
#include <stdint.h>
#include "hwy/contrib/sort/order.h" // SortDescending
#include "hwy/contrib/sort/shared-inl.h"
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace detail {
// Also used by HeapSort, so do not require VQSORT_ENABLED.
#if HWY_TARGET != HWY_SCALAR || HWY_IDE
// Highway does not provide a lane type for 128-bit keys, so we use uint64_t
// along with an abstraction layer for single-lane vs. lane-pair, which is
// independent of the order.
struct KeyAny128 {
static constexpr bool Is128() { return true; }
constexpr size_t LanesPerKey() const { return 2; }
// What type bench_sort should allocate for generating inputs.
using LaneType = uint64_t;
// KeyType and KeyString are defined by derived classes.
HWY_INLINE void Swap(LaneType* a, LaneType* b) const {
const FixedTag<LaneType, 2> d;
const auto temp = LoadU(d, a);
StoreU(LoadU(d, b), d, a);
StoreU(temp, d, b);
}
template <class V, class M>
HWY_INLINE V CompressKeys(V keys, M mask) const {
return CompressBlocksNot(keys, mask);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> SetKey(D d, const TFromD<D>* key) const {
return LoadDup128(d, key);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> ReverseKeys(D d, Vec<D> v) const {
return ReverseBlocks(d, v);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> ReverseKeys2(D /* tag */, const Vec<D> v) const {
HWY_DASSERT(Lanes(D()) >= 4); // at least 2 keys
return SwapAdjacentBlocks(v);
}
// Only called for 4 keys because we do not support >512-bit vectors.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> ReverseKeys4(D d, const Vec<D> v) const {
HWY_DASSERT(Lanes(D()) == 8); // exactly 4 keys: the 512-bit limit
return ReverseKeys(d, v);
}
// Only called for 4 keys because we do not support >512-bit vectors.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> OddEvenPairs(D d, const Vec<D> odd,
const Vec<D> even) const {
HWY_DASSERT(Lanes(D()) == 8); // exactly 4 keys: the 512-bit limit
return ConcatUpperLower(d, odd, even);
}
template <class V>
HWY_INLINE V OddEvenKeys(const V odd, const V even) const {
return OddEvenBlocks(odd, even);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> ReverseKeys8(D, Vec<D>) const {
HWY_ASSERT(0); // not supported: would require 1024-bit vectors
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> ReverseKeys16(D, Vec<D>) const {
HWY_ASSERT(0); // not supported: would require 2048-bit vectors
}
// This is only called for 8/16 col networks (not supported).
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> SwapAdjacentPairs(D, Vec<D>) const {
HWY_ASSERT(0);
}
// This is only called for 16 col networks (not supported).
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> SwapAdjacentQuads(D, Vec<D>) const {
HWY_ASSERT(0);
}
// This is only called for 8 col networks (not supported).
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> OddEvenQuads(D, Vec<D>, Vec<D>) const {
HWY_ASSERT(0);
}
};
// Base class shared between OrderAscending128, OrderDescending128.
struct Key128 : public KeyAny128 {
// False indicates the entire key should be compared. KV means key-value.
static constexpr bool IsKV() { return false; }
// What type to pass to VQSort.
using KeyType = hwy::uint128_t;
const char* KeyString() const { return "U128"; }
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> EqualKeys(D d, Vec<D> a, Vec<D> b) const {
return Eq128(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> NotEqualKeys(D d, Vec<D> a, Vec<D> b) const {
return Ne128(d, a, b);
}
// For keys=entire 128 bits, any difference counts.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE bool NoKeyDifference(D /*tag*/, Vec<D> diff) const {
// Must avoid floating-point comparisons (for -0)
const RebindToUnsigned<D> du;
return AllTrue(du, Eq(BitCast(du, diff), Zero(du)));
}
HWY_INLINE bool Equal1(const LaneType* a, const LaneType* b) const {
return a[0] == b[0] && a[1] == b[1];
}
// Returns vector with only the top half of each block valid. This allows
// fusing the "replicate upper to lower half" step with a subsequent permute.
template <class Order, class D>
HWY_INLINE HWY_MAYBE_UNUSED Vec<D> CompareTop(D d, Vec<D> a, Vec<D> b) const {
const Mask<D> eqHL = Eq(a, b);
const Vec<D> ltHL = VecFromMask(d, Order().CompareLanes(a, b));
#if HWY_TARGET <= HWY_AVX2 // slightly faster
const Vec<D> ltLX = ShiftLeftLanes<1>(ltHL);
return OrAnd(ltHL, VecFromMask(d, eqHL), ltLX);
#else
return IfThenElse(eqHL, DupEven(ltHL), ltHL);
#endif
}
};
// Anything order-related depends on the key traits *and* the order (see
// FirstOfLanes). We cannot implement just one Compare function because Lt128
// only compiles if the lane type is u64. Thus we need either overloaded
// functions with a tag type, class specializations, or separate classes.
// We avoid overloaded functions because we want all functions to be callable
// from a SortTraits without per-function wrappers. Specializing would work, but
// we are anyway going to specialize at a higher level.
struct OrderAscending128 : public Key128 {
using Order = SortAscending;
using OrderForSortingNetwork = OrderAscending128;
HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) const {
return (a[1] == b[1]) ? a[0] < b[0] : a[1] < b[1];
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> Compare(D d, Vec<D> a, Vec<D> b) const {
return Lt128(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> First(D d, const Vec<D> a, const Vec<D> b) const {
return Min128(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> Last(D d, const Vec<D> a, const Vec<D> b) const {
return Max128(d, a, b);
}
// FirstOfLanes/LastOfLanes are implemented in Traits128.
// Same as for regular lanes because 128-bit keys are u64.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> FirstValue(D d) const {
return Set(d, hwy::LowestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> LastValue(D d) const {
return Set(d, hwy::HighestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
const Vec<D> k0 = Zero(d);
const Vec<D> k1 = OddEven(k0, Set(d, uint64_t{1}));
const Mask<D> borrow = Eq(v, k0); // don't-care, lo == 0
// lo == 0? 1 : 0, 0
const Vec<D> adjust = ShiftLeftLanes<1>(IfThenElseZero(borrow, k1));
return Sub(Sub(v, k1), adjust);
}
// 'Private', used by base class Key128::CompareTop.
template <class V>
HWY_INLINE Mask<DFromV<V> > CompareLanes(V a, V b) const {
return Lt(a, b);
}
};
struct OrderDescending128 : public Key128 {
using Order = SortDescending;
using OrderForSortingNetwork = OrderDescending128;
HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) const {
return (a[1] == b[1]) ? b[0] < a[0] : b[1] < a[1];
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> Compare(D d, Vec<D> a, Vec<D> b) const {
return Lt128(d, b, a);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> First(D d, const Vec<D> a, const Vec<D> b) const {
return Max128(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> Last(D d, const Vec<D> a, const Vec<D> b) const {
return Min128(d, a, b);
}
// FirstOfLanes/LastOfLanes are implemented in Traits128.
// Same as for regular lanes because 128-bit keys are u64.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> FirstValue(D d) const {
return Set(d, hwy::HighestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> LastValue(D d) const {
return Set(d, hwy::LowestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
const Vec<D> k1 = OddEven(Zero(d), Set(d, uint64_t{1}));
const Vec<D> added = Add(v, k1);
const Mask<D> overflowed = Lt(added, v); // false, overflowed
// overflowed? 1 : 0, 0
const Vec<D> adjust = ShiftLeftLanes<1>(IfThenElseZero(overflowed, k1));
return Add(added, adjust);
}
// 'Private', used by base class Key128::CompareTop.
template <class V>
HWY_INLINE Mask<DFromV<V> > CompareLanes(V a, V b) const {
return Lt(b, a);
}
};
// Base class shared between OrderAscendingKV128, OrderDescendingKV128.
struct KeyValue128 : public KeyAny128 {
// True indicates only part of the key (the more significant lane) should be
// compared. KV stands for key-value.
static constexpr bool IsKV() { return true; }
// What type to pass to VQSort.
using KeyType = K64V64;
const char* KeyString() const { return "k+v=128"; }
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> EqualKeys(D d, Vec<D> a, Vec<D> b) const {
return Eq128Upper(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> NotEqualKeys(D d, Vec<D> a, Vec<D> b) const {
return Ne128Upper(d, a, b);
}
HWY_INLINE bool Equal1(const LaneType* a, const LaneType* b) const {
return a[1] == b[1];
}
// Only count differences in the actual key, not the value.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE bool NoKeyDifference(D /*tag*/, Vec<D> diff) const {
// Must avoid floating-point comparisons (for -0)
const RebindToUnsigned<D> du;
const Vec<decltype(du)> zero = Zero(du);
const Vec<decltype(du)> keys = OddEven(diff, zero); // clear values
return AllTrue(du, Eq(BitCast(du, keys), zero));
}
// Returns vector with only the top half of each block valid. This allows
// fusing the "replicate upper to lower half" step with a subsequent permute.
template <class Order, class D>
HWY_INLINE HWY_MAYBE_UNUSED Vec<D> CompareTop(D d, Vec<D> a, Vec<D> b) const {
// Only the upper lane of each block is a key, and only that lane is
// required to be valid, so comparing all lanes is sufficient.
return VecFromMask(d, Order().CompareLanes(a, b));
}
};
struct OrderAscendingKV128 : public KeyValue128 {
using Order = SortAscending;
using OrderForSortingNetwork = OrderAscending128;
HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) const {
return a[1] < b[1];
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> Compare(D d, Vec<D> a, Vec<D> b) const {
return Lt128Upper(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> First(D d, const Vec<D> a, const Vec<D> b) const {
return Min128Upper(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> Last(D d, const Vec<D> a, const Vec<D> b) const {
return Max128Upper(d, a, b);
}
// FirstOfLanes/LastOfLanes are implemented in Traits128.
// Same as for regular lanes because 128-bit keys are u64.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> FirstValue(D d) const {
return Set(d, hwy::LowestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> LastValue(D d) const {
return Set(d, hwy::HighestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
const Vec<D> k1 = OddEven(Set(d, uint64_t{1}), Zero(d));
return Sub(v, k1);
}
// 'Private', used by base class KeyValue128::CompareTop.
template <class V>
HWY_INLINE Mask<DFromV<V> > CompareLanes(V a, V b) const {
return Lt(a, b);
}
};
struct OrderDescendingKV128 : public KeyValue128 {
using Order = SortDescending;
using OrderForSortingNetwork = OrderDescending128;
HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) const {
return b[1] < a[1];
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Mask<D> Compare(D d, Vec<D> a, Vec<D> b) const {
return Lt128Upper(d, b, a);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> First(D d, const Vec<D> a, const Vec<D> b) const {
return Max128Upper(d, a, b);
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> Last(D d, const Vec<D> a, const Vec<D> b) const {
return Min128Upper(d, a, b);
}
// FirstOfLanes/LastOfLanes are implemented in Traits128.
// Same as for regular lanes because 128-bit keys are u64.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> FirstValue(D d) const {
return Set(d, hwy::HighestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> LastValue(D d) const {
return Set(d, hwy::LowestValue<TFromD<D> >());
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
const Vec<D> k1 = OddEven(Set(d, uint64_t{1}), Zero(d));
return Add(v, k1);
}
// 'Private', used by base class KeyValue128::CompareTop.
template <class V>
HWY_INLINE Mask<DFromV<V> > CompareLanes(V a, V b) const {
return Lt(b, a);
}
};
// We want to swap 2 u128, i.e. 4 u64 lanes, based on the 0 or FF..FF mask in
// the most-significant of those lanes (the result of CompareTop), so
// replicate it 4x. Only called for >= 256-bit vectors.
#if HWY_TARGET <= HWY_AVX3
template <class V, HWY_IF_V_SIZE_V(V, 64)>
HWY_INLINE V ReplicateTop4x(V v) {
return V{_mm512_permutex_epi64(v.raw, _MM_SHUFFLE(3, 3, 3, 3))};
}
#endif // HWY_TARGET <= HWY_AVX3
#if HWY_TARGET <= HWY_AVX2
template <class V, HWY_IF_V_SIZE_V(V, 32)>
HWY_INLINE V ReplicateTop4x(V v) {
return V{_mm256_permute4x64_epi64(v.raw, _MM_SHUFFLE(3, 3, 3, 3))};
}
#else // HWY_TARGET > HWY_AVX2
template <class V>
HWY_INLINE V ReplicateTop4x(V v) {
#if HWY_TARGET == HWY_SVE_256
return svdup_lane_u64(v, 3);
#else
const ScalableTag<uint64_t> d;
HWY_DASSERT(Lanes(d) == 4 || Lanes(d) == 8); // for table below
HWY_ALIGN static constexpr uint64_t kIndices[8] = {3, 3, 3, 3, 7, 7, 7, 7};
return TableLookupLanes(v, SetTableIndices(d, kIndices));
#endif
}
#endif // HWY_TARGET <= HWY_AVX2
// Shared code that depends on Order.
template <class Base>
struct Traits128 : public Base {
using TraitsForSortingNetwork =
Traits128<typename Base::OrderForSortingNetwork>;
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
TFromD<D>* HWY_RESTRICT buf) const {
const Base* base = static_cast<const Base*>(this);
const size_t N = Lanes(d);
Store(v, d, buf);
v = base->SetKey(d, buf + 0); // result must be broadcasted
for (size_t i = base->LanesPerKey(); i < N; i += base->LanesPerKey()) {
v = base->First(d, v, base->SetKey(d, buf + i));
}
return v;
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
TFromD<D>* HWY_RESTRICT buf) const {
const Base* base = static_cast<const Base*>(this);
const size_t N = Lanes(d);
Store(v, d, buf);
v = base->SetKey(d, buf + 0); // result must be broadcasted
for (size_t i = base->LanesPerKey(); i < N; i += base->LanesPerKey()) {
v = base->Last(d, v, base->SetKey(d, buf + i));
}
return v;
}
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE void Sort2(D d, Vec<D>& a, Vec<D>& b) const {
const Base* base = static_cast<const Base*>(this);
const Vec<D> a_copy = a;
const auto lt = base->Compare(d, a, b);
a = IfThenElse(lt, a, b);
b = IfThenElse(lt, b, a_copy);
}
// Conditionally swaps even-numbered keys with their odd-numbered neighbor.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> SortPairsDistance1(D d, Vec<D> v) const {
HWY_DASSERT(Lanes(d) >= 4); // required by ReplicateTop4x
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->ReverseKeys2(d, v);
const Vec<D> cmpHx = base->template CompareTop<Base>(d, v, swapped);
return IfVecThenElse(ReplicateTop4x(cmpHx), swapped, v);
}
// Swaps with the vector formed by reversing contiguous groups of four 128-bit
// keys, which implies 512-bit vectors (we do not support more than that).
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> SortPairsReverse4(D d, Vec<D> v) const {
HWY_DASSERT(Lanes(d) == 8); // For TableLookupLanes below
const Base* base = static_cast<const Base*>(this);
Vec<D> swapped = base->ReverseKeys4(d, v);
const Vec<D> cmpHx = base->template CompareTop<Base>(d, v, swapped);
// Similar to ReplicateTop4x, we want to gang together 2 comparison results
// (4 lanes). They are not contiguous, so use permute to replicate 4x.
HWY_ALIGN uint64_t kIndices[8] = {7, 7, 5, 5, 5, 5, 7, 7};
const Vec<D> select = TableLookupLanes(cmpHx, SetTableIndices(d, kIndices));
return IfVecThenElse(select, swapped, v);
}
// Conditionally swaps lane 0 with 4, 1 with 5 etc.
template <class D, HWY_IF_U64_D(D)>
HWY_INLINE Vec<D> SortPairsDistance4(D, Vec<D>) const {
// Only used by Merge16, which would require 2048 bit vectors (unsupported).
HWY_ASSERT(0);
}
};
#endif // HWY_TARGET != HWY_SCALAR
} // namespace detail
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_SORT_TRAITS128_TOGGLE

2243
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort-inl.h vendored Normal file
View File

File diff suppressed because it is too large Load Diff

217
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort.cc vendored Normal file
View File

@ -0,0 +1,217 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h"
#include "hwy/base.h"
#include "hwy/contrib/sort/vqsort-inl.h"
#include "hwy/per_target.h"
// Check if we have getrandom from <sys/random.h>. Because <features.h> is
// unavailable on Android and non-Linux RVV, we assume that those systems lack
// getrandom. Note that the only supported sources of entropy are getrandom or
// Windows, thus VQSORT_SECURE_SEED=0 when this is 0 and we are not on Windows.
#if defined(ANDROID) || defined(__ANDROID__) || \
(HWY_ARCH_RISCV && !HWY_OS_LINUX)
#define VQSORT_GETRANDOM 0
#endif
#if !defined(VQSORT_GETRANDOM) && HWY_OS_LINUX
#include <features.h>
// ---- which libc
#if defined(__UCLIBC__)
#define VQSORT_GETRANDOM 1 // added Mar 2015, before uclibc-ng 1.0
#elif defined(__GLIBC__) && defined(__GLIBC_PREREQ)
#if __GLIBC_PREREQ(2, 25)
#define VQSORT_GETRANDOM 1
#else
#define VQSORT_GETRANDOM 0
#endif
#else
// Assume MUSL, which has getrandom since 2018. There is no macro to test, see
// https://www.openwall.com/lists/musl/2013/03/29/13.
#define VQSORT_GETRANDOM 1
#endif // ---- which libc
#endif // linux
#if !defined(VQSORT_GETRANDOM)
#define VQSORT_GETRANDOM 0
#endif
// Choose a seed source for SFC generator: 1=getrandom, 2=CryptGenRandom.
// Allow user override - not all Android support the getrandom wrapper.
#ifndef VQSORT_SECURE_SEED
#if VQSORT_GETRANDOM
#define VQSORT_SECURE_SEED 1
#elif defined(_WIN32) || defined(_WIN64)
#define VQSORT_SECURE_SEED 2
#else
#define VQSORT_SECURE_SEED 0
#endif
#endif // VQSORT_SECURE_SEED
// Pull in dependencies of the chosen seed source.
#if VQSORT_SECURE_SEED == 1
#include <sys/random.h>
#elif VQSORT_SECURE_SEED == 2
#include <windows.h>
#if HWY_COMPILER_MSVC || HWY_COMPILER_CLANGCL
#pragma comment(lib, "advapi32.lib")
#endif // HWY_COMPILER_MSVC || HWY_COMPILER_CLANGCL
// Must come after windows.h.
#include <wincrypt.h>
#endif // VQSORT_SECURE_SEED
namespace hwy {
// Returns false or performs the equivalent of `memcpy(bytes, r, 16)`, where r
// is high-quality (unpredictable, uniformly distributed) random bits.
bool Fill16BytesSecure(void* bytes) {
#if VQSORT_SECURE_SEED == 1
// May block if urandom is not yet initialized.
const ssize_t ret = getrandom(bytes, 16, /*flags=*/0);
if (ret == 16) return true;
#elif VQSORT_SECURE_SEED == 2
HCRYPTPROV hProvider{};
if (CryptAcquireContextA(&hProvider, nullptr, nullptr, PROV_RSA_FULL,
CRYPT_VERIFYCONTEXT)) {
const BOOL ok =
CryptGenRandom(hProvider, 16, reinterpret_cast<BYTE*>(bytes));
CryptReleaseContext(hProvider, 0);
if (ok) return true;
}
#else
(void)bytes;
#endif
return false;
}
void Sorter::operator()(uint16_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint16_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint32_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint32_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint64_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint64_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(int16_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(int16_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(int32_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(int32_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(int64_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(int64_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(float16_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(float16_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(float* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(float* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(double* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(double* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint128_t* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(uint128_t* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(K64V64* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(K64V64* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(K32V32* HWY_RESTRICT keys, size_t n,
SortAscending tag) const {
VQSort(keys, n, tag);
}
void Sorter::operator()(K32V32* HWY_RESTRICT keys, size_t n,
SortDescending tag) const {
VQSort(keys, n, tag);
}
// Unused, only for ABI compatibility
void Sorter::Fill24Bytes(const void*, size_t, void*) {}
bool Sorter::HaveFloat64() { return hwy::HaveFloat64(); }
Sorter::Sorter() {}
void Sorter::Delete() {}
uint64_t* GetGeneratorState() { return hwy::detail::GetGeneratorStateStatic(); }
} // namespace hwy

303
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort.h vendored Normal file
View File

@ -0,0 +1,303 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Interface to vectorized quicksort with dynamic dispatch. For static dispatch
// without any DLLEXPORT, avoid including this header and instead define
// VQSORT_ONLY_STATIC, then call VQSortStatic* in vqsort-inl.h.
//
// Blog post: https://tinyurl.com/vqsort-blog
// Paper with measurements: https://arxiv.org/abs/2205.05982
//
// To ensure the overhead of using wide vectors (e.g. AVX2 or AVX-512) is
// worthwhile, we recommend using this code for sorting arrays whose size is at
// least 100 KiB. See the README for details.
#ifndef HIGHWAY_HWY_CONTRIB_SORT_VQSORT_H_
#define HIGHWAY_HWY_CONTRIB_SORT_VQSORT_H_
// IWYU pragma: begin_exports
#include <stddef.h>
#include "hwy/base.h"
#include "hwy/contrib/sort/order.h" // SortAscending
// IWYU pragma: end_exports
namespace hwy {
// Vectorized Quicksort: sorts keys[0, n). Does not preserve the ordering of
// equivalent keys (defined as: neither greater nor less than another).
// Dispatches to the best available instruction set. Does not allocate memory.
// Uses about 1.2 KiB stack plus an internal 3-word TLS cache for random state.
HWY_CONTRIB_DLLEXPORT void VQSort(uint16_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(uint16_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(uint32_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(uint32_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(uint64_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(uint64_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(int16_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(int16_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(int32_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(int32_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(int64_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(int64_t* HWY_RESTRICT keys, size_t n,
SortDescending);
// These two must only be called if hwy::HaveFloat16() is true.
HWY_CONTRIB_DLLEXPORT void VQSort(float16_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(float16_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(float* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(float* HWY_RESTRICT keys, size_t n,
SortDescending);
// These two must only be called if hwy::HaveFloat64() is true.
HWY_CONTRIB_DLLEXPORT void VQSort(double* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(double* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(K32V32* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(K32V32* HWY_RESTRICT keys, size_t n,
SortDescending);
// 128-bit types: `n` is still in units of the 128-bit keys.
HWY_CONTRIB_DLLEXPORT void VQSort(uint128_t* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(uint128_t* HWY_RESTRICT keys, size_t n,
SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSort(K64V64* HWY_RESTRICT keys, size_t n,
SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSort(K64V64* HWY_RESTRICT keys, size_t n,
SortDescending);
// Vectorized partial Quicksort:
// Rearranges elements such that the range [0, k) contains the sorted first k
// elements in the range [0, n). Does not preserve the ordering of equivalent
// keys (defined as: neither greater nor less than another).
// Dispatches to the best available instruction set. Does not allocate memory.
// Uses about 1.2 KiB stack plus an internal 3-word TLS cache for random state.
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(int16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(int16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(int32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(int32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(int64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(int64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// These two must only be called if hwy::HaveFloat16() is true.
HWY_CONTRIB_DLLEXPORT void VQPartialSort(float16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(float16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(float* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(float* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// These two must only be called if hwy::HaveFloat64() is true.
HWY_CONTRIB_DLLEXPORT void VQPartialSort(double* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(double* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(K32V32* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(K32V32* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// 128-bit types: `n` and `k` are still in units of the 128-bit keys.
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint128_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(uint128_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(K64V64* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQPartialSort(K64V64* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// Vectorized Quickselect:
// rearranges elements in [0, n) such that:
// The element pointed at by kth is changed to whatever element would occur in
// that position if [0, n) were sorted. All of the elements before this new kth
// element are less than or equal to the elements after the new kth element.
HWY_CONTRIB_DLLEXPORT void VQSelect(uint16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(uint16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(uint32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(uint32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(uint64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(uint64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(int16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(int16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(int32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(int32_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(int64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(int64_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// These two must only be called if hwy::HaveFloat16() is true.
HWY_CONTRIB_DLLEXPORT void VQSelect(float16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(float16_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(float* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(float* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// These two must only be called if hwy::HaveFloat64() is true.
HWY_CONTRIB_DLLEXPORT void VQSelect(double* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(double* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(K32V32* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(K32V32* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// 128-bit types: `n` and `k` are still in units of the 128-bit keys.
HWY_CONTRIB_DLLEXPORT void VQSelect(uint128_t* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(uint128_t* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
HWY_CONTRIB_DLLEXPORT void VQSelect(K64V64* HWY_RESTRICT keys, size_t n,
size_t k, SortAscending);
HWY_CONTRIB_DLLEXPORT void VQSelect(K64V64* HWY_RESTRICT keys, size_t n,
size_t k, SortDescending);
// User-level caching is no longer required, so this class is no longer
// beneficial. We recommend using the simpler VQSort() interface instead, and
// retain this class only for compatibility. It now just calls VQSort.
class HWY_CONTRIB_DLLEXPORT Sorter {
public:
Sorter();
~Sorter() { Delete(); }
// Move-only
Sorter(const Sorter&) = delete;
Sorter& operator=(const Sorter&) = delete;
Sorter(Sorter&& /*other*/) {}
Sorter& operator=(Sorter&& /*other*/) { return *this; }
void operator()(uint16_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(uint16_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(uint32_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(uint32_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(uint64_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(uint64_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(int16_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(int16_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(int32_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(int32_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(int64_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(int64_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
// These two must only be called if hwy::HaveFloat16() is true.
void operator()(float16_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(float16_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(float* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(float* HWY_RESTRICT keys, size_t n, SortDescending) const;
// These two must only be called if hwy::HaveFloat64() is true.
void operator()(double* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(double* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(uint128_t* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(uint128_t* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(K64V64* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(K64V64* HWY_RESTRICT keys, size_t n, SortDescending) const;
void operator()(K32V32* HWY_RESTRICT keys, size_t n, SortAscending) const;
void operator()(K32V32* HWY_RESTRICT keys, size_t n, SortDescending) const;
// Unused
static void Fill24Bytes(const void*, size_t, void*);
static bool HaveFloat64(); // Can also use hwy::HaveFloat64 directly.
private:
void Delete();
template <typename T>
T* Get() const {
return unused_;
}
#if HWY_COMPILER_CLANG
HWY_DIAGNOSTICS(push)
HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wunused-private-field")
#endif
void* unused_ = nullptr;
#if HWY_COMPILER_CLANG
HWY_DIAGNOSTICS(pop)
#endif
};
// Used by vqsort-inl.h unless VQSORT_ONLY_STATIC.
HWY_CONTRIB_DLLEXPORT bool Fill16BytesSecure(void* bytes);
// Unused, only provided for binary compatibility.
HWY_CONTRIB_DLLEXPORT uint64_t* GetGeneratorState();
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_SORT_VQSORT_H_

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_128a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_128a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void Sort128Asc(uint128_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSort128Asc(uint128_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void Select128Asc(uint128_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(Sort128Asc);
HWY_EXPORT(PartialSort128Asc);
HWY_EXPORT(Select128Asc);
} // namespace
void VQSort(uint128_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(Sort128Asc)(keys, n);
}
void VQPartialSort(uint128_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSort128Asc)(keys, n, k);
}
void VQSelect(uint128_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(Select128Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_128d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_128d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void Sort128Desc(uint128_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSort128Desc(uint128_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void Select128Desc(uint128_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(Sort128Desc);
HWY_EXPORT(PartialSort128Desc);
HWY_EXPORT(Select128Desc);
} // namespace
void VQSort(uint128_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(Sort128Desc)(keys, n);
}
void VQPartialSort(uint128_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSort128Desc)(keys, n, k);
}
void VQSelect(uint128_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(Select128Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

91
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_f16a.cc vendored Normal file
View File

@ -0,0 +1,91 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_f16a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortF16Asc(float16_t* HWY_RESTRICT keys, const size_t num) {
#if HWY_HAVE_FLOAT16
return VQSortStatic(keys, num, SortAscending());
#else
(void)keys;
(void)num;
HWY_ASSERT(0);
#endif
}
void PartialSortF16Asc(float16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT16
return VQPartialSortStatic(keys, num, k, SortAscending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
void SelectF16Asc(float16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT16
return VQSelectStatic(keys, num, k, SortAscending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortF16Asc);
HWY_EXPORT(PartialSortF16Asc);
HWY_EXPORT(SelectF16Asc);
} // namespace
void VQSort(float16_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortF16Asc)(keys, n);
}
void VQPartialSort(float16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortF16Asc)(keys, n, k);
}
void VQSelect(float16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectF16Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

91
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_f16d.cc vendored Normal file
View File

@ -0,0 +1,91 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_f16d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortF16Desc(float16_t* HWY_RESTRICT keys, const size_t num) {
#if HWY_HAVE_FLOAT16
return VQSortStatic(keys, num, SortDescending());
#else
(void)keys;
(void)num;
HWY_ASSERT(0);
#endif
}
void PartialSortF16Desc(float16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT16
return VQPartialSortStatic(keys, num, k, SortDescending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
void SelectF16Desc(float16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT16
return VQSelectStatic(keys, num, k, SortDescending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortF16Desc);
HWY_EXPORT(PartialSortF16Desc);
HWY_EXPORT(SelectF16Desc);
} // namespace
void VQSort(float16_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortF16Desc)(keys, n);
}
void VQPartialSort(float16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortF16Desc)(keys, n, k);
}
void VQSelect(float16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectF16Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

70
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_f32a.cc vendored Normal file
View File

@ -0,0 +1,70 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_f32a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortF32Asc(float* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortF32Asc(float* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectF32Asc(float* HWY_RESTRICT keys, const size_t num, const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortF32Asc);
HWY_EXPORT(PartialSortF32Asc);
HWY_EXPORT(SelectF32Asc);
} // namespace
void VQSort(float* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortF32Asc)(keys, n);
}
void VQPartialSort(float* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortF32Asc)(keys, n, k);
}
void VQSelect(float* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectF32Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

70
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_f32d.cc vendored Normal file
View File

@ -0,0 +1,70 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_f32d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortF32Desc(float* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortF32Desc(float* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectF32Desc(float* HWY_RESTRICT keys, const size_t num, const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortF32Desc);
HWY_EXPORT(PartialSortF32Desc);
HWY_EXPORT(SelectF32Desc);
} // namespace
void VQSort(float* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortF32Desc)(keys, n);
}
void VQPartialSort(float* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortF32Desc)(keys, n, k);
}
void VQSelect(float* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectF32Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

90
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_f64a.cc vendored Normal file
View File

@ -0,0 +1,90 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_f64a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortF64Asc(double* HWY_RESTRICT keys, const size_t num) {
#if HWY_HAVE_FLOAT64
return VQSortStatic(keys, num, SortAscending());
#else
(void)keys;
(void)num;
HWY_ASSERT(0);
#endif
}
void PartialSortF64Asc(double* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT64
return VQPartialSortStatic(keys, num, k, SortAscending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
void SelectF64Asc(double* HWY_RESTRICT keys, const size_t num, const size_t k) {
#if HWY_HAVE_FLOAT64
return VQSelectStatic(keys, num, k, SortAscending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortF64Asc);
HWY_EXPORT(PartialSortF64Asc);
HWY_EXPORT(SelectF64Asc);
} // namespace
void VQSort(double* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortF64Asc)(keys, n);
}
void VQPartialSort(double* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortF64Asc)(keys, n, k);
}
void VQSelect(double* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectF64Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

91
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_f64d.cc vendored Normal file
View File

@ -0,0 +1,91 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_f64d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortF64Desc(double* HWY_RESTRICT keys, const size_t num) {
#if HWY_HAVE_FLOAT64
return VQSortStatic(keys, num, SortDescending());
#else
(void)keys;
(void)num;
HWY_ASSERT(0);
#endif
}
void PartialSortF64Desc(double* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT64
return VQPartialSortStatic(keys, num, k, SortDescending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
void SelectF64Desc(double* HWY_RESTRICT keys, const size_t num,
const size_t k) {
#if HWY_HAVE_FLOAT64
return VQSelectStatic(keys, num, k, SortDescending());
#else
(void)keys;
(void)num;
(void)k;
HWY_ASSERT(0);
#endif
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortF64Desc);
HWY_EXPORT(PartialSortF64Desc);
HWY_EXPORT(SelectF64Desc);
} // namespace
void VQSort(double* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortF64Desc)(keys, n);
}
void VQPartialSort(double* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortF64Desc)(keys, n, k);
}
void VQSelect(double* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectF64Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_i16a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_i16a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortI16Asc(int16_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortI16Asc(int16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectI16Asc(int16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortI16Asc);
HWY_EXPORT(PartialSortI16Asc);
HWY_EXPORT(SelectI16Asc);
} // namespace
void VQSort(int16_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortI16Asc)(keys, n);
}
void VQPartialSort(int16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortI16Asc)(keys, n, k);
}
void VQSelect(int16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectI16Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_i16d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_i16d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortI16Desc(int16_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortI16Desc(int16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectI16Desc(int16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortI16Desc);
HWY_EXPORT(PartialSortI16Desc);
HWY_EXPORT(SelectI16Desc);
} // namespace
void VQSort(int16_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortI16Desc)(keys, n);
}
void VQPartialSort(int16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortI16Desc)(keys, n, k);
}
void VQSelect(int16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectI16Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_i32a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_i32a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortI32Asc(int32_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortI32Asc(int32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectI32Asc(int32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortI32Asc);
HWY_EXPORT(PartialSortI32Asc);
HWY_EXPORT(SelectI32Asc);
} // namespace
void VQSort(int32_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortI32Asc)(keys, n);
}
void VQPartialSort(int32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortI32Asc)(keys, n, k);
}
void VQSelect(int32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectI32Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_i32d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_i32d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortI32Desc(int32_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortI32Desc(int32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectI32Desc(int32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortI32Desc);
HWY_EXPORT(PartialSortI32Desc);
HWY_EXPORT(SelectI32Desc);
} // namespace
void VQSort(int32_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortI32Desc)(keys, n);
}
void VQPartialSort(int32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortI32Desc)(keys, n, k);
}
void VQSelect(int32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectI32Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_i64a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_i64a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortI64Asc(int64_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortI64Asc(int64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectI64Asc(int64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortI64Asc);
HWY_EXPORT(PartialSortI64Asc);
HWY_EXPORT(SelectI64Asc);
} // namespace
void VQSort(int64_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortI64Asc)(keys, n);
}
void VQPartialSort(int64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortI64Asc)(keys, n, k);
}
void VQSelect(int64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectI64Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_i64d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_i64d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortI64Desc(int64_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortI64Desc(int64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectI64Desc(int64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortI64Desc);
HWY_EXPORT(PartialSortI64Desc);
HWY_EXPORT(SelectI64Desc);
} // namespace
void VQSort(int64_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortI64Desc)(keys, n);
}
void VQPartialSort(int64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortI64Desc)(keys, n, k);
}
void VQSelect(int64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectI64Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

74
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_kv128a.cc vendored Normal file
View File

@ -0,0 +1,74 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
// clang-format off
// (avoid line break, which would prevent Copybara rules from matching)
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_kv128a.cc" //NOLINT
// clang-format on
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortKV128Asc(K64V64* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortKV128Asc(K64V64* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectKV128Asc(K64V64* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortKV128Asc);
HWY_EXPORT(PartialSortKV128Asc);
HWY_EXPORT(SelectKV128Asc);
} // namespace
void VQSort(K64V64* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortKV128Asc)(keys, n);
}
void VQPartialSort(K64V64* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortKV128Asc)(keys, n, k);
}
void VQSelect(K64V64* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectKV128Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

74
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_kv128d.cc vendored Normal file
View File

@ -0,0 +1,74 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
// clang-format off
// (avoid line break, which would prevent Copybara rules from matching)
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_kv128d.cc" //NOLINT
// clang-format on
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortKV128Desc(K64V64* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortKV128Desc(K64V64* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectKV128Desc(K64V64* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortKV128Desc);
HWY_EXPORT(PartialSortKV128Desc);
HWY_EXPORT(SelectKV128Desc);
} // namespace
void VQSort(K64V64* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortKV128Desc)(keys, n);
}
void VQPartialSort(K64V64* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortKV128Desc)(keys, n, k);
}
void VQSelect(K64V64* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectKV128Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

74
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_kv64a.cc vendored Normal file
View File

@ -0,0 +1,74 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
// clang-format off
// (avoid line break, which would prevent Copybara rules from matching)
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_kv64a.cc" //NOLINT
// clang-format on
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortKV64Asc(K32V32* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortKV64Asc(K32V32* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectKV64Asc(K32V32* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortKV64Asc);
HWY_EXPORT(PartialSortKV64Asc);
HWY_EXPORT(SelectKV64Asc);
} // namespace
void VQSort(K32V32* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortKV64Asc)(keys, n);
}
void VQPartialSort(K32V32* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortKV64Asc)(keys, n, k);
}
void VQSelect(K32V32* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectKV64Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

74
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_kv64d.cc vendored Normal file
View File

@ -0,0 +1,74 @@
// Copyright 2022 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
// clang-format off
// (avoid line break, which would prevent Copybara rules from matching)
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_kv64d.cc" //NOLINT
// clang-format on
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortKV64Desc(K32V32* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortKV64Desc(K32V32* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectKV64Desc(K32V32* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortKV64Desc);
HWY_EXPORT(PartialSortKV64Desc);
HWY_EXPORT(SelectKV64Desc);
} // namespace
void VQSort(K32V32* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortKV64Desc)(keys, n);
}
void VQPartialSort(K32V32* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortKV64Desc)(keys, n, k);
}
void VQSelect(K32V32* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectKV64Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_u16a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_u16a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortU16Asc(uint16_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortU16Asc(uint16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectU16Asc(uint16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortU16Asc);
HWY_EXPORT(PartialSortU16Asc);
HWY_EXPORT(SelectU16Asc);
} // namespace
void VQSort(uint16_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortU16Asc)(keys, n);
}
void VQPartialSort(uint16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortU16Asc)(keys, n, k);
}
void VQSelect(uint16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectU16Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_u16d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_u16d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortU16Desc(uint16_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortU16Desc(uint16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectU16Desc(uint16_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortU16Desc);
HWY_EXPORT(PartialSortU16Desc);
HWY_EXPORT(SelectU16Desc);
} // namespace
void VQSort(uint16_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortU16Desc)(keys, n);
}
void VQPartialSort(uint16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortU16Desc)(keys, n, k);
}
void VQSelect(uint16_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectU16Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_u32a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_u32a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortU32Asc(uint32_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortU32Asc(uint32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectU32Asc(uint32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortU32Asc);
HWY_EXPORT(PartialSortU32Asc);
HWY_EXPORT(SelectU32Asc);
} // namespace
void VQSort(uint32_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortU32Asc)(keys, n);
}
void VQPartialSort(uint32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortU32Asc)(keys, n, k);
}
void VQSelect(uint32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectU32Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_u32d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_u32d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortU32Desc(uint32_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortU32Desc(uint32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectU32Desc(uint32_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortU32Desc);
HWY_EXPORT(PartialSortU32Desc);
HWY_EXPORT(SelectU32Desc);
} // namespace
void VQSort(uint32_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortU32Desc)(keys, n);
}
void VQPartialSort(uint32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortU32Desc)(keys, n, k);
}
void VQSelect(uint32_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectU32Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_u64a.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_u64a.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortU64Asc(uint64_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortAscending());
}
void PartialSortU64Asc(uint64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortAscending());
}
void SelectU64Asc(uint64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortAscending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortU64Asc);
HWY_EXPORT(PartialSortU64Asc);
HWY_EXPORT(SelectU64Asc);
} // namespace
void VQSort(uint64_t* HWY_RESTRICT keys, const size_t n, SortAscending) {
HWY_DYNAMIC_DISPATCH(SortU64Asc)(keys, n);
}
void VQPartialSort(uint64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(PartialSortU64Asc)(keys, n, k);
}
void VQSelect(uint64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortAscending) {
HWY_DYNAMIC_DISPATCH(SelectU64Asc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

71
deps/v8/third_party/highway/src/hwy/contrib/sort/vqsort_u64d.cc vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/sort/vqsort.h" // VQSort
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/sort/vqsort_u64d.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// After foreach_target
#include "hwy/contrib/sort/vqsort-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
void SortU64Desc(uint64_t* HWY_RESTRICT keys, const size_t num) {
return VQSortStatic(keys, num, SortDescending());
}
void PartialSortU64Desc(uint64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQPartialSortStatic(keys, num, k, SortDescending());
}
void SelectU64Desc(uint64_t* HWY_RESTRICT keys, const size_t num,
const size_t k) {
return VQSelectStatic(keys, num, k, SortDescending());
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_EXPORT(SortU64Desc);
HWY_EXPORT(PartialSortU64Desc);
HWY_EXPORT(SelectU64Desc);
} // namespace
void VQSort(uint64_t* HWY_RESTRICT keys, const size_t n, SortDescending) {
HWY_DYNAMIC_DISPATCH(SortU64Desc)(keys, n);
}
void VQPartialSort(uint64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(PartialSortU64Desc)(keys, n, k);
}
void VQSelect(uint64_t* HWY_RESTRICT keys, const size_t n, const size_t k,
SortDescending) {
HWY_DYNAMIC_DISPATCH(SelectU64Desc)(keys, n, k);
}
} // namespace hwy
#endif // HWY_ONCE

199
deps/v8/third_party/highway/src/hwy/contrib/thread_pool/futex.h vendored Normal file
View File

@ -0,0 +1,199 @@
// Copyright 2024 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_CONTRIB_THREAD_POOL_FUTEX_H_
#define HIGHWAY_HWY_CONTRIB_THREAD_POOL_FUTEX_H_
// Keyed event (futex): kernel queue of blocked threads, identified by the
// address of an atomic u32 called `current` within the same process (do NOT
// use with shared-memory mappings).
//
// Futex equivalents: https://outerproduct.net/futex-dictionary.html; we
// support Linux/Emscripten/Apple/Windows and C++20 std::atomic::wait, plus a
// usleep fallback.
#include <atomic>
#include <climits> // INT_MAX
#include "hwy/base.h"
#if HWY_ARCH_WASM
#include <emscripten/threading.h>
#include <math.h> // INFINITY
#elif HWY_OS_LINUX
#include <errno.h> // IWYU pragma: keep
#include <linux/futex.h> // FUTEX_*
#include <pthread.h>
#include <sys/syscall.h> // SYS_*
#include <unistd.h>
// Android may not declare these:
#ifndef SYS_futex
#ifdef SYS_futex_time64 // 32-bit with 64-bit time_t
#define SYS_futex SYS_futex_time64
#else
#define SYS_futex __NR_futex
#endif // SYS_futex_time64
#endif // SYS_futex
#ifndef FUTEX_WAIT_PRIVATE
#define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | 128)
#endif
#ifndef FUTEX_WAKE_PRIVATE
#define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | 128)
#endif
#elif HWY_OS_APPLE && !defined(HWY_DISABLE_FUTEX)
// These are private APIs, so add an opt-out.
extern "C" {
int __ulock_wait(uint32_t op, void* address, uint64_t val, uint32_t max_us);
int __ulock_wake(uint32_t op, void* address, uint64_t zero);
} // extern "C"
#define UL_COMPARE_AND_WAIT 1
#define ULF_WAKE_ALL 0x00000100
#elif HWY_OS_WIN && !defined(HWY_DISABLE_FUTEX)
// WakeByAddressAll requires Windows 8, so add an opt-out.
#include <windows.h>
#pragma comment(lib, "synchronization.lib")
#elif HWY_CXX_LANG < 202002L // NOT C++20, which has native support
#define HWY_FUTEX_SLEEP
#include <chrono> // NOLINT (sleep_for)
#endif
namespace hwy {
// Waits until `current != prev` and returns the new value. May return
// immediately if `current` already changed, or after blocking and waking.
static inline uint32_t BlockUntilDifferent(
const uint32_t prev, const std::atomic<uint32_t>& current) {
const auto acq = std::memory_order_acquire;
#if HWY_ARCH_WASM
// It is always safe to cast to void.
volatile void* address =
const_cast<volatile void*>(static_cast<const volatile void*>(&current));
const double max_ms = INFINITY;
for (;;) {
const uint32_t next = current.load(acq);
if (next != prev) return next;
const int ret = emscripten_futex_wait(address, prev, max_ms);
HWY_DASSERT(ret >= 0);
(void)ret;
}
#elif HWY_OS_LINUX
// Safe to cast because std::atomic is a standard layout type.
const uint32_t* address = reinterpret_cast<const uint32_t*>(&current);
// _PRIVATE requires this only be used in the same process, and avoids
// virtual->physical lookups and atomic reference counting.
const int op = FUTEX_WAIT_PRIVATE;
for (;;) {
const uint32_t next = current.load(acq);
if (next != prev) return next;
// timeout=null may prevent interrupts via signal. No lvalue because
// the timespec type is only standardized since C++17 or C11.
const auto ret = syscall(SYS_futex, address, op, prev, nullptr, nullptr, 0);
if (ret == -1) {
HWY_DASSERT(errno == EAGAIN); // otherwise an actual error
}
}
#elif HWY_OS_WIN && !defined(HWY_DISABLE_FUTEX)
// It is always safe to cast to void.
volatile void* address =
const_cast<volatile void*>(static_cast<const volatile void*>(&current));
// API is not const-correct, but only loads from the pointer.
PVOID pprev = const_cast<void*>(static_cast<const void*>(&prev));
const DWORD max_ms = INFINITE;
for (;;) {
const uint32_t next = current.load(acq);
if (next != prev) return next;
const BOOL ok = WaitOnAddress(address, pprev, sizeof(prev), max_ms);
HWY_DASSERT(ok);
(void)ok;
}
#elif HWY_OS_APPLE && !defined(HWY_DISABLE_FUTEX)
// It is always safe to cast to void.
void* address = const_cast<void*>(static_cast<const void*>(&current));
for (;;) {
const uint32_t next = current.load(acq);
if (next != prev) return next;
__ulock_wait(UL_COMPARE_AND_WAIT, address, prev, 0);
}
#elif defined(HWY_FUTEX_SLEEP)
for (;;) {
const uint32_t next = current.load(acq);
if (next != prev) return next;
std::this_thread::sleep_for(std::chrono::microseconds(2));
}
#elif HWY_CXX_LANG >= 202002L
current.wait(prev, acq); // No spurious wakeup.
const uint32_t next = current.load(acq);
HWY_DASSERT(next != prev);
return next;
#else
#error "Logic error, should have reached HWY_FUTEX_SLEEP"
#endif // HWY_OS_*
} // BlockUntilDifferent
// Wakes all threads, if any, that are waiting because they called
// `BlockUntilDifferent` with the same `current`.
static inline void WakeAll(std::atomic<uint32_t>& current) {
#if HWY_ARCH_WASM
// It is always safe to cast to void.
volatile void* address = static_cast<volatile void*>(&current);
const int max_to_wake = INT_MAX; // actually signed
const int ret = emscripten_futex_wake(address, max_to_wake);
HWY_DASSERT(ret >= 0);
(void)ret;
#elif HWY_OS_LINUX
// Safe to cast because std::atomic is a standard layout type.
uint32_t* address = reinterpret_cast<uint32_t*>(&current);
const int max_to_wake = INT_MAX; // actually signed
const auto ret = syscall(SYS_futex, address, FUTEX_WAKE_PRIVATE, max_to_wake,
nullptr, nullptr, 0);
HWY_DASSERT(ret >= 0); // number woken
(void)ret;
#elif HWY_OS_WIN && !defined(HWY_DISABLE_FUTEX)
// It is always safe to cast to void.
void* address = static_cast<void*>(&current);
WakeByAddressAll(address);
#elif HWY_OS_APPLE && !defined(HWY_DISABLE_FUTEX)
// It is always safe to cast to void.
void* address = static_cast<void*>(&current);
__ulock_wake(UL_COMPARE_AND_WAIT | ULF_WAKE_ALL, address, 0);
#elif defined(HWY_FUTEX_SLEEP)
// Sleep loop does not require wakeup.
(void)current;
#elif HWY_CXX_LANG >= 202002L
current.notify_all();
#else
#error "Logic error, should have reached HWY_FUTEX_SLEEP"
#endif
} // WakeAll
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_THREAD_POOL_FUTEX_H_

693
deps/v8/third_party/highway/src/hwy/contrib/thread_pool/thread_pool.h vendored Normal file
View File

@ -0,0 +1,693 @@
// Copyright 2023 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified from BSD-licensed code
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
// See https://github.com/libjxl/libjxl/blob/main/LICENSE.
#ifndef HIGHWAY_HWY_CONTRIB_THREAD_POOL_THREAD_POOL_H_
#define HIGHWAY_HWY_CONTRIB_THREAD_POOL_THREAD_POOL_H_
// IWYU pragma: begin_exports
#include <stddef.h>
#include <stdint.h>
#include <stdio.h> // snprintf
#include <array>
#include <new>
#include <thread> //NOLINT
// IWYU pragma: end_exports
#include <atomic>
#include <vector>
#include "hwy/aligned_allocator.h" // HWY_ALIGNMENT
#include "hwy/base.h"
#include "hwy/cache_control.h" // Pause
#include "hwy/contrib/thread_pool/futex.h"
#include "hwy/contrib/thread_pool/topology.h"
// Temporary NOINLINE for profiling.
#define HWY_POOL_INLINE HWY_NOINLINE
#ifndef HWY_POOL_SETRANGE_INLINE
#if HWY_ARCH_ARM
// Workaround for invalid codegen on Arm (begin_ is larger than expected).
#define HWY_POOL_SETRANGE_INLINE HWY_NOINLINE
#else
#define HWY_POOL_SETRANGE_INLINE
#endif
#endif // HWY_POOL_SETRANGE_INLINE
namespace hwy {
// Generates a random permutation of [0, size). O(1) storage.
class ShuffledIota {
public:
ShuffledIota() : coprime_(1) {} // for PoolWorker
explicit ShuffledIota(uint32_t coprime) : coprime_(coprime) {}
// Returns the next after `current`, using an LCG-like generator.
uint32_t Next(uint32_t current, const Divisor& divisor) const {
HWY_DASSERT(current < divisor.GetDivisor());
// (coprime * i + current) % size, see https://lemire.me/blog/2017/09/18/.
return divisor.Remainder(current + coprime_);
}
// Returns true if a and b have no common denominator except 1. Based on
// binary GCD. Assumes a and b are nonzero. Also used in tests.
static bool CoprimeNonzero(uint32_t a, uint32_t b) {
const size_t trailing_a = Num0BitsBelowLS1Bit_Nonzero32(a);
const size_t trailing_b = Num0BitsBelowLS1Bit_Nonzero32(b);
// If both have at least one trailing zero, they are both divisible by 2.
if (HWY_MIN(trailing_a, trailing_b) != 0) return false;
// If one of them has a trailing zero, shift it out.
a >>= trailing_a;
b >>= trailing_b;
for (;;) {
// Swap such that a >= b.
const uint32_t tmp_a = a;
a = HWY_MAX(tmp_a, b);
b = HWY_MIN(tmp_a, b);
// When the smaller number is 1, they were coprime.
if (b == 1) return true;
a -= b;
// a == b means there was a common factor, so not coprime.
if (a == 0) return false;
a >>= Num0BitsBelowLS1Bit_Nonzero32(a);
}
}
// Returns another coprime >= `start`, or 1 for small `size`.
// Used to seed independent ShuffledIota instances.
static uint32_t FindAnotherCoprime(uint32_t size, uint32_t start) {
if (size <= 2) {
return 1;
}
// Avoids even x for even sizes, which are sure to be rejected.
const uint32_t inc = (size & 1) ? 1 : 2;
for (uint32_t x = start | 1; x < start + size * 16; x += inc) {
if (CoprimeNonzero(x, static_cast<uint32_t>(size))) {
return x;
}
}
HWY_ABORT("unreachable");
}
uint32_t coprime_;
};
// We want predictable struct/class sizes so we can reason about cache lines.
#pragma pack(push, 1)
enum class PoolWaitMode : uint32_t { kBlock, kSpin };
// Worker's private working set.
class PoolWorker { // HWY_ALIGNMENT bytes
static constexpr size_t kMaxVictims = 4;
public:
PoolWorker(size_t thread, size_t num_workers) {
wait_mode_ = PoolWaitMode::kBlock;
num_victims_ = static_cast<uint32_t>(HWY_MIN(kMaxVictims, num_workers));
const Divisor div_workers(static_cast<uint32_t>(num_workers));
// Increase gap between coprimes to reduce collisions.
const uint32_t coprime = ShuffledIota::FindAnotherCoprime(
static_cast<uint32_t>(num_workers),
static_cast<uint32_t>((thread + 1) * 257 + thread * 13));
const ShuffledIota shuffled_iota(coprime);
// To simplify WorkerRun, our own thread is the first to 'steal' from.
victims_[0] = static_cast<uint32_t>(thread);
for (uint32_t i = 1; i < num_victims_; ++i) {
victims_[i] = shuffled_iota.Next(victims_[i - 1], div_workers);
HWY_DASSERT(victims_[i] != thread);
}
(void)padding_;
}
~PoolWorker() = default;
void SetWaitMode(PoolWaitMode wait_mode) {
wait_mode_.store(wait_mode, std::memory_order_release);
}
PoolWaitMode WorkerGetWaitMode() const {
return wait_mode_.load(std::memory_order_acquire);
}
hwy::Span<const uint32_t> Victims() const {
return hwy::Span<const uint32_t>(victims_.data(),
static_cast<size_t>(num_victims_));
}
// Called from main thread in Plan().
HWY_POOL_SETRANGE_INLINE void SetRange(uint64_t begin, uint64_t end) {
const auto rel = std::memory_order_release;
begin_.store(begin, rel);
end_.store(end, rel);
}
// Returns the STL-style end of this worker's assigned range.
uint64_t WorkerGetEnd() const { return end_.load(std::memory_order_acquire); }
// Returns the next task to execute. If >= WorkerGetEnd(), it must be skipped.
uint64_t WorkerReserveTask() {
return begin_.fetch_add(1, std::memory_order_relaxed);
}
private:
std::atomic<uint64_t> begin_;
std::atomic<uint64_t> end_; // only changes during SetRange
std::atomic<PoolWaitMode> wait_mode_; // (32-bit)
uint32_t num_victims_; // <= kPoolMaxVictims
std::array<uint32_t, kMaxVictims> victims_;
uint8_t padding_[HWY_ALIGNMENT - 16 - 8 - sizeof(victims_)];
};
static_assert(sizeof(PoolWorker) == HWY_ALIGNMENT, "");
// Modified by main thread, shared with all workers.
class PoolTasks { // 32 bytes
// Signature of the (internal) function called from workers(s) for each
// `task` in the [`begin`, `end`) passed to Run(). Closures (lambdas) do not
// receive the first argument, which points to the lambda object.
typedef void (*RunFunc)(const void* opaque, uint64_t task, size_t thread_id);
// Calls closure(task, thread). Signature must match RunFunc.
template <class Closure>
static void CallClosure(const void* opaque, uint64_t task, size_t thread) {
(*reinterpret_cast<const Closure*>(opaque))(task, thread);
}
public:
// Called from main thread in Plan().
template <class Closure>
void Store(const Closure& closure, uint64_t begin, uint64_t end) {
const auto rel = std::memory_order_release;
func_.store(static_cast<RunFunc>(&CallClosure<Closure>), rel);
opaque_.store(reinterpret_cast<const void*>(&closure), rel);
begin_.store(begin, rel);
end_.store(end, rel);
}
RunFunc WorkerGet(uint64_t& begin, uint64_t& end, const void*& opaque) const {
const auto acq = std::memory_order_acquire;
begin = begin_.load(acq);
end = end_.load(acq);
opaque = opaque_.load(acq);
return func_.load(acq);
}
private:
std::atomic<RunFunc> func_;
std::atomic<const void*> opaque_;
std::atomic<uint64_t> begin_;
std::atomic<uint64_t> end_;
};
// Modified by main thread, shared with all workers.
class PoolCommands { // 16 bytes
static constexpr uint32_t kInitial = 0;
static constexpr uint32_t kMask = 0xF; // for command, rest is ABA counter.
static constexpr size_t kShift = hwy::CeilLog2(kMask);
public:
static constexpr uint32_t kTerminate = 1;
static constexpr uint32_t kWork = 2;
static constexpr uint32_t kNop = 3;
// Workers must initialize their copy to this so that they wait for the first
// command as intended.
static uint32_t WorkerInitialSeqCmd() { return kInitial; }
// Sends `cmd` to all workers.
void Broadcast(uint32_t cmd) {
HWY_DASSERT(cmd <= kMask);
const uint32_t epoch = ++epoch_;
const uint32_t seq_cmd = (epoch << kShift) | cmd;
seq_cmd_.store(seq_cmd, std::memory_order_release);
// Wake any worker whose wait_mode_ is or was kBlock.
WakeAll(seq_cmd_);
// Workers are either starting up, or waiting for a command. Either way,
// they will not miss this command, so no need to wait for them here.
}
// Returns the command, i.e., one of the public constants, e.g., kTerminate.
uint32_t WorkerWaitForNewCommand(PoolWaitMode wait_mode,
uint32_t& prev_seq_cmd) {
uint32_t seq_cmd;
if (HWY_LIKELY(wait_mode == PoolWaitMode::kSpin)) {
seq_cmd = SpinUntilDifferent(prev_seq_cmd, seq_cmd_);
} else {
seq_cmd = BlockUntilDifferent(prev_seq_cmd, seq_cmd_);
}
prev_seq_cmd = seq_cmd;
return seq_cmd & kMask;
}
private:
static HWY_INLINE uint32_t SpinUntilDifferent(
const uint32_t prev_seq_cmd, std::atomic<uint32_t>& current) {
for (;;) {
hwy::Pause();
const uint32_t seq_cmd = current.load(std::memory_order_acquire);
if (seq_cmd != prev_seq_cmd) return seq_cmd;
}
}
// Counter for ABA-proofing WorkerWaitForNewCommand. Stored next to seq_cmd_
// because both are written at the same time by the main thread. Sharding this
// 4x (one per cache line) is not helpful.
uint32_t epoch_{0};
std::atomic<uint32_t> seq_cmd_{kInitial};
};
// Modified by main thread AND workers.
// TODO(janwas): more scalable tree
class alignas(HWY_ALIGNMENT) PoolBarrier { // 4 * HWY_ALIGNMENT bytes
static constexpr size_t kU64PerCacheLine = HWY_ALIGNMENT / sizeof(uint64_t);
public:
void Reset() {
for (size_t i = 0; i < 4; ++i) {
num_finished_[i * kU64PerCacheLine].store(0, std::memory_order_release);
}
}
void WorkerArrive(size_t thread) {
const size_t i = (thread & 3);
num_finished_[i * kU64PerCacheLine].fetch_add(1, std::memory_order_release);
}
// Spin until all have called Arrive(). Note that workers spin for a new
// command, not the barrier itself.
HWY_POOL_INLINE void WaitAll(size_t num_workers) {
const auto acq = std::memory_order_acquire;
for (;;) {
hwy::Pause();
const uint64_t sum = num_finished_[0 * kU64PerCacheLine].load(acq) +
num_finished_[1 * kU64PerCacheLine].load(acq) +
num_finished_[2 * kU64PerCacheLine].load(acq) +
num_finished_[3 * kU64PerCacheLine].load(acq);
if (sum == num_workers) break;
}
}
private:
// Sharded to reduce contention. Four counters, each in their own cache line.
std::atomic<uint64_t> num_finished_[4 * kU64PerCacheLine];
};
// All mutable pool and worker state.
struct alignas(HWY_ALIGNMENT) PoolMem {
PoolWorker& Worker(size_t thread) {
return *reinterpret_cast<PoolWorker*>(reinterpret_cast<uint8_t*>(&barrier) +
sizeof(barrier) +
thread * sizeof(PoolWorker));
}
PoolTasks tasks;
PoolCommands commands;
// barrier is more write-heavy, hence keep in another cache line.
uint8_t padding[HWY_ALIGNMENT - sizeof(tasks) - sizeof(commands)];
PoolBarrier barrier;
static_assert(sizeof(barrier) % HWY_ALIGNMENT == 0, "");
// Followed by `num_workers` PoolWorker.
};
// Aligned allocation and initialization of variable-length PoolMem.
class PoolMemOwner {
public:
explicit PoolMemOwner(size_t num_threads)
// The main thread also participates.
: num_workers_(num_threads + 1) {
const size_t size = sizeof(PoolMem) + num_workers_ * sizeof(PoolWorker);
bytes_ = hwy::AllocateAligned<uint8_t>(size);
HWY_ASSERT(bytes_);
mem_ = new (bytes_.get()) PoolMem();
for (size_t thread = 0; thread < num_workers_; ++thread) {
new (&mem_->Worker(thread)) PoolWorker(thread, num_workers_);
}
// Publish non-atomic stores in mem_ - that is the only shared state workers
// access before they call WorkerWaitForNewCommand.
std::atomic_thread_fence(std::memory_order_release);
}
~PoolMemOwner() {
for (size_t thread = 0; thread < num_workers_; ++thread) {
mem_->Worker(thread).~PoolWorker();
}
mem_->~PoolMem();
}
size_t NumWorkers() const { return num_workers_; }
PoolMem* Mem() const { return mem_; }
private:
const size_t num_workers_; // >= 1
// Aligned allocation ensures we do not straddle cache lines.
hwy::AlignedFreeUniquePtr<uint8_t[]> bytes_;
PoolMem* mem_;
};
// Plans and executes parallel-for loops with work-stealing. No synchronization
// because there is no mutable shared state.
class ParallelFor { // 0 bytes
// A prior version of this code attempted to assign only as much work as a
// thread will actually use. As with OpenMP's 'guided' strategy, we assigned
// remaining/(k*num_threads) in each iteration. Although the worst-case
// imbalance is bounded, this required several rounds of work allocation, and
// the atomic counter did not scale to > 30 threads.
//
// We now use work stealing instead, where already-finished threads look for
// and perform work from others, as if they were that thread. This deals with
// imbalances as they arise, but care is required to reduce contention. We
// randomize the order in which threads choose victims to steal from.
//
// Results: across 10K calls Run(), we observe a mean of 5.1 tasks per
// thread, and standard deviation 0.67, indicating good load-balance.
public:
// Make preparations for workers to later run `closure(i)` for all `i` in
// `[begin, end)`. Called from the main thread; workers are initializing or
// spinning for a command. Returns false if there are no tasks or workers.
template <class Closure>
static bool Plan(uint64_t begin, uint64_t end, size_t num_workers,
const Closure& closure, PoolMem& mem) {
// If there are no tasks, we are done.
HWY_DASSERT(begin <= end);
const size_t num_tasks = static_cast<size_t>(end - begin);
if (HWY_UNLIKELY(num_tasks == 0)) return false;
// If there are no workers, run all tasks already on the main thread without
// the overhead of planning.
if (HWY_UNLIKELY(num_workers <= 1)) {
for (uint64_t task = begin; task < end; ++task) {
closure(task, /*thread=*/0);
}
return false;
}
// Store for later retrieval by all workers in WorkerRun. Must happen after
// the loop above because it may be re-entered by concurrent threads.
mem.tasks.Store(closure, begin, end);
// Assigning all remainders to the last thread causes imbalance. We instead
// give one more to each thread whose index is less.
const size_t remainder = num_tasks % num_workers;
const size_t min_tasks = num_tasks / num_workers;
uint64_t task = begin;
for (size_t thread = 0; thread < num_workers; ++thread) {
const uint64_t my_end = task + min_tasks + (thread < remainder);
mem.Worker(thread).SetRange(task, my_end);
task = my_end;
}
HWY_DASSERT(task == end);
return true;
}
// Must be called for each `thread` in [0, num_workers), but only if
// Plan returned true.
static HWY_POOL_INLINE void WorkerRun(const size_t thread, size_t num_workers,
PoolMem& mem) {
// Nonzero, otherwise Plan returned false and this should not be called.
HWY_DASSERT(num_workers != 0);
HWY_DASSERT(thread < num_workers);
const PoolTasks& tasks = mem.tasks;
uint64_t begin, end;
const void* opaque;
const auto func = tasks.WorkerGet(begin, end, opaque);
// Special case for <= 1 task per worker - avoid any shared state.
if (HWY_UNLIKELY(end <= begin + num_workers)) {
const uint64_t task = begin + thread;
if (HWY_LIKELY(task < end)) {
func(opaque, task, thread);
}
return;
}
// For each worker in random order, attempt to do all their work.
for (uint32_t victim : mem.Worker(thread).Victims()) {
PoolWorker* other_worker = &mem.Worker(victim);
// Until all of other_worker's work is done:
const uint64_t other_end = other_worker->WorkerGetEnd();
for (;;) {
// On x86 this generates a LOCK prefix, but that is only expensive if
// there is actually contention, which is unlikely because we shard the
// counters, threads do not quite proceed in lockstep due to memory
// traffic, and stealing happens in semi-random order.
uint64_t task = other_worker->WorkerReserveTask();
// The worker that first sets `task` to `other_end` exits this loop.
// After that, `task` can be incremented up to `num_workers - 1` times,
// once per other worker.
HWY_DASSERT(task < other_end + num_workers);
if (HWY_UNLIKELY(task >= other_end)) {
hwy::Pause(); // Reduce coherency traffic while stealing.
break;
}
// `thread` is the one we are actually running on; this is important
// because it is the TLS index for user code.
func(opaque, task, thread);
}
}
}
};
#pragma pack(pop)
// Sets the name of the current thread to the format string `format`, which must
// include %d for `thread`. Currently only implemented for pthreads (*nix and
// OSX); Windows involves throwing an exception.
static inline void SetThreadName(const char* format, int thread) {
#if HWY_OS_LINUX
char buf[16] = {}; // Linux limit, including \0
const int chars_written = snprintf(buf, sizeof(buf), format, thread);
HWY_ASSERT(0 < chars_written &&
chars_written <= static_cast<int>(sizeof(buf) - 1));
HWY_ASSERT(0 == pthread_setname_np(pthread_self(), buf));
#else
(void)format;
(void)thread;
#endif
}
// Highly efficient parallel-for, intended for workloads with thousands of
// fork-join regions which consist of calling tasks[t](i) for a few hundred i,
// using dozens of threads.
//
// To reduce scheduling overhead, we assume that tasks are statically known and
// that threads do not schedule new work themselves. This allows us to avoid
// queues and only store a counter plus the current task. The latter is a
// pointer to a lambda function, without the allocation/indirection required for
// std::function.
//
// To reduce fork/join latency, we use an efficient barrier, optionally
// support spin-waits via SetWaitMode, and avoid any mutex/lock.
//
// To eliminate false sharing and enable reasoning about cache line traffic, the
// worker state uses a single aligned allocation.
//
// For load-balancing, we use work stealing in random order.
class ThreadPool {
static void ThreadFunc(size_t thread, size_t num_workers, PoolMem* mem) {
HWY_DASSERT(thread < num_workers);
SetThreadName("worker%03zu", static_cast<int>(thread));
// Ensure mem is ready to use (synchronize with PoolMemOwner's fence).
std::atomic_thread_fence(std::memory_order_acquire);
PoolWorker& worker = mem->Worker(thread);
PoolCommands& commands = mem->commands;
uint32_t prev_seq_cmd = PoolCommands::WorkerInitialSeqCmd();
for (;;) {
const PoolWaitMode wait_mode = worker.WorkerGetWaitMode();
const uint32_t command =
commands.WorkerWaitForNewCommand(wait_mode, prev_seq_cmd);
if (HWY_UNLIKELY(command == PoolCommands::kTerminate)) {
return; // exits thread
} else if (HWY_LIKELY(command == PoolCommands::kWork)) {
ParallelFor::WorkerRun(thread, num_workers, *mem);
mem->barrier.WorkerArrive(thread);
} else if (command == PoolCommands::kNop) {
// do nothing - used to change wait mode
} else {
HWY_DASSERT(false); // unknown command
}
}
}
public:
// This typically includes hyperthreads, hence it is a loose upper bound.
// -1 because these are in addition to the main thread.
static size_t MaxThreads() {
LogicalProcessorSet lps;
// This is OS dependent, but more accurate if available because it takes
// into account restrictions set by cgroups or numactl/taskset.
if (GetThreadAffinity(lps)) {
return lps.Count() - 1;
}
return static_cast<size_t>(std::thread::hardware_concurrency() - 1);
}
// `num_threads` should not exceed `MaxThreads()`. If `num_threads` <= 1,
// Run() runs only on the main thread. Otherwise, we launch `num_threads - 1`
// threads because the main thread also participates.
explicit ThreadPool(size_t num_threads) : owner_(num_threads) {
(void)busy_; // unused in non-debug builds, avoid warning
const size_t num_workers = owner_.NumWorkers();
// Launch threads without waiting afterwards: they will receive the next
// PoolCommands once ready.
threads_.reserve(num_workers - 1);
for (size_t thread = 0; thread < num_workers - 1; ++thread) {
threads_.emplace_back(ThreadFunc, thread, num_workers, owner_.Mem());
}
}
// Waits for all threads to exit.
~ThreadPool() {
PoolMem& mem = *owner_.Mem();
mem.commands.Broadcast(PoolCommands::kTerminate); // requests threads exit
for (std::thread& thread : threads_) {
HWY_ASSERT(thread.joinable());
thread.join();
}
}
ThreadPool(const ThreadPool&) = delete;
ThreadPool& operator&(const ThreadPool&) = delete;
// Returns number of PoolWorker, i.e., one more than the largest `thread`
// argument. Useful for callers that want to allocate thread-local storage.
size_t NumWorkers() const { return owner_.NumWorkers(); }
// `mode` is initially `kBlock`, which means futex. Switching to `kSpin`
// reduces fork-join overhead especially when there are many calls to `Run`,
// but wastes power when waiting over long intervals. Inexpensive, OK to call
// multiple times, but not concurrently with any `Run`.
void SetWaitMode(PoolWaitMode mode) {
// Run must not be active, otherwise we may overwrite the previous command
// before it is seen by all workers.
HWY_DASSERT(busy_.fetch_add(1) == 0);
PoolMem& mem = *owner_.Mem();
// For completeness/consistency, set on all workers, including the main
// thread, even though it will never wait for a command.
for (size_t thread = 0; thread < owner_.NumWorkers(); ++thread) {
mem.Worker(thread).SetWaitMode(mode);
}
// Send a no-op command so that workers wake as soon as possible. Skip the
// expensive barrier - workers may miss this command, but it is fine for
// them to wake up later and get the next actual command.
mem.commands.Broadcast(PoolCommands::kNop);
HWY_DASSERT(busy_.fetch_add(-1) == 1);
}
// parallel-for: Runs `closure(task, thread)` on worker thread(s) for every
// `task` in `[begin, end)`. Note that the unit of work should be large
// enough to amortize the function call overhead, but small enough that each
// worker processes a few tasks. Thus each `task` is usually a loop.
//
// Not thread-safe - concurrent calls to `Run` in the same ThreadPool are
// forbidden unless NumWorkers() == 0. We check for that in debug builds.
template <class Closure>
void Run(uint64_t begin, uint64_t end, const Closure& closure) {
const size_t num_workers = NumWorkers();
PoolMem& mem = *owner_.Mem();
if (HWY_LIKELY(ParallelFor::Plan(begin, end, num_workers, closure, mem))) {
// Only check if we are going to fork/join.
HWY_DASSERT(busy_.fetch_add(1) == 0);
mem.barrier.Reset();
mem.commands.Broadcast(PoolCommands::kWork);
// Also perform work on main thread instead of busy-waiting.
const size_t thread = num_workers - 1;
ParallelFor::WorkerRun(thread, num_workers, mem);
mem.barrier.WorkerArrive(thread);
mem.barrier.WaitAll(num_workers);
HWY_DASSERT(busy_.fetch_add(-1) == 1);
}
}
// Can pass this as init_closure when no initialization is needed.
// DEPRECATED, better to call the Run() overload without the init_closure arg.
static bool NoInit(size_t /*num_threads*/) { return true; } // DEPRECATED
// DEPRECATED equivalent of NumWorkers. Note that this is not the same as the
// ctor argument because num_threads = 0 has the same effect as 1.
size_t NumThreads() const { return NumWorkers(); } // DEPRECATED
// DEPRECATED prior interface with 32-bit tasks and first calling
// `init_closure(num_threads)`. Instead, perform any init before this, calling
// NumWorkers() for an upper bound on the thread indices, then call the
// other overload.
template <class InitClosure, class RunClosure>
bool Run(uint64_t begin, uint64_t end, const InitClosure& init_closure,
const RunClosure& run_closure) {
if (!init_closure(NumThreads())) return false;
Run(begin, end, run_closure);
return true;
}
// Only for use in tests.
PoolMem& InternalMem() const { return *owner_.Mem(); }
private:
// Unmodified after ctor, but cannot be const because we call thread::join().
std::vector<std::thread> threads_;
PoolMemOwner owner_;
// In debug builds, detects if functions are re-entered; always present so
// that the memory layout does not change.
std::atomic<int> busy_{0};
};
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_THREAD_POOL_THREAD_POOL_H_

400
deps/v8/third_party/highway/src/hwy/contrib/thread_pool/thread_pool_test.cc vendored Normal file
View File

@ -0,0 +1,400 @@
// Copyright 2023 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Modified from BSD-licensed code
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
// See https://github.com/libjxl/libjxl/blob/main/LICENSE.
#include "hwy/contrib/thread_pool/thread_pool.h"
#include <math.h> // sqrtf
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <atomic>
#include <vector>
#include "hwy/base.h" // PopCount
#include "hwy/contrib/thread_pool/topology.h"
#include "hwy/tests/hwy_gtest.h"
#include "hwy/tests/test_util-inl.h" // AdjustedReps
namespace hwy {
namespace {
using HWY_NAMESPACE::AdjustedReps;
TEST(ThreadPoolTest, TestCoprime) {
// 1 is coprime with anything
for (uint32_t i = 1; i < 500; ++i) {
HWY_ASSERT(ShuffledIota::CoprimeNonzero(1, i));
HWY_ASSERT(ShuffledIota::CoprimeNonzero(i, 1));
}
// Powers of two >= 2 are not coprime
for (size_t i = 1; i < 20; ++i) {
for (size_t j = 1; j < 20; ++j) {
HWY_ASSERT(!ShuffledIota::CoprimeNonzero(1u << i, 1u << j));
}
}
// 2^x and 2^x +/- 1 are coprime
for (size_t i = 1; i < 30; ++i) {
const uint32_t pow2 = 1u << i;
HWY_ASSERT(ShuffledIota::CoprimeNonzero(pow2, pow2 + 1));
HWY_ASSERT(ShuffledIota::CoprimeNonzero(pow2, pow2 - 1));
HWY_ASSERT(ShuffledIota::CoprimeNonzero(pow2 + 1, pow2));
HWY_ASSERT(ShuffledIota::CoprimeNonzero(pow2 - 1, pow2));
}
// Random number x * random y (both >= 2) is not co-prime with x nor y.
RandomState rng;
for (size_t i = 1; i < 5000; ++i) {
const uint32_t x = (Random32(&rng) & 0xFFF7) + 2;
const uint32_t y = (Random32(&rng) & 0xFFF7) + 2;
HWY_ASSERT(!ShuffledIota::CoprimeNonzero(x * y, x));
HWY_ASSERT(!ShuffledIota::CoprimeNonzero(x * y, y));
HWY_ASSERT(!ShuffledIota::CoprimeNonzero(x, x * y));
HWY_ASSERT(!ShuffledIota::CoprimeNonzero(y, x * y));
}
// Primes are all coprime (list from https://oeis.org/A000040)
static constexpr uint32_t primes[] = {
2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47,
53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113,
127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181, 191, 193, 197,
199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271};
for (size_t i = 0; i < sizeof(primes) / sizeof(primes[0]); ++i) {
for (size_t j = i + 1; j < sizeof(primes) / sizeof(primes[0]); ++j) {
HWY_ASSERT(ShuffledIota::CoprimeNonzero(primes[i], primes[j]));
HWY_ASSERT(ShuffledIota::CoprimeNonzero(primes[j], primes[i]));
}
}
}
// Ensures `shuffled` visits [0, size) exactly once starting from `current`.
void VerifyPermutation(uint32_t size, const Divisor& divisor,
const ShuffledIota& shuffled, uint32_t current,
uint32_t* visited) {
for (size_t i = 0; i < size; i++) {
visited[i] = 0;
}
for (size_t i = 0; i < size; i++) {
++visited[current];
current = shuffled.Next(current, divisor);
}
for (size_t i = 0; i < size; i++) {
HWY_ASSERT(visited[i] == 1);
}
}
// Verifies ShuffledIota generates a permutation of [0, size).
TEST(ThreadPoolTest, TestRandomPermutation) {
constexpr size_t kMaxSize = 40;
uint32_t visited[kMaxSize];
// Exhaustive enumeration of size and starting point.
for (uint32_t size = 1; size < kMaxSize; ++size) {
const Divisor divisor(size);
const uint32_t coprime = ShuffledIota::FindAnotherCoprime(size, 1);
const ShuffledIota shuffled(coprime);
for (uint32_t start = 0; start < size; ++start) {
VerifyPermutation(size, divisor, shuffled, start, visited);
}
}
}
// Verifies multiple ShuffledIota are relatively independent.
TEST(ThreadPoolTest, TestMultiplePermutations) {
constexpr size_t kMaxSize = 40;
uint32_t coprimes[kMaxSize];
// One per ShuffledIota; initially the starting value, then its Next().
uint32_t current[kMaxSize];
for (uint32_t size = 1; size < kMaxSize; ++size) {
const Divisor divisor(size);
// Create `size` ShuffledIota instances with unique coprimes.
std::vector<ShuffledIota> shuffled;
for (size_t i = 0; i < size; ++i) {
coprimes[i] = ShuffledIota::FindAnotherCoprime(
size, static_cast<uint32_t>((i + 1) * 257 + i * 13));
shuffled.emplace_back(coprimes[i]);
}
// ShuffledIota[i] starts at i to match the worker thread use case.
for (uint32_t i = 0; i < size; ++i) {
current[i] = i;
}
size_t num_bad = 0;
uint32_t all_visited[kMaxSize] = {0};
// For each step, ensure there are few non-unique current[].
for (size_t step = 0; step < size; ++step) {
// How many times is each number visited?
uint32_t visited[kMaxSize] = {0};
for (size_t i = 0; i < size; ++i) {
visited[current[i]] += 1;
all_visited[current[i]] = 1; // visited at all across all steps?
}
// How many numbers are visited multiple times?
size_t num_contended = 0;
uint32_t max_contention = 0;
for (size_t i = 0; i < size; ++i) {
num_contended += visited[i] > 1;
max_contention = HWY_MAX(max_contention, visited[i]);
}
// Count/print if excessive collisions.
const size_t expected =
static_cast<size_t>(sqrtf(static_cast<float>(size)) * 2.0f);
if ((num_contended > expected) && (max_contention > 3)) {
++num_bad;
if (true) {
fprintf(stderr, "size %u step %zu contended %zu max contention %u\n",
size, step, num_contended, max_contention);
for (size_t i = 0; i < size; ++i) {
fprintf(stderr, " %u\n", current[i]);
}
fprintf(stderr, "coprimes\n");
for (size_t i = 0; i < size; ++i) {
fprintf(stderr, " %u\n", coprimes[i]);
}
}
}
// Advance all ShuffledIota generators.
for (size_t i = 0; i < size; ++i) {
current[i] = shuffled[i].Next(current[i], divisor);
}
} // step
// Ensure each task was visited during at least one step.
for (size_t i = 0; i < size; ++i) {
HWY_ASSERT(all_visited[i] != 0);
}
if (num_bad != 0) {
fprintf(stderr, "size %u total bad: %zu\n", size, num_bad);
}
HWY_ASSERT(num_bad < kMaxSize / 10);
} // size
}
// Ensures all tasks are run. Similar to TestPool below but without threads.
TEST(ThreadPoolTest, TestTasks) {
for (size_t num_threads = 0; num_threads <= 8; ++num_threads) {
PoolMemOwner owner(num_threads);
PoolMem& mem = *owner.Mem();
const size_t num_workers = owner.NumWorkers();
constexpr uint64_t kMaxTasks = 20;
uint64_t mementos[kMaxTasks];
for (uint64_t num_tasks = 0; num_tasks < 20; ++num_tasks) {
for (uint64_t begin = 0; begin < AdjustedReps(32); ++begin) {
const uint64_t end = begin + num_tasks;
ZeroBytes(mementos, sizeof(mementos));
const auto func = [begin, end, &mementos](uint64_t task,
size_t /*thread*/) {
HWY_ASSERT(begin <= task && task < end);
// Store mementos ensure we visited each task.
mementos[task - begin] = 1000 + task;
};
if (ParallelFor::Plan(begin, end, num_workers, func, mem)) {
// The `tasks < workers` special case requires running by all workers.
for (size_t thread = 0; thread < num_workers; ++thread) {
ParallelFor::WorkerRun(thread, num_workers, mem);
}
}
// Ensure all tasks were run.
for (uint64_t task = begin; task < end; ++task) {
HWY_ASSERT_EQ(1000 + task, mementos[task - begin]);
}
}
}
}
}
// Ensures old code with 32-bit tasks and InitClosure still compiles.
TEST(ThreadPoolTest, TestDeprecated) {
ThreadPool pool(0);
pool.Run(1, 10, &ThreadPool::NoInit,
[&](const uint64_t /*task*/, size_t /*thread*/) {});
}
// Ensures task parameter is in bounds, every parameter is reached,
// pool can be reused (multiple consecutive Run calls), pool can be destroyed
// (joining with its threads), num_threads=0 works (runs on current thread).
TEST(ThreadPoolTest, TestPool) {
if (!HaveThreadingSupport()) return;
ThreadPool inner(0);
for (size_t num_threads = 0; num_threads <= 6; num_threads += 3) {
ThreadPool pool(HWY_MIN(ThreadPool::MaxThreads(), num_threads));
constexpr uint64_t kMaxTasks = 20;
std::atomic<uint64_t> mementos[kMaxTasks];
for (uint64_t num_tasks = 0; num_tasks < kMaxTasks; ++num_tasks) {
for (uint64_t begin = 0; begin < AdjustedReps(32); ++begin) {
const uint64_t end = begin + num_tasks;
std::atomic<uint64_t> a_begin;
std::atomic<uint64_t> a_end;
a_begin.store(begin, std::memory_order_release);
a_end.store(end, std::memory_order_release);
for (size_t i = 0; i < kMaxTasks; ++i) {
mementos[i].store(0);
}
pool.Run(begin, end,
[&a_begin, &a_end, &mementos, &inner](uint64_t task,
size_t /*thread*/) {
const uint64_t begin =
a_begin.load(std::memory_order_acquire);
const uint64_t end = a_end.load(std::memory_order_acquire);
HWY_ASSERT(begin <= task && task < end);
// Store mementos ensure we visited each task.
mementos[task - begin].store(1000 + task);
// Re-entering Run is fine on a 0-worker pool.
inner.Run(begin, end,
[begin, end](uint64_t task, size_t /*thread*/) {
HWY_ASSERT(begin <= task && task < end);
});
});
for (uint64_t task = begin; task < end; ++task) {
HWY_ASSERT_EQ(1000 + task, mementos[task - begin].load());
}
}
}
}
}
// Debug tsan builds seem to generate incorrect codegen for [&] of atomics, so
// use a pointer to a state object instead.
struct SmallAssignmentState {
// (Avoid mutex because it may perturb the worker thread scheduling)
std::atomic<uint64_t> num_tasks{0};
std::atomic<uint64_t> num_workers{0};
std::atomic<uint64_t> id_bits{0};
std::atomic<uint64_t> num_calls{0};
};
// Verify "thread" parameter when processing few tasks.
TEST(ThreadPoolTest, TestSmallAssignments) {
if (!HaveThreadingSupport()) return;
static SmallAssignmentState state;
for (size_t num_threads :
{size_t{0}, size_t{1}, size_t{3}, size_t{5}, size_t{8}}) {
ThreadPool pool(HWY_MIN(ThreadPool::MaxThreads(), num_threads));
state.num_workers.store(pool.NumWorkers());
for (size_t mul = 1; mul <= 2; ++mul) {
const size_t num_tasks = pool.NumWorkers() * mul;
state.num_tasks.store(num_tasks);
state.id_bits.store(0);
state.num_calls.store(0);
pool.Run(0, num_tasks, [](uint64_t task, size_t thread) {
HWY_ASSERT(task < state.num_tasks.load());
HWY_ASSERT(thread < state.num_workers.load());
state.num_calls.fetch_add(1);
uint64_t bits = state.id_bits.load();
while (!state.id_bits.compare_exchange_weak(bits,
bits | (1ULL << thread))) {
}
});
// Correct number of tasks.
const uint64_t actual_calls = state.num_calls.load();
HWY_ASSERT(num_tasks == actual_calls);
const size_t num_participants = PopCount(state.id_bits.load());
// <= because some workers may not manage to run any tasks.
HWY_ASSERT(num_participants <= pool.NumWorkers());
}
}
}
struct Counter {
Counter() {
// Suppress "unused-field" warning.
(void)padding;
}
void Assimilate(const Counter& victim) { counter += victim.counter; }
std::atomic<uint64_t> counter{0};
uint64_t padding[15];
};
// Can switch between any wait mode, and multiple times.
TEST(ThreadPoolTest, TestWaitMode) {
if (!HaveThreadingSupport()) return;
const size_t kNumThreads = 9;
ThreadPool pool(kNumThreads);
RandomState rng;
for (size_t i = 0; i < 10; ++i) {
pool.SetWaitMode(Random32(&rng) ? PoolWaitMode::kSpin
: PoolWaitMode::kBlock);
}
}
TEST(ThreadPoolTest, TestCounter) {
if (!HaveThreadingSupport()) return;
const size_t kNumThreads = 12;
ThreadPool pool(kNumThreads);
for (PoolWaitMode mode : {PoolWaitMode::kSpin, PoolWaitMode::kBlock}) {
pool.SetWaitMode(mode);
alignas(128) Counter counters[1+kNumThreads];
const uint64_t kNumTasks = kNumThreads * 19;
pool.Run(0, kNumTasks,
[&counters](const uint64_t task, const size_t thread) {
counters[thread].counter.fetch_add(task);
});
uint64_t expected = 0;
for (uint64_t i = 0; i < kNumTasks; ++i) {
expected += i;
}
for (size_t i = 1; i < pool.NumWorkers(); ++i) {
counters[0].Assimilate(counters[i]);
}
HWY_ASSERT_EQ(expected, counters[0].counter.load());
}
}
} // namespace
} // namespace hwy
HWY_TEST_MAIN();

530
deps/v8/third_party/highway/src/hwy/contrib/thread_pool/topology.cc vendored Normal file
View File

@ -0,0 +1,530 @@
// Copyright 2024 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/thread_pool/topology.h"
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h> // strchr
#include <map>
#include <vector>
#include "hwy/detect_compiler_arch.h" // HWY_OS_WIN
#if HWY_OS_WIN
#ifndef NOMINMAX
#define NOMINMAX
#endif
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
#endif // HWY_OS_WIN
#if HWY_OS_LINUX || HWY_OS_FREEBSD
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <sched.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h> // sysconf
#endif // HWY_OS_LINUX || HWY_OS_FREEBSD
#if HWY_OS_FREEBSD
// must come after sys/types.h.
#include <sys/cpuset.h> // CPU_SET
#endif // HWY_OS_FREEBSD
#if HWY_ARCH_WASM
#include <emscripten/threading.h>
#endif
#include "hwy/base.h"
namespace hwy {
HWY_CONTRIB_DLLEXPORT bool HaveThreadingSupport() {
#if HWY_ARCH_WASM
return emscripten_has_threading_support() != 0;
#else
return true;
#endif
}
HWY_CONTRIB_DLLEXPORT size_t TotalLogicalProcessors() {
size_t lp = 0;
#if HWY_ARCH_WASM
const int num_cores = emscripten_num_logical_cores();
if (num_cores > 0) lp = static_cast<size_t>(num_cores);
#elif HWY_OS_WIN
SYSTEM_INFO sysinfo;
GetSystemInfo(&sysinfo); // always succeeds
// WARNING: this is only for the current group, hence limited to 64.
lp = static_cast<size_t>(sysinfo.dwNumberOfProcessors);
#elif HWY_OS_LINUX
// Use configured, not "online" (_SC_NPROCESSORS_ONLN), because we want an
// upper bound.
const long ret = sysconf(_SC_NPROCESSORS_CONF); // NOLINT(runtime/int)
if (ret < 0) {
fprintf(stderr, "Unexpected value of _SC_NPROCESSORS_CONF: %d\n",
static_cast<int>(ret));
} else {
lp = static_cast<size_t>(ret);
}
#endif
if (HWY_UNLIKELY(lp == 0)) { // Failed to detect.
HWY_IF_CONSTEXPR(HWY_IS_DEBUG_BUILD) {
fprintf(stderr,
"Unknown TotalLogicalProcessors, assuming 1. "
"HWY_OS_: WIN=%d LINUX=%d APPLE=%d;\n"
"HWY_ARCH_: WASM=%d X86=%d PPC=%d ARM=%d RISCV=%d S390X=%d\n",
HWY_OS_WIN, HWY_OS_LINUX, HWY_OS_APPLE, HWY_ARCH_WASM,
HWY_ARCH_X86, HWY_ARCH_PPC, HWY_ARCH_ARM, HWY_ARCH_RISCV,
HWY_ARCH_S390X);
}
return 1;
}
// Warn that we are clamping.
if (HWY_UNLIKELY(lp > kMaxLogicalProcessors)) {
HWY_IF_CONSTEXPR(HWY_IS_DEBUG_BUILD) {
fprintf(stderr, "OS reports %zu processors but clamping to %zu\n", lp,
kMaxLogicalProcessors);
}
lp = kMaxLogicalProcessors;
}
return lp;
}
#ifdef __ANDROID__
#include <sys/syscall.h>
#endif
HWY_CONTRIB_DLLEXPORT bool GetThreadAffinity(LogicalProcessorSet& lps) {
#if HWY_OS_WIN
// Only support the first 64 because WINE does not support processor groups.
const HANDLE hThread = GetCurrentThread();
const DWORD_PTR prev = SetThreadAffinityMask(hThread, ~DWORD_PTR(0));
if (!prev) return false;
(void)SetThreadAffinityMask(hThread, prev);
lps = LogicalProcessorSet(); // clear all
lps.SetNonzeroBitsFrom64(prev);
return true;
#elif HWY_OS_LINUX
cpu_set_t set;
CPU_ZERO(&set);
const pid_t pid = 0; // current thread
#ifdef __ANDROID__
const int err = syscall(__NR_sched_getaffinity, pid, sizeof(cpu_set_t), &set);
#else
const int err = sched_getaffinity(pid, sizeof(cpu_set_t), &set);
#endif // __ANDROID__
if (err != 0) return false;
for (size_t lp = 0; lp < kMaxLogicalProcessors; ++lp) {
#if HWY_COMPILER_GCC_ACTUAL
// Workaround for GCC compiler warning with CPU_ISSET macro
HWY_DIAGNOSTICS(push)
HWY_DIAGNOSTICS_OFF(disable : 4305 4309, ignored "-Wsign-conversion")
#endif
if (CPU_ISSET(static_cast<int>(lp), &set)) {
lps.Set(lp);
}
#if HWY_COMPILER_GCC_ACTUAL
HWY_DIAGNOSTICS(pop)
#endif
}
return true;
#elif HWY_OS_FREEBSD
cpuset_t set;
CPU_ZERO(&set);
const pid_t pid = getpid(); // current thread
const int err = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid,
sizeof(cpuset_t), &set);
if (err != 0) return false;
for (size_t lp = 0; lp < kMaxLogicalProcessors; ++lp) {
#if HWY_COMPILER_GCC_ACTUAL
// Workaround for GCC compiler warning with CPU_ISSET macro
HWY_DIAGNOSTICS(push)
HWY_DIAGNOSTICS_OFF(disable : 4305 4309, ignored "-Wsign-conversion")
#endif
if (CPU_ISSET(static_cast<int>(lp), &set)) {
lps.Set(lp);
}
#if HWY_COMPILER_GCC_ACTUAL
HWY_DIAGNOSTICS(pop)
#endif
}
return true;
#else
// Do not even set lp=0 to force callers to handle this case.
(void)lps;
return false;
#endif
}
HWY_CONTRIB_DLLEXPORT bool SetThreadAffinity(const LogicalProcessorSet& lps) {
#if HWY_OS_WIN
const HANDLE hThread = GetCurrentThread();
const DWORD_PTR prev = SetThreadAffinityMask(hThread, lps.Get64());
return prev != 0;
#elif HWY_OS_LINUX
cpu_set_t set;
CPU_ZERO(&set);
#if HWY_COMPILER_GCC_ACTUAL
// Workaround for GCC compiler warning with CPU_SET macro
HWY_DIAGNOSTICS(push)
HWY_DIAGNOSTICS_OFF(disable : 4305 4309, ignored "-Wsign-conversion")
#endif
lps.Foreach([&set](size_t lp) { CPU_SET(static_cast<int>(lp), &set); });
#if HWY_COMPILER_GCC_ACTUAL
HWY_DIAGNOSTICS(pop)
#endif
const pid_t pid = 0; // current thread
#ifdef __ANDROID__
const int err = syscall(__NR_sched_setaffinity, pid, sizeof(cpu_set_t), &set);
#else
const int err = sched_setaffinity(pid, sizeof(cpu_set_t), &set);
#endif // __ANDROID__
if (err != 0) return false;
return true;
#elif HWY_OS_FREEBSD
cpuset_t set;
CPU_ZERO(&set);
#if HWY_COMPILER_GCC_ACTUAL
// Workaround for GCC compiler warning with CPU_SET macro
HWY_DIAGNOSTICS(push)
HWY_DIAGNOSTICS_OFF(disable : 4305 4309, ignored "-Wsign-conversion")
#endif
lps.Foreach([&set](size_t lp) { CPU_SET(static_cast<int>(lp), &set); });
#if HWY_COMPILER_GCC_ACTUAL
HWY_DIAGNOSTICS(pop)
#endif
const pid_t pid = getpid(); // current thread
const int err = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid,
sizeof(cpuset_t), &set);
if (err != 0) return false;
return true;
#else
// Apple THREAD_AFFINITY_POLICY is only an (often ignored) hint.
(void)lps;
return false;
#endif
}
#if HWY_OS_LINUX
namespace {
class File {
public:
explicit File(const char* path) {
for (;;) {
fd_ = open(path, O_RDONLY);
if (fd_ > 0) return; // success
if (errno == EINTR) continue; // signal: retry
if (errno == ENOENT) return; // not found, give up
if (HWY_IS_DEBUG_BUILD) {
fprintf(stderr, "Unexpected error opening %s: %d\n", path, errno);
}
return; // unknown error, give up
}
}
~File() {
if (fd_ > 0) {
for (;;) {
const int ret = close(fd_);
if (ret == 0) break; // success
if (errno == EINTR) continue; // signal: retry
if (HWY_IS_DEBUG_BUILD) {
fprintf(stderr, "Unexpected error closing file: %d\n", errno);
}
return; // unknown error, ignore
}
}
}
// Returns number of bytes read or 0 on failure.
size_t Read(char* buf200) const {
if (fd_ < 0) return 0;
size_t pos = 0;
for (;;) {
// read instead of `pread`, which might not work for sysfs.
const auto bytes_read = read(fd_, buf200 + pos, 200 - pos);
if (bytes_read == 0) { // EOF: done
buf200[pos++] = '\0';
return pos;
}
if (bytes_read == -1) {
if (errno == EINTR) continue; // signal: retry
if (HWY_IS_DEBUG_BUILD) {
fprintf(stderr, "Unexpected error reading file: %d\n", errno);
}
return 0;
}
pos += static_cast<size_t>(bytes_read);
HWY_ASSERT(pos <= 200);
}
}
private:
int fd_;
};
// Returns bytes read, or 0 on failure.
size_t ReadSysfs(const char* format, size_t lp, char* buf200) {
char path[200];
const int bytes_written = snprintf(path, sizeof(path), format, lp);
HWY_ASSERT(0 < bytes_written &&
bytes_written < static_cast<int>(sizeof(path) - 1));
const File file(path);
return file.Read(buf200);
}
// Interprets [str + pos, str + end) as base-10 ASCII. Stops when any non-digit
// is found, or at end. Returns false if no digits found.
bool ParseDigits(const char* str, const size_t end, size_t& pos, size_t* out) {
HWY_ASSERT(pos <= end);
// 9 digits cannot overflow even 32-bit size_t.
const size_t stop = pos + 9;
*out = 0;
for (; pos < HWY_MIN(end, stop); ++pos) {
const int c = str[pos];
if (c < '0' || c > '9') break;
*out *= 10;
*out += static_cast<size_t>(c - '0');
}
if (pos == 0) { // No digits found
*out = 0;
return false;
}
return true;
}
// Number, plus optional K or M suffix, plus terminator.
bool ParseNumberWithOptionalSuffix(const char* str, size_t len, size_t* out) {
size_t pos = 0;
if (!ParseDigits(str, len, pos, out)) return false;
if (str[pos] == 'K') {
*out <<= 10;
++pos;
}
if (str[pos] == 'M') {
*out <<= 20;
++pos;
}
if (str[pos] != '\0' && str[pos] != '\n') {
HWY_ABORT("Expected [suffix] terminator at %zu %s\n", pos, str);
}
return true;
}
bool ReadNumberWithOptionalSuffix(const char* format, size_t lp, size_t* out) {
char buf200[200];
const size_t pos = ReadSysfs(format, lp, buf200);
if (pos == 0) return false;
return ParseNumberWithOptionalSuffix(buf200, pos, out);
}
const char* kPackage =
"/sys/devices/system/cpu/cpu%zu/topology/physical_package_id";
const char* kCluster = "/sys/devices/system/cpu/cpu%zu/cache/index3/id";
const char* kCore = "/sys/devices/system/cpu/cpu%zu/topology/core_id";
const char* kL2Size = "/sys/devices/system/cpu/cpu%zu/cache/index2/size";
const char* kL3Size = "/sys/devices/system/cpu/cpu%zu/cache/index3/size";
const char* kNode = "/sys/devices/system/node/node%zu/cpulist";
// sysfs values can be arbitrarily large, so store in a map and replace with
// indices in order of appearance.
class Remapper {
public:
// Returns false on error, or sets `out_index` to the index of the sysfs
// value selected by `format` and `lp`.
template <typename T>
bool operator()(const char* format, size_t lp, T* HWY_RESTRICT out_index) {
size_t opaque;
if (!ReadNumberWithOptionalSuffix(format, lp, &opaque)) return false;
const auto ib = indices_.insert({opaque, num_});
num_ += ib.second; // increment if inserted
const size_t index = ib.first->second; // new or existing
HWY_ASSERT(index < num_);
HWY_ASSERT(index < hwy::LimitsMax<T>());
*out_index = static_cast<T>(index);
return true;
}
size_t Num() const { return num_; }
private:
std::map<size_t, size_t> indices_;
size_t num_ = 0;
};
// Stores the global cluster/core values separately for each package so we can
// return per-package arrays.
struct PerPackage {
Remapper clusters;
Remapper cores;
uint8_t smt_per_core[kMaxLogicalProcessors] = {0};
};
// Initializes `lps` and returns a PerPackage vector (empty on failure).
std::vector<PerPackage> DetectPackages(std::vector<Topology::LP>& lps) {
std::vector<PerPackage> empty;
Remapper packages;
for (size_t lp = 0; lp < lps.size(); ++lp) {
if (!packages(kPackage, lp, &lps[lp].package)) return empty;
}
std::vector<PerPackage> per_package(packages.Num());
for (size_t lp = 0; lp < lps.size(); ++lp) {
PerPackage& pp = per_package[lps[lp].package];
if (!pp.clusters(kCluster, lp, &lps[lp].cluster)) return empty;
if (!pp.cores(kCore, lp, &lps[lp].core)) return empty;
// SMT ID is how many LP we have already seen assigned to the same core.
HWY_ASSERT(lps[lp].core < kMaxLogicalProcessors);
lps[lp].smt = pp.smt_per_core[lps[lp].core]++;
HWY_ASSERT(lps[lp].smt < 16);
}
return per_package;
}
// Sets LP.node for all `lps`.
void SetNodes(std::vector<Topology::LP>& lps) {
// For each NUMA node found via sysfs:
for (size_t node = 0;; node++) {
// Read its cpulist so we can scatter `node` to all its `lps`.
char buf200[200];
const size_t bytes_read = ReadSysfs(kNode, node, buf200);
if (bytes_read == 0) break;
constexpr size_t kNotFound = ~size_t{0};
size_t pos = 0;
// Returns first `found_pos >= pos` where `buf200[found_pos] == c`, or
// `kNotFound`.
const auto find = [buf200, &pos](char c) -> size_t {
const char* found_ptr = strchr(buf200 + pos, c);
if (found_ptr == nullptr) return kNotFound;
HWY_ASSERT(found_ptr >= buf200);
const size_t found_pos = static_cast<size_t>(found_ptr - buf200);
HWY_ASSERT(found_pos >= pos && buf200[found_pos] == c);
return found_pos;
};
// Reads LP number and advances `pos`. `end` is for verifying we did not
// read past a known terminator, or the end of string.
const auto parse_lp = [buf200, bytes_read, &pos,
&lps](size_t end) -> size_t {
end = HWY_MIN(end, bytes_read);
size_t lp;
HWY_ASSERT(ParseDigits(buf200, end, pos, &lp));
HWY_IF_CONSTEXPR(HWY_ARCH_RISCV) {
// On RISC-V, both TotalLogicalProcessors and GetThreadAffinity may
// under-report the count, hence clamp.
lp = HWY_MIN(lp, lps.size() - 1);
}
HWY_ASSERT(lp < lps.size());
HWY_ASSERT(pos <= end);
return lp;
};
// Parse all [first-]last separated by commas.
for (;;) {
// Single number or first of range: ends with dash, comma, or end.
const size_t lp_range_first = parse_lp(HWY_MIN(find('-'), find(',')));
if (buf200[pos] == '-') { // range
++pos; // skip dash
// Last of range ends with comma or end.
const size_t lp_range_last = parse_lp(find(','));
for (size_t lp = lp_range_first; lp <= lp_range_last; ++lp) {
lps[lp].node = static_cast<uint8_t>(node);
}
} else { // single number
lps[lp_range_first].node = static_cast<uint8_t>(node);
}
// Done if reached end of string.
if (pos == bytes_read || buf200[pos] == '\0' || buf200[pos] == '\n') {
break;
}
// Comma means at least one more term is coming.
if (buf200[pos] == ',') {
++pos;
continue;
}
HWY_ABORT("Unexpected character at %zu in %s\n", pos, buf200);
} // for pos
} // for node
}
} // namespace
#endif // HWY_OS_LINUX
HWY_CONTRIB_DLLEXPORT Topology::Topology() {
#if HWY_OS_LINUX
lps.resize(TotalLogicalProcessors());
const std::vector<PerPackage>& per_package = DetectPackages(lps);
if (per_package.empty()) return;
SetNodes(lps);
// Allocate per-package/cluster/core vectors. This indicates to callers that
// detection succeeded.
packages.resize(per_package.size());
for (size_t p = 0; p < packages.size(); ++p) {
packages[p].clusters.resize(per_package[p].clusters.Num());
packages[p].cores.resize(per_package[p].cores.Num());
}
// Populate the per-cluster/core sets of LP.
for (size_t lp = 0; lp < lps.size(); ++lp) {
Package& p = packages[lps[lp].package];
p.clusters[lps[lp].cluster].lps.Set(lp);
p.cores[lps[lp].core].lps.Set(lp);
}
// Detect cache sizes (only once per cluster)
for (size_t ip = 0; ip < packages.size(); ++ip) {
Package& p = packages[ip];
for (size_t ic = 0; ic < p.clusters.size(); ++ic) {
Cluster& c = p.clusters[ic];
const size_t lp = c.lps.First();
size_t bytes;
if (ReadNumberWithOptionalSuffix(kL2Size, lp, &bytes)) {
c.private_kib = bytes >> 10;
}
if (ReadNumberWithOptionalSuffix(kL3Size, lp, &bytes)) {
c.shared_kib = bytes >> 10;
}
}
}
#endif
}
} // namespace hwy

108
deps/v8/third_party/highway/src/hwy/contrib/thread_pool/topology.h vendored Normal file
View File

@ -0,0 +1,108 @@
// Copyright 2024 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_CONTRIB_THREAD_POOL_TOPOLOGY_H_
#define HIGHWAY_HWY_CONTRIB_THREAD_POOL_TOPOLOGY_H_
// OS-specific functions for processor topology and thread affinity.
#include <stddef.h>
#include <vector>
#include "hwy/base.h"
#include "hwy/bit_set.h"
namespace hwy {
// Returns false if std::thread should not be used.
HWY_CONTRIB_DLLEXPORT bool HaveThreadingSupport();
// Upper bound on logical processors, including hyperthreads.
static constexpr size_t kMaxLogicalProcessors = 1024; // matches glibc
// Set used by Get/SetThreadAffinity.
using LogicalProcessorSet = BitSet4096<kMaxLogicalProcessors>;
// Returns false, or sets `lps` to all logical processors which are online and
// available to the current thread.
HWY_CONTRIB_DLLEXPORT bool GetThreadAffinity(LogicalProcessorSet& lps);
// Ensures the current thread can only run on the logical processors in `lps`.
// Returns false if not supported (in particular on Apple), or if the
// intersection between `lps` and `GetThreadAffinity` is the empty set.
HWY_CONTRIB_DLLEXPORT bool SetThreadAffinity(const LogicalProcessorSet& lps);
// Returns false, or ensures the current thread will only run on `lp`, which
// must not exceed `TotalLogicalProcessors`. Note that this merely calls
// `SetThreadAffinity`, see the comment there.
static inline bool PinThreadToLogicalProcessor(size_t lp) {
LogicalProcessorSet lps;
lps.Set(lp);
return SetThreadAffinity(lps);
}
// Returns 1 if unknown, otherwise the total number of logical processors
// provided by the hardware clamped to `kMaxLogicalProcessors`.
// These processors are not necessarily all usable; you can determine which are
// via GetThreadAffinity().
HWY_CONTRIB_DLLEXPORT size_t TotalLogicalProcessors();
struct Topology {
// Caller must check packages.empty(); if so, do not use any fields.
HWY_CONTRIB_DLLEXPORT Topology();
// Clique of cores with lower latency to each other. On Apple M1 these are
// four cores sharing an L2. On Zen4 these 'CCX' are up to eight cores sharing
// an L3 and a memory controller, or for Zen4c up to 16 and half the L3 size.
struct Cluster {
LogicalProcessorSet lps;
uint64_t private_kib = 0; // 0 if unknown
uint64_t shared_kib = 0; // 0 if unknown
uint64_t reserved1 = 0;
uint64_t reserved2 = 0;
uint64_t reserved3 = 0;
};
struct Core {
LogicalProcessorSet lps;
uint64_t reserved = 0;
};
struct Package {
std::vector<Cluster> clusters;
std::vector<Core> cores;
};
std::vector<Package> packages;
// Several hundred instances, so prefer a compact representation.
#pragma pack(push, 1)
struct LP {
uint16_t cluster = 0; // < packages[package].clusters.size()
uint16_t core = 0; // < packages[package].cores.size()
uint8_t package = 0; // < packages.size()
uint8_t smt = 0; // < packages[package].cores[core].lps.Count()
uint8_t node = 0;
uint8_t reserved = 0;
};
#pragma pack(pop)
std::vector<LP> lps; // size() == TotalLogicalProcessors().
};
} // namespace hwy
#endif // HIGHWAY_HWY_CONTRIB_THREAD_POOL_TOPOLOGY_H_

89
deps/v8/third_party/highway/src/hwy/contrib/thread_pool/topology_test.cc vendored Normal file
View File

@ -0,0 +1,89 @@
// Copyright 2024 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/contrib/thread_pool/topology.h"
#include <stddef.h>
#include <stdio.h>
#include <vector>
#include "hwy/base.h"
#include "hwy/tests/hwy_gtest.h"
#include "hwy/tests/test_util-inl.h"
#include "hwy/timer.h"
namespace hwy {
namespace {
TEST(TopologyTest, TestNum) {
const size_t total = TotalLogicalProcessors();
fprintf(stderr, "TotalLogical %zu\n", total);
LogicalProcessorSet lps;
if (GetThreadAffinity(lps)) {
fprintf(stderr, "Active %zu\n", lps.Count());
HWY_ASSERT(lps.Count() <= total);
}
}
TEST(TopologyTest, TestTopology) {
char cpu100[100];
if (hwy::platform::GetCpuString(cpu100)) {
fprintf(stderr, "%s\n", cpu100);
}
Topology topology;
if (topology.packages.empty()) return;
HWY_ASSERT(!topology.lps.empty());
LogicalProcessorSet nodes;
for (size_t lp = 0; lp < topology.lps.size(); ++lp) {
const size_t node = static_cast<size_t>(topology.lps[lp].node);
if (!nodes.Get(node)) {
fprintf(stderr, "Found NUMA node %zu, LP %zu\n", node, lp);
nodes.Set(node);
}
}
size_t lps_by_cluster = 0;
size_t lps_by_core = 0;
LogicalProcessorSet all_lps;
for (size_t p = 0; p < topology.packages.size(); ++p) {
const Topology::Package& pkg = topology.packages[p];
HWY_ASSERT(!pkg.clusters.empty());
HWY_ASSERT(!pkg.cores.empty());
HWY_ASSERT(pkg.clusters.size() <= pkg.cores.size());
for (const Topology::Cluster& c : pkg.clusters) {
lps_by_cluster += c.lps.Count();
c.lps.Foreach([&all_lps](size_t lp) { all_lps.Set(lp); });
}
for (const Topology::Core& c : pkg.cores) {
lps_by_core += c.lps.Count();
c.lps.Foreach([&all_lps](size_t lp) { all_lps.Set(lp); });
}
}
// Ensure the per-cluster and per-core sets sum to the total.
HWY_ASSERT(lps_by_cluster == topology.lps.size());
HWY_ASSERT(lps_by_core == topology.lps.size());
// .. and are a partition of unity (all LPs are covered)
HWY_ASSERT(all_lps.Count() == topology.lps.size());
}
} // namespace
} // namespace hwy
HWY_TEST_MAIN();

31
deps/v8/third_party/highway/src/hwy/contrib/unroller/README.md vendored Normal file
View File

@ -0,0 +1,31 @@
# Unroller
All contents of the `unroller` folder are experimental and subject to changes.
`Unroller` is a templated function that automatically implements common optimizations that are usually handled by compilers when writing scalar code. Modern CPUs operate much more efficiently when non-dependent calculations are packed into an instruction pipeline. For scalar code, this often means a compiler will take a one-line loop, and compile it down to hundreds of lines of machine code in order to fully capture these efficiencies.
As of today (2023-07-06), compilers are not nearly as good at implementing these optimizations for code written in SIMD intrinsics. `Unroller` is a templated function that takes in an `UnrollerUnit` of SIMD instructions, and then implements unrolling, reordering, hoisting and tail-handling (URHT optimizations) of arrays of data being processed with SIMD intrinsics.
### `UnrollerUnit`
`UnrollerUnit` and `UnrollerUnit2D` are a base classes of functions that `Unroller` needs implemented in order to properly handle URHT. `UnrollerUnit` has default implementations for all but the `Func` method, which defines the SIMD operation to be applied. Many examples of how to implement these functions are in the tests.
### Doubling values of an array example
```
struct DoubleUnit : UnrollerUnit<DoubleUnit, int, int> {
using TT = ScalableTag<int>;
inline Vec<TT> Func(ptrdiff_t idx, Vec<TT> x, Vec<TT> y) {
TT d;
return Mul(x, Set(d, 2));
}
};
```
Leaving all other methods in their default state, the following code will double all the values in array `a` and place them in `r`
```
DoubleUnit dblunit;
int r[N];
Unroller(dblunit, a, r, N);
```

470
deps/v8/third_party/highway/src/hwy/contrib/unroller/unroller-inl.h vendored Normal file
View File

@ -0,0 +1,470 @@
// Copyright 2023 Matthew Kolbe
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#if defined(HIGHWAY_HWY_CONTRIB_UNROLLER_UNROLLER_INL_H_) == \
defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_CONTRIB_UNROLLER_UNROLLER_INL_H_
#undef HIGHWAY_HWY_CONTRIB_UNROLLER_UNROLLER_INL_H_
#else
#define HIGHWAY_HWY_CONTRIB_UNROLLER_UNROLLER_INL_H_
#endif
#include <cstdlib> // std::abs
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace hn = hwy::HWY_NAMESPACE;
template <class DERIVED, typename IN_T, typename OUT_T>
struct UnrollerUnit {
static constexpr size_t kMaxTSize = HWY_MAX(sizeof(IN_T), sizeof(OUT_T));
using LargerT = SignedFromSize<kMaxTSize>; // only the size matters.
DERIVED* me() { return static_cast<DERIVED*>(this); }
static constexpr size_t MaxUnitLanes() {
return HWY_MAX_LANES_D(hn::ScalableTag<LargerT>);
}
static size_t ActualLanes() { return Lanes(hn::ScalableTag<LargerT>()); }
using LargerD = hn::CappedTag<LargerT, MaxUnitLanes()>;
using IT = hn::Rebind<IN_T, LargerD>;
using OT = hn::Rebind<OUT_T, LargerD>;
IT d_in;
OT d_out;
using Y_VEC = hn::Vec<OT>;
using X_VEC = hn::Vec<IT>;
Y_VEC Func(const ptrdiff_t idx, const X_VEC x, const Y_VEC y) {
return me()->Func(idx, x, y);
}
X_VEC X0Init() { return me()->X0InitImpl(); }
X_VEC X0InitImpl() { return hn::Zero(d_in); }
Y_VEC YInit() { return me()->YInitImpl(); }
Y_VEC YInitImpl() { return hn::Zero(d_out); }
X_VEC Load(const ptrdiff_t idx, IN_T* from) {
return me()->LoadImpl(idx, from);
}
X_VEC LoadImpl(const ptrdiff_t idx, IN_T* from) {
return hn::LoadU(d_in, from + idx);
}
// MaskLoad can take in either a positive or negative number for `places`. if
// the number is positive, then it loads the top `places` values, and if it's
// negative, it loads the bottom |places| values. example: places = 3
// | o | o | o | x | x | x | x | x |
// example places = -3
// | x | x | x | x | x | o | o | o |
X_VEC MaskLoad(const ptrdiff_t idx, IN_T* from, const ptrdiff_t places) {
return me()->MaskLoadImpl(idx, from, places);
}
X_VEC MaskLoadImpl(const ptrdiff_t idx, IN_T* from, const ptrdiff_t places) {
auto mask = hn::FirstN(d_in, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d_in,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
return hn::MaskedLoad(mask, d_in, from + idx);
}
bool StoreAndShortCircuit(const ptrdiff_t idx, OUT_T* to, const Y_VEC x) {
return me()->StoreAndShortCircuitImpl(idx, to, x);
}
bool StoreAndShortCircuitImpl(const ptrdiff_t idx, OUT_T* to, const Y_VEC x) {
hn::StoreU(x, d_out, to + idx);
return true;
}
ptrdiff_t MaskStore(const ptrdiff_t idx, OUT_T* to, const Y_VEC x,
ptrdiff_t const places) {
return me()->MaskStoreImpl(idx, to, x, places);
}
ptrdiff_t MaskStoreImpl(const ptrdiff_t idx, OUT_T* to, const Y_VEC x,
const ptrdiff_t places) {
auto mask = hn::FirstN(d_out, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d_out,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
hn::BlendedStore(x, mask, d_out, to + idx);
return std::abs(places);
}
ptrdiff_t Reduce(const Y_VEC x, OUT_T* to) { return me()->ReduceImpl(x, to); }
ptrdiff_t ReduceImpl(const Y_VEC x, OUT_T* to) {
// default does nothing
(void)x;
(void)to;
return 0;
}
void Reduce(const Y_VEC x0, const Y_VEC x1, const Y_VEC x2, Y_VEC* y) {
me()->ReduceImpl(x0, x1, x2, y);
}
void ReduceImpl(const Y_VEC x0, const Y_VEC x1, const Y_VEC x2, Y_VEC* y) {
// default does nothing
(void)x0;
(void)x1;
(void)x2;
(void)y;
}
};
template <class DERIVED, typename IN0_T, typename IN1_T, typename OUT_T>
struct UnrollerUnit2D {
DERIVED* me() { return static_cast<DERIVED*>(this); }
static constexpr size_t kMaxTSize =
HWY_MAX(sizeof(IN0_T), HWY_MAX(sizeof(IN1_T), sizeof(OUT_T)));
using LargerT = SignedFromSize<kMaxTSize>; // only the size matters.
static constexpr size_t MaxUnitLanes() {
return HWY_MAX_LANES_D(hn::ScalableTag<LargerT>);
}
static size_t ActualLanes() { return Lanes(hn::ScalableTag<LargerT>()); }
using LargerD = hn::CappedTag<LargerT, MaxUnitLanes()>;
using I0T = hn::Rebind<IN0_T, LargerD>;
using I1T = hn::Rebind<IN1_T, LargerD>;
using OT = hn::Rebind<OUT_T, LargerD>;
I0T d_in0;
I1T d_in1;
OT d_out;
using Y_VEC = hn::Vec<OT>;
using X0_VEC = hn::Vec<I0T>;
using X1_VEC = hn::Vec<I1T>;
hn::Vec<OT> Func(const ptrdiff_t idx, const hn::Vec<I0T> x0,
const hn::Vec<I1T> x1, const Y_VEC y) {
return me()->Func(idx, x0, x1, y);
}
X0_VEC X0Init() { return me()->X0InitImpl(); }
X0_VEC X0InitImpl() { return hn::Zero(d_in0); }
X1_VEC X1Init() { return me()->X1InitImpl(); }
X1_VEC X1InitImpl() { return hn::Zero(d_in1); }
Y_VEC YInit() { return me()->YInitImpl(); }
Y_VEC YInitImpl() { return hn::Zero(d_out); }
X0_VEC Load0(const ptrdiff_t idx, IN0_T* from) {
return me()->Load0Impl(idx, from);
}
X0_VEC Load0Impl(const ptrdiff_t idx, IN0_T* from) {
return hn::LoadU(d_in0, from + idx);
}
X1_VEC Load1(const ptrdiff_t idx, IN1_T* from) {
return me()->Load1Impl(idx, from);
}
X1_VEC Load1Impl(const ptrdiff_t idx, IN1_T* from) {
return hn::LoadU(d_in1, from + idx);
}
// maskload can take in either a positive or negative number for `places`. if
// the number is positive, then it loads the top `places` values, and if it's
// negative, it loads the bottom |places| values. example: places = 3
// | o | o | o | x | x | x | x | x |
// example places = -3
// | x | x | x | x | x | o | o | o |
X0_VEC MaskLoad0(const ptrdiff_t idx, IN0_T* from, const ptrdiff_t places) {
return me()->MaskLoad0Impl(idx, from, places);
}
X0_VEC MaskLoad0Impl(const ptrdiff_t idx, IN0_T* from,
const ptrdiff_t places) {
auto mask = hn::FirstN(d_in0, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d_in0,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
return hn::MaskedLoad(mask, d_in0, from + idx);
}
hn::Vec<I1T> MaskLoad1(const ptrdiff_t idx, IN1_T* from,
const ptrdiff_t places) {
return me()->MaskLoad1Impl(idx, from, places);
}
hn::Vec<I1T> MaskLoad1Impl(const ptrdiff_t idx, IN1_T* from,
const ptrdiff_t places) {
auto mask = hn::FirstN(d_in1, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d_in1,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
return hn::MaskedLoad(mask, d_in1, from + idx);
}
// store returns a bool that is `false` when
bool StoreAndShortCircuit(const ptrdiff_t idx, OUT_T* to, const Y_VEC x) {
return me()->StoreAndShortCircuitImpl(idx, to, x);
}
bool StoreAndShortCircuitImpl(const ptrdiff_t idx, OUT_T* to, const Y_VEC x) {
hn::StoreU(x, d_out, to + idx);
return true;
}
ptrdiff_t MaskStore(const ptrdiff_t idx, OUT_T* to, const Y_VEC x,
const ptrdiff_t places) {
return me()->MaskStoreImpl(idx, to, x, places);
}
ptrdiff_t MaskStoreImpl(const ptrdiff_t idx, OUT_T* to, const Y_VEC x,
const ptrdiff_t places) {
auto mask = hn::FirstN(d_out, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d_out,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
hn::BlendedStore(x, mask, d_out, to + idx);
return std::abs(places);
}
ptrdiff_t Reduce(const Y_VEC x, OUT_T* to) { return me()->ReduceImpl(x, to); }
ptrdiff_t ReduceImpl(const Y_VEC x, OUT_T* to) {
// default does nothing
(void)x;
(void)to;
return 0;
}
void Reduce(const Y_VEC x0, const Y_VEC x1, const Y_VEC x2, Y_VEC* y) {
me()->ReduceImpl(x0, x1, x2, y);
}
void ReduceImpl(const Y_VEC x0, const Y_VEC x1, const Y_VEC x2, Y_VEC* y) {
// default does nothing
(void)x0;
(void)x1;
(void)x2;
(void)y;
}
};
template <class FUNC, typename IN_T, typename OUT_T>
inline void Unroller(FUNC& f, IN_T* HWY_RESTRICT x, OUT_T* HWY_RESTRICT y,
const ptrdiff_t n) {
auto xx = f.X0Init();
auto yy = f.YInit();
ptrdiff_t i = 0;
#if HWY_MEM_OPS_MIGHT_FAULT
constexpr auto lane_sz =
static_cast<ptrdiff_t>(RemoveRef<FUNC>::MaxUnitLanes());
if (n < lane_sz) {
const DFromV<decltype(yy)> d;
// this may not fit on the stack for HWY_RVV, but we do not reach this code
// there
HWY_ALIGN IN_T xtmp[static_cast<size_t>(lane_sz)];
HWY_ALIGN OUT_T ytmp[static_cast<size_t>(lane_sz)];
CopyBytes(x, xtmp, static_cast<size_t>(n) * sizeof(IN_T));
xx = f.MaskLoad(0, xtmp, n);
yy = f.Func(0, xx, yy);
Store(Zero(d), d, ytmp);
i += f.MaskStore(0, ytmp, yy, n);
i += f.Reduce(yy, ytmp);
CopyBytes(ytmp, y, static_cast<size_t>(i) * sizeof(OUT_T));
return;
}
#endif
const ptrdiff_t actual_lanes =
static_cast<ptrdiff_t>(RemoveRef<FUNC>::ActualLanes());
if (n > 4 * actual_lanes) {
auto xx1 = f.X0Init();
auto yy1 = f.YInit();
auto xx2 = f.X0Init();
auto yy2 = f.YInit();
auto xx3 = f.X0Init();
auto yy3 = f.YInit();
while (i + 4 * actual_lanes - 1 < n) {
xx = f.Load(i, x);
i += actual_lanes;
xx1 = f.Load(i, x);
i += actual_lanes;
xx2 = f.Load(i, x);
i += actual_lanes;
xx3 = f.Load(i, x);
i -= 3 * actual_lanes;
yy = f.Func(i, xx, yy);
yy1 = f.Func(i + actual_lanes, xx1, yy1);
yy2 = f.Func(i + 2 * actual_lanes, xx2, yy2);
yy3 = f.Func(i + 3 * actual_lanes, xx3, yy3);
if (!f.StoreAndShortCircuit(i, y, yy)) return;
i += actual_lanes;
if (!f.StoreAndShortCircuit(i, y, yy1)) return;
i += actual_lanes;
if (!f.StoreAndShortCircuit(i, y, yy2)) return;
i += actual_lanes;
if (!f.StoreAndShortCircuit(i, y, yy3)) return;
i += actual_lanes;
}
f.Reduce(yy3, yy2, yy1, &yy);
}
while (i + actual_lanes - 1 < n) {
xx = f.Load(i, x);
yy = f.Func(i, xx, yy);
if (!f.StoreAndShortCircuit(i, y, yy)) return;
i += actual_lanes;
}
if (i != n) {
xx = f.MaskLoad(n - actual_lanes, x, i - n);
yy = f.Func(n - actual_lanes, xx, yy);
f.MaskStore(n - actual_lanes, y, yy, i - n);
}
f.Reduce(yy, y);
}
template <class FUNC, typename IN0_T, typename IN1_T, typename OUT_T>
inline void Unroller(FUNC& HWY_RESTRICT f, IN0_T* HWY_RESTRICT x0,
IN1_T* HWY_RESTRICT x1, OUT_T* HWY_RESTRICT y,
const ptrdiff_t n) {
const ptrdiff_t lane_sz =
static_cast<ptrdiff_t>(RemoveRef<FUNC>::ActualLanes());
auto xx00 = f.X0Init();
auto xx10 = f.X1Init();
auto yy = f.YInit();
ptrdiff_t i = 0;
#if HWY_MEM_OPS_MIGHT_FAULT
if (n < lane_sz) {
const DFromV<decltype(yy)> d;
// this may not fit on the stack for HWY_RVV, but we do not reach this code
// there
constexpr auto max_lane_sz =
static_cast<ptrdiff_t>(RemoveRef<FUNC>::MaxUnitLanes());
HWY_ALIGN IN0_T xtmp0[static_cast<size_t>(max_lane_sz)];
HWY_ALIGN IN1_T xtmp1[static_cast<size_t>(max_lane_sz)];
HWY_ALIGN OUT_T ytmp[static_cast<size_t>(max_lane_sz)];
CopyBytes(x0, xtmp0, static_cast<size_t>(n) * sizeof(IN0_T));
CopyBytes(x1, xtmp1, static_cast<size_t>(n) * sizeof(IN1_T));
xx00 = f.MaskLoad0(0, xtmp0, n);
xx10 = f.MaskLoad1(0, xtmp1, n);
yy = f.Func(0, xx00, xx10, yy);
Store(Zero(d), d, ytmp);
i += f.MaskStore(0, ytmp, yy, n);
i += f.Reduce(yy, ytmp);
CopyBytes(ytmp, y, static_cast<size_t>(i) * sizeof(OUT_T));
return;
}
#endif
if (n > 4 * lane_sz) {
auto xx01 = f.X0Init();
auto xx11 = f.X1Init();
auto yy1 = f.YInit();
auto xx02 = f.X0Init();
auto xx12 = f.X1Init();
auto yy2 = f.YInit();
auto xx03 = f.X0Init();
auto xx13 = f.X1Init();
auto yy3 = f.YInit();
while (i + 4 * lane_sz - 1 < n) {
xx00 = f.Load0(i, x0);
xx10 = f.Load1(i, x1);
i += lane_sz;
xx01 = f.Load0(i, x0);
xx11 = f.Load1(i, x1);
i += lane_sz;
xx02 = f.Load0(i, x0);
xx12 = f.Load1(i, x1);
i += lane_sz;
xx03 = f.Load0(i, x0);
xx13 = f.Load1(i, x1);
i -= 3 * lane_sz;
yy = f.Func(i, xx00, xx10, yy);
yy1 = f.Func(i + lane_sz, xx01, xx11, yy1);
yy2 = f.Func(i + 2 * lane_sz, xx02, xx12, yy2);
yy3 = f.Func(i + 3 * lane_sz, xx03, xx13, yy3);
if (!f.StoreAndShortCircuit(i, y, yy)) return;
i += lane_sz;
if (!f.StoreAndShortCircuit(i, y, yy1)) return;
i += lane_sz;
if (!f.StoreAndShortCircuit(i, y, yy2)) return;
i += lane_sz;
if (!f.StoreAndShortCircuit(i, y, yy3)) return;
i += lane_sz;
}
f.Reduce(yy3, yy2, yy1, &yy);
}
while (i + lane_sz - 1 < n) {
xx00 = f.Load0(i, x0);
xx10 = f.Load1(i, x1);
yy = f.Func(i, xx00, xx10, yy);
if (!f.StoreAndShortCircuit(i, y, yy)) return;
i += lane_sz;
}
if (i != n) {
xx00 = f.MaskLoad0(n - lane_sz, x0, i - n);
xx10 = f.MaskLoad1(n - lane_sz, x1, i - n);
yy = f.Func(n - lane_sz, xx00, xx10, yy);
f.MaskStore(n - lane_sz, y, yy, i - n);
}
f.Reduce(yy, y);
}
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#endif // HIGHWAY_HWY_CONTRIB_UNROLLER_UNROLLER_INL_H_

491
deps/v8/third_party/highway/src/hwy/contrib/unroller/unroller_test.cc vendored Normal file
View File

@ -0,0 +1,491 @@
// Copyright Google LLC 2021
// Matthew Kolbe 2023
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <vector>
#include "hwy/base.h"
// clang-format off
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/contrib/unroller/unroller_test.cc" //NOLINT
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/contrib/unroller/unroller-inl.h"
#include "hwy/tests/test_util-inl.h"
// clang-format on
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
template <typename T>
T SimpleDot(const T* pa, const T* pb, size_t num) {
T sum = 0;
for (size_t i = 0; i < num; ++i) {
// For reasons unknown, fp16 += does not compile on clang (Arm).
sum = ConvertScalarTo<T>(sum + pa[i] * pb[i]);
}
return sum;
}
template <typename T>
T SimpleAcc(const T* pa, size_t num) {
T sum = 0;
for (size_t i = 0; i < num; ++i) {
sum += pa[i];
}
return sum;
}
template <typename T>
T SimpleMin(const T* pa, size_t num) {
T min = HighestValue<T>();
for (size_t i = 0; i < num; ++i) {
if (min > pa[i]) min = pa[i];
}
return min;
}
template <typename T>
struct MultiplyUnit : UnrollerUnit2D<MultiplyUnit<T>, T, T, T> {
using TT = hn::ScalableTag<T>;
HWY_INLINE hn::Vec<TT> Func(ptrdiff_t idx, const hn::Vec<TT> x0,
const hn::Vec<TT> x1, const hn::Vec<TT> y) {
(void)idx;
(void)y;
return hn::Mul(x0, x1);
}
};
template <typename FROM_T, typename TO_T>
struct ConvertUnit : UnrollerUnit<ConvertUnit<FROM_T, TO_T>, FROM_T, TO_T> {
using Base = UnrollerUnit<ConvertUnit<FROM_T, TO_T>, FROM_T, TO_T>;
using Base::MaxUnitLanes;
using typename Base::LargerD;
using TT_FROM = hn::Rebind<FROM_T, LargerD>;
using TT_TO = hn::Rebind<TO_T, LargerD>;
template <
class ToD, class FromV,
hwy::EnableIf<(sizeof(TFromV<FromV>) > sizeof(TFromD<ToD>))>* = nullptr>
static HWY_INLINE hn::Vec<ToD> DoConvertVector(ToD d, FromV v) {
return hn::DemoteTo(d, v);
}
template <
class ToD, class FromV,
hwy::EnableIf<(sizeof(TFromV<FromV>) == sizeof(TFromD<ToD>))>* = nullptr>
static HWY_INLINE hn::Vec<ToD> DoConvertVector(ToD d, FromV v) {
return hn::ConvertTo(d, v);
}
template <
class ToD, class FromV,
hwy::EnableIf<(sizeof(TFromV<FromV>) < sizeof(TFromD<ToD>))>* = nullptr>
static HWY_INLINE hn::Vec<ToD> DoConvertVector(ToD d, FromV v) {
return hn::PromoteTo(d, v);
}
hn::Vec<TT_TO> Func(ptrdiff_t idx, const hn::Vec<TT_FROM> x,
const hn::Vec<TT_TO> y) {
(void)idx;
(void)y;
TT_TO d;
return DoConvertVector(d, x);
}
};
// Returns a value that does not compare equal to `value`.
template <class D, HWY_IF_FLOAT_D(D)>
HWY_INLINE Vec<D> OtherValue(D d, TFromD<D> /*value*/) {
return NaN(d);
}
template <class D, HWY_IF_NOT_FLOAT_D(D)>
HWY_INLINE Vec<D> OtherValue(D d, TFromD<D> value) {
return hn::Set(d, hwy::AddWithWraparound(value, 1));
}
// Caveat: stores lane indices as MakeSigned<T>, which may overflow for 8-bit T
// on HWY_RVV.
template <typename T>
struct FindUnit : UnrollerUnit<FindUnit<T>, T, MakeSigned<T>> {
using TI = MakeSigned<T>;
using Base = UnrollerUnit<FindUnit<T>, T, TI>;
using Base::ActualLanes;
using Base::MaxUnitLanes;
using D = hn::CappedTag<T, MaxUnitLanes()>;
T to_find;
D d;
using DI = RebindToSigned<D>;
DI di;
FindUnit(T find) : to_find(find) {}
hn::Vec<DI> Func(ptrdiff_t idx, const hn::Vec<D> x, const hn::Vec<DI> y) {
const Mask<D> msk = hn::Eq(x, hn::Set(d, to_find));
const TI first_idx = static_cast<TI>(hn::FindFirstTrue(d, msk));
if (first_idx > -1)
return hn::Set(di, static_cast<TI>(static_cast<TI>(idx) + first_idx));
else
return y;
}
hn::Vec<D> X0InitImpl() { return OtherValue(D(), to_find); }
hn::Vec<DI> YInitImpl() { return hn::Set(di, TI{-1}); }
hn::Vec<D> MaskLoadImpl(const ptrdiff_t idx, T* from,
const ptrdiff_t places) {
auto mask = hn::FirstN(d, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
return hn::IfThenElse(mask, hn::MaskedLoad(mask, d, from + idx),
X0InitImpl());
}
bool StoreAndShortCircuitImpl(const ptrdiff_t idx, TI* to,
const hn::Vec<DI> x) {
(void)idx;
TI a = hn::GetLane(x);
to[0] = a;
if (a == -1) return true;
return false;
}
ptrdiff_t MaskStoreImpl(const ptrdiff_t idx, TI* to, const hn::Vec<DI> x,
const ptrdiff_t places) {
(void)idx;
(void)places;
TI a = hn::GetLane(x);
to[0] = a;
return 1;
}
};
template <typename T>
struct AccumulateUnit : UnrollerUnit<AccumulateUnit<T>, T, T> {
using TT = hn::ScalableTag<T>;
hn::Vec<TT> Func(ptrdiff_t idx, const hn::Vec<TT> x, const hn::Vec<TT> y) {
(void)idx;
return hn::Add(x, y);
}
bool StoreAndShortCircuitImpl(const ptrdiff_t idx, T* to,
const hn::Vec<TT> x) {
// no stores in a reducer
(void)idx;
(void)to;
(void)x;
return true;
}
ptrdiff_t MaskStoreImpl(const ptrdiff_t idx, T* to, const hn::Vec<TT> x,
const ptrdiff_t places) {
// no stores in a reducer
(void)idx;
(void)to;
(void)x;
(void)places;
return 0;
}
ptrdiff_t ReduceImpl(const hn::Vec<TT> x, T* to) {
const hn::ScalableTag<T> d;
(*to) = hn::ReduceSum(d, x);
return 1;
}
void ReduceImpl(const hn::Vec<TT> x0, const hn::Vec<TT> x1,
const hn::Vec<TT> x2, hn::Vec<TT>* y) {
(*y) = hn::Add(hn::Add(*y, x0), hn::Add(x1, x2));
}
};
template <typename T>
struct MinUnit : UnrollerUnit<MinUnit<T>, T, T> {
using Base = UnrollerUnit<MinUnit<T>, T, T>;
using Base::ActualLanes;
using TT = hn::ScalableTag<T>;
TT d;
hn::Vec<TT> Func(const ptrdiff_t idx, const hn::Vec<TT> x,
const hn::Vec<TT> y) {
(void)idx;
return hn::Min(y, x);
}
hn::Vec<TT> YInitImpl() { return hn::Set(d, HighestValue<T>()); }
hn::Vec<TT> MaskLoadImpl(const ptrdiff_t idx, T* from,
const ptrdiff_t places) {
auto mask = hn::FirstN(d, static_cast<size_t>(places));
auto maskneg = hn::Not(hn::FirstN(
d,
static_cast<size_t>(places + static_cast<ptrdiff_t>(ActualLanes()))));
if (places < 0) mask = maskneg;
auto def = YInitImpl();
return hn::MaskedLoadOr(def, mask, d, from + idx);
}
bool StoreAndShortCircuitImpl(const ptrdiff_t idx, T* to,
const hn::Vec<TT> x) {
// no stores in a reducer
(void)idx;
(void)to;
(void)x;
return true;
}
ptrdiff_t MaskStoreImpl(const ptrdiff_t idx, T* to, const hn::Vec<TT> x,
const ptrdiff_t places) {
// no stores in a reducer
(void)idx;
(void)to;
(void)x;
(void)places;
return 0;
}
ptrdiff_t ReduceImpl(const hn::Vec<TT> x, T* to) {
auto minvect = hn::MinOfLanes(d, x);
(*to) = hn::ExtractLane(minvect, 0);
return 1;
}
void ReduceImpl(const hn::Vec<TT> x0, const hn::Vec<TT> x1,
const hn::Vec<TT> x2, hn::Vec<TT>* y) {
auto a = hn::Min(x1, x0);
auto b = hn::Min(*y, x2);
(*y) = hn::Min(a, b);
}
};
template <typename T>
struct DotUnit : UnrollerUnit2D<DotUnit<T>, T, T, T> {
using TT = hn::ScalableTag<T>;
hn::Vec<TT> Func(const ptrdiff_t idx, const hn::Vec<TT> x0,
const hn::Vec<TT> x1, const hn::Vec<TT> y) {
(void)idx;
return hn::MulAdd(x0, x1, y);
}
bool StoreAndShortCircuitImpl(const ptrdiff_t idx, T* to,
const hn::Vec<TT> x) {
// no stores in a reducer
(void)idx;
(void)to;
(void)x;
return true;
}
ptrdiff_t MaskStoreImpl(const ptrdiff_t idx, T* to, const hn::Vec<TT> x,
const ptrdiff_t places) {
// no stores in a reducer
(void)idx;
(void)to;
(void)x;
(void)places;
return 0;
}
ptrdiff_t ReduceImpl(const hn::Vec<TT> x, T* to) {
const hn::ScalableTag<T> d;
(*to) = hn::ReduceSum(d, x);
return 1;
}
void ReduceImpl(const hn::Vec<TT> x0, const hn::Vec<TT> x1,
const hn::Vec<TT> x2, hn::Vec<TT>* y) {
(*y) = hn::Add(hn::Add(*y, x0), hn::Add(x1, x2));
}
};
template <class D>
std::vector<size_t> Counts(D d) {
const size_t N = Lanes(d);
return std::vector<size_t>{1,
3,
7,
16,
HWY_MAX(N / 2, 1),
HWY_MAX(2 * N / 3, 1),
N,
N + 1,
4 * N / 3,
3 * N,
8 * N,
8 * N + 2,
256 * N - 1,
256 * N};
}
struct TestDot {
template <typename T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
// TODO(janwas): avoid internal compiler error
#if HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2 || HWY_COMPILER_MSVC
(void)d;
#else
RandomState rng;
const auto random_t = [&rng]() {
const int32_t bits = static_cast<int32_t>(Random32(&rng)) & 1023;
return static_cast<float>(bits - 512) * (1.0f / 64);
};
for (size_t num : Counts(d)) {
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(num);
AlignedFreeUniquePtr<T[]> pb = AllocateAligned<T>(num);
AlignedFreeUniquePtr<T[]> py = AllocateAligned<T>(num);
HWY_ASSERT(pa && pb && py);
T* a = pa.get();
T* b = pb.get();
T* y = py.get();
size_t i = 0;
for (; i < num; ++i) {
a[i] = ConvertScalarTo<T>(random_t());
b[i] = ConvertScalarTo<T>(random_t());
}
const T expected_dot = SimpleDot(a, b, num);
MultiplyUnit<T> multfn;
Unroller(multfn, a, b, y, static_cast<ptrdiff_t>(num));
AccumulateUnit<T> accfn;
T dot_via_mul_acc;
Unroller(accfn, y, &dot_via_mul_acc, static_cast<ptrdiff_t>(num));
const double tolerance = 48.0 *
ConvertScalarTo<double>(hwy::Epsilon<T>()) *
ScalarAbs(expected_dot);
HWY_ASSERT(ScalarAbs(expected_dot - dot_via_mul_acc) < tolerance);
DotUnit<T> dotfn;
T dotr;
Unroller(dotfn, a, b, &dotr, static_cast<ptrdiff_t>(num));
HWY_ASSERT(ConvertScalarTo<double>(ScalarAbs((expected_dot - dotr))) <
tolerance);
auto expected_min = SimpleMin(a, num);
MinUnit<T> minfn;
T minr;
Unroller(minfn, a, &minr, static_cast<ptrdiff_t>(num));
HWY_ASSERT(ConvertScalarTo<double>(ScalarAbs(expected_min - minr)) <
1e-7);
}
#endif
}
};
void TestAllDot() { ForFloatTypes(ForPartialVectors<TestDot>()); }
struct TestConvert {
template <typename T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
// TODO(janwas): avoid internal compiler error
#if HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2 || HWY_COMPILER_MSVC
(void)d;
#else
for (size_t num : Counts(d)) {
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(num);
AlignedFreeUniquePtr<int[]> pto = AllocateAligned<int>(num);
HWY_ASSERT(pa && pto);
T* HWY_RESTRICT a = pa.get();
int* HWY_RESTRICT to = pto.get();
for (size_t i = 0; i < num; ++i) {
a[i] = ConvertScalarTo<T>(static_cast<double>(i) * 0.25);
}
ConvertUnit<T, int> cvtfn;
Unroller(cvtfn, a, to, static_cast<ptrdiff_t>(num));
for (size_t i = 0; i < num; ++i) {
// TODO(janwas): RVV QEMU fcvt_rtz appears to 'truncate' 4.75 to 5.
HWY_ASSERT(
static_cast<int>(a[i]) == to[i] ||
(HWY_TARGET == HWY_RVV && static_cast<int>(a[i]) == to[i] - 1));
}
ConvertUnit<int, T> cvtbackfn;
Unroller(cvtbackfn, to, a, static_cast<ptrdiff_t>(num));
for (size_t i = 0; i < num; ++i) {
HWY_ASSERT_EQ(ConvertScalarTo<T>(to[i]), a[i]);
}
}
#endif
}
};
void TestAllConvert() { ForFloat3264Types(ForPartialVectors<TestConvert>()); }
struct TestFind {
template <typename T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
for (size_t num : Counts(d)) {
AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(num);
HWY_ASSERT(pa);
T* a = pa.get();
for (size_t i = 0; i < num; ++i) a[i] = ConvertScalarTo<T>(i);
FindUnit<T> cvtfn(ConvertScalarTo<T>(num - 1));
MakeSigned<T> idx = 0;
Unroller(cvtfn, a, &idx, static_cast<ptrdiff_t>(num));
HWY_ASSERT(static_cast<MakeUnsigned<T>>(idx) < num);
HWY_ASSERT(a[idx] == ConvertScalarTo<T>(num - 1));
FindUnit<T> cvtfnzero((T)(0));
Unroller(cvtfnzero, a, &idx, static_cast<ptrdiff_t>(num));
HWY_ASSERT(static_cast<MakeUnsigned<T>>(idx) < num);
HWY_ASSERT(a[idx] == (T)(0));
// For f16, we cannot search for `num` because it may round to a value
// that is actually in the (large) array.
FindUnit<T> cvtfnnotin(HighestValue<T>());
Unroller(cvtfnnotin, a, &idx, static_cast<ptrdiff_t>(num));
HWY_ASSERT(idx == -1);
}
}
};
void TestAllFind() { ForFloatTypes(ForPartialVectors<TestFind>()); }
} // namespace
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(UnrollerTest);
HWY_EXPORT_AND_TEST_P(UnrollerTest, TestAllDot);
HWY_EXPORT_AND_TEST_P(UnrollerTest, TestAllConvert);
HWY_EXPORT_AND_TEST_P(UnrollerTest, TestAllFind);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

376
deps/v8/third_party/highway/src/hwy/detect_compiler_arch.h vendored Normal file
View File

@ -0,0 +1,376 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_DETECT_COMPILER_ARCH_H_
#define HIGHWAY_HWY_DETECT_COMPILER_ARCH_H_
// Detects compiler and arch from predefined macros. Zero dependencies for
// inclusion by foreach_target.h.
// Add to #if conditions to prevent IDE from graying out code.
#if (defined __CDT_PARSER__) || (defined __INTELLISENSE__) || \
(defined Q_CREATOR_RUN) || (defined __CLANGD__) || \
(defined GROK_ELLIPSIS_BUILD)
#define HWY_IDE 1
#else
#define HWY_IDE 0
#endif
//------------------------------------------------------------------------------
// Compiler
// Actual MSVC, not clang-cl, which defines _MSC_VER but doesn't behave like
// MSVC in other aspects (e.g. HWY_DIAGNOSTICS).
#if defined(_MSC_VER) && !defined(__clang__)
#define HWY_COMPILER_MSVC _MSC_VER
#else
#define HWY_COMPILER_MSVC 0
#endif
#if defined(_MSC_VER) && defined(__clang__)
#define HWY_COMPILER_CLANGCL _MSC_VER
#else
#define HWY_COMPILER_CLANGCL 0
#endif
#ifdef __INTEL_COMPILER
#define HWY_COMPILER_ICC __INTEL_COMPILER
#else
#define HWY_COMPILER_ICC 0
#endif
#ifdef __INTEL_LLVM_COMPILER
#define HWY_COMPILER_ICX __INTEL_LLVM_COMPILER
#else
#define HWY_COMPILER_ICX 0
#endif
// HWY_COMPILER_GCC is a generic macro for all compilers implementing the GNU
// compiler extensions (eg. Clang, Intel...)
#ifdef __GNUC__
#define HWY_COMPILER_GCC (__GNUC__ * 100 + __GNUC_MINOR__)
#else
#define HWY_COMPILER_GCC 0
#endif
// Clang or clang-cl, not GCC.
#ifdef __clang__
// In case of Apple LLVM (whose version number is unrelated to that of LLVM) or
// an invalid version number, deduce it from the presence of warnings.
// Originally based on
// https://github.com/simd-everywhere/simde/blob/47d6e603de9d04ee05cdfbc57cf282a02be1bf2a/simde/simde-detect-clang.h#L59.
// Please send updates below to them as well, thanks!
#if defined(__apple_build_version__) || __clang_major__ >= 999
#if __has_warning("-Woverriding-option")
#define HWY_COMPILER_CLANG 1801
// No new warnings in 17.0, and Apple LLVM 15.3, which should be 1600, already
// has the unsafe_buffer_usage attribute, so we instead check for new builtins.
#elif __has_builtin(__builtin_nondeterministic_value)
#define HWY_COMPILER_CLANG 1700
#elif __has_attribute(nouwtable) // no new warnings in 16.0
#define HWY_COMPILER_CLANG 1600
#elif __has_warning("-Warray-parameter")
#define HWY_COMPILER_CLANG 1500
#elif __has_warning("-Wbitwise-instead-of-logical")
#define HWY_COMPILER_CLANG 1400
#elif __has_warning("-Wreserved-identifier")
#define HWY_COMPILER_CLANG 1300
#elif __has_warning("-Wformat-insufficient-args")
#define HWY_COMPILER_CLANG 1200
#elif __has_warning("-Wimplicit-const-int-float-conversion")
#define HWY_COMPILER_CLANG 1100
#elif __has_warning("-Wmisleading-indentation")
#define HWY_COMPILER_CLANG 1000
#elif defined(__FILE_NAME__)
#define HWY_COMPILER_CLANG 900
#elif __has_warning("-Wextra-semi-stmt") || \
__has_builtin(__builtin_rotateleft32)
#define HWY_COMPILER_CLANG 800
// For reasons unknown, XCode 10.3 (Apple LLVM version 10.0.1) is apparently
// based on Clang 7, but does not support the warning we test.
// See https://en.wikipedia.org/wiki/Xcode#Toolchain_versions and
// https://trac.macports.org/wiki/XcodeVersionInfo.
#elif __has_warning("-Wc++98-compat-extra-semi") || \
(defined(__apple_build_version__) && __apple_build_version__ >= 10010000)
#define HWY_COMPILER_CLANG 700
#else // Anything older than 7.0 is not recommended for Highway.
#define HWY_COMPILER_CLANG 600
#endif // __has_warning chain
#define HWY_COMPILER3_CLANG (HWY_COMPILER_CLANG * 100)
#else // use normal version
#define HWY_COMPILER_CLANG (__clang_major__ * 100 + __clang_minor__)
#define HWY_COMPILER3_CLANG \
(__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__)
#endif
#else // Not clang
#define HWY_COMPILER_CLANG 0
#define HWY_COMPILER3_CLANG 0
#endif
#if HWY_COMPILER_GCC && !HWY_COMPILER_CLANG && !HWY_COMPILER_ICC && \
!HWY_COMPILER_ICX
#define HWY_COMPILER_GCC_ACTUAL HWY_COMPILER_GCC
#else
#define HWY_COMPILER_GCC_ACTUAL 0
#endif
// More than one may be nonzero, but we want at least one.
#if 0 == (HWY_COMPILER_MSVC + HWY_COMPILER_CLANGCL + HWY_COMPILER_ICC + \
HWY_COMPILER_ICX + HWY_COMPILER_GCC + HWY_COMPILER_CLANG)
#error "Unsupported compiler"
#endif
// We should only detect one of these (only clang/clangcl/icx overlap)
#if 1 < (!!HWY_COMPILER_MSVC + (!!HWY_COMPILER_ICC & !HWY_COMPILER_ICX) + \
!!HWY_COMPILER_GCC_ACTUAL + \
!!(HWY_COMPILER_ICX | HWY_COMPILER_CLANGCL | HWY_COMPILER_CLANG))
#error "Detected multiple compilers"
#endif
//------------------------------------------------------------------------------
// Compiler features and C++ version
#ifdef __has_builtin
#define HWY_HAS_BUILTIN(name) __has_builtin(name)
#else
#define HWY_HAS_BUILTIN(name) 0
#endif
#ifdef __has_attribute
#define HWY_HAS_ATTRIBUTE(name) __has_attribute(name)
#else
#define HWY_HAS_ATTRIBUTE(name) 0
#endif
#ifdef __has_cpp_attribute
#define HWY_HAS_CPP_ATTRIBUTE(name) __has_cpp_attribute(name)
#else
#define HWY_HAS_CPP_ATTRIBUTE(name) 0
#endif
#ifdef __has_feature
#define HWY_HAS_FEATURE(name) __has_feature(name)
#else
#define HWY_HAS_FEATURE(name) 0
#endif
// NOTE: clang ~17 does not correctly handle wrapping __has_include in a macro.
#if HWY_COMPILER_MSVC && defined(_MSVC_LANG) && _MSVC_LANG > __cplusplus
#define HWY_CXX_LANG _MSVC_LANG
#else
#define HWY_CXX_LANG __cplusplus
#endif
#if defined(__cpp_constexpr) && __cpp_constexpr >= 201603L
#define HWY_CXX17_CONSTEXPR constexpr
#else
#define HWY_CXX17_CONSTEXPR
#endif
#if defined(__cpp_constexpr) && __cpp_constexpr >= 201304L
#define HWY_CXX14_CONSTEXPR constexpr
#else
#define HWY_CXX14_CONSTEXPR
#endif
#if HWY_CXX_LANG >= 201703L
#define HWY_IF_CONSTEXPR if constexpr
#else
#define HWY_IF_CONSTEXPR if
#endif
//------------------------------------------------------------------------------
// Architecture
#if defined(__i386__) || defined(_M_IX86)
#define HWY_ARCH_X86_32 1
#else
#define HWY_ARCH_X86_32 0
#endif
#if defined(__x86_64__) || defined(_M_X64)
#define HWY_ARCH_X86_64 1
#else
#define HWY_ARCH_X86_64 0
#endif
#if HWY_ARCH_X86_32 && HWY_ARCH_X86_64
#error "Cannot have both x86-32 and x86-64"
#endif
#if HWY_ARCH_X86_32 || HWY_ARCH_X86_64
#define HWY_ARCH_X86 1
#else
#define HWY_ARCH_X86 0
#endif
#if defined(__powerpc64__) || defined(_M_PPC) || defined(__powerpc__)
#define HWY_ARCH_PPC 1
#else
#define HWY_ARCH_PPC 0
#endif
#if defined(__powerpc64__) || (HWY_ARCH_PPC && defined(__64BIT__))
#define HWY_ARCH_PPC_64 1
#else
#define HWY_ARCH_PPC_64 0
#endif
// aarch32 is currently not supported; please raise an issue if you want it.
#if defined(__ARM_ARCH_ISA_A64) || defined(__aarch64__) || defined(_M_ARM64)
#define HWY_ARCH_ARM_A64 1
#else
#define HWY_ARCH_ARM_A64 0
#endif
#if (defined(__ARM_ARCH) && __ARM_ARCH == 7) || (defined(_M_ARM) && _M_ARM == 7)
#define HWY_ARCH_ARM_V7 1
#else
#define HWY_ARCH_ARM_V7 0
#endif
#if HWY_ARCH_ARM_A64 && HWY_ARCH_ARM_V7
#error "Cannot have both A64 and V7"
#endif
// Any *supported* version of Arm, i.e. 7 or later
#if HWY_ARCH_ARM_A64 || HWY_ARCH_ARM_V7
#define HWY_ARCH_ARM 1
#else
#define HWY_ARCH_ARM 0
#endif
// Older than Armv7 (e.g. armel aka Armv5) => we do not support SIMD.
#if (defined(__arm__) || defined(_M_ARM)) && !HWY_ARCH_ARM
#define HWY_ARCH_ARM_OLD 1
#else
#define HWY_ARCH_ARM_OLD 0
#endif
#if defined(__EMSCRIPTEN__) || defined(__wasm__) || defined(__WASM__)
#define HWY_ARCH_WASM 1
#else
#define HWY_ARCH_WASM 0
#endif
#ifdef __riscv
#define HWY_ARCH_RISCV 1
#else
#define HWY_ARCH_RISCV 0
#endif
// DEPRECATED names; please use HWY_ARCH_RISCV instead.
#define HWY_ARCH_RVV HWY_ARCH_RISCV
#if HWY_ARCH_RISCV && defined(__riscv_xlen)
#if __riscv_xlen == 32
#define HWY_ARCH_RISCV_32 1
#else
#define HWY_ARCH_RISCV_32 0
#endif
#if __riscv_xlen == 64
#define HWY_ARCH_RISCV_64 1
#else
#define HWY_ARCH_RISCV_64 0
#endif
#else // !HWY_ARCH_RISCV || !defined(__riscv_xlen)
#define HWY_ARCH_RISCV_32 0
#define HWY_ARCH_RISCV_64 0
#endif // HWY_ARCH_RISCV && defined(__riscv_xlen)
#if HWY_ARCH_RISCV_32 && HWY_ARCH_RISCV_64
#error "Cannot have both RISCV_32 and RISCV_64"
#endif
#if defined(__s390x__)
#define HWY_ARCH_S390X 1
#else
#define HWY_ARCH_S390X 0
#endif
// It is an error to detect multiple architectures at the same time, but OK to
// detect none of the above.
#if (HWY_ARCH_X86 + HWY_ARCH_PPC + HWY_ARCH_ARM + HWY_ARCH_ARM_OLD + \
HWY_ARCH_WASM + HWY_ARCH_RISCV + HWY_ARCH_S390X) > 1
#error "Must not detect more than one architecture"
#endif
//------------------------------------------------------------------------------
// Operating system
#if defined(_WIN32) || defined(_WIN64)
#define HWY_OS_WIN 1
#else
#define HWY_OS_WIN 0
#endif
#if defined(linux) || defined(__linux__)
#define HWY_OS_LINUX 1
#else
#define HWY_OS_LINUX 0
#endif
// iOS or Mac
#if defined(__APPLE__)
#define HWY_OS_APPLE 1
#else
#define HWY_OS_APPLE 0
#endif
#if defined(__FreeBSD__)
#define HWY_OS_FREEBSD 1
#else
#define HWY_OS_FREEBSD 0
#endif
// It is an error to detect multiple OSes at the same time, but OK to
// detect none of the above.
#if (HWY_OS_WIN + HWY_OS_LINUX + HWY_OS_APPLE + HWY_OS_FREEBSD) > 1
#error "Must not detect more than one OS"
#endif
//------------------------------------------------------------------------------
// Endianness
#if HWY_COMPILER_MSVC
#if HWY_ARCH_PPC && defined(_XBOX_VER) && _XBOX_VER >= 200
// XBox 360 is big-endian
#define HWY_IS_LITTLE_ENDIAN 0
#define HWY_IS_BIG_ENDIAN 1
#else
// All other targets supported by MSVC are little-endian
#define HWY_IS_LITTLE_ENDIAN 1
#define HWY_IS_BIG_ENDIAN 0
#endif // HWY_ARCH_PPC && defined(_XBOX_VER) && _XBOX_VER >= 200
#elif defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \
__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define HWY_IS_LITTLE_ENDIAN 1
#define HWY_IS_BIG_ENDIAN 0
#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \
__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#define HWY_IS_LITTLE_ENDIAN 0
#define HWY_IS_BIG_ENDIAN 1
#else
#error "Unable to detect endianness or unsupported byte order"
#endif
#if (HWY_IS_LITTLE_ENDIAN + HWY_IS_BIG_ENDIAN) != 1
#error "Must only detect one byte order"
#endif
#endif // HIGHWAY_HWY_DETECT_COMPILER_ARCH_H_

797
deps/v8/third_party/highway/src/hwy/detect_targets.h vendored Normal file
View File

@ -0,0 +1,797 @@
// Copyright 2021 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_DETECT_TARGETS_H_
#define HIGHWAY_HWY_DETECT_TARGETS_H_
// Defines targets and chooses which to enable.
#include "hwy/detect_compiler_arch.h"
//------------------------------------------------------------------------------
// Optional configuration
// See g3doc/quick_reference.md for documentation of these macros.
// Uncomment to override the default baseline determined from predefined macros:
// #define HWY_BASELINE_TARGETS (HWY_SSE4 | HWY_SCALAR)
// Uncomment to override the default blocklist:
// #define HWY_BROKEN_TARGETS HWY_AVX3
// Uncomment to definitely avoid generating those target(s):
// #define HWY_DISABLED_TARGETS HWY_SSE4
// Uncomment to avoid emitting BMI/BMI2/FMA instructions (allows generating
// AVX2 target for VMs which support AVX2 but not the other instruction sets)
// #define HWY_DISABLE_BMI2_FMA
// Uncomment to enable these on MSVC even if the predefined macros are not set.
// #define HWY_WANT_SSE2 1
// #define HWY_WANT_SSSE3 1
// #define HWY_WANT_SSE4 1
//------------------------------------------------------------------------------
// Targets
// Unique bit value for each target. A lower value is "better" (e.g. more lanes)
// than a higher value within the same group/platform - see HWY_STATIC_TARGET.
//
// All values are unconditionally defined so we can test HWY_TARGETS without
// first checking the HWY_ARCH_*.
//
// The C99 preprocessor evaluates #if expressions using intmax_t types. This
// holds at least 64 bits in practice (verified 2022-07-18 via Godbolt on
// 32-bit clang/GCC/MSVC compilers for x86/Arm7/AArch32/RISC-V/WASM). We now
// avoid overflow when computing HWY_TARGETS (subtracting one instead of
// left-shifting 2^62), but still do not use bit 63 because it is the sign bit.
// --------------------------- x86: 15 targets (+ one fallback)
// Bits 0..3 reserved (4 targets)
#define HWY_AVX3_SPR (1LL << 4)
// Bit 5 reserved (likely AVX10.2 with 256-bit vectors)
// Currently HWY_AVX3_DL plus AVX512BF16 and a special case for CompressStore
// (10x as fast).
// We may later also use VPCONFLICT.
#define HWY_AVX3_ZEN4 (1LL << 6) // see HWY_WANT_AVX3_ZEN4 below
// Currently satisfiable by Ice Lake (VNNI, VPCLMULQDQ, VPOPCNTDQ, VBMI, VBMI2,
// VAES, BITALG, GFNI). Later to be added: BF16 (Cooper Lake). VP2INTERSECT is
// only in Tiger Lake?
#define HWY_AVX3_DL (1LL << 7) // see HWY_WANT_AVX3_DL below
#define HWY_AVX3 (1LL << 8) // HWY_AVX2 plus AVX-512F/BW/CD/DQ/VL
#define HWY_AVX2 (1LL << 9) // HWY_SSE4 plus BMI2 + F16 + FMA
// Bit 10: reserved
#define HWY_SSE4 (1LL << 11) // SSE4.2 plus AES + CLMUL
#define HWY_SSSE3 (1LL << 12) // S-SSE3
// Bit 13: reserved for SSE3
#define HWY_SSE2 (1LL << 14)
// The highest bit in the HWY_TARGETS mask that a x86 target can have. Used for
// dynamic dispatch. All x86 target bits must be lower or equal to
// (1 << HWY_HIGHEST_TARGET_BIT_X86) and they can only use
// HWY_MAX_DYNAMIC_TARGETS in total.
#define HWY_HIGHEST_TARGET_BIT_X86 14
// --------------------------- Arm: 15 targets (+ one fallback)
// Bits 15..17 reserved (3 targets)
#define HWY_SVE2_128 (1LL << 18) // specialized (e.g. Neoverse V2/N2/N3)
#define HWY_SVE_256 (1LL << 19) // specialized (Neoverse V1)
// Bits 20-22 reserved for later SVE (3 targets)
#define HWY_SVE2 (1LL << 23)
#define HWY_SVE (1LL << 24)
// Bit 25 reserved for NEON
#define HWY_NEON_BF16 (1LL << 26) // fp16/dot/bf16 (e.g. Neoverse V2/N2/N3)
// Bit 27 reserved for NEON
#define HWY_NEON (1LL << 28) // Implies support for AES
#define HWY_NEON_WITHOUT_AES (1LL << 29)
#define HWY_HIGHEST_TARGET_BIT_ARM 29
#define HWY_ALL_NEON (HWY_NEON_WITHOUT_AES | HWY_NEON | HWY_NEON_BF16)
#define HWY_ALL_SVE (HWY_SVE | HWY_SVE2 | HWY_SVE_256 | HWY_SVE2_128)
// --------------------------- RISC-V: 9 targets (+ one fallback)
// Bits 30..36 reserved (7 targets)
#define HWY_RVV (1LL << 37)
// Bit 38 reserved
#define HWY_HIGHEST_TARGET_BIT_RVV 38
// --------------------------- Future expansion: 4 targets
// Bits 39..42 reserved
// --------------------------- IBM Power/ZSeries: 9 targets (+ one fallback)
// Bits 43..46 reserved (4 targets)
#define HWY_PPC10 (1LL << 47) // v3.1
#define HWY_PPC9 (1LL << 48) // v3.0
#define HWY_PPC8 (1LL << 49) // v2.07
#define HWY_Z15 (1LL << 50) // Z15
#define HWY_Z14 (1LL << 51) // Z14
#define HWY_HIGHEST_TARGET_BIT_PPC 51
#define HWY_ALL_PPC (HWY_PPC8 | HWY_PPC9 | HWY_PPC10)
// --------------------------- WebAssembly: 9 targets (+ one fallback)
// Bits 52..57 reserved (6 targets)
#define HWY_WASM_EMU256 (1LL << 58) // Experimental
#define HWY_WASM (1LL << 59)
// Bits 60 reserved
#define HWY_HIGHEST_TARGET_BIT_WASM 60
// --------------------------- Emulation: 2 targets
#define HWY_EMU128 (1LL << 61)
// We do not add/left-shift, so this will not overflow to a negative number.
#define HWY_SCALAR (1LL << 62)
#define HWY_HIGHEST_TARGET_BIT_SCALAR 62
// Do not use bit 63 - would be confusing to have negative numbers.
//------------------------------------------------------------------------------
// Set default blocklists
// Disabled means excluded from enabled at user's request. A separate config
// macro allows disabling without deactivating the blocklist below.
#ifndef HWY_DISABLED_TARGETS
#define HWY_DISABLED_TARGETS 0
#endif
// Broken means excluded from enabled due to known compiler issues. We define
// separate HWY_BROKEN_* and then OR them together (more than one might apply).
// x86 clang-6: we saw multiple AVX2/3 compile errors and in one case invalid
// SSE4 codegen (possibly only for msan), so disable all those targets.
#if HWY_ARCH_X86 && (HWY_COMPILER_CLANG != 0 && HWY_COMPILER_CLANG < 700)
#define HWY_BROKEN_CLANG6 (HWY_SSE4 | (HWY_SSE4 - 1))
// This entails a major speed reduction, so warn unless the user explicitly
// opts in to scalar-only.
#if !defined(HWY_COMPILE_ONLY_SCALAR)
#pragma message("x86 Clang <= 6: define HWY_COMPILE_ONLY_SCALAR or upgrade.")
#endif
#else
#define HWY_BROKEN_CLANG6 0
#endif
// 32-bit may fail to compile AVX2/3.
#if HWY_ARCH_X86_32
#define HWY_BROKEN_32BIT (HWY_AVX2 | (HWY_AVX2 - 1))
#else
#define HWY_BROKEN_32BIT 0
#endif
// MSVC AVX3 support is buggy: https://github.com/Mysticial/Flops/issues/16
#if HWY_COMPILER_MSVC != 0
#define HWY_BROKEN_MSVC (HWY_AVX3 | (HWY_AVX3 - 1))
#else
#define HWY_BROKEN_MSVC 0
#endif
// AVX3_DL and AVX3_ZEN4 require clang >= 7 (ensured above), gcc >= 8.1 or ICC
// 2021.
#if (HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 801) || \
(HWY_COMPILER_ICC && HWY_COMPILER_ICC < 2021)
#define HWY_BROKEN_AVX3_DL_ZEN4 (HWY_AVX3_DL | HWY_AVX3_ZEN4)
#else
#define HWY_BROKEN_AVX3_DL_ZEN4 0
#endif
// AVX3_SPR requires clang >= 14, gcc >= 12, or ICC 2021.
#if (HWY_COMPILER_CLANG != 0 && HWY_COMPILER_CLANG < 1400) || \
(HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1200) || \
(HWY_COMPILER_ICC && HWY_COMPILER_ICC < 2021)
#define HWY_BROKEN_AVX3_SPR (HWY_AVX3_SPR)
#else
#define HWY_BROKEN_AVX3_SPR 0
#endif
// armv7be has not been tested and is not yet supported.
#if HWY_ARCH_ARM_V7 && HWY_IS_BIG_ENDIAN
#define HWY_BROKEN_ARM7_BIG_ENDIAN HWY_ALL_NEON
#else
#define HWY_BROKEN_ARM7_BIG_ENDIAN 0
#endif
// armv7-a without a detected vfpv4 is not supported
// (for example Cortex-A8, Cortex-A9)
// vfpv4 always have neon half-float _and_ FMA.
#if HWY_ARCH_ARM_V7 && (__ARM_ARCH_PROFILE == 'A') && \
!defined(__ARM_VFPV4__) && \
!((__ARM_NEON_FP & 0x2 /* half-float */) && (__ARM_FEATURE_FMA == 1))
#define HWY_BROKEN_ARM7_WITHOUT_VFP4 HWY_ALL_NEON
#else
#define HWY_BROKEN_ARM7_WITHOUT_VFP4 0
#endif
// HWY_NEON_BF16 requires recent compilers.
#if (HWY_COMPILER_CLANG != 0 && HWY_COMPILER_CLANG < 1700) || \
(HWY_COMPILER_GCC_ACTUAL != 0 && HWY_COMPILER_GCC_ACTUAL < 1302)
#define HWY_BROKEN_NEON_BF16 (HWY_NEON_BF16)
#else
#define HWY_BROKEN_NEON_BF16 0
#endif
// SVE[2] require recent clang or gcc versions.
#if (HWY_COMPILER_CLANG && HWY_COMPILER_CLANG < 1900) || \
(HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1000)
#define HWY_BROKEN_SVE (HWY_SVE | HWY_SVE2 | HWY_SVE_256 | HWY_SVE2_128)
#else
#define HWY_BROKEN_SVE 0
#endif
#if (HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1100)
// GCC 10 supports the -mcpu=power10 option but does not support the PPC10
// vector intrinsics
#define HWY_BROKEN_PPC10 (HWY_PPC10)
#elif HWY_ARCH_PPC && HWY_IS_BIG_ENDIAN && \
((HWY_COMPILER3_CLANG && HWY_COMPILER3_CLANG < 160001) || \
(HWY_COMPILER_GCC_ACTUAL >= 1200 && HWY_COMPILER_GCC_ACTUAL <= 1203) || \
(HWY_COMPILER_GCC_ACTUAL >= 1300 && HWY_COMPILER_GCC_ACTUAL <= 1301))
// GCC 12.0 through 12.3 and GCC 13.0 through 13.1 have a compiler bug where the
// vsldoi instruction is sometimes incorrectly optimized out (and this causes
// some of the Highway unit tests to fail on big-endian PPC10). Details about
// this compiler bug can be found at
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=109069, and this bug will be
// fixed in the upcoming GCC 12.4 and 13.2 releases.
// Clang 16.0.0 and earlier (but not Clang 16.0.1 and later) have a compiler
// bug in the LLVM DAGCombiner that causes a zero-extend followed by an
// element insert into a vector, followed by a vector shuffle to be incorrectly
// optimized on big-endian PPC (and which caused some of the Highway unit tests
// to fail on big-endian PPC10).
// Details about this bug, which has already been fixed in Clang 16.0.1 and
// later, can be found at https://github.com/llvm/llvm-project/issues/61315.
#define HWY_BROKEN_PPC10 (HWY_PPC10)
#else
#define HWY_BROKEN_PPC10 0
#endif
// PPC8/PPC9/PPC10 targets may fail to compile on 32-bit PowerPC
#if HWY_ARCH_PPC && !HWY_ARCH_PPC_64
#define HWY_BROKEN_PPC_32BIT (HWY_PPC8 | HWY_PPC9 | HWY_PPC10)
#else
#define HWY_BROKEN_PPC_32BIT 0
#endif
// HWY_RVV fails to compile with GCC < 13 or Clang < 16.
#if HWY_ARCH_RISCV && \
((HWY_COMPILER_CLANG && HWY_COMPILER_CLANG < 1600) || \
(HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1300))
#define HWY_BROKEN_RVV (HWY_RVV)
#else
#define HWY_BROKEN_RVV 0
#endif
// Allow the user to override this without any guarantee of success.
#ifndef HWY_BROKEN_TARGETS
#define HWY_BROKEN_TARGETS \
(HWY_BROKEN_CLANG6 | HWY_BROKEN_32BIT | HWY_BROKEN_MSVC | \
HWY_BROKEN_AVX3_DL_ZEN4 | HWY_BROKEN_AVX3_SPR | \
HWY_BROKEN_ARM7_BIG_ENDIAN | HWY_BROKEN_ARM7_WITHOUT_VFP4 | \
HWY_BROKEN_NEON_BF16 | HWY_BROKEN_SVE | HWY_BROKEN_PPC10 | \
HWY_BROKEN_PPC_32BIT | HWY_BROKEN_RVV)
#endif // HWY_BROKEN_TARGETS
// Enabled means not disabled nor blocklisted.
#define HWY_ENABLED(targets) \
((targets) & ~((HWY_DISABLED_TARGETS) | (HWY_BROKEN_TARGETS)))
// Opt-out for EMU128 (affected by a GCC bug on multiple arches, fixed in 12.3:
// see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106322). An issue still
// remains with 13.2, see #1683. This is separate from HWY_BROKEN_TARGETS
// because it affects the fallback target, which must always be enabled. If 1,
// we instead choose HWY_SCALAR even without HWY_COMPILE_ONLY_SCALAR being set.
#if !defined(HWY_BROKEN_EMU128) // allow overriding
#if (HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1400) || \
defined(HWY_NO_LIBCXX)
#define HWY_BROKEN_EMU128 1
#else
#define HWY_BROKEN_EMU128 0
#endif
#endif // HWY_BROKEN_EMU128
//------------------------------------------------------------------------------
// Detect baseline targets using predefined macros
// Baseline means the targets for which the compiler is allowed to generate
// instructions, implying the target CPU would have to support them. This does
// not take the blocklist into account.
#if defined(HWY_COMPILE_ONLY_SCALAR) || HWY_BROKEN_EMU128
#define HWY_BASELINE_SCALAR HWY_SCALAR
#else
#define HWY_BASELINE_SCALAR HWY_EMU128
#endif
// Also check HWY_ARCH to ensure that simulating unknown platforms ends up with
// HWY_TARGET == HWY_BASELINE_SCALAR.
#if HWY_ARCH_WASM && defined(__wasm_simd128__)
#if defined(HWY_WANT_WASM2)
#define HWY_BASELINE_WASM HWY_WASM_EMU256
#else
#define HWY_BASELINE_WASM HWY_WASM
#endif // HWY_WANT_WASM2
#else
#define HWY_BASELINE_WASM 0
#endif
// GCC or Clang.
#if HWY_ARCH_PPC && HWY_COMPILER_GCC && defined(__ALTIVEC__) && \
defined(__VSX__) && defined(__POWER8_VECTOR__) && \
(defined(__CRYPTO__) || defined(HWY_DISABLE_PPC8_CRYPTO))
#define HWY_BASELINE_PPC8 HWY_PPC8
#else
#define HWY_BASELINE_PPC8 0
#endif
#if HWY_BASELINE_PPC8 != 0 && defined(__POWER9_VECTOR__)
#define HWY_BASELINE_PPC9 HWY_PPC9
#else
#define HWY_BASELINE_PPC9 0
#endif
#if HWY_BASELINE_PPC9 != 0 && \
(defined(_ARCH_PWR10) || defined(__POWER10_VECTOR__))
#define HWY_BASELINE_PPC10 HWY_PPC10
#else
#define HWY_BASELINE_PPC10 0
#endif
#if HWY_ARCH_S390X && defined(__VEC__) && defined(__ARCH__) && __ARCH__ >= 12
#define HWY_BASELINE_Z14 HWY_Z14
#else
#define HWY_BASELINE_Z14 0
#endif
#if HWY_BASELINE_Z14 && __ARCH__ >= 13
#define HWY_BASELINE_Z15 HWY_Z15
#else
#define HWY_BASELINE_Z15 0
#endif
#define HWY_BASELINE_SVE2 0
#define HWY_BASELINE_SVE 0
#define HWY_BASELINE_NEON 0
#if HWY_ARCH_ARM
// Also check compiler version as done for HWY_ATTAINABLE_SVE2 because the
// static target (influenced here) must be one of the attainable targets.
#if defined(__ARM_FEATURE_SVE2) && \
(HWY_COMPILER_CLANG >= 1400 || HWY_COMPILER_GCC_ACTUAL >= 1200)
#undef HWY_BASELINE_SVE2 // was 0, will be re-defined
// If user specified -msve-vector-bits=128, they assert the vector length is
// 128 bits and we should use the HWY_SVE2_128 (more efficient for some ops).
#if defined(__ARM_FEATURE_SVE_BITS) && __ARM_FEATURE_SVE_BITS == 128
#define HWY_BASELINE_SVE2 HWY_SVE2_128
// Otherwise we're not sure what the vector length will be. The baseline must be
// unconditionally valid, so we can only assume HWY_SVE2. However, when running
// on a CPU with 128-bit vectors, user code that supports dynamic dispatch will
// still benefit from HWY_SVE2_128 because we add it to HWY_ATTAINABLE_TARGETS.
#else
#define HWY_BASELINE_SVE2 HWY_SVE2
#endif // __ARM_FEATURE_SVE_BITS
#endif // __ARM_FEATURE_SVE2
#if defined(__ARM_FEATURE_SVE) && \
(HWY_COMPILER_CLANG >= 900 || HWY_COMPILER_GCC_ACTUAL >= 800)
#undef HWY_BASELINE_SVE // was 0, will be re-defined
// See above. If user-specified vector length matches our optimization, use it.
#if defined(__ARM_FEATURE_SVE_BITS) && __ARM_FEATURE_SVE_BITS == 256
#define HWY_BASELINE_SVE HWY_SVE_256
#else
#define HWY_BASELINE_SVE HWY_SVE
#endif // __ARM_FEATURE_SVE_BITS
#endif // __ARM_FEATURE_SVE
// GCC 4.5.4 only defines __ARM_NEON__; 5.4 defines both.
#if defined(__ARM_NEON__) || defined(__ARM_NEON)
#undef HWY_BASELINE_NEON
#if defined(__ARM_FEATURE_AES) && \
defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && \
defined(__ARM_FEATURE_DOTPROD) && \
defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)
#define HWY_BASELINE_NEON HWY_ALL_NEON
#elif defined(__ARM_FEATURE_AES)
#define HWY_BASELINE_NEON (HWY_NEON_WITHOUT_AES | HWY_NEON)
#else
#define HWY_BASELINE_NEON (HWY_NEON_WITHOUT_AES)
#endif // __ARM_FEATURE*
#endif // __ARM_NEON
#endif // HWY_ARCH_ARM
// Special handling for MSVC because it has fewer predefined macros:
#if HWY_COMPILER_MSVC
#if HWY_ARCH_X86_32
#if _M_IX86_FP >= 2
#define HWY_CHECK_SSE2 1
#else
#define HWY_CHECK_SSE2 0
#endif
#elif HWY_ARCH_X86_64
#define HWY_CHECK_SSE2 1
#else
#define HWY_CHECK_SSE2 0
#endif
// 1) We can only be sure SSSE3/SSE4 are enabled if AVX is:
// https://stackoverflow.com/questions/18563978/.
#if defined(__AVX__)
#define HWY_CHECK_SSSE3 1
#define HWY_CHECK_SSE4 1
#else
#define HWY_CHECK_SSSE3 0
#define HWY_CHECK_SSE4 0
#endif
// 2) Cannot check for PCLMUL/AES and BMI2/FMA/F16C individually; we assume
// PCLMUL/AES are available if SSE4 is, and BMI2/FMA/F16C if AVX2 is.
#define HWY_CHECK_PCLMUL_AES 1
#define HWY_CHECK_BMI2_FMA 1
#define HWY_CHECK_F16C 1
#else // non-MSVC
#if defined(__SSE2__)
#define HWY_CHECK_SSE2 1
#else
#define HWY_CHECK_SSE2 0
#endif
#if defined(__SSSE3__)
#define HWY_CHECK_SSSE3 1
#else
#define HWY_CHECK_SSSE3 0
#endif
#if defined(__SSE4_1__) && defined(__SSE4_2__)
#define HWY_CHECK_SSE4 1
#else
#define HWY_CHECK_SSE4 0
#endif
// If these are disabled, they should not gate the availability of SSE4/AVX2.
#if defined(HWY_DISABLE_PCLMUL_AES) || (defined(__PCLMUL__) && defined(__AES__))
#define HWY_CHECK_PCLMUL_AES 1
#else
#define HWY_CHECK_PCLMUL_AES 0
#endif
#if defined(HWY_DISABLE_BMI2_FMA) || (defined(__BMI2__) && defined(__FMA__))
#define HWY_CHECK_BMI2_FMA 1
#else
#define HWY_CHECK_BMI2_FMA 0
#endif
#if defined(HWY_DISABLE_F16C) || defined(__F16C__)
#define HWY_CHECK_F16C 1
#else
#define HWY_CHECK_F16C 0
#endif
#endif // non-MSVC
#if HWY_ARCH_X86 && (HWY_WANT_SSE2 || HWY_CHECK_SSE2)
#define HWY_BASELINE_SSE2 HWY_SSE2
#else
#define HWY_BASELINE_SSE2 0
#endif
#if HWY_ARCH_X86 && (HWY_WANT_SSSE3 || HWY_CHECK_SSSE3)
#define HWY_BASELINE_SSSE3 HWY_SSSE3
#else
#define HWY_BASELINE_SSSE3 0
#endif
#if HWY_ARCH_X86 && (HWY_WANT_SSE4 || (HWY_CHECK_SSE4 && HWY_CHECK_PCLMUL_AES))
#define HWY_BASELINE_SSE4 HWY_SSE4
#else
#define HWY_BASELINE_SSE4 0
#endif
#if HWY_BASELINE_SSE4 != 0 && HWY_CHECK_BMI2_FMA && HWY_CHECK_F16C && \
defined(__AVX2__)
#define HWY_BASELINE_AVX2 HWY_AVX2
#else
#define HWY_BASELINE_AVX2 0
#endif
// Require everything in AVX2 plus AVX-512 flags (also set by MSVC)
#if HWY_BASELINE_AVX2 != 0 && defined(__AVX512F__) && defined(__AVX512BW__) && \
defined(__AVX512DQ__) && defined(__AVX512VL__)
#define HWY_BASELINE_AVX3 HWY_AVX3
#else
#define HWY_BASELINE_AVX3 0
#endif
// TODO(janwas): not yet known whether these will be set by MSVC
#if HWY_BASELINE_AVX3 != 0 && defined(__AVX512VNNI__) && defined(__VAES__) && \
defined(__VPCLMULQDQ__) && defined(__AVX512VBMI__) && \
defined(__AVX512VBMI2__) && defined(__AVX512VPOPCNTDQ__) && \
defined(__AVX512BITALG__)
#define HWY_BASELINE_AVX3_DL HWY_AVX3_DL
#else
#define HWY_BASELINE_AVX3_DL 0
#endif
// The ZEN4-optimized AVX3 target is numerically lower than AVX3_DL and is thus
// considered better. Do not enable it unless the user explicitly requests it -
// we do not want to choose the ZEN4 path on Intel because it could be slower.
#if defined(HWY_WANT_AVX3_ZEN4) && HWY_BASELINE_AVX3_DL != 0
#define HWY_BASELINE_AVX3_ZEN4 HWY_AVX3_ZEN4
#else
#define HWY_BASELINE_AVX3_ZEN4 0
#endif
#if HWY_BASELINE_AVX3_DL != 0 && defined(__AVX512BF16__) && \
defined(__AVX512FP16__)
#define HWY_BASELINE_AVX3_SPR HWY_AVX3_SPR
#else
#define HWY_BASELINE_AVX3_SPR 0
#endif
// RVV requires intrinsics 0.11 or later, see #1156.
#if HWY_ARCH_RISCV && defined(__riscv_v_intrinsic) && \
__riscv_v_intrinsic >= 11000
#define HWY_BASELINE_RVV HWY_RVV
#else
#define HWY_BASELINE_RVV 0
#endif
// Allow the user to override this without any guarantee of success.
#ifndef HWY_BASELINE_TARGETS
#define HWY_BASELINE_TARGETS \
(HWY_BASELINE_SCALAR | HWY_BASELINE_WASM | HWY_BASELINE_PPC8 | \
HWY_BASELINE_PPC9 | HWY_BASELINE_PPC10 | HWY_BASELINE_Z14 | \
HWY_BASELINE_Z15 | HWY_BASELINE_SVE2 | HWY_BASELINE_SVE | \
HWY_BASELINE_NEON | HWY_BASELINE_SSE2 | HWY_BASELINE_SSSE3 | \
HWY_BASELINE_SSE4 | HWY_BASELINE_AVX2 | HWY_BASELINE_AVX3 | \
HWY_BASELINE_AVX3_DL | HWY_BASELINE_AVX3_ZEN4 | HWY_BASELINE_AVX3_SPR | \
HWY_BASELINE_RVV)
#endif // HWY_BASELINE_TARGETS
//------------------------------------------------------------------------------
// Choose target for static dispatch
#define HWY_ENABLED_BASELINE HWY_ENABLED(HWY_BASELINE_TARGETS)
#if HWY_ENABLED_BASELINE == 0
#error "At least one baseline target must be defined and enabled"
#endif
// Best baseline, used for static dispatch. This is the least-significant 1-bit
// within HWY_ENABLED_BASELINE and lower bit values imply "better".
#define HWY_STATIC_TARGET (HWY_ENABLED_BASELINE & -HWY_ENABLED_BASELINE)
// Start by assuming static dispatch. If we later use dynamic dispatch, this
// will be defined to other targets during the multiple-inclusion, and finally
// return to the initial value. Defining this outside begin/end_target ensures
// inl headers successfully compile by themselves (required by Bazel).
#define HWY_TARGET HWY_STATIC_TARGET
//------------------------------------------------------------------------------
// Choose targets for dynamic dispatch according to one of four policies
#if 1 < (defined(HWY_COMPILE_ONLY_SCALAR) + defined(HWY_COMPILE_ONLY_EMU128) + \
defined(HWY_COMPILE_ONLY_STATIC))
#error "Can only define one of HWY_COMPILE_ONLY_{SCALAR|EMU128|STATIC} - bug?"
#endif
// Defining one of HWY_COMPILE_ONLY_* will trump HWY_COMPILE_ALL_ATTAINABLE.
#ifndef HWY_HAVE_AUXV // allow override
#ifdef TOOLCHAIN_MISS_SYS_AUXV_H
#define HWY_HAVE_AUXV 0 // CMake failed to find the header
// glibc 2.16 added auxv, but checking for that requires features.h, and we do
// not want to include system headers here. Instead check for the header
// directly, which has been supported at least since GCC 5.4 and Clang 3.
#elif defined(__has_include) // note: wrapper macro fails on Clang ~17
// clang-format off
#if __has_include(<sys/auxv.h>)
// clang-format on
#define HWY_HAVE_AUXV 1 // header present
#else
#define HWY_HAVE_AUXV 0 // header not present
#endif // __has_include
#else // compiler lacks __has_include
#define HWY_HAVE_AUXV 0
#endif
#endif // HWY_HAVE_AUXV
#ifndef HWY_HAVE_RUNTIME_DISPATCH_RVV // allow override
// The riscv_vector.h in Clang 16-18 requires compiler flags, and 19 still has
// some missing intrinsics, see
// https://github.com/llvm/llvm-project/issues/56592. GCC 13.3 also has an
// #error check, whereas 14.1 fails with "argument type 'vuint16m8_t' requires
// the V ISA extension": https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115325.
#if HWY_ARCH_RISCV && HWY_COMPILER_CLANG >= 1900 && 0
#define HWY_HAVE_RUNTIME_DISPATCH_RVV 1
#else
#define HWY_HAVE_RUNTIME_DISPATCH_RVV 0
#endif
#endif // HWY_HAVE_RUNTIME_DISPATCH_RVV
#ifndef HWY_HAVE_RUNTIME_DISPATCH_APPLE // allow override
#if HWY_ARCH_ARM_A64 && HWY_OS_APPLE && \
(HWY_COMPILER_GCC_ACTUAL || HWY_COMPILER_CLANG >= 1700)
#define HWY_HAVE_RUNTIME_DISPATCH_APPLE 1
#else
#define HWY_HAVE_RUNTIME_DISPATCH_APPLE 0
#endif
#endif // HWY_HAVE_RUNTIME_DISPATCH_APPLE
#ifndef HWY_HAVE_RUNTIME_DISPATCH_LINUX // allow override
#if (HWY_ARCH_ARM || HWY_ARCH_PPC || HWY_ARCH_S390X) && HWY_OS_LINUX && \
(HWY_COMPILER_GCC_ACTUAL || HWY_COMPILER_CLANG >= 1700) && HWY_HAVE_AUXV
#define HWY_HAVE_RUNTIME_DISPATCH_LINUX 1
#else
#define HWY_HAVE_RUNTIME_DISPATCH_LINUX 0
#endif
#endif // HWY_HAVE_RUNTIME_DISPATCH_LINUX
// Allow opting out, and without a guarantee of success, opting-in.
#ifndef HWY_HAVE_RUNTIME_DISPATCH
// Clang, GCC and MSVC allow OS-independent runtime dispatch on x86.
#if HWY_ARCH_X86 || HWY_HAVE_RUNTIME_DISPATCH_RVV || \
HWY_HAVE_RUNTIME_DISPATCH_APPLE || HWY_HAVE_RUNTIME_DISPATCH_LINUX
#define HWY_HAVE_RUNTIME_DISPATCH 1
#else
#define HWY_HAVE_RUNTIME_DISPATCH 0
#endif
#endif // HWY_HAVE_RUNTIME_DISPATCH
// AVX3_DL is not widely available yet. To reduce code size and compile time,
// only include it in the set of attainable targets (for dynamic dispatch) if
// the user opts in, OR it is in the baseline (we check whether enabled below).
#if defined(HWY_WANT_AVX3_DL) || (HWY_BASELINE_TARGETS & HWY_AVX3_DL)
#define HWY_ATTAINABLE_AVX3_DL (HWY_AVX3_DL)
#else
#define HWY_ATTAINABLE_AVX3_DL 0
#endif
#if HWY_ARCH_ARM_A64 && HWY_HAVE_RUNTIME_DISPATCH
#define HWY_ATTAINABLE_NEON HWY_ALL_NEON
#elif HWY_ARCH_ARM // static dispatch, or HWY_ARCH_ARM_V7
#define HWY_ATTAINABLE_NEON (HWY_BASELINE_NEON)
#else
#define HWY_ATTAINABLE_NEON 0
#endif
#if HWY_ARCH_ARM_A64 && \
(HWY_COMPILER_CLANG >= 900 || HWY_COMPILER_GCC_ACTUAL >= 800) && \
(HWY_HAVE_RUNTIME_DISPATCH || \
(HWY_ENABLED_BASELINE & (HWY_SVE | HWY_SVE_256)))
#define HWY_ATTAINABLE_SVE (HWY_SVE | HWY_SVE_256)
#else
#define HWY_ATTAINABLE_SVE 0
#endif
#if HWY_ARCH_ARM_A64 && \
(HWY_COMPILER_CLANG >= 1400 || HWY_COMPILER_GCC_ACTUAL >= 1200) && \
(HWY_HAVE_RUNTIME_DISPATCH || \
(HWY_ENABLED_BASELINE & (HWY_SVE2 | HWY_SVE2_128)))
#define HWY_ATTAINABLE_SVE2 (HWY_SVE2 | HWY_SVE2_128)
#else
#define HWY_ATTAINABLE_SVE2 0
#endif
#if HWY_ARCH_PPC && defined(__ALTIVEC__) && \
(!HWY_COMPILER_CLANG || HWY_BASELINE_PPC8 != 0)
#if (HWY_BASELINE_PPC9 | HWY_BASELINE_PPC10) && \
!defined(HWY_SKIP_NON_BEST_BASELINE)
// On POWER with -m flags, we get compile errors (#1707) for targets older than
// the baseline specified via -m, so only generate the static target and better.
// Note that some Linux distros actually do set POWER9 as the baseline.
// This works by skipping case 3 below, so case 4 is reached.
#define HWY_SKIP_NON_BEST_BASELINE
#endif
#define HWY_ATTAINABLE_PPC (HWY_PPC8 | HWY_PPC9 | HWY_PPC10)
#else
#define HWY_ATTAINABLE_PPC 0
#endif
#if HWY_ARCH_S390X && HWY_BASELINE_Z14 != 0
#define HWY_ATTAINABLE_S390X (HWY_Z14 | HWY_Z15)
#else
#define HWY_ATTAINABLE_S390X 0
#endif
#if HWY_ARCH_RISCV && HWY_HAVE_RUNTIME_DISPATCH
#define HWY_ATTAINABLE_RISCV HWY_RVV
#else
#define HWY_ATTAINABLE_RISCV HWY_BASELINE_RVV
#endif
#ifndef HWY_ATTAINABLE_TARGETS_X86 // allow override
#if HWY_COMPILER_MSVC && defined(HWY_SLOW_MSVC)
// Fewer targets for faster builds.
#define HWY_ATTAINABLE_TARGETS_X86 \
HWY_ENABLED(HWY_BASELINE_SCALAR | HWY_STATIC_TARGET | HWY_AVX2)
#else // !HWY_COMPILER_MSVC
#define HWY_ATTAINABLE_TARGETS_X86 \
HWY_ENABLED(HWY_BASELINE_SCALAR | HWY_SSE2 | HWY_SSSE3 | HWY_SSE4 | \
HWY_AVX2 | HWY_AVX3 | HWY_ATTAINABLE_AVX3_DL | HWY_AVX3_ZEN4 | \
HWY_AVX3_SPR)
#endif // !HWY_COMPILER_MSVC
#endif // HWY_ATTAINABLE_TARGETS_X86
// Attainable means enabled and the compiler allows intrinsics (even when not
// allowed to autovectorize). Used in 3 and 4.
#if HWY_ARCH_X86
#define HWY_ATTAINABLE_TARGETS HWY_ATTAINABLE_TARGETS_X86
#elif HWY_ARCH_ARM
#define HWY_ATTAINABLE_TARGETS \
HWY_ENABLED(HWY_BASELINE_SCALAR | HWY_ATTAINABLE_NEON | HWY_ATTAINABLE_SVE | \
HWY_ATTAINABLE_SVE2)
#elif HWY_ARCH_PPC
#define HWY_ATTAINABLE_TARGETS \
HWY_ENABLED(HWY_BASELINE_SCALAR | HWY_ATTAINABLE_PPC)
#elif HWY_ARCH_S390X
#define HWY_ATTAINABLE_TARGETS \
HWY_ENABLED(HWY_BASELINE_SCALAR | HWY_ATTAINABLE_S390X)
#elif HWY_ARCH_RISCV
#define HWY_ATTAINABLE_TARGETS \
HWY_ENABLED(HWY_BASELINE_SCALAR | HWY_ATTAINABLE_RISCV)
#else
#define HWY_ATTAINABLE_TARGETS (HWY_ENABLED_BASELINE)
#endif // HWY_ARCH_*
// 1) For older compilers: avoid SIMD intrinsics, but still support all ops.
#if defined(HWY_COMPILE_ONLY_EMU128) && !HWY_BROKEN_EMU128
#undef HWY_STATIC_TARGET
#define HWY_STATIC_TARGET HWY_EMU128 // override baseline
#define HWY_TARGETS HWY_EMU128
// 1b) HWY_SCALAR is less capable than HWY_EMU128 (which supports all ops), but
// we currently still support it for backwards compatibility.
#elif defined(HWY_COMPILE_ONLY_SCALAR) || \
(defined(HWY_COMPILE_ONLY_EMU128) && HWY_BROKEN_EMU128)
#undef HWY_STATIC_TARGET
#define HWY_STATIC_TARGET HWY_SCALAR // override baseline
#define HWY_TARGETS HWY_SCALAR
// 2) For forcing static dispatch without code changes (removing HWY_EXPORT)
#elif defined(HWY_COMPILE_ONLY_STATIC)
#define HWY_TARGETS HWY_STATIC_TARGET
// 3) For tests: include all attainable targets (in particular: scalar)
#elif (defined(HWY_COMPILE_ALL_ATTAINABLE) || defined(HWY_IS_TEST)) && \
!defined(HWY_SKIP_NON_BEST_BASELINE)
#define HWY_TARGETS HWY_ATTAINABLE_TARGETS
// 4) Default: attainable WITHOUT non-best baseline. This reduces code size by
// excluding superseded targets, in particular scalar. Note: HWY_STATIC_TARGET
// may be 2^62 (HWY_SCALAR), so we must not left-shift/add it. Subtracting one
// sets all lower bits (better targets), then we also include the static target.
#else
#define HWY_TARGETS \
(HWY_ATTAINABLE_TARGETS & ((HWY_STATIC_TARGET - 1LL) | HWY_STATIC_TARGET))
#endif // target policy
// HWY_ONCE and the multiple-inclusion mechanism rely on HWY_STATIC_TARGET being
// one of the dynamic targets. This also implies HWY_TARGETS != 0 and
// (HWY_TARGETS & HWY_ENABLED_BASELINE) != 0.
#if (HWY_TARGETS & HWY_STATIC_TARGET) == 0
#error "Logic error: best baseline should be included in dynamic targets"
#endif
#endif // HIGHWAY_HWY_DETECT_TARGETS_H_

253
deps/v8/third_party/highway/src/hwy/examples/benchmark.cc vendored Normal file
View File

@ -0,0 +1,253 @@
// Copyright 2019 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdio.h>
#include <stdlib.h> // abort
#include <cmath> // std::abs
#include <memory>
#include <numeric> // std::iota, std::inner_product
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/examples/benchmark.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// Must come after foreach_target.h to avoid redefinition errors.
#include "hwy/aligned_allocator.h"
#include "hwy/highway.h"
#include "hwy/nanobenchmark.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
// These templates are not found via ADL.
#if HWY_TARGET != HWY_SCALAR
using hwy::HWY_NAMESPACE::CombineShiftRightLanes;
#endif
class TwoArray {
public:
// Must be a multiple of the vector lane count * 8.
static size_t NumItems() { return 3456; }
TwoArray()
: a_(AllocateAligned<float>(NumItems() * 2)), b_(a_.get() + NumItems()) {
// = 1, but compiler doesn't know
const float init = static_cast<float>(Unpredictable1());
std::iota(a_.get(), a_.get() + NumItems(), init);
std::iota(b_, b_ + NumItems(), init);
}
protected:
AlignedFreeUniquePtr<float[]> a_;
float* b_;
};
// Measures durations, verifies results, prints timings.
template <class Benchmark>
void RunBenchmark(const char* caption) {
printf("%10s: ", caption);
const size_t kNumInputs = 1;
const size_t num_items = Benchmark::NumItems() * size_t(Unpredictable1());
const FuncInput inputs[kNumInputs] = {num_items};
Result results[kNumInputs];
Benchmark benchmark;
Params p;
p.verbose = false;
p.max_evals = 7;
p.target_rel_mad = 0.002;
const size_t num_results = MeasureClosure(
[&benchmark](const FuncInput input) { return benchmark(input); }, inputs,
kNumInputs, results, p);
if (num_results != kNumInputs) {
fprintf(stderr, "MeasureClosure failed.\n");
}
benchmark.Verify(num_items);
for (size_t i = 0; i < num_results; ++i) {
const double cycles_per_item =
results[i].ticks / static_cast<double>(results[i].input);
const double mad = results[i].variability * cycles_per_item;
printf("%6d: %6.3f (+/- %5.3f)\n", static_cast<int>(results[i].input),
cycles_per_item, mad);
}
}
void Intro() {
const float in[16] = {1, 2, 3, 4, 5, 6};
float out[16];
const ScalableTag<float> d; // largest possible vector
for (size_t i = 0; i < 16; i += Lanes(d)) {
const auto vec = LoadU(d, in + i); // no alignment requirement
auto result = Mul(vec, vec);
result = Add(result, result); // can update if not const
StoreU(result, d, out + i);
}
printf("\nF(x)->2*x^2, F(%.0f) = %.1f\n", in[2], out[2]);
}
// BEGINNER: dot product
// 0.4 cyc/float = bronze, 0.25 = silver, 0.15 = gold!
class BenchmarkDot : public TwoArray {
public:
BenchmarkDot() : dot_{-1.0f} {}
FuncOutput operator()(const size_t num_items) {
const ScalableTag<float> d;
const size_t N = Lanes(d);
using V = decltype(Zero(d));
// Compiler doesn't make independent sum* accumulators, so unroll manually.
// We cannot use an array because V might be a sizeless type. For reasonable
// code, we unroll 4x, but 8x might help (2 FMA ports * 4 cycle latency).
V sum0 = Zero(d);
V sum1 = Zero(d);
V sum2 = Zero(d);
V sum3 = Zero(d);
const float* const HWY_RESTRICT pa = &a_[0];
const float* const HWY_RESTRICT pb = b_;
for (size_t i = 0; i < num_items; i += 4 * N) {
const auto a0 = Load(d, pa + i + 0 * N);
const auto b0 = Load(d, pb + i + 0 * N);
sum0 = MulAdd(a0, b0, sum0);
const auto a1 = Load(d, pa + i + 1 * N);
const auto b1 = Load(d, pb + i + 1 * N);
sum1 = MulAdd(a1, b1, sum1);
const auto a2 = Load(d, pa + i + 2 * N);
const auto b2 = Load(d, pb + i + 2 * N);
sum2 = MulAdd(a2, b2, sum2);
const auto a3 = Load(d, pa + i + 3 * N);
const auto b3 = Load(d, pb + i + 3 * N);
sum3 = MulAdd(a3, b3, sum3);
}
// Reduction tree: sum of all accumulators by pairs into sum0.
sum0 = Add(sum0, sum1);
sum2 = Add(sum2, sum3);
sum0 = Add(sum0, sum2);
// Remember to store the result in `dot_` for verification; see `Verify`.
dot_ = ReduceSum(d, sum0);
// Return the result so that the benchmarking framework can ensure that the
// computation is not elided by the compiler.
return static_cast<FuncOutput>(dot_);
}
void Verify(size_t num_items) {
if (dot_ == -1.0f) {
fprintf(stderr, "Dot: must call Verify after benchmark");
abort();
}
const float expected =
std::inner_product(a_.get(), a_.get() + num_items, b_, 0.0f);
const float rel_err = std::abs(expected - dot_) / expected;
if (rel_err > 1.1E-6f) {
fprintf(stderr, "Dot: expected %e actual %e (%e)\n", expected, dot_,
rel_err);
abort();
}
}
private:
float dot_; // for Verify
};
// INTERMEDIATE: delta coding
// 1.0 cycles/float = bronze, 0.7 = silver, 0.4 = gold!
struct BenchmarkDelta : public TwoArray {
FuncOutput operator()(const size_t num_items) const {
#if HWY_TARGET == HWY_SCALAR
b_[0] = a_[0];
for (size_t i = 1; i < num_items; ++i) {
b_[i] = a_[i] - a_[i - 1];
}
#elif HWY_CAP_GE256
// Larger vectors are split into 128-bit blocks, easiest to use the
// unaligned load support to shift between them.
const ScalableTag<float> df;
const size_t N = Lanes(df);
size_t i;
b_[0] = a_[0];
for (i = 1; i < N; ++i) {
b_[i] = a_[i] - a_[i - 1];
}
for (; i < num_items; i += N) {
const auto a = Load(df, &a_[i]);
const auto shifted = LoadU(df, &a_[i - 1]);
Store(a - shifted, df, &b_[i]);
}
#else // 128-bit
// Slightly better than unaligned loads
const HWY_CAPPED(float, 4) df;
const size_t N = Lanes(df);
size_t i;
b_[0] = a_[0];
for (i = 1; i < N; ++i) {
b_[i] = a_[i] - a_[i - 1];
}
auto prev = Load(df, &a_[0]);
for (; i < num_items; i += Lanes(df)) {
const auto a = Load(df, &a_[i]);
const auto shifted = CombineShiftRightLanes<3>(df, a, prev);
prev = a;
Store(Sub(a, shifted), df, &b_[i]);
}
#endif
return static_cast<FuncOutput>(b_[num_items - 1]);
}
void Verify(size_t num_items) {
for (size_t i = 0; i < num_items; ++i) {
const float expected = (i == 0) ? a_[0] : a_[i] - a_[i - 1];
const float err = std::abs(expected - b_[i]);
if (err > 1E-6f) {
fprintf(stderr, "Delta: expected %e, actual %e\n", expected, b_[i]);
}
}
}
};
void RunBenchmarks() {
Intro();
printf("------------------------ %s\n", TargetName(HWY_TARGET));
RunBenchmark<BenchmarkDot>("dot");
RunBenchmark<BenchmarkDelta>("delta");
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
HWY_EXPORT(RunBenchmarks);
void Run() {
for (int64_t target : SupportedAndGeneratedTargets()) {
SetSupportedTargetsForTest(target);
HWY_DYNAMIC_DISPATCH(RunBenchmarks)();
}
SetSupportedTargetsForTest(0); // Reset the mask afterwards.
}
} // namespace hwy
int main(int /*argc*/, char** /*argv*/) {
hwy::Run();
return 0;
}
#endif // HWY_ONCE

95
deps/v8/third_party/highway/src/hwy/examples/profiler_example.cc vendored Normal file
View File

@ -0,0 +1,95 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cmath>
#include "hwy/base.h" // Abort
#include "hwy/profiler.h"
#include "hwy/timer.h"
namespace hwy {
namespace {
void Spin(const double min_time) {
const double t0 = hwy::platform::Now();
for (;;) {
const double elapsed = hwy::platform::Now() - t0;
if (elapsed > min_time) {
break;
}
}
}
void Spin10() {
PROFILER_FUNC;
Spin(10E-6);
}
void Spin20() {
PROFILER_FUNC;
Spin(20E-6);
}
void Spin3060() {
{
PROFILER_ZONE("spin30");
Spin(30E-6);
}
{
PROFILER_ZONE("spin60");
Spin(60E-6);
}
}
void Level3() {
PROFILER_FUNC;
for (int rep = 0; rep < 10; ++rep) {
double total = 0.0;
for (int i = 0; i < 100 - rep; ++i) {
total += std::pow(0.9, i);
}
if (std::abs(total - 9.999) > 1E-2) {
HWY_ABORT("unexpected total %f", total);
}
}
}
void Level2() {
PROFILER_FUNC;
Level3();
}
void Level1() {
PROFILER_FUNC;
Level2();
}
void ProfilerExample() {
{
PROFILER_FUNC;
Spin10();
Spin20();
Spin3060();
Level1();
}
PROFILER_PRINT_RESULTS();
}
} // namespace
} // namespace hwy
int main(int /*argc*/, char* /*argv*/[]) {
hwy::ProfilerExample();
return 0;
}

64
deps/v8/third_party/highway/src/hwy/examples/skeleton-inl.h vendored Normal file
View File

@ -0,0 +1,64 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Demo of functions that might be called from multiple SIMD modules (either
// other -inl.h files, or a .cc file between begin/end_target-inl). This is
// optional - all SIMD code can reside in .cc files. However, this allows
// splitting code into different files while still inlining instead of requiring
// calling through function pointers.
// Per-target include guard. This is only required when using dynamic dispatch,
// i.e. including foreach_target.h. For static dispatch, a normal include
// guard would be fine because the header is only compiled once.
#if defined(HIGHWAY_HWY_EXAMPLES_SKELETON_INL_H_) == defined(HWY_TARGET_TOGGLE)
#ifdef HIGHWAY_HWY_EXAMPLES_SKELETON_INL_H_
#undef HIGHWAY_HWY_EXAMPLES_SKELETON_INL_H_
#else
#define HIGHWAY_HWY_EXAMPLES_SKELETON_INL_H_
#endif
// It is fine to #include normal or *-inl headers.
#include "hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace skeleton {
namespace HWY_NAMESPACE {
// Highway ops reside here; ADL does not find templates nor builtins.
namespace hn = hwy::HWY_NAMESPACE;
// Example of a type-agnostic (caller-specified lane type) and width-agnostic
// (uses best available instruction set) function in a header.
//
// Computes x[i] = mul_array[i] * x_array[i] + add_array[i] for i < size.
template <class D, typename T>
HWY_MAYBE_UNUSED void MulAddLoop(const D d, const T* HWY_RESTRICT mul_array,
const T* HWY_RESTRICT add_array,
const size_t size, T* HWY_RESTRICT x_array) {
for (size_t i = 0; i < size; i += hn::Lanes(d)) {
const auto mul = hn::Load(d, mul_array + i);
const auto add = hn::Load(d, add_array + i);
auto x = hn::Load(d, x_array + i);
x = hn::MulAdd(mul, x, add);
hn::Store(x, d, x_array + i);
}
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace skeleton
HWY_AFTER_NAMESPACE();
#endif // include guard

129
deps/v8/third_party/highway/src/hwy/examples/skeleton.cc vendored Normal file
View File

@ -0,0 +1,129 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/examples/skeleton.h"
#include <stdio.h>
// >>>> for dynamic dispatch only, skip if you want static dispatch
// First undef to prevent error when re-included.
#undef HWY_TARGET_INCLUDE
// For dynamic dispatch, specify the name of the current file (unfortunately
// __FILE__ is not reliable) so that foreach_target.h can re-include it.
#define HWY_TARGET_INCLUDE "hwy/examples/skeleton.cc"
// Generates code for each enabled target by re-including this source file.
#include "hwy/foreach_target.h" // IWYU pragma: keep
// <<<< end of dynamic dispatch
// Must come after foreach_target.h to avoid redefinition errors.
#include "hwy/highway.h"
// Optional, can instead add HWY_ATTR to all functions.
HWY_BEFORE_NAMESPACE();
namespace skeleton {
// This namespace name is unique per target, which allows code for multiple
// targets to co-exist in the same translation unit. Required when using dynamic
// dispatch, otherwise optional.
namespace HWY_NAMESPACE {
// Highway ops reside here; ADL does not find templates nor builtins.
namespace hn = hwy::HWY_NAMESPACE;
// Computes log2 by converting to a vector of floats. Compiled once per target.
template <class DF>
HWY_ATTR_NO_MSAN void OneFloorLog2(const DF df,
const uint8_t* HWY_RESTRICT values,
uint8_t* HWY_RESTRICT log2) {
// Type tags for converting to other element types (Rebind = same count).
const hn::RebindToSigned<DF> d32;
const hn::Rebind<uint8_t, DF> d8;
using VI32 = hn::Vec<decltype(d32)>;
const VI32 vi32 = hn::PromoteTo(d32, hn::Load(d8, values));
const VI32 bits = hn::BitCast(d32, hn::ConvertTo(df, vi32));
const VI32 exponent = hn::Sub(hn::ShiftRight<23>(bits), hn::Set(d32, 127));
hn::Store(hn::DemoteTo(d8, exponent), d8, log2);
}
void CodepathDemo() {
// Highway defaults to portability, but per-target codepaths may be selected
// via #if HWY_TARGET == HWY_SSE4 or by testing capability macros:
#if HWY_HAVE_INTEGER64
const char* gather = "Has int64";
#else
const char* gather = "No int64";
#endif
printf("Target %s: %s\n", hwy::TargetName(HWY_TARGET), gather);
}
void FloorLog2(const uint8_t* HWY_RESTRICT values, size_t count,
uint8_t* HWY_RESTRICT log2) {
CodepathDemo();
const hn::ScalableTag<float> df;
const size_t N = hn::Lanes(df);
size_t i = 0;
for (; i + N <= count; i += N) {
OneFloorLog2(df, values + i, log2 + i);
}
for (; i < count; ++i) {
hn::CappedTag<float, 1> d1;
OneFloorLog2(d1, values + i, log2 + i);
}
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace skeleton
HWY_AFTER_NAMESPACE();
// The table of pointers to the various implementations in HWY_NAMESPACE must
// be compiled only once (foreach_target #includes this file multiple times).
// HWY_ONCE is true for only one of these 'compilation passes'.
#if HWY_ONCE
namespace skeleton {
// This macro declares a static array used for dynamic dispatch; it resides in
// the same outer namespace that contains FloorLog2.
HWY_EXPORT(FloorLog2);
// This function is optional and only needed in the case of exposing it in the
// header file. Otherwise using HWY_DYNAMIC_DISPATCH(FloorLog2) in this module
// is equivalent to inlining this function.
HWY_DLLEXPORT void CallFloorLog2(const uint8_t* HWY_RESTRICT in,
const size_t count,
uint8_t* HWY_RESTRICT out) {
// This must reside outside of HWY_NAMESPACE because it references (calls the
// appropriate one from) the per-target implementations there.
// For static dispatch, use HWY_STATIC_DISPATCH.
return HWY_DYNAMIC_DISPATCH(FloorLog2)(in, count, out);
}
HWY_DLLEXPORT void SavedCallFloorLog2(const uint8_t* HWY_RESTRICT in,
const size_t count,
uint8_t* HWY_RESTRICT out) {
const auto ptr = HWY_DYNAMIC_POINTER(FloorLog2);
return ptr(in, count, out);
}
// Optional: anything to compile only once, e.g. non-SIMD implementations of
// public functions provided by this module, can go inside #if HWY_ONCE.
} // namespace skeleton
#endif // HWY_ONCE

38
deps/v8/third_party/highway/src/hwy/examples/skeleton.h vendored Normal file
View File

@ -0,0 +1,38 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Demo interface to target-specific code in skeleton.cc
// Normal header with include guard and namespace.
#ifndef HIGHWAY_HWY_EXAMPLES_SKELETON_H_
#define HIGHWAY_HWY_EXAMPLES_SKELETON_H_
// Platform-specific definitions used for declaring an interface, independent of
// the SIMD instruction set.
#include "hwy/base.h" // HWY_RESTRICT
namespace skeleton {
// Computes base-2 logarithm by converting to float. Supports dynamic dispatch.
HWY_DLLEXPORT void CallFloorLog2(const uint8_t* HWY_RESTRICT in, size_t count,
uint8_t* HWY_RESTRICT out);
// Same, but uses HWY_DYNAMIC_POINTER to save a function pointer and call it.
HWY_DLLEXPORT void SavedCallFloorLog2(const uint8_t* HWY_RESTRICT in,
size_t count, uint8_t* HWY_RESTRICT out);
} // namespace skeleton
#endif // HIGHWAY_HWY_EXAMPLES_SKELETON_H_

149
deps/v8/third_party/highway/src/hwy/examples/skeleton_test.cc vendored Normal file
View File

@ -0,0 +1,149 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Example of unit test for the "skeleton" library.
#include "hwy/examples/skeleton.h"
#include <stdint.h>
#include <stdio.h>
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "examples/skeleton_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
// Must come after foreach_target.h to avoid redefinition errors.
#include "hwy/highway.h"
#include "hwy/nanobenchmark.h" // Unpredictable1
#include "hwy/tests/test_util-inl.h"
// Optional: factor out parts of the implementation into *-inl.h
// (must also come after foreach_target.h to avoid redefinition errors)
#include "hwy/examples/skeleton-inl.h"
HWY_BEFORE_NAMESPACE();
namespace skeleton {
namespace HWY_NAMESPACE {
namespace {
namespace hn = hwy::HWY_NAMESPACE;
// Calls function defined in skeleton.cc.
struct TestFloorLog2 {
template <class T, class DF>
HWY_NOINLINE void operator()(T /*unused*/, DF df) {
const size_t count = 5 * hn::Lanes(df);
auto in = hwy::AllocateAligned<uint8_t>(count);
auto expected = hwy::AllocateAligned<uint8_t>(count);
auto out = hwy::AllocateAligned<uint8_t>(count);
HWY_ASSERT(in && expected && out);
hwy::RandomState rng;
for (size_t i = 0; i < count; ++i) {
expected[i] = Random32(&rng) & 7;
in[i] = static_cast<uint8_t>(1u << expected[i]);
}
CallFloorLog2(in.get(), count, out.get());
int sum = 0;
for (size_t i = 0; i < count; ++i) {
HWY_ASSERT_EQ(expected[i], out[i]);
sum += out[i];
}
for (size_t i = 0; i < count; ++i) {
out[i] = static_cast<uint8_t>(hwy::Unpredictable1());
}
SavedCallFloorLog2(in.get(), count, out.get());
for (size_t i = 0; i < count; ++i) {
HWY_ASSERT_EQ(expected[i], out[i]);
sum += out[i];
}
hwy::PreventElision(sum);
}
};
HWY_NOINLINE void TestAllFloorLog2() {
hn::ForPartialVectors<TestFloorLog2>()(float());
}
// Calls function defined in skeleton-inl.h.
struct TestSumMulAdd {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
hwy::RandomState rng;
const size_t count = 4096;
HWY_ASSERT_EQ(size_t{0}, count % hn::Lanes(d));
auto mul = hwy::AllocateAligned<T>(count);
auto x = hwy::AllocateAligned<T>(count);
auto add = hwy::AllocateAligned<T>(count);
HWY_ASSERT(mul && x && add);
for (size_t i = 0; i < count; ++i) {
mul[i] = hwy::ConvertScalarTo<T>(Random32(&rng) & 0xF);
x[i] = hwy::ConvertScalarTo<T>(Random32(&rng) & 0xFF);
add[i] = hwy::ConvertScalarTo<T>(Random32(&rng) & 0xFF);
}
double expected_sum = 0.0;
for (size_t i = 0; i < count; ++i) {
expected_sum += hwy::ConvertScalarTo<double>(mul[i]) *
hwy::ConvertScalarTo<double>(x[i]) +
hwy::ConvertScalarTo<double>(add[i]);
}
MulAddLoop(d, mul.get(), add.get(), count, x.get());
double vector_sum = 0.0;
for (size_t i = 0; i < count; ++i) {
vector_sum += hwy::ConvertScalarTo<double>(x[i]);
}
if (hwy::IsSame<T, hwy::float16_t>()) {
// The expected value for float16 will vary based on the underlying
// implementation (compiler emulation, ARM ACLE __fp16 vs _Float16, etc).
// In some cases the scalar and vector paths will have different results;
// we check them against known values where possible, else we ignore them.
#if HWY_COMPILER_CLANG && HWY_NEON_HAVE_F16C
HWY_ASSERT_EQ(4344240.0, expected_sum); // Full-width float
HWY_ASSERT_EQ(4344235.0, vector_sum); // __fp16
#endif
return;
}
HWY_ASSERT_EQ(4344240.0, expected_sum);
HWY_ASSERT_EQ(expected_sum, vector_sum);
}
};
HWY_NOINLINE void TestAllSumMulAdd() {
hn::ForFloatTypes(hn::ForPartialVectors<TestSumMulAdd>());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace skeleton
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace skeleton {
namespace {
HWY_BEFORE_TEST(SkeletonTest);
HWY_EXPORT_AND_TEST_P(SkeletonTest, TestAllFloorLog2);
HWY_EXPORT_AND_TEST_P(SkeletonTest, TestAllSumMulAdd);
HWY_AFTER_TEST();
} // namespace
} // namespace skeleton
HWY_TEST_MAIN();
#endif // HWY_ONCE

375
deps/v8/third_party/highway/src/hwy/foreach_target.h vendored Normal file
View File

@ -0,0 +1,375 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_FOREACH_TARGET_H_
#define HIGHWAY_HWY_FOREACH_TARGET_H_
// Re-includes the translation unit zero or more times to compile for any
// targets except HWY_STATIC_TARGET. Defines unique HWY_TARGET each time so that
// highway.h defines the corresponding macro/namespace.
#include "hwy/detect_targets.h"
// *_inl.h may include other headers, which requires include guards to prevent
// repeated inclusion. The guards must be reset after compiling each target, so
// the header is again visible. This is done by flipping HWY_TARGET_TOGGLE,
// defining it if undefined and vice versa. This macro is initially undefined
// so that IDEs don't gray out the contents of each header.
#ifdef HWY_TARGET_TOGGLE
#error "This macro must not be defined outside foreach_target.h"
#endif
#ifdef HWY_HIGHWAY_INCLUDED // highway.h include guard
// Trigger fixup at the bottom of this header.
#define HWY_ALREADY_INCLUDED
// The next highway.h must re-include set_macros-inl.h because the first
// highway.h chose the static target instead of what we will set below.
#undef HWY_SET_MACROS_PER_TARGET
#endif
// Disable HWY_EXPORT in user code until we have generated all targets. Note
// that a subsequent highway.h will not override this definition.
#undef HWY_ONCE
#define HWY_ONCE (0 || HWY_IDE)
// Avoid warnings on #include HWY_TARGET_INCLUDE by hiding them from the IDE;
// also skip if only 1 target defined (no re-inclusion will be necessary).
#if !HWY_IDE && (HWY_TARGETS != HWY_STATIC_TARGET)
#if !defined(HWY_TARGET_INCLUDE)
#error ">1 target enabled => define HWY_TARGET_INCLUDE before foreach_target.h"
#endif
// ------------------------------ HWY_ARCH_X86
#if (HWY_TARGETS & HWY_SSE2) && (HWY_STATIC_TARGET != HWY_SSE2)
#undef HWY_TARGET
#define HWY_TARGET HWY_SSE2
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_SSSE3) && (HWY_STATIC_TARGET != HWY_SSSE3)
#undef HWY_TARGET
#define HWY_TARGET HWY_SSSE3
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_SSE4) && (HWY_STATIC_TARGET != HWY_SSE4)
#undef HWY_TARGET
#define HWY_TARGET HWY_SSE4
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_AVX2) && (HWY_STATIC_TARGET != HWY_AVX2)
#undef HWY_TARGET
#define HWY_TARGET HWY_AVX2
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_AVX3) && (HWY_STATIC_TARGET != HWY_AVX3)
#undef HWY_TARGET
#define HWY_TARGET HWY_AVX3
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_AVX3_DL) && (HWY_STATIC_TARGET != HWY_AVX3_DL)
#undef HWY_TARGET
#define HWY_TARGET HWY_AVX3_DL
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_AVX3_ZEN4) && (HWY_STATIC_TARGET != HWY_AVX3_ZEN4)
#undef HWY_TARGET
#define HWY_TARGET HWY_AVX3_ZEN4
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_AVX3_SPR) && (HWY_STATIC_TARGET != HWY_AVX3_SPR)
#undef HWY_TARGET
#define HWY_TARGET HWY_AVX3_SPR
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
// ------------------------------ HWY_ARCH_ARM
#if (HWY_TARGETS & HWY_NEON_WITHOUT_AES) && \
(HWY_STATIC_TARGET != HWY_NEON_WITHOUT_AES)
#undef HWY_TARGET
#define HWY_TARGET HWY_NEON_WITHOUT_AES
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_NEON) && (HWY_STATIC_TARGET != HWY_NEON)
#undef HWY_TARGET
#define HWY_TARGET HWY_NEON
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_NEON_BF16) && (HWY_STATIC_TARGET != HWY_NEON_BF16)
#undef HWY_TARGET
#define HWY_TARGET HWY_NEON_BF16
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_SVE) && (HWY_STATIC_TARGET != HWY_SVE)
#undef HWY_TARGET
#define HWY_TARGET HWY_SVE
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_SVE2) && (HWY_STATIC_TARGET != HWY_SVE2)
#undef HWY_TARGET
#define HWY_TARGET HWY_SVE2
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_SVE_256) && (HWY_STATIC_TARGET != HWY_SVE_256)
#undef HWY_TARGET
#define HWY_TARGET HWY_SVE_256
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_SVE2_128) && (HWY_STATIC_TARGET != HWY_SVE2_128)
#undef HWY_TARGET
#define HWY_TARGET HWY_SVE2_128
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
// ------------------------------ HWY_ARCH_WASM
#if (HWY_TARGETS & HWY_WASM_EMU256) && (HWY_STATIC_TARGET != HWY_WASM_EMU256)
#undef HWY_TARGET
#define HWY_TARGET HWY_WASM_EMU256
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_WASM) && (HWY_STATIC_TARGET != HWY_WASM)
#undef HWY_TARGET
#define HWY_TARGET HWY_WASM
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
// ------------------------------ HWY_ARCH_PPC
#if (HWY_TARGETS & HWY_PPC8) && (HWY_STATIC_TARGET != HWY_PPC8)
#undef HWY_TARGET
#define HWY_TARGET HWY_PPC8
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_PPC9) && (HWY_STATIC_TARGET != HWY_PPC9)
#undef HWY_TARGET
#define HWY_TARGET HWY_PPC9
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_PPC10) && (HWY_STATIC_TARGET != HWY_PPC10)
#undef HWY_TARGET
#define HWY_TARGET HWY_PPC10
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
// ------------------------------ HWY_ARCH_S390X
#if (HWY_TARGETS & HWY_Z14) && (HWY_STATIC_TARGET != HWY_Z14)
#undef HWY_TARGET
#define HWY_TARGET HWY_Z14
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_Z15) && (HWY_STATIC_TARGET != HWY_Z15)
#undef HWY_TARGET
#define HWY_TARGET HWY_Z15
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
// ------------------------------ HWY_ARCH_RISCV
#if (HWY_TARGETS & HWY_RVV) && (HWY_STATIC_TARGET != HWY_RVV)
#undef HWY_TARGET
#define HWY_TARGET HWY_RVV
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
// ------------------------------ Scalar
#if (HWY_TARGETS & HWY_EMU128) && (HWY_STATIC_TARGET != HWY_EMU128)
#undef HWY_TARGET
#define HWY_TARGET HWY_EMU128
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#if (HWY_TARGETS & HWY_SCALAR) && (HWY_STATIC_TARGET != HWY_SCALAR)
#undef HWY_TARGET
#define HWY_TARGET HWY_SCALAR
#include HWY_TARGET_INCLUDE
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
#endif
#endif // !HWY_IDE && (HWY_TARGETS != HWY_STATIC_TARGET)
// Now that all but the static target have been generated, re-enable HWY_EXPORT.
#undef HWY_ONCE
#define HWY_ONCE 1
// If we re-include once per enabled target, the translation unit's
// implementation would have to be skipped via #if to avoid redefining symbols.
// We instead skip the re-include for HWY_STATIC_TARGET, and generate its
// implementation when resuming compilation of the translation unit.
#undef HWY_TARGET
#define HWY_TARGET HWY_STATIC_TARGET
#ifdef HWY_ALREADY_INCLUDED
// Revert the previous toggle to prevent redefinitions for the static target.
#ifdef HWY_TARGET_TOGGLE
#undef HWY_TARGET_TOGGLE
#else
#define HWY_TARGET_TOGGLE
#endif
// Force re-inclusion of set_macros-inl.h now that HWY_TARGET is restored.
#ifdef HWY_SET_MACROS_PER_TARGET
#undef HWY_SET_MACROS_PER_TARGET
#else
#define HWY_SET_MACROS_PER_TARGET
#endif
#endif
#endif // HIGHWAY_HWY_FOREACH_TARGET_H_

607
deps/v8/third_party/highway/src/hwy/highway.h vendored Normal file
View File

@ -0,0 +1,607 @@
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Main header required before using vector types.
// IWYU pragma: begin_exports
#include "hwy/base.h"
#include "hwy/detect_compiler_arch.h"
#include "hwy/detect_targets.h"
#include "hwy/highway_export.h"
#include "hwy/targets.h"
// IWYU pragma: end_exports
#if HWY_CXX_LANG < 201703L
#define HWY_DISPATCH_MAP 1
#else
#define HWY_DISPATCH_MAP 0
#endif
// This include guard is checked by foreach_target, so avoid the usual _H_
// suffix to prevent copybara from renaming it. NOTE: ops/*-inl.h are included
// after/outside this include guard.
#ifndef HWY_HIGHWAY_INCLUDED
#define HWY_HIGHWAY_INCLUDED
namespace hwy {
//------------------------------------------------------------------------------
// Shorthand for tags (defined in shared-inl.h) used to select overloads.
// Note that ScalableTag<T> is preferred over HWY_FULL, and CappedTag<T, N> over
// HWY_CAPPED(T, N).
// HWY_FULL(T[,LMUL=1]) is a native vector/group. LMUL is the number of
// registers in the group, and is ignored on targets that do not support groups.
#define HWY_FULL1(T) hwy::HWY_NAMESPACE::ScalableTag<T>
#define HWY_FULL2(T, LMUL) \
hwy::HWY_NAMESPACE::ScalableTag<T, hwy::CeilLog2(HWY_MAX(0, LMUL))>
#define HWY_3TH_ARG(arg1, arg2, arg3, ...) arg3
// Workaround for MSVC grouping __VA_ARGS__ into a single argument
#define HWY_FULL_RECOMPOSER(args_with_paren) HWY_3TH_ARG args_with_paren
// Trailing comma avoids -pedantic false alarm
#define HWY_CHOOSE_FULL(...) \
HWY_FULL_RECOMPOSER((__VA_ARGS__, HWY_FULL2, HWY_FULL1, ))
#define HWY_FULL(...) HWY_CHOOSE_FULL(__VA_ARGS__())(__VA_ARGS__)
// Vector of up to MAX_N lanes. It's better to use full vectors where possible.
#define HWY_CAPPED(T, MAX_N) hwy::HWY_NAMESPACE::CappedTag<T, MAX_N>
//------------------------------------------------------------------------------
// Export user functions for static/dynamic dispatch
// Evaluates to 0 inside a translation unit if it is generating anything but the
// static target (the last one if multiple targets are enabled). Used to prevent
// redefinitions of HWY_EXPORT. Unless foreach_target.h is included, we only
// compile once anyway, so this is 1 unless it is or has been included.
#ifndef HWY_ONCE
#define HWY_ONCE 1
#endif
// HWY_STATIC_DISPATCH(FUNC_NAME) is the namespace-qualified FUNC_NAME for
// HWY_STATIC_TARGET (the only defined namespace unless HWY_TARGET_INCLUDE is
// defined), and can be used to deduce the return type of Choose*.
#if HWY_STATIC_TARGET == HWY_SCALAR
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SCALAR::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_EMU128
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_EMU128::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_RVV
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_RVV::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_WASM_EMU256
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM_EMU256::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_WASM
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_NEON_WITHOUT_AES
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON_WITHOUT_AES::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_NEON
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_NEON_BF16
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON_BF16::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_SVE
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_SVE2
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE2::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_SVE_256
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE_256::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_SVE2_128
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE2_128::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_PPC8
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC8::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_PPC9
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC9::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_PPC10
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC10::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_Z14
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_Z14::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_Z15
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_Z15::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_SSE2
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSE2::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_SSSE3
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSSE3::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_SSE4
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSE4::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_AVX2
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX2::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_AVX3
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_AVX3_DL
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_DL::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_AVX3_ZEN4
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_ZEN4::FUNC_NAME
#elif HWY_STATIC_TARGET == HWY_AVX3_SPR
#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_SPR::FUNC_NAME
#endif
// HWY_CHOOSE_*(FUNC_NAME) expands to the function pointer for that target or
// nullptr is that target was not compiled.
#if HWY_TARGETS & HWY_EMU128
#define HWY_CHOOSE_FALLBACK(FUNC_NAME) &N_EMU128::FUNC_NAME
#elif HWY_TARGETS & HWY_SCALAR
#define HWY_CHOOSE_FALLBACK(FUNC_NAME) &N_SCALAR::FUNC_NAME
#else
// When HWY_SCALAR/HWY_EMU128 are not present and other targets were disabled at
// runtime, fall back to the baseline with HWY_STATIC_DISPATCH().
#define HWY_CHOOSE_FALLBACK(FUNC_NAME) &HWY_STATIC_DISPATCH(FUNC_NAME)
#endif
#if HWY_TARGETS & HWY_WASM_EMU256
#define HWY_CHOOSE_WASM_EMU256(FUNC_NAME) &N_WASM_EMU256::FUNC_NAME
#else
#define HWY_CHOOSE_WASM_EMU256(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_WASM
#define HWY_CHOOSE_WASM(FUNC_NAME) &N_WASM::FUNC_NAME
#else
#define HWY_CHOOSE_WASM(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_RVV
#define HWY_CHOOSE_RVV(FUNC_NAME) &N_RVV::FUNC_NAME
#else
#define HWY_CHOOSE_RVV(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_NEON_WITHOUT_AES
#define HWY_CHOOSE_NEON_WITHOUT_AES(FUNC_NAME) &N_NEON_WITHOUT_AES::FUNC_NAME
#else
#define HWY_CHOOSE_NEON_WITHOUT_AES(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_NEON
#define HWY_CHOOSE_NEON(FUNC_NAME) &N_NEON::FUNC_NAME
#else
#define HWY_CHOOSE_NEON(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_NEON_BF16
#define HWY_CHOOSE_NEON_BF16(FUNC_NAME) &N_NEON_BF16::FUNC_NAME
#else
#define HWY_CHOOSE_NEON_BF16(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_SVE
#define HWY_CHOOSE_SVE(FUNC_NAME) &N_SVE::FUNC_NAME
#else
#define HWY_CHOOSE_SVE(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_SVE2
#define HWY_CHOOSE_SVE2(FUNC_NAME) &N_SVE2::FUNC_NAME
#else
#define HWY_CHOOSE_SVE2(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_SVE_256
#define HWY_CHOOSE_SVE_256(FUNC_NAME) &N_SVE_256::FUNC_NAME
#else
#define HWY_CHOOSE_SVE_256(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_SVE2_128
#define HWY_CHOOSE_SVE2_128(FUNC_NAME) &N_SVE2_128::FUNC_NAME
#else
#define HWY_CHOOSE_SVE2_128(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_PPC8
#define HWY_CHOOSE_PPC8(FUNC_NAME) &N_PPC8::FUNC_NAME
#else
#define HWY_CHOOSE_PPC8(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_PPC9
#define HWY_CHOOSE_PPC9(FUNC_NAME) &N_PPC9::FUNC_NAME
#else
#define HWY_CHOOSE_PPC9(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_PPC10
#define HWY_CHOOSE_PPC10(FUNC_NAME) &N_PPC10::FUNC_NAME
#else
#define HWY_CHOOSE_PPC10(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_Z14
#define HWY_CHOOSE_Z14(FUNC_NAME) &N_Z14::FUNC_NAME
#else
#define HWY_CHOOSE_Z14(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_Z15
#define HWY_CHOOSE_Z15(FUNC_NAME) &N_Z15::FUNC_NAME
#else
#define HWY_CHOOSE_Z15(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_SSE2
#define HWY_CHOOSE_SSE2(FUNC_NAME) &N_SSE2::FUNC_NAME
#else
#define HWY_CHOOSE_SSE2(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_SSSE3
#define HWY_CHOOSE_SSSE3(FUNC_NAME) &N_SSSE3::FUNC_NAME
#else
#define HWY_CHOOSE_SSSE3(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_SSE4
#define HWY_CHOOSE_SSE4(FUNC_NAME) &N_SSE4::FUNC_NAME
#else
#define HWY_CHOOSE_SSE4(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_AVX2
#define HWY_CHOOSE_AVX2(FUNC_NAME) &N_AVX2::FUNC_NAME
#else
#define HWY_CHOOSE_AVX2(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_AVX3
#define HWY_CHOOSE_AVX3(FUNC_NAME) &N_AVX3::FUNC_NAME
#else
#define HWY_CHOOSE_AVX3(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_AVX3_DL
#define HWY_CHOOSE_AVX3_DL(FUNC_NAME) &N_AVX3_DL::FUNC_NAME
#else
#define HWY_CHOOSE_AVX3_DL(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_AVX3_ZEN4
#define HWY_CHOOSE_AVX3_ZEN4(FUNC_NAME) &N_AVX3_ZEN4::FUNC_NAME
#else
#define HWY_CHOOSE_AVX3_ZEN4(FUNC_NAME) nullptr
#endif
#if HWY_TARGETS & HWY_AVX3_SPR
#define HWY_CHOOSE_AVX3_SPR(FUNC_NAME) &N_AVX3_SPR::FUNC_NAME
#else
#define HWY_CHOOSE_AVX3_SPR(FUNC_NAME) nullptr
#endif
// MSVC 2017 workaround: the non-type template parameter to ChooseAndCall
// apparently cannot be an array. Use a function pointer instead, which has the
// disadvantage that we call the static (not best) target on the first call to
// any HWY_DYNAMIC_DISPATCH.
#if (HWY_COMPILER_MSVC && HWY_COMPILER_MSVC < 1915) || \
(HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 700)
#define HWY_DISPATCH_WORKAROUND 1
#else
#define HWY_DISPATCH_WORKAROUND 0
#endif
#if HWY_DISPATCH_MAP
struct AllExports {
template <class FuncPtr, class ExportsKey, uint64_t kHash>
static const FuncPtr*& GetRefToExportsPtr() {
static const FuncPtr* s_exports = nullptr;
return s_exports;
}
};
#endif
// Provides a static member function which is what is called during the first
// HWY_DYNAMIC_DISPATCH, where GetIndex is still zero, and instantiations of
// this function are the first entry in the tables created by HWY_EXPORT[_T].
template <typename RetType, typename... Args>
struct FunctionCache {
public:
typedef RetType(FuncType)(Args...);
using FuncPtr = FuncType*;
// A template function that when instantiated has the same signature as the
// function being called. This function initializes the bit array of targets
// supported by the current CPU and then calls the appropriate entry within
// the HWY_EXPORT table. Subsequent calls via HWY_DYNAMIC_DISPATCH to any
// exported functions, even those defined by different translation units,
// will dispatch directly to the best available target.
#if HWY_DISPATCH_MAP
template <class ExportsKey, uint64_t kHash>
static RetType ChooseAndCall(Args... args) {
ChosenTarget& chosen_target = GetChosenTarget();
chosen_target.Update(SupportedTargets());
const FuncPtr* table = AllExports::template GetRefToExportsPtr<
FuncPtr, RemoveCvRef<ExportsKey>, kHash>();
HWY_ASSERT(table);
return (table[chosen_target.GetIndex()])(args...);
}
#if !HWY_DISPATCH_WORKAROUND
template <const FuncPtr* table>
static RetType TableChooseAndCall(Args... args) {
ChosenTarget& chosen_target = GetChosenTarget();
chosen_target.Update(SupportedTargets());
return (table[chosen_target.GetIndex()])(args...);
}
#endif // !HWY_DISPATCH_WORKAROUND
#else // !HWY_DISPATCH_MAP: zero-overhead, but requires C++17
template <const FuncPtr* table>
static RetType ChooseAndCall(Args... args) {
ChosenTarget& chosen_target = GetChosenTarget();
chosen_target.Update(SupportedTargets());
return (table[chosen_target.GetIndex()])(args...);
}
#endif // HWY_DISPATCH_MAP
};
// Used to deduce the template parameters RetType and Args from a function.
template <typename RetType, typename... Args>
FunctionCache<RetType, Args...> DeduceFunctionCache(RetType (*)(Args...)) {
return FunctionCache<RetType, Args...>();
}
#define HWY_DISPATCH_TABLE(FUNC_NAME) \
HWY_CONCAT(FUNC_NAME, HighwayDispatchTable)
// HWY_EXPORT(FUNC_NAME); expands to a static array that is used by
// HWY_DYNAMIC_DISPATCH() to call the appropriate function at runtime.
// After being exported, it can be called from other parts of the same source
// file using HWY_DYNAMIC_DISPATCH(), in particular from a function wrapper
// like in the following example:
//
// #include "hwy/highway.h"
// HWY_BEFORE_NAMESPACE();
// namespace skeleton {
// namespace HWY_NAMESPACE {
//
// void MyFunction(int a, char b, const char* c) { ... }
//
// // NOLINTNEXTLINE(google-readability-namespace-comments)
// } // namespace HWY_NAMESPACE
// } // namespace skeleton
// HWY_AFTER_NAMESPACE();
//
// namespace skeleton {
// HWY_EXPORT(MyFunction); // Defines the dispatch table in this scope.
//
// void MyFunction(int a, char b, const char* c) {
// return HWY_DYNAMIC_DISPATCH(MyFunction)(a, b, c);
// }
// } // namespace skeleton
//
// For templated code with a single type parameter, instead use HWY_EXPORT_T and
// its HWY_DYNAMIC_DISPATCH_T counterpart:
//
// template <typename T>
// void MyFunctionCaller(T ...) {
// // First argument to both HWY_EXPORT_T and HWY_DYNAMIC_DISPATCH_T is an
// // arbitrary table name; you must provide the same name for each call.
// // It is fine to have multiple HWY_EXPORT_T in a function, but a 64-bit
// // FNV hash collision among *any* table names will trigger HWY_ABORT.
// HWY_EXPORT_T(Table1, MyFunction<T>)
// HWY_DYNAMIC_DISPATCH_T(Table1)(a, b, c);
// }
//
// Note that HWY_EXPORT_T must be invoked inside a template (in the above
// example: `MyFunctionCaller`), so that a separate table will be created for
// each template instantiation. For convenience, we also provide a macro that
// combines both steps and avoids the need to pick a table name:
//
// template <typename T>
// void MyFunctionCaller(T ...) {
// // Table name is automatically chosen. Note that this variant must be
// // called in statement context; it is not a valid expression.
// HWY_EXPORT_AND_DYNAMIC_DISPATCH_T(MyFunction<T>)(a, b, c);
// }
// Simplified version for IDE or the dynamic dispatch case with only one target.
#if HWY_IDE || ((HWY_TARGETS & (HWY_TARGETS - 1)) == 0)
// We use a table to provide the same compile error conditions as with the
// non-simplified case, but the table only has a single entry.
#define HWY_EXPORT_T(TABLE_NAME, FUNC_NAME) \
HWY_MAYBE_UNUSED static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const \
HWY_DISPATCH_TABLE(TABLE_NAME)[1] = {&HWY_STATIC_DISPATCH(FUNC_NAME)}
// Use the table, not just STATIC_DISPATCH as in DYNAMIC_DISPATCH, because
// TABLE_NAME might not match the function name.
#define HWY_DYNAMIC_POINTER_T(TABLE_NAME) (HWY_DISPATCH_TABLE(TABLE_NAME)[0])
#define HWY_DYNAMIC_DISPATCH_T(TABLE_NAME) \
(*(HWY_DYNAMIC_POINTER_T(TABLE_NAME)))
#define HWY_EXPORT(FUNC_NAME) HWY_EXPORT_T(FUNC_NAME, FUNC_NAME)
#define HWY_DYNAMIC_POINTER(FUNC_NAME) &HWY_STATIC_DISPATCH(FUNC_NAME)
#define HWY_DYNAMIC_DISPATCH(FUNC_NAME) HWY_STATIC_DISPATCH(FUNC_NAME)
#else // not simplified: full table
// Pre-C++17 workaround: non-type template arguments must have linkage, which
// means we cannot pass &table as a template argument to ChooseAndCall.
// ChooseAndCall must find a way to access the table in order to dispatch to the
// chosen target:
// 0) Skipping this by dispatching to the static target would be surprising to
// users and may have serious performance implications.
// 1) An extra function parameter would be unacceptable because it changes the
// user-visible function signature.
// 2) Declaring a table, then defining a pointer to it would work, but requires
// an additional DECLARE step outside the function so that the pointer has
// linkage, which breaks existing code.
// 3) We instead associate the function with the table using an instance of an
// unnamed struct and the hash of the table name as the key. Because
// ChooseAndCall has the type information, it can then cast to the function
// pointer type. However, we cannot simply pass the name as a template
// argument to ChooseAndCall because this requires char*, which hits the same
// linkage problem. We instead hash the table name, which assumes the
// function names do not have collisions.
#if HWY_DISPATCH_MAP
static constexpr uint64_t FNV(const char* name) {
return *name ? static_cast<uint64_t>(static_cast<uint8_t>(*name)) ^
(0x100000001b3ULL * FNV(name + 1))
: 0xcbf29ce484222325ULL;
}
template <uint64_t kHash>
struct AddExport {
template <class ExportsKey, class FuncPtr>
AddExport(ExportsKey /*exports_key*/, const char* table_name,
const FuncPtr* table) {
using FuncCache = decltype(DeduceFunctionCache(hwy::DeclVal<FuncPtr>()));
static_assert(
hwy::IsSame<RemoveCvRef<FuncPtr>, typename FuncCache::FuncPtr>(),
"FuncPtr should be same type as FuncCache::FuncPtr");
const FuncPtr*& exports_ptr = AllExports::template GetRefToExportsPtr<
RemoveCvRef<FuncPtr>, RemoveCvRef<ExportsKey>, kHash>();
if (exports_ptr && exports_ptr != table) {
HWY_ABORT("Hash collision for %s, rename the function\n", table_name);
} else {
exports_ptr = table;
}
}
};
// Dynamic dispatch: defines table of function pointers. This must be invoked
// from inside the function template that calls the template we are exporting.
// TABLE_NAME must match the one passed to HWY_DYNAMIC_DISPATCH_T. This
// argument allows multiple exports within one function.
#define HWY_EXPORT_T(TABLE_NAME, FUNC_NAME) \
static const struct { \
} HWY_CONCAT(TABLE_NAME, HighwayDispatchExportsKey) = {}; \
static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const HWY_DISPATCH_TABLE( \
TABLE_NAME)[static_cast<size_t>(HWY_MAX_DYNAMIC_TARGETS + 2)] = { \
/* The first entry in the table initializes the global cache and \
* calls the appropriate function. */ \
&decltype(hwy::DeduceFunctionCache(&HWY_STATIC_DISPATCH(FUNC_NAME))):: \
template ChooseAndCall<decltype(HWY_CONCAT( \
TABLE_NAME, HighwayDispatchExportsKey)), \
hwy::FNV(#TABLE_NAME)>, \
HWY_CHOOSE_TARGET_LIST(FUNC_NAME), \
HWY_CHOOSE_FALLBACK(FUNC_NAME), \
}; \
HWY_MAYBE_UNUSED static hwy::AddExport<hwy::FNV(#TABLE_NAME)> HWY_CONCAT( \
HighwayAddTable, __LINE__)( \
HWY_CONCAT(TABLE_NAME, HighwayDispatchExportsKey), #TABLE_NAME, \
HWY_DISPATCH_TABLE(TABLE_NAME))
// For non-template functions. Not necessarily invoked within a function, hence
// we derive the string and variable names from FUNC_NAME, not HWY_FUNCTION.
#if HWY_DISPATCH_WORKAROUND
#define HWY_EXPORT(FUNC_NAME) HWY_EXPORT_T(FUNC_NAME, FUNC_NAME)
#else
#define HWY_EXPORT(FUNC_NAME) \
static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const HWY_DISPATCH_TABLE( \
FUNC_NAME)[static_cast<size_t>(HWY_MAX_DYNAMIC_TARGETS + 2)] = { \
/* The first entry in the table initializes the global cache and \
* calls the appropriate function. */ \
&decltype(hwy::DeduceFunctionCache(&HWY_STATIC_DISPATCH(FUNC_NAME))):: \
template TableChooseAndCall<HWY_DISPATCH_TABLE(FUNC_NAME)>, \
HWY_CHOOSE_TARGET_LIST(FUNC_NAME), \
HWY_CHOOSE_FALLBACK(FUNC_NAME), \
}
#endif // HWY_DISPATCH_WORKAROUND
#else // !HWY_DISPATCH_MAP
// Zero-overhead, but requires C++17 for non-type template arguments without
// linkage, because HWY_EXPORT_T tables are local static variables.
#define HWY_EXPORT_T(TABLE_NAME, FUNC_NAME) \
static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const HWY_DISPATCH_TABLE( \
TABLE_NAME)[static_cast<size_t>(HWY_MAX_DYNAMIC_TARGETS + 2)] = { \
/* The first entry in the table initializes the global cache and \
* calls the appropriate function. */ \
&decltype(hwy::DeduceFunctionCache(&HWY_STATIC_DISPATCH(FUNC_NAME))):: \
template ChooseAndCall<HWY_DISPATCH_TABLE(TABLE_NAME)>, \
HWY_CHOOSE_TARGET_LIST(FUNC_NAME), \
HWY_CHOOSE_FALLBACK(FUNC_NAME), \
}
#define HWY_EXPORT(FUNC_NAME) HWY_EXPORT_T(FUNC_NAME, FUNC_NAME)
#endif // HWY_DISPATCH_MAP
// HWY_DISPATCH_MAP only affects how tables are created, not their usage.
// Evaluates to the function pointer for the chosen target.
#define HWY_DYNAMIC_POINTER(FUNC_NAME) \
(HWY_DISPATCH_TABLE(FUNC_NAME)[hwy::GetChosenTarget().GetIndex()])
// Calls the function pointer for the chosen target.
#define HWY_DYNAMIC_DISPATCH(FUNC_NAME) (*(HWY_DYNAMIC_POINTER(FUNC_NAME)))
// Same as DISPATCH, but provide a different arg name to clarify usage.
#define HWY_DYNAMIC_DISPATCH_T(TABLE_NAME) HWY_DYNAMIC_DISPATCH(TABLE_NAME)
#define HWY_DYNAMIC_POINTER_T(TABLE_NAME) HWY_DYNAMIC_POINTER(TABLE_NAME)
#endif // HWY_IDE || ((HWY_TARGETS & (HWY_TARGETS - 1)) == 0)
// Returns the name of an anonymous dispatch table that is only shared with
// macro invocations coming from the same source line.
#define HWY_DISPATCH_TABLE_T() HWY_CONCAT(HighwayDispatchTableT, __LINE__)
// For templated code, combines export and dispatch using an anonymous table.
#define HWY_EXPORT_AND_DYNAMIC_DISPATCH_T(FUNC_NAME) \
HWY_EXPORT_T(HWY_DISPATCH_TABLE_T(), FUNC_NAME); \
HWY_DYNAMIC_DISPATCH_T(HWY_DISPATCH_TABLE_T())
// DEPRECATED names; please use HWY_HAVE_* instead.
#define HWY_CAP_INTEGER64 HWY_HAVE_INTEGER64
#define HWY_CAP_FLOAT16 HWY_HAVE_FLOAT16
#define HWY_CAP_FLOAT64 HWY_HAVE_FLOAT64
} // namespace hwy
#endif // HWY_HIGHWAY_INCLUDED
//------------------------------------------------------------------------------
// NOTE: the following definitions and ops/*.h depend on HWY_TARGET, so we want
// to include them once per target, which is ensured by the toggle check.
// Because ops/*.h are included under it, they do not need their own guard.
#if defined(HWY_HIGHWAY_PER_TARGET) == defined(HWY_TARGET_TOGGLE)
#ifdef HWY_HIGHWAY_PER_TARGET
#undef HWY_HIGHWAY_PER_TARGET
#else
#define HWY_HIGHWAY_PER_TARGET
#endif
// These define ops inside namespace hwy::HWY_NAMESPACE.
#if HWY_TARGET == HWY_SSE2 || HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4
#include "hwy/ops/x86_128-inl.h"
#elif HWY_TARGET == HWY_AVX2
#include "hwy/ops/x86_256-inl.h"
#elif HWY_TARGET == HWY_AVX3 || HWY_TARGET == HWY_AVX3_DL || \
HWY_TARGET == HWY_AVX3_ZEN4 || HWY_TARGET == HWY_AVX3_SPR
#include "hwy/ops/x86_512-inl.h"
#elif HWY_TARGET == HWY_Z14 || HWY_TARGET == HWY_Z15 || \
(HWY_TARGET & HWY_ALL_PPC)
#include "hwy/ops/ppc_vsx-inl.h"
#elif HWY_TARGET & HWY_ALL_NEON
#include "hwy/ops/arm_neon-inl.h"
#elif HWY_TARGET & HWY_ALL_SVE
#include "hwy/ops/arm_sve-inl.h"
#elif HWY_TARGET == HWY_WASM_EMU256
#include "hwy/ops/wasm_256-inl.h"
#elif HWY_TARGET == HWY_WASM
#include "hwy/ops/wasm_128-inl.h"
#elif HWY_TARGET == HWY_RVV
#include "hwy/ops/rvv-inl.h"
#elif HWY_TARGET == HWY_EMU128
#include "hwy/ops/emu128-inl.h"
#elif HWY_TARGET == HWY_SCALAR
#include "hwy/ops/scalar-inl.h"
#else
#pragma message("HWY_TARGET does not match any known target")
#endif // HWY_TARGET
#include "hwy/ops/generic_ops-inl.h"
#endif // HWY_HIGHWAY_PER_TARGET

74
deps/v8/third_party/highway/src/hwy/highway_export.h vendored Normal file
View File

@ -0,0 +1,74 @@
// Pseudo-generated file to handle both cmake & bazel build system.
// Initial generation done using cmake code:
// include(GenerateExportHeader)
// generate_export_header(hwy EXPORT_MACRO_NAME HWY_DLLEXPORT EXPORT_FILE_NAME
// hwy/highway_export.h)
// code reformatted using clang-format --style=Google
#ifndef HWY_DLLEXPORT_H
#define HWY_DLLEXPORT_H
#if !defined(HWY_SHARED_DEFINE)
#define HWY_DLLEXPORT
#define HWY_CONTRIB_DLLEXPORT
#define HWY_TEST_DLLEXPORT
#else // !HWY_SHARED_DEFINE
#ifndef HWY_DLLEXPORT
#if defined(hwy_EXPORTS)
/* We are building this library */
#ifdef _WIN32
#define HWY_DLLEXPORT __declspec(dllexport)
#else
#define HWY_DLLEXPORT __attribute__((visibility("default")))
#endif
#else // defined(hwy_EXPORTS)
/* We are using this library */
#ifdef _WIN32
#define HWY_DLLEXPORT __declspec(dllimport)
#else
#define HWY_DLLEXPORT __attribute__((visibility("default")))
#endif
#endif // defined(hwy_EXPORTS)
#endif // HWY_DLLEXPORT
#ifndef HWY_CONTRIB_DLLEXPORT
#if defined(hwy_contrib_EXPORTS)
/* We are building this library */
#ifdef _WIN32
#define HWY_CONTRIB_DLLEXPORT __declspec(dllexport)
#else
#define HWY_CONTRIB_DLLEXPORT __attribute__((visibility("default")))
#endif
#else // defined(hwy_contrib_EXPORTS)
/* We are using this library */
#ifdef _WIN32
#define HWY_CONTRIB_DLLEXPORT __declspec(dllimport)
#else
#define HWY_CONTRIB_DLLEXPORT __attribute__((visibility("default")))
#endif
#endif // defined(hwy_contrib_EXPORTS)
#endif // HWY_CONTRIB_DLLEXPORT
#ifndef HWY_TEST_DLLEXPORT
#if defined(hwy_test_EXPORTS)
/* We are building this library */
#ifdef _WIN32
#define HWY_TEST_DLLEXPORT __declspec(dllexport)
#else
#define HWY_TEST_DLLEXPORT __attribute__((visibility("default")))
#endif
#else // defined(hwy_test_EXPORTS)
/* We are using this library */
#ifdef _WIN32
#define HWY_TEST_DLLEXPORT __declspec(dllimport)
#else
#define HWY_TEST_DLLEXPORT __attribute__((visibility("default")))
#endif
#endif // defined(hwy_test_EXPORTS)
#endif // HWY_TEST_DLLEXPORT
#endif // !HWY_SHARED_DEFINE
#endif /* HWY_DLLEXPORT_H */

610
deps/v8/third_party/highway/src/hwy/highway_test.cc vendored Normal file
View File

@ -0,0 +1,610 @@
// Copyright 2019 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <bitset>
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "highway_test.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/nanobenchmark.h" // Unpredictable1
#include "hwy/tests/test_util-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
template <size_t kLimit, typename T>
HWY_NOINLINE void TestCappedLimit(T /* tag */) {
CappedTag<T, kLimit> d;
// Ensure two ops compile
const T k0 = ConvertScalarTo<T>(0);
const T k1 = ConvertScalarTo<T>(1);
HWY_ASSERT_VEC_EQ(d, Zero(d), Set(d, k0));
// Ensure we do not write more than kLimit lanes
const size_t N = Lanes(d);
if (kLimit < N) {
auto lanes = AllocateAligned<T>(N);
HWY_ASSERT(lanes);
ZeroBytes(lanes.get(), N * sizeof(T));
Store(Set(d, k1), d, lanes.get());
for (size_t i = kLimit; i < N; ++i) {
HWY_ASSERT_EQ(lanes[i], k0);
}
}
}
// Adapter for ForAllTypes - we are constructing our own Simd<> and thus do not
// use ForPartialVectors etc.
struct TestCapped {
template <typename T>
void operator()(T t) const {
TestCappedLimit<1>(t);
TestCappedLimit<3>(t);
TestCappedLimit<5>(t);
TestCappedLimit<1ull << 15>(t);
}
};
HWY_NOINLINE void TestAllCapped() { ForAllTypes(TestCapped()); }
// For testing that ForPartialVectors reaches every possible size:
using NumLanesSet = std::bitset<HWY_MAX_BYTES + 1>;
// Monostate pattern because ForPartialVectors takes a template argument, not a
// functor by reference.
static NumLanesSet* NumLanesForSize(size_t sizeof_t) {
HWY_ASSERT(sizeof_t <= sizeof(uint64_t));
static NumLanesSet num_lanes[sizeof(uint64_t) + 1];
return num_lanes + sizeof_t;
}
static size_t* MaxLanesForSize(size_t sizeof_t) {
HWY_ASSERT(sizeof_t <= sizeof(uint64_t));
static size_t num_lanes[sizeof(uint64_t) + 1] = {0};
return num_lanes + sizeof_t;
}
struct TestMaxLanes {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) const {
const size_t N = Lanes(d);
const size_t kMax = MaxLanes(d); // for RVV, includes LMUL
HWY_ASSERT(N <= kMax);
HWY_ASSERT(kMax <= (HWY_MAX_BYTES / sizeof(T)));
NumLanesForSize(sizeof(T))->set(N);
*MaxLanesForSize(sizeof(T)) = HWY_MAX(*MaxLanesForSize(sizeof(T)), N);
}
};
class TestFracNLanes {
private:
template <int kNewPow2, class D>
using DWithPow2 =
Simd<TFromD<D>, D::template NewN<kNewPow2, HWY_MAX_LANES_D(D)>(),
kNewPow2>;
template <typename T1, size_t N1, int kPow2, typename T2, size_t N2>
static HWY_INLINE void DoTestFracNLanes(Simd<T1, N1, 0> /*d1*/,
Simd<T2, N2, kPow2> d2) {
using D2 = Simd<T2, N2, kPow2>;
static_assert(IsSame<T1, T2>(), "T1 and T2 should be the same type");
static_assert(N2 > HWY_MAX_BYTES, "N2 > HWY_MAX_BYTES should be true");
static_assert(HWY_MAX_LANES_D(D2) == N1,
"HWY_MAX_LANES_D(D2) should be equal to N1");
static_assert(N1 <= HWY_LANES(T2), "N1 <= HWY_LANES(T2) should be true");
TestMaxLanes()(T2(), d2);
}
#if HWY_TARGET != HWY_SCALAR
template <class T, HWY_IF_LANES_LE(4, HWY_LANES(T))>
static HWY_INLINE void DoTest4LanesWithPow3(T /*unused*/) {
// If HWY_LANES(T) >= 4 is true, do DoTestFracNLanes for the
// MaxLanes(d) == 4, kPow2 == 3 case
const Simd<T, 4, 0> d;
DoTestFracNLanes(d, DWithPow2<3, decltype(d)>());
}
template <class T, HWY_IF_LANES_GT(4, HWY_LANES(T))>
static HWY_INLINE void DoTest4LanesWithPow3(T /*unused*/) {
// If HWY_LANES(T) < 4, do nothing
}
#endif
public:
template <class T>
HWY_NOINLINE void operator()(T /*unused*/) const {
const Simd<T, 1, 0> d1;
DoTestFracNLanes(d1, DWithPow2<1, decltype(d1)>());
DoTestFracNLanes(d1, DWithPow2<2, decltype(d1)>());
DoTestFracNLanes(d1, DWithPow2<3, decltype(d1)>());
#if HWY_TARGET != HWY_SCALAR
const Simd<T, 2, 0> d2;
DoTestFracNLanes(d2, DWithPow2<2, decltype(d2)>());
DoTestFracNLanes(d2, DWithPow2<3, decltype(d2)>());
DoTest4LanesWithPow3(T());
#endif
}
};
HWY_NOINLINE void TestAllMaxLanes() {
ForAllTypes(ForPartialVectors<TestMaxLanes>());
// Ensure ForPartialVectors visited all powers of two [1, N].
for (size_t sizeof_t : {sizeof(uint8_t), sizeof(uint16_t), sizeof(uint32_t),
sizeof(uint64_t)}) {
const size_t N = *MaxLanesForSize(sizeof_t);
for (size_t i = 1; i <= N; i += i) {
if (!NumLanesForSize(sizeof_t)->test(i)) {
fprintf(stderr, "T=%d: did not visit for N=%d, max=%d\n",
static_cast<int>(sizeof_t), static_cast<int>(i),
static_cast<int>(N));
HWY_ASSERT(false);
}
}
}
ForAllTypes(TestFracNLanes());
}
struct TestSet {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
// Zero
const Vec<D> v0 = Zero(d);
const size_t N = Lanes(d);
auto expected = AllocateAligned<T>(N);
HWY_ASSERT(expected);
ZeroBytes(expected.get(), N * sizeof(T));
HWY_ASSERT_VEC_EQ(d, expected.get(), v0);
// Set
const Vec<D> v2 = Set(d, ConvertScalarTo<T>(2));
for (size_t i = 0; i < N; ++i) {
expected[i] = ConvertScalarTo<T>(2);
}
HWY_ASSERT_VEC_EQ(d, expected.get(), v2);
// Iota
const Vec<D> vi = IotaForSpecial(d, 5);
for (size_t i = 0; i < N; ++i) {
expected[i] = ConvertScalarTo<T>(5 + i);
}
HWY_ASSERT_VEC_EQ(d, expected.get(), vi);
// Undefined. This may result in a 'using uninitialized memory' warning
// here, even though we already suppress warnings in Undefined.
HWY_DIAGNOSTICS(push)
HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wuninitialized")
#if HWY_COMPILER_GCC_ACTUAL
HWY_DIAGNOSTICS_OFF(disable : 4701, ignored "-Wmaybe-uninitialized")
#endif
const Vec<D> vu = Undefined(d);
Store(vu, d, expected.get());
HWY_DIAGNOSTICS(pop)
}
};
HWY_NOINLINE void TestAllSet() {
ForAllTypesAndSpecial(ForPartialVectors<TestSet>());
}
// Ensures wraparound (mod 2^bits)
struct TestOverflow {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
const Vec<D> v1 = Set(d, static_cast<T>(1));
const Vec<D> vmax = Set(d, LimitsMax<T>());
const Vec<D> vmin = Set(d, LimitsMin<T>());
// Unsigned underflow / negative -> positive
HWY_ASSERT_VEC_EQ(d, vmax, Sub(vmin, v1));
// Unsigned overflow / positive -> negative
HWY_ASSERT_VEC_EQ(d, vmin, Add(vmax, v1));
}
};
HWY_NOINLINE void TestAllOverflow() {
ForIntegerTypes(ForPartialVectors<TestOverflow>());
}
struct TestClamp {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
const Vec<D> v0 = Zero(d);
const Vec<D> v1 = Set(d, ConvertScalarTo<T>(1));
const Vec<D> v2 = Set(d, ConvertScalarTo<T>(2));
HWY_ASSERT_VEC_EQ(d, v1, Clamp(v2, v0, v1));
HWY_ASSERT_VEC_EQ(d, v1, Clamp(v0, v1, v2));
}
};
HWY_NOINLINE void TestAllClamp() {
ForAllTypes(ForPartialVectors<TestClamp>());
}
struct TestSignBitInteger {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
const Vec<D> v0 = Zero(d);
const Vec<D> all = VecFromMask(d, Eq(v0, v0));
const Vec<D> vs = SignBit(d);
const Vec<D> other = Sub(vs, Set(d, ConvertScalarTo<T>(1)));
// Shifting left by one => overflow, equal zero
HWY_ASSERT_VEC_EQ(d, v0, Add(vs, vs));
// Verify the lower bits are zero (only +/- and logical ops are available
// for all types)
HWY_ASSERT_VEC_EQ(d, all, Add(vs, other));
}
};
struct TestSignBitFloat {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
const Vec<D> v0 = Zero(d);
const Vec<D> vs = SignBit(d);
const Vec<D> vp = Set(d, ConvertScalarTo<T>(2.25));
const Vec<D> vn = Set(d, ConvertScalarTo<T>(-2.25));
HWY_ASSERT_VEC_EQ(d, Or(vp, vs), vn);
HWY_ASSERT_VEC_EQ(d, AndNot(vs, vn), vp);
HWY_ASSERT_VEC_EQ(d, v0, vs);
}
};
HWY_NOINLINE void TestAllSignBit() {
ForIntegerTypes(ForPartialVectors<TestSignBitInteger>());
ForFloatTypes(ForPartialVectors<TestSignBitFloat>());
}
// TODO(b/287462770): inline to work around incorrect SVE codegen
template <class D, class V>
HWY_INLINE void AssertNaN(D d, VecArg<V> v, const char* file, int line) {
using T = TFromD<D>;
const size_t N = Lanes(d);
if (!AllTrue(d, IsNaN(v))) {
Print(d, "not all NaN", v, 0, N);
Print(d, "mask", VecFromMask(d, IsNaN(v)), 0, N);
// RVV lacks PRIu64 and MSYS still has problems with %zu, so print bytes to
// avoid truncating doubles.
uint8_t bytes[HWY_MAX(sizeof(T), 8)] = {0};
const T lane = GetLane(v);
CopyBytes<sizeof(T)>(&lane, bytes);
Abort(file, line,
"Expected %s NaN, got %E (bytes %02x %02x %02x %02x %02x %02x %02x "
"%02x)",
TypeName(T(), N).c_str(), ConvertScalarTo<double>(lane), bytes[0],
bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]);
}
}
#define HWY_ASSERT_NAN(d, v) AssertNaN(d, v, __FILE__, __LINE__)
struct TestNaN {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
const Vec<D> v1 = Set(d, ConvertScalarTo<T>(Unpredictable1()));
const Vec<D> nan =
IfThenElse(Eq(v1, Set(d, ConvertScalarTo<T>(1))), NaN(d), v1);
HWY_ASSERT_NAN(d, nan);
// Arithmetic
HWY_ASSERT_NAN(d, Add(nan, v1));
HWY_ASSERT_NAN(d, Add(v1, nan));
HWY_ASSERT_NAN(d, Sub(nan, v1));
HWY_ASSERT_NAN(d, Sub(v1, nan));
HWY_ASSERT_NAN(d, Mul(nan, v1));
HWY_ASSERT_NAN(d, Mul(v1, nan));
HWY_ASSERT_NAN(d, Div(nan, v1));
HWY_ASSERT_NAN(d, Div(v1, nan));
// FMA
HWY_ASSERT_NAN(d, MulAdd(nan, v1, v1));
HWY_ASSERT_NAN(d, MulAdd(v1, nan, v1));
HWY_ASSERT_NAN(d, MulAdd(v1, v1, nan));
HWY_ASSERT_NAN(d, MulSub(nan, v1, v1));
HWY_ASSERT_NAN(d, MulSub(v1, nan, v1));
HWY_ASSERT_NAN(d, MulSub(v1, v1, nan));
HWY_ASSERT_NAN(d, NegMulAdd(nan, v1, v1));
HWY_ASSERT_NAN(d, NegMulAdd(v1, nan, v1));
HWY_ASSERT_NAN(d, NegMulAdd(v1, v1, nan));
HWY_ASSERT_NAN(d, NegMulSub(nan, v1, v1));
HWY_ASSERT_NAN(d, NegMulSub(v1, nan, v1));
HWY_ASSERT_NAN(d, NegMulSub(v1, v1, nan));
// Rcp/Sqrt
HWY_ASSERT_NAN(d, Sqrt(nan));
// Sign manipulation
HWY_ASSERT_NAN(d, Abs(nan));
HWY_ASSERT_NAN(d, Neg(nan));
HWY_ASSERT_NAN(d, CopySign(nan, v1));
HWY_ASSERT_NAN(d, CopySignToAbs(nan, v1));
// Rounding
HWY_ASSERT_NAN(d, Ceil(nan));
HWY_ASSERT_NAN(d, Floor(nan));
HWY_ASSERT_NAN(d, Round(nan));
HWY_ASSERT_NAN(d, Trunc(nan));
// Logical (And/AndNot/Xor will clear NaN!)
HWY_ASSERT_NAN(d, Or(nan, v1));
// Comparison
HWY_ASSERT(AllFalse(d, Eq(nan, v1)));
HWY_ASSERT(AllFalse(d, Gt(nan, v1)));
HWY_ASSERT(AllFalse(d, Lt(nan, v1)));
HWY_ASSERT(AllFalse(d, Ge(nan, v1)));
HWY_ASSERT(AllFalse(d, Le(nan, v1)));
HWY_ASSERT(AllTrue(d, IsEitherNaN(nan, nan)));
HWY_ASSERT(AllTrue(d, IsEitherNaN(nan, v1)));
HWY_ASSERT(AllTrue(d, IsEitherNaN(v1, nan)));
HWY_ASSERT(AllFalse(d, IsEitherNaN(v1, v1)));
// Reduction
HWY_ASSERT_NAN(d, SumOfLanes(d, nan));
HWY_ASSERT_NAN(d, Set(d, ReduceSum(d, nan)));
// TODO(janwas): re-enable after QEMU/Spike are fixed
#if HWY_TARGET != HWY_RVV
HWY_ASSERT_NAN(d, MinOfLanes(d, nan));
HWY_ASSERT_NAN(d, Set(d, ReduceMin(d, nan)));
HWY_ASSERT_NAN(d, MaxOfLanes(d, nan));
HWY_ASSERT_NAN(d, Set(d, ReduceMax(d, nan)));
#endif
// Min/Max
#if (HWY_ARCH_X86 || HWY_ARCH_WASM) && (HWY_TARGET < HWY_EMU128)
// Native WASM or x86 SIMD return the second operand if any input is NaN.
HWY_ASSERT_VEC_EQ(d, v1, Min(nan, v1));
HWY_ASSERT_VEC_EQ(d, v1, Max(nan, v1));
HWY_ASSERT_NAN(d, Min(v1, nan));
HWY_ASSERT_NAN(d, Max(v1, nan));
#elif HWY_TARGET <= HWY_NEON_WITHOUT_AES && HWY_ARCH_ARM_V7
// Armv7 NEON returns NaN if any input is NaN.
HWY_ASSERT_NAN(d, Min(v1, nan));
HWY_ASSERT_NAN(d, Max(v1, nan));
HWY_ASSERT_NAN(d, Min(nan, v1));
HWY_ASSERT_NAN(d, Max(nan, v1));
#else
// IEEE 754-2019 minimumNumber is defined as the other argument if exactly
// one is NaN, and qNaN if both are.
HWY_ASSERT_VEC_EQ(d, v1, Min(nan, v1));
HWY_ASSERT_VEC_EQ(d, v1, Max(nan, v1));
HWY_ASSERT_VEC_EQ(d, v1, Min(v1, nan));
HWY_ASSERT_VEC_EQ(d, v1, Max(v1, nan));
#endif
HWY_ASSERT_NAN(d, Min(nan, nan));
HWY_ASSERT_NAN(d, Max(nan, nan));
// AbsDiff
HWY_ASSERT_NAN(d, AbsDiff(nan, v1));
HWY_ASSERT_NAN(d, AbsDiff(v1, nan));
// Approximate*
HWY_ASSERT_NAN(d, ApproximateReciprocal(nan));
HWY_ASSERT_NAN(d, ApproximateReciprocalSqrt(nan));
}
};
HWY_NOINLINE void TestAllNaN() { ForFloatTypes(ForPartialVectors<TestNaN>()); }
struct TestIsNaN {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
const Vec<D> v1 = Set(d, ConvertScalarTo<T>(Unpredictable1()));
const Vec<D> inf =
IfThenElse(Eq(v1, Set(d, ConvertScalarTo<T>(1))), Inf(d), v1);
const Vec<D> nan =
IfThenElse(Eq(v1, Set(d, ConvertScalarTo<T>(1))), NaN(d), v1);
const Vec<D> neg = Set(d, ConvertScalarTo<T>(-1));
HWY_ASSERT_NAN(d, nan);
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsNaN(inf));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsNaN(CopySign(inf, neg)));
HWY_ASSERT_MASK_EQ(d, MaskTrue(d), IsNaN(nan));
HWY_ASSERT_MASK_EQ(d, MaskTrue(d), IsNaN(CopySign(nan, neg)));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsNaN(v1));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsNaN(Zero(d)));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsNaN(Set(d, hwy::LowestValue<T>())));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsNaN(Set(d, hwy::HighestValue<T>())));
}
};
HWY_NOINLINE void TestAllIsNaN() {
ForFloatTypes(ForPartialVectors<TestIsNaN>());
}
struct TestIsInf {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
const Vec<D> k1 = Set(d, ConvertScalarTo<T>(1));
const Vec<D> v1 = Set(d, ConvertScalarTo<T>(Unpredictable1()));
const Vec<D> inf = IfThenElse(Eq(v1, k1), Inf(d), v1);
const Vec<D> nan = IfThenElse(Eq(v1, k1), NaN(d), v1);
const Vec<D> neg = Neg(k1);
HWY_ASSERT_MASK_EQ(d, MaskTrue(d), IsInf(inf));
HWY_ASSERT_MASK_EQ(d, MaskTrue(d), IsInf(CopySign(inf, neg)));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsInf(nan));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsInf(CopySign(nan, neg)));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsInf(v1));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsInf(Zero(d)));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsInf(Set(d, hwy::LowestValue<T>())));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsInf(Set(d, hwy::HighestValue<T>())));
}
};
HWY_NOINLINE void TestAllIsInf() {
ForFloatTypes(ForPartialVectors<TestIsInf>());
}
struct TestIsFinite {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
const Vec<D> k1 = Set(d, ConvertScalarTo<T>(1));
const Vec<D> v1 = Set(d, ConvertScalarTo<T>(Unpredictable1()));
const Vec<D> inf = IfThenElse(Eq(v1, k1), Inf(d), v1);
const Vec<D> nan = IfThenElse(Eq(v1, k1), NaN(d), v1);
const Vec<D> neg = Neg(k1);
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsFinite(inf));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsFinite(CopySign(inf, neg)));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsFinite(nan));
HWY_ASSERT_MASK_EQ(d, MaskFalse(d), IsFinite(CopySign(nan, neg)));
HWY_ASSERT_MASK_EQ(d, MaskTrue(d), IsFinite(v1));
HWY_ASSERT_MASK_EQ(d, MaskTrue(d), IsFinite(Zero(d)));
HWY_ASSERT_MASK_EQ(d, MaskTrue(d), IsFinite(Set(d, hwy::LowestValue<T>())));
HWY_ASSERT_MASK_EQ(d, MaskTrue(d),
IsFinite(Set(d, hwy::HighestValue<T>())));
}
};
HWY_NOINLINE void TestAllIsFinite() {
ForFloatTypes(ForPartialVectors<TestIsFinite>());
}
struct TestCopyAndAssign {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
// copy V
const Vec<D> v3 = Iota(d, 3);
auto v3b(v3);
HWY_ASSERT_VEC_EQ(d, v3, v3b);
// assign V
auto v3c = Undefined(d);
v3c = v3;
HWY_ASSERT_VEC_EQ(d, v3, v3c);
}
};
HWY_NOINLINE void TestAllCopyAndAssign() {
ForAllTypes(ForPartialVectors<TestCopyAndAssign>());
}
struct TestGetLane {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
const T k1 = ConvertScalarTo<T>(1);
HWY_ASSERT_EQ(ConvertScalarTo<T>(0), GetLane(Zero(d)));
HWY_ASSERT_EQ(k1, GetLane(Set(d, k1)));
}
};
HWY_NOINLINE void TestAllGetLane() {
ForAllTypes(ForPartialVectors<TestGetLane>());
}
struct TestDFromV {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
const Vec<D> v0 = Zero(d);
// This deduced type is not necessarily the same as D.
using D0 = DFromV<decltype(v0)>;
// The two types of vectors can be used interchangeably.
const Vec<D> v0b = And(v0, Set(D0(), ConvertScalarTo<T>(1)));
HWY_ASSERT_VEC_EQ(d, v0, v0b);
}
};
HWY_NOINLINE void TestAllDFromV() {
ForAllTypes(ForPartialVectors<TestDFromV>());
}
struct TestBlocks {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
const size_t N = Lanes(d);
const size_t num_of_blocks = Blocks(d);
static constexpr size_t kNumOfLanesPer16ByteBlk = 16 / sizeof(T);
HWY_ASSERT(num_of_blocks >= 1);
HWY_ASSERT(num_of_blocks <= d.MaxBlocks());
HWY_ASSERT(
num_of_blocks ==
((N < kNumOfLanesPer16ByteBlk) ? 1 : (N / kNumOfLanesPer16ByteBlk)));
}
};
HWY_NOINLINE void TestAllBlocks() {
ForAllTypes(ForPartialVectors<TestDFromV>());
}
struct TestBlockDFromD {
template <class T, class D>
HWY_NOINLINE void operator()(T /*unused*/, D d) {
const BlockDFromD<decltype(d)> d_block;
static_assert(d_block.MaxBytes() <= 16,
"d_block.MaxBytes() <= 16 must be true");
static_assert(d_block.MaxBytes() <= d.MaxBytes(),
"d_block.MaxBytes() <= d.MaxBytes() must be true");
static_assert(d.MaxBytes() > 16 || d_block.MaxBytes() == d.MaxBytes(),
"d_block.MaxBytes() == d.MaxBytes() must be true if "
"d.MaxBytes() is less than or equal to 16");
static_assert(d.MaxBytes() < 16 || d_block.MaxBytes() == 16,
"d_block.MaxBytes() == 16 must be true if d.MaxBytes() is "
"greater than or equal to 16");
static_assert(
IsSame<Vec<decltype(d_block)>, decltype(ExtractBlock<0>(Zero(d)))>(),
"Vec<decltype(d_block)> should be the same vector type as "
"decltype(ExtractBlock<0>(Zero(d)))");
const size_t d_bytes = Lanes(d) * sizeof(T);
const size_t d_block_bytes = Lanes(d_block) * sizeof(T);
HWY_ASSERT(d_block_bytes >= 1);
HWY_ASSERT(d_block_bytes <= d_bytes);
HWY_ASSERT(d_block_bytes <= 16);
HWY_ASSERT(d_bytes > 16 || d_block_bytes == d_bytes);
HWY_ASSERT(d_bytes < 16 || d_block_bytes == 16);
}
};
HWY_NOINLINE void TestAllBlockDFromD() {
ForAllTypes(ForPartialVectors<TestBlockDFromD>());
}
} // namespace
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
namespace {
HWY_BEFORE_TEST(HighwayTest);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllCapped);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllMaxLanes);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllSet);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllOverflow);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllClamp);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllSignBit);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllNaN);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllIsNaN);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllIsInf);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllIsFinite);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllCopyAndAssign);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllGetLane);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllDFromV);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllBlocks);
HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllBlockDFromD);
HWY_AFTER_TEST();
} // namespace
} // namespace hwy
HWY_TEST_MAIN();
#endif // HWY_ONCE

19
deps/v8/third_party/highway/src/hwy/hwy.version vendored Normal file
View File

@ -0,0 +1,19 @@
HWY_0 {
global:
extern "C++" {
*hwy::*;
};
local:
# Hide all the std namespace symbols. std namespace is explicitly marked
# as visibility(default) and header-only functions or methods (such as those
# from templates) should be exposed in shared libraries as weak symbols but
# this is only needed when we expose those types in the shared library API
# in any way. We don't use C++ std types in the API and we also don't
# support exceptions in the library.
# See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=36022 for a discussion
# about this.
extern "C++" {
*std::*;
};
};

300
deps/v8/third_party/highway/src/hwy/nanobenchmark.cc vendored Normal file
View File

@ -0,0 +1,300 @@
// Copyright 2019 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/nanobenchmark.h"
#include <stdio.h>
#include <stdlib.h>
#include <time.h> // clock_gettime
#include <algorithm> // std::sort, std::find_if
#include <numeric> // std::iota
#include <random>
#include <vector>
#include "hwy/robust_statistics.h"
#include "hwy/timer-inl.h"
#include "hwy/timer.h"
namespace hwy {
namespace {
namespace timer = hwy::HWY_NAMESPACE::timer;
static const timer::Ticks timer_resolution = platform::TimerResolution();
// Estimates the expected value of "lambda" values with a variable number of
// samples until the variability "rel_mad" is less than "max_rel_mad".
template <class Lambda>
timer::Ticks SampleUntilStable(const double max_rel_mad, double* rel_mad,
const Params& p, const Lambda& lambda) {
// Choose initial samples_per_eval based on a single estimated duration.
timer::Ticks t0 = timer::Start();
lambda();
timer::Ticks t1 = timer::Stop(); // Caller checks HaveTimerStop
timer::Ticks est = t1 - t0;
static const double ticks_per_second = platform::InvariantTicksPerSecond();
const size_t ticks_per_eval =
static_cast<size_t>(ticks_per_second * p.seconds_per_eval);
size_t samples_per_eval = est == 0
? p.min_samples_per_eval
: static_cast<size_t>(ticks_per_eval / est);
samples_per_eval = HWY_MAX(samples_per_eval, p.min_samples_per_eval);
std::vector<timer::Ticks> samples;
samples.reserve(1 + samples_per_eval);
samples.push_back(est);
// Percentage is too strict for tiny differences, so also allow a small
// absolute "median absolute deviation".
const timer::Ticks max_abs_mad = (timer_resolution + 99) / 100;
*rel_mad = 0.0; // ensure initialized
for (size_t eval = 0; eval < p.max_evals; ++eval, samples_per_eval *= 2) {
samples.reserve(samples.size() + samples_per_eval);
for (size_t i = 0; i < samples_per_eval; ++i) {
t0 = timer::Start();
lambda();
t1 = timer::Stop(); // Caller checks HaveTimerStop
samples.push_back(t1 - t0);
}
if (samples.size() >= p.min_mode_samples) {
est = robust_statistics::Mode(samples.data(), samples.size());
} else {
// For "few" (depends also on the variance) samples, Median is safer.
est = robust_statistics::Median(samples.data(), samples.size());
}
NANOBENCHMARK_CHECK(est != 0);
// Median absolute deviation (mad) is a robust measure of 'variability'.
const timer::Ticks abs_mad = robust_statistics::MedianAbsoluteDeviation(
samples.data(), samples.size(), est);
*rel_mad = static_cast<double>(abs_mad) / static_cast<double>(est);
if (*rel_mad <= max_rel_mad || abs_mad <= max_abs_mad) {
if (p.verbose) {
printf("%6d samples => %5d (abs_mad=%4d, rel_mad=%4.2f%%)\n",
static_cast<int>(samples.size()), static_cast<int>(est),
static_cast<int>(abs_mad), *rel_mad * 100.0);
}
return est;
}
}
if (p.verbose) {
printf("WARNING: rel_mad=%4.2f%% still exceeds %4.2f%% after %6d samples\n",
*rel_mad * 100.0, max_rel_mad * 100.0,
static_cast<int>(samples.size()));
}
return est;
}
using InputVec = std::vector<FuncInput>;
// Returns vector of unique input values.
InputVec UniqueInputs(const FuncInput* inputs, const size_t num_inputs) {
InputVec unique(inputs, inputs + num_inputs);
std::sort(unique.begin(), unique.end());
unique.erase(std::unique(unique.begin(), unique.end()), unique.end());
return unique;
}
// Returns how often we need to call func for sufficient precision.
size_t NumSkip(const Func func, const uint8_t* arg, const InputVec& unique,
const Params& p) {
// Min elapsed ticks for any input.
timer::Ticks min_duration = ~timer::Ticks(0);
for (const FuncInput input : unique) {
double rel_mad;
const timer::Ticks total = SampleUntilStable(
p.target_rel_mad, &rel_mad, p,
[func, arg, input]() { PreventElision(func(arg, input)); });
min_duration = HWY_MIN(min_duration, total - timer_resolution);
}
// Number of repetitions required to reach the target resolution.
const size_t max_skip = p.precision_divisor;
// Number of repetitions given the estimated duration.
const size_t num_skip =
min_duration == 0
? 0
: static_cast<size_t>((max_skip + min_duration - 1) / min_duration);
if (p.verbose) {
printf("res=%d max_skip=%d min_dur=%d num_skip=%d\n",
static_cast<int>(timer_resolution), static_cast<int>(max_skip),
static_cast<int>(min_duration), static_cast<int>(num_skip));
}
return num_skip;
}
// Replicates inputs until we can omit "num_skip" occurrences of an input.
InputVec ReplicateInputs(const FuncInput* inputs, const size_t num_inputs,
const size_t num_unique, const size_t num_skip,
const Params& p) {
InputVec full;
if (num_unique == 1) {
full.assign(p.subset_ratio * num_skip, inputs[0]);
return full;
}
full.reserve(p.subset_ratio * num_skip * num_inputs);
for (size_t i = 0; i < p.subset_ratio * num_skip; ++i) {
full.insert(full.end(), inputs, inputs + num_inputs);
}
std::mt19937 rng;
std::shuffle(full.begin(), full.end(), rng);
return full;
}
// Copies the "full" to "subset" in the same order, but with "num_skip"
// randomly selected occurrences of "input_to_skip" removed.
void FillSubset(const InputVec& full, const FuncInput input_to_skip,
const size_t num_skip, InputVec* subset) {
const size_t count =
static_cast<size_t>(std::count(full.begin(), full.end(), input_to_skip));
// Generate num_skip random indices: which occurrence to skip.
std::vector<uint32_t> omit(count);
std::iota(omit.begin(), omit.end(), 0);
// omit[] is the same on every call, but that's OK because they identify the
// Nth instance of input_to_skip, so the position within full[] differs.
std::mt19937 rng;
std::shuffle(omit.begin(), omit.end(), rng);
omit.resize(num_skip);
std::sort(omit.begin(), omit.end());
uint32_t occurrence = ~0u; // 0 after preincrement
size_t idx_omit = 0; // cursor within omit[]
size_t idx_subset = 0; // cursor within *subset
for (const FuncInput next : full) {
if (next == input_to_skip) {
++occurrence;
// Haven't removed enough already
if (idx_omit < num_skip) {
// This one is up for removal
if (occurrence == omit[idx_omit]) {
++idx_omit;
continue;
}
}
}
if (idx_subset < subset->size()) {
(*subset)[idx_subset++] = next;
}
}
NANOBENCHMARK_CHECK(idx_subset == subset->size());
NANOBENCHMARK_CHECK(idx_omit == omit.size());
NANOBENCHMARK_CHECK(occurrence == count - 1);
}
// Returns total ticks elapsed for all inputs.
timer::Ticks TotalDuration(const Func func, const uint8_t* arg,
const InputVec* inputs, const Params& p,
double* max_rel_mad) {
double rel_mad;
const timer::Ticks duration =
SampleUntilStable(p.target_rel_mad, &rel_mad, p, [func, arg, inputs]() {
for (const FuncInput input : *inputs) {
PreventElision(func(arg, input));
}
});
*max_rel_mad = HWY_MAX(*max_rel_mad, rel_mad);
return duration;
}
// (Nearly) empty Func for measuring timer overhead/resolution.
HWY_NOINLINE FuncOutput EmptyFunc(const void* /*arg*/, const FuncInput input) {
return input;
}
// Returns overhead of accessing inputs[] and calling a function; this will
// be deducted from future TotalDuration return values.
timer::Ticks Overhead(const uint8_t* arg, const InputVec* inputs,
const Params& p) {
double rel_mad;
// Zero tolerance because repeatability is crucial and EmptyFunc is fast.
return SampleUntilStable(0.0, &rel_mad, p, [arg, inputs]() {
for (const FuncInput input : *inputs) {
PreventElision(EmptyFunc(arg, input));
}
});
}
} // namespace
HWY_DLLEXPORT int Unpredictable1() { return timer::Start() != ~0ULL; }
HWY_DLLEXPORT size_t Measure(const Func func, const uint8_t* arg,
const FuncInput* inputs, const size_t num_inputs,
Result* results, const Params& p) {
NANOBENCHMARK_CHECK(num_inputs != 0);
char cpu100[100];
if (!platform::HaveTimerStop(cpu100)) {
fprintf(stderr, "CPU '%s' does not support RDTSCP, skipping benchmark.\n",
cpu100);
return 0;
}
const InputVec& unique = UniqueInputs(inputs, num_inputs);
const size_t num_skip = NumSkip(func, arg, unique, p); // never 0
if (num_skip == 0) return 0; // NumSkip already printed error message
// (slightly less work on x86 to cast from signed integer)
const float mul = 1.0f / static_cast<float>(static_cast<int>(num_skip));
const InputVec& full =
ReplicateInputs(inputs, num_inputs, unique.size(), num_skip, p);
InputVec subset(full.size() - num_skip);
const timer::Ticks overhead = Overhead(arg, &full, p);
const timer::Ticks overhead_skip = Overhead(arg, &subset, p);
if (overhead < overhead_skip) {
fprintf(stderr, "Measurement failed: overhead %d < %d\n",
static_cast<int>(overhead), static_cast<int>(overhead_skip));
return 0;
}
if (p.verbose) {
printf("#inputs=%5d,%5d overhead=%5d,%5d\n", static_cast<int>(full.size()),
static_cast<int>(subset.size()), static_cast<int>(overhead),
static_cast<int>(overhead_skip));
}
double max_rel_mad = 0.0;
const timer::Ticks total = TotalDuration(func, arg, &full, p, &max_rel_mad);
for (size_t i = 0; i < unique.size(); ++i) {
FillSubset(full, unique[i], num_skip, &subset);
const timer::Ticks total_skip =
TotalDuration(func, arg, &subset, p, &max_rel_mad);
if (total < total_skip) {
fprintf(stderr, "Measurement failed: total %f < %f\n",
static_cast<double>(total), static_cast<double>(total_skip));
return 0;
}
const timer::Ticks duration =
(total - overhead) - (total_skip - overhead_skip);
results[i].input = unique[i];
results[i].ticks = static_cast<float>(duration) * mul;
results[i].variability = static_cast<float>(max_rel_mad);
}
return unique.size();
}
} // namespace hwy

171
deps/v8/third_party/highway/src/hwy/nanobenchmark.h vendored Normal file
View File

@ -0,0 +1,171 @@
// Copyright 2019 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_NANOBENCHMARK_H_
#define HIGHWAY_HWY_NANOBENCHMARK_H_
// Benchmarks functions of a single integer argument with realistic branch
// prediction hit rates. Uses a robust estimator to summarize the measurements.
// The precision is about 0.2%.
//
// Examples: see nanobenchmark_test.cc.
//
// Background: Microbenchmarks such as http://github.com/google/benchmark
// can measure elapsed times on the order of a microsecond. Shorter functions
// are typically measured by repeating them thousands of times and dividing
// the total elapsed time by this count. Unfortunately, repetition (especially
// with the same input parameter!) influences the runtime. In time-critical
// code, it is reasonable to expect warm instruction/data caches and TLBs,
// but a perfect record of which branches will be taken is unrealistic.
// Unless the application also repeatedly invokes the measured function with
// the same parameter, the benchmark is measuring something very different -
// a best-case result, almost as if the parameter were made a compile-time
// constant. This may lead to erroneous conclusions about branch-heavy
// algorithms outperforming branch-free alternatives.
//
// Our approach differs in three ways. Adding fences to the timer functions
// reduces variability due to instruction reordering, improving the timer
// resolution to about 40 CPU cycles. However, shorter functions must still
// be invoked repeatedly. For more realistic branch prediction performance,
// we vary the input parameter according to a user-specified distribution.
// Thus, instead of VaryInputs(Measure(Repeat(func))), we change the
// loop nesting to Measure(Repeat(VaryInputs(func))). We also estimate the
// central tendency of the measurement samples with the "half sample mode",
// which is more robust to outliers and skewed data than the mean or median.
#include <stddef.h>
#include <stdint.h>
#include "hwy/highway_export.h"
#include "hwy/timer.h"
// Enables sanity checks that verify correct operation at the cost of
// longer benchmark runs.
#ifndef NANOBENCHMARK_ENABLE_CHECKS
#define NANOBENCHMARK_ENABLE_CHECKS 0
#endif
#define NANOBENCHMARK_CHECK_ALWAYS(condition) \
while (!(condition)) { \
fprintf(stderr, "Nanobenchmark check failed at line %d\n", __LINE__); \
abort(); \
}
#if NANOBENCHMARK_ENABLE_CHECKS
#define NANOBENCHMARK_CHECK(condition) NANOBENCHMARK_CHECK_ALWAYS(condition)
#else
#define NANOBENCHMARK_CHECK(condition)
#endif
namespace hwy {
// Returns 1, but without the compiler knowing what the value is. This prevents
// optimizing out code.
HWY_DLLEXPORT int Unpredictable1();
// Input influencing the function being measured (e.g. number of bytes to copy).
using FuncInput = size_t;
// "Proof of work" returned by Func to ensure the compiler does not elide it.
using FuncOutput = uint64_t;
// Function to measure: either 1) a captureless lambda or function with two
// arguments or 2) a lambda with capture, in which case the first argument
// is reserved for use by MeasureClosure.
using Func = FuncOutput (*)(const void*, FuncInput);
// Internal parameters that determine precision/resolution/measuring time.
struct Params {
// Best-case precision, expressed as a divisor of the timer resolution.
// Larger => more calls to Func and higher precision.
size_t precision_divisor = 1024;
// Ratio between full and subset input distribution sizes. Cannot be less
// than 2; larger values increase measurement time but more faithfully
// model the given input distribution.
size_t subset_ratio = 2;
// Together with the estimated Func duration, determines how many times to
// call Func before checking the sample variability. Larger values increase
// measurement time, memory/cache use and precision.
double seconds_per_eval = 4E-3;
// The minimum number of samples before estimating the central tendency.
size_t min_samples_per_eval = 7;
// The mode is better than median for estimating the central tendency of
// skewed/fat-tailed distributions, but it requires sufficient samples
// relative to the width of half-ranges.
size_t min_mode_samples = 64;
// Maximum permissible variability (= median absolute deviation / center).
double target_rel_mad = 0.002;
// Abort after this many evals without reaching target_rel_mad. This
// prevents infinite loops.
size_t max_evals = 9;
// Whether to print additional statistics to stdout.
bool verbose = true;
};
// Measurement result for each unique input.
struct Result {
FuncInput input;
// Robust estimate (mode or median) of duration.
float ticks;
// Measure of variability (median absolute deviation relative to "ticks").
float variability;
};
// Precisely measures the number of ticks elapsed when calling "func" with the
// given inputs, shuffled to ensure realistic branch prediction hit rates.
//
// "func" returns a 'proof of work' to ensure its computations are not elided.
// "arg" is passed to Func, or reserved for internal use by MeasureClosure.
// "inputs" is an array of "num_inputs" (not necessarily unique) arguments to
// "func". The values should be chosen to maximize coverage of "func". This
// represents a distribution, so a value's frequency should reflect its
// probability in the real application. Order does not matter; for example, a
// uniform distribution over [0, 4) could be represented as {3,0,2,1}.
// Returns how many Result were written to "results": one per unique input, or
// zero if the measurement failed (an error message goes to stderr).
HWY_DLLEXPORT size_t Measure(Func func, const uint8_t* arg,
const FuncInput* inputs, size_t num_inputs,
Result* results, const Params& p = Params());
// Calls operator() of the given closure (lambda function).
template <class Closure>
static FuncOutput CallClosure(const Closure* f, const FuncInput input) {
return (*f)(input);
}
// Same as Measure, except "closure" is typically a lambda function of
// FuncInput -> FuncOutput with a capture list.
template <class Closure>
static inline size_t MeasureClosure(const Closure& closure,
const FuncInput* inputs,
const size_t num_inputs, Result* results,
const Params& p = Params()) {
return Measure(reinterpret_cast<Func>(&CallClosure<Closure>),
reinterpret_cast<const uint8_t*>(&closure), inputs, num_inputs,
results, p);
}
} // namespace hwy
#endif // HIGHWAY_HWY_NANOBENCHMARK_H_

Some files were not shown because too many files have changed in this diff Show More