forked from Onek8/LNXSDK
250 lines
7.5 KiB
C
250 lines
7.5 KiB
C
#define HL_NAME(n) datachannel_##n
|
|
|
|
#include <hl.h>
|
|
#include <string.h>
|
|
#include <stdint.h>
|
|
|
|
// AEGIS-128X4 (RFC 10032, section 5.4) keystream DRBG.
|
|
// VAES-512 path processes all four lanes per instruction; AES-NI path
|
|
// runs the lanes sequentially. Non-x86 builds report unavailable and the
|
|
// caller falls back to the portable implementation.
|
|
|
|
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
|
|
#define AEGIS_X86 1
|
|
#include <immintrin.h>
|
|
#if defined(_MSC_VER)
|
|
#include <intrin.h>
|
|
#else
|
|
#include <cpuid.h>
|
|
#endif
|
|
#endif
|
|
|
|
#if defined(__GNUC__) || defined(__clang__)
|
|
#define TARGET_AVX512 __attribute__((target("avx512f,vaes")))
|
|
#define TARGET_AESNI __attribute__((target("aes")))
|
|
#else
|
|
#define TARGET_AVX512
|
|
#define TARGET_AESNI
|
|
#endif
|
|
|
|
#ifdef AEGIS_X86
|
|
|
|
static void cpuidex(int leaf, int subleaf, int regs[4])
|
|
{
|
|
#if defined(_MSC_VER)
|
|
__cpuidex(regs, leaf, subleaf);
|
|
#else
|
|
__cpuid_count(leaf, subleaf, regs[0], regs[1], regs[2], regs[3]);
|
|
#endif
|
|
}
|
|
|
|
static uint64_t xgetbv0(void)
|
|
{
|
|
#if defined(_MSC_VER)
|
|
return _xgetbv(0);
|
|
#else
|
|
uint32_t eax, edx;
|
|
__asm__ volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0));
|
|
return ((uint64_t)edx << 32) | eax;
|
|
#endif
|
|
}
|
|
|
|
// 0 = no AES-NI, 1 = AES-NI, 2 = VAES + AVX-512F with OS ZMM support
|
|
static int detect(void)
|
|
{
|
|
int r[4];
|
|
cpuidex(1, 0, r);
|
|
if (!((r[2] >> 25) & 1)) return 0; // AES-NI
|
|
int osxsave = (r[2] >> 27) & 1;
|
|
int avx = (r[2] >> 28) & 1;
|
|
if (!(osxsave && avx)) return 1;
|
|
uint64_t xcr0 = xgetbv0();
|
|
if ((xcr0 & 0x6) != 0x6) return 1; // SSE+AVX state
|
|
cpuidex(7, 0, r);
|
|
int avx512f = (r[1] >> 16) & 1;
|
|
int vaes = (r[2] >> 9) & 1;
|
|
if (avx512f && vaes && (xcr0 & 0xE0) == 0xE0) return 2; // opmask+ZMM state
|
|
return 1;
|
|
}
|
|
|
|
static int mode = 0;
|
|
static uint8_t buf[128];
|
|
static int bufpos = 128;
|
|
|
|
// --- VAES-512: lane i of register j holds V[j,i] ---
|
|
|
|
static __m512i s4[8];
|
|
|
|
TARGET_AVX512 static void update512(__m512i m0, __m512i m1)
|
|
{
|
|
__m512i a0 = s4[0], a1 = s4[1], a2 = s4[2], a3 = s4[3];
|
|
__m512i a4 = s4[4], a5 = s4[5], a6 = s4[6], a7 = s4[7];
|
|
s4[0] = _mm512_aesenc_epi128(a7, _mm512_xor_si512(a0, m0));
|
|
s4[1] = _mm512_aesenc_epi128(a0, a1);
|
|
s4[2] = _mm512_aesenc_epi128(a1, a2);
|
|
s4[3] = _mm512_aesenc_epi128(a2, a3);
|
|
s4[4] = _mm512_aesenc_epi128(a3, _mm512_xor_si512(a4, m1));
|
|
s4[5] = _mm512_aesenc_epi128(a4, a5);
|
|
s4[6] = _mm512_aesenc_epi128(a5, a6);
|
|
s4[7] = _mm512_aesenc_epi128(a6, a7);
|
|
}
|
|
|
|
TARGET_AVX512 static void init512(const uint8_t* key, const uint8_t* nonce)
|
|
{
|
|
const __m128i c0_128 = _mm_setr_epi8(
|
|
0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d,
|
|
0x15, 0x22, 0x37, 0x59, (char)0x90, (char)0xe9, 0x79, 0x62);
|
|
const __m128i c1_128 = _mm_setr_epi8(
|
|
(char)0xdb, 0x3d, 0x18, 0x55, 0x6d, (char)0xc2, 0x2f, (char)0xf1,
|
|
0x20, 0x11, 0x31, 0x42, 0x73, (char)0xb5, 0x28, (char)0xdd);
|
|
__m512i kv = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)key));
|
|
__m512i nv = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)nonce));
|
|
__m512i c0 = _mm512_broadcast_i32x4(c0_128);
|
|
__m512i c1 = _mm512_broadcast_i32x4(c1_128);
|
|
__m512i kn = _mm512_xor_si512(kv, nv);
|
|
s4[0] = kn;
|
|
s4[1] = c1;
|
|
s4[2] = c0;
|
|
s4[3] = c1;
|
|
s4[4] = kn;
|
|
s4[5] = _mm512_xor_si512(kv, c0);
|
|
s4[6] = _mm512_xor_si512(kv, c1);
|
|
s4[7] = _mm512_xor_si512(kv, c0);
|
|
// ctx[i] = Byte(i) || Byte(D-1) || zeros, one per 128-bit lane
|
|
__m512i ctx = _mm512_set_epi32(
|
|
0, 0, 0, 0x303, 0, 0, 0, 0x302, 0, 0, 0, 0x301, 0, 0, 0, 0x300);
|
|
for (int r = 0; r < 10; r++) {
|
|
s4[3] = _mm512_xor_si512(s4[3], ctx);
|
|
s4[7] = _mm512_xor_si512(s4[7], ctx);
|
|
update512(nv, kv);
|
|
}
|
|
}
|
|
|
|
TARGET_AVX512 static void block512(uint8_t* out)
|
|
{
|
|
__m512i z0 = _mm512_xor_si512(_mm512_xor_si512(s4[1], s4[6]),
|
|
_mm512_and_si512(s4[2], s4[3]));
|
|
__m512i z1 = _mm512_xor_si512(_mm512_xor_si512(s4[2], s4[5]),
|
|
_mm512_and_si512(s4[6], s4[7]));
|
|
_mm512_storeu_si512((void*)out, z0);
|
|
_mm512_storeu_si512((void*)(out + 64), z1);
|
|
update512(_mm512_setzero_si512(), _mm512_setzero_si512());
|
|
}
|
|
|
|
// --- AES-NI: s1[j][i] holds V[j,i] ---
|
|
|
|
static __m128i s1[8][4];
|
|
|
|
TARGET_AESNI static void update128(__m128i m0[4], __m128i m1[4])
|
|
{
|
|
for (int i = 0; i < 4; i++) {
|
|
__m128i a0 = s1[0][i], a1 = s1[1][i], a2 = s1[2][i], a3 = s1[3][i];
|
|
__m128i a4 = s1[4][i], a5 = s1[5][i], a6 = s1[6][i], a7 = s1[7][i];
|
|
s1[0][i] = _mm_aesenc_si128(a7, _mm_xor_si128(a0, m0[i]));
|
|
s1[1][i] = _mm_aesenc_si128(a0, a1);
|
|
s1[2][i] = _mm_aesenc_si128(a1, a2);
|
|
s1[3][i] = _mm_aesenc_si128(a2, a3);
|
|
s1[4][i] = _mm_aesenc_si128(a3, _mm_xor_si128(a4, m1[i]));
|
|
s1[5][i] = _mm_aesenc_si128(a4, a5);
|
|
s1[6][i] = _mm_aesenc_si128(a5, a6);
|
|
s1[7][i] = _mm_aesenc_si128(a6, a7);
|
|
}
|
|
}
|
|
|
|
TARGET_AESNI static void init128(const uint8_t* key, const uint8_t* nonce)
|
|
{
|
|
const __m128i c0 = _mm_setr_epi8(
|
|
0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d,
|
|
0x15, 0x22, 0x37, 0x59, (char)0x90, (char)0xe9, 0x79, 0x62);
|
|
const __m128i c1 = _mm_setr_epi8(
|
|
(char)0xdb, 0x3d, 0x18, 0x55, 0x6d, (char)0xc2, 0x2f, (char)0xf1,
|
|
0x20, 0x11, 0x31, 0x42, 0x73, (char)0xb5, 0x28, (char)0xdd);
|
|
__m128i k = _mm_loadu_si128((const __m128i*)key);
|
|
__m128i n = _mm_loadu_si128((const __m128i*)nonce);
|
|
__m128i kn = _mm_xor_si128(k, n);
|
|
__m128i m0[4], m1[4];
|
|
for (int i = 0; i < 4; i++) {
|
|
s1[0][i] = kn;
|
|
s1[1][i] = c1;
|
|
s1[2][i] = c0;
|
|
s1[3][i] = c1;
|
|
s1[4][i] = kn;
|
|
s1[5][i] = _mm_xor_si128(k, c0);
|
|
s1[6][i] = _mm_xor_si128(k, c1);
|
|
s1[7][i] = _mm_xor_si128(k, c0);
|
|
m0[i] = n;
|
|
m1[i] = k;
|
|
}
|
|
for (int r = 0; r < 10; r++) {
|
|
for (int i = 0; i < 4; i++) {
|
|
__m128i ctx = _mm_set_epi32(0, 0, 0, 0x300 + i);
|
|
s1[3][i] = _mm_xor_si128(s1[3][i], ctx);
|
|
s1[7][i] = _mm_xor_si128(s1[7][i], ctx);
|
|
}
|
|
update128(m0, m1);
|
|
}
|
|
}
|
|
|
|
TARGET_AESNI static void block128(uint8_t* out)
|
|
{
|
|
for (int i = 0; i < 4; i++) {
|
|
__m128i z0 = _mm_xor_si128(_mm_xor_si128(s1[1][i], s1[6][i]),
|
|
_mm_and_si128(s1[2][i], s1[3][i]));
|
|
__m128i z1 = _mm_xor_si128(_mm_xor_si128(s1[2][i], s1[5][i]),
|
|
_mm_and_si128(s1[6][i], s1[7][i]));
|
|
_mm_storeu_si128((__m128i*)(out + i * 16), z0);
|
|
_mm_storeu_si128((__m128i*)(out + 64 + i * 16), z1);
|
|
}
|
|
__m128i zero[4];
|
|
for (int i = 0; i < 4; i++) zero[i] = _mm_setzero_si128();
|
|
update128(zero, zero);
|
|
}
|
|
|
|
#endif // AEGIS_X86
|
|
|
|
HL_PRIM bool HL_NAME(aegis_available)()
|
|
{
|
|
#ifdef AEGIS_X86
|
|
return detect() != 0;
|
|
#else
|
|
return false;
|
|
#endif
|
|
}
|
|
|
|
HL_PRIM bool HL_NAME(aegis_init)(vbyte* key, vbyte* nonce)
|
|
{
|
|
#ifdef AEGIS_X86
|
|
mode = detect();
|
|
if (mode == 2) init512((const uint8_t*)key, (const uint8_t*)nonce);
|
|
else if (mode == 1) init128((const uint8_t*)key, (const uint8_t*)nonce);
|
|
else return false;
|
|
bufpos = 128;
|
|
return true;
|
|
#else
|
|
return false;
|
|
#endif
|
|
}
|
|
|
|
HL_PRIM void HL_NAME(aegis_random)(vbyte* out, int len)
|
|
{
|
|
#ifdef AEGIS_X86
|
|
int off = 0;
|
|
while (off < len) {
|
|
if (bufpos >= 128) {
|
|
if (mode == 2) block512(buf);
|
|
else block128(buf);
|
|
bufpos = 0;
|
|
}
|
|
int n = len - off;
|
|
if (n > 128 - bufpos) n = 128 - bufpos;
|
|
memcpy(out + off, buf + bufpos, n);
|
|
bufpos += n;
|
|
off += n;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
DEFINE_PRIM(_BOOL, aegis_available, _NO_ARG);
|
|
DEFINE_PRIM(_BOOL, aegis_init, _BYTES _BYTES);
|
|
DEFINE_PRIM(_VOID, aegis_random, _BYTES _I32);
|