#define HL_NAME(n) datachannel_##n #include #include #include // AEGIS-128X4 (RFC 10032, section 5.4) keystream DRBG. // VAES-512 path processes all four lanes per instruction; AES-NI path // runs the lanes sequentially. Non-x86 builds report unavailable and the // caller falls back to the portable implementation. #if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) #define AEGIS_X86 1 #include #if defined(_MSC_VER) #include #else #include #endif #endif #if defined(__GNUC__) || defined(__clang__) #define TARGET_AVX512 __attribute__((target("avx512f,vaes"))) #define TARGET_AESNI __attribute__((target("aes"))) #else #define TARGET_AVX512 #define TARGET_AESNI #endif #ifdef AEGIS_X86 static void cpuidex(int leaf, int subleaf, int regs[4]) { #if defined(_MSC_VER) __cpuidex(regs, leaf, subleaf); #else __cpuid_count(leaf, subleaf, regs[0], regs[1], regs[2], regs[3]); #endif } static uint64_t xgetbv0(void) { #if defined(_MSC_VER) return _xgetbv(0); #else uint32_t eax, edx; __asm__ volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0)); return ((uint64_t)edx << 32) | eax; #endif } // 0 = no AES-NI, 1 = AES-NI, 2 = VAES + AVX-512F with OS ZMM support static int detect(void) { int r[4]; cpuidex(1, 0, r); if (!((r[2] >> 25) & 1)) return 0; // AES-NI int osxsave = (r[2] >> 27) & 1; int avx = (r[2] >> 28) & 1; if (!(osxsave && avx)) return 1; uint64_t xcr0 = xgetbv0(); if ((xcr0 & 0x6) != 0x6) return 1; // SSE+AVX state cpuidex(7, 0, r); int avx512f = (r[1] >> 16) & 1; int vaes = (r[2] >> 9) & 1; if (avx512f && vaes && (xcr0 & 0xE0) == 0xE0) return 2; // opmask+ZMM state return 1; } static int mode = 0; static uint8_t buf[128]; static int bufpos = 128; // --- VAES-512: lane i of register j holds V[j,i] --- static __m512i s4[8]; TARGET_AVX512 static void update512(__m512i m0, __m512i m1) { __m512i a0 = s4[0], a1 = s4[1], a2 = s4[2], a3 = s4[3]; __m512i a4 = s4[4], a5 = s4[5], a6 = s4[6], a7 = s4[7]; s4[0] = _mm512_aesenc_epi128(a7, _mm512_xor_si512(a0, m0)); s4[1] = _mm512_aesenc_epi128(a0, a1); s4[2] = _mm512_aesenc_epi128(a1, a2); s4[3] = _mm512_aesenc_epi128(a2, a3); s4[4] = _mm512_aesenc_epi128(a3, _mm512_xor_si512(a4, m1)); s4[5] = _mm512_aesenc_epi128(a4, a5); s4[6] = _mm512_aesenc_epi128(a5, a6); s4[7] = _mm512_aesenc_epi128(a6, a7); } TARGET_AVX512 static void init512(const uint8_t* key, const uint8_t* nonce) { const __m128i c0_128 = _mm_setr_epi8( 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d, 0x15, 0x22, 0x37, 0x59, (char)0x90, (char)0xe9, 0x79, 0x62); const __m128i c1_128 = _mm_setr_epi8( (char)0xdb, 0x3d, 0x18, 0x55, 0x6d, (char)0xc2, 0x2f, (char)0xf1, 0x20, 0x11, 0x31, 0x42, 0x73, (char)0xb5, 0x28, (char)0xdd); __m512i kv = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)key)); __m512i nv = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)nonce)); __m512i c0 = _mm512_broadcast_i32x4(c0_128); __m512i c1 = _mm512_broadcast_i32x4(c1_128); __m512i kn = _mm512_xor_si512(kv, nv); s4[0] = kn; s4[1] = c1; s4[2] = c0; s4[3] = c1; s4[4] = kn; s4[5] = _mm512_xor_si512(kv, c0); s4[6] = _mm512_xor_si512(kv, c1); s4[7] = _mm512_xor_si512(kv, c0); // ctx[i] = Byte(i) || Byte(D-1) || zeros, one per 128-bit lane __m512i ctx = _mm512_set_epi32( 0, 0, 0, 0x303, 0, 0, 0, 0x302, 0, 0, 0, 0x301, 0, 0, 0, 0x300); for (int r = 0; r < 10; r++) { s4[3] = _mm512_xor_si512(s4[3], ctx); s4[7] = _mm512_xor_si512(s4[7], ctx); update512(nv, kv); } } TARGET_AVX512 static void block512(uint8_t* out) { __m512i z0 = _mm512_xor_si512(_mm512_xor_si512(s4[1], s4[6]), _mm512_and_si512(s4[2], s4[3])); __m512i z1 = _mm512_xor_si512(_mm512_xor_si512(s4[2], s4[5]), _mm512_and_si512(s4[6], s4[7])); _mm512_storeu_si512((void*)out, z0); _mm512_storeu_si512((void*)(out + 64), z1); update512(_mm512_setzero_si512(), _mm512_setzero_si512()); } // --- AES-NI: s1[j][i] holds V[j,i] --- static __m128i s1[8][4]; TARGET_AESNI static void update128(__m128i m0[4], __m128i m1[4]) { for (int i = 0; i < 4; i++) { __m128i a0 = s1[0][i], a1 = s1[1][i], a2 = s1[2][i], a3 = s1[3][i]; __m128i a4 = s1[4][i], a5 = s1[5][i], a6 = s1[6][i], a7 = s1[7][i]; s1[0][i] = _mm_aesenc_si128(a7, _mm_xor_si128(a0, m0[i])); s1[1][i] = _mm_aesenc_si128(a0, a1); s1[2][i] = _mm_aesenc_si128(a1, a2); s1[3][i] = _mm_aesenc_si128(a2, a3); s1[4][i] = _mm_aesenc_si128(a3, _mm_xor_si128(a4, m1[i])); s1[5][i] = _mm_aesenc_si128(a4, a5); s1[6][i] = _mm_aesenc_si128(a5, a6); s1[7][i] = _mm_aesenc_si128(a6, a7); } } TARGET_AESNI static void init128(const uint8_t* key, const uint8_t* nonce) { const __m128i c0 = _mm_setr_epi8( 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d, 0x15, 0x22, 0x37, 0x59, (char)0x90, (char)0xe9, 0x79, 0x62); const __m128i c1 = _mm_setr_epi8( (char)0xdb, 0x3d, 0x18, 0x55, 0x6d, (char)0xc2, 0x2f, (char)0xf1, 0x20, 0x11, 0x31, 0x42, 0x73, (char)0xb5, 0x28, (char)0xdd); __m128i k = _mm_loadu_si128((const __m128i*)key); __m128i n = _mm_loadu_si128((const __m128i*)nonce); __m128i kn = _mm_xor_si128(k, n); __m128i m0[4], m1[4]; for (int i = 0; i < 4; i++) { s1[0][i] = kn; s1[1][i] = c1; s1[2][i] = c0; s1[3][i] = c1; s1[4][i] = kn; s1[5][i] = _mm_xor_si128(k, c0); s1[6][i] = _mm_xor_si128(k, c1); s1[7][i] = _mm_xor_si128(k, c0); m0[i] = n; m1[i] = k; } for (int r = 0; r < 10; r++) { for (int i = 0; i < 4; i++) { __m128i ctx = _mm_set_epi32(0, 0, 0, 0x300 + i); s1[3][i] = _mm_xor_si128(s1[3][i], ctx); s1[7][i] = _mm_xor_si128(s1[7][i], ctx); } update128(m0, m1); } } TARGET_AESNI static void block128(uint8_t* out) { for (int i = 0; i < 4; i++) { __m128i z0 = _mm_xor_si128(_mm_xor_si128(s1[1][i], s1[6][i]), _mm_and_si128(s1[2][i], s1[3][i])); __m128i z1 = _mm_xor_si128(_mm_xor_si128(s1[2][i], s1[5][i]), _mm_and_si128(s1[6][i], s1[7][i])); _mm_storeu_si128((__m128i*)(out + i * 16), z0); _mm_storeu_si128((__m128i*)(out + 64 + i * 16), z1); } __m128i zero[4]; for (int i = 0; i < 4; i++) zero[i] = _mm_setzero_si128(); update128(zero, zero); } #endif // AEGIS_X86 HL_PRIM bool HL_NAME(aegis_available)() { #ifdef AEGIS_X86 return detect() != 0; #else return false; #endif } HL_PRIM bool HL_NAME(aegis_init)(vbyte* key, vbyte* nonce) { #ifdef AEGIS_X86 mode = detect(); if (mode == 2) init512((const uint8_t*)key, (const uint8_t*)nonce); else if (mode == 1) init128((const uint8_t*)key, (const uint8_t*)nonce); else return false; bufpos = 128; return true; #else return false; #endif } HL_PRIM void HL_NAME(aegis_random)(vbyte* out, int len) { #ifdef AEGIS_X86 int off = 0; while (off < len) { if (bufpos >= 128) { if (mode == 2) block512(buf); else block128(buf); bufpos = 0; } int n = len - off; if (n > 128 - bufpos) n = 128 - bufpos; memcpy(out + off, buf + bufpos, n); bufpos += n; off += n; } #endif } DEFINE_PRIM(_BOOL, aegis_available, _NO_ARG); DEFINE_PRIM(_BOOL, aegis_init, _BYTES _BYTES); DEFINE_PRIM(_VOID, aegis_random, _BYTES _I32);