/* Blake2s.c -- BLAKE2sp Hash 2024-05-18 : Igor Pavlov : Public domain 2015-2019 : Samuel Neves : original code : CC0 1.0 Universal (CC0 1.0). */ #include "Precomp.h" // #include #include #include "Blake2.h" #include "RotateDefs.h" #include "Compiler.h" #include "CpuArch.h" /* if defined(__AVX512F__) && defined(__AVX512VL__) { we define Z7_BLAKE2S_USE_AVX512_ALWAYS, but the compiler can use avx512 for any code. } else if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) { we use avx512 only for sse* and avx* branches of code. } */ // #define Z7_BLAKE2S_USE_AVX512_ALWAYS // for debug #if defined(__SSE2__) #define Z7_BLAKE2S_USE_VECTORS #elif defined(MY_CPU_X86_OR_AMD64) #if defined(_MSC_VER) && _MSC_VER > 1200 \ || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 30300) \ || defined(__clang__) \ || defined(__INTEL_COMPILER) #define Z7_BLAKE2S_USE_VECTORS #endif #endif #ifdef Z7_BLAKE2S_USE_VECTORS #define Z7_BLAKE2SP_USE_FUNCTIONS // define Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED, if CBlake2sp can be non aligned for 32-bytes. // #define Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED // SSSE3 : for _mm_shuffle_epi8 (pshufb) that improves the performance for 5-15%. #if defined(__SSSE3__) #define Z7_BLAKE2S_USE_SSSE3 #elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1500) \ || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40300) \ || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40000) \ || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 20300) \ || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1000) #define Z7_BLAKE2S_USE_SSSE3 #endif #ifdef Z7_BLAKE2S_USE_SSSE3 /* SSE41 : for _mm_insert_epi32 (pinsrd) it can slightly reduce code size and improves the performance in some cases. it's used only for last 512-1024 bytes, if FAST versions (2 or 3) of vector algos are used. it can be used for all blocks in another algos (4+). */ #if defined(__SSE4_1__) #define Z7_BLAKE2S_USE_SSE41 #elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1500) \ || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40300) \ || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40000) \ || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 20300) \ || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1000) #define Z7_BLAKE2S_USE_SSE41 #endif #endif // SSSE3 #if defined(__GNUC__) || defined(__clang__) #if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) && !(defined(__AVX512F__) && defined(__AVX512VL__)) #define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("avx512vl,avx512f"))) #else #if defined(Z7_BLAKE2S_USE_SSE41) #define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("sse4.1"))) #elif defined(Z7_BLAKE2S_USE_SSSE3) #define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("ssse3"))) #else #define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("sse2"))) #endif #endif #endif #if defined(__AVX2__) #define Z7_BLAKE2S_USE_AVX2 #else #if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \ || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100) #define Z7_BLAKE2S_USE_AVX2 #ifdef Z7_BLAKE2S_USE_AVX2 #if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) && !(defined(__AVX512F__) && defined(__AVX512VL__)) #define BLAKE2S_ATTRIB_AVX2 __attribute__((__target__("avx512vl,avx512f"))) #else #define BLAKE2S_ATTRIB_AVX2 __attribute__((__target__("avx2"))) #endif #endif #elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \ || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400) #if (Z7_MSC_VER_ORIGINAL == 1900) #pragma warning(disable : 4752) // found Intel(R) Advanced Vector Extensions; consider using /arch:AVX #endif #define Z7_BLAKE2S_USE_AVX2 #endif #endif #ifdef Z7_BLAKE2S_USE_SSE41 #include // SSE4.1 #elif defined(Z7_BLAKE2S_USE_SSSE3) #include // SSSE3 #else #include // SSE2 #endif #ifdef Z7_BLAKE2S_USE_AVX2 #include #if defined(__clang__) #include #include #endif #endif // avx2 #if defined(__AVX512F__) && defined(__AVX512VL__) // && defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL > 1930) #ifndef Z7_BLAKE2S_USE_AVX512_ALWAYS #define Z7_BLAKE2S_USE_AVX512_ALWAYS #endif // #pragma message ("=== Blake2s AVX512") #endif #define Z7_BLAKE2S_USE_V128_FAST // for speed optimization for small messages: // #define Z7_BLAKE2S_USE_V128_WAY2 #ifdef Z7_BLAKE2S_USE_AVX2 // for debug: // gather is slow // #define Z7_BLAKE2S_USE_GATHER #define Z7_BLAKE2S_USE_AVX2_FAST // for speed optimization for small messages: // #define Z7_BLAKE2S_USE_AVX2_WAY2 // #define Z7_BLAKE2S_USE_AVX2_WAY4 #if defined(Z7_BLAKE2S_USE_AVX2_WAY2) || \ defined(Z7_BLAKE2S_USE_AVX2_WAY4) #define Z7_BLAKE2S_USE_AVX2_WAY_SLOW #endif #endif #define Z7_BLAKE2SP_ALGO_DEFAULT 0 #define Z7_BLAKE2SP_ALGO_SCALAR 1 #ifdef Z7_BLAKE2S_USE_V128_FAST #define Z7_BLAKE2SP_ALGO_V128_FAST 2 #endif #ifdef Z7_BLAKE2S_USE_AVX2_FAST #define Z7_BLAKE2SP_ALGO_V256_FAST 3 #endif #define Z7_BLAKE2SP_ALGO_V128_WAY1 4 #ifdef Z7_BLAKE2S_USE_V128_WAY2 #define Z7_BLAKE2SP_ALGO_V128_WAY2 5 #endif #ifdef Z7_BLAKE2S_USE_AVX2_WAY2 #define Z7_BLAKE2SP_ALGO_V256_WAY2 6 #endif #ifdef Z7_BLAKE2S_USE_AVX2_WAY4 #define Z7_BLAKE2SP_ALGO_V256_WAY4 7 #endif #endif // Z7_BLAKE2S_USE_VECTORS #define BLAKE2S_FINAL_FLAG (~(UInt32)0) #define NSW Z7_BLAKE2SP_NUM_STRUCT_WORDS #define SUPER_BLOCK_SIZE (Z7_BLAKE2S_BLOCK_SIZE * Z7_BLAKE2SP_PARALLEL_DEGREE) #define SUPER_BLOCK_MASK (SUPER_BLOCK_SIZE - 1) #define V_INDEX_0_0 0 #define V_INDEX_1_0 1 #define V_INDEX_2_0 2 #define V_INDEX_3_0 3 #define V_INDEX_0_1 4 #define V_INDEX_1_1 5 #define V_INDEX_2_1 6 #define V_INDEX_3_1 7 #define V_INDEX_0_2 8 #define V_INDEX_1_2 9 #define V_INDEX_2_2 10 #define V_INDEX_3_2 11 #define V_INDEX_0_3 12 #define V_INDEX_1_3 13 #define V_INDEX_2_3 14 #define V_INDEX_3_3 15 #define V_INDEX_4_0 0 #define V_INDEX_5_0 1 #define V_INDEX_6_0 2 #define V_INDEX_7_0 3 #define V_INDEX_7_1 4 #define V_INDEX_4_1 5 #define V_INDEX_5_1 6 #define V_INDEX_6_1 7 #define V_INDEX_6_2 8 #define V_INDEX_7_2 9 #define V_INDEX_4_2 10 #define V_INDEX_5_2 11 #define V_INDEX_5_3 12 #define V_INDEX_6_3 13 #define V_INDEX_7_3 14 #define V_INDEX_4_3 15 #define V(row, col) v[V_INDEX_ ## row ## _ ## col] #define k_Blake2s_IV_0 0x6A09E667UL #define k_Blake2s_IV_1 0xBB67AE85UL #define k_Blake2s_IV_2 0x3C6EF372UL #define k_Blake2s_IV_3 0xA54FF53AUL #define k_Blake2s_IV_4 0x510E527FUL #define k_Blake2s_IV_5 0x9B05688CUL #define k_Blake2s_IV_6 0x1F83D9ABUL #define k_Blake2s_IV_7 0x5BE0CD19UL #define KIV(n) (k_Blake2s_IV_## n) #ifdef Z7_BLAKE2S_USE_VECTORS MY_ALIGN(16) static const UInt32 k_Blake2s_IV[8] = { KIV(0), KIV(1), KIV(2), KIV(3), KIV(4), KIV(5), KIV(6), KIV(7) }; #endif #define STATE_T(s) ((s) + 8) #define STATE_F(s) ((s) + 10) #ifdef Z7_BLAKE2S_USE_VECTORS #define LOAD_128(p) _mm_load_si128 ((const __m128i *)(const void *)(p)) #define LOADU_128(p) _mm_loadu_si128((const __m128i *)(const void *)(p)) #ifdef Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED // here we use unaligned load and stores // use this branch if CBlake2sp can be unaligned for 16 bytes #define STOREU_128(p, r) _mm_storeu_si128((__m128i *)(void *)(p), r) #define LOAD_128_FROM_STRUCT(p) LOADU_128(p) #define STORE_128_TO_STRUCT(p, r) STOREU_128(p, r) #else // here we use aligned load and stores // use this branch if CBlake2sp is aligned for 16 bytes #define STORE_128(p, r) _mm_store_si128((__m128i *)(void *)(p), r) #define LOAD_128_FROM_STRUCT(p) LOAD_128(p) #define STORE_128_TO_STRUCT(p, r) STORE_128(p, r) #endif #endif // Z7_BLAKE2S_USE_VECTORS #if 0 static void PrintState(const UInt32 *s, unsigned num) { unsigned i; printf("\n"); for (i = 0; i < num; i++) printf(" %08x", (unsigned)s[i]); } static void PrintStates2(const UInt32 *s, unsigned x, unsigned y) { unsigned i; for (i = 0; i < y; i++) PrintState(s + i * x, x); printf("\n"); } #endif #define REP8_MACRO(m) { m(0) m(1) m(2) m(3) m(4) m(5) m(6) m(7) } #define BLAKE2S_NUM_ROUNDS 10 #if defined(Z7_BLAKE2S_USE_VECTORS) #define ROUNDS_LOOP(mac) \ { unsigned r; for (r = 0; r < BLAKE2S_NUM_ROUNDS; r++) mac(r) } #endif /* #define ROUNDS_LOOP_2(mac) \ { unsigned r; for (r = 0; r < BLAKE2S_NUM_ROUNDS; r += 2) { mac(r) mac(r + 1) } } */ #if 0 || 1 && !defined(Z7_BLAKE2S_USE_VECTORS) #define ROUNDS_LOOP_UNROLLED(m) \ { m(0) m(1) m(2) m(3) m(4) m(5) m(6) m(7) m(8) m(9) } #endif #define SIGMA_TABLE(M) \ M( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ), \ M( 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 ), \ M( 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 ), \ M( 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 ), \ M( 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 ), \ M( 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 ), \ M( 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 ), \ M( 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 ), \ M( 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 ), \ M( 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 ) #define SIGMA_TABLE_MULT(m, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ { a0*m,a1*m,a2*m,a3*m,a4*m,a5*m,a6*m,a7*m,a8*m,a9*m,a10*m,a11*m,a12*m,a13*m,a14*m,a15*m } #define SIGMA_TABLE_MULT_4( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ SIGMA_TABLE_MULT(4, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) // MY_ALIGN(32) MY_ALIGN(16) static const Byte k_Blake2s_Sigma_4[BLAKE2S_NUM_ROUNDS][16] = { SIGMA_TABLE(SIGMA_TABLE_MULT_4) }; #define GET_SIGMA_PTR(p, index) \ ((const void *)((const Byte *)(const void *)(p) + (index))) #define GET_STATE_TABLE_PTR_FROM_BYTE_POS(s, pos) \ ((UInt32 *)(void *)((Byte *)(void *)(s) + (pos))) #ifdef Z7_BLAKE2S_USE_VECTORS #if 0 // use loading constants from memory // is faster for some compilers. #define KK4(n) KIV(n), KIV(n), KIV(n), KIV(n) MY_ALIGN(64) static const UInt32 k_Blake2s_IV_WAY4[]= { KK4(0), KK4(1), KK4(2), KK4(3), KK4(4), KK4(5), KK4(6), KK4(7) }; #define GET_128_IV_WAY4(i) LOAD_128(k_Blake2s_IV_WAY4 + 4 * (i)) #else // use constant generation: #define GET_128_IV_WAY4(i) _mm_set1_epi32((Int32)KIV(i)) #endif #ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW #define GET_CONST_128_FROM_ARRAY32(k) \ _mm_set_epi32((Int32)(k)[3], (Int32)(k)[2], (Int32)(k)[1], (Int32)(k)[0]) #endif #if 0 #define k_r8 _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1) #define k_r16 _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2) #define k_inc _mm_set_epi32(0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE) #define k_iv0_128 GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 0) #define k_iv4_128 GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 4) #else #if defined(Z7_BLAKE2S_USE_SSSE3) && \ !defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) MY_ALIGN(16) static const Byte k_r8_arr [16] = { 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12 }; MY_ALIGN(16) static const Byte k_r16_arr[16] = { 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 }; #define k_r8 LOAD_128(k_r8_arr) #define k_r16 LOAD_128(k_r16_arr) #endif MY_ALIGN(16) static const UInt32 k_inc_arr[4] = { Z7_BLAKE2S_BLOCK_SIZE, 0, 0, 0 }; #define k_inc LOAD_128(k_inc_arr) #define k_iv0_128 LOAD_128(k_Blake2s_IV + 0) #define k_iv4_128 LOAD_128(k_Blake2s_IV + 4) #endif #ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW #ifdef Z7_BLAKE2S_USE_AVX2 #if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION < 80000) #define MY_mm256_set_m128i(hi, lo) _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1) #else #define MY_mm256_set_m128i _mm256_set_m128i #endif #define SET_FROM_128(a) MY_mm256_set_m128i(a, a) #ifndef Z7_BLAKE2S_USE_AVX512_ALWAYS MY_ALIGN(32) static const Byte k_r8_arr_256 [32] = { 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12, 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12 }; MY_ALIGN(32) static const Byte k_r16_arr_256[32] = { 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13, 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 }; #define k_r8_256 LOAD_256(k_r8_arr_256) #define k_r16_256 LOAD_256(k_r16_arr_256) #endif // #define k_r8_256 SET_FROM_128(_mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)) // #define k_r16_256 SET_FROM_128(_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)) // #define k_inc_256 SET_FROM_128(_mm_set_epi32(0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE)) // #define k_iv0_256 SET_FROM_128(GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 0)) #define k_iv4_256 SET_FROM_128(GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 4)) #endif // Z7_BLAKE2S_USE_AVX2_WAY_SLOW #endif /* IPC(TP) ports: 1 p__5 : skl- : SSE : shufps : _mm_shuffle_ps 2 p_15 : icl+ 1 p__5 : nhm-bdw : SSE : xorps : _mm_xor_ps 3 p015 : skl+ 3 p015 : SSE2 : pxor : _mm_xor_si128 2 p_15: snb-bdw : SSE2 : padd : _mm_add_epi32 2 p0_5: mrm-wsm : 3 p015 : skl+ 2 p_15 : ivb-,icl+ : SSE2 : punpcklqdq, punpckhqdq, punpckldq, punpckhdq 2 p_15 : : SSE2 : pshufd : _mm_shuffle_epi32 2 p_15 : : SSE2 : pshuflw : _mm_shufflelo_epi16 2 p_15 : : SSE2 : psrldq : 2 p_15 : : SSE3 : pshufb : _mm_shuffle_epi8 2 p_15 : : SSE4 : pblendw : _mm_blend_epi16 1 p__5 : hsw-skl : * 1 p0 : SSE2 : pslld (i8) : _mm_slli_si128 2 p01 : skl+ : 2 p_15 : ivb- : SSE3 : palignr 1 p__5 : hsw+ 2 p_15 + p23 : ivb-, icl+ : SSE4 : pinsrd : _mm_insert_epi32(xmm, m32, i8) 1 p__5 + p23 : hsw-skl 1 p_15 + p5 : ivb-, ice+ : SSE4 : pinsrd : _mm_insert_epi32(xmm, r32, i8) 0.5 2*p5 : hsw-skl 2 p23 : SSE2 : movd (m32) 3 p23A : adl : 1 p5: : SSE2 : movd (r32) */ #if 0 && defined(__XOP__) // we must debug and test __XOP__ instruction #include #include #define LOAD_ROTATE_CONSTS #define MM_ROR_EPI32(r, c) _mm_roti_epi32(r, -(c)) #define Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED #elif 1 && defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) #define LOAD_ROTATE_CONSTS #define MM_ROR_EPI32(r, c) _mm_ror_epi32(r, c) #define Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED #else // MSVC_1937+ uses "orps" instruction for _mm_or_si128(). // But "orps" has low throughput: TP=1 for bdw-nhm. // So it can be better to use _mm_add_epi32()/"paddd" (TP=2 for bdw-nhm) instead of "xorps". // But "orps" is fast for modern cpus (skl+). // So we are default with "or" version: #if 0 || 0 && defined(Z7_MSC_VER_ORIGINAL) && Z7_MSC_VER_ORIGINAL > 1937 // minor optimization for some old cpus, if "xorps" is slow. #define MM128_EPI32_OR_or_ADD _mm_add_epi32 #else #define MM128_EPI32_OR_or_ADD _mm_or_si128 #endif #define MM_ROR_EPI32_VIA_SHIFT(r, c)( \ MM128_EPI32_OR_or_ADD( \ _mm_srli_epi32((r), (c)), \ _mm_slli_epi32((r), 32-(c)))) #if defined(Z7_BLAKE2S_USE_SSSE3) || defined(Z7_BLAKE2S_USE_SSE41) #define LOAD_ROTATE_CONSTS \ const __m128i r8 = k_r8; \ const __m128i r16 = k_r16; #define MM_ROR_EPI32(r, c) ( \ ( 8==(c)) ? _mm_shuffle_epi8(r,r8) \ : (16==(c)) ? _mm_shuffle_epi8(r,r16) \ : MM_ROR_EPI32_VIA_SHIFT(r, c)) #else #define LOAD_ROTATE_CONSTS #define MM_ROR_EPI32(r, c) ( \ (16==(c)) ? _mm_shufflehi_epi16(_mm_shufflelo_epi16(r, 0xb1), 0xb1) \ : MM_ROR_EPI32_VIA_SHIFT(r, c)) #endif #endif /* we have 3 main ways to load 4 32-bit integers to __m128i: 1) SSE2: _mm_set_epi32() 2) SSE2: _mm_unpacklo_epi64() / _mm_unpacklo_epi32 / _mm_cvtsi32_si128() 3) SSE41: _mm_insert_epi32() and _mm_cvtsi32_si128() good compiler for _mm_set_epi32() generates these instructions: { movd xmm, [m32]; vpunpckldq; vpunpckldq; vpunpcklqdq; } good new compiler generates one instruction { for _mm_insert_epi32() : { pinsrd xmm, [m32], i } for _mm_cvtsi32_si128() : { movd xmm, [m32] } } but vc2010 generates slow pair of instructions: { for _mm_insert_epi32() : { mov r32, [m32]; pinsrd xmm, r32, i } for _mm_cvtsi32_si128() : { mov r32, [m32]; movd xmm, r32 } } _mm_insert_epi32() (pinsrd) code reduces xmm register pressure in comparison with _mm_set_epi32() (movd + vpunpckld) code. Note that variant with "movd xmm, r32" can be more slow, but register pressure can be more important. So we can force to "pinsrd" always. */ // #if !defined(Z7_MSC_VER_ORIGINAL) || Z7_MSC_VER_ORIGINAL > 1600 || defined(MY_CPU_X86) #ifdef Z7_BLAKE2S_USE_SSE41 /* _mm_set_epi32() can be more effective for GCC and CLANG _mm_insert_epi32() is more effective for MSVC */ #if 0 || 1 && defined(Z7_MSC_VER_ORIGINAL) #define Z7_BLAKE2S_USE_INSERT_INSTRUCTION #endif #endif // USE_SSE41 // #endif #ifdef Z7_BLAKE2S_USE_INSERT_INSTRUCTION // for SSE4.1 #define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \ _mm_insert_epi32( \ _mm_insert_epi32( \ _mm_insert_epi32( \ _mm_cvtsi32_si128( \ *(const Int32 *)p0), \ *(const Int32 *)p1, 1), \ *(const Int32 *)p2, 2), \ *(const Int32 *)p3, 3) #elif 0 || 1 && defined(Z7_MSC_VER_ORIGINAL) /* MSVC 1400 implements _mm_set_epi32() via slow memory write/read. Also _mm_unpacklo_epi32 is more effective for another MSVC compilers. But _mm_set_epi32() is more effective for GCC and CLANG. So we use _mm_unpacklo_epi32 for MSVC only */ #define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \ _mm_unpacklo_epi64( \ _mm_unpacklo_epi32( _mm_cvtsi32_si128(*(const Int32 *)p0), \ _mm_cvtsi32_si128(*(const Int32 *)p1)), \ _mm_unpacklo_epi32( _mm_cvtsi32_si128(*(const Int32 *)p2), \ _mm_cvtsi32_si128(*(const Int32 *)p3))) #else #define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \ _mm_set_epi32( \ *(const Int32 *)p3, \ *(const Int32 *)p2, \ *(const Int32 *)p1, \ *(const Int32 *)p0) #endif #define SET_ROW_FROM_SIGMA_BASE(input, i0, i1, i2, i3) \ MM_LOAD_EPI32_FROM_4_POINTERS( \ GET_SIGMA_PTR(input, i0), \ GET_SIGMA_PTR(input, i1), \ GET_SIGMA_PTR(input, i2), \ GET_SIGMA_PTR(input, i3)) #define SET_ROW_FROM_SIGMA(input, sigma_index) \ SET_ROW_FROM_SIGMA_BASE(input, \ sigma[(sigma_index) ], \ sigma[(sigma_index) + 2 * 1], \ sigma[(sigma_index) + 2 * 2], \ sigma[(sigma_index) + 2 * 3]) \ #define ADD_128(a, b) _mm_add_epi32(a, b) #define XOR_128(a, b) _mm_xor_si128(a, b) #define D_ADD_128(dest, src) dest = ADD_128(dest, src) #define D_XOR_128(dest, src) dest = XOR_128(dest, src) #define D_ROR_128(dest, shift) dest = MM_ROR_EPI32(dest, shift) #define D_ADD_EPI64_128(dest, src) dest = _mm_add_epi64(dest, src) #define AXR(a, b, d, shift) \ D_ADD_128(a, b); \ D_XOR_128(d, a); \ D_ROR_128(d, shift); #define AXR2(a, b, c, d, input, sigma_index, shift1, shift2) \ a = _mm_add_epi32 (a, SET_ROW_FROM_SIGMA(input, sigma_index)); \ AXR(a, b, d, shift1) \ AXR(c, d, b, shift2) #define ROTATE_WORDS_TO_RIGHT(a, n) \ a = _mm_shuffle_epi32(a, _MM_SHUFFLE((3+n)&3, (2+n)&3, (1+n)&3, (0+n)&3)); #define AXR4(a, b, c, d, input, sigma_index) \ AXR2(a, b, c, d, input, sigma_index, 16, 12) \ AXR2(a, b, c, d, input, sigma_index + 1, 8, 7) \ #define RR2(a, b, c, d, input) \ { \ AXR4(a, b, c, d, input, 0) \ ROTATE_WORDS_TO_RIGHT(b, 1) \ ROTATE_WORDS_TO_RIGHT(c, 2) \ ROTATE_WORDS_TO_RIGHT(d, 3) \ AXR4(a, b, c, d, input, 8) \ ROTATE_WORDS_TO_RIGHT(b, 3) \ ROTATE_WORDS_TO_RIGHT(c, 2) \ ROTATE_WORDS_TO_RIGHT(d, 1) \ } /* Way1: per 64 bytes block: 10 rounds * 4 iters * (7 + 2) = 360 cycles = if pslld TP=1 * (7 + 1) = 320 cycles = if pslld TP=2 (skl+) additional operations per 7_op_iter : 4 movzx byte mem 1 movd mem 3 pinsrd mem 1.5 pshufd */ static #if 0 || 0 && (defined(Z7_BLAKE2S_USE_V128_WAY2) || \ defined(Z7_BLAKE2S_USE_V256_WAY2)) Z7_NO_INLINE #else Z7_FORCE_INLINE #endif #ifdef BLAKE2S_ATTRIB_128BIT BLAKE2S_ATTRIB_128BIT #endif void Z7_FASTCALL Blake2s_Compress_V128_Way1(UInt32 * const s, const Byte * const input) { __m128i a, b, c, d; __m128i f0, f1; LOAD_ROTATE_CONSTS d = LOAD_128_FROM_STRUCT(STATE_T(s)); c = k_iv0_128; a = f0 = LOAD_128_FROM_STRUCT(s); b = f1 = LOAD_128_FROM_STRUCT(s + 4); D_ADD_EPI64_128(d, k_inc); STORE_128_TO_STRUCT (STATE_T(s), d); D_XOR_128(d, k_iv4_128); #define RR(r) { const Byte * const sigma = k_Blake2s_Sigma_4[r]; \ RR2(a, b, c, d, input) } ROUNDS_LOOP(RR) #undef RR STORE_128_TO_STRUCT(s , XOR_128(f0, XOR_128(a, c))); STORE_128_TO_STRUCT(s + 4, XOR_128(f1, XOR_128(b, d))); } static Z7_NO_INLINE #ifdef BLAKE2S_ATTRIB_128BIT BLAKE2S_ATTRIB_128BIT #endif void Z7_FASTCALL Blake2sp_Compress2_V128_Way1(UInt32 *s_items, const Byte *data, const Byte *end) { size_t pos = 0; do { UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); Blake2s_Compress_V128_Way1(s, data); data += Z7_BLAKE2S_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE; pos &= SUPER_BLOCK_MASK; } while (data != end); } #if defined(Z7_BLAKE2S_USE_V128_WAY2) || \ defined(Z7_BLAKE2S_USE_AVX2_WAY2) #if 1 #define Z7_BLAKE2S_CompressSingleBlock(s, data) \ Blake2sp_Compress2_V128_Way1(s, data, \ (const Byte *)(const void *)(data) + Z7_BLAKE2S_BLOCK_SIZE) #else #define Z7_BLAKE2S_CompressSingleBlock Blake2s_Compress_V128_Way1 #endif #endif #if (defined(Z7_BLAKE2S_USE_AVX2_WAY_SLOW) || \ defined(Z7_BLAKE2S_USE_V128_WAY2)) && \ !defined(Z7_BLAKE2S_USE_GATHER) #define AXR2_LOAD_INDEXES(sigma_index) \ const unsigned i0 = sigma[(sigma_index)]; \ const unsigned i1 = sigma[(sigma_index) + 2 * 1]; \ const unsigned i2 = sigma[(sigma_index) + 2 * 2]; \ const unsigned i3 = sigma[(sigma_index) + 2 * 3]; \ #define SET_ROW_FROM_SIGMA_W(input) \ SET_ROW_FROM_SIGMA_BASE(input, i0, i1, i2, i3) #endif #ifdef Z7_BLAKE2S_USE_V128_WAY2 #if 1 || !defined(Z7_BLAKE2S_USE_SSE41) /* we use SET_ROW_FROM_SIGMA_BASE, that uses (SSE4) _mm_insert_epi32(), if Z7_BLAKE2S_USE_INSERT_INSTRUCTION is defined (SSE2) _mm_set_epi32() MSVC can be faster for this branch: */ #define AXR2_W(sigma_index, shift1, shift2) \ { \ AXR2_LOAD_INDEXES(sigma_index) \ a0 = _mm_add_epi32(a0, SET_ROW_FROM_SIGMA_W(data)); \ a1 = _mm_add_epi32(a1, SET_ROW_FROM_SIGMA_W(data + Z7_BLAKE2S_BLOCK_SIZE)); \ AXR(a0, b0, d0, shift1) \ AXR(a1, b1, d1, shift1) \ AXR(c0, d0, b0, shift2) \ AXR(c1, d1, b1, shift2) \ } #else /* we use interleaved _mm_insert_epi32(): GCC can be faster for this branch: */ #define AXR2_W_PRE_INSERT(sigma_index, i) \ { const unsigned ii = sigma[(sigma_index) + i * 2]; \ t0 = _mm_insert_epi32(t0, *(const Int32 *)GET_SIGMA_PTR(data, ii), i); \ t1 = _mm_insert_epi32(t1, *(const Int32 *)GET_SIGMA_PTR(data, Z7_BLAKE2S_BLOCK_SIZE + ii), i); \ } #define AXR2_W(sigma_index, shift1, shift2) \ { __m128i t0, t1; \ { const unsigned ii = sigma[sigma_index]; \ t0 = _mm_cvtsi32_si128(*(const Int32 *)GET_SIGMA_PTR(data, ii)); \ t1 = _mm_cvtsi32_si128(*(const Int32 *)GET_SIGMA_PTR(data, Z7_BLAKE2S_BLOCK_SIZE + ii)); \ } \ AXR2_W_PRE_INSERT(sigma_index, 1) \ AXR2_W_PRE_INSERT(sigma_index, 2) \ AXR2_W_PRE_INSERT(sigma_index, 3) \ a0 = _mm_add_epi32(a0, t0); \ a1 = _mm_add_epi32(a1, t1); \ AXR(a0, b0, d0, shift1) \ AXR(a1, b1, d1, shift1) \ AXR(c0, d0, b0, shift2) \ AXR(c1, d1, b1, shift2) \ } #endif #define AXR4_W(sigma_index) \ AXR2_W(sigma_index, 16, 12) \ AXR2_W(sigma_index + 1, 8, 7) \ #define WW(r) \ { const Byte * const sigma = k_Blake2s_Sigma_4[r]; \ AXR4_W(0) \ ROTATE_WORDS_TO_RIGHT(b0, 1) \ ROTATE_WORDS_TO_RIGHT(b1, 1) \ ROTATE_WORDS_TO_RIGHT(c0, 2) \ ROTATE_WORDS_TO_RIGHT(c1, 2) \ ROTATE_WORDS_TO_RIGHT(d0, 3) \ ROTATE_WORDS_TO_RIGHT(d1, 3) \ AXR4_W(8) \ ROTATE_WORDS_TO_RIGHT(b0, 3) \ ROTATE_WORDS_TO_RIGHT(b1, 3) \ ROTATE_WORDS_TO_RIGHT(c0, 2) \ ROTATE_WORDS_TO_RIGHT(c1, 2) \ ROTATE_WORDS_TO_RIGHT(d0, 1) \ ROTATE_WORDS_TO_RIGHT(d1, 1) \ } static Z7_NO_INLINE #ifdef BLAKE2S_ATTRIB_128BIT BLAKE2S_ATTRIB_128BIT #endif void Z7_FASTCALL Blake2sp_Compress2_V128_Way2(UInt32 *s_items, const Byte *data, const Byte *end) { size_t pos = 0; end -= Z7_BLAKE2S_BLOCK_SIZE; if (data != end) { LOAD_ROTATE_CONSTS do { UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); __m128i a0, b0, c0, d0; __m128i a1, b1, c1, d1; { const __m128i inc = k_inc; const __m128i temp = k_iv4_128; d0 = LOAD_128_FROM_STRUCT (STATE_T(s)); d1 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW)); D_ADD_EPI64_128(d0, inc); D_ADD_EPI64_128(d1, inc); STORE_128_TO_STRUCT (STATE_T(s ), d0); STORE_128_TO_STRUCT (STATE_T(s + NSW), d1); D_XOR_128(d0, temp); D_XOR_128(d1, temp); } c1 = c0 = k_iv0_128; a0 = LOAD_128_FROM_STRUCT(s); b0 = LOAD_128_FROM_STRUCT(s + 4); a1 = LOAD_128_FROM_STRUCT(s + NSW); b1 = LOAD_128_FROM_STRUCT(s + NSW + 4); ROUNDS_LOOP (WW) #undef WW D_XOR_128(a0, c0); D_XOR_128(b0, d0); D_XOR_128(a1, c1); D_XOR_128(b1, d1); D_XOR_128(a0, LOAD_128_FROM_STRUCT(s)); D_XOR_128(b0, LOAD_128_FROM_STRUCT(s + 4)); D_XOR_128(a1, LOAD_128_FROM_STRUCT(s + NSW)); D_XOR_128(b1, LOAD_128_FROM_STRUCT(s + NSW + 4)); STORE_128_TO_STRUCT(s, a0); STORE_128_TO_STRUCT(s + 4, b0); STORE_128_TO_STRUCT(s + NSW, a1); STORE_128_TO_STRUCT(s + NSW + 4, b1); data += Z7_BLAKE2S_BLOCK_SIZE * 2; pos += Z7_BLAKE2S_BLOCK_SIZE * 2; pos &= SUPER_BLOCK_MASK; } while (data < end); if (data != end) return; } { UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); Z7_BLAKE2S_CompressSingleBlock(s, data); } } #endif // Z7_BLAKE2S_USE_V128_WAY2 #ifdef Z7_BLAKE2S_USE_V128_WAY2 #define Z7_BLAKE2S_Compress2_V128 Blake2sp_Compress2_V128_Way2 #else #define Z7_BLAKE2S_Compress2_V128 Blake2sp_Compress2_V128_Way1 #endif #ifdef Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED #define ROT_128_8(x) MM_ROR_EPI32(x, 8) #define ROT_128_16(x) MM_ROR_EPI32(x, 16) #define ROT_128_7(x) MM_ROR_EPI32(x, 7) #define ROT_128_12(x) MM_ROR_EPI32(x, 12) #else #if defined(Z7_BLAKE2S_USE_SSSE3) || defined(Z7_BLAKE2S_USE_SSE41) #define ROT_128_8(x) _mm_shuffle_epi8(x, r8) // k_r8 #define ROT_128_16(x) _mm_shuffle_epi8(x, r16) // k_r16 #else #define ROT_128_8(x) MM_ROR_EPI32_VIA_SHIFT(x, 8) #define ROT_128_16(x) MM_ROR_EPI32_VIA_SHIFT(x, 16) #endif #define ROT_128_7(x) MM_ROR_EPI32_VIA_SHIFT(x, 7) #define ROT_128_12(x) MM_ROR_EPI32_VIA_SHIFT(x, 12) #endif #if 1 // this branch can provide similar speed on x86* in most cases, // because [base + index*4] provides same speed as [base + index]. // but some compilers can generate different code with this branch, that can be faster sometimes. // this branch uses additional table of 10*16=160 bytes. #define SIGMA_TABLE_MULT_16( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ SIGMA_TABLE_MULT(16, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) MY_ALIGN(16) static const Byte k_Blake2s_Sigma_16[BLAKE2S_NUM_ROUNDS][16] = { SIGMA_TABLE(SIGMA_TABLE_MULT_16) }; #define GET_SIGMA_PTR_128(r) const Byte * const sigma = k_Blake2s_Sigma_16[r]; #define GET_SIGMA_VAL_128(n) (sigma[n]) #else #define GET_SIGMA_PTR_128(r) const Byte * const sigma = k_Blake2s_Sigma_4[r]; #define GET_SIGMA_VAL_128(n) (4 * (size_t)sigma[n]) #endif #ifdef Z7_BLAKE2S_USE_AVX2_FAST #if 1 #define SIGMA_TABLE_MULT_32( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ SIGMA_TABLE_MULT(32, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) MY_ALIGN(64) static const UInt16 k_Blake2s_Sigma_32[BLAKE2S_NUM_ROUNDS][16] = { SIGMA_TABLE(SIGMA_TABLE_MULT_32) }; #define GET_SIGMA_PTR_256(r) const UInt16 * const sigma = k_Blake2s_Sigma_32[r]; #define GET_SIGMA_VAL_256(n) (sigma[n]) #else #define GET_SIGMA_PTR_256(r) const Byte * const sigma = k_Blake2s_Sigma_4[r]; #define GET_SIGMA_VAL_256(n) (8 * (size_t)sigma[n]) #endif #endif // Z7_BLAKE2S_USE_AVX2_FAST #define D_ROT_128_7(dest) dest = ROT_128_7(dest) #define D_ROT_128_8(dest) dest = ROT_128_8(dest) #define D_ROT_128_12(dest) dest = ROT_128_12(dest) #define D_ROT_128_16(dest) dest = ROT_128_16(dest) #define OP_L(a, i) D_ADD_128 (V(a, 0), \ LOAD_128((const Byte *)(w) + GET_SIGMA_VAL_128(2*(a)+(i)))); #define OP_0(a) OP_L(a, 0) #define OP_7(a) OP_L(a, 1) #define OP_1(a) D_ADD_128 (V(a, 0), V(a, 1)); #define OP_2(a) D_XOR_128 (V(a, 3), V(a, 0)); #define OP_4(a) D_ADD_128 (V(a, 2), V(a, 3)); #define OP_5(a) D_XOR_128 (V(a, 1), V(a, 2)); #define OP_3(a) D_ROT_128_16 (V(a, 3)); #define OP_6(a) D_ROT_128_12 (V(a, 1)); #define OP_8(a) D_ROT_128_8 (V(a, 3)); #define OP_9(a) D_ROT_128_7 (V(a, 1)); // for 32-bit x86 : interleave mode works slower, because of register pressure. #if 0 || 1 && (defined(MY_CPU_X86) \ || defined(__GNUC__) && !defined(__clang__)) // non-inteleaved version: // is fast for x86 32-bit. // is fast for GCC x86-64. #define V4G(a) \ OP_0 (a) \ OP_1 (a) \ OP_2 (a) \ OP_3 (a) \ OP_4 (a) \ OP_5 (a) \ OP_6 (a) \ OP_7 (a) \ OP_1 (a) \ OP_2 (a) \ OP_8 (a) \ OP_4 (a) \ OP_5 (a) \ OP_9 (a) \ #define V4R \ { \ V4G (0) \ V4G (1) \ V4G (2) \ V4G (3) \ V4G (4) \ V4G (5) \ V4G (6) \ V4G (7) \ } #elif 0 || 1 && defined(MY_CPU_X86) #define OP_INTER_2(op, a,b) \ op (a) \ op (b) \ #define V4G(a,b) \ OP_INTER_2 (OP_0, a,b) \ OP_INTER_2 (OP_1, a,b) \ OP_INTER_2 (OP_2, a,b) \ OP_INTER_2 (OP_3, a,b) \ OP_INTER_2 (OP_4, a,b) \ OP_INTER_2 (OP_5, a,b) \ OP_INTER_2 (OP_6, a,b) \ OP_INTER_2 (OP_7, a,b) \ OP_INTER_2 (OP_1, a,b) \ OP_INTER_2 (OP_2, a,b) \ OP_INTER_2 (OP_8, a,b) \ OP_INTER_2 (OP_4, a,b) \ OP_INTER_2 (OP_5, a,b) \ OP_INTER_2 (OP_9, a,b) \ #define V4R \ { \ V4G (0, 1) \ V4G (2, 3) \ V4G (4, 5) \ V4G (6, 7) \ } #else // iterleave-4 version is fast for x64 (MSVC/CLANG) #define OP_INTER_4(op, a,b,c,d) \ op (a) \ op (b) \ op (c) \ op (d) \ #define V4G(a,b,c,d) \ OP_INTER_4 (OP_0, a,b,c,d) \ OP_INTER_4 (OP_1, a,b,c,d) \ OP_INTER_4 (OP_2, a,b,c,d) \ OP_INTER_4 (OP_3, a,b,c,d) \ OP_INTER_4 (OP_4, a,b,c,d) \ OP_INTER_4 (OP_5, a,b,c,d) \ OP_INTER_4 (OP_6, a,b,c,d) \ OP_INTER_4 (OP_7, a,b,c,d) \ OP_INTER_4 (OP_1, a,b,c,d) \ OP_INTER_4 (OP_2, a,b,c,d) \ OP_INTER_4 (OP_8, a,b,c,d) \ OP_INTER_4 (OP_4, a,b,c,d) \ OP_INTER_4 (OP_5, a,b,c,d) \ OP_INTER_4 (OP_9, a,b,c,d) \ #define V4R \ { \ V4G (0, 1, 2, 3) \ V4G (4, 5, 6, 7) \ } #endif #define V4_ROUND(r) { GET_SIGMA_PTR_128(r); V4R } #define V4_LOAD_MSG_1(w, m, i) \ { \ __m128i m0, m1, m2, m3; \ __m128i t0, t1, t2, t3; \ m0 = LOADU_128((m) + ((i) + 0 * 4) * 16); \ m1 = LOADU_128((m) + ((i) + 1 * 4) * 16); \ m2 = LOADU_128((m) + ((i) + 2 * 4) * 16); \ m3 = LOADU_128((m) + ((i) + 3 * 4) * 16); \ t0 = _mm_unpacklo_epi32(m0, m1); \ t1 = _mm_unpackhi_epi32(m0, m1); \ t2 = _mm_unpacklo_epi32(m2, m3); \ t3 = _mm_unpackhi_epi32(m2, m3); \ w[(i) * 4 + 0] = _mm_unpacklo_epi64(t0, t2); \ w[(i) * 4 + 1] = _mm_unpackhi_epi64(t0, t2); \ w[(i) * 4 + 2] = _mm_unpacklo_epi64(t1, t3); \ w[(i) * 4 + 3] = _mm_unpackhi_epi64(t1, t3); \ } #define V4_LOAD_MSG(w, m) \ { \ V4_LOAD_MSG_1 (w, m, 0) \ V4_LOAD_MSG_1 (w, m, 1) \ V4_LOAD_MSG_1 (w, m, 2) \ V4_LOAD_MSG_1 (w, m, 3) \ } #define V4_LOAD_UNPACK_PAIR_128(src32, i, d0, d1) \ { \ const __m128i v0 = LOAD_128_FROM_STRUCT((src32) + (i ) * 4); \ const __m128i v1 = LOAD_128_FROM_STRUCT((src32) + (i + 1) * 4); \ d0 = _mm_unpacklo_epi32(v0, v1); \ d1 = _mm_unpackhi_epi32(v0, v1); \ } #define V4_UNPACK_PAIR_128(dest32, i, s0, s1) \ { \ STORE_128_TO_STRUCT((dest32) + i * 4 , _mm_unpacklo_epi64(s0, s1)); \ STORE_128_TO_STRUCT((dest32) + i * 4 + 16, _mm_unpackhi_epi64(s0, s1)); \ } #define V4_UNPACK_STATE(dest32, src32) \ { \ __m128i t0, t1, t2, t3, t4, t5, t6, t7; \ V4_LOAD_UNPACK_PAIR_128(src32, 0, t0, t1) \ V4_LOAD_UNPACK_PAIR_128(src32, 2, t2, t3) \ V4_LOAD_UNPACK_PAIR_128(src32, 4, t4, t5) \ V4_LOAD_UNPACK_PAIR_128(src32, 6, t6, t7) \ V4_UNPACK_PAIR_128(dest32, 0, t0, t2) \ V4_UNPACK_PAIR_128(dest32, 8, t1, t3) \ V4_UNPACK_PAIR_128(dest32, 1, t4, t6) \ V4_UNPACK_PAIR_128(dest32, 9, t5, t7) \ } static Z7_NO_INLINE #ifdef BLAKE2S_ATTRIB_128BIT BLAKE2S_ATTRIB_128BIT #endif void Z7_FASTCALL Blake2sp_Compress2_V128_Fast(UInt32 *s_items, const Byte *data, const Byte *end) { // PrintStates2(s_items, 8, 16); size_t pos = 0; pos /= 2; do { #if defined(Z7_BLAKE2S_USE_SSSE3) && \ !defined(Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED) const __m128i r8 = k_r8; const __m128i r16 = k_r16; #endif __m128i w[16]; __m128i v[16]; UInt32 *s; V4_LOAD_MSG(w, data) s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); { __m128i ctr = LOAD_128_FROM_STRUCT(s + 64); D_ADD_EPI64_128 (ctr, k_inc); STORE_128_TO_STRUCT(s + 64, ctr); v[12] = XOR_128 (GET_128_IV_WAY4(4), _mm_shuffle_epi32(ctr, _MM_SHUFFLE(0, 0, 0, 0))); v[13] = XOR_128 (GET_128_IV_WAY4(5), _mm_shuffle_epi32(ctr, _MM_SHUFFLE(1, 1, 1, 1))); } v[ 8] = GET_128_IV_WAY4(0); v[ 9] = GET_128_IV_WAY4(1); v[10] = GET_128_IV_WAY4(2); v[11] = GET_128_IV_WAY4(3); v[14] = GET_128_IV_WAY4(6); v[15] = GET_128_IV_WAY4(7); #define LOAD_STATE_128_FROM_STRUCT(i) \ v[i] = LOAD_128_FROM_STRUCT(s + (i) * 4); #define UPDATE_STATE_128_IN_STRUCT(i) \ STORE_128_TO_STRUCT(s + (i) * 4, XOR_128( \ XOR_128(v[i], v[(i) + 8]), \ LOAD_128_FROM_STRUCT(s + (i) * 4))); REP8_MACRO (LOAD_STATE_128_FROM_STRUCT) ROUNDS_LOOP (V4_ROUND) REP8_MACRO (UPDATE_STATE_128_IN_STRUCT) data += Z7_BLAKE2S_BLOCK_SIZE * 4; pos += Z7_BLAKE2S_BLOCK_SIZE * 4 / 2; pos &= SUPER_BLOCK_SIZE / 2 - 1; } while (data != end); } static Z7_NO_INLINE #ifdef BLAKE2S_ATTRIB_128BIT BLAKE2S_ATTRIB_128BIT #endif void Z7_FASTCALL Blake2sp_Final_V128_Fast(UInt32 *states) { const __m128i ctr = LOAD_128_FROM_STRUCT(states + 64); // printf("\nBlake2sp_Compress2_V128_Fast_Final4\n"); // PrintStates2(states, 8, 16); { ptrdiff_t pos = 8 * 4; do { UInt32 *src32 = states + (size_t)(pos * 1); UInt32 *dest32 = states + (size_t)(pos * 2); V4_UNPACK_STATE(dest32, src32) pos -= 8 * 4; } while (pos >= 0); } { unsigned k; for (k = 0; k < 8; k++) { UInt32 *s = states + (size_t)k * 16; STORE_128_TO_STRUCT (STATE_T(s), ctr); } } // PrintStates2(states, 8, 16); } #ifdef Z7_BLAKE2S_USE_AVX2 #define ADD_256(a, b) _mm256_add_epi32(a, b) #define XOR_256(a, b) _mm256_xor_si256(a, b) #if 1 && defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) #define MM256_ROR_EPI32 _mm256_ror_epi32 #define Z7_MM256_ROR_EPI32_IS_SUPPORTED #ifdef Z7_BLAKE2S_USE_AVX2_WAY2 #define LOAD_ROTATE_CONSTS_256 #endif #else #ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW #ifdef Z7_BLAKE2S_USE_AVX2_WAY2 #define LOAD_ROTATE_CONSTS_256 \ const __m256i r8 = k_r8_256; \ const __m256i r16 = k_r16_256; #endif // AVX2_WAY2 #define MM256_ROR_EPI32(r, c) ( \ ( 8==(c)) ? _mm256_shuffle_epi8(r,r8) \ : (16==(c)) ? _mm256_shuffle_epi8(r,r16) \ : _mm256_or_si256( \ _mm256_srli_epi32((r), (c)), \ _mm256_slli_epi32((r), 32-(c)))) #endif // WAY_SLOW #endif #define D_ADD_256(dest, src) dest = ADD_256(dest, src) #define D_XOR_256(dest, src) dest = XOR_256(dest, src) #define LOADU_256(p) _mm256_loadu_si256((const __m256i *)(const void *)(p)) #ifdef Z7_BLAKE2S_USE_AVX2_FAST #ifdef Z7_MM256_ROR_EPI32_IS_SUPPORTED #define ROT_256_16(x) MM256_ROR_EPI32((x), 16) #define ROT_256_12(x) MM256_ROR_EPI32((x), 12) #define ROT_256_8(x) MM256_ROR_EPI32((x), 8) #define ROT_256_7(x) MM256_ROR_EPI32((x), 7) #else #define ROTATE8 _mm256_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1, \ 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1) #define ROTATE16 _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, \ 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2) #define ROT_256_16(x) _mm256_shuffle_epi8((x), ROTATE16) #define ROT_256_12(x) _mm256_or_si256(_mm256_srli_epi32((x), 12), _mm256_slli_epi32((x), 20)) #define ROT_256_8(x) _mm256_shuffle_epi8((x), ROTATE8) #define ROT_256_7(x) _mm256_or_si256(_mm256_srli_epi32((x), 7), _mm256_slli_epi32((x), 25)) #endif #define D_ROT_256_7(dest) dest = ROT_256_7(dest) #define D_ROT_256_8(dest) dest = ROT_256_8(dest) #define D_ROT_256_12(dest) dest = ROT_256_12(dest) #define D_ROT_256_16(dest) dest = ROT_256_16(dest) #define LOAD_256(p) _mm256_load_si256((const __m256i *)(const void *)(p)) #ifdef Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED #define STOREU_256(p, r) _mm256_storeu_si256((__m256i *)(void *)(p), r) #define LOAD_256_FROM_STRUCT(p) LOADU_256(p) #define STORE_256_TO_STRUCT(p, r) STOREU_256(p, r) #else // if struct is aligned for 32-bytes #define STORE_256(p, r) _mm256_store_si256((__m256i *)(void *)(p), r) #define LOAD_256_FROM_STRUCT(p) LOAD_256(p) #define STORE_256_TO_STRUCT(p, r) STORE_256(p, r) #endif #endif // Z7_BLAKE2S_USE_AVX2_FAST #ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW #if 0 #define DIAG_PERM2(s) \ { \ const __m256i a = LOAD_256_FROM_STRUCT((s) ); \ const __m256i b = LOAD_256_FROM_STRUCT((s) + NSW); \ STORE_256_TO_STRUCT((s ), _mm256_permute2x128_si256(a, b, 0x20)); \ STORE_256_TO_STRUCT((s + NSW), _mm256_permute2x128_si256(a, b, 0x31)); \ } #else #define DIAG_PERM2(s) \ { \ const __m128i a = LOAD_128_FROM_STRUCT((s) + 4); \ const __m128i b = LOAD_128_FROM_STRUCT((s) + NSW); \ STORE_128_TO_STRUCT((s) + NSW, a); \ STORE_128_TO_STRUCT((s) + 4 , b); \ } #endif #define DIAG_PERM8(s_items) \ { \ DIAG_PERM2(s_items) \ DIAG_PERM2(s_items + NSW * 2) \ DIAG_PERM2(s_items + NSW * 4) \ DIAG_PERM2(s_items + NSW * 6) \ } #define AXR256(a, b, d, shift) \ D_ADD_256(a, b); \ D_XOR_256(d, a); \ d = MM256_ROR_EPI32(d, shift); \ #ifdef Z7_BLAKE2S_USE_GATHER #define TABLE_GATHER_256_4(a0,a1,a2,a3) \ a0,a1,a2,a3, a0+16,a1+16,a2+16,a3+16 #define TABLE_GATHER_256( \ a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ { TABLE_GATHER_256_4(a0,a2,a4,a6), \ TABLE_GATHER_256_4(a1,a3,a5,a7), \ TABLE_GATHER_256_4(a8,a10,a12,a14), \ TABLE_GATHER_256_4(a9,a11,a13,a15) } MY_ALIGN(64) static const UInt32 k_Blake2s_Sigma_gather256[BLAKE2S_NUM_ROUNDS][16 * 2] = { SIGMA_TABLE(TABLE_GATHER_256) }; #define GET_SIGMA(r) \ const UInt32 * const sigma = k_Blake2s_Sigma_gather256[r]; #define AXR2_LOAD_INDEXES_AVX(sigma_index) \ const __m256i i01234567 = LOAD_256(sigma + (sigma_index)); #define SET_ROW_FROM_SIGMA_AVX(in) \ _mm256_i32gather_epi32((const void *)(in), i01234567, 4) #define SIGMA_INTERLEAVE 8 #define SIGMA_HALF_ROW_SIZE 16 #else // !Z7_BLAKE2S_USE_GATHER #define GET_SIGMA(r) \ const Byte * const sigma = k_Blake2s_Sigma_4[r]; #define AXR2_LOAD_INDEXES_AVX(sigma_index) \ AXR2_LOAD_INDEXES(sigma_index) #define SET_ROW_FROM_SIGMA_AVX(in) \ MY_mm256_set_m128i( \ SET_ROW_FROM_SIGMA_W((in) + Z7_BLAKE2S_BLOCK_SIZE), \ SET_ROW_FROM_SIGMA_W(in)) #define SIGMA_INTERLEAVE 1 #define SIGMA_HALF_ROW_SIZE 8 #endif // !Z7_BLAKE2S_USE_GATHER #define ROTATE_WORDS_TO_RIGHT_256(a, n) \ a = _mm256_shuffle_epi32(a, _MM_SHUFFLE((3+n)&3, (2+n)&3, (1+n)&3, (0+n)&3)); #ifdef Z7_BLAKE2S_USE_AVX2_WAY2 #define AXR2_A(sigma_index, shift1, shift2) \ AXR2_LOAD_INDEXES_AVX(sigma_index) \ D_ADD_256( a0, SET_ROW_FROM_SIGMA_AVX(data)); \ AXR256(a0, b0, d0, shift1) \ AXR256(c0, d0, b0, shift2) \ #define AXR4_A(sigma_index) \ { AXR2_A(sigma_index, 16, 12) } \ { AXR2_A(sigma_index + SIGMA_INTERLEAVE, 8, 7) } #define EE1(r) \ { GET_SIGMA(r) \ AXR4_A(0) \ ROTATE_WORDS_TO_RIGHT_256(b0, 1) \ ROTATE_WORDS_TO_RIGHT_256(c0, 2) \ ROTATE_WORDS_TO_RIGHT_256(d0, 3) \ AXR4_A(SIGMA_HALF_ROW_SIZE) \ ROTATE_WORDS_TO_RIGHT_256(b0, 3) \ ROTATE_WORDS_TO_RIGHT_256(c0, 2) \ ROTATE_WORDS_TO_RIGHT_256(d0, 1) \ } static Z7_NO_INLINE #ifdef BLAKE2S_ATTRIB_AVX2 BLAKE2S_ATTRIB_AVX2 #endif void Z7_FASTCALL Blake2sp_Compress2_AVX2_Way2(UInt32 *s_items, const Byte *data, const Byte *end) { size_t pos = 0; end -= Z7_BLAKE2S_BLOCK_SIZE; if (data != end) { LOAD_ROTATE_CONSTS_256 DIAG_PERM8(s_items) do { UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); __m256i a0, b0, c0, d0; { const __m128i inc = k_inc; __m128i d0_128 = LOAD_128_FROM_STRUCT (STATE_T(s)); __m128i d1_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW)); D_ADD_EPI64_128(d0_128, inc); D_ADD_EPI64_128(d1_128, inc); STORE_128_TO_STRUCT (STATE_T(s ), d0_128); STORE_128_TO_STRUCT (STATE_T(s + NSW), d1_128); d0 = MY_mm256_set_m128i(d1_128, d0_128); D_XOR_256(d0, k_iv4_256); } c0 = SET_FROM_128(k_iv0_128); a0 = LOAD_256_FROM_STRUCT(s + NSW * 0); b0 = LOAD_256_FROM_STRUCT(s + NSW * 1); ROUNDS_LOOP (EE1) D_XOR_256(a0, c0); D_XOR_256(b0, d0); D_XOR_256(a0, LOAD_256_FROM_STRUCT(s + NSW * 0)); D_XOR_256(b0, LOAD_256_FROM_STRUCT(s + NSW * 1)); STORE_256_TO_STRUCT(s + NSW * 0, a0); STORE_256_TO_STRUCT(s + NSW * 1, b0); data += Z7_BLAKE2S_BLOCK_SIZE * 2; pos += Z7_BLAKE2S_BLOCK_SIZE * 2; pos &= SUPER_BLOCK_MASK; } while (data < end); DIAG_PERM8(s_items) if (data != end) return; } { UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); Z7_BLAKE2S_CompressSingleBlock(s, data); } } #endif // Z7_BLAKE2S_USE_AVX2_WAY2 #ifdef Z7_BLAKE2S_USE_AVX2_WAY4 #define AXR2_X(sigma_index, shift1, shift2) \ AXR2_LOAD_INDEXES_AVX(sigma_index) \ D_ADD_256( a0, SET_ROW_FROM_SIGMA_AVX(data)); \ D_ADD_256( a1, SET_ROW_FROM_SIGMA_AVX((data) + Z7_BLAKE2S_BLOCK_SIZE * 2)); \ AXR256(a0, b0, d0, shift1) \ AXR256(a1, b1, d1, shift1) \ AXR256(c0, d0, b0, shift2) \ AXR256(c1, d1, b1, shift2) \ #define AXR4_X(sigma_index) \ { AXR2_X(sigma_index, 16, 12) } \ { AXR2_X(sigma_index + SIGMA_INTERLEAVE, 8, 7) } #define EE2(r) \ { GET_SIGMA(r) \ AXR4_X(0) \ ROTATE_WORDS_TO_RIGHT_256(b0, 1) \ ROTATE_WORDS_TO_RIGHT_256(b1, 1) \ ROTATE_WORDS_TO_RIGHT_256(c0, 2) \ ROTATE_WORDS_TO_RIGHT_256(c1, 2) \ ROTATE_WORDS_TO_RIGHT_256(d0, 3) \ ROTATE_WORDS_TO_RIGHT_256(d1, 3) \ AXR4_X(SIGMA_HALF_ROW_SIZE) \ ROTATE_WORDS_TO_RIGHT_256(b0, 3) \ ROTATE_WORDS_TO_RIGHT_256(b1, 3) \ ROTATE_WORDS_TO_RIGHT_256(c0, 2) \ ROTATE_WORDS_TO_RIGHT_256(c1, 2) \ ROTATE_WORDS_TO_RIGHT_256(d0, 1) \ ROTATE_WORDS_TO_RIGHT_256(d1, 1) \ } static Z7_NO_INLINE #ifdef BLAKE2S_ATTRIB_AVX2 BLAKE2S_ATTRIB_AVX2 #endif void Z7_FASTCALL Blake2sp_Compress2_AVX2_Way4(UInt32 *s_items, const Byte *data, const Byte *end) { size_t pos = 0; if ((size_t)(end - data) >= Z7_BLAKE2S_BLOCK_SIZE * 4) { #ifndef Z7_MM256_ROR_EPI32_IS_SUPPORTED const __m256i r8 = k_r8_256; const __m256i r16 = k_r16_256; #endif end -= Z7_BLAKE2S_BLOCK_SIZE * 3; DIAG_PERM8(s_items) do { UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); __m256i a0, b0, c0, d0; __m256i a1, b1, c1, d1; { const __m128i inc = k_inc; __m128i d0_128 = LOAD_128_FROM_STRUCT (STATE_T(s)); __m128i d1_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW)); __m128i d2_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW * 2)); __m128i d3_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW * 3)); D_ADD_EPI64_128(d0_128, inc); D_ADD_EPI64_128(d1_128, inc); D_ADD_EPI64_128(d2_128, inc); D_ADD_EPI64_128(d3_128, inc); STORE_128_TO_STRUCT (STATE_T(s ), d0_128); STORE_128_TO_STRUCT (STATE_T(s + NSW * 1), d1_128); STORE_128_TO_STRUCT (STATE_T(s + NSW * 2), d2_128); STORE_128_TO_STRUCT (STATE_T(s + NSW * 3), d3_128); d0 = MY_mm256_set_m128i(d1_128, d0_128); d1 = MY_mm256_set_m128i(d3_128, d2_128); D_XOR_256(d0, k_iv4_256); D_XOR_256(d1, k_iv4_256); } c1 = c0 = SET_FROM_128(k_iv0_128); a0 = LOAD_256_FROM_STRUCT(s + NSW * 0); b0 = LOAD_256_FROM_STRUCT(s + NSW * 1); a1 = LOAD_256_FROM_STRUCT(s + NSW * 2); b1 = LOAD_256_FROM_STRUCT(s + NSW * 3); ROUNDS_LOOP (EE2) D_XOR_256(a0, c0); D_XOR_256(b0, d0); D_XOR_256(a1, c1); D_XOR_256(b1, d1); D_XOR_256(a0, LOAD_256_FROM_STRUCT(s + NSW * 0)); D_XOR_256(b0, LOAD_256_FROM_STRUCT(s + NSW * 1)); D_XOR_256(a1, LOAD_256_FROM_STRUCT(s + NSW * 2)); D_XOR_256(b1, LOAD_256_FROM_STRUCT(s + NSW * 3)); STORE_256_TO_STRUCT(s + NSW * 0, a0); STORE_256_TO_STRUCT(s + NSW * 1, b0); STORE_256_TO_STRUCT(s + NSW * 2, a1); STORE_256_TO_STRUCT(s + NSW * 3, b1); data += Z7_BLAKE2S_BLOCK_SIZE * 4; pos += Z7_BLAKE2S_BLOCK_SIZE * 4; pos &= SUPER_BLOCK_MASK; } while (data < end); DIAG_PERM8(s_items) end += Z7_BLAKE2S_BLOCK_SIZE * 3; } if (data == end) return; // Z7_BLAKE2S_Compress2_V128(s_items, data, end, pos); do { UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); Z7_BLAKE2S_CompressSingleBlock(s, data); data += Z7_BLAKE2S_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE; pos &= SUPER_BLOCK_MASK; } while (data != end); } #endif // Z7_BLAKE2S_USE_AVX2_WAY4 #endif // Z7_BLAKE2S_USE_AVX2_WAY_SLOW // --------------------------------------------------------- #ifdef Z7_BLAKE2S_USE_AVX2_FAST #define OP256_L(a, i) D_ADD_256 (V(a, 0), \ LOAD_256((const Byte *)(w) + GET_SIGMA_VAL_256(2*(a)+(i)))); #define OP256_0(a) OP256_L(a, 0) #define OP256_7(a) OP256_L(a, 1) #define OP256_1(a) D_ADD_256 (V(a, 0), V(a, 1)); #define OP256_2(a) D_XOR_256 (V(a, 3), V(a, 0)); #define OP256_4(a) D_ADD_256 (V(a, 2), V(a, 3)); #define OP256_5(a) D_XOR_256 (V(a, 1), V(a, 2)); #define OP256_3(a) D_ROT_256_16 (V(a, 3)); #define OP256_6(a) D_ROT_256_12 (V(a, 1)); #define OP256_8(a) D_ROT_256_8 (V(a, 3)); #define OP256_9(a) D_ROT_256_7 (V(a, 1)); #if 0 || 1 && defined(MY_CPU_X86) #define V8_G(a) \ OP256_0 (a) \ OP256_1 (a) \ OP256_2 (a) \ OP256_3 (a) \ OP256_4 (a) \ OP256_5 (a) \ OP256_6 (a) \ OP256_7 (a) \ OP256_1 (a) \ OP256_2 (a) \ OP256_8 (a) \ OP256_4 (a) \ OP256_5 (a) \ OP256_9 (a) \ #define V8R { \ V8_G (0); \ V8_G (1); \ V8_G (2); \ V8_G (3); \ V8_G (4); \ V8_G (5); \ V8_G (6); \ V8_G (7); \ } #else #define OP256_INTER_4(op, a,b,c,d) \ op (a) \ op (b) \ op (c) \ op (d) \ #define V8_G(a,b,c,d) \ OP256_INTER_4 (OP256_0, a,b,c,d) \ OP256_INTER_4 (OP256_1, a,b,c,d) \ OP256_INTER_4 (OP256_2, a,b,c,d) \ OP256_INTER_4 (OP256_3, a,b,c,d) \ OP256_INTER_4 (OP256_4, a,b,c,d) \ OP256_INTER_4 (OP256_5, a,b,c,d) \ OP256_INTER_4 (OP256_6, a,b,c,d) \ OP256_INTER_4 (OP256_7, a,b,c,d) \ OP256_INTER_4 (OP256_1, a,b,c,d) \ OP256_INTER_4 (OP256_2, a,b,c,d) \ OP256_INTER_4 (OP256_8, a,b,c,d) \ OP256_INTER_4 (OP256_4, a,b,c,d) \ OP256_INTER_4 (OP256_5, a,b,c,d) \ OP256_INTER_4 (OP256_9, a,b,c,d) \ #define V8R { \ V8_G (0, 1, 2, 3) \ V8_G (4, 5, 6, 7) \ } #endif #define V8_ROUND(r) { GET_SIGMA_PTR_256(r); V8R } // for debug: // #define Z7_BLAKE2S_PERMUTE_WITH_GATHER #if defined(Z7_BLAKE2S_PERMUTE_WITH_GATHER) // gather instruction is slow. #define V8_LOAD_MSG(w, m) \ { \ unsigned i; \ for (i = 0; i < 16; ++i) { \ w[i] = _mm256_i32gather_epi32( \ (const void *)((m) + i * sizeof(UInt32)),\ _mm256_set_epi32(0x70, 0x60, 0x50, 0x40, 0x30, 0x20, 0x10, 0x00), \ sizeof(UInt32)); \ } \ } #else // !Z7_BLAKE2S_PERMUTE_WITH_GATHER #define V8_LOAD_MSG_2(w, a0, a1) \ { \ (w)[0] = _mm256_permute2x128_si256(a0, a1, 0x20); \ (w)[4] = _mm256_permute2x128_si256(a0, a1, 0x31); \ } #define V8_LOAD_MSG_4(w, z0, z1, z2, z3) \ { \ __m256i s0, s1, s2, s3; \ s0 = _mm256_unpacklo_epi64(z0, z1); \ s1 = _mm256_unpackhi_epi64(z0, z1); \ s2 = _mm256_unpacklo_epi64(z2, z3); \ s3 = _mm256_unpackhi_epi64(z2, z3); \ V8_LOAD_MSG_2((w) + 0, s0, s2) \ V8_LOAD_MSG_2((w) + 1, s1, s3) \ } #define V8_LOAD_MSG_0(t0, t1, m) \ { \ __m256i m0, m1; \ m0 = LOADU_256(m); \ m1 = LOADU_256((m) + 2 * 32); \ t0 = _mm256_unpacklo_epi32(m0, m1); \ t1 = _mm256_unpackhi_epi32(m0, m1); \ } #define V8_LOAD_MSG_8(w, m) \ { \ __m256i t0, t1, t2, t3, t4, t5, t6, t7; \ V8_LOAD_MSG_0(t0, t4, (m) + 0 * 4 * 32) \ V8_LOAD_MSG_0(t1, t5, (m) + 1 * 4 * 32) \ V8_LOAD_MSG_0(t2, t6, (m) + 2 * 4 * 32) \ V8_LOAD_MSG_0(t3, t7, (m) + 3 * 4 * 32) \ V8_LOAD_MSG_4((w) , t0, t1, t2, t3) \ V8_LOAD_MSG_4((w) + 2, t4, t5, t6, t7) \ } #define V8_LOAD_MSG(w, m) \ { \ V8_LOAD_MSG_8(w, m) \ V8_LOAD_MSG_8((w) + 8, (m) + 32) \ } #endif // !Z7_BLAKE2S_PERMUTE_WITH_GATHER #define V8_PERM_PAIR_STORE(u, a0, a2) \ { \ STORE_256_TO_STRUCT((u), _mm256_permute2x128_si256(a0, a2, 0x20)); \ STORE_256_TO_STRUCT((u) + 8, _mm256_permute2x128_si256(a0, a2, 0x31)); \ } #define V8_UNPACK_STORE_4(u, z0, z1, z2, z3) \ { \ __m256i s0, s1, s2, s3; \ s0 = _mm256_unpacklo_epi64(z0, z1); \ s1 = _mm256_unpackhi_epi64(z0, z1); \ s2 = _mm256_unpacklo_epi64(z2, z3); \ s3 = _mm256_unpackhi_epi64(z2, z3); \ V8_PERM_PAIR_STORE(u + 0, s0, s2) \ V8_PERM_PAIR_STORE(u + 2, s1, s3) \ } #define V8_UNPACK_STORE_0(src32, d0, d1) \ { \ const __m256i v0 = LOAD_256_FROM_STRUCT ((src32) ); \ const __m256i v1 = LOAD_256_FROM_STRUCT ((src32) + 8); \ d0 = _mm256_unpacklo_epi32(v0, v1); \ d1 = _mm256_unpackhi_epi32(v0, v1); \ } #define V8_UNPACK_STATE(dest32, src32) \ { \ __m256i t0, t1, t2, t3, t4, t5, t6, t7; \ V8_UNPACK_STORE_0 ((src32) + 16 * 0, t0, t4) \ V8_UNPACK_STORE_0 ((src32) + 16 * 1, t1, t5) \ V8_UNPACK_STORE_0 ((src32) + 16 * 2, t2, t6) \ V8_UNPACK_STORE_0 ((src32) + 16 * 3, t3, t7) \ V8_UNPACK_STORE_4 ((__m256i *)(void *)(dest32) , t0, t1, t2, t3) \ V8_UNPACK_STORE_4 ((__m256i *)(void *)(dest32) + 4, t4, t5, t6, t7) \ } #define V8_LOAD_STATE_256_FROM_STRUCT(i) \ v[i] = LOAD_256_FROM_STRUCT(s_items + (i) * 8); #if 0 || 0 && defined(MY_CPU_X86) #define Z7_BLAKE2S_AVX2_FAST_USE_STRUCT #endif #ifdef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT // this branch doesn't use (iv) array // so register pressure can be lower. // it can be faster sometimes #define V8_LOAD_STATE_256(i) V8_LOAD_STATE_256_FROM_STRUCT(i) #define V8_UPDATE_STATE_256(i) \ { \ STORE_256_TO_STRUCT(s_items + (i) * 8, XOR_256( \ XOR_256(v[i], v[(i) + 8]), \ LOAD_256_FROM_STRUCT(s_items + (i) * 8))); \ } #else // it uses more variables (iv) registers // it's better for gcc // maybe that branch is better, if register pressure will be lower (avx512) #define V8_LOAD_STATE_256(i) { iv[i] = v[i]; } #define V8_UPDATE_STATE_256(i) { v[i] = XOR_256(XOR_256(v[i], v[i + 8]), iv[i]); } #define V8_STORE_STATE_256(i) { STORE_256_TO_STRUCT(s_items + (i) * 8, v[i]); } #endif #if 0 // use loading constants from memory #define KK8(n) KIV(n), KIV(n), KIV(n), KIV(n), KIV(n), KIV(n), KIV(n), KIV(n) MY_ALIGN(64) static const UInt32 k_Blake2s_IV_WAY8[]= { KK8(0), KK8(1), KK8(2), KK8(3), KK8(4), KK8(5), KK8(6), KK8(7) }; #define GET_256_IV_WAY8(i) LOAD_256(k_Blake2s_IV_WAY8 + 8 * (i)) #else // use constant generation: #define GET_256_IV_WAY8(i) _mm256_set1_epi32((Int32)KIV(i)) #endif static Z7_NO_INLINE #ifdef BLAKE2S_ATTRIB_AVX2 BLAKE2S_ATTRIB_AVX2 #endif void Z7_FASTCALL Blake2sp_Compress2_AVX2_Fast(UInt32 *s_items, const Byte *data, const Byte *end) { #ifndef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT __m256i v[16]; #endif // PrintStates2(s_items, 8, 16); #ifndef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT REP8_MACRO (V8_LOAD_STATE_256_FROM_STRUCT) #endif do { __m256i w[16]; #ifdef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT __m256i v[16]; #else __m256i iv[8]; #endif V8_LOAD_MSG(w, data) { // we use load/store ctr inside loop to reduce register pressure: #if 1 || 1 && defined(MY_CPU_X86) const __m256i ctr = _mm256_add_epi64( LOAD_256_FROM_STRUCT(s_items + 64), _mm256_set_epi32( 0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE, 0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE)); STORE_256_TO_STRUCT(s_items + 64, ctr); #else const UInt64 ctr64 = *(const UInt64 *)(const void *)(s_items + 64) + Z7_BLAKE2S_BLOCK_SIZE; const __m256i ctr = _mm256_set_epi64x(0, (Int64)ctr64, 0, (Int64)ctr64); *(UInt64 *)(void *)(s_items + 64) = ctr64; #endif v[12] = XOR_256 (GET_256_IV_WAY8(4), _mm256_shuffle_epi32(ctr, _MM_SHUFFLE(0, 0, 0, 0))); v[13] = XOR_256 (GET_256_IV_WAY8(5), _mm256_shuffle_epi32(ctr, _MM_SHUFFLE(1, 1, 1, 1))); } v[ 8] = GET_256_IV_WAY8(0); v[ 9] = GET_256_IV_WAY8(1); v[10] = GET_256_IV_WAY8(2); v[11] = GET_256_IV_WAY8(3); v[14] = GET_256_IV_WAY8(6); v[15] = GET_256_IV_WAY8(7); REP8_MACRO (V8_LOAD_STATE_256) ROUNDS_LOOP (V8_ROUND) REP8_MACRO (V8_UPDATE_STATE_256) data += SUPER_BLOCK_SIZE; } while (data != end); #ifndef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT REP8_MACRO (V8_STORE_STATE_256) #endif } static Z7_NO_INLINE #ifdef BLAKE2S_ATTRIB_AVX2 BLAKE2S_ATTRIB_AVX2 #endif void Z7_FASTCALL Blake2sp_Final_AVX2_Fast(UInt32 *states) { const __m128i ctr = LOAD_128_FROM_STRUCT(states + 64); // PrintStates2(states, 8, 16); V8_UNPACK_STATE(states, states) // PrintStates2(states, 8, 16); { unsigned k; for (k = 0; k < 8; k++) { UInt32 *s = states + (size_t)k * 16; STORE_128_TO_STRUCT (STATE_T(s), ctr); } } // PrintStates2(states, 8, 16); // printf("\nafter V8_UNPACK_STATE \n"); } #endif // Z7_BLAKE2S_USE_AVX2_FAST #endif // avx2 #endif // vector /* #define Blake2s_Increment_Counter(s, inc) \ { STATE_T(s)[0] += (inc); STATE_T(s)[1] += (STATE_T(s)[0] < (inc)); } #define Blake2s_Increment_Counter_Small(s, inc) \ { STATE_T(s)[0] += (inc); } */ #define Blake2s_Set_LastBlock(s) \ { STATE_F(s)[0] = BLAKE2S_FINAL_FLAG; /* STATE_F(s)[1] = p->u.header.lastNode_f1; */ } #if 0 || 1 && defined(Z7_MSC_VER_ORIGINAL) && Z7_MSC_VER_ORIGINAL >= 1600 // good for vs2022 #define LOOP_8(mac) { unsigned kkk; for (kkk = 0; kkk < 8; kkk++) mac(kkk) } #else // good for Z7_BLAKE2S_UNROLL for GCC9 (arm*/x86*) and MSC_VER_1400-x64. #define LOOP_8(mac) { REP8_MACRO(mac) } #endif static Z7_FORCE_INLINE // Z7_NO_INLINE void Z7_FASTCALL Blake2s_Compress(UInt32 *s, const Byte *input) { UInt32 m[16]; UInt32 v[16]; { unsigned i; for (i = 0; i < 16; i++) m[i] = GetUi32(input + i * 4); } #define INIT_v_FROM_s(i) v[i] = s[i]; LOOP_8(INIT_v_FROM_s) // Blake2s_Increment_Counter(s, Z7_BLAKE2S_BLOCK_SIZE) { const UInt32 t0 = STATE_T(s)[0] + Z7_BLAKE2S_BLOCK_SIZE; const UInt32 t1 = STATE_T(s)[1] + (t0 < Z7_BLAKE2S_BLOCK_SIZE); STATE_T(s)[0] = t0; STATE_T(s)[1] = t1; v[12] = t0 ^ KIV(4); v[13] = t1 ^ KIV(5); } // v[12] = STATE_T(s)[0] ^ KIV(4); // v[13] = STATE_T(s)[1] ^ KIV(5); v[14] = STATE_F(s)[0] ^ KIV(6); v[15] = STATE_F(s)[1] ^ KIV(7); v[ 8] = KIV(0); v[ 9] = KIV(1); v[10] = KIV(2); v[11] = KIV(3); // PrintStates2((const UInt32 *)v, 1, 16); #define ADD_SIGMA(a, index) V(a, 0) += *(const UInt32 *)GET_SIGMA_PTR(m, sigma[index]); #define ADD32M(dest, src, a) V(a, dest) += V(a, src); #define XOR32M(dest, src, a) V(a, dest) ^= V(a, src); #define RTR32M(dest, shift, a) V(a, dest) = rotrFixed(V(a, dest), shift); // big interleaving can provides big performance gain, if scheduler queues are small. #if 0 || 1 && defined(MY_CPU_X86) // interleave-1: for small register number (x86-32bit) #define G2(index, a, x, y) \ ADD_SIGMA (a, (index) + 2 * 0) \ ADD32M (0, 1, a) \ XOR32M (3, 0, a) \ RTR32M (3, x, a) \ ADD32M (2, 3, a) \ XOR32M (1, 2, a) \ RTR32M (1, y, a) \ #define G(a) \ G2(a * 2 , a, 16, 12) \ G2(a * 2 + 1, a, 8, 7) \ #define R2 \ G(0) \ G(1) \ G(2) \ G(3) \ G(4) \ G(5) \ G(6) \ G(7) \ #elif 0 || 1 && defined(MY_CPU_X86_OR_AMD64) // interleave-2: is good if the number of registers is not big (x86-64). #define REP2(mac, dest, src, a, b) \ mac(dest, src, a) \ mac(dest, src, b) #define G2(index, a, b, x, y) \ ADD_SIGMA (a, (index) + 2 * 0) \ ADD_SIGMA (b, (index) + 2 * 1) \ REP2 (ADD32M, 0, 1, a, b) \ REP2 (XOR32M, 3, 0, a, b) \ REP2 (RTR32M, 3, x, a, b) \ REP2 (ADD32M, 2, 3, a, b) \ REP2 (XOR32M, 1, 2, a, b) \ REP2 (RTR32M, 1, y, a, b) \ #define G(a, b) \ G2(a * 2 , a, b, 16, 12) \ G2(a * 2 + 1, a, b, 8, 7) \ #define R2 \ G(0, 1) \ G(2, 3) \ G(4, 5) \ G(6, 7) \ #else // interleave-4: // it has big register pressure for x86/x64. // and MSVC compilers for x86/x64 are slow for this branch. // but if we have big number of registers, this branch can be faster. #define REP4(mac, dest, src, a, b, c, d) \ mac(dest, src, a) \ mac(dest, src, b) \ mac(dest, src, c) \ mac(dest, src, d) #define G2(index, a, b, c, d, x, y) \ ADD_SIGMA (a, (index) + 2 * 0) \ ADD_SIGMA (b, (index) + 2 * 1) \ ADD_SIGMA (c, (index) + 2 * 2) \ ADD_SIGMA (d, (index) + 2 * 3) \ REP4 (ADD32M, 0, 1, a, b, c, d) \ REP4 (XOR32M, 3, 0, a, b, c, d) \ REP4 (RTR32M, 3, x, a, b, c, d) \ REP4 (ADD32M, 2, 3, a, b, c, d) \ REP4 (XOR32M, 1, 2, a, b, c, d) \ REP4 (RTR32M, 1, y, a, b, c, d) \ #define G(a, b, c, d) \ G2(a * 2 , a, b, c, d, 16, 12) \ G2(a * 2 + 1, a, b, c, d, 8, 7) \ #define R2 \ G(0, 1, 2, 3) \ G(4, 5, 6, 7) \ #endif #define R(r) { const Byte *sigma = k_Blake2s_Sigma_4[r]; R2 } // Z7_BLAKE2S_UNROLL gives 5-6 KB larger code, but faster: // 20-40% faster for (x86/x64) VC2010+/GCC/CLANG. // 30-60% faster for (arm64-arm32) GCC. // 5-11% faster for (arm64) CLANG-MAC. // so Z7_BLAKE2S_UNROLL is good optimization, if there is no vector branch. // But if there is vectors branch (for x86*), this scalar code will be unused mostly. // So we want smaller code (without unrolling) in that case (x86*). #if 0 || 1 && !defined(Z7_BLAKE2S_USE_VECTORS) #define Z7_BLAKE2S_UNROLL #endif #ifdef Z7_BLAKE2S_UNROLL ROUNDS_LOOP_UNROLLED (R) #else ROUNDS_LOOP (R) #endif #undef G #undef G2 #undef R #undef R2 // printf("\n v after: \n"); // PrintStates2((const UInt32 *)v, 1, 16); #define XOR_s_PAIR_v(i) s[i] ^= v[i] ^ v[i + 8]; LOOP_8(XOR_s_PAIR_v) // printf("\n s after:\n"); // PrintStates2((const UInt32 *)s, 1, 16); } static Z7_NO_INLINE void Z7_FASTCALL Blake2sp_Compress2(UInt32 *s_items, const Byte *data, const Byte *end) { size_t pos = 0; // PrintStates2(s_items, 8, 16); do { UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); Blake2s_Compress(s, data); data += Z7_BLAKE2S_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE; pos &= SUPER_BLOCK_MASK; } while (data != end); } #ifdef Z7_BLAKE2S_USE_VECTORS static Z7_BLAKE2SP_FUNC_COMPRESS g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast = Blake2sp_Compress2; static Z7_BLAKE2SP_FUNC_COMPRESS g_Z7_BLAKE2SP_FUNC_COMPRESS_Single = Blake2sp_Compress2; static Z7_BLAKE2SP_FUNC_INIT g_Z7_BLAKE2SP_FUNC_INIT_Init; static Z7_BLAKE2SP_FUNC_INIT g_Z7_BLAKE2SP_FUNC_INIT_Final; static unsigned g_z7_Blake2sp_SupportedFlags; #define Z7_BLAKE2SP_Compress_Fast(p) (p)->u.header.func_Compress_Fast #define Z7_BLAKE2SP_Compress_Single(p) (p)->u.header.func_Compress_Single #else #define Z7_BLAKE2SP_Compress_Fast(p) Blake2sp_Compress2 #define Z7_BLAKE2SP_Compress_Single(p) Blake2sp_Compress2 #endif // Z7_BLAKE2S_USE_VECTORS #if 1 && defined(MY_CPU_LE) #define GET_DIGEST(_s, _digest) \ { memcpy(_digest, _s, Z7_BLAKE2S_DIGEST_SIZE); } #else #define GET_DIGEST(_s, _digest) \ { unsigned _i; for (_i = 0; _i < 8; _i++) \ { SetUi32((_digest) + 4 * _i, (_s)[_i]) } \ } #endif /* ---------- BLAKE2s ---------- */ /* // we need to xor CBlake2s::h[i] with input parameter block after Blake2s_Init0() typedef struct { Byte digest_length; Byte key_length; Byte fanout; // = 1 : in sequential mode Byte depth; // = 1 : in sequential mode UInt32 leaf_length; Byte node_offset[6]; // 0 for the first, leftmost, leaf, or in sequential mode Byte node_depth; // 0 for the leaves, or in sequential mode Byte inner_length; // [0, 32], 0 in sequential mode Byte salt[BLAKE2S_SALTBYTES]; Byte personal[BLAKE2S_PERSONALBYTES]; } CBlake2sParam; */ #define k_Blake2sp_IV_0 \ (KIV(0) ^ (Z7_BLAKE2S_DIGEST_SIZE | ((UInt32)Z7_BLAKE2SP_PARALLEL_DEGREE << 16) | ((UInt32)2 << 24))) #define k_Blake2sp_IV_3_FROM_NODE_DEPTH(node_depth) \ (KIV(3) ^ ((UInt32)(node_depth) << 16) ^ ((UInt32)Z7_BLAKE2S_DIGEST_SIZE << 24)) Z7_FORCE_INLINE static void Blake2sp_Init_Spec(UInt32 *s, unsigned node_offset, unsigned node_depth) { s[0] = k_Blake2sp_IV_0; s[1] = KIV(1); s[2] = KIV(2) ^ (UInt32)node_offset; s[3] = k_Blake2sp_IV_3_FROM_NODE_DEPTH(node_depth); s[4] = KIV(4); s[5] = KIV(5); s[6] = KIV(6); s[7] = KIV(7); STATE_T(s)[0] = 0; STATE_T(s)[1] = 0; STATE_F(s)[0] = 0; STATE_F(s)[1] = 0; } #ifdef Z7_BLAKE2S_USE_V128_FAST static Z7_NO_INLINE #ifdef BLAKE2S_ATTRIB_128BIT BLAKE2S_ATTRIB_128BIT #endif void Z7_FASTCALL Blake2sp_InitState_V128_Fast(UInt32 *states) { #define STORE_128_PAIR_INIT_STATES_2(i, t0, t1) \ { STORE_128_TO_STRUCT(states + 0 + 4 * (i), (t0)); \ STORE_128_TO_STRUCT(states + 32 + 4 * (i), (t1)); \ } #define STORE_128_PAIR_INIT_STATES_1(i, mac) \ { const __m128i t = mac; \ STORE_128_PAIR_INIT_STATES_2(i, t, t) \ } #define STORE_128_PAIR_INIT_STATES_IV(i) \ STORE_128_PAIR_INIT_STATES_1(i, GET_128_IV_WAY4(i)) STORE_128_PAIR_INIT_STATES_1 (0, _mm_set1_epi32((Int32)k_Blake2sp_IV_0)) STORE_128_PAIR_INIT_STATES_IV (1) { const __m128i t = GET_128_IV_WAY4(2); STORE_128_PAIR_INIT_STATES_2 (2, XOR_128(t, _mm_set_epi32(3, 2, 1, 0)), XOR_128(t, _mm_set_epi32(7, 6, 5, 4))) } STORE_128_PAIR_INIT_STATES_1 (3, _mm_set1_epi32((Int32)k_Blake2sp_IV_3_FROM_NODE_DEPTH(0))) STORE_128_PAIR_INIT_STATES_IV (4) STORE_128_PAIR_INIT_STATES_IV (5) STORE_128_PAIR_INIT_STATES_IV (6) STORE_128_PAIR_INIT_STATES_IV (7) STORE_128_PAIR_INIT_STATES_1 (16, _mm_set_epi32(0, 0, 0, 0)) // printf("\n== exit Blake2sp_InitState_V128_Fast ctr=%d\n", states[64]); } #endif // Z7_BLAKE2S_USE_V128_FAST #ifdef Z7_BLAKE2S_USE_AVX2_FAST static Z7_NO_INLINE #ifdef BLAKE2S_ATTRIB_AVX2 BLAKE2S_ATTRIB_AVX2 #endif void Z7_FASTCALL Blake2sp_InitState_AVX2_Fast(UInt32 *states) { #define STORE_256_INIT_STATES(i, t) \ STORE_256_TO_STRUCT(states + 8 * (i), t); #define STORE_256_INIT_STATES_IV(i) \ STORE_256_INIT_STATES(i, GET_256_IV_WAY8(i)) STORE_256_INIT_STATES (0, _mm256_set1_epi32((Int32)k_Blake2sp_IV_0)) STORE_256_INIT_STATES_IV (1) STORE_256_INIT_STATES (2, XOR_256( GET_256_IV_WAY8(2), _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0))) STORE_256_INIT_STATES (3, _mm256_set1_epi32((Int32)k_Blake2sp_IV_3_FROM_NODE_DEPTH(0))) STORE_256_INIT_STATES_IV (4) STORE_256_INIT_STATES_IV (5) STORE_256_INIT_STATES_IV (6) STORE_256_INIT_STATES_IV (7) STORE_256_INIT_STATES (8, _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, 0)) // printf("\n== exit Blake2sp_InitState_AVX2_Fast\n"); } #endif // Z7_BLAKE2S_USE_AVX2_FAST Z7_NO_INLINE void Blake2sp_InitState(CBlake2sp *p) { size_t i; // memset(p->states, 0, sizeof(p->states)); // for debug p->u.header.cycPos = 0; #ifdef Z7_BLAKE2SP_USE_FUNCTIONS if (p->u.header.func_Init) { p->u.header.func_Init(p->states); return; } #endif for (i = 0; i < Z7_BLAKE2SP_PARALLEL_DEGREE; i++) Blake2sp_Init_Spec(p->states + i * NSW, (unsigned)i, 0); } void Blake2sp_Init(CBlake2sp *p) { #ifdef Z7_BLAKE2SP_USE_FUNCTIONS p->u.header.func_Compress_Fast = #ifdef Z7_BLAKE2S_USE_VECTORS g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast; #else NULL; #endif p->u.header.func_Compress_Single = #ifdef Z7_BLAKE2S_USE_VECTORS g_Z7_BLAKE2SP_FUNC_COMPRESS_Single; #else NULL; #endif p->u.header.func_Init = #ifdef Z7_BLAKE2S_USE_VECTORS g_Z7_BLAKE2SP_FUNC_INIT_Init; #else NULL; #endif p->u.header.func_Final = #ifdef Z7_BLAKE2S_USE_VECTORS g_Z7_BLAKE2SP_FUNC_INIT_Final; #else NULL; #endif #endif Blake2sp_InitState(p); } void Blake2sp_Update(CBlake2sp *p, const Byte *data, size_t size) { size_t pos; // printf("\nsize = 0x%6x, cycPos = %5u data = %p\n", (unsigned)size, (unsigned)p->u.header.cycPos, data); if (size == 0) return; pos = p->u.header.cycPos; // pos < SUPER_BLOCK_SIZE * 2 : is expected // pos == SUPER_BLOCK_SIZE * 2 : is not expected, but is supported also { const size_t pos2 = pos & SUPER_BLOCK_MASK; if (pos2) { const size_t rem = SUPER_BLOCK_SIZE - pos2; if (rem > size) { p->u.header.cycPos = (unsigned)(pos + size); // cycPos < SUPER_BLOCK_SIZE * 2 memcpy((Byte *)(void *)p->buf32 + pos, data, size); /* to simpilify the code here we don't try to process first superblock, if (cycPos > SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE) */ return; } // (rem <= size) memcpy((Byte *)(void *)p->buf32 + pos, data, rem); pos += rem; data += rem; size -= rem; } } // pos <= SUPER_BLOCK_SIZE * 2 // pos % SUPER_BLOCK_SIZE == 0 if (pos) { /* pos == SUPER_BLOCK_SIZE || pos == SUPER_BLOCK_SIZE * 2 */ size_t end = pos; if (size > SUPER_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE || (end -= SUPER_BLOCK_SIZE)) { Z7_BLAKE2SP_Compress_Fast(p)(p->states, (const Byte *)(const void *)p->buf32, (const Byte *)(const void *)p->buf32 + end); if (pos -= end) memcpy(p->buf32, (const Byte *)(const void *)p->buf32 + SUPER_BLOCK_SIZE, SUPER_BLOCK_SIZE); } } // pos == 0 || (pos == SUPER_BLOCK_SIZE && size <= SUPER_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE) if (size > SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE) { // pos == 0 const Byte *end; const size_t size2 = (size - (SUPER_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE + 1)) & ~(size_t)SUPER_BLOCK_MASK; size -= size2; // size < SUPER_BLOCK_SIZE * 2 end = data + size2; Z7_BLAKE2SP_Compress_Fast(p)(p->states, data, end); data = end; } if (size != 0) { memcpy((Byte *)(void *)p->buf32 + pos, data, size); pos += size; } p->u.header.cycPos = (unsigned)pos; // cycPos < SUPER_BLOCK_SIZE * 2 } void Blake2sp_Final(CBlake2sp *p, Byte *digest) { // UInt32 * const R_states = p->states; // printf("\nBlake2sp_Final \n"); #ifdef Z7_BLAKE2SP_USE_FUNCTIONS if (p->u.header.func_Final) p->u.header.func_Final(p->states); #endif // printf("\n=====\nBlake2sp_Final \n"); // PrintStates(p->states, 32); // (p->u.header.cycPos == SUPER_BLOCK_SIZE) can be processed in any branch: if (p->u.header.cycPos <= SUPER_BLOCK_SIZE) { unsigned pos; memset((Byte *)(void *)p->buf32 + p->u.header.cycPos, 0, SUPER_BLOCK_SIZE - p->u.header.cycPos); STATE_F(&p->states[(Z7_BLAKE2SP_PARALLEL_DEGREE - 1) * NSW])[1] = BLAKE2S_FINAL_FLAG; for (pos = 0; pos < SUPER_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE) { UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, pos); Blake2s_Set_LastBlock(s) if (pos + Z7_BLAKE2S_BLOCK_SIZE > p->u.header.cycPos) { UInt32 delta = Z7_BLAKE2S_BLOCK_SIZE; if (pos < p->u.header.cycPos) delta -= p->u.header.cycPos & (Z7_BLAKE2S_BLOCK_SIZE - 1); // 0 < delta <= Z7_BLAKE2S_BLOCK_SIZE { const UInt32 v = STATE_T(s)[0]; STATE_T(s)[1] -= v < delta; // (v < delta) is same condition here as (v == 0) STATE_T(s)[0] = v - delta; } } } // PrintStates(p->states, 16); Z7_BLAKE2SP_Compress_Single(p)(p->states, (Byte *)(void *)p->buf32, (Byte *)(void *)p->buf32 + SUPER_BLOCK_SIZE); // PrintStates(p->states, 16); } else { // (p->u.header.cycPos > SUPER_BLOCK_SIZE) unsigned pos; for (pos = 0; pos < SUPER_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE) { UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, pos); if (pos + SUPER_BLOCK_SIZE >= p->u.header.cycPos) Blake2s_Set_LastBlock(s) } if (p->u.header.cycPos <= SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE) STATE_F(&p->states[(Z7_BLAKE2SP_PARALLEL_DEGREE - 1) * NSW])[1] = BLAKE2S_FINAL_FLAG; Z7_BLAKE2SP_Compress_Single(p)(p->states, (Byte *)(void *)p->buf32, (Byte *)(void *)p->buf32 + SUPER_BLOCK_SIZE); // if (p->u.header.cycPos > SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE; STATE_F(&p->states[(Z7_BLAKE2SP_PARALLEL_DEGREE - 1) * NSW])[1] = BLAKE2S_FINAL_FLAG; // if (p->u.header.cycPos != SUPER_BLOCK_SIZE) { pos = SUPER_BLOCK_SIZE; for (;;) { UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, pos & SUPER_BLOCK_MASK); Blake2s_Set_LastBlock(s) pos += Z7_BLAKE2S_BLOCK_SIZE; if (pos >= p->u.header.cycPos) { if (pos != p->u.header.cycPos) { const UInt32 delta = pos - p->u.header.cycPos; const UInt32 v = STATE_T(s)[0]; STATE_T(s)[1] -= v < delta; STATE_T(s)[0] = v - delta; memset((Byte *)(void *)p->buf32 + p->u.header.cycPos, 0, delta); } break; } } Z7_BLAKE2SP_Compress_Single(p)(p->states, (Byte *)(void *)p->buf32 + SUPER_BLOCK_SIZE, (Byte *)(void *)p->buf32 + pos); } } { size_t pos; for (pos = 0; pos < SUPER_BLOCK_SIZE / 2; pos += Z7_BLAKE2S_BLOCK_SIZE / 2) { const UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, (pos * 2)); Byte *dest = (Byte *)(void *)p->buf32 + pos; GET_DIGEST(s, dest) } } Blake2sp_Init_Spec(p->states, 0, 1); { size_t pos; for (pos = 0; pos < (Z7_BLAKE2SP_PARALLEL_DEGREE * Z7_BLAKE2S_DIGEST_SIZE) - Z7_BLAKE2S_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE) { Z7_BLAKE2SP_Compress_Single(p)(p->states, (const Byte *)(const void *)p->buf32 + pos, (const Byte *)(const void *)p->buf32 + pos + Z7_BLAKE2S_BLOCK_SIZE); } } // Blake2s_Final(p->states, 0, digest, p, (Byte *)(void *)p->buf32 + i); Blake2s_Set_LastBlock(p->states) STATE_F(p->states)[1] = BLAKE2S_FINAL_FLAG; { Z7_BLAKE2SP_Compress_Single(p)(p->states, (const Byte *)(const void *)p->buf32 + Z7_BLAKE2SP_PARALLEL_DEGREE / 2 * Z7_BLAKE2S_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE, (const Byte *)(const void *)p->buf32 + Z7_BLAKE2SP_PARALLEL_DEGREE / 2 * Z7_BLAKE2S_BLOCK_SIZE); } GET_DIGEST(p->states, digest) // printf("\n Blake2sp_Final 555 numDataInBufs = %5u\n", (unsigned)p->u.header.numDataInBufs); } BoolInt Blake2sp_SetFunction(CBlake2sp *p, unsigned algo) { // printf("\n========== setfunction = %d ======== \n", algo); #ifdef Z7_BLAKE2SP_USE_FUNCTIONS Z7_BLAKE2SP_FUNC_COMPRESS func = NULL; Z7_BLAKE2SP_FUNC_COMPRESS func_Single = NULL; Z7_BLAKE2SP_FUNC_INIT func_Final = NULL; Z7_BLAKE2SP_FUNC_INIT func_Init = NULL; #else UNUSED_VAR(p) #endif #ifdef Z7_BLAKE2S_USE_VECTORS func = func_Single = Blake2sp_Compress2; if (algo != Z7_BLAKE2SP_ALGO_SCALAR) { // printf("\n========== setfunction NON-SCALER ======== \n"); if (algo == Z7_BLAKE2SP_ALGO_DEFAULT) { func = g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast; func_Single = g_Z7_BLAKE2SP_FUNC_COMPRESS_Single; func_Init = g_Z7_BLAKE2SP_FUNC_INIT_Init; func_Final = g_Z7_BLAKE2SP_FUNC_INIT_Final; } else { if ((g_z7_Blake2sp_SupportedFlags & (1u << algo)) == 0) return False; #ifdef Z7_BLAKE2S_USE_AVX2 func_Single = #if defined(Z7_BLAKE2S_USE_AVX2_WAY2) Blake2sp_Compress2_AVX2_Way2; #else Z7_BLAKE2S_Compress2_V128; #endif #ifdef Z7_BLAKE2S_USE_AVX2_FAST if (algo == Z7_BLAKE2SP_ALGO_V256_FAST) { func = Blake2sp_Compress2_AVX2_Fast; func_Final = Blake2sp_Final_AVX2_Fast; func_Init = Blake2sp_InitState_AVX2_Fast; } else #endif #ifdef Z7_BLAKE2S_USE_AVX2_WAY2 if (algo == Z7_BLAKE2SP_ALGO_V256_WAY2) func = Blake2sp_Compress2_AVX2_Way2; else #endif #ifdef Z7_BLAKE2S_USE_AVX2_WAY4 if (algo == Z7_BLAKE2SP_ALGO_V256_WAY4) { func_Single = func = Blake2sp_Compress2_AVX2_Way4; } else #endif #endif // avx2 { if (algo == Z7_BLAKE2SP_ALGO_V128_FAST) { func = Blake2sp_Compress2_V128_Fast; func_Final = Blake2sp_Final_V128_Fast; func_Init = Blake2sp_InitState_V128_Fast; func_Single = Z7_BLAKE2S_Compress2_V128; } else #ifdef Z7_BLAKE2S_USE_V128_WAY2 if (algo == Z7_BLAKE2SP_ALGO_V128_WAY2) func = func_Single = Blake2sp_Compress2_V128_Way2; else #endif { if (algo != Z7_BLAKE2SP_ALGO_V128_WAY1) return False; func = func_Single = Blake2sp_Compress2_V128_Way1; } } } } #else // !VECTORS if (algo > 1) // Z7_BLAKE2SP_ALGO_SCALAR return False; #endif // !VECTORS #ifdef Z7_BLAKE2SP_USE_FUNCTIONS p->u.header.func_Compress_Fast = func; p->u.header.func_Compress_Single = func_Single; p->u.header.func_Final = func_Final; p->u.header.func_Init = func_Init; #endif // printf("\n p->u.header.func_Compress = %p", p->u.header.func_Compress); return True; } void z7_Black2sp_Prepare(void) { #ifdef Z7_BLAKE2S_USE_VECTORS unsigned flags = 0; // (1u << Z7_BLAKE2SP_ALGO_V128_SCALAR); Z7_BLAKE2SP_FUNC_COMPRESS func_Fast = Blake2sp_Compress2; Z7_BLAKE2SP_FUNC_COMPRESS func_Single = Blake2sp_Compress2; Z7_BLAKE2SP_FUNC_INIT func_Init = NULL; Z7_BLAKE2SP_FUNC_INIT func_Final = NULL; #if defined(MY_CPU_X86_OR_AMD64) #if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) // optional check #if 0 || !(defined(__AVX512F__) && defined(__AVX512VL__)) if (CPU_IsSupported_AVX512F_AVX512VL()) #endif #elif defined(Z7_BLAKE2S_USE_SSE41) if (CPU_IsSupported_SSE41()) #elif defined(Z7_BLAKE2S_USE_SSSE3) if (CPU_IsSupported_SSSE3()) #elif !defined(MY_CPU_AMD64) if (CPU_IsSupported_SSE2()) #endif #endif { #if defined(Z7_BLAKE2S_USE_SSE41) // printf("\n========== Blake2s SSE41 128-bit\n"); #elif defined(Z7_BLAKE2S_USE_SSSE3) // printf("\n========== Blake2s SSSE3 128-bit\n"); #else // printf("\n========== Blake2s SSE2 128-bit\n"); #endif // func_Fast = f_vector = Blake2sp_Compress2_V128_Way2; // printf("\n========== Blake2sp_Compress2_V128_Way2\n"); func_Fast = func_Single = Z7_BLAKE2S_Compress2_V128; flags |= (1u << Z7_BLAKE2SP_ALGO_V128_WAY1); #ifdef Z7_BLAKE2S_USE_V128_WAY2 flags |= (1u << Z7_BLAKE2SP_ALGO_V128_WAY2); #endif #ifdef Z7_BLAKE2S_USE_V128_FAST flags |= (1u << Z7_BLAKE2SP_ALGO_V128_FAST); func_Fast = Blake2sp_Compress2_V128_Fast; func_Init = Blake2sp_InitState_V128_Fast; func_Final = Blake2sp_Final_V128_Fast; #endif #ifdef Z7_BLAKE2S_USE_AVX2 #if defined(MY_CPU_X86_OR_AMD64) #if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) #if 0 if (CPU_IsSupported_AVX512F_AVX512VL()) #endif #else if (CPU_IsSupported_AVX2()) #endif #endif { // #pragma message ("=== Blake2s AVX2") // printf("\n========== Blake2s AVX2\n"); #ifdef Z7_BLAKE2S_USE_AVX2_WAY2 func_Single = Blake2sp_Compress2_AVX2_Way2; flags |= (1u << Z7_BLAKE2SP_ALGO_V256_WAY2); #endif #ifdef Z7_BLAKE2S_USE_AVX2_WAY4 flags |= (1u << Z7_BLAKE2SP_ALGO_V256_WAY4); #endif #ifdef Z7_BLAKE2S_USE_AVX2_FAST flags |= (1u << Z7_BLAKE2SP_ALGO_V256_FAST); func_Fast = Blake2sp_Compress2_AVX2_Fast; func_Init = Blake2sp_InitState_AVX2_Fast; func_Final = Blake2sp_Final_AVX2_Fast; #elif defined(Z7_BLAKE2S_USE_AVX2_WAY4) func_Fast = Blake2sp_Compress2_AVX2_Way4; #elif defined(Z7_BLAKE2S_USE_AVX2_WAY2) func_Fast = Blake2sp_Compress2_AVX2_Way2; #endif } // avx2 #endif // avx2 } // sse* g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast = func_Fast; g_Z7_BLAKE2SP_FUNC_COMPRESS_Single = func_Single; g_Z7_BLAKE2SP_FUNC_INIT_Init = func_Init; g_Z7_BLAKE2SP_FUNC_INIT_Final = func_Final; g_z7_Blake2sp_SupportedFlags = flags; // printf("\nflags=%x\n", flags); #endif // vectors } /* #ifdef Z7_BLAKE2S_USE_VECTORS void align_test2(CBlake2sp *sp); void align_test2(CBlake2sp *sp) { __m128i a = LOAD_128(sp->states); D_XOR_128(a, LOAD_128(sp->states + 4)); STORE_128(sp->states, a); } void align_test2(void); void align_test2(void) { CBlake2sp sp; Blake2sp_Init(&sp); Blake2sp_Update(&sp, NULL, 0); } #endif */