Branch data Line data Source code
1 : : // Copyright (c) 2018-present The Bitcoin Core developers
2 : : // Distributed under the MIT software license, see the accompanying
3 : : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 : : //
5 : : // Based on https://github.com/noloader/SHA-Intrinsics/blob/master/sha256-x86.c,
6 : : // Written and placed in public domain by Jeffrey Walton.
7 : : // Based on code from Intel, and by Sean Gulley for the miTLS project.
8 : :
9 : : #if defined(ENABLE_SSE41) && defined(ENABLE_X86_SHANI)
10 : :
11 : : #include <cstddef>
12 : : #include <cstdint>
13 : : #include <immintrin.h>
14 : :
15 : : #include <attributes.h>
16 : :
17 : : namespace {
18 : :
19 : : alignas(__m128i) const uint8_t MASK[16] = {0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04, 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c};
20 : : alignas(__m128i) const uint8_t INIT0[16] = {0x8c, 0x68, 0x05, 0x9b, 0x7f, 0x52, 0x0e, 0x51, 0x85, 0xae, 0x67, 0xbb, 0x67, 0xe6, 0x09, 0x6a};
21 : : alignas(__m128i) const uint8_t INIT1[16] = {0x19, 0xcd, 0xe0, 0x5b, 0xab, 0xd9, 0x83, 0x1f, 0x3a, 0xf5, 0x4f, 0xa5, 0x72, 0xf3, 0x6e, 0x3c};
22 : :
23 : 169172 : void ALWAYS_INLINE QuadRound(__m128i& state0, __m128i& state1, uint64_t k1, uint64_t k0)
24 : : {
25 : 169172 : const __m128i msg = _mm_set_epi64x(k1, k0);
26 : 169172 : state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
27 : 169172 : state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e));
28 : : }
29 : :
30 : 514513056 : void ALWAYS_INLINE QuadRound(__m128i& state0, __m128i& state1, __m128i m, uint64_t k1, uint64_t k0)
31 : : {
32 : 514513056 : const __m128i msg = _mm_add_epi32(m, _mm_set_epi64x(k1, k0));
33 : 514343884 : state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
34 : 514343884 : state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e));
35 : : }
36 : :
37 : 514513056 : void ALWAYS_INLINE ShiftMessageA(__m128i& m0, __m128i m1)
38 : : {
39 : 514513056 : m0 = _mm_sha256msg1_epu32(m0, m1);
40 : : }
41 : :
42 : 514513056 : void ALWAYS_INLINE ShiftMessageC(__m128i& m0, __m128i m1, __m128i& m2)
43 : : {
44 : 514513056 : m2 = _mm_sha256msg2_epu32(_mm_add_epi32(m2, _mm_alignr_epi8(m1, m0, 4)), m1);
45 : : }
46 : :
47 : 514513056 : void ALWAYS_INLINE ShiftMessageB(__m128i& m0, __m128i m1, __m128i& m2)
48 : : {
49 : 514513056 : ShiftMessageC(m0, m1, m2);
50 : 514513056 : ShiftMessageA(m0, m1);
51 : : }
52 : :
53 : 52226658 : void ALWAYS_INLINE Shuffle(__m128i& s0, __m128i& s1)
54 : : {
55 : 52226658 : const __m128i t1 = _mm_shuffle_epi32(s0, 0xB1);
56 : 52226658 : const __m128i t2 = _mm_shuffle_epi32(s1, 0x1B);
57 : 52226658 : s0 = _mm_alignr_epi8(t1, t2, 0x08);
58 : 52226658 : s1 = _mm_blend_epi16(t2, t1, 0xF0);
59 : 52226658 : }
60 : :
61 : 52395830 : void ALWAYS_INLINE Unshuffle(__m128i& s0, __m128i& s1)
62 : : {
63 : 52395830 : const __m128i t1 = _mm_shuffle_epi32(s0, 0x1B);
64 : 52395830 : const __m128i t2 = _mm_shuffle_epi32(s1, 0xB1);
65 : 52395830 : s0 = _mm_blend_epi16(t1, t2, 0xF0);
66 : 52395830 : s1 = _mm_alignr_epi8(t2, t1, 0x08);
67 : : }
68 : :
69 : 514513056 : __m128i ALWAYS_INLINE Load(const unsigned char* in)
70 : : {
71 : 514513056 : return _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)in), _mm_load_si128((const __m128i*)MASK));
72 : : }
73 : :
74 : 169172 : void ALWAYS_INLINE Save(unsigned char* out, __m128i s)
75 : : {
76 : 169172 : _mm_storeu_si128((__m128i*)out, _mm_shuffle_epi8(s, _mm_load_si128((const __m128i*)MASK)));
77 : : }
78 : : }
79 : :
80 : : namespace sha256_x86_shani {
81 : 52226658 : void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks)
82 : : {
83 : 52226658 : __m128i m0, m1, m2, m3, s0, s1, so0, so1;
84 : :
85 : : /* Load state */
86 : 52226658 : s0 = _mm_loadu_si128((const __m128i*)s);
87 : 52226658 : s1 = _mm_loadu_si128((const __m128i*)(s + 4));
88 : 52226658 : Shuffle(s0, s1);
89 : :
90 [ + + ]: 566570542 : while (blocks--) {
91 : : /* Remember old state */
92 : 514343884 : so0 = s0;
93 : 514343884 : so1 = s1;
94 : :
95 : : /* Load data and transform */
96 : 514343884 : m0 = Load(chunk);
97 : 514343884 : QuadRound(s0, s1, m0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull);
98 : 514343884 : m1 = Load(chunk + 16);
99 : 514343884 : QuadRound(s0, s1, m1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
100 : 514343884 : ShiftMessageA(m0, m1);
101 : 514343884 : m2 = Load(chunk + 32);
102 : 514343884 : QuadRound(s0, s1, m2, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);
103 : 514343884 : ShiftMessageA(m1, m2);
104 : 514343884 : m3 = Load(chunk + 48);
105 : 514343884 : QuadRound(s0, s1, m3, 0xc19bf1749bdc06a7ull, 0x80deb1fe72be5d74ull);
106 : 514343884 : ShiftMessageB(m2, m3, m0);
107 : 514343884 : QuadRound(s0, s1, m0, 0x240ca1cc0fc19dc6ull, 0xefbe4786E49b69c1ull);
108 : 514343884 : ShiftMessageB(m3, m0, m1);
109 : 514343884 : QuadRound(s0, s1, m1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);
110 : 514343884 : ShiftMessageB(m0, m1, m2);
111 : 514343884 : QuadRound(s0, s1, m2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);
112 : 514343884 : ShiftMessageB(m1, m2, m3);
113 : 514343884 : QuadRound(s0, s1, m3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);
114 : 514343884 : ShiftMessageB(m2, m3, m0);
115 : 514343884 : QuadRound(s0, s1, m0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);
116 : 514343884 : ShiftMessageB(m3, m0, m1);
117 : 514343884 : QuadRound(s0, s1, m1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);
118 : 514343884 : ShiftMessageB(m0, m1, m2);
119 : 514343884 : QuadRound(s0, s1, m2, 0xc76c51A3c24b8b70ull, 0xa81a664ba2bfe8a1ull);
120 : 514343884 : ShiftMessageB(m1, m2, m3);
121 : 514343884 : QuadRound(s0, s1, m3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);
122 : 514343884 : ShiftMessageB(m2, m3, m0);
123 : 514343884 : QuadRound(s0, s1, m0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);
124 : 514343884 : ShiftMessageB(m3, m0, m1);
125 : 514343884 : QuadRound(s0, s1, m1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);
126 : 514343884 : ShiftMessageC(m0, m1, m2);
127 : 514343884 : QuadRound(s0, s1, m2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);
128 : 514343884 : ShiftMessageC(m1, m2, m3);
129 : 514343884 : QuadRound(s0, s1, m3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull);
130 : :
131 : : /* Combine with old state */
132 : 514343884 : s0 = _mm_add_epi32(s0, so0);
133 : 514343884 : s1 = _mm_add_epi32(s1, so1);
134 : :
135 : : /* Advance */
136 : 514343884 : chunk += 64;
137 : : }
138 : :
139 : 52226658 : Unshuffle(s0, s1);
140 : 52226658 : _mm_storeu_si128((__m128i*)s, s0);
141 : 52226658 : _mm_storeu_si128((__m128i*)(s + 4), s1);
142 : 52226658 : }
143 : : }
144 : :
145 : : namespace sha256d64_x86_shani {
146 : :
147 : 169172 : void Transform_2way(unsigned char* out, const unsigned char* in)
148 : : {
149 : 169172 : __m128i am0, am1, am2, am3, as0, as1, aso0, aso1;
150 : 169172 : __m128i bm0, bm1, bm2, bm3, bs0, bs1, bso0, bso1;
151 : :
152 : : /* Transform 1 */
153 : 169172 : bs0 = as0 = _mm_load_si128((const __m128i*)INIT0);
154 : 169172 : bs1 = as1 = _mm_load_si128((const __m128i*)INIT1);
155 : 169172 : am0 = Load(in);
156 : 169172 : bm0 = Load(in + 64);
157 : 169172 : QuadRound(as0, as1, am0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull);
158 : 169172 : QuadRound(bs0, bs1, bm0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull);
159 : 169172 : am1 = Load(in + 16);
160 : 169172 : bm1 = Load(in + 80);
161 : 169172 : QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
162 : 169172 : QuadRound(bs0, bs1, bm1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
163 : 169172 : ShiftMessageA(am0, am1);
164 : 169172 : ShiftMessageA(bm0, bm1);
165 : 169172 : am2 = Load(in + 32);
166 : 169172 : bm2 = Load(in + 96);
167 : 169172 : QuadRound(as0, as1, am2, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);
168 : 169172 : QuadRound(bs0, bs1, bm2, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);
169 : 169172 : ShiftMessageA(am1, am2);
170 : 169172 : ShiftMessageA(bm1, bm2);
171 : 169172 : am3 = Load(in + 48);
172 : 169172 : bm3 = Load(in + 112);
173 : 169172 : QuadRound(as0, as1, am3, 0xc19bf1749bdc06a7ull, 0x80deb1fe72be5d74ull);
174 : 169172 : QuadRound(bs0, bs1, bm3, 0xc19bf1749bdc06a7ull, 0x80deb1fe72be5d74ull);
175 : 169172 : ShiftMessageB(am2, am3, am0);
176 : 169172 : ShiftMessageB(bm2, bm3, bm0);
177 : 169172 : QuadRound(as0, as1, am0, 0x240ca1cc0fc19dc6ull, 0xefbe4786E49b69c1ull);
178 : 169172 : QuadRound(bs0, bs1, bm0, 0x240ca1cc0fc19dc6ull, 0xefbe4786E49b69c1ull);
179 : 169172 : ShiftMessageB(am3, am0, am1);
180 : 169172 : ShiftMessageB(bm3, bm0, bm1);
181 : 169172 : QuadRound(as0, as1, am1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);
182 : 169172 : QuadRound(bs0, bs1, bm1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);
183 : 169172 : ShiftMessageB(am0, am1, am2);
184 : 169172 : ShiftMessageB(bm0, bm1, bm2);
185 : 169172 : QuadRound(as0, as1, am2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);
186 : 169172 : QuadRound(bs0, bs1, bm2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);
187 : 169172 : ShiftMessageB(am1, am2, am3);
188 : 169172 : ShiftMessageB(bm1, bm2, bm3);
189 : 169172 : QuadRound(as0, as1, am3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);
190 : 169172 : QuadRound(bs0, bs1, bm3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);
191 : 169172 : ShiftMessageB(am2, am3, am0);
192 : 169172 : ShiftMessageB(bm2, bm3, bm0);
193 : 169172 : QuadRound(as0, as1, am0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);
194 : 169172 : QuadRound(bs0, bs1, bm0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);
195 : 169172 : ShiftMessageB(am3, am0, am1);
196 : 169172 : ShiftMessageB(bm3, bm0, bm1);
197 : 169172 : QuadRound(as0, as1, am1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);
198 : 169172 : QuadRound(bs0, bs1, bm1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);
199 : 169172 : ShiftMessageB(am0, am1, am2);
200 : 169172 : ShiftMessageB(bm0, bm1, bm2);
201 : 169172 : QuadRound(as0, as1, am2, 0xc76c51A3c24b8b70ull, 0xa81a664ba2bfe8a1ull);
202 : 169172 : QuadRound(bs0, bs1, bm2, 0xc76c51A3c24b8b70ull, 0xa81a664ba2bfe8a1ull);
203 : 169172 : ShiftMessageB(am1, am2, am3);
204 : 169172 : ShiftMessageB(bm1, bm2, bm3);
205 : 169172 : QuadRound(as0, as1, am3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);
206 : 169172 : QuadRound(bs0, bs1, bm3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);
207 : 169172 : ShiftMessageB(am2, am3, am0);
208 : 169172 : ShiftMessageB(bm2, bm3, bm0);
209 : 169172 : QuadRound(as0, as1, am0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);
210 : 169172 : QuadRound(bs0, bs1, bm0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);
211 : 169172 : ShiftMessageB(am3, am0, am1);
212 : 169172 : ShiftMessageB(bm3, bm0, bm1);
213 : 169172 : QuadRound(as0, as1, am1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);
214 : 169172 : QuadRound(bs0, bs1, bm1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);
215 : 169172 : ShiftMessageC(am0, am1, am2);
216 : 169172 : ShiftMessageC(bm0, bm1, bm2);
217 : 169172 : QuadRound(as0, as1, am2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);
218 : 169172 : QuadRound(bs0, bs1, bm2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);
219 : 169172 : ShiftMessageC(am1, am2, am3);
220 : 169172 : ShiftMessageC(bm1, bm2, bm3);
221 : 169172 : QuadRound(as0, as1, am3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull);
222 : 169172 : QuadRound(bs0, bs1, bm3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull);
223 : 169172 : as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i*)INIT0));
224 : 169172 : bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i*)INIT0));
225 : 169172 : as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i*)INIT1));
226 : 169172 : bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i*)INIT1));
227 : :
228 : : /* Transform 2 */
229 : 169172 : aso0 = as0;
230 : 169172 : bso0 = bs0;
231 : 169172 : aso1 = as1;
232 : 169172 : bso1 = bs1;
233 : 169172 : QuadRound(as0, as1, 0xe9b5dba5b5c0fbcfull, 0x71374491c28a2f98ull);
234 : 169172 : QuadRound(bs0, bs1, 0xe9b5dba5b5c0fbcfull, 0x71374491c28a2f98ull);
235 : 169172 : QuadRound(as0, as1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
236 : 169172 : QuadRound(bs0, bs1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
237 : 169172 : QuadRound(as0, as1, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);
238 : 169172 : QuadRound(bs0, bs1, 0x550c7dc3243185beull, 0x12835b01d807aa98ull);
239 : 169172 : QuadRound(as0, as1, 0xc19bf3749bdc06a7ull, 0x80deb1fe72be5d74ull);
240 : 169172 : QuadRound(bs0, bs1, 0xc19bf3749bdc06a7ull, 0x80deb1fe72be5d74ull);
241 : 169172 : QuadRound(as0, as1, 0x240cf2540fe1edc6ull, 0xf0fe4786649b69c1ull);
242 : 169172 : QuadRound(bs0, bs1, 0x240cf2540fe1edc6ull, 0xf0fe4786649b69c1ull);
243 : 169172 : QuadRound(as0, as1, 0x16f988fa61b9411eull, 0x6cc984be4fe9346full);
244 : 169172 : QuadRound(bs0, bs1, 0x16f988fa61b9411eull, 0x6cc984be4fe9346full);
245 : 169172 : QuadRound(as0, as1, 0xb9d99ec7b019fc65ull, 0xa88e5a6df2c65152ull);
246 : 169172 : QuadRound(bs0, bs1, 0xb9d99ec7b019fc65ull, 0xa88e5a6df2c65152ull);
247 : 169172 : QuadRound(as0, as1, 0xc7353eb0fdb1232bull, 0xe70eeaa09a1231c3ull);
248 : 169172 : QuadRound(bs0, bs1, 0xc7353eb0fdb1232bull, 0xe70eeaa09a1231c3ull);
249 : 169172 : QuadRound(as0, as1, 0xdc1eeefd5a0f118full, 0xcb976d5f3069bad5ull);
250 : 169172 : QuadRound(bs0, bs1, 0xdc1eeefd5a0f118full, 0xcb976d5f3069bad5ull);
251 : 169172 : QuadRound(as0, as1, 0xe15d5b1658f4ca9dull, 0xde0b7a040a35b689ull);
252 : 169172 : QuadRound(bs0, bs1, 0xe15d5b1658f4ca9dull, 0xde0b7a040a35b689ull);
253 : 169172 : QuadRound(as0, as1, 0x6fab9537a507ea32ull, 0x37088980007f3e86ull);
254 : 169172 : QuadRound(bs0, bs1, 0x6fab9537a507ea32ull, 0x37088980007f3e86ull);
255 : 169172 : QuadRound(as0, as1, 0xc0bbbe37cdaa3b6dull, 0x0d8cd6f117406110ull);
256 : 169172 : QuadRound(bs0, bs1, 0xc0bbbe37cdaa3b6dull, 0x0d8cd6f117406110ull);
257 : 169172 : QuadRound(as0, as1, 0x6fd15ca70b02e931ull, 0xdb48a36383613bdaull);
258 : 169172 : QuadRound(bs0, bs1, 0x6fd15ca70b02e931ull, 0xdb48a36383613bdaull);
259 : 169172 : QuadRound(as0, as1, 0x6d4378906ed41a95ull, 0x31338431521afacaull);
260 : 169172 : QuadRound(bs0, bs1, 0x6d4378906ed41a95ull, 0x31338431521afacaull);
261 : 169172 : QuadRound(as0, as1, 0x532fb63cb5c9a0e6ull, 0x9eccabbdc39c91f2ull);
262 : 169172 : QuadRound(bs0, bs1, 0x532fb63cb5c9a0e6ull, 0x9eccabbdc39c91f2ull);
263 : 169172 : QuadRound(as0, as1, 0x4c191d76a4954b68ull, 0x07237ea3d2c741c6ull);
264 : 169172 : QuadRound(bs0, bs1, 0x4c191d76a4954b68ull, 0x07237ea3d2c741c6ull);
265 : 169172 : as0 = _mm_add_epi32(as0, aso0);
266 : 169172 : bs0 = _mm_add_epi32(bs0, bso0);
267 : 169172 : as1 = _mm_add_epi32(as1, aso1);
268 : 169172 : bs1 = _mm_add_epi32(bs1, bso1);
269 : :
270 : : /* Extract hash */
271 : 169172 : Unshuffle(as0, as1);
272 : 169172 : Unshuffle(bs0, bs1);
273 : 169172 : am0 = as0;
274 : 169172 : bm0 = bs0;
275 : 169172 : am1 = as1;
276 : 169172 : bm1 = bs1;
277 : :
278 : : /* Transform 3 */
279 : 169172 : bs0 = as0 = _mm_load_si128((const __m128i*)INIT0);
280 : 169172 : bs1 = as1 = _mm_load_si128((const __m128i*)INIT1);
281 : 169172 : QuadRound(as0, as1, am0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull);
282 : 169172 : QuadRound(bs0, bs1, bm0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull);
283 : 169172 : QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
284 : 169172 : QuadRound(bs0, bs1, bm1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull);
285 : 169172 : ShiftMessageA(am0, am1);
286 : 169172 : ShiftMessageA(bm0, bm1);
287 : 169172 : bm2 = am2 = _mm_set_epi64x(0x0ull, 0x80000000ull);
288 : 169172 : QuadRound(as0, as1, 0x550c7dc3243185beull, 0x12835b015807aa98ull);
289 : 169172 : QuadRound(bs0, bs1, 0x550c7dc3243185beull, 0x12835b015807aa98ull);
290 : 169172 : ShiftMessageA(am1, am2);
291 : 169172 : ShiftMessageA(bm1, bm2);
292 : 169172 : bm3 = am3 = _mm_set_epi64x(0x10000000000ull, 0x0ull);
293 : 169172 : QuadRound(as0, as1, 0xc19bf2749bdc06a7ull, 0x80deb1fe72be5d74ull);
294 : 169172 : QuadRound(bs0, bs1, 0xc19bf2749bdc06a7ull, 0x80deb1fe72be5d74ull);
295 : 169172 : ShiftMessageB(am2, am3, am0);
296 : 169172 : ShiftMessageB(bm2, bm3, bm0);
297 : 169172 : QuadRound(as0, as1, am0, 0x240ca1cc0fc19dc6ull, 0xefbe4786e49b69c1ull);
298 : 169172 : QuadRound(bs0, bs1, bm0, 0x240ca1cc0fc19dc6ull, 0xefbe4786e49b69c1ull);
299 : 169172 : ShiftMessageB(am3, am0, am1);
300 : 169172 : ShiftMessageB(bm3, bm0, bm1);
301 : 169172 : QuadRound(as0, as1, am1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);
302 : 169172 : QuadRound(bs0, bs1, bm1, 0x76f988da5cb0a9dcull, 0x4a7484aa2de92c6full);
303 : 169172 : ShiftMessageB(am0, am1, am2);
304 : 169172 : ShiftMessageB(bm0, bm1, bm2);
305 : 169172 : QuadRound(as0, as1, am2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);
306 : 169172 : QuadRound(bs0, bs1, bm2, 0xbf597fc7b00327c8ull, 0xa831c66d983e5152ull);
307 : 169172 : ShiftMessageB(am1, am2, am3);
308 : 169172 : ShiftMessageB(bm1, bm2, bm3);
309 : 169172 : QuadRound(as0, as1, am3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);
310 : 169172 : QuadRound(bs0, bs1, bm3, 0x1429296706ca6351ull, 0xd5a79147c6e00bf3ull);
311 : 169172 : ShiftMessageB(am2, am3, am0);
312 : 169172 : ShiftMessageB(bm2, bm3, bm0);
313 : 169172 : QuadRound(as0, as1, am0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);
314 : 169172 : QuadRound(bs0, bs1, bm0, 0x53380d134d2c6dfcull, 0x2e1b213827b70a85ull);
315 : 169172 : ShiftMessageB(am3, am0, am1);
316 : 169172 : ShiftMessageB(bm3, bm0, bm1);
317 : 169172 : QuadRound(as0, as1, am1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);
318 : 169172 : QuadRound(bs0, bs1, bm1, 0x92722c8581c2c92eull, 0x766a0abb650a7354ull);
319 : 169172 : ShiftMessageB(am0, am1, am2);
320 : 169172 : ShiftMessageB(bm0, bm1, bm2);
321 : 169172 : QuadRound(as0, as1, am2, 0xc76c51a3c24b8b70ull, 0xa81a664ba2bfe8A1ull);
322 : 169172 : QuadRound(bs0, bs1, bm2, 0xc76c51a3c24b8b70ull, 0xa81a664ba2bfe8A1ull);
323 : 169172 : ShiftMessageB(am1, am2, am3);
324 : 169172 : ShiftMessageB(bm1, bm2, bm3);
325 : 169172 : QuadRound(as0, as1, am3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);
326 : 169172 : QuadRound(bs0, bs1, bm3, 0x106aa070f40e3585ull, 0xd6990624d192e819ull);
327 : 169172 : ShiftMessageB(am2, am3, am0);
328 : 169172 : ShiftMessageB(bm2, bm3, bm0);
329 : 169172 : QuadRound(as0, as1, am0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);
330 : 169172 : QuadRound(bs0, bs1, bm0, 0x34b0bcb52748774cull, 0x1e376c0819a4c116ull);
331 : 169172 : ShiftMessageB(am3, am0, am1);
332 : 169172 : ShiftMessageB(bm3, bm0, bm1);
333 : 169172 : QuadRound(as0, as1, am1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);
334 : 169172 : QuadRound(bs0, bs1, bm1, 0x682e6ff35b9cca4full, 0x4ed8aa4a391c0cb3ull);
335 : 169172 : ShiftMessageC(am0, am1, am2);
336 : 169172 : ShiftMessageC(bm0, bm1, bm2);
337 : 169172 : QuadRound(as0, as1, am2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);
338 : 169172 : QuadRound(bs0, bs1, bm2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull);
339 : 169172 : ShiftMessageC(am1, am2, am3);
340 : 169172 : ShiftMessageC(bm1, bm2, bm3);
341 : 169172 : QuadRound(as0, as1, am3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull);
342 : 169172 : QuadRound(bs0, bs1, bm3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull);
343 : 169172 : as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i*)INIT0));
344 : 169172 : bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i*)INIT0));
345 : 169172 : as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i*)INIT1));
346 : 169172 : bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i*)INIT1));
347 : :
348 : : /* Extract hash into out */
349 : 169172 : Unshuffle(as0, as1);
350 : 169172 : Unshuffle(bs0, bs1);
351 : 169172 : Save(out, as0);
352 : 169172 : Save(out + 16, as1);
353 : 169172 : Save(out + 32, bs0);
354 : 169172 : Save(out + 48, bs1);
355 : 169172 : }
356 : :
357 : : }
358 : :
359 : : #endif
|
