1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
|
/*
* Hardware-accelerated implementation of SHA-512 using Arm NEON.
*/
#include "ssh.h"
#include "sha512.h"
#if USE_ARM64_NEON_H
#include <arm64_neon.h>
#else
#include <arm_neon.h>
#endif
static bool sha512_neon_available(void)
{
/*
* For Arm, we delegate to a per-platform detection function (see
* explanation in aes-neon.c).
*/
return platform_sha512_neon_available();
}
#if !HAVE_NEON_SHA512_INTRINSICS
/*
* clang 12 and before do not provide the SHA-512 NEON intrinsics, but
* do provide assembler support for the underlying instructions. So I
* define the intrinsic functions myself, using inline assembler.
*/
static inline uint64x2_t vsha512su0q_u64(uint64x2_t x, uint64x2_t y)
{
__asm__("sha512su0 %0.2D,%1.2D" : "+w" (x) : "w" (y));
return x;
}
static inline uint64x2_t vsha512su1q_u64(uint64x2_t x, uint64x2_t y,
uint64x2_t z)
{
__asm__("sha512su1 %0.2D,%1.2D,%2.2D" : "+w" (x) : "w" (y), "w" (z));
return x;
}
static inline uint64x2_t vsha512hq_u64(uint64x2_t x, uint64x2_t y,
uint64x2_t z)
{
__asm__("sha512h %0,%1,%2.2D" : "+w" (x) : "w" (y), "w" (z));
return x;
}
static inline uint64x2_t vsha512h2q_u64(uint64x2_t x, uint64x2_t y,
uint64x2_t z)
{
__asm__("sha512h2 %0,%1,%2.2D" : "+w" (x) : "w" (y), "w" (z));
return x;
}
#endif /* HAVE_NEON_SHA512_INTRINSICS */
typedef struct sha512_neon_core sha512_neon_core;
struct sha512_neon_core {
uint64x2_t ab, cd, ef, gh;
};
static inline uint64x2_t sha512_neon_load_input(const uint8_t *p)
{
return vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p)));
}
static inline uint64x2_t sha512_neon_schedule_update(
uint64x2_t m8, uint64x2_t m7, uint64x2_t m4, uint64x2_t m3, uint64x2_t m1)
{
/*
* vsha512su0q_u64() takes words from a long way back in the
* schedule and performs the sigma_0 half of the computation of
* the next two 64-bit message-schedule words.
*
* vsha512su1q_u64() combines the result of that with the sigma_1
* steps, to output the finished version of those two words. The
* total amount of input data it requires fits nicely into three
* 128-bit vector registers, but one of those registers is
* misaligned compared to the 128-bit chunks that the message
* schedule is stored in. So we use vextq_u64 to make one of its
* input words out of the second half of m4 and the first half of
* m3.
*/
return vsha512su1q_u64(vsha512su0q_u64(m8, m7), m1, vextq_u64(m4, m3, 1));
}
static inline void sha512_neon_round2(
unsigned round_index, uint64x2_t schedule_words,
uint64x2_t *ab, uint64x2_t *cd, uint64x2_t *ef, uint64x2_t *gh)
{
/*
* vsha512hq_u64 performs the Sigma_1 and Ch half of the
* computation of two rounds of SHA-512 (including feeding back
* one of the outputs from the first of those half-rounds into the
* second one).
*
* vsha512h2q_u64 combines the result of that with the Sigma_0 and
* Maj steps, and outputs one 128-bit vector that replaces the gh
* piece of the input hash state, and a second that updates cd by
* addition.
*
* Similarly to vsha512su1q_u64 above, some of the input registers
* expected by these instructions are misaligned by 64 bits
* relative to the chunks we've divided the hash state into, so we
* have to start by making 'de' and 'fg' words out of our input
* cd,ef,gh, using vextq_u64.
*
* Also, one of the inputs to vsha512hq_u64 is expected to contain
* the results of summing gh + two round constants + two words of
* message schedule, but the two words of the message schedule
* have to be the opposite way round in the vector register from
* the way that vsha512su1q_u64 output them. Hence, there's
* another vextq_u64 in here that swaps the two halves of the
* initial_sum vector register.
*
* (This also means that I don't have to prepare a specially
* reordered version of the sha512_round_constants[] array: as
* long as I'm unavoidably doing a swap at run time _anyway_, I
* can load from the normally ordered version of that array, and
* just take care to fold in that data _before_ the swap rather
* than after.)
*/
/* Load two round constants, with the first one in the low half */
uint64x2_t round_constants = vld1q_u64(
sha512_round_constants + round_index);
/* Add schedule words to round constants */
uint64x2_t initial_sum = vaddq_u64(schedule_words, round_constants);
/* Swap that sum around so the word used in the first of the two
* rounds is in the _high_ half of the vector, matching where h
* lives in the gh vector */
uint64x2_t swapped_initial_sum = vextq_u64(initial_sum, initial_sum, 1);
/* Add gh to that, now that they're matching ways round */
uint64x2_t sum = vaddq_u64(swapped_initial_sum, *gh);
/* Make the misaligned de and fg words */
uint64x2_t de = vextq_u64(*cd, *ef, 1);
uint64x2_t fg = vextq_u64(*ef, *gh, 1);
/* Now we're ready to put all the pieces together. The output from
* vsha512h2q_u64 can be used directly as the new gh, and the
* output from vsha512hq_u64 is simultaneously the intermediate
* value passed to h2 and the thing you have to add on to cd. */
uint64x2_t intermed = vsha512hq_u64(sum, fg, de);
*gh = vsha512h2q_u64(intermed, *cd, *ab);
*cd = vaddq_u64(*cd, intermed);
}
static inline void sha512_neon_block(sha512_neon_core *core, const uint8_t *p)
{
uint64x2_t s0, s1, s2, s3, s4, s5, s6, s7;
uint64x2_t ab = core->ab, cd = core->cd, ef = core->ef, gh = core->gh;
s0 = sha512_neon_load_input(p + 16*0);
sha512_neon_round2(0, s0, &ab, &cd, &ef, &gh);
s1 = sha512_neon_load_input(p + 16*1);
sha512_neon_round2(2, s1, &gh, &ab, &cd, &ef);
s2 = sha512_neon_load_input(p + 16*2);
sha512_neon_round2(4, s2, &ef, &gh, &ab, &cd);
s3 = sha512_neon_load_input(p + 16*3);
sha512_neon_round2(6, s3, &cd, &ef, &gh, &ab);
s4 = sha512_neon_load_input(p + 16*4);
sha512_neon_round2(8, s4, &ab, &cd, &ef, &gh);
s5 = sha512_neon_load_input(p + 16*5);
sha512_neon_round2(10, s5, &gh, &ab, &cd, &ef);
s6 = sha512_neon_load_input(p + 16*6);
sha512_neon_round2(12, s6, &ef, &gh, &ab, &cd);
s7 = sha512_neon_load_input(p + 16*7);
sha512_neon_round2(14, s7, &cd, &ef, &gh, &ab);
s0 = sha512_neon_schedule_update(s0, s1, s4, s5, s7);
sha512_neon_round2(16, s0, &ab, &cd, &ef, &gh);
s1 = sha512_neon_schedule_update(s1, s2, s5, s6, s0);
sha512_neon_round2(18, s1, &gh, &ab, &cd, &ef);
s2 = sha512_neon_schedule_update(s2, s3, s6, s7, s1);
sha512_neon_round2(20, s2, &ef, &gh, &ab, &cd);
s3 = sha512_neon_schedule_update(s3, s4, s7, s0, s2);
sha512_neon_round2(22, s3, &cd, &ef, &gh, &ab);
s4 = sha512_neon_schedule_update(s4, s5, s0, s1, s3);
sha512_neon_round2(24, s4, &ab, &cd, &ef, &gh);
s5 = sha512_neon_schedule_update(s5, s6, s1, s2, s4);
sha512_neon_round2(26, s5, &gh, &ab, &cd, &ef);
s6 = sha512_neon_schedule_update(s6, s7, s2, s3, s5);
sha512_neon_round2(28, s6, &ef, &gh, &ab, &cd);
s7 = sha512_neon_schedule_update(s7, s0, s3, s4, s6);
sha512_neon_round2(30, s7, &cd, &ef, &gh, &ab);
s0 = sha512_neon_schedule_update(s0, s1, s4, s5, s7);
sha512_neon_round2(32, s0, &ab, &cd, &ef, &gh);
s1 = sha512_neon_schedule_update(s1, s2, s5, s6, s0);
sha512_neon_round2(34, s1, &gh, &ab, &cd, &ef);
s2 = sha512_neon_schedule_update(s2, s3, s6, s7, s1);
sha512_neon_round2(36, s2, &ef, &gh, &ab, &cd);
s3 = sha512_neon_schedule_update(s3, s4, s7, s0, s2);
sha512_neon_round2(38, s3, &cd, &ef, &gh, &ab);
s4 = sha512_neon_schedule_update(s4, s5, s0, s1, s3);
sha512_neon_round2(40, s4, &ab, &cd, &ef, &gh);
s5 = sha512_neon_schedule_update(s5, s6, s1, s2, s4);
sha512_neon_round2(42, s5, &gh, &ab, &cd, &ef);
s6 = sha512_neon_schedule_update(s6, s7, s2, s3, s5);
sha512_neon_round2(44, s6, &ef, &gh, &ab, &cd);
s7 = sha512_neon_schedule_update(s7, s0, s3, s4, s6);
sha512_neon_round2(46, s7, &cd, &ef, &gh, &ab);
s0 = sha512_neon_schedule_update(s0, s1, s4, s5, s7);
sha512_neon_round2(48, s0, &ab, &cd, &ef, &gh);
s1 = sha512_neon_schedule_update(s1, s2, s5, s6, s0);
sha512_neon_round2(50, s1, &gh, &ab, &cd, &ef);
s2 = sha512_neon_schedule_update(s2, s3, s6, s7, s1);
sha512_neon_round2(52, s2, &ef, &gh, &ab, &cd);
s3 = sha512_neon_schedule_update(s3, s4, s7, s0, s2);
sha512_neon_round2(54, s3, &cd, &ef, &gh, &ab);
s4 = sha512_neon_schedule_update(s4, s5, s0, s1, s3);
sha512_neon_round2(56, s4, &ab, &cd, &ef, &gh);
s5 = sha512_neon_schedule_update(s5, s6, s1, s2, s4);
sha512_neon_round2(58, s5, &gh, &ab, &cd, &ef);
s6 = sha512_neon_schedule_update(s6, s7, s2, s3, s5);
sha512_neon_round2(60, s6, &ef, &gh, &ab, &cd);
s7 = sha512_neon_schedule_update(s7, s0, s3, s4, s6);
sha512_neon_round2(62, s7, &cd, &ef, &gh, &ab);
s0 = sha512_neon_schedule_update(s0, s1, s4, s5, s7);
sha512_neon_round2(64, s0, &ab, &cd, &ef, &gh);
s1 = sha512_neon_schedule_update(s1, s2, s5, s6, s0);
sha512_neon_round2(66, s1, &gh, &ab, &cd, &ef);
s2 = sha512_neon_schedule_update(s2, s3, s6, s7, s1);
sha512_neon_round2(68, s2, &ef, &gh, &ab, &cd);
s3 = sha512_neon_schedule_update(s3, s4, s7, s0, s2);
sha512_neon_round2(70, s3, &cd, &ef, &gh, &ab);
s4 = sha512_neon_schedule_update(s4, s5, s0, s1, s3);
sha512_neon_round2(72, s4, &ab, &cd, &ef, &gh);
s5 = sha512_neon_schedule_update(s5, s6, s1, s2, s4);
sha512_neon_round2(74, s5, &gh, &ab, &cd, &ef);
s6 = sha512_neon_schedule_update(s6, s7, s2, s3, s5);
sha512_neon_round2(76, s6, &ef, &gh, &ab, &cd);
s7 = sha512_neon_schedule_update(s7, s0, s3, s4, s6);
sha512_neon_round2(78, s7, &cd, &ef, &gh, &ab);
core->ab = vaddq_u64(core->ab, ab);
core->cd = vaddq_u64(core->cd, cd);
core->ef = vaddq_u64(core->ef, ef);
core->gh = vaddq_u64(core->gh, gh);
}
typedef struct sha512_neon {
sha512_neon_core core;
sha512_block blk;
BinarySink_IMPLEMENTATION;
ssh_hash hash;
} sha512_neon;
static void sha512_neon_write(BinarySink *bs, const void *vp, size_t len);
static ssh_hash *sha512_neon_new(const ssh_hashalg *alg)
{
const struct sha512_extra *extra = (const struct sha512_extra *)alg->extra;
if (!check_availability(extra))
return NULL;
sha512_neon *s = snew(sha512_neon);
s->hash.vt = alg;
BinarySink_INIT(s, sha512_neon_write);
BinarySink_DELEGATE_INIT(&s->hash, s);
return &s->hash;
}
static void sha512_neon_reset(ssh_hash *hash)
{
sha512_neon *s = container_of(hash, sha512_neon, hash);
const struct sha512_extra *extra =
(const struct sha512_extra *)hash->vt->extra;
s->core.ab = vld1q_u64(extra->initial_state);
s->core.cd = vld1q_u64(extra->initial_state+2);
s->core.ef = vld1q_u64(extra->initial_state+4);
s->core.gh = vld1q_u64(extra->initial_state+6);
sha512_block_setup(&s->blk);
}
static void sha512_neon_copyfrom(ssh_hash *hcopy, ssh_hash *horig)
{
sha512_neon *copy = container_of(hcopy, sha512_neon, hash);
sha512_neon *orig = container_of(horig, sha512_neon, hash);
*copy = *orig; /* structure copy */
BinarySink_COPIED(copy);
BinarySink_DELEGATE_INIT(©->hash, copy);
}
static void sha512_neon_free(ssh_hash *hash)
{
sha512_neon *s = container_of(hash, sha512_neon, hash);
smemclr(s, sizeof(*s));
sfree(s);
}
static void sha512_neon_write(BinarySink *bs, const void *vp, size_t len)
{
sha512_neon *s = BinarySink_DOWNCAST(bs, sha512_neon);
while (len > 0)
if (sha512_block_write(&s->blk, &vp, &len))
sha512_neon_block(&s->core, s->blk.block);
}
static void sha512_neon_digest(ssh_hash *hash, uint8_t *digest)
{
sha512_neon *s = container_of(hash, sha512_neon, hash);
sha512_block_pad(&s->blk, BinarySink_UPCAST(s));
vst1q_u8(digest, vrev64q_u8(vreinterpretq_u8_u64(s->core.ab)));
vst1q_u8(digest+16, vrev64q_u8(vreinterpretq_u8_u64(s->core.cd)));
vst1q_u8(digest+32, vrev64q_u8(vreinterpretq_u8_u64(s->core.ef)));
vst1q_u8(digest+48, vrev64q_u8(vreinterpretq_u8_u64(s->core.gh)));
}
static void sha384_neon_digest(ssh_hash *hash, uint8_t *digest)
{
sha512_neon *s = container_of(hash, sha512_neon, hash);
sha512_block_pad(&s->blk, BinarySink_UPCAST(s));
vst1q_u8(digest, vrev64q_u8(vreinterpretq_u8_u64(s->core.ab)));
vst1q_u8(digest+16, vrev64q_u8(vreinterpretq_u8_u64(s->core.cd)));
vst1q_u8(digest+32, vrev64q_u8(vreinterpretq_u8_u64(s->core.ef)));
}
SHA512_VTABLES(neon, "NEON accelerated");
|