diff options
Diffstat (limited to 'crypto/sha512-neon.c')
| -rw-r--r-- | crypto/sha512-neon.c | 329 |
1 files changed, 329 insertions, 0 deletions
diff --git a/crypto/sha512-neon.c b/crypto/sha512-neon.c new file mode 100644 index 00000000..849a79d7 --- /dev/null +++ b/crypto/sha512-neon.c @@ -0,0 +1,329 @@ +/* + * Hardware-accelerated implementation of SHA-512 using Arm NEON. + */ + +#include "ssh.h" +#include "sha512.h" + +#if USE_ARM64_NEON_H +#include <arm64_neon.h> +#else +#include <arm_neon.h> +#endif + +static bool sha512_neon_available(void) +{ + /* + * For Arm, we delegate to a per-platform detection function (see + * explanation in aes-neon.c). + */ + return platform_sha512_neon_available(); +} + +#if !HAVE_NEON_SHA512_INTRINSICS +/* + * clang 12 and before do not provide the SHA-512 NEON intrinsics, but + * do provide assembler support for the underlying instructions. So I + * define the intrinsic functions myself, using inline assembler. + */ +static inline uint64x2_t vsha512su0q_u64(uint64x2_t x, uint64x2_t y) +{ + __asm__("sha512su0 %0.2D,%1.2D" : "+w" (x) : "w" (y)); + return x; +} +static inline uint64x2_t vsha512su1q_u64(uint64x2_t x, uint64x2_t y, + uint64x2_t z) +{ + __asm__("sha512su1 %0.2D,%1.2D,%2.2D" : "+w" (x) : "w" (y), "w" (z)); + return x; +} +static inline uint64x2_t vsha512hq_u64(uint64x2_t x, uint64x2_t y, + uint64x2_t z) +{ + __asm__("sha512h %0,%1,%2.2D" : "+w" (x) : "w" (y), "w" (z)); + return x; +} +static inline uint64x2_t vsha512h2q_u64(uint64x2_t x, uint64x2_t y, + uint64x2_t z) +{ + __asm__("sha512h2 %0,%1,%2.2D" : "+w" (x) : "w" (y), "w" (z)); + return x; +} +#endif /* HAVE_NEON_SHA512_INTRINSICS */ + +typedef struct sha512_neon_core sha512_neon_core; +struct sha512_neon_core { + uint64x2_t ab, cd, ef, gh; +}; + +static inline uint64x2_t sha512_neon_load_input(const uint8_t *p) +{ + return vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p))); +} + +static inline uint64x2_t sha512_neon_schedule_update( + uint64x2_t m8, uint64x2_t m7, uint64x2_t m4, uint64x2_t m3, uint64x2_t m1) +{ + /* + * vsha512su0q_u64() takes words from a long way back in the + * schedule and performs the sigma_0 half of the computation of + * the next two 64-bit message-schedule words. + * + * vsha512su1q_u64() combines the result of that with the sigma_1 + * steps, to output the finished version of those two words. The + * total amount of input data it requires fits nicely into three + * 128-bit vector registers, but one of those registers is + * misaligned compared to the 128-bit chunks that the message + * schedule is stored in. So we use vextq_u64 to make one of its + * input words out of the second half of m4 and the first half of + * m3. + */ + return vsha512su1q_u64(vsha512su0q_u64(m8, m7), m1, vextq_u64(m4, m3, 1)); +} + +static inline void sha512_neon_round2( + unsigned round_index, uint64x2_t schedule_words, + uint64x2_t *ab, uint64x2_t *cd, uint64x2_t *ef, uint64x2_t *gh) +{ + /* + * vsha512hq_u64 performs the Sigma_1 and Ch half of the + * computation of two rounds of SHA-512 (including feeding back + * one of the outputs from the first of those half-rounds into the + * second one). + * + * vsha512h2q_u64 combines the result of that with the Sigma_0 and + * Maj steps, and outputs one 128-bit vector that replaces the gh + * piece of the input hash state, and a second that updates cd by + * addition. + * + * Similarly to vsha512su1q_u64 above, some of the input registers + * expected by these instructions are misaligned by 64 bits + * relative to the chunks we've divided the hash state into, so we + * have to start by making 'de' and 'fg' words out of our input + * cd,ef,gh, using vextq_u64. + * + * Also, one of the inputs to vsha512hq_u64 is expected to contain + * the results of summing gh + two round constants + two words of + * message schedule, but the two words of the message schedule + * have to be the opposite way round in the vector register from + * the way that vsha512su1q_u64 output them. Hence, there's + * another vextq_u64 in here that swaps the two halves of the + * initial_sum vector register. + * + * (This also means that I don't have to prepare a specially + * reordered version of the sha512_round_constants[] array: as + * long as I'm unavoidably doing a swap at run time _anyway_, I + * can load from the normally ordered version of that array, and + * just take care to fold in that data _before_ the swap rather + * than after.) + */ + + /* Load two round constants, with the first one in the low half */ + uint64x2_t round_constants = vld1q_u64( + sha512_round_constants + round_index); + + /* Add schedule words to round constants */ + uint64x2_t initial_sum = vaddq_u64(schedule_words, round_constants); + + /* Swap that sum around so the word used in the first of the two + * rounds is in the _high_ half of the vector, matching where h + * lives in the gh vector */ + uint64x2_t swapped_initial_sum = vextq_u64(initial_sum, initial_sum, 1); + + /* Add gh to that, now that they're matching ways round */ + uint64x2_t sum = vaddq_u64(swapped_initial_sum, *gh); + + /* Make the misaligned de and fg words */ + uint64x2_t de = vextq_u64(*cd, *ef, 1); + uint64x2_t fg = vextq_u64(*ef, *gh, 1); + + /* Now we're ready to put all the pieces together. The output from + * vsha512h2q_u64 can be used directly as the new gh, and the + * output from vsha512hq_u64 is simultaneously the intermediate + * value passed to h2 and the thing you have to add on to cd. */ + uint64x2_t intermed = vsha512hq_u64(sum, fg, de); + *gh = vsha512h2q_u64(intermed, *cd, *ab); + *cd = vaddq_u64(*cd, intermed); +} + +static inline void sha512_neon_block(sha512_neon_core *core, const uint8_t *p) +{ + uint64x2_t s0, s1, s2, s3, s4, s5, s6, s7; + + uint64x2_t ab = core->ab, cd = core->cd, ef = core->ef, gh = core->gh; + + s0 = sha512_neon_load_input(p + 16*0); + sha512_neon_round2(0, s0, &ab, &cd, &ef, &gh); + s1 = sha512_neon_load_input(p + 16*1); + sha512_neon_round2(2, s1, &gh, &ab, &cd, &ef); + s2 = sha512_neon_load_input(p + 16*2); + sha512_neon_round2(4, s2, &ef, &gh, &ab, &cd); + s3 = sha512_neon_load_input(p + 16*3); + sha512_neon_round2(6, s3, &cd, &ef, &gh, &ab); + s4 = sha512_neon_load_input(p + 16*4); + sha512_neon_round2(8, s4, &ab, &cd, &ef, &gh); + s5 = sha512_neon_load_input(p + 16*5); + sha512_neon_round2(10, s5, &gh, &ab, &cd, &ef); + s6 = sha512_neon_load_input(p + 16*6); + sha512_neon_round2(12, s6, &ef, &gh, &ab, &cd); + s7 = sha512_neon_load_input(p + 16*7); + sha512_neon_round2(14, s7, &cd, &ef, &gh, &ab); + s0 = sha512_neon_schedule_update(s0, s1, s4, s5, s7); + sha512_neon_round2(16, s0, &ab, &cd, &ef, &gh); + s1 = sha512_neon_schedule_update(s1, s2, s5, s6, s0); + sha512_neon_round2(18, s1, &gh, &ab, &cd, &ef); + s2 = sha512_neon_schedule_update(s2, s3, s6, s7, s1); + sha512_neon_round2(20, s2, &ef, &gh, &ab, &cd); + s3 = sha512_neon_schedule_update(s3, s4, s7, s0, s2); + sha512_neon_round2(22, s3, &cd, &ef, &gh, &ab); + s4 = sha512_neon_schedule_update(s4, s5, s0, s1, s3); + sha512_neon_round2(24, s4, &ab, &cd, &ef, &gh); + s5 = sha512_neon_schedule_update(s5, s6, s1, s2, s4); + sha512_neon_round2(26, s5, &gh, &ab, &cd, &ef); + s6 = sha512_neon_schedule_update(s6, s7, s2, s3, s5); + sha512_neon_round2(28, s6, &ef, &gh, &ab, &cd); + s7 = sha512_neon_schedule_update(s7, s0, s3, s4, s6); + sha512_neon_round2(30, s7, &cd, &ef, &gh, &ab); + s0 = sha512_neon_schedule_update(s0, s1, s4, s5, s7); + sha512_neon_round2(32, s0, &ab, &cd, &ef, &gh); + s1 = sha512_neon_schedule_update(s1, s2, s5, s6, s0); + sha512_neon_round2(34, s1, &gh, &ab, &cd, &ef); + s2 = sha512_neon_schedule_update(s2, s3, s6, s7, s1); + sha512_neon_round2(36, s2, &ef, &gh, &ab, &cd); + s3 = sha512_neon_schedule_update(s3, s4, s7, s0, s2); + sha512_neon_round2(38, s3, &cd, &ef, &gh, &ab); + s4 = sha512_neon_schedule_update(s4, s5, s0, s1, s3); + sha512_neon_round2(40, s4, &ab, &cd, &ef, &gh); + s5 = sha512_neon_schedule_update(s5, s6, s1, s2, s4); + sha512_neon_round2(42, s5, &gh, &ab, &cd, &ef); + s6 = sha512_neon_schedule_update(s6, s7, s2, s3, s5); + sha512_neon_round2(44, s6, &ef, &gh, &ab, &cd); + s7 = sha512_neon_schedule_update(s7, s0, s3, s4, s6); + sha512_neon_round2(46, s7, &cd, &ef, &gh, &ab); + s0 = sha512_neon_schedule_update(s0, s1, s4, s5, s7); + sha512_neon_round2(48, s0, &ab, &cd, &ef, &gh); + s1 = sha512_neon_schedule_update(s1, s2, s5, s6, s0); + sha512_neon_round2(50, s1, &gh, &ab, &cd, &ef); + s2 = sha512_neon_schedule_update(s2, s3, s6, s7, s1); + sha512_neon_round2(52, s2, &ef, &gh, &ab, &cd); + s3 = sha512_neon_schedule_update(s3, s4, s7, s0, s2); + sha512_neon_round2(54, s3, &cd, &ef, &gh, &ab); + s4 = sha512_neon_schedule_update(s4, s5, s0, s1, s3); + sha512_neon_round2(56, s4, &ab, &cd, &ef, &gh); + s5 = sha512_neon_schedule_update(s5, s6, s1, s2, s4); + sha512_neon_round2(58, s5, &gh, &ab, &cd, &ef); + s6 = sha512_neon_schedule_update(s6, s7, s2, s3, s5); + sha512_neon_round2(60, s6, &ef, &gh, &ab, &cd); + s7 = sha512_neon_schedule_update(s7, s0, s3, s4, s6); + sha512_neon_round2(62, s7, &cd, &ef, &gh, &ab); + s0 = sha512_neon_schedule_update(s0, s1, s4, s5, s7); + sha512_neon_round2(64, s0, &ab, &cd, &ef, &gh); + s1 = sha512_neon_schedule_update(s1, s2, s5, s6, s0); + sha512_neon_round2(66, s1, &gh, &ab, &cd, &ef); + s2 = sha512_neon_schedule_update(s2, s3, s6, s7, s1); + sha512_neon_round2(68, s2, &ef, &gh, &ab, &cd); + s3 = sha512_neon_schedule_update(s3, s4, s7, s0, s2); + sha512_neon_round2(70, s3, &cd, &ef, &gh, &ab); + s4 = sha512_neon_schedule_update(s4, s5, s0, s1, s3); + sha512_neon_round2(72, s4, &ab, &cd, &ef, &gh); + s5 = sha512_neon_schedule_update(s5, s6, s1, s2, s4); + sha512_neon_round2(74, s5, &gh, &ab, &cd, &ef); + s6 = sha512_neon_schedule_update(s6, s7, s2, s3, s5); + sha512_neon_round2(76, s6, &ef, &gh, &ab, &cd); + s7 = sha512_neon_schedule_update(s7, s0, s3, s4, s6); + sha512_neon_round2(78, s7, &cd, &ef, &gh, &ab); + + core->ab = vaddq_u64(core->ab, ab); + core->cd = vaddq_u64(core->cd, cd); + core->ef = vaddq_u64(core->ef, ef); + core->gh = vaddq_u64(core->gh, gh); +} + +typedef struct sha512_neon { + sha512_neon_core core; + sha512_block blk; + BinarySink_IMPLEMENTATION; + ssh_hash hash; +} sha512_neon; + +static void sha512_neon_write(BinarySink *bs, const void *vp, size_t len); + +static ssh_hash *sha512_neon_new(const ssh_hashalg *alg) +{ + const struct sha512_extra *extra = (const struct sha512_extra *)alg->extra; + if (!check_availability(extra)) + return NULL; + + sha512_neon *s = snew(sha512_neon); + + s->hash.vt = alg; + BinarySink_INIT(s, sha512_neon_write); + BinarySink_DELEGATE_INIT(&s->hash, s); + return &s->hash; +} + +static void sha512_neon_reset(ssh_hash *hash) +{ + sha512_neon *s = container_of(hash, sha512_neon, hash); + const struct sha512_extra *extra = + (const struct sha512_extra *)hash->vt->extra; + + s->core.ab = vld1q_u64(extra->initial_state); + s->core.cd = vld1q_u64(extra->initial_state+2); + s->core.ef = vld1q_u64(extra->initial_state+4); + s->core.gh = vld1q_u64(extra->initial_state+6); + + sha512_block_setup(&s->blk); +} + +static void sha512_neon_copyfrom(ssh_hash *hcopy, ssh_hash *horig) +{ + sha512_neon *copy = container_of(hcopy, sha512_neon, hash); + sha512_neon *orig = container_of(horig, sha512_neon, hash); + + *copy = *orig; /* structure copy */ + + BinarySink_COPIED(copy); + BinarySink_DELEGATE_INIT(©->hash, copy); +} + +static void sha512_neon_free(ssh_hash *hash) +{ + sha512_neon *s = container_of(hash, sha512_neon, hash); + smemclr(s, sizeof(*s)); + sfree(s); +} + +static void sha512_neon_write(BinarySink *bs, const void *vp, size_t len) +{ + sha512_neon *s = BinarySink_DOWNCAST(bs, sha512_neon); + + while (len > 0) + if (sha512_block_write(&s->blk, &vp, &len)) + sha512_neon_block(&s->core, s->blk.block); +} + +static void sha512_neon_digest(ssh_hash *hash, uint8_t *digest) +{ + sha512_neon *s = container_of(hash, sha512_neon, hash); + + sha512_block_pad(&s->blk, BinarySink_UPCAST(s)); + + vst1q_u8(digest, vrev64q_u8(vreinterpretq_u8_u64(s->core.ab))); + vst1q_u8(digest+16, vrev64q_u8(vreinterpretq_u8_u64(s->core.cd))); + vst1q_u8(digest+32, vrev64q_u8(vreinterpretq_u8_u64(s->core.ef))); + vst1q_u8(digest+48, vrev64q_u8(vreinterpretq_u8_u64(s->core.gh))); +} + +static void sha384_neon_digest(ssh_hash *hash, uint8_t *digest) +{ + sha512_neon *s = container_of(hash, sha512_neon, hash); + + sha512_block_pad(&s->blk, BinarySink_UPCAST(s)); + + vst1q_u8(digest, vrev64q_u8(vreinterpretq_u8_u64(s->core.ab))); + vst1q_u8(digest+16, vrev64q_u8(vreinterpretq_u8_u64(s->core.cd))); + vst1q_u8(digest+32, vrev64q_u8(vreinterpretq_u8_u64(s->core.ef))); +} + +SHA512_VTABLES(neon, "NEON accelerated"); |