Use a different way to check for big endian

endian.h doesn't seem to work on macOS, let's use the compiler
defined macros.

Also assume little endian if the macros aren't defined to be on the safe side.

2 files changed, 37 insertions, 39 deletions

diff --git a/crypto/ec/p224-64.c b/crypto/ec/p224-64.c
index d424a475..795e9640 100644
--- a/crypto/ec/p224-64.c
+++ b/crypto/ec/p224-64.c
@@ -17,7 +17,6 @@
  * Inspired by Daniel J. Bernstein's public domain nistp224 implementation
  * and Adam Langley's public domain 64-bit C implementation of curve25519. */
 
-#include <endian.h>
 #include <openssl/base.h>
 
 #if defined(OPENSSL_64_BIT) && !defined(OPENSSL_WINDOWS) && \
@@ -186,32 +185,32 @@ static const felem g_pre_comp[2][16][3] = {
 
 /* Helper functions to convert field elements to/from internal representation */
 static void bin28_to_felem(felem out, const u8 in[28]) {
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-   out[0] = *((const uint64_t *)(in)) & 0x00ffffffffffffff;
-   out[1] = (*((const uint64_t *)(in + 7))) & 0x00ffffffffffffff;
-   out[2] = (*((const uint64_t *)(in + 14))) & 0x00ffffffffffffff;
-   out[3] = (*((const uint64_t *)(in + 20))) >> 8;
-#else
+#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
    out[0] = (*((const uint64_t *)(in + 20))) << 8 >> 8;
    out[1] = (*((const uint64_t *)(in + 14))) >> 8;
    out[2] = (*((const uint64_t *)(in + 7))) >> 8;
    out[3] = *((const uint64_t *)(in)) >>8;
+#else
+   out[0] = *((const uint64_t *)(in)) & 0x00ffffffffffffff;
+   out[1] = (*((const uint64_t *)(in + 7))) & 0x00ffffffffffffff;
+   out[2] = (*((const uint64_t *)(in + 14))) & 0x00ffffffffffffff;
+   out[3] = (*((const uint64_t *)(in + 20))) >> 8;
 #endif
 }
 
 static void felem_to_bin28(u8 out[28], const felem in) {
   size_t i;
   for (i = 0; i < 7; ++i) {
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-    out[i] = in[0] >> (8 * i);
-    out[i + 7] = in[1] >> (8 * i);
-    out[i + 14] = in[2] >> (8 * i);
-    out[i + 21] = in[3] >> (8 * i);
-#else
+#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
     out[i] = *((u8 *)&in[3] + i + 1);
     out[i + 7] = *((u8 *)&in[2] + i + 1);
     out[i + 14] = *((u8 *)&in[1] + i + 1);
     out[i + 21] = *((u8 *)&in[0] + i + 1);
+#else
+    out[i] = in[0] >> (8 * i);
+    out[i + 7] = in[1] >> (8 * i);
+    out[i + 14] = in[2] >> (8 * i);
+    out[i + 21] = in[3] >> (8 * i);
 #endif
   }
 }
@@ -238,11 +237,11 @@ static int BN_to_felem(felem out, const BIGNUM *bn) {
 
   felem_bytearray b_in;
   num_bytes = BN_bn2bin(bn, b_in);
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-  flip_endian(b_out, b_in, num_bytes);
-#else
+#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
   memcpy(b_out+sizeof(b_out)-num_bytes, b_in, num_bytes);
   memset(b_out, 0, sizeof(b_out)-num_bytes);
+#else
+  flip_endian(b_out, b_in, num_bytes);
 #endif
   bin28_to_felem(out, b_out);
   return 1;
@@ -251,12 +250,12 @@ static int BN_to_felem(felem out, const BIGNUM *bn) {
 /* From internal representation to OpenSSL BIGNUM */
 static BIGNUM *felem_to_BN(BIGNUM *out, const felem in) {
   felem_bytearray b_out;
-#if __BYTE_ORDER == __LITTLE_ENDIAN
+#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+  felem_to_bin28(b_out, in);
+#else
   felem_bytearray b_in;
   felem_to_bin28(b_in, in);
   flip_endian(b_out, b_in, sizeof(b_out));
-#else
-  felem_to_bin28(b_out, in);
 #endif
   return BN_bin2bn(b_out, sizeof(b_out), out);
 }
diff --git a/crypto/ec/p256-64.c b/crypto/ec/p256-64.c
index 9ee2bb84..d3b413b2 100644
--- a/crypto/ec/p256-64.c
+++ b/crypto/ec/p256-64.c
@@ -19,7 +19,6 @@
  * Otherwise based on Emilia's P224 work, which was inspired by my curve25519
  * work which got its smarts from Daniel J. Bernstein's work on the same. */
 
-#include <endian.h>
 #include <openssl/base.h>
 
 #if defined(OPENSSL_64_BIT) && !defined(OPENSSL_WINDOWS)
@@ -78,32 +77,32 @@ static const u64 bottom63bits = 0x7ffffffffffffffful;
 /* bin32_to_felem takes a little-endian byte array and converts it into felem
  * form. This assumes no particular CPU endianess. */
 static void bin32_to_felem(felem out, const u8 in[32]) {
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-  out[0] = *((const u64 *)&in[0]);
-  out[1] = *((const u64 *)&in[8]);
-  out[2] = *((const u64 *)&in[16]);
-  out[3] = *((const u64 *)&in[24]);
-#else
+#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
   out[0] = *((const u64 *)&in[24]);
   out[1] = *((const u64 *)&in[16]);
   out[2] = *((const u64 *)&in[8]);
   out[3] = *((const u64 *)&in[0]);
+#else
+  out[0] = *((const u64 *)&in[0]);
+  out[1] = *((const u64 *)&in[8]);
+  out[2] = *((const u64 *)&in[16]);
+  out[3] = *((const u64 *)&in[24]);
 #endif
 }
 
 /* smallfelem_to_bin32 takes a smallfelem and serialises into a little endian,
  * 32 byte array. This assumes no particular CPU endianess. */
 static void smallfelem_to_bin32(u8 out[32], const smallfelem in) {
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-  *((u64 *)&out[0]) = in[0];
-  *((u64 *)&out[8]) = in[1];
-  *((u64 *)&out[16]) = in[2];
-  *((u64 *)&out[24]) = in[3];
-#else
+#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
   *((u64 *)&out[0]) = in[3];
   *((u64 *)&out[8]) = in[2];
   *((u64 *)&out[16]) = in[1];
   *((u64 *)&out[24]) = in[0];
+#else
+  *((u64 *)&out[0]) = in[0];
+  *((u64 *)&out[8]) = in[1];
+  *((u64 *)&out[16]) = in[2];
+  *((u64 *)&out[24]) = in[3];
 #endif
 }
 
@@ -133,12 +132,12 @@ static int BN_to_felem(felem out, const BIGNUM *bn) {
 
   felem_bytearray b_in;
   num_bytes = BN_bn2bin(bn, b_in);
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-  flip_endian(b_out, b_in, num_bytes);
-  bin32_to_felem(out, b_out);
-#else
+#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
   memcpy(b_out+sizeof(b_out)-num_bytes, b_in, num_bytes);
   memset(b_out, 0, sizeof(b_out)-num_bytes);
+#else
+  flip_endian(b_out, b_in, num_bytes);
+  bin32_to_felem(out, b_out);
 #endif
   return 1;
 }
@@ -146,12 +145,12 @@ static int BN_to_felem(felem out, const BIGNUM *bn) {
 /* felem_to_BN converts an felem into an OpenSSL BIGNUM. */
 static BIGNUM *smallfelem_to_BN(BIGNUM *out, const smallfelem in) {
   felem_bytearray b_out;
-#if __BYTE_ORDER == __LITTLE_ENDIAN
+#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+  smallfelem_to_bin32(b_out, in);
+#else
   felem_bytearray b_in;
   smallfelem_to_bin32(b_in, in);
   flip_endian(b_out, b_in, sizeof(b_out));
-#else
-  smallfelem_to_bin32(b_out, in);
 #endif
   return BN_bin2bn(b_out, sizeof(b_out), out);
 }