Expose ECDSA_SIG_max_len to size a DER ECDSA-Sig-Value.

Also implement it without reference to crypto/asn1 or fake ASN1_INTEGERs and
add a test. Some platform crypto APIs only give back the key size, and not the
encoded signature length. No sense in implementing it twice.

BUG=347404,499653

Change-Id: I9aa27d52674375f8b036e57bb5850f091c9b25dd
Reviewed-on: https://boringssl-review.googlesource.com/5080
Reviewed-by: Adam Langley <agl@google.com>

2 files changed, 83 insertions, 26 deletions

diff --git a/crypto/ecdsa/ecdsa_asn1.c b/crypto/ecdsa/ecdsa_asn1.c
index f557ca77..93be405d 100644
--- a/crypto/ecdsa/ecdsa_asn1.c
+++ b/crypto/ecdsa/ecdsa_asn1.c
@@ -71,26 +71,20 @@ ASN1_SEQUENCE(ECDSA_SIG) = {
 IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(ECDSA_SIG, ECDSA_SIG, ECDSA_SIG);
 
 size_t ECDSA_size(const EC_KEY *key) {
-  size_t ret, i, group_order_size;
-  ASN1_INTEGER bs;
-  BIGNUM *order = NULL;
-  unsigned char buf[4];
-  const EC_GROUP *group;
+  if (key == NULL) {
+    return 0;
+  }
 
+  size_t group_order_size;
   if (key->ecdsa_meth && key->ecdsa_meth->group_order_size) {
     group_order_size = key->ecdsa_meth->group_order_size(key);
   } else {
-    size_t num_bits;
-
-    if (key == NULL) {
-      return 0;
-    }
-    group = EC_KEY_get0_group(key);
+    const EC_GROUP *group = EC_KEY_get0_group(key);
     if (group == NULL) {
       return 0;
     }
 
-    order = BN_new();
+    BIGNUM *order = BN_new();
     if (order == NULL) {
       return 0;
     }
@@ -99,21 +93,11 @@ size_t ECDSA_size(const EC_KEY *key) {
       return 0;
     }
 
-    num_bits = BN_num_bits(order);
-    group_order_size = (num_bits + 7) / 8;
+    group_order_size = BN_num_bytes(order);
+    BN_clear_free(order);
   }
 
-  bs.length = group_order_size;
-  bs.data = buf;
-  bs.type = V_ASN1_INTEGER;
-  /* If the top bit is set the ASN.1 encoding is 1 larger. */
-  buf[0] = 0xff;
-
-  i = i2d_ASN1_INTEGER(&bs, NULL);
-  i += i; /* r and s */
-  ret = ASN1_object_size(1, i, V_ASN1_SEQUENCE);
-  BN_clear_free(order);
-  return ret;
+  return ECDSA_SIG_max_len(group_order_size);
 }
 
 ECDSA_SIG *ECDSA_SIG_new(void) {
@@ -139,3 +123,37 @@ void ECDSA_SIG_free(ECDSA_SIG *sig) {
   BN_free(sig->s);
   OPENSSL_free(sig);
 }
+
+/* der_len_len returns the number of bytes needed to represent a length of |len|
+ * in DER. */
+static size_t der_len_len(size_t len) {
+  if (len < 0x80) {
+    return 1;
+  }
+  size_t ret = 1;
+  while (len > 0) {
+    ret++;
+    len >>= 8;
+  }
+  return ret;
+}
+
+size_t ECDSA_SIG_max_len(size_t order_len) {
+  /* Compute the maximum length of an |order_len| byte integer. Defensively
+   * assume that the leading 0x00 is included. */
+  size_t integer_len = 1 /* tag */ + der_len_len(order_len + 1) + 1 + order_len;
+  if (integer_len < order_len) {
+    return 0;
+  }
+  /* An ECDSA signature is two INTEGERs. */
+  size_t value_len = 2 * integer_len;
+  if (value_len < integer_len) {
+    return 0;
+  }
+  /* Add the header. */
+  size_t ret = 1 /* tag */ + der_len_len(value_len) + value_len;
+  if (ret < value_len) {
+    return 0;
+  }
+  return ret;
+}
diff --git a/crypto/ecdsa/ecdsa_test.cc b/crypto/ecdsa/ecdsa_test.cc
index a6bd7a19..615741ff 100644
--- a/crypto/ecdsa/ecdsa_test.cc
+++ b/crypto/ecdsa/ecdsa_test.cc
@@ -325,11 +325,50 @@ static bool TestBuiltin(FILE *out) {
   return true;
 }
 
+static bool TestECDSA_SIG_max_len(size_t order_len) {
+  /* Create the largest possible |ECDSA_SIG| of the given constraints. */
+  ScopedECDSA_SIG sig(ECDSA_SIG_new());
+  if (!sig) {
+    return false;
+  }
+  std::vector<uint8_t> bytes(order_len, 0xff);
+  if (!BN_bin2bn(bssl::vector_data(&bytes), bytes.size(), sig->r) ||
+      !BN_bin2bn(bssl::vector_data(&bytes), bytes.size(), sig->s)) {
+    return false;
+  }
+  /* Serialize it. */
+  int len = i2d_ECDSA_SIG(sig.get(), nullptr);
+  if (len < 0) {
+    return false;
+  }
+  std::vector<uint8_t> der(len);
+  uint8_t *ptr = bssl::vector_data(&der);
+  len = i2d_ECDSA_SIG(sig.get(), &ptr);
+  if (len < 0) {
+    return false;
+  }
+  der.resize(len);
+
+  size_t max_len = ECDSA_SIG_max_len(order_len);
+  if (max_len != static_cast<size_t>(len)) {
+    fprintf(stderr, "ECDSA_SIG_max_len(%u) returned %u, wanted %d\n",
+            static_cast<unsigned>(order_len), static_cast<unsigned>(max_len),
+            len);
+    return false;
+  }
+  return true;
+}
+
 int main(void) {
   CRYPTO_library_init();
   ERR_load_crypto_strings();
 
-  if (!TestBuiltin(stdout)) {
+  if (!TestBuiltin(stdout) ||
+      !TestECDSA_SIG_max_len(224/8) ||
+      !TestECDSA_SIG_max_len(256/8) ||
+      !TestECDSA_SIG_max_len(384/8) ||
+      !TestECDSA_SIG_max_len(512/8) ||
+      !TestECDSA_SIG_max_len(10000)) {
     printf("\nECDSA test failed\n");
     ERR_print_errors_fp(stdout);
     return 1;