Deduplicate the rest of ASN1_INTEGER and ASN1_ENUMERATED.

These functions need some work, but first avoid the duplicate versions.
See also upstream's 6c5b6cb035666d46495ccbe4a4f3d5e3a659cd40.

Update-Note: ASN1_INTEGER_to_BN and ASN1_ENUMERATED_to_BN will now fail
when called on an ASN1_STRING/ASN1_INTEGER/ASN1_ENUMERATED (they're all
the same type) with the wrong runtime type value. Previously, callers
that mixed them up would get the right answer on positive values and
silently misinterpret the input on negative values. This change matches
OpenSSL's 1.1.0's behavior.

Change-Id: Ie01366003f7b2e49477cb73eaf7eaac26d86675d
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/51631
Reviewed-by: Adam Langley <agl@google.com>
Commit-Queue: David Benjamin <davidben@google.com>
diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt
index 6ab74b8..79802c6 100644
--- a/crypto/CMakeLists.txt
+++ b/crypto/CMakeLists.txt
@@ -203,7 +203,6 @@
   asn1/a_bool.c
   asn1/a_d2i_fp.c
   asn1/a_dup.c
-  asn1/a_enum.c
   asn1/a_gentm.c
   asn1/a_i2d_fp.c
   asn1/a_int.c
diff --git a/crypto/asn1/a_enum.c b/crypto/asn1/a_enum.c
deleted file mode 100644
index d7a7357..0000000
--- a/crypto/asn1/a_enum.c
+++ /dev/null
@@ -1,195 +0,0 @@
-/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
- * All rights reserved.
- *
- * This package is an SSL implementation written
- * by Eric Young (eay@cryptsoft.com).
- * The implementation was written so as to conform with Netscapes SSL.
- *
- * This library is free for commercial and non-commercial use as long as
- * the following conditions are aheared to.  The following conditions
- * apply to all code found in this distribution, be it the RC4, RSA,
- * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
- * included with this distribution is covered by the same copyright terms
- * except that the holder is Tim Hudson (tjh@cryptsoft.com).
- *
- * Copyright remains Eric Young's, and as such any Copyright notices in
- * the code are not to be removed.
- * If this package is used in a product, Eric Young should be given attribution
- * as the author of the parts of the library used.
- * This can be in the form of a textual message at program startup or
- * in documentation (online or textual) provided with the package.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *    "This product includes cryptographic software written by
- *     Eric Young (eay@cryptsoft.com)"
- *    The word 'cryptographic' can be left out if the rouines from the library
- *    being used are not cryptographic related :-).
- * 4. If you include any Windows specific code (or a derivative thereof) from
- *    the apps directory (application code) you must include an acknowledgement:
- *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
- *
- * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * The licence and distribution terms for any publically available version or
- * derivative of this code cannot be changed.  i.e. this code cannot simply be
- * copied and put under another distribution licence
- * [including the GNU Public Licence.] */
-
-#include <openssl/asn1.h>
-
-#include <limits.h>
-#include <string.h>
-
-#include <openssl/err.h>
-#include <openssl/mem.h>
-
-#include "../internal.h"
-
-
-/*
- * Code for ENUMERATED type: identical to INTEGER apart from a different tag.
- * for comments on encoding see a_int.c
- */
-
-int ASN1_ENUMERATED_set(ASN1_ENUMERATED *a, long v)
-{
-    int j, k;
-    unsigned int i;
-    unsigned char buf[sizeof(long) + 1];
-    long d;
-
-    a->type = V_ASN1_ENUMERATED;
-    if (a->length < (int)(sizeof(long) + 1)) {
-        if (a->data != NULL)
-            OPENSSL_free(a->data);
-        if ((a->data =
-             (unsigned char *)OPENSSL_malloc(sizeof(long) + 1)) != NULL)
-            OPENSSL_memset((char *)a->data, 0, sizeof(long) + 1);
-    }
-    if (a->data == NULL) {
-        OPENSSL_PUT_ERROR(ASN1, ERR_R_MALLOC_FAILURE);
-        return (0);
-    }
-    d = v;
-    if (d < 0) {
-        d = -d;
-        a->type = V_ASN1_NEG_ENUMERATED;
-    }
-
-    for (i = 0; i < sizeof(long); i++) {
-        if (d == 0)
-            break;
-        buf[i] = (int)d & 0xff;
-        d >>= 8;
-    }
-    j = 0;
-    for (k = i - 1; k >= 0; k--)
-        a->data[j++] = buf[k];
-    a->length = j;
-    return (1);
-}
-
-long ASN1_ENUMERATED_get(const ASN1_ENUMERATED *a)
-{
-    int neg = 0, i;
-
-    if (a == NULL)
-        return (0L);
-    i = a->type;
-    if (i == V_ASN1_NEG_ENUMERATED)
-        neg = 1;
-    else if (i != V_ASN1_ENUMERATED)
-        return -1;
-
-    OPENSSL_STATIC_ASSERT(sizeof(uint64_t) >= sizeof(long),
-                          "long larger than uint64_t");
-
-    if (a->length > (int)sizeof(uint64_t)) {
-        /* hmm... a bit ugly */
-        return -1;
-    }
-
-    uint64_t r64 = 0;
-    if (a->data != NULL) {
-      for (i = 0; i < a->length; i++) {
-          r64 <<= 8;
-          r64 |= (unsigned char)a->data[i];
-      }
-
-      if (r64 > LONG_MAX) {
-          return -1;
-      }
-    }
-
-    long r = (long) r64;
-    if (neg)
-        r = -r;
-
-    return r;
-}
-
-ASN1_ENUMERATED *BN_to_ASN1_ENUMERATED(const BIGNUM *bn, ASN1_ENUMERATED *ai)
-{
-    ASN1_ENUMERATED *ret;
-    int len, j;
-
-    if (ai == NULL)
-        ret = ASN1_ENUMERATED_new();
-    else
-        ret = ai;
-    if (ret == NULL) {
-        OPENSSL_PUT_ERROR(ASN1, ASN1_R_NESTED_ASN1_ERROR);
-        goto err;
-    }
-    if (BN_is_negative(bn))
-        ret->type = V_ASN1_NEG_ENUMERATED;
-    else
-        ret->type = V_ASN1_ENUMERATED;
-    j = BN_num_bits(bn);
-    len = ((j == 0) ? 0 : ((j / 8) + 1));
-    if (ret->length < len + 4) {
-        unsigned char *new_data = OPENSSL_realloc(ret->data, len + 4);
-        if (!new_data) {
-            OPENSSL_PUT_ERROR(ASN1, ERR_R_MALLOC_FAILURE);
-            goto err;
-        }
-        ret->data = new_data;
-    }
-
-    ret->length = BN_bn2bin(bn, ret->data);
-    return (ret);
- err:
-    if (ret != ai)
-        ASN1_ENUMERATED_free(ret);
-    return (NULL);
-}
-
-BIGNUM *ASN1_ENUMERATED_to_BN(const ASN1_ENUMERATED *ai, BIGNUM *bn)
-{
-    BIGNUM *ret;
-
-    if ((ret = BN_bin2bn(ai->data, ai->length, bn)) == NULL)
-        OPENSSL_PUT_ERROR(ASN1, ASN1_R_BN_LIB);
-    else if (ai->type == V_ASN1_NEG_ENUMERATED)
-        BN_set_negative(ret, 1);
-    return (ret);
-}
diff --git a/crypto/asn1/a_int.c b/crypto/asn1/a_int.c
index c94f574..833ca7c 100644
--- a/crypto/asn1/a_int.c
+++ b/crypto/asn1/a_int.c
@@ -304,7 +304,21 @@
     return 1;
 }
 
-int ASN1_INTEGER_set_uint64(ASN1_INTEGER *out, uint64_t v)
+int ASN1_ENUMERATED_set(ASN1_ENUMERATED *a, long v)
+{
+    if (v >= 0) {
+        return ASN1_ENUMERATED_set_uint64(a, (uint64_t) v);
+    }
+
+    if (!ASN1_ENUMERATED_set_uint64(a, 0 - (uint64_t) v)) {
+        return 0;
+    }
+
+    a->type = V_ASN1_NEG_ENUMERATED;
+    return 1;
+}
+
+static int asn1_string_set_uint64(ASN1_STRING *out, uint64_t v, int type)
 {
     uint8_t *const newdata = OPENSSL_malloc(sizeof(uint64_t));
     if (newdata == NULL) {
@@ -317,7 +331,7 @@
     v = CRYPTO_bswap8(v);
     memcpy(out->data, &v, sizeof(v));
 
-    out->type = V_ASN1_INTEGER;
+    out->type = type;
 
     size_t leading_zeros;
     for (leading_zeros = 0; leading_zeros < sizeof(uint64_t) - 1;
@@ -333,16 +347,26 @@
     return 1;
 }
 
-long ASN1_INTEGER_get(const ASN1_INTEGER *a)
+int ASN1_INTEGER_set_uint64(ASN1_INTEGER *out, uint64_t v)
+{
+    return asn1_string_set_uint64(out, v, V_ASN1_INTEGER);
+}
+
+int ASN1_ENUMERATED_set_uint64(ASN1_ENUMERATED *out, uint64_t v)
+{
+    return asn1_string_set_uint64(out, v, V_ASN1_ENUMERATED);
+}
+
+static long asn1_string_get_long(const ASN1_STRING *a, int type)
 {
     int neg = 0, i;
 
     if (a == NULL)
         return (0L);
     i = a->type;
-    if (i == V_ASN1_NEG_INTEGER)
+    if (i == (type | V_ASN1_NEG))
         neg = 1;
-    else if (i != V_ASN1_INTEGER)
+    else if (i != type)
         return -1;
 
     OPENSSL_STATIC_ASSERT(sizeof(uint64_t) >= sizeof(long),
@@ -372,13 +396,24 @@
     return r;
 }
 
-ASN1_INTEGER *BN_to_ASN1_INTEGER(const BIGNUM *bn, ASN1_INTEGER *ai)
+long ASN1_INTEGER_get(const ASN1_INTEGER *a)
+{
+    return asn1_string_get_long(a, V_ASN1_INTEGER);
+}
+
+long ASN1_ENUMERATED_get(const ASN1_ENUMERATED *a)
+{
+    return asn1_string_get_long(a, V_ASN1_ENUMERATED);
+}
+
+static ASN1_STRING *bn_to_asn1_string(const BIGNUM *bn, ASN1_STRING *ai,
+                                      int type)
 {
     ASN1_INTEGER *ret;
     int len, j;
 
     if (ai == NULL)
-        ret = ASN1_INTEGER_new();
+        ret = ASN1_STRING_type_new(type);
     else
         ret = ai;
     if (ret == NULL) {
@@ -386,9 +421,9 @@
         goto err;
     }
     if (BN_is_negative(bn) && !BN_is_zero(bn))
-        ret->type = V_ASN1_NEG_INTEGER;
+        ret->type = type | V_ASN1_NEG;
     else
-        ret->type = V_ASN1_INTEGER;
+        ret->type = type;
     j = BN_num_bits(bn);
     len = ((j == 0) ? 0 : ((j / 8) + 1));
     if (ret->length < len + 4) {
@@ -408,17 +443,41 @@
     return (ret);
  err:
     if (ret != ai)
-        ASN1_INTEGER_free(ret);
+        ASN1_STRING_free(ret);
     return (NULL);
 }
 
+ASN1_INTEGER *BN_to_ASN1_INTEGER(const BIGNUM *bn, ASN1_INTEGER *ai)
+{
+    return bn_to_asn1_string(bn, ai, V_ASN1_INTEGER);
+}
+
+ASN1_ENUMERATED *BN_to_ASN1_ENUMERATED(const BIGNUM *bn, ASN1_ENUMERATED *ai)
+{
+    return bn_to_asn1_string(bn, ai, V_ASN1_ENUMERATED);
+}
+
+static BIGNUM *asn1_string_to_bn(const ASN1_STRING *ai, BIGNUM *bn, int type)
+{
+    if ((ai->type & ~V_ASN1_NEG) != type) {
+        OPENSSL_PUT_ERROR(ASN1, ASN1_R_WRONG_INTEGER_TYPE);
+        return NULL;
+    }
+
+    BIGNUM *ret;
+    if ((ret = BN_bin2bn(ai->data, ai->length, bn)) == NULL)
+        OPENSSL_PUT_ERROR(ASN1, ASN1_R_BN_LIB);
+    else if (ai->type & V_ASN1_NEG)
+        BN_set_negative(ret, 1);
+    return (ret);
+}
+
 BIGNUM *ASN1_INTEGER_to_BN(const ASN1_INTEGER *ai, BIGNUM *bn)
 {
-    BIGNUM *ret;
+    return asn1_string_to_bn(ai, bn, V_ASN1_INTEGER);
+}
 
-    if ((ret = BN_bin2bn(ai->data, ai->length, bn)) == NULL)
-        OPENSSL_PUT_ERROR(ASN1, ASN1_R_BN_LIB);
-    else if (ai->type == V_ASN1_NEG_INTEGER)
-        BN_set_negative(ret, 1);
-    return (ret);
+BIGNUM *ASN1_ENUMERATED_to_BN(const ASN1_ENUMERATED *ai, BIGNUM *bn)
+{
+    return asn1_string_to_bn(ai, bn, V_ASN1_ENUMERATED);
 }
diff --git a/crypto/err/asn1.errordata b/crypto/err/asn1.errordata
index 9344621..9490db2 100644
--- a/crypto/err/asn1.errordata
+++ b/crypto/err/asn1.errordata
@@ -90,6 +90,7 @@
 ASN1,186,UNSUPPORTED_ANY_DEFINED_BY_TYPE
 ASN1,187,UNSUPPORTED_PUBLIC_KEY_TYPE
 ASN1,188,UNSUPPORTED_TYPE
+ASN1,195,WRONG_INTEGER_TYPE
 ASN1,189,WRONG_PUBLIC_KEY_TYPE
 ASN1,190,WRONG_TAG
 ASN1,191,WRONG_TYPE
diff --git a/include/openssl/asn1.h b/include/openssl/asn1.h
index d8a371c..5554a72 100644
--- a/include/openssl/asn1.h
+++ b/include/openssl/asn1.h
@@ -1127,6 +1127,10 @@
 // on success and zero on error.
 OPENSSL_EXPORT int ASN1_ENUMERATED_set(ASN1_ENUMERATED *a, long v);
 
+// ASN1_ENUMERATED_set_uint64 sets |a| to an ENUMERATED with value |v|. It
+// returns one on success and zero on error.
+OPENSSL_EXPORT int ASN1_ENUMERATED_set_uint64(ASN1_ENUMERATED *out, uint64_t v);
+
 // ASN1_ENUMERATED_get returns the value of |a| as a |long|, or -1 if |a| is out
 // of range or the wrong type.
 OPENSSL_EXPORT long ASN1_ENUMERATED_get(const ASN1_ENUMERATED *a);
@@ -2035,5 +2039,6 @@
 #define ASN1_R_NESTED_TOO_DEEP 192
 #define ASN1_R_BAD_TEMPLATE 193
 #define ASN1_R_INVALID_BIT_STRING_PADDING 194
+#define ASN1_R_WRONG_INTEGER_TYPE 195
 
 #endif