crypto: support openssl 1.1.0 api and add backward compatibility

4 files changed, 155 insertions, 70 deletions

diff --git a/libfreerdp-core/certificate.c b/libfreerdp-core/certificate.c
index 4187d88..cfdb8ba 100644
--- a/libfreerdp-core/certificate.c
+++ b/libfreerdp-core/certificate.c
@@ -120,6 +120,25 @@ static const char certificate_known_hosts_file[] = "known_hosts";
  *
  */
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+void RSA_get0_key(const RSA *rsa, const BIGNUM **out_n, const BIGNUM **out_e,
+                  const BIGNUM **out_d)
+{
+  if (out_n != NULL) 
+  {
+    *out_n = rsa->n;
+  }
+  if (out_e != NULL)
+  {
+    *out_e = rsa->e;
+  }
+  if (out_d != NULL)
+  {
+    *out_d = rsa->d;
+  }
+}
+#endif
+
 /**
  * Read X.509 Certificate
  * @param certificate certificate module
@@ -483,6 +502,7 @@ rdpKey* key_new(const char* keyfile)
 	rdpKey* key;
 	RSA *rsa;
 	FILE *fp;
+	const BIGNUM *n, *e, *d;
 
 	key = (rdpKey*) xzalloc(sizeof(rdpKey));
 
@@ -525,21 +545,23 @@ rdpKey* key_new(const char* keyfile)
 			return NULL;
 	}
 
-	if (BN_num_bytes(rsa->e) > 4)
+	RSA_get0_key(rsa, &n, &e, &d);
+
+	if (BN_num_bytes(e) > 4)
 	{
 		RSA_free(rsa);
 		printf("RSA public exponent too large in %s", keyfile);
 		return NULL;
 	}
 
-	freerdp_blob_alloc(&key->modulus, BN_num_bytes(rsa->n));
-	BN_bn2bin(rsa->n, key->modulus.data);
+	freerdp_blob_alloc(&key->modulus, BN_num_bytes(n));
+	BN_bn2bin(n, key->modulus.data);
 	crypto_reverse(key->modulus.data, key->modulus.length);
-	freerdp_blob_alloc(&key->private_exponent, BN_num_bytes(rsa->d));
-	BN_bn2bin(rsa->d, key->private_exponent.data);
+	freerdp_blob_alloc(&key->private_exponent, BN_num_bytes(d));
+	BN_bn2bin(d, key->private_exponent.data);
 	crypto_reverse(key->private_exponent.data, key->private_exponent.length);
 	memset(key->exponent, 0, sizeof(key->exponent));
-	BN_bn2bin(rsa->e, key->exponent + sizeof(key->exponent) - BN_num_bytes(rsa->e));
+	BN_bn2bin(e, key->exponent + sizeof(key->exponent) - BN_num_bytes(e));
 	crypto_reverse(key->exponent, sizeof(key->exponent));
 
 	RSA_free(rsa);
diff --git a/libfreerdp-core/crypto.c b/libfreerdp-core/crypto.c
index f6fa50a..6a44311 100644
--- a/libfreerdp-core/crypto.c
+++ b/libfreerdp-core/crypto.c
@@ -19,6 +19,31 @@
 
 #include "crypto.h"
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+static HMAC_CTX *HMAC_CTX_new(void)
+{
+	HMAC_CTX *ctx = xmalloc(sizeof(*ctx));
+	if (ctx == NULL) 
+	{
+		return NULL;
+	}
+
+	HMAC_CTX_init(ctx);
+	return ctx;
+}
+
+static void HMAC_CTX_free(HMAC_CTX *ctx)
+{
+	if (ctx == NULL) 
+	{
+		return;
+	}
+
+	HMAC_CTX_cleanup(ctx);
+	xfree(ctx);
+}
+#endif /* OPENSSL_VERSION_NUMBER < 0x10100000L */
+
 CryptoSha1 crypto_sha1_init(void)
 {
 	CryptoSha1 sha1 = xmalloc(sizeof(*sha1));
@@ -97,31 +122,31 @@ void crypto_rc4_free(CryptoRc4 rc4)
 CryptoDes3 crypto_des3_encrypt_init(const uint8* key, const uint8* ivec)
 {
 	CryptoDes3 des3 = xmalloc(sizeof(*des3));
-	EVP_CIPHER_CTX_init(&des3->des3_ctx);
-	EVP_EncryptInit_ex(&des3->des3_ctx, EVP_des_ede3_cbc(), NULL, key, ivec);
-	EVP_CIPHER_CTX_set_padding(&des3->des3_ctx, 0);
+	des3->des3_ctx = EVP_CIPHER_CTX_new();
+	EVP_EncryptInit_ex(des3->des3_ctx, EVP_des_ede3_cbc(), NULL, key, ivec);
+	EVP_CIPHER_CTX_set_padding(des3->des3_ctx, 0);
 	return des3;
 }
 
 CryptoDes3 crypto_des3_decrypt_init(const uint8* key, const uint8* ivec)
 {
 	CryptoDes3 des3 = xmalloc(sizeof(*des3));
-	EVP_CIPHER_CTX_init(&des3->des3_ctx);
-	EVP_DecryptInit_ex(&des3->des3_ctx, EVP_des_ede3_cbc(), NULL, key, ivec);
-	EVP_CIPHER_CTX_set_padding(&des3->des3_ctx, 0);
+	des3->des3_ctx = EVP_CIPHER_CTX_new();
+	EVP_DecryptInit_ex(des3->des3_ctx, EVP_des_ede3_cbc(), NULL, key, ivec);
+	EVP_CIPHER_CTX_set_padding(des3->des3_ctx, 0);
 	return des3;
 }
 
 void crypto_des3_encrypt(CryptoDes3 des3, uint32 length, const uint8* in_data, uint8* out_data)
 {
 	int len;
-	EVP_EncryptUpdate(&des3->des3_ctx, out_data, &len, in_data, length);
+	EVP_EncryptUpdate(des3->des3_ctx, out_data, &len, in_data, length);
 }
 
 void crypto_des3_decrypt(CryptoDes3 des3, uint32 length, const uint8* in_data, uint8* out_data)
 {
 	int len;
-	EVP_DecryptUpdate(&des3->des3_ctx, out_data, &len, in_data, length);
+	EVP_DecryptUpdate(des3->des3_ctx, out_data, &len, in_data, length);
 
 	if (length != len)
 		abort(); /* TODO */
@@ -129,35 +154,35 @@ void crypto_des3_decrypt(CryptoDes3 des3, uint32 length, const uint8* in_data, u
 
 void crypto_des3_free(CryptoDes3 des3)
 {
-	EVP_CIPHER_CTX_cleanup(&des3->des3_ctx);
+	EVP_CIPHER_CTX_free(des3->des3_ctx);
 	xfree(des3);
 }
 
 CryptoHmac crypto_hmac_new(void)
 {
 	CryptoHmac hmac = xmalloc(sizeof(*hmac));
-	HMAC_CTX_init(&hmac->hmac_ctx);
+	hmac->hmac_ctx = HMAC_CTX_new();
 	return hmac;
 }
 
 void crypto_hmac_sha1_init(CryptoHmac hmac, const uint8* data, uint32 length)
 {
-	HMAC_Init_ex(&hmac->hmac_ctx, data, length, EVP_sha1(), NULL);
+	HMAC_Init_ex(hmac->hmac_ctx, data, length, EVP_sha1(), NULL);
 }
 
 void crypto_hmac_update(CryptoHmac hmac, const uint8* data, uint32 length)
 {
-	HMAC_Update(&hmac->hmac_ctx, data, length);
+	HMAC_Update(hmac->hmac_ctx, data, length);
 }
 
 void crypto_hmac_final(CryptoHmac hmac, uint8* out_data, uint32 length)
 {
-	HMAC_Final(&hmac->hmac_ctx, out_data, &length);
+	HMAC_Final(hmac->hmac_ctx, out_data, &length);
 }
 
 void crypto_hmac_free(CryptoHmac hmac)
 {
-	HMAC_CTX_cleanup(&hmac->hmac_ctx);
+	HMAC_CTX_free(hmac->hmac_ctx);
 	xfree(hmac);
 }
 
@@ -245,14 +270,15 @@ const uint8 tssk_exponent[] =
 	0x5b, 0x7b, 0x88, 0xc0
 };
 
-static void crypto_rsa_common(const uint8* input, int length, uint32 key_length, const uint8* modulus, const uint8* exponent, int exponent_size, uint8* output)
+static void crypto_rsa_common(const uint8* input, int length, uint32 key_length,
+	const uint8* modulus, const uint8* exponent, int exponent_size, uint8* output)
 {
 	BN_CTX* ctx;
 	int output_length;
 	uint8* input_reverse;
 	uint8* modulus_reverse;
 	uint8* exponent_reverse;
-	BIGNUM mod, exp, x, y;
+	BIGNUM *mod, *exp, *x, *y;
 
 	input_reverse = (uint8*) xmalloc(2 * key_length + exponent_size);
 	modulus_reverse = input_reverse + key_length;
@@ -266,69 +292,79 @@ static void crypto_rsa_common(const uint8* input, int length, uint32 key_length,
 	crypto_reverse(input_reverse, length);
 
 	ctx = BN_CTX_new();
-	BN_init(&mod);
-	BN_init(&exp);
-	BN_init(&x);
-	BN_init(&y);
+	mod = BN_new();
+	exp = BN_new();
+	x = BN_new();
+	y = BN_new();
 
-	BN_bin2bn(modulus_reverse, key_length, &mod);
-	BN_bin2bn(exponent_reverse, exponent_size, &exp);
-	BN_bin2bn(input_reverse, length, &x);
-	BN_mod_exp(&y, &x, &exp, &mod, ctx);
+	BN_bin2bn(modulus_reverse, key_length, mod);
+	BN_bin2bn(exponent_reverse, exponent_size, exp);
+	BN_bin2bn(input_reverse, length, x);
+	BN_mod_exp(y, x, exp, mod, ctx);
 
-	output_length = BN_bn2bin(&y, output);
+	output_length = BN_bn2bin(y, output);
 	crypto_reverse(output, output_length);
 
 	if (output_length < (int) key_length)
 		memset(output + output_length, 0, key_length - output_length);
 
-	BN_free(&y);
-	BN_clear_free(&x);
-	BN_free(&exp);
-	BN_free(&mod);
+	BN_free(y);
+	BN_clear_free(x);
+	BN_free(exp);
+	BN_free(mod);
 	BN_CTX_free(ctx);
 	xfree(input_reverse);
 }
 
-static void crypto_rsa_public(const uint8* input, int length, uint32 key_length, const uint8* modulus, const uint8* exponent, uint8* output)
+static void crypto_rsa_public(const uint8* input, int length, uint32 key_length,
+	const uint8* modulus, const uint8* exponent, uint8* output)
 {
 	crypto_rsa_common(input, length, key_length, modulus, exponent, EXPONENT_MAX_SIZE, output);
 }
 
-static void crypto_rsa_private(const uint8* input, int length, uint32 key_length, const uint8* modulus, const uint8* private_exponent, uint8* output)
+static void crypto_rsa_private(const uint8* input, int length, uint32 key_length,
+	const uint8* modulus, const uint8* private_exponent, uint8* output)
 {
 
-	crypto_rsa_common(input, length, key_length, modulus, private_exponent, key_length, output);
+	crypto_rsa_common(input, length, key_length, modulus, private_exponent,
+		key_length, output);
 }
 
-void crypto_rsa_public_encrypt(const uint8* input, int length, uint32 key_length, const uint8* modulus, const uint8* exponent, uint8* output)
+void crypto_rsa_public_encrypt(const uint8* input, int length, uint32 key_length,
+	const uint8* modulus, const uint8* exponent, uint8* output)
 {
 
 	crypto_rsa_public(input, length, key_length, modulus, exponent, output);
 }
 
-void crypto_rsa_public_decrypt(const uint8* input, int length, uint32 key_length, const uint8* modulus, const uint8* exponent, uint8* output)
+void crypto_rsa_public_decrypt(const uint8* input, int length, uint32 key_length,
+	const uint8* modulus, const uint8* exponent, uint8* output)
 {
 
 	crypto_rsa_public(input, length, key_length, modulus, exponent, output);
 }
 
-void crypto_rsa_private_encrypt(const uint8* input, int length, uint32 key_length, const uint8* modulus, const uint8* private_exponent, uint8* output)
+void crypto_rsa_private_encrypt(const uint8* input, int length, uint32 key_length,
+	const uint8* modulus, const uint8* private_exponent, uint8* output)
 {
 
 	crypto_rsa_private(input, length, key_length, modulus, private_exponent, output);
 }
 
-void crypto_rsa_private_decrypt(const uint8* input, int length, uint32 key_length, const uint8* modulus, const uint8* private_exponent, uint8* output)
+void crypto_rsa_private_decrypt(const uint8* input, int length, uint32 key_length,
+	const uint8* modulus, const uint8* private_exponent, uint8* output)
 {
 
-	crypto_rsa_private(input, length, key_length, modulus, private_exponent, output);
+	crypto_rsa_private(input, length, key_length, modulus, private_exponent,
+		output);
 }
 
-void crypto_rsa_decrypt(const uint8* input, int length, uint32 key_length, const uint8* modulus, const uint8* private_exponent, uint8* output)
+void crypto_rsa_decrypt(const uint8* input, int length, uint32 key_length,
+	const uint8* modulus, const uint8* private_exponent, uint8* output)
 {
 
-	crypto_rsa_common(input, length, key_length, modulus, private_exponent, key_length, output);
+	crypto_rsa_common(input, length, key_length, modulus, private_exponent,
+		key_length, output);
 }
 
 void crypto_reverse(uint8* data, int length)
diff --git a/libfreerdp-core/crypto.h b/libfreerdp-core/crypto.h
index 4552c68..cf65e69 100644
--- a/libfreerdp-core/crypto.h
+++ b/libfreerdp-core/crypto.h
@@ -33,6 +33,7 @@
 #include <openssl/bn.h>
 #include <openssl/x509v3.h>
 #include <openssl/rand.h>
+#include <openssl/evp.h>
 
 #if defined(OPENSSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x0090800f)
 #define D2I_X509_CONST const
@@ -64,12 +65,12 @@ struct crypto_rc4_struct
 
 struct crypto_des3_struct
 {
-	EVP_CIPHER_CTX des3_ctx;
+	EVP_CIPHER_CTX *des3_ctx;
 };
 
 struct crypto_hmac_struct
 {
-	HMAC_CTX hmac_ctx;
+	HMAC_CTX *hmac_ctx;
 };
 
 struct crypto_cert_struct
diff --git a/libfreerdp-core/ntlmssp.c b/libfreerdp-core/ntlmssp.c
index 5a175f0..ba5b482 100644
--- a/libfreerdp-core/ntlmssp.c
+++ b/libfreerdp-core/ntlmssp.c
@@ -126,6 +126,31 @@ static const char* const AV_PAIRS_STRINGS[] =
 	"MsvChannelBindings"
 };
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+static HMAC_CTX *HMAC_CTX_new(void)
+{
+	HMAC_CTX *ctx = xmalloc(sizeof(*ctx));
+	if (ctx == NULL) 
+	{
+		return NULL;
+	}
+
+	HMAC_CTX_init(ctx);
+	return ctx;
+}
+
+static void HMAC_CTX_free(HMAC_CTX *ctx)
+{
+	if (ctx == NULL) 
+	{
+		return;
+	}
+
+	HMAC_CTX_cleanup(ctx);
+	xfree(ctx);
+}
+#endif /* OPENSSL_VERSION_NUMBER < 0x10100000L */
+
 /**
  * Set NTLMSSP username.
  * @param ntlmssp
@@ -456,7 +481,7 @@ void ntlmssp_compute_lm_hash(char* password, char* hash)
 	char text[14];
 	char des_key1[8];
 	char des_key2[8];
-	des_key_schedule ks;
+	DES_key_schedule ks;
 
 	/* LM("password") = E52CAC67419A9A224A3B108F3FA6CB6D */
 
@@ -530,7 +555,7 @@ void ntlmssp_compute_lm_response(char* password, char* challenge, char* response
 	char des_key1[8];
 	char des_key2[8];
 	char des_key3[8];
-	des_key_schedule ks;
+	DES_key_schedule ks;
 
 	/* A LM hash is 16-bytes long, but the LM response uses a LM hash null-padded to 21 bytes */
 	memset(hash, '\0', 21);
@@ -1154,19 +1179,20 @@ static void ntlmssp_output_version(STREAM* s)
 
 void ntlmssp_compute_message_integrity_check(NTLMSSP* ntlmssp)
 {
-	HMAC_CTX hmac_ctx;
+	HMAC_CTX *hmac_ctx;
 
 	/*
 	 * Compute the HMAC-MD5 hash of ConcatenationOf(NEGOTIATE_MESSAGE,
 	 * CHALLENGE_MESSAGE, AUTHENTICATE_MESSAGE) using the ExportedSessionKey
 	 */
 
-	HMAC_CTX_init(&hmac_ctx);
-	HMAC_Init_ex(&hmac_ctx, ntlmssp->exported_session_key, 16, EVP_md5(), NULL);
-	HMAC_Update(&hmac_ctx, ntlmssp->negotiate_message.data, ntlmssp->negotiate_message.length);
-	HMAC_Update(&hmac_ctx, ntlmssp->challenge_message.data, ntlmssp->challenge_message.length);
-	HMAC_Update(&hmac_ctx, ntlmssp->authenticate_message.data, ntlmssp->authenticate_message.length);
-	HMAC_Final(&hmac_ctx, ntlmssp->message_integrity_check, NULL);
+	hmac_ctx = HMAC_CTX_new();
+	HMAC_Init_ex(hmac_ctx, ntlmssp->exported_session_key, 16, EVP_md5(), NULL);
+	HMAC_Update(hmac_ctx, ntlmssp->negotiate_message.data, ntlmssp->negotiate_message.length);
+	HMAC_Update(hmac_ctx, ntlmssp->challenge_message.data, ntlmssp->challenge_message.length);
+	HMAC_Update(hmac_ctx, ntlmssp->authenticate_message.data, ntlmssp->authenticate_message.length);
+	HMAC_Final(hmac_ctx, ntlmssp->message_integrity_check, NULL);
+	HMAC_CTX_free(hmac_ctx);
 }
 
 /**
@@ -1181,17 +1207,17 @@ void ntlmssp_compute_message_integrity_check(NTLMSSP* ntlmssp)
 
 void ntlmssp_encrypt_message(NTLMSSP* ntlmssp, rdpBlob* msg, rdpBlob* encrypted_msg, uint8* signature)
 {
-	HMAC_CTX hmac_ctx;
+	HMAC_CTX *hmac_ctx;
 	uint8 digest[16];
 	uint8 checksum[8];
 	uint32 version = 1;
 
 	/* Compute the HMAC-MD5 hash of ConcatenationOf(seq_num,msg) using the client signing key */
-	HMAC_CTX_init(&hmac_ctx);
-	HMAC_Init_ex(&hmac_ctx, ntlmssp->client_signing_key, 16, EVP_md5(), NULL);
-	HMAC_Update(&hmac_ctx, (void*) &ntlmssp->send_seq_num, 4);
-	HMAC_Update(&hmac_ctx, msg->data, msg->length);
-	HMAC_Final(&hmac_ctx, digest, NULL);
+	hmac_ctx = HMAC_CTX_new();
+	HMAC_Init_ex(hmac_ctx, ntlmssp->client_signing_key, 16, EVP_md5(), NULL);
+	HMAC_Update(hmac_ctx, (void*) &ntlmssp->send_seq_num, 4);
+	HMAC_Update(hmac_ctx, msg->data, msg->length);
+	HMAC_Final(hmac_ctx, digest, NULL);
 
 	if (encrypted_msg != NULL)
 	{
@@ -1210,7 +1236,7 @@ void ntlmssp_encrypt_message(NTLMSSP* ntlmssp, rdpBlob* msg, rdpBlob* encrypted_
 	memcpy(&signature[4], (void*) checksum, 8);
 	memcpy(&signature[12], (void*) &(ntlmssp->send_seq_num), 4);
 
-	HMAC_CTX_cleanup(&hmac_ctx);
+	HMAC_CTX_free(hmac_ctx);
 
 	ntlmssp->send_seq_num++;
 }
@@ -1228,7 +1254,7 @@ void ntlmssp_encrypt_message(NTLMSSP* ntlmssp, rdpBlob* msg, rdpBlob* encrypted_
 
 int ntlmssp_decrypt_message(NTLMSSP* ntlmssp, rdpBlob* encrypted_msg, rdpBlob* msg, uint8* signature)
 {
-	HMAC_CTX hmac_ctx;
+	HMAC_CTX *hmac_ctx;
 	uint8 digest[16];
 	uint8 checksum[8];
 	uint32 version = 1;
@@ -1241,11 +1267,11 @@ int ntlmssp_decrypt_message(NTLMSSP* ntlmssp, rdpBlob* encrypted_msg, rdpBlob* m
 	crypto_rc4(ntlmssp->recv_rc4_seal, encrypted_msg->length, encrypted_msg->data, msg->data);
 
 	/* Compute the HMAC-MD5 hash of ConcatenationOf(seq_num,msg) using the client signing key */
-	HMAC_CTX_init(&hmac_ctx);
-	HMAC_Init_ex(&hmac_ctx, ntlmssp->server_signing_key, 16, EVP_md5(), NULL);
-	HMAC_Update(&hmac_ctx, (void*) &ntlmssp->recv_seq_num, 4);
-	HMAC_Update(&hmac_ctx, msg->data, msg->length);
-	HMAC_Final(&hmac_ctx, digest, NULL);
+	hmac_ctx = HMAC_CTX_new();
+	HMAC_Init_ex(hmac_ctx, ntlmssp->server_signing_key, 16, EVP_md5(), NULL);
+	HMAC_Update(hmac_ctx, (void*) &ntlmssp->recv_seq_num, 4);
+	HMAC_Update(hmac_ctx, msg->data, msg->length);
+	HMAC_Final(hmac_ctx, digest, NULL);
 
 	/* RC4-encrypt first 8 bytes of digest */
 	crypto_rc4(ntlmssp->recv_rc4_seal, 8, digest, checksum);
@@ -1262,7 +1288,7 @@ int ntlmssp_decrypt_message(NTLMSSP* ntlmssp, rdpBlob* encrypted_msg, rdpBlob* m
 		return 0;
 	}
 
-	HMAC_CTX_cleanup(&hmac_ctx);
+	HMAC_CTX_free(hmac_ctx);
 
 	ntlmssp->recv_seq_num++;
 	return 1;