librtmp: Add mbedtls 3 compatibility
Since Mbed TLS 3 doesn't support RC4 algorithm, encrypted RTMP is disabled if OBS is built with the version 3 or later of Mbed TLS.
This commit is contained in:
tytan652
Jim
parent
9170e0a9b1
commit
179ad9e67b
@ -26,7 +26,9 @@
|
||||
|
||||
#if defined(USE_MBEDTLS)
|
||||
#include <mbedtls/md.h>
|
||||
#if MBEDTLS_VERSION_MAJOR < 3
|
||||
#include <mbedtls/arc4.h>
|
||||
#endif
|
||||
#ifndef SHA256_DIGEST_LENGTH
|
||||
#define SHA256_DIGEST_LENGTH 32
|
||||
#endif
|
||||
@ -38,12 +40,14 @@ typedef mbedtls_md_context_t *HMAC_CTX;
|
||||
#define HMAC_finish(ctx, dig) mbedtls_md_hmac_finish(ctx, dig)
|
||||
#define HMAC_close(ctx) mbedtls_md_free(ctx); free(ctx); ctx = NULL
|
||||
|
||||
#if MBEDTLS_VERSION_MAJOR < 3
|
||||
typedef mbedtls_arc4_context* RC4_handle;
|
||||
#define RC4_alloc(h) *h = malloc(sizeof(mbedtls_arc4_context)); mbedtls_arc4_init(*h)
|
||||
#define RC4_setkey(h,l,k) mbedtls_arc4_setup(h,k,l)
|
||||
#define RC4_encrypt(h,l,d) mbedtls_arc4_crypt(h,l,(unsigned char *)d,(unsigned char *)d)
|
||||
#define RC4_encrypt2(h,l,s,d) mbedtls_arc4_crypt(h,l,(unsigned char *)s,(unsigned char *)d)
|
||||
#define RC4_free(h) mbedtls_arc4_free(h); free(h); h = NULL
|
||||
#endif
|
||||
|
||||
#elif defined(USE_POLARSSL)
|
||||
#include <polarssl/sha2.h>
|
||||
@ -104,7 +108,9 @@ typedef RC4_KEY * RC4_handle;
|
||||
|
||||
#define FP10
|
||||
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
#include "dh.h"
|
||||
#endif
|
||||
|
||||
static const uint8_t GenuineFMSKey[] =
|
||||
{
|
||||
@ -133,6 +139,7 @@ static const uint8_t GenuineFPKey[] =
|
||||
0x31, 0xAE
|
||||
}; /* 62 */
|
||||
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
static void InitRC4Encryption
|
||||
(uint8_t * secretKey,
|
||||
uint8_t * pubKeyIn,
|
||||
@ -173,9 +180,11 @@ static void InitRC4Encryption
|
||||
|
||||
RC4_setkey(*rc4keyIn, 16, digest);
|
||||
}
|
||||
#endif
|
||||
|
||||
typedef unsigned int (getoff)(uint8_t *buf, unsigned int len);
|
||||
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
static unsigned int
|
||||
GetDHOffset2(uint8_t *handshake, unsigned int len)
|
||||
{
|
||||
@ -206,6 +215,7 @@ GetDHOffset2(uint8_t *handshake, unsigned int len)
|
||||
}
|
||||
return res;
|
||||
}
|
||||
#endif
|
||||
|
||||
static unsigned int
|
||||
GetDigestOffset2(uint8_t *handshake, unsigned int len)
|
||||
@ -238,6 +248,7 @@ GetDigestOffset2(uint8_t *handshake, unsigned int len)
|
||||
return res;
|
||||
}
|
||||
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
static unsigned int
|
||||
GetDHOffset1(uint8_t *handshake, unsigned int len)
|
||||
{
|
||||
@ -268,6 +279,7 @@ GetDHOffset1(uint8_t *handshake, unsigned int len)
|
||||
|
||||
return res;
|
||||
}
|
||||
#endif
|
||||
|
||||
static unsigned int
|
||||
GetDigestOffset1(uint8_t *handshake, unsigned int len)
|
||||
@ -302,7 +314,9 @@ GetDigestOffset1(uint8_t *handshake, unsigned int len)
|
||||
}
|
||||
|
||||
static getoff *digoff[] = {GetDigestOffset1, GetDigestOffset2};
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
static getoff *dhoff[] = {GetDHOffset1, GetDHOffset2};
|
||||
#endif
|
||||
|
||||
static void
|
||||
HMACsha256(const uint8_t *message, size_t messageLen, const uint8_t *key,
|
||||
@ -805,12 +819,16 @@ static int
|
||||
HandShake(RTMP * r, int FP9HandShake)
|
||||
{
|
||||
int i, offalg = 0;
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
int dhposClient = 0;
|
||||
#endif
|
||||
int digestPosClient = 0;
|
||||
int encrypted = r->Link.protocol & RTMP_FEATURE_ENC;
|
||||
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
RC4_handle keyIn = 0;
|
||||
RC4_handle keyOut = 0;
|
||||
#endif
|
||||
|
||||
#ifndef _DEBUG
|
||||
int32_t *ip;
|
||||
@ -820,15 +838,31 @@ HandShake(RTMP * r, int FP9HandShake)
|
||||
uint8_t clientbuf[RTMP_SIG_SIZE + 4], *clientsig=clientbuf+4;
|
||||
uint8_t serversig[RTMP_SIG_SIZE], client2[RTMP_SIG_SIZE], *reply;
|
||||
uint8_t type;
|
||||
getoff *getdh = NULL, *getdig = NULL;
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
getoff *getdh = NULL;
|
||||
#endif
|
||||
getoff *getdig = NULL;
|
||||
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
if (encrypted || r->Link.SWFSize)
|
||||
FP9HandShake = TRUE;
|
||||
else
|
||||
FP9HandShake = FALSE;
|
||||
|
||||
r->Link.rc4keyIn = r->Link.rc4keyOut = 0;
|
||||
#else
|
||||
if (encrypted)
|
||||
{
|
||||
RTMP_Log(RTMP_LOGWARNING, "%s: encrypted RTMP is no longer supported with mbedtls 3 and later", __FUNCTION__);
|
||||
return FALSE;
|
||||
}
|
||||
else if (r->Link.SWFSize)
|
||||
FP9HandShake = TRUE;
|
||||
else
|
||||
FP9HandShake = FALSE;
|
||||
#endif
|
||||
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
if (encrypted)
|
||||
{
|
||||
clientsig[-1] = 0x06; /* 0x08 is RTMPE as well */
|
||||
@ -836,12 +870,16 @@ HandShake(RTMP * r, int FP9HandShake)
|
||||
}
|
||||
else
|
||||
clientsig[-1] = 0x03;
|
||||
#else
|
||||
clientsig[-1] = 0x03;
|
||||
#endif
|
||||
|
||||
uptime = htonl(RTMP_GetTime());
|
||||
memcpy(clientsig, &uptime, 4);
|
||||
|
||||
if (FP9HandShake)
|
||||
{
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
/* set version to at least 9.0.115.0 */
|
||||
if (encrypted)
|
||||
{
|
||||
@ -853,12 +891,18 @@ HandShake(RTMP * r, int FP9HandShake)
|
||||
clientsig[4] = 10;
|
||||
clientsig[6] = 45;
|
||||
}
|
||||
#else
|
||||
clientsig[4] = 10;
|
||||
clientsig[6] = 45;
|
||||
#endif
|
||||
clientsig[5] = 0;
|
||||
clientsig[7] = 2;
|
||||
|
||||
RTMP_Log(RTMP_LOGDEBUG, "%s: Client type: %02X", __FUNCTION__, clientsig[-1]);
|
||||
getdig = digoff[offalg];
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
getdh = dhoff[offalg];
|
||||
#endif
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -877,6 +921,7 @@ HandShake(RTMP * r, int FP9HandShake)
|
||||
/* set handshake digest */
|
||||
if (FP9HandShake)
|
||||
{
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
if (encrypted)
|
||||
{
|
||||
/* generate Diffie-Hellmann parameters */
|
||||
@ -904,6 +949,7 @@ HandShake(RTMP * r, int FP9HandShake)
|
||||
return FALSE;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
digestPosClient = getdig(clientsig, RTMP_SIG_SIZE); /* reuse this value in verification */
|
||||
RTMP_Log(RTMP_LOGDEBUG, "%s: Client digest offset: %d", __FUNCTION__,
|
||||
@ -966,7 +1012,9 @@ HandShake(RTMP * r, int FP9HandShake)
|
||||
RTMP_Log(RTMP_LOGWARNING, "Trying different position for server digest!");
|
||||
offalg ^= 1;
|
||||
getdig = digoff[offalg];
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
getdh = dhoff[offalg];
|
||||
#endif
|
||||
digestPosServer = getdig(serversig, RTMP_SIG_SIZE);
|
||||
|
||||
if (!VerifyDigest(digestPosServer, serversig, GenuineFMSKey, 36))
|
||||
@ -991,6 +1039,7 @@ HandShake(RTMP * r, int FP9HandShake)
|
||||
(uint8_t *)&r->Link.SWFVerificationResponse[10]);
|
||||
}
|
||||
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
/* do Diffie-Hellmann Key exchange for encrypted RTMP */
|
||||
if (encrypted)
|
||||
{
|
||||
@ -1017,6 +1066,7 @@ HandShake(RTMP * r, int FP9HandShake)
|
||||
(uint8_t *) & clientsig[dhposClient],
|
||||
&keyIn, &keyOut);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
reply = client2;
|
||||
@ -1146,6 +1196,7 @@ HandShake(RTMP * r, int FP9HandShake)
|
||||
RTMP_Log(RTMP_LOGDEBUG, "%s: Genuine Adobe Flash Media Server", __FUNCTION__);
|
||||
}
|
||||
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
if (encrypted)
|
||||
{
|
||||
char buff[RTMP_SIG_SIZE];
|
||||
@ -1165,6 +1216,7 @@ HandShake(RTMP * r, int FP9HandShake)
|
||||
RC4_encrypt(r->Link.rc4keyOut, RTMP_SIG_SIZE, (uint8_t *) buff);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -954,7 +954,11 @@ RTMP_Connect1(RTMP *r, RTMPPacket *cp)
|
||||
|
||||
#if defined(USE_MBEDTLS)
|
||||
mbedtls_net_context *server_fd = &r->RTMP_TLS_ctx->net;
|
||||
#if MBEDTLS_VERSION_NUMBER == 0x03000000
|
||||
server_fd->MBEDTLS_PRIVATE(fd) = r->m_sb.sb_socket;
|
||||
#else
|
||||
server_fd->fd = r->m_sb.sb_socket;
|
||||
#endif
|
||||
TLS_setfd(r->m_sb.sb_ssl, server_fd);
|
||||
|
||||
// make sure we verify the certificate hostname
|
||||
@ -1540,7 +1544,7 @@ ReadN(RTMP *r, char *buffer, int n)
|
||||
if (r->Link.protocol & RTMP_FEATURE_HTTP)
|
||||
r->m_resplen -= nBytes;
|
||||
|
||||
#ifdef CRYPTO
|
||||
#if defined(CRYPTO) && (!defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3)
|
||||
if (r->Link.rc4keyIn)
|
||||
{
|
||||
RC4_encrypt(r->Link.rc4keyIn, nBytes, ptr);
|
||||
@ -1562,6 +1566,7 @@ WriteN(RTMP *r, const char *buffer, int n)
|
||||
char *encrypted = 0;
|
||||
char buf[RTMP_BUFFER_CACHE_SIZE];
|
||||
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
if (r->Link.rc4keyOut)
|
||||
{
|
||||
if (n > (int)sizeof(buf))
|
||||
@ -1571,6 +1576,7 @@ WriteN(RTMP *r, const char *buffer, int n)
|
||||
ptr = encrypted;
|
||||
RC4_encrypt2(r->Link.rc4keyOut, n, buffer, ptr);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
while (n > 0)
|
||||
@ -2607,7 +2613,7 @@ b64enc(const unsigned char *input, int length, char *output, int maxsize)
|
||||
#if defined(USE_MBEDTLS)
|
||||
typedef mbedtls_md5_context MD5_CTX;
|
||||
|
||||
#if MBEDTLS_VERSION_NUMBER >= 0x02070000
|
||||
#if MBEDTLS_VERSION_NUMBER >= 0x02070000 && MBEDTLS_VERSION_MAJOR < 3
|
||||
#define MD5_Init(ctx) mbedtls_md5_init(ctx); mbedtls_md5_starts_ret(ctx)
|
||||
#define MD5_Update(ctx,data,len) mbedtls_md5_update_ret(ctx,(unsigned char *)data,len)
|
||||
#define MD5_Final(dig,ctx) mbedtls_md5_finish_ret(ctx,dig); mbedtls_md5_free(ctx)
|
||||
@ -4409,7 +4415,7 @@ RTMP_Close(RTMP *r)
|
||||
free(r->Link.tcUrl.av_val);
|
||||
r->Link.tcUrl.av_val = NULL;
|
||||
}
|
||||
#elif defined(CRYPTO)
|
||||
#elif defined(CRYPTO) && (!defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3)
|
||||
if (r->Link.dh)
|
||||
{
|
||||
MDH_free(r->Link.dh);
|
||||
|
||||
@ -342,9 +342,11 @@ extern "C"
|
||||
|
||||
#ifdef CRYPTO
|
||||
#define RTMP_SWF_HASHLEN 32
|
||||
#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
|
||||
void *dh; /* for encryption */
|
||||
void *rc4keyIn;
|
||||
void *rc4keyOut;
|
||||
#endif
|
||||
|
||||
uint32_t SWFSize;
|
||||
uint8_t SWFHash[RTMP_SWF_HASHLEN];
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user