From d500ef3bb52d3989476e9ab41170ac138dcfd5f2 Mon Sep 17 00:00:00 2001 From: Shaolin Date: Tue, 7 Aug 2018 22:24:54 -0300 Subject: [PATCH 1/3] obs-outputs: Make rtmp packet alloc code path clearer On 64bit systems, this check will always evaluate to false due to SIZE_MAX type and triggers a compiler warning. This both makes it clearer that its only needed on 32bit system and clear the compiler warning. --- plugins/obs-outputs/librtmp/rtmp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/obs-outputs/librtmp/rtmp.c b/plugins/obs-outputs/librtmp/rtmp.c index cb2aa765f..71e7c1900 100644 --- a/plugins/obs-outputs/librtmp/rtmp.c +++ b/plugins/obs-outputs/librtmp/rtmp.c @@ -242,8 +242,10 @@ int RTMPPacket_Alloc(RTMPPacket *p, uint32_t nSize) { char *ptr; +#if ARCH_BITS == 32 if (nSize > SIZE_MAX - RTMP_MAX_HEADER_SIZE) return FALSE; +#endif ptr = calloc(1, nSize + RTMP_MAX_HEADER_SIZE); if (!ptr) From bf72435db8cc15a15155153435233bff3aca670b Mon Sep 17 00:00:00 2001 From: Shaolin Date: Tue, 7 Aug 2018 23:32:16 -0300 Subject: [PATCH 2/3] obs-outputs: Fix unused variable compiler warning Some security layer libraries code path used by the rtmp output had a not used variable and not used param on HMAC_finish macro that was triggering warnings during compilation. --- plugins/obs-outputs/librtmp/handshake.h | 28 +++++++++++++++++++------ plugins/obs-outputs/librtmp/hashswf.c | 16 ++++++++------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/plugins/obs-outputs/librtmp/handshake.h b/plugins/obs-outputs/librtmp/handshake.h index 8cd3e3224..f14a0a8e8 100644 --- a/plugins/obs-outputs/librtmp/handshake.h +++ b/plugins/obs-outputs/librtmp/handshake.h @@ -34,9 +34,9 @@ typedef mbedtls_md_context_t *HMAC_CTX; #define HMAC_setup(ctx, key, len) ctx = malloc(sizeof(mbedtls_md_context_t)); mbedtls_md_init(ctx); \ mbedtls_md_setup(ctx, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1); \ mbedtls_md_hmac_starts(ctx, (const unsigned char *)key, len) -#define HMAC_crunch(ctx, buf, len) mbedtls_md_hmac_update(ctx, buf, len) -#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; mbedtls_md_hmac_finish(ctx, dig) -#define HMAC_close(ctx) mbedtls_md_free(ctx); free(ctx); ctx = NULL +#define HMAC_crunch(ctx, buf, len) mbedtls_md_hmac_update(ctx, buf, len) +#define HMAC_finish(ctx, dig) mbedtls_md_hmac_finish(ctx, dig) +#define HMAC_close(ctx) mbedtls_md_free(ctx); free(ctx); ctx = NULL typedef mbedtls_arc4_context* RC4_handle; #define RC4_alloc(h) *h = malloc(sizeof(mbedtls_arc4_context)); mbedtls_arc4_init(*h) @@ -54,7 +54,7 @@ typedef mbedtls_arc4_context* RC4_handle; #define HMAC_CTX sha2_context #define HMAC_setup(ctx, key, len) sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0) #define HMAC_crunch(ctx, buf, len) sha2_hmac_update(&ctx, buf, len) -#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig) +#define HMAC_finish(ctx, dig) sha2_hmac_finish(&ctx, dig) typedef arc4_context * RC4_handle; #define RC4_alloc(h) *h = malloc(sizeof(arc4_context)) @@ -73,7 +73,7 @@ typedef arc4_context * RC4_handle; #define HMAC_CTX struct hmac_sha256_ctx #define HMAC_setup(ctx, key, len) hmac_sha256_set_key(&ctx, len, key) #define HMAC_crunch(ctx, buf, len) hmac_sha256_update(&ctx, len, buf) -#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig) +#define HMAC_finish(ctx, dig) hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig) #define HMAC_close(ctx) typedef struct arcfour_ctx* RC4_handle; @@ -92,7 +92,7 @@ typedef struct arcfour_ctx* RC4_handle; #endif #define HMAC_setup(ctx, key, len) HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, key, len, EVP_sha256(), 0) #define HMAC_crunch(ctx, buf, len) HMAC_Update(&ctx, buf, len) -#define HMAC_finish(ctx, dig, dlen) HMAC_Final(&ctx, dig, &dlen); HMAC_CTX_cleanup(&ctx) +#define HMAC_finish(ctx, dig, len) HMAC_Final(&ctx, dig, &len); HMAC_CTX_cleanup(&ctx) typedef RC4_KEY * RC4_handle; #define RC4_alloc(h) *h = malloc(sizeof(RC4_KEY)) @@ -139,7 +139,9 @@ static void InitRC4Encryption uint8_t * pubKeyOut, RC4_handle *rc4keyIn, RC4_handle *rc4keyOut) { uint8_t digest[SHA256_DIGEST_LENGTH]; +#if !(defined(USE_MBEDTLS) || defined(USE_POLARSSL) || defined(USE_GNUTLS)) unsigned int digestLen = 0; +#endif HMAC_CTX ctx; RC4_alloc(rc4keyIn); @@ -147,7 +149,11 @@ static void InitRC4Encryption HMAC_setup(ctx, secretKey, 128); HMAC_crunch(ctx, pubKeyIn, 128); +#if defined(USE_MBEDTLS) || defined(USE_POLARSSL) || defined(USE_GNUTLS) + HMAC_finish(ctx, digest); +#else HMAC_finish(ctx, digest, digestLen); +#endif RTMP_Log(RTMP_LOGDEBUG, "RC4 Out Key: "); RTMP_LogHex(RTMP_LOGDEBUG, digest, 16); @@ -156,7 +162,11 @@ static void InitRC4Encryption HMAC_setup(ctx, secretKey, 128); HMAC_crunch(ctx, pubKeyOut, 128); +#if defined(USE_MBEDTLS) || defined(USE_POLARSSL) || defined(USE_GNUTLS) + HMAC_finish(ctx, digest); +#else HMAC_finish(ctx, digest, digestLen); +#endif RTMP_Log(RTMP_LOGDEBUG, "RC4 In Key: "); RTMP_LogHex(RTMP_LOGDEBUG, digest, 16); @@ -303,7 +313,13 @@ HMACsha256(const uint8_t *message, size_t messageLen, const uint8_t *key, HMAC_setup(ctx, key, keylen); HMAC_crunch(ctx, message, messageLen); + +#if defined(USE_MBEDTLS) || defined(USE_POLARSSL) || defined(USE_GNUTLS) + digestLen = SHA256_DIGEST_LENGTH; + HMAC_finish(ctx, digest); +#else HMAC_finish(ctx, digest, digestLen); +#endif assert(digestLen == 32); } diff --git a/plugins/obs-outputs/librtmp/hashswf.c b/plugins/obs-outputs/librtmp/hashswf.c index b6c2b61fb..bbc8d31aa 100644 --- a/plugins/obs-outputs/librtmp/hashswf.c +++ b/plugins/obs-outputs/librtmp/hashswf.c @@ -39,9 +39,9 @@ typedef mbedtls_md_context_t *HMAC_CTX; #define HMAC_setup(ctx, key, len) ctx = malloc(sizeof(mbedtls_md_context_t)); mbedtls_md_init(ctx); \ mbedtls_md_setup(ctx, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1); \ mbedtls_md_hmac_starts(ctx, (const unsigned char *)key, len) -#define HMAC_crunch(ctx, buf, len) mbedtls_md_hmac_update(ctx, buf, len) -#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; mbedtls_md_hmac_finish(ctx, dig) -#define HMAC_close(ctx) free(ctx); mbedtls_md_free(ctx); ctx = NULL +#define HMAC_crunch(ctx, buf, len) mbedtls_md_hmac_update(ctx, buf, len) +#define HMAC_finish(ctx, dig) mbedtls_md_hmac_finish(ctx, dig) +#define HMAC_close(ctx) free(ctx); mbedtls_md_free(ctx); ctx = NULL #elif defined(USE_POLARSSL) #include @@ -51,7 +51,7 @@ typedef mbedtls_md_context_t *HMAC_CTX; #define HMAC_CTX sha2_context #define HMAC_setup(ctx, key, len) sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0) #define HMAC_crunch(ctx, buf, len) sha2_hmac_update(&ctx, buf, len) -#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig) +#define HMAC_finish(ctx, dig) sha2_hmac_finish(&ctx, dig) #define HMAC_close(ctx) #elif defined(USE_GNUTLS) @@ -63,7 +63,7 @@ typedef mbedtls_md_context_t *HMAC_CTX; #define HMAC_CTX struct hmac_sha256_ctx #define HMAC_setup(ctx, key, len) hmac_sha256_set_key(&ctx, len, key) #define HMAC_crunch(ctx, buf, len) hmac_sha256_update(&ctx, len, buf) -#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig) +#define HMAC_finish(ctx, dig) hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig) #define HMAC_close(ctx) #else /* USE_OPENSSL */ @@ -73,7 +73,7 @@ typedef mbedtls_md_context_t *HMAC_CTX; #include #define HMAC_setup(ctx, key, len) HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, (unsigned char *)key, len, EVP_sha256(), 0) #define HMAC_crunch(ctx, buf, len) HMAC_Update(&ctx, (unsigned char *)buf, len) -#define HMAC_finish(ctx, dig, dlen) HMAC_Final(&ctx, (unsigned char *)dig, &dlen); +#define HMAC_finish(ctx, dig, len) HMAC_Final(&ctx, (unsigned char *)dig, &len); #define HMAC_close(ctx) HMAC_CTX_cleanup(&ctx) #endif @@ -658,7 +658,11 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash, if (!in.first) { +#if defined(USE_MBEDTLS) || defined(USE_POLARSSL) || defined(USE_GNUTLS) + HMAC_finish(in.ctx, hash); +#else HMAC_finish(in.ctx, hash, hlen); +#endif *size = in.size; fprintf(f, "date: %s\n", date); From b5ee8a9a5da92c061261c0072939cf18bfe7b494 Mon Sep 17 00:00:00 2001 From: Shaolin Date: Wed, 8 Aug 2018 01:47:54 -0300 Subject: [PATCH 3/3] UI: Prevent format-truncation compiler warning Variables path and dir with the same declared size triggers a compiler warning about possible truncation. Although, depending on the path where profiles are stored, its still possible to go beyong the declared size and the path will be invalid. This will not trigger any compilation warning and all invalid paths will be cought by the already check in the os_glob function. --- UI/window-basic-main-profiles.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UI/window-basic-main-profiles.cpp b/UI/window-basic-main-profiles.cpp index f5db4cd01..82f66d75e 100644 --- a/UI/window-basic-main-profiles.cpp +++ b/UI/window-basic-main-profiles.cpp @@ -143,7 +143,7 @@ static bool GetProfileName(QWidget *parent, std::string &name, static bool CopyProfile(const char *fromPartial, const char *to) { os_glob_t *glob; - char path[512]; + char path[514]; char dir[512]; int ret;