From 6b441a180c9054c7936dc154550f61ed93674359 Mon Sep 17 00:00:00 2001
From: Richard Stanway <r.stanway@gmail.com>
Date: Sun, 26 Apr 2020 23:35:28 +0200
Subject: [PATCH] obs-ffmpeg: Preserve error message from new NVENC

If we fallback to ffmpeg NVENC, the error from new NVENC might still be
present in the encoder structure. Given that this provides a lot more
actionable information to the user, let's use it if possible.
---
 plugins/obs-ffmpeg/obs-ffmpeg-nvenc.c | 46 ++++++++++++++++-----------
 1 file changed, 27 insertions(+), 19 deletions(-)

diff --git a/plugins/obs-ffmpeg/obs-ffmpeg-nvenc.c b/plugins/obs-ffmpeg/obs-ffmpeg-nvenc.c
index 171935e7a..31c734b18 100644
--- a/plugins/obs-ffmpeg/obs-ffmpeg-nvenc.c
+++ b/plugins/obs-ffmpeg/obs-ffmpeg-nvenc.c
@@ -90,27 +90,35 @@ static bool nvenc_init_codec(struct nvenc_encoder *enc)
 
 	ret = avcodec_open2(enc->context, enc->nvenc, NULL);
 	if (ret < 0) {
-		struct dstr error_message = {0};
+		// if we were a fallback from jim-nvenc, there may already be a
+		// more useful error returned from that, so don't overwrite.
+		// this can be removed if / when ffmpeg fallback is removed.
+		if (!obs_encoder_get_last_error(enc->encoder)) {
+			struct dstr error_message = {0};
 
-		// special case for common NVENC error
-		if (ret == AVERROR_EXTERNAL) {
-			dstr_printf(&error_message,
-				    "Failed to open NVENC codec: %s\r\n\r\n"
-				    "Check your video drivers are up to "
-				    "date. Disable other software that may "
-				    "be using NVENC such as NVIDIA "
-				    "ShadowPlay or Windows 10 Game "
-				    "DVR.",
-				    av_err2str(ret));
-		} else {
-			dstr_printf(
-				&error_message,
-				"Failed to open NVENC codec: %s\r\n\r\n"
-				"Please check your video drivers are up to date.",
-				av_err2str(ret));
+			// special case for common NVENC error
+			if (ret == AVERROR_EXTERNAL) {
+				dstr_printf(
+					&error_message,
+					"Failed to open NVENC codec: %s\r\n\r\n"
+					"Check your video drivers are up to "
+					"date. Disable other software that may "
+					"be using NVENC such as NVIDIA "
+					"ShadowPlay or Windows 10 Game "
+					"DVR.",
+					av_err2str(ret));
+			} else {
+				dstr_printf(
+					&error_message,
+					"Failed to open NVENC codec: %s\r\n\r\n"
+					"Please check your video drivers are up to date.",
+					av_err2str(ret));
+			}
+
+			obs_encoder_set_last_error(enc->encoder,
+						   error_message.array);
+			dstr_free(&error_message);
 		}
-		obs_encoder_set_last_error(enc->encoder, error_message.array);
-		dstr_free(&error_message);
 		warn("Failed to open NVENC codec: %s", av_err2str(ret));
 		return false;
 	}