win-dshow, obs-ffmpeg: Add hardware decoding support

Fixes hardware decoding support and updates it to FFmpeg 4.0.

plugins/win-dshow/ffmpeg-decode.c

@@ -19,7 +19,57 @@
 #include "obs-ffmpeg-compat.h"
 #include <obs-avc.h>
 
-int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id)
+#if LIBAVCODEC_VERSION_INT > AV_VERSION_INT(58, 4, 100)
+#define USE_NEW_HARDWARE_CODEC_METHOD
+#endif
+
+#ifdef USE_NEW_HARDWARE_CODEC_METHOD
+enum AVHWDeviceType hw_priority[] = {
+	AV_HWDEVICE_TYPE_D3D11VA, AV_HWDEVICE_TYPE_DXVA2, AV_HWDEVICE_TYPE_QSV,
+	AV_HWDEVICE_TYPE_CUDA,    AV_HWDEVICE_TYPE_NONE,
+};
+
+static bool has_hw_type(AVCodec *c, enum AVHWDeviceType type)
+{
+	for (int i = 0;; i++) {
+		const AVCodecHWConfig *config = avcodec_get_hw_config(c, i);
+		if (!config) {
+			break;
+		}
+
+		if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX &&
+		    config->device_type == type)
+			return true;
+	}
+
+	return false;
+}
+
+static void init_hw_decoder(struct ffmpeg_decode *d)
+{
+	enum AVHWDeviceType *priority = hw_priority;
+	AVBufferRef *hw_ctx = NULL;
+
+	while (*priority != AV_HWDEVICE_TYPE_NONE) {
+		if (has_hw_type(d->codec, *priority)) {
+			int ret = av_hwdevice_ctx_create(&hw_ctx, *priority,
+							 NULL, NULL, 0);
+			if (ret == 0)
+				break;
+		}
+
+		priority++;
+	}
+
+	if (hw_ctx) {
+		d->decoder->hw_device_ctx = av_buffer_ref(hw_ctx);
+		d->hw = true;
+	}
+}
+#endif
+
+int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id,
+		       bool use_hw)
 {
 	int ret;
 
@@ -32,6 +82,15 @@ int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id)
 
 	decode->decoder = avcodec_alloc_context3(decode->codec);
 
+	decode->decoder->thread_count = 0;
+
+#ifdef USE_NEW_HARDWARE_CODEC_METHOD
+	if (use_hw)
+		init_hw_decoder(decode);
+#else
+	(void)use_hw;
+#endif
+
 	ret = avcodec_open2(decode->decoder, decode->codec, NULL);
 	if (ret < 0) {
 		ffmpeg_decode_free(decode);
@@ -46,6 +105,9 @@ int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id)
 
 void ffmpeg_decode_free(struct ffmpeg_decode *decode)
 {
+	if (decode->hw_frame)
+		av_free(decode->hw_frame);
+
 	if (decode->decoder) {
 		avcodec_close(decode->decoder);
 		av_free(decode->decoder);
@@ -214,6 +276,7 @@ bool ffmpeg_decode_video(struct ffmpeg_decode *decode, uint8_t *data,
 	AVPacket packet = {0};
 	int got_frame = false;
 	enum video_format new_format;
+	AVFrame *out_frame;
 	int ret;
 
 	*got_output = false;
@@ -233,11 +296,20 @@ bool ffmpeg_decode_video(struct ffmpeg_decode *decode, uint8_t *data,
 		decode->frame = av_frame_alloc();
 		if (!decode->frame)
 			return false;
+
+		if (decode->hw && !decode->hw_frame) {
+			decode->hw_frame = av_frame_alloc();
+			if (!decode->hw_frame)
+				return false;
+		}
 	}
 
+	out_frame = decode->hw ? decode->hw_frame : decode->frame;
+
 	ret = avcodec_send_packet(decode->decoder, &packet);
-	if (ret == 0)
-		ret = avcodec_receive_frame(decode->decoder, decode->frame);
+	if (ret == 0) {
+		ret = avcodec_receive_frame(decode->decoder, out_frame);
+	}
 
 	got_frame = (ret == 0);
 
@@ -249,6 +321,15 @@ bool ffmpeg_decode_video(struct ffmpeg_decode *decode, uint8_t *data,
 	else if (!got_frame)
 		return true;
 
+#ifdef USE_NEW_HARDWARE_CODEC_METHOD
+	if (got_frame && decode->hw) {
+		ret = av_hwframe_transfer_data(decode->frame, out_frame, 0);
+		if (ret < 0) {
+			return false;
+		}
+	}
+#endif
+
 	for (size_t i = 0; i < MAX_AV_PLANES; i++) {
 		frame->data[i] = decode->frame->data[i];
 		frame->linesize[i] = decode->frame->linesize[i];

plugins/win-dshow/ffmpeg-decode.h

@@ -40,13 +40,16 @@ struct ffmpeg_decode {
 	AVCodecContext *decoder;
 	AVCodec *codec;
 
+	AVFrame *hw_frame;
 	AVFrame *frame;
+	bool hw;
 
 	uint8_t *packet_buffer;
 	size_t packet_size;
 };
 
-extern int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id);
+extern int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id,
+			      bool use_hw);
 extern void ffmpeg_decode_free(struct ffmpeg_decode *decode);
 
 extern bool ffmpeg_decode_audio(struct ffmpeg_decode *decode, uint8_t *data,

plugins/win-dshow/win-dshow.cpp

@@ -462,11 +462,18 @@ static inline enum speaker_layout convert_speaker_layout(uint8_t channels)
 //#define LOG_ENCODED_VIDEO_TS 1
 //#define LOG_ENCODED_AUDIO_TS 1
 
+#define MAX_SW_RES_INT (1920 * 1080)
+
 void DShowInput::OnEncodedVideoData(enum AVCodecID id, unsigned char *data,
 				    size_t size, long long ts)
 {
 	if (!ffmpeg_decode_valid(video_decoder)) {
-		if (ffmpeg_decode_init(video_decoder, id) < 0) {
+		/* Only use MJPEG hardware decoding on resolutions higher
+		 * than 1920x1080.  The reason why is because we want to strike
+		 * a reasonable balance between hardware and CPU usage. */
+		bool useHW = videoConfig.format != VideoFormat::MJPEG ||
+			     (videoConfig.cx * videoConfig.cy) > MAX_SW_RES_INT;
+		if (ffmpeg_decode_init(video_decoder, id, useHW) < 0) {
 			blog(LOG_WARNING, "Could not initialize video decoder");
 			return;
 		}
@@ -571,7 +578,7 @@ void DShowInput::OnEncodedAudioData(enum AVCodecID id, unsigned char *data,
 				    size_t size, long long ts)
 {
 	if (!ffmpeg_decode_valid(audio_decoder)) {
-		if (ffmpeg_decode_init(audio_decoder, id) < 0) {
+		if (ffmpeg_decode_init(audio_decoder, id, false) < 0) {
 			blog(LOG_WARNING, "Could not initialize audio decoder");
 			return;
 		}