From 00408b6b3bd18eab761079ae37ca33e54756dedf Mon Sep 17 00:00:00 2001
From: Richard Stanway <r1ch@r1ch.net>
Date: Fri, 22 Jul 2022 01:03:30 +0200
Subject: [PATCH] obs-ffmpeg: Don't load AMF DLL before amf-test

In the event that simply loading the AMF DLL causes initialization code
to run that might crash, don't load the DLL before the AMF test process
has succeeded. Instead, we load the DLL as data to see if it exists,
then run the AMF test, then load the DLL for real.
---
 plugins/obs-ffmpeg/texture-amf.cpp | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/plugins/obs-ffmpeg/texture-amf.cpp b/plugins/obs-ffmpeg/texture-amf.cpp
index 9654c533c..7da77965c 100644
--- a/plugins/obs-ffmpeg/texture-amf.cpp
+++ b/plugins/obs-ffmpeg/texture-amf.cpp
@@ -1602,10 +1602,15 @@ static void register_hevc()
 extern "C" void amf_load(void)
 try {
 	AMF_RESULT res;
+	HMODULE amf_module_test;
 
-	amf_module = LoadLibraryW(AMF_DLL_NAME);
-	if (!amf_module)
+	/* Check if the DLL is present before running the more expensive */
+	/* obs-amf-test.exe, but load it as data so it can't crash us    */
+	amf_module_test =
+		LoadLibraryExW(AMF_DLL_NAME, nullptr, LOAD_LIBRARY_AS_DATAFILE);
+	if (!amf_module_test)
 		throw "No AMF library";
+	FreeLibrary(amf_module_test);
 
 	/* ----------------------------------- */
 	/* Check for AVC/HEVC support          */
@@ -1667,6 +1672,10 @@ try {
 	/* ----------------------------------- */
 	/* Init AMF                            */
 
+	amf_module = LoadLibraryW(AMF_DLL_NAME);
+	if (!amf_module)
+		throw "AMF library failed to load";
+
 	AMFInit_Fn init =
 		(AMFInit_Fn)GetProcAddress(amf_module, AMF_INIT_FUNCTION_NAME);
 	if (!init)