libobs/util: Add get_plugin_info (internal)

Allows the ability to get additional information about the plugin beyond
just whether it's an OBS plugin. For this case, whether it can load,
which is primarily used on Windows and mostly just determines whether it
has an incompatible Qt version.

libobs/util/platform-nix.c

@@ -96,6 +96,13 @@ void os_dlclose(void *module)
 		dlclose(module);
 }
 
+void get_plugin_info(const char *path, bool *is_obs_plugin, bool *can_load)
+{
+	*is_obs_plugin = true;
+	*can_load = true;
+	UNUSED_PARAMETER(path);
+}
+
 bool os_is_obs_plugin(const char *path)
 {
 	UNUSED_PARAMETER(path);

libobs/util/platform-windows.c

@@ -26,6 +26,7 @@
 #include "platform.h"
 #include "darray.h"
 #include "dstr.h"
+#include "obsconfig.h"
 #include "util_uint64.h"
 #include "windows/win-registry.h"
 #include "windows/win-version.h"
@@ -136,6 +137,71 @@ void os_dlclose(void *module)
 	FreeLibrary(module);
 }
 
+#if OBS_QT_VERSION == 6
+static bool has_qt5_import(VOID *base, PIMAGE_NT_HEADERS nt_headers)
+{
+	__try {
+		PIMAGE_DATA_DIRECTORY data_dir;
+		data_dir =
+			&nt_headers->OptionalHeader
+				 .DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT];
+
+		if (data_dir->Size == 0)
+			return false;
+
+		PIMAGE_SECTION_HEADER section, last_section;
+		section = IMAGE_FIRST_SECTION(nt_headers);
+		last_section = section;
+
+		/* find the section that contains the export directory */
+		int i;
+		for (i = 0; i < nt_headers->FileHeader.NumberOfSections; i++) {
+			if (section->VirtualAddress <=
+			    data_dir->VirtualAddress) {
+				last_section = section;
+				section++;
+				continue;
+			} else {
+				break;
+			}
+		}
+
+		/* double check in case we exited early */
+		if (last_section->VirtualAddress > data_dir->VirtualAddress ||
+		    section->VirtualAddress <= data_dir->VirtualAddress)
+			return false;
+
+		section = last_section;
+
+		/* get a pointer to the import directory */
+		PIMAGE_IMPORT_DESCRIPTOR import;
+		import = (PIMAGE_IMPORT_DESCRIPTOR)((byte *)base +
+						    data_dir->VirtualAddress -
+						    section->VirtualAddress +
+						    section->PointerToRawData);
+
+		while (import->Name != 0) {
+			char *name = (char *)((byte *)base + import->Name -
+					      section->VirtualAddress +
+					      section->PointerToRawData);
+
+			/* qt5? bingo, reject this library */
+			if (astrcmpi_n(name, "qt5", 3) == 0) {
+				return true;
+			}
+
+			import++;
+		}
+
+	} __except (EXCEPTION_EXECUTE_HANDLER) {
+		/* we failed somehow, for compatibility assume no qt5 import */
+		return false;
+	}
+
+	return false;
+}
+#endif
+
 static bool has_obs_export(VOID *base, PIMAGE_NT_HEADERS nt_headers)
 {
 	__try {
@@ -209,7 +275,7 @@ static bool has_obs_export(VOID *base, PIMAGE_NT_HEADERS nt_headers)
 	return false;
 }
 
-bool os_is_obs_plugin(const char *path)
+void get_plugin_info(const char *path, bool *is_obs_plugin, bool *can_load)
 {
 	struct dstr dll_name;
 	wchar_t *wpath;
@@ -221,10 +287,11 @@ bool os_is_obs_plugin(const char *path)
 	PIMAGE_DOS_HEADER dos_header;
 	PIMAGE_NT_HEADERS nt_headers;
 
-	bool ret = false;
+	*is_obs_plugin = false;
+	*can_load = false;
 
 	if (!path)
-		return false;
+		return;
 
 	dstr_init_copy(&dll_name, path);
 	dstr_replace(&dll_name, "\\", "/");
@@ -265,12 +332,21 @@ bool os_is_obs_plugin(const char *path)
 		if (nt_headers->Signature != IMAGE_NT_SIGNATURE)
 			goto cleanup;
 
-		ret = has_obs_export(base, nt_headers);
+		*is_obs_plugin = has_obs_export(base, nt_headers);
+
+#if OBS_QT_VERSION == 6
+		if (*is_obs_plugin) {
+			*can_load = !has_qt5_import(base, nt_headers);
+		}
+#else
+		*can_load = true;
+#endif
 
 	} __except (EXCEPTION_EXECUTE_HANDLER) {
 		/* we failed somehow, for compatibility let's assume it
 		 * was a valid plugin and let the loader deal with it */
-		ret = true;
+		*is_obs_plugin = true;
+		*can_load = true;
 		goto cleanup;
 	}
 
@@ -283,8 +359,16 @@ cleanup:
 
 	if (hFile != INVALID_HANDLE_VALUE)
 		CloseHandle(hFile);
+}
 
-	return ret;
+bool os_is_obs_plugin(const char *path)
+{
+	bool is_obs_plugin;
+	bool can_load;
+
+	get_plugin_info(path, &is_obs_plugin, &can_load);
+
+	return is_obs_plugin && can_load;
 }
 
 union time_data {