From e55a2b4331f2783e1b006fd992e4736e0d416ee3 Mon Sep 17 00:00:00 2001 From: jp9000 Date: Thu, 27 Feb 2020 05:13:39 -0800 Subject: [PATCH 1/5] cmake: Add FindVulkan module --- cmake/Modules/FindVulkan.cmake | 70 ++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 cmake/Modules/FindVulkan.cmake diff --git a/cmake/Modules/FindVulkan.cmake b/cmake/Modules/FindVulkan.cmake new file mode 100644 index 000000000..5bea7771d --- /dev/null +++ b/cmake/Modules/FindVulkan.cmake @@ -0,0 +1,70 @@ +# Once done these will be defined: +# +# VULKAN_FOUND +# VULKAN_INCLUDE_DIRS +# VULKAN_LIBRARIES +# +# For use in OBS: +# +# VULKAN_INCLUDE_DIR + +find_package(PkgConfig QUIET) +if (PKG_CONFIG_FOUND) + pkg_check_modules(_VULKAN QUIET vulkan) +endif() + +if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(_lib_suffix 64) +else() + set(_lib_suffix 32) +endif() + +find_path(VULKAN_INCLUDE_DIR + NAMES vulkan/vulkan.h + HINTS + ENV vulkanPath${_lib_suffix} + ENV vulkanPath + ENV DepsPath${_lib_suffix} + ENV DepsPath + ENV VULKAN_SDK + ${vulkanPath${_lib_suffix}} + ${vulkanPath} + ${DepsPath${_lib_suffix}} + ${DepsPath} + ${_VULKAN_INCLUDE_DIRS} + PATHS + /usr/include /usr/local/include /opt/local/include /sw/include + PATH_SUFFIXES + include) + +find_library(VULKAN_LIB + NAMES ${_VULKAN_LIBRARIES} vulkan-1 vulkan libvulkan + HINTS + ENV vulkanPath${_lib_suffix} + ENV vulkanPath + ENV DepsPath${_lib_suffix} + ENV DepsPath + ENV VULKAN_SDK + ${vulkanPath${_lib_suffix}} + ${vulkanPath} + ${DepsPath${_lib_suffix}} + ${DepsPath} + ${_VULKAN_LIBRARY_DIRS} + PATHS + /usr/lib /usr/local/lib /opt/local/lib /sw/lib + PATH_SUFFIXES + lib${_lib_suffix} lib + libs${_lib_suffix} libs + bin${_lib_suffix} bin + ../lib${_lib_suffix} ../lib + ../libs${_lib_suffix} ../libs + ../bin${_lib_suffix} ../bin) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(vulkan DEFAULT_MSG VULKAN_LIB VULKAN_INCLUDE_DIR) +mark_as_advanced(VULKAN_INCLUDE_DIR VULKAN_LIB) + +if(VULKAN_FOUND) + set(VULKAN_INCLUDE_DIRS ${VULKAN_INCLUDE_DIR}) + set(VULKAN_LIBRARIES ${VULKAN_LIB}) +endif() From a20f1168a7d8491fcecf1a022739f23a0dc4e353 Mon Sep 17 00:00:00 2001 From: jp9000 Date: Thu, 27 Feb 2020 05:09:24 -0800 Subject: [PATCH 2/5] win-capture/graphics-hook: Don't allow multiple hooks Prevents multiple separate hook DLLs from being loaded in to a target at once. This is done just in the off-chance that someone might add another hook to the Vulkan layer registry. --- .../win-capture/graphics-hook/graphics-hook.c | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/plugins/win-capture/graphics-hook/graphics-hook.c b/plugins/win-capture/graphics-hook/graphics-hook.c index d939d8afd..2d6f25014 100644 --- a/plugins/win-capture/graphics-hook/graphics-hook.c +++ b/plugins/win-capture/graphics-hook/graphics-hook.c @@ -37,6 +37,7 @@ static HANDLE filemap_hook_info = NULL; static HINSTANCE dll_inst = NULL; static volatile bool stop_loop = false; +static HANDLE dup_hook_mutex = NULL; static HANDLE capture_thread = NULL; char system_path[MAX_PATH] = {0}; char process_name[MAX_PATH] = {0}; @@ -272,6 +273,7 @@ static void free_hook(void) close_handle(&signal_ready); close_handle(&signal_stop); close_handle(&signal_restart); + close_handle(&dup_hook_mutex); ipc_pipe_client_free(&pipe); } @@ -786,6 +788,30 @@ void capture_free(void) active = false; } +#define HOOK_NAME L"graphics_hook_dup_mutex" + +static inline HANDLE open_mutex_plus_id(const wchar_t *name, DWORD id) +{ + wchar_t new_name[64]; + _snwprintf(new_name, 64, L"%s%lu", name, id); + return open_mutex(new_name); +} + +static bool init_dll(void) +{ + DWORD pid = GetCurrentProcessId(); + HANDLE h; + + h = open_mutex_plus_id(HOOK_NAME, pid); + if (h) { + CloseHandle(h); + return false; + } + + dup_hook_mutex = create_mutex_plus_id(HOOK_NAME, pid); + return !!dup_hook_mutex; +} + BOOL WINAPI DllMain(HINSTANCE hinst, DWORD reason, LPVOID unused1) { if (reason == DLL_PROCESS_ATTACH) { @@ -793,6 +819,11 @@ BOOL WINAPI DllMain(HINSTANCE hinst, DWORD reason, LPVOID unused1) dll_inst = hinst; + if (!init_dll()) { + DbgOut("Duplicate hook library"); + return false; + } + HANDLE cur_thread; bool success = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), From 72e770458f7dda648ebc80545bf07e72875f839a Mon Sep 17 00:00:00 2001 From: jp9000 Date: Thu, 27 Feb 2020 05:14:48 -0800 Subject: [PATCH 3/5] win-capture: Use full DLL path for inject helper The inject helper should be able to specify the full path rather than assume the path of the hook DLL. This change allows us to modify the hook's location. This needs to be done because the hook needs to be relocated to ProgramData to prevent the possibility of multiple Vulkan capture hooks. --- plugins/win-capture/game-capture.c | 2 +- plugins/win-capture/inject-helper/inject-helper.c | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/plugins/win-capture/game-capture.c b/plugins/win-capture/game-capture.c index a47f4912e..fb28336f1 100644 --- a/plugins/win-capture/game-capture.c +++ b/plugins/win-capture/game-capture.c @@ -930,7 +930,7 @@ static inline bool inject_hook(struct game_capture *gc) } else { info("using helper (%s hook)", use_anticheat(gc) ? "compatibility" : "direct"); - success = create_inject_process(gc, inject_path, hook_dll); + success = create_inject_process(gc, inject_path, hook_path); } cleanup: diff --git a/plugins/win-capture/inject-helper/inject-helper.c b/plugins/win-capture/inject-helper/inject-helper.c index e35042b48..636350ce6 100644 --- a/plugins/win-capture/inject-helper/inject-helper.c +++ b/plugins/win-capture/inject-helper/inject-helper.c @@ -109,12 +109,7 @@ int main(int argc, char *argv_ansi[]) if (argv && argc == 4) { DWORD size = GetModuleFileNameW(NULL, dll_path, MAX_PATH); if (size) { - wchar_t *name_start = wcsrchr(dll_path, '\\'); - if (name_start) { - *(++name_start) = 0; - wcscpy(name_start, argv[1]); - ret = inject_helper(argv, dll_path); - } + ret = inject_helper(argv, argv[1]); } } LocalFree(argv); From 74acbd7ed002ef6159b8159e739d83f6c74cf0c9 Mon Sep 17 00:00:00 2001 From: Matthieu Cunzi Date: Thu, 27 Feb 2020 05:28:59 -0800 Subject: [PATCH 4/5] win-capture: Add Vulkan capture Co-authored-by: jp9000 Co-authored-by: jpark37 --- UI/win-update/updater/CMakeLists.txt | 1 + UI/win-update/updater/init-hook-files.c | 135 ++ UI/win-update/updater/updater.cpp | 10 + plugins/win-capture/CMakeLists.txt | 2 + plugins/win-capture/game-capture-file-init.c | 238 +++ plugins/win-capture/game-capture.c | 7 +- plugins/win-capture/graphics-hook-ver.h | 22 + .../win-capture/graphics-hook/CMakeLists.txt | 20 + .../graphics-hook/graphics-hook-config.h.in | 2 + .../win-capture/graphics-hook/graphics-hook.c | 25 +- .../win-capture/graphics-hook/graphics-hook.h | 3 + .../graphics-hook/graphics-hook.rc | 39 + .../graphics-hook/obs-vulkan32.json | 17 + .../graphics-hook/obs-vulkan64.json | 17 + .../graphics-hook/vulkan-capture.c | 1533 +++++++++++++++++ .../graphics-hook/vulkan-capture.h | 874 ++++++++++ plugins/win-capture/plugin-main.c | 3 + 17 files changed, 2943 insertions(+), 5 deletions(-) create mode 100644 UI/win-update/updater/init-hook-files.c create mode 100644 plugins/win-capture/game-capture-file-init.c create mode 100644 plugins/win-capture/graphics-hook-ver.h create mode 100644 plugins/win-capture/graphics-hook/graphics-hook.rc create mode 100644 plugins/win-capture/graphics-hook/obs-vulkan32.json create mode 100644 plugins/win-capture/graphics-hook/obs-vulkan64.json create mode 100644 plugins/win-capture/graphics-hook/vulkan-capture.c create mode 100644 plugins/win-capture/graphics-hook/vulkan-capture.h diff --git a/UI/win-update/updater/CMakeLists.txt b/UI/win-update/updater/CMakeLists.txt index e7cfd8262..a65ceab3a 100644 --- a/UI/win-update/updater/CMakeLists.txt +++ b/UI/win-update/updater/CMakeLists.txt @@ -23,6 +23,7 @@ set(updater_HEADERS ) set(updater_SOURCES ../win-update-helpers.cpp + init-hook-files.c updater.cpp patch.cpp http.cpp diff --git a/UI/win-update/updater/init-hook-files.c b/UI/win-update/updater/init-hook-files.c new file mode 100644 index 000000000..bbc0ab599 --- /dev/null +++ b/UI/win-update/updater/init-hook-files.c @@ -0,0 +1,135 @@ +#include +#include +#include +#include + +static inline bool file_exists(const wchar_t *path) +{ + WIN32_FIND_DATAW wfd; + HANDLE h = FindFirstFileW(path, &wfd); + if (h == INVALID_HANDLE_VALUE) + return false; + FindClose(h); + return true; +} + +static LSTATUS get_reg(HKEY hkey, LPCWSTR sub_key, LPCWSTR value_name, bool b64) +{ + HKEY key; + LSTATUS status; + DWORD flags = b64 ? KEY_WOW64_64KEY : KEY_WOW64_32KEY; + DWORD size = sizeof(DWORD); + DWORD val; + + status = RegOpenKeyEx(hkey, sub_key, 0, KEY_READ | flags, &key); + if (status == ERROR_SUCCESS) { + status = RegQueryValueExW(key, value_name, NULL, NULL, + (LPBYTE)&val, &size); + RegCloseKey(key); + } + return status; +} + +#define get_programdata_path(path, subpath) \ + do { \ + SHGetFolderPathW(NULL, CSIDL_COMMON_APPDATA, NULL, \ + SHGFP_TYPE_CURRENT, path); \ + StringCbCatW(path, sizeof(path), L"\\"); \ + StringCbCatW(path, sizeof(path), subpath); \ + } while (false) + +#define make_filename(str, name, ext) \ + do { \ + StringCbCatW(str, sizeof(str), name); \ + StringCbCatW(str, sizeof(str), b64 ? L"64" : L"32"); \ + StringCbCatW(str, sizeof(str), ext); \ + } while (false) + +#define IMPLICIT_LAYERS L"SOFTWARE\\Khronos\\Vulkan\\ImplicitLayers" +#define HOOK_LOCATION L"\\data\\obs-plugins\\win-capture\\" + +static bool update_hook_file(bool b64) +{ + wchar_t temp[MAX_PATH]; + wchar_t src[MAX_PATH]; + wchar_t dst[MAX_PATH]; + wchar_t src_json[MAX_PATH]; + wchar_t dst_json[MAX_PATH]; + + GetCurrentDirectoryW(_countof(src_json), src_json); + StringCbCat(src_json, sizeof(src_json), HOOK_LOCATION); + make_filename(src_json, L"obs-vulkan", L".json"); + + GetCurrentDirectoryW(_countof(src), src); + StringCbCat(src, sizeof(src), HOOK_LOCATION); + make_filename(src, L"graphics-hook", L".dll"); + + get_programdata_path(temp, L"obs-studio-hook\\"); + StringCbCopyW(dst_json, sizeof(dst_json), temp); + StringCbCopyW(dst, sizeof(dst), temp); + make_filename(dst_json, L"obs-vulkan", L".json"); + make_filename(dst, L"graphics-hook", L".dll"); + + if (!file_exists(src)) { + return false; + } + + CreateDirectoryW(temp, NULL); + CopyFileW(src, dst, false); + CopyFileW(src_json, dst_json, false); + return true; +} + +static void update_vulkan_registry(bool b64) +{ + DWORD flags = b64 ? KEY_WOW64_64KEY : KEY_WOW64_32KEY; + wchar_t path[MAX_PATH]; + DWORD temp; + LSTATUS s; + HKEY key; + + get_programdata_path(path, L"obs-studio-hook\\"); + make_filename(path, L"obs-vulkan", L".json"); + + s = get_reg(HKEY_CURRENT_USER, IMPLICIT_LAYERS, path, b64); + if (s == ERROR_SUCCESS) { + s = RegOpenKeyEx(HKEY_CURRENT_USER, IMPLICIT_LAYERS, 0, + KEY_WRITE | flags, &key); + if (s == ERROR_SUCCESS) { + RegDeleteValueW(key, path); + RegCloseKey(key); + } + } + + s = get_reg(HKEY_LOCAL_MACHINE, IMPLICIT_LAYERS, path, b64); + if (s == ERROR_SUCCESS) { + return; + } + + /* ------------------- */ + + s = RegCreateKeyExW(HKEY_LOCAL_MACHINE, IMPLICIT_LAYERS, 0, NULL, 0, + KEY_WRITE | flags, NULL, &key, &temp); + if (s != ERROR_SUCCESS) { + goto finish; + } + + DWORD zero = 0; + s = RegSetValueExW(key, path, 0, REG_DWORD, (const BYTE *)&zero, + sizeof(zero)); + if (s != ERROR_SUCCESS) { + goto finish; + } + +finish: + if (key) + RegCloseKey(key); +} + +void UpdateHookFiles(void) +{ + if (update_hook_file(true)) + update_vulkan_registry(true); + if (update_hook_file(false)) + update_vulkan_registry(false); +} diff --git a/UI/win-update/updater/updater.cpp b/UI/win-update/updater/updater.cpp index 66670f4d3..564b32bae 100644 --- a/UI/win-update/updater/updater.cpp +++ b/UI/win-update/updater/updater.cpp @@ -1071,6 +1071,8 @@ static bool UpdateVS2017Redists(json_t *root) return success; } +extern "C" void UpdateHookFiles(void); + static bool Update(wchar_t *cmdLine) { /* ------------------------------------- * @@ -1408,6 +1410,14 @@ static bool Update(wchar_t *cmdLine) } } + /* ------------------------------------- * + * Update hook files and vulkan registry */ + + UpdateHookFiles(); + + /* ------------------------------------- * + * Finish */ + /* If we get here, all updates installed successfully so we can purge * the old versions */ for (update_t &update : updates) { diff --git a/plugins/win-capture/CMakeLists.txt b/plugins/win-capture/CMakeLists.txt index a0965bd93..768d27e92 100644 --- a/plugins/win-capture/CMakeLists.txt +++ b/plugins/win-capture/CMakeLists.txt @@ -8,6 +8,7 @@ set(win-capture_HEADERS inject-library.h cursor-capture.h graphics-hook-info.h + graphics-hook-ver.h window-helpers.h dc-capture.h) @@ -22,6 +23,7 @@ set(win-capture_SOURCES monitor-capture.c window-capture.c load-graphics-offsets.c + game-capture-file-init.c duplicator-monitor-capture.c plugin-main.c) diff --git a/plugins/win-capture/game-capture-file-init.c b/plugins/win-capture/game-capture-file-init.c new file mode 100644 index 000000000..2be48806c --- /dev/null +++ b/plugins/win-capture/game-capture-file-init.c @@ -0,0 +1,238 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +/* ------------------------------------------------------------------------- */ +/* helper funcs */ + +static bool has_elevation() +{ + SID_IDENTIFIER_AUTHORITY sia = SECURITY_NT_AUTHORITY; + PSID sid = NULL; + BOOL elevated = false; + BOOL success; + + success = AllocateAndInitializeSid(&sia, 2, SECURITY_BUILTIN_DOMAIN_RID, + DOMAIN_ALIAS_RID_ADMINS, 0, 0, 0, 0, + 0, 0, &sid); + if (success && sid) { + CheckTokenMembership(NULL, sid, &elevated); + FreeSid(sid); + } + + return elevated; +} + +static inline bool file_exists(const wchar_t *path) +{ + WIN32_FIND_DATAW wfd; + HANDLE h = FindFirstFileW(path, &wfd); + if (h == INVALID_HANDLE_VALUE) + return false; + FindClose(h); + return true; +} + +static LSTATUS get_reg(HKEY hkey, LPCWSTR sub_key, LPCWSTR value_name, bool b64) +{ + HKEY key; + LSTATUS status; + DWORD flags = b64 ? KEY_WOW64_64KEY : KEY_WOW64_32KEY; + DWORD size = sizeof(DWORD); + DWORD val; + + status = RegOpenKeyEx(hkey, sub_key, 0, KEY_READ | flags, &key); + if (status == ERROR_SUCCESS) { + status = RegQueryValueExW(key, value_name, NULL, NULL, + (LPBYTE)&val, &size); + RegCloseKey(key); + } + return status; +} + +#define get_programdata_path(path, subpath) \ + do { \ + SHGetFolderPathW(NULL, CSIDL_COMMON_APPDATA, NULL, \ + SHGFP_TYPE_CURRENT, path); \ + StringCbCatW(path, sizeof(path), L"\\"); \ + StringCbCatW(path, sizeof(path), subpath); \ + } while (false) + +#define make_filename(str, name, ext) \ + do { \ + StringCbCatW(str, sizeof(str), name); \ + StringCbCatW(str, sizeof(str), b64 ? L"64" : L"32"); \ + StringCbCatW(str, sizeof(str), ext); \ + } while (false) + +/* ------------------------------------------------------------------------- */ +/* function to get the path to the hook */ + +static bool programdata64_hook_exists = false; +static bool programdata32_hook_exists = false; + +char *get_hook_path(bool b64) +{ + wchar_t path[MAX_PATH]; + + get_programdata_path(path, L"obs-studio-hook\\"); + make_filename(path, L"graphics-hook", L".dll"); + + if ((b64 && programdata64_hook_exists) || + (!b64 && programdata32_hook_exists)) { + char *path_utf8 = NULL; + os_wcs_to_utf8_ptr(path, 0, &path_utf8); + return path_utf8; + } + + return obs_module_file(b64 ? "graphics-hook64.dll" + : "graphics-hook32.dll"); +} + +/* ------------------------------------------------------------------------- */ +/* initialization */ + +#define IMPLICIT_LAYERS L"SOFTWARE\\Khronos\\Vulkan\\ImplicitLayers" + +static bool update_hook_file(bool b64) +{ + wchar_t temp[MAX_PATH]; + wchar_t src[MAX_PATH]; + wchar_t dst[MAX_PATH]; + wchar_t src_json[MAX_PATH]; + wchar_t dst_json[MAX_PATH]; + + StringCbCopyW(temp, sizeof(temp), + L"..\\..\\data\\obs-plugins\\" + L"win-capture\\"); + make_filename(temp, L"obs-vulkan", L".json"); + + if (_wfullpath(src_json, temp, MAX_PATH) == NULL) + return false; + + StringCbCopyW(temp, sizeof(temp), + L"..\\..\\data\\obs-plugins\\" + L"win-capture\\"); + make_filename(temp, L"graphics-hook", L".dll"); + + if (_wfullpath(src, temp, MAX_PATH) == NULL) + return false; + + get_programdata_path(temp, L"obs-studio-hook\\"); + StringCbCopyW(dst_json, sizeof(dst_json), temp); + StringCbCopyW(dst, sizeof(dst), temp); + make_filename(dst_json, L"obs-vulkan", L".json"); + make_filename(dst, L"graphics-hook", L".dll"); + + if (!file_exists(src)) { + return false; + } + if (!file_exists(src_json)) { + return false; + } + if (!file_exists(dst) || !file_exists(dst_json)) { + CreateDirectoryW(temp, NULL); + if (!CopyFileW(src_json, dst_json, false)) + return false; + if (!CopyFileW(src, dst, false)) + return false; + return true; + } + + struct win_version_info ver_src = {0}; + struct win_version_info ver_dst = {0}; + if (!get_dll_ver(src, &ver_src)) + return false; + if (!get_dll_ver(dst, &ver_dst)) + return false; + + /* if source is greater than dst, overwrite new file */ + while (win_version_compare(&ver_dst, &ver_src) < 0) { + if (!CopyFileW(src_json, dst_json, false)) + return false; + if (!CopyFileW(src, dst, false)) + return false; + + if (!get_dll_ver(dst, &ver_dst)) + return false; + } + + /* do not use if major version incremented in target compared to + * ours */ + if (ver_dst.major > ver_src.major) { + return false; + } + + return true; +} + +#define warn(format, ...) \ + blog(LOG_WARNING, "%s: " format, "[Vulkan Capture Init]", ##__VA_ARGS__) + +/* Sets vulkan layer registry if it doesn't already exist */ +static void init_vulkan_registry(bool b64) +{ + HKEY key = NULL; + LSTATUS s; + + wchar_t path[MAX_PATH]; + get_programdata_path(path, L"obs-studio-hook\\"); + + s = get_reg(HKEY_LOCAL_MACHINE, IMPLICIT_LAYERS, path, b64); + make_filename(path, L"obs-vulkan", L".json"); + + if (s == ERROR_FILE_NOT_FOUND) { + s = get_reg(HKEY_CURRENT_USER, IMPLICIT_LAYERS, path, b64); + + if (s != ERROR_FILE_NOT_FOUND && s != ERROR_SUCCESS) { + warn("Failed to query registry keys: %d", (int)s); + goto finish; + } + } else if (s != ERROR_SUCCESS) { + warn("Failed to query registry keys: %d", (int)s); + goto finish; + } + + if (s == ERROR_SUCCESS) { + goto finish; + } + + HKEY type = has_elevation() ? HKEY_LOCAL_MACHINE : HKEY_CURRENT_USER; + DWORD flags = b64 ? KEY_WOW64_64KEY : KEY_WOW64_32KEY; + DWORD temp; + + s = RegCreateKeyExW(type, IMPLICIT_LAYERS, 0, NULL, 0, + KEY_WRITE | flags, NULL, &key, &temp); + if (s != ERROR_SUCCESS) { + warn("Failed to create registry key"); + goto finish; + } + + DWORD zero = 0; + s = RegSetValueExW(key, path, 0, REG_DWORD, (const BYTE *)&zero, + sizeof(zero)); + if (s != ERROR_SUCCESS) { + warn("Failed to set registry value"); + } + +finish: + if (key) + RegCloseKey(key); +} + +void init_hook_files() +{ + if (update_hook_file(true)) { + programdata64_hook_exists = true; + init_vulkan_registry(true); + } + if (update_hook_file(false)) { + programdata32_hook_exists = true; + init_vulkan_registry(false); + } +} diff --git a/plugins/win-capture/game-capture.c b/plugins/win-capture/game-capture.c index fb28336f1..6ca146d9d 100644 --- a/plugins/win-capture/game-capture.c +++ b/plugins/win-capture/game-capture.c @@ -893,23 +893,22 @@ static inline bool create_inject_process(struct game_capture *gc, return success; } +extern char *get_hook_path(bool b64); + static inline bool inject_hook(struct game_capture *gc) { bool matching_architecture; bool success = false; - const char *hook_dll; char *inject_path; char *hook_path; if (gc->process_is_64bit) { - hook_dll = "graphics-hook64.dll"; inject_path = obs_module_file("inject-helper64.exe"); } else { - hook_dll = "graphics-hook32.dll"; inject_path = obs_module_file("inject-helper32.exe"); } - hook_path = obs_module_file(hook_dll); + hook_path = get_hook_path(gc->process_is_64bit); if (!check_file_integrity(gc, inject_path, "inject helper")) { goto cleanup; diff --git a/plugins/win-capture/graphics-hook-ver.h b/plugins/win-capture/graphics-hook-ver.h new file mode 100644 index 000000000..ef29591ea --- /dev/null +++ b/plugins/win-capture/graphics-hook-ver.h @@ -0,0 +1,22 @@ +/* DO NOT MODIFY THIS FILE WITHOUT CONSULTING JIM. OTHERWISE, JIM WILL DO + * EVERYTHING IN HIS POWER TO MAKE YOUR LIFE MISERABLE FROM THEREON OUT WITH A + * LEVEL OF DETERMINATION UNLIKE ANYTHING ANYONE HAS EVER SEEN IN THE HISTORY + * OF MANKIND. + * + * YES, THAT MEANS YOU READING THIS RIGHT NOW. + * + * IF YOU HAVE A FORK AND FEEL YOU NEED TO MODIFY THIS, SUBMIT A PULL REQUEST + * AND WAIT UNTIL IT HAS BEEN MERGED AND FULLY RELEASED IN THE CORE PROJECT + * BEFORE USING IT. + * + * THIS IS YOUR ONLY WARNING. */ + +#define HOOK_VER_MAJOR 1 +#define HOOK_VER_MINOR 0 +#define HOOK_VER_PATCH 2 + +#define STRINGIFY(s) #s +#define MAKE_VERSION_NAME(major, minor, patch) \ + STRINGIFY(major) "." STRINGIFY(minor) "." STRINGIFY(patch) ".0" +#define HOOK_VERSION_NAME \ + MAKE_VERSION_NAME(HOOK_VER_MAJOR, HOOK_VER_MINOR, HOOK_VER_PATCH) diff --git a/plugins/win-capture/graphics-hook/CMakeLists.txt b/plugins/win-capture/graphics-hook/CMakeLists.txt index f8bf6309f..4141861f8 100644 --- a/plugins/win-capture/graphics-hook/CMakeLists.txt +++ b/plugins/win-capture/graphics-hook/CMakeLists.txt @@ -2,6 +2,9 @@ project(graphics-hook) set(COMPILE_D3D12_HOOK FALSE CACHE BOOL "Compile D3D12 hook support (required windows 10 SDK)") +find_package(Vulkan REQUIRED) +include_directories(${VULKAN_INCLUDE_DIR}) + configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/graphics-hook-config.h.in" "${CMAKE_BINARY_DIR}/plugins/win-capture/graphics-hook/config/graphics-hook-config.h") @@ -10,6 +13,7 @@ configure_file( set(graphics-hook_HEADERS "${CMAKE_BINARY_DIR}/plugins/win-capture/graphics-hook/config/graphics-hook-config.h" graphics-hook.h + ../graphics-hook-ver.h ../graphics-hook-info.h ../hook-helpers.h ../funchook.h @@ -33,7 +37,15 @@ if(MSVC) add_compile_options("$,/MTd,/MT>") endif() +if (VULKAN_FOUND) + list(APPEND graphics-hook_SOURCES + vulkan-capture.c) + list(APPEND graphics-hook_HEADERS + vulkan-capture.h) +endif() + add_library(graphics-hook MODULE + graphics-hook.rc ${graphics-hook_SOURCES} ${graphics-hook_HEADERS}) @@ -53,4 +65,12 @@ set_target_properties(graphics-hook PROPERTIES OUTPUT_NAME "graphics-hook${_output_suffix}") +if (VULKAN_FOUND) + add_custom_command(TARGET graphics-hook POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy + "${CMAKE_CURRENT_SOURCE_DIR}/obs-vulkan64.json" "${OBS_OUTPUT_DIR}/$/data/obs-plugins/win-capture/obs-vulkan64.json" + COMMAND "${CMAKE_COMMAND}" -E copy + "${CMAKE_CURRENT_SOURCE_DIR}/obs-vulkan32.json" "${OBS_OUTPUT_DIR}/$/data/obs-plugins/win-capture/obs-vulkan32.json" + VERBATIM) +endif() install_obs_datatarget(graphics-hook "obs-plugins/win-capture") diff --git a/plugins/win-capture/graphics-hook/graphics-hook-config.h.in b/plugins/win-capture/graphics-hook/graphics-hook-config.h.in index 6753669f1..a22ddd7b0 100644 --- a/plugins/win-capture/graphics-hook/graphics-hook-config.h.in +++ b/plugins/win-capture/graphics-hook/graphics-hook-config.h.in @@ -13,3 +13,5 @@ #endif #define COMPILE_D3D12_HOOK @COMPILE_D3D12_HOOK@ + +#define COMPILE_VULKAN_HOOK @VULKAN_FOUND@ diff --git a/plugins/win-capture/graphics-hook/graphics-hook.c b/plugins/win-capture/graphics-hook/graphics-hook.c index 2d6f25014..abe789131 100644 --- a/plugins/win-capture/graphics-hook/graphics-hook.c +++ b/plugins/win-capture/graphics-hook/graphics-hook.c @@ -314,6 +314,15 @@ static inline bool attempt_hook(void) static bool d3d9_hooked = false; static bool dxgi_hooked = false; static bool gl_hooked = false; +#if COMPILE_VULKAN_HOOK + static bool vulkan_hooked = false; + if (!vulkan_hooked) { + vulkan_hooked = hook_vulkan(); + if (vulkan_hooked) { + return true; + } + } +#endif //COMPILE_VULKAN_HOOK if (!d3d9_hooked) { if (!d3d9_hookable()) { @@ -788,6 +797,9 @@ void capture_free(void) active = false; } +BOOL init_vk_layer(); +BOOL shutdown_vk_layer(); + #define HOOK_NAME L"graphics_hook_dup_mutex" static inline HANDLE open_mutex_plus_id(const wchar_t *name, DWORD id) @@ -845,7 +857,11 @@ BOOL WINAPI DllMain(HINSTANCE hinst, DWORD reason, LPVOID unused1) if (!init_mutexes()) { return false; } - +#if COMPILE_VULKAN_HOOK + if (!init_vk_layer()) { + return false; + } +#endif /* this prevents the library from being automatically unloaded * by the next FreeLibrary call */ GetModuleFileNameW(hinst, name, MAX_PATH); @@ -860,12 +876,19 @@ BOOL WINAPI DllMain(HINSTANCE hinst, DWORD reason, LPVOID unused1) } } else if (reason == DLL_PROCESS_DETACH) { + if (!dup_hook_mutex) { + return true; + } + if (capture_thread) { stop_loop = true; WaitForSingleObject(capture_thread, 300); CloseHandle(capture_thread); } +#if COMPILE_VULKAN_HOOK + shutdown_vk_layer(); +#endif free_hook(); } diff --git a/plugins/win-capture/graphics-hook/graphics-hook.h b/plugins/win-capture/graphics-hook/graphics-hook.h index 7bfe3a057..e4e649d88 100644 --- a/plugins/win-capture/graphics-hook/graphics-hook.h +++ b/plugins/win-capture/graphics-hook/graphics-hook.h @@ -45,6 +45,9 @@ extern bool hook_d3d8(void); extern bool hook_d3d9(void); extern bool hook_dxgi(void); extern bool hook_gl(void); +#if COMPILE_VULKAN_HOOK +extern bool hook_vulkan(void); +#endif extern void d3d10_capture(void *swap, void *backbuffer, bool capture_overlay); extern void d3d10_free(void); diff --git a/plugins/win-capture/graphics-hook/graphics-hook.rc b/plugins/win-capture/graphics-hook/graphics-hook.rc new file mode 100644 index 000000000..a9b283cb2 --- /dev/null +++ b/plugins/win-capture/graphics-hook/graphics-hook.rc @@ -0,0 +1,39 @@ +/* DO NOT MODIFY THIS FILE WITHOUT CONSULTING JIM. OTHERWISE, JIM WILL DO + * EVERYTHING IN HIS POWER TO MAKE YOUR LIFE MISERABLE FROM THEREON OUT WITH A + * LEVEL OF DETERMINATION UNLIKE ANYTHING ANYONE HAS EVER SEEN IN THE HISTORY + * OF MANKIND. + * + * YES, THAT MEANS YOU READING THIS RIGHT NOW. + * + * IF YOU HAVE A FORK AND FEEL YOU NEED TO MODIFY THIS, SUBMIT A PULL REQUEST + * AND WAIT UNTIL IT HAS BEEN MERGED AND FULLY RELEASED IN THE CORE PROJECT + * BEFORE USING IT. + * + * THIS IS YOUR ONLY WARNING. */ + +#include "../graphics-hook-ver.h" + +1 VERSIONINFO +FILEVERSION HOOK_VER_MAJOR,HOOK_VER_MINOR,HOOK_VER_PATCH,0 +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904B0" + BEGIN + VALUE "CompanyName", "OBS Project" + VALUE "FileDescription", "OBS Graphics Hook" + VALUE "FileVersion", HOOK_VERSION_NAME + VALUE "InternalName", "graphics-hook" + VALUE "OriginalFilename", "graphics-hook" + VALUE "ProductName", "OBS Graphics Hook" + VALUE "ProductVersion", HOOK_VERSION_NAME + VALUE "Comments", "Free and open source software for video recording and live streaming" + VALUE "LegalCopyright", "(C) Hugh Bailey" + END + END + + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0409, 0x04B0 + END +END diff --git a/plugins/win-capture/graphics-hook/obs-vulkan32.json b/plugins/win-capture/graphics-hook/obs-vulkan32.json new file mode 100644 index 000000000..a4b187aa3 --- /dev/null +++ b/plugins/win-capture/graphics-hook/obs-vulkan32.json @@ -0,0 +1,17 @@ +{ + "file_format_version": "1.1.2", + "layer": { + "name": "VK_LAYER_OBS_hook", + "type": "GLOBAL", + "library_path": ".\\graphics-hook32.dll", + "api_version": "1.2.131", + "implementation_version": "1", + "description": "Open Broadcaster Software hook", + "functions": { + "vkNegotiateLoaderLayerInterfaceVersion": "OBS_Negotiate" + }, + "disable_environment": { + "DISABLE_VULKAN_OBS_CAPTURE": "1" + } + } +} diff --git a/plugins/win-capture/graphics-hook/obs-vulkan64.json b/plugins/win-capture/graphics-hook/obs-vulkan64.json new file mode 100644 index 000000000..d7b318d55 --- /dev/null +++ b/plugins/win-capture/graphics-hook/obs-vulkan64.json @@ -0,0 +1,17 @@ +{ + "file_format_version": "1.1.2", + "layer": { + "name": "VK_LAYER_OBS_hook", + "type": "GLOBAL", + "library_path": ".\\graphics-hook64.dll", + "api_version": "1.2.131", + "implementation_version": "1", + "description": "Open Broadcaster Software hook", + "functions": { + "vkNegotiateLoaderLayerInterfaceVersion": "OBS_Negotiate" + }, + "disable_environment": { + "DISABLE_VULKAN_OBS_CAPTURE": "1" + } + } +} diff --git a/plugins/win-capture/graphics-hook/vulkan-capture.c b/plugins/win-capture/graphics-hook/vulkan-capture.c new file mode 100644 index 000000000..b17a6deef --- /dev/null +++ b/plugins/win-capture/graphics-hook/vulkan-capture.c @@ -0,0 +1,1533 @@ +#include +#include "graphics-hook.h" + +#define VK_USE_PLATFORM_WIN32_KHR + +#include +#include +#include + +#undef VK_LAYER_EXPORT +#if defined(WIN32) +#define VK_LAYER_EXPORT __declspec(dllexport) +#else +#define VK_LAYER_EXPORT +#endif + +#include + +#define COBJMACROS +#include +#include + +#include "vulkan-capture.h" + +/* ======================================================================== */ +/* defs/statics */ + +/* shorten stuff because dear GOD is vulkan unclean. */ +#define VKAPI VKAPI_CALL +#define VkFunc PFN_vkVoidFunction +#define EXPORT VK_LAYER_EXPORT + +#define OBJ_MAX 16 + +/* use the loader's dispatch table pointer as a key for internal data maps */ +#define GET_LDT(x) (*(void **)x) + +#define DUMMY_WINDOW_CLASS_NAME L"graphics_hook_vk_dummy_window" + +/* clang-format off */ +static const GUID dxgi_factory1_guid = +{0x770aae78, 0xf26f, 0x4dba, {0xa8, 0x29, 0x25, 0x3c, 0x83, 0xd1, 0xb3, 0x87}}; +static const GUID dxgi_resource_guid = +{0x035f3ab4, 0x482e, 0x4e50, {0xb4, 0x1f, 0x8a, 0x7f, 0x8b, 0xd8, 0x96, 0x0b}}; +/* clang-format on */ + +static bool vulkan_seen = false; +static CRITICAL_SECTION mutex; + +/* ======================================================================== */ +/* hook data */ + +struct vk_swap_data { + VkSwapchainKHR sc; + VkExtent2D image_extent; + VkFormat format; + VkSurfaceKHR surf; + VkImage export_image; + bool layout_initialized; + VkDeviceMemory export_mem; + VkImage swap_images[OBJ_MAX]; + uint32_t image_count; + + HANDLE handle; + struct shtex_data *shtex_info; + ID3D11Texture2D *d3d11_tex; + bool captured; +}; + +struct vk_queue_data { + VkQueue queue; + uint32_t fam_idx; +}; + +struct vk_cmd_pool_data { + VkCommandPool cmd_pool; + VkCommandBuffer cmd_buffers[OBJ_MAX]; + VkFence fences[OBJ_MAX]; + bool cmd_buffer_busy[OBJ_MAX]; + uint32_t image_count; +}; + +struct vk_data { + bool valid; + + struct vk_device_funcs funcs; + VkPhysicalDevice phy_device; + VkDevice device; + struct vk_swap_data swaps[OBJ_MAX]; + struct vk_swap_data *cur_swap; + uint32_t swap_idx; + + struct vk_queue_data queues[OBJ_MAX]; + uint32_t queue_count; + + struct vk_cmd_pool_data cmd_pools[OBJ_MAX]; + VkExternalMemoryProperties external_mem_props; + + ID3D11Device *d3d11_device; + ID3D11DeviceContext *d3d11_context; + IDXGISwapChain *dxgi_swap; + HWND dummy_hwnd; +}; + +static struct vk_swap_data *get_swap_data(struct vk_data *data, + VkSwapchainKHR sc) +{ + for (int i = 0; i < OBJ_MAX; i++) { + if (data->swaps[i].sc == sc) { + return &data->swaps[i]; + } + } + + debug("get_swap_data failed, swapchain not found"); + return NULL; +} + +static struct vk_swap_data *get_new_swap_data(struct vk_data *data) +{ + for (int i = 0; i < OBJ_MAX; i++) { + if (data->swaps[i].surf == NULL && data->swaps[i].sc == NULL) { + return &data->swaps[i]; + } + } + + debug("get_new_swap_data failed, no more free slot"); + return NULL; +} + +/* ------------------------------------------------------------------------- */ + +static inline size_t find_obj_idx(void *objs[], void *obj) +{ + size_t idx = SIZE_MAX; + + EnterCriticalSection(&mutex); + for (size_t i = 0; i < OBJ_MAX; i++) { + if (objs[i] == obj) { + idx = i; + break; + } + } + LeaveCriticalSection(&mutex); + + return idx; +} + +static size_t get_obj_idx(void *objs[], void *obj) +{ + size_t idx = SIZE_MAX; + + EnterCriticalSection(&mutex); + for (size_t i = 0; i < OBJ_MAX; i++) { + if (objs[i] == obj) { + idx = i; + break; + } + if (!objs[i] && idx == SIZE_MAX) { + idx = i; + } + } + LeaveCriticalSection(&mutex); + return idx; +} + +/* ------------------------------------------------------------------------- */ + +static struct vk_data device_data[OBJ_MAX] = {0}; +static void *devices[OBJ_MAX] = {0}; + +static inline struct vk_data *get_device_data(void *dev) +{ + size_t idx = get_obj_idx(devices, GET_LDT(dev)); + if (idx == SIZE_MAX) { + debug("out of device slots"); + return NULL; + } + + return &device_data[idx]; +} + +static void vk_shtex_clear_fence(struct vk_data *data, + struct vk_cmd_pool_data *pool_data, + uint32_t image_idx) +{ + VkFence fence = pool_data->fences[image_idx]; + if (pool_data->cmd_buffer_busy[image_idx]) { + VkDevice device = data->device; + struct vk_device_funcs *funcs = &data->funcs; + funcs->WaitForFences(device, 1, &fence, VK_TRUE, ~0ull); + funcs->ResetFences(device, 1, &fence); + pool_data->cmd_buffer_busy[image_idx] = false; + } +} + +static void vk_shtex_wait_until_pool_idle(struct vk_data *data, + struct vk_cmd_pool_data *pool_data) +{ + for (uint32_t image_idx = 0; image_idx < pool_data->image_count; + image_idx++) { + vk_shtex_clear_fence(data, pool_data, image_idx); + } +} + +static void vk_shtex_wait_until_idle(struct vk_data *data) +{ + for (uint32_t fam_idx = 0; fam_idx < _countof(data->cmd_pools); + fam_idx++) { + struct vk_cmd_pool_data *pool_data = &data->cmd_pools[fam_idx]; + if (pool_data->cmd_pool != VK_NULL_HANDLE) + vk_shtex_wait_until_pool_idle(data, pool_data); + } +} + +static void vk_shtex_free(struct vk_data *data) +{ + capture_free(); + + vk_shtex_wait_until_idle(data); + + for (int swap_idx = 0; swap_idx < OBJ_MAX; swap_idx++) { + struct vk_swap_data *swap = &data->swaps[swap_idx]; + + if (swap->export_image) + data->funcs.DestroyImage(data->device, + swap->export_image, NULL); + + if (swap->export_mem) + data->funcs.FreeMemory(data->device, swap->export_mem, + NULL); + + if (swap->d3d11_tex) { + ID3D11Resource_Release(swap->d3d11_tex); + } + + swap->handle = INVALID_HANDLE_VALUE; + swap->d3d11_tex = NULL; + swap->export_mem = NULL; + swap->export_image = NULL; + + swap->captured = false; + } + + if (data->d3d11_context) { + ID3D11DeviceContext_Release(data->d3d11_context); + data->d3d11_context = NULL; + } + if (data->d3d11_device) { + ID3D11Device_Release(data->d3d11_device); + data->d3d11_device = NULL; + } + if (data->dxgi_swap) { + IDXGISwapChain_Release(data->dxgi_swap); + data->dxgi_swap = NULL; + } + + data->cur_swap = NULL; + + hlog("------------------ vulkan capture freed ------------------"); +} + +static void vk_remove_device(void *dev) +{ + size_t idx = find_obj_idx(devices, GET_LDT(dev)); + if (idx == SIZE_MAX) { + return; + } + + struct vk_data *data = &device_data[idx]; + + memset(data, 0, sizeof(*data)); + + EnterCriticalSection(&mutex); + devices[idx] = NULL; + LeaveCriticalSection(&mutex); +} + +/* ------------------------------------------------------------------------- */ + +struct vk_surf_data { + VkSurfaceKHR surf; + HINSTANCE hinstance; + HWND hwnd; +}; + +struct vk_inst_data { + bool valid; + + struct vk_inst_funcs funcs; + struct vk_surf_data surfaces[OBJ_MAX]; +}; + +static struct vk_surf_data *find_surf_data(struct vk_inst_data *data, + VkSurfaceKHR surf) +{ + int idx = OBJ_MAX; + for (int i = 0; i < OBJ_MAX; i++) { + if (data->surfaces[i].surf == surf) { + return &data->surfaces[i]; + } else if (data->surfaces[i].surf == NULL && idx == OBJ_MAX) { + idx = i; + } + } + if (idx != OBJ_MAX) { + data->surfaces[idx].surf = surf; + return &data->surfaces[idx]; + } + + debug("find_surf_data failed, no more free slots"); + return NULL; +} + +/* ------------------------------------------------------------------------- */ + +static struct vk_inst_data inst_data[OBJ_MAX] = {0}; +static void *instances[OBJ_MAX] = {0}; + +static struct vk_inst_data *get_inst_data(void *inst) +{ + size_t idx = get_obj_idx(instances, GET_LDT(inst)); + if (idx == SIZE_MAX) { + debug("out of instance slots"); + return NULL; + } + + vulkan_seen = true; + return &inst_data[idx]; +} + +static inline struct vk_inst_funcs *get_inst_funcs(void *inst) +{ + struct vk_inst_data *data = get_inst_data(inst); + return &data->funcs; +} + +static void remove_instance(void *inst) +{ + size_t idx = find_obj_idx(instances, inst); + if (idx == SIZE_MAX) { + return; + } + + struct vk_inst_data *data = &inst_data[idx]; + memset(data, 0, sizeof(*data)); + + EnterCriticalSection(&mutex); + instances[idx] = NULL; + LeaveCriticalSection(&mutex); +} + +/* ======================================================================== */ +/* capture */ + +static bool vk_register_window(void) +{ + WNDCLASSW wc = {0}; + wc.style = CS_OWNDC; + wc.hInstance = GetModuleHandle(NULL); + wc.lpfnWndProc = DefWindowProc; + wc.lpszClassName = DUMMY_WINDOW_CLASS_NAME; + + if (!RegisterClassW(&wc)) { + flog("failed to register window class: %d", GetLastError()); + return false; + } + + return true; +} + +static inline bool vk_shtex_init_window(struct vk_data *data) +{ + static bool registered = false; + if (!registered) { + static bool failure = false; + if (failure || !vk_register_window()) { + failure = true; + return false; + } + registered = true; + } + + data->dummy_hwnd = CreateWindowExW( + 0, DUMMY_WINDOW_CLASS_NAME, L"Dummy VK window, ignore", + WS_POPUP | WS_CLIPCHILDREN | WS_CLIPSIBLINGS, 0, 0, 2, 2, NULL, + NULL, GetModuleHandle(NULL), NULL); + if (!data->dummy_hwnd) { + flog("failed to create window: %d", GetLastError()); + return false; + } + + return true; +} + +static inline bool vk_shtex_init_d3d11(struct vk_data *data) +{ + D3D_FEATURE_LEVEL level_used; + IDXGIFactory1 *factory; + IDXGIAdapter *adapter; + HRESULT hr; + + HMODULE d3d11 = load_system_library("d3d11.dll"); + if (!d3d11) { + flog("failed to load d3d11: %d", GetLastError()); + return false; + } + + HMODULE dxgi = load_system_library("dxgi.dll"); + if (!dxgi) { + flog("failed to load dxgi: %d", GetLastError()); + return false; + } + + DXGI_SWAP_CHAIN_DESC desc = {0}; + desc.BufferCount = 2; + desc.BufferDesc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.BufferDesc.Width = 2; + desc.BufferDesc.Height = 2; + desc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT; + desc.SampleDesc.Count = 1; + desc.Windowed = true; + desc.OutputWindow = data->dummy_hwnd; + + HRESULT(WINAPI * create_factory) + (REFIID, void **) = (void *)GetProcAddress(dxgi, "CreateDXGIFactory1"); + if (!create_factory) { + flog("failed to get CreateDXGIFactory1 address: %d", + GetLastError()); + return false; + } + + PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN create = + (void *)GetProcAddress(d3d11, "D3D11CreateDeviceAndSwapChain"); + if (!create) { + flog("failed to get D3D11CreateDeviceAndSwapChain address: %d", + GetLastError()); + return false; + } + + hr = create_factory(&dxgi_factory1_guid, (void **)&factory); + if (FAILED(hr)) { + flog_hr("failed to create factory", hr); + return false; + } + + hr = IDXGIFactory1_EnumAdapters1(factory, 0, + (IDXGIAdapter1 **)&adapter); + IDXGIFactory1_Release(factory); + + if (FAILED(hr)) { + flog_hr("failed to create adapter", hr); + return false; + } + + static const D3D_FEATURE_LEVEL feature_levels[] = { + D3D_FEATURE_LEVEL_11_0, + D3D_FEATURE_LEVEL_10_1, + D3D_FEATURE_LEVEL_10_0, + D3D_FEATURE_LEVEL_9_3, + }; + + hr = create(adapter, D3D_DRIVER_TYPE_UNKNOWN, NULL, 0, feature_levels, + sizeof(feature_levels) / sizeof(D3D_FEATURE_LEVEL), + D3D11_SDK_VERSION, &desc, &data->dxgi_swap, + &data->d3d11_device, &level_used, &data->d3d11_context); + IDXGIAdapter_Release(adapter); + + if (FAILED(hr)) { + flog_hr("failed to create device", hr); + return false; + } + + return true; +} + +static inline bool vk_shtex_init_d3d11_tex(struct vk_data *data, + struct vk_swap_data *swap) +{ + IDXGIResource *dxgi_res; + HRESULT hr; + + D3D11_TEXTURE2D_DESC desc = {0}; + desc.Width = swap->image_extent.width; + desc.Height = swap->image_extent.height; + desc.MipLevels = 1; + desc.ArraySize = 1; + + flog("OBS requesting %s texture format", + vk_format_to_str(swap->format)); + + desc.Format = vk_format_to_dxgi(swap->format); + desc.SampleDesc.Count = 1; + desc.SampleDesc.Quality = 0; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; + desc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE; + + hr = ID3D11Device_CreateTexture2D(data->d3d11_device, &desc, NULL, + &swap->d3d11_tex); + if (FAILED(hr)) { + flog_hr("failed to create texture", hr); + return false; + } + + hr = ID3D11Device_QueryInterface(swap->d3d11_tex, &dxgi_resource_guid, + (void **)&dxgi_res); + if (FAILED(hr)) { + flog_hr("failed to get IDXGIResource", hr); + return false; + } + + hr = IDXGIResource_GetSharedHandle(dxgi_res, &swap->handle); + IDXGIResource_Release(dxgi_res); + + if (FAILED(hr)) { + flog_hr("failed to get shared handle", hr); + return false; + } + + return true; +} + +static inline bool vk_shtex_init_vulkan_tex(struct vk_data *data, + struct vk_swap_data *swap) +{ + struct vk_device_funcs *funcs = &data->funcs; + VkExternalMemoryFeatureFlags f = + data->external_mem_props.externalMemoryFeatures; + + /* -------------------------------------------------------- */ + /* create texture */ + + VkExternalMemoryImageCreateInfo emici; + emici.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO; + emici.pNext = NULL; + emici.handleTypes = + VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT; + + VkImageCreateInfo ici; + ici.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + ici.pNext = &emici; + ici.flags = 0; + ici.imageType = VK_IMAGE_TYPE_2D; + ici.format = swap->format; + ici.extent.width = swap->image_extent.width; + ici.extent.height = swap->image_extent.height; + ici.extent.depth = 1; + ici.mipLevels = 1; + ici.arrayLayers = 1; + ici.samples = VK_SAMPLE_COUNT_1_BIT; + ici.tiling = VK_IMAGE_TILING_OPTIMAL; + ici.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | + VK_IMAGE_USAGE_SAMPLED_BIT; + ici.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + ici.queueFamilyIndexCount = 0; + ici.pQueueFamilyIndices = 0; + ici.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + + VkResult res; + res = funcs->CreateImage(data->device, &ici, NULL, &swap->export_image); + if (VK_SUCCESS != res) { + flog("failed to CreateImage: %s", result_to_str(res)); + swap->export_image = NULL; + return false; + } + + swap->layout_initialized = false; + + /* -------------------------------------------------------- */ + /* get image memory requirements */ + + VkMemoryRequirements mr; + bool use_gimr2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT; + + if (use_gimr2) { + VkMemoryDedicatedRequirements mdr = {0}; + mdr.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS; + mdr.pNext = NULL; + + VkMemoryRequirements2 mr2 = {0}; + mr2.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2; + mr2.pNext = &mdr; + + VkImageMemoryRequirementsInfo2 imri2 = {0}; + imri2.sType = + VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2; + imri2.pNext = NULL; + imri2.image = swap->export_image; + + funcs->GetImageMemoryRequirements2(data->device, &imri2, &mr2); + mr = mr2.memoryRequirements; + } else { + funcs->GetImageMemoryRequirements(data->device, + swap->export_image, &mr); + } + + /* -------------------------------------------------------- */ + /* get memory type index */ + + struct vk_inst_funcs *ifuncs = get_inst_funcs(data->phy_device); + + VkPhysicalDeviceMemoryProperties pdmp; + ifuncs->GetPhysicalDeviceMemoryProperties(data->phy_device, &pdmp); + + uint32_t mem_type_idx = 0; + + for (; mem_type_idx < pdmp.memoryTypeCount; mem_type_idx++) { + if ((mr.memoryTypeBits & (1 << mem_type_idx)) && + (pdmp.memoryTypes[mem_type_idx].propertyFlags & + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) == + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) { + break; + } + } + + if (mem_type_idx == pdmp.memoryTypeCount) { + flog("failed to get memory type index"); + funcs->DestroyImage(data->device, swap->export_image, NULL); + swap->export_image = NULL; + return false; + } + + /* -------------------------------------------------------- */ + /* allocate memory */ + + VkImportMemoryWin32HandleInfoKHR imw32hi; + imw32hi.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_WIN32_HANDLE_INFO_KHR; + imw32hi.pNext = NULL; + imw32hi.name = NULL; + imw32hi.handleType = + VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT; + imw32hi.handle = swap->handle; + + VkMemoryAllocateInfo mai; + mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + mai.pNext = &imw32hi; + mai.allocationSize = mr.size; + mai.memoryTypeIndex = mem_type_idx; + + VkMemoryDedicatedAllocateInfo mdai; + mdai.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO; + mdai.pNext = NULL; + mdai.buffer = VK_NULL_HANDLE; + + if (data->external_mem_props.externalMemoryFeatures & + VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT) { + mdai.image = swap->export_image; + imw32hi.pNext = &mdai; + } + + res = funcs->AllocateMemory(data->device, &mai, NULL, + &swap->export_mem); + if (VK_SUCCESS != res) { + flog("failed to AllocateMemory: %s", result_to_str(res)); + funcs->DestroyImage(data->device, swap->export_image, NULL); + swap->export_image = NULL; + return false; + } + + /* -------------------------------------------------------- */ + /* bind image memory */ + + bool use_bi2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT; + + if (use_bi2) { + VkBindImageMemoryInfo bimi = {0}; + bimi.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO; + bimi.image = swap->export_image; + bimi.memory = swap->export_mem; + bimi.memoryOffset = 0; + res = funcs->BindImageMemory2(data->device, 1, &bimi); + } else { + res = funcs->BindImageMemory(data->device, swap->export_image, + swap->export_mem, 0); + } + if (VK_SUCCESS != res) { + flog("%s failed: %s", + use_bi2 ? "BindImageMemory2" : "BindImageMemory", + result_to_str(res)); + funcs->DestroyImage(data->device, swap->export_image, NULL); + swap->export_image = NULL; + return false; + } + return true; +} + +static bool vk_shtex_init(struct vk_data *data, HWND window, + struct vk_swap_data *swap) +{ + if (!vk_shtex_init_window(data)) { + return false; + } + if (!vk_shtex_init_d3d11(data)) { + return false; + } + if (!vk_shtex_init_d3d11_tex(data, swap)) { + return false; + } + if (!vk_shtex_init_vulkan_tex(data, swap)) { + return false; + } + + data->cur_swap = swap; + + swap->captured = capture_init_shtex( + &swap->shtex_info, window, swap->image_extent.width, + swap->image_extent.height, swap->image_extent.width, + swap->image_extent.height, (uint32_t)swap->format, false, + (uintptr_t)swap->handle); + + if (swap->captured) { + if (global_hook_info->force_shmem) { + flog("shared memory capture currently " + "unsupported; ignoring"); + } + + hlog("vulkan shared texture capture successful"); + return true; + } + + return false; +} + +static void vk_shtex_create_cmd_pool_objects(struct vk_data *data, + uint32_t fam_idx, + uint32_t image_count) +{ + struct vk_cmd_pool_data *pool_data = &data->cmd_pools[fam_idx]; + + VkCommandPoolCreateInfo cpci; + cpci.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + cpci.pNext = NULL; + cpci.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + cpci.queueFamilyIndex = fam_idx; + + VkResult res = data->funcs.CreateCommandPool(data->device, &cpci, NULL, + &pool_data->cmd_pool); + debug_res("CreateCommandPool", res); + + VkCommandBufferAllocateInfo cbai; + cbai.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + cbai.pNext = NULL; + cbai.commandPool = pool_data->cmd_pool; + cbai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cbai.commandBufferCount = image_count; + + res = data->funcs.AllocateCommandBuffers(data->device, &cbai, + pool_data->cmd_buffers); + debug_res("AllocateCommandBuffers", res); + for (uint32_t image_index = 0; image_index < image_count; + image_index++) { + /* Dispatch table something or other. Well-designed API. */ + VkCommandBuffer cmd_buffer = + pool_data->cmd_buffers[image_index]; + *(void **)cmd_buffer = *(void **)(data->device); + + VkFence *fence = &pool_data->fences[image_index]; + VkFenceCreateInfo fci = {0}; + fci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + fci.pNext = NULL; + fci.flags = 0; + res = data->funcs.CreateFence(data->device, &fci, NULL, fence); + debug_res("CreateFence", res); + } + + pool_data->image_count = image_count; +} + +static void vk_shtex_destroy_fence(struct vk_data *data, bool *cmd_buffer_busy, + VkFence *fence) +{ + VkDevice device = data->device; + + if (*cmd_buffer_busy) { + data->funcs.WaitForFences(device, 1, fence, VK_TRUE, ~0ull); + *cmd_buffer_busy = false; + } + + data->funcs.DestroyFence(device, *fence, NULL); + *fence = VK_NULL_HANDLE; +} + +static void +vk_shtex_destroy_cmd_pool_objects(struct vk_data *data, + struct vk_cmd_pool_data *pool_data) +{ + for (uint32_t image_idx = 0; image_idx < pool_data->image_count; + image_idx++) { + bool *cmd_buffer_busy = &pool_data->cmd_buffer_busy[image_idx]; + VkFence *fence = &pool_data->fences[image_idx]; + vk_shtex_destroy_fence(data, cmd_buffer_busy, fence); + } + + data->funcs.DestroyCommandPool(data->device, pool_data->cmd_pool, NULL); + pool_data->cmd_pool = VK_NULL_HANDLE; + pool_data->image_count = 0; +} + +static void vk_shtex_capture(struct vk_data *data, + struct vk_device_funcs *funcs, + struct vk_swap_data *swap, uint32_t idx, + VkQueue queue, const VkPresentInfoKHR *info) +{ + VkResult res = VK_SUCCESS; + + VkCommandBufferBeginInfo begin_info; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.pNext = NULL; + begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + begin_info.pInheritanceInfo = NULL; + + VkImageMemoryBarrier mb[2]; + VkImageMemoryBarrier *src_mb = &mb[0]; + VkImageMemoryBarrier *dst_mb = &mb[1]; + + /* ------------------------------------------------------ */ + /* do image copy */ + + const uint32_t image_index = info->pImageIndices[idx]; + VkImage cur_backbuffer = swap->swap_images[image_index]; + + uint32_t fam_idx = 0; + for (uint32_t i = 0; i < data->queue_count; i++) { + if (data->queues[i].queue == queue) + fam_idx = data->queues[i].fam_idx; + } + + if (fam_idx >= _countof(data->cmd_pools)) + return; + + struct vk_cmd_pool_data *pool_data = &data->cmd_pools[fam_idx]; + VkCommandPool *pool = &pool_data->cmd_pool; + const uint32_t image_count = swap->image_count; + if (pool_data->image_count < image_count) { + if (*pool != VK_NULL_HANDLE) + vk_shtex_destroy_cmd_pool_objects(data, pool_data); + vk_shtex_create_cmd_pool_objects(data, fam_idx, image_count); + } + + vk_shtex_clear_fence(data, pool_data, image_index); + + VkCommandBuffer cmd_buffer = pool_data->cmd_buffers[image_index]; + res = funcs->BeginCommandBuffer(cmd_buffer, &begin_info); + debug_res("BeginCommandBuffer", res); + + /* ------------------------------------------------------ */ + /* transition shared texture if necessary */ + + if (!swap->layout_initialized) { + VkImageMemoryBarrier imb; + imb.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + imb.pNext = NULL; + imb.srcAccessMask = 0; + imb.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + imb.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + imb.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + imb.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + imb.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + imb.image = swap->export_image; + imb.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + imb.subresourceRange.baseMipLevel = 0; + imb.subresourceRange.levelCount = 1; + imb.subresourceRange.baseArrayLayer = 0; + imb.subresourceRange.layerCount = 1; + + funcs->CmdPipelineBarrier(cmd_buffer, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, + NULL, 0, NULL, 1, &imb); + + swap->layout_initialized = true; + } + + /* ------------------------------------------------------ */ + /* transition cur_backbuffer to transfer source state */ + + src_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + src_mb->pNext = NULL; + src_mb->srcAccessMask = VK_ACCESS_MEMORY_READ_BIT; + src_mb->dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + src_mb->oldLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + src_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + src_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + src_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + src_mb->image = cur_backbuffer; + src_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + src_mb->subresourceRange.baseMipLevel = 0; + src_mb->subresourceRange.levelCount = 1; + src_mb->subresourceRange.baseArrayLayer = 0; + src_mb->subresourceRange.layerCount = 1; + + /* ------------------------------------------------------ */ + /* transition exportedTexture to transfer dest state */ + + dst_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + dst_mb->pNext = NULL; + dst_mb->srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + dst_mb->dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + dst_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + dst_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + dst_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; + dst_mb->dstQueueFamilyIndex = fam_idx; + dst_mb->image = swap->export_image; + dst_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + dst_mb->subresourceRange.baseMipLevel = 0; + dst_mb->subresourceRange.levelCount = 1; + dst_mb->subresourceRange.baseArrayLayer = 0; + dst_mb->subresourceRange.layerCount = 1; + + funcs->CmdPipelineBarrier(cmd_buffer, + VK_PIPELINE_STAGE_TRANSFER_BIT | + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, NULL, 0, + NULL, 2, mb); + + /* ------------------------------------------------------ */ + /* copy cur_backbuffer's content to our interop image */ + + VkImageCopy cpy; + cpy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + cpy.srcSubresource.mipLevel = 0; + cpy.srcSubresource.baseArrayLayer = 0; + cpy.srcSubresource.layerCount = 1; + cpy.srcOffset.x = 0; + cpy.srcOffset.y = 0; + cpy.srcOffset.z = 0; + cpy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + cpy.dstSubresource.mipLevel = 0; + cpy.dstSubresource.baseArrayLayer = 0; + cpy.dstSubresource.layerCount = 1; + cpy.dstOffset.x = 0; + cpy.dstOffset.y = 0; + cpy.dstOffset.z = 0; + cpy.extent.width = swap->image_extent.width; + cpy.extent.height = swap->image_extent.height; + cpy.extent.depth = 1; + funcs->CmdCopyImage(cmd_buffer, cur_backbuffer, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + swap->export_image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &cpy); + + /* ------------------------------------------------------ */ + /* Restore the swap chain image layout to what it was + * before. This may not be strictly needed, but it is + * generally good to restore things to their original + * state. */ + + src_mb->srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + src_mb->dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + src_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + src_mb->newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + + dst_mb->srcQueueFamilyIndex = fam_idx; + dst_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; + + funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT | + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, + 0, 0, NULL, 0, NULL, 2, mb); + + funcs->EndCommandBuffer(cmd_buffer); + + /* ------------------------------------------------------ */ + + VkSubmitInfo submit_info; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info.pNext = NULL; + submit_info.waitSemaphoreCount = 0; + submit_info.pWaitSemaphores = NULL; + submit_info.pWaitDstStageMask = NULL; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &cmd_buffer; + submit_info.signalSemaphoreCount = 0; + submit_info.pSignalSemaphores = NULL; + + VkFence fence = pool_data->fences[image_index]; + res = funcs->QueueSubmit(queue, 1, &submit_info, fence); + debug_res("QueueSubmit", res); + + if (res == VK_SUCCESS) + pool_data->cmd_buffer_busy[image_index] = true; +} + +static inline HWND get_swap_window(struct vk_swap_data *swap) +{ + for (size_t i = 0; i < OBJ_MAX; i++) { + struct vk_surf_data *surf_data = + find_surf_data(&inst_data[i], swap->surf); + + if (!!surf_data && surf_data->surf == swap->surf) { + return surf_data->hwnd; + } + } + + return NULL; +} + +static void vk_capture(struct vk_data *data, VkQueue queue, + const VkPresentInfoKHR *info) +{ + struct vk_swap_data *swap = NULL; + HWND window = NULL; + uint32_t idx = 0; + + debug("QueuePresentKHR called on " + "devicekey %p, swapchain count %d", + &data->funcs, info->swapchainCount); + + /* use first swap chain associated with a window */ + for (; idx < info->swapchainCount; idx++) { + struct vk_swap_data *cur_swap = + get_swap_data(data, info->pSwapchains[idx]); + window = get_swap_window(cur_swap); + if (!!window) { + swap = cur_swap; + break; + } + } + + if (!window) { + return; + } + + if (capture_should_stop()) { + vk_shtex_free(data); + } + if (capture_should_init()) { + if (!vk_shtex_init(data, window, swap)) { + vk_shtex_free(data); + data->valid = false; + } + } + if (capture_ready()) { + if (swap != data->cur_swap) { + vk_shtex_free(data); + return; + } + + vk_shtex_capture(data, &data->funcs, swap, idx, queue, info); + } +} + +static VkResult VKAPI OBS_QueuePresentKHR(VkQueue queue, + const VkPresentInfoKHR *info) +{ + struct vk_data *data = get_device_data(queue); + struct vk_device_funcs *funcs = &data->funcs; + + if (data->valid) { + vk_capture(data, queue, info); + } + return funcs->QueuePresentKHR(queue, info); +} + +/* ======================================================================== */ +/* setup hooks */ + +static inline bool is_inst_link_info(VkLayerInstanceCreateInfo *lici) +{ + return lici->sType == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO && + lici->function == VK_LAYER_LINK_INFO; +} + +static VkResult VKAPI OBS_CreateInstance(const VkInstanceCreateInfo *cinfo, + const VkAllocationCallbacks *ac, + VkInstance *p_inst) +{ + VkInstanceCreateInfo info = *cinfo; + bool funcs_not_found = false; + + /* -------------------------------------------------------- */ + /* step through chain until we get to the link info */ + + VkLayerInstanceCreateInfo *lici = (void *)info.pNext; + while (lici && !is_inst_link_info(lici)) { + lici = (VkLayerInstanceCreateInfo *)lici->pNext; + } + + if (lici == NULL) { + return VK_ERROR_INITIALIZATION_FAILED; + } + + PFN_vkGetInstanceProcAddr gpa = + lici->u.pLayerInfo->pfnNextGetInstanceProcAddr; + + /* -------------------------------------------------------- */ + /* move chain on for next layer */ + + lici->u.pLayerInfo = lici->u.pLayerInfo->pNext; + + /* -------------------------------------------------------- */ + /* (HACK) Set api version to 1.1 if set to 1.0 */ + /* We do this to get our extensions working properly */ + + VkApplicationInfo ai = *info.pApplicationInfo; + if (ai.apiVersion < VK_API_VERSION_1_1) { + info.pApplicationInfo = &ai; + ai.apiVersion = VK_API_VERSION_1_1; + } + + /* -------------------------------------------------------- */ + /* create instance */ + + PFN_vkCreateInstance create = (void *)gpa(NULL, "vkCreateInstance"); + + VkResult res = create(&info, ac, p_inst); + VkInstance inst = *p_inst; + + /* -------------------------------------------------------- */ + /* fetch the functions we need */ + + struct vk_inst_data *data = get_inst_data(inst); + struct vk_inst_funcs *funcs = &data->funcs; + +#define GETADDR(x) \ + do { \ + funcs->x = (void *)gpa(inst, "vk" #x); \ + if (!funcs->x) { \ + flog("could not get instance " \ + "address for %s", \ + #x); \ + funcs_not_found = true; \ + } \ + } while (false) + + GETADDR(GetInstanceProcAddr); + GETADDR(DestroyInstance); + GETADDR(CreateWin32SurfaceKHR); + GETADDR(GetPhysicalDeviceMemoryProperties); + GETADDR(GetPhysicalDeviceImageFormatProperties2); +#undef GETADDR + + data->valid = !funcs_not_found; + return res; +} + +static VkResult VKAPI OBS_DestroyInstance(VkInstance instance, + const VkAllocationCallbacks *ac) +{ + struct vk_inst_funcs *funcs = get_inst_funcs(instance); + funcs->DestroyInstance(instance, ac); + remove_instance(instance); + return VK_SUCCESS; +} + +static bool +vk_shared_tex_supported(struct vk_inst_funcs *funcs, + VkPhysicalDevice phy_device, VkFormat format, + VkImageUsageFlags usage, + VkExternalMemoryProperties *external_mem_props) +{ + VkPhysicalDeviceImageFormatInfo2 info; + VkPhysicalDeviceExternalImageFormatInfo external_info; + + external_info.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO; + external_info.pNext = NULL; + external_info.handleType = + VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT; + + info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2; + info.pNext = &external_info; + info.format = format; + info.type = VK_IMAGE_TYPE_2D; + info.tiling = VK_IMAGE_TILING_OPTIMAL; + info.flags = 0; + info.usage = usage; + + VkExternalImageFormatProperties external_props = {0}; + external_props.sType = + VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES; + external_props.pNext = NULL; + + VkImageFormatProperties2 props = {0}; + props.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2; + props.pNext = &external_props; + + VkResult result = funcs->GetPhysicalDeviceImageFormatProperties2( + phy_device, &info, &props); + + *external_mem_props = external_props.externalMemoryProperties; + + const VkExternalMemoryFeatureFlags features = + external_mem_props->externalMemoryFeatures; + + return ((VK_SUCCESS == result) && + (features & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT)); +} + +static inline bool is_device_link_info(VkLayerDeviceCreateInfo *lici) +{ + return lici->sType == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO && + lici->function == VK_LAYER_LINK_INFO; +} + +static VkResult VKAPI OBS_CreateDevice(VkPhysicalDevice phy_device, + const VkDeviceCreateInfo *cinfo, + const VkAllocationCallbacks *ac, + VkDevice *p_device) +{ + VkDeviceCreateInfo info = *cinfo; + struct vk_inst_data *idata = get_inst_data(phy_device); + struct vk_inst_funcs *ifuncs = &idata->funcs; + struct vk_data *data = NULL; + + VkResult ret = VK_ERROR_INITIALIZATION_FAILED; + + VkLayerDeviceCreateInfo *ldci = (void *)info.pNext; + + /* -------------------------------------------------------- */ + /* step through chain until we get to the link info */ + + while (ldci && !is_device_link_info(ldci)) { + ldci = (VkLayerDeviceCreateInfo *)ldci->pNext; + } + + if (!ldci) { + goto fail; + } + + PFN_vkGetInstanceProcAddr gipa; + PFN_vkGetDeviceProcAddr gdpa; + + gipa = ldci->u.pLayerInfo->pfnNextGetInstanceProcAddr; + gdpa = ldci->u.pLayerInfo->pfnNextGetDeviceProcAddr; + + /* -------------------------------------------------------- */ + /* move chain on for next layer */ + + ldci->u.pLayerInfo = ldci->u.pLayerInfo->pNext; + + /* -------------------------------------------------------- */ + /* create device and initialize hook data */ + + PFN_vkCreateDevice createFunc = + (PFN_vkCreateDevice)gipa(VK_NULL_HANDLE, "vkCreateDevice"); + + ret = createFunc(phy_device, idata->valid ? &info : cinfo, ac, + p_device); + if (ret != VK_SUCCESS) { + goto fail; + } + + VkDevice device = *p_device; + + data = get_device_data(*p_device); + struct vk_device_funcs *dfuncs = &data->funcs; + + data->valid = false; /* set true below if it doesn't go to fail */ + data->phy_device = phy_device; + data->device = device; + + /* -------------------------------------------------------- */ + /* fetch the functions we need */ + + bool funcs_not_found = false; + +#define GETADDR(x) \ + do { \ + dfuncs->x = (void *)gdpa(device, "vk" #x); \ + if (!dfuncs->x) { \ + flog("could not get device " \ + "address for %s", \ + #x); \ + funcs_not_found = true; \ + } \ + } while (false) + +#define GETADDR_OPTIONAL(x) \ + do { \ + dfuncs->x = (void *)gdpa(device, "vk" #x); \ + } while (false) + + GETADDR(GetDeviceProcAddr); + GETADDR(DestroyDevice); + GETADDR(CreateSwapchainKHR); + GETADDR(DestroySwapchainKHR); + GETADDR(QueuePresentKHR); + GETADDR(AllocateMemory); + GETADDR(FreeMemory); + GETADDR(BindImageMemory); + GETADDR(BindImageMemory2); + GETADDR(GetSwapchainImagesKHR); + GETADDR(CreateImage); + GETADDR(DestroyImage); + GETADDR(GetImageMemoryRequirements); + GETADDR(GetImageMemoryRequirements2); + GETADDR(BeginCommandBuffer); + GETADDR(EndCommandBuffer); + GETADDR(CmdCopyImage); + GETADDR(CmdPipelineBarrier); + GETADDR(GetDeviceQueue); + GETADDR(QueueSubmit); + GETADDR(CreateCommandPool); + GETADDR(DestroyCommandPool); + GETADDR(AllocateCommandBuffers); + GETADDR(CreateFence); + GETADDR(DestroyFence); + GETADDR(WaitForFences); + GETADDR(ResetFences); +#undef GETADDR_OPTIONAL +#undef GETADDR + + if (funcs_not_found) { + goto fail; + } + if (!idata->valid) { + goto fail; + } + + VkFormat format = VK_FORMAT_R8G8B8A8_UNORM; + VkImageUsageFlags usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | + VK_IMAGE_USAGE_TRANSFER_DST_BIT; + + if (!vk_shared_tex_supported(ifuncs, phy_device, format, usage, + &data->external_mem_props)) { + flog("texture sharing is not supported"); + goto fail; + } + + data->valid = true; + +fail: + return ret; +} + +static void VKAPI OBS_DestroyDevice(VkDevice device, + const VkAllocationCallbacks *ac) +{ + struct vk_data *data = get_device_data(device); + if (!data) + return; + + for (uint32_t fam_idx = 0; fam_idx < _countof(data->cmd_pools); + fam_idx++) { + struct vk_cmd_pool_data *pool_data = &data->cmd_pools[fam_idx]; + if (pool_data->cmd_pool != VK_NULL_HANDLE) { + vk_shtex_destroy_cmd_pool_objects(data, pool_data); + } + } + + data->queue_count = 0; + + vk_remove_device(device); + data->funcs.DestroyDevice(device, ac); +} + +static VkResult VKAPI +OBS_CreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR *cinfo, + const VkAllocationCallbacks *ac, VkSwapchainKHR *p_sc) +{ + struct vk_data *data = get_device_data(device); + struct vk_device_funcs *funcs = &data->funcs; + + struct vk_swap_data *swap = get_new_swap_data(data); + + swap->surf = cinfo->surface; + swap->image_extent = cinfo->imageExtent; + swap->format = cinfo->imageFormat; + + VkSwapchainCreateInfoKHR info = *cinfo; + info.imageUsage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + + VkResult res = funcs->CreateSwapchainKHR(device, &info, ac, p_sc); + VkSwapchainKHR sc = *p_sc; + + uint32_t count = 0; + res = funcs->GetSwapchainImagesKHR(data->device, sc, &count, NULL); + debug_res("GetSwapchainImagesKHR", res); + + if (count > 0) { + if (count > OBJ_MAX) + count = OBJ_MAX; + + res = funcs->GetSwapchainImagesKHR(data->device, sc, &count, + swap->swap_images); + debug_res("GetSwapchainImagesKHR", res); + swap->image_count = count; + } + + swap->sc = sc; + return res; +} + +static void VKAPI OBS_DestroySwapchainKHR(VkDevice device, VkSwapchainKHR sc, + const VkAllocationCallbacks *ac) +{ + struct vk_data *data = get_device_data(device); + struct vk_device_funcs *funcs = &data->funcs; + + struct vk_swap_data *swap = get_swap_data(data, sc); + if (swap) { + if (data->cur_swap == swap) { + vk_shtex_free(data); + } + + swap->sc = VK_NULL_HANDLE; + swap->surf = NULL; + } + + funcs->DestroySwapchainKHR(device, sc, ac); +} + +static void VKAPI OBS_GetDeviceQueue(VkDevice device, uint32_t queueFamilyIndex, + uint32_t queueIndex, VkQueue *pQueue) +{ + struct vk_data *data = get_device_data(device); + struct vk_device_funcs *funcs = &data->funcs; + + funcs->GetDeviceQueue(device, queueFamilyIndex, queueIndex, pQueue); + + for (uint32_t i = 0; i < data->queue_count; ++i) { + if (data->queues[i].queue == *pQueue) + return; + } + + if (data->queue_count < _countof(data->queues)) { + data->queues[data->queue_count].queue = *pQueue; + data->queues[data->queue_count].fam_idx = queueFamilyIndex; + ++data->queue_count; + } +} + +static VkResult VKAPI OBS_CreateWin32SurfaceKHR( + VkInstance inst, const VkWin32SurfaceCreateInfoKHR *info, + const VkAllocationCallbacks *ac, VkSurfaceKHR *surf) +{ + struct vk_inst_data *data = get_inst_data(inst); + struct vk_inst_funcs *funcs = &data->funcs; + + VkResult res = funcs->CreateWin32SurfaceKHR(inst, info, ac, surf); + if (NULL != surf && VK_NULL_HANDLE != *surf) { + struct vk_surf_data *surf_data = find_surf_data(data, *surf); + + surf_data->hinstance = info->hinstance; + surf_data->hwnd = info->hwnd; + } + return res; +} + +#define GETPROCADDR(func) \ + if (!strcmp(name, "vk" #func)) \ + return (VkFunc)&OBS_##func; + +static VkFunc VKAPI OBS_GetDeviceProcAddr(VkDevice dev, const char *name) +{ + struct vk_data *data = get_device_data(dev); + struct vk_device_funcs *funcs = &data->funcs; + + debug_procaddr("vkGetDeviceProcAddr(%p, \"%s\")", dev, name); + + GETPROCADDR(GetDeviceProcAddr); + GETPROCADDR(CreateDevice); + GETPROCADDR(DestroyDevice); + GETPROCADDR(CreateSwapchainKHR); + GETPROCADDR(DestroySwapchainKHR); + GETPROCADDR(QueuePresentKHR); + GETPROCADDR(GetDeviceQueue); + + if (funcs->GetDeviceProcAddr == NULL) + return NULL; + return funcs->GetDeviceProcAddr(dev, name); +} + +static VkFunc VKAPI OBS_GetInstanceProcAddr(VkInstance inst, const char *name) +{ + debug_procaddr("vkGetInstanceProcAddr(%p, \"%s\")", inst, name); + + /* instance chain functions we intercept */ + GETPROCADDR(GetInstanceProcAddr); + GETPROCADDR(CreateInstance); + GETPROCADDR(DestroyInstance); + GETPROCADDR(CreateWin32SurfaceKHR); + + /* device chain functions we intercept */ + GETPROCADDR(GetDeviceProcAddr); + GETPROCADDR(CreateDevice); + GETPROCADDR(DestroyDevice); + + struct vk_inst_funcs *funcs = get_inst_funcs(inst); + if (funcs->GetInstanceProcAddr == NULL) + return NULL; + return funcs->GetInstanceProcAddr(inst, name); +} + +#undef GETPROCADDR + +EXPORT VkResult VKAPI OBS_Negotiate(VkNegotiateLayerInterface *nli) +{ + if (nli->loaderLayerInterfaceVersion >= 2) { + nli->sType = LAYER_NEGOTIATE_INTERFACE_STRUCT; + nli->pNext = NULL; + nli->pfnGetInstanceProcAddr = OBS_GetInstanceProcAddr; + nli->pfnGetDeviceProcAddr = OBS_GetDeviceProcAddr; + nli->pfnGetPhysicalDeviceProcAddr = NULL; + } + + const uint32_t cur_ver = CURRENT_LOADER_LAYER_INTERFACE_VERSION; + + if (nli->loaderLayerInterfaceVersion > cur_ver) { + nli->loaderLayerInterfaceVersion = cur_ver; + } + + return VK_SUCCESS; +} + +bool hook_vulkan(void) +{ + static bool hooked = false; + if (!hooked && vulkan_seen) { + hlog("Hooked Vulkan"); + hooked = true; + } + return hooked; +} + +static bool vulkan_initialized = false; + +bool init_vk_layer() +{ + if (!vulkan_initialized) { + InitializeCriticalSection(&mutex); + vulkan_initialized = true; + } + return true; +} + +bool shutdown_vk_layer() +{ + if (vulkan_initialized) { + DeleteCriticalSection(&mutex); + } + return true; +} diff --git a/plugins/win-capture/graphics-hook/vulkan-capture.h b/plugins/win-capture/graphics-hook/vulkan-capture.h new file mode 100644 index 000000000..da2444951 --- /dev/null +++ b/plugins/win-capture/graphics-hook/vulkan-capture.h @@ -0,0 +1,874 @@ +#pragma once + +#define DEF_FUNC(x) PFN_vk##x x + +struct vk_inst_funcs { + DEF_FUNC(GetInstanceProcAddr); + DEF_FUNC(DestroyInstance); + DEF_FUNC(CreateWin32SurfaceKHR); + DEF_FUNC(GetPhysicalDeviceMemoryProperties); + DEF_FUNC(GetPhysicalDeviceImageFormatProperties2); +}; + +struct vk_device_funcs { + DEF_FUNC(GetDeviceProcAddr); + DEF_FUNC(DestroyDevice); + DEF_FUNC(CreateSwapchainKHR); + DEF_FUNC(DestroySwapchainKHR); + DEF_FUNC(QueuePresentKHR); + DEF_FUNC(AllocateMemory); + DEF_FUNC(FreeMemory); + DEF_FUNC(BindImageMemory); + DEF_FUNC(BindImageMemory2); + DEF_FUNC(GetSwapchainImagesKHR); + DEF_FUNC(CreateImage); + DEF_FUNC(DestroyImage); + DEF_FUNC(GetImageMemoryRequirements); + DEF_FUNC(GetImageMemoryRequirements2); + DEF_FUNC(BeginCommandBuffer); + DEF_FUNC(EndCommandBuffer); + DEF_FUNC(CmdCopyImage); + DEF_FUNC(CmdPipelineBarrier); + DEF_FUNC(GetDeviceQueue); + DEF_FUNC(QueueSubmit); + DEF_FUNC(CreateCommandPool); + DEF_FUNC(DestroyCommandPool); + DEF_FUNC(AllocateCommandBuffers); + DEF_FUNC(CreateFence); + DEF_FUNC(DestroyFence); + DEF_FUNC(WaitForFences); + DEF_FUNC(ResetFences); +}; + +#undef DEF_FUNC + +const char *vk_format_to_str(VkFormat format) +{ + switch (format) { + default: +#define VAL(x) \ + case x: \ + return #x + + VAL(VK_FORMAT_UNDEFINED); + VAL(VK_FORMAT_R4G4_UNORM_PACK8); + VAL(VK_FORMAT_R4G4B4A4_UNORM_PACK16); + VAL(VK_FORMAT_B4G4R4A4_UNORM_PACK16); + VAL(VK_FORMAT_R5G6B5_UNORM_PACK16); + VAL(VK_FORMAT_B5G6R5_UNORM_PACK16); + VAL(VK_FORMAT_R5G5B5A1_UNORM_PACK16); + VAL(VK_FORMAT_B5G5R5A1_UNORM_PACK16); + VAL(VK_FORMAT_A1R5G5B5_UNORM_PACK16); + VAL(VK_FORMAT_R8_UNORM); + VAL(VK_FORMAT_R8_SNORM); + VAL(VK_FORMAT_R8_USCALED); + VAL(VK_FORMAT_R8_SSCALED); + VAL(VK_FORMAT_R8_UINT); + VAL(VK_FORMAT_R8_SINT); + VAL(VK_FORMAT_R8_SRGB); + VAL(VK_FORMAT_R8G8_UNORM); + VAL(VK_FORMAT_R8G8_SNORM); + VAL(VK_FORMAT_R8G8_USCALED); + VAL(VK_FORMAT_R8G8_SSCALED); + VAL(VK_FORMAT_R8G8_UINT); + VAL(VK_FORMAT_R8G8_SINT); + VAL(VK_FORMAT_R8G8_SRGB); + VAL(VK_FORMAT_R8G8B8_UNORM); + VAL(VK_FORMAT_R8G8B8_SNORM); + VAL(VK_FORMAT_R8G8B8_USCALED); + VAL(VK_FORMAT_R8G8B8_SSCALED); + VAL(VK_FORMAT_R8G8B8_UINT); + VAL(VK_FORMAT_R8G8B8_SINT); + VAL(VK_FORMAT_R8G8B8_SRGB); + VAL(VK_FORMAT_B8G8R8_UNORM); + VAL(VK_FORMAT_B8G8R8_SNORM); + VAL(VK_FORMAT_B8G8R8_USCALED); + VAL(VK_FORMAT_B8G8R8_SSCALED); + VAL(VK_FORMAT_B8G8R8_UINT); + VAL(VK_FORMAT_B8G8R8_SINT); + VAL(VK_FORMAT_B8G8R8_SRGB); + VAL(VK_FORMAT_R8G8B8A8_UNORM); + VAL(VK_FORMAT_R8G8B8A8_SNORM); + VAL(VK_FORMAT_R8G8B8A8_USCALED); + VAL(VK_FORMAT_R8G8B8A8_SSCALED); + VAL(VK_FORMAT_R8G8B8A8_UINT); + VAL(VK_FORMAT_R8G8B8A8_SINT); + VAL(VK_FORMAT_R8G8B8A8_SRGB); /* dota 2 */ + VAL(VK_FORMAT_B8G8R8A8_UNORM); + VAL(VK_FORMAT_B8G8R8A8_SNORM); + VAL(VK_FORMAT_B8G8R8A8_USCALED); + VAL(VK_FORMAT_B8G8R8A8_SSCALED); + VAL(VK_FORMAT_B8G8R8A8_UINT); + VAL(VK_FORMAT_B8G8R8A8_SINT); + VAL(VK_FORMAT_B8G8R8A8_SRGB); + VAL(VK_FORMAT_A8B8G8R8_UNORM_PACK32); + VAL(VK_FORMAT_A8B8G8R8_SNORM_PACK32); + VAL(VK_FORMAT_A8B8G8R8_USCALED_PACK32); + VAL(VK_FORMAT_A8B8G8R8_SSCALED_PACK32); + VAL(VK_FORMAT_A8B8G8R8_UINT_PACK32); + VAL(VK_FORMAT_A8B8G8R8_SINT_PACK32); + VAL(VK_FORMAT_A8B8G8R8_SRGB_PACK32); + VAL(VK_FORMAT_A2R10G10B10_UNORM_PACK32); + VAL(VK_FORMAT_A2R10G10B10_SNORM_PACK32); + VAL(VK_FORMAT_A2R10G10B10_USCALED_PACK32); + VAL(VK_FORMAT_A2R10G10B10_SSCALED_PACK32); + VAL(VK_FORMAT_A2R10G10B10_UINT_PACK32); + VAL(VK_FORMAT_A2R10G10B10_SINT_PACK32); + VAL(VK_FORMAT_A2B10G10R10_UNORM_PACK32); + VAL(VK_FORMAT_A2B10G10R10_SNORM_PACK32); + VAL(VK_FORMAT_A2B10G10R10_USCALED_PACK32); + VAL(VK_FORMAT_A2B10G10R10_SSCALED_PACK32); + VAL(VK_FORMAT_A2B10G10R10_UINT_PACK32); + VAL(VK_FORMAT_A2B10G10R10_SINT_PACK32); + VAL(VK_FORMAT_R16_UNORM); + VAL(VK_FORMAT_R16_SNORM); + VAL(VK_FORMAT_R16_USCALED); + VAL(VK_FORMAT_R16_SSCALED); + VAL(VK_FORMAT_R16_UINT); + VAL(VK_FORMAT_R16_SINT); + VAL(VK_FORMAT_R16_SFLOAT); + VAL(VK_FORMAT_R16G16_UNORM); + VAL(VK_FORMAT_R16G16_SNORM); + VAL(VK_FORMAT_R16G16_USCALED); + VAL(VK_FORMAT_R16G16_SSCALED); + VAL(VK_FORMAT_R16G16_UINT); + VAL(VK_FORMAT_R16G16_SINT); + VAL(VK_FORMAT_R16G16_SFLOAT); + VAL(VK_FORMAT_R16G16B16_UNORM); + VAL(VK_FORMAT_R16G16B16_SNORM); + VAL(VK_FORMAT_R16G16B16_USCALED); + VAL(VK_FORMAT_R16G16B16_SSCALED); + VAL(VK_FORMAT_R16G16B16_UINT); + VAL(VK_FORMAT_R16G16B16_SINT); + VAL(VK_FORMAT_R16G16B16_SFLOAT); + VAL(VK_FORMAT_R16G16B16A16_UNORM); + VAL(VK_FORMAT_R16G16B16A16_SNORM); + VAL(VK_FORMAT_R16G16B16A16_USCALED); + VAL(VK_FORMAT_R16G16B16A16_SSCALED); + VAL(VK_FORMAT_R16G16B16A16_UINT); + VAL(VK_FORMAT_R16G16B16A16_SINT); + VAL(VK_FORMAT_R16G16B16A16_SFLOAT); + VAL(VK_FORMAT_R32_UINT); + VAL(VK_FORMAT_R32_SINT); + VAL(VK_FORMAT_R32_SFLOAT); + VAL(VK_FORMAT_R32G32_UINT); + VAL(VK_FORMAT_R32G32_SINT); + VAL(VK_FORMAT_R32G32_SFLOAT); + VAL(VK_FORMAT_R32G32B32_UINT); + VAL(VK_FORMAT_R32G32B32_SINT); + VAL(VK_FORMAT_R32G32B32_SFLOAT); + VAL(VK_FORMAT_R32G32B32A32_UINT); + VAL(VK_FORMAT_R32G32B32A32_SINT); + VAL(VK_FORMAT_R32G32B32A32_SFLOAT); + VAL(VK_FORMAT_R64_UINT); + VAL(VK_FORMAT_R64_SINT); + VAL(VK_FORMAT_R64_SFLOAT); + VAL(VK_FORMAT_R64G64_UINT); + VAL(VK_FORMAT_R64G64_SINT); + VAL(VK_FORMAT_R64G64_SFLOAT); + VAL(VK_FORMAT_R64G64B64_UINT); + VAL(VK_FORMAT_R64G64B64_SINT); + VAL(VK_FORMAT_R64G64B64_SFLOAT); + VAL(VK_FORMAT_R64G64B64A64_UINT); + VAL(VK_FORMAT_R64G64B64A64_SINT); + VAL(VK_FORMAT_R64G64B64A64_SFLOAT); + VAL(VK_FORMAT_B10G11R11_UFLOAT_PACK32); + VAL(VK_FORMAT_E5B9G9R9_UFLOAT_PACK32); + VAL(VK_FORMAT_D16_UNORM); + VAL(VK_FORMAT_X8_D24_UNORM_PACK32); + VAL(VK_FORMAT_D32_SFLOAT); + VAL(VK_FORMAT_S8_UINT); + VAL(VK_FORMAT_D16_UNORM_S8_UINT); + VAL(VK_FORMAT_D24_UNORM_S8_UINT); + VAL(VK_FORMAT_D32_SFLOAT_S8_UINT); + VAL(VK_FORMAT_BC1_RGB_UNORM_BLOCK); + VAL(VK_FORMAT_BC1_RGB_SRGB_BLOCK); + VAL(VK_FORMAT_BC1_RGBA_UNORM_BLOCK); + VAL(VK_FORMAT_BC1_RGBA_SRGB_BLOCK); + VAL(VK_FORMAT_BC2_UNORM_BLOCK); + VAL(VK_FORMAT_BC2_SRGB_BLOCK); + VAL(VK_FORMAT_BC3_UNORM_BLOCK); + VAL(VK_FORMAT_BC3_SRGB_BLOCK); + VAL(VK_FORMAT_BC4_UNORM_BLOCK); + VAL(VK_FORMAT_BC4_SNORM_BLOCK); + VAL(VK_FORMAT_BC5_UNORM_BLOCK); + VAL(VK_FORMAT_BC5_SNORM_BLOCK); + VAL(VK_FORMAT_BC6H_UFLOAT_BLOCK); + VAL(VK_FORMAT_BC6H_SFLOAT_BLOCK); + VAL(VK_FORMAT_BC7_UNORM_BLOCK); + VAL(VK_FORMAT_BC7_SRGB_BLOCK); + VAL(VK_FORMAT_ETC2_R8G8B8_UNORM_BLOCK); + VAL(VK_FORMAT_ETC2_R8G8B8_SRGB_BLOCK); + VAL(VK_FORMAT_ETC2_R8G8B8A1_UNORM_BLOCK); + VAL(VK_FORMAT_ETC2_R8G8B8A1_SRGB_BLOCK); + VAL(VK_FORMAT_ETC2_R8G8B8A8_UNORM_BLOCK); + VAL(VK_FORMAT_ETC2_R8G8B8A8_SRGB_BLOCK); + VAL(VK_FORMAT_EAC_R11_UNORM_BLOCK); + VAL(VK_FORMAT_EAC_R11_SNORM_BLOCK); + VAL(VK_FORMAT_EAC_R11G11_UNORM_BLOCK); + VAL(VK_FORMAT_EAC_R11G11_SNORM_BLOCK); + VAL(VK_FORMAT_ASTC_4x4_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_4x4_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_5x4_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_5x4_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_5x5_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_5x5_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_6x5_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_6x5_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_6x6_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_6x6_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_8x5_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_8x5_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_8x6_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_8x6_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_8x8_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_8x8_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_10x5_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_10x5_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_10x6_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_10x6_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_10x8_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_10x8_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_10x10_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_10x10_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_12x10_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_12x10_SRGB_BLOCK); + VAL(VK_FORMAT_ASTC_12x12_UNORM_BLOCK); + VAL(VK_FORMAT_ASTC_12x12_SRGB_BLOCK); + VAL(VK_FORMAT_G8B8G8R8_422_UNORM); + VAL(VK_FORMAT_B8G8R8G8_422_UNORM); + VAL(VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM); + VAL(VK_FORMAT_G8_B8R8_2PLANE_420_UNORM); + VAL(VK_FORMAT_G8_B8_R8_3PLANE_422_UNORM); + VAL(VK_FORMAT_G8_B8R8_2PLANE_422_UNORM); + VAL(VK_FORMAT_G8_B8_R8_3PLANE_444_UNORM); + VAL(VK_FORMAT_R10X6_UNORM_PACK16); + VAL(VK_FORMAT_R10X6G10X6_UNORM_2PACK16); + VAL(VK_FORMAT_R10X6G10X6B10X6A10X6_UNORM_4PACK16); + VAL(VK_FORMAT_G10X6B10X6G10X6R10X6_422_UNORM_4PACK16); + VAL(VK_FORMAT_B10X6G10X6R10X6G10X6_422_UNORM_4PACK16); + VAL(VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_420_UNORM_3PACK16); + VAL(VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16); + VAL(VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_422_UNORM_3PACK16); + VAL(VK_FORMAT_G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16); + VAL(VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_444_UNORM_3PACK16); + VAL(VK_FORMAT_R12X4_UNORM_PACK16); + VAL(VK_FORMAT_R12X4G12X4_UNORM_2PACK16); + VAL(VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16); + VAL(VK_FORMAT_G12X4B12X4G12X4R12X4_422_UNORM_4PACK16); + VAL(VK_FORMAT_B12X4G12X4R12X4G12X4_422_UNORM_4PACK16); + VAL(VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_420_UNORM_3PACK16); + VAL(VK_FORMAT_G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16); + VAL(VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_422_UNORM_3PACK16); + VAL(VK_FORMAT_G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16); + VAL(VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_444_UNORM_3PACK16); + VAL(VK_FORMAT_G16B16G16R16_422_UNORM); + VAL(VK_FORMAT_B16G16R16G16_422_UNORM); + VAL(VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM); + VAL(VK_FORMAT_G16_B16R16_2PLANE_420_UNORM); + VAL(VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM); + VAL(VK_FORMAT_G16_B16R16_2PLANE_422_UNORM); + VAL(VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM); + VAL(VK_FORMAT_PVRTC1_2BPP_UNORM_BLOCK_IMG); + VAL(VK_FORMAT_PVRTC1_4BPP_UNORM_BLOCK_IMG); + VAL(VK_FORMAT_PVRTC2_2BPP_UNORM_BLOCK_IMG); + VAL(VK_FORMAT_PVRTC2_4BPP_UNORM_BLOCK_IMG); + VAL(VK_FORMAT_PVRTC1_2BPP_SRGB_BLOCK_IMG); + VAL(VK_FORMAT_PVRTC1_4BPP_SRGB_BLOCK_IMG); + VAL(VK_FORMAT_PVRTC2_2BPP_SRGB_BLOCK_IMG); + VAL(VK_FORMAT_PVRTC2_4BPP_SRGB_BLOCK_IMG); + } +} + +const char *result_to_str(VkResult result) +{ + switch (result) { + VAL(VK_SUCCESS); + VAL(VK_NOT_READY); + VAL(VK_TIMEOUT); + VAL(VK_EVENT_SET); + VAL(VK_EVENT_RESET); + VAL(VK_INCOMPLETE); + VAL(VK_ERROR_OUT_OF_HOST_MEMORY); + VAL(VK_ERROR_OUT_OF_DEVICE_MEMORY); + VAL(VK_ERROR_INITIALIZATION_FAILED); + VAL(VK_ERROR_DEVICE_LOST); + VAL(VK_ERROR_MEMORY_MAP_FAILED); + VAL(VK_ERROR_LAYER_NOT_PRESENT); + VAL(VK_ERROR_EXTENSION_NOT_PRESENT); + VAL(VK_ERROR_FEATURE_NOT_PRESENT); + VAL(VK_ERROR_INCOMPATIBLE_DRIVER); + VAL(VK_ERROR_TOO_MANY_OBJECTS); + VAL(VK_ERROR_FORMAT_NOT_SUPPORTED); + VAL(VK_ERROR_FRAGMENTED_POOL); + VAL(VK_ERROR_OUT_OF_POOL_MEMORY); + VAL(VK_ERROR_INVALID_EXTERNAL_HANDLE); + VAL(VK_ERROR_SURFACE_LOST_KHR); + VAL(VK_ERROR_NATIVE_WINDOW_IN_USE_KHR); + VAL(VK_SUBOPTIMAL_KHR); + VAL(VK_ERROR_OUT_OF_DATE_KHR); + VAL(VK_ERROR_INCOMPATIBLE_DISPLAY_KHR); + VAL(VK_ERROR_VALIDATION_FAILED_EXT); + VAL(VK_ERROR_INVALID_SHADER_NV); + VAL(VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT); + VAL(VK_ERROR_FRAGMENTATION_EXT); + VAL(VK_ERROR_NOT_PERMITTED_EXT); + VAL(VK_ERROR_INVALID_DEVICE_ADDRESS_EXT); + VAL(VK_ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT); + //VAL(VK_ERROR_OUT_OF_POOL_MEMORY_KHR); + //VAL(VK_ERROR_INVALID_EXTERNAL_HANDLE_KHR); + //VAL(VK_RESULT_BEGIN_RANGE); + //VAL(VK_RESULT_END_RANGE); + VAL(VK_RESULT_RANGE_SIZE); + VAL(VK_RESULT_MAX_ENUM); +#undef VAL + + default: + return "Unknown VkResult"; + break; + } +} + +DXGI_FORMAT vk_format_to_dxgi(VkFormat format) +{ + //this is not a real format matching ! + //ex: we need to avoid stacking srg + DXGI_FORMAT dxgi_format = DXGI_FORMAT_UNKNOWN; + switch (format) { + default: + case VK_FORMAT_UNDEFINED: + break; + case VK_FORMAT_R4G4_UNORM_PACK8: + break; + case VK_FORMAT_R4G4B4A4_UNORM_PACK16: + break; + case VK_FORMAT_B4G4R4A4_UNORM_PACK16: + dxgi_format = DXGI_FORMAT_B4G4R4A4_UNORM; + break; + case VK_FORMAT_R5G6B5_UNORM_PACK16: + break; + case VK_FORMAT_B5G6R5_UNORM_PACK16: + dxgi_format = DXGI_FORMAT_B5G6R5_UNORM; + break; + case VK_FORMAT_R5G5B5A1_UNORM_PACK16: + break; + case VK_FORMAT_B5G5R5A1_UNORM_PACK16: + dxgi_format = DXGI_FORMAT_B5G5R5A1_UNORM; + break; + case VK_FORMAT_A1R5G5B5_UNORM_PACK16: + break; + case VK_FORMAT_R8_UNORM: + dxgi_format = DXGI_FORMAT_R8_UNORM; + break; + case VK_FORMAT_R8_SNORM: + dxgi_format = DXGI_FORMAT_R8_SNORM; + break; + case VK_FORMAT_R8_USCALED: + break; + case VK_FORMAT_R8_SSCALED: + break; + case VK_FORMAT_R8_UINT: + dxgi_format = DXGI_FORMAT_R8_UINT; + break; + case VK_FORMAT_R8_SINT: + dxgi_format = DXGI_FORMAT_R8_SINT; + break; + case VK_FORMAT_R8_SRGB: + break; + case VK_FORMAT_R8G8_UNORM: + dxgi_format = DXGI_FORMAT_R8G8_UNORM; + break; + case VK_FORMAT_R8G8_SNORM: + dxgi_format = DXGI_FORMAT_R8G8_SNORM; + break; + case VK_FORMAT_R8G8_USCALED: + break; + case VK_FORMAT_R8G8_SSCALED: + break; + case VK_FORMAT_R8G8_UINT: + dxgi_format = DXGI_FORMAT_R8G8_UINT; + break; + case VK_FORMAT_R8G8_SINT: + dxgi_format = DXGI_FORMAT_R8G8_UINT; + break; + case VK_FORMAT_R8G8_SRGB: + break; + case VK_FORMAT_R8G8B8_UNORM: + break; + case VK_FORMAT_R8G8B8_SNORM: + break; + case VK_FORMAT_R8G8B8_USCALED: + break; + case VK_FORMAT_R8G8B8_SSCALED: + break; + case VK_FORMAT_R8G8B8_UINT: + break; + case VK_FORMAT_R8G8B8_SINT: + break; + case VK_FORMAT_R8G8B8_SRGB: + break; + case VK_FORMAT_B8G8R8_UNORM: + break; + case VK_FORMAT_B8G8R8_SNORM: + break; + case VK_FORMAT_B8G8R8_USCALED: + break; + case VK_FORMAT_B8G8R8_SSCALED: + break; + case VK_FORMAT_B8G8R8_UINT: + break; + case VK_FORMAT_B8G8R8_SINT: + break; + case VK_FORMAT_B8G8R8_SRGB: + break; + case VK_FORMAT_R8G8B8A8_UNORM: + dxgi_format = DXGI_FORMAT_R8G8B8A8_UNORM; + break; + case VK_FORMAT_R8G8B8A8_SNORM: + dxgi_format = DXGI_FORMAT_R8G8B8A8_SNORM; + break; + case VK_FORMAT_R8G8B8A8_USCALED: + break; + case VK_FORMAT_R8G8B8A8_SSCALED: + break; + case VK_FORMAT_R8G8B8A8_UINT: + dxgi_format = DXGI_FORMAT_R8G8B8A8_UINT; + break; + case VK_FORMAT_R8G8B8A8_SINT: + dxgi_format = DXGI_FORMAT_R8G8B8A8_SINT; + break; + case VK_FORMAT_R8G8B8A8_SRGB: + dxgi_format = DXGI_FORMAT_R8G8B8A8_UNORM; + break; //dota2 + case VK_FORMAT_B8G8R8A8_UNORM: + dxgi_format = DXGI_FORMAT_B8G8R8A8_UNORM; + break; + case VK_FORMAT_B8G8R8A8_SNORM: + break; + case VK_FORMAT_B8G8R8A8_USCALED: + break; + case VK_FORMAT_B8G8R8A8_SSCALED: + break; + case VK_FORMAT_B8G8R8A8_UINT: + break; + case VK_FORMAT_B8G8R8A8_SINT: + break; + case VK_FORMAT_B8G8R8A8_SRGB: + dxgi_format = DXGI_FORMAT_B8G8R8A8_UNORM; + break; + case VK_FORMAT_A8B8G8R8_UNORM_PACK32: + break; + case VK_FORMAT_A8B8G8R8_SNORM_PACK32: + break; + case VK_FORMAT_A8B8G8R8_USCALED_PACK32: + break; + case VK_FORMAT_A8B8G8R8_SSCALED_PACK32: + break; + case VK_FORMAT_A8B8G8R8_UINT_PACK32: + break; + case VK_FORMAT_A8B8G8R8_SINT_PACK32: + break; + case VK_FORMAT_A8B8G8R8_SRGB_PACK32: + break; + case VK_FORMAT_A2R10G10B10_UNORM_PACK32: + dxgi_format = DXGI_FORMAT_R10G10B10A2_UNORM; + break; + case VK_FORMAT_A2R10G10B10_SNORM_PACK32: + break; + case VK_FORMAT_A2R10G10B10_USCALED_PACK32: + break; + case VK_FORMAT_A2R10G10B10_SSCALED_PACK32: + break; + case VK_FORMAT_A2R10G10B10_UINT_PACK32: + dxgi_format = DXGI_FORMAT_R10G10B10A2_UINT; + break; + case VK_FORMAT_A2R10G10B10_SINT_PACK32: + break; + case VK_FORMAT_A2B10G10R10_UNORM_PACK32: + dxgi_format = DXGI_FORMAT_R10G10B10A2_UNORM; + break; //no man sky + case VK_FORMAT_A2B10G10R10_SNORM_PACK32: + break; + case VK_FORMAT_A2B10G10R10_USCALED_PACK32: + break; + case VK_FORMAT_A2B10G10R10_SSCALED_PACK32: + break; + case VK_FORMAT_A2B10G10R10_UINT_PACK32: + break; + case VK_FORMAT_A2B10G10R10_SINT_PACK32: + break; + case VK_FORMAT_R16_UNORM: + dxgi_format = DXGI_FORMAT_R16_UNORM; + break; + case VK_FORMAT_R16_SNORM: + dxgi_format = DXGI_FORMAT_R16_SNORM; + break; + case VK_FORMAT_R16_USCALED: + break; + case VK_FORMAT_R16_SSCALED: + break; + case VK_FORMAT_R16_UINT: + dxgi_format = DXGI_FORMAT_R16_UINT; + break; + case VK_FORMAT_R16_SINT: + dxgi_format = DXGI_FORMAT_R16_SINT; + break; + case VK_FORMAT_R16_SFLOAT: + dxgi_format = DXGI_FORMAT_R16_FLOAT; + break; + case VK_FORMAT_R16G16_UNORM: + dxgi_format = DXGI_FORMAT_R16G16_UNORM; + break; + case VK_FORMAT_R16G16_SNORM: + dxgi_format = DXGI_FORMAT_R16G16_SNORM; + break; + case VK_FORMAT_R16G16_USCALED: + break; + case VK_FORMAT_R16G16_SSCALED: + break; + case VK_FORMAT_R16G16_UINT: + dxgi_format = DXGI_FORMAT_R16G16_UINT; + break; + case VK_FORMAT_R16G16_SINT: + dxgi_format = DXGI_FORMAT_R16G16_SINT; + break; + case VK_FORMAT_R16G16_SFLOAT: + dxgi_format = DXGI_FORMAT_R16G16_FLOAT; + break; + case VK_FORMAT_R16G16B16_UNORM: + break; + case VK_FORMAT_R16G16B16_SNORM: + break; + case VK_FORMAT_R16G16B16_USCALED: + break; + case VK_FORMAT_R16G16B16_SSCALED: + break; + case VK_FORMAT_R16G16B16_UINT: + break; + case VK_FORMAT_R16G16B16_SINT: + break; + case VK_FORMAT_R16G16B16_SFLOAT: + break; + case VK_FORMAT_R16G16B16A16_UNORM: + dxgi_format = DXGI_FORMAT_R16G16B16A16_UNORM; + break; + case VK_FORMAT_R16G16B16A16_SNORM: + dxgi_format = DXGI_FORMAT_R16G16B16A16_SNORM; + break; + case VK_FORMAT_R16G16B16A16_USCALED: + break; + case VK_FORMAT_R16G16B16A16_SSCALED: + break; + case VK_FORMAT_R16G16B16A16_UINT: + dxgi_format = DXGI_FORMAT_R16G16B16A16_UINT; + break; + case VK_FORMAT_R16G16B16A16_SINT: + dxgi_format = DXGI_FORMAT_R16G16B16A16_SINT; + break; + case VK_FORMAT_R16G16B16A16_SFLOAT: + dxgi_format = DXGI_FORMAT_R16G16B16A16_FLOAT; + break; + case VK_FORMAT_R32_UINT: + dxgi_format = DXGI_FORMAT_R32_UINT; + break; + case VK_FORMAT_R32_SINT: + dxgi_format = DXGI_FORMAT_R32_SINT; + break; + case VK_FORMAT_R32_SFLOAT: + dxgi_format = DXGI_FORMAT_R32_FLOAT; + break; + case VK_FORMAT_R32G32_UINT: + dxgi_format = DXGI_FORMAT_R32G32_UINT; + break; + case VK_FORMAT_R32G32_SINT: + dxgi_format = DXGI_FORMAT_R32G32_SINT; + break; + case VK_FORMAT_R32G32_SFLOAT: + dxgi_format = DXGI_FORMAT_R32G32_FLOAT; + break; + case VK_FORMAT_R32G32B32_UINT: + dxgi_format = DXGI_FORMAT_R32G32B32_UINT; + break; + case VK_FORMAT_R32G32B32_SINT: + dxgi_format = DXGI_FORMAT_R32G32B32_SINT; + break; + case VK_FORMAT_R32G32B32_SFLOAT: + dxgi_format = DXGI_FORMAT_R32G32B32_FLOAT; + break; + case VK_FORMAT_R32G32B32A32_UINT: + dxgi_format = DXGI_FORMAT_R32G32B32A32_UINT; + break; + case VK_FORMAT_R32G32B32A32_SINT: + dxgi_format = DXGI_FORMAT_R32G32B32A32_SINT; + break; + case VK_FORMAT_R32G32B32A32_SFLOAT: + dxgi_format = DXGI_FORMAT_R32G32B32A32_FLOAT; + break; + case VK_FORMAT_R64_UINT: + break; + case VK_FORMAT_R64_SINT: + break; + case VK_FORMAT_R64_SFLOAT: + break; + case VK_FORMAT_R64G64_UINT: + break; + case VK_FORMAT_R64G64_SINT: + break; + case VK_FORMAT_R64G64_SFLOAT: + break; + case VK_FORMAT_R64G64B64_UINT: + break; + case VK_FORMAT_R64G64B64_SINT: + break; + case VK_FORMAT_R64G64B64_SFLOAT: + break; + case VK_FORMAT_R64G64B64A64_UINT: + break; + case VK_FORMAT_R64G64B64A64_SINT: + break; + case VK_FORMAT_R64G64B64A64_SFLOAT: + break; + case VK_FORMAT_B10G11R11_UFLOAT_PACK32: + break; + case VK_FORMAT_E5B9G9R9_UFLOAT_PACK32: + break; + case VK_FORMAT_D16_UNORM: + break; + case VK_FORMAT_X8_D24_UNORM_PACK32: + break; + case VK_FORMAT_D32_SFLOAT: + break; + case VK_FORMAT_S8_UINT: + break; + case VK_FORMAT_D16_UNORM_S8_UINT: + break; + case VK_FORMAT_D24_UNORM_S8_UINT: + break; + case VK_FORMAT_D32_SFLOAT_S8_UINT: + break; + case VK_FORMAT_BC1_RGB_UNORM_BLOCK: + break; + case VK_FORMAT_BC1_RGB_SRGB_BLOCK: + break; + case VK_FORMAT_BC1_RGBA_UNORM_BLOCK: + break; + case VK_FORMAT_BC1_RGBA_SRGB_BLOCK: + break; + case VK_FORMAT_BC2_UNORM_BLOCK: + break; + case VK_FORMAT_BC2_SRGB_BLOCK: + break; + case VK_FORMAT_BC3_UNORM_BLOCK: + break; + case VK_FORMAT_BC3_SRGB_BLOCK: + break; + case VK_FORMAT_BC4_UNORM_BLOCK: + break; + case VK_FORMAT_BC4_SNORM_BLOCK: + break; + case VK_FORMAT_BC5_UNORM_BLOCK: + break; + case VK_FORMAT_BC5_SNORM_BLOCK: + break; + case VK_FORMAT_BC6H_UFLOAT_BLOCK: + break; + case VK_FORMAT_BC6H_SFLOAT_BLOCK: + break; + case VK_FORMAT_BC7_UNORM_BLOCK: + break; + case VK_FORMAT_BC7_SRGB_BLOCK: + break; + case VK_FORMAT_ETC2_R8G8B8_UNORM_BLOCK: + break; + case VK_FORMAT_ETC2_R8G8B8_SRGB_BLOCK: + break; + case VK_FORMAT_ETC2_R8G8B8A1_UNORM_BLOCK: + break; + case VK_FORMAT_ETC2_R8G8B8A1_SRGB_BLOCK: + break; + case VK_FORMAT_ETC2_R8G8B8A8_UNORM_BLOCK: + break; + case VK_FORMAT_ETC2_R8G8B8A8_SRGB_BLOCK: + break; + case VK_FORMAT_EAC_R11_UNORM_BLOCK: + break; + case VK_FORMAT_EAC_R11_SNORM_BLOCK: + break; + case VK_FORMAT_EAC_R11G11_UNORM_BLOCK: + break; + case VK_FORMAT_EAC_R11G11_SNORM_BLOCK: + break; + case VK_FORMAT_ASTC_4x4_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_4x4_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_5x4_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_5x4_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_5x5_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_5x5_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_6x5_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_6x5_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_6x6_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_6x6_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_8x5_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_8x5_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_8x6_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_8x6_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_8x8_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_8x8_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_10x5_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_10x5_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_10x6_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_10x6_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_10x8_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_10x8_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_10x10_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_10x10_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_12x10_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_12x10_SRGB_BLOCK: + break; + case VK_FORMAT_ASTC_12x12_UNORM_BLOCK: + break; + case VK_FORMAT_ASTC_12x12_SRGB_BLOCK: + break; + case VK_FORMAT_G8B8G8R8_422_UNORM: + break; + case VK_FORMAT_B8G8R8G8_422_UNORM: + break; + case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM: + break; + case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM: + break; + case VK_FORMAT_G8_B8_R8_3PLANE_422_UNORM: + break; + case VK_FORMAT_G8_B8R8_2PLANE_422_UNORM: + break; + case VK_FORMAT_G8_B8_R8_3PLANE_444_UNORM: + break; + case VK_FORMAT_R10X6_UNORM_PACK16: + break; + case VK_FORMAT_R10X6G10X6_UNORM_2PACK16: + break; + case VK_FORMAT_R10X6G10X6B10X6A10X6_UNORM_4PACK16: + break; + case VK_FORMAT_G10X6B10X6G10X6R10X6_422_UNORM_4PACK16: + break; + case VK_FORMAT_B10X6G10X6R10X6G10X6_422_UNORM_4PACK16: + break; + case VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_420_UNORM_3PACK16: + break; + case VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16: + break; + case VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_422_UNORM_3PACK16: + break; + case VK_FORMAT_G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16: + break; + case VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_444_UNORM_3PACK16: + break; + case VK_FORMAT_R12X4_UNORM_PACK16: + break; + case VK_FORMAT_R12X4G12X4_UNORM_2PACK16: + break; + case VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16: + break; + case VK_FORMAT_G12X4B12X4G12X4R12X4_422_UNORM_4PACK16: + break; + case VK_FORMAT_B12X4G12X4R12X4G12X4_422_UNORM_4PACK16: + break; + case VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_420_UNORM_3PACK16: + break; + case VK_FORMAT_G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16: + break; + case VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_422_UNORM_3PACK16: + break; + case VK_FORMAT_G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16: + break; + case VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_444_UNORM_3PACK16: + break; + case VK_FORMAT_G16B16G16R16_422_UNORM: + break; + case VK_FORMAT_B16G16R16G16_422_UNORM: + break; + case VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM: + break; + case VK_FORMAT_G16_B16R16_2PLANE_420_UNORM: + break; + case VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM: + break; + case VK_FORMAT_G16_B16R16_2PLANE_422_UNORM: + break; + case VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM: + break; + case VK_FORMAT_PVRTC1_2BPP_UNORM_BLOCK_IMG: + break; + case VK_FORMAT_PVRTC1_4BPP_UNORM_BLOCK_IMG: + break; + case VK_FORMAT_PVRTC2_2BPP_UNORM_BLOCK_IMG: + break; + case VK_FORMAT_PVRTC2_4BPP_UNORM_BLOCK_IMG: + break; + case VK_FORMAT_PVRTC1_2BPP_SRGB_BLOCK_IMG: + break; + case VK_FORMAT_PVRTC1_4BPP_SRGB_BLOCK_IMG: + break; + case VK_FORMAT_PVRTC2_2BPP_SRGB_BLOCK_IMG: + break; + case VK_FORMAT_PVRTC2_4BPP_SRGB_BLOCK_IMG: + break; + } + if (dxgi_format == DXGI_FORMAT_UNKNOWN) { + flog("unknown swapchain format, " + "defaulting to B8G8R8A8_UNORM"); + dxgi_format = DXGI_FORMAT_B8G8R8A8_UNORM; + } + return dxgi_format; +} + +//#define DEBUG_PRINT +//#define DEBUG_PRINT_PROCADDR + +#ifdef DEBUG_PRINT +#include +#define debug(format, ...) \ + do { \ + char str[256]; \ + snprintf(str, sizeof(str) - 1, "%s " format "\n", \ + "[OBS graphics-hook]", ##__VA_ARGS__); \ + OutputDebugStringA(str); \ + } while (false) + +#define debug_res(x, y) debug("%s result: %s", x, result_to_str(y)) + +#else +#define debug(x, ...) +#define debug_res(x, y) +#endif + +#ifdef DEBUG_PRINT_PROCADDR +#define debug_procaddr(format, ...) debug(format, ##__VA_ARGS__) +#else +#define debug_procaddr(format, ...) +#endif diff --git a/plugins/win-capture/plugin-main.c b/plugins/win-capture/plugin-main.c index b382e8fd6..748ac2caa 100644 --- a/plugins/win-capture/plugin-main.c +++ b/plugins/win-capture/plugin-main.c @@ -65,6 +65,8 @@ void wait_for_hook_initialization(void) } } +void init_hook_files(void); + bool obs_module_load(void) { struct win_version_info ver; @@ -94,6 +96,7 @@ bool obs_module_load(void) char *config_path = obs_module_config_path(NULL); + init_hook_files(); init_hooks_thread = CreateThread(NULL, 0, init_hooks, config_path, 0, NULL); obs_register_source(&game_capture_info); From 7e78c17ace4fd4d5ae58166af9fbfb45cb7123ad Mon Sep 17 00:00:00 2001 From: jp9000 Date: Fri, 28 Feb 2020 23:24:18 -0800 Subject: [PATCH 5/5] win-capture: Check hook version before capture init Checks the hook version to ensure compatibility with hook DLL. It's unlikely it'll ever be necessary to increment the hook version, but this is just a precautionary thing that allows a hook DLL to make sure it's rejected by an older OBS version if needed. Again however, very unlikely that the major version will ever be incremented. --- plugins/win-capture/game-capture.c | 12 ++++++++++++ plugins/win-capture/graphics-hook-info.h | 6 ++++++ plugins/win-capture/graphics-hook/graphics-hook.c | 5 +++++ 3 files changed, 23 insertions(+) diff --git a/plugins/win-capture/game-capture.c b/plugins/win-capture/game-capture.c index 6ca146d9d..5530e5629 100644 --- a/plugins/win-capture/game-capture.c +++ b/plugins/win-capture/game-capture.c @@ -10,6 +10,7 @@ #include "obfuscate.h" #include "inject-library.h" #include "graphics-hook-info.h" +#include "graphics-hook-ver.h" #include "window-helpers.h" #include "cursor-capture.h" #include "app-helpers.h" @@ -1610,6 +1611,17 @@ static bool start_capture(struct game_capture *gc) { debug("Starting capture"); + /* prevent from using a DLL version that's higher than current */ + if (gc->global_hook_info->hook_ver_major > HOOK_VER_MAJOR) { + warn("cannot initialize hook, DLL hook version is " + "%" PRIu32 ".%" PRIu32 + ", current plugin hook major version is %d.%d", + gc->global_hook_info->hook_ver_major, + gc->global_hook_info->hook_ver_minor, HOOK_VER_MAJOR, + HOOK_VER_MINOR); + return false; + } + if (gc->global_hook_info->type == CAPTURE_TYPE_MEMORY) { if (!init_shmem_capture(gc)) { return false; diff --git a/plugins/win-capture/graphics-hook-info.h b/plugins/win-capture/graphics-hook-info.h index 04546e489..17f93574b 100644 --- a/plugins/win-capture/graphics-hook-info.h +++ b/plugins/win-capture/graphics-hook-info.h @@ -80,6 +80,10 @@ struct graphics_offsets { }; struct hook_info { + /* hook version */ + uint32_t hook_ver_major; + uint32_t hook_ver_minor; + /* capture info */ enum capture_type type; uint32_t window; @@ -101,6 +105,8 @@ struct hook_info { /* hook addresses */ struct graphics_offsets offsets; + + uint32_t reserved[128]; }; #pragma pack(pop) diff --git a/plugins/win-capture/graphics-hook/graphics-hook.c b/plugins/win-capture/graphics-hook/graphics-hook.c index abe789131..2dc83a80c 100644 --- a/plugins/win-capture/graphics-hook/graphics-hook.c +++ b/plugins/win-capture/graphics-hook/graphics-hook.c @@ -1,6 +1,7 @@ #include #include #include "graphics-hook.h" +#include "../graphics-hook-ver.h" #include "../obfuscate.h" #include "../funchook.h" @@ -539,6 +540,8 @@ bool capture_init_shtex(struct shtex_data **data, HWND window, uint32_t base_cx, *data = shmem_info; (*data)->tex_handle = (uint32_t)handle; + global_hook_info->hook_ver_major = HOOK_VER_MAJOR; + global_hook_info->hook_ver_minor = HOOK_VER_MINOR; global_hook_info->window = (uint32_t)(uintptr_t)window; global_hook_info->type = CAPTURE_TYPE_TEXTURE; global_hook_info->format = format; @@ -732,6 +735,8 @@ bool capture_init_shmem(struct shmem_data **data, HWND window, uint32_t base_cx, (*data)->tex1_offset = (uint32_t)align_pos; (*data)->tex2_offset = (*data)->tex1_offset + aligned_tex; + global_hook_info->hook_ver_major = HOOK_VER_MAJOR; + global_hook_info->hook_ver_minor = HOOK_VER_MINOR; global_hook_info->window = (uint32_t)(uintptr_t)window; global_hook_info->type = CAPTURE_TYPE_MEMORY; global_hook_info->format = format;