From 9f15514c1aad4869eb4412cb7f94e107b2219d0a Mon Sep 17 00:00:00 2001 From: jpark37 Date: Fri, 13 Mar 2020 04:36:19 -0700 Subject: [PATCH] win-capture: Vulkan surface refactor Make sure HWND tracking is cleaned up when Vulkan surfaces are destroyed. Also use unbounded linked list to fix games that leak surfaces on Alt+Tab like Doom. Also replace CRITICAL_SECTION with SRWLOCK, both for claimed speed benefit, and to remove initialization code. --- plugins/win-capture/graphics-hook-ver.h | 2 +- .../win-capture/graphics-hook/graphics-hook.c | 12 +- .../graphics-hook/vulkan-capture.c | 157 ++++++++++-------- .../graphics-hook/vulkan-capture.h | 1 + 4 files changed, 87 insertions(+), 85 deletions(-) diff --git a/plugins/win-capture/graphics-hook-ver.h b/plugins/win-capture/graphics-hook-ver.h index 809d6a6d2..4ac3b2d68 100644 --- a/plugins/win-capture/graphics-hook-ver.h +++ b/plugins/win-capture/graphics-hook-ver.h @@ -13,7 +13,7 @@ #define HOOK_VER_MAJOR 1 #define HOOK_VER_MINOR 1 -#define HOOK_VER_PATCH 2 +#define HOOK_VER_PATCH 3 #define STRINGIFY(s) #s #define MAKE_VERSION_NAME(major, minor, patch) \ diff --git a/plugins/win-capture/graphics-hook/graphics-hook.c b/plugins/win-capture/graphics-hook/graphics-hook.c index 6855951d3..e6171b15b 100644 --- a/plugins/win-capture/graphics-hook/graphics-hook.c +++ b/plugins/win-capture/graphics-hook/graphics-hook.c @@ -805,9 +805,6 @@ void capture_free(void) active = false; } -BOOL init_vk_layer(); -BOOL shutdown_vk_layer(); - #define HOOK_NAME L"graphics_hook_dup_mutex" static inline HANDLE open_mutex_plus_id(const wchar_t *name, DWORD id) @@ -865,11 +862,7 @@ BOOL WINAPI DllMain(HINSTANCE hinst, DWORD reason, LPVOID unused1) if (!init_mutexes()) { return false; } -#if COMPILE_VULKAN_HOOK - if (!init_vk_layer()) { - return false; - } -#endif + /* this prevents the library from being automatically unloaded * by the next FreeLibrary call */ GetModuleFileNameW(hinst, name, MAX_PATH); @@ -894,9 +887,6 @@ BOOL WINAPI DllMain(HINSTANCE hinst, DWORD reason, LPVOID unused1) CloseHandle(capture_thread); } -#if COMPILE_VULKAN_HOOK - shutdown_vk_layer(); -#endif free_hook(); } diff --git a/plugins/win-capture/graphics-hook/vulkan-capture.c b/plugins/win-capture/graphics-hook/vulkan-capture.c index 5a71f9983..62e98505d 100644 --- a/plugins/win-capture/graphics-hook/vulkan-capture.c +++ b/plugins/win-capture/graphics-hook/vulkan-capture.c @@ -43,7 +43,7 @@ static const GUID dxgi_resource_guid = /* clang-format on */ static bool vulkan_seen = false; -static CRITICAL_SECTION mutex; +static SRWLOCK mutex = SRWLOCK_INIT; // Faster CRITICAL_SECTION /* ======================================================================== */ /* hook data */ @@ -52,7 +52,7 @@ struct vk_swap_data { VkSwapchainKHR sc; VkExtent2D image_extent; VkFormat format; - VkSurfaceKHR surf; + HWND hwnd; VkImage export_image; bool layout_initialized; VkDeviceMemory export_mem; @@ -94,6 +94,8 @@ struct vk_data { struct vk_cmd_pool_data cmd_pools[OBJ_MAX]; VkExternalMemoryProperties external_mem_props; + struct vk_inst_data *inst_data; + ID3D11Device *d3d11_device; ID3D11DeviceContext *d3d11_context; }; @@ -114,8 +116,7 @@ static struct vk_swap_data *get_swap_data(struct vk_data *data, static struct vk_swap_data *get_new_swap_data(struct vk_data *data) { for (int i = 0; i < OBJ_MAX; i++) { - if (data->swaps[i].surf == VK_NULL_HANDLE && - data->swaps[i].sc == VK_NULL_HANDLE) { + if (data->swaps[i].sc == VK_NULL_HANDLE) { return &data->swaps[i]; } } @@ -130,14 +131,14 @@ static inline size_t find_obj_idx(void *objs[], void *obj) { size_t idx = SIZE_MAX; - EnterCriticalSection(&mutex); + AcquireSRWLockExclusive(&mutex); for (size_t i = 0; i < OBJ_MAX; i++) { if (objs[i] == obj) { idx = i; break; } } - LeaveCriticalSection(&mutex); + ReleaseSRWLockExclusive(&mutex); return idx; } @@ -146,7 +147,7 @@ static size_t get_obj_idx(void *objs[], void *obj) { size_t idx = SIZE_MAX; - EnterCriticalSection(&mutex); + AcquireSRWLockExclusive(&mutex); for (size_t i = 0; i < OBJ_MAX; i++) { if (objs[i] == obj) { idx = i; @@ -156,7 +157,7 @@ static size_t get_obj_idx(void *objs[], void *obj) idx = i; } } - LeaveCriticalSection(&mutex); + ReleaseSRWLockExclusive(&mutex); return idx; } @@ -263,45 +264,79 @@ static void vk_remove_device(void *dev) memset(data, 0, sizeof(*data)); - EnterCriticalSection(&mutex); + AcquireSRWLockExclusive(&mutex); devices[idx] = NULL; - LeaveCriticalSection(&mutex); + ReleaseSRWLockExclusive(&mutex); } /* ------------------------------------------------------------------------- */ struct vk_surf_data { VkSurfaceKHR surf; - HINSTANCE hinstance; HWND hwnd; + struct vk_surf_data *next; }; struct vk_inst_data { bool valid; struct vk_inst_funcs funcs; - struct vk_surf_data surfaces[OBJ_MAX]; + struct vk_surf_data *surfaces; }; -static struct vk_surf_data *find_surf_data(struct vk_inst_data *data, - VkSurfaceKHR surf) +static void insert_surf_data(struct vk_inst_data *data, VkSurfaceKHR surf, + HWND hwnd) { - int idx = OBJ_MAX; - for (int i = 0; i < OBJ_MAX; i++) { - if (data->surfaces[i].surf == surf) { - return &data->surfaces[i]; - } else if (data->surfaces[i].surf == VK_NULL_HANDLE && - idx == OBJ_MAX) { - idx = i; - } - } - if (idx != OBJ_MAX) { - data->surfaces[idx].surf = surf; - return &data->surfaces[idx]; - } + struct vk_surf_data *surf_data = malloc(sizeof(struct vk_surf_data)); + if (surf_data) { + surf_data->surf = surf; + surf_data->hwnd = hwnd; - debug("find_surf_data failed, no more free slots"); - return NULL; + AcquireSRWLockExclusive(&mutex); + struct vk_surf_data *next = data->surfaces; + surf_data->next = next; + data->surfaces = surf_data; + ReleaseSRWLockExclusive(&mutex); + } +} + +static HWND find_surf_hwnd(struct vk_inst_data *data, VkSurfaceKHR surf) +{ + HWND hwnd = NULL; + + AcquireSRWLockExclusive(&mutex); + struct vk_surf_data *surf_data = data->surfaces; + while (surf_data) { + if (surf_data->surf == surf) { + hwnd = surf_data->hwnd; + break; + } + surf_data = surf_data->next; + } + ReleaseSRWLockExclusive(&mutex); + + return hwnd; +} + +static void erase_surf_data(struct vk_inst_data *data, VkSurfaceKHR surf) +{ + AcquireSRWLockExclusive(&mutex); + struct vk_surf_data *current = data->surfaces; + if (current->surf == surf) { + data->surfaces = current->next; + } else { + struct vk_surf_data *previous; + do { + previous = current; + current = current->next; + } while (current && current->surf != surf); + + if (current) + previous->next = current->next; + } + ReleaseSRWLockExclusive(&mutex); + + free(current); } /* ------------------------------------------------------------------------- */ @@ -337,9 +372,9 @@ static void remove_instance(void *inst) struct vk_inst_data *data = &inst_data[idx]; memset(data, 0, sizeof(*data)); - EnterCriticalSection(&mutex); + AcquireSRWLockExclusive(&mutex); instances[idx] = NULL; - LeaveCriticalSection(&mutex); + ReleaseSRWLockExclusive(&mutex); } /* ======================================================================== */ @@ -921,20 +956,6 @@ static void vk_shtex_capture(struct vk_data *data, pool_data->cmd_buffer_busy[image_index] = true; } -static inline HWND get_swap_window(struct vk_swap_data *swap) -{ - for (size_t i = 0; i < OBJ_MAX; i++) { - struct vk_surf_data *surf_data = - find_surf_data(&inst_data[i], swap->surf); - - if (!!surf_data && surf_data->surf == swap->surf) { - return surf_data->hwnd; - } - } - - return NULL; -} - static inline bool valid_rect(struct vk_swap_data *swap) { return !!swap->image_extent.width && !!swap->image_extent.height; @@ -955,7 +976,7 @@ static void vk_capture(struct vk_data *data, VkQueue queue, for (; idx < info->swapchainCount; idx++) { struct vk_swap_data *cur_swap = get_swap_data(data, info->pSwapchains[idx]); - window = get_swap_window(cur_swap); + window = cur_swap->hwnd; if (!!window) { swap = cur_swap; break; @@ -1083,6 +1104,7 @@ static VkResult VKAPI OBS_CreateInstance(const VkInstanceCreateInfo *cinfo, GETADDR(GetInstanceProcAddr); GETADDR(DestroyInstance); GETADDR(CreateWin32SurfaceKHR); + GETADDR(DestroySurfaceKHR); GETADDR(GetPhysicalDeviceMemoryProperties); GETADDR(GetPhysicalDeviceImageFormatProperties2); #undef GETADDR @@ -1276,6 +1298,7 @@ static VkResult VKAPI OBS_CreateDevice(VkPhysicalDevice phy_device, goto fail; } + data->inst_data = idata; data->valid = true; fail: @@ -1336,7 +1359,7 @@ OBS_CreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR *cinfo, swap->sc = sc; swap->image_extent = cinfo->imageExtent; swap->format = cinfo->imageFormat; - swap->surf = cinfo->surface; + swap->hwnd = find_surf_hwnd(data->inst_data, cinfo->surface); swap->image_count = count; return VK_SUCCESS; @@ -1355,7 +1378,7 @@ static void VKAPI OBS_DestroySwapchainKHR(VkDevice device, VkSwapchainKHR sc, } swap->sc = VK_NULL_HANDLE; - swap->surf = VK_NULL_HANDLE; + swap->hwnd = NULL; } funcs->DestroySwapchainKHR(device, sc, ac); @@ -1389,15 +1412,21 @@ static VkResult VKAPI OBS_CreateWin32SurfaceKHR( struct vk_inst_funcs *funcs = &data->funcs; VkResult res = funcs->CreateWin32SurfaceKHR(inst, info, ac, surf); - if (NULL != surf && VK_NULL_HANDLE != *surf) { - struct vk_surf_data *surf_data = find_surf_data(data, *surf); - - surf_data->hinstance = info->hinstance; - surf_data->hwnd = info->hwnd; - } + if (res == VK_SUCCESS) + insert_surf_data(data, *surf, info->hwnd); return res; } +static void VKAPI OBS_DestroySurfaceKHR(VkInstance inst, VkSurfaceKHR surf, + const VkAllocationCallbacks *ac) +{ + struct vk_inst_data *data = get_inst_data(inst); + struct vk_inst_funcs *funcs = &data->funcs; + + erase_surf_data(data, surf); + funcs->DestroySurfaceKHR(inst, surf, ac); +} + #define GETPROCADDR(func) \ if (!strcmp(name, "vk" #func)) \ return (VkFunc)&OBS_##func; @@ -1431,6 +1460,7 @@ static VkFunc VKAPI OBS_GetInstanceProcAddr(VkInstance inst, const char *name) GETPROCADDR(CreateInstance); GETPROCADDR(DestroyInstance); GETPROCADDR(CreateWin32SurfaceKHR); + GETPROCADDR(DestroySurfaceKHR); /* device chain functions we intercept */ GETPROCADDR(GetDeviceProcAddr); @@ -1473,22 +1503,3 @@ bool hook_vulkan(void) } return hooked; } - -static bool vulkan_initialized = false; - -bool init_vk_layer() -{ - if (!vulkan_initialized) { - InitializeCriticalSection(&mutex); - vulkan_initialized = true; - } - return true; -} - -bool shutdown_vk_layer() -{ - if (vulkan_initialized) { - DeleteCriticalSection(&mutex); - } - return true; -} diff --git a/plugins/win-capture/graphics-hook/vulkan-capture.h b/plugins/win-capture/graphics-hook/vulkan-capture.h index a51327869..986ec186a 100644 --- a/plugins/win-capture/graphics-hook/vulkan-capture.h +++ b/plugins/win-capture/graphics-hook/vulkan-capture.h @@ -6,6 +6,7 @@ struct vk_inst_funcs { DEF_FUNC(GetInstanceProcAddr); DEF_FUNC(DestroyInstance); DEF_FUNC(CreateWin32SurfaceKHR); + DEF_FUNC(DestroySurfaceKHR); DEF_FUNC(GetPhysicalDeviceMemoryProperties); DEF_FUNC(GetPhysicalDeviceImageFormatProperties2); };