From dad861b036849f569c54cd111e71b0d18a7b7229 Mon Sep 17 00:00:00 2001 From: jpark37 Date: Wed, 9 Sep 2020 22:35:11 -0700 Subject: [PATCH] win-capture: Fix D3D leaks on swap chain release For game capture, hook DXGI release function to release D3D objects if the related swap chain is also being destroyed. An added bonus is that the game capture hook will handle swap chain recreation for applications that don't use ResizeBuffers. --- .../get-graphics-offsets/dxgi-offsets.cpp | 5 +++- .../get-graphics-offsets.c | 4 ++- .../get-graphics-offsets.h | 3 +- plugins/win-capture/graphics-hook-info.h | 9 +++++- .../graphics-hook/dxgi-capture.cpp | 28 +++++++++++++++++++ plugins/win-capture/load-graphics-offsets.c | 2 ++ 6 files changed, 47 insertions(+), 4 deletions(-) diff --git a/plugins/win-capture/get-graphics-offsets/dxgi-offsets.cpp b/plugins/win-capture/get-graphics-offsets/dxgi-offsets.cpp index 87dbdff43..dc48293b5 100644 --- a/plugins/win-capture/get-graphics-offsets/dxgi-offsets.cpp +++ b/plugins/win-capture/get-graphics-offsets/dxgi-offsets.cpp @@ -102,7 +102,8 @@ static inline void dxgi_free(dxgi_info &info) DestroyWindow(info.hwnd); } -void get_dxgi_offsets(struct dxgi_offsets *offsets) +void get_dxgi_offsets(struct dxgi_offsets *offsets, + struct dxgi_offsets2 *offsets2) { dxgi_info info = {}; bool success = dxgi_init(info); @@ -120,6 +121,8 @@ void get_dxgi_offsets(struct dxgi_offsets *offsets) vtable_offset(info.module, swap1, 22); swap1->Release(); } + + offsets2->release = vtable_offset(info.module, info.swap, 2); } dxgi_free(info); diff --git a/plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c b/plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c index 6dfb5ba3d..a813617d9 100644 --- a/plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c +++ b/plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c @@ -8,6 +8,7 @@ int main(int argc, char *argv[]) struct d3d8_offsets d3d8 = {0}; struct d3d9_offsets d3d9 = {0}; struct dxgi_offsets dxgi = {0}; + struct dxgi_offsets2 dxgi2 = {0}; WNDCLASSA wc = {0}; wc.style = CS_OWNDC; @@ -24,7 +25,7 @@ int main(int argc, char *argv[]) get_d3d9_offsets(&d3d9); get_d3d8_offsets(&d3d8); - get_dxgi_offsets(&dxgi); + get_dxgi_offsets(&dxgi, &dxgi2); printf("[d3d8]\n"); printf("present=0x%" PRIx32 "\n", d3d8.present); @@ -38,6 +39,7 @@ int main(int argc, char *argv[]) printf("present=0x%" PRIx32 "\n", dxgi.present); printf("present1=0x%" PRIx32 "\n", dxgi.present1); printf("resize=0x%" PRIx32 "\n", dxgi.resize); + printf("release=0x%" PRIx32 "\n", dxgi2.release); (void)argc; (void)argv; diff --git a/plugins/win-capture/get-graphics-offsets/get-graphics-offsets.h b/plugins/win-capture/get-graphics-offsets/get-graphics-offsets.h index 7b0275fdf..53a6ee407 100644 --- a/plugins/win-capture/get-graphics-offsets/get-graphics-offsets.h +++ b/plugins/win-capture/get-graphics-offsets/get-graphics-offsets.h @@ -21,7 +21,8 @@ static inline uint32_t vtable_offset(HMODULE module, void *cls, return (uint32_t)(vtable[offset] - (uintptr_t)module); } -extern void get_dxgi_offsets(struct dxgi_offsets *offsets); +extern void get_dxgi_offsets(struct dxgi_offsets *offsets, + struct dxgi_offsets2 *offsets2); extern void get_d3d9_offsets(struct d3d9_offsets *offsets); extern void get_d3d8_offsets(struct d3d8_offsets *offsets); diff --git a/plugins/win-capture/graphics-hook-info.h b/plugins/win-capture/graphics-hook-info.h index 9e741105b..af43cf4e6 100644 --- a/plugins/win-capture/graphics-hook-info.h +++ b/plugins/win-capture/graphics-hook-info.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -46,6 +47,10 @@ struct dxgi_offsets { uint32_t present1; }; +struct dxgi_offsets2 { + uint32_t release; +}; + struct ddraw_offsets { uint32_t surface_create; uint32_t surface_restore; @@ -77,6 +82,7 @@ struct graphics_offsets { struct d3d9_offsets d3d9; struct dxgi_offsets dxgi; struct ddraw_offsets ddraw; + struct dxgi_offsets2 dxgi2; }; struct hook_info { @@ -106,8 +112,9 @@ struct hook_info { /* hook addresses */ struct graphics_offsets offsets; - uint32_t reserved[128]; + uint32_t reserved[127]; }; +static_assert(sizeof(struct hook_info) == 648, "ABI compatibility"); #pragma pack(pop) diff --git a/plugins/win-capture/graphics-hook/dxgi-capture.cpp b/plugins/win-capture/graphics-hook/dxgi-capture.cpp index b5c82f6c4..98bec3669 100644 --- a/plugins/win-capture/graphics-hook/dxgi-capture.cpp +++ b/plugins/win-capture/graphics-hook/dxgi-capture.cpp @@ -10,6 +10,7 @@ #include #endif +typedef ULONG(STDMETHODCALLTYPE *release_t)(IUnknown *); typedef HRESULT(STDMETHODCALLTYPE *resize_buffers_t)(IDXGISwapChain *, UINT, UINT, UINT, DXGI_FORMAT, UINT); @@ -17,6 +18,7 @@ typedef HRESULT(STDMETHODCALLTYPE *present_t)(IDXGISwapChain *, UINT, UINT); typedef HRESULT(STDMETHODCALLTYPE *present1_t)(IDXGISwapChain1 *, UINT, UINT, const DXGI_PRESENT_PARAMETERS *); +static struct func_hook release; static struct func_hook resize_buffers; static struct func_hook present; static struct func_hook present1; @@ -80,6 +82,23 @@ static bool setup_dxgi(IDXGISwapChain *swap) return false; } +static ULONG STDMETHODCALLTYPE hook_release(IUnknown *unknown) +{ + unhook(&release); + release_t call = (release_t)release.call_addr; + ULONG refs = call(unknown); + rehook(&release); + + if (unknown == data.swap && refs == 0) { + data.free(); + data.swap = nullptr; + data.free = nullptr; + data.capture = nullptr; + } + + return refs; +} + static bool resize_buffers_called = false; static HRESULT STDMETHODCALLTYPE hook_resize_buffers(IDXGISwapChain *swap, @@ -223,6 +242,7 @@ bool hook_dxgi(void) void *present_addr; void *resize_addr; void *present1_addr = nullptr; + void *release_addr = nullptr; if (!dxgi_module) { return false; @@ -237,6 +257,9 @@ bool hook_dxgi(void) if (global_hook_info->offsets.dxgi.present1) present1_addr = get_offset_addr( dxgi_module, global_hook_info->offsets.dxgi.present1); + if (global_hook_info->offsets.dxgi2.release) + release_addr = get_offset_addr( + dxgi_module, global_hook_info->offsets.dxgi2.release); hook_init(&present, present_addr, (void *)hook_present, "IDXGISwapChain::Present"); @@ -245,11 +268,16 @@ bool hook_dxgi(void) if (present1_addr) hook_init(&present1, present1_addr, (void *)hook_present1, "IDXGISwapChain1::Present1"); + if (release_addr) + hook_init(&release, release_addr, (void *)hook_release, + "IDXGISwapChain::Release"); rehook(&resize_buffers); rehook(&present); if (present1_addr) rehook(&present1); + if (release_addr) + rehook(&release); hlog("Hooked DXGI"); return true; diff --git a/plugins/win-capture/load-graphics-offsets.c b/plugins/win-capture/load-graphics-offsets.c index 9155e2312..fb46e8035 100644 --- a/plugins/win-capture/load-graphics-offsets.c +++ b/plugins/win-capture/load-graphics-offsets.c @@ -40,6 +40,8 @@ static inline bool load_offsets_from_string(struct graphics_offsets *offsets, (uint32_t)config_get_uint(config, "dxgi", "present1"); offsets->dxgi.resize = (uint32_t)config_get_uint(config, "dxgi", "resize"); + offsets->dxgi2.release = + (uint32_t)config_get_uint(config, "dxgi", "release"); config_close(config); return true;