From 1755511b2f5afd95d6ae6657eff2e2ddc387f0bc Mon Sep 17 00:00:00 2001 From: jp9000 Date: Fri, 20 Nov 2015 09:32:44 -0800 Subject: [PATCH] win-capture: Hook Reset/ResetEx in Present* funcs The virtual address table values for Reset/ResetEx can sometimes point to functions that are in libraries outside of D3D8.dll and D3D9.dll, and will cause a crash if used. Instead, just hook Reset/ResetEx when one of the Present* functions are called. --- .../get-graphics-offsets/d3d8-offsets.cpp | 1 - .../get-graphics-offsets/d3d9-offsets.cpp | 3 -- .../get-graphics-offsets.c | 3 -- plugins/win-capture/graphics-hook-info.h | 3 -- .../graphics-hook/d3d8-capture.cpp | 22 +++++--- .../graphics-hook/d3d9-capture.cpp | 50 ++++++++++++++----- .../win-capture/graphics-hook/graphics-hook.c | 7 +-- plugins/win-capture/load-graphics-offsets.c | 6 --- 8 files changed, 55 insertions(+), 40 deletions(-) diff --git a/plugins/win-capture/get-graphics-offsets/d3d8-offsets.cpp b/plugins/win-capture/get-graphics-offsets/d3d8-offsets.cpp index 86e0e6a6b..41e49aca4 100644 --- a/plugins/win-capture/get-graphics-offsets/d3d8-offsets.cpp +++ b/plugins/win-capture/get-graphics-offsets/d3d8-offsets.cpp @@ -76,7 +76,6 @@ void get_d3d8_offsets(struct d3d8_offsets *offsets) if (success) { offsets->present = vtable_offset(info.module, info.device, 15); - offsets->reset = vtable_offset(info.module, info.device, 14); } d3d8_free(info); diff --git a/plugins/win-capture/get-graphics-offsets/d3d9-offsets.cpp b/plugins/win-capture/get-graphics-offsets/d3d9-offsets.cpp index 657d7a2b7..b6d7ddc3f 100644 --- a/plugins/win-capture/get-graphics-offsets/d3d9-offsets.cpp +++ b/plugins/win-capture/get-graphics-offsets/d3d9-offsets.cpp @@ -87,11 +87,8 @@ void get_d3d9_offsets(struct d3d9_offsets *offsets) if (success) { offsets->present = vtable_offset(info.module, info.device, 17); - offsets->reset = vtable_offset(info.module, info.device, 16); offsets->present_ex = vtable_offset(info.module, info.device, 121); - offsets->reset_ex = vtable_offset(info.module, info.device, - 132); offsets->present_swap = vtable_offset(info.module, info.swap, 3); } diff --git a/plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c b/plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c index d05a4e98a..fc12f6116 100644 --- a/plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c +++ b/plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c @@ -27,13 +27,10 @@ int main(int argc, char *argv[]) printf("[d3d8]\n"); printf("present=0x%"PRIx32"\n", d3d8.present); - printf("reset=0x%"PRIx32"\n", d3d8.reset); printf("[d3d9]\n"); printf("present=0x%"PRIx32"\n", d3d9.present); printf("present_ex=0x%"PRIx32"\n", d3d9.present_ex); printf("present_swap=0x%"PRIx32"\n", d3d9.present_swap); - printf("reset=0x%"PRIx32"\n", d3d9.reset); - printf("reset_ex=0x%"PRIx32"\n", d3d9.reset_ex); printf("[dxgi]\n"); printf("present=0x%"PRIx32"\n", dxgi.present); printf("resize=0x%"PRIx32"\n", dxgi.resize); diff --git a/plugins/win-capture/graphics-hook-info.h b/plugins/win-capture/graphics-hook-info.h index 3240c9091..ff21372fc 100644 --- a/plugins/win-capture/graphics-hook-info.h +++ b/plugins/win-capture/graphics-hook-info.h @@ -27,15 +27,12 @@ struct d3d8_offsets { uint32_t present; - uint32_t reset; }; struct d3d9_offsets { uint32_t present; uint32_t present_ex; uint32_t present_swap; - uint32_t reset; - uint32_t reset_ex; }; struct dxgi_offsets { diff --git a/plugins/win-capture/graphics-hook/d3d8-capture.cpp b/plugins/win-capture/graphics-hook/d3d8-capture.cpp index 9edf7e39f..94f608bed 100644 --- a/plugins/win-capture/graphics-hook/d3d8-capture.cpp +++ b/plugins/win-capture/graphics-hook/d3d8-capture.cpp @@ -252,6 +252,19 @@ static HRESULT STDMETHODCALLTYPE hook_reset(IDirect3DDevice8 *device, return hr; } +static bool hooked_reset = false; + +static void setup_reset_hooks(IDirect3DDevice8 *device) +{ + uintptr_t *vtable = *(uintptr_t**)device; + + hook_init(&reset, (void*)vtable[14], (void*)hook_reset, + "IDirect3DDevice8::Reset"); + rehook(&reset); + + hooked_reset = true; +} + static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice8 *device, CONST RECT *src_rect, CONST RECT *dst_rect, HWND override_window, CONST RGNDATA *dirty_region) @@ -259,6 +272,9 @@ static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice8 *device, IDirect3DSurface8 *backbuffer; HRESULT hr; + if (!hooked_reset) + setup_reset_hooks(device); + backbuffer = d3d8_get_backbuffer(device); if (backbuffer) { d3d8_capture(device, backbuffer); @@ -277,7 +293,6 @@ bool hook_d3d8(void) { HMODULE d3d8_module = get_system_module("d3d8.dll"); void *present_addr; - void *reset_addr; if (!d3d8_module) { return false; @@ -285,16 +300,11 @@ bool hook_d3d8(void) present_addr = get_offset_addr(d3d8_module, global_hook_info->offsets.d3d8.present); - reset_addr = get_offset_addr(d3d8_module, - global_hook_info->offsets.d3d8.reset); hook_init(&present, present_addr, (void*)hook_present, "IDirect3DDevice8::Present"); - hook_init(&reset, reset_addr, (void*)hook_reset, - "IDirect3DDevice8::Reset"); rehook(&present); - rehook(&reset); hlog("Hooked D3D8"); diff --git a/plugins/win-capture/graphics-hook/d3d9-capture.cpp b/plugins/win-capture/graphics-hook/d3d9-capture.cpp index effaf5f3a..200e8f280 100644 --- a/plugins/win-capture/graphics-hook/d3d9-capture.cpp +++ b/plugins/win-capture/graphics-hook/d3d9-capture.cpp @@ -648,6 +648,9 @@ static inline void present_end(IDirect3DDevice9 *device, } } +static bool hooked_reset = false; +static void setup_reset_hooks(IDirect3DDevice9 *device); + static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice9 *device, CONST RECT *src_rect, CONST RECT *dst_rect, HWND override_window, CONST RGNDATA *dirty_region) @@ -655,6 +658,9 @@ static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice9 *device, IDirect3DSurface9 *backbuffer = nullptr; HRESULT hr; + if (!hooked_reset) + setup_reset_hooks(device); + present_begin(device, backbuffer); unhook(&present); @@ -674,6 +680,9 @@ static HRESULT STDMETHODCALLTYPE hook_present_ex(IDirect3DDevice9 *device, IDirect3DSurface9 *backbuffer = nullptr; HRESULT hr; + if (!hooked_reset) + setup_reset_hooks(device); + present_begin(device, backbuffer); unhook(&present_ex); @@ -702,8 +711,12 @@ static HRESULT STDMETHODCALLTYPE hook_present_swap(IDirect3DSwapChain9 *swap, } } - if (device) + if (device) { + if (!hooked_reset) + setup_reset_hooks(device); + present_begin(device, backbuffer); + } unhook(&present_swap); present_swap_t call = (present_swap_t)present_swap.call_addr; @@ -749,14 +762,35 @@ static HRESULT STDMETHODCALLTYPE hook_reset_ex(IDirect3DDevice9 *device, return hr; } +static void setup_reset_hooks(IDirect3DDevice9 *device) +{ + IDirect3DDevice9Ex *d3d9ex = nullptr; + uintptr_t *vtable = *(uintptr_t**)device; + HRESULT hr; + + hook_init(&reset, (void*)vtable[16], (void*)hook_reset, + "IDirect3DDevice9::Reset"); + rehook(&reset); + + hr = device->QueryInterface(__uuidof(IDirect3DDevice9Ex), + (void**)&d3d9ex); + if (SUCCEEDED(hr)) { + hook_init(&reset_ex, (void*)vtable[132], (void*)hook_reset_ex, + "IDirect3DDevice9Ex::ResetEx"); + rehook(&reset_ex); + + d3d9ex->Release(); + } + + hooked_reset = true; +} + bool hook_d3d9(void) { HMODULE d3d9_module = get_system_module("d3d9.dll"); void *present_addr; void *present_ex_addr; void *present_swap_addr; - void *reset_addr; - void *reset_ex_addr; if (!d3d9_module) { return false; @@ -768,10 +802,6 @@ bool hook_d3d9(void) global_hook_info->offsets.d3d9.present_ex); present_swap_addr = get_offset_addr(d3d9_module, global_hook_info->offsets.d3d9.present_swap); - reset_addr = get_offset_addr(d3d9_module, - global_hook_info->offsets.d3d9.reset); - reset_ex_addr = get_offset_addr(d3d9_module, - global_hook_info->offsets.d3d9.reset_ex); hook_init(&present, present_addr, (void*)hook_present, "IDirect3DDevice9::Present"); @@ -779,13 +809,7 @@ bool hook_d3d9(void) "IDirect3DDevice9Ex::PresentEx"); hook_init(&present_swap, present_swap_addr, (void*)hook_present_swap, "IDirect3DSwapChain9::Present"); - hook_init(&reset, reset_addr, (void*)hook_reset, - "IDirect3DDevice9::Reset"); - hook_init(&reset_ex, reset_ex_addr, (void*)hook_reset_ex, - "IDirect3DDevice9Ex::ResetEx"); - rehook(&reset_ex); - rehook(&reset); rehook(&present_swap); rehook(&present_ex); rehook(&present); diff --git a/plugins/win-capture/graphics-hook/graphics-hook.c b/plugins/win-capture/graphics-hook/graphics-hook.c index a5797114f..529114641 100644 --- a/plugins/win-capture/graphics-hook/graphics-hook.c +++ b/plugins/win-capture/graphics-hook/graphics-hook.c @@ -238,8 +238,7 @@ static void free_hook(void) static inline bool d3d8_hookable(void) { - return !!global_hook_info->offsets.d3d8.present && - !!global_hook_info->offsets.d3d8.reset; + return !!global_hook_info->offsets.d3d8.present; } static inline bool ddraw_hookable(void) @@ -258,9 +257,7 @@ static inline bool d3d9_hookable(void) { return !!global_hook_info->offsets.d3d9.present && !!global_hook_info->offsets.d3d9.present_ex && - !!global_hook_info->offsets.d3d9.present_swap && - !!global_hook_info->offsets.d3d9.reset && - !!global_hook_info->offsets.d3d9.reset_ex; + !!global_hook_info->offsets.d3d9.present_swap; } static inline bool dxgi_hookable(void) diff --git a/plugins/win-capture/load-graphics-offsets.c b/plugins/win-capture/load-graphics-offsets.c index 9b23305a9..0602a1cb9 100644 --- a/plugins/win-capture/load-graphics-offsets.c +++ b/plugins/win-capture/load-graphics-offsets.c @@ -23,8 +23,6 @@ static inline bool load_offsets_from_string(struct graphics_offsets *offsets, offsets->d3d8.present = (uint32_t)config_get_uint(config, "d3d8", "present"); - offsets->d3d8.reset = - (uint32_t)config_get_uint(config, "d3d8", "reset"); offsets->d3d9.present = (uint32_t)config_get_uint(config, "d3d9", "present"); @@ -32,10 +30,6 @@ static inline bool load_offsets_from_string(struct graphics_offsets *offsets, (uint32_t)config_get_uint(config, "d3d9", "present_ex"); offsets->d3d9.present_swap = (uint32_t)config_get_uint(config, "d3d9", "present_swap"); - offsets->d3d9.reset = - (uint32_t)config_get_uint(config, "d3d9", "reset"); - offsets->d3d9.reset_ex = - (uint32_t)config_get_uint(config, "d3d9", "reset_ex"); offsets->dxgi.present = (uint32_t)config_get_uint(config, "dxgi", "present");