jp9000/obs-studio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

win-capture: Hook Reset/ResetEx in Present* funcs

Browse Source 
The virtual address table values for Reset/ResetEx can sometimes point
to functions that are in libraries outside of D3D8.dll and D3D9.dll, and
will cause a crash if used.  Instead, just hook Reset/ResetEx when one
of the Present* functions are called.
This commit is contained in:
jp9000
2015-11-20 09:32:44 -08:00
parent d1cbb2742e
commit 1755511b2f
8 changed files with 55 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
plugins/win-capture/graphics-hook/d3d8-capture.cpp
View File

@@ -252,6 +252,19 @@ static HRESULT STDMETHODCALLTYPE hook_reset(IDirect3DDevice8 *device,
return hr;
}
static bool hooked_reset = false;
static void setup_reset_hooks(IDirect3DDevice8 *device)
{
uintptr_t *vtable = *(uintptr_t**)device;
hook_init(&reset, (void*)vtable[14], (void*)hook_reset,
"IDirect3DDevice8::Reset");
rehook(&reset);
hooked_reset = true;
}
static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice8 *device,
CONST RECT *src_rect, CONST RECT *dst_rect,
HWND override_window, CONST RGNDATA *dirty_region)
@@ -259,6 +272,9 @@ static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice8 *device,
IDirect3DSurface8 *backbuffer;
HRESULT hr;
if (!hooked_reset)
setup_reset_hooks(device);
backbuffer = d3d8_get_backbuffer(device);
if (backbuffer) {
d3d8_capture(device, backbuffer);
@@ -277,7 +293,6 @@ bool hook_d3d8(void)
{
HMODULE d3d8_module = get_system_module("d3d8.dll");
void *present_addr;
void *reset_addr;
if (!d3d8_module) {
return false;
@@ -285,16 +300,11 @@ bool hook_d3d8(void)
present_addr = get_offset_addr(d3d8_module,
global_hook_info->offsets.d3d8.present);
reset_addr = get_offset_addr(d3d8_module,
global_hook_info->offsets.d3d8.reset);
hook_init(&present, present_addr, (void*)hook_present,
"IDirect3DDevice8::Present");
hook_init(&reset, reset_addr, (void*)hook_reset,
"IDirect3DDevice8::Reset");
rehook(&present);
rehook(&reset);
hlog("Hooked D3D8");

50
plugins/win-capture/graphics-hook/d3d9-capture.cpp
View File

@@ -648,6 +648,9 @@ static inline void present_end(IDirect3DDevice9 *device,
}
}
static bool hooked_reset = false;
static void setup_reset_hooks(IDirect3DDevice9 *device);
static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice9 *device,
CONST RECT *src_rect, CONST RECT *dst_rect,
HWND override_window, CONST RGNDATA *dirty_region)
@@ -655,6 +658,9 @@ static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice9 *device,
IDirect3DSurface9 *backbuffer = nullptr;
HRESULT hr;
if (!hooked_reset)
setup_reset_hooks(device);
present_begin(device, backbuffer);
unhook(&present);
@@ -674,6 +680,9 @@ static HRESULT STDMETHODCALLTYPE hook_present_ex(IDirect3DDevice9 *device,
IDirect3DSurface9 *backbuffer = nullptr;
HRESULT hr;
if (!hooked_reset)
setup_reset_hooks(device);
present_begin(device, backbuffer);
unhook(&present_ex);
@@ -702,8 +711,12 @@ static HRESULT STDMETHODCALLTYPE hook_present_swap(IDirect3DSwapChain9 *swap,
}
}
if (device)
if (device) {
if (!hooked_reset)
setup_reset_hooks(device);
present_begin(device, backbuffer);
}
unhook(&present_swap);
present_swap_t call = (present_swap_t)present_swap.call_addr;
@@ -749,14 +762,35 @@ static HRESULT STDMETHODCALLTYPE hook_reset_ex(IDirect3DDevice9 *device,
return hr;
}
static void setup_reset_hooks(IDirect3DDevice9 *device)
{
IDirect3DDevice9Ex *d3d9ex = nullptr;
uintptr_t *vtable = *(uintptr_t**)device;
HRESULT hr;
hook_init(&reset, (void*)vtable[16], (void*)hook_reset,
"IDirect3DDevice9::Reset");
rehook(&reset);
hr = device->QueryInterface(__uuidof(IDirect3DDevice9Ex),
(void**)&d3d9ex);
if (SUCCEEDED(hr)) {
hook_init(&reset_ex, (void*)vtable[132], (void*)hook_reset_ex,
"IDirect3DDevice9Ex::ResetEx");
rehook(&reset_ex);
d3d9ex->Release();
}
hooked_reset = true;
}
bool hook_d3d9(void)
{
HMODULE d3d9_module = get_system_module("d3d9.dll");
void *present_addr;
void *present_ex_addr;
void *present_swap_addr;
void *reset_addr;
void *reset_ex_addr;
if (!d3d9_module) {
return false;
@@ -768,10 +802,6 @@ bool hook_d3d9(void)
global_hook_info->offsets.d3d9.present_ex);
present_swap_addr = get_offset_addr(d3d9_module,
global_hook_info->offsets.d3d9.present_swap);
reset_addr = get_offset_addr(d3d9_module,
global_hook_info->offsets.d3d9.reset);
reset_ex_addr = get_offset_addr(d3d9_module,
global_hook_info->offsets.d3d9.reset_ex);
hook_init(&present, present_addr, (void*)hook_present,
"IDirect3DDevice9::Present");
@@ -779,13 +809,7 @@ bool hook_d3d9(void)
"IDirect3DDevice9Ex::PresentEx");
hook_init(&present_swap, present_swap_addr, (void*)hook_present_swap,
"IDirect3DSwapChain9::Present");
hook_init(&reset, reset_addr, (void*)hook_reset,
"IDirect3DDevice9::Reset");
hook_init(&reset_ex, reset_ex_addr, (void*)hook_reset_ex,
"IDirect3DDevice9Ex::ResetEx");
rehook(&reset_ex);
rehook(&reset);
rehook(&present_swap);
rehook(&present_ex);
rehook(&present);

7
plugins/win-capture/graphics-hook/graphics-hook.c
View File

@@ -238,8 +238,7 @@ static void free_hook(void)
static inline bool d3d8_hookable(void)
{
return !!global_hook_info->offsets.d3d8.present &&
!!global_hook_info->offsets.d3d8.reset;
return !!global_hook_info->offsets.d3d8.present;
}
static inline bool ddraw_hookable(void)
@@ -258,9 +257,7 @@ static inline bool d3d9_hookable(void)
{
return !!global_hook_info->offsets.d3d9.present &&
!!global_hook_info->offsets.d3d9.present_ex &&
!!global_hook_info->offsets.d3d9.present_swap &&
!!global_hook_info->offsets.d3d9.reset &&
!!global_hook_info->offsets.d3d9.reset_ex;
!!global_hook_info->offsets.d3d9.present_swap;
}
static inline bool dxgi_hookable(void)