jp9000
/
obs-studio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

win-capture: Hook Reset/ResetEx in Present* funcs 

The virtual address table values for Reset/ResetEx can sometimes point
to functions that are in libraries outside of D3D8.dll and D3D9.dll, and
will cause a crash if used.  Instead, just hook Reset/ResetEx when one
of the Present* functions are called.
master
jp9000 2015-11-20 09:32:44 -08:00
parent d1cbb2742e
commit 1755511b2f
8 changed files with 55 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
plugins/win-capture/get-graphics-offsets/d3d8-offsets.cpp
View File

@ -76,7 +76,6 @@ void get_d3d8_offsets(struct d3d8_offsets *offsets)
if (success) { if (success) {
offsets->present = vtable_offset(info.module, info.device, 15); offsets->present = vtable_offset(info.module, info.device, 15);
offsets->reset = vtable_offset(info.module, info.device, 14);
} }
d3d8_free(info); d3d8_free(info);

3
plugins/win-capture/get-graphics-offsets/d3d9-offsets.cpp
View File

@ -87,11 +87,8 @@ void get_d3d9_offsets(struct d3d9_offsets *offsets)
if (success) { if (success) {
offsets->present = vtable_offset(info.module, info.device, 17); offsets->present = vtable_offset(info.module, info.device, 17);
offsets->reset = vtable_offset(info.module, info.device, 16);
offsets->present_ex = vtable_offset(info.module, info.device, offsets->present_ex = vtable_offset(info.module, info.device,
121); 121);
offsets->reset_ex = vtable_offset(info.module, info.device,
132);
offsets->present_swap = vtable_offset(info.module, info.swap, offsets->present_swap = vtable_offset(info.module, info.swap,
3); 3);
} }

3
plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c
View File

@ -27,13 +27,10 @@ int main(int argc, char *argv[])
printf("[d3d8]\n"); printf("[d3d8]\n");
printf("present=0x%"PRIx32"\n", d3d8.present); printf("present=0x%"PRIx32"\n", d3d8.present);
printf("reset=0x%"PRIx32"\n", d3d8.reset);
printf("[d3d9]\n"); printf("[d3d9]\n");
printf("present=0x%"PRIx32"\n", d3d9.present); printf("present=0x%"PRIx32"\n", d3d9.present);
printf("present_ex=0x%"PRIx32"\n", d3d9.present_ex); printf("present_ex=0x%"PRIx32"\n", d3d9.present_ex);
printf("present_swap=0x%"PRIx32"\n", d3d9.present_swap); printf("present_swap=0x%"PRIx32"\n", d3d9.present_swap);
printf("reset=0x%"PRIx32"\n", d3d9.reset);
printf("reset_ex=0x%"PRIx32"\n", d3d9.reset_ex);
printf("[dxgi]\n"); printf("[dxgi]\n");
printf("present=0x%"PRIx32"\n", dxgi.present); printf("present=0x%"PRIx32"\n", dxgi.present);
printf("resize=0x%"PRIx32"\n", dxgi.resize); printf("resize=0x%"PRIx32"\n", dxgi.resize);

3
plugins/win-capture/graphics-hook-info.h
View File

@ -27,15 +27,12 @@
struct d3d8_offsets { struct d3d8_offsets {
uint32_t present; uint32_t present;
uint32_t reset;
}; };
struct d3d9_offsets { struct d3d9_offsets {
uint32_t present; uint32_t present;
uint32_t present_ex; uint32_t present_ex;
uint32_t present_swap; uint32_t present_swap;
uint32_t reset;
uint32_t reset_ex;
}; };
struct dxgi_offsets { struct dxgi_offsets {

22
plugins/win-capture/graphics-hook/d3d8-capture.cpp
View File

@ -252,6 +252,19 @@ static HRESULT STDMETHODCALLTYPE hook_reset(IDirect3DDevice8 *device,
return hr; return hr;
} }
static bool hooked_reset = false;
static void setup_reset_hooks(IDirect3DDevice8 *device)
{
uintptr_t *vtable = *(uintptr_t**)device;
hook_init(&reset, (void*)vtable[14], (void*)hook_reset,
"IDirect3DDevice8::Reset");
rehook(&reset);
hooked_reset = true;
}
static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice8 *device, static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice8 *device,
CONST RECT *src_rect, CONST RECT *dst_rect, CONST RECT *src_rect, CONST RECT *dst_rect,
HWND override_window, CONST RGNDATA *dirty_region) HWND override_window, CONST RGNDATA *dirty_region)
@ -259,6 +272,9 @@ static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice8 *device,
IDirect3DSurface8 *backbuffer; IDirect3DSurface8 *backbuffer;
HRESULT hr; HRESULT hr;
if (!hooked_reset)
setup_reset_hooks(device);
backbuffer = d3d8_get_backbuffer(device); backbuffer = d3d8_get_backbuffer(device);
if (backbuffer) { if (backbuffer) {
d3d8_capture(device, backbuffer); d3d8_capture(device, backbuffer);
@ -277,7 +293,6 @@ bool hook_d3d8(void)
{ {
HMODULE d3d8_module = get_system_module("d3d8.dll"); HMODULE d3d8_module = get_system_module("d3d8.dll");
void *present_addr; void *present_addr;
void *reset_addr;
if (!d3d8_module) { if (!d3d8_module) {
return false; return false;
@ -285,16 +300,11 @@ bool hook_d3d8(void)
present_addr = get_offset_addr(d3d8_module, present_addr = get_offset_addr(d3d8_module,
global_hook_info->offsets.d3d8.present); global_hook_info->offsets.d3d8.present);
reset_addr = get_offset_addr(d3d8_module,
global_hook_info->offsets.d3d8.reset);
hook_init(&present, present_addr, (void*)hook_present, hook_init(&present, present_addr, (void*)hook_present,
"IDirect3DDevice8::Present"); "IDirect3DDevice8::Present");
hook_init(&reset, reset_addr, (void*)hook_reset,
"IDirect3DDevice8::Reset");
rehook(&present); rehook(&present);
rehook(&reset);
hlog("Hooked D3D8"); hlog("Hooked D3D8");

50
plugins/win-capture/graphics-hook/d3d9-capture.cpp
View File

@ -648,6 +648,9 @@ static inline void present_end(IDirect3DDevice9 *device,
} }
} }
static bool hooked_reset = false;
static void setup_reset_hooks(IDirect3DDevice9 *device);
static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice9 *device, static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice9 *device,
CONST RECT *src_rect, CONST RECT *dst_rect, CONST RECT *src_rect, CONST RECT *dst_rect,
HWND override_window, CONST RGNDATA *dirty_region) HWND override_window, CONST RGNDATA *dirty_region)
@ -655,6 +658,9 @@ static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice9 *device,
IDirect3DSurface9 *backbuffer = nullptr; IDirect3DSurface9 *backbuffer = nullptr;
HRESULT hr; HRESULT hr;
if (!hooked_reset)
setup_reset_hooks(device);
present_begin(device, backbuffer); present_begin(device, backbuffer);
unhook(&present); unhook(&present);
@ -674,6 +680,9 @@ static HRESULT STDMETHODCALLTYPE hook_present_ex(IDirect3DDevice9 *device,
IDirect3DSurface9 *backbuffer = nullptr; IDirect3DSurface9 *backbuffer = nullptr;
HRESULT hr; HRESULT hr;
if (!hooked_reset)
setup_reset_hooks(device);
present_begin(device, backbuffer); present_begin(device, backbuffer);
unhook(&present_ex); unhook(&present_ex);
@ -702,8 +711,12 @@ static HRESULT STDMETHODCALLTYPE hook_present_swap(IDirect3DSwapChain9 *swap,
} }
} }
if (device) if (device) {
if (!hooked_reset)
setup_reset_hooks(device);
present_begin(device, backbuffer); present_begin(device, backbuffer);
}
unhook(&present_swap); unhook(&present_swap);
present_swap_t call = (present_swap_t)present_swap.call_addr; present_swap_t call = (present_swap_t)present_swap.call_addr;
@ -749,14 +762,35 @@ static HRESULT STDMETHODCALLTYPE hook_reset_ex(IDirect3DDevice9 *device,
return hr; return hr;
} }
static void setup_reset_hooks(IDirect3DDevice9 *device)
{
IDirect3DDevice9Ex *d3d9ex = nullptr;
uintptr_t *vtable = *(uintptr_t**)device;
HRESULT hr;
hook_init(&reset, (void*)vtable[16], (void*)hook_reset,
"IDirect3DDevice9::Reset");
rehook(&reset);
hr = device->QueryInterface(__uuidof(IDirect3DDevice9Ex),
(void**)&d3d9ex);
if (SUCCEEDED(hr)) {
hook_init(&reset_ex, (void*)vtable[132], (void*)hook_reset_ex,
"IDirect3DDevice9Ex::ResetEx");
rehook(&reset_ex);
d3d9ex->Release();
}
hooked_reset = true;
}
bool hook_d3d9(void) bool hook_d3d9(void)
{ {
HMODULE d3d9_module = get_system_module("d3d9.dll"); HMODULE d3d9_module = get_system_module("d3d9.dll");
void *present_addr; void *present_addr;
void *present_ex_addr; void *present_ex_addr;
void *present_swap_addr; void *present_swap_addr;
void *reset_addr;
void *reset_ex_addr;
if (!d3d9_module) { if (!d3d9_module) {
return false; return false;
@ -768,10 +802,6 @@ bool hook_d3d9(void)
global_hook_info->offsets.d3d9.present_ex); global_hook_info->offsets.d3d9.present_ex);
present_swap_addr = get_offset_addr(d3d9_module, present_swap_addr = get_offset_addr(d3d9_module,
global_hook_info->offsets.d3d9.present_swap); global_hook_info->offsets.d3d9.present_swap);
reset_addr = get_offset_addr(d3d9_module,
global_hook_info->offsets.d3d9.reset);
reset_ex_addr = get_offset_addr(d3d9_module,
global_hook_info->offsets.d3d9.reset_ex);
hook_init(&present, present_addr, (void*)hook_present, hook_init(&present, present_addr, (void*)hook_present,
"IDirect3DDevice9::Present"); "IDirect3DDevice9::Present");
@ -779,13 +809,7 @@ bool hook_d3d9(void)
"IDirect3DDevice9Ex::PresentEx"); "IDirect3DDevice9Ex::PresentEx");
hook_init(&present_swap, present_swap_addr, (void*)hook_present_swap, hook_init(&present_swap, present_swap_addr, (void*)hook_present_swap,
"IDirect3DSwapChain9::Present"); "IDirect3DSwapChain9::Present");
hook_init(&reset, reset_addr, (void*)hook_reset,
"IDirect3DDevice9::Reset");
hook_init(&reset_ex, reset_ex_addr, (void*)hook_reset_ex,
"IDirect3DDevice9Ex::ResetEx");
rehook(&reset_ex);
rehook(&reset);
rehook(&present_swap); rehook(&present_swap);
rehook(&present_ex); rehook(&present_ex);
rehook(&present); rehook(&present);

7
plugins/win-capture/graphics-hook/graphics-hook.c
View File

@ -238,8 +238,7 @@ static void free_hook(void)
static inline bool d3d8_hookable(void) static inline bool d3d8_hookable(void)
{ {
return !!global_hook_info->offsets.d3d8.present && return !!global_hook_info->offsets.d3d8.present;
!!global_hook_info->offsets.d3d8.reset;
} }
static inline bool ddraw_hookable(void) static inline bool ddraw_hookable(void)
@ -258,9 +257,7 @@ static inline bool d3d9_hookable(void)
{ {
return !!global_hook_info->offsets.d3d9.present && return !!global_hook_info->offsets.d3d9.present &&
!!global_hook_info->offsets.d3d9.present_ex && !!global_hook_info->offsets.d3d9.present_ex &&
!!global_hook_info->offsets.d3d9.present_swap && !!global_hook_info->offsets.d3d9.present_swap;
!!global_hook_info->offsets.d3d9.reset &&
!!global_hook_info->offsets.d3d9.reset_ex;
} }
static inline bool dxgi_hookable(void) static inline bool dxgi_hookable(void)

6
plugins/win-capture/load-graphics-offsets.c
View File

@ -23,8 +23,6 @@ static inline bool load_offsets_from_string(struct graphics_offsets *offsets,
offsets->d3d8.present = offsets->d3d8.present =
(uint32_t)config_get_uint(config, "d3d8", "present"); (uint32_t)config_get_uint(config, "d3d8", "present");
offsets->d3d8.reset =
(uint32_t)config_get_uint(config, "d3d8", "reset");
offsets->d3d9.present = offsets->d3d9.present =
(uint32_t)config_get_uint(config, "d3d9", "present"); (uint32_t)config_get_uint(config, "d3d9", "present");
@ -32,10 +30,6 @@ static inline bool load_offsets_from_string(struct graphics_offsets *offsets,
(uint32_t)config_get_uint(config, "d3d9", "present_ex"); (uint32_t)config_get_uint(config, "d3d9", "present_ex");
offsets->d3d9.present_swap = offsets->d3d9.present_swap =
(uint32_t)config_get_uint(config, "d3d9", "present_swap"); (uint32_t)config_get_uint(config, "d3d9", "present_swap");
offsets->d3d9.reset =
(uint32_t)config_get_uint(config, "d3d9", "reset");
offsets->d3d9.reset_ex =
(uint32_t)config_get_uint(config, "d3d9", "reset_ex");
offsets->dxgi.present = offsets->dxgi.present =
(uint32_t)config_get_uint(config, "dxgi", "present"); (uint32_t)config_get_uint(config, "dxgi", "present");