jp9000
/
obs-studio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

win-capture: Fall back to creating d3d contexts if offsets bad

master
jp9000 2015-11-20 13:55:06 -08:00
parent 1755511b2f
commit 50c61898d0
4 changed files with 257 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
plugins/win-capture/graphics-hook/d3d8-capture.cpp
View File

@ -289,17 +289,90 @@ static HRESULT STDMETHODCALLTYPE hook_present(IDirect3DDevice8 *device,
return hr;
}
typedef IDirect3D8 *(WINAPI *d3d8create_t)(UINT);
static bool manually_get_d3d8_present_addr(HMODULE d3d8_module,
void **present_addr)
{
d3d8create_t create;
D3DPRESENT_PARAMETERS pp;
HRESULT hr;
IDirect3DDevice8 *device;
IDirect3D8 *d3d8;
hlog("D3D8 value invalid, manually obtaining");
create = (d3d8create_t)GetProcAddress(d3d8_module, "Direct3DCreate8");
if (!create) {
hlog("Failed to load Direct3DCreate8");
return false;
}
d3d8 = create(D3D_SDK_VERSION);
if (!d3d8) {
hlog("Failed to create D3D8 context");
return false;
}
memset(&pp, 0, sizeof(pp));
pp.Windowed = true;
pp.SwapEffect = D3DSWAPEFFECT_FLIP;
pp.BackBufferFormat = D3DFMT_A8R8G8B8;
pp.BackBufferWidth = 2;
pp.BackBufferHeight = 2;
pp.BackBufferCount = 1;
pp.hDeviceWindow = dummy_window;
hr = d3d8->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL,
dummy_window, D3DCREATE_HARDWARE_VERTEXPROCESSING, &pp,
&device);
d3d8->Release();
if (SUCCEEDED(hr)) {
uintptr_t *vtable = *(uintptr_t**)device;
*present_addr = (void*)vtable[15];
device->Release();
} else {
hlog("Failed to create D3D8 device");
return false;
}
return true;
}
bool hook_d3d8(void)
{
HMODULE d3d8_module = get_system_module("d3d8.dll");
void *present_addr;
uint32_t d3d8_size;
void *present_addr = nullptr;
if (!d3d8_module) {
return false;
}
present_addr = get_offset_addr(d3d8_module,
global_hook_info->offsets.d3d8.present);
d3d8_size = module_size(d3d8_module);
if (global_hook_info->offsets.d3d8.present < d3d8_size) {
present_addr = get_offset_addr(d3d8_module,
global_hook_info->offsets.d3d8.present);
} else {
if (!dummy_window) {
return false;
}
if (!manually_get_d3d8_present_addr(d3d8_module,
&present_addr)) {
hlog("Failed to get D3D8 value");
return true;
}
}
if (!present_addr) {
hlog("Invalid D3D8 value");
return true;
}
hook_init(&present, present_addr, (void*)hook_present,
"IDirect3DDevice8::Present");
@ -307,6 +380,5 @@ bool hook_d3d8(void)
rehook(&present);
hlog("Hooked D3D8");
return true;
}

134
plugins/win-capture/graphics-hook/d3d9-capture.cpp
View File

@ -785,34 +785,132 @@ static void setup_reset_hooks(IDirect3DDevice9 *device)
hooked_reset = true;
}
typedef HRESULT (WINAPI *d3d9create_ex_t)(UINT, IDirect3D9Ex**);
static bool manually_get_d3d9_addrs(HMODULE d3d9_module,
void **present_addr,
void **present_ex_addr,
void **present_swap_addr)
{
d3d9create_ex_t create_ex;
D3DPRESENT_PARAMETERS pp;
HRESULT hr;
IDirect3DDevice9Ex *device;
IDirect3D9Ex *d3d9ex;
hlog("D3D9 values invalid, manually obtaining");
create_ex = (d3d9create_ex_t)GetProcAddress(d3d9_module,
"Direct3DCreate9Ex");
if (!create_ex) {
hlog("Failed to load Direct3DCreate9Ex");
return false;
}
if (FAILED(create_ex(D3D_SDK_VERSION, &d3d9ex))) {
hlog("Failed to create D3D9 context");
return false;
}
memset(&pp, 0, sizeof(pp));
pp.Windowed = 1;
pp.SwapEffect = D3DSWAPEFFECT_FLIP;
pp.BackBufferFormat = D3DFMT_A8R8G8B8;
pp.BackBufferCount = 1;
pp.hDeviceWindow = (HWND)dummy_window;
pp.PresentationInterval = D3DPRESENT_INTERVAL_IMMEDIATE;
hr = d3d9ex->CreateDeviceEx(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL,
dummy_window,
D3DCREATE_HARDWARE_VERTEXPROCESSING |
D3DCREATE_NOWINDOWCHANGES, &pp, NULL, &device);
d3d9ex->Release();
if (SUCCEEDED(hr)) {
uintptr_t *vtable = *(uintptr_t**)device;
IDirect3DSwapChain9 *swap;
*present_addr = (void*)vtable[17];
*present_ex_addr = (void*)vtable[121];
hr = device->GetSwapChain(0, &swap);
if (SUCCEEDED(hr)) {
vtable = *(uintptr_t**)swap;
*present_swap_addr = (void*)vtable[3];
swap->Release();
}
device->Release();
} else {
hlog("Failed to create D3D9 device");
return false;
}
return true;
}
bool hook_d3d9(void)
{
HMODULE d3d9_module = get_system_module("d3d9.dll");
void *present_addr;
void *present_ex_addr;
void *present_swap_addr;
uint32_t d3d9_size;
void *present_addr = nullptr;
void *present_ex_addr = nullptr;
void *present_swap_addr = nullptr;
if (!d3d9_module) {
return false;
}
present_addr = get_offset_addr(d3d9_module,
global_hook_info->offsets.d3d9.present);
present_ex_addr = get_offset_addr(d3d9_module,
global_hook_info->offsets.d3d9.present_ex);
present_swap_addr = get_offset_addr(d3d9_module,
global_hook_info->offsets.d3d9.present_swap);
d3d9_size = module_size(d3d9_module);
hook_init(&present, present_addr, (void*)hook_present,
"IDirect3DDevice9::Present");
hook_init(&present_ex, present_ex_addr, (void*)hook_present_ex,
"IDirect3DDevice9Ex::PresentEx");
hook_init(&present_swap, present_swap_addr, (void*)hook_present_swap,
"IDirect3DSwapChain9::Present");
if (global_hook_info->offsets.d3d9.present < d3d9_size &&
global_hook_info->offsets.d3d9.present_ex < d3d9_size &&
global_hook_info->offsets.d3d9.present_swap < d3d9_size) {
rehook(&present_swap);
rehook(&present_ex);
rehook(&present);
present_addr = get_offset_addr(d3d9_module,
global_hook_info->offsets.d3d9.present);
present_ex_addr = get_offset_addr(d3d9_module,
global_hook_info->offsets.d3d9.present_ex);
present_swap_addr = get_offset_addr(d3d9_module,
global_hook_info->offsets.d3d9.present_swap);
} else {
if (!dummy_window) {
return false;
}
if (!manually_get_d3d9_addrs(d3d9_module,
&present_addr,
&present_ex_addr,
&present_swap_addr)) {
hlog("Failed to get D3D9 values");
return true;
}
}
if (!present_addr && !present_ex_addr && !present_swap_addr) {
hlog("Invalid D3D9 values");
return true;
}
if (present_swap_addr) {
hook_init(&present_swap, present_swap_addr,
(void*)hook_present_swap,
"IDirect3DSwapChain9::Present");
rehook(&present_swap);
}
if (present_ex_addr) {
hook_init(&present_ex, present_ex_addr,
(void*)hook_present_ex,
"IDirect3DDevice9Ex::PresentEx");
rehook(&present_ex);
}
if (present_addr) {
hook_init(&present, present_addr,
(void*)hook_present,
"IDirect3DDevice9::Present");
rehook(&present);
}
hlog("Hooked D3D9");
return true;

55
plugins/win-capture/graphics-hook/graphics-hook.c
View File

@ -35,11 +35,13 @@ HANDLE signal_exit = NULL;
HANDLE tex_mutexes[2] = {NULL, NULL};
static HANDLE filemap_hook_info = NULL;
static HINSTANCE dll_inst = NULL;
static volatile bool stop_loop = false;
static HANDLE capture_thread = NULL;
char system_path[MAX_PATH] = {0};
char process_name[MAX_PATH] = {0};
char keepalive_name[64] = {0};
HWND dummy_window = NULL;
static unsigned int shmem_id_counter = 0;
static void *shmem_info = NULL;
@ -50,6 +52,7 @@ static struct thread_data thread_data = {0};
volatile bool active = false;
struct hook_info *global_hook_info = NULL;
static inline void wait_for_dll_main_finish(HANDLE thread_handle)
{
if (thread_handle) {
@ -179,6 +182,55 @@ static inline bool init_hook_info(void)
return true;
}
#define DEF_FLAGS (WS_POPUP | WS_CLIPCHILDREN | WS_CLIPSIBLINGS)
static DWORD WINAPI dummy_window_thread(LPVOID *unused)
{
static const wchar_t dummy_window_class[] = L"temp_d3d_window_4039785";
WNDCLASSW wc;
MSG msg;
memset(&wc, 0, sizeof(wc));
wc.style = CS_OWNDC;
wc.hInstance = dll_inst;
wc.lpfnWndProc = (WNDPROC)DefWindowProc;
wc.lpszClassName = dummy_window_class;
if (!RegisterClass(&wc)) {
hlog("Failed to create temp D3D window class: %lu",
GetLastError());
return 0;
}
dummy_window = CreateWindowExW(0, dummy_window_class, L"Temp Window",
DEF_FLAGS, 0, 0, 1, 1, NULL, NULL, dll_inst, NULL);
if (!dummy_window) {
hlog("Failed to create temp D3D window: %lu", GetLastError());
return 0;
}
while (GetMessage(&msg, NULL, 0, 0)) {
TranslateMessage(&msg);
DispatchMessage(&msg);
}
(void)unused;
return 0;
}
static inline void init_dummy_window_thread(void)
{
HANDLE thread = CreateThread(NULL, 0, dummy_window_thread, NULL, 0,
NULL);
if (!thread) {
hlog("Failed to create temp D3D window thread: %lu",
GetLastError());
return;
}
CloseHandle(thread);
}
static inline bool init_hook(HANDLE thread_handle)
{
wait_for_dll_main_finish(thread_handle);
@ -202,6 +254,7 @@ static inline bool init_hook(HANDLE thread_handle)
return false;
}
init_dummy_window_thread();
log_current_process();
SetEvent(signal_restart);
@ -737,6 +790,8 @@ BOOL WINAPI DllMain(HINSTANCE hinst, DWORD reason, LPVOID unused1)
if (reason == DLL_PROCESS_ATTACH) {
wchar_t name[MAX_PATH];
dll_inst = hinst;
HANDLE cur_thread = OpenThread(THREAD_ALL_ACCESS, false,
GetCurrentThreadId());

10
plugins/win-capture/graphics-hook/graphics-hook.h
View File

@ -7,6 +7,7 @@
#include "../graphics-hook-info.h"
#include <ipc-util/pipe.h>
#include <psapi.h>
#ifdef __cplusplus
extern "C" {
@ -91,6 +92,7 @@ extern HANDLE tex_mutexes[2];
extern char system_path[MAX_PATH];
extern char process_name[MAX_PATH];
extern char keepalive_name[64];
extern HWND dummy_window;
extern volatile bool active;
static inline const char *get_process_name(void)
@ -108,6 +110,14 @@ static inline HMODULE get_system_module(const char *module)
return GetModuleHandleA(base_path);
}
static inline uint32_t module_size(HMODULE module)
{
MODULEINFO info;
bool success = !!GetModuleInformation(GetCurrentProcess(), module,
&info, sizeof(info));
return success ? info.SizeOfImage : 0;
}
static inline HMODULE load_system_library(const char *name)
{
char base_path[MAX_PATH];