jp9000
/
obs-studio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
obs-studio/plugins/win-capture/funchook.c

283 lines
6.8 KiB
C
Raw Normal View History

win-capture: Add hooking functions These functions allow the safe hooking of windows functions, specifically windows API functions that may or may not have built-in machine code to help aid in reverse chain hooks. If a new hook is applied to an existing forward hook, that hook will be preserved to prevent that new hook's data from being removed unintentionally. Hopefully with all these precautions this will reduce the likelihood of crashes and abnormal hook behavior, while allowing existing hooks to be preserved, and allowing new hooks to be applied.
2014-11-10 01:40:15 -08:00
#include <windows.h>
#include <stdlib.h>
#include "funchook.h"
#define JMP_64_SIZE 14
#define JMP_32_SIZE 5
#define X86_NOP 0x90
#define X86_JMP_NEG_5 0xF9EB
static inline void fix_permissions(void *addr, size_t size)
{
DWORD protect_val;
VirtualProtect(addr, size, PAGE_EXECUTE_READWRITE, &protect_val);
}
void hook_init(struct func_hook *hook,
void *func_addr, void *hook_addr, const char *name)
{
memset(hook, 0, sizeof(*hook));
hook->func_addr = (uintptr_t)func_addr;
hook->hook_addr = (uintptr_t)hook_addr;
hook->name = name;
fix_permissions((void*)(hook->func_addr - JMP_32_SIZE),
JMP_64_SIZE + JMP_32_SIZE);
memcpy(hook->unhook_data, func_addr, JMP_64_SIZE);
}
static inline size_t patch_size(struct func_hook *hook)
{
return hook->is_64bit_jump ? JMP_64_SIZE : JMP_32_SIZE;
}
static const uint8_t longjmp64[6] = {0xFF, 0x25, 0x00, 0x00, 0x00, 0x00};
static inline void rehook64(struct func_hook *hook)
{
uint8_t data[JMP_64_SIZE];
uintptr_t *ptr_loc = (uintptr_t*)((uint8_t*)data + sizeof(longjmp64));
fix_permissions((void*)hook->func_addr, JMP_64_SIZE);
memcpy(data, (void*)hook->func_addr, JMP_64_SIZE);
memcpy(data, longjmp64, sizeof(longjmp64));
*ptr_loc = hook->hook_addr;
hook->call_addr = (void*)hook->func_addr;
hook->type = HOOKTYPE_FORWARD_OVERWRITE;
hook->hooked = true;
memcpy((void*)hook->func_addr, data, JMP_64_SIZE);
}
static inline void hook_reverse_new(struct func_hook *hook, uint8_t *p)
{
hook->call_addr = (void*)(hook->func_addr + 2);
hook->type = HOOKTYPE_REVERSE_CHAIN;
hook->hooked = true;
p[0] = 0xE9;
*((uint32_t*)&p[1]) = (uint32_t)(hook->hook_addr - hook->func_addr);
*((uint16_t*)&p[5]) = X86_JMP_NEG_5;
}
static inline void hook_reverse_chain(struct func_hook *hook, uint8_t *p)
{
if (hook->type != HOOKTYPE_FORWARD_OVERWRITE)
return;
hook->call_addr = (void*)(hook->func_addr + *((int32_t*)&p[1]));
hook->type = HOOKTYPE_REVERSE_CHAIN;
hook->hooked = true;
*((uint32_t*)&p[1]) = (uint32_t)(hook->hook_addr - hook->func_addr);
}
static inline void hook_forward_chain(struct func_hook *hook, uint8_t *p,
intptr_t offset)
{
int32_t cur_offset = *(int32_t*)&p[6];
if (hook->type != HOOKTYPE_FORWARD_OVERWRITE)
return;
hook->call_addr = (void*)(hook->func_addr + JMP_32_SIZE + cur_offset);
hook->type = HOOKTYPE_FORWARD_CHAIN;
hook->hooked = true;
*((int32_t*)&p[6]) = (int32_t)offset;
}
static inline void hook_forward_overwrite(struct func_hook *hook,
intptr_t offset)
{
uint8_t *ptr = (uint8_t*)hook->func_addr;
hook->call_addr = (void*)hook->func_addr;
hook->type = HOOKTYPE_FORWARD_OVERWRITE;
hook->hooked = true;
*(ptr++) = 0xE9;
*((int32_t*)ptr) = (int32_t)offset;
}
static inline void rehook32(struct func_hook *hook, bool force, intptr_t offset)
{
fix_permissions((void*)(hook->func_addr - JMP_32_SIZE),
JMP_32_SIZE * 2);
if (force || !hook->started) {
uint8_t *p = (uint8_t*)hook->func_addr - JMP_32_SIZE;
size_t nop_count = 0;
/* check for reverse chain hook availability */
for (size_t i = 0; i < JMP_32_SIZE; i++) {
if (p[i] == X86_NOP)
nop_count++;
}
if (nop_count == JMP_32_SIZE && p[5] == 0x8B && p[6] == 0xFF) {
hook_reverse_new(hook, p);
} else if (p[0] == 0xE9 && *(uint16_t*)&p[5] == X86_JMP_NEG_5) {
hook_reverse_chain(hook, p);
} else if (p[5] == 0xE9) {
hook_forward_chain(hook, p, offset);
} else if (hook->type != HOOKTYPE_FORWARD_OVERWRITE) {
hook->type = HOOKTYPE_FORWARD_OVERWRITE;
}
hook->started = true;
}
if (hook->type == HOOKTYPE_FORWARD_OVERWRITE) {
hook_forward_overwrite(hook, offset);
}
}
win-capture: Add 64bit bounce to func_hook When hooking 64bit functions, sometimes the offset between the function being hooked and the hook itself can be large enough to where it requires a 64bit offset to be used. However, because a 64bit jump requires overwriting so many code instructions in the function, it can sometimes overwrite code in to an adjacent function, thereby causing a crash. The 64bit hook bounce (created by R1CH) is designed to prevent using very long jumps in the target by creating executable memory within a 32bit offset of that target, and then writing it with the 64bit long jump instruction instead. Then in the target function, it will jump to that memory instead, thus forcing the actual hooked function to use a 32bit hook instead of a 64bit hook, and using at most 5 bytes for the actual hook, preventing any likelihood of it overwriting an adjacent function.
2015-07-03 11:55:35 -07:00
/*
* Creates memory close to the target function, used to force the actual hook
* to use a 32bit jump instead of a 64bit jump, thus preventing the chance of
* overwriting adjacent functions, which can cause a crash. (by R1CH)
*/
win-capture: Fix crash with 64bit bounce hook The new 'offset' value was not being passed back to the caller, which caused the caller to continue to use the old value and thus would cause an invalid hook and crash.
2015-09-22 18:03:18 -07:00
static void setup_64bit_bounce(struct func_hook *hook, intptr_t *offset)
win-capture: Add 64bit bounce to func_hook When hooking 64bit functions, sometimes the offset between the function being hooked and the hook itself can be large enough to where it requires a 64bit offset to be used. However, because a 64bit jump requires overwriting so many code instructions in the function, it can sometimes overwrite code in to an adjacent function, thereby causing a crash. The 64bit hook bounce (created by R1CH) is designed to prevent using very long jumps in the target by creating executable memory within a 32bit offset of that target, and then writing it with the 64bit long jump instruction instead. Then in the target function, it will jump to that memory instead, thus forcing the actual hooked function to use a 32bit hook instead of a 64bit hook, and using at most 5 bytes for the actual hook, preventing any likelihood of it overwriting an adjacent function.
2015-07-03 11:55:35 -07:00
{
MEMORY_BASIC_INFORMATION mbi;
uintptr_t address;
uintptr_t newdiff;
SYSTEM_INFO si;
bool success;
int pagesize;
int i;
success = VirtualQueryEx(GetCurrentProcess(),
(const void*)hook->func_addr, &mbi, sizeof(mbi));
if (!success)
return;
GetSystemInfo(&si);
pagesize = (int)si.dwAllocationGranularity;
address = (uintptr_t)mbi.AllocationBase - pagesize;
for (i = 0; i < 256; i++, address -= pagesize) {
hook->bounce_addr = VirtualAlloc((LPVOID)address, pagesize,
MEM_RESERVE | MEM_COMMIT,
PAGE_EXECUTE_READWRITE);
if (hook->bounce_addr)
break;
}
if (!hook->bounce_addr) {
address = (uintptr_t)mbi.AllocationBase + mbi.RegionSize +
pagesize;
for (i = 0; i < 256; i++, address += pagesize) {
hook->bounce_addr = VirtualAlloc((LPVOID)address,
pagesize, MEM_RESERVE | MEM_COMMIT,
PAGE_EXECUTE_READWRITE);
if (hook->bounce_addr)
break;
}
}
if (!hook->bounce_addr)
return;
if ((hook->func_addr + 5) > (uintptr_t)hook->bounce_addr)
newdiff = hook->func_addr + 5 - (uintptr_t)hook->bounce_addr;
else
newdiff = (uintptr_t)hook->bounce_addr - hook->func_addr + 5;
if (newdiff <= 0x7ffffff0) {
uint8_t *addr = (uint8_t*)hook->bounce_addr;
FillMemory(hook->bounce_addr, pagesize, 0xCC);
*(addr++) = 0xFF;
*(addr++) = 0x25;
*((uint32_t*)addr) = 0;
*((uint64_t*)(addr + 4)) = hook->hook_addr;
hook->hook_addr = (uint64_t)hook->bounce_addr;
win-capture: Fix crash with 64bit bounce hook The new 'offset' value was not being passed back to the caller, which caused the caller to continue to use the old value and thus would cause an invalid hook and crash.
2015-09-22 18:03:18 -07:00
*offset = hook->hook_addr - hook->func_addr - JMP_32_SIZE;
hook->is_64bit_jump = false;
win-capture: Add 64bit bounce to func_hook When hooking 64bit functions, sometimes the offset between the function being hooked and the hook itself can be large enough to where it requires a 64bit offset to be used. However, because a 64bit jump requires overwriting so many code instructions in the function, it can sometimes overwrite code in to an adjacent function, thereby causing a crash. The 64bit hook bounce (created by R1CH) is designed to prevent using very long jumps in the target by creating executable memory within a 32bit offset of that target, and then writing it with the 64bit long jump instruction instead. Then in the target function, it will jump to that memory instead, thus forcing the actual hooked function to use a 32bit hook instead of a 64bit hook, and using at most 5 bytes for the actual hook, preventing any likelihood of it overwriting an adjacent function.
2015-07-03 11:55:35 -07:00
}
}
win-capture: Add hooking functions These functions allow the safe hooking of windows functions, specifically windows API functions that may or may not have built-in machine code to help aid in reverse chain hooks. If a new hook is applied to an existing forward hook, that hook will be preserved to prevent that new hook's data from being removed unintentionally. Hopefully with all these precautions this will reduce the likelihood of crashes and abnormal hook behavior, while allowing existing hooks to be preserved, and allowing new hooks to be applied.
2014-11-10 01:40:15 -08:00
void do_hook(struct func_hook *hook, bool force)
{
intptr_t offset;
if (!force && hook->hooked)
return;
win-capture: Fix case where hook only captures one frame When using a chain hook method (forward or reverse), it was unwisely assumed that the previous hook in the chain would not overwrite new hooks when it's called. When the game capture hook calls the previous hook in the chain, certain other programs that hook (in this case, rivatuner on-screen display) would overwrite the hook with older data and erase the game capture hook, causing it to only capture the first frame and then never capture again. This patch ensures that the hook is always saved before calling the next hook in the chain and then restored after the call returns. It also preserves any new hooks that may be added on top of it at any point.
2016-04-14 22:20:41 -07:00
/* copy back the memory that was previously encountered to preserve
* the current hook and any newer hooks on top */
if (hook->started && !force) {
uintptr_t addr;
size_t size;
if (hook->type == HOOKTYPE_REVERSE_CHAIN) {
addr = hook->func_addr - JMP_32_SIZE;
size = JMP_32_SIZE;
} else {
addr = hook->func_addr;
size = patch_size(hook);
}
memcpy((void*)addr, hook->rehook_data, size);
win-capture: Add hooking functions These functions allow the safe hooking of windows functions, specifically windows API functions that may or may not have built-in machine code to help aid in reverse chain hooks. If a new hook is applied to an existing forward hook, that hook will be preserved to prevent that new hook's data from being removed unintentionally. Hopefully with all these precautions this will reduce the likelihood of crashes and abnormal hook behavior, while allowing existing hooks to be preserved, and allowing new hooks to be applied.
2014-11-10 01:40:15 -08:00
hook->hooked = true;
return;
}
offset = hook->hook_addr - hook->func_addr - JMP_32_SIZE;
#ifdef _WIN64
hook->is_64bit_jump = (llabs(offset) >= 0x7fffffff);
if (hook->is_64bit_jump) {
win-capture: Add 64bit bounce to func_hook When hooking 64bit functions, sometimes the offset between the function being hooked and the hook itself can be large enough to where it requires a 64bit offset to be used. However, because a 64bit jump requires overwriting so many code instructions in the function, it can sometimes overwrite code in to an adjacent function, thereby causing a crash. The 64bit hook bounce (created by R1CH) is designed to prevent using very long jumps in the target by creating executable memory within a 32bit offset of that target, and then writing it with the 64bit long jump instruction instead. Then in the target function, it will jump to that memory instead, thus forcing the actual hooked function to use a 32bit hook instead of a 64bit hook, and using at most 5 bytes for the actual hook, preventing any likelihood of it overwriting an adjacent function.
2015-07-03 11:55:35 -07:00
if (!hook->attempted_bounce) {
hook->attempted_bounce = true;
win-capture: Fix crash with 64bit bounce hook The new 'offset' value was not being passed back to the caller, which caused the caller to continue to use the old value and thus would cause an invalid hook and crash.
2015-09-22 18:03:18 -07:00
setup_64bit_bounce(hook, &offset);
win-capture: Add 64bit bounce to func_hook When hooking 64bit functions, sometimes the offset between the function being hooked and the hook itself can be large enough to where it requires a 64bit offset to be used. However, because a 64bit jump requires overwriting so many code instructions in the function, it can sometimes overwrite code in to an adjacent function, thereby causing a crash. The 64bit hook bounce (created by R1CH) is designed to prevent using very long jumps in the target by creating executable memory within a 32bit offset of that target, and then writing it with the 64bit long jump instruction instead. Then in the target function, it will jump to that memory instead, thus forcing the actual hooked function to use a 32bit hook instead of a 64bit hook, and using at most 5 bytes for the actual hook, preventing any likelihood of it overwriting an adjacent function.
2015-07-03 11:55:35 -07:00
}
if (hook->is_64bit_jump) {
rehook64(hook);
return;
}
win-capture: Add hooking functions These functions allow the safe hooking of windows functions, specifically windows API functions that may or may not have built-in machine code to help aid in reverse chain hooks. If a new hook is applied to an existing forward hook, that hook will be preserved to prevent that new hook's data from being removed unintentionally. Hopefully with all these precautions this will reduce the likelihood of crashes and abnormal hook behavior, while allowing existing hooks to be preserved, and allowing new hooks to be applied.
2014-11-10 01:40:15 -08:00
}
#endif
rehook32(hook, force, offset);
}
void unhook(struct func_hook *hook)
{
win-capture: Fix case where hook only captures one frame When using a chain hook method (forward or reverse), it was unwisely assumed that the previous hook in the chain would not overwrite new hooks when it's called. When the game capture hook calls the previous hook in the chain, certain other programs that hook (in this case, rivatuner on-screen display) would overwrite the hook with older data and erase the game capture hook, causing it to only capture the first frame and then never capture again. This patch ensures that the hook is always saved before calling the next hook in the chain and then restored after the call returns. It also preserves any new hooks that may be added on top of it at any point.
2016-04-14 22:20:41 -07:00
uintptr_t addr;
win-capture: Add hooking functions These functions allow the safe hooking of windows functions, specifically windows API functions that may or may not have built-in machine code to help aid in reverse chain hooks. If a new hook is applied to an existing forward hook, that hook will be preserved to prevent that new hook's data from being removed unintentionally. Hopefully with all these precautions this will reduce the likelihood of crashes and abnormal hook behavior, while allowing existing hooks to be preserved, and allowing new hooks to be applied.
2014-11-10 01:40:15 -08:00
size_t size;
win-capture: Fix case where hook only captures one frame When using a chain hook method (forward or reverse), it was unwisely assumed that the previous hook in the chain would not overwrite new hooks when it's called. When the game capture hook calls the previous hook in the chain, certain other programs that hook (in this case, rivatuner on-screen display) would overwrite the hook with older data and erase the game capture hook, causing it to only capture the first frame and then never capture again. This patch ensures that the hook is always saved before calling the next hook in the chain and then restored after the call returns. It also preserves any new hooks that may be added on top of it at any point.
2016-04-14 22:20:41 -07:00
if (!hook->hooked)
win-capture: Add hooking functions These functions allow the safe hooking of windows functions, specifically windows API functions that may or may not have built-in machine code to help aid in reverse chain hooks. If a new hook is applied to an existing forward hook, that hook will be preserved to prevent that new hook's data from being removed unintentionally. Hopefully with all these precautions this will reduce the likelihood of crashes and abnormal hook behavior, while allowing existing hooks to be preserved, and allowing new hooks to be applied.
2014-11-10 01:40:15 -08:00
return;
win-capture: Fix case where hook only captures one frame When using a chain hook method (forward or reverse), it was unwisely assumed that the previous hook in the chain would not overwrite new hooks when it's called. When the game capture hook calls the previous hook in the chain, certain other programs that hook (in this case, rivatuner on-screen display) would overwrite the hook with older data and erase the game capture hook, causing it to only capture the first frame and then never capture again. This patch ensures that the hook is always saved before calling the next hook in the chain and then restored after the call returns. It also preserves any new hooks that may be added on top of it at any point.
2016-04-14 22:20:41 -07:00
if (hook->type == HOOKTYPE_REVERSE_CHAIN) {
size = JMP_32_SIZE;
addr = (hook->func_addr - JMP_32_SIZE);
} else {
size = patch_size(hook);
addr = hook->func_addr;
}
fix_permissions((void*)addr, size);
memcpy(hook->rehook_data, (void*)addr, size);
if (hook->type == HOOKTYPE_FORWARD_OVERWRITE)
memcpy((void*)hook->func_addr, hook->unhook_data, size);
win-capture: Add hooking functions These functions allow the safe hooking of windows functions, specifically windows API functions that may or may not have built-in machine code to help aid in reverse chain hooks. If a new hook is applied to an existing forward hook, that hook will be preserved to prevent that new hook's data from being removed unintentionally. Hopefully with all these precautions this will reduce the likelihood of crashes and abnormal hook behavior, while allowing existing hooks to be preserved, and allowing new hooks to be applied.
2014-11-10 01:40:15 -08:00
hook->hooked = false;
}