win-capture: Add 64bit bounce to func_hook

When hooking 64bit functions, sometimes the offset between the function
being hooked and the hook itself can be large enough to where it
requires a 64bit offset to be used.  However, because a 64bit jump
requires overwriting so many code instructions in the function, it can
sometimes overwrite code in to an adjacent function, thereby causing a
crash.

The 64bit hook bounce (created by R1CH) is designed to prevent using
very long jumps in the target by creating executable memory within a
32bit offset of that target, and then writing it with the 64bit long
jump instruction instead.  Then in the target function, it will jump to
that memory instead, thus forcing the actual hooked function to use a
32bit hook instead of a 64bit hook, and using at most 5 bytes for the
actual hook, preventing any likelihood of it overwriting an adjacent
function.

plugins/win-capture/funchook.c

@@ -141,6 +141,75 @@ static inline void rehook32(struct func_hook *hook, bool force, intptr_t offset)
 	}
 }
 
+/*
+ * Creates memory close to the target function, used to force the actual hook
+ * to use a 32bit jump instead of a 64bit jump, thus preventing the chance of
+ * overwriting adjacent functions, which can cause a crash.  (by R1CH)
+ */
+static void setup_64bit_bounce(struct func_hook *hook)
+{
+	MEMORY_BASIC_INFORMATION mbi;
+	uintptr_t address;
+	uintptr_t newdiff;
+	SYSTEM_INFO si;
+	bool success;
+	int pagesize;
+	int i;
+
+	success = VirtualQueryEx(GetCurrentProcess(),
+			(const void*)hook->func_addr, &mbi, sizeof(mbi));
+	if (!success)
+		return;
+
+	GetSystemInfo(&si);
+	pagesize = (int)si.dwAllocationGranularity;
+
+	address = (uintptr_t)mbi.AllocationBase - pagesize;
+	for (i = 0; i < 256; i++, address -= pagesize) {
+		hook->bounce_addr = VirtualAlloc((LPVOID)address, pagesize,
+				MEM_RESERVE | MEM_COMMIT,
+				PAGE_EXECUTE_READWRITE);
+		if (hook->bounce_addr)
+			break;
+	}
+
+	if (!hook->bounce_addr) {
+		address = (uintptr_t)mbi.AllocationBase + mbi.RegionSize +
+			pagesize;
+		for (i = 0; i < 256; i++, address += pagesize) {
+			hook->bounce_addr = VirtualAlloc((LPVOID)address,
+					pagesize, MEM_RESERVE | MEM_COMMIT,
+					PAGE_EXECUTE_READWRITE);
+			if (hook->bounce_addr)
+				break;
+		}
+	}
+
+	if (!hook->bounce_addr)
+		return;
+
+	if ((hook->func_addr + 5) > (uintptr_t)hook->bounce_addr)
+		newdiff = hook->func_addr + 5 - (uintptr_t)hook->bounce_addr;
+	else
+		newdiff = (uintptr_t)hook->bounce_addr - hook->func_addr + 5;
+
+	if (newdiff <= 0x7ffffff0) {
+		uint8_t *addr = (uint8_t*)hook->bounce_addr;
+		intptr_t offset;
+
+		FillMemory(hook->bounce_addr, pagesize, 0xCC);
+
+		*(addr++) = 0xFF;
+		*(addr++) = 0x25;
+		*((uint32_t*)addr) = 0;
+		*((uint64_t*)(addr + 4)) = hook->hook_addr;
+
+		hook->hook_addr = (uint64_t)hook->bounce_addr;
+		offset = hook->hook_addr - hook->func_addr - JMP_32_SIZE;
+		hook->is_64bit_jump = (llabs(offset) >= 0x7ffffff0);
+	}
+}
+
 void do_hook(struct func_hook *hook, bool force)
 {
 	intptr_t offset;
@@ -165,8 +234,15 @@ void do_hook(struct func_hook *hook, bool force)
 	hook->is_64bit_jump = (llabs(offset) >= 0x7fffffff);
 
 	if (hook->is_64bit_jump) {
-		rehook64(hook);
-		return;
+		if (!hook->attempted_bounce) {
+			hook->attempted_bounce = true;
+			setup_64bit_bounce(hook);
+		}
+
+		if (hook->is_64bit_jump) {
+			rehook64(hook);
+			return;
+		}
 	}
 #endif