win-dshow: Fix crashing when using a custom vcam placeholder

The scaler assumed the placeholder was the same size as the camera which
caused crashes if the user replaced the placeholder with a smaller
resolution image (or if the camera was potentially running at > 1080p).
This adds a separate scaler for the placeholder and uses the resolution
of the virtual camera instead of defaulting to 1080p.

plugins/win-dshow/virtualcam-module/placeholder.cpp

@@ -7,7 +7,10 @@
 using namespace Gdiplus;
 
 extern HINSTANCE dll_inst;
+
 static std::vector<uint8_t> placeholder;
+static bool initialized = false;
+int cx, cy;
 
 /* XXX: optimize this later.  or don't, it's only called once. */
 
@@ -101,46 +104,50 @@ static bool load_placeholder_internal()
 		return false;
 	}
 
+	cx = bmp.GetWidth();
+	cy = bmp.GetHeight();
+
 	BitmapData bmd = {};
-	Rect r(0, 0, bmp.GetWidth(), bmp.GetHeight());
+	Rect r(0, 0, cx, cy);
 
 	s = bmp.LockBits(&r, ImageLockModeRead, PixelFormat24bppRGB, &bmd);
 	if (s != Status::Ok) {
 		return false;
 	}
 
-	convert_placeholder((const uint8_t *)bmd.Scan0, bmp.GetWidth(),
-			    bmp.GetHeight());
+	convert_placeholder((const uint8_t *)bmd.Scan0, cx, cy);
 
 	bmp.UnlockBits(&bmd);
 	return true;
 }
 
-static bool load_placeholder()
+bool initialize_placeholder()
 {
 	GdiplusStartupInput si;
 	ULONG_PTR token;
 	GdiplusStartup(&token, &si, nullptr);
 
-	bool success = load_placeholder_internal();
+	initialized = load_placeholder_internal();
 
 	GdiplusShutdown(token);
-	return success;
+	return initialized;
 }
 
-const uint8_t *get_placeholder()
+const uint8_t *get_placeholder_ptr()
 {
-	static bool failed = false;
-	static bool initialized = false;
-
-	if (initialized) {
+	if (initialized)
 		return placeholder.data();
-	} else if (failed) {
+
 	return nullptr;
 }
 
-	initialized = load_placeholder();
-	failed = !initialized;
-
-	return initialized ? placeholder.data() : nullptr;
+const bool get_placeholder_size(int *out_cx, int *out_cy)
+{
+	if (initialized) {
+		*out_cx = cx;
+		*out_cy = cy;
+		return true;
+	}
+
+	return false;
 }

plugins/win-dshow/virtualcam-module/virtualcam-filter.cpp

@@ -7,7 +7,9 @@
 
 using namespace DShow;
 
-extern const uint8_t *get_placeholder();
+extern bool initialize_placeholder();
+extern const uint8_t *get_placeholder_ptr();
+extern const bool get_placeholder_size(int *out_cx, int *out_cy);
 
 /* ========================================================================= */
 
@@ -26,7 +28,12 @@ VCamFilter::VCamFilter()
 	/* ---------------------------------------- */
 	/* load placeholder image                   */
 
-	placeholder = get_placeholder();
+	if (initialize_placeholder()) {
+		placeholder.data = get_placeholder_ptr();
+		get_placeholder_size(&placeholder.cx, &placeholder.cy);
+	} else {
+		placeholder.data = nullptr;
+	}
 
 	/* ---------------------------------------- */
 	/* detect if this filter is within obs      */
@@ -102,6 +109,10 @@ VCamFilter::VCamFilter()
 	}
 
 	nv12_scale_init(&scaler, TARGET_FORMAT_NV12, cx, cy, cx, cy);
+	if (placeholder.data)
+		nv12_scale_init(&placeholder.scaler, TARGET_FORMAT_NV12,
+				GetCX(), GetCY(), placeholder.cx,
+				placeholder.cy);
 
 	/* ---------------------------------------- */
 
@@ -164,6 +175,11 @@ void VCamFilter::Thread()
 
 	nv12_scale_init(&scaler, TARGET_FORMAT_NV12, GetCX(), GetCY(), cx, cy);
 
+	if (placeholder.data)
+		nv12_scale_init(&placeholder.scaler, TARGET_FORMAT_NV12,
+				GetCX(), GetCY(), placeholder.cx,
+				placeholder.cy);
+
 	while (!stopped()) {
 		Frame(filter_time);
 		sleepto_100ns(cur_time += interval);
@@ -195,9 +211,9 @@ void VCamFilter::Frame(uint64_t ts)
 	}
 
 	if (state != SHARED_QUEUE_STATE_READY) {
-		new_cx = DEFAULT_CX;
-		new_cy = DEFAULT_CY;
-		new_interval = DEFAULT_INTERVAL;
+		new_cx = GetCX();
+		new_cy = GetCY();
+		new_interval = GetInterval();
 	}
 
 	if (new_cx != cx || new_cy != cy || new_interval != interval) {
@@ -209,17 +225,22 @@ void VCamFilter::Frame(uint64_t ts)
 		nv12_scale_init(&scaler, TARGET_FORMAT_NV12, GetCX(), GetCY(),
 				new_cx, new_cy);
 
+		if (placeholder.data)
+			nv12_scale_init(&placeholder.scaler, TARGET_FORMAT_NV12,
+					GetCX(), GetCY(), placeholder.cx,
+					placeholder.cy);
+
 		cx = new_cx;
 		cy = new_cy;
 		interval = new_interval;
 	}
 
 	if (GetVideoFormat() == VideoFormat::I420)
-		scaler.format = TARGET_FORMAT_I420;
+		scaler.format = placeholder.scaler.format = TARGET_FORMAT_I420;
 	else if (GetVideoFormat() == VideoFormat::YUY2)
-		scaler.format = TARGET_FORMAT_YUY2;
+		scaler.format = placeholder.scaler.format = TARGET_FORMAT_YUY2;
 	else
-		scaler.format = TARGET_FORMAT_NV12;
+		scaler.format = placeholder.scaler.format = TARGET_FORMAT_NV12;
 
 	uint8_t *ptr;
 	if (LockSampleData(&ptr)) {
@@ -243,8 +264,8 @@ void VCamFilter::ShowOBSFrame(uint8_t *ptr)
 
 void VCamFilter::ShowDefaultFrame(uint8_t *ptr)
 {
-	if (placeholder) {
-		nv12_do_scale(&scaler, ptr, placeholder);
+	if (placeholder.data) {
+		nv12_do_scale(&placeholder.scaler, ptr, placeholder.data);
 	} else {
 		memset(ptr, 127, GetCX() * GetCY() * 3 / 2);
 	}

plugins/win-dshow/virtualcam-module/virtualcam-filter.hpp

@@ -12,6 +12,13 @@
 #define DEFAULT_CY 1080
 #define DEFAULT_INTERVAL 333333ULL
 
+typedef struct {
+	int cx;
+	int cy;
+	nv12_scale_t scaler;
+	const uint8_t *data;
+} placeholder_t;
+
 class VCamFilter : public DShow::OutputFilter {
 	std::thread th;
 
@@ -19,7 +26,7 @@ class VCamFilter : public DShow::OutputFilter {
 	int queue_mode = 0;
 	bool in_obs = false;
 	enum queue_state prev_state = SHARED_QUEUE_STATE_INVALID;
-	const uint8_t *placeholder;
+	placeholder_t placeholder;
 	uint32_t cx = DEFAULT_CX;
 	uint32_t cy = DEFAULT_CY;
 	uint64_t interval = DEFAULT_INTERVAL;