obs-filters: Apply luma key filter in linear space

Also switch from 601 to 709 luma coefficients to match other filters.

plugins/obs-filters/data/luma_key_filter_v2.effect

@@ -0,0 +1,51 @@
+uniform float4x4 ViewProj;
+uniform texture2d image;
+
+uniform float lumaMax;
+uniform float lumaMin;
+uniform float lumaMaxSmooth;
+uniform float lumaMinSmooth;
+
+sampler_state textureSampler {
+	Filter    = Linear;
+	AddressU  = Clamp;
+	AddressV  = Clamp;
+};
+
+struct VertData {
+	float4 pos : POSITION;
+	float2 uv  : TEXCOORD0;
+};
+
+VertData VSDefault(VertData v_in)
+{
+	VertData vert_out;
+	vert_out.pos = mul(float4(v_in.pos.xyz, 1.0), ViewProj);
+	vert_out.uv  = v_in.uv;
+	return vert_out;
+}
+
+float4 PSALumaKeyRGBA(VertData v_in) : TARGET
+{
+	float4 rgba = image.Sample(textureSampler, v_in.uv);
+
+	float3 lumaCoef = float3(0.2126, 0.7152, 0.0722);
+
+	float luminance = dot(rgba.rgb, lumaCoef);
+
+	float clo = smoothstep(lumaMin, lumaMin + lumaMinSmooth, luminance);
+	float chi = 1. - smoothstep(lumaMax - lumaMaxSmooth, lumaMax, luminance);
+
+	float amask = clo * chi;
+
+	return float4(rgba.rgb, rgba.a * amask);
+}
+
+technique Draw
+{
+	pass
+	{
+		vertex_shader = VSDefault(v_in);
+		pixel_shader  = PSALumaKeyRGBA(v_in);
+	}
+}

plugins/obs-filters/luma-key-filter.c

@@ -66,11 +66,13 @@ static void luma_key_destroy(void *data)
 	bfree(data);
 }
 
-static void *luma_key_create(obs_data_t *settings, obs_source_t *context)
+static void *luma_key_create_internal(obs_data_t *settings,
+				      obs_source_t *context,
+				      const char *effect_name)
 {
 	struct luma_key_filter_data *filter =
 		bzalloc(sizeof(struct luma_key_filter_data));
-	char *effect_path = obs_module_file("luma_key_filter.effect");
+	char *effect_path = obs_module_file(effect_name);
 
 	filter->context = context;
 
@@ -101,7 +103,19 @@ static void *luma_key_create(obs_data_t *settings, obs_source_t *context)
 	return filter;
 }
 
-static void luma_key_render(void *data, gs_effect_t *effect)
+static void *luma_key_create_v1(obs_data_t *settings, obs_source_t *context)
+{
+	return luma_key_create_internal(settings, context,
+					"luma_key_filter.effect");
+}
+
+static void *luma_key_create_v2(obs_data_t *settings, obs_source_t *context)
+{
+	return luma_key_create_internal(settings, context,
+					"luma_key_filter_v2.effect");
+}
+
+static void luma_key_render_internal(void *data, bool srgb)
 {
 	struct luma_key_filter_data *filter = data;
 
@@ -116,9 +130,19 @@ static void luma_key_render(void *data, gs_effect_t *effect)
 	gs_effect_set_float(filter->luma_min_smooth_param,
 			    filter->luma_min_smooth);
 
+	const bool previous = gs_set_linear_srgb(srgb);
 	obs_source_process_filter_end(filter->context, filter->effect, 0, 0);
+	gs_set_linear_srgb(previous);
+}
 
-	UNUSED_PARAMETER(effect);
+static void luma_key_render_v1(void *data, gs_effect_t *effect)
+{
+	luma_key_render_internal(data, false);
+}
+
+static void luma_key_render_v2(void *data, gs_effect_t *effect)
+{
+	luma_key_render_internal(data, true);
 }
 
 static obs_properties_t *luma_key_properties(void *data)
@@ -149,11 +173,25 @@ static void luma_key_defaults(obs_data_t *settings)
 struct obs_source_info luma_key_filter = {
 	.id = "luma_key_filter",
 	.type = OBS_SOURCE_TYPE_FILTER,
-	.output_flags = OBS_SOURCE_VIDEO,
+	.output_flags = OBS_SOURCE_VIDEO | OBS_SOURCE_CAP_OBSOLETE,
 	.get_name = luma_key_name,
-	.create = luma_key_create,
+	.create = luma_key_create_v1,
 	.destroy = luma_key_destroy,
-	.video_render = luma_key_render,
+	.video_render = luma_key_render_v1,
+	.update = luma_key_update,
+	.get_properties = luma_key_properties,
+	.get_defaults = luma_key_defaults,
+};
+
+struct obs_source_info luma_key_filter_v2 = {
+	.id = "luma_key_filter",
+	.version = 2,
+	.type = OBS_SOURCE_TYPE_FILTER,
+	.output_flags = OBS_SOURCE_VIDEO,
+	.get_name = luma_key_name,
+	.create = luma_key_create_v2,
+	.destroy = luma_key_destroy,
+	.video_render = luma_key_render_v2,
 	.update = luma_key_update,
 	.get_properties = luma_key_properties,
 	.get_defaults = luma_key_defaults,

plugins/obs-filters/obs-filters.c

@@ -33,6 +33,7 @@ extern struct obs_source_info compressor_filter;
 extern struct obs_source_info limiter_filter;
 extern struct obs_source_info expander_filter;
 extern struct obs_source_info luma_key_filter;
+extern struct obs_source_info luma_key_filter_v2;
 
 bool obs_module_load(void)
 {
@@ -61,5 +62,6 @@ bool obs_module_load(void)
 	obs_register_source(&limiter_filter);
 	obs_register_source(&expander_filter);
 	obs_register_source(&luma_key_filter);
+	obs_register_source(&luma_key_filter_v2);
 	return true;
 }