libobs: Fix various alpha issues

There are cases where alpha is multiplied unnecessarily. This change
attempts to use premultiplied alpha blending for composition.

To keep this change simple, The filter chain will continue to use
straight alpha. Otherwise, every source would need to modified to output
premultiplied, and every filter modified for premultiplied input.

"DrawAlphaDivide" shader techniques have been added to convert from
premultiplied alpha to straight alpha for final output. "DrawMatrix"
techniques ignore alpha, so they do not appear to need changing.

One remaining issue is that scale effects are set up here to use the
same shader logic for both scale filters (straight alpha - incorrectly),
and output composition (premultiplied alpha - correctly). A fix could be
made to add additional shaders for straight alpha, but the "real" fix
may be to eliminate the straight alpha path at some point.

For graphics, SrcBlendAlpha and DestBlendAlpha were both ONE, and could
combine together to form alpha values greater than one. This is not as
noticeable of a problem for UNORM targets because the channels are
clamped, but it will likely become a problem in more situations if FLOAT
targets are used.

This change switches DestBlendAlpha to INVSRCALPHA. The blending
behavior of stacked transparents is preserved without overflowing the
alpha channel.

obs-transitions: Use premultiplied alpha blend, and simplify shaders
because both inputs and outputs use premultiplied alpha now.

Fixes https://obsproject.com/mantis/view.php?id=1108

plugins/obs-transitions/data/fade_to_color_transition.effect

@@ -14,8 +14,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -26,7 +24,8 @@ VertData VSDefault(VertData v_in)
 
 float4 PSFadeToColor(VertData v_in) : TARGET
 {
-	return lerp(convert_pmalpha(tex.Sample(textureSampler, v_in.uv)), color, swp);
+	float4 premultiplied = float4(color.rgb * color.a, color.a);
+	return lerp(tex.Sample(textureSampler, v_in.uv), premultiplied, swp);
 }
 
 technique FadeToColor

plugins/obs-transitions/data/fade_transition.effect

@@ -14,8 +14,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -26,8 +24,8 @@ VertData VSDefault(VertData v_in)
 
 float4 PSFade(VertData v_in) : TARGET
 {
-	float4 a_val = convert_pmalpha(tex_a.Sample(textureSampler, v_in.uv));
-	float4 b_val = convert_pmalpha(tex_b.Sample(textureSampler, v_in.uv));
+	float4 a_val = tex_a.Sample(textureSampler, v_in.uv);
+	float4 b_val = tex_b.Sample(textureSampler, v_in.uv);
 	return lerp(a_val, b_val, fade_val);
 }
 

plugins/obs-transitions/data/luma_wipe_transition.effect

@@ -20,8 +20,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -33,8 +31,8 @@ VertData VSDefault(VertData v_in)
 float4 PSLumaWipe(VertData v_in) : TARGET
 {
 	float2 uv = v_in.uv;
-	float4 a_color = convert_pmalpha(a_tex.Sample(textureSampler, uv));
-	float4 b_color = convert_pmalpha(b_tex.Sample(textureSampler, uv));
+	float4 a_color = a_tex.Sample(textureSampler, uv);
+	float4 b_color = b_tex.Sample(textureSampler, uv);
 	float luma     = l_tex.Sample(textureSampler, uv).x;
 
 	if (invert)

plugins/obs-transitions/data/premultiplied.inc

@@ -1,9 +0,0 @@
-float4 convert_pmalpha(float4 color)
-{
-	float4 ret = color;
-	if (color.a >= 0.001)
-		ret.xyz /= color.a;
-	else
-		ret = float4(0.0, 0.0, 0.0, 0.0);
-	return ret;
-}

plugins/obs-transitions/data/slide_transition.effect

@@ -16,8 +16,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -37,7 +35,7 @@ float4 PSSlide(VertData v_in) : TARGET
 		   ? tex_b.Sample(textureSampler, tex_b_uv)
 		   : tex_a.Sample(textureSampler, tex_a_uv);
 
-	return convert_pmalpha(outc);
+	return outc;
 }
 
 technique Slide

plugins/obs-transitions/data/swipe_transition.effect

@@ -14,8 +14,6 @@ struct VertData {
 	float2 uv  : TEXCOORD0;
 };
 
-#include "premultiplied.inc"
-
 VertData VSDefault(VertData v_in)
 {
 	VertData vert_out;
@@ -34,7 +32,7 @@ float4 PSSwipe(VertData v_in) : TARGET
 		   ? tex_b.Sample(textureSampler, v_in.uv)
 		   : tex_a.Sample(textureSampler, swipe_uv);
 
-	return convert_pmalpha(outc);
+	return outc;
 }
 
 technique Swipe