From 2fa37a1f2e96c14230ac5b422c6001868c50fef0 Mon Sep 17 00:00:00 2001
From: jp9000 <obs.jim@gmail.com>
Date: Thu, 26 Feb 2015 20:01:52 -0800
Subject: [PATCH] libobs-opengl: Fix render targets being flipped

When render targets are used, they output to the render target inverted
due to the way that opengl works.  This fixes that issue by inverting
the projection matrix so that it renders the image upside down and
inverting the front face from counterclockwise to clockwise.
---
 libobs-opengl/gl-subsystem.c         | 21 ++++++++++++++++++---
 libobs/data/format_conversion.effect | 17 -----------------
 2 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/libobs-opengl/gl-subsystem.c b/libobs-opengl/gl-subsystem.c
index 80b5d3229..ddcea29df 100644
--- a/libobs-opengl/gl-subsystem.c
+++ b/libobs-opengl/gl-subsystem.c
@@ -877,10 +877,25 @@ static inline bool can_render(const gs_device_t *device)
 static void update_viewproj_matrix(struct gs_device *device)
 {
 	struct gs_shader *vs = device->cur_vertex_shader;
-	gs_matrix_get(&device->cur_view);
+	struct matrix4 cur_proj;
 
-	matrix4_mul(&device->cur_viewproj, &device->cur_view,
-			&device->cur_proj);
+	gs_matrix_get(&device->cur_view);
+	matrix4_copy(&cur_proj, &device->cur_proj);
+
+	if (device->cur_fbo) {
+		cur_proj.x.y = -cur_proj.x.y;
+		cur_proj.y.y = -cur_proj.y.y;
+		cur_proj.z.y = -cur_proj.z.y;
+		cur_proj.t.y = -cur_proj.t.y;
+
+		glFrontFace(GL_CW);
+	} else {
+		glFrontFace(GL_CCW);
+	}
+
+	gl_success("glFrontFace");
+
+	matrix4_mul(&device->cur_viewproj, &device->cur_view, &cur_proj);
 	matrix4_transpose(&device->cur_viewproj, &device->cur_viewproj);
 
 	if (vs->viewproj)
diff --git a/libobs/data/format_conversion.effect b/libobs/data/format_conversion.effect
index e58996003..74f29d698 100644
--- a/libobs/data/format_conversion.effect
+++ b/libobs/data/format_conversion.effect
@@ -63,11 +63,7 @@ VertInOut VSDefault(VertInOut vert_in)
 
 float4 PSNV12(VertInOut vert_in) : TARGET
 {
-#ifdef _OPENGL
-	float v_mul = floor((1.0 - vert_in.uv.y) * input_height);
-#else
 	float v_mul = floor(vert_in.uv.y * input_height);
-#endif
 
 	float byte_offset = floor((v_mul + vert_in.uv.x) * width) * 4.0;
 	byte_offset += PRECISION_OFFSET;
@@ -127,11 +123,7 @@ float4 PSNV12(VertInOut vert_in) : TARGET
 
 float4 PSPlanar420(VertInOut vert_in) : TARGET
 {
-#ifdef _OPENGL
-	float v_mul = floor((1.0 - vert_in.uv.y) * input_height);
-#else
 	float v_mul = floor(vert_in.uv.y * input_height);
-#endif
 
 	float byte_offset = floor((v_mul + vert_in.uv.x) * width) * 4.0;
 	byte_offset += PRECISION_OFFSET;
@@ -202,9 +194,6 @@ float4 PSPacked422_Reverse(VertInOut vert_in, int u_pos, int v_pos,
 		int y0_pos, int y1_pos) : TARGET
 {
 	float y = vert_in.uv.y;
-#ifdef _OPENGL
-	y = 1. - y;
-#endif
 	float odd = floor(fmod(width * vert_in.uv.x + PRECISION_OFFSET, 2.0));
 	float x = floor(width_d2 * vert_in.uv.x + PRECISION_OFFSET) *
 			width_d2_i;
@@ -233,9 +222,6 @@ float4 PSPlanar420_Reverse(VertInOut vert_in) : TARGET
 {
 	float x = vert_in.uv.x;
 	float y = vert_in.uv.y;
-#ifdef _OPENGL
-	y = 1. - y;
-#endif
 	float x_offset   = floor(x * width  + PRECISION_OFFSET);
 	float y_offset   = floor(y * height + PRECISION_OFFSET);
 
@@ -258,9 +244,6 @@ float4 PSNV12_Reverse(VertInOut vert_in) : TARGET
 {
 	float x = vert_in.uv.x;
 	float y = vert_in.uv.y;
-#ifdef _OPENGL
-	y = 1. - y;
-#endif
 	float x_offset   = floor(x * width  + PRECISION_OFFSET);
 	float y_offset   = floor(y * height + PRECISION_OFFSET);