From bb42c8dfb608da944875cb6370c3fc03da0f39af Mon Sep 17 00:00:00 2001
From: Philip Haynes <singularity.haynes@gmail.com>
Date: Mon, 8 Oct 2018 12:18:40 -0500
Subject: [PATCH] linux-capture: XCompCap now chooses glXFBConfigs based on
 window depth

Rather than inheriting the exact visuals from the window,
find whether the color buffer has alpha, find a config with
the necessary attributes for capture, and match its depth to
the depth of the window instead of the color buffer. Also,
cleanup glxpixmap texture binding before destroying the texture.

Update: Style conformancy changes
---
 plugins/linux-capture/xcompcap-main.cpp | 141 ++++++++++++++----------
 1 file changed, 80 insertions(+), 61 deletions(-)

diff --git a/plugins/linux-capture/xcompcap-main.cpp b/plugins/linux-capture/xcompcap-main.cpp
index df78e3dcc..bbfe2eb0d 100644
--- a/plugins/linux-capture/xcompcap-main.cpp
+++ b/plugins/linux-capture/xcompcap-main.cpp
@@ -252,6 +252,9 @@ static void xcc_cleanup(XCompcapMain_private *p)
 	XDisplayLock xlock;
 
 	if (p->gltex) {
+		GLuint gltex = *(GLuint*)gs_texture_get_obj(p->gltex);
+		glBindTexture(GL_TEXTURE_2D, gltex);
+		glXReleaseTexImageEXT(xdisp, p->glxpixmap, GLX_FRONT_LEFT_EXT);
 		gs_texture_destroy(p->gltex);
 		p->gltex = 0;
 	}
@@ -351,18 +354,88 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 
 	bool has_alpha = true;
 
-	if (attr.depth < 32) {
-		cf = GS_BGRX;
-		has_alpha = false;
+	const int attrs[] =
+	{
+		GLX_BIND_TO_TEXTURE_RGBA_EXT, GL_TRUE,
+		GLX_DRAWABLE_TYPE, GLX_PIXMAP_BIT,
+		GLX_BIND_TO_TEXTURE_TARGETS_EXT, GLX_TEXTURE_2D_BIT_EXT,
+		GLX_ALPHA_SIZE, 8,
+		GLX_DOUBLEBUFFER, GL_FALSE,
+		None
+	};
+
+	int nelem = 0;
+	GLXFBConfig *configs = glXGetFBConfigs(xdisp,
+			XCompcap::getRootWindowScreen(attr.root),
+			&nelem);
+
+	if (nelem <= 0) {
+		blog(LOG_ERROR, "no fb configs available");
+		p->win = 0;
+		p->height = 0;
+		p->width = 0;
+		return;
 	}
 
-	if (cf == GS_BGRX) {
-		if (settings) {
-			p->swapRedBlue = !p->swapRedBlue;
+	GLXFBConfig config;
+	for (int i = 0; i < nelem; i++) {
+		config = configs[i];
+		XVisualInfo *visual = glXGetVisualFromFBConfig(xdisp, config);
+		if (!visual)
+			continue;
+
+		if (attr.visual->visualid != visual->visualid) {
+			XFree(visual);
+			continue;
 		}
+		XFree(visual);
+
+		int value;
+		glXGetFBConfigAttrib(xdisp, config, GLX_ALPHA_SIZE, &value);
+		if (value != 8)
+			has_alpha = false;
+
+		break;
+	}
+
+	XFree(configs);
+	configs = glXChooseFBConfig(xdisp,
+			XCompcap::getRootWindowScreen(attr.root),
+			attrs, &nelem);
+
+	if (nelem <= 0) {
+		blog(LOG_ERROR, "no matching fb config found");
+		p->win = 0;
+		p->height = 0;
+		p->width = 0;
+		return;
+	}
+	bool found = false;
+	for (int i = 0; i < nelem; i++) {
+		config = configs[i];
+		XVisualInfo *visual = glXGetVisualFromFBConfig(xdisp, config);
+		if (!visual)
+			continue;
+
+		if (attr.depth != visual->depth) {
+			XFree(visual);
+			continue;
+		}
+		XFree(visual);
+		found = true;
+		break;
+	}
+	if (!found)
+		config = configs[0];
+
+	if (cf == GS_BGRX || !has_alpha) {
 		p->draw_opaque = true;
 	}
 
+	int inverted;
+	glXGetFBConfigAttrib(xdisp, config, GLX_Y_INVERTED_EXT, &inverted);
+	p->inverted = inverted != 0;
+
 	p->border = attr.border_width;
 
 	if (p->include_border) {
@@ -411,60 +484,6 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 		glBindTexture(GL_TEXTURE_2D, 0);
 	}
 
-	const int attrs_alpha[] =
-	{
-		GLX_BIND_TO_TEXTURE_RGBA_EXT, GL_TRUE,
-		GLX_DRAWABLE_TYPE, GLX_PIXMAP_BIT,
-		GLX_BIND_TO_TEXTURE_TARGETS_EXT, GLX_TEXTURE_2D_BIT_EXT,
-		GLX_ALPHA_SIZE, 8,
-		None
-	};
-
-	const int attrs_no_alpha[] =
-	{
-		GLX_BIND_TO_TEXTURE_RGB_EXT, GL_TRUE,
-		GLX_DRAWABLE_TYPE, GLX_PIXMAP_BIT,
-		GLX_BIND_TO_TEXTURE_TARGETS_EXT, GLX_TEXTURE_2D_BIT_EXT,
-		None
-	};
-
-	const int *attrs = has_alpha ? attrs_alpha : attrs_no_alpha;
-
-	int nelem = 0;
-	GLXFBConfig* configs = glXChooseFBConfig(xdisp,
-			XCompcap::getRootWindowScreen(attr.root),
-			attrs, &nelem);
-
-	if (nelem <= 0) {
-		blog(LOG_ERROR, "no matching fb config found");
-		p->win = 0;
-		p->height = 0;
-		p->width = 0;
-		return;
-	}
-
-	GLXFBConfig config;
-	bool found = false;
-	for (int i = 0; i < nelem; i++) {
-		config = configs[i];
-		XVisualInfo *visual = glXGetVisualFromFBConfig(xdisp, config);
-		if (attr.visual->visualid == visual->visualid) {
-			found = true;
-			XFree(visual);
-			break;
-		}
-		XFree(visual);
-	}
-
-	if (!found) {
-		config = configs[0];
-		p->draw_opaque = true;
-	}
-
-	int inverted;
-	glXGetFBConfigAttrib(xdisp, config, GLX_Y_INVERTED_EXT, &inverted);
-	p->inverted = inverted != 0;
-
 	xlock.resetError();
 
 	p->pixmap = XCompositeNameWindowPixmap(xdisp, p->win);
@@ -521,7 +540,7 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 				"\ttitle: %s\n"
 				"\tclass: %s\n"
 				"\tHas alpha: %s\n"
-				"\tFound exact GLXFBConfig: %s",
+				"\tFound proper GLXFBConfig: %s\n",
 				obs_source_get_name(p->source),
 				XCompcap::getWindowName(p->win).c_str(),
 				XCompcap::getWindowClass(p->win).c_str(),