From 316f858c68d3ef3a73166191b6f89b7bc3f3387c Mon Sep 17 00:00:00 2001
From: Tatsuyuki Ishi <ishitatsuyuki@gmail.com>
Date: Sun, 7 Nov 2021 20:22:33 +0900
Subject: [PATCH] linux-capture: Fix capturing on software rasterization setups

The GLX_EXT_texture_from_pixmap spec says:

The contents of the texture after the drawable has been bound are defined
as the result of all rendering that has completed before the call to
glXBindTexImageEXT.  In other words, the results of any operation which
has caused damage on the drawable prior to the glXBindTexImageEXT call
will be represented in the texture.

Rendering to the drawable while it is bound to a texture will leave the
contents of the texture in an undefined state.  However, no
synchronization between rendering and texturing is done by GLX.  It is
the application's responsibility to implement any synchronization
required.

In practice, on most systems with a GPU this kept a directly binding to
the framebuffer, which made it work with the previous code. However,
on software rasterization setups using llvmpipe, a blit was done inside
the call to glXBindTexImageEXT. This was the cause of a notoriously
ignored bug where the captured image would "freeze" until the source
was reconfigured.
---
 plugins/linux-capture/xcompcap-main.cpp | 37 ++++++++++++++++++-------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/plugins/linux-capture/xcompcap-main.cpp b/plugins/linux-capture/xcompcap-main.cpp
index 9f969b9ca..27558661a 100644
--- a/plugins/linux-capture/xcompcap-main.cpp
+++ b/plugins/linux-capture/xcompcap-main.cpp
@@ -200,6 +200,7 @@ struct XCompcapMain_private {
 	bool show_cursor = true;
 	bool cursor_outside = false;
 	xcursor_t *cursor = nullptr;
+	bool tick_error_suppressed = false;
 };
 
 XCompcapMain::XCompcapMain(obs_data_t *settings, obs_source_t *source)
@@ -300,14 +301,6 @@ static void xcc_cleanup(XCompcapMain_private *p)
 		GLuint gltex = *(GLuint *)gs_texture_get_obj(p->gltex);
 		glBindTexture(GL_TEXTURE_2D, gltex);
 		if (p->glxpixmap) {
-			glXReleaseTexImageEXT(xdisp, p->glxpixmap,
-					      GLX_FRONT_LEFT_EXT);
-			if (xlock.gotError()) {
-				blog(LOG_ERROR,
-				     "cleanup glXReleaseTexImageEXT failed: %s",
-				     xlock.getErrorText().c_str());
-				xlock.resetError();
-			}
 			glXDestroyPixmap(xdisp, p->glxpixmap);
 			if (xlock.gotError()) {
 				blog(LOG_ERROR,
@@ -398,6 +391,8 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 
 	xcc_cleanup(p);
 
+	p->tick_error_suppressed = false;
+
 	if (settings) {
 		/* Settings initialized or changed */
 		const char *windowName =
@@ -554,7 +549,7 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 				     GS_GL_DUMMYTEX);
 	GLuint gltex = *(GLuint *)gs_texture_get_obj(p->gltex);
 	glBindTexture(GL_TEXTURE_2D, gltex);
-	glXBindTexImageEXT(xdisp, p->glxpixmap, GLX_FRONT_LEFT_EXT, NULL);
+	glXBindTexImageEXT(xdisp, p->glxpixmap, GLX_FRONT_EXT, nullptr);
 	if (xlock.gotError()) {
 		blog(LOG_ERROR, "glXBindTexImageEXT failed: %s",
 		     xlock.getErrorText().c_str());
@@ -568,6 +563,12 @@ void XCompcapMain::updateSettings(obs_data_t *settings)
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
 	// glxBindTexImageEXT might modify the textures format.
 	gs_color_format format = gs_format_from_tex();
+	glXReleaseTexImageEXT(xdisp, p->glxpixmap, GLX_FRONT_EXT);
+	if (xlock.gotError()) {
+		blog(LOG_ERROR, "glXReleaseTexImageEXT failed: %s",
+		     xlock.getErrorText().c_str());
+		xlock.resetError();
+	}
 	glBindTexture(GL_TEXTURE_2D, 0);
 	// sync OBS texture format based on any glxBindTexImageEXT changes
 	p->gltex->format = format;
@@ -620,7 +621,7 @@ void XCompcapMain::tick(float seconds)
 		p->win = 0;
 	}
 
-	XDisplayLock xlock;
+	XErrorLock xlock;
 	XWindowAttributes attr;
 
 	if (!p->win || !XGetWindowAttributes(xdisp, p->win, &attr)) {
@@ -651,6 +652,14 @@ void XCompcapMain::tick(float seconds)
 		XSync(xdisp, 0);
 	}
 
+	glBindTexture(GL_TEXTURE_2D, *(GLuint *)gs_texture_get_obj(p->gltex));
+	glXBindTexImageEXT(xdisp, p->glxpixmap, GLX_FRONT_EXT, nullptr);
+	if (xlock.gotError() && !p->tick_error_suppressed) {
+		blog(LOG_ERROR, "glXBindTexImageEXT failed: %s",
+		     xlock.getErrorText().c_str());
+		p->tick_error_suppressed = true;
+	}
+
 	if (p->include_border) {
 		gs_copy_texture_region(p->tex, 0, 0, p->gltex, p->cur_cut_left,
 				       p->cur_cut_top, width(), height());
@@ -661,6 +670,14 @@ void XCompcapMain::tick(float seconds)
 				       height());
 	}
 
+	glXReleaseTexImageEXT(xdisp, p->glxpixmap, GLX_FRONT_EXT);
+	if (xlock.gotError() && !p->tick_error_suppressed) {
+		blog(LOG_ERROR, "glXReleaseTexImageEXT failed: %s",
+		     xlock.getErrorText().c_str());
+		p->tick_error_suppressed = true;
+	}
+	glBindTexture(GL_TEXTURE_2D, 0);
+
 	if (p->cursor && p->show_cursor) {
 		xcursor_tick(p->cursor);