Fix #402, negative determinant fails, only 0 or not finite should fail

src/gd_matrix.c

@@ -84,7 +84,10 @@ BGD_DECLARE(int) gdAffineInvert (double dst[6], const double src[6])
 {
 	double r_det = (src[0] * src[3] - src[1] * src[2]);
 
-	if (fabs(r_det) <= 0.0) {
+	if (!isfinite(r_det)) {
+		return GD_FALSE;
+	}
+	if (r_det == 0) {
 		return GD_FALSE;
 	}
 

tests/CMakeLists.txt

@@ -78,6 +78,7 @@ if (BUILD_TEST)
 		gdimagestringup16
 		gdimagetruecolortopalette
 		gdinterpolatedscale
+		gdmatrix
 		gdnewfilectx
 		gdtest
 		gdtiled

tests/gdmatrix/CMakeLists.txt

@@ -0,0 +1,5 @@
+LIST(APPEND TESTS_FILES
+	bug00402
+)
+
+ADD_GD_TESTS()

tests/gdmatrix/bug00402.c

@@ -0,0 +1,30 @@
+#include "gd.h"
+#include <stdio.h>
+#include "gdtest.h"
+#include <math.h>
+int main()
+{
+    double matrix[6] = {
+        0.000000,	0.150000,
+        0.150000,	-0.000000,
+        0.000000,	0.000000
+    };
+    double matrix_inv[6];
+    double matrix_inv_exp[6] = {
+        0.0000000000, 6.6666666667,
+        6.6666666667, -0.0000000000,
+        -0.0000000000, 0.0000000000
+    };
+
+    int res = gdAffineInvert(matrix_inv, matrix);
+
+    for (int i=0; i < 6; i++) {
+        double rounded_res = round(matrix_inv[i] * 10);
+        double rounded_exp = round(matrix_inv_exp[i] * 10);
+        if (rounded_res != rounded_exp) {
+            printf("%i failed %f exp %f", i, matrix_inv[i], matrix_inv_exp[i]);
+            return -1;
+        }
+    }
+	return res;
+}