Turn debugger off in programs launched by the program being debugged (#9594)

* Undefine the CAML_DEBUG_SOCKET variable early

So that if the debugged program creates or executes another program
that happens to be an OCaml bytecode executable, said program does
not try to connect to the debugger at beginning of execution.

Fixes: #8678

* Check availability of setenv() and unsetenv()

And guard the use of unsetenv() in runtime/debugger.c.

Changes

@@ -309,6 +309,13 @@ OCaml 4.11
 
 ### Tools:
 
+- #6969: Argument -nocwd added to ocamldep
+  (Muskan Garg, review by Florian Angeletti)
+
+- #8676, #9594: turn debugger off in programs launched by the program
+  being debugged
+  (Xavier Leroy, report by Michael Soegtrop, review by Gabriel Scherer)
+
 - #9057: aid debugging the debugger by preserving backtraces of unhandled
   exceptions.
   (David Allsopp, review by Gabriel Scherer)
@@ -338,9 +345,6 @@ OCaml 4.11
   to the toplevel.
   (Gabriel Scherer, review by Armaël Guéneau)
 
-- #6969: Argument -nocwd added to ocamldep
-  (Muskan Garg, review by Florian Angeletti)
-
 - #9207, #9210: fix ocamlyacc to work correctly with up to 255 entry
   points to the grammar.
   (Andreas Abel, review by Xavier Leroy)

configure

@@ -14903,6 +14903,19 @@ if test "x$ac_cv_func_putenv" = xyes; then :
 fi
 
 
+## setenv and unsetenv
+
+ac_fn_c_check_func "$LINENO" "setenv" "ac_cv_func_setenv"
+if test "x$ac_cv_func_setenv" = xyes; then :
+  ac_fn_c_check_func "$LINENO" "unsetenv" "ac_cv_func_unsetenv"
+if test "x$ac_cv_func_unsetenv" = xyes; then :
+  $as_echo "#define HAS_SETENV_UNSETENV 1" >>confdefs.h
+
+fi
+
+fi
+
+
 ## newlocale() and <locale.h>
 # Note: the detection fails on msvc so we hardcode the result
 # (should be debugged later)

configure.ac

@@ -1429,6 +1429,11 @@ AS_CASE([$host],
 
 AC_CHECK_FUNC([putenv], [AC_DEFINE([HAS_PUTENV])])
 
+## setenv and unsetenv
+
+AC_CHECK_FUNC([setenv],
+  [AC_CHECK_FUNC([unsetenv], [AC_DEFINE([HAS_SETENV_UNSETENV])])])
+
 ## newlocale() and <locale.h>
 # Note: the detection fails on msvc so we hardcode the result
 # (should be debugged later)

runtime/caml/s.h.in

@@ -188,6 +188,10 @@
 
 /* Define HAS_PUTENV if you have putenv(). */
 
+#undef HAS_SETENV_UNSETENV
+
+/* Define HAS_SETENV_UNSETENV if you have setenv() and unsetenv(). */
+
 #undef HAS_LOCALE_H
 
 /* Define HAS_LOCALE_H if you have the include file <locale.h> and the

runtime/debugger.c

@@ -53,6 +53,7 @@ void caml_debugger_cleanup_fork(void)
 #include <unistd.h>
 #endif
 #include <errno.h>
+#include <stdlib.h>
 #include <sys/types.h>
 #ifndef _WIN32
 #include <sys/wait.h>
@@ -188,6 +189,15 @@ void caml_debugger_init(void)
   if (dbg_addr != NULL) caml_stat_free(dbg_addr);
   dbg_addr = address;
 
+  /* #8676: erase the CAML_DEBUG_SOCKET variable so that processes
+     created by the program being debugged do not try to connect with
+     the debugger. */
+#if defined(_WIN32)
+  _wputenv(L"CAML_DEBUG_SOCKET=");
+#elif defined(HAS_SETENV_UNSETENV)
+  unsetenv("CAML_DEBUG_SOCKET");
+#endif
+
   caml_ext_table_init(&breakpoints_table, 16);
 
 #ifdef _WIN32