Work around LuaJIT issues on aarch64 (#9614)

- Move the text segment below the 47-bit limit,
  needed for script_exception_wrapper which must be lightuserdata
- Replace CUSTOM_RIDX_SCRIPTAPI with full userdata
- Fix FreeBSD aarch64 faulies when use luajit
* FreeBSD uses lld, and lld does not support -Ttext-segment,
  suggesting --image-base instead. Not sure if it's equivalent change
  for the purpose at least if fixes build on FreeBSD/aarch64.
  Note that the code checks for FreeBSD, while it should really check
  for lld on any system, however I don't know any CMake facilities which allow this
* https://codeberg.org/minenux/minetest-engine/issues/16

src/CMakeLists.txt

@@ -706,7 +706,7 @@ endif()
 
 # Set some optimizations and tweaks
 
-include(CheckCXXCompilerFlag)
+include(CheckCSourceCompiles)
 
 if(MSVC)
 	# Visual Studio
@@ -731,9 +731,6 @@ if(MSVC)
 	set(CMAKE_C_FLAGS_RELEASE "/O2 /Ob2 /MT")
 else()
 	# Probably GCC
-	if(APPLE)
-		SET( CMAKE_EXE_LINKER_FLAGS  "${CMAKE_EXE_LINKER_FLAGS} -pagezero_size 10000 -image_base 100000000" )
-	endif()
 	if(WARN_ALL)
 		set(RELEASE_WARNING_FLAGS "-Wall")
 	else()
@@ -746,6 +743,24 @@ else()
 		set(OTHER_FLAGS "${OTHER_FLAGS} -Wsign-compare")
 	endif()
 
+	if(APPLE AND USE_LUAJIT)
+		SET( CMAKE_EXE_LINKER_FLAGS  "${CMAKE_EXE_LINKER_FLAGS} -pagezero_size 10000 -image_base 100000000" )
+	elseif(UNIX AND USE_LUAJIT)
+		check_c_source_compiles("#ifndef __aarch64__\n#error\n#endif\nint main(){}" IS_AARCH64)
+		if(IS_AARCH64)
+			if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+			# FreeBSD uses lld, and lld does not support -Ttext-segment, suggesting
+			# --image-base instead. Not sure if it's equivalent change for the purpose 
+			# but at least if fixes build on FreeBSD/aarch64
+			# XXX: the condition should also be changed to check for lld regardless of
+			# os, bit CMake doesn't have anything like CMAKE_LINKER_IS_LLD yet
+				SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--image-base=0x200000000")
+			else()
+			# Move text segment below LuaJIT's 47-bit limit (see issue #9367)
+				SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-Ttext-segment=0x200000000")
+			endif()
+		endif()
+	endif()
 	if(WIN32 AND NOT ZLIBWAPI_DLL AND CMAKE_SIZEOF_VOID_P EQUAL 4)
 		set(OTHER_FLAGS "${OTHER_FLAGS} -DWIN32_NO_ZLIB_WINAPI")
 		message(WARNING "Defaulting to cdecl for zlib on win32 because ZLIBWAPI_DLL"

src/script/common/c_internal.h

@@ -55,6 +55,15 @@ extern "C" {
 #define CUSTOM_RIDX_CURRENT_MOD_NAME    (CUSTOM_RIDX_BASE + 2)
 #define CUSTOM_RIDX_ERROR_HANDLER       (CUSTOM_RIDX_BASE + 3)
 
+// Determine if CUSTOM_RIDX_SCRIPTAPI will hold a light or full userdata
+#if defined(__aarch64__) && USE_LUAJIT
+/* LuaJIT has a 47-bit limit for lightuserdata on this platform and we cannot
+ * assume that the ScriptApi class was allocated at a fitting address. */
+#define INDIRECT_SCRIPTAPI_RIDX 1
+#else
+#define INDIRECT_SCRIPTAPI_RIDX 0
+#endif
+
 // Pushes the error handler onto the stack and returns its index
 #define PUSH_ERROR_HANDLER(L) \
 	(lua_rawgeti((L), LUA_REGISTRYINDEX, CUSTOM_RIDX_ERROR_HANDLER), lua_gettop((L)))

src/script/cpp_api/s_base.cpp

@@ -87,7 +87,11 @@ ScriptApiBase::ScriptApiBase() :
 	luaL_openlibs(m_luastack);
 
 	// Make the ScriptApiBase* accessible to ModApiBase
+#if INDIRECT_SCRIPTAPI_RIDX
+	*(void **)(lua_newuserdata(m_luastack, sizeof(void *))) = this;
+#else
 	lua_pushlightuserdata(m_luastack, this);
+#endif
 	lua_rawseti(m_luastack, LUA_REGISTRYINDEX, CUSTOM_RIDX_SCRIPTAPI);
 
 	// Add and save an error handler

src/script/cpp_api/s_security.cpp

@@ -499,7 +499,12 @@ bool ScriptApiSecurity::checkPath(lua_State *L, const char *path,
 
 	// Get server from registry
 	lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_SCRIPTAPI);
-	ScriptApiBase *script = (ScriptApiBase *) lua_touserdata(L, -1);
+	ScriptApiBase *script;
+#if INDIRECT_SCRIPTAPI_RIDX
+	script = (ScriptApiBase *) *(void**)(lua_touserdata(L, -1));
+#else
+	script = (ScriptApiBase *) lua_touserdata(L, -1);
+#endif
 	lua_pop(L, 1);
 	const IGameDef *gamedef = script->getGameDef();
 	if (!gamedef)

src/script/lua_api/l_base.cpp

@@ -27,7 +27,12 @@ ScriptApiBase *ModApiBase::getScriptApiBase(lua_State *L)
 {
 	// Get server from registry
 	lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_SCRIPTAPI);
-	ScriptApiBase *sapi_ptr = (ScriptApiBase*) lua_touserdata(L, -1);
+	ScriptApiBase *sapi_ptr;
+#if INDIRECT_SCRIPTAPI_RIDX
+	sapi_ptr = (ScriptApiBase*) *(void**)(lua_touserdata(L, -1));
+#else
+	sapi_ptr = (ScriptApiBase*) lua_touserdata(L, -1);
+#endif
 	lua_pop(L, 1);
 	return sapi_ptr;
 }