Merge pull request #318 from qarmin/master

Add test CI and fuzzer

.github/workflows/fuzzer.yml

@@ -0,0 +1,166 @@
+name: 🐺️ Fuzzer
+on: 
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 0 * * *'
+
+# Global Cache Settings
+env:
+  GODOT_BASE_BRANCH: 3.x
+  SCONS_CACHE_LIMIT: 4096
+
+jobs:
+  test-project:
+    runs-on: "ubuntu-20.04"
+    name: Editor
+
+    steps:
+      # Clone Godot
+      - uses: actions/checkout@v2
+        with:
+          repository: godotengine/godot
+          ref: 3.x
+
+      # Clone our module under the correct directory
+      - uses: actions/checkout@v2
+        with:
+          path: modules/voxel
+
+      # Clone fuzzer
+      - name: Checkout Goost fuzzer backend
+        uses: actions/checkout@v2
+        with:
+          repository: qarmin/Qarminer
+          ref: 3.x
+          path: fuzzer
+
+      # Currently configuration is done by modifing source code
+      - name: Configure fuzzer
+        working-directory: fuzzer
+        run: |
+          sed -i 's/var use_parent_methods: bool = false/var use_parent_methods: bool = true/' FunctionExecutor.gd
+          sed -i 's/var add_to_tree: bool = false/var add_to_tree: bool = true/' FunctionExecutor.gd
+          sed -i 's/var delay_removing_added_nodes_to_next_frame: bool = false/var delay_removing_added_nodes_to_next_frame: bool = true/' FunctionExecutor.gd
+          echo "FastNoiseLite" >> classes.txt
+          echo "FastNoiseLiteGradient" >> classes.txt
+          echo "Voxel" >> classes.txt
+          echo "VoxelBlockSerializer" >> classes.txt
+          echo "VoxelBoxMover" >> classes.txt
+          echo "VoxelBuffer" >> classes.txt
+          echo "VoxelColorPalette" >> classes.txt
+          echo "VoxelGenerator" >> classes.txt
+          echo "VoxelGeneratorFlat" >> classes.txt
+          echo "VoxelGeneratorGraph" >> classes.txt
+          echo "VoxelGeneratorHeightmap" >> classes.txt
+          echo "VoxelGeneratorImage" >> classes.txt
+          echo "VoxelGeneratorNoise" >> classes.txt
+          echo "VoxelGeneratorNoise2D" >> classes.txt
+          echo "VoxelGeneratorScript" >> classes.txt
+          echo "VoxelGeneratorWaves" >> classes.txt
+          echo "VoxelInstanceComponent" >> classes.txt
+          echo "VoxelInstanceGenerator" >> classes.txt
+          echo "VoxelInstanceLibrary" >> classes.txt
+          echo "VoxelInstanceLibraryItem" >> classes.txt
+          echo "VoxelInstanceLibraryItemBase" >> classes.txt
+          echo "VoxelInstanceLibrarySceneItem" >> classes.txt
+          echo "VoxelInstancer" >> classes.txt
+          echo "VoxelLibrary" >> classes.txt
+          echo "VoxelLodTerrain" >> classes.txt
+          echo "VoxelMesher" >> classes.txt
+          echo "VoxelMesherBlocky" >> classes.txt
+          echo "VoxelMesherCubes" >> classes.txt
+          echo "VoxelMesherDMC" >> classes.txt
+          echo "VoxelMesherTransvoxel" >> classes.txt
+          echo "VoxelNode" >> classes.txt
+          echo "VoxelRaycastResult" >> classes.txt
+          echo "VoxelServer" >> classes.txt
+          echo "VoxelStream" >> classes.txt
+          echo "VoxelStreamBlockFiles" >> classes.txt
+          echo "VoxelStreamRegionFiles" >> classes.txt
+          echo "VoxelStreamSQLite" >> classes.txt
+          echo "VoxelStreamScript" >> classes.txt
+          echo "VoxelTerrain" >> classes.txt
+          echo "VoxelTool" >> classes.txt
+          echo "VoxelToolBuffer" >> classes.txt
+          echo "VoxelToolLodTerrain" >> classes.txt
+          echo "VoxelToolTerrain" >> classes.txt
+          echo "VoxelViewer" >> classes.txt
+          echo "VoxelVoxLoader" >> classes.txt
+
+      # Azure repositories are not reliable, we need to prevent azure giving us packages.
+      - name: Make apt sources.list use the default Ubuntu repositories
+        run: |
+          sudo rm -f /etc/apt/sources.list.d/*
+          sudo cp -f misc/ci/sources.list /etc/apt/sources.list
+          sudo apt-get update
+
+      # Install all packages (except scons)
+      - name: Configure dependencies
+        run: |
+          sudo apt-get install build-essential pkg-config libx11-dev libxcursor-dev \
+            libxinerama-dev libgl1-mesa-dev libglu-dev libasound2-dev libpulse-dev libudev-dev libxi-dev libxrandr-dev yasm \
+            xvfb wget unzip
+
+      # Upload cache on completion and check it out now
+      - name: Load .scons_cache directory
+        id: fuzzer-cache
+        uses: actions/cache@v2
+        with:
+          path: ${{github.workspace}}/.scons_cache/
+          key: ${{github.job}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+          restore-keys: |
+            ${{github.job}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+            ${{github.job}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}
+            ${{github.job}}-${{env.GODOT_BASE_BRANCH}}
+
+      # Use python 3.x release (works cross platform; best to keep self contained in it's own step)
+      - name: Set up Python 3.x
+        uses: actions/setup-python@v2
+        with:
+          # Semantic version range syntax or exact version of a Python version
+          python-version: '3.x'
+          # Optional - x64 or x86 architecture, defaults to x64
+          architecture: 'x64'
+
+      # Setup scons, print python version and scons version info, so if anything is broken it won't run the build.
+      - name: Configuring Python packages
+        run: |
+          python -c "import sys; print(sys.version)"
+          python -m pip install scons
+          python --version
+          scons --version
+
+      # We should always be explicit with our flags usage here since it's gonna be sure to always set those flags
+      - name: Compilation
+        env:
+          SCONS_CACHE: ${{github.workspace}}/.scons_cache/
+        run: |
+          scons -j2 use_asan=yes use_ubsan=yes
+
+      - name: Run fuzzer
+        run: |
+          DRI_PRIME=0 xvfb-run bin/godot.x11.tools.64s FunctionExecutor.tscn 600 --audio-driver Dummy --video-driver GLES3 --path fuzzer 2>&1 | tee sanitizers_log.txt || true
+          tail -n 300 sanitizers_log.txt > project_results.txt
+
+      - name: Store project results
+        uses: actions/upload-artifact@v2
+        with:
+          name: project-results
+          path: project_results.txt
+
+      - name: Store test results
+        uses: actions/upload-artifact@v2
+        with:
+          name: last-run-results
+          path: fuzzer/results.txt
+
+      - name: Store time results
+        uses: actions/upload-artifact@v2
+        with:
+          name: timer
+          path: fuzzer/timer.txt
+
+      - name: Check fuzzer output
+        run: |
+          fuzzer/misc/check_ci_log.py sanitizers_log.txt

.github/workflows/test_project.yml

@@ -0,0 +1,97 @@
+name: ✈️ Test Project
+on:
+  push:
+    branches: [ master, github_actions ]
+  pull_request:
+    branches: [ master ]
+
+# Global Cache Settings
+env:
+  GODOT_BASE_BRANCH: 3.x
+  SCONS_CACHE_LIMIT: 4096
+
+jobs:
+  test-project:
+    runs-on: "ubuntu-20.04"
+    name: Editor
+
+    steps:
+      # Clone Godot
+      - uses: actions/checkout@v2
+        with:
+          repository: godotengine/godot
+          ref: 3.x
+
+      # Clone our module under the correct directory
+      - uses: actions/checkout@v2
+        with:
+          path: modules/voxel
+
+      # Azure repositories are not reliable, we need to prevent azure giving us packages.
+      - name: Make apt sources.list use the default Ubuntu repositories
+        run: |
+          sudo rm -f /etc/apt/sources.list.d/*
+          sudo cp -f misc/ci/sources.list /etc/apt/sources.list
+          sudo apt-get update
+
+      # Install all packages (except scons)
+      - name: Configure dependencies
+        run: |
+          sudo apt-get install build-essential pkg-config libx11-dev libxcursor-dev \
+            libxinerama-dev libgl1-mesa-dev libglu-dev libasound2-dev libpulse-dev libudev-dev libxi-dev libxrandr-dev yasm \
+            xvfb wget unzip
+
+      # Upload cache on completion and check it out now
+      - name: Load .scons_cache directory
+        id: test-project-cache
+        uses: actions/cache@v2
+        with:
+          path: ${{github.workspace}}/.scons_cache/
+          key: ${{github.job}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+          restore-keys: |
+            ${{github.job}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+            ${{github.job}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}
+            ${{github.job}}-${{env.GODOT_BASE_BRANCH}}
+
+      # Use python 3.x release (works cross platform; best to keep self contained in it's own step)
+      - name: Set up Python 3.x
+        uses: actions/setup-python@v2
+        with:
+          # Semantic version range syntax or exact version of a Python version
+          python-version: '3.x'
+          # Optional - x64 or x86 architecture, defaults to x64
+          architecture: 'x64'
+
+      # Setup scons, print python version and scons version info, so if anything is broken it won't run the build.
+      - name: Configuring Python packages
+        run: |
+          python -c "import sys; print(sys.version)"
+          python -m pip install scons
+          python --version
+          scons --version
+
+      # We should always be explicit with our flags usage here since it's gonna be sure to always set those flags
+      - name: Compilation
+        env:
+          SCONS_CACHE: ${{github.workspace}}/.scons_cache/
+        run: |
+          scons -j2 use_asan=yes use_ubsan=yes
+
+      # Download and extract zip archive with project, folder is renamed to be able to easy change used project
+      - name: Download test project
+        run: |
+          wget https://github.com/qarmin/RegressionTestProject/archive/3.x.zip
+          unzip 3.x.zip
+          mv "RegressionTestProject-3.x" "test_project"
+
+      # Editor is quite complicated piece of software, so it is easy to introduce bug here
+      - name: Open and close editor
+        run: |
+          DRI_PRIME=0 xvfb-run bin/godot.x11.tools.64s --audio-driver Dummy -e -q --path test_project 2>&1 | tee sanitizers_log.txt || true
+          modules/voxel/misc/check_ci_log.py sanitizers_log.txt
+
+      # Run test project
+      - name: Run project
+        run: |
+          DRI_PRIME=0 xvfb-run bin/godot.x11.tools.64s 30 --video-driver GLES3 --audio-driver Dummy --path test_project 2>&1 | tee sanitizers_log.txt || true
+          modules/voxel/misc/check_ci_log.py sanitizers_log.txt

misc/check_ci_log.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+
+if len(sys.argv) < 2:
+    print("ERROR: You must run program with file name as argument.")
+    sys.exit(50)
+
+fname = sys.argv[1]
+
+fileread = open(fname.strip(), "r")
+file_contents = fileread.read()
+
+# If find "ERROR: AddressSanitizer:", then happens invalid read or write
+# This is critical bug, so we need to fix this as fast as possible
+# Example - https://github.com/godotengine/godot/issues/48114#issue-865502805
+
+if file_contents.find("ERROR: AddressSanitizer:") != -1:
+    print("FATAL ERROR: An incorrectly used memory was found.")
+    sys.exit(51)
+
+# In test project may be put several assert functions which will control if
+# project is executed with right parameters etc. which normally will not stop
+# execution of project
+# Example - https://github.com/godotengine/godot/issues/37980#issue-602414919
+
+if file_contents.find("Assertion failed") != -1:
+    print(
+        'ERROR: Assertion failed in project, check execution log for more info (search for "Assertion failed" in CI log)'
+    )
+    sys.exit(55)
+
+# There is also possible, that program crashed with or without backtrace.
+# Example - https://github.com/godotengine/godot/issues/51620#issue-970308653
+
+if (
+    file_contents.find("Program crashed with signal") != -1
+    or file_contents.find("Dumping the backtrace") != -1
+    or file_contents.find("Segmentation fault (core dumped)") != -1
+    or file_contents.find("Aborted (core dumped)") != -1
+    or file_contents.find("(core dumped)") != -1
+    or file_contents.find("Aborted") != -1
+    or file_contents.find("Assertion") != -1
+
+):
+    print("FATAL ERROR: Godot has been crashed.")
+    sys.exit(52)
+
+# Finding memory leaks in Godot is quite difficult, because we need to take into
+# account leaks also in external libraries. They are usually provided without
+# debugging symbols, so the leak report from it usually has only 2/3 lines,
+# so searching for 5 element - "#4 0x" - should correctly detect the vast
+# majority of memory leaks
+# Example - https://github.com/godotengine/godot/issues/34495#issue-541154242
+
+if file_contents.find("ERROR: LeakSanitizer:") != -1:
+    if file_contents.find("#4 0x") != -1:
+        print("ERROR: Memory leak was found")
+        sys.exit(53)
+
+# It may happen that Godot detects leaking nodes/resources and removes them, so
+# this possibility should also be handled as a potential error, even if
+# LeakSanitizer doesn't report anything
+# Example - https://github.com/godotengine/godot/issues/49438#issue-915348285
+
+if file_contents.find("ObjectDB instances leaked at exit") != -1:
+    print('ERROR: Memory leak was found (search for "ObjectDB instances leaked at exit" in CI log)')
+    sys.exit(54)
+
+# Sometimes pointers points at objects of different types, which may cause
+# to show errors like "runtime error: member call on address 0x1 which does not point to an object of type"
+# Example - https://github.com/godotengine/godot/issues/51351#issue-963170661
+
+# Waiting for https://github.com/godotengine/godot/issues/51888
+
+if file_contents.find("vptr for") != -1:
+    print('WARNING: Found pointer which not point at valid object (search for "vptr for" in CI log)')
+    sys.exit(56)
+
+# By default overflow or underflow of signed values are in C++ just
+# undefined behavior(most of the time value is wrapped)
+# Example - https://github.com/godotengine/godot/issues/33644#issue-523623547
+
+if file_contents.find("cannot be represented in type") != -1 or file_contents.find("is outside the range") != -1:
+    print(
+        'ERROR: Found pointer which not point at valid object (search for "cannot be represented in type" or "is outside the range" in CI log)'
+    )
+    #sys.exit(57)
+
+# Some functions like memcpy doesn't expect that its argument is null pointer.
+# This may later be cause of bugs or crashes.
+# Example - https://github.com/godotengine/godot/issues/48215#issue-867854743
+
+if file_contents.find("null pointer passed as argument") != -1:
+    print(
+        'ERROR: Found null pointer passed as argument to function which not expect it (search for "null pointer passed as argument" in CI log)'
+    )
+    #sys.exit(58)
+
+# Casting or pointer moving caused that code trying to violate alignement rules
+# Example - https://github.com/godotengine/godot/issues/31203#issue-478487290
+
+if file_contents.find("misaligned address") != -1:
+    print('ERROR: Found usage of misaligned pointer (search for "misaligned address" in CI log)')
+    sys.exit(59)
+
+# For now Godot leaks a lot of rendering stuff so for now we just show info
+# about it and this needs to be re-enabled after fixing this memory leaks.
+# Example - https://github.com/godotengine/godot/issues/47941#issue-859356605
+
+# Blocked by https://github.com/godotengine/godot/issues/46833
+
+if file_contents.find("were leaked") != -1 or file_contents.find("were never freed") != -1:
+    print('WARNING: Memory leak was found (search for "were leaked" or "were never freed" in CI log)')
+    sys.exit(60)
+
+# Usually error about trying to free invalid ID is caused by removing wrong object
+# Example - https://github.com/godotengine/godot/issues/49623#issue-921610423
+
+# Blocked by Swiftshader bugs in CI
+
+if file_contents.find("Attempted to free invalid ID") != -1:
+    print('WARNING: Trying to free invalid object (search for "Attempted to free invalid ID" in CI log)')
+    #sys.exit(61)
+
+
+sys.exit(0)