convert debug safety tests to zig build system

build.zig

@@ -39,4 +39,5 @@ pub fn build(b: &Builder) {
     test_step.dependOn(tests.addBuildExampleTests(b, test_filter));
     test_step.dependOn(tests.addCompileErrorTests(b, test_filter));
     test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter));
+    test_step.dependOn(tests.addDebugSafetyTests(b, test_filter));
 }

test/debug_safety.zig

@@ -0,0 +1,234 @@
+const tests = @import("tests.zig");
+
+pub fn addCases(cases: &tests.CompareOutputContext) {
+    cases.addDebugSafety("calling panic",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\pub fn main() -> %void {
+        \\    @panic("oh no");
+        \\}
+    );
+
+    cases.addDebugSafety("out of bounds slice access",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\pub fn main() -> %void {
+        \\    const a = []i32{1, 2, 3, 4};
+        \\    baz(bar(a));
+        \\}
+        \\fn bar(a: []const i32) -> i32 {
+        \\    a[4]
+        \\}
+        \\fn baz(a: i32) { }
+    );
+
+    cases.addDebugSafety("integer addition overflow",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    const x = add(65530, 10);
+        \\    if (x == 0) return error.Whatever;
+        \\}
+        \\fn add(a: u16, b: u16) -> u16 {
+        \\    a + b
+        \\}
+    );
+
+    cases.addDebugSafety("integer subtraction overflow",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    const x = sub(10, 20);
+        \\    if (x == 0) return error.Whatever;
+        \\}
+        \\fn sub(a: u16, b: u16) -> u16 {
+        \\    a - b
+        \\}
+    );
+
+    cases.addDebugSafety("integer multiplication overflow",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    const x = mul(300, 6000);
+        \\    if (x == 0) return error.Whatever;
+        \\}
+        \\fn mul(a: u16, b: u16) -> u16 {
+        \\    a * b
+        \\}
+    );
+
+    cases.addDebugSafety("integer negation overflow",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    const x = neg(-32768);
+        \\    if (x == 32767) return error.Whatever;
+        \\}
+        \\fn neg(a: i16) -> i16 {
+        \\    -a
+        \\}
+    );
+
+    cases.addDebugSafety("signed integer division overflow",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    const x = div(-32768, -1);
+        \\    if (x == 32767) return error.Whatever;
+        \\}
+        \\fn div(a: i16, b: i16) -> i16 {
+        \\    a / b
+        \\}
+    );
+
+    cases.addDebugSafety("signed shift left overflow",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    const x = shl(-16385, 1);
+        \\    if (x == 0) return error.Whatever;
+        \\}
+        \\fn shl(a: i16, b: i16) -> i16 {
+        \\    a << b
+        \\}
+    );
+
+    cases.addDebugSafety("unsigned shift left overflow",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    const x = shl(0b0010111111111111, 3);
+        \\    if (x == 0) return error.Whatever;
+        \\}
+        \\fn shl(a: u16, b: u16) -> u16 {
+        \\    a << b
+        \\}
+    );
+
+    cases.addDebugSafety("integer division by zero",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    const x = div0(999, 0);
+        \\}
+        \\fn div0(a: i32, b: i32) -> i32 {
+        \\    a / b
+        \\}
+    );
+
+    cases.addDebugSafety("exact division failure",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    const x = divExact(10, 3);
+        \\    if (x == 0) return error.Whatever;
+        \\}
+        \\fn divExact(a: i32, b: i32) -> i32 {
+        \\    @divExact(a, b)
+        \\}
+    );
+
+    cases.addDebugSafety("cast []u8 to bigger slice of wrong size",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    const x = widenSlice([]u8{1, 2, 3, 4, 5});
+        \\    if (x.len == 0) return error.Whatever;
+        \\}
+        \\fn widenSlice(slice: []const u8) -> []const i32 {
+        \\    ([]const i32)(slice)
+        \\}
+    );
+
+    cases.addDebugSafety("value does not fit in shortening cast",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    const x = shorten_cast(200);
+        \\    if (x == 0) return error.Whatever;
+        \\}
+        \\fn shorten_cast(x: i32) -> i8 {
+        \\    i8(x)
+        \\}
+    );
+
+    cases.addDebugSafety("signed integer not fitting in cast to unsigned integer",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    const x = unsigned_cast(-10);
+        \\    if (x == 0) return error.Whatever;
+        \\}
+        \\fn unsigned_cast(x: i32) -> u32 {
+        \\    u32(x)
+        \\}
+    );
+
+    cases.addDebugSafety("unwrap error",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\error Whatever;
+        \\pub fn main() -> %void {
+        \\    %%bar();
+        \\}
+        \\fn bar() -> %void {
+        \\    return error.Whatever;
+        \\}
+    );
+
+    cases.addDebugSafety("cast integer to error and no code matches",
+        \\pub fn panic(message: []const u8) -> noreturn {
+        \\    @breakpoint();
+        \\    while (true) {}
+        \\}
+        \\pub fn main() -> %void {
+        \\    _ = bar(9999);
+        \\}
+        \\fn bar(x: u32) -> error {
+        \\    return error(x);
+        \\}
+    );
+}

test/run_tests.cpp

@@ -40,7 +40,6 @@ struct TestCase {
     bool is_parseh;
     TestSpecial special;
     bool is_release_mode;
-    bool is_debug_safety;
     AllowWarnings allow_warnings;
 };
 
@@ -58,26 +57,6 @@ static const char *zig_exe = "./zig";
 #define NL "\n"
 #endif
 
-static void add_debug_safety_case(const char *case_name, const char *source) {
-    TestCase *test_case = allocate<TestCase>(1);
-    test_case->is_debug_safety = true;
-    test_case->case_name = buf_ptr(buf_sprintf("%s", case_name));
-    test_case->source_files.resize(1);
-    test_case->source_files.at(0).relative_path = tmp_source_path;
-    test_case->source_files.at(0).source_code = source;
-
-    test_case->compiler_args.append("build_exe");
-    test_case->compiler_args.append(tmp_source_path);
-
-    test_case->compiler_args.append("--name");
-    test_case->compiler_args.append("test");
-
-    test_case->compiler_args.append("--output");
-    test_case->compiler_args.append(tmp_exe_path);
-
-    test_cases.append(test_case);
-}
-
 static TestCase *add_parseh_case(const char *case_name, AllowWarnings allow_warnings,
     const char *source, size_t count, ...)
 {
@@ -107,240 +86,6 @@ static TestCase *add_parseh_case(const char *case_name, AllowWarnings allow_warn
     va_end(ap);
     return test_case;
 }
-//////////////////////////////////////////////////////////////////////////////
-
-static void add_debug_safety_test_cases(void) {
-    add_debug_safety_case("calling panic", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-pub fn main() -> %void {
-    @panic("oh no");
-}
-    )SOURCE");
-
-    add_debug_safety_case("out of bounds slice access", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-pub fn main() -> %void {
-    const a = []i32{1, 2, 3, 4};
-    baz(bar(a));
-}
-fn bar(a: []const i32) -> i32 {
-    a[4]
-}
-fn baz(a: i32) { }
-    )SOURCE");
-
-    add_debug_safety_case("integer addition overflow", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    const x = add(65530, 10);
-    if (x == 0) return error.Whatever;
-}
-fn add(a: u16, b: u16) -> u16 {
-    a + b
-}
-    )SOURCE");
-
-    add_debug_safety_case("integer subtraction overflow", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    const x = sub(10, 20);
-    if (x == 0) return error.Whatever;
-}
-fn sub(a: u16, b: u16) -> u16 {
-    a - b
-}
-    )SOURCE");
-
-    add_debug_safety_case("integer multiplication overflow", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    const x = mul(300, 6000);
-    if (x == 0) return error.Whatever;
-}
-fn mul(a: u16, b: u16) -> u16 {
-    a * b
-}
-    )SOURCE");
-
-    add_debug_safety_case("integer negation overflow", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    const x = neg(-32768);
-    if (x == 32767) return error.Whatever;
-}
-fn neg(a: i16) -> i16 {
-    -a
-}
-    )SOURCE");
-
-    add_debug_safety_case("signed integer division overflow", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    const x = div(-32768, -1);
-    if (x == 32767) return error.Whatever;
-}
-fn div(a: i16, b: i16) -> i16 {
-    a / b
-}
-    )SOURCE");
-
-    add_debug_safety_case("signed shift left overflow", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    const x = shl(-16385, 1);
-    if (x == 0) return error.Whatever;
-}
-fn shl(a: i16, b: i16) -> i16 {
-    a << b
-}
-    )SOURCE");
-
-    add_debug_safety_case("unsigned shift left overflow", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    const x = shl(0b0010111111111111, 3);
-    if (x == 0) return error.Whatever;
-}
-fn shl(a: u16, b: u16) -> u16 {
-    a << b
-}
-    )SOURCE");
-
-    add_debug_safety_case("integer division by zero", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    const x = div0(999, 0);
-}
-fn div0(a: i32, b: i32) -> i32 {
-    a / b
-}
-    )SOURCE");
-
-    add_debug_safety_case("exact division failure", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    const x = divExact(10, 3);
-    if (x == 0) return error.Whatever;
-}
-fn divExact(a: i32, b: i32) -> i32 {
-    @divExact(a, b)
-}
-    )SOURCE");
-
-    add_debug_safety_case("cast []u8 to bigger slice of wrong size", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    const x = widenSlice([]u8{1, 2, 3, 4, 5});
-    if (x.len == 0) return error.Whatever;
-}
-fn widenSlice(slice: []const u8) -> []const i32 {
-    ([]const i32)(slice)
-}
-    )SOURCE");
-
-    add_debug_safety_case("value does not fit in shortening cast", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    const x = shorten_cast(200);
-    if (x == 0) return error.Whatever;
-}
-fn shorten_cast(x: i32) -> i8 {
-    i8(x)
-}
-    )SOURCE");
-
-    add_debug_safety_case("signed integer not fitting in cast to unsigned integer", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    const x = unsigned_cast(-10);
-    if (x == 0) return error.Whatever;
-}
-fn unsigned_cast(x: i32) -> u32 {
-    u32(x)
-}
-    )SOURCE");
-
-    add_debug_safety_case("unwrap error", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-error Whatever;
-pub fn main() -> %void {
-    %%bar();
-}
-fn bar() -> %void {
-    return error.Whatever;
-}
-    )SOURCE");
-
-    add_debug_safety_case("cast integer to error and no code matches", R"SOURCE(
-pub fn panic(message: []const u8) -> noreturn {
-    @breakpoint();
-    while (true) {}
-}
-pub fn main() -> %void {
-    _ = bar(9999);
-}
-fn bar(x: u32) -> error {
-    return error(x);
-}
-    )SOURCE");
-}
 
 //////////////////////////////////////////////////////////////////////////////
 
@@ -653,43 +398,24 @@ static void run_test(TestCase *test_case) {
         Buf program_stdout = BUF_INIT;
         os_exec_process(tmp_exe_path, test_case->program_args, &term, &program_stderr, &program_stdout);
 
-        if (test_case->is_debug_safety) {
-            int debug_trap_signal = 5;
-            if (term.how != TerminationIdSignaled || term.code != debug_trap_signal) {
-                if (term.how == TerminationIdClean) {
-                    printf("\nProgram expected to hit debug trap (signal %d) but exited with return code %d\n",
-                            debug_trap_signal, term.code);
-                } else if (term.how == TerminationIdSignaled) {
-                    printf("\nProgram expected to hit debug trap (signal %d) but signaled with code %d\n",
-                            debug_trap_signal, term.code);
-                } else {
-                    printf("\nProgram expected to hit debug trap (signal %d) exited in an unexpected way\n",
-                            debug_trap_signal);
-                }
-                print_compiler_invocation(test_case);
-                print_exe_invocation(test_case);
-                exit(1);
-            }
-        } else {
-            if (term.how != TerminationIdClean || term.code != 0) {
-                printf("\nProgram exited with error\n");
-                print_compiler_invocation(test_case);
-                print_exe_invocation(test_case);
-                printf("%s\n", buf_ptr(&program_stderr));
-                exit(1);
-            }
+        if (term.how != TerminationIdClean || term.code != 0) {
+            printf("\nProgram exited with error\n");
+            print_compiler_invocation(test_case);
+            print_exe_invocation(test_case);
+            printf("%s\n", buf_ptr(&program_stderr));
+            exit(1);
+        }
 
-            if (test_case->output != nullptr && !buf_eql_str(&program_stdout, test_case->output)) {
-                printf("\n");
-                print_compiler_invocation(test_case);
-                print_exe_invocation(test_case);
-                printf("==== Test failed. Expected output: ====\n");
-                printf("%s\n", test_case->output);
-                printf("========= Actual output: ==============\n");
-                printf("%s\n", buf_ptr(&program_stdout));
-                printf("=======================================\n");
-                exit(1);
-            }
+        if (test_case->output != nullptr && !buf_eql_str(&program_stdout, test_case->output)) {
+            printf("\n");
+            print_compiler_invocation(test_case);
+            print_exe_invocation(test_case);
+            printf("==== Test failed. Expected output: ====\n");
+            printf("%s\n", test_case->output);
+            printf("========= Actual output: ==============\n");
+            printf("%s\n", buf_ptr(&program_stdout));
+            printf("=======================================\n");
+            exit(1);
         }
     }
 
@@ -740,7 +466,6 @@ int main(int argc, char **argv) {
             }
         }
     }
-    add_debug_safety_test_cases();
     add_parseh_test_cases();
     run_all_tests(grep_text);
     cleanup();

test/tests.zig

@@ -16,6 +16,7 @@ pub const compare_output = @import("compare_output.zig");
 pub const build_examples = @import("build_examples.zig");
 pub const compile_errors = @import("compile_errors.zig");
 pub const assemble_and_link = @import("assemble_and_link.zig");
+pub const debug_safety = @import("debug_safety.zig");
 
 pub fn addCompareOutputTests(b: &build.Builder, test_filter: ?[]const u8) -> &build.Step {
     const cases = %%b.allocator.create(CompareOutputContext);
@@ -31,6 +32,20 @@ pub fn addCompareOutputTests(b: &build.Builder, test_filter: ?[]const u8) -> &bu
     return cases.step;
 }
 
+pub fn addDebugSafetyTests(b: &build.Builder, test_filter: ?[]const u8) -> &build.Step {
+    const cases = %%b.allocator.create(CompareOutputContext);
+    *cases = CompareOutputContext {
+        .b = b,
+        .step = b.step("test-debug-safety", "Run the debug safety tests"),
+        .test_index = 0,
+        .test_filter = test_filter,
+    };
+
+    debug_safety.addCases(cases);
+
+    return cases.step;
+}
+
 pub fn addCompileErrorTests(b: &build.Builder, test_filter: ?[]const u8) -> &build.Step {
     const cases = %%b.allocator.create(CompileErrorContext);
     *cases = CompileErrorContext {
@@ -79,12 +94,18 @@ pub const CompareOutputContext = struct {
     test_index: usize,
     test_filter: ?[]const u8,
 
+    const Special = enum {
+        None,
+        Asm,
+        DebugSafety,
+    };
+
     const TestCase = struct {
         name: []const u8,
         sources: List(SourceFile),
         expected_output: []const u8,
         link_libc: bool,
-        is_asm: bool,
+        special: Special,
 
         const SourceFile = struct {
             filename: []const u8,
@@ -175,17 +196,83 @@ pub const CompareOutputContext = struct {
         }
     };
 
+    const DebugSafetyRunStep = struct {
+        step: build.Step,
+        context: &CompareOutputContext,
+        exe_path: []const u8,
+        name: []const u8,
+        test_index: usize,
+
+        pub fn create(context: &CompareOutputContext, exe_path: []const u8,
+            name: []const u8) -> &DebugSafetyRunStep
+        {
+            const allocator = context.b.allocator;
+            const ptr = %%allocator.create(DebugSafetyRunStep);
+            *ptr = DebugSafetyRunStep {
+                .context = context,
+                .exe_path = exe_path,
+                .name = name,
+                .test_index = context.test_index,
+                .step = build.Step.init("DebugSafetyRun", allocator, make),
+            };
+            context.test_index += 1;
+            return ptr;
+        }
+
+        fn make(step: &build.Step) -> %void {
+            const self = @fieldParentPtr(DebugSafetyRunStep, "step", step);
+            const b = self.context.b;
+
+            const full_exe_path = b.pathFromRoot(self.exe_path);
+
+            %%io.stderr.printf("Test {}/{} {}...", self.test_index+1, self.context.test_index, self.name);
+
+            var child = os.ChildProcess.spawn(full_exe_path, [][]u8{}, &b.env_map,
+                StdIo.Ignore, StdIo.Pipe, StdIo.Pipe, b.allocator) %% |err|
+            {
+                debug.panic("Unable to spawn {}: {}\n", full_exe_path, @errorName(err));
+            };
+
+            const term = child.wait() %% |err| {
+                debug.panic("Unable to spawn {}: {}\n", full_exe_path, @errorName(err));
+            };
+
+            const debug_trap_signal: i32 = 5;
+            switch (term) {
+                Term.Clean => |code| {
+                    %%io.stderr.printf("\nProgram expected to hit debug trap (signal {}) " ++
+                        "but exited with return code {}\n", debug_trap_signal, code);
+                    return error.TestFailed;
+                },
+                Term.Signal => |sig| {
+                    if (sig != debug_trap_signal) {
+                        %%io.stderr.printf("\nProgram expected to hit debug trap (signal {}) " ++
+                            "but instead signaled {}\n", debug_trap_signal, sig);
+                        return error.TestFailed;
+                    }
+                },
+                else => {
+                    %%io.stderr.printf("\nProgram expected to hit debug trap (signal {}) " ++
+                        " but exited in an unexpected way\n", debug_trap_signal);
+                    return error.TestFailed;
+                },
+            }
+
+            %%io.stderr.printf("OK\n");
+        }
+    };
+
     pub fn createExtra(self: &CompareOutputContext, name: []const u8, source: []const u8,
-        expected_output: []const u8, is_asm: bool) -> TestCase
+        expected_output: []const u8, special: Special) -> TestCase
     {
         var tc = TestCase {
             .name = name,
             .sources = List(TestCase.SourceFile).init(self.b.allocator),
             .expected_output = expected_output,
             .link_libc = false,
-            .is_asm = is_asm,
+            .special = special,
         };
-        const root_src_name = if (is_asm) "source.s" else "source.zig";
+        const root_src_name = if (special == Special.Asm) "source.s" else "source.zig";
         tc.addSourceFile(root_src_name, source);
         return tc;
     }
@@ -193,7 +280,7 @@ pub const CompareOutputContext = struct {
     pub fn create(self: &CompareOutputContext, name: []const u8, source: []const u8,
         expected_output: []const u8) -> TestCase
     {
-        return createExtra(self, name, source, expected_output, false);
+        return createExtra(self, name, source, expected_output, Special.None);
     }
 
     pub fn addC(self: &CompareOutputContext, name: []const u8, source: []const u8, expected_output: []const u8) {
@@ -208,7 +295,12 @@ pub const CompareOutputContext = struct {
     }
 
     pub fn addAsm(self: &CompareOutputContext, name: []const u8, source: []const u8, expected_output: []const u8) {
-        const tc = self.createExtra(name, source, expected_output, true);
+        const tc = self.createExtra(name, source, expected_output, Special.Asm);
+        self.addCase(tc);
+    }
+
+    pub fn addDebugSafety(self: &CompareOutputContext, name: []const u8, source: []const u8) {
+        const tc = self.createExtra(name, source, undefined, Special.DebugSafety);
         self.addCase(tc);
     }
 
@@ -218,45 +310,74 @@ pub const CompareOutputContext = struct {
         const root_src = %%os.path.join(b.allocator, "test_artifacts", case.sources.items[0].filename);
         const exe_path = %%os.path.join(b.allocator, "test_artifacts", "test");
 
-        if (case.is_asm) {
-            const obj_path = %%os.path.join(b.allocator, "test_artifacts", "test.o");
-            const annotated_case_name = %%fmt.allocPrint(self.b.allocator, "assemble-and-link {}", case.name);
-            if (const filter ?= self.test_filter) {
-                if (mem.indexOf(u8, annotated_case_name, filter) == null)
-                    return;
-            }
-
-            const obj = b.addAssemble("test", root_src);
-            obj.setOutputPath(obj_path);
-
-            for (case.sources.toSliceConst()) |src_file| {
-                const expanded_src_path = %%os.path.join(b.allocator, "test_artifacts", src_file.filename);
-                const write_src = b.addWriteFile(expanded_src_path, src_file.source);
-                obj.step.dependOn(&write_src.step);
-            }
-
-            const exe = b.addLinkExecutable("test");
-            exe.step.dependOn(&obj.step);
-            exe.addObjectFile(obj_path);
-            exe.setOutputPath(exe_path);
-
-            const run_and_cmp_output = RunCompareOutputStep.create(self, exe_path, annotated_case_name,
-                case.expected_output);
-            run_and_cmp_output.step.dependOn(&exe.step);
-
-            self.step.dependOn(&run_and_cmp_output.step);
-        } else {
-            for ([]bool{false, true}) |release| {
-                const annotated_case_name = %%fmt.allocPrint(self.b.allocator, "{} ({})",
-                    case.name, if (release) "release" else "debug");
+        switch (case.special) {
+            Special.Asm => {
+                const obj_path = %%os.path.join(b.allocator, "test_artifacts", "test.o");
+                const annotated_case_name = %%fmt.allocPrint(self.b.allocator, "assemble-and-link {}", case.name);
                 if (const filter ?= self.test_filter) {
                     if (mem.indexOf(u8, annotated_case_name, filter) == null)
-                        continue;
+                        return;
+                }
+
+                const obj = b.addAssemble("test", root_src);
+                obj.setOutputPath(obj_path);
+
+                for (case.sources.toSliceConst()) |src_file| {
+                    const expanded_src_path = %%os.path.join(b.allocator, "test_artifacts", src_file.filename);
+                    const write_src = b.addWriteFile(expanded_src_path, src_file.source);
+                    obj.step.dependOn(&write_src.step);
+                }
+
+                const exe = b.addLinkExecutable("test");
+                exe.step.dependOn(&obj.step);
+                exe.addObjectFile(obj_path);
+                exe.setOutputPath(exe_path);
+
+                const run_and_cmp_output = RunCompareOutputStep.create(self, exe_path, annotated_case_name,
+                    case.expected_output);
+                run_and_cmp_output.step.dependOn(&exe.step);
+
+                self.step.dependOn(&run_and_cmp_output.step);
+            },
+            Special.None => {
+                for ([]bool{false, true}) |release| {
+                    const annotated_case_name = %%fmt.allocPrint(self.b.allocator, "{} ({})",
+                        case.name, if (release) "release" else "debug");
+                    if (const filter ?= self.test_filter) {
+                        if (mem.indexOf(u8, annotated_case_name, filter) == null)
+                            continue;
+                    }
+
+                    const exe = b.addExecutable("test", root_src);
+                    exe.setOutputPath(exe_path);
+                    exe.setRelease(release);
+                    if (case.link_libc) {
+                        exe.linkLibrary("c");
+                    }
+
+                    for (case.sources.toSliceConst()) |src_file| {
+                        const expanded_src_path = %%os.path.join(b.allocator, "test_artifacts", src_file.filename);
+                        const write_src = b.addWriteFile(expanded_src_path, src_file.source);
+                        exe.step.dependOn(&write_src.step);
+                    }
+
+                    const run_and_cmp_output = RunCompareOutputStep.create(self, exe_path, annotated_case_name,
+                        case.expected_output);
+                    run_and_cmp_output.step.dependOn(&exe.step);
+
+                    self.step.dependOn(&run_and_cmp_output.step);
+                }
+            },
+            Special.DebugSafety => {
+                const obj_path = %%os.path.join(b.allocator, "test_artifacts", "test.o");
+                const annotated_case_name = %%fmt.allocPrint(self.b.allocator, "debug-safety {}", case.name);
+                if (const filter ?= self.test_filter) {
+                    if (mem.indexOf(u8, annotated_case_name, filter) == null)
+                        return;
                 }
 
                 const exe = b.addExecutable("test", root_src);
                 exe.setOutputPath(exe_path);
-                exe.setRelease(release);
                 if (case.link_libc) {
                     exe.linkLibrary("c");
                 }
@@ -267,14 +388,12 @@ pub const CompareOutputContext = struct {
                     exe.step.dependOn(&write_src.step);
                 }
 
-                const run_and_cmp_output = RunCompareOutputStep.create(self, exe_path, annotated_case_name,
-                    case.expected_output);
+                const run_and_cmp_output = DebugSafetyRunStep.create(self, exe_path, annotated_case_name);
                 run_and_cmp_output.step.dependOn(&exe.step);
 
                 self.step.dependOn(&run_and_cmp_output.step);
-            }
-        };
-
+            },
+        }
     }
 };