From a19e73d8ae0bc5b7ac19b1bc72aa1fdb8f86d80f Mon Sep 17 00:00:00 2001 From: Michael Dusan Date: Mon, 27 May 2019 20:07:05 -0400 Subject: [PATCH] test: add compare-panic `zig build test-compare-panic` Create basic tests to compare panic output. The address field is replaced by a symbolic constant and each expected output is specific to os. Tests will only run for explicitly defined platforms. see also #2485 --- build.zig | 1 + test/compare_panic.zig | 277 +++++++++++++++++++++++++++++++++++++++++ test/tests.zig | 210 +++++++++++++++++++++++++++++++ 3 files changed, 488 insertions(+) create mode 100644 test/compare_panic.zig diff --git a/build.zig b/build.zig index 45758d407..cc0405c21 100644 --- a/build.zig +++ b/build.zig @@ -138,6 +138,7 @@ pub fn build(b: *Builder) !void { test_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes)); test_step.dependOn(tests.addStandaloneTests(b, test_filter, modes)); + test_step.dependOn(tests.addComparePanicTests(b, test_filter, modes)); test_step.dependOn(tests.addCliTests(b, test_filter, modes)); test_step.dependOn(tests.addCompileErrorTests(b, test_filter, modes)); test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, modes)); diff --git a/test/compare_panic.zig b/test/compare_panic.zig new file mode 100644 index 000000000..3c71b9626 --- /dev/null +++ b/test/compare_panic.zig @@ -0,0 +1,277 @@ +const builtin = @import("builtin"); +const std = @import("std"); +const os = std.os; +const tests = @import("tests.zig"); + +pub fn addCases(cases: *tests.ComparePanicContext) void { + const source_return = + \\const std = @import("std"); + \\ + \\pub fn main() !void { + \\ return error.TheSkyIsFalling; + \\} + ; + const source_try_return = + \\const std = @import("std"); + \\ + \\fn foo() !void { + \\ return error.TheSkyIsFalling; + \\} + \\ + \\pub fn main() !void { + \\ try foo(); + \\} + ; + const source_try_try_return_return = + \\const std = @import("std"); + \\ + \\fn foo() !void { + \\ try bar(); + \\} + \\ + \\fn bar() !void { + \\ return make_error(); + \\} + \\ + \\fn make_error() !void { + \\ return error.TheSkyIsFalling; + \\} + \\ + \\pub fn main() !void { + \\ try foo(); + \\} + ; + switch (builtin.os) { + .linux => { + cases.addCase( + "return", + source_return, + [][]const u8{ + // debug + \\error: TheSkyIsFalling + \\source.zig:4:5: [address] in main (test) + \\ + , + // release-safe + \\error: TheSkyIsFalling + \\source.zig:4:5: [address] in std.special.posixCallMainAndExit (test) + \\ + , + // release-fast + \\error: TheSkyIsFalling + \\ + , + // release-small + \\error: TheSkyIsFalling + \\ + }, + ); + cases.addCase( + "try return", + source_try_return, + [][]const u8{ + // debug + \\error: TheSkyIsFalling + \\source.zig:4:5: [address] in foo (test) + \\source.zig:8:5: [address] in main (test) + \\ + , + // release-safe + \\error: TheSkyIsFalling + \\source.zig:4:5: [address] in std.special.posixCallMainAndExit (test) + \\source.zig:8:5: [address] in std.special.posixCallMainAndExit (test) + \\ + , + // release-fast + \\error: TheSkyIsFalling + \\ + , + // release-small + \\error: TheSkyIsFalling + \\ + }, + ); + cases.addCase( + "try try return return", + source_try_try_return_return, + [][]const u8{ + // debug + \\error: TheSkyIsFalling + \\source.zig:12:5: [address] in make_error (test) + \\source.zig:8:5: [address] in bar (test) + \\source.zig:4:5: [address] in foo (test) + \\source.zig:16:5: [address] in main (test) + \\ + , + // release-safe + \\error: TheSkyIsFalling + \\source.zig:12:5: [address] in std.special.posixCallMainAndExit (test) + \\source.zig:8:5: [address] in std.special.posixCallMainAndExit (test) + \\source.zig:4:5: [address] in std.special.posixCallMainAndExit (test) + \\source.zig:16:5: [address] in std.special.posixCallMainAndExit (test) + \\ + , + // release-fast + \\error: TheSkyIsFalling + \\ + , + // release-small + \\error: TheSkyIsFalling + \\ + }, + ); + }, + .macosx => { + cases.addCase( + "return", + source_return, + [][]const u8{ + // debug + \\error: TheSkyIsFalling + \\source.zig:4:5: [address] in _main.0 (test.o) + \\ + , + // release-safe + \\error: TheSkyIsFalling + \\source.zig:4:5: [address] in _main (test.o) + \\ + , + // release-fast + \\error: TheSkyIsFalling + \\ + , + // release-small + \\error: TheSkyIsFalling + \\ + }, + ); + cases.addCase( + "try return", + source_try_return, + [][]const u8{ + // debug + \\error: TheSkyIsFalling + \\source.zig:4:5: [address] in _foo (test.o) + \\source.zig:8:5: [address] in _main.0 (test.o) + \\ + , + // release-safe + \\error: TheSkyIsFalling + \\source.zig:4:5: [address] in _main (test.o) + \\source.zig:8:5: [address] in _main (test.o) + \\ + , + // release-fast + \\error: TheSkyIsFalling + \\ + , + // release-small + \\error: TheSkyIsFalling + \\ + }, + ); + cases.addCase( + "try try return return", + source_try_try_return_return, + [][]const u8{ + // debug + \\error: TheSkyIsFalling + \\source.zig:12:5: [address] in _make_error (test.o) + \\source.zig:8:5: [address] in _bar (test.o) + \\source.zig:4:5: [address] in _foo (test.o) + \\source.zig:16:5: [address] in _main.0 (test.o) + \\ + , + // release-safe + \\error: TheSkyIsFalling + \\source.zig:12:5: [address] in _main (test.o) + \\source.zig:8:5: [address] in _main (test.o) + \\source.zig:4:5: [address] in _main (test.o) + \\source.zig:16:5: [address] in _main (test.o) + \\ + , + // release-fast + \\error: TheSkyIsFalling + \\ + , + // release-small + \\error: TheSkyIsFalling + \\ + }, + ); + }, + .windows => { + cases.addCase( + "return", + source_return, + [][]const u8{ + // debug + \\error: TheSkyIsFalling + \\source.zig:4:5: [address] in main (test.obj) + \\ + , + // release-safe + // --disabled-- results in segmenetation fault + "" + , + // release-fast + \\error: TheSkyIsFalling + \\ + , + // release-small + \\error: TheSkyIsFalling + \\ + }, + ); + cases.addCase( + "try return", + source_try_return, + [][]const u8{ + // debug + \\error: TheSkyIsFalling + \\source.zig:4:5: [address] in foo (test.obj) + \\source.zig:8:5: [address] in main (test.obj) + \\ + , + // release-safe + // --disabled-- results in segmenetation fault + "" + , + // release-fast + \\error: TheSkyIsFalling + \\ + , + // release-small + \\error: TheSkyIsFalling + \\ + }, + ); + cases.addCase( + "try try return return", + source_try_try_return_return, + [][]const u8{ + // debug + \\error: TheSkyIsFalling + \\source.zig:12:5: [address] in make_error (test.obj) + \\source.zig:8:5: [address] in bar (test.obj) + \\source.zig:4:5: [address] in foo (test.obj) + \\source.zig:16:5: [address] in main (test.obj) + \\ + , + // release-safe + // --disabled-- results in segmenetation fault + "" + , + // release-fast + \\error: TheSkyIsFalling + \\ + , + // release-small + \\error: TheSkyIsFalling + \\ + }, + ); + }, + else => {}, + } +} diff --git a/test/tests.zig b/test/tests.zig index f7ef05d3c..1bb9e417e 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -16,6 +16,7 @@ const LibExeObjStep = build.LibExeObjStep; const compare_output = @import("compare_output.zig"); const standalone = @import("standalone.zig"); +const compare_panic = @import("compare_panic.zig"); const compile_errors = @import("compile_errors.zig"); const assemble_and_link = @import("assemble_and_link.zig"); const runtime_safety = @import("runtime_safety.zig"); @@ -57,6 +58,21 @@ pub fn addCompareOutputTests(b: *build.Builder, test_filter: ?[]const u8, modes: return cases.step; } +pub fn addComparePanicTests(b: *build.Builder, test_filter: ?[]const u8, modes: []const Mode) *build.Step { + const cases = b.allocator.create(ComparePanicContext) catch unreachable; + cases.* = ComparePanicContext{ + .b = b, + .step = b.step("test-compare-panic", "Run the compare panic tests"), + .test_index = 0, + .test_filter = test_filter, + .modes = modes, + }; + + compare_panic.addCases(cases); + + return cases.step; +} + pub fn addRuntimeSafetyTests(b: *build.Builder, test_filter: ?[]const u8, modes: []const Mode) *build.Step { const cases = b.allocator.create(CompareOutputContext) catch unreachable; cases.* = CompareOutputContext{ @@ -549,6 +565,200 @@ pub const CompareOutputContext = struct { } }; +pub const ComparePanicContext = struct { + b: *build.Builder, + step: *build.Step, + test_index: usize, + test_filter: ?[]const u8, + modes: []const Mode, + + const Expect = [@typeInfo(Mode).Enum.fields.len][]const u8; + + pub fn addCase( + self: *ComparePanicContext, + name: []const u8, + source: []const u8, + expect: Expect, + ) void { + const b = self.b; + + const source_pathname = fs.path.join( + b.allocator, + [][]const u8{ b.cache_root, "source.zig" }, + ) catch unreachable; + + for (self.modes) |mode| { + const expect_for_mode = expect[@enumToInt(mode)]; + if (expect_for_mode.len == 0) continue; + + const annotated_case_name = fmt.allocPrint(self.b.allocator, "{} {} ({})", "compare-panic", name, @tagName(mode)) catch unreachable; + if (self.test_filter) |filter| { + if (mem.indexOf(u8, annotated_case_name, filter) == null) continue; + } + + const exe = b.addExecutable("test", source_pathname); + exe.setBuildMode(mode); + + const write_source = b.addWriteFile(source_pathname, source); + exe.step.dependOn(&write_source.step); + + const run_and_compare = RunAndCompareStep.create( + self, + exe, + annotated_case_name, + mode, + expect_for_mode, + ); + + self.step.dependOn(&run_and_compare.step); + } + } + + const RunAndCompareStep = struct { + step: build.Step, + context: *ComparePanicContext, + exe: *LibExeObjStep, + name: []const u8, + mode: Mode, + expect_output: []const u8, + test_index: usize, + + pub fn create( + context: *ComparePanicContext, + exe: *LibExeObjStep, + name: []const u8, + mode: Mode, + expect_output: []const u8, + ) *RunAndCompareStep { + const allocator = context.b.allocator; + const ptr = allocator.create(RunAndCompareStep) catch unreachable; + ptr.* = RunAndCompareStep{ + .step = build.Step.init("PanicCompareOutputStep", allocator, make), + .context = context, + .exe = exe, + .name = name, + .mode = mode, + .expect_output = expect_output, + .test_index = context.test_index, + }; + ptr.step.dependOn(&exe.step); + context.test_index += 1; + return ptr; + } + + fn make(step: *build.Step) !void { + const self = @fieldParentPtr(RunAndCompareStep, "step", step); + const b = self.context.b; + + const full_exe_path = self.exe.getOutputPath(); + var args = ArrayList([]const u8).init(b.allocator); + defer args.deinit(); + args.append(full_exe_path) catch unreachable; + + warn("Test {}/{} {}...", self.test_index + 1, self.context.test_index, self.name); + + const child = std.ChildProcess.init(args.toSliceConst(), b.allocator) catch unreachable; + defer child.deinit(); + + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; + child.env_map = b.env_map; + + child.spawn() catch |err| debug.panic("Unable to spawn {}: {}\n", full_exe_path, @errorName(err)); + + var stdout = Buffer.initNull(b.allocator); + var stderr = Buffer.initNull(b.allocator); + + var stdout_file_in_stream = child.stdout.?.inStream(); + var stderr_file_in_stream = child.stderr.?.inStream(); + + stdout_file_in_stream.stream.readAllBuffer(&stdout, max_stdout_size) catch unreachable; + stderr_file_in_stream.stream.readAllBuffer(&stderr, max_stdout_size) catch unreachable; + + const term = child.wait() catch |err| { + debug.panic("Unable to spawn {}: {}\n", full_exe_path, @errorName(err)); + }; + + switch (term) { + .Exited => |code| { + const expect_code: u32 = 1; + if (code != expect_code) { + warn("Process {} exited with error code {} but expected code {}\n", full_exe_path, code, expect_code); + printInvocation(args.toSliceConst()); + return error.TestFailed; + } + }, + .Signal => |signum| { + warn("Process {} terminated on signal {}\n", full_exe_path, signum); + printInvocation(args.toSliceConst()); + return error.TestFailed; + }, + .Stopped => |signum| { + warn("Process {} stopped on signal {}\n", full_exe_path, signum); + printInvocation(args.toSliceConst()); + return error.TestFailed; + }, + .Unknown => |code| { + warn("Process {} terminated unexpectedly with error code {}\n", full_exe_path, code); + printInvocation(args.toSliceConst()); + return error.TestFailed; + }, + } + + // process result + // - keep only basename of source file path + // - replace address with symbolic string + // - skip empty lines + const got: []const u8 = got_result: { + var buf = try Buffer.initSize(b.allocator, 0); + defer buf.deinit(); + var bytes = stderr.toSliceConst(); + if (bytes.len != 0 and bytes[bytes.len - 1] == '\n') bytes = bytes[0 .. bytes.len - 1]; + var it = mem.separate(bytes, "\n"); + process_lines: while (it.next()) |line| { + if (line.len == 0) continue; + const delims = []const []const u8{ ":", ":", ":", " in " }; + var marks = []usize{0} ** 4; + // offset search past `[drive]:` on windows + var pos: usize = if (builtin.os == .windows) 2 else 0; + for (delims) |delim, i| { + marks[i] = mem.indexOfPos(u8, line, pos, delim) orelse { + try buf.append(line); + try buf.append("\n"); + continue :process_lines; + }; + pos = marks[i] + delim.len; + } + pos = mem.lastIndexOfScalar(u8, line[0..marks[0]], fs.path.sep) orelse { + try buf.append(line); + try buf.append("\n"); + continue :process_lines; + }; + try buf.append(line[pos + 1 .. marks[2] + delims[2].len]); + try buf.append(" [address]"); + try buf.append(line[marks[3]..]); + try buf.append("\n"); + } + break :got_result buf.toOwnedSlice(); + }; + + if (!mem.eql(u8, self.expect_output, got)) { + warn( + \\ + \\========= Expected this output: ========= + \\{} + \\================================================ + \\{} + \\ + , self.expect_output, got); + return error.TestFailed; + } + warn("OK\n"); + } + }; +}; + pub const CompileErrorContext = struct { b: *build.Builder, step: *build.Step,