test: add compare-panic
`zig build test-compare-panic` Create basic tests to compare panic output. The address field is replaced by a symbolic constant and each expected output is specific to os. Tests will only run for explicitly defined platforms. see also #2485master
parent
d74b8567cf
commit
a19e73d8ae
|
@ -138,6 +138,7 @@ pub fn build(b: *Builder) !void {
|
|||
|
||||
test_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes));
|
||||
test_step.dependOn(tests.addStandaloneTests(b, test_filter, modes));
|
||||
test_step.dependOn(tests.addComparePanicTests(b, test_filter, modes));
|
||||
test_step.dependOn(tests.addCliTests(b, test_filter, modes));
|
||||
test_step.dependOn(tests.addCompileErrorTests(b, test_filter, modes));
|
||||
test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, modes));
|
||||
|
|
|
@ -0,0 +1,277 @@
|
|||
const builtin = @import("builtin");
|
||||
const std = @import("std");
|
||||
const os = std.os;
|
||||
const tests = @import("tests.zig");
|
||||
|
||||
pub fn addCases(cases: *tests.ComparePanicContext) void {
|
||||
const source_return =
|
||||
\\const std = @import("std");
|
||||
\\
|
||||
\\pub fn main() !void {
|
||||
\\ return error.TheSkyIsFalling;
|
||||
\\}
|
||||
;
|
||||
const source_try_return =
|
||||
\\const std = @import("std");
|
||||
\\
|
||||
\\fn foo() !void {
|
||||
\\ return error.TheSkyIsFalling;
|
||||
\\}
|
||||
\\
|
||||
\\pub fn main() !void {
|
||||
\\ try foo();
|
||||
\\}
|
||||
;
|
||||
const source_try_try_return_return =
|
||||
\\const std = @import("std");
|
||||
\\
|
||||
\\fn foo() !void {
|
||||
\\ try bar();
|
||||
\\}
|
||||
\\
|
||||
\\fn bar() !void {
|
||||
\\ return make_error();
|
||||
\\}
|
||||
\\
|
||||
\\fn make_error() !void {
|
||||
\\ return error.TheSkyIsFalling;
|
||||
\\}
|
||||
\\
|
||||
\\pub fn main() !void {
|
||||
\\ try foo();
|
||||
\\}
|
||||
;
|
||||
switch (builtin.os) {
|
||||
.linux => {
|
||||
cases.addCase(
|
||||
"return",
|
||||
source_return,
|
||||
[][]const u8{
|
||||
// debug
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:4:5: [address] in main (test)
|
||||
\\
|
||||
,
|
||||
// release-safe
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:4:5: [address] in std.special.posixCallMainAndExit (test)
|
||||
\\
|
||||
,
|
||||
// release-fast
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
,
|
||||
// release-small
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
},
|
||||
);
|
||||
cases.addCase(
|
||||
"try return",
|
||||
source_try_return,
|
||||
[][]const u8{
|
||||
// debug
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:4:5: [address] in foo (test)
|
||||
\\source.zig:8:5: [address] in main (test)
|
||||
\\
|
||||
,
|
||||
// release-safe
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:4:5: [address] in std.special.posixCallMainAndExit (test)
|
||||
\\source.zig:8:5: [address] in std.special.posixCallMainAndExit (test)
|
||||
\\
|
||||
,
|
||||
// release-fast
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
,
|
||||
// release-small
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
},
|
||||
);
|
||||
cases.addCase(
|
||||
"try try return return",
|
||||
source_try_try_return_return,
|
||||
[][]const u8{
|
||||
// debug
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:12:5: [address] in make_error (test)
|
||||
\\source.zig:8:5: [address] in bar (test)
|
||||
\\source.zig:4:5: [address] in foo (test)
|
||||
\\source.zig:16:5: [address] in main (test)
|
||||
\\
|
||||
,
|
||||
// release-safe
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:12:5: [address] in std.special.posixCallMainAndExit (test)
|
||||
\\source.zig:8:5: [address] in std.special.posixCallMainAndExit (test)
|
||||
\\source.zig:4:5: [address] in std.special.posixCallMainAndExit (test)
|
||||
\\source.zig:16:5: [address] in std.special.posixCallMainAndExit (test)
|
||||
\\
|
||||
,
|
||||
// release-fast
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
,
|
||||
// release-small
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
},
|
||||
);
|
||||
},
|
||||
.macosx => {
|
||||
cases.addCase(
|
||||
"return",
|
||||
source_return,
|
||||
[][]const u8{
|
||||
// debug
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:4:5: [address] in _main.0 (test.o)
|
||||
\\
|
||||
,
|
||||
// release-safe
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:4:5: [address] in _main (test.o)
|
||||
\\
|
||||
,
|
||||
// release-fast
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
,
|
||||
// release-small
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
},
|
||||
);
|
||||
cases.addCase(
|
||||
"try return",
|
||||
source_try_return,
|
||||
[][]const u8{
|
||||
// debug
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:4:5: [address] in _foo (test.o)
|
||||
\\source.zig:8:5: [address] in _main.0 (test.o)
|
||||
\\
|
||||
,
|
||||
// release-safe
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:4:5: [address] in _main (test.o)
|
||||
\\source.zig:8:5: [address] in _main (test.o)
|
||||
\\
|
||||
,
|
||||
// release-fast
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
,
|
||||
// release-small
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
},
|
||||
);
|
||||
cases.addCase(
|
||||
"try try return return",
|
||||
source_try_try_return_return,
|
||||
[][]const u8{
|
||||
// debug
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:12:5: [address] in _make_error (test.o)
|
||||
\\source.zig:8:5: [address] in _bar (test.o)
|
||||
\\source.zig:4:5: [address] in _foo (test.o)
|
||||
\\source.zig:16:5: [address] in _main.0 (test.o)
|
||||
\\
|
||||
,
|
||||
// release-safe
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:12:5: [address] in _main (test.o)
|
||||
\\source.zig:8:5: [address] in _main (test.o)
|
||||
\\source.zig:4:5: [address] in _main (test.o)
|
||||
\\source.zig:16:5: [address] in _main (test.o)
|
||||
\\
|
||||
,
|
||||
// release-fast
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
,
|
||||
// release-small
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
},
|
||||
);
|
||||
},
|
||||
.windows => {
|
||||
cases.addCase(
|
||||
"return",
|
||||
source_return,
|
||||
[][]const u8{
|
||||
// debug
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:4:5: [address] in main (test.obj)
|
||||
\\
|
||||
,
|
||||
// release-safe
|
||||
// --disabled-- results in segmenetation fault
|
||||
""
|
||||
,
|
||||
// release-fast
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
,
|
||||
// release-small
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
},
|
||||
);
|
||||
cases.addCase(
|
||||
"try return",
|
||||
source_try_return,
|
||||
[][]const u8{
|
||||
// debug
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:4:5: [address] in foo (test.obj)
|
||||
\\source.zig:8:5: [address] in main (test.obj)
|
||||
\\
|
||||
,
|
||||
// release-safe
|
||||
// --disabled-- results in segmenetation fault
|
||||
""
|
||||
,
|
||||
// release-fast
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
,
|
||||
// release-small
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
},
|
||||
);
|
||||
cases.addCase(
|
||||
"try try return return",
|
||||
source_try_try_return_return,
|
||||
[][]const u8{
|
||||
// debug
|
||||
\\error: TheSkyIsFalling
|
||||
\\source.zig:12:5: [address] in make_error (test.obj)
|
||||
\\source.zig:8:5: [address] in bar (test.obj)
|
||||
\\source.zig:4:5: [address] in foo (test.obj)
|
||||
\\source.zig:16:5: [address] in main (test.obj)
|
||||
\\
|
||||
,
|
||||
// release-safe
|
||||
// --disabled-- results in segmenetation fault
|
||||
""
|
||||
,
|
||||
// release-fast
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
,
|
||||
// release-small
|
||||
\\error: TheSkyIsFalling
|
||||
\\
|
||||
},
|
||||
);
|
||||
},
|
||||
else => {},
|
||||
}
|
||||
}
|
210
test/tests.zig
210
test/tests.zig
|
@ -16,6 +16,7 @@ const LibExeObjStep = build.LibExeObjStep;
|
|||
|
||||
const compare_output = @import("compare_output.zig");
|
||||
const standalone = @import("standalone.zig");
|
||||
const compare_panic = @import("compare_panic.zig");
|
||||
const compile_errors = @import("compile_errors.zig");
|
||||
const assemble_and_link = @import("assemble_and_link.zig");
|
||||
const runtime_safety = @import("runtime_safety.zig");
|
||||
|
@ -57,6 +58,21 @@ pub fn addCompareOutputTests(b: *build.Builder, test_filter: ?[]const u8, modes:
|
|||
return cases.step;
|
||||
}
|
||||
|
||||
pub fn addComparePanicTests(b: *build.Builder, test_filter: ?[]const u8, modes: []const Mode) *build.Step {
|
||||
const cases = b.allocator.create(ComparePanicContext) catch unreachable;
|
||||
cases.* = ComparePanicContext{
|
||||
.b = b,
|
||||
.step = b.step("test-compare-panic", "Run the compare panic tests"),
|
||||
.test_index = 0,
|
||||
.test_filter = test_filter,
|
||||
.modes = modes,
|
||||
};
|
||||
|
||||
compare_panic.addCases(cases);
|
||||
|
||||
return cases.step;
|
||||
}
|
||||
|
||||
pub fn addRuntimeSafetyTests(b: *build.Builder, test_filter: ?[]const u8, modes: []const Mode) *build.Step {
|
||||
const cases = b.allocator.create(CompareOutputContext) catch unreachable;
|
||||
cases.* = CompareOutputContext{
|
||||
|
@ -549,6 +565,200 @@ pub const CompareOutputContext = struct {
|
|||
}
|
||||
};
|
||||
|
||||
pub const ComparePanicContext = struct {
|
||||
b: *build.Builder,
|
||||
step: *build.Step,
|
||||
test_index: usize,
|
||||
test_filter: ?[]const u8,
|
||||
modes: []const Mode,
|
||||
|
||||
const Expect = [@typeInfo(Mode).Enum.fields.len][]const u8;
|
||||
|
||||
pub fn addCase(
|
||||
self: *ComparePanicContext,
|
||||
name: []const u8,
|
||||
source: []const u8,
|
||||
expect: Expect,
|
||||
) void {
|
||||
const b = self.b;
|
||||
|
||||
const source_pathname = fs.path.join(
|
||||
b.allocator,
|
||||
[][]const u8{ b.cache_root, "source.zig" },
|
||||
) catch unreachable;
|
||||
|
||||
for (self.modes) |mode| {
|
||||
const expect_for_mode = expect[@enumToInt(mode)];
|
||||
if (expect_for_mode.len == 0) continue;
|
||||
|
||||
const annotated_case_name = fmt.allocPrint(self.b.allocator, "{} {} ({})", "compare-panic", name, @tagName(mode)) catch unreachable;
|
||||
if (self.test_filter) |filter| {
|
||||
if (mem.indexOf(u8, annotated_case_name, filter) == null) continue;
|
||||
}
|
||||
|
||||
const exe = b.addExecutable("test", source_pathname);
|
||||
exe.setBuildMode(mode);
|
||||
|
||||
const write_source = b.addWriteFile(source_pathname, source);
|
||||
exe.step.dependOn(&write_source.step);
|
||||
|
||||
const run_and_compare = RunAndCompareStep.create(
|
||||
self,
|
||||
exe,
|
||||
annotated_case_name,
|
||||
mode,
|
||||
expect_for_mode,
|
||||
);
|
||||
|
||||
self.step.dependOn(&run_and_compare.step);
|
||||
}
|
||||
}
|
||||
|
||||
const RunAndCompareStep = struct {
|
||||
step: build.Step,
|
||||
context: *ComparePanicContext,
|
||||
exe: *LibExeObjStep,
|
||||
name: []const u8,
|
||||
mode: Mode,
|
||||
expect_output: []const u8,
|
||||
test_index: usize,
|
||||
|
||||
pub fn create(
|
||||
context: *ComparePanicContext,
|
||||
exe: *LibExeObjStep,
|
||||
name: []const u8,
|
||||
mode: Mode,
|
||||
expect_output: []const u8,
|
||||
) *RunAndCompareStep {
|
||||
const allocator = context.b.allocator;
|
||||
const ptr = allocator.create(RunAndCompareStep) catch unreachable;
|
||||
ptr.* = RunAndCompareStep{
|
||||
.step = build.Step.init("PanicCompareOutputStep", allocator, make),
|
||||
.context = context,
|
||||
.exe = exe,
|
||||
.name = name,
|
||||
.mode = mode,
|
||||
.expect_output = expect_output,
|
||||
.test_index = context.test_index,
|
||||
};
|
||||
ptr.step.dependOn(&exe.step);
|
||||
context.test_index += 1;
|
||||
return ptr;
|
||||
}
|
||||
|
||||
fn make(step: *build.Step) !void {
|
||||
const self = @fieldParentPtr(RunAndCompareStep, "step", step);
|
||||
const b = self.context.b;
|
||||
|
||||
const full_exe_path = self.exe.getOutputPath();
|
||||
var args = ArrayList([]const u8).init(b.allocator);
|
||||
defer args.deinit();
|
||||
args.append(full_exe_path) catch unreachable;
|
||||
|
||||
warn("Test {}/{} {}...", self.test_index + 1, self.context.test_index, self.name);
|
||||
|
||||
const child = std.ChildProcess.init(args.toSliceConst(), b.allocator) catch unreachable;
|
||||
defer child.deinit();
|
||||
|
||||
child.stdin_behavior = .Ignore;
|
||||
child.stdout_behavior = .Pipe;
|
||||
child.stderr_behavior = .Pipe;
|
||||
child.env_map = b.env_map;
|
||||
|
||||
child.spawn() catch |err| debug.panic("Unable to spawn {}: {}\n", full_exe_path, @errorName(err));
|
||||
|
||||
var stdout = Buffer.initNull(b.allocator);
|
||||
var stderr = Buffer.initNull(b.allocator);
|
||||
|
||||
var stdout_file_in_stream = child.stdout.?.inStream();
|
||||
var stderr_file_in_stream = child.stderr.?.inStream();
|
||||
|
||||
stdout_file_in_stream.stream.readAllBuffer(&stdout, max_stdout_size) catch unreachable;
|
||||
stderr_file_in_stream.stream.readAllBuffer(&stderr, max_stdout_size) catch unreachable;
|
||||
|
||||
const term = child.wait() catch |err| {
|
||||
debug.panic("Unable to spawn {}: {}\n", full_exe_path, @errorName(err));
|
||||
};
|
||||
|
||||
switch (term) {
|
||||
.Exited => |code| {
|
||||
const expect_code: u32 = 1;
|
||||
if (code != expect_code) {
|
||||
warn("Process {} exited with error code {} but expected code {}\n", full_exe_path, code, expect_code);
|
||||
printInvocation(args.toSliceConst());
|
||||
return error.TestFailed;
|
||||
}
|
||||
},
|
||||
.Signal => |signum| {
|
||||
warn("Process {} terminated on signal {}\n", full_exe_path, signum);
|
||||
printInvocation(args.toSliceConst());
|
||||
return error.TestFailed;
|
||||
},
|
||||
.Stopped => |signum| {
|
||||
warn("Process {} stopped on signal {}\n", full_exe_path, signum);
|
||||
printInvocation(args.toSliceConst());
|
||||
return error.TestFailed;
|
||||
},
|
||||
.Unknown => |code| {
|
||||
warn("Process {} terminated unexpectedly with error code {}\n", full_exe_path, code);
|
||||
printInvocation(args.toSliceConst());
|
||||
return error.TestFailed;
|
||||
},
|
||||
}
|
||||
|
||||
// process result
|
||||
// - keep only basename of source file path
|
||||
// - replace address with symbolic string
|
||||
// - skip empty lines
|
||||
const got: []const u8 = got_result: {
|
||||
var buf = try Buffer.initSize(b.allocator, 0);
|
||||
defer buf.deinit();
|
||||
var bytes = stderr.toSliceConst();
|
||||
if (bytes.len != 0 and bytes[bytes.len - 1] == '\n') bytes = bytes[0 .. bytes.len - 1];
|
||||
var it = mem.separate(bytes, "\n");
|
||||
process_lines: while (it.next()) |line| {
|
||||
if (line.len == 0) continue;
|
||||
const delims = []const []const u8{ ":", ":", ":", " in " };
|
||||
var marks = []usize{0} ** 4;
|
||||
// offset search past `[drive]:` on windows
|
||||
var pos: usize = if (builtin.os == .windows) 2 else 0;
|
||||
for (delims) |delim, i| {
|
||||
marks[i] = mem.indexOfPos(u8, line, pos, delim) orelse {
|
||||
try buf.append(line);
|
||||
try buf.append("\n");
|
||||
continue :process_lines;
|
||||
};
|
||||
pos = marks[i] + delim.len;
|
||||
}
|
||||
pos = mem.lastIndexOfScalar(u8, line[0..marks[0]], fs.path.sep) orelse {
|
||||
try buf.append(line);
|
||||
try buf.append("\n");
|
||||
continue :process_lines;
|
||||
};
|
||||
try buf.append(line[pos + 1 .. marks[2] + delims[2].len]);
|
||||
try buf.append(" [address]");
|
||||
try buf.append(line[marks[3]..]);
|
||||
try buf.append("\n");
|
||||
}
|
||||
break :got_result buf.toOwnedSlice();
|
||||
};
|
||||
|
||||
if (!mem.eql(u8, self.expect_output, got)) {
|
||||
warn(
|
||||
\\
|
||||
\\========= Expected this output: =========
|
||||
\\{}
|
||||
\\================================================
|
||||
\\{}
|
||||
\\
|
||||
, self.expect_output, got);
|
||||
return error.TestFailed;
|
||||
}
|
||||
warn("OK\n");
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
pub const CompileErrorContext = struct {
|
||||
b: *build.Builder,
|
||||
step: *build.Step,
|
||||
|
|
Loading…
Reference in New Issue