test: add compare-panic

`zig build test-compare-panic`

Create basic tests to compare panic output. The address field
is replaced by a symbolic constant and each expected output is
specific to os. Tests will only run for explicitly defined
platforms.

see also #2485
master
Michael Dusan 2019-05-27 20:07:05 -04:00 committed by Andrew Kelley
parent d74b8567cf
commit a19e73d8ae
No known key found for this signature in database
GPG Key ID: 7C5F548F728501A9
3 changed files with 488 additions and 0 deletions

View File

@ -138,6 +138,7 @@ pub fn build(b: *Builder) !void {
test_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes)); test_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes));
test_step.dependOn(tests.addStandaloneTests(b, test_filter, modes)); test_step.dependOn(tests.addStandaloneTests(b, test_filter, modes));
test_step.dependOn(tests.addComparePanicTests(b, test_filter, modes));
test_step.dependOn(tests.addCliTests(b, test_filter, modes)); test_step.dependOn(tests.addCliTests(b, test_filter, modes));
test_step.dependOn(tests.addCompileErrorTests(b, test_filter, modes)); test_step.dependOn(tests.addCompileErrorTests(b, test_filter, modes));
test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, modes)); test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, modes));

277
test/compare_panic.zig Normal file
View File

@ -0,0 +1,277 @@
const builtin = @import("builtin");
const std = @import("std");
const os = std.os;
const tests = @import("tests.zig");
pub fn addCases(cases: *tests.ComparePanicContext) void {
const source_return =
\\const std = @import("std");
\\
\\pub fn main() !void {
\\ return error.TheSkyIsFalling;
\\}
;
const source_try_return =
\\const std = @import("std");
\\
\\fn foo() !void {
\\ return error.TheSkyIsFalling;
\\}
\\
\\pub fn main() !void {
\\ try foo();
\\}
;
const source_try_try_return_return =
\\const std = @import("std");
\\
\\fn foo() !void {
\\ try bar();
\\}
\\
\\fn bar() !void {
\\ return make_error();
\\}
\\
\\fn make_error() !void {
\\ return error.TheSkyIsFalling;
\\}
\\
\\pub fn main() !void {
\\ try foo();
\\}
;
switch (builtin.os) {
.linux => {
cases.addCase(
"return",
source_return,
[][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in main (test)
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in std.special.posixCallMainAndExit (test)
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try return",
source_try_return,
[][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in foo (test)
\\source.zig:8:5: [address] in main (test)
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in std.special.posixCallMainAndExit (test)
\\source.zig:8:5: [address] in std.special.posixCallMainAndExit (test)
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try try return return",
source_try_try_return_return,
[][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:12:5: [address] in make_error (test)
\\source.zig:8:5: [address] in bar (test)
\\source.zig:4:5: [address] in foo (test)
\\source.zig:16:5: [address] in main (test)
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:12:5: [address] in std.special.posixCallMainAndExit (test)
\\source.zig:8:5: [address] in std.special.posixCallMainAndExit (test)
\\source.zig:4:5: [address] in std.special.posixCallMainAndExit (test)
\\source.zig:16:5: [address] in std.special.posixCallMainAndExit (test)
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
},
.macosx => {
cases.addCase(
"return",
source_return,
[][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in _main.0 (test.o)
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in _main (test.o)
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try return",
source_try_return,
[][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in _foo (test.o)
\\source.zig:8:5: [address] in _main.0 (test.o)
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in _main (test.o)
\\source.zig:8:5: [address] in _main (test.o)
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try try return return",
source_try_try_return_return,
[][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:12:5: [address] in _make_error (test.o)
\\source.zig:8:5: [address] in _bar (test.o)
\\source.zig:4:5: [address] in _foo (test.o)
\\source.zig:16:5: [address] in _main.0 (test.o)
\\
,
// release-safe
\\error: TheSkyIsFalling
\\source.zig:12:5: [address] in _main (test.o)
\\source.zig:8:5: [address] in _main (test.o)
\\source.zig:4:5: [address] in _main (test.o)
\\source.zig:16:5: [address] in _main (test.o)
\\
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
},
.windows => {
cases.addCase(
"return",
source_return,
[][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in main (test.obj)
\\
,
// release-safe
// --disabled-- results in segmenetation fault
""
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try return",
source_try_return,
[][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:4:5: [address] in foo (test.obj)
\\source.zig:8:5: [address] in main (test.obj)
\\
,
// release-safe
// --disabled-- results in segmenetation fault
""
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
cases.addCase(
"try try return return",
source_try_try_return_return,
[][]const u8{
// debug
\\error: TheSkyIsFalling
\\source.zig:12:5: [address] in make_error (test.obj)
\\source.zig:8:5: [address] in bar (test.obj)
\\source.zig:4:5: [address] in foo (test.obj)
\\source.zig:16:5: [address] in main (test.obj)
\\
,
// release-safe
// --disabled-- results in segmenetation fault
""
,
// release-fast
\\error: TheSkyIsFalling
\\
,
// release-small
\\error: TheSkyIsFalling
\\
},
);
},
else => {},
}
}

View File

@ -16,6 +16,7 @@ const LibExeObjStep = build.LibExeObjStep;
const compare_output = @import("compare_output.zig"); const compare_output = @import("compare_output.zig");
const standalone = @import("standalone.zig"); const standalone = @import("standalone.zig");
const compare_panic = @import("compare_panic.zig");
const compile_errors = @import("compile_errors.zig"); const compile_errors = @import("compile_errors.zig");
const assemble_and_link = @import("assemble_and_link.zig"); const assemble_and_link = @import("assemble_and_link.zig");
const runtime_safety = @import("runtime_safety.zig"); const runtime_safety = @import("runtime_safety.zig");
@ -57,6 +58,21 @@ pub fn addCompareOutputTests(b: *build.Builder, test_filter: ?[]const u8, modes:
return cases.step; return cases.step;
} }
pub fn addComparePanicTests(b: *build.Builder, test_filter: ?[]const u8, modes: []const Mode) *build.Step {
const cases = b.allocator.create(ComparePanicContext) catch unreachable;
cases.* = ComparePanicContext{
.b = b,
.step = b.step("test-compare-panic", "Run the compare panic tests"),
.test_index = 0,
.test_filter = test_filter,
.modes = modes,
};
compare_panic.addCases(cases);
return cases.step;
}
pub fn addRuntimeSafetyTests(b: *build.Builder, test_filter: ?[]const u8, modes: []const Mode) *build.Step { pub fn addRuntimeSafetyTests(b: *build.Builder, test_filter: ?[]const u8, modes: []const Mode) *build.Step {
const cases = b.allocator.create(CompareOutputContext) catch unreachable; const cases = b.allocator.create(CompareOutputContext) catch unreachable;
cases.* = CompareOutputContext{ cases.* = CompareOutputContext{
@ -549,6 +565,200 @@ pub const CompareOutputContext = struct {
} }
}; };
pub const ComparePanicContext = struct {
b: *build.Builder,
step: *build.Step,
test_index: usize,
test_filter: ?[]const u8,
modes: []const Mode,
const Expect = [@typeInfo(Mode).Enum.fields.len][]const u8;
pub fn addCase(
self: *ComparePanicContext,
name: []const u8,
source: []const u8,
expect: Expect,
) void {
const b = self.b;
const source_pathname = fs.path.join(
b.allocator,
[][]const u8{ b.cache_root, "source.zig" },
) catch unreachable;
for (self.modes) |mode| {
const expect_for_mode = expect[@enumToInt(mode)];
if (expect_for_mode.len == 0) continue;
const annotated_case_name = fmt.allocPrint(self.b.allocator, "{} {} ({})", "compare-panic", name, @tagName(mode)) catch unreachable;
if (self.test_filter) |filter| {
if (mem.indexOf(u8, annotated_case_name, filter) == null) continue;
}
const exe = b.addExecutable("test", source_pathname);
exe.setBuildMode(mode);
const write_source = b.addWriteFile(source_pathname, source);
exe.step.dependOn(&write_source.step);
const run_and_compare = RunAndCompareStep.create(
self,
exe,
annotated_case_name,
mode,
expect_for_mode,
);
self.step.dependOn(&run_and_compare.step);
}
}
const RunAndCompareStep = struct {
step: build.Step,
context: *ComparePanicContext,
exe: *LibExeObjStep,
name: []const u8,
mode: Mode,
expect_output: []const u8,
test_index: usize,
pub fn create(
context: *ComparePanicContext,
exe: *LibExeObjStep,
name: []const u8,
mode: Mode,
expect_output: []const u8,
) *RunAndCompareStep {
const allocator = context.b.allocator;
const ptr = allocator.create(RunAndCompareStep) catch unreachable;
ptr.* = RunAndCompareStep{
.step = build.Step.init("PanicCompareOutputStep", allocator, make),
.context = context,
.exe = exe,
.name = name,
.mode = mode,
.expect_output = expect_output,
.test_index = context.test_index,
};
ptr.step.dependOn(&exe.step);
context.test_index += 1;
return ptr;
}
fn make(step: *build.Step) !void {
const self = @fieldParentPtr(RunAndCompareStep, "step", step);
const b = self.context.b;
const full_exe_path = self.exe.getOutputPath();
var args = ArrayList([]const u8).init(b.allocator);
defer args.deinit();
args.append(full_exe_path) catch unreachable;
warn("Test {}/{} {}...", self.test_index + 1, self.context.test_index, self.name);
const child = std.ChildProcess.init(args.toSliceConst(), b.allocator) catch unreachable;
defer child.deinit();
child.stdin_behavior = .Ignore;
child.stdout_behavior = .Pipe;
child.stderr_behavior = .Pipe;
child.env_map = b.env_map;
child.spawn() catch |err| debug.panic("Unable to spawn {}: {}\n", full_exe_path, @errorName(err));
var stdout = Buffer.initNull(b.allocator);
var stderr = Buffer.initNull(b.allocator);
var stdout_file_in_stream = child.stdout.?.inStream();
var stderr_file_in_stream = child.stderr.?.inStream();
stdout_file_in_stream.stream.readAllBuffer(&stdout, max_stdout_size) catch unreachable;
stderr_file_in_stream.stream.readAllBuffer(&stderr, max_stdout_size) catch unreachable;
const term = child.wait() catch |err| {
debug.panic("Unable to spawn {}: {}\n", full_exe_path, @errorName(err));
};
switch (term) {
.Exited => |code| {
const expect_code: u32 = 1;
if (code != expect_code) {
warn("Process {} exited with error code {} but expected code {}\n", full_exe_path, code, expect_code);
printInvocation(args.toSliceConst());
return error.TestFailed;
}
},
.Signal => |signum| {
warn("Process {} terminated on signal {}\n", full_exe_path, signum);
printInvocation(args.toSliceConst());
return error.TestFailed;
},
.Stopped => |signum| {
warn("Process {} stopped on signal {}\n", full_exe_path, signum);
printInvocation(args.toSliceConst());
return error.TestFailed;
},
.Unknown => |code| {
warn("Process {} terminated unexpectedly with error code {}\n", full_exe_path, code);
printInvocation(args.toSliceConst());
return error.TestFailed;
},
}
// process result
// - keep only basename of source file path
// - replace address with symbolic string
// - skip empty lines
const got: []const u8 = got_result: {
var buf = try Buffer.initSize(b.allocator, 0);
defer buf.deinit();
var bytes = stderr.toSliceConst();
if (bytes.len != 0 and bytes[bytes.len - 1] == '\n') bytes = bytes[0 .. bytes.len - 1];
var it = mem.separate(bytes, "\n");
process_lines: while (it.next()) |line| {
if (line.len == 0) continue;
const delims = []const []const u8{ ":", ":", ":", " in " };
var marks = []usize{0} ** 4;
// offset search past `[drive]:` on windows
var pos: usize = if (builtin.os == .windows) 2 else 0;
for (delims) |delim, i| {
marks[i] = mem.indexOfPos(u8, line, pos, delim) orelse {
try buf.append(line);
try buf.append("\n");
continue :process_lines;
};
pos = marks[i] + delim.len;
}
pos = mem.lastIndexOfScalar(u8, line[0..marks[0]], fs.path.sep) orelse {
try buf.append(line);
try buf.append("\n");
continue :process_lines;
};
try buf.append(line[pos + 1 .. marks[2] + delims[2].len]);
try buf.append(" [address]");
try buf.append(line[marks[3]..]);
try buf.append("\n");
}
break :got_result buf.toOwnedSlice();
};
if (!mem.eql(u8, self.expect_output, got)) {
warn(
\\
\\========= Expected this output: =========
\\{}
\\================================================
\\{}
\\
, self.expect_output, got);
return error.TestFailed;
}
warn("OK\n");
}
};
};
pub const CompileErrorContext = struct { pub const CompileErrorContext = struct {
b: *build.Builder, b: *build.Builder,
step: *build.Step, step: *build.Step,