self-hosted: update main.zig

After this commit there are no more bit rotted files.
The testing program that was in ir.zig has been moved to main.zig
Unsupported command line options have been deleted, or error messages
added.

The compiler repl is available from the build-exe, build-lib,
build-obj commands with the --watch option.

The main zig build script now builds the self-hosted compiler
unconditionally. Linking against LLVM is behind a -Denable-llvm
flag that defaults to off.
This commit is contained in:
Andrew Kelley 2020-05-15 15:20:42 -04:00
parent ebb81ebe59
commit e1d4b59c5b
7 changed files with 450 additions and 2605 deletions

View File

@ -51,6 +51,9 @@ pub fn build(b: *Builder) !void {
var exe = b.addExecutable("zig", "src-self-hosted/main.zig");
exe.setBuildMode(mode);
test_step.dependOn(&exe.step);
b.default_step.dependOn(&exe.step);
exe.install();
const skip_release = b.option(bool, "skip-release", "Main test suite skips release builds") orelse false;
const skip_release_small = b.option(bool, "skip-release-small", "Main test suite skips release-small builds") orelse skip_release;
@ -58,21 +61,17 @@ pub fn build(b: *Builder) !void {
const skip_release_safe = b.option(bool, "skip-release-safe", "Main test suite skips release-safe builds") orelse skip_release;
const skip_non_native = b.option(bool, "skip-non-native", "Main test suite skips non-native builds") orelse false;
const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false;
const skip_self_hosted = (b.option(bool, "skip-self-hosted", "Main test suite skips building self hosted compiler") orelse false) or true; // TODO evented I/O good enough that this passes everywhere
if (!skip_self_hosted) {
test_step.dependOn(&exe.step);
}
const only_install_lib_files = b.option(bool, "lib-files-only", "Only install library files") orelse false;
if (!only_install_lib_files and !skip_self_hosted) {
const enable_llvm = b.option(bool, "enable-llvm", "Build self-hosted compiler with LLVM backend enabled") orelse false;
if (enable_llvm) {
var ctx = parseConfigH(b, config_h_text);
ctx.llvm = try findLLVM(b, ctx.llvm_config_exe);
try configureStage2(b, exe, ctx);
b.default_step.dependOn(&exe.step);
exe.install();
}
const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse false;
if (link_libc) exe.linkLibC();
b.installDirectory(InstallDirectoryOptions{
.source_dir = "lib",

File diff suppressed because it is too large Load Diff

View File

@ -1,284 +0,0 @@
const std = @import("std");
const mem = std.mem;
const fs = std.fs;
const process = std.process;
const Token = std.zig.Token;
const ast = std.zig.ast;
const TokenIndex = std.zig.ast.TokenIndex;
const Compilation = @import("compilation.zig").Compilation;
const Scope = @import("scope.zig").Scope;
pub const Color = enum {
Auto,
Off,
On,
};
pub const Span = struct {
first: ast.TokenIndex,
last: ast.TokenIndex,
pub fn token(i: TokenIndex) Span {
return Span{
.first = i,
.last = i,
};
}
pub fn node(n: *ast.Node) Span {
return Span{
.first = n.firstToken(),
.last = n.lastToken(),
};
}
};
pub const Msg = struct {
text: []u8,
realpath: []u8,
data: Data,
const Data = union(enum) {
Cli: Cli,
PathAndTree: PathAndTree,
ScopeAndComp: ScopeAndComp,
};
const PathAndTree = struct {
span: Span,
tree: *ast.Tree,
allocator: *mem.Allocator,
};
const ScopeAndComp = struct {
span: Span,
tree_scope: *Scope.AstTree,
compilation: *Compilation,
};
const Cli = struct {
allocator: *mem.Allocator,
};
pub fn destroy(self: *Msg) void {
switch (self.data) {
.Cli => |cli| {
cli.allocator.free(self.text);
cli.allocator.free(self.realpath);
cli.allocator.destroy(self);
},
.PathAndTree => |path_and_tree| {
path_and_tree.allocator.free(self.text);
path_and_tree.allocator.free(self.realpath);
path_and_tree.allocator.destroy(self);
},
.ScopeAndComp => |scope_and_comp| {
scope_and_comp.tree_scope.base.deref(scope_and_comp.compilation);
scope_and_comp.compilation.gpa().free(self.text);
scope_and_comp.compilation.gpa().free(self.realpath);
scope_and_comp.compilation.gpa().destroy(self);
},
}
}
fn getAllocator(self: *const Msg) *mem.Allocator {
switch (self.data) {
.Cli => |cli| return cli.allocator,
.PathAndTree => |path_and_tree| {
return path_and_tree.allocator;
},
.ScopeAndComp => |scope_and_comp| {
return scope_and_comp.compilation.gpa();
},
}
}
pub fn getTree(self: *const Msg) *ast.Tree {
switch (self.data) {
.Cli => unreachable,
.PathAndTree => |path_and_tree| {
return path_and_tree.tree;
},
.ScopeAndComp => |scope_and_comp| {
return scope_and_comp.tree_scope.tree;
},
}
}
pub fn getSpan(self: *const Msg) Span {
return switch (self.data) {
.Cli => unreachable,
.PathAndTree => |path_and_tree| path_and_tree.span,
.ScopeAndComp => |scope_and_comp| scope_and_comp.span,
};
}
/// Takes ownership of text
/// References tree_scope, and derefs when the msg is freed
pub fn createFromScope(comp: *Compilation, tree_scope: *Scope.AstTree, span: Span, text: []u8) !*Msg {
const realpath = try mem.dupe(comp.gpa(), u8, tree_scope.root().realpath);
errdefer comp.gpa().free(realpath);
const msg = try comp.gpa().create(Msg);
msg.* = Msg{
.text = text,
.realpath = realpath,
.data = Data{
.ScopeAndComp = ScopeAndComp{
.tree_scope = tree_scope,
.compilation = comp,
.span = span,
},
},
};
tree_scope.base.ref();
return msg;
}
/// Caller owns returned Msg and must free with `allocator`
/// allocator will additionally be used for printing messages later.
pub fn createFromCli(comp: *Compilation, realpath: []const u8, text: []u8) !*Msg {
const realpath_copy = try mem.dupe(comp.gpa(), u8, realpath);
errdefer comp.gpa().free(realpath_copy);
const msg = try comp.gpa().create(Msg);
msg.* = Msg{
.text = text,
.realpath = realpath_copy,
.data = Data{
.Cli = Cli{ .allocator = comp.gpa() },
},
};
return msg;
}
pub fn createFromParseErrorAndScope(
comp: *Compilation,
tree_scope: *Scope.AstTree,
parse_error: *const ast.Error,
) !*Msg {
const loc_token = parse_error.loc();
var text_buf = std.ArrayList(u8).init(comp.gpa());
defer text_buf.deinit();
const realpath_copy = try mem.dupe(comp.gpa(), u8, tree_scope.root().realpath);
errdefer comp.gpa().free(realpath_copy);
try parse_error.render(&tree_scope.tree.tokens, text_buf.outStream());
const msg = try comp.gpa().create(Msg);
msg.* = Msg{
.text = undefined,
.realpath = realpath_copy,
.data = Data{
.ScopeAndComp = ScopeAndComp{
.tree_scope = tree_scope,
.compilation = comp,
.span = Span{
.first = loc_token,
.last = loc_token,
},
},
},
};
tree_scope.base.ref();
msg.text = text_buf.toOwnedSlice();
return msg;
}
/// `realpath` must outlive the returned Msg
/// `tree` must outlive the returned Msg
/// Caller owns returned Msg and must free with `allocator`
/// allocator will additionally be used for printing messages later.
pub fn createFromParseError(
allocator: *mem.Allocator,
parse_error: *const ast.Error,
tree: *ast.Tree,
realpath: []const u8,
) !*Msg {
const loc_token = parse_error.loc();
var text_buf = std.ArrayList(u8).init(allocator);
defer text_buf.deinit();
const realpath_copy = try mem.dupe(allocator, u8, realpath);
errdefer allocator.free(realpath_copy);
try parse_error.render(&tree.tokens, text_buf.outStream());
const msg = try allocator.create(Msg);
msg.* = Msg{
.text = undefined,
.realpath = realpath_copy,
.data = Data{
.PathAndTree = PathAndTree{
.allocator = allocator,
.tree = tree,
.span = Span{
.first = loc_token,
.last = loc_token,
},
},
},
};
msg.text = text_buf.toOwnedSlice();
errdefer allocator.destroy(msg);
return msg;
}
pub fn printToStream(msg: *const Msg, stream: var, color_on: bool) !void {
switch (msg.data) {
.Cli => {
try stream.print("{}:-:-: error: {}\n", .{ msg.realpath, msg.text });
return;
},
else => {},
}
const allocator = msg.getAllocator();
const tree = msg.getTree();
const cwd = try process.getCwdAlloc(allocator);
defer allocator.free(cwd);
const relpath = try fs.path.relative(allocator, cwd, msg.realpath);
defer allocator.free(relpath);
const path = if (relpath.len < msg.realpath.len) relpath else msg.realpath;
const span = msg.getSpan();
const first_token = tree.tokens.at(span.first);
const last_token = tree.tokens.at(span.last);
const start_loc = tree.tokenLocationPtr(0, first_token);
const end_loc = tree.tokenLocationPtr(first_token.end, last_token);
if (!color_on) {
try stream.print("{}:{}:{}: error: {}\n", .{
path,
start_loc.line + 1,
start_loc.column + 1,
msg.text,
});
return;
}
try stream.print("{}:{}:{}: error: {}\n{}\n", .{
path,
start_loc.line + 1,
start_loc.column + 1,
msg.text,
tree.source[start_loc.line_start..start_loc.line_end],
});
try stream.writeByteNTimes(' ', start_loc.column);
try stream.writeByteNTimes('~', last_token.end - first_token.start);
try stream.writeAll("\n");
}
pub fn printToFile(msg: *const Msg, file: fs.File, color: Color) !void {
const color_on = switch (color) {
.Auto => file.isTty(),
.On => true,
.Off => false,
};
return msg.printToStream(file.outStream(), color_on);
}
};

View File

@ -922,7 +922,6 @@ pub const Module = struct {
if (self.decl_table.get(hash)) |kv| {
return kv.value;
} else {
std.debug.warn("creating new decl for {}\n", .{old_inst.name});
const new_decl = blk: {
try self.decl_table.ensureCapacity(self.decl_table.size + 1);
const new_decl = try self.allocator.create(Decl);
@ -2161,101 +2160,3 @@ pub const ErrorMsg = struct {
self.* = undefined;
}
};
pub fn main() anyerror!void {
var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
defer arena.deinit();
const allocator = if (std.builtin.link_libc) std.heap.c_allocator else &arena.allocator;
const args = try std.process.argsAlloc(allocator);
defer std.process.argsFree(allocator, args);
const src_path = args[1];
const bin_path = args[2];
const debug_error_trace = false;
const output_zir = false;
const object_format: ?std.builtin.ObjectFormat = null;
const native_info = try std.zig.system.NativeTargetInfo.detect(allocator, .{});
var bin_file = try link.openBinFilePath(allocator, std.fs.cwd(), bin_path, .{
.target = native_info.target,
.output_mode = .Exe,
.link_mode = .Static,
.object_format = object_format orelse native_info.target.getObjectFormat(),
});
defer bin_file.deinit();
var module = blk: {
const root_pkg = try Package.create(allocator, std.fs.cwd(), ".", src_path);
errdefer root_pkg.destroy();
const root_scope = try allocator.create(Module.Scope.ZIRModule);
errdefer allocator.destroy(root_scope);
root_scope.* = .{
.sub_file_path = root_pkg.root_src_path,
.source = .{ .unloaded = {} },
.contents = .{ .not_available = {} },
.status = .never_loaded,
};
break :blk Module{
.allocator = allocator,
.root_pkg = root_pkg,
.root_scope = root_scope,
.bin_file = &bin_file,
.optimize_mode = .Debug,
.decl_table = std.AutoHashMap(Module.Decl.Hash, *Module.Decl).init(allocator),
.decl_exports = std.AutoHashMap(*Module.Decl, []*Module.Export).init(allocator),
.export_owners = std.AutoHashMap(*Module.Decl, []*Module.Export).init(allocator),
.failed_decls = std.AutoHashMap(*Module.Decl, *ErrorMsg).init(allocator),
.failed_files = std.AutoHashMap(*Module.Scope.ZIRModule, *ErrorMsg).init(allocator),
.failed_exports = std.AutoHashMap(*Module.Export, *ErrorMsg).init(allocator),
.work_queue = std.fifo.LinearFifo(Module.WorkItem, .Dynamic).init(allocator),
};
};
defer module.deinit();
const stdin = std.io.getStdIn().inStream();
const stderr = std.io.getStdErr().outStream();
var repl_buf: [1024]u8 = undefined;
while (true) {
try module.update();
var errors = try module.getAllErrorsAlloc();
defer errors.deinit(allocator);
if (errors.list.len != 0) {
for (errors.list) |full_err_msg| {
std.debug.warn("{}:{}:{}: error: {}\n", .{
full_err_msg.src_path,
full_err_msg.line + 1,
full_err_msg.column + 1,
full_err_msg.msg,
});
}
if (debug_error_trace) return error.AnalysisFail;
}
try stderr.print("🦎 ", .{});
if (try stdin.readUntilDelimiterOrEof(&repl_buf, '\n')) |line| {
if (mem.eql(u8, line, "update")) {
continue;
} else {
try stderr.print("unknown command: {}\n", .{line});
}
} else {
break;
}
}
if (output_zir) {
var new_zir_module = try text.emit_zir(allocator, module);
defer new_zir_module.deinit(allocator);
var bos = std.io.bufferedOutStream(std.io.getStdOut().outStream());
try new_zir_module.writeToStream(allocator, bos.outStream());
try bos.flush();
}
}

View File

@ -20,7 +20,7 @@ pub const Inst = struct {
name: []const u8,
/// Slice into the source of the part after the = and before the next instruction.
contents: []const u8,
contents: []const u8 = &[0]u8{},
/// These names are used directly as the instruction names in the text format.
pub const Tag = enum {
@ -825,7 +825,6 @@ const Parser = struct {
.name = inst_name,
.src = self.i,
.tag = InstType.base_tag,
.contents = undefined,
};
if (@hasField(InstType, "ty")) {
@ -960,7 +959,6 @@ const Parser = struct {
.name = try self.generateName(),
.src = src,
.tag = Inst.Str.base_tag,
.contents = undefined,
},
.positionals = .{ .bytes = ident },
.kw_args = .{},
@ -971,7 +969,6 @@ const Parser = struct {
.name = try self.generateName(),
.src = src,
.tag = Inst.DeclRef.base_tag,
.contents = undefined,
},
.positionals = .{ .name = &name.base },
.kw_args = .{},

File diff suppressed because it is too large Load Diff

View File

@ -12,7 +12,6 @@ const ArrayListSentineled = std.ArrayListSentineled;
const Target = std.Target;
const CrossTarget = std.zig.CrossTarget;
const self_hosted_main = @import("main.zig");
const errmsg = @import("errmsg.zig");
const DepTokenizer = @import("dep_tokenizer.zig").Tokenizer;
const assert = std.debug.assert;
const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
@ -168,8 +167,6 @@ export fn stage2_render_ast(tree: *ast.Tree, output_file: *FILE) Error {
return .None;
}
// TODO: just use the actual self-hosted zig fmt. Until https://github.com/ziglang/zig/issues/2377,
// we use a blocking implementation.
export fn stage2_fmt(argc: c_int, argv: [*]const [*:0]const u8) c_int {
if (std.debug.runtime_safety) {
fmtMain(argc, argv) catch unreachable;
@ -191,258 +188,9 @@ fn fmtMain(argc: c_int, argv: [*]const [*:0]const u8) !void {
try args_list.append(mem.spanZ(argv[arg_i]));
}
stdout = std.io.getStdOut().outStream();
stderr_file = std.io.getStdErr();
stderr = stderr_file.outStream();
const args = args_list.span()[2..];
var color: errmsg.Color = .Auto;
var stdin_flag: bool = false;
var check_flag: bool = false;
var input_files = ArrayList([]const u8).init(allocator);
{
var i: usize = 0;
while (i < args.len) : (i += 1) {
const arg = args[i];
if (mem.startsWith(u8, arg, "-")) {
if (mem.eql(u8, arg, "--help")) {
try stdout.writeAll(self_hosted_main.usage_fmt);
process.exit(0);
} else if (mem.eql(u8, arg, "--color")) {
if (i + 1 >= args.len) {
try stderr.writeAll("expected [auto|on|off] after --color\n");
process.exit(1);
}
i += 1;
const next_arg = args[i];
if (mem.eql(u8, next_arg, "auto")) {
color = .Auto;
} else if (mem.eql(u8, next_arg, "on")) {
color = .On;
} else if (mem.eql(u8, next_arg, "off")) {
color = .Off;
} else {
try stderr.print("expected [auto|on|off] after --color, found '{}'\n", .{next_arg});
process.exit(1);
}
} else if (mem.eql(u8, arg, "--stdin")) {
stdin_flag = true;
} else if (mem.eql(u8, arg, "--check")) {
check_flag = true;
} else {
try stderr.print("unrecognized parameter: '{}'", .{arg});
process.exit(1);
}
} else {
try input_files.append(arg);
}
}
}
if (stdin_flag) {
if (input_files.items.len != 0) {
try stderr.writeAll("cannot use --stdin with positional arguments\n");
process.exit(1);
}
const stdin_file = io.getStdIn();
var stdin = stdin_file.inStream();
const source_code = try stdin.readAllAlloc(allocator, self_hosted_main.max_src_size);
defer allocator.free(source_code);
const tree = std.zig.parse(allocator, source_code) catch |err| {
try stderr.print("error parsing stdin: {}\n", .{err});
process.exit(1);
};
defer tree.deinit();
var error_it = tree.errors.iterator(0);
while (error_it.next()) |parse_error| {
try printErrMsgToFile(allocator, parse_error, tree, "<stdin>", stderr_file, color);
}
if (tree.errors.len != 0) {
process.exit(1);
}
if (check_flag) {
const anything_changed = try std.zig.render(allocator, io.null_out_stream, tree);
const code = if (anything_changed) @as(u8, 1) else @as(u8, 0);
process.exit(code);
}
_ = try std.zig.render(allocator, stdout, tree);
return;
}
if (input_files.items.len == 0) {
try stderr.writeAll("expected at least one source file argument\n");
process.exit(1);
}
var fmt = Fmt{
.seen = Fmt.SeenMap.init(allocator),
.any_error = false,
.color = color,
.allocator = allocator,
};
for (input_files.span()) |file_path| {
try fmtPath(&fmt, file_path, check_flag);
}
if (fmt.any_error) {
process.exit(1);
}
}
const FmtError = error{
SystemResources,
OperationAborted,
IoPending,
BrokenPipe,
Unexpected,
WouldBlock,
FileClosed,
DestinationAddressRequired,
DiskQuota,
FileTooBig,
InputOutput,
NoSpaceLeft,
AccessDenied,
OutOfMemory,
RenameAcrossMountPoints,
ReadOnlyFileSystem,
LinkQuotaExceeded,
FileBusy,
} || fs.File.OpenError;
fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool) FmtError!void {
// get the real path here to avoid Windows failing on relative file paths with . or .. in them
var real_path = fs.realpathAlloc(fmt.allocator, file_path) catch |err| {
try stderr.print("unable to open '{}': {}\n", .{ file_path, err });
fmt.any_error = true;
return;
};
defer fmt.allocator.free(real_path);
if (fmt.seen.exists(real_path)) return;
try fmt.seen.put(real_path);
const source_code = fs.cwd().readFileAlloc(fmt.allocator, real_path, self_hosted_main.max_src_size) catch |err| switch (err) {
error.IsDir, error.AccessDenied => {
// TODO make event based (and dir.next())
var dir = try fs.cwd().openDir(file_path, .{ .iterate = true });
defer dir.close();
var dir_it = dir.iterate();
while (try dir_it.next()) |entry| {
if (entry.kind == .Directory or mem.endsWith(u8, entry.name, ".zig")) {
const full_path = try fs.path.join(fmt.allocator, &[_][]const u8{ file_path, entry.name });
try fmtPath(fmt, full_path, check_mode);
}
}
return;
},
else => {
// TODO lock stderr printing
try stderr.print("unable to open '{}': {}\n", .{ file_path, err });
fmt.any_error = true;
return;
},
};
defer fmt.allocator.free(source_code);
const tree = std.zig.parse(fmt.allocator, source_code) catch |err| {
try stderr.print("error parsing file '{}': {}\n", .{ file_path, err });
fmt.any_error = true;
return;
};
defer tree.deinit();
var error_it = tree.errors.iterator(0);
while (error_it.next()) |parse_error| {
try printErrMsgToFile(fmt.allocator, parse_error, tree, file_path, stderr_file, fmt.color);
}
if (tree.errors.len != 0) {
fmt.any_error = true;
return;
}
if (check_mode) {
const anything_changed = try std.zig.render(fmt.allocator, io.null_out_stream, tree);
if (anything_changed) {
try stderr.print("{}\n", .{file_path});
fmt.any_error = true;
}
} else {
const baf = try io.BufferedAtomicFile.create(fmt.allocator, fs.cwd(), real_path, .{});
defer baf.destroy();
const anything_changed = try std.zig.render(fmt.allocator, baf.stream(), tree);
if (anything_changed) {
try stderr.print("{}\n", .{file_path});
try baf.finish();
}
}
}
const Fmt = struct {
seen: SeenMap,
any_error: bool,
color: errmsg.Color,
allocator: *mem.Allocator,
const SeenMap = std.BufSet;
};
fn printErrMsgToFile(
allocator: *mem.Allocator,
parse_error: *const ast.Error,
tree: *ast.Tree,
path: []const u8,
file: fs.File,
color: errmsg.Color,
) !void {
const color_on = switch (color) {
.Auto => file.isTty(),
.On => true,
.Off => false,
};
const lok_token = parse_error.loc();
const span = errmsg.Span{
.first = lok_token,
.last = lok_token,
};
const first_token = tree.tokens.at(span.first);
const last_token = tree.tokens.at(span.last);
const start_loc = tree.tokenLocationPtr(0, first_token);
const end_loc = tree.tokenLocationPtr(first_token.end, last_token);
var text_buf = std.ArrayList(u8).init(allocator);
defer text_buf.deinit();
const out_stream = text_buf.outStream();
try parse_error.render(&tree.tokens, out_stream);
const text = text_buf.span();
const stream = file.outStream();
try stream.print("{}:{}:{}: error: {}\n", .{ path, start_loc.line + 1, start_loc.column + 1, text });
if (!color_on) return;
// Print \r and \t as one space each so that column counts line up
for (tree.source[start_loc.line_start..start_loc.line_end]) |byte| {
try stream.writeByte(switch (byte) {
'\r', '\t' => ' ',
else => byte,
});
}
try stream.writeByte('\n');
try stream.writeByteNTimes(' ', start_loc.column);
try stream.writeByteNTimes('~', last_token.end - first_token.start);
try stream.writeByte('\n');
return self_hosted_main.cmdFmt(allocator, args);
}
export fn stage2_DepTokenizer_init(input: [*]const u8, len: usize) stage2_DepTokenizer {