diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 77a2b8cd6..903dc051e 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -6,6 +6,7 @@ const mem = std.mem; pub const Node = struct { id: Id, + comment: ?&NodeLineComment, pub const Id = enum { Root, @@ -20,6 +21,7 @@ pub const Node = struct { FloatLiteral, StringLiteral, BuiltinCall, + LineComment, }; pub fn iterate(base: &Node, index: usize) ?&Node { @@ -36,6 +38,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), + Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), }; } @@ -53,6 +56,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), + Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), }; } @@ -70,6 +74,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), + Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), }; } }; @@ -454,3 +459,20 @@ pub const NodeStringLiteral = struct { return self.token; } }; + +pub const NodeLineComment = struct { + base: Node, + lines: ArrayList(Token), + + pub fn iterate(self: &NodeLineComment, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeLineComment) Token { + return self.lines.at(0); + } + + pub fn lastToken(self: &NodeLineComment) Token { + return self.lines.at(self.lines.len - 1); + } +}; diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 35bba378d..533ad754a 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -18,6 +18,7 @@ pub const Parser = struct { put_back_tokens: [2]Token, put_back_count: usize, source_file_name: []const u8, + pending_line_comment_node: ?&ast.NodeLineComment, pub const Tree = struct { root_node: &ast.NodeRoot, @@ -43,6 +44,7 @@ pub const Parser = struct { .put_back_count = 0, .source_file_name = source_file_name, .utility_bytes = []align(utility_bytes_align) u8{}, + .pending_line_comment_node = null, }; } @@ -131,6 +133,33 @@ pub const Parser = struct { // warn("\n"); //} + // look for line comments + while (true) { + const token = self.getNextToken(); + if (token.id == Token.Id.LineComment) { + const node = blk: { + if (self.pending_line_comment_node) |comment_node| { + break :blk comment_node; + } else { + const comment_node = try arena.create(ast.NodeLineComment); + *comment_node = ast.NodeLineComment { + .base = ast.Node { + .id = ast.Node.Id.LineComment, + .comment = null, + }, + .lines = ArrayList(Token).init(arena), + }; + self.pending_line_comment_node = comment_node; + break :blk comment_node; + } + }; + try node.lines.append(token); + continue; + } + self.putBackToken(token); + break; + } + // This gives us 1 free append that can't fail const state = stack.pop(); @@ -329,7 +358,7 @@ pub const Parser = struct { Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { - .base = ast.Node {.id = ast.Node.Id.BuiltinCall}, + .base = self.initNode(ast.Node.Id.BuiltinCall), .builtin_token = token, .params = ArrayList(&ast.Node).init(arena), .rparen_token = undefined, @@ -350,7 +379,7 @@ pub const Parser = struct { Token.Id.StringLiteral => { const node = try arena.create(ast.NodeStringLiteral); *node = ast.NodeStringLiteral { - .base = ast.Node {.id = ast.Node.Id.StringLiteral}, + .base = self.initNode(ast.Node.Id.StringLiteral), .token = token, }; try stack.append(State { @@ -359,6 +388,7 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), } }, @@ -660,11 +690,19 @@ pub const Parser = struct { } } + fn initNode(self: &Parser, id: ast.Node.Id) ast.Node { + if (self.pending_line_comment_node) |comment_node| { + self.pending_line_comment_node = null; + return ast.Node {.id = id, .comment = comment_node}; + } + return ast.Node {.id = id, .comment = null }; + } + fn createRoot(self: &Parser, arena: &mem.Allocator) !&ast.NodeRoot { const node = try arena.create(ast.NodeRoot); *node = ast.NodeRoot { - .base = ast.Node {.id = ast.Node.Id.Root}, + .base = self.initNode(ast.Node.Id.Root), .decls = ArrayList(&ast.Node).init(arena), // initialized when we get the eof token .eof_token = undefined, @@ -678,7 +716,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeVarDecl); *node = ast.NodeVarDecl { - .base = ast.Node {.id = ast.Node.Id.VarDecl}, + .base = self.initNode(ast.Node.Id.VarDecl), .visib_token = *visib_token, .mut_token = *mut_token, .comptime_token = *comptime_token, @@ -701,7 +739,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeFnProto); *node = ast.NodeFnProto { - .base = ast.Node {.id = ast.Node.Id.FnProto}, + .base = self.initNode(ast.Node.Id.FnProto), .visib_token = *visib_token, .name_token = null, .fn_token = *fn_token, @@ -722,7 +760,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeParamDecl); *node = ast.NodeParamDecl { - .base = ast.Node {.id = ast.Node.Id.ParamDecl}, + .base = self.initNode(ast.Node.Id.ParamDecl), .comptime_token = null, .noalias_token = null, .name_token = null, @@ -736,7 +774,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeBlock); *node = ast.NodeBlock { - .base = ast.Node {.id = ast.Node.Id.Block}, + .base = self.initNode(ast.Node.Id.Block), .begin_token = *begin_token, .end_token = undefined, .statements = ArrayList(&ast.Node).init(arena), @@ -748,7 +786,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeInfixOp); *node = ast.NodeInfixOp { - .base = ast.Node {.id = ast.Node.Id.InfixOp}, + .base = self.initNode(ast.Node.Id.InfixOp), .op_token = *op_token, .lhs = undefined, .op = *op, @@ -761,7 +799,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodePrefixOp); *node = ast.NodePrefixOp { - .base = ast.Node {.id = ast.Node.Id.PrefixOp}, + .base = self.initNode(ast.Node.Id.PrefixOp), .op_token = *op_token, .op = *op, .rhs = undefined, @@ -773,7 +811,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeIdentifier); *node = ast.NodeIdentifier { - .base = ast.Node {.id = ast.Node.Id.Identifier}, + .base = self.initNode(ast.Node.Id.Identifier), .name_token = *name_token, }; return node; @@ -783,7 +821,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeIntegerLiteral); *node = ast.NodeIntegerLiteral { - .base = ast.Node {.id = ast.Node.Id.IntegerLiteral}, + .base = self.initNode(ast.Node.Id.IntegerLiteral), .token = *token, }; return node; @@ -793,7 +831,7 @@ pub const Parser = struct { const node = try arena.create(ast.NodeFloatLiteral); *node = ast.NodeFloatLiteral { - .base = ast.Node {.id = ast.Node.Id.FloatLiteral}, + .base = self.initNode(ast.Node.Id.FloatLiteral), .token = *token, }; return node; @@ -1158,9 +1196,11 @@ pub const Parser = struct { } } }, + ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), + ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), + ast.Node.Id.Root, ast.Node.Id.VarDecl, - ast.Node.Id.FnProto, ast.Node.Id.ParamDecl => unreachable, }, RenderState.FnProtoRParen => |fn_proto| { @@ -1187,6 +1227,12 @@ pub const Parser = struct { } }, RenderState.Statement => |base| { + if (base.comment) |comment| { + for (comment.lines.toSliceConst()) |line_token| { + try stream.print("{}\n", self.tokenizer.getTokenSlice(line_token)); + try stream.writeByteNTimes(' ', indent); + } + } switch (base.id) { ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base); @@ -1279,6 +1325,17 @@ fn testCanonical(source: []const u8) !void { } test "zig fmt" { + try testCanonical( + \\const std = @import("std"); + \\ + \\pub fn main() !void { + \\ // If this program is run without stdout attached, exit with an error. + \\ // another comment + \\ var stdout_file = try std.io.getStdOut; + \\} + \\ + ); + try testCanonical( \\const std = @import("std"); \\ diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index bdd82e4a4..4af6c20ca 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -99,6 +99,7 @@ pub const Token = struct { AmpersandEqual, IntegerLiteral, FloatLiteral, + LineComment, Keyword_align, Keyword_and, Keyword_asm, @@ -470,7 +471,7 @@ pub const Tokenizer = struct { State.Slash => switch (c) { '/' => { - result.id = undefined; + result.id = Token.Id.LineComment; state = State.LineComment; }, else => { @@ -479,16 +480,7 @@ pub const Tokenizer = struct { }, }, State.LineComment => switch (c) { - '\n' => { - state = State.Start; - result = Token { - .id = Token.Id.Eof, - .start = self.index + 1, - .column = 0, - .line = self.line + 1, - .end = undefined, - }; - }, + '\n' => break, else => self.checkLiteralCharacter(), }, State.Zero => switch (c) {