From 295bca9b5f397ff98e5a0162fcb2a9f5e0a3e35c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 22 May 2020 00:28:07 -0400 Subject: [PATCH] stage2 parser: don't append doc comments to the list The DocComment AST node now only points to the first doc comment token. API users are expected to iterate over the following tokens directly. After this commit there are no more linked lists in use in the self-hosted AST API. Performance impact is negligible. Memory usage slightly reduced. --- lib/std/zig/ast.zig | 17 +++++++------- lib/std/zig/parse.zig | 45 ++++++++++++------------------------ lib/std/zig/render.zig | 52 ++++++++++++++++++++++++++++++------------ 3 files changed, 60 insertions(+), 54 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index eff461fdd..eb22ac233 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -1,7 +1,6 @@ const std = @import("../std.zig"); const assert = std.debug.assert; const testing = std.testing; -const LinkedList = std.SinglyLinkedList; const mem = std.mem; const Token = std.zig.Token; @@ -3013,9 +3012,10 @@ pub const Node = struct { pub const DocComment = struct { base: Node = Node{ .id = .DocComment }, - lines: LineList, - - pub const LineList = LinkedList(TokenIndex); + /// Points to the first doc comment token. API users are expected to iterate over the + /// tokens array, looking for more doc comments, ignoring line comments, and stopping + /// at the first other token. + first_line: TokenIndex, pub fn iterate(self: *const DocComment) Node.Iterator { return .{ .parent_node = &self.base, .index = 0 }; @@ -3026,14 +3026,13 @@ pub const Node = struct { } pub fn firstToken(self: *const DocComment) TokenIndex { - return self.lines.first.?.data; + return self.first_line; } + /// Returns the first doc comment line. Be careful, this may not be the desired behavior, + /// which would require the tokens array. pub fn lastToken(self: *const DocComment) TokenIndex { - var node = self.lines.first.?; - while (true) { - node = node.next orelse return node.data; - } + return self.first_line; } }; diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 818446efd..6d568cf5e 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -58,6 +58,8 @@ const Parser = struct { arena: std.heap.ArenaAllocator, gpa: *Allocator, source: []const u8, + /// TODO: Optimization idea: have this be several arrays of the token fields rather + /// than an array of structs. tokens: []const Token, tok_i: TokenIndex, errors: std.ArrayListUnmanaged(AstError), @@ -367,20 +369,13 @@ const Parser = struct { /// Eat a multiline container doc comment fn parseContainerDocComments(p: *Parser) !?*Node { - var lines = Node.DocComment.LineList{}; - var lines_it: *?*Node.DocComment.LineList.Node = &lines.first; - - while (p.eatToken(.ContainerDocComment)) |line| { - lines_it = try p.llpush(TokenIndex, lines_it, line); + if (p.eatToken(.ContainerDocComment)) |first_line| { + while (p.eatToken(.ContainerDocComment)) |_| {} + const node = try p.arena.allocator.create(Node.DocComment); + node.* = .{ .first_line = first_line }; + return &node.base; } - - if (lines.first == null) return null; - - const node = try p.arena.allocator.create(Node.DocComment); - node.* = .{ - .lines = lines, - }; - return &node.base; + return null; } /// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block @@ -3210,20 +3205,13 @@ const Parser = struct { /// Eat a multiline doc comment fn parseDocComment(p: *Parser) !?*Node.DocComment { - var lines = Node.DocComment.LineList{}; - var lines_it = &lines.first; - - while (p.eatToken(.DocComment)) |line| { - lines_it = try p.llpush(TokenIndex, lines_it, line); + if (p.eatToken(.DocComment)) |first_line| { + while (p.eatToken(.DocComment)) |_| {} + const node = try p.arena.allocator.create(Node.DocComment); + node.* = .{ .first_line = first_line }; + return node; } - - if (lines.first == null) return null; - - const node = try p.arena.allocator.create(Node.DocComment); - node.* = .{ - .lines = lines, - }; - return node; + return null; } fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool { @@ -3234,11 +3222,8 @@ const Parser = struct { fn parseAppendedDocComment(p: *Parser, after_token: TokenIndex) !?*Node.DocComment { const comment_token = p.eatToken(.DocComment) orelse return null; if (p.tokensOnSameLine(after_token, comment_token)) { - var lines = Node.DocComment.LineList{}; - _ = try p.llpush(TokenIndex, &lines.first, comment_token); - const node = try p.arena.allocator.create(Node.DocComment); - node.* = .{ .lines = lines }; + node.* = .{ .first_line = comment_token }; return node; } p.putBackToken(comment_token); diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 236e645d4..bc38cf406 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -327,11 +327,18 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, .DocComment => { const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl); - var it = comment.lines.first; - while (it) |node| : (it = node.next) { - try renderToken(tree, stream, node.data, indent, start_col, .Newline); - if (node.next != null) { + const kind = tree.tokens[comment.first_line].id; + try renderToken(tree, stream, comment.first_line, indent, start_col, .Newline); + var tok_i = comment.first_line + 1; + while (true) : (tok_i += 1) { + const tok_id = tree.tokens[tok_i].id; + if (tok_id == kind) { try stream.writeByteNTimes(' ', indent); + try renderToken(tree, stream, tok_i, indent, start_col, .Newline); + } else if (tok_id == .LineComment) { + continue; + } else { + break; } } }, @@ -2428,17 +2435,32 @@ fn renderDocComments( start_col: *usize, ) (@TypeOf(stream).Error || Error)!void { const comment = node.doc_comments orelse return; - var it = comment.lines.first; - const first_token = node.firstToken(); - while (it) |line_token_index_node| : (it = line_token_index_node.next) { - const line_token_index = line_token_index_node.data; - if (line_token_index < first_token) { - try renderToken(tree, stream, line_token_index, indent, start_col, Space.Newline); - try stream.writeByteNTimes(' ', indent); - } else { - try renderToken(tree, stream, line_token_index, indent, start_col, Space.NoComment); - try stream.writeAll("\n"); - try stream.writeByteNTimes(' ', indent); + return renderDocCommentsToken(tree, stream, comment, node.firstToken(), indent, start_col); +} + +fn renderDocCommentsToken( + tree: *ast.Tree, + stream: var, + comment: *ast.Node.DocComment, + first_token: ast.TokenIndex, + indent: usize, + start_col: *usize, +) (@TypeOf(stream).Error || Error)!void { + var tok_i = comment.first_line; + while (true) : (tok_i += 1) { + switch (tree.tokens[tok_i].id) { + .DocComment, .ContainerDocComment => { + if (comment.first_line < first_token) { + try renderToken(tree, stream, tok_i, indent, start_col, Space.Newline); + try stream.writeByteNTimes(' ', indent); + } else { + try renderToken(tree, stream, tok_i, indent, start_col, Space.NoComment); + try stream.writeAll("\n"); + try stream.writeByteNTimes(' ', indent); + } + }, + .LineComment => continue, + else => break, } } }