stage2 parser: token ids in their own array
To prevent cache misses, token ids go in their own array, and the
start/end offsets go in a different one.
perf measurement before:
2,667,914 cache-misses:u
2,139,139,935 instructions:u
894,167,331 cycles:u
perf measurement after:
1,757,723 cache-misses:u
2,069,932,298 instructions:u
858,105,570 cycles:u
master
Andrew Kelley
parent
295bca9b5f
commit
8df0841d6a
|
|
@ -10,7 +10,8 @@ pub const NodeIndex = usize;
|
|||
pub const Tree = struct {
|
||||
/// Reference to externally-owned data.
|
||||
source: []const u8,
|
||||
tokens: []const Token,
|
||||
token_ids: []const Token.Id,
|
||||
token_locs: []const Token.Loc,
|
||||
errors: []const Error,
|
||||
/// undefined on parse error (when errors field is not empty)
|
||||
root_node: *Node.Root,
|
||||
|
|
@ -23,26 +24,27 @@ pub const Tree = struct {
|
|||
generated: bool = false,
|
||||
|
||||
pub fn deinit(self: *Tree) void {
|
||||
self.gpa.free(self.tokens);
|
||||
self.gpa.free(self.token_ids);
|
||||
self.gpa.free(self.token_locs);
|
||||
self.gpa.free(self.errors);
|
||||
self.arena.promote(self.gpa).deinit();
|
||||
}
|
||||
|
||||
pub fn renderError(self: *Tree, parse_error: *const Error, stream: var) !void {
|
||||
return parse_error.render(self.tokens, stream);
|
||||
return parse_error.render(self.token_ids, stream);
|
||||
}
|
||||
|
||||
pub fn tokenSlice(self: *Tree, token_index: TokenIndex) []const u8 {
|
||||
return self.tokenSlicePtr(self.tokens[token_index]);
|
||||
return self.tokenSliceLoc(self.token_locs[token_index]);
|
||||
}
|
||||
|
||||
pub fn tokenSlicePtr(self: *Tree, token: Token) []const u8 {
|
||||
pub fn tokenSliceLoc(self: *Tree, token: Token.Loc) []const u8 {
|
||||
return self.source[token.start..token.end];
|
||||
}
|
||||
|
||||
pub fn getNodeSource(self: *const Tree, node: *const Node) []const u8 {
|
||||
const first_token = self.tokens[node.firstToken()];
|
||||
const last_token = self.tokens[node.lastToken()];
|
||||
const first_token = self.token_locs[node.firstToken()];
|
||||
const last_token = self.token_locs[node.lastToken()];
|
||||
return self.source[first_token.start..last_token.end];
|
||||
}
|
||||
|
||||
|
|
@ -54,7 +56,7 @@ pub const Tree = struct {
|
|||
};
|
||||
|
||||
/// Return the Location of the token relative to the offset specified by `start_index`.
|
||||
pub fn tokenLocationPtr(self: *Tree, start_index: usize, token: Token) Location {
|
||||
pub fn tokenLocationLoc(self: *Tree, start_index: usize, token: Token.Loc) Location {
|
||||
var loc = Location{
|
||||
.line = 0,
|
||||
.column = 0,
|
||||
|
|
@ -82,14 +84,14 @@ pub const Tree = struct {
|
|||
}
|
||||
|
||||
pub fn tokenLocation(self: *Tree, start_index: usize, token_index: TokenIndex) Location {
|
||||
return self.tokenLocationPtr(start_index, self.tokens[token_index]);
|
||||
return self.tokenLocationLoc(start_index, self.token_locs[token_index]);
|
||||
}
|
||||
|
||||
pub fn tokensOnSameLine(self: *Tree, token1_index: TokenIndex, token2_index: TokenIndex) bool {
|
||||
return self.tokensOnSameLinePtr(self.tokens[token1_index], self.tokens[token2_index]);
|
||||
return self.tokensOnSameLineLoc(self.token_locs[token1_index], self.token_locs[token2_index]);
|
||||
}
|
||||
|
||||
pub fn tokensOnSameLinePtr(self: *Tree, token1: Token, token2: Token) bool {
|
||||
pub fn tokensOnSameLineLoc(self: *Tree, token1: Token.Loc, token2: Token.Loc) bool {
|
||||
return mem.indexOfScalar(u8, self.source[token1.end..token2.start], '\n') == null;
|
||||
}
|
||||
|
||||
|
|
@ -100,7 +102,7 @@ pub const Tree = struct {
|
|||
/// Skips over comments
|
||||
pub fn prevToken(self: *Tree, token_index: TokenIndex) TokenIndex {
|
||||
var index = token_index - 1;
|
||||
while (self.tokens[index].id == Token.Id.LineComment) {
|
||||
while (self.token_ids[index] == Token.Id.LineComment) {
|
||||
index -= 1;
|
||||
}
|
||||
return index;
|
||||
|
|
@ -109,7 +111,7 @@ pub const Tree = struct {
|
|||
/// Skips over comments
|
||||
pub fn nextToken(self: *Tree, token_index: TokenIndex) TokenIndex {
|
||||
var index = token_index + 1;
|
||||
while (self.tokens[index].id == Token.Id.LineComment) {
|
||||
while (self.token_ids[index] == Token.Id.LineComment) {
|
||||
index += 1;
|
||||
}
|
||||
return index;
|
||||
|
|
@ -166,7 +168,7 @@ pub const Error = union(enum) {
|
|||
DeclBetweenFields: DeclBetweenFields,
|
||||
InvalidAnd: InvalidAnd,
|
||||
|
||||
pub fn render(self: *const Error, tokens: []const Token, stream: var) !void {
|
||||
pub fn render(self: *const Error, tokens: []const Token.Id, stream: var) !void {
|
||||
switch (self.*) {
|
||||
.InvalidToken => |*x| return x.render(tokens, stream),
|
||||
.ExpectedContainerMembers => |*x| return x.render(tokens, stream),
|
||||
|
|
@ -321,7 +323,7 @@ pub const Error = union(enum) {
|
|||
pub const ExpectedCall = struct {
|
||||
node: *Node,
|
||||
|
||||
pub fn render(self: *const ExpectedCall, tokens: []const Token, stream: var) !void {
|
||||
pub fn render(self: *const ExpectedCall, tokens: []const Token.Id, stream: var) !void {
|
||||
return stream.print("expected " ++ @tagName(Node.Id.Call) ++ ", found {}", .{
|
||||
@tagName(self.node.id),
|
||||
});
|
||||
|
|
@ -331,7 +333,7 @@ pub const Error = union(enum) {
|
|||
pub const ExpectedCallOrFnProto = struct {
|
||||
node: *Node,
|
||||
|
||||
pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token, stream: var) !void {
|
||||
pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token.Id, stream: var) !void {
|
||||
return stream.print("expected " ++ @tagName(Node.Id.Call) ++ " or " ++
|
||||
@tagName(Node.Id.FnProto) ++ ", found {}", .{@tagName(self.node.id)});
|
||||
}
|
||||
|
|
@ -341,14 +343,14 @@ pub const Error = union(enum) {
|
|||
token: TokenIndex,
|
||||
expected_id: Token.Id,
|
||||
|
||||
pub fn render(self: *const ExpectedToken, tokens: []const Token, stream: var) !void {
|
||||
pub fn render(self: *const ExpectedToken, tokens: []const Token.Id, stream: var) !void {
|
||||
const found_token = tokens[self.token];
|
||||
switch (found_token.id) {
|
||||
switch (found_token) {
|
||||
.Invalid => {
|
||||
return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});
|
||||
},
|
||||
else => {
|
||||
const token_name = found_token.id.symbol();
|
||||
const token_name = found_token.symbol();
|
||||
return stream.print("expected '{}', found '{}'", .{ self.expected_id.symbol(), token_name });
|
||||
},
|
||||
}
|
||||
|
|
@ -359,11 +361,11 @@ pub const Error = union(enum) {
|
|||
token: TokenIndex,
|
||||
end_id: Token.Id,
|
||||
|
||||
pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token, stream: var) !void {
|
||||
pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token.Id, stream: var) !void {
|
||||
const actual_token = tokens[self.token];
|
||||
return stream.print("expected ',' or '{}', found '{}'", .{
|
||||
self.end_id.symbol(),
|
||||
actual_token.id.symbol(),
|
||||
actual_token.symbol(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
|
@ -374,9 +376,9 @@ pub const Error = union(enum) {
|
|||
|
||||
token: TokenIndex,
|
||||
|
||||
pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void {
|
||||
pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: var) !void {
|
||||
const actual_token = tokens[self.token];
|
||||
return stream.print(msg, .{actual_token.id.symbol()});
|
||||
return stream.print(msg, .{actual_token.symbol()});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -387,7 +389,7 @@ pub const Error = union(enum) {
|
|||
|
||||
token: TokenIndex,
|
||||
|
||||
pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void {
|
||||
pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: var) !void {
|
||||
return stream.writeAll(msg);
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -16,28 +16,32 @@ pub const Error = error{ParseError} || Allocator.Error;
|
|||
pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
|
||||
// TODO optimization idea: ensureCapacity on the tokens list and
|
||||
// then appendAssumeCapacity inside the loop.
|
||||
var tokens = std.ArrayList(Token).init(gpa);
|
||||
defer tokens.deinit();
|
||||
var token_ids = std.ArrayList(Token.Id).init(gpa);
|
||||
defer token_ids.deinit();
|
||||
var token_locs = std.ArrayList(Token.Loc).init(gpa);
|
||||
defer token_locs.deinit();
|
||||
|
||||
var tokenizer = std.zig.Tokenizer.init(source);
|
||||
while (true) {
|
||||
const tree_token = try tokens.addOne();
|
||||
tree_token.* = tokenizer.next();
|
||||
if (tree_token.id == .Eof) break;
|
||||
const token = tokenizer.next();
|
||||
try token_ids.append(token.id);
|
||||
try token_locs.append(token.loc);
|
||||
if (token.id == .Eof) break;
|
||||
}
|
||||
|
||||
var parser: Parser = .{
|
||||
.source = source,
|
||||
.arena = std.heap.ArenaAllocator.init(gpa),
|
||||
.gpa = gpa,
|
||||
.tokens = tokens.items,
|
||||
.token_ids = token_ids.items,
|
||||
.token_locs = token_locs.items,
|
||||
.errors = .{},
|
||||
.tok_i = 0,
|
||||
};
|
||||
defer parser.errors.deinit(gpa);
|
||||
errdefer parser.arena.deinit();
|
||||
|
||||
while (tokens.items[parser.tok_i].id == .LineComment) parser.tok_i += 1;
|
||||
while (token_ids.items[parser.tok_i] == .LineComment) parser.tok_i += 1;
|
||||
|
||||
const root_node = try parser.parseRoot();
|
||||
|
||||
|
|
@ -45,7 +49,8 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
|
|||
tree.* = .{
|
||||
.gpa = gpa,
|
||||
.source = source,
|
||||
.tokens = tokens.toOwnedSlice(),
|
||||
.token_ids = token_ids.toOwnedSlice(),
|
||||
.token_locs = token_locs.toOwnedSlice(),
|
||||
.errors = parser.errors.toOwnedSlice(gpa),
|
||||
.root_node = root_node,
|
||||
.arena = parser.arena.state,
|
||||
|
|
@ -58,9 +63,8 @@ const Parser = struct {
|
|||
arena: std.heap.ArenaAllocator,
|
||||
gpa: *Allocator,
|
||||
source: []const u8,
|
||||
/// TODO: Optimization idea: have this be several arrays of the token fields rather
|
||||
/// than an array of structs.
|
||||
tokens: []const Token,
|
||||
token_ids: []const Token.Id,
|
||||
token_locs: []const Token.Loc,
|
||||
tok_i: TokenIndex,
|
||||
errors: std.ArrayListUnmanaged(AstError),
|
||||
|
||||
|
|
@ -80,19 +84,6 @@ const Parser = struct {
|
|||
return node;
|
||||
}
|
||||
|
||||
/// Helper function for appending elements to a singly linked list.
|
||||
fn llpush(
|
||||
p: *Parser,
|
||||
comptime T: type,
|
||||
it: *?*std.SinglyLinkedList(T).Node,
|
||||
data: T,
|
||||
) !*?*std.SinglyLinkedList(T).Node {
|
||||
const llnode = try p.arena.allocator.create(std.SinglyLinkedList(T).Node);
|
||||
llnode.* = .{ .data = data };
|
||||
it.* = llnode;
|
||||
return &llnode.next;
|
||||
}
|
||||
|
||||
/// ContainerMembers
|
||||
/// <- TestDecl ContainerMembers
|
||||
/// / TopLevelComptime ContainerMembers
|
||||
|
|
@ -228,7 +219,7 @@ const Parser = struct {
|
|||
// try to continue parsing
|
||||
const index = p.tok_i;
|
||||
p.findNextContainerMember();
|
||||
const next = p.tokens[p.tok_i].id;
|
||||
const next = p.token_ids[p.tok_i];
|
||||
switch (next) {
|
||||
.Eof => break,
|
||||
else => {
|
||||
|
|
@ -257,7 +248,7 @@ const Parser = struct {
|
|||
});
|
||||
}
|
||||
|
||||
const next = p.tokens[p.tok_i].id;
|
||||
const next = p.token_ids[p.tok_i];
|
||||
switch (next) {
|
||||
.Eof => break,
|
||||
.Keyword_comptime => {
|
||||
|
|
@ -291,7 +282,7 @@ const Parser = struct {
|
|||
var level: u32 = 0;
|
||||
while (true) {
|
||||
const tok = p.nextToken();
|
||||
switch (tok.ptr.id) {
|
||||
switch (p.token_ids[tok]) {
|
||||
// any of these can start a new top level declaration
|
||||
.Keyword_test,
|
||||
.Keyword_comptime,
|
||||
|
|
@ -308,7 +299,7 @@ const Parser = struct {
|
|||
.Identifier,
|
||||
=> {
|
||||
if (level == 0) {
|
||||
p.putBackToken(tok.index);
|
||||
p.putBackToken(tok);
|
||||
return;
|
||||
}
|
||||
},
|
||||
|
|
@ -325,13 +316,13 @@ const Parser = struct {
|
|||
.RBrace => {
|
||||
if (level == 0) {
|
||||
// end of container, exit
|
||||
p.putBackToken(tok.index);
|
||||
p.putBackToken(tok);
|
||||
return;
|
||||
}
|
||||
level -= 1;
|
||||
},
|
||||
.Eof => {
|
||||
p.putBackToken(tok.index);
|
||||
p.putBackToken(tok);
|
||||
return;
|
||||
},
|
||||
else => {},
|
||||
|
|
@ -344,11 +335,11 @@ const Parser = struct {
|
|||
var level: u32 = 0;
|
||||
while (true) {
|
||||
const tok = p.nextToken();
|
||||
switch (tok.ptr.id) {
|
||||
switch (p.token_ids[tok]) {
|
||||
.LBrace => level += 1,
|
||||
.RBrace => {
|
||||
if (level == 0) {
|
||||
p.putBackToken(tok.index);
|
||||
p.putBackToken(tok);
|
||||
return;
|
||||
}
|
||||
level -= 1;
|
||||
|
|
@ -359,7 +350,7 @@ const Parser = struct {
|
|||
}
|
||||
},
|
||||
.Eof => {
|
||||
p.putBackToken(tok.index);
|
||||
p.putBackToken(tok);
|
||||
return;
|
||||
},
|
||||
else => {},
|
||||
|
|
@ -454,8 +445,8 @@ const Parser = struct {
|
|||
}
|
||||
|
||||
if (extern_export_inline_token) |token| {
|
||||
if (p.tokens[token].id == .Keyword_inline or
|
||||
p.tokens[token].id == .Keyword_noinline)
|
||||
if (p.token_ids[token] == .Keyword_inline or
|
||||
p.token_ids[token] == .Keyword_noinline)
|
||||
{
|
||||
try p.errors.append(p.gpa, .{
|
||||
.ExpectedFn = .{ .token = p.tok_i },
|
||||
|
|
@ -722,7 +713,7 @@ const Parser = struct {
|
|||
|
||||
const defer_token = p.eatToken(.Keyword_defer) orelse p.eatToken(.Keyword_errdefer);
|
||||
if (defer_token) |token| {
|
||||
const payload = if (p.tokens[token].id == .Keyword_errdefer)
|
||||
const payload = if (p.token_ids[token] == .Keyword_errdefer)
|
||||
try p.parsePayload()
|
||||
else
|
||||
null;
|
||||
|
|
@ -2269,7 +2260,7 @@ const Parser = struct {
|
|||
/// / EQUAL
|
||||
fn parseAssignOp(p: *Parser) !?*Node {
|
||||
const token = p.nextToken();
|
||||
const op: Node.InfixOp.Op = switch (token.ptr.id) {
|
||||
const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
|
||||
.AsteriskEqual => .AssignMul,
|
||||
.SlashEqual => .AssignDiv,
|
||||
.PercentEqual => .AssignMod,
|
||||
|
|
@ -2285,14 +2276,14 @@ const Parser = struct {
|
|||
.MinusPercentEqual => .AssignSubWrap,
|
||||
.Equal => .Assign,
|
||||
else => {
|
||||
p.putBackToken(token.index);
|
||||
p.putBackToken(token);
|
||||
return null;
|
||||
},
|
||||
};
|
||||
|
||||
const node = try p.arena.allocator.create(Node.InfixOp);
|
||||
node.* = .{
|
||||
.op_token = token.index,
|
||||
.op_token = token,
|
||||
.lhs = undefined, // set by caller
|
||||
.op = op,
|
||||
.rhs = undefined, // set by caller
|
||||
|
|
@ -2309,7 +2300,7 @@ const Parser = struct {
|
|||
/// / RARROWEQUAL
|
||||
fn parseCompareOp(p: *Parser) !?*Node {
|
||||
const token = p.nextToken();
|
||||
const op: Node.InfixOp.Op = switch (token.ptr.id) {
|
||||
const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
|
||||
.EqualEqual => .EqualEqual,
|
||||
.BangEqual => .BangEqual,
|
||||
.AngleBracketLeft => .LessThan,
|
||||
|
|
@ -2317,12 +2308,12 @@ const Parser = struct {
|
|||
.AngleBracketLeftEqual => .LessOrEqual,
|
||||
.AngleBracketRightEqual => .GreaterOrEqual,
|
||||
else => {
|
||||
p.putBackToken(token.index);
|
||||
p.putBackToken(token);
|
||||
return null;
|
||||
},
|
||||
};
|
||||
|
||||
return p.createInfixOp(token.index, op);
|
||||
return p.createInfixOp(token, op);
|
||||
}
|
||||
|
||||
/// BitwiseOp
|
||||
|
|
@ -2333,19 +2324,19 @@ const Parser = struct {
|
|||
/// / KEYWORD_catch Payload?
|
||||
fn parseBitwiseOp(p: *Parser) !?*Node {
|
||||
const token = p.nextToken();
|
||||
const op: Node.InfixOp.Op = switch (token.ptr.id) {
|
||||
const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
|
||||
.Ampersand => .BitAnd,
|
||||
.Caret => .BitXor,
|
||||
.Pipe => .BitOr,
|
||||
.Keyword_orelse => .UnwrapOptional,
|
||||
.Keyword_catch => .{ .Catch = try p.parsePayload() },
|
||||
else => {
|
||||
p.putBackToken(token.index);
|
||||
p.putBackToken(token);
|
||||
return null;
|
||||
},
|
||||
};
|
||||
|
||||
return p.createInfixOp(token.index, op);
|
||||
return p.createInfixOp(token, op);
|
||||
}
|
||||
|
||||
/// BitShiftOp
|
||||
|
|
@ -2353,16 +2344,16 @@ const Parser = struct {
|
|||
/// / RARROW2
|
||||
fn parseBitShiftOp(p: *Parser) !?*Node {
|
||||
const token = p.nextToken();
|
||||
const op: Node.InfixOp.Op = switch (token.ptr.id) {
|
||||
const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
|
||||
.AngleBracketAngleBracketLeft => .BitShiftLeft,
|
||||
.AngleBracketAngleBracketRight => .BitShiftRight,
|
||||
else => {
|
||||
p.putBackToken(token.index);
|
||||
p.putBackToken(token);
|
||||
return null;
|
||||
},
|
||||
};
|
||||
|
||||
return p.createInfixOp(token.index, op);
|
||||
return p.createInfixOp(token, op);
|
||||
}
|
||||
|
||||
/// AdditionOp
|
||||
|
|
@ -2373,19 +2364,19 @@ const Parser = struct {
|
|||
/// / MINUSPERCENT
|
||||
fn parseAdditionOp(p: *Parser) !?*Node {
|
||||
const token = p.nextToken();
|
||||
const op: Node.InfixOp.Op = switch (token.ptr.id) {
|
||||
const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
|
||||
.Plus => .Add,
|
||||
.Minus => .Sub,
|
||||
.PlusPlus => .ArrayCat,
|
||||
.PlusPercent => .AddWrap,
|
||||
.MinusPercent => .SubWrap,
|
||||
else => {
|
||||
p.putBackToken(token.index);
|
||||
p.putBackToken(token);
|
||||
return null;
|
||||
},
|
||||
};
|
||||
|
||||
return p.createInfixOp(token.index, op);
|
||||
return p.createInfixOp(token, op);
|
||||
}
|
||||
|
||||
/// MultiplyOp
|
||||
|
|
@ -2397,7 +2388,7 @@ const Parser = struct {
|
|||
/// / ASTERISKPERCENT
|
||||
fn parseMultiplyOp(p: *Parser) !?*Node {
|
||||
const token = p.nextToken();
|
||||
const op: Node.InfixOp.Op = switch (token.ptr.id) {
|
||||
const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
|
||||
.PipePipe => .MergeErrorSets,
|
||||
.Asterisk => .Mul,
|
||||
.Slash => .Div,
|
||||
|
|
@ -2405,12 +2396,12 @@ const Parser = struct {
|
|||
.AsteriskAsterisk => .ArrayMult,
|
||||
.AsteriskPercent => .MulWrap,
|
||||
else => {
|
||||
p.putBackToken(token.index);
|
||||
p.putBackToken(token);
|
||||
return null;
|
||||
},
|
||||
};
|
||||
|
||||
return p.createInfixOp(token.index, op);
|
||||
return p.createInfixOp(token, op);
|
||||
}
|
||||
|
||||
/// PrefixOp
|
||||
|
|
@ -2423,7 +2414,7 @@ const Parser = struct {
|
|||
/// / KEYWORD_await
|
||||
fn parsePrefixOp(p: *Parser) !?*Node {
|
||||
const token = p.nextToken();
|
||||
const op: Node.PrefixOp.Op = switch (token.ptr.id) {
|
||||
const op: Node.PrefixOp.Op = switch (p.token_ids[token]) {
|
||||
.Bang => .BoolNot,
|
||||
.Minus => .Negation,
|
||||
.Tilde => .BitNot,
|
||||
|
|
@ -2432,14 +2423,14 @@ const Parser = struct {
|
|||
.Keyword_try => .Try,
|
||||
.Keyword_await => .Await,
|
||||
else => {
|
||||
p.putBackToken(token.index);
|
||||
p.putBackToken(token);
|
||||
return null;
|
||||
},
|
||||
};
|
||||
|
||||
const node = try p.arena.allocator.create(Node.PrefixOp);
|
||||
node.* = .{
|
||||
.op_token = token.index,
|
||||
.op_token = token,
|
||||
.op = op,
|
||||
.rhs = undefined, // set by caller
|
||||
};
|
||||
|
|
@ -2493,7 +2484,7 @@ const Parser = struct {
|
|||
// If the token encountered was **, there will be two nodes instead of one.
|
||||
// The attributes should be applied to the rightmost operator.
|
||||
const prefix_op = node.cast(Node.PrefixOp).?;
|
||||
var ptr_info = if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk)
|
||||
var ptr_info = if (p.token_ids[prefix_op.op_token] == .AsteriskAsterisk)
|
||||
&prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType
|
||||
else
|
||||
&prefix_op.op.PtrType;
|
||||
|
|
@ -2812,7 +2803,8 @@ const Parser = struct {
|
|||
return null;
|
||||
};
|
||||
if (p.eatToken(.Identifier)) |ident| {
|
||||
const token_slice = p.source[p.tokens[ident].start..p.tokens[ident].end];
|
||||
const token_loc = p.token_locs[ident];
|
||||
const token_slice = p.source[token_loc.start..token_loc.end];
|
||||
if (!std.mem.eql(u8, token_slice, "c")) {
|
||||
p.putBackToken(ident);
|
||||
} else {
|
||||
|
|
@ -2879,7 +2871,7 @@ const Parser = struct {
|
|||
fn parseContainerDeclType(p: *Parser) !?ContainerDeclType {
|
||||
const kind_token = p.nextToken();
|
||||
|
||||
const init_arg_expr = switch (kind_token.ptr.id) {
|
||||
const init_arg_expr = switch (p.token_ids[kind_token]) {
|
||||
.Keyword_struct => Node.ContainerDecl.InitArg{ .None = {} },
|
||||
.Keyword_enum => blk: {
|
||||
if (p.eatToken(.LParen) != null) {
|
||||
|
|
@ -2914,13 +2906,13 @@ const Parser = struct {
|
|||
break :blk Node.ContainerDecl.InitArg{ .None = {} };
|
||||
},
|
||||
else => {
|
||||
p.putBackToken(kind_token.index);
|
||||
p.putBackToken(kind_token);
|
||||
return null;
|
||||
},
|
||||
};
|
||||
|
||||
return ContainerDeclType{
|
||||
.kind_token = kind_token.index,
|
||||
.kind_token = kind_token,
|
||||
.init_arg_expr = init_arg_expr,
|
||||
};
|
||||
}
|
||||
|
|
@ -2973,7 +2965,7 @@ const Parser = struct {
|
|||
while (try nodeParseFn(p)) |item| {
|
||||
try list.append(item);
|
||||
|
||||
switch (p.tokens[p.tok_i].id) {
|
||||
switch (p.token_ids[p.tok_i]) {
|
||||
.Comma => _ = p.nextToken(),
|
||||
// all possible delimiters
|
||||
.Colon, .RParen, .RBrace, .RBracket => break,
|
||||
|
|
@ -2994,13 +2986,13 @@ const Parser = struct {
|
|||
fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn {
|
||||
return struct {
|
||||
pub fn parse(p: *Parser) Error!?*Node {
|
||||
const op_token = if (token == .Keyword_and) switch (p.tokens[p.tok_i].id) {
|
||||
.Keyword_and => p.nextToken().index,
|
||||
const op_token = if (token == .Keyword_and) switch (p.token_ids[p.tok_i]) {
|
||||
.Keyword_and => p.nextToken(),
|
||||
.Invalid_ampersands => blk: {
|
||||
try p.errors.append(p.gpa, .{
|
||||
.InvalidAnd = .{ .token = p.tok_i },
|
||||
});
|
||||
break :blk p.nextToken().index;
|
||||
break :blk p.nextToken();
|
||||
},
|
||||
else => return null,
|
||||
} else p.eatToken(token) orelse return null;
|
||||
|
|
@ -3104,7 +3096,7 @@ const Parser = struct {
|
|||
var tok_i = start_tok_i;
|
||||
var count: usize = 1; // including first_line
|
||||
while (true) : (tok_i += 1) {
|
||||
switch (p.tokens[tok_i].id) {
|
||||
switch (p.token_ids[tok_i]) {
|
||||
.LineComment => continue,
|
||||
.MultilineStringLiteralLine => count += 1,
|
||||
else => break,
|
||||
|
|
@ -3118,7 +3110,7 @@ const Parser = struct {
|
|||
lines[0] = first_line;
|
||||
count = 1;
|
||||
while (true) : (tok_i += 1) {
|
||||
switch (p.tokens[tok_i].id) {
|
||||
switch (p.token_ids[tok_i]) {
|
||||
.LineComment => continue,
|
||||
.MultilineStringLiteralLine => {
|
||||
lines[count] = tok_i;
|
||||
|
|
@ -3215,7 +3207,7 @@ const Parser = struct {
|
|||
}
|
||||
|
||||
fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool {
|
||||
return std.mem.indexOfScalar(u8, p.source[p.tokens[token1].end..p.tokens[token2].start], '\n') == null;
|
||||
return std.mem.indexOfScalar(u8, p.source[p.token_locs[token1].end..p.token_locs[token2].start], '\n') == null;
|
||||
}
|
||||
|
||||
/// Eat a single-line doc comment on the same line as another node
|
||||
|
|
@ -3239,7 +3231,7 @@ const Parser = struct {
|
|||
.PrefixOp => {
|
||||
var prefix_op = rightmost_op.cast(Node.PrefixOp).?;
|
||||
// If the token encountered was **, there will be two nodes
|
||||
if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk) {
|
||||
if (p.token_ids[prefix_op.op_token] == .AsteriskAsterisk) {
|
||||
rightmost_op = prefix_op.rhs;
|
||||
prefix_op = rightmost_op.cast(Node.PrefixOp).?;
|
||||
}
|
||||
|
|
@ -3328,11 +3320,7 @@ const Parser = struct {
|
|||
}
|
||||
|
||||
fn eatToken(p: *Parser, id: Token.Id) ?TokenIndex {
|
||||
return if (p.eatAnnotatedToken(id)) |token| token.index else null;
|
||||
}
|
||||
|
||||
fn eatAnnotatedToken(p: *Parser, id: Token.Id) ?AnnotatedToken {
|
||||
return if (p.tokens[p.tok_i].id == id) p.nextToken() else null;
|
||||
return if (p.token_ids[p.tok_i] == id) p.nextToken() else null;
|
||||
}
|
||||
|
||||
fn expectToken(p: *Parser, id: Token.Id) Error!TokenIndex {
|
||||
|
|
@ -3341,29 +3329,25 @@ const Parser = struct {
|
|||
|
||||
fn expectTokenRecoverable(p: *Parser, id: Token.Id) !?TokenIndex {
|
||||
const token = p.nextToken();
|
||||
if (token.ptr.id != id) {
|
||||
if (p.token_ids[token] != id) {
|
||||
try p.errors.append(p.gpa, .{
|
||||
.ExpectedToken = .{ .token = token.index, .expected_id = id },
|
||||
.ExpectedToken = .{ .token = token, .expected_id = id },
|
||||
});
|
||||
// go back so that we can recover properly
|
||||
p.putBackToken(token.index);
|
||||
p.putBackToken(token);
|
||||
return null;
|
||||
}
|
||||
return token.index;
|
||||
return token;
|
||||
}
|
||||
|
||||
fn nextToken(p: *Parser) AnnotatedToken {
|
||||
const result = AnnotatedToken{
|
||||
.index = p.tok_i,
|
||||
.ptr = &p.tokens[p.tok_i],
|
||||
};
|
||||
fn nextToken(p: *Parser) TokenIndex {
|
||||
const result = p.tok_i;
|
||||
p.tok_i += 1;
|
||||
assert(result.ptr.id != .LineComment);
|
||||
if (p.tok_i >= p.tokens.len) return result;
|
||||
assert(p.token_ids[result] != .LineComment);
|
||||
if (p.tok_i >= p.token_ids.len) return result;
|
||||
|
||||
while (true) {
|
||||
const next_tok = p.tokens[p.tok_i];
|
||||
if (next_tok.id != .LineComment) return result;
|
||||
if (p.token_ids[p.tok_i] != .LineComment) return result;
|
||||
p.tok_i += 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -3371,18 +3355,12 @@ const Parser = struct {
|
|||
fn putBackToken(p: *Parser, putting_back: TokenIndex) void {
|
||||
while (p.tok_i > 0) {
|
||||
p.tok_i -= 1;
|
||||
const prev_tok = p.tokens[p.tok_i];
|
||||
if (prev_tok.id == .LineComment) continue;
|
||||
if (p.token_ids[p.tok_i] == .LineComment) continue;
|
||||
assert(putting_back == p.tok_i);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const AnnotatedToken = struct {
|
||||
index: TokenIndex,
|
||||
ptr: *const Token,
|
||||
};
|
||||
|
||||
fn expectNode(
|
||||
p: *Parser,
|
||||
parseFn: NodeParseFn,
|
||||
|
|
|
|||
|
|
@ -3181,7 +3181,7 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b
|
|||
defer tree.deinit();
|
||||
|
||||
for (tree.errors) |*parse_error| {
|
||||
const token = tree.tokens[parse_error.loc()];
|
||||
const token = tree.token_locs[parse_error.loc()];
|
||||
const loc = tree.tokenLocation(0, parse_error.loc());
|
||||
try stderr.print("(memory buffer):{}:{}: error: ", .{ loc.line + 1, loc.column + 1 });
|
||||
try tree.renderError(parse_error, stderr);
|
||||
|
|
|
|||
|
|
@ -68,11 +68,12 @@ fn renderRoot(
|
|||
tree: *ast.Tree,
|
||||
) (@TypeOf(stream).Error || Error)!void {
|
||||
// render all the line comments at the beginning of the file
|
||||
for (tree.tokens) |token, i| {
|
||||
if (token.id != .LineComment) break;
|
||||
try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSlicePtr(token), " ")});
|
||||
const next_token = &tree.tokens[i + 1];
|
||||
const loc = tree.tokenLocationPtr(token.end, next_token.*);
|
||||
for (tree.token_ids) |token_id, i| {
|
||||
if (token_id != .LineComment) break;
|
||||
const token_loc = tree.token_locs[i];
|
||||
try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSliceLoc(token_loc), " ")});
|
||||
const next_token = tree.token_locs[i + 1];
|
||||
const loc = tree.tokenLocationLoc(token_loc.end, next_token);
|
||||
if (loc.line >= 2) {
|
||||
try stream.writeByte('\n');
|
||||
}
|
||||
|
|
@ -101,8 +102,8 @@ fn renderRoot(
|
|||
|
||||
while (token_index != 0) {
|
||||
token_index -= 1;
|
||||
const token = tree.tokens[token_index];
|
||||
switch (token.id) {
|
||||
const token_id = tree.token_ids[token_index];
|
||||
switch (token_id) {
|
||||
.LineComment => {},
|
||||
.DocComment => {
|
||||
copy_start_token_index = token_index;
|
||||
|
|
@ -111,12 +112,13 @@ fn renderRoot(
|
|||
else => break,
|
||||
}
|
||||
|
||||
if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: off")) {
|
||||
const token_loc = tree.token_locs[token_index];
|
||||
if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: off")) {
|
||||
if (!found_fmt_directive) {
|
||||
fmt_active = false;
|
||||
found_fmt_directive = true;
|
||||
}
|
||||
} else if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: on")) {
|
||||
} else if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: on")) {
|
||||
if (!found_fmt_directive) {
|
||||
fmt_active = true;
|
||||
found_fmt_directive = true;
|
||||
|
|
@ -135,7 +137,7 @@ fn renderRoot(
|
|||
if (decl_i >= root_decls.len) {
|
||||
// If there's no next reformatted `decl`, just copy the
|
||||
// remaining input tokens and bail out.
|
||||
const start = tree.tokens[copy_start_token_index].start;
|
||||
const start = tree.token_locs[copy_start_token_index].start;
|
||||
try copyFixingWhitespace(stream, tree.source[start..]);
|
||||
return;
|
||||
}
|
||||
|
|
@ -143,15 +145,16 @@ fn renderRoot(
|
|||
var decl_first_token_index = decl.firstToken();
|
||||
|
||||
while (token_index < decl_first_token_index) : (token_index += 1) {
|
||||
const token = tree.tokens[token_index];
|
||||
switch (token.id) {
|
||||
const token_id = tree.token_ids[token_index];
|
||||
switch (token_id) {
|
||||
.LineComment => {},
|
||||
.Eof => unreachable,
|
||||
else => continue,
|
||||
}
|
||||
if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: on")) {
|
||||
const token_loc = tree.token_locs[token_index];
|
||||
if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: on")) {
|
||||
fmt_active = true;
|
||||
} else if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: off")) {
|
||||
} else if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: off")) {
|
||||
fmt_active = false;
|
||||
}
|
||||
}
|
||||
|
|
@ -163,8 +166,8 @@ fn renderRoot(
|
|||
token_index = copy_end_token_index;
|
||||
while (token_index != 0) {
|
||||
token_index -= 1;
|
||||
const token = tree.tokens[token_index];
|
||||
switch (token.id) {
|
||||
const token_id = tree.token_ids[token_index];
|
||||
switch (token_id) {
|
||||
.LineComment => {},
|
||||
.DocComment => {
|
||||
copy_end_token_index = token_index;
|
||||
|
|
@ -174,8 +177,8 @@ fn renderRoot(
|
|||
}
|
||||
}
|
||||
|
||||
const start = tree.tokens[copy_start_token_index].start;
|
||||
const end = tree.tokens[copy_end_token_index].start;
|
||||
const start = tree.token_locs[copy_start_token_index].start;
|
||||
const end = tree.token_locs[copy_end_token_index].start;
|
||||
try copyFixingWhitespace(stream, tree.source[start..end]);
|
||||
}
|
||||
|
||||
|
|
@ -194,13 +197,13 @@ fn renderExtraNewlineToken(tree: *ast.Tree, stream: var, start_col: *usize, firs
|
|||
var prev_token = first_token;
|
||||
if (prev_token == 0) return;
|
||||
var newline_threshold: usize = 2;
|
||||
while (tree.tokens[prev_token - 1].id == .DocComment) {
|
||||
if (tree.tokenLocation(tree.tokens[prev_token - 1].end, prev_token).line == 1) {
|
||||
while (tree.token_ids[prev_token - 1] == .DocComment) {
|
||||
if (tree.tokenLocation(tree.token_locs[prev_token - 1].end, prev_token).line == 1) {
|
||||
newline_threshold += 1;
|
||||
}
|
||||
prev_token -= 1;
|
||||
}
|
||||
const prev_token_end = tree.tokens[prev_token - 1].end;
|
||||
const prev_token_end = tree.token_locs[prev_token - 1].end;
|
||||
const loc = tree.tokenLocation(prev_token_end, first_token);
|
||||
if (loc.line >= newline_threshold) {
|
||||
try stream.writeByte('\n');
|
||||
|
|
@ -265,7 +268,7 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree,
|
|||
|
||||
const src_has_trailing_comma = blk: {
|
||||
const maybe_comma = tree.nextToken(field.lastToken());
|
||||
break :blk tree.tokens[maybe_comma].id == .Comma;
|
||||
break :blk tree.token_ids[maybe_comma] == .Comma;
|
||||
};
|
||||
|
||||
// The trailing comma is emitted at the end, but if it's not present
|
||||
|
|
@ -327,11 +330,11 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree,
|
|||
|
||||
.DocComment => {
|
||||
const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl);
|
||||
const kind = tree.tokens[comment.first_line].id;
|
||||
const kind = tree.token_ids[comment.first_line];
|
||||
try renderToken(tree, stream, comment.first_line, indent, start_col, .Newline);
|
||||
var tok_i = comment.first_line + 1;
|
||||
while (true) : (tok_i += 1) {
|
||||
const tok_id = tree.tokens[tok_i].id;
|
||||
const tok_id = tree.token_ids[tok_i];
|
||||
if (tok_id == kind) {
|
||||
try stream.writeByteNTimes(' ', indent);
|
||||
try renderToken(tree, stream, tok_i, indent, start_col, .Newline);
|
||||
|
|
@ -436,13 +439,13 @@ fn renderExpression(
|
|||
try renderExpression(allocator, stream, tree, indent, start_col, infix_op_node.lhs, op_space);
|
||||
|
||||
const after_op_space = blk: {
|
||||
const loc = tree.tokenLocation(tree.tokens[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token));
|
||||
const loc = tree.tokenLocation(tree.token_locs[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token));
|
||||
break :blk if (loc.line == 0) op_space else Space.Newline;
|
||||
};
|
||||
|
||||
try renderToken(tree, stream, infix_op_node.op_token, indent, start_col, after_op_space);
|
||||
if (after_op_space == Space.Newline and
|
||||
tree.tokens[tree.nextToken(infix_op_node.op_token)].id != .MultilineStringLiteralLine)
|
||||
tree.token_ids[tree.nextToken(infix_op_node.op_token)] != .MultilineStringLiteralLine)
|
||||
{
|
||||
try stream.writeByteNTimes(' ', indent + indent_delta);
|
||||
start_col.* = indent + indent_delta;
|
||||
|
|
@ -463,10 +466,10 @@ fn renderExpression(
|
|||
|
||||
switch (prefix_op_node.op) {
|
||||
.PtrType => |ptr_info| {
|
||||
const op_tok_id = tree.tokens[prefix_op_node.op_token].id;
|
||||
const op_tok_id = tree.token_ids[prefix_op_node.op_token];
|
||||
switch (op_tok_id) {
|
||||
.Asterisk, .AsteriskAsterisk => try stream.writeByte('*'),
|
||||
.LBracket => if (tree.tokens[prefix_op_node.op_token + 2].id == .Identifier)
|
||||
.LBracket => if (tree.token_ids[prefix_op_node.op_token + 2] == .Identifier)
|
||||
try stream.writeAll("[*c")
|
||||
else
|
||||
try stream.writeAll("[*"),
|
||||
|
|
@ -578,8 +581,8 @@ fn renderExpression(
|
|||
|
||||
try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [
|
||||
|
||||
const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment;
|
||||
const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment;
|
||||
const starts_with_comment = tree.token_ids[lbracket + 1] == .LineComment;
|
||||
const ends_with_comment = tree.token_ids[rbracket - 1] == .LineComment;
|
||||
const new_indent = if (ends_with_comment) indent + indent_delta else indent;
|
||||
const new_space = if (ends_with_comment) Space.Newline else Space.None;
|
||||
try renderExpression(allocator, stream, tree, new_indent, start_col, array_info.len_expr, new_space);
|
||||
|
|
@ -653,7 +656,7 @@ fn renderExpression(
|
|||
return renderToken(tree, stream, rtoken, indent, start_col, space);
|
||||
}
|
||||
|
||||
if (exprs.len == 1 and tree.tokens[exprs[0].lastToken() + 1].id == .RBrace) {
|
||||
if (exprs.len == 1 and tree.token_ids[exprs[0].lastToken() + 1] == .RBrace) {
|
||||
const expr = exprs[0];
|
||||
switch (lhs) {
|
||||
.dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None),
|
||||
|
|
@ -675,17 +678,17 @@ fn renderExpression(
|
|||
for (exprs) |expr, i| {
|
||||
if (i + 1 < exprs.len) {
|
||||
const expr_last_token = expr.lastToken() + 1;
|
||||
const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, exprs[i+1].firstToken());
|
||||
const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, exprs[i+1].firstToken());
|
||||
if (loc.line != 0) break :blk count;
|
||||
count += 1;
|
||||
} else {
|
||||
const expr_last_token = expr.lastToken();
|
||||
const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, rtoken);
|
||||
const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, rtoken);
|
||||
if (loc.line == 0) {
|
||||
// all on one line
|
||||
const src_has_trailing_comma = trailblk: {
|
||||
const maybe_comma = tree.prevToken(rtoken);
|
||||
break :trailblk tree.tokens[maybe_comma].id == .Comma;
|
||||
break :trailblk tree.token_ids[maybe_comma] == .Comma;
|
||||
};
|
||||
if (src_has_trailing_comma) {
|
||||
break :blk 1; // force row size 1
|
||||
|
|
@ -723,7 +726,7 @@ fn renderExpression(
|
|||
|
||||
var new_indent = indent + indent_delta;
|
||||
|
||||
if (tree.tokens[tree.nextToken(lbrace)].id != .MultilineStringLiteralLine) {
|
||||
if (tree.token_ids[tree.nextToken(lbrace)] != .MultilineStringLiteralLine) {
|
||||
try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline);
|
||||
try stream.writeByteNTimes(' ', new_indent);
|
||||
} else {
|
||||
|
|
@ -750,7 +753,7 @@ fn renderExpression(
|
|||
}
|
||||
col = 1;
|
||||
|
||||
if (tree.tokens[tree.nextToken(comma)].id != .MultilineStringLiteralLine) {
|
||||
if (tree.token_ids[tree.nextToken(comma)] != .MultilineStringLiteralLine) {
|
||||
try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // ,
|
||||
} else {
|
||||
try renderToken(tree, stream, comma, new_indent, start_col, Space.None); // ,
|
||||
|
|
@ -819,11 +822,11 @@ fn renderExpression(
|
|||
|
||||
const src_has_trailing_comma = blk: {
|
||||
const maybe_comma = tree.prevToken(rtoken);
|
||||
break :blk tree.tokens[maybe_comma].id == .Comma;
|
||||
break :blk tree.token_ids[maybe_comma] == .Comma;
|
||||
};
|
||||
|
||||
const src_same_line = blk: {
|
||||
const loc = tree.tokenLocation(tree.tokens[lbrace].end, rtoken);
|
||||
const loc = tree.tokenLocation(tree.token_locs[lbrace].end, rtoken);
|
||||
break :blk loc.line == 0;
|
||||
};
|
||||
|
||||
|
|
@ -929,7 +932,7 @@ fn renderExpression(
|
|||
|
||||
const src_has_trailing_comma = blk: {
|
||||
const maybe_comma = tree.prevToken(call.rtoken);
|
||||
break :blk tree.tokens[maybe_comma].id == .Comma;
|
||||
break :blk tree.token_ids[maybe_comma] == .Comma;
|
||||
};
|
||||
|
||||
if (src_has_trailing_comma) {
|
||||
|
|
@ -983,8 +986,8 @@ fn renderExpression(
|
|||
try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None);
|
||||
try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [
|
||||
|
||||
const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment;
|
||||
const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment;
|
||||
const starts_with_comment = tree.token_ids[lbracket + 1] == .LineComment;
|
||||
const ends_with_comment = tree.token_ids[rbracket - 1] == .LineComment;
|
||||
const new_indent = if (ends_with_comment) indent + indent_delta else indent;
|
||||
const new_space = if (ends_with_comment) Space.Newline else Space.None;
|
||||
try renderExpression(allocator, stream, tree, new_indent, start_col, index_expr, new_space);
|
||||
|
|
@ -1226,9 +1229,9 @@ fn renderExpression(
|
|||
var maybe_comma = tree.prevToken(container_decl.lastToken());
|
||||
// Doc comments for a field may also appear after the comma, eg.
|
||||
// field_name: T, // comment attached to field_name
|
||||
if (tree.tokens[maybe_comma].id == .DocComment)
|
||||
if (tree.token_ids[maybe_comma] == .DocComment)
|
||||
maybe_comma = tree.prevToken(maybe_comma);
|
||||
break :blk tree.tokens[maybe_comma].id == .Comma;
|
||||
break :blk tree.token_ids[maybe_comma] == .Comma;
|
||||
};
|
||||
|
||||
const fields_and_decls = container_decl.fieldsAndDecls();
|
||||
|
|
@ -1321,7 +1324,7 @@ fn renderExpression(
|
|||
|
||||
const src_has_trailing_comma = blk: {
|
||||
const maybe_comma = tree.prevToken(err_set_decl.rbrace_token);
|
||||
break :blk tree.tokens[maybe_comma].id == .Comma;
|
||||
break :blk tree.token_ids[maybe_comma] == .Comma;
|
||||
};
|
||||
|
||||
if (src_has_trailing_comma) {
|
||||
|
|
@ -1353,7 +1356,7 @@ fn renderExpression(
|
|||
try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None);
|
||||
|
||||
const comma_token = tree.nextToken(node.lastToken());
|
||||
assert(tree.tokens[comma_token].id == .Comma);
|
||||
assert(tree.token_ids[comma_token] == .Comma);
|
||||
try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // ,
|
||||
try renderExtraNewline(tree, stream, start_col, decls[i + 1]);
|
||||
} else {
|
||||
|
|
@ -1378,7 +1381,7 @@ fn renderExpression(
|
|||
const multiline_str_literal = @fieldParentPtr(ast.Node.MultilineStringLiteral, "base", base);
|
||||
|
||||
var skip_first_indent = true;
|
||||
if (tree.tokens[multiline_str_literal.firstToken() - 1].id != .LineComment) {
|
||||
if (tree.token_ids[multiline_str_literal.firstToken() - 1] != .LineComment) {
|
||||
try stream.print("\n", .{});
|
||||
skip_first_indent = false;
|
||||
}
|
||||
|
|
@ -1406,7 +1409,7 @@ fn renderExpression(
|
|||
if (builtin_call.params_len < 2) break :blk false;
|
||||
const last_node = builtin_call.params()[builtin_call.params_len - 1];
|
||||
const maybe_comma = tree.nextToken(last_node.lastToken());
|
||||
break :blk tree.tokens[maybe_comma].id == .Comma;
|
||||
break :blk tree.token_ids[maybe_comma] == .Comma;
|
||||
};
|
||||
|
||||
const lparen = tree.nextToken(builtin_call.builtin_token);
|
||||
|
|
@ -1443,8 +1446,8 @@ fn renderExpression(
|
|||
const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", base);
|
||||
|
||||
if (fn_proto.visib_token) |visib_token_index| {
|
||||
const visib_token = tree.tokens[visib_token_index];
|
||||
assert(visib_token.id == .Keyword_pub or visib_token.id == .Keyword_export);
|
||||
const visib_token = tree.token_ids[visib_token_index];
|
||||
assert(visib_token == .Keyword_pub or visib_token == .Keyword_export);
|
||||
|
||||
try renderToken(tree, stream, visib_token_index, indent, start_col, Space.Space); // pub
|
||||
}
|
||||
|
|
@ -1466,7 +1469,7 @@ fn renderExpression(
|
|||
try renderToken(tree, stream, fn_proto.fn_token, indent, start_col, Space.Space); // fn
|
||||
break :blk tree.nextToken(fn_proto.fn_token);
|
||||
};
|
||||
assert(tree.tokens[lparen].id == .LParen);
|
||||
assert(tree.token_ids[lparen] == .LParen);
|
||||
|
||||
const rparen = tree.prevToken(
|
||||
// the first token for the annotation expressions is the left
|
||||
|
|
@ -1482,10 +1485,10 @@ fn renderExpression(
|
|||
.InferErrorSet => |node| tree.prevToken(node.firstToken()),
|
||||
.Invalid => unreachable,
|
||||
});
|
||||
assert(tree.tokens[rparen].id == .RParen);
|
||||
assert(tree.token_ids[rparen] == .RParen);
|
||||
|
||||
const src_params_trailing_comma = blk: {
|
||||
const maybe_comma = tree.tokens[rparen - 1].id;
|
||||
const maybe_comma = tree.token_ids[rparen - 1];
|
||||
break :blk maybe_comma == .Comma or maybe_comma == .LineComment;
|
||||
};
|
||||
|
||||
|
|
@ -1622,7 +1625,7 @@ fn renderExpression(
|
|||
const src_has_trailing_comma = blk: {
|
||||
const last_node = switch_case.items()[switch_case.items_len - 1];
|
||||
const maybe_comma = tree.nextToken(last_node.lastToken());
|
||||
break :blk tree.tokens[maybe_comma].id == .Comma;
|
||||
break :blk tree.token_ids[maybe_comma] == .Comma;
|
||||
};
|
||||
|
||||
if (switch_case.items_len == 1 or !src_has_trailing_comma) {
|
||||
|
|
@ -1967,7 +1970,7 @@ fn renderExpression(
|
|||
try renderAsmOutput(allocator, stream, tree, indent_extra, start_col, asm_output, Space.Newline);
|
||||
try stream.writeByteNTimes(' ', indent_once);
|
||||
const comma_or_colon = tree.nextToken(asm_output.lastToken());
|
||||
break :blk switch (tree.tokens[comma_or_colon].id) {
|
||||
break :blk switch (tree.token_ids[comma_or_colon]) {
|
||||
.Comma => tree.nextToken(comma_or_colon),
|
||||
else => comma_or_colon,
|
||||
};
|
||||
|
|
@ -2002,7 +2005,7 @@ fn renderExpression(
|
|||
try renderAsmInput(allocator, stream, tree, indent_extra, start_col, asm_input, Space.Newline);
|
||||
try stream.writeByteNTimes(' ', indent_once);
|
||||
const comma_or_colon = tree.nextToken(asm_input.lastToken());
|
||||
break :blk switch (tree.tokens[comma_or_colon].id) {
|
||||
break :blk switch (tree.token_ids[comma_or_colon]) {
|
||||
.Comma => tree.nextToken(comma_or_colon),
|
||||
else => comma_or_colon,
|
||||
};
|
||||
|
|
@ -2205,7 +2208,7 @@ fn renderStatement(
|
|||
try renderExpression(allocator, stream, tree, indent, start_col, base, Space.None);
|
||||
|
||||
const semicolon_index = tree.nextToken(base.lastToken());
|
||||
assert(tree.tokens[semicolon_index].id == .Semicolon);
|
||||
assert(tree.token_ids[semicolon_index] == .Semicolon);
|
||||
try renderToken(tree, stream, semicolon_index, indent, start_col, Space.Newline);
|
||||
} else {
|
||||
try renderExpression(allocator, stream, tree, indent, start_col, base, Space.Newline);
|
||||
|
|
@ -2243,22 +2246,25 @@ fn renderTokenOffset(
|
|||
return;
|
||||
}
|
||||
|
||||
var token = tree.tokens[token_index];
|
||||
try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(token)[token_skip_bytes..], " "));
|
||||
var token_loc = tree.token_locs[token_index];
|
||||
try stream.writeAll(mem.trimRight(u8, tree.tokenSliceLoc(token_loc)[token_skip_bytes..], " "));
|
||||
|
||||
if (space == Space.NoComment)
|
||||
return;
|
||||
|
||||
var next_token = tree.tokens[token_index + 1];
|
||||
var next_token_id = tree.token_ids[token_index + 1];
|
||||
var next_token_loc = tree.token_locs[token_index + 1];
|
||||
|
||||
if (space == Space.Comma) switch (next_token.id) {
|
||||
if (space == Space.Comma) switch (next_token_id) {
|
||||
.Comma => return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline),
|
||||
.LineComment => {
|
||||
try stream.writeAll(", ");
|
||||
return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline);
|
||||
},
|
||||
else => {
|
||||
if (token_index + 2 < tree.tokens.len and tree.tokens[token_index + 2].id == .MultilineStringLiteralLine) {
|
||||
if (token_index + 2 < tree.token_ids.len and
|
||||
tree.token_ids[token_index + 2] == .MultilineStringLiteralLine)
|
||||
{
|
||||
try stream.writeAll(",");
|
||||
return;
|
||||
} else {
|
||||
|
|
@ -2271,19 +2277,20 @@ fn renderTokenOffset(
|
|||
|
||||
// Skip over same line doc comments
|
||||
var offset: usize = 1;
|
||||
if (next_token.id == .DocComment) {
|
||||
const loc = tree.tokenLocationPtr(token.end, next_token);
|
||||
if (next_token_id == .DocComment) {
|
||||
const loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
|
||||
if (loc.line == 0) {
|
||||
offset += 1;
|
||||
next_token = tree.tokens[token_index + offset];
|
||||
next_token_id = tree.token_ids[token_index + offset];
|
||||
next_token_loc = tree.token_locs[token_index + offset];
|
||||
}
|
||||
}
|
||||
|
||||
if (next_token.id != .LineComment) blk: {
|
||||
if (next_token_id != .LineComment) blk: {
|
||||
switch (space) {
|
||||
Space.None, Space.NoNewline => return,
|
||||
Space.Newline => {
|
||||
if (next_token.id == .MultilineStringLiteralLine) {
|
||||
if (next_token_id == .MultilineStringLiteralLine) {
|
||||
return;
|
||||
} else {
|
||||
try stream.writeAll("\n");
|
||||
|
|
@ -2292,7 +2299,7 @@ fn renderTokenOffset(
|
|||
}
|
||||
},
|
||||
Space.Space, Space.SpaceOrOutdent => {
|
||||
if (next_token.id == .MultilineStringLiteralLine)
|
||||
if (next_token_id == .MultilineStringLiteralLine)
|
||||
return;
|
||||
try stream.writeByte(' ');
|
||||
return;
|
||||
|
|
@ -2302,14 +2309,15 @@ fn renderTokenOffset(
|
|||
}
|
||||
|
||||
while (true) {
|
||||
const comment_is_empty = mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ").len == 2;
|
||||
const comment_is_empty = mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ").len == 2;
|
||||
if (comment_is_empty) {
|
||||
switch (space) {
|
||||
Space.Newline => {
|
||||
offset += 1;
|
||||
token = next_token;
|
||||
next_token = tree.tokens[token_index + offset];
|
||||
if (next_token.id != .LineComment) {
|
||||
token_loc = next_token_loc;
|
||||
next_token_id = tree.token_ids[token_index + offset];
|
||||
next_token_loc = tree.token_locs[token_index + offset];
|
||||
if (next_token_id != .LineComment) {
|
||||
try stream.writeByte('\n');
|
||||
start_col.* = 0;
|
||||
return;
|
||||
|
|
@ -2322,18 +2330,19 @@ fn renderTokenOffset(
|
|||
}
|
||||
}
|
||||
|
||||
var loc = tree.tokenLocationPtr(token.end, next_token);
|
||||
var loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
|
||||
if (loc.line == 0) {
|
||||
try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ")});
|
||||
try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ")});
|
||||
offset = 2;
|
||||
token = next_token;
|
||||
next_token = tree.tokens[token_index + offset];
|
||||
if (next_token.id != .LineComment) {
|
||||
token_loc = next_token_loc;
|
||||
next_token_loc = tree.token_locs[token_index + offset];
|
||||
next_token_id = tree.token_ids[token_index + offset];
|
||||
if (next_token_id != .LineComment) {
|
||||
switch (space) {
|
||||
Space.None, Space.Space => {
|
||||
try stream.writeByte('\n');
|
||||
const after_comment_token = tree.tokens[token_index + offset];
|
||||
const next_line_indent = switch (after_comment_token.id) {
|
||||
const after_comment_token = tree.token_ids[token_index + offset];
|
||||
const next_line_indent = switch (after_comment_token) {
|
||||
.RParen, .RBrace, .RBracket => indent,
|
||||
else => indent + indent_delta,
|
||||
};
|
||||
|
|
@ -2346,7 +2355,7 @@ fn renderTokenOffset(
|
|||
start_col.* = indent;
|
||||
},
|
||||
Space.Newline => {
|
||||
if (next_token.id == .MultilineStringLiteralLine) {
|
||||
if (next_token_id == .MultilineStringLiteralLine) {
|
||||
return;
|
||||
} else {
|
||||
try stream.writeAll("\n");
|
||||
|
|
@ -2359,7 +2368,7 @@ fn renderTokenOffset(
|
|||
}
|
||||
return;
|
||||
}
|
||||
loc = tree.tokenLocationPtr(token.end, next_token);
|
||||
loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
|
||||
}
|
||||
|
||||
while (true) {
|
||||
|
|
@ -2369,15 +2378,16 @@ fn renderTokenOffset(
|
|||
const newline_count = if (loc.line <= 1) @as(u8, 1) else @as(u8, 2);
|
||||
try stream.writeByteNTimes('\n', newline_count);
|
||||
try stream.writeByteNTimes(' ', indent);
|
||||
try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(next_token), " "));
|
||||
try stream.writeAll(mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " "));
|
||||
|
||||
offset += 1;
|
||||
token = next_token;
|
||||
next_token = tree.tokens[token_index + offset];
|
||||
if (next_token.id != .LineComment) {
|
||||
token_loc = next_token_loc;
|
||||
next_token_loc = tree.token_locs[token_index + offset];
|
||||
next_token_id = tree.token_ids[token_index + offset];
|
||||
if (next_token_id != .LineComment) {
|
||||
switch (space) {
|
||||
Space.Newline => {
|
||||
if (next_token.id == .MultilineStringLiteralLine) {
|
||||
if (next_token_id == .MultilineStringLiteralLine) {
|
||||
return;
|
||||
} else {
|
||||
try stream.writeAll("\n");
|
||||
|
|
@ -2388,8 +2398,8 @@ fn renderTokenOffset(
|
|||
Space.None, Space.Space => {
|
||||
try stream.writeByte('\n');
|
||||
|
||||
const after_comment_token = tree.tokens[token_index + offset];
|
||||
const next_line_indent = switch (after_comment_token.id) {
|
||||
const after_comment_token = tree.token_ids[token_index + offset];
|
||||
const next_line_indent = switch (after_comment_token) {
|
||||
.RParen, .RBrace, .RBracket => blk: {
|
||||
if (indent > indent_delta) {
|
||||
break :blk indent - indent_delta;
|
||||
|
|
@ -2412,7 +2422,7 @@ fn renderTokenOffset(
|
|||
}
|
||||
return;
|
||||
}
|
||||
loc = tree.tokenLocationPtr(token.end, next_token);
|
||||
loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2448,7 +2458,7 @@ fn renderDocCommentsToken(
|
|||
) (@TypeOf(stream).Error || Error)!void {
|
||||
var tok_i = comment.first_line;
|
||||
while (true) : (tok_i += 1) {
|
||||
switch (tree.tokens[tok_i].id) {
|
||||
switch (tree.token_ids[tok_i]) {
|
||||
.DocComment, .ContainerDocComment => {
|
||||
if (comment.first_line < first_token) {
|
||||
try renderToken(tree, stream, tok_i, indent, start_col, Space.Newline);
|
||||
|
|
|
|||
|
|
@ -3,8 +3,12 @@ const mem = std.mem;
|
|||
|
||||
pub const Token = struct {
|
||||
id: Id,
|
||||
start: usize,
|
||||
end: usize,
|
||||
loc: Loc,
|
||||
|
||||
pub const Loc = struct {
|
||||
start: usize,
|
||||
end: usize,
|
||||
};
|
||||
|
||||
pub const Keyword = struct {
|
||||
bytes: []const u8,
|
||||
|
|
@ -426,8 +430,10 @@ pub const Tokenizer = struct {
|
|||
var state: State = .start;
|
||||
var result = Token{
|
||||
.id = .Eof,
|
||||
.start = self.index,
|
||||
.end = undefined,
|
||||
.loc = .{
|
||||
.start = self.index,
|
||||
.end = undefined,
|
||||
},
|
||||
};
|
||||
var seen_escape_digits: usize = undefined;
|
||||
var remaining_code_units: usize = undefined;
|
||||
|
|
@ -436,7 +442,7 @@ pub const Tokenizer = struct {
|
|||
switch (state) {
|
||||
.start => switch (c) {
|
||||
' ', '\n', '\t', '\r' => {
|
||||
result.start = self.index + 1;
|
||||
result.loc.start = self.index + 1;
|
||||
},
|
||||
'"' => {
|
||||
state = .string_literal;
|
||||
|
|
@ -686,7 +692,7 @@ pub const Tokenizer = struct {
|
|||
.identifier => switch (c) {
|
||||
'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
|
||||
else => {
|
||||
if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
|
||||
if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |id| {
|
||||
result.id = id;
|
||||
}
|
||||
break;
|
||||
|
|
@ -1313,7 +1319,7 @@ pub const Tokenizer = struct {
|
|||
=> {},
|
||||
|
||||
.identifier => {
|
||||
if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
|
||||
if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |id| {
|
||||
result.id = id;
|
||||
}
|
||||
},
|
||||
|
|
@ -1420,7 +1426,7 @@ pub const Tokenizer = struct {
|
|||
}
|
||||
}
|
||||
|
||||
result.end = self.index;
|
||||
result.loc.end = self.index;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
@ -1430,8 +1436,10 @@ pub const Tokenizer = struct {
|
|||
if (invalid_length == 0) return;
|
||||
self.pending_invalid_token = .{
|
||||
.id = .Invalid,
|
||||
.start = self.index,
|
||||
.end = self.index + invalid_length,
|
||||
.loc = .{
|
||||
.start = self.index,
|
||||
.end = self.index + invalid_length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -247,27 +247,6 @@ pub const Context = struct {
|
|||
}
|
||||
};
|
||||
|
||||
/// Helper function to append items to a singly linked list.
|
||||
fn llpusher(c: *Context, list: *std.SinglyLinkedList(*ast.Node)) LinkedListPusher {
|
||||
assert(list.first == null);
|
||||
return .{
|
||||
.c = c,
|
||||
.it = &list.first,
|
||||
};
|
||||
}
|
||||
|
||||
fn llpush(
|
||||
c: *Context,
|
||||
comptime T: type,
|
||||
it: *?*std.SinglyLinkedList(T).Node,
|
||||
data: T,
|
||||
) !*?*std.SinglyLinkedList(T).Node {
|
||||
const llnode = try c.arena.create(std.SinglyLinkedList(T).Node);
|
||||
llnode.* = .{ .data = data };
|
||||
it.* = llnode;
|
||||
return &llnode.next;
|
||||
}
|
||||
|
||||
fn getMangle(c: *Context) u32 {
|
||||
c.mangle_count += 1;
|
||||
return c.mangle_count;
|
||||
|
|
|
|||
Loading…
Reference in New Issue