Merge pull request #3957 from xackus/stage2_parser_3799

stage2 parser: fix segfault on extern block

lib/std/zig/ast.zig

@@ -10,6 +10,7 @@ pub const TokenIndex = usize;
 pub const Tree = struct {
     source: []const u8,
     tokens: TokenList,
+    /// undefined on parse error (errors not empty)
     root_node: *Node.Root,
     arena_allocator: std.heap.ArenaAllocator,
     errors: ErrorList,
@@ -612,7 +613,7 @@ pub const Node = struct {
         visib_token: ?TokenIndex,
         thread_local_token: ?TokenIndex,
         name_token: TokenIndex,
-        eq_token: TokenIndex,
+        eq_token: ?TokenIndex,
         mut_token: TokenIndex,
         comptime_token: ?TokenIndex,
         extern_export_token: ?TokenIndex,

lib/std/zig/parse.zig

@@ -13,7 +13,7 @@ pub const Error = error{ParseError} || Allocator.Error;
 
 /// Result should be freed with tree.deinit() when there are
 /// no more references to any of the tokens or nodes.
-pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {
+pub fn parse(allocator: *Allocator, source: []const u8) Allocator.Error!*Tree {
     const tree = blk: {
         // This block looks unnecessary, but is a "foot-shield" to prevent the SegmentedLists
         // from being initialized with a pointer to this `arena`, which is created on
@@ -48,29 +48,32 @@ pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {
 
     while (it.peek().?.id == .LineComment) _ = it.next();
 
-    tree.root_node = try parseRoot(arena, &it, tree);
+    tree.root_node = parseRoot(arena, &it, tree) catch |err| blk: {
+        switch (err) {
+            error.ParseError => {
+                assert(tree.errors.len != 0);
+                break :blk undefined;
+            },
+            error.OutOfMemory => {
+                return error.OutOfMemory;
+            },
+        }
+    };
+
     return tree;
 }
 
 /// Root <- skip ContainerMembers eof
-fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!*Node.Root {
+fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!*Node.Root {
     const node = try arena.create(Node.Root);
     node.* = Node.Root{
-        .decls = undefined,
-        .eof_token = undefined,
-    };
-    node.decls = parseContainerMembers(arena, it, tree) catch |err| {
-        // TODO: Switch on the error type
-        // https://github.com/ziglang/zig/issues/2473
-        if (err == error.ParseError) return node;
-        assert(err == Allocator.Error.OutOfMemory);
-        return Allocator.Error.OutOfMemory;
-    };
-    node.eof_token = eatToken(it, .Eof) orelse {
-        try tree.errors.push(AstError{
-            .ExpectedContainerMembers = AstError.ExpectedContainerMembers{ .token = it.index },
-        });
-        return node;
+        .decls = try parseContainerMembers(arena, it, tree),
+        .eof_token = eatToken(it, .Eof) orelse {
+            try tree.errors.push(AstError{
+                .ExpectedContainerMembers = .{ .token = it.index },
+            });
+            return error.ParseError;
+        },
     };
     return node;
 }
@@ -303,7 +306,17 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
 fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
     const cc = parseFnCC(arena, it, tree);
     const fn_token = eatToken(it, .Keyword_fn) orelse {
-        if (cc == null) return null else return error.ParseError;
+        if (cc) |fnCC| {
+            if (fnCC == .Extern) {
+                putBackToken(it, fnCC.Extern); // 'extern' is also used in ContainerDecl
+            } else {
+                try tree.errors.push(AstError{
+                    .ExpectedToken = .{ .token = it.index, .expected_id = .Keyword_fn },
+                });
+                return error.ParseError;
+            }
+        }
+        return null;
     };
     const name_token = eatToken(it, .Identifier);
     const lparen = try expectToken(it, tree, .LParen);
@@ -390,7 +403,7 @@ fn parseVarDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
         .visib_token = null,
         .thread_local_token = null,
         .name_token = name_token,
-        .eq_token = eq_token orelse undefined,
+        .eq_token = eq_token,
         .mut_token = mut_token,
         .comptime_token = null,
         .extern_export_token = null,
@@ -2177,7 +2190,7 @@ fn parsePrefixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
     node.* = Node.PrefixOp{
         .op_token = token.index,
         .op = op,
-        .rhs = undefined,
+        .rhs = undefined, // set by caller
     };
     return &node.base;
 }
@@ -2819,8 +2832,8 @@ fn parseUse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
         .doc_comments = null,
         .visib_token = null,
         .use_token = token,
-        .expr = undefined,
-        .semicolon_token = undefined,
+        .expr = undefined, // set by caller
+        .semicolon_token = undefined, // set by caller
     };
     return &node.base;
 }
@@ -2979,9 +2992,9 @@ fn createInfixOp(arena: *Allocator, index: TokenIndex, op: Node.InfixOp.Op) !*No
     const node = try arena.create(Node.InfixOp);
     node.* = Node.InfixOp{
         .op_token = index,
-        .lhs = undefined,
+        .lhs = undefined, // set by caller
         .op = op,
-        .rhs = undefined,
+        .rhs = undefined, // set by caller
     };
     return &node.base;
 }

lib/std/zig/parser_test.zig

@@ -2714,6 +2714,13 @@ test "zig fmt: top level doc comments" {
     );
 }
 
+test "zig fmt: extern without container keyword returns error" {
+    try testError(
+        \\const container = extern {};
+        \\
+    );
+}
+
 const std = @import("std");
 const mem = std.mem;
 const warn = std.debug.warn;
@@ -2820,3 +2827,11 @@ fn testTransform(source: []const u8, expected_source: []const u8) !void {
 fn testCanonical(source: []const u8) !void {
     return testTransform(source, source);
 }
+
+fn testError(source: []const u8) !void {
+    var fixed_allocator = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
+    const tree = try std.zig.parse(&fixed_allocator.allocator, source);
+    defer tree.deinit();
+
+    std.testing.expect(tree.errors.len != 0);
+}

lib/std/zig/render.zig

@@ -2020,7 +2020,7 @@ fn renderVarDecl(
 
     if (var_decl.init_node) |init_node| {
         const s = if (init_node.id == .MultilineStringLiteral) Space.None else Space.Space;
-        try renderToken(tree, stream, var_decl.eq_token, indent, start_col, s); // =
+        try renderToken(tree, stream, var_decl.eq_token.?, indent, start_col, s); // =
         try renderExpression(allocator, stream, tree, indent, start_col, init_node, Space.None);
     }