From e4d0b46c0c07f3151b4959c5e2e0d4c304981ead Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sun, 8 Apr 2018 17:05:08 +0200 Subject: [PATCH] std.zig.parser WIP generalizing parsing of payloads * Note, it doesn't work :) --- std/zig/ast.zig | 108 ++++++++++++++++++++++++++++++++++++++++----- std/zig/parser.zig | 104 ++++++++++++++++++++++++++----------------- 2 files changed, 161 insertions(+), 51 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index ca384efed..0d060eb68 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,9 +20,11 @@ pub const Node = struct { FnProto, ParamDecl, Block, + Payload, Switch, SwitchCase, SwitchElse, + While, InfixOp, PrefixOp, SuffixOp, @@ -58,9 +60,11 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), + Id.While => @fieldParentPtr(NodeWhile, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -97,9 +101,11 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), + Id.While => @fieldParentPtr(NodeWhile, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -136,9 +142,11 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), - Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), + Id.While => @fieldParentPtr(NodeWhile, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -545,6 +553,31 @@ pub const NodeBlock = struct { } }; +pub const NodePayload = struct { + base: Node, + lpipe: Token, + is_ptr: bool, + symbol: &NodeIdentifier, + rpipe: Token, + + pub fn iterate(self: &NodePayload, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.symbol.base; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodePayload) Token { + return self.lpipe; + } + + pub fn lastToken(self: &NodePayload) Token { + return self.rpipe; + } +}; + pub const NodeSwitch = struct { base: Node, switch_token: Token, @@ -576,22 +609,17 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - capture: ?Capture, + payload: ?&Node, expr: &Node, - const Capture = struct { - symbol: &NodeIdentifier, - is_ptr: bool, - }; - pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { var i = index; if (i < self.items.len) return self.items.at(i); i -= self.items.len; - if (self.capture) |capture| { - if (i < 1) return &capture.base; + if (self.payload) |payload| { + if (i < 1) return payload; i -= 1; } @@ -627,6 +655,66 @@ pub const NodeSwitchElse = struct { } }; +pub const NodeWhile = struct { + base: Node, + while_token: Token, + condition: &Node, + payload: ?&NodePayload, + continue_expr: ?&Node, + body: &Node, + @"else": ?Else, + + const Else = struct { + capture: ?&NodeIdentifier, + body: &Node, + }; + + + pub fn iterate(self: &NodeWhile, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.condition; + i -= 1; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (self.continue_expr) |continue_expr| { + if (i < 1) return continue_expr; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + if (self.@"else") |@"else"| { + if (@"else".capture) |capture| { + if (i < 1) return &capture.base; + i -= 1; + } + + if (i < 1) return @"else".body; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeWhile) Token { + return self.while_token; + } + + pub fn lastToken(self: &NodeWhile) Token { + if (self.@"else") |@"else"| { + return @"else".body.lastToken(); + } + + return self.body.lastToken(); + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, @@ -661,7 +749,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&NodeIdentifier, + Catch: ?&Node, Div, EqualEqual, ErrorUnion, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 09f782750..0d8b8a324 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -104,6 +104,11 @@ pub const Parser = struct { ptr: &Token, }; + const ElseCtx = struct { + payload: ?DestPtr, + body: DestPtr, + }; + fn ListSave(comptime T: type) type { return struct { list: &ArrayList(T), @@ -140,7 +145,7 @@ pub const Parser = struct { FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - SwitchCaseCapture: &?ast.NodeSwitchCase.Capture, + Payload: DestPtr, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -635,9 +640,6 @@ pub const Parser = struct { Token.Id.Keyword_await => { @panic("TODO: await"); }, - Token.Id.Keyword_suspend => { - @panic("TODO: suspend"); - }, else => { self.putBackToken(token); stack.append(State { .AssignmentExpressionBegin = dest_ptr }) catch unreachable; @@ -705,21 +707,14 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - - const next = self.getNextToken(); - if (next.id != Token.Id.Pipe) { - self.putBackToken(next); - continue; - } - - node.op.Catch = try self.createIdentifier(arena, Token(undefined)); - try stack.append(State { .ExpectToken = Token.Id.Pipe }); try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.Identifier, - .ptr = &(??node.op.Catch).name_token + .Optional = RevertState { + .tokenizer = *self.tokenizer, + .parser = *self, + .ptr = &node.op.Catch, } }); + try stack.append(State { .Payload = DestPtr { .NullableField = &node.op.Catch } }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1404,12 +1399,25 @@ pub const Parser = struct { @panic("TODO: inline if"); }, Token.Id.Keyword_while => { + const node = try arena.create(ast.NodeWhile); + *node = ast.NodeWhile { + .base = self.initNode(ast.Node.Id.While), + .while_token = token, + .condition = undefined, + .payload = null, + .continue_expr = null, + .body = undefined, + .@"else" = null, + }; + dest_ptr.store(&node.base); + @panic("TODO: inline while"); }, Token.Id.Keyword_for => { @panic("TODO: inline for"); }, Token.Id.Keyword_switch => { + @breakpoint(); const node = try arena.create(ast.NodeSwitch); *node = ast.NodeSwitch { .base = self.initNode(ast.Node.Id.Switch), @@ -1434,9 +1442,6 @@ pub const Parser = struct { Token.Id.Keyword_comptime => { @panic("TODO: inline comptime"); }, - Token.Id.Keyword_suspend => { - @panic("TODO: inline suspend"); - }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; @@ -1547,13 +1552,21 @@ pub const Parser = struct { *node = ast.NodeSwitchCase { .base = self.initNode(ast.Node.Id.SwitchCase), .items = ArrayList(&ast.Node).init(arena), - .capture = null, + .payload = null, .expr = undefined, }; try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { .SwitchCaseCapture = &node.capture }); + try stack.append(State { + .Optional = RevertState { + .tokenizer = *self.tokenizer, + .parser = *self, + .ptr = &node.payload, + } + }); + try stack.append(State { .Payload = DestPtr { .NullableField = &node.payload } }); + try stack.append(State.Required); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1572,12 +1585,8 @@ pub const Parser = struct { } }, - State.SwitchCaseCapture => |capture| { - const token = self.getNextToken(); - if (token.id != Token.Id.Pipe) { - self.putBackToken(token); - continue; - } + State.Payload => |dest_ptr| { + const lpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; const is_ptr = blk: { const asterik = self.getNextToken(); @@ -1590,11 +1599,16 @@ pub const Parser = struct { }; const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - _ = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - *capture = ast.NodeSwitchCase.Capture { + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodePayload); + *node = ast.NodePayload { + .base = self.initNode(ast.Node.Id.Payload), + .lpipe = lpipe, + .is_ptr = is_ptr, .symbol = try self.createIdentifier(arena, ident), - .is_ptr = is_ptr + .rpipe = rpipe }; + dest_ptr.store(&node.base); }, State.SwitchCaseItem => |case_items| { @@ -1856,6 +1870,8 @@ pub const Parser = struct { stack.append(State { .Block = inner_block }) catch unreachable; continue; }, + Token.Id.Keyword_suspend, Token.Id.Keyword_if, + Token.Id.Keyword_while, Token.Id.Keyword_for, Token.Id.Keyword_switch => { self.putBackToken(next); stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }) catch unreachable; @@ -2572,9 +2588,8 @@ pub const Parser = struct { if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { if (prefix_op_node.op.Catch) |payload| { - try stack.append(RenderState { .Text = "| " }); - try stack.append(RenderState { .Expression = &payload.base }); - try stack.append(RenderState { .Text = "|" }); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = payload }); } try stack.append(RenderState { .Text = " catch " }); } else { @@ -2764,6 +2779,17 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = rhs }); } }, + ast.Node.Id.Payload => { + const payload = @fieldParentPtr(ast.NodePayload, "base", base); + try stack.append(RenderState { .Text = "|"}); + try stack.append(RenderState { .Expression = &payload.symbol.base }); + + if (payload.is_ptr) { + try stack.append(RenderState { .Text = "*"}); + } + + try stack.append(RenderState { .Text = "|"}); + }, ast.Node.Id.GroupedExpression => { const grouped_expr = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); try stack.append(RenderState { .Text = ")"}); @@ -2985,14 +3011,9 @@ pub const Parser = struct { const switch_case = @fieldParentPtr(ast.NodeSwitchCase, "base", base); try stack.append(RenderState { .Expression = switch_case.expr }); - if (switch_case.capture) |capture| { - try stack.append(RenderState { .Text = "| "}); - try stack.append(RenderState { .Expression = &capture.symbol.base }); - - if (capture.is_ptr) { - try stack.append(RenderState { .Text = "*"}); - } - try stack.append(RenderState { .Text = "|"}); + if (switch_case.payload) |payload| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = payload }); } try stack.append(RenderState { .Text = " => "}); @@ -3011,6 +3032,7 @@ pub const Parser = struct { const switch_else = @fieldParentPtr(ast.NodeSwitchElse, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(switch_else.token)); }, + ast.Node.Id.While => @panic("TODO: Render while"), ast.Node.Id.StructField, ast.Node.Id.UnionTag,