std.zig Major Refactor

* parser now parses expression like the C++ compiler does
* This makes initializers work
* Added control flow expression (only return is parsed)
* Added catch parsing (It doesn't quite work)
* The parse can now specify states as optional.
  * The parse will roll back on error if states are optional
  * This can be overriden by State.Required

std/zig/ast.zig

@@ -24,6 +24,7 @@ pub const Node = struct {
         PrefixOp,
         SuffixOp,
         GroupedExpression,
+        ControlFlowExpression,
         FieldInitializer,
         IntegerLiteral,
         FloatLiteral,
@@ -58,6 +59,7 @@ pub const Node = struct {
             Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index),
             Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index),
             Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).iterate(index),
+            Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).iterate(index),
             Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index),
             Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
             Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
@@ -93,6 +95,7 @@ pub const Node = struct {
             Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(),
             Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(),
             Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).firstToken(),
+            Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).firstToken(),
             Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(),
             Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),
             Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
@@ -128,6 +131,7 @@ pub const Node = struct {
             Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(),
             Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(),
             Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).lastToken(),
+            Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).lastToken(),
             Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(),
             Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),
             Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
@@ -531,7 +535,7 @@ pub const NodeInfixOp = struct {
     op: InfixOp,
     rhs: &Node,
 
-    const InfixOp = enum {
+    const InfixOp = union(enum) {
         Add,
         AddWrap,
         ArrayCat,
@@ -558,6 +562,7 @@ pub const NodeInfixOp = struct {
         BitXor,
         BoolAnd,
         BoolOr,
+        Catch: ?&NodeIdentifier,
         Div,
         EqualEqual,
         ErrorUnion,
@@ -651,9 +656,9 @@ pub const NodePrefixOp = struct {
         BitNot,
         BoolNot,
         Deref,
+        MaybeType,
         Negation,
         NegationWrap,
-        Return,
         ArrayType: &Node,
         SliceType: AddrOfInfo,
         Try,
@@ -754,6 +759,7 @@ pub const NodeSuffixOp = struct {
     const CallInfo = struct {
         params: ArrayList(&Node),
         is_async: bool,
+        allocator: ?&Node,
     };
 
     const SliceRange = struct {
@@ -831,6 +837,56 @@ pub const NodeGroupedExpression = struct {
     }
 };
 
+pub const NodeControlFlowExpression = struct {
+    base: Node,
+    ltoken: Token,
+    kind: Kind,
+    rhs: ?&Node,
+
+    const Kind = union(enum) {
+        Break: ?Token,
+        Continue: ?Token,
+        Return,
+    };
+
+    pub fn iterate(self: &NodeControlFlowExpression, index: usize) ?&Node {
+        var i = index;
+
+        if (self.rhs) |rhs| {
+            if (i < 1) return rhs;
+            i -= 1;
+        }
+
+        return null;
+    }
+
+    pub fn firstToken(self: &NodeControlFlowExpression) Token {
+        return self.ltoken;
+    }
+
+    pub fn lastToken(self: &NodeControlFlowExpression) Token {
+        if (self.rhs) |rhs| {
+            return rhs.lastToken();
+        }
+
+        switch (self.kind) {
+            Kind.Break => |maybe_blk_token| {
+                if (maybe_blk_token) |blk_token| {
+                    return blk_token;
+                }
+            },
+            Kind.Continue => |maybe_blk_token| {
+                if (maybe_blk_token) |blk_token| {
+                    return blk_token;
+                }
+            },
+            Kind.Return => return self.ltoken,
+        }
+
+        return self.ltoken;
+    }
+};
+
 pub const NodeIntegerLiteral = struct {
     base: Node,
     token: Token,

std/zig/tokenizer.zig

@@ -15,8 +15,11 @@ pub const Token = struct {
         KeywordId{.bytes="align", .id = Id.Keyword_align},
         KeywordId{.bytes="and", .id = Id.Keyword_and},
         KeywordId{.bytes="asm", .id = Id.Keyword_asm},
+        KeywordId{.bytes="async", .id = Id.Keyword_async},
+        KeywordId{.bytes="await", .id = Id.Keyword_await},
         KeywordId{.bytes="break", .id = Id.Keyword_break},
         KeywordId{.bytes="catch", .id = Id.Keyword_catch},
+        KeywordId{.bytes="cancel", .id = Id.Keyword_cancel},
         KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
         KeywordId{.bytes="const", .id = Id.Keyword_const},
         KeywordId{.bytes="continue", .id = Id.Keyword_continue},
@@ -37,10 +40,12 @@ pub const Token = struct {
         KeywordId{.bytes="or", .id = Id.Keyword_or},
         KeywordId{.bytes="packed", .id = Id.Keyword_packed},
         KeywordId{.bytes="pub", .id = Id.Keyword_pub},
+        KeywordId{.bytes="resume", .id = Id.Keyword_resume},
         KeywordId{.bytes="return", .id = Id.Keyword_return},
         KeywordId{.bytes="section", .id = Id.Keyword_section},
         KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
         KeywordId{.bytes="struct", .id = Id.Keyword_struct},
+        KeywordId{.bytes="suspend", .id = Id.Keyword_suspend},
         KeywordId{.bytes="switch", .id = Id.Keyword_switch},
         KeywordId{.bytes="test", .id = Id.Keyword_test},
         KeywordId{.bytes="this", .id = Id.Keyword_this},
@@ -134,7 +139,10 @@ pub const Token = struct {
         Keyword_align,
         Keyword_and,
         Keyword_asm,
+        Keyword_async,
+        Keyword_await,
         Keyword_break,
+        Keyword_cancel,
         Keyword_catch,
         Keyword_comptime,
         Keyword_const,
@@ -156,10 +164,12 @@ pub const Token = struct {
         Keyword_or,
         Keyword_packed,
         Keyword_pub,
+        Keyword_resume,
         Keyword_return,
         Keyword_section,
         Keyword_stdcallcc,
         Keyword_struct,
+        Keyword_suspend,
         Keyword_switch,
         Keyword_test,
         Keyword_this,