diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 75003b06c..1c84e1264 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -21,6 +21,7 @@ pub const Node = struct { FloatLiteral, StringLiteral, MultilineStringLiteral, + CharLiteral, UndefinedLiteral, BuiltinCall, Call, @@ -42,6 +43,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index), + Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), @@ -64,6 +66,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(), + Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), @@ -86,6 +89,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(), + Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), @@ -608,6 +612,23 @@ pub const NodeMultilineStringLiteral = struct { } }; +pub const NodeCharLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeCharLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeCharLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeCharLiteral) Token { + return self.token; + } +}; + pub const NodeUndefinedLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index fa09a26db..a05328660 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -456,6 +456,18 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.CharLiteral => { + const node = try arena.create(ast.NodeCharLiteral); + *node = ast.NodeCharLiteral { + .base = self.initNode(ast.Node.Id.CharLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, Token.Id.MultilineStringLiteralLine => { const node = try arena.create(ast.NodeMultilineStringLiteral); *node = ast.NodeMultilineStringLiteral { @@ -1451,6 +1463,10 @@ pub const Parser = struct { const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token)); }, + ast.Node.Id.CharLiteral => { + const char_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(char_literal.token)); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 5647fcb86..64eb62759 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -73,6 +73,7 @@ pub const Token = struct { Identifier, StringLiteral: StrLitKind, MultilineStringLiteralLine: StrLitKind, + CharLiteral, StringIdentifier, Eof, Builtin, @@ -228,6 +229,9 @@ pub const Tokenizer = struct { StringLiteralBackslash, MultilineStringLiteralLine, MultilineStringLiteralLineBackslash, + CharLiteral, + CharLiteralBackslash, + CharLiteralEnd, Backslash, Equal, Bang, @@ -294,6 +298,9 @@ pub const Tokenizer = struct { state = State.StringLiteral; result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal }; }, + '\'' => { + state = State.CharLiteral; + }, 'a'...'b', 'd'...'z', 'A'...'Z', '_' => { state = State.Identifier; result.id = Token.Id.Identifier; @@ -579,6 +586,35 @@ pub const Tokenizer = struct { }, }, + State.CharLiteral => switch (c) { + '\\' => { + state = State.CharLiteralBackslash; + }, + '\'' => break, // Look for this error later. + else => { + if (c < 0x20 or c == 0x7f) + break; // Look for this error later. + + state = State.CharLiteralEnd; + } + }, + + State.CharLiteralBackslash => switch (c) { + '\n' => break, // Look for this error later. + else => { + state = State.CharLiteralEnd; + }, + }, + + State.CharLiteralEnd => switch (c) { + '\'' => { + result.id = Token.Id.CharLiteral; + self.index += 1; + break; + }, + else => break, // Look for this error later. + }, + State.MultilineStringLiteralLine => switch (c) { '\\' => { state = State.MultilineStringLiteralLineBackslash; @@ -847,6 +883,7 @@ pub const Tokenizer = struct { State.FloatExponentNumber, State.StringLiteral, // find this error later State.MultilineStringLiteralLine, + State.CharLiteralEnd, State.Builtin => {}, State.Identifier => { @@ -863,6 +900,8 @@ pub const Tokenizer = struct { State.SawAtSign, State.Backslash, State.MultilineStringLiteralLineBackslash, + State.CharLiteral, + State.CharLiteralBackslash, State.StringLiteralBackslash => { result.id = Token.Id.Invalid; }, @@ -1006,9 +1045,16 @@ test "tokenizer" { }); } +test "tokenizer - chars" { + testTokenize("'c'", []Token.Id {Token.Id.CharLiteral}); +} + test "tokenizer - invalid token characters" { testTokenize("#", []Token.Id{Token.Id.Invalid}); testTokenize("`", []Token.Id{Token.Id.Invalid}); + testTokenize("'c", []Token.Id {Token.Id.Invalid}); + testTokenize("'", []Token.Id {Token.Id.Invalid}); + testTokenize("''", []Token.Id {Token.Id.Invalid}); } test "tokenizer - invalid literal/comment characters" {