Ekdohibs
/
zig
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

std.zig.parser now parses char literals

master
Jimmi Holst Christensen 2018-03-31 21:28:40 +02:00
parent 4d8f9e2295
commit 975dc5a390
3 changed files with 83 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
std/zig/ast.zig
View File

@ -21,6 +21,7 @@ pub const Node = struct {
FloatLiteral,
StringLiteral,
MultilineStringLiteral,
CharLiteral,
UndefinedLiteral,
BuiltinCall,
Call,
@ -42,6 +43,7 @@ pub const Node = struct {
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index),
Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).iterate(index),
Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index),
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index),
@ -64,6 +66,7 @@ pub const Node = struct {
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(),
Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).firstToken(),
Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(),
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(),
@ -86,6 +89,7 @@ pub const Node = struct {
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(),
Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).lastToken(),
Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(),
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(),
@ -608,6 +612,23 @@ pub const NodeMultilineStringLiteral = struct {
}
};
pub const NodeCharLiteral = struct {
base: Node,
token: Token,
pub fn iterate(self: &NodeCharLiteral, index: usize) ?&Node {
return null;
}
pub fn firstToken(self: &NodeCharLiteral) Token {
return self.token;
}
pub fn lastToken(self: &NodeCharLiteral) Token {
return self.token;
}
};
pub const NodeUndefinedLiteral = struct {
base: Node,
token: Token,

16
std/zig/parser.zig
View File

@ -456,6 +456,18 @@ pub const Parser = struct {
try stack.append(State.AfterOperand);
continue;
},
Token.Id.CharLiteral => {
const node = try arena.create(ast.NodeCharLiteral);
*node = ast.NodeCharLiteral {
.base = self.initNode(ast.Node.Id.CharLiteral),
.token = token,
};
try stack.append(State {
.Operand = &node.base
});
try stack.append(State.AfterOperand);
continue;
},
Token.Id.MultilineStringLiteralLine => {
const node = try arena.create(ast.NodeMultilineStringLiteral);
*node = ast.NodeMultilineStringLiteral {
@ -1451,6 +1463,10 @@ pub const Parser = struct {
const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base);
try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token));
},
ast.Node.Id.CharLiteral => {
const char_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base);
try stream.print("{}", self.tokenizer.getTokenSlice(char_literal.token));
},
ast.Node.Id.MultilineStringLiteral => {
const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base);
try stream.print("\n");

46
std/zig/tokenizer.zig
View File

@ -73,6 +73,7 @@ pub const Token = struct {
Identifier,
StringLiteral: StrLitKind,
MultilineStringLiteralLine: StrLitKind,
CharLiteral,
StringIdentifier,
Eof,
Builtin,
@ -228,6 +229,9 @@ pub const Tokenizer = struct {
StringLiteralBackslash,
MultilineStringLiteralLine,
MultilineStringLiteralLineBackslash,
CharLiteral,
CharLiteralBackslash,
CharLiteralEnd,
Backslash,
Equal,
Bang,
@ -294,6 +298,9 @@ pub const Tokenizer = struct {
state = State.StringLiteral;
result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
},
'\'' => {
state = State.CharLiteral;
},
'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
state = State.Identifier;
result.id = Token.Id.Identifier;
@ -579,6 +586,35 @@ pub const Tokenizer = struct {
},
},
State.CharLiteral => switch (c) {
'\\' => {
state = State.CharLiteralBackslash;
},
'\'' => break, // Look for this error later.
else => {
if (c < 0x20 or c == 0x7f)
break; // Look for this error later.
state = State.CharLiteralEnd;
}
},
State.CharLiteralBackslash => switch (c) {
'\n' => break, // Look for this error later.
else => {
state = State.CharLiteralEnd;
},
},
State.CharLiteralEnd => switch (c) {
'\'' => {
result.id = Token.Id.CharLiteral;
self.index += 1;
break;
},
else => break, // Look for this error later.
},
State.MultilineStringLiteralLine => switch (c) {
'\\' => {
state = State.MultilineStringLiteralLineBackslash;
@ -847,6 +883,7 @@ pub const Tokenizer = struct {
State.FloatExponentNumber,
State.StringLiteral, // find this error later
State.MultilineStringLiteralLine,
State.CharLiteralEnd,
State.Builtin => {},
State.Identifier => {
@ -863,6 +900,8 @@ pub const Tokenizer = struct {
State.SawAtSign,
State.Backslash,
State.MultilineStringLiteralLineBackslash,
State.CharLiteral,
State.CharLiteralBackslash,
State.StringLiteralBackslash => {
result.id = Token.Id.Invalid;
},
@ -1006,9 +1045,16 @@ test "tokenizer" {
});
}
test "tokenizer - chars" {
testTokenize("'c'", []Token.Id {Token.Id.CharLiteral});
}
test "tokenizer - invalid token characters" {
testTokenize("#", []Token.Id{Token.Id.Invalid});
testTokenize("`", []Token.Id{Token.Id.Invalid});
testTokenize("'c", []Token.Id {Token.Id.Invalid});
testTokenize("'", []Token.Id {Token.Id.Invalid});
testTokenize("''", []Token.Id {Token.Id.Invalid});
}
test "tokenizer - invalid literal/comment characters" {