Ekdohibs
/
zig
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

std-c tokenizer string concatenation

master
Vexu 2020-01-04 13:16:37 +02:00
parent a5d1fb1e49
commit 2183c4bb44
No known key found for this signature in database
GPG Key ID: 59AEB8936E16A6AC
1 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
lib/std/c/tokenizer.zig
View File

@ -272,6 +272,7 @@ pub const Tokenizer = struct {
U,
L,
StringLiteral,
AfterStringLiteral,
CharLiteralStart,
CharLiteral,
EscapeSequence,
@ -565,8 +566,7 @@ pub const Tokenizer = struct {
state = .EscapeSequence;
},
'"' => {
self.index += 1;
break;
state = .AfterStringLiteral;
},
'\n', '\r' => {
result.id = .Invalid;
@ -574,6 +574,15 @@ pub const Tokenizer = struct {
},
else => {},
},
.AfterStringLiteral => switch (c) {
'"' => {
state = .StringLiteral;
},
'\n'...'\r', ' ' => {},
else => {
break;
},
},
.CharLiteralStart => switch (c) {
'\\' => {
string = false;
@ -1109,6 +1118,7 @@ pub const Tokenizer = struct {
}
} else if (self.index == self.source.buffer.len) {
switch (state) {
.AfterStringLiteral,
.Start => {},
.u, .u8, .U, .L, .Identifier => {
result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
@ -1351,11 +1361,11 @@ test "line continuation" {
test "string prefix" {
expectTokens(
\\"foo"
\\u"foo"
\\u8"foo"
\\U"foo"
\\L"foo"
\\"foo" "bar"
\\u"foo" "bar"
\\u8"foo" "bar"
\\U"foo" "bar"
\\L"foo" "bar"
\\'foo'
\\u'foo'
\\U'foo'