133 lines
2.9 KiB
Zig
133 lines
2.9 KiB
Zig
const std = @import("std");
|
|
const expect = std.testing.expect;
|
|
|
|
pub const Source = struct {
|
|
buffer: []const u8,
|
|
file_name: []const u8,
|
|
};
|
|
|
|
pub const Token = struct {
|
|
id: Id,
|
|
num_suffix: NumSuffix = .None,
|
|
start: usize,
|
|
end: usize,
|
|
source: *Source,
|
|
|
|
pub const Id = enum {
|
|
Invalid,
|
|
Eof,
|
|
Nl,
|
|
Identifier,
|
|
StringLiteral,
|
|
CharLiteral,
|
|
IntegerLiteral,
|
|
FloatLiteral,
|
|
Bang,
|
|
BangEqual,
|
|
Pipe,
|
|
PipePipe,
|
|
PipeEqual,
|
|
Equal,
|
|
EqualEqual,
|
|
EqualAngleBracketRight,
|
|
LParen,
|
|
RParen,
|
|
LBrace,
|
|
RBrace,
|
|
LBracket,
|
|
RBracket,
|
|
Period,
|
|
PeriodAsterisk,
|
|
Ellipsis,
|
|
Caret,
|
|
CaretEqual,
|
|
Plus,
|
|
PlusPlus,
|
|
PlusEqual,
|
|
Minus,
|
|
MinusMinus,
|
|
MinusEqual,
|
|
Asterisk,
|
|
AsteriskEqual,
|
|
Percent,
|
|
PercentEqual,
|
|
Arrow,
|
|
Colon,
|
|
Semicolon,
|
|
Slash,
|
|
SlashEqual,
|
|
Comma,
|
|
Ampersand,
|
|
AmpersandAmpersand,
|
|
AmpersandEqual,
|
|
QuestionMark,
|
|
AngleBracketLeft,
|
|
AngleBracketLeftEqual,
|
|
AngleBracketAngleBracketLeft,
|
|
AngleBracketAngleBracketLeftEqual,
|
|
AngleBracketRight,
|
|
AngleBracketRightEqual,
|
|
AngleBracketAngleBracketRight,
|
|
AngleBracketAngleBracketRightEqual,
|
|
Tilde,
|
|
LineComment,
|
|
MultiLineComment,
|
|
Hash,
|
|
HashHash,
|
|
};
|
|
|
|
pub const NumSuffix = enum {
|
|
None,
|
|
F,
|
|
L,
|
|
U,
|
|
LU,
|
|
LL,
|
|
LLU,
|
|
};
|
|
};
|
|
|
|
pub const Tokenizer = struct {
|
|
source: *Source,
|
|
index: usize = 0,
|
|
|
|
pub fn next(self: *Tokenizer) Token {
|
|
const start_index = self.index;
|
|
var result = Token{
|
|
.id = .Eof,
|
|
.start = self.index,
|
|
.end = undefined,
|
|
.source = self.source,
|
|
};
|
|
var state: enum {
|
|
Start,
|
|
} = .Start;
|
|
while (self.index < self.source.buffer.len) : (self.index += 1) {
|
|
const c = self.source.buffer[self.index];
|
|
switch (state) {
|
|
.Start => switch (c) {
|
|
else => @panic("TODO"),
|
|
},
|
|
else => @panic("TODO"),
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
fn expectTokens(source: []const u8, expected_tokens: []const Token.Id) void {
|
|
var tokenizer = Tokenizer{
|
|
.source = .{
|
|
.buffer = source,
|
|
.file_name = undefined,
|
|
},
|
|
};
|
|
for (expected_tokens) |expected_token_id| {
|
|
const token = tokenizer.next();
|
|
if (token.id != expected_token_id) {
|
|
std.debug.panic("expected {}, found {}\n", .{ @tagName(expected_token_id), @tagName(token.id) });
|
|
}
|
|
}
|
|
const last_token = tokenizer.next();
|
|
std.testing.expect(last_token.id == .Eof);
|
|
}
|