self-hosted: refactor into multiple files
add return expression add number literalmaster
Andrew Kelley
parent
a3a590a32a
commit
d8d379faf1
|
|
@ -0,0 +1,245 @@
|
|||
const std = @import("std");
|
||||
const assert = std.debug.assert;
|
||||
const ArrayList = std.ArrayList;
|
||||
const Token = @import("tokenizer.zig").Token;
|
||||
const mem = std.mem;
|
||||
|
||||
pub const Node = struct {
|
||||
id: Id,
|
||||
|
||||
pub const Id = enum {
|
||||
Root,
|
||||
VarDecl,
|
||||
Identifier,
|
||||
FnProto,
|
||||
ParamDecl,
|
||||
AddrOfExpr,
|
||||
Block,
|
||||
Return,
|
||||
IntegerLiteral,
|
||||
FloatLiteral,
|
||||
};
|
||||
|
||||
pub fn iterate(base: &Node, index: usize) -> ?&Node {
|
||||
return switch (base.id) {
|
||||
Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index),
|
||||
Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index),
|
||||
Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index),
|
||||
Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index),
|
||||
Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index),
|
||||
Id.AddrOfExpr => @fieldParentPtr(NodeAddrOfExpr, "base", base).iterate(index),
|
||||
Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index),
|
||||
Id.Return => @fieldParentPtr(NodeReturn, "base", base).iterate(index),
|
||||
Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
|
||||
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
|
||||
};
|
||||
}
|
||||
|
||||
pub fn destroy(base: &Node, allocator: &mem.Allocator) {
|
||||
return switch (base.id) {
|
||||
Id.Root => allocator.destroy(@fieldParentPtr(NodeRoot, "base", base)),
|
||||
Id.VarDecl => allocator.destroy(@fieldParentPtr(NodeVarDecl, "base", base)),
|
||||
Id.Identifier => allocator.destroy(@fieldParentPtr(NodeIdentifier, "base", base)),
|
||||
Id.FnProto => allocator.destroy(@fieldParentPtr(NodeFnProto, "base", base)),
|
||||
Id.ParamDecl => allocator.destroy(@fieldParentPtr(NodeParamDecl, "base", base)),
|
||||
Id.AddrOfExpr => allocator.destroy(@fieldParentPtr(NodeAddrOfExpr, "base", base)),
|
||||
Id.Block => allocator.destroy(@fieldParentPtr(NodeBlock, "base", base)),
|
||||
Id.Return => allocator.destroy(@fieldParentPtr(NodeReturn, "base", base)),
|
||||
Id.IntegerLiteral => allocator.destroy(@fieldParentPtr(NodeIntegerLiteral, "base", base)),
|
||||
Id.FloatLiteral => allocator.destroy(@fieldParentPtr(NodeFloatLiteral, "base", base)),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
pub const NodeRoot = struct {
|
||||
base: Node,
|
||||
decls: ArrayList(&Node),
|
||||
|
||||
pub fn iterate(self: &NodeRoot, index: usize) -> ?&Node {
|
||||
if (index < self.decls.len) {
|
||||
return self.decls.items[self.decls.len - index - 1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
pub const NodeVarDecl = struct {
|
||||
base: Node,
|
||||
visib_token: ?Token,
|
||||
name_token: Token,
|
||||
eq_token: Token,
|
||||
mut_token: Token,
|
||||
comptime_token: ?Token,
|
||||
extern_token: ?Token,
|
||||
lib_name: ?&Node,
|
||||
type_node: ?&Node,
|
||||
align_node: ?&Node,
|
||||
init_node: ?&Node,
|
||||
|
||||
pub fn iterate(self: &NodeVarDecl, index: usize) -> ?&Node {
|
||||
var i = index;
|
||||
|
||||
if (self.type_node) |type_node| {
|
||||
if (i < 1) return type_node;
|
||||
i -= 1;
|
||||
}
|
||||
|
||||
if (self.align_node) |align_node| {
|
||||
if (i < 1) return align_node;
|
||||
i -= 1;
|
||||
}
|
||||
|
||||
if (self.init_node) |init_node| {
|
||||
if (i < 1) return init_node;
|
||||
i -= 1;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
pub const NodeIdentifier = struct {
|
||||
base: Node,
|
||||
name_token: Token,
|
||||
|
||||
pub fn iterate(self: &NodeIdentifier, index: usize) -> ?&Node {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
pub const NodeFnProto = struct {
|
||||
base: Node,
|
||||
visib_token: ?Token,
|
||||
fn_token: Token,
|
||||
name_token: ?Token,
|
||||
params: ArrayList(&Node),
|
||||
return_type: ?&Node,
|
||||
var_args_token: ?Token,
|
||||
extern_token: ?Token,
|
||||
inline_token: ?Token,
|
||||
cc_token: ?Token,
|
||||
body_node: ?&Node,
|
||||
lib_name: ?&Node, // populated if this is an extern declaration
|
||||
align_expr: ?&Node, // populated if align(A) is present
|
||||
|
||||
pub fn iterate(self: &NodeFnProto, index: usize) -> ?&Node {
|
||||
var i = index;
|
||||
|
||||
if (self.body_node) |body_node| {
|
||||
if (i < 1) return body_node;
|
||||
i -= 1;
|
||||
}
|
||||
|
||||
if (self.return_type) |return_type| {
|
||||
if (i < 1) return return_type;
|
||||
i -= 1;
|
||||
}
|
||||
|
||||
if (self.align_expr) |align_expr| {
|
||||
if (i < 1) return align_expr;
|
||||
i -= 1;
|
||||
}
|
||||
|
||||
if (i < self.params.len) return self.params.items[self.params.len - i - 1];
|
||||
i -= self.params.len;
|
||||
|
||||
if (self.lib_name) |lib_name| {
|
||||
if (i < 1) return lib_name;
|
||||
i -= 1;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
pub const NodeParamDecl = struct {
|
||||
base: Node,
|
||||
comptime_token: ?Token,
|
||||
noalias_token: ?Token,
|
||||
name_token: ?Token,
|
||||
type_node: &Node,
|
||||
var_args_token: ?Token,
|
||||
|
||||
pub fn iterate(self: &NodeParamDecl, index: usize) -> ?&Node {
|
||||
var i = index;
|
||||
|
||||
if (i < 1) return self.type_node;
|
||||
i -= 1;
|
||||
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
pub const NodeAddrOfExpr = struct {
|
||||
base: Node,
|
||||
op_token: Token,
|
||||
align_expr: ?&Node,
|
||||
bit_offset_start_token: ?Token,
|
||||
bit_offset_end_token: ?Token,
|
||||
const_token: ?Token,
|
||||
volatile_token: ?Token,
|
||||
op_expr: &Node,
|
||||
|
||||
pub fn iterate(self: &NodeAddrOfExpr, index: usize) -> ?&Node {
|
||||
var i = index;
|
||||
|
||||
if (self.align_expr) |align_expr| {
|
||||
if (i < 1) return align_expr;
|
||||
i -= 1;
|
||||
}
|
||||
|
||||
if (i < 1) return self.op_expr;
|
||||
i -= 1;
|
||||
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
pub const NodeBlock = struct {
|
||||
base: Node,
|
||||
begin_token: Token,
|
||||
end_token: Token,
|
||||
statements: ArrayList(&Node),
|
||||
|
||||
pub fn iterate(self: &NodeBlock, index: usize) -> ?&Node {
|
||||
var i = index;
|
||||
|
||||
if (i < self.statements.len) return self.statements.items[i];
|
||||
i -= self.statements.len;
|
||||
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
pub const NodeReturn = struct {
|
||||
base: Node,
|
||||
return_token: Token,
|
||||
expr: &Node,
|
||||
|
||||
pub fn iterate(self: &NodeReturn, index: usize) -> ?&Node {
|
||||
var i = index;
|
||||
|
||||
if (i < 1) return self.expr;
|
||||
i -= 1;
|
||||
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
pub const NodeIntegerLiteral = struct {
|
||||
base: Node,
|
||||
token: Token,
|
||||
|
||||
pub fn iterate(self: &NodeIntegerLiteral, index: usize) -> ?&Node {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
pub const NodeFloatLiteral = struct {
|
||||
base: Node,
|
||||
token: Token,
|
||||
|
||||
pub fn iterate(self: &NodeFloatLiteral, index: usize) -> ?&Node {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,479 @@
|
|||
const std = @import("std");
|
||||
const mem = std.mem;
|
||||
|
||||
pub const Token = struct {
|
||||
id: Id,
|
||||
start: usize,
|
||||
end: usize,
|
||||
|
||||
const KeywordId = struct {
|
||||
bytes: []const u8,
|
||||
id: Id,
|
||||
};
|
||||
|
||||
const keywords = []KeywordId {
|
||||
KeywordId{.bytes="align", .id = Id.Keyword_align},
|
||||
KeywordId{.bytes="and", .id = Id.Keyword_and},
|
||||
KeywordId{.bytes="asm", .id = Id.Keyword_asm},
|
||||
KeywordId{.bytes="break", .id = Id.Keyword_break},
|
||||
KeywordId{.bytes="coldcc", .id = Id.Keyword_coldcc},
|
||||
KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
|
||||
KeywordId{.bytes="const", .id = Id.Keyword_const},
|
||||
KeywordId{.bytes="continue", .id = Id.Keyword_continue},
|
||||
KeywordId{.bytes="defer", .id = Id.Keyword_defer},
|
||||
KeywordId{.bytes="else", .id = Id.Keyword_else},
|
||||
KeywordId{.bytes="enum", .id = Id.Keyword_enum},
|
||||
KeywordId{.bytes="error", .id = Id.Keyword_error},
|
||||
KeywordId{.bytes="export", .id = Id.Keyword_export},
|
||||
KeywordId{.bytes="extern", .id = Id.Keyword_extern},
|
||||
KeywordId{.bytes="false", .id = Id.Keyword_false},
|
||||
KeywordId{.bytes="fn", .id = Id.Keyword_fn},
|
||||
KeywordId{.bytes="for", .id = Id.Keyword_for},
|
||||
KeywordId{.bytes="goto", .id = Id.Keyword_goto},
|
||||
KeywordId{.bytes="if", .id = Id.Keyword_if},
|
||||
KeywordId{.bytes="inline", .id = Id.Keyword_inline},
|
||||
KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
|
||||
KeywordId{.bytes="noalias", .id = Id.Keyword_noalias},
|
||||
KeywordId{.bytes="null", .id = Id.Keyword_null},
|
||||
KeywordId{.bytes="or", .id = Id.Keyword_or},
|
||||
KeywordId{.bytes="packed", .id = Id.Keyword_packed},
|
||||
KeywordId{.bytes="pub", .id = Id.Keyword_pub},
|
||||
KeywordId{.bytes="return", .id = Id.Keyword_return},
|
||||
KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
|
||||
KeywordId{.bytes="struct", .id = Id.Keyword_struct},
|
||||
KeywordId{.bytes="switch", .id = Id.Keyword_switch},
|
||||
KeywordId{.bytes="test", .id = Id.Keyword_test},
|
||||
KeywordId{.bytes="this", .id = Id.Keyword_this},
|
||||
KeywordId{.bytes="true", .id = Id.Keyword_true},
|
||||
KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
|
||||
KeywordId{.bytes="union", .id = Id.Keyword_union},
|
||||
KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
|
||||
KeywordId{.bytes="use", .id = Id.Keyword_use},
|
||||
KeywordId{.bytes="var", .id = Id.Keyword_var},
|
||||
KeywordId{.bytes="volatile", .id = Id.Keyword_volatile},
|
||||
KeywordId{.bytes="while", .id = Id.Keyword_while},
|
||||
};
|
||||
|
||||
fn getKeyword(bytes: []const u8) -> ?Id {
|
||||
for (keywords) |kw| {
|
||||
if (mem.eql(u8, kw.bytes, bytes)) {
|
||||
return kw.id;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const StrLitKind = enum {Normal, C};
|
||||
|
||||
pub const Id = union(enum) {
|
||||
Invalid,
|
||||
Identifier,
|
||||
StringLiteral: StrLitKind,
|
||||
Eof,
|
||||
Builtin,
|
||||
Equal,
|
||||
LParen,
|
||||
RParen,
|
||||
Semicolon,
|
||||
Percent,
|
||||
LBrace,
|
||||
RBrace,
|
||||
Period,
|
||||
Ellipsis2,
|
||||
Ellipsis3,
|
||||
Minus,
|
||||
Arrow,
|
||||
Colon,
|
||||
Slash,
|
||||
Comma,
|
||||
Ampersand,
|
||||
AmpersandEqual,
|
||||
IntegerLiteral,
|
||||
FloatLiteral,
|
||||
Keyword_align,
|
||||
Keyword_and,
|
||||
Keyword_asm,
|
||||
Keyword_break,
|
||||
Keyword_coldcc,
|
||||
Keyword_comptime,
|
||||
Keyword_const,
|
||||
Keyword_continue,
|
||||
Keyword_defer,
|
||||
Keyword_else,
|
||||
Keyword_enum,
|
||||
Keyword_error,
|
||||
Keyword_export,
|
||||
Keyword_extern,
|
||||
Keyword_false,
|
||||
Keyword_fn,
|
||||
Keyword_for,
|
||||
Keyword_goto,
|
||||
Keyword_if,
|
||||
Keyword_inline,
|
||||
Keyword_nakedcc,
|
||||
Keyword_noalias,
|
||||
Keyword_null,
|
||||
Keyword_or,
|
||||
Keyword_packed,
|
||||
Keyword_pub,
|
||||
Keyword_return,
|
||||
Keyword_stdcallcc,
|
||||
Keyword_struct,
|
||||
Keyword_switch,
|
||||
Keyword_test,
|
||||
Keyword_this,
|
||||
Keyword_true,
|
||||
Keyword_undefined,
|
||||
Keyword_union,
|
||||
Keyword_unreachable,
|
||||
Keyword_use,
|
||||
Keyword_var,
|
||||
Keyword_volatile,
|
||||
Keyword_while,
|
||||
};
|
||||
};
|
||||
|
||||
pub const Tokenizer = struct {
|
||||
buffer: []const u8,
|
||||
index: usize,
|
||||
|
||||
pub const Location = struct {
|
||||
line: usize,
|
||||
column: usize,
|
||||
line_start: usize,
|
||||
line_end: usize,
|
||||
};
|
||||
|
||||
pub fn getTokenLocation(self: &Tokenizer, token: &const Token) -> Location {
|
||||
var loc = Location {
|
||||
.line = 0,
|
||||
.column = 0,
|
||||
.line_start = 0,
|
||||
.line_end = 0,
|
||||
};
|
||||
for (self.buffer) |c, i| {
|
||||
if (i == token.start) {
|
||||
loc.line_end = i;
|
||||
while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
|
||||
return loc;
|
||||
}
|
||||
if (c == '\n') {
|
||||
loc.line += 1;
|
||||
loc.column = 0;
|
||||
loc.line_start = i + 1;
|
||||
} else {
|
||||
loc.column += 1;
|
||||
}
|
||||
}
|
||||
return loc;
|
||||
}
|
||||
|
||||
/// For debugging purposes
|
||||
pub fn dump(self: &Tokenizer, token: &const Token) {
|
||||
std.debug.warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
|
||||
}
|
||||
|
||||
pub fn init(buffer: []const u8) -> Tokenizer {
|
||||
return Tokenizer {
|
||||
.buffer = buffer,
|
||||
.index = 0,
|
||||
};
|
||||
}
|
||||
|
||||
const State = enum {
|
||||
Start,
|
||||
Identifier,
|
||||
Builtin,
|
||||
C,
|
||||
StringLiteral,
|
||||
StringLiteralBackslash,
|
||||
Minus,
|
||||
Slash,
|
||||
LineComment,
|
||||
Zero,
|
||||
IntegerLiteral,
|
||||
NumberDot,
|
||||
FloatFraction,
|
||||
FloatExponentUnsigned,
|
||||
FloatExponentNumber,
|
||||
Ampersand,
|
||||
Period,
|
||||
Period2,
|
||||
};
|
||||
|
||||
pub fn next(self: &Tokenizer) -> Token {
|
||||
var state = State.Start;
|
||||
var result = Token {
|
||||
.id = Token.Id.Eof,
|
||||
.start = self.index,
|
||||
.end = undefined,
|
||||
};
|
||||
while (self.index < self.buffer.len) : (self.index += 1) {
|
||||
const c = self.buffer[self.index];
|
||||
switch (state) {
|
||||
State.Start => switch (c) {
|
||||
' ', '\n' => {
|
||||
result.start = self.index + 1;
|
||||
},
|
||||
'c' => {
|
||||
state = State.C;
|
||||
result.id = Token.Id.Identifier;
|
||||
},
|
||||
'"' => {
|
||||
state = State.StringLiteral;
|
||||
result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
|
||||
},
|
||||
'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
|
||||
state = State.Identifier;
|
||||
result.id = Token.Id.Identifier;
|
||||
},
|
||||
'@' => {
|
||||
state = State.Builtin;
|
||||
result.id = Token.Id.Builtin;
|
||||
},
|
||||
'=' => {
|
||||
result.id = Token.Id.Equal;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'(' => {
|
||||
result.id = Token.Id.LParen;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
')' => {
|
||||
result.id = Token.Id.RParen;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
';' => {
|
||||
result.id = Token.Id.Semicolon;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
',' => {
|
||||
result.id = Token.Id.Comma;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
':' => {
|
||||
result.id = Token.Id.Colon;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'%' => {
|
||||
result.id = Token.Id.Percent;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'{' => {
|
||||
result.id = Token.Id.LBrace;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'}' => {
|
||||
result.id = Token.Id.RBrace;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'.' => {
|
||||
state = State.Period;
|
||||
},
|
||||
'-' => {
|
||||
state = State.Minus;
|
||||
},
|
||||
'/' => {
|
||||
state = State.Slash;
|
||||
},
|
||||
'&' => {
|
||||
state = State.Ampersand;
|
||||
},
|
||||
'0' => {
|
||||
state = State.Zero;
|
||||
result.id = Token.Id.IntegerLiteral;
|
||||
},
|
||||
'1'...'9' => {
|
||||
state = State.IntegerLiteral;
|
||||
result.id = Token.Id.IntegerLiteral;
|
||||
},
|
||||
else => {
|
||||
result.id = Token.Id.Invalid;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
},
|
||||
State.Ampersand => switch (c) {
|
||||
'=' => {
|
||||
result.id = Token.Id.AmpersandEqual;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
else => {
|
||||
result.id = Token.Id.Ampersand;
|
||||
break;
|
||||
},
|
||||
},
|
||||
State.Identifier => switch (c) {
|
||||
'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
|
||||
else => {
|
||||
if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
|
||||
result.id = id;
|
||||
}
|
||||
break;
|
||||
},
|
||||
},
|
||||
State.Builtin => switch (c) {
|
||||
'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
|
||||
else => break,
|
||||
},
|
||||
State.C => switch (c) {
|
||||
'\\' => @panic("TODO"),
|
||||
'"' => {
|
||||
state = State.StringLiteral;
|
||||
result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
|
||||
},
|
||||
'a'...'z', 'A'...'Z', '_', '0'...'9' => {
|
||||
state = State.Identifier;
|
||||
},
|
||||
else => break,
|
||||
},
|
||||
State.StringLiteral => switch (c) {
|
||||
'\\' => {
|
||||
state = State.StringLiteralBackslash;
|
||||
},
|
||||
'"' => {
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
'\n' => break, // Look for this error later.
|
||||
else => {},
|
||||
},
|
||||
|
||||
State.StringLiteralBackslash => switch (c) {
|
||||
'\n' => break, // Look for this error later.
|
||||
else => {
|
||||
state = State.StringLiteral;
|
||||
},
|
||||
},
|
||||
|
||||
State.Minus => switch (c) {
|
||||
'>' => {
|
||||
result.id = Token.Id.Arrow;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
else => {
|
||||
result.id = Token.Id.Minus;
|
||||
break;
|
||||
},
|
||||
},
|
||||
|
||||
State.Period => switch (c) {
|
||||
'.' => {
|
||||
state = State.Period2;
|
||||
},
|
||||
else => {
|
||||
result.id = Token.Id.Period;
|
||||
break;
|
||||
},
|
||||
},
|
||||
|
||||
State.Period2 => switch (c) {
|
||||
'.' => {
|
||||
result.id = Token.Id.Ellipsis3;
|
||||
self.index += 1;
|
||||
break;
|
||||
},
|
||||
else => {
|
||||
result.id = Token.Id.Ellipsis2;
|
||||
break;
|
||||
},
|
||||
},
|
||||
|
||||
State.Slash => switch (c) {
|
||||
'/' => {
|
||||
result.id = undefined;
|
||||
state = State.LineComment;
|
||||
},
|
||||
else => {
|
||||
result.id = Token.Id.Slash;
|
||||
break;
|
||||
},
|
||||
},
|
||||
State.LineComment => switch (c) {
|
||||
'\n' => {
|
||||
state = State.Start;
|
||||
result = Token {
|
||||
.id = Token.Id.Eof,
|
||||
.start = self.index + 1,
|
||||
.end = undefined,
|
||||
};
|
||||
},
|
||||
else => {},
|
||||
},
|
||||
State.Zero => switch (c) {
|
||||
'b', 'o', 'x' => {
|
||||
state = State.IntegerLiteral;
|
||||
},
|
||||
else => {
|
||||
// reinterpret as a normal number
|
||||
self.index -= 1;
|
||||
state = State.IntegerLiteral;
|
||||
},
|
||||
},
|
||||
State.IntegerLiteral => switch (c) {
|
||||
'.' => {
|
||||
state = State.NumberDot;
|
||||
},
|
||||
'p', 'P', 'e', 'E' => {
|
||||
state = State.FloatExponentUnsigned;
|
||||
},
|
||||
'0'...'9', 'a'...'f', 'A'...'F' => {},
|
||||
else => break,
|
||||
},
|
||||
State.NumberDot => switch (c) {
|
||||
'.' => {
|
||||
self.index -= 1;
|
||||
state = State.Start;
|
||||
break;
|
||||
},
|
||||
else => {
|
||||
self.index -= 1;
|
||||
result.id = Token.Id.FloatLiteral;
|
||||
state = State.FloatFraction;
|
||||
},
|
||||
},
|
||||
State.FloatFraction => switch (c) {
|
||||
'p', 'P', 'e', 'E' => {
|
||||
state = State.FloatExponentUnsigned;
|
||||
},
|
||||
'0'...'9', 'a'...'f', 'A'...'F' => {},
|
||||
else => break,
|
||||
},
|
||||
State.FloatExponentUnsigned => switch (c) {
|
||||
'+', '-' => {
|
||||
state = State.FloatExponentNumber;
|
||||
},
|
||||
else => {
|
||||
// reinterpret as a normal exponent number
|
||||
self.index -= 1;
|
||||
state = State.FloatExponentNumber;
|
||||
}
|
||||
},
|
||||
State.FloatExponentNumber => switch (c) {
|
||||
'0'...'9', 'a'...'f', 'A'...'F' => {},
|
||||
else => break,
|
||||
},
|
||||
}
|
||||
}
|
||||
result.end = self.index;
|
||||
// TODO check state when returning EOF
|
||||
return result;
|
||||
}
|
||||
|
||||
pub fn getTokenSlice(self: &const Tokenizer, token: &const Token) -> []const u8 {
|
||||
return self.buffer[token.start..token.end];
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Loading…
Reference in New Issue