Ekdohibs/zig
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

zig fmt: support sentinel-terminated pointer syntax

Browse Source
This commit is contained in:
Andrew Kelley 2019-11-23 21:58:28 -05:00
parent f25182f46d
commit 00878a15d7
No known key found for this signature in database
GPG Key ID: 7C5F548F728501A9
5 changed files with 203 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
lib/std/zig/ast.zig
View File

@ -137,7 +137,6 @@ pub const Error = union(enum) {
ExpectedCallOrFnProto: ExpectedCallOrFnProto,
ExpectedSliceOrRBracket: ExpectedSliceOrRBracket,
ExtraAlignQualifier: ExtraAlignQualifier,
ExtraNullQualifier: ExtraNullQualifier,
ExtraConstQualifier: ExtraConstQualifier,
ExtraVolatileQualifier: ExtraVolatileQualifier,
ExtraAllowZeroQualifier: ExtraAllowZeroQualifier,
@ -185,7 +184,6 @@ pub const Error = union(enum) {
.ExpectedCallOrFnProto => |*x| return x.render(tokens, stream),
.ExpectedSliceOrRBracket => |*x| return x.render(tokens, stream),
.ExtraAlignQualifier => |*x| return x.render(tokens, stream),
.ExtraNullQualifier => |*x| return x.render(tokens, stream),
.ExtraConstQualifier => |*x| return x.render(tokens, stream),
.ExtraVolatileQualifier => |*x| return x.render(tokens, stream),
.ExtraAllowZeroQualifier => |*x| return x.render(tokens, stream),
@ -235,7 +233,6 @@ pub const Error = union(enum) {
.ExpectedCallOrFnProto => |x| return x.node.firstToken(),
.ExpectedSliceOrRBracket => |x| return x.token,
.ExtraAlignQualifier => |x| return x.token,
.ExtraNullQualifier => |x| return x.token,
.ExtraConstQualifier => |x| return x.token,
.ExtraVolatileQualifier => |x| return x.token,
.ExtraAllowZeroQualifier => |x| return x.token,
@ -296,7 +293,6 @@ pub const Error = union(enum) {
pub const ExpectedPubItem = SimpleError("Expected function or variable declaration after pub");
pub const UnattachedDocComment = SimpleError("Unattached documentation comment");
pub const ExtraAlignQualifier = SimpleError("Extra align qualifier");
pub const ExtraNullQualifier = SimpleError("Extra null qualifier");
pub const ExtraConstQualifier = SimpleError("Extra const qualifier");
pub const ExtraVolatileQualifier = SimpleError("Extra volatile qualifier");
pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier");
@ -1535,7 +1531,7 @@ pub const Node = struct {
};
pub const PrefixOp = struct {
base: Node,
base: Node = Node{ .id = .PrefixOp },
op_token: TokenIndex,
op: Op,
rhs: *Node,
@ -1558,15 +1554,15 @@ pub const Node = struct {
pub const ArrayInfo = struct {
len_expr: *Node,
null_token: ?TokenIndex,
sentinel: ?*Node,
};
pub const PtrInfo = struct {
allowzero_token: ?TokenIndex,
align_info: ?Align,
const_token: ?TokenIndex,
volatile_token: ?TokenIndex,
null_token: ?TokenIndex,
allowzero_token: ?TokenIndex = null,
align_info: ?Align = null,
const_token: ?TokenIndex = null,
volatile_token: ?TokenIndex = null,
sentinel: ?*Node = null,
pub const Align = struct {
node: *Node,
@ -1585,6 +1581,11 @@ pub const Node = struct {
switch (self.op) {
// TODO https://github.com/ziglang/zig/issues/1107
Op.SliceType => |addr_of_info| {
if (addr_of_info.sentinel) |sentinel| {
if (i < 1) return sentinel;
i -= 1;
}
if (addr_of_info.align_info) |align_info| {
if (i < 1) return align_info.node;
i -= 1;
@ -1601,6 +1602,10 @@ pub const Node = struct {
Op.ArrayType => |array_info| {
if (i < 1) return array_info.len_expr;
i -= 1;
if (array_info.sentinel) |sentinel| {
if (i < 1) return sentinel;
i -= 1;
}
},
Op.AddressOf,

238
lib/std/zig/parse.zig
View File

@ -2246,22 +2246,70 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
return &node.base;
}
if (try parsePtrTypeStart(arena, it, tree)) |node| {
// If the token encountered was **, there will be two nodes instead of one.
// The attributes should be applied to the rightmost operator.
const prefix_op = node.cast(Node.PrefixOp).?;
var ptr_info = if (tree.tokens.at(prefix_op.op_token).id == .AsteriskAsterisk)
&prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType
else
&prefix_op.op.PtrType;
while (true) {
if (eatToken(it, .Keyword_align)) |align_token| {
const lparen = try expectToken(it, tree, .LParen);
const expr_node = try expectNode(arena, it, tree, parseExpr, AstError{
.ExpectedExpr = AstError.ExpectedExpr{ .token = it.index },
});
// Optional bit range
const bit_range = if (eatToken(it, .Colon)) |_| bit_range_value: {
const range_start = try expectNode(arena, it, tree, parseIntegerLiteral, AstError{
.ExpectedIntegerLiteral = AstError.ExpectedIntegerLiteral{ .token = it.index },
});
_ = try expectToken(it, tree, .Colon);
const range_end = try expectNode(arena, it, tree, parseIntegerLiteral, AstError{
.ExpectedIntegerLiteral = AstError.ExpectedIntegerLiteral{ .token = it.index },
});
break :bit_range_value Node.PrefixOp.PtrInfo.Align.BitRange{
.start = range_start,
.end = range_end,
};
} else null;
_ = try expectToken(it, tree, .RParen);
ptr_info.align_info = Node.PrefixOp.PtrInfo.Align{
.node = expr_node,
.bit_range = bit_range,
};
continue;
}
if (eatToken(it, .Keyword_const)) |const_token| {
ptr_info.const_token = const_token;
continue;
}
if (eatToken(it, .Keyword_volatile)) |volatile_token| {
ptr_info.volatile_token = volatile_token;
continue;
}
if (eatToken(it, .Keyword_allowzero)) |allowzero_token| {
ptr_info.allowzero_token = allowzero_token;
continue;
}
break;
}
return node;
}
if (try parseArrayTypeStart(arena, it, tree)) |node| {
switch (node.cast(Node.PrefixOp).?.op) {
.ArrayType => {},
.SliceType => |*slice_type| {
// Collect pointer qualifiers in any order, but disallow duplicates
while (true) {
if (eatToken(it, .Keyword_null)) |null_token| {
if (slice_type.null_token != null) {
try tree.errors.push(AstError{
.ExtraNullQualifier = AstError.ExtraNullQualifier{ .token = it.index },
});
return error.ParseError;
}
slice_type.null_token = null_token;
continue;
}
if (try parseByteAlign(arena, it, tree)) |align_expr| {
if (slice_type.align_info != null) {
try tree.errors.push(AstError{
@ -2313,68 +2361,6 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
return node;
}
if (try parsePtrTypeStart(arena, it, tree)) |node| {
// If the token encountered was **, there will be two nodes instead of one.
// The attributes should be applied to the rightmost operator.
const prefix_op = node.cast(Node.PrefixOp).?;
var ptr_info = if (tree.tokens.at(prefix_op.op_token).id == .AsteriskAsterisk)
&prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType
else
&prefix_op.op.PtrType;
while (true) {
if (eatToken(it, .Keyword_null)) |null_token| {
ptr_info.null_token = null_token;
continue;
}
if (eatToken(it, .Keyword_align)) |align_token| {
const lparen = try expectToken(it, tree, .LParen);
const expr_node = try expectNode(arena, it, tree, parseExpr, AstError{
.ExpectedExpr = AstError.ExpectedExpr{ .token = it.index },
});
// Optional bit range
const bit_range = if (eatToken(it, .Colon)) |_| bit_range_value: {
const range_start = try expectNode(arena, it, tree, parseIntegerLiteral, AstError{
.ExpectedIntegerLiteral = AstError.ExpectedIntegerLiteral{ .token = it.index },
});
_ = try expectToken(it, tree, .Colon);
const range_end = try expectNode(arena, it, tree, parseIntegerLiteral, AstError{
.ExpectedIntegerLiteral = AstError.ExpectedIntegerLiteral{ .token = it.index },
});
break :bit_range_value Node.PrefixOp.PtrInfo.Align.BitRange{
.start = range_start,
.end = range_end,
};
} else null;
_ = try expectToken(it, tree, .RParen);
ptr_info.align_info = Node.PrefixOp.PtrInfo.Align{
.node = expr_node,
.bit_range = bit_range,
};
continue;
}
if (eatToken(it, .Keyword_const)) |const_token| {
ptr_info.const_token = const_token;
continue;
}
if (eatToken(it, .Keyword_volatile)) |volatile_token| {
ptr_info.volatile_token = volatile_token;
continue;
}
if (eatToken(it, .Keyword_allowzero)) |allowzero_token| {
ptr_info.allowzero_token = allowzero_token;
continue;
}
break;
}
return node;
}
return null;
}
@ -2473,14 +2459,19 @@ const AnnotatedParamList = struct {
fn parseArrayTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
const lbracket = eatToken(it, .LBracket) orelse return null;
const expr = try parseExpr(arena, it, tree);
const sentinel = if (eatToken(it, .Colon)) |_|
try expectNode(arena, it, tree, parseExpr, AstError{
.ExpectedExpr = .{ .token = it.index },
})
else
null;
const rbracket = try expectToken(it, tree, .RBracket);
const null_token = eatToken(it, .Keyword_null);
const op = if (expr) |len_expr|
Node.PrefixOp.Op{
.ArrayType = .{
.len_expr = len_expr,
.null_token = null_token,
.sentinel = sentinel,
},
}
else
@ -2490,7 +2481,7 @@ fn parseArrayTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No
.align_info = null,
.const_token = null,
.volatile_token = null,
.null_token = null,
.sentinel = sentinel,
},
};
@ -2510,49 +2501,76 @@ fn parseArrayTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No
/// / PTRUNKNOWN
/// / PTRC
fn parsePtrTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
const token = eatAnnotatedToken(it, .Asterisk) orelse
eatAnnotatedToken(it, .AsteriskAsterisk) orelse
eatAnnotatedToken(it, .BracketStarBracket) orelse
eatAnnotatedToken(it, .BracketStarCBracket) orelse
return null;
if (eatToken(it, .Asterisk)) |asterisk| {
const sentinel = if (eatToken(it, .Colon)) |_|
try expectNode(arena, it, tree, parseExpr, AstError{
.ExpectedExpr = .{ .token = it.index },
})
else
null;
const node = try arena.create(Node.PrefixOp);
node.* = .{
.op_token = asterisk,
.op = .{ .PtrType = .{ .sentinel = sentinel } },
.rhs = undefined, // set by caller
};
return &node.base;
}
const node = try arena.create(Node.PrefixOp);
node.* = Node.PrefixOp{
.base = Node{ .id = .PrefixOp },
.op_token = token.index,
.op = Node.PrefixOp.Op{
.PtrType = Node.PrefixOp.PtrInfo{
.allowzero_token = null,
.align_info = null,
.const_token = null,
.volatile_token = null,
.null_token = null,
},
},
.rhs = undefined, // set by caller
};
if (eatToken(it, .AsteriskAsterisk)) |double_asterisk| {
const node = try arena.create(Node.PrefixOp);
node.* = Node.PrefixOp{
.op_token = double_asterisk,
.op = Node.PrefixOp.Op{ .PtrType = .{} },
.rhs = undefined, // set by caller
};
// Special case for **, which is its own token
if (token.ptr.id == .AsteriskAsterisk) {
// Special case for **, which is its own token
const child = try arena.create(Node.PrefixOp);
child.* = Node.PrefixOp{
.base = Node{ .id = .PrefixOp },
.op_token = token.index,
.op = Node.PrefixOp.Op{
.PtrType = Node.PrefixOp.PtrInfo{
.allowzero_token = null,
.align_info = null,
.const_token = null,
.volatile_token = null,
.null_token = null,
},
},
.op_token = double_asterisk,
.op = Node.PrefixOp.Op{ .PtrType = .{} },
.rhs = undefined, // set by caller
};
node.rhs = &child.base;
}
return &node.base;
return &node.base;
}
if (eatToken(it, .LBracket)) |lbracket| {
const asterisk = eatToken(it, .Asterisk) orelse {
putBackToken(it, lbracket);
return null;
};
if (eatToken(it, .Identifier)) |ident| {
if (!std.mem.eql(u8, tree.tokenSlice(ident), "c")) {
putBackToken(it, ident);
} else {
_ = try expectToken(it, tree, .RBracket);
const node = try arena.create(Node.PrefixOp);
node.* = .{
.op_token = ident,
.op = .{ .PtrType = .{} },
.rhs = undefined, // set by caller
};
return &node.base;
}
}
const sentinel = if (eatToken(it, .Colon)) |_|
try expectNode(arena, it, tree, parseExpr, AstError{
.ExpectedExpr = .{ .token = it.index },
})
else
null;
_ = try expectToken(it, tree, .RBracket);
const node = try arena.create(Node.PrefixOp);
node.* = .{
.op_token = lbracket,
.op = .{ .PtrType = .{ .sentinel = sentinel } },
.rhs = undefined, // set by caller
};
return &node.base;
}
return null;
}
/// ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE

40
lib/std/zig/render.zig
View File

@ -418,13 +418,26 @@ fn renderExpression(
switch (prefix_op_node.op) {
ast.Node.PrefixOp.Op.PtrType => |ptr_info| {
const star_offset = switch (tree.tokens.at(prefix_op_node.op_token).id) {
Token.Id.AsteriskAsterisk => @as(usize, 1),
else => @as(usize, 0),
};
try renderTokenOffset(tree, stream, prefix_op_node.op_token, indent, start_col, Space.None, star_offset); // *
if (ptr_info.null_token) |null_token| {
try renderToken(tree, stream, null_token, indent, start_col, Space.Space); // null
const op_tok_id = tree.tokens.at(prefix_op_node.op_token).id;
switch (op_tok_id) {
.Asterisk, .AsteriskAsterisk => try stream.writeByte('*'),
.Identifier => try stream.write("[*c]"),
.LBracket => try stream.write("[*"),
else => unreachable,
}
if (ptr_info.sentinel) |sentinel| {
const colon_token = tree.prevToken(sentinel.firstToken());
try renderToken(tree, stream, colon_token, indent, start_col, Space.None); // :
const sentinel_space = switch (op_tok_id) {
.LBracket => Space.None,
else => Space.Space,
};
try renderExpression(allocator, stream, tree, indent, start_col, sentinel, sentinel_space);
}
switch (op_tok_id) {
.Asterisk, .AsteriskAsterisk, .Identifier => {},
.LBracket => try stream.writeByte(']'),
else => unreachable,
}
if (ptr_info.allowzero_token) |allowzero_token| {
try renderToken(tree, stream, allowzero_token, indent, start_col, Space.Space); // allowzero
@ -504,7 +517,10 @@ fn renderExpression(
ast.Node.PrefixOp.Op.ArrayType => |array_info| {
const lbracket = prefix_op_node.op_token;
const rbracket = tree.nextToken(array_info.len_expr.lastToken());
const rbracket = tree.nextToken(if (array_info.sentinel) |sentinel|
sentinel.lastToken()
else
array_info.len_expr.lastToken());
try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [
@ -519,10 +535,12 @@ fn renderExpression(
if (ends_with_comment or starts_with_comment) {
try stream.writeByteNTimes(' ', indent);
}
try renderToken(tree, stream, rbracket, indent, start_col, Space.None); // ]
if (array_info.null_token) |null_token| {
try renderToken(tree, stream, null_token, indent, start_col, Space.Space); // null
if (array_info.sentinel) |sentinel| {
const colon_token = tree.prevToken(sentinel.firstToken());
try renderToken(tree, stream, colon_token, indent, start_col, Space.None); // :
try renderExpression(allocator, stream, tree, indent, start_col, sentinel, Space.None);
}
try renderToken(tree, stream, rbracket, indent, start_col, Space.None); // ]
},
ast.Node.PrefixOp.Op.BitNot,
ast.Node.PrefixOp.Op.BoolNot,

62
lib/std/zig/tokenizer.zig
View File

@ -143,8 +143,6 @@ pub const Token = struct {
LineComment,
DocComment,
ContainerDocComment,
BracketStarBracket,
BracketStarCBracket,
ShebangLine,
Keyword_align,
Keyword_allowzero,
@ -269,8 +267,6 @@ pub const Token = struct {
.AngleBracketAngleBracketRight => ">>",
.AngleBracketAngleBracketRightEqual => ">>=",
.Tilde => "~",
.BracketStarBracket => "[*]",
.BracketStarCBracket => "[*c]",
.Keyword_align => "align",
.Keyword_allowzero => "allowzero",
.Keyword_and => "and",
@ -400,9 +396,6 @@ pub const Tokenizer = struct {
Period,
Period2,
SawAtSign,
LBracket,
LBracketStar,
LBracketStarC,
};
pub fn next(self: *Tokenizer) Token {
@ -460,7 +453,9 @@ pub const Tokenizer = struct {
break;
},
'[' => {
state = State.LBracket;
result.id = .LBracket;
self.index += 1;
break;
},
']' => {
result.id = Token.Id.RBracket;
@ -564,43 +559,6 @@ pub const Tokenizer = struct {
},
},
State.LBracket => switch (c) {
'*' => {
state = State.LBracketStar;
},
else => {
result.id = Token.Id.LBracket;
break;
},
},
State.LBracketStar => switch (c) {
'c' => {
state = State.LBracketStarC;
},
']' => {
result.id = Token.Id.BracketStarBracket;
self.index += 1;
break;
},
else => {
result.id = Token.Id.Invalid;
break;
},
},
State.LBracketStarC => switch (c) {
']' => {
result.id = Token.Id.BracketStarCBracket;
self.index += 1;
break;
},
else => {
result.id = Token.Id.Invalid;
break;
},
},
State.Ampersand => switch (c) {
'&' => {
result.id = Token.Id.Invalid_ampersands;
@ -1227,8 +1185,6 @@ pub const Tokenizer = struct {
State.CharLiteralEnd,
State.CharLiteralUnicode,
State.StringLiteralBackslash,
State.LBracketStar,
State.LBracketStarC,
=> {
result.id = Token.Id.Invalid;
},
@ -1245,9 +1201,6 @@ pub const Tokenizer = struct {
State.Slash => {
result.id = Token.Id.Slash;
},
State.LBracket => {
result.id = Token.Id.LBracket;
},
State.Zero => {
result.id = Token.Id.IntegerLiteral;
},
@ -1368,9 +1321,14 @@ test "tokenizer - unknown length pointer and then c pointer" {
\\[*]u8
\\[*c]u8
, [_]Token.Id{
Token.Id.BracketStarBracket,
Token.Id.LBracket,
Token.Id.Asterisk,
Token.Id.RBracket,
Token.Id.Identifier,
Token.Id.BracketStarCBracket,
Token.Id.LBracket,
Token.Id.Asterisk,
Token.Id.Identifier,
Token.Id.RBracket,
Token.Id.Identifier,
});
}

29
src-self-hosted/translate_c.zig
View File

@ -1105,19 +1105,32 @@ fn transCreateNodePtrType(
is_const: bool,
is_volatile: bool,
op_tok_id: std.zig.Token.Id,
bytes: []const u8,
) !*ast.Node.PrefixOp {
const node = try c.a().create(ast.Node.PrefixOp);
const op_token = switch (op_tok_id) {
.LBracket => blk: {
const lbracket = try appendToken(c, .LBracket, "[");
_ = try appendToken(c, .Asterisk, "*");
_ = try appendToken(c, .RBracket, "]");
break :blk lbracket;
},
.Identifier => blk: {
_ = try appendToken(c, .LBracket, "[");
_ = try appendToken(c, .Asterisk, "*");
const c_ident = try appendToken(c, .Identifier, "c");
_ = try appendToken(c, .RBracket, "]");
break :blk c_ident;
},
.Asterisk => try appendToken(c, .Asterisk, "*"),
else => unreachable,
};
node.* = ast.Node.PrefixOp{
.base = ast.Node{ .id = .PrefixOp },
.op_token = try appendToken(c, op_tok_id, bytes),
.op_token = op_token,
.op = ast.Node.PrefixOp.Op{
.PtrType = ast.Node.PrefixOp.PtrInfo{
.allowzero_token = null,
.align_info = null,
.PtrType = .{
.const_token = if (is_const) try appendToken(c, .Keyword_const, "const") else null,
.volatile_token = if (is_volatile) try appendToken(c, .Keyword_volatile, "volatile") else null,
.null_token = null,
},
},
.rhs = undefined, // translate and set afterward
@ -1226,7 +1239,6 @@ fn transType(rp: RestorePoint, ty: *const ZigClangType, source_loc: ZigClangSour
ZigClangQualType_isConstQualified(child_qt),
ZigClangQualType_isVolatileQualified(child_qt),
.Asterisk,
"*",
);
optional_node.rhs = &pointer_node.base;
pointer_node.rhs = try transQualType(rp, child_qt, source_loc);
@ -1236,8 +1248,7 @@ fn transType(rp: RestorePoint, ty: *const ZigClangType, source_loc: ZigClangSour
rp.c,
ZigClangQualType_isConstQualified(child_qt),
ZigClangQualType_isVolatileQualified(child_qt),
.BracketStarCBracket,
"[*c]",
.Identifier,
);
pointer_node.rhs = try transQualType(rp, child_qt, source_loc);
return &pointer_node.base;