std-c redo scoping, do string concatanation in parser

2020-01-19 20:41:44 +02:00 · 2020-01-19 20:41:44 +02:00 · ad327fed05
parent 28daddae81
commit ad327fed05
3 changed files with 69 additions and 63 deletions
--- a/lib/std/c/ast.zig
+++ b/lib/std/c/ast.zig
@ -576,7 +576,7 @@ pub const Node = struct {
                asterisk: ?TokenIndex,
                static: ?TokenIndex,
                qual: TypeQual,
-                // expr: *Expr,
+                expr: *Expr,
            },
        },
        rbracket: TokenIndex,
--- a/lib/std/c/parse.zig
+++ b/lib/std/c/parse.zig
@ -75,9 +75,12 @@ pub fn parse(allocator: *Allocator, source: []const u8, options: Options) !*Tree
        }
    }

+    var parse_arena = std.heap.ArenaAllocator.init(allocator);
+    defer parse_arena.deinit();
+
    var parser = Parser{
-        .symbols = Parser.SymbolList.init(allocator),
-        .arena = arena,
+        .scopes = Parser.SymbolList.init(allocator),
+        .arena = &parse_arena.allocator,
        .it = &it,
        .tree = tree,
        .options = options,
@ -93,11 +96,17 @@ const Parser = struct {
    it: *TokenIterator,
    tree: *Tree,

-    /// only used for scopes
-    symbols: SymbolList,
+    arena: *Allocator,
+    scopes: ScopeList,
    options: Options,

-    const SymbolList = std.ArrayList(Symbol);
+    const ScopeList = std.SegmentedLists(Scope);
+    const SymbolList = std.SegmentedLists(Symbol);
+
+    const Scope = struct {
+        kind: ScopeKind,
+        syms: SymbolList,
+    };

    const Symbol = struct {
        name: []const u8,
@ -111,21 +120,27 @@ const Parser = struct {
        Switch,
    };

-    fn pushScope(parser: *Parser, kind: ScopeKind) usize {
-        return parser.symbols.len;
+    fn pushScope(parser: *Parser, kind: ScopeKind) !void {
+        const new = try parser.scopes.addOne();
+        new.* = .{
+            .kind = kind,
+            .syms = SymbolList.init(parser.arena),
+        };
    }

    fn popScope(parser: *Parser, len: usize) void {
-        parser.symbols.resize(len) catch unreachable;
+        _ = parser.scopes.pop();
    }

-    fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Type {
+    fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Symbol {
        const name = parser.tree.tokenSlice(tok);
-        const syms = parser.symbols.toSliceConst();
-        var i = syms.len;
-        while (i > 0) : (i -= 1) {
-            if (mem.eql(u8, name, syms[i].name)) {
-                return syms[i].ty;
+        var scope_it = parser.scopes.iterator(parser.scopes.len);
+        while (scope_it.prev()) |scope| {
+            var sym_it = scope.syms.iterator(scope.syms.len);
+            while (sym_it.prev()) |sym| {
+                if (mem.eql(u8, sym.name, name)) {
+                    return sym;
+                }
            }
        }
        return null;
@ -137,8 +152,8 @@ const Parser = struct {

    /// Root <- ExternalDeclaration* eof
    fn root(parser: *Parser) Allocator.Error!*Node.Root {
-        const scope = parser.pushScope(.Root);
-        defer parser.popScope(scope);
+        try parser.pushScope(.Root);
+        defer parser.popScope();
        const node = try parser.arena.create(Node.Root);
        node.* = .{
            .decls = Node.Root.DeclList.init(parser.arena),
@ -782,8 +797,8 @@ const Parser = struct {
                .ty = ty,
            });
        if (parser.eatToken(.LBrace)) |lbrace| {
-            const scope = parser.pushScope(.Block);
-            defer parser.popScope(scope);
+            try parser.pushScope(.Block);
+            defer parser.popScope();
            var fields = Node.RecordType.FieldList.init(parser.arena);
            while (true) {
                if (parser.eatToken(.RBrace)) |rbrace| {
@ -996,15 +1011,14 @@ const Parser = struct {
    fn assignmentExpr(parser: *Parser) !*Node {}

    /// ConstExpr <- ConditionalExpr
-    fn constExpr(parser: *Parser) Error!*Node {
+    fn constExpr(parser: *Parser) Error!?*Expr {
        const start = parser.it.index;
        const expression = try parser.conditionalExpr();
-        // TODO
-        // if (expression == nullor expression.?.value == null)
-        //     return parser.err(.{
-        //         .ConsExpr = start,
-        //     });
-        return expression.?;
+        if (expression != null and expression.?.value == .None)
+            return parser.err(.{
+                .ConsExpr = start,
+            });
+        return expression;
    }

    /// ConditionalExpr <- LogicalOrExpr (QUESTIONMARK Expr COLON ConditionalExpr)?
@ -1085,8 +1099,8 @@ const Parser = struct {
    /// CompoundStmt <- LBRACE (Declaration / Stmt)* RBRACE
    fn compoundStmt(parser: *Parser) Error!?*Node {
        const lbrace = parser.eatToken(.LBrace) orelse return null;
-        const scope = parser.pushScope(.Block);
-        defer parser.popScope(scope);
+        try parser.pushScope(.Block);
+        defer parser.popScope();
        const body_node = try parser.arena.create(Node.CompoundStmt);
        body_node.* = .{
            .lbrace = lbrace,
@ -1142,8 +1156,8 @@ const Parser = struct {
            return &node.base;
        }
        if (parser.eatToken(.Keyword_while)) |tok| {
-            const scope = parser.pushScope(.Loop);
-            defer parser.popScope(scope);
+            try parser.pushScope(.Loop);
+            defer parser.popScope();
            _ = try parser.expectToken(.LParen);
            const cond = (try parser.expr()) orelse return parser.err(.{
                .ExpectedExpr = .{ .token = parser.it.index },
@ -1160,8 +1174,8 @@ const Parser = struct {
            return &node.base;
        }
        if (parser.eatToken(.Keyword_do)) |tok| {
-            const scope = parser.pushScope(.Loop);
-            defer parser.popScope(scope);
+            try parser.pushScope(.Loop);
+            defer parser.popScope();
            const body = try parser.stmt();
            _ = try parser.expectToken(.LParen);
            const cond = (try parser.expr()) orelse return parser.err(.{
@ -1179,8 +1193,8 @@ const Parser = struct {
            return &node.base;
        }
        if (parser.eatToken(.Keyword_for)) |tok| {
-            const scope = parser.pushScope(.Loop);
-            defer parser.popScope(scope);
+            try parser.pushScope(.Loop);
+            defer parser.popScope();
            _ = try parser.expectToken(.LParen);
            const init = if (try parser.declaration()) |decl| blk:{
                // TODO disallow storage class other than auto and register
@ -1203,8 +1217,8 @@ const Parser = struct {
            return &node.base;
        }
        if (parser.eatToken(.Keyword_switch)) |tok| {
-            const scope = parser.pushScope(.Switch);
-            defer parser.popScope(scope);
+            try parser.pushScope(.Switch);
+            defer parser.popScope();
            _ = try parser.expectToken(.LParen);
            const switch_expr = try parser.exprStmt();
            const rparen = try parser.expectToken(.RParen);
--- a/lib/std/c/tokenizer.zig
+++ b/lib/std/c/tokenizer.zig
@ -401,7 +401,6 @@ pub const Tokenizer = struct {
            U,
            L,
            StringLiteral,
-            AfterStringLiteral,
            CharLiteralStart,
            CharLiteral,
            EscapeSequence,
@ -617,7 +616,7 @@ pub const Tokenizer = struct {
                },
                .BackSlash => switch (c) {
                    '\n' => {
-                        state = if (string) .AfterStringLiteral else .Start;
+                        state = .Start;
                    },
                    '\r' => {
                        state = .BackSlashCr;
@ -632,7 +631,7 @@ pub const Tokenizer = struct {
                },
                .BackSlashCr => switch (c) {
                    '\n' => {
-                        state = if (string) .AfterStringLiteral else .Start;
+                        state =  .Start;
                    },
                    else => {
                        result.id = .Invalid;
@ -696,7 +695,8 @@ pub const Tokenizer = struct {
                        state = .EscapeSequence;
                    },
                    '"' => {
-                        state = .AfterStringLiteral;
+                        self.index += 1;
+                        break;
                    },
                    '\n', '\r' => {
                        result.id = .Invalid;
@ -704,22 +704,6 @@ pub const Tokenizer = struct {
                    },
                    else => {},
                },
-                .AfterStringLiteral => switch (c) {
-                    '"' => {
-                        state = .StringLiteral;
-                    },
-                    '\\' => {
-                        state = .BackSlash;
-                    },
-                    '\n', '\r' => {
-                        if (self.pp_directive)
-                            break;
-                    },
-                    '\t', '\x0B', '\x0C', ' ' => {},
-                    else => {
-                        break;
-                    },
-                },
                .CharLiteralStart => switch (c) {
                    '\\' => {
                        string = false;
@ -1255,7 +1239,7 @@ pub const Tokenizer = struct {
            }
        } else if (self.index == self.source.buffer.len) {
            switch (state) {
-                .AfterStringLiteral, .Start => {},
+                .Start => {},
                .u, .u8, .U, .L, .Identifier => {
                    result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
                },
@ -1322,7 +1306,7 @@ pub const Tokenizer = struct {

 test "operators" {
    expectTokens(
-        \\ ! != | || |= = ==
+        \\ ! != | || |= = ==
        \\ ( ) { } [ ] . .. ...
        \\ ^ ^= + ++ += - -- -=
        \\ * *= % %= -> : ; / /=
@ -1505,24 +1489,27 @@ test "line continuation" {
        .Identifier,
        .Nl,
        .{ .StringLiteral = .None },
+        .Nl,
        .Hash,
        .Keyword_define,
        .{ .StringLiteral = .None },
        .Nl,
        .{ .StringLiteral = .None },
+        .Nl,
        .Hash,
        .Keyword_define,
        .{ .StringLiteral = .None },
+        .{ .StringLiteral = .None },
    });
 }

 test "string prefix" {
    expectTokens(
-        \\"foo" "bar"
-        \\u"foo" "bar"
-        \\u8"foo" "bar"
-        \\U"foo" "bar"
-        \\L"foo" "bar"
+        \\"foo"
+        \\u"foo"
+        \\u8"foo"
+        \\U"foo"
+        \\L"foo"
        \\'foo'
        \\u'foo'
        \\U'foo'
@ -1530,10 +1517,15 @@ test "string prefix" {
        \\
    , &[_]Token.Id{
        .{ .StringLiteral = .None },
+        .Nl,
        .{ .StringLiteral = .Utf16 },
+        .Nl,
        .{ .StringLiteral = .Utf8 },
+        .Nl,
        .{ .StringLiteral = .Utf32 },
+        .Nl,
        .{ .StringLiteral = .Wide },
+        .Nl,
        .{ .CharLiteral = .None },
        .Nl,
        .{ .CharLiteral = .Utf16 },