Merge pull request #6878 from frmdstryr/multiline-string-comments

Support comments in multiline string literals

src/stage1/parser.cpp

@@ -493,6 +493,33 @@ static AstNode *ast_parse_root(ParseContext *pc) {
     return node;
 }
 
+static Token *ast_parse_multiline_string_literal(ParseContext *pc, Buf *buf) {
+    Token *first_str_token = nullptr;
+    Token *str_token = nullptr;
+    while ((str_token = eat_token_if(pc, TokenIdMultilineStringLiteral))) {
+        if (first_str_token == nullptr) {
+            first_str_token = str_token;
+        }
+        if (buf->list.length == 0) {
+            buf_resize(buf, 0);
+        }
+        buf_append_buf(buf, token_buf(str_token));
+
+        // Ignore inline comments
+        size_t cur_token = pc->current_token;
+        while (eat_token_if(pc, TokenIdDocComment));
+
+        // Lookahead to see if there's another multilne string literal,
+        // if not, we have to revert back to before the doc comment
+        if (peek_token(pc)->id != TokenIdMultilineStringLiteral) {
+            pc->current_token = cur_token;
+        } else {
+            buf_append_char(buf, '\n'); // Add a newline between comments
+        }
+    }
+    return first_str_token;
+}
+
 static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) {
     Token *first_doc_token = nullptr;
     Token *doc_token = nullptr;
@@ -605,7 +632,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
                 case ContainerFieldStateSeen:
                     break;
                 case ContainerFieldStateEnd:
-                    ast_error(pc, first_token, "declarations are not allowed between container fields");                    
+                    ast_error(pc, first_token, "declarations are not allowed between container fields");
             }
 
             assert(container_field->type == NodeTypeStructField);
@@ -1754,12 +1781,21 @@ static AstNode *ast_parse_primary_type_expr(ParseContext *pc) {
     if (unreachable != nullptr)
         return ast_create_node(pc, NodeTypeUnreachable, unreachable);
 
+
+    Buf *string_buf;
     Token *string_lit = eat_token_if(pc, TokenIdStringLiteral);
-    if (string_lit == nullptr)
-        string_lit = eat_token_if(pc, TokenIdMultilineStringLiteral);
+    if (string_lit != nullptr) {
+      string_buf = token_buf(string_lit);
+    } else {
+        Buf multiline_string_buf = BUF_INIT;
+        string_lit = ast_parse_multiline_string_literal(pc, &multiline_string_buf);
+        if (string_lit != nullptr) {
+            string_buf = buf_create_from_buf(&multiline_string_buf);
+        }
+    }
     if (string_lit != nullptr) {
         AstNode *res = ast_create_node(pc, NodeTypeStringLiteral, string_lit);
-        res->data.string_literal.buf = token_buf(string_lit);
+        res->data.string_literal.buf = string_buf;
         return res;
     }
 

test/stage1/behavior/misc.zig

@@ -166,6 +166,48 @@ test "multiline string" {
     expect(mem.eql(u8, s1, s2));
 }
 
+test "multiline string comments at start" {
+    const s1 =
+        //\\one
+        \\two)
+        \\three
+    ;
+    const s2 = "two)\nthree";
+    expect(mem.eql(u8, s1, s2));
+}
+
+test "multiline string comments at end" {
+    const s1 =
+        \\one
+        \\two)
+        //\\three
+    ;
+    const s2 = "one\ntwo)";
+    expect(mem.eql(u8, s1, s2));
+}
+
+test "multiline string comments in middle" {
+    const s1 =
+        \\one
+        //\\two)
+        \\three
+    ;
+    const s2 = "one\nthree";
+    expect(mem.eql(u8, s1, s2));
+}
+
+test "multiline string comments at multiple places" {
+    const s1 =
+        \\one
+        //\\two
+        \\three
+        //\\four
+        \\five
+    ;
+    const s2 = "one\nthree\nfive";
+    expect(mem.eql(u8, s1, s2));
+}
+
 test "multiline C string" {
     const s1 =
         \\one