new .d file parser for C compilation

- wip for #2046
- clang .d output must be created with `clang -MV` switch
- implemented in Zig
- hybridized for zig stage0 and stage1
- zig test src-self-hosted/dep_tokenizer.zig

CMakeLists.txt

@@ -6726,6 +6726,7 @@ add_custom_command(
         "-Doutput-dir=${CMAKE_BINARY_DIR}"
     WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
     DEPENDS
+        "${CMAKE_SOURCE_DIR}/src-self-hosted/dep_tokenizer.zig"
         "${CMAKE_SOURCE_DIR}/src-self-hosted/stage1.zig"
         "${CMAKE_SOURCE_DIR}/src-self-hosted/translate_c.zig"
         "${CMAKE_SOURCE_DIR}/build.zig"

src-self-hosted/stage1.zig

@@ -20,6 +20,10 @@ var stderr_file: fs.File = undefined;
 var stderr: *io.OutStream(fs.File.WriteError) = undefined;
 var stdout: *io.OutStream(fs.File.WriteError) = undefined;
 
+comptime {
+    _ =  @import("dep_tokenizer.zig");
+}
+
 // ABI warning
 export fn stage2_zen(ptr: *[*]const u8, len: *usize) void {
     const info_zen = @import("main.zig").info_zen;

src/cache_hash.cpp

@@ -5,6 +5,7 @@
  * See http://opensource.org/licenses/MIT
  */
 
+#include "userland.h"
 #include "cache_hash.hpp"
 #include "all_types.hpp"
 #include "buffer.hpp"
@@ -473,71 +474,62 @@ Error cache_add_dep_file(CacheHash *ch, Buf *dep_file_path, bool verbose) {
         if (err == ErrorFileNotFound)
             return err;
         if (verbose) {
-            fprintf(stderr, "unable to read .d file: %s\n", err_str(err));
+            fprintf(stderr, "%s: unable to read .d file: %s\n", err_str(err), buf_ptr(dep_file_path));
         }
         return ErrorReadingDepFile;
     }
-    SplitIterator it = memSplit(buf_to_slice(contents), str("\r\n"));
+    auto it = stage2_DepTokenizer_init(buf_ptr(contents), buf_len(contents));
-    // skip first line
+    // skip first token: target
-    SplitIterator_next(&it);
+    {
-    for (;;) {
+        auto result = stage2_DepTokenizer_next(&it);
-        Optional<Slice<uint8_t>> opt_line = SplitIterator_next(&it);
+        switch (result.ent) {
-        if (!opt_line.is_some)
+            case stage2_DepNextResult::error:
-            break;
-        if (opt_line.value.len == 0)
-            continue;
-        // skip over indentation
-        while (opt_line.value.len != 0 && (opt_line.value.ptr[0] == ' ' || opt_line.value.ptr[0] == '\t')) {
-            opt_line.value.ptr += 1;
-            opt_line.value.len -= 1;
-        }
-        if (opt_line.value.len == 0)
-            continue;
-
-        if (opt_line.value.ptr[0] == '"') {
-            if (opt_line.value.len < 2) {
                 if (verbose) {
-                    fprintf(stderr, "unable to process invalid .d file %s: line too short\n", buf_ptr(dep_file_path));
+                    fprintf(stderr, "%s: failed processing .d file: %s\n", result.textz, buf_ptr(dep_file_path));
                 }
-                return ErrorInvalidDepFile;
+                err = ErrorInvalidDepFile;
-            }
+                goto finish;
-            opt_line.value.ptr += 1;
+            case stage2_DepNextResult::null:
-            opt_line.value.len -= 2;
+                err = ErrorNone;
-            while (opt_line.value.len != 0 && opt_line.value.ptr[opt_line.value.len] != '"') {
+                goto finish;
-                opt_line.value.len -= 1;
+            case stage2_DepNextResult::target:
-            }
+            case stage2_DepNextResult::prereq:
-            if (opt_line.value.len == 0) {
+                err = ErrorNone;
-                if (verbose) {
+                break;
-                    fprintf(stderr, "unable to process invalid .d file %s: missing double quote\n", buf_ptr(dep_file_path));
-                }
-                return ErrorInvalidDepFile;
-            }
-            Buf *filename_buf = buf_create_from_slice(opt_line.value);
-            if ((err = cache_add_file(ch, filename_buf))) {
-                if (verbose) {
-                    fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(filename_buf), err_str(err));
-                    fprintf(stderr, "when processing .d file: %s\n", buf_ptr(dep_file_path));
-                }
-                return err;
-            }
-        } else {
-            // sometimes there are multiple files on the same line; we actually need space tokenization.
-            SplitIterator line_it = memSplit(opt_line.value, str(" \t"));
-            Slice<uint8_t> filename;
-            while (SplitIterator_next(&line_it).unwrap(&filename)) {
-                Buf *filename_buf = buf_create_from_slice(filename);
-                if (buf_eql_str(filename_buf, "\\")) continue;
-                if ((err = cache_add_file(ch, filename_buf))) {
-                    if (verbose) {
-                        fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(filename_buf), err_str(err));
-                        fprintf(stderr, "when processing .d file: %s\n", buf_ptr(dep_file_path));
-                    }
-                    return err;
-                }
-            }
         }
     }
-    return ErrorNone;
+    // Process 0+ preqreqs.
+    // clang is invoked in single-source mode so we never get more targets.
+    for (;;) {
+        auto result = stage2_DepTokenizer_next(&it);
+        switch (result.ent) {
+            case stage2_DepNextResult::error:
+                if (verbose) {
+                    fprintf(stderr, "%s: failed processing .d file: %s\n", result.textz, buf_ptr(dep_file_path));
+                }
+                err = ErrorInvalidDepFile;
+                goto finish;
+            case stage2_DepNextResult::null:
+            case stage2_DepNextResult::target:
+                err = ErrorNone;
+                goto finish;
+            case stage2_DepNextResult::prereq:
+                break;
+        }
+        auto textbuf = buf_alloc();
+        buf_init_from_str(textbuf, result.textz);
+        if ((err = cache_add_file(ch, textbuf))) {
+            if (verbose) {
+                fprintf(stderr, "unable to add %s to cache: %s\n", result.textz, err_str(err));
+                fprintf(stderr, "when processing .d file: %s\n", buf_ptr(dep_file_path));
+            }
+            goto finish;
+        }
+    }
+
+    finish:
+    stage2_DepTokenizer_deinit(&it);
+    return err;
 }
 
 static Error write_manifest_file(CacheHash *ch) {

src/userland.cpp

@@ -42,3 +42,18 @@ int stage2_fmt(int argc, char **argv) {
     const char *msg = "stage0 called stage2_fmt";
     stage2_panic(msg, strlen(msg));
 }
+
+stage2_DepTokenizer stage2_DepTokenizer_init(const char *input, size_t len) {
+    const char *msg = "stage0 called stage2_DepTokenizer_init";
+    stage2_panic(msg, strlen(msg));
+}
+
+void stage2_DepTokenizer_deinit(stage2_DepTokenizer *self) {
+    const char *msg = "stage0 called stage2_DepTokenizer_deinit";
+    stage2_panic(msg, strlen(msg));
+}
+
+stage2_DepNextResult stage2_DepTokenizer_next(stage2_DepTokenizer *self) {
+    const char *msg = "stage0 called stage2_DepTokenizer_next";
+    stage2_panic(msg, strlen(msg));
+}

src/userland.h

@@ -9,6 +9,7 @@
 #define ZIG_USERLAND_H
 
 #include <stddef.h>
+#include <stdint.h>
 #include <stdio.h>
 
 #ifdef __cplusplus
@@ -118,4 +119,36 @@ ZIG_EXTERN_C ZIG_ATTRIBUTE_NORETURN void stage2_panic(const char *ptr, size_t le
 // ABI warning
 ZIG_EXTERN_C int stage2_fmt(int argc, char **argv);
 
+// ABI warning
+struct stage2_DepTokenizer {
+    void *handle;
+};
+
+// ABI warning
+struct stage2_DepNextResult {
+    enum Ent : uint8_t {
+        error,
+        null,
+        target,
+        prereq,
+    };
+
+    Ent ent;
+
+    // when ent == error --> error text
+    // when ent == null --> undefined
+    // when ent == target --> target pathname
+    // when ent == prereq --> prereq pathname
+    const char *textz;
+};
+
+// ABI warning
+ZIG_EXTERN_C stage2_DepTokenizer stage2_DepTokenizer_init(const char *input, size_t len);
+
+// ABI warning
+ZIG_EXTERN_C void stage2_DepTokenizer_deinit(stage2_DepTokenizer *self);
+
+// ABI warning
+ZIG_EXTERN_C stage2_DepNextResult stage2_DepTokenizer_next(stage2_DepTokenizer *self);
+
 #endif