initial support of response files

See #4833

It doesn't support comments or quotes yet.

src-self-hosted/stage2.zig

@@ -114,6 +114,7 @@ const Error = extern enum {
     InvalidAbiVersion,
     InvalidOperatingSystemVersion,
     UnknownClangOption,
+    NestedResponseFile,
 };
 
 const FILE = std.c.FILE;
@@ -1259,6 +1260,7 @@ pub const ClangArgIterator = extern struct {
     argv_ptr: [*]const [*:0]const u8,
     argv_len: usize,
     next_index: usize,
+    root_args: ?*Args,
 
     // ABI warning
     pub const ZigEquivalent = extern enum {
@@ -1289,7 +1291,13 @@ pub const ClangArgIterator = extern struct {
         no_rtti,
     };
 
-    fn init(argv: []const [*:0]const u8) ClangArgIterator {
+    const Args = struct {
+        next_index: usize,
+        argv_ptr: [*]const [*:0]const u8,
+        argv_len: usize,
+    };
+
+    pub fn init(argv: []const [*:0]const u8) ClangArgIterator {
         return .{
             .next_index = 2, // `zig cc foo` this points to `foo`
             .has_next = argv.len > 2,
@@ -1300,22 +1308,74 @@ pub const ClangArgIterator = extern struct {
             .other_args_len = undefined,
             .argv_ptr = argv.ptr,
             .argv_len = argv.len,
+            .root_args = null,
         };
     }
 
-    fn next(self: *ClangArgIterator) !void {
+    pub fn next(self: *ClangArgIterator) !void {
         assert(self.has_next);
         assert(self.next_index < self.argv_len);
         // In this state we know that the parameter we are looking at is a root parameter
         // rather than an argument to a parameter.
         self.other_args_ptr = self.argv_ptr + self.next_index;
         self.other_args_len = 1; // We adjust this value below when necessary.
-        const arg = mem.span(self.argv_ptr[self.next_index]);
-        self.next_index += 1;
-        defer {
-            if (self.next_index >= self.argv_len) self.has_next = false;
-        }
+        var arg = mem.span(self.argv_ptr[self.next_index]);
+        self.incrementArgIndex();
 
+        if (mem.startsWith(u8, arg, "@")) {
+            if (self.root_args != null) return error.NestedResponseFile;
+
+            // This is a "compiler response file". We must parse the file and treat its
+            // contents as command line parameters.
+            const allocator = std.heap.c_allocator;
+            const max_bytes = 10 * 1024 * 1024; // 10 MiB of command line arguments is a reasonable limit
+            const resp_file_path = arg[1..];
+            const resp_contents = fs.cwd().readFileAlloc(allocator, resp_file_path, max_bytes) catch |err| {
+                std.debug.warn("unable to read response file '{}': {}\n", .{ resp_file_path, @errorName(err) });
+                process.exit(1);
+            };
+            defer allocator.free(resp_contents);
+            // TODO is there a specification for this file format? Let's find it and make this parsing more robust
+            // at the very least I'm guessing this needs to handle quotes and `#` comments.
+            var it = mem.tokenize(resp_contents, " \t\r\n");
+            var resp_arg_list = std.ArrayList([*:0]const u8).init(allocator);
+            defer resp_arg_list.deinit();
+            {
+                errdefer {
+                    for (resp_arg_list.span()) |item| {
+                        allocator.free(mem.span(item));
+                    }
+                }
+                while (it.next()) |token| {
+                    const dupe_token = try mem.dupeZ(allocator, u8, token);
+                    errdefer allocator.free(dupe_token);
+                    try resp_arg_list.append(dupe_token);
+                }
+                const args = try allocator.create(Args);
+                errdefer allocator.destroy(args);
+                args.* = .{
+                    .next_index = self.next_index,
+                    .argv_ptr = self.argv_ptr,
+                    .argv_len = self.argv_len,
+                };
+                self.root_args = args;
+            }
+            const resp_arg_slice = resp_arg_list.toOwnedSlice();
+            self.next_index = 0;
+            self.argv_ptr = resp_arg_slice.ptr;
+            self.argv_len = resp_arg_slice.len;
+
+            if (resp_arg_slice.len == 0) {
+                self.resolveRespFileArgs();
+                return;
+            }
+
+            self.has_next = true;
+            self.other_args_ptr = self.argv_ptr + self.next_index;
+            self.other_args_len = 1; // We adjust this value below when necessary.
+            arg = mem.span(self.argv_ptr[self.next_index]);
+            self.incrementArgIndex();
+        }
         if (!mem.startsWith(u8, arg, "-")) {
             self.zig_equivalent = .positional;
             self.only_arg = arg.ptr;
@@ -1352,7 +1412,7 @@ pub const ClangArgIterator = extern struct {
                         process.exit(1);
                     }
                     self.only_arg = self.argv_ptr[self.next_index];
-                    self.next_index += 1;
+                    self.incrementArgIndex();
                     self.other_args_len += 1;
                     self.zig_equivalent = clang_arg.zig_equivalent;
 
@@ -1374,7 +1434,7 @@ pub const ClangArgIterator = extern struct {
                         process.exit(1);
                     }
                     self.second_arg = self.argv_ptr[self.next_index];
-                    self.next_index += 1;
+                    self.incrementArgIndex();
                     self.other_args_len += 1;
                     self.zig_equivalent = clang_arg.zig_equivalent;
                     break :find_clang_arg;
@@ -1386,7 +1446,7 @@ pub const ClangArgIterator = extern struct {
                     process.exit(1);
                 }
                 self.only_arg = self.argv_ptr[self.next_index];
-                self.next_index += 1;
+                self.incrementArgIndex();
                 self.other_args_len += 1;
                 self.zig_equivalent = clang_arg.zig_equivalent;
                 break :find_clang_arg;
@@ -1406,6 +1466,28 @@ pub const ClangArgIterator = extern struct {
             process.exit(1);
         }
     }
+
+    fn incrementArgIndex(self: *ClangArgIterator) void {
+        self.next_index += 1;
+        self.resolveRespFileArgs();
+    }
+
+    fn resolveRespFileArgs(self: *ClangArgIterator) void {
+        const allocator = std.heap.c_allocator;
+        if (self.next_index >= self.argv_len) {
+            if (self.root_args) |root_args| {
+                self.next_index = root_args.next_index;
+                self.argv_ptr = root_args.argv_ptr;
+                self.argv_len = root_args.argv_len;
+
+                allocator.destroy(root_args);
+                self.root_args = null;
+            }
+            if (self.next_index >= self.argv_len) {
+                self.has_next = false;
+            }
+        }
+    }
 };
 
 export fn stage2_clang_arg_iterator(
@@ -1418,7 +1500,8 @@ export fn stage2_clang_arg_iterator(
 
 export fn stage2_clang_arg_next(it: *ClangArgIterator) Error {
     it.next() catch |err| switch (err) {
-        error.UnknownClangOption => return .UnknownClangOption,
+        error.NestedResponseFile => return .NestedResponseFile,
+        error.OutOfMemory => return .OutOfMemory,
     };
     return .None;
 }

src/error.cpp

@@ -84,6 +84,7 @@ const char *err_str(Error err) {
         case ErrorInvalidAbiVersion: return "invalid C ABI version";
         case ErrorInvalidOperatingSystemVersion: return "invalid operating system version";
         case ErrorUnknownClangOption: return "unknown Clang option";
+        case ErrorNestedResponseFile: return "nested response file";
     }
     return "(invalid error)";
 }

src/stage2.h

@@ -106,6 +106,7 @@ enum Error {
     ErrorInvalidAbiVersion,
     ErrorInvalidOperatingSystemVersion,
     ErrorUnknownClangOption,
+    ErrorNestedResponseFile,
 };
 
 // ABI warning
@@ -361,6 +362,7 @@ struct Stage2ClangArgIterator {
     const char **argv_ptr;
     size_t argv_len;
     size_t next_index;
+    size_t root_args;
 };
 
 // ABI warning