From 66e76a0209586000a78fe896071e73202a80b81f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 13 Mar 2020 21:06:07 -0400 Subject: [PATCH 01/10] zig build system: correctly handle multiple output artifacts Previously the zig build system incorrectly assumed that the only build artifact was a binary. Now, when you enable the cache, only the output dir is printed to stdout, and the zig build system iterates over the files in that directory, copying them to the output directory. To support this change: * Add `std.os.renameat`, `std.os.renameatZ`, and `std.os.renameatW`. * Fix `std.os.linux.renameat` not compiling due to typos. * Deprecate `std.fs.updateFile` and `std.fs.updateFileMode`. * Add `std.fs.Dir.updateFile`, which supports using open directory handles for both the source and destination paths, as well as an options parameter which allows overriding the mode. * Update `std.fs.AtomicFile` to support operating based on an open directory handle. Instead of `std.fs.AtomicFile.init`, use `std.fs.Dir.atomicFile`. * `std.fs.AtomicFile` deinit() better handles the situation when the rename fails but the temporary file still exists, by still attempting to remove the temporary file. * `std.fs.Dir.openFileWindows` is moved to `std.os.windows.OpenFileW`. * `std.os.RenameError` gains the error codes `NoDevice`, `SharingViolation`, and `PipeBusy` which have been observed from Windows. Closes #4733 --- lib/std/build.zig | 21 +-- lib/std/c.zig | 1 + lib/std/fs.zig | 261 ++++++++++++++++-------------------- lib/std/os.zig | 114 +++++++++++++++- lib/std/os/linux.zig | 8 +- lib/std/os/windows.zig | 76 +++++++++++ lib/std/os/windows/bits.zig | 7 + src/main.cpp | 3 +- 8 files changed, 333 insertions(+), 158 deletions(-) diff --git a/lib/std/build.zig b/lib/std/build.zig index e8484e9d1..8fb5eaeeb 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -2144,17 +2144,22 @@ pub const LibExeObjStep = struct { try zig_args.append("--cache"); try zig_args.append("on"); - const output_path_nl = try builder.execFromStep(zig_args.toSliceConst(), &self.step); - const output_path = mem.trimRight(u8, output_path_nl, "\r\n"); + const output_dir_nl = try builder.execFromStep(zig_args.toSliceConst(), &self.step); + const build_output_dir = mem.trimRight(u8, output_dir_nl, "\r\n"); if (self.output_dir) |output_dir| { - const full_dest = try fs.path.join(builder.allocator, &[_][]const u8{ - output_dir, - fs.path.basename(output_path), - }); - try builder.updateFile(output_path, full_dest); + var src_dir = try std.fs.cwd().openDirTraverse(build_output_dir); + defer src_dir.close(); + + var dest_dir = try std.fs.cwd().openDirList(output_dir); + defer dest_dir.close(); + + var it = src_dir.iterate(); + while (try it.next()) |entry| { + _ = try src_dir.updateFile(entry.name, dest_dir, entry.name, .{}); + } } else { - self.output_dir = fs.path.dirname(output_path).?; + self.output_dir = build_output_dir; } } diff --git a/lib/std/c.zig b/lib/std/c.zig index 39a865ebb..43b3d7f31 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -106,6 +106,7 @@ pub extern "c" fn mkdir(path: [*:0]const u8, mode: c_uint) c_int; pub extern "c" fn mkdirat(dirfd: fd_t, path: [*:0]const u8, mode: u32) c_int; pub extern "c" fn symlink(existing: [*:0]const u8, new: [*:0]const u8) c_int; pub extern "c" fn rename(old: [*:0]const u8, new: [*:0]const u8) c_int; +pub extern "c" fn renameat(olddirfd: fd_t, old: [*:0]const u8, newdirfd: fd_t, new: [*:0]const u8) c_int; pub extern "c" fn chdir(path: [*:0]const u8) c_int; pub extern "c" fn fchdir(fd: fd_t) c_int; pub extern "c" fn execve(path: [*:0]const u8, argv: [*:null]const ?[*:0]const u8, envp: [*:null]const ?[*:0]const u8) c_int; diff --git a/lib/std/fs.zig b/lib/std/fs.zig index de6be91f7..231235baf 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -81,60 +81,21 @@ pub fn atomicSymLink(allocator: *Allocator, existing_path: []const u8, new_path: } } -// TODO fix enum literal not casting to error union -const PrevStatus = enum { +pub const PrevStatus = enum { stale, fresh, }; +/// Deprecated; use `Dir.updateFile`. pub fn updateFile(source_path: []const u8, dest_path: []const u8) !PrevStatus { - return updateFileMode(source_path, dest_path, null); + const my_cwd = cwd(); + return Dir.updateFile(my_cwd, source_path, my_cwd, dest_path, .{}); } -/// Check the file size, mtime, and mode of `source_path` and `dest_path`. If they are equal, does nothing. -/// Otherwise, atomically copies `source_path` to `dest_path`. The destination file gains the mtime, -/// atime, and mode of the source file so that the next call to `updateFile` will not need a copy. -/// Returns the previous status of the file before updating. -/// If any of the directories do not exist for dest_path, they are created. -/// TODO rework this to integrate with Dir -pub fn updateFileMode(source_path: []const u8, dest_path: []const u8, mode: ?File.Mode) !PrevStatus { +/// Deprecated; use `Dir.updateFile`. +pub fn updateFileMode(source_path: []const u8, dest_path: []const u8, mode: ?File.Mode) !Dir.PrevStatus { const my_cwd = cwd(); - - var src_file = try my_cwd.openFile(source_path, .{}); - defer src_file.close(); - - const src_stat = try src_file.stat(); - check_dest_stat: { - const dest_stat = blk: { - var dest_file = my_cwd.openFile(dest_path, .{}) catch |err| switch (err) { - error.FileNotFound => break :check_dest_stat, - else => |e| return e, - }; - defer dest_file.close(); - - break :blk try dest_file.stat(); - }; - - if (src_stat.size == dest_stat.size and - src_stat.mtime == dest_stat.mtime and - src_stat.mode == dest_stat.mode) - { - return PrevStatus.fresh; - } - } - const actual_mode = mode orelse src_stat.mode; - - if (path.dirname(dest_path)) |dirname| { - try cwd().makePath(dirname); - } - - var atomic_file = try AtomicFile.init(dest_path, actual_mode); - defer atomic_file.deinit(); - - try atomic_file.file.writeFileAll(src_file, .{ .in_len = src_stat.size }); - try atomic_file.file.updateTimes(src_stat.atime, src_stat.mtime); - try atomic_file.finish(); - return PrevStatus.stale; + return Dir.updateFile(my_cwd, source_path, my_cwd, dest_path, .{ .override_mode = mode }); } /// Guaranteed to be atomic. @@ -172,43 +133,40 @@ pub fn copyFileMode(source_path: []const u8, dest_path: []const u8, mode: File.M return atomic_file.finish(); } -/// TODO update this API to avoid a getrandom syscall for every operation. It -/// should accept a random interface. -/// TODO rework this to integrate with Dir +/// TODO update this API to avoid a getrandom syscall for every operation. pub const AtomicFile = struct { file: File, - tmp_path_buf: [MAX_PATH_BYTES]u8, + tmp_path_buf: [MAX_PATH_BYTES - 1:0]u8, dest_path: []const u8, - finished: bool, + file_open: bool, + file_exists: bool, + dir: Dir, const InitError = File.OpenError; - /// dest_path must remain valid for the lifetime of AtomicFile - /// call finish to atomically replace dest_path with contents - pub fn init(dest_path: []const u8, mode: File.Mode) InitError!AtomicFile { + /// TODO rename this. Callers should go through Dir API + pub fn init2(dest_path: []const u8, mode: File.Mode, dir: Dir) InitError!AtomicFile { const dirname = path.dirname(dest_path); var rand_buf: [12]u8 = undefined; const dirname_component_len = if (dirname) |d| d.len + 1 else 0; const encoded_rand_len = comptime base64.Base64Encoder.calcSize(rand_buf.len); const tmp_path_len = dirname_component_len + encoded_rand_len; - var tmp_path_buf: [MAX_PATH_BYTES]u8 = undefined; - if (tmp_path_len >= tmp_path_buf.len) return error.NameTooLong; + var tmp_path_buf: [MAX_PATH_BYTES - 1:0]u8 = undefined; + if (tmp_path_len > tmp_path_buf.len) return error.NameTooLong; - if (dirname) |dir| { - mem.copy(u8, tmp_path_buf[0..], dir); - tmp_path_buf[dir.len] = path.sep; + if (dirname) |dn| { + mem.copy(u8, tmp_path_buf[0..], dn); + tmp_path_buf[dn.len] = path.sep; } tmp_path_buf[tmp_path_len] = 0; const tmp_path_slice = tmp_path_buf[0..tmp_path_len :0]; - const my_cwd = cwd(); - while (true) { try crypto.randomBytes(rand_buf[0..]); base64_encoder.encode(tmp_path_slice[dirname_component_len..tmp_path_len], &rand_buf); - const file = my_cwd.createFileC( + const file = dir.createFileC( tmp_path_slice, .{ .mode = mode, .exclusive = true }, ) catch |err| switch (err) { @@ -220,33 +178,46 @@ pub const AtomicFile = struct { .file = file, .tmp_path_buf = tmp_path_buf, .dest_path = dest_path, - .finished = false, + .file_open = true, + .file_exists = true, + .dir = dir, }; } } + /// Deprecated. Use `Dir.atomicFile`. + pub fn init(dest_path: []const u8, mode: File.Mode) InitError!AtomicFile { + return init2(dest_path, mode, cwd()); + } + /// always call deinit, even after successful finish() pub fn deinit(self: *AtomicFile) void { - if (!self.finished) { + if (self.file_open) { self.file.close(); - cwd().deleteFileC(@ptrCast([*:0]u8, &self.tmp_path_buf)) catch {}; - self.finished = true; + self.file_open = false; } + if (self.file_exists) { + self.dir.deleteFileC(&self.tmp_path_buf) catch {}; + self.file_exists = false; + } + self.* = undefined; } pub fn finish(self: *AtomicFile) !void { - assert(!self.finished); + assert(self.file_exists); + if (self.file_open) { + self.file.close(); + self.file_open = false; + } if (std.Target.current.os.tag == .windows) { const dest_path_w = try os.windows.sliceToPrefixedFileW(self.dest_path); - const tmp_path_w = try os.windows.cStrToPrefixedFileW(@ptrCast([*:0]u8, &self.tmp_path_buf)); - self.file.close(); - self.finished = true; - return os.renameW(&tmp_path_w, &dest_path_w); + const tmp_path_w = try os.windows.cStrToPrefixedFileW(&self.tmp_path_buf); + try os.renameatW(self.dir.fd, &tmp_path_w, self.dir.fd, &dest_path_w, os.windows.TRUE); + self.file_exists = false; } else { const dest_path_c = try os.toPosixPath(self.dest_path); - self.file.close(); - self.finished = true; - return os.renameC(@ptrCast([*:0]u8, &self.tmp_path_buf), &dest_path_c); + try os.renameatZ(self.dir.fd, &self.tmp_path_buf, self.dir.fd, &dest_path_c); + self.file_exists = false; } } }; @@ -694,7 +665,10 @@ pub const Dir = struct { const access_mask = w.SYNCHRONIZE | (if (flags.read) @as(u32, w.GENERIC_READ) else 0) | (if (flags.write) @as(u32, w.GENERIC_WRITE) else 0); - return self.openFileWindows(sub_path_w, access_mask, w.FILE_OPEN); + return @as(File, .{ + .handle = try os.windows.OpenFileW(self.fd, sub_path_w, null, access_mask, w.FILE_OPEN), + .io_mode = .blocking, + }); } /// Creates, opens, or overwrites a file with write access. @@ -739,7 +713,10 @@ pub const Dir = struct { @as(u32, w.FILE_OVERWRITE_IF) else @as(u32, w.FILE_OPEN_IF); - return self.openFileWindows(sub_path_w, access_mask, creation); + return @as(File, .{ + .handle = try os.windows.OpenFileW(self.fd, sub_path_w, null, access_mask, creation), + .io_mode = .blocking, + }); } /// Deprecated; call `openFile` directly. @@ -757,72 +734,6 @@ pub const Dir = struct { return self.openFileW(sub_path, .{}); } - pub fn openFileWindows( - self: Dir, - sub_path_w: [*:0]const u16, - access_mask: os.windows.ACCESS_MASK, - creation: os.windows.ULONG, - ) File.OpenError!File { - const w = os.windows; - - if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { - return error.IsDir; - } - if (sub_path_w[0] == '.' and sub_path_w[1] == '.' and sub_path_w[2] == 0) { - return error.IsDir; - } - - var result = File{ - .handle = undefined, - .io_mode = .blocking, - }; - - const path_len_bytes = math.cast(u16, mem.toSliceConst(u16, sub_path_w).len * 2) catch |err| switch (err) { - error.Overflow => return error.NameTooLong, - }; - var nt_name = w.UNICODE_STRING{ - .Length = path_len_bytes, - .MaximumLength = path_len_bytes, - .Buffer = @intToPtr([*]u16, @ptrToInt(sub_path_w)), - }; - var attr = w.OBJECT_ATTRIBUTES{ - .Length = @sizeOf(w.OBJECT_ATTRIBUTES), - .RootDirectory = if (path.isAbsoluteWindowsW(sub_path_w)) null else self.fd, - .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. - .ObjectName = &nt_name, - .SecurityDescriptor = null, - .SecurityQualityOfService = null, - }; - var io: w.IO_STATUS_BLOCK = undefined; - const rc = w.ntdll.NtCreateFile( - &result.handle, - access_mask, - &attr, - &io, - null, - w.FILE_ATTRIBUTE_NORMAL, - w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE, - creation, - w.FILE_NON_DIRECTORY_FILE | w.FILE_SYNCHRONOUS_IO_NONALERT, - null, - 0, - ); - switch (rc) { - .SUCCESS => return result, - .OBJECT_NAME_INVALID => unreachable, - .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, - .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, - .NO_MEDIA_IN_DEVICE => return error.NoDevice, - .INVALID_PARAMETER => unreachable, - .SHARING_VIOLATION => return error.SharingViolation, - .ACCESS_DENIED => return error.AccessDenied, - .PIPE_BUSY => return error.PipeBusy, - .OBJECT_PATH_SYNTAX_BAD => unreachable, - .OBJECT_NAME_COLLISION => return error.PathAlreadyExists, - else => return w.unexpectedStatus(rc), - } - } - pub fn makeDir(self: Dir, sub_path: []const u8) !void { try os.mkdirat(self.fd, sub_path, default_new_dir_mode); } @@ -898,6 +809,7 @@ pub const Dir = struct { /// Call `close` on the result when done. /// /// Asserts that the path parameter has no null bytes. + /// TODO collapse this and `openDirList` into one function with an options parameter pub fn openDirTraverse(self: Dir, sub_path: []const u8) OpenError!Dir { if (builtin.os.tag == .windows) { const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); @@ -915,6 +827,7 @@ pub const Dir = struct { /// Call `close` on the result when done. /// /// Asserts that the path parameter has no null bytes. + /// TODO collapse this and `openDirTraverse` into one function with an options parameter pub fn openDirList(self: Dir, sub_path: []const u8) OpenError!Dir { if (builtin.os.tag == .windows) { const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); @@ -1370,6 +1283,70 @@ pub const Dir = struct { pub fn accessW(self: Dir, sub_path_w: [*:0]const u16, flags: File.OpenFlags) AccessError!void { return os.faccessatW(self.fd, sub_path_w, 0, 0); } + + pub const UpdateFileOptions = struct { + override_mode: ?File.Mode = null, + }; + + /// Check the file size, mtime, and mode of `source_path` and `dest_path`. If they are equal, does nothing. + /// Otherwise, atomically copies `source_path` to `dest_path`. The destination file gains the mtime, + /// atime, and mode of the source file so that the next call to `updateFile` will not need a copy. + /// Returns the previous status of the file before updating. + /// If any of the directories do not exist for dest_path, they are created. + /// If `override_mode` is provided, then that value is used rather than the source path's mode. + pub fn updateFile( + source_dir: Dir, + source_path: []const u8, + dest_dir: Dir, + dest_path: []const u8, + options: UpdateFileOptions, + ) !PrevStatus { + var src_file = try source_dir.openFile(source_path, .{}); + defer src_file.close(); + + const src_stat = try src_file.stat(); + const actual_mode = options.override_mode orelse src_stat.mode; + check_dest_stat: { + const dest_stat = blk: { + var dest_file = dest_dir.openFile(dest_path, .{}) catch |err| switch (err) { + error.FileNotFound => break :check_dest_stat, + else => |e| return e, + }; + defer dest_file.close(); + + break :blk try dest_file.stat(); + }; + + if (src_stat.size == dest_stat.size and + src_stat.mtime == dest_stat.mtime and + actual_mode == dest_stat.mode) + { + return PrevStatus.fresh; + } + } + + if (path.dirname(dest_path)) |dirname| { + try dest_dir.makePath(dirname); + } + + var atomic_file = try dest_dir.atomicFile(dest_path, .{ .mode = actual_mode }); + defer atomic_file.deinit(); + + try atomic_file.file.writeFileAll(src_file, .{ .in_len = src_stat.size }); + try atomic_file.file.updateTimes(src_stat.atime, src_stat.mtime); + try atomic_file.finish(); + return PrevStatus.stale; + } + + pub const AtomicFileOptions = struct { + mode: File.Mode = File.default_mode, + }; + + /// `dest_path` must remain valid for the lifetime of `AtomicFile`. + /// Call `AtomicFile.finish` to atomically replace `dest_path` with contents. + pub fn atomicFile(self: Dir, dest_path: []const u8, options: AtomicFileOptions) !AtomicFile { + return AtomicFile.init2(dest_path, options.mode, self); + } }; /// Returns an handle to the current working directory that is open for traversal. diff --git a/lib/std/os.zig b/lib/std/os.zig index ed99c4802..0186e1b0b 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -461,13 +461,11 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void { ); switch (rc) { - .SUCCESS => {}, + .SUCCESS => return, .INVALID_HANDLE => unreachable, // Handle not open for writing .ACCESS_DENIED => return error.CannotTruncate, else => return windows.unexpectedStatus(rc), } - - return; } while (true) { @@ -852,6 +850,7 @@ pub const OpenError = error{ /// Open and possibly create a file. Keeps trying if it gets interrupted. /// See also `openC`. +/// TODO support windows pub fn open(file_path: []const u8, flags: u32, perm: usize) OpenError!fd_t { const file_path_c = try toPosixPath(file_path); return openC(&file_path_c, flags, perm); @@ -859,6 +858,7 @@ pub fn open(file_path: []const u8, flags: u32, perm: usize) OpenError!fd_t { /// Open and possibly create a file. Keeps trying if it gets interrupted. /// See also `open`. +/// TODO support windows pub fn openC(file_path: [*:0]const u8, flags: u32, perm: usize) OpenError!fd_t { while (true) { const rc = system.open(file_path, flags, perm); @@ -892,6 +892,7 @@ pub fn openC(file_path: [*:0]const u8, flags: u32, perm: usize) OpenError!fd_t { /// Open and possibly create a file. Keeps trying if it gets interrupted. /// `file_path` is relative to the open directory handle `dir_fd`. /// See also `openatC`. +/// TODO support windows pub fn openat(dir_fd: fd_t, file_path: []const u8, flags: u32, mode: mode_t) OpenError!fd_t { const file_path_c = try toPosixPath(file_path); return openatC(dir_fd, &file_path_c, flags, mode); @@ -900,6 +901,7 @@ pub fn openat(dir_fd: fd_t, file_path: []const u8, flags: u32, mode: mode_t) Ope /// Open and possibly create a file. Keeps trying if it gets interrupted. /// `file_path` is relative to the open directory handle `dir_fd`. /// See also `openat`. +/// TODO support windows pub fn openatC(dir_fd: fd_t, file_path: [*:0]const u8, flags: u32, mode: mode_t) OpenError!fd_t { while (true) { const rc = system.openat(dir_fd, file_path, flags, mode); @@ -1527,6 +1529,9 @@ const RenameError = error{ RenameAcrossMountPoints, InvalidUtf8, BadPathName, + NoDevice, + SharingViolation, + PipeBusy, } || UnexpectedError; /// Change the name or location of a file. @@ -1580,6 +1585,108 @@ pub fn renameW(old_path: [*:0]const u16, new_path: [*:0]const u16) RenameError!v return windows.MoveFileExW(old_path, new_path, flags); } +/// Change the name or location of a file based on an open directory handle. +pub fn renameat( + old_dir_fd: fd_t, + old_path: []const u8, + new_dir_fd: fd_t, + new_path: []const u8, +) RenameError!void { + if (builtin.os.tag == .windows) { + const old_path_w = try windows.sliceToPrefixedFileW(old_path); + const new_path_w = try windows.sliceToPrefixedFileW(new_path); + return renameatW(old_dir_fd, &old_path_w, new_dir_fd, &new_path_w, windows.TRUE); + } else { + const old_path_c = try toPosixPath(old_path); + const new_path_c = try toPosixPath(new_path); + return renameatZ(old_dir_fd, &old_path_c, new_dir_fd, &new_path_c); + } +} + +/// Same as `renameat` except the parameters are null-terminated byte arrays. +pub fn renameatZ( + old_dir_fd: fd_t, + old_path: [*:0]const u8, + new_dir_fd: fd_t, + new_path: [*:0]const u8, +) RenameError!void { + if (builtin.os.tag == .windows) { + const old_path_w = try windows.cStrToPrefixedFileW(old_path); + const new_path_w = try windows.cStrToPrefixedFileW(new_path); + return renameatW(old_dir_fd, &old_path_w, new_dir_fd, &new_path_w, windows.TRUE); + } + + switch (errno(system.renameat(old_dir_fd, old_path, new_dir_fd, new_path))) { + 0 => return, + EACCES => return error.AccessDenied, + EPERM => return error.AccessDenied, + EBUSY => return error.FileBusy, + EDQUOT => return error.DiskQuota, + EFAULT => unreachable, + EINVAL => unreachable, + EISDIR => return error.IsDir, + ELOOP => return error.SymLinkLoop, + EMLINK => return error.LinkQuotaExceeded, + ENAMETOOLONG => return error.NameTooLong, + ENOENT => return error.FileNotFound, + ENOTDIR => return error.NotDir, + ENOMEM => return error.SystemResources, + ENOSPC => return error.NoSpaceLeft, + EEXIST => return error.PathAlreadyExists, + ENOTEMPTY => return error.PathAlreadyExists, + EROFS => return error.ReadOnlyFileSystem, + EXDEV => return error.RenameAcrossMountPoints, + else => |err| return unexpectedErrno(err), + } +} + +/// Same as `renameat` except the parameters are null-terminated UTF16LE encoded byte arrays. +/// Assumes target is Windows. +/// TODO these args can actually be slices when using ntdll. audit the rest of the W functions too. +pub fn renameatW( + old_dir_fd: fd_t, + old_path: [*:0]const u16, + new_dir_fd: fd_t, + new_path_w: [*:0]const u16, + ReplaceIfExists: windows.BOOLEAN, +) RenameError!void { + const access_mask = windows.SYNCHRONIZE | windows.GENERIC_WRITE; + const src_fd = try windows.OpenFileW(old_dir_fd, old_path, null, access_mask, windows.FILE_OPEN); + defer windows.CloseHandle(src_fd); + + const struct_buf_len = @sizeOf(windows.FILE_RENAME_INFORMATION) + (MAX_PATH_BYTES - 1); + var rename_info_buf: [struct_buf_len]u8 align(@alignOf(windows.FILE_RENAME_INFORMATION)) = undefined; + const new_path = mem.span(new_path_w); + const struct_len = @sizeOf(windows.FILE_RENAME_INFORMATION) - 1 + new_path.len * 2; + if (struct_len > struct_buf_len) return error.NameTooLong; + + const rename_info = @ptrCast(*windows.FILE_RENAME_INFORMATION, &rename_info_buf); + + rename_info.* = .{ + .ReplaceIfExists = ReplaceIfExists, + .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(new_path_w)) null else new_dir_fd, + .FileNameLength = @intCast(u32, new_path.len * 2), // already checked error.NameTooLong + .FileName = undefined, + }; + std.mem.copy(u16, @as([*]u16, &rename_info.FileName)[0..new_path.len], new_path); + + var io_status_block: windows.IO_STATUS_BLOCK = undefined; + + const rc = windows.ntdll.NtSetInformationFile( + src_fd, + &io_status_block, + rename_info, + @intCast(u32, struct_len), // already checked for error.NameTooLong + .FileRenameInformation, + ); + + switch (rc) { + .SUCCESS => return, + .INVALID_HANDLE => unreachable, + else => return windows.unexpectedStatus(rc), + } +} + pub const MakeDirError = error{ AccessDenied, DiskQuota, @@ -3125,6 +3232,7 @@ pub fn realpathC(pathname: [*:0]const u8, out_buffer: *[MAX_PATH_BYTES]u8) RealP } /// Same as `realpath` except `pathname` is null-terminated and UTF16LE-encoded. +/// TODO use ntdll for better semantics pub fn realpathW(pathname: [*:0]const u16, out_buffer: *[MAX_PATH_BYTES]u8) RealPathError![]u8 { const h_file = try windows.CreateFileW( pathname, diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index ba7356d62..f3265eaa6 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -465,17 +465,17 @@ pub fn renameat(oldfd: i32, oldpath: [*]const u8, newfd: i32, newpath: [*]const return syscall4( SYS_renameat, @bitCast(usize, @as(isize, oldfd)), - @ptrToInt(old), + @ptrToInt(oldpath), @bitCast(usize, @as(isize, newfd)), - @ptrToInt(new), + @ptrToInt(newpath), ); } else { return syscall5( SYS_renameat2, @bitCast(usize, @as(isize, oldfd)), - @ptrToInt(old), + @ptrToInt(oldpath), @bitCast(usize, @as(isize, newfd)), - @ptrToInt(new), + @ptrToInt(newpath), 0, ); } diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 6c9a1f24b..f5acb70ad 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -88,6 +88,82 @@ pub fn CreateFileW( return result; } +pub const OpenError = error{ + IsDir, + FileNotFound, + NoDevice, + SharingViolation, + AccessDenied, + PipeBusy, + PathAlreadyExists, + Unexpected, + NameTooLong, +}; + +/// TODO rename to CreateFileW +/// TODO actually we don't need the path parameter to be null terminated +pub fn OpenFileW( + dir: ?HANDLE, + sub_path_w: [*:0]const u16, + sa: ?*SECURITY_ATTRIBUTES, + access_mask: ACCESS_MASK, + creation: ULONG, +) OpenError!HANDLE { + if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + return error.IsDir; + } + if (sub_path_w[0] == '.' and sub_path_w[1] == '.' and sub_path_w[2] == 0) { + return error.IsDir; + } + + var result: HANDLE = undefined; + + const path_len_bytes = math.cast(u16, mem.toSliceConst(u16, sub_path_w).len * 2) catch |err| switch (err) { + error.Overflow => return error.NameTooLong, + }; + var nt_name = UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + .Buffer = @intToPtr([*]u16, @ptrToInt(sub_path_w)), + }; + var attr = OBJECT_ATTRIBUTES{ + .Length = @sizeOf(OBJECT_ATTRIBUTES), + .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(sub_path_w)) null else dir, + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. + .ObjectName = &nt_name, + .SecurityDescriptor = if (sa) |ptr| ptr.lpSecurityDescriptor else null, + .SecurityQualityOfService = null, + }; + var io: IO_STATUS_BLOCK = undefined; + const rc = ntdll.NtCreateFile( + &result, + access_mask, + &attr, + &io, + null, + FILE_ATTRIBUTE_NORMAL, + FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE, + creation, + FILE_NON_DIRECTORY_FILE | FILE_SYNCHRONOUS_IO_NONALERT, + null, + 0, + ); + switch (rc) { + .SUCCESS => return result, + .OBJECT_NAME_INVALID => unreachable, + .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, + .NO_MEDIA_IN_DEVICE => return error.NoDevice, + .INVALID_PARAMETER => unreachable, + .SHARING_VIOLATION => return error.SharingViolation, + .ACCESS_DENIED => return error.AccessDenied, + .PIPE_BUSY => return error.PipeBusy, + .OBJECT_PATH_SYNTAX_BAD => unreachable, + .OBJECT_NAME_COLLISION => return error.PathAlreadyExists, + else => return unexpectedStatus(rc), + } +} + pub const CreatePipeError = error{Unexpected}; pub fn CreatePipe(rd: *HANDLE, wr: *HANDLE, sattr: *const SECURITY_ATTRIBUTES) CreatePipeError!void { diff --git a/lib/std/os/windows/bits.zig b/lib/std/os/windows/bits.zig index 1e49b903c..16cc6d744 100644 --- a/lib/std/os/windows/bits.zig +++ b/lib/std/os/windows/bits.zig @@ -242,6 +242,13 @@ pub const FILE_NAME_INFORMATION = extern struct { FileName: [1]WCHAR, }; +pub const FILE_RENAME_INFORMATION = extern struct { + ReplaceIfExists: BOOLEAN, + RootDirectory: ?HANDLE, + FileNameLength: ULONG, + FileName: [1]WCHAR, +}; + pub const IO_STATUS_BLOCK = extern struct { // "DUMMYUNIONNAME" expands to "u" u: extern union { diff --git a/src/main.cpp b/src/main.cpp index 66138dd04..06d132da8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1290,6 +1290,7 @@ static int main0(int argc, char **argv) { if (g->enable_cache) { #if defined(ZIG_OS_WINDOWS) buf_replace(&g->bin_file_output_path, '/', '\\'); + buf_replace(g->output_dir, '/', '\\'); #endif if (final_output_dir_step != nullptr) { Buf *dest_basename = buf_alloc(); @@ -1303,7 +1304,7 @@ static int main0(int argc, char **argv) { return main_exit(root_progress_node, EXIT_FAILURE); } } else { - if (g->emit_bin && printf("%s\n", buf_ptr(&g->bin_file_output_path)) < 0) + if (printf("%s\n", buf_ptr(g->output_dir)) < 0) return main_exit(root_progress_node, EXIT_FAILURE); } } From 66d7370facc63648eb85fb7f1b94753ab0823ff3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 13 Mar 2020 23:59:36 -0400 Subject: [PATCH 02/10] special case when doing build-obj with just one source file When building an object file from only one source file, instead of having a two-stage cache system, we special case it and use the cache directory that the .o file is output to as the final cache directory for all the build artifacts. When there are more than 1 source file, the linker has to merge objects into one, and so the two stage approach makes sens. But in the case of only one source file, this prevents needlessly copying the object file. This commit fixes an issue with the previous one, where zig with cache enabled would print a directory that actually did not have any build artifacts in it. --- src/codegen.cpp | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index aba8a4903..924ea9baa 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -10482,10 +10482,6 @@ static void resolve_out_paths(CodeGen *g) { case OutTypeUnknown: zig_unreachable(); case OutTypeObj: - if (g->enable_cache && g->link_objects.length == 1 && !need_llvm_module(g)) { - buf_init_from_buf(&g->bin_file_output_path, g->link_objects.at(0)); - return; - } if (need_llvm_module(g) && g->link_objects.length != 0 && !g->enable_cache && buf_eql_buf(o_basename, out_basename)) { @@ -10580,6 +10576,20 @@ static void output_type_information(CodeGen *g) { } } +static bool main_output_dir_is_just_one_c_object(CodeGen *g) { + return g->enable_cache && g->link_objects.length == 1 && !need_llvm_module(g); +} + +static void init_output_dir(CodeGen *g, Buf *digest) { + if (main_output_dir_is_just_one_c_object(g)) { + g->output_dir = buf_alloc(); + os_path_dirname(g->link_objects.at(0), g->output_dir); + } else { + g->output_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR OS_SEP "%s", + buf_ptr(g->cache_dir), buf_ptr(digest)); + } +} + void codegen_build_and_link(CodeGen *g) { Error err; assert(g->out_type != OutTypeUnknown); @@ -10622,8 +10632,7 @@ void codegen_build_and_link(CodeGen *g) { } if (g->enable_cache && buf_len(&digest) != 0) { - g->output_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR OS_SEP "%s", - buf_ptr(g->cache_dir), buf_ptr(&digest)); + init_output_dir(g, &digest); resolve_out_paths(g); } else { if (need_llvm_module(g)) { @@ -10644,8 +10653,7 @@ void codegen_build_and_link(CodeGen *g) { exit(1); } } - g->output_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR OS_SEP "%s", - buf_ptr(g->cache_dir), buf_ptr(&digest)); + init_output_dir(g, &digest); if ((err = os_make_path(g->output_dir))) { fprintf(stderr, "Unable to create output directory: %s\n", err_str(err)); From faa3c40b542f7e6f4d98e8ca542bb47dd603e358 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 14 Mar 2020 00:46:40 -0400 Subject: [PATCH 03/10] fix docgen, which relied on stdout being path to binary --- doc/docgen.zig | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index 4d2625f54..4b9b94dbe 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -1096,6 +1096,9 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var try build_args.append("-lc"); try out.print(" -lc", .{}); } + const target = try std.zig.CrossTarget.parse(.{ + .arch_os_abi = code.target_str orelse "native", + }); if (code.target_str) |triple| { try build_args.appendSlice(&[_][]const u8{ "-target", triple }); if (!code.is_inline) { @@ -1150,7 +1153,15 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var } } - const path_to_exe = mem.trim(u8, exec_result.stdout, " \r\n"); + const path_to_exe_dir = mem.trim(u8, exec_result.stdout, " \r\n"); + const path_to_exe_basename = try std.fmt.allocPrint(allocator, "{}{}", .{ + code.name, + target.exeFileExt(), + }); + const path_to_exe = try fs.path.join(allocator, &[_][]const u8{ + path_to_exe_dir, + path_to_exe_basename, + }); const run_args = &[_][]const u8{path_to_exe}; var exited_with_signal = false; @@ -1486,7 +1497,12 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var } fn exec(allocator: *mem.Allocator, env_map: *std.BufMap, args: []const []const u8) !ChildProcess.ExecResult { - const result = try ChildProcess.exec(allocator, args, null, env_map, max_doc_file_size); + const result = try ChildProcess.exec2(.{ + .allocator = allocator, + .argv = args, + .env_map = env_map, + .max_output_bytes = max_doc_file_size, + }); switch (result.term) { .Exited => |exit_code| { if (exit_code != 0) { From 4a8e766ef5d0c98b27ffad7907c6ce30c54a1ba8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 14 Mar 2020 01:26:49 -0400 Subject: [PATCH 04/10] fix mismatch between expected and actual output name --- src/link.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/link.cpp b/src/link.cpp index 5b6d3a4fd..3f7772bb0 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -566,6 +566,7 @@ static const char *build_libc_object(CodeGen *parent_gen, const char *name, CFil Stage2ProgressNode *progress_node) { CodeGen *child_gen = create_child_codegen(parent_gen, nullptr, OutTypeObj, nullptr, name, progress_node); + child_gen->root_out_name = buf_create_from_str(name); ZigList c_source_files = {0}; c_source_files.append(c_file); child_gen->c_source_files = c_source_files; From a77386eb9847a121bc15af33e5a40bd62f3a67e5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 14 Mar 2020 17:11:51 -0400 Subject: [PATCH 05/10] for build-obj with only 1 C file, name .o file after root_out_name --- src/codegen.cpp | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 924ea9baa..dc6fe04cb 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -9650,6 +9650,21 @@ Error create_c_object_cache(CodeGen *g, CacheHash **out_cache_hash, bool verbose return ErrorNone; } +static bool need_llvm_module(CodeGen *g) { + return buf_len(&g->main_pkg->root_src_path) != 0; +} + +// before gen_c_objects +static bool main_output_dir_is_just_one_c_object_pre(CodeGen *g) { + return g->enable_cache && g->c_source_files.length == 1 && !need_llvm_module(g) && + g->out_type == OutTypeObj && g->link_objects.length == 0; +} + +// after gen_c_objects +static bool main_output_dir_is_just_one_c_object_post(CodeGen *g) { + return g->enable_cache && g->link_objects.length == 1 && !need_llvm_module(g) && g->out_type == OutTypeObj; +} + // returns true if it was a cache miss static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) { Error err; @@ -9667,7 +9682,12 @@ static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) { buf_len(c_source_basename), 0); Buf *final_o_basename = buf_alloc(); - os_path_extname(c_source_basename, final_o_basename, nullptr); + // We special case when doing build-obj for just one C file + if (main_output_dir_is_just_one_c_object_pre(g)) { + buf_init_from_buf(final_o_basename, g->root_out_name); + } else { + os_path_extname(c_source_basename, final_o_basename, nullptr); + } buf_append_str(final_o_basename, target_o_file_ext(g->zig_target)); CacheHash *cache_hash; @@ -10467,10 +10487,6 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { return ErrorNone; } -static bool need_llvm_module(CodeGen *g) { - return buf_len(&g->main_pkg->root_src_path) != 0; -} - static void resolve_out_paths(CodeGen *g) { assert(g->output_dir != nullptr); assert(g->root_out_name != nullptr); @@ -10576,12 +10592,8 @@ static void output_type_information(CodeGen *g) { } } -static bool main_output_dir_is_just_one_c_object(CodeGen *g) { - return g->enable_cache && g->link_objects.length == 1 && !need_llvm_module(g); -} - static void init_output_dir(CodeGen *g, Buf *digest) { - if (main_output_dir_is_just_one_c_object(g)) { + if (main_output_dir_is_just_one_c_object_post(g)) { g->output_dir = buf_alloc(); os_path_dirname(g->link_objects.at(0), g->output_dir); } else { From 701aaf0ddf618edffa182db1e888172b6cae4ab1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 15 Mar 2020 14:46:09 -0400 Subject: [PATCH 06/10] renameatW: handle more windows nt status codes --- lib/std/os.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/std/os.zig b/lib/std/os.zig index 0186e1b0b..9f18f1225 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -1683,6 +1683,11 @@ pub fn renameatW( switch (rc) { .SUCCESS => return, .INVALID_HANDLE => unreachable, + .INVALID_PARAMETER => unreachable, + .OBJECT_PATH_SYNTAX_BAD => unreachable, + .ACCESS_DENIED => return error.AccessDenied, + .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, else => return windows.unexpectedStatus(rc), } } From 6c2b23593b97aef6f375bd7d81beeaf0f66f5682 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 15 Mar 2020 15:46:56 -0400 Subject: [PATCH 07/10] fix std.mem.span handling of sentinel-terminated arrays previously this function would use the array length, but now it scans the array looking for the first sentinel that occurs. --- lib/std/mem.zig | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/std/mem.zig b/lib/std/mem.zig index c76776565..07fbebfb3 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -567,12 +567,20 @@ test "span" { /// Takes a pointer to an array, an array, a sentinel-terminated pointer, /// or a slice, and returns the length. +/// In the case of a sentinel-terminated array, it scans the array +/// for a sentinel and uses that for the length, rather than using the array length. pub fn len(ptr: var) usize { return switch (@typeInfo(@TypeOf(ptr))) { - .Array => |info| info.len, + .Array => |info| if (info.sentinel) |sentinel| + indexOfSentinel(info.child, sentinel, &ptr) + else + info.len, .Pointer => |info| switch (info.size) { .One => switch (@typeInfo(info.child)) { - .Array => |x| x.len, + .Array => |x| if (x.sentinel) |sentinel| + indexOfSentinel(x.child, sentinel, ptr) + else + ptr.len, else => @compileError("invalid type given to std.mem.length"), }, .Many => if (info.sentinel) |sentinel| @@ -597,6 +605,12 @@ test "len" { const ptr = array[0..2 :0].ptr; testing.expect(len(ptr) == 2); } + { + var array: [5:0]u16 = [_]u16{ 1, 2, 3, 4, 5 }; + testing.expect(len(&array) == 5); + array[2] = 0; + testing.expect(len(&array) == 2); + } } pub fn indexOfSentinel(comptime Elem: type, comptime sentinel: Elem, ptr: [*:sentinel]const Elem) usize { From e36978906214cb8c3dba693a073914394a90d0d8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 15 Mar 2020 15:47:42 -0400 Subject: [PATCH 08/10] fix std.os.renameatW Ask for DELETE access when opening the source file. Additionally, when the source and dest dir are the same, pass null for RootDirectory. --- lib/std/os.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 9f18f1225..5a56224e6 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -1650,7 +1650,7 @@ pub fn renameatW( new_path_w: [*:0]const u16, ReplaceIfExists: windows.BOOLEAN, ) RenameError!void { - const access_mask = windows.SYNCHRONIZE | windows.GENERIC_WRITE; + const access_mask = windows.SYNCHRONIZE | windows.GENERIC_WRITE | windows.DELETE; const src_fd = try windows.OpenFileW(old_dir_fd, old_path, null, access_mask, windows.FILE_OPEN); defer windows.CloseHandle(src_fd); @@ -1664,7 +1664,7 @@ pub fn renameatW( rename_info.* = .{ .ReplaceIfExists = ReplaceIfExists, - .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(new_path_w)) null else new_dir_fd, + .RootDirectory = if (old_dir_fd == new_dir_fd or std.fs.path.isAbsoluteWindowsW(new_path_w)) null else new_dir_fd, .FileNameLength = @intCast(u32, new_path.len * 2), // already checked error.NameTooLong .FileName = undefined, }; From 7e45a3ef6ac9a5a24276d26cde11fdbf94e373e4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 15 Mar 2020 15:57:51 -0400 Subject: [PATCH 09/10] fix typo in new mem.len test --- lib/std/mem.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 07fbebfb3..9d2232d3e 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -606,7 +606,7 @@ test "len" { testing.expect(len(ptr) == 2); } { - var array: [5:0]u16 = [_]u16{ 1, 2, 3, 4, 5 }; + var array: [5:0]u16 = [_:0]u16{ 1, 2, 3, 4, 5 }; testing.expect(len(&array) == 5); array[2] = 0; testing.expect(len(&array) == 2); From a27a8561e9387b22d7c694af924e8f2ed0f17290 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 15 Mar 2020 17:26:29 -0400 Subject: [PATCH 10/10] adjust renameatW to always supply dest root dir this fixes tests for wine --- lib/std/mem.zig | 2 +- lib/std/os.zig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 9d2232d3e..438c15c2e 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -116,7 +116,7 @@ pub const Allocator = struct { pub fn allocSentinel(self: *Allocator, comptime Elem: type, n: usize, comptime sentinel: Elem) Error![:sentinel]Elem { var ptr = try self.alloc(Elem, n + 1); ptr[n] = sentinel; - return ptr[0 .. n :sentinel]; + return ptr[0..n :sentinel]; } pub fn alignedAlloc( diff --git a/lib/std/os.zig b/lib/std/os.zig index 5a56224e6..377f7d707 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -1664,7 +1664,7 @@ pub fn renameatW( rename_info.* = .{ .ReplaceIfExists = ReplaceIfExists, - .RootDirectory = if (old_dir_fd == new_dir_fd or std.fs.path.isAbsoluteWindowsW(new_path_w)) null else new_dir_fd, + .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(new_path_w)) null else new_dir_fd, .FileNameLength = @intCast(u32, new_path.len * 2), // already checked error.NameTooLong .FileName = undefined, };