From c664692bdd62765ab444a79a8704d2161c1809f1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 19 Feb 2020 01:24:34 -0500 Subject: [PATCH] make the CLI support depending on system headers and libraries (include and lib search paths) The detection of native system paths is self-hosted. closes #2041 --- lib/std/build.zig | 123 ------------------------------ lib/std/zig.zig | 1 + lib/std/zig/system.zig | 152 +++++++++++++++++++++++++++++++++++++ src-self-hosted/stage2.zig | 38 ++++++++++ src/main.cpp | 28 ++++++- src/stage2.cpp | 16 ++++ src/stage2.h | 16 ++++ 7 files changed, 250 insertions(+), 124 deletions(-) create mode 100644 lib/std/zig/system.zig diff --git a/lib/std/build.zig b/lib/std/build.zig index 5d13a9923..f6e847470 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -27,9 +27,6 @@ pub const Builder = struct { install_tls: TopLevelStep, uninstall_tls: TopLevelStep, allocator: *Allocator, - native_system_lib_paths: ArrayList([]const u8), - native_system_include_dirs: ArrayList([]const u8), - native_system_rpaths: ArrayList([]const u8), user_input_options: UserInputOptionsMap, available_options_map: AvailableOptionsMap, available_options_list: ArrayList(AvailableOption), @@ -139,9 +136,6 @@ pub const Builder = struct { .verbose_cimport = false, .invalid_user_input = false, .allocator = allocator, - .native_system_lib_paths = ArrayList([]const u8).init(allocator), - .native_system_include_dirs = ArrayList([]const u8).init(allocator), - .native_system_rpaths = ArrayList([]const u8).init(allocator), .user_input_options = UserInputOptionsMap.init(allocator), .available_options_map = AvailableOptionsMap.init(allocator), .available_options_list = ArrayList(AvailableOption).init(allocator), @@ -172,15 +166,11 @@ pub const Builder = struct { }; try self.top_level_steps.append(&self.install_tls); try self.top_level_steps.append(&self.uninstall_tls); - self.detectNativeSystemPaths(); self.default_step = &self.install_tls.step; return self; } pub fn destroy(self: *Builder) void { - self.native_system_lib_paths.deinit(); - self.native_system_include_dirs.deinit(); - self.native_system_rpaths.deinit(); self.env_map.deinit(); self.top_level_steps.deinit(); self.allocator.destroy(self); @@ -347,18 +337,6 @@ pub const Builder = struct { }; } - pub fn addNativeSystemIncludeDir(self: *Builder, path: []const u8) void { - self.native_system_include_dirs.append(path) catch unreachable; - } - - pub fn addNativeSystemRPath(self: *Builder, path: []const u8) void { - self.native_system_rpaths.append(path) catch unreachable; - } - - pub fn addNativeSystemLibPath(self: *Builder, path: []const u8) void { - self.native_system_lib_paths.append(path) catch unreachable; - } - pub fn make(self: *Builder, step_names: []const []const u8) !void { try self.makePath(self.cache_root); @@ -433,87 +411,6 @@ pub const Builder = struct { return error.InvalidStepName; } - fn detectNativeSystemPaths(self: *Builder) void { - var is_nixos = false; - if (process.getEnvVarOwned(self.allocator, "NIX_CFLAGS_COMPILE")) |nix_cflags_compile| { - is_nixos = true; - var it = mem.tokenize(nix_cflags_compile, " "); - while (true) { - const word = it.next() orelse break; - if (mem.eql(u8, word, "-isystem")) { - const include_path = it.next() orelse { - warn("Expected argument after -isystem in NIX_CFLAGS_COMPILE\n", .{}); - break; - }; - self.addNativeSystemIncludeDir(include_path); - } else { - warn("Unrecognized C flag from NIX_CFLAGS_COMPILE: {}\n", .{word}); - break; - } - } - } else |err| { - assert(err == error.EnvironmentVariableNotFound); - } - if (process.getEnvVarOwned(self.allocator, "NIX_LDFLAGS")) |nix_ldflags| { - is_nixos = true; - var it = mem.tokenize(nix_ldflags, " "); - while (true) { - const word = it.next() orelse break; - if (mem.eql(u8, word, "-rpath")) { - const rpath = it.next() orelse { - warn("Expected argument after -rpath in NIX_LDFLAGS\n", .{}); - break; - }; - self.addNativeSystemRPath(rpath); - } else if (word.len > 2 and word[0] == '-' and word[1] == 'L') { - const lib_path = word[2..]; - self.addNativeSystemLibPath(lib_path); - } else { - warn("Unrecognized C flag from NIX_LDFLAGS: {}\n", .{word}); - break; - } - } - } else |err| { - assert(err == error.EnvironmentVariableNotFound); - } - if (is_nixos) return; - switch (builtin.os) { - .windows => {}, - else => { - const triple = (Target{ - .Cross = CrossTarget{ - .arch = builtin.arch, - .os = builtin.os, - .abi = builtin.abi, - .cpu_features = builtin.cpu_features, - }, - }).linuxTriple(self.allocator); - - // TODO: $ ld --verbose | grep SEARCH_DIR - // the output contains some paths that end with lib64, maybe include them too? - // also, what is the best possible order of things? - - self.addNativeSystemIncludeDir("/usr/local/include"); - self.addNativeSystemLibPath("/usr/local/lib"); - self.addNativeSystemLibPath("/usr/local/lib64"); - - self.addNativeSystemIncludeDir(self.fmt("/usr/include/{}", .{triple})); - self.addNativeSystemLibPath(self.fmt("/usr/lib/{}", .{triple})); - - self.addNativeSystemIncludeDir("/usr/include"); - self.addNativeSystemLibPath("/lib"); - self.addNativeSystemLibPath("/lib64"); - self.addNativeSystemLibPath("/usr/lib"); - self.addNativeSystemLibPath("/usr/lib64"); - - // example: on a 64-bit debian-based linux distro, with zlib installed from apt: - // zlib.h is in /usr/include (added above) - // libz.so.1 is in /lib/x86_64-linux-gnu (added here) - self.addNativeSystemLibPath(self.fmt("/lib/{}", .{triple})); - }, - } - } - pub fn option(self: *Builder, comptime T: type, name: []const u8, description: []const u8) ?T { const type_id = comptime typeToEnum(T); const available_option = AvailableOption{ @@ -1185,7 +1082,6 @@ pub const LibExeObjStep = struct { include_dirs: ArrayList(IncludeDir), c_macros: ArrayList([]const u8), output_dir: ?[]const u8, - need_system_paths: bool, is_linking_libc: bool = false, vcpkg_bin_path: ?[]const u8 = null, @@ -1323,7 +1219,6 @@ pub const LibExeObjStep = struct { .disable_stack_probing = false, .disable_sanitize_c = false, .output_dir = null, - .need_system_paths = false, .single_threaded = false, .installed_path = null, .install_step = null, @@ -1499,7 +1394,6 @@ pub const LibExeObjStep = struct { /// Prefer to use `linkSystemLibrary` instead. pub fn linkSystemLibraryName(self: *LibExeObjStep, name: []const u8) void { self.link_objects.append(LinkObject{ .SystemLib = self.builder.dupe(name) }) catch unreachable; - self.need_system_paths = true; } /// This links against a system library, exclusively using pkg-config to find the library. @@ -2159,23 +2053,6 @@ pub const LibExeObjStep = struct { try zig_args.append(lib_path); } - if (self.need_system_paths and self.target == Target.Native) { - for (builder.native_system_include_dirs.toSliceConst()) |include_path| { - zig_args.append("-isystem") catch unreachable; - zig_args.append(builder.pathFromRoot(include_path)) catch unreachable; - } - - for (builder.native_system_rpaths.toSliceConst()) |rpath| { - zig_args.append("-rpath") catch unreachable; - zig_args.append(rpath) catch unreachable; - } - - for (builder.native_system_lib_paths.toSliceConst()) |lib_path| { - zig_args.append("--library-path") catch unreachable; - zig_args.append(lib_path) catch unreachable; - } - } - for (self.c_macros.toSliceConst()) |c_macro| { try zig_args.append("-D"); try zig_args.append(c_macro); diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 494c9d005..d76ed9dfd 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -5,6 +5,7 @@ pub const parse = @import("zig/parse.zig").parse; pub const parseStringLiteral = @import("zig/parse_string_literal.zig").parseStringLiteral; pub const render = @import("zig/render.zig").render; pub const ast = @import("zig/ast.zig"); +pub const system = @import("zig/system.zig"); test "std.zig tests" { _ = @import("zig/ast.zig"); diff --git a/lib/std/zig/system.zig b/lib/std/zig/system.zig new file mode 100644 index 000000000..74a9ffdc7 --- /dev/null +++ b/lib/std/zig/system.zig @@ -0,0 +1,152 @@ +const std = @import("../std.zig"); +const mem = std.mem; +const Allocator = std.mem.Allocator; +const ArrayList = std.ArrayList; +const assert = std.debug.assert; +const process = std.process; + +pub const NativePaths = struct { + include_dirs: ArrayList([:0]u8), + lib_dirs: ArrayList([:0]u8), + rpaths: ArrayList([:0]u8), + warnings: ArrayList([:0]u8), + + pub fn detect(allocator: *Allocator) !NativePaths { + var self: NativePaths = .{ + .include_dirs = ArrayList([:0]u8).init(allocator), + .lib_dirs = ArrayList([:0]u8).init(allocator), + .rpaths = ArrayList([:0]u8).init(allocator), + .warnings = ArrayList([:0]u8).init(allocator), + }; + errdefer self.deinit(); + + var is_nix = false; + if (std.os.getenvZ("NIX_CFLAGS_COMPILE")) |nix_cflags_compile| { + is_nix = true; + var it = mem.tokenize(nix_cflags_compile, " "); + while (true) { + const word = it.next() orelse break; + if (mem.eql(u8, word, "-isystem")) { + const include_path = it.next() orelse { + try self.addWarning("Expected argument after -isystem in NIX_CFLAGS_COMPILE"); + break; + }; + try self.addIncludeDir(include_path); + } else { + try self.addWarningFmt("Unrecognized C flag from NIX_CFLAGS_COMPILE: {}", .{word}); + break; + } + } + } + if (std.os.getenvZ("NIX_LDFLAGS")) |nix_ldflags| { + is_nix = true; + var it = mem.tokenize(nix_ldflags, " "); + while (true) { + const word = it.next() orelse break; + if (mem.eql(u8, word, "-rpath")) { + const rpath = it.next() orelse { + try self.addWarning("Expected argument after -rpath in NIX_LDFLAGS"); + break; + }; + try self.addRPath(rpath); + } else if (word.len > 2 and word[0] == '-' and word[1] == 'L') { + const lib_path = word[2..]; + try self.addLibDir(lib_path); + } else { + try self.addWarningFmt("Unrecognized C flag from NIX_LDFLAGS: {}", .{word}); + break; + } + } + } + if (is_nix) { + return self; + } + + switch (std.builtin.os) { + .windows => {}, + else => { + const triple = try std.Target.current.linuxTriple(allocator); + + // TODO: $ ld --verbose | grep SEARCH_DIR + // the output contains some paths that end with lib64, maybe include them too? + // TODO: what is the best possible order of things? + // TODO: some of these are suspect and should only be added on some systems. audit needed. + + try self.addIncludeDir("/usr/local/include"); + try self.addLibDir("/usr/local/lib"); + try self.addLibDir("/usr/local/lib64"); + + try self.addIncludeDirFmt("/usr/include/{}", .{triple}); + try self.addLibDirFmt("/usr/lib/{}", .{triple}); + + try self.addIncludeDir("/usr/include"); + try self.addLibDir("/lib"); + try self.addLibDir("/lib64"); + try self.addLibDir("/usr/lib"); + try self.addLibDir("/usr/lib64"); + + // example: on a 64-bit debian-based linux distro, with zlib installed from apt: + // zlib.h is in /usr/include (added above) + // libz.so.1 is in /lib/x86_64-linux-gnu (added here) + try self.addLibDirFmt("/lib/{}", .{triple}); + }, + } + + return self; + } + + pub fn deinit(self: *NativePaths) void { + deinitArray(&self.include_dirs); + deinitArray(&self.lib_dirs); + deinitArray(&self.rpaths); + deinitArray(&self.warnings); + self.* = undefined; + } + + fn deinitArray(array: *ArrayList([:0]u8)) void { + for (array.toSlice()) |item| { + array.allocator.free(item); + } + array.deinit(); + } + + pub fn addIncludeDir(self: *NativePaths, s: []const u8) !void { + return self.appendArray(&self.include_dirs, s); + } + + pub fn addIncludeDirFmt(self: *NativePaths, comptime fmt: []const u8, args: var) !void { + const item = try std.fmt.allocPrint0(self.include_dirs.allocator, fmt, args); + errdefer self.include_dirs.allocator.free(item); + try self.include_dirs.append(item); + } + + pub fn addLibDir(self: *NativePaths, s: []const u8) !void { + return self.appendArray(&self.lib_dirs, s); + } + + pub fn addLibDirFmt(self: *NativePaths, comptime fmt: []const u8, args: var) !void { + const item = try std.fmt.allocPrint0(self.lib_dirs.allocator, fmt, args); + errdefer self.lib_dirs.allocator.free(item); + try self.lib_dirs.append(item); + } + + pub fn addWarning(self: *NativePaths, s: []const u8) !void { + return self.appendArray(&self.warnings, s); + } + + pub fn addWarningFmt(self: *NativePaths, comptime fmt: []const u8, args: var) !void { + const item = try std.fmt.allocPrint0(self.warnings.allocator, fmt, args); + errdefer self.warnings.allocator.free(item); + try self.warnings.append(item); + } + + pub fn addRPath(self: *NativePaths, s: []const u8) !void { + return self.appendArray(&self.rpaths, s); + } + + fn appendArray(self: *NativePaths, array: *ArrayList([:0]u8), s: []const u8) !void { + const item = try std.mem.dupeZ(array.allocator, u8, s); + errdefer array.allocator.free(item); + try array.append(item); + } +}; diff --git a/src-self-hosted/stage2.zig b/src-self-hosted/stage2.zig index 37816464a..b278bb9f7 100644 --- a/src-self-hosted/stage2.zig +++ b/src-self-hosted/stage2.zig @@ -1116,3 +1116,41 @@ export fn stage2_detect_dynamic_linker(in_target: *const Stage2Target, out_ptr: fn enumInt(comptime Enum: type, int: c_int) Enum { return @intToEnum(Enum, @intCast(@TagType(Enum), int)); } + +// ABI warning +const Stage2NativePaths = extern struct { + include_dirs_ptr: [*][*:0]u8, + include_dirs_len: usize, + lib_dirs_ptr: [*][*:0]u8, + lib_dirs_len: usize, + rpaths_ptr: [*][*:0]u8, + rpaths_len: usize, + warnings_ptr: [*][*:0]u8, + warnings_len: usize, +}; +// ABI warning +export fn stage2_detect_native_paths(stage1_paths: *Stage2NativePaths) Error { + stage2DetectNativePaths(stage1_paths) catch |err| switch (err) { + error.OutOfMemory => return .OutOfMemory, + }; + return .None; +} + +fn stage2DetectNativePaths(stage1_paths: *Stage2NativePaths) !void { + var paths = try std.zig.system.NativePaths.detect(std.heap.c_allocator); + errdefer paths.deinit(); + + try convertSlice(paths.include_dirs.toSlice(), &stage1_paths.include_dirs_ptr, &stage1_paths.include_dirs_len); + try convertSlice(paths.lib_dirs.toSlice(), &stage1_paths.lib_dirs_ptr, &stage1_paths.lib_dirs_len); + try convertSlice(paths.rpaths.toSlice(), &stage1_paths.rpaths_ptr, &stage1_paths.rpaths_len); + try convertSlice(paths.warnings.toSlice(), &stage1_paths.warnings_ptr, &stage1_paths.warnings_len); +} + +fn convertSlice(slice: [][:0]u8, ptr: *[*][*:0]u8, len: *usize) !void { + len.* = slice.len; + const new_slice = try std.heap.c_allocator.alloc([*:0]u8, slice.len); + for (slice) |item, i| { + new_slice[i] = item.ptr; + } + ptr.* = new_slice.ptr; +} diff --git a/src/main.cpp b/src/main.cpp index 5117348e3..d7f5e5b7b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1120,6 +1120,33 @@ static int main0(int argc, char **argv) { return print_error_usage(arg0); } + if (target.is_native && link_libs.length != 0) { + Error err; + Stage2NativePaths paths; + if ((err = stage2_detect_native_paths(&paths))) { + fprintf(stderr, "unable to detect native system paths: %s\n", err_str(err)); + exit(1); + } + for (size_t i = 0; i < paths.warnings_len; i += 1) { + const char *warning = paths.warnings_ptr[i]; + fprintf(stderr, "warning: %s\n", warning); + } + for (size_t i = 0; i < paths.include_dirs_len; i += 1) { + const char *include_dir = paths.include_dirs_ptr[i]; + clang_argv.append("-I"); + clang_argv.append(include_dir); + } + for (size_t i = 0; i < paths.lib_dirs_len; i += 1) { + const char *lib_dir = paths.lib_dirs_ptr[i]; + lib_dirs.append(lib_dir); + } + for (size_t i = 0; i < paths.rpaths_len; i += 1) { + const char *rpath = paths.rpaths_ptr[i]; + rpath_list.append(rpath); + } + } + + assert(cmd != CmdBuild || out_type != OutTypeUnknown); bool need_name = (cmd == CmdBuild || cmd == CmdTranslateC); @@ -1226,7 +1253,6 @@ static int main0(int argc, char **argv) { g->function_sections = function_sections; g->code_model = code_model; - for (size_t i = 0; i < lib_dirs.length; i += 1) { codegen_add_lib_dir(g, lib_dirs.at(i)); } diff --git a/src/stage2.cpp b/src/stage2.cpp index 7f6832175..c3abe54d7 100644 --- a/src/stage2.cpp +++ b/src/stage2.cpp @@ -175,3 +175,19 @@ enum Error stage2_detect_dynamic_linker(const struct ZigTarget *target, char **o const char *msg = "stage0 called stage2_detect_dynamic_linker"; stage2_panic(msg, strlen(msg)); } + +enum Error stage2_detect_native_paths(struct Stage2NativePaths *native_paths) { + native_paths->include_dirs_ptr = nullptr; + native_paths->include_dirs_len = 0; + + native_paths->lib_dirs_ptr = nullptr; + native_paths->lib_dirs_len = 0; + + native_paths->rpaths_ptr = nullptr; + native_paths->rpaths_len = 0; + + native_paths->warnings_ptr = nullptr; + native_paths->warnings_len = 0; + + return ErrorNone; +} diff --git a/src/stage2.h b/src/stage2.h index b6e4cebba..302cffafc 100644 --- a/src/stage2.h +++ b/src/stage2.h @@ -312,4 +312,20 @@ struct ZigTarget { ZIG_EXTERN_C enum Error stage2_detect_dynamic_linker(const struct ZigTarget *target, char **out_ptr, size_t *out_len); + + +// ABI warning +struct Stage2NativePaths { + const char **include_dirs_ptr; + size_t include_dirs_len; + const char **lib_dirs_ptr; + size_t lib_dirs_len; + const char **rpaths_ptr; + size_t rpaths_len; + const char **warnings_ptr; + size_t warnings_len; +}; +// ABI warning +ZIG_EXTERN_C enum Error stage2_detect_native_paths(struct Stage2NativePaths *native_paths); + #endif