Ekdohibs/zig
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

stage2: building musl libc from source

Browse Source
This commit is contained in:
Andrew Kelley 2020-09-23 20:48:47 -07:00
parent 64deb46859
commit b08fd0e8fc
6 changed files with 426 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
BRANCH_TODO
View File

@ -1,7 +1,6 @@
* musl
* repair @cImport
* tests passing with -Dskip-non-native
* windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj]
* repair @cImport
* make sure zig cc works
- using it as a preprocessor (-E)
- try building some software
@ -20,6 +19,7 @@
* WASM LLD linking
* skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd)
(maybe make it an explicit option and have main.zig disable it)
- make sure that `zig cc -o hello hello.c -target native-native-musl` and `zig build-exe hello.zig -lc -target native-native-musl` will share the same libc build.
* audit the CLI options for stage2
* audit the base cache hash
* On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process.
@ -58,3 +58,5 @@
* rename std.builtin.Mode to std.builtin.OptimizeMode
* implement `zig run` and `zig test` when combined with `--watch`
* close the --pkg-begin --pkg-end Package directory handles
* make std.Progress support multithreaded
* update musl.zig static data to use native path separator in static data rather than replacing '/' at runtime

6
lib/std/zig/system.zig
View File

@ -393,6 +393,12 @@ pub const NativeTargetInfo = struct {
if (!native_target_has_ld or have_all_info or os_is_non_native) {
return defaultAbiAndDynamicLinker(cpu, os, cross_target);
}
if (cross_target.abi) |abi| {
if (abi.isMusl()) {
// musl implies static linking.
return defaultAbiAndDynamicLinker(cpu, os, cross_target);
}
}
// The current target's ABI cannot be relied on for this. For example, we may build the zig
// compiler for target riscv64-linux-musl and provide a tarball for users to download.
// A user could then run that zig compiler on riscv64-linux-gnu. This use case is well-defined

110
src/Compilation.zig
View File

@ -15,6 +15,7 @@ const liveness = @import("liveness.zig");
const build_options = @import("build_options");
const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
const glibc = @import("glibc.zig");
const musl = @import("musl.zig");
const libunwind = @import("libunwind.zig");
const libcxx = @import("libcxx.zig");
const fatal = @import("main.zig").fatal;
@ -140,6 +141,8 @@ const Job = union(enum) {
glibc_crt_file: glibc.CRTFile,
/// all of the glibc shared objects
glibc_shared_objects,
/// one of the glibc static objects
musl_crt_file: musl.CRTFile,
/// libunwind.a, usually needed when linking libc
libunwind: void,
libcxx: void,
@ -778,6 +781,18 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
if (comp.wantBuildGLibCFromSource()) {
try comp.addBuildingGLibCJobs();
}
if (comp.wantBuildMuslFromSource()) {
try comp.work_queue.write(&[_]Job{
.{ .musl_crt_file = .crti_o },
.{ .musl_crt_file = .crtn_o },
.{ .musl_crt_file = .crt1_o },
.{ .musl_crt_file = .scrt1_o },
.{ .musl_crt_file = .libc_a },
});
}
if (comp.wantBuildMinGWW64FromSource()) {
@panic("TODO");
}
if (comp.wantBuildLibUnwindFromSource()) {
try comp.work_queue.writeItem(.{ .libunwind = {} });
}
@ -822,6 +837,7 @@ pub fn destroy(self: *Compilation) void {
{
var it = self.crt_files.iterator();
while (it.next()) |entry| {
gpa.free(entry.key);
entry.value.deinit(gpa);
}
self.crt_files.deinit(gpa);
@ -1128,6 +1144,12 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void {
fatal("unable to build glibc shared objects: {}", .{@errorName(err)});
};
},
.musl_crt_file => |crt_file| {
musl.buildCRTFile(self, crt_file) catch |err| {
// TODO Expose this as a normal compile error rather than crashing here.
fatal("unable to build musl CRT file: {}", .{@errorName(err)});
};
},
.libunwind => {
libunwind.buildStaticLib(self) catch |err| {
// TODO Expose this as a normal compile error rather than crashing here.
@ -1846,7 +1868,10 @@ fn detectLibCFromLibCInstallation(arena: *Allocator, target: Target, lci: *const
}
pub fn get_libc_crt_file(comp: *Compilation, arena: *Allocator, basename: []const u8) ![]const u8 {
if (comp.wantBuildGLibCFromSource()) {
if (comp.wantBuildGLibCFromSource() or
comp.wantBuildMuslFromSource() or
comp.wantBuildMinGWW64FromSource())
{
return comp.crt_files.get(basename).?.full_object_path;
}
const lci = comp.bin_file.options.libc_installation orelse return error.LibCInstallationNotAvailable;
@ -1865,15 +1890,26 @@ fn addBuildingGLibCJobs(comp: *Compilation) !void {
});
}
fn wantBuildGLibCFromSource(comp: *Compilation) bool {
fn wantBuildLibCFromSource(comp: Compilation) bool {
const is_exe_or_dyn_lib = switch (comp.bin_file.options.output_mode) {
.Obj => false,
.Lib => comp.bin_file.options.link_mode == .Dynamic,
.Exe => true,
};
return comp.bin_file.options.link_libc and is_exe_or_dyn_lib and
comp.bin_file.options.libc_installation == null and
comp.bin_file.options.target.isGnuLibC();
comp.bin_file.options.libc_installation == null;
}
fn wantBuildGLibCFromSource(comp: Compilation) bool {
return comp.wantBuildLibCFromSource() and comp.getTarget().isGnuLibC();
}
fn wantBuildMuslFromSource(comp: Compilation) bool {
return comp.wantBuildLibCFromSource() and comp.getTarget().isMusl();
}
fn wantBuildMinGWW64FromSource(comp: Compilation) bool {
return comp.wantBuildLibCFromSource() and comp.getTarget().isMinGW();
}
fn wantBuildLibUnwindFromSource(comp: *Compilation) bool {
@ -2362,3 +2398,69 @@ fn createStage1Pkg(
};
return child_pkg;
}
pub fn build_crt_file(
comp: *Compilation,
root_name: []const u8,
output_mode: std.builtin.OutputMode,
c_source_files: []const Compilation.CSourceFile,
) !void {
const tracy = trace(@src());
defer tracy.end();
const target = comp.getTarget();
const basename = try std.zig.binNameAlloc(comp.gpa, root_name, target, output_mode, null, null);
errdefer comp.gpa.free(basename);
// TODO: This is extracted into a local variable to work around a stage1 miscompilation.
const emit_bin = Compilation.EmitLoc{
.directory = null, // Put it in the cache directory.
.basename = basename,
};
const sub_compilation = try Compilation.create(comp.gpa, .{
.local_cache_directory = comp.global_cache_directory,
.global_cache_directory = comp.global_cache_directory,
.zig_lib_directory = comp.zig_lib_directory,
.target = target,
.root_name = root_name,
.root_pkg = null,
.output_mode = output_mode,
.rand = comp.rand,
.libc_installation = comp.bin_file.options.libc_installation,
.emit_bin = emit_bin,
.optimize_mode = comp.bin_file.options.optimize_mode,
.want_sanitize_c = false,
.want_stack_check = false,
.want_valgrind = false,
.want_pic = comp.bin_file.options.pic,
.emit_h = null,
.strip = comp.bin_file.options.strip,
.is_native_os = comp.bin_file.options.is_native_os,
.self_exe_path = comp.self_exe_path,
.c_source_files = c_source_files,
.verbose_cc = comp.verbose_cc,
.verbose_link = comp.bin_file.options.verbose_link,
.verbose_tokenize = comp.verbose_tokenize,
.verbose_ast = comp.verbose_ast,
.verbose_ir = comp.verbose_ir,
.verbose_llvm_ir = comp.verbose_llvm_ir,
.verbose_cimport = comp.verbose_cimport,
.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
.clang_passthrough_mode = comp.clang_passthrough_mode,
.is_compiler_rt_or_libc = true,
});
defer sub_compilation.destroy();
try sub_compilation.updateSubCompilation();
try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1);
const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p|
try std.fs.path.join(comp.gpa, &[_][]const u8{ p, basename })
else
try comp.gpa.dupe(u8, basename);
comp.crt_files.putAssumeCapacityNoClobber(basename, .{
.full_object_path = artifact_path,
.lock = sub_compilation.bin_file.toOwnedLock(),
});
}

70
src/glibc.zig
View File

@ -274,7 +274,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
"-g",
"-Wa,--noexecstack",
});
return build_crt_file(comp, "crti.o", .Obj, &[1]Compilation.CSourceFile{
return comp.build_crt_file("crti", .Obj, &[1]Compilation.CSourceFile{
.{
.src_path = try start_asm_path(comp, arena, "crti.S"),
.extra_flags = args.items,
@ -292,7 +292,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
"-g",
"-Wa,--noexecstack",
});
return build_crt_file(comp, "crtn.o", .Obj, &[1]Compilation.CSourceFile{
return comp.build_crt_file("crtn", .Obj, &[1]Compilation.CSourceFile{
.{
.src_path = try start_asm_path(comp, arena, "crtn.S"),
.extra_flags = args.items,
@ -343,7 +343,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
.extra_flags = args.items,
};
};
return build_crt_file(comp, "Scrt1.o", .Obj, &[_]Compilation.CSourceFile{ start_os, abi_note_o });
return comp.build_crt_file("Scrt1", .Obj, &[_]Compilation.CSourceFile{ start_os, abi_note_o });
},
.libc_nonshared_a => {
const deps = [_][]const u8{
@ -433,7 +433,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
.extra_flags = args.items,
};
}
return build_crt_file(comp, "libc_nonshared.a", .Lib, &c_source_files);
return comp.build_crt_file("c_nonshared", .Lib, &c_source_files);
},
}
}
@ -676,68 +676,6 @@ fn lib_path(comp: *Compilation, arena: *Allocator, sub_path: []const u8) ![]cons
return path.join(arena, &[_][]const u8{ comp.zig_lib_directory.path.?, sub_path });
}
fn build_crt_file(
comp: *Compilation,
basename: []const u8,
output_mode: std.builtin.OutputMode,
c_source_files: []const Compilation.CSourceFile,
) !void {
const tracy = trace(@src());
defer tracy.end();
// TODO: This is extracted into a local variable to work around a stage1 miscompilation.
const emit_bin = Compilation.EmitLoc{
.directory = null, // Put it in the cache directory.
.basename = basename,
};
const sub_compilation = try Compilation.create(comp.gpa, .{
.local_cache_directory = comp.global_cache_directory,
.global_cache_directory = comp.global_cache_directory,
.zig_lib_directory = comp.zig_lib_directory,
.target = comp.getTarget(),
.root_name = mem.split(basename, ".").next().?,
.root_pkg = null,
.output_mode = output_mode,
.rand = comp.rand,
.libc_installation = comp.bin_file.options.libc_installation,
.emit_bin = emit_bin,
.optimize_mode = comp.bin_file.options.optimize_mode,
.want_sanitize_c = false,
.want_stack_check = false,
.want_valgrind = false,
.want_pic = comp.bin_file.options.pic,
.emit_h = null,
.strip = comp.bin_file.options.strip,
.is_native_os = comp.bin_file.options.is_native_os,
.self_exe_path = comp.self_exe_path,
.c_source_files = c_source_files,
.verbose_cc = comp.verbose_cc,
.verbose_link = comp.bin_file.options.verbose_link,
.verbose_tokenize = comp.verbose_tokenize,
.verbose_ast = comp.verbose_ast,
.verbose_ir = comp.verbose_ir,
.verbose_llvm_ir = comp.verbose_llvm_ir,
.verbose_cimport = comp.verbose_cimport,
.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
.clang_passthrough_mode = comp.clang_passthrough_mode,
.is_compiler_rt_or_libc = true,
});
defer sub_compilation.destroy();
try sub_compilation.updateSubCompilation();
try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1);
const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p|
try path.join(comp.gpa, &[_][]const u8{ p, basename })
else
try comp.gpa.dupe(u8, basename);
comp.crt_files.putAssumeCapacityNoClobber(basename, .{
.full_object_path = artifact_path,
.lock = sub_compilation.bin_file.toOwnedLock(),
});
}
pub const BuiltSharedObjects = struct {
lock: Cache.Lock,
dir_path: []u8,

2
src/link/Elf.zig
View File

@ -1548,7 +1548,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
try argv.append(try comp.get_libc_crt_file(arena, "libc_nonshared.a"));
} else if (target.isMusl()) {
try argv.append(comp.libunwind_static_lib.?.full_object_path);
try argv.append(comp.libc_static_lib.?.full_object_path);
try argv.append(try comp.get_libc_crt_file(arena, "libc.a"));
} else if (self.base.options.link_libcpp) {
try argv.append(comp.libunwind_static_lib.?.full_object_path);
} else {

308
src/musl.zig
View File

@ -1,6 +1,308 @@
//! TODO build musl libc from source
const std = @import("std");
const Allocator = std.mem.Allocator;
const mem = std.mem;
const path = std.fs.path;
const assert = std.debug.assert;
pub const src_files = [_][]const u8{
const target_util = @import("target.zig");
const Compilation = @import("Compilation.zig");
const build_options = @import("build_options");
const trace = @import("tracy.zig").trace;
const Cache = @import("Cache.zig");
const Package = @import("Package.zig");
pub const CRTFile = enum {
crti_o,
crtn_o,
crt1_o,
scrt1_o,
libc_a,
};
pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
if (!build_options.have_llvm) {
return error.ZigCompilerNotBuiltWithLLVMExtensions;
}
const gpa = comp.gpa;
var arena_allocator = std.heap.ArenaAllocator.init(gpa);
defer arena_allocator.deinit();
const arena = &arena_allocator.allocator;
switch (crt_file) {
.crti_o => {
var args = std.ArrayList([]const u8).init(arena);
try add_cc_args(comp, arena, &args, false);
try args.appendSlice(&[_][]const u8{
"-Qunused-arguments",
});
return comp.build_crt_file("crti", .Obj, &[1]Compilation.CSourceFile{
.{
.src_path = try start_asm_path(comp, arena, "crti.s"),
.extra_flags = args.items,
},
});
},
.crtn_o => {
var args = std.ArrayList([]const u8).init(arena);
try add_cc_args(comp, arena, &args, false);
try args.appendSlice(&[_][]const u8{
"-Qunused-arguments",
});
return comp.build_crt_file("crtn", .Obj, &[1]Compilation.CSourceFile{
.{
.src_path = try start_asm_path(comp, arena, "crtn.s"),
.extra_flags = args.items,
},
});
},
.crt1_o => {
var args = std.ArrayList([]const u8).init(arena);
try add_cc_args(comp, arena, &args, false);
try args.appendSlice(&[_][]const u8{
"-fno-stack-protector",
"-DCRT",
});
return comp.build_crt_file("crt1", .Obj, &[1]Compilation.CSourceFile{
.{
.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
"libc", "musl", "crt", "crt1.c",
}),
.extra_flags = args.items,
},
});
},
.scrt1_o => {
var args = std.ArrayList([]const u8).init(arena);
try add_cc_args(comp, arena, &args, false);
try args.appendSlice(&[_][]const u8{
"-fPIC",
"-fno-stack-protector",
"-DCRT",
});
return comp.build_crt_file("Scrt1", .Obj, &[1]Compilation.CSourceFile{
.{
.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
"libc", "musl", "crt", "Scrt1.c",
}),
.extra_flags = args.items,
},
});
},
.libc_a => {
// When there is a src/<arch>/foo.* then it should substitute for src/foo.*
// Even a .s file can substitute for a .c file.
const target = comp.getTarget();
const arch_name = target_util.archMuslName(target.cpu.arch);
var source_table = std.StringArrayHashMap(Ext).init(comp.gpa);
defer source_table.deinit();
try source_table.ensureCapacity(compat_time32_files.len + src_files.len);
for (src_files) |src_file| {
try addSrcFile(arena, &source_table, src_file);
}
const time32_compat_arch_list = [_][]const u8{ "arm", "i386", "mips", "powerpc" };
for (time32_compat_arch_list) |time32_compat_arch| {
if (mem.eql(u8, arch_name, time32_compat_arch)) {
for (compat_time32_files) |compat_time32_file| {
try addSrcFile(arena, &source_table, compat_time32_file);
}
}
}
var c_source_files = std.ArrayList(Compilation.CSourceFile).init(comp.gpa);
defer c_source_files.deinit();
var override_path = std.ArrayList(u8).init(comp.gpa);
defer override_path.deinit();
const s = path.sep_str;
for (source_table.items()) |entry| {
const src_file = entry.key;
const ext = entry.value;
const dirname = path.dirname(src_file).?;
const basename = path.basename(src_file);
const noextbasename = mem.split(basename, ".").next().?;
const before_arch_dir = path.dirname(dirname).?;
const dirbasename = path.basename(dirname);
var is_arch_specific = false;
// Architecture-specific implementations are under a <arch>/ folder.
if (is_musl_arch_name(dirbasename)) {
if (!mem.eql(u8, dirbasename, arch_name))
continue; // Not the architecture we're compiling for.
is_arch_specific = true;
}
if (!is_arch_specific) {
// Look for an arch specific override.
override_path.shrinkRetainingCapacity(0);
try override_path.writer().print("{}" ++ s ++ "{}" ++ s ++ "{}.s", .{
dirname, arch_name, noextbasename,
});
if (source_table.contains(override_path.items))
continue;
override_path.shrinkRetainingCapacity(0);
try override_path.writer().print("{}" ++ s ++ "{}" ++ s ++ "{}.S", .{
dirname, arch_name, noextbasename,
});
if (source_table.contains(override_path.items))
continue;
override_path.shrinkRetainingCapacity(0);
try override_path.writer().print("{}" ++ s ++ "{}" ++ s ++ "{}.c", .{
dirname, arch_name, noextbasename,
});
if (source_table.contains(override_path.items))
continue;
}
var args = std.ArrayList([]const u8).init(arena);
try add_cc_args(comp, arena, &args, ext == .o3);
try args.appendSlice(&[_][]const u8{
"-Qunused-arguments",
"-w", // disable all warnings
});
const c_source_file = try c_source_files.addOne();
c_source_file.* = .{
.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", src_file }),
.extra_flags = args.items,
};
}
return comp.build_crt_file("c", .Lib, c_source_files.items);
},
}
}
fn is_musl_arch_name(name: []const u8) bool {
const musl_arch_names = [_][]const u8{
"aarch64",
"arm",
"generic",
"i386",
"m68k",
"microblaze",
"mips",
"mips64",
"mipsn32",
"or1k",
"powerpc",
"powerpc64",
"riscv64",
"s390x",
"sh",
"x32",
"x86_64",
};
for (musl_arch_names) |musl_arch_name| {
if (mem.eql(u8, musl_arch_name, name)) {
return true;
}
}
return false;
}
const Ext = enum {
assembly,
normal,
o3,
};
fn addSrcFile(arena: *Allocator, source_table: *std.StringArrayHashMap(Ext), file_path: []const u8) !void {
const ext: Ext = ext: {
if (mem.endsWith(u8, file_path, ".c")) {
if (mem.startsWith(u8, file_path, "musl/src/malloc/") or
mem.startsWith(u8, file_path, "musl/src/string/") or
mem.startsWith(u8, file_path, "musl/src/internal/"))
{
break :ext .o3;
} else {
break :ext .assembly;
}
} else if (mem.endsWith(u8, file_path, ".s") or mem.endsWith(u8, file_path, ".S")) {
break :ext .assembly;
} else {
unreachable;
}
};
// TODO do this at comptime on the comptime data rather than at runtime
// probably best to wait until self-hosted is done and our comptime execution
// is faster and uses less memory.
const key = if (path.sep != '/') blk: {
const mutable_file_path = try arena.dupe(u8, file_path);
for (mutable_file_path) |*c| {
if (c.* == '/') {
c.* == path.sep;
}
}
break :blk mutable_file_path;
} else file_path;
source_table.putAssumeCapacityNoClobber(key, ext);
}
fn add_cc_args(
comp: *Compilation,
arena: *Allocator,
args: *std.ArrayList([]const u8),
want_O3: bool,
) error{OutOfMemory}!void {
const target = comp.getTarget();
const arch_name = target_util.archMuslName(target.cpu.arch);
const os_name = @tagName(target.os.tag);
const triple = try std.fmt.allocPrint(arena, "{}-{}-musl", .{ arch_name, os_name });
const o_arg = if (want_O3) "-O3" else "-Os";
try args.appendSlice(&[_][]const u8{
"-std=c99",
"-ffreestanding",
// Musl adds these args to builds with gcc but clang does not support them.
//"-fexcess-precision=standard",
//"-frounding-math",
"-Wa,--noexecstack",
"-D_XOPEN_SOURCE=700",
"-I",
try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "arch", arch_name }),
"-I",
try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "arch", "generic" }),
"-I",
try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "src", "include" }),
"-I",
try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "src", "internal" }),
"-I",
try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "include" }),
"-I",
try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "include", triple }),
"-I",
try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "include", "generic-musl" }),
o_arg,
"-fomit-frame-pointer",
"-fno-unwind-tables",
"-fno-asynchronous-unwind-tables",
"-ffunction-sections",
"-fdata-sections",
});
}
fn start_asm_path(comp: *Compilation, arena: *Allocator, basename: []const u8) ![]const u8 {
const target = comp.getTarget();
return comp.zig_lib_directory.join(arena, &[_][]const u8{
"libc", "musl", "crt", target_util.archMuslName(target.cpu.arch), basename,
});
}
const src_files = [_][]const u8{
"musl/src/aio/aio.c",
"musl/src/aio/aio_suspend.c",
"musl/src/aio/lio_listio.c",
@ -1776,7 +2078,7 @@ pub const src_files = [_][]const u8{
"musl/src/unistd/writev.c",
"musl/src/unistd/x32/lseek.c",
};
pub const compat_time32_files = [_][]const u8{
const compat_time32_files = [_][]const u8{
"musl/compat/time32/__xstat.c",
"musl/compat/time32/adjtime32.c",
"musl/compat/time32/adjtimex_time32.c",