From ec0d775524e0b8b8f0bc41baa3e6d4e11c50557d Mon Sep 17 00:00:00 2001 From: Shawn Anastasio Date: Wed, 3 Jun 2020 19:37:14 -0500 Subject: [PATCH 01/89] Implement std.os for powerpc64{,le} --- lib/std/os/bits/linux.zig | 1 + lib/std/os/bits/linux/powerpc64.zig | 602 ++++++++++++++++++++++++++++ lib/std/os/linux.zig | 1 + lib/std/os/linux/powerpc64.zig | 127 ++++++ lib/std/os/linux/tls.zig | 13 +- 5 files changed, 741 insertions(+), 3 deletions(-) create mode 100644 lib/std/os/bits/linux/powerpc64.zig create mode 100644 lib/std/os/linux/powerpc64.zig diff --git a/lib/std/os/bits/linux.zig b/lib/std/os/bits/linux.zig index 64832673f..9ec7038b0 100644 --- a/lib/std/os/bits/linux.zig +++ b/lib/std/os/bits/linux.zig @@ -15,6 +15,7 @@ pub usingnamespace switch (builtin.arch) { .arm => @import("linux/arm-eabi.zig"), .riscv64 => @import("linux/riscv64.zig"), .mips, .mipsel => @import("linux/mips.zig"), + .powerpc64, .powerpc64le => @import("linux/powerpc64.zig"), else => struct {}, }; diff --git a/lib/std/os/bits/linux/powerpc64.zig b/lib/std/os/bits/linux/powerpc64.zig new file mode 100644 index 000000000..a6884aa02 --- /dev/null +++ b/lib/std/os/bits/linux/powerpc64.zig @@ -0,0 +1,602 @@ +const std = @import("../../../std.zig"); +const linux = std.os.linux; +const socklen_t = linux.socklen_t; +const iovec = linux.iovec; +const iovec_const = linux.iovec_const; +const uid_t = linux.uid_t; +const gid_t = linux.gid_t; +const pid_t = linux.pid_t; +const stack_t = linux.stack_t; +const sigset_t = linux.sigset_t; +pub const SYS = extern enum(usize) { + restart_syscall = 0, + exit = 1, + fork = 2, + read = 3, + write = 4, + open = 5, + close = 6, + waitpid = 7, + creat = 8, + link = 9, + unlink = 10, + execve = 11, + chdir = 12, + time = 13, + mknod = 14, + chmod = 15, + lchown = 16, + sys_break = 17, // sys_ prepended to avoid clashing with keyword + oldstat = 18, + lseek = 19, + getpid = 20, + mount = 21, + umount = 22, + setuid = 23, + getuid = 24, + stime = 25, + ptrace = 26, + alarm = 27, + oldfstat = 28, + pause = 29, + utime = 30, + stty = 31, + gtty = 32, + access = 33, + nice = 34, + ftime = 35, + sync = 36, + kill = 37, + rename = 38, + mkdir = 39, + rmdir = 40, + dup = 41, + pipe = 42, + times = 43, + prof = 44, + brk = 45, + setgid = 46, + getgid = 47, + signal = 48, + geteuid = 49, + getegid = 50, + acct = 51, + umount2 = 52, + lock = 53, + ioctl = 54, + fcntl = 55, + mpx = 56, + setpgid = 57, + ulimit = 58, + oldolduname = 59, + umask = 60, + chroot = 61, + ustat = 62, + dup2 = 63, + getppid = 64, + getpgrp = 65, + setsid = 66, + sigaction = 67, + sgetmask = 68, + ssetmask = 69, + setreuid = 70, + setregid = 71, + sigsuspend = 72, + sigpending = 73, + sethostname = 74, + setrlimit = 75, + getrlimit = 76, + getrusage = 77, + gettimeofday = 78, + settimeofday = 79, + getgroups = 80, + setgroups = 81, + select = 82, + symlink = 83, + oldlstat = 84, + readlink = 85, + uselib = 86, + swapon = 87, + reboot = 88, + readdir = 89, + mmap = 90, + munmap = 91, + truncate = 92, + ftruncate = 93, + fchmod = 94, + fchown = 95, + getpriority = 96, + setpriority = 97, + profil = 98, + statfs = 99, + fstatfs = 100, + ioperm = 101, + socketcall = 102, + syslog = 103, + setitimer = 104, + getitimer = 105, + stat = 106, + lstat = 107, + fstat = 108, + olduname = 109, + iopl = 110, + vhangup = 111, + idle = 112, + vm86 = 113, + wait4 = 114, + swapoff = 115, + sysinfo = 116, + ipc = 117, + fsync = 118, + sigreturn = 119, + clone = 120, + setdomainname = 121, + uname = 122, + modify_ldt = 123, + adjtimex = 124, + mprotect = 125, + sigprocmask = 126, + create_module = 127, + init_module = 128, + delete_module = 129, + get_kernel_syms = 130, + quotactl = 131, + getpgid = 132, + fchdir = 133, + bdflush = 134, + sysfs = 135, + personality = 136, + afs_syscall = 137, + setfsuid = 138, + setfsgid = 139, + _llseek = 140, + getdents = 141, + _newselect = 142, + flock = 143, + msync = 144, + readv = 145, + writev = 146, + getsid = 147, + fdatasync = 148, + _sysctl = 149, + mlock = 150, + munlock = 151, + mlockall = 152, + munlockall = 153, + sched_setparam = 154, + sched_getparam = 155, + sched_setscheduler = 156, + sched_getscheduler = 157, + sched_yield = 158, + sched_get_priority_max = 159, + sched_get_priority_min = 160, + sched_rr_get_interval = 161, + nanosleep = 162, + mremap = 163, + setresuid = 164, + getresuid = 165, + query_module = 166, + poll = 167, + nfsservctl = 168, + setresgid = 169, + getresgid = 170, + prctl = 171, + rt_sigreturn = 172, + rt_sigaction = 173, + rt_sigprocmask = 174, + rt_sigpending = 175, + rt_sigtimedwait = 176, + rt_sigqueueinfo = 177, + rt_sigsuspend = 178, + pread64 = 179, + pwrite64 = 180, + chown = 181, + getcwd = 182, + capget = 183, + capset = 184, + sigaltstack = 185, + sendfile = 186, + getpmsg = 187, + putpmsg = 188, + vfork = 189, + ugetrlimit = 190, + readahead = 191, + pciconfig_read = 198, + pciconfig_write = 199, + pciconfig_iobase = 200, + multiplexer = 201, + getdents64 = 202, + pivot_root = 203, + madvise = 205, + mincore = 206, + gettid = 207, + tkill = 208, + setxattr = 209, + lsetxattr = 210, + fsetxattr = 211, + getxattr = 212, + lgetxattr = 213, + fgetxattr = 214, + listxattr = 215, + llistxattr = 216, + flistxattr = 217, + removexattr = 218, + lremovexattr = 219, + fremovexattr = 220, + futex = 221, + sched_setaffinity = 222, + sched_getaffinity = 223, + tuxcall = 225, + io_setup = 227, + io_destroy = 228, + io_getevents = 229, + io_submit = 230, + io_cancel = 231, + set_tid_address = 232, + fadvise64 = 233, + exit_group = 234, + lookup_dcookie = 235, + epoll_create = 236, + epoll_ctl = 237, + epoll_wait = 238, + remap_file_pages = 239, + timer_create = 240, + timer_settime = 241, + timer_gettime = 242, + timer_getoverrun = 243, + timer_delete = 244, + clock_settime = 245, + clock_gettime = 246, + clock_getres = 247, + clock_nanosleep = 248, + swapcontext = 249, + tgkill = 250, + utimes = 251, + statfs64 = 252, + fstatfs64 = 253, + rtas = 255, + sys_debug_setcontext = 256, + migrate_pages = 258, + mbind = 259, + get_mempolicy = 260, + set_mempolicy = 261, + mq_open = 262, + mq_unlink = 263, + mq_timedsend = 264, + mq_timedreceive = 265, + mq_notify = 266, + mq_getsetattr = 267, + kexec_load = 268, + add_key = 269, + request_key = 270, + keyctl = 271, + waitid = 272, + ioprio_set = 273, + ioprio_get = 274, + inotify_init = 275, + inotify_add_watch = 276, + inotify_rm_watch = 277, + spu_run = 278, + spu_create = 279, + pselect6 = 280, + ppoll = 281, + unshare = 282, + splice = 283, + tee = 284, + vmsplice = 285, + openat = 286, + mkdirat = 287, + mknodat = 288, + fchownat = 289, + futimesat = 290, + newfstatat = 291, + unlinkat = 292, + renameat = 293, + linkat = 294, + symlinkat = 295, + readlinkat = 296, + fchmodat = 297, + faccessat = 298, + get_robust_list = 299, + set_robust_list = 300, + move_pages = 301, + getcpu = 302, + epoll_pwait = 303, + utimensat = 304, + signalfd = 305, + timerfd_create = 306, + eventfd = 307, + sync_file_range2 = 308, + fallocate = 309, + subpage_prot = 310, + timerfd_settime = 311, + timerfd_gettime = 312, + signalfd4 = 313, + eventfd2 = 314, + epoll_create1 = 315, + dup3 = 316, + pipe2 = 317, + inotify_init1 = 318, + perf_event_open = 319, + preadv = 320, + pwritev = 321, + rt_tgsigqueueinfo = 322, + fanotify_init = 323, + fanotify_mark = 324, + prlimit64 = 325, + socket = 326, + bind = 327, + connect = 328, + listen = 329, + accept = 330, + getsockname = 331, + getpeername = 332, + socketpair = 333, + send = 334, + sendto = 335, + recv = 336, + recvfrom = 337, + shutdown = 338, + setsockopt = 339, + getsockopt = 340, + sendmsg = 341, + recvmsg = 342, + recvmmsg = 343, + accept4 = 344, + name_to_handle_at = 345, + open_by_handle_at = 346, + clock_adjtime = 347, + syncfs = 348, + sendmmsg = 349, + setns = 350, + process_vm_readv = 351, + process_vm_writev = 352, + finit_module = 353, + kcmp = 354, + sched_setattr = 355, + sched_getattr = 356, + renameat2 = 357, + seccomp = 358, + getrandom = 359, + memfd_create = 360, + bpf = 361, + execveat = 362, + switch_endian = 363, + userfaultfd = 364, + membarrier = 365, + mlock2 = 378, + copy_file_range = 379, + preadv2 = 380, + pwritev2 = 381, + kexec_file_load = 382, + statx = 383, + pkey_alloc = 384, + pkey_free = 385, + pkey_mprotect = 386, + rseq = 387, + io_pgetevents = 388, + semtimedop = 392, + semget = 393, + semctl = 394, + shmget = 395, + shmctl = 396, + shmat = 397, + shmdt = 398, + msgget = 399, + msgsnd = 400, + msgrcv = 401, + msgctl = 402, + pidfd_send_signal = 424, + io_uring_setup = 425, + io_uring_enter = 426, + io_uring_register = 427, + open_tree = 428, + move_mount = 429, + fsopen = 430, + fsconfig = 431, + fsmount = 432, + fspick = 433, + pidfd_open = 434, + clone3 = 435, + openat2 = 437, + pidfd_getfd = 438, + + _, +}; + +pub const O_CREAT = 0o100; +pub const O_EXCL = 0o200; +pub const O_NOCTTY = 0o400; +pub const O_TRUNC = 0o1000; +pub const O_APPEND = 0o2000; +pub const O_NONBLOCK = 0o4000; +pub const O_DSYNC = 0o10000; +pub const O_SYNC = 0o4010000; +pub const O_RSYNC = 0o4010000; +pub const O_DIRECTORY = 0o40000; +pub const O_NOFOLLOW = 0o100000; +pub const O_CLOEXEC = 0o2000000; + +pub const O_ASYNC = 0o20000; +pub const O_DIRECT = 0o400000; +pub const O_LARGEFILE = 0o200000; +pub const O_NOATIME = 0o1000000; +pub const O_PATH = 0o10000000; +pub const O_TMPFILE = 0o20200000; +pub const O_NDELAY = O_NONBLOCK; + +pub const F_DUPFD = 0; +pub const F_GETFD = 1; +pub const F_SETFD = 2; +pub const F_GETFL = 3; +pub const F_SETFL = 4; + +pub const F_SETOWN = 8; +pub const F_GETOWN = 9; +pub const F_SETSIG = 10; +pub const F_GETSIG = 11; + +pub const F_GETLK = 5; +pub const F_SETLK = 6; +pub const F_SETLKW = 7; + +pub const F_RDLCK = 0; +pub const F_WRLCK = 1; +pub const F_UNLCK = 2; + +pub const LOCK_SH = 1; +pub const LOCK_EX = 2; +pub const LOCK_UN = 8; +pub const LOCK_NB = 4; + +pub const F_SETOWN_EX = 15; +pub const F_GETOWN_EX = 16; + +pub const F_GETOWNER_UIDS = 17; + +/// stack-like segment +pub const MAP_GROWSDOWN = 0x0100; + +/// ETXTBSY +pub const MAP_DENYWRITE = 0x0800; + +/// mark it as an executable +pub const MAP_EXECUTABLE = 0x1000; + +/// pages are locked +pub const MAP_LOCKED = 0x0080; + +/// don't check for reservations +pub const MAP_NORESERVE = 0x0040; + +pub const VDSO_CGT_SYM = "__kernel_clock_gettime"; +pub const VDSO_CGT_VER = "LINUX_2.6.15"; + +pub const Flock = extern struct { + l_type: i16, + l_whence: i16, + l_start: off_t, + l_len: off_t, + l_pid: pid_t, + __unused: [4]u8, +}; + +pub const msghdr = extern struct { + msg_name: ?*sockaddr, + msg_namelen: socklen_t, + msg_iov: [*]iovec, + msg_iovlen: usize, + msg_control: ?*c_void, + msg_controllen: usize, + msg_flags: i32, +}; + +pub const msghdr_const = extern struct { + msg_name: ?*const sockaddr, + msg_namelen: socklen_t, + msg_iov: [*]iovec_const, + msg_iovlen: usize, + msg_control: ?*c_void, + msg_controllen: usize, + msg_flags: i32, +}; + +pub const blksize_t = i64; +pub const nlink_t = u64; +pub const time_t = i64; +pub const mode_t = u32; +pub const off_t = i64; +pub const ino_t = u64; +pub const dev_t = u64; +pub const blkcnt_t = i64; + +/// Renamed to Stat to not conflict with the stat function. +/// atime, mtime, and ctime have functions to return `timespec`, +/// because although this is a POSIX API, the layout and names of +/// the structs are inconsistent across operating systems, and +/// in C, macros are used to hide the differences. Here we use +/// methods to accomplish this. +pub const Stat = extern struct { + dev: dev_t, + ino: ino_t, + nlink: nlink_t, + mode: mode_t, + uid: uid_t, + gid: gid_t, + rdev: dev_t, + size: off_t, + blksize: blksize_t, + blocks: blkcnt_t, + atim: timespec, + mtim: timespec, + ctim: timespec, + __unused: [3]u64, + + pub fn atime(self: Stat) timespec { + return self.atim; + } + + pub fn mtime(self: Stat) timespec { + return self.mtim; + } + + pub fn ctime(self: Stat) timespec { + return self.ctim; + } +}; + +pub const timespec = extern struct { + tv_sec: time_t, + tv_nsec: isize, +}; + +pub const timeval = extern struct { + tv_sec: isize, + tv_usec: isize, +}; + +pub const timezone = extern struct { + tz_minuteswest: i32, + tz_dsttime: i32, +}; + +pub const greg_t = u64; +pub const gregset_t = [48]greg_t; +pub const fpregset_t = [33]f64; + +/// The position of the vscr register depends on endianness. +/// On C, macros are used to change vscr_word's offset to +/// account for this. Here we'll just define vscr_word_le +/// and vscr_word_be. Code must take care to use the correct one. +pub const vrregset = extern struct { + vrregs: [32][4]u32 align(16), + vscr_word_le: u32, + _pad1: [2]u32, + vscr_word_be: u32, + vrsave: u32, + _pad2: [3]u32 +}; +pub const vrregset_t = vrregset; + +pub const mcontext_t = extern struct { + __unused: [4]u64, + signal: i32, + _pad0: i32, + handler: u64, + oldmask: u64, + regs: ?*c_void, + gp_regs: gregset_t, + fp_regs: fpregset_t, + v_regs: *vrregset_t, + vmx_reserve: [34+34+32+1]i64, +}; + +pub const ucontext_t = extern struct { + flags: u32, + link: *ucontext_t, + stack: stack_t, + sigmask: sigset_t, + mcontext: mcontext_t, +}; + +pub const Elf_Symndx = u32; diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 3b8df3d17..67c6fc5d3 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -20,6 +20,7 @@ pub usingnamespace switch (builtin.arch) { .arm => @import("linux/arm-eabi.zig"), .riscv64 => @import("linux/riscv64.zig"), .mips, .mipsel => @import("linux/mips.zig"), + .powerpc64, .powerpc64le => @import("linux/powerpc64.zig"), else => struct {}, }; pub usingnamespace @import("bits.zig"); diff --git a/lib/std/os/linux/powerpc64.zig b/lib/std/os/linux/powerpc64.zig new file mode 100644 index 000000000..337a6aa30 --- /dev/null +++ b/lib/std/os/linux/powerpc64.zig @@ -0,0 +1,127 @@ +usingnamespace @import("../bits.zig"); + +pub fn syscall0(number: SYS) usize { + return asm volatile ( + \\ sc + \\ bns+ 1f + \\ neg 3, 3 + \\ 1: + : [ret] "={r3}" (-> usize) + : [number] "{r0}" (@enumToInt(number)) + : "memory", "cr0", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12" + ); +} + +pub fn syscall1(number: SYS, arg1: usize) usize { + return asm volatile ( + \\ sc + \\ bns+ 1f + \\ neg 3, 3 + \\ 1: + : [ret] "={r3}" (-> usize) + : [number] "{r0}" (@enumToInt(number)), + [arg1] "{r3}" (arg1) + : "memory", "cr0", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12" + ); +} + +pub fn syscall2(number: SYS, arg1: usize, arg2: usize) usize { + return asm volatile ( + \\ sc + \\ bns+ 1f + \\ neg 3, 3 + \\ 1: + : [ret] "={r3}" (-> usize) + : [number] "{r0}" (@enumToInt(number)), + [arg1] "{r3}" (arg1), + [arg2] "{r4}" (arg2) + : "memory", "cr0", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12" + ); +} + +pub fn syscall3(number: SYS, arg1: usize, arg2: usize, arg3: usize) usize { + return asm volatile ( + \\ sc + \\ bns+ 1f + \\ neg 3, 3 + \\ 1: + : [ret] "={r3}" (-> usize) + : [number] "{r0}" (@enumToInt(number)), + [arg1] "{r3}" (arg1), + [arg2] "{r4}" (arg2), + [arg3] "{r5}" (arg3) + : "memory", "cr0", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12" + ); +} + +pub fn syscall4(number: SYS, arg1: usize, arg2: usize, arg3: usize, arg4: usize) usize { + return asm volatile ( + \\ sc + \\ bns+ 1f + \\ neg 3, 3 + \\ 1: + : [ret] "={r3}" (-> usize) + : [number] "{r0}" (@enumToInt(number)), + [arg1] "{r3}" (arg1), + [arg2] "{r4}" (arg2), + [arg3] "{r5}" (arg3), + [arg4] "{r6}" (arg4) + : "memory", "cr0", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12" + ); +} + +pub fn syscall5(number: SYS, arg1: usize, arg2: usize, arg3: usize, arg4: usize, arg5: usize) usize { + return asm volatile ( + \\ sc + \\ bns+ 1f + \\ neg 3, 3 + \\ 1: + : [ret] "={r3}" (-> usize) + : [number] "{r0}" (@enumToInt(number)), + [arg1] "{r3}" (arg1), + [arg2] "{r4}" (arg2), + [arg3] "{r5}" (arg3), + [arg4] "{r6}" (arg4), + [arg5] "{r7}" (arg5) + : "memory", "cr0", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12" + ); +} + +pub fn syscall6( + number: SYS, + arg1: usize, + arg2: usize, + arg3: usize, + arg4: usize, + arg5: usize, + arg6: usize, +) usize { + return asm volatile ( + \\ sc + \\ bns+ 1f + \\ neg 3, 3 + \\ 1: + : [ret] "={r3}" (-> usize) + : [number] "{r0}" (@enumToInt(number)), + [arg1] "{r3}" (arg1), + [arg2] "{r4}" (arg2), + [arg3] "{r5}" (arg3), + [arg4] "{r6}" (arg4), + [arg5] "{r7}" (arg5), + [arg6] "{r8}" (arg6) + : "memory", "cr0", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12" + ); +} + +/// This matches the libc clone function. +pub extern fn clone(func: fn (arg: usize) callconv(.C) u8, stack: usize, flags: usize, arg: usize, ptid: *i32, tls: usize, ctid: *i32) usize; + +pub const restore = restore_rt; + +pub fn restore_rt() callconv(.Naked) void { + return asm volatile ("sc" + : + : [number] "{r0}" (@enumToInt(SYS.rt_sigreturn)) + : "memory", "cr0", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12" + ); +} diff --git a/lib/std/os/linux/tls.zig b/lib/std/os/linux/tls.zig index 8cba45d4b..9c79bc3a9 100644 --- a/lib/std/os/linux/tls.zig +++ b/lib/std/os/linux/tls.zig @@ -48,7 +48,7 @@ const TLSVariant = enum { }; const tls_variant = switch (builtin.arch) { - .arm, .armeb, .aarch64, .aarch64_be, .riscv32, .riscv64, .mips, .mipsel => TLSVariant.VariantI, + .arm, .armeb, .aarch64, .aarch64_be, .riscv32, .riscv64, .mips, .mipsel, .powerpc, .powerpc64, .powerpc64le => TLSVariant.VariantI, .x86_64, .i386 => TLSVariant.VariantII, else => @compileError("undefined tls_variant for this architecture"), }; @@ -72,12 +72,12 @@ const tls_tp_points_past_tcb = switch (builtin.arch) { // make the generated code more efficient const tls_tp_offset = switch (builtin.arch) { - .mips, .mipsel => 0x7000, + .mips, .mipsel, .powerpc, .powerpc64, .powerpc64le => 0x7000, else => 0, }; const tls_dtv_offset = switch (builtin.arch) { - .mips, .mipsel => 0x8000, + .mips, .mipsel, .powerpc, .powerpc64, .powerpc64le => 0x8000, .riscv32, .riscv64 => 0x800, else => 0, }; @@ -160,6 +160,13 @@ pub fn setThreadPointer(addr: usize) void { const rc = std.os.linux.syscall1(.set_thread_area, addr); assert(rc == 0); }, + .powerpc, .powerpc64, .powerpc64le => { + asm volatile ( + \\ mr 13, %[addr] + : + : [addr] "r" (addr) + ); + }, else => @compileError("Unsupported architecture"), } } From 15371775d18e6425d1528643cedb70d6f0b7e6ea Mon Sep 17 00:00:00 2001 From: Shawn Anastasio Date: Wed, 3 Jun 2020 20:55:33 -0500 Subject: [PATCH 02/89] Implement clone() for powerpc64{,le} Implementation borrowed from musl, as most (all?) of the other ones seem to be. --- lib/std/special/c.zig | 55 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/lib/std/special/c.zig b/lib/std/special/c.zig index c769bc358..13012e16a 100644 --- a/lib/std/special/c.zig +++ b/lib/std/special/c.zig @@ -389,6 +389,61 @@ fn clone() callconv(.Naked) void { \\ syscall ); }, + + .powerpc64, .powerpc64le => { + asm volatile ( + \\ # store non-volatile regs r30, r31 on stack in order to put our + \\ # start func and its arg there + \\ stwu 30, -16(1) + \\ stw 31, 4(1) + \\ # save r3 (func) into r30, and r6(arg) into r31 + \\ mr 30, 3 + \\ mr 31, 6 + \\ # create initial stack frame for new thread + \\ clrrwi 4, 4, 4 + \\ li 0, 0 + \\ stwu 0, -16(4) + \\ #move c into first arg + \\ mr 3, 5 + \\ mr 5, 7 + \\ mr 6, 8 + \\ mr 7, 9 + \\ # move syscall number into r0 + \\ li 0, 120 + \\ sc + + \\ # check for syscall error + \\ bns+ 1f # jump to label 1 if no summary overflow. + \\ #else + \\ neg 3, 3 #negate the result (errno) + \\1: + \\ # compare sc result with 0 + \\ cmpwi cr7, 3, 0 + + \\ # if not 0, jump to end + \\ bne cr7, 2f + + \\ #else: we're the child + \\ #call funcptr: move arg (d) into r3 + \\ mr 3, 31 + \\ #move r30 (funcptr) into CTR reg + \\ mtctr 30 + \\ # call CTR reg + \\ bctrl + \\ # mov SYS_exit into r0 (the exit param is already in r3) + \\ li 0, 1 + \\ sc + + \\2: + \\ # restore stack + \\ lwz 30, 0(1) + \\ lwz 31, 4(1) + \\ addi 1, 1, 16 + + \\ blr + ); + }, + else => @compileError("Implement clone() for this arch."), } } From 8574861ca0bc5e8103c27e3c78866e3a4eb56f98 Mon Sep 17 00:00:00 2001 From: Shawn Anastasio Date: Wed, 1 Jul 2020 16:12:27 -0500 Subject: [PATCH 03/89] Implement required ABI bits for powerpc{,64,64le} --- src/analyze.cpp | 3 ++- src/target.cpp | 11 ++++++++--- src/target.hpp | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/analyze.cpp b/src/analyze.cpp index 9062c1fb1..72762f422 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -1003,7 +1003,8 @@ bool want_first_arg_sret(CodeGen *g, FnTypeId *fn_type_id) { g->zig_target->arch == ZigLLVM_x86_64 || target_is_arm(g->zig_target) || target_is_riscv(g->zig_target) || - target_is_wasm(g->zig_target)) + target_is_wasm(g->zig_target) || + target_is_ppc(g->zig_target)) { X64CABIClass abi_class = type_c_abi_x86_64_class(g, fn_type_id->return_type); return abi_class == X64CABIClass_MEMORY || abi_class == X64CABIClass_MEMORY_nobyval; diff --git a/src/target.cpp b/src/target.cpp index 73c0924ab..a4cb03aab 100644 --- a/src/target.cpp +++ b/src/target.cpp @@ -853,6 +853,9 @@ const char *arch_stack_pointer_register_name(ZigLLVM_ArchType arch) { case ZigLLVM_riscv32: case ZigLLVM_riscv64: case ZigLLVM_mipsel: + case ZigLLVM_ppc: + case ZigLLVM_ppc64: + case ZigLLVM_ppc64le: return "sp"; case ZigLLVM_wasm32: @@ -879,7 +882,6 @@ const char *arch_stack_pointer_register_name(ZigLLVM_ArchType arch) { case ZigLLVM_msp430: case ZigLLVM_nvptx: case ZigLLVM_nvptx64: - case ZigLLVM_ppc64le: case ZigLLVM_r600: case ZigLLVM_renderscript32: case ZigLLVM_renderscript64: @@ -893,8 +895,6 @@ const char *arch_stack_pointer_register_name(ZigLLVM_ArchType arch) { case ZigLLVM_tce: case ZigLLVM_tcele: case ZigLLVM_xcore: - case ZigLLVM_ppc: - case ZigLLVM_ppc64: case ZigLLVM_ve: zig_panic("TODO populate this table with stack pointer register name for this CPU architecture"); } @@ -1318,6 +1318,11 @@ bool target_is_mips(const ZigTarget *target) { target->arch == ZigLLVM_mips64 || target->arch == ZigLLVM_mips64el; } +bool target_is_ppc(const ZigTarget *target) { + return target->arch == ZigLLVM_ppc || target->arch == ZigLLVM_ppc64 || + target->arch == ZigLLVM_ppc64le; +} + unsigned target_fn_align(const ZigTarget *target) { return 16; } diff --git a/src/target.hpp b/src/target.hpp index 898fa9020..9a8e79de9 100644 --- a/src/target.hpp +++ b/src/target.hpp @@ -95,6 +95,7 @@ ZigLLVM_OSType get_llvm_os_type(Os os_type); bool target_is_arm(const ZigTarget *target); bool target_is_mips(const ZigTarget *target); +bool target_is_ppc(const ZigTarget *target); bool target_allows_addr_zero(const ZigTarget *target); bool target_has_valgrind_support(const ZigTarget *target); bool target_os_is_darwin(Os os); From 51fcf949f97d068e917c0c3301ba63a4a305ab6e Mon Sep 17 00:00:00 2001 From: Shawn Anastasio Date: Wed, 1 Jul 2020 16:13:14 -0500 Subject: [PATCH 04/89] Implement std.start for powerpc64le This is a bit hacky since we end up doing more than just grabbing the stack pointer in the inline assembly block. Ideally _start would be implemented in pure asm for powerpc64le, but this will do for now. Still to be implemented is powerpc, powerpc64, and powerpc64 (ELFv2) support. The latter will just require correctly determing target ABI for powerpc64 and enabling the existing powerpc64le implementation for it. --- lib/std/start.zig | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/std/start.zig b/lib/std/start.zig index 604c22101..0af7ea640 100644 --- a/lib/std/start.zig +++ b/lib/std/start.zig @@ -116,6 +116,21 @@ fn _start() callconv(.Naked) noreturn { : [argc] "=r" (-> [*]usize) ); }, + .powerpc64le => { + // Before returning the stack pointer, we have to set up a backchain + // and a few other registers required by the ELFv2 ABI. + // TODO: Support powerpc64 (big endian) on ELFv2. + starting_stack_ptr = asm ( + \\ mr 4, 1 + \\ subi 1, 1, 32 + \\ li 5, 0 + \\ std 5, 0(1) + \\ mr %[argc], 4 + : [argc] "=r" (-> [*]usize) + : + : "r4", "r5" + ); + }, else => @compileError("unsupported arch"), } // If LLVM inlines stack variables into _start, they will overwrite From 295f09eadcb39b79e46676ff0ea503103439aae2 Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Sun, 6 Sep 2020 16:12:27 -0700 Subject: [PATCH 05/89] implemented and testing op codes for instructions documented in the unofficial bpf insn reference --- lib/std/os/linux/bpf.zig | 336 +++++++++++++++++++++++++++++++++------ 1 file changed, 290 insertions(+), 46 deletions(-) diff --git a/lib/std/os/linux/bpf.zig b/lib/std/os/linux/bpf.zig index 928c157c4..e5227b384 100644 --- a/lib/std/os/linux/bpf.zig +++ b/lib/std/os/linux/bpf.zig @@ -5,6 +5,7 @@ // and substantial portions of the software. usingnamespace std.os; const std = @import("../../std.zig"); +const builtin = @import("builtin"); const expectEqual = std.testing.expectEqual; // instruction classes @@ -328,6 +329,8 @@ pub const Helper = enum(i32) { _, }; +// TODO: determine that this is the expected bit layout for both little and big +// endian systems /// a single BPF instruction pub const Insn = packed struct { code: u8, @@ -340,19 +343,30 @@ pub const Insn = packed struct { /// frame pub const Reg = packed enum(u4) { r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10 }; const Source = packed enum(u1) { reg, imm }; + + const Mode = packed enum(u8) { + imm = IMM, + abs = ABS, + ind = IND, + mem = MEM, + len = LEN, + msh = MSH, + }; + const AluOp = packed enum(u8) { add = ADD, sub = SUB, mul = MUL, div = DIV, - op_or = OR, - op_and = AND, + alu_or = OR, + alu_and = AND, lsh = LSH, rsh = RSH, neg = NEG, mod = MOD, xor = XOR, mov = MOV, + arsh = ARSH, }; pub const Size = packed enum(u8) { @@ -368,6 +382,13 @@ pub const Insn = packed struct { jgt = JGT, jge = JGE, jset = JSET, + jlt = JLT, + jle = JLE, + jne = JNE, + jsgt = JSGT, + jsge = JSGE, + jslt = JSLT, + jsle = JSLE, }; const ImmOrReg = union(Source) { @@ -419,14 +440,102 @@ pub const Insn = packed struct { return alu(64, .add, dst, src); } + pub fn sub(dst: Reg, src: anytype) Insn { + return alu(64, .sub, dst, src); + } + + pub fn mul(dst: Reg, src: anytype) Insn { + return alu(64, .mul, dst, src); + } + + pub fn div(dst: Reg, src: anytype) Insn { + return alu(64, .div, dst, src); + } + + pub fn alu_or(dst: Reg, src: anytype) Insn { + return alu(64, .alu_or, dst, src); + } + + pub fn alu_and(dst: Reg, src: anytype) Insn { + return alu(64, .alu_and, dst, src); + } + + pub fn lsh(dst: Reg, src: anytype) Insn { + return alu(64, .lsh, dst, src); + } + + pub fn rsh(dst: Reg, src: anytype) Insn { + return alu(64, .rsh, dst, src); + } + + pub fn neg(dst: Reg) Insn { + return alu(64, .neg, dst, 0); + } + + pub fn mod(dst: Reg, src: anytype) Insn { + return alu(64, .mod, dst, src); + } + + pub fn xor(dst: Reg, src: anytype) Insn { + return alu(64, .xor, dst, src); + } + + pub fn arsh(dst: Reg, src: anytype) Insn { + return alu(64, .arsh, dst, src); + } + fn jmp(op: JmpOp, dst: Reg, src: anytype, off: i16) Insn { return imm_reg(JMP | @enumToInt(op), dst, src, off); } + pub fn ja(off: i16) Insn { + return jmp(.ja, .r0, 0, off); + } + pub fn jeq(dst: Reg, src: anytype, off: i16) Insn { return jmp(.jeq, dst, src, off); } + pub fn jgt(dst: Reg, src: anytype, off: i16) Insn { + return jmp(.jgt, dst, src, off); + } + + pub fn jge(dst: Reg, src: anytype, off: i16) Insn { + return jmp(.jge, dst, src, off); + } + + pub fn jlt(dst: Reg, src: anytype, off: i16) Insn { + return jmp(.jlt, dst, src, off); + } + + pub fn jle(dst: Reg, src: anytype, off: i16) Insn { + return jmp(.jle, dst, src, off); + } + + pub fn jset(dst: Reg, src: anytype, off: i16) Insn { + return jmp(.jset, dst, src, off); + } + + pub fn jne(dst: Reg, src: anytype, off: i16) Insn { + return jmp(.jne, dst, src, off); + } + + pub fn jsgt(dst: Reg, src: anytype, off: i16) Insn { + return jmp(.jsgt, dst, src, off); + } + + pub fn jsge(dst: Reg, src: anytype, off: i16) Insn { + return jmp(.jsge, dst, src, off); + } + + pub fn jslt(dst: Reg, src: anytype, off: i16) Insn { + return jmp(.jslt, dst, src, off); + } + + pub fn jsle(dst: Reg, src: anytype, off: i16) Insn { + return jmp(.jsle, dst, src, off); + } + pub fn stx_mem(size: Size, dst: Reg, src: Reg, off: i16) Insn { return Insn{ .code = STX | @enumToInt(size) | MEM, @@ -447,17 +556,34 @@ pub const Insn = packed struct { }; } - /// direct packet access, R0 = *(uint *)(skb->data + imm32) - pub fn ld_abs(size: Size, imm: i32) Insn { + fn ld(mode: Mode, size: Size, dst: Reg, src: Reg, imm: i32) Insn { return Insn{ - .code = LD | @enumToInt(size) | ABS, - .dst = 0, - .src = 0, + .code = @enumToInt(mode) | @enumToInt(size) | LD, + .dst = @enumToInt(dst), + .src = @enumToInt(src), .off = 0, .imm = imm, }; } + pub fn ld_abs(size: Size, dst: Reg, src: Reg, imm: i32) Insn { + return ld(.abs, size, dst, src, imm); + } + + pub fn ld_ind(size: Size, dst: Reg, src: Reg, imm: i32) Insn { + return ld(.ind, size, dst, src, imm); + } + + pub fn ldx(size: Size, dst: Reg, src: Reg, off: i16) Insn { + return Insn{ + .code = MEM | @enumToInt(size) | LDX, + .dst = @enumToInt(dst), + .src = @enumToInt(src), + .off = off, + .imm = 0, + }; + } + fn ld_imm_impl1(dst: Reg, src: Reg, imm: u64) Insn { return Insn{ .code = LD | DW | IMM, @@ -478,6 +604,14 @@ pub const Insn = packed struct { }; } + pub fn ld_dw1(dst: Reg, imm: u64) Insn { + return ld_imm_impl1(dst, .r0, imm); + } + + pub fn ld_dw2(imm: u64) Insn { + return ld_imm_impl2(imm); + } + pub fn ld_map_fd1(dst: Reg, map_fd: fd_t) Insn { return ld_imm_impl1(dst, @intToEnum(Reg, PSEUDO_MAP_FD), @intCast(u64, map_fd)); } @@ -486,6 +620,53 @@ pub const Insn = packed struct { return ld_imm_impl2(@intCast(u64, map_fd)); } + pub fn st(comptime size: Size, dst: Reg, off: i16, imm: i32) Insn { + if (size == .double_word) @compileError("TODO: implement st_dw"); + return Insn{ + .code = MEM | @enumToInt(size) | ST, + .dst = @enumToInt(dst), + .src = 0, + .off = off, + .imm = imm, + }; + } + + pub fn stx(size: Size, dst: Reg, off: i16, src: Reg) Insn { + return Insn{ + .code = MEM | @enumToInt(size) | STX, + .dst = @enumToInt(dst), + .src = @enumToInt(src), + .off = off, + .imm = 0, + }; + } + + fn endian_swap(endian: builtin.Endian, comptime size: Size, dst: Reg) Insn { + return Insn{ + .code = switch (endian) { + .Big => 0xdc, + .Little => 0xd4, + }, + .dst = @enumToInt(dst), + .src = 0, + .off = 0, + .imm = switch (size) { + .byte => @compileError("can't swap a single byte"), + .half_word => 16, + .word => 32, + .double_word => 64, + }, + }; + } + + pub fn le(comptime size: Size, dst: Reg) Insn { + return endian_swap(.Little, size, dst); + } + + pub fn be(comptime size: Size, dst: Reg) Insn { + return endian_swap(.Big, size, dst); + } + pub fn call(helper: Helper) Insn { return Insn{ .code = JMP | CALL, @@ -508,59 +689,122 @@ pub const Insn = packed struct { } }; -fn expect_insn(insn: Insn, val: u64) void { - expectEqual(@bitCast(u64, insn), val); -} - test "insn bitsize" { expectEqual(@bitSizeOf(Insn), 64); } -// mov instructions -test "mov imm" { - expect_insn(Insn.mov(.r1, 1), 0x00000001000001b7); +fn expect_opcode(code: u8, insn: Insn) void { + expectEqual(code, insn.code); } -test "mov reg" { - expect_insn(Insn.mov(.r6, .r1), 0x00000000000016bf); -} +// The opcodes were grabbed from https://github.com/iovisor/bpf-docs/blob/master/eBPF.md +test "opcodes" { + // instructions that have a name that end with 1 or 2 are consecutive for + // loading 64-bit immediates (imm is only 32 bits wide) -// alu instructions -test "add imm" { - expect_insn(Insn.add(.r2, -4), 0xfffffffc00000207); -} + // alu instructions + expect_opcode(0x07, Insn.add(.r1, 0)); + expect_opcode(0x0f, Insn.add(.r1, .r2)); + expect_opcode(0x17, Insn.sub(.r1, 0)); + expect_opcode(0x1f, Insn.sub(.r1, .r2)); + expect_opcode(0x27, Insn.mul(.r1, 0)); + expect_opcode(0x2f, Insn.mul(.r1, .r2)); + expect_opcode(0x37, Insn.div(.r1, 0)); + expect_opcode(0x3f, Insn.div(.r1, .r2)); + expect_opcode(0x47, Insn.alu_or(.r1, 0)); + expect_opcode(0x4f, Insn.alu_or(.r1, .r2)); + expect_opcode(0x57, Insn.alu_and(.r1, 0)); + expect_opcode(0x5f, Insn.alu_and(.r1, .r2)); + expect_opcode(0x67, Insn.lsh(.r1, 0)); + expect_opcode(0x6f, Insn.lsh(.r1, .r2)); + expect_opcode(0x77, Insn.rsh(.r1, 0)); + expect_opcode(0x7f, Insn.rsh(.r1, .r2)); + expect_opcode(0x87, Insn.neg(.r1)); + expect_opcode(0x97, Insn.mod(.r1, 0)); + expect_opcode(0x9f, Insn.mod(.r1, .r2)); + expect_opcode(0xa7, Insn.xor(.r1, 0)); + expect_opcode(0xaf, Insn.xor(.r1, .r2)); + expect_opcode(0xb7, Insn.mov(.r1, 0)); + expect_opcode(0xbf, Insn.mov(.r1, .r2)); + expect_opcode(0xc7, Insn.arsh(.r1, 0)); + expect_opcode(0xcf, Insn.arsh(.r1, .r2)); -// ld instructions -test "ld_abs" { - expect_insn(Insn.ld_abs(.byte, 42), 0x0000002a00000030); -} + // atomic instructions: might be more of these not documented in the wild + expect_opcode(0xdb, Insn.xadd(.r1, .r2)); -test "ld_map_fd" { - expect_insn(Insn.ld_map_fd1(.r1, 42), 0x0000002a00001118); - expect_insn(Insn.ld_map_fd2(42), 0x0000000000000000); -} + // TODO: byteswap instructions + expect_opcode(0xd4, Insn.le(.half_word, .r1)); + expectEqual(@intCast(i32, 16), Insn.le(.half_word, .r1).imm); + expect_opcode(0xd4, Insn.le(.word, .r1)); + expectEqual(@intCast(i32, 32), Insn.le(.word, .r1).imm); + expect_opcode(0xd4, Insn.le(.double_word, .r1)); + expectEqual(@intCast(i32, 64), Insn.le(.double_word, .r1).imm); + expect_opcode(0xdc, Insn.be(.half_word, .r1)); + expectEqual(@intCast(i32, 16), Insn.be(.half_word, .r1).imm); + expect_opcode(0xdc, Insn.be(.word, .r1)); + expectEqual(@intCast(i32, 32), Insn.be(.word, .r1).imm); + expect_opcode(0xdc, Insn.be(.double_word, .r1)); + expectEqual(@intCast(i32, 64), Insn.be(.double_word, .r1).imm); -// st instructions -test "stx_mem" { - expect_insn(Insn.stx_mem(.word, .r10, .r0, -4), 0x00000000fffc0a63); -} + // memory instructions + expect_opcode(0x18, Insn.ld_dw1(.r1, 0)); + expect_opcode(0x00, Insn.ld_dw2(0)); -test "xadd" { - expect_insn(Insn.xadd(.r0, .r1), 0x00000000000010db); -} + // loading a map fd + expect_opcode(0x18, Insn.ld_map_fd1(.r1, 0)); + expectEqual(@intCast(u4, PSEUDO_MAP_FD), Insn.ld_map_fd1(.r1, 0).src); + expect_opcode(0x00, Insn.ld_map_fd2(0)); -// jmp instructions -test "jeq imm" { - expect_insn(Insn.jeq(.r0, 0, 2), 0x0000000000020015); -} + expect_opcode(0x38, Insn.ld_abs(.double_word, .r1, .r2, 0)); + expect_opcode(0x20, Insn.ld_abs(.word, .r1, .r2, 0)); + expect_opcode(0x28, Insn.ld_abs(.half_word, .r1, .r2, 0)); + expect_opcode(0x30, Insn.ld_abs(.byte, .r1, .r2, 0)); -// other instructions -test "call" { - expect_insn(Insn.call(.map_lookup_elem), 0x0000000100000085); -} + expect_opcode(0x58, Insn.ld_ind(.double_word, .r1, .r2, 0)); + expect_opcode(0x40, Insn.ld_ind(.word, .r1, .r2, 0)); + expect_opcode(0x48, Insn.ld_ind(.half_word, .r1, .r2, 0)); + expect_opcode(0x50, Insn.ld_ind(.byte, .r1, .r2, 0)); -test "exit" { - expect_insn(Insn.exit(), 0x0000000000000095); + expect_opcode(0x79, Insn.ldx(.double_word, .r1, .r2, 0)); + expect_opcode(0x61, Insn.ldx(.word, .r1, .r2, 0)); + expect_opcode(0x69, Insn.ldx(.half_word, .r1, .r2, 0)); + expect_opcode(0x71, Insn.ldx(.byte, .r1, .r2, 0)); + + expect_opcode(0x62, Insn.st(.word, .r1, 0, 0)); + expect_opcode(0x6a, Insn.st(.half_word, .r1, 0, 0)); + expect_opcode(0x72, Insn.st(.byte, .r1, 0, 0)); + + expect_opcode(0x63, Insn.stx(.word, .r1, 0, .r2)); + expect_opcode(0x6b, Insn.stx(.half_word, .r1, 0, .r2)); + expect_opcode(0x73, Insn.stx(.byte, .r1, 0, .r2)); + expect_opcode(0x7b, Insn.stx(.double_word, .r1, 0, .r2)); + + // branch instructions + expect_opcode(0x05, Insn.ja(0)); + expect_opcode(0x15, Insn.jeq(.r1, 0, 0)); + expect_opcode(0x1d, Insn.jeq(.r1, .r2, 0)); + expect_opcode(0x25, Insn.jgt(.r1, 0, 0)); + expect_opcode(0x2d, Insn.jgt(.r1, .r2, 0)); + expect_opcode(0x35, Insn.jge(.r1, 0, 0)); + expect_opcode(0x3d, Insn.jge(.r1, .r2, 0)); + expect_opcode(0xa5, Insn.jlt(.r1, 0, 0)); + expect_opcode(0xad, Insn.jlt(.r1, .r2, 0)); + expect_opcode(0xb5, Insn.jle(.r1, 0, 0)); + expect_opcode(0xbd, Insn.jle(.r1, .r2, 0)); + expect_opcode(0x45, Insn.jset(.r1, 0, 0)); + expect_opcode(0x4d, Insn.jset(.r1, .r2, 0)); + expect_opcode(0x55, Insn.jne(.r1, 0, 0)); + expect_opcode(0x5d, Insn.jne(.r1, .r2, 0)); + expect_opcode(0x65, Insn.jsgt(.r1, 0, 0)); + expect_opcode(0x6d, Insn.jsgt(.r1, .r2, 0)); + expect_opcode(0x75, Insn.jsge(.r1, 0, 0)); + expect_opcode(0x7d, Insn.jsge(.r1, .r2, 0)); + expect_opcode(0xc5, Insn.jslt(.r1, 0, 0)); + expect_opcode(0xcd, Insn.jslt(.r1, .r2, 0)); + expect_opcode(0xd5, Insn.jsle(.r1, 0, 0)); + expect_opcode(0xdd, Insn.jsle(.r1, .r2, 0)); + expect_opcode(0x85, Insn.call(.unspec)); + expect_opcode(0x95, Insn.exit()); } pub const Cmd = extern enum(usize) { From a993c7dd1bf7d6ffa8e4a235ef2133e5d5bef254 Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Sun, 6 Sep 2020 16:19:49 -0700 Subject: [PATCH 06/89] removed redundant pseudo insn --- lib/std/os/linux/bpf.zig | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/std/os/linux/bpf.zig b/lib/std/os/linux/bpf.zig index e5227b384..0cef923f7 100644 --- a/lib/std/os/linux/bpf.zig +++ b/lib/std/os/linux/bpf.zig @@ -536,16 +536,6 @@ pub const Insn = packed struct { return jmp(.jsle, dst, src, off); } - pub fn stx_mem(size: Size, dst: Reg, src: Reg, off: i16) Insn { - return Insn{ - .code = STX | @enumToInt(size) | MEM, - .dst = @enumToInt(dst), - .src = @enumToInt(src), - .off = off, - .imm = 0, - }; - } - pub fn xadd(dst: Reg, src: Reg) Insn { return Insn{ .code = STX | XADD | DW, From cf06817768a347353735ddc908c74ca17c1d7d15 Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Sun, 6 Sep 2020 16:21:05 -0700 Subject: [PATCH 07/89] improved compile error message --- lib/std/os/linux/bpf.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/os/linux/bpf.zig b/lib/std/os/linux/bpf.zig index 0cef923f7..146bd04ce 100644 --- a/lib/std/os/linux/bpf.zig +++ b/lib/std/os/linux/bpf.zig @@ -611,7 +611,7 @@ pub const Insn = packed struct { } pub fn st(comptime size: Size, dst: Reg, off: i16, imm: i32) Insn { - if (size == .double_word) @compileError("TODO: implement st_dw"); + if (size == .double_word) @compileError("TODO: need to determine how to correctly handle double words"); return Insn{ .code = MEM | @enumToInt(size) | ST, .dst = @enumToInt(dst), From 67817b230f9da645eee22bdb98e715b1505c8f16 Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Sun, 6 Sep 2020 17:09:25 -0700 Subject: [PATCH 08/89] fixed improper builtin import --- lib/std/os/linux/bpf.zig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/std/os/linux/bpf.zig b/lib/std/os/linux/bpf.zig index 146bd04ce..226394980 100644 --- a/lib/std/os/linux/bpf.zig +++ b/lib/std/os/linux/bpf.zig @@ -5,7 +5,6 @@ // and substantial portions of the software. usingnamespace std.os; const std = @import("../../std.zig"); -const builtin = @import("builtin"); const expectEqual = std.testing.expectEqual; // instruction classes @@ -631,7 +630,7 @@ pub const Insn = packed struct { }; } - fn endian_swap(endian: builtin.Endian, comptime size: Size, dst: Reg) Insn { + fn endian_swap(endian: std.builtin.Endian, comptime size: Size, dst: Reg) Insn { return Insn{ .code = switch (endian) { .Big => 0xdc, From caaa9ad2afe7ca21ff48b9fd5c82be99057b46e4 Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Sun, 6 Sep 2020 21:44:16 -0700 Subject: [PATCH 09/89] added btf --- lib/std/os/linux/bpf.zig | 2 + lib/std/os/linux/bpf/btf.zig | 151 +++++++++++++++++++++++++++++++ lib/std/os/linux/bpf/btf_ext.zig | 19 ++++ 3 files changed, 172 insertions(+) create mode 100644 lib/std/os/linux/bpf/btf.zig create mode 100644 lib/std/os/linux/bpf/btf_ext.zig diff --git a/lib/std/os/linux/bpf.zig b/lib/std/os/linux/bpf.zig index 928c157c4..870e405f1 100644 --- a/lib/std/os/linux/bpf.zig +++ b/lib/std/os/linux/bpf.zig @@ -7,6 +7,8 @@ usingnamespace std.os; const std = @import("../../std.zig"); const expectEqual = std.testing.expectEqual; +pub const btf = @import("bpf/btf.zig"); + // instruction classes pub const LD = 0x00; pub const LDX = 0x01; diff --git a/lib/std/os/linux/bpf/btf.zig b/lib/std/os/linux/bpf/btf.zig new file mode 100644 index 000000000..36560416f --- /dev/null +++ b/lib/std/os/linux/bpf/btf.zig @@ -0,0 +1,151 @@ +const magic = 0xeb9f; +const version = 1; + +pub const ext = @import("ext.zig"); + +/// All offsets are in bytes relative to the end of this header +pub const Header = packed struct { + magic: u16, + version: u8, + flags: u8, + hdr_len: u32, + + /// offset of type section + type_off: u32, + + /// length of type section + type_len: u32, + + /// offset of string section + str_off: u32, + + /// length of string section + str_len: u32, +}; + +/// Max number of type identifiers +pub const max_type = 0xfffff; + +/// Max offset into string section +pub const max_name_offset = 0xffffff; + +/// Max number of struct/union/enum member of func args +pub const max_vlen = 0xffff; + +pub const Type = packed struct { + name_off: u32, + info: struct { + /// number of struct's members + vlen: u16, + + unused_1: u8, + kind: Kind, + unused_2: u3, + + /// used by Struct, Union, and Fwd + kind_flag: bool, + }, + + /// size is used by Int, Enum, Struct, Union, and DataSec, it tells the size + /// of the type it is describing + /// + /// type is used by Ptr, Typedef, Volatile, Const, Restrict, Func, + /// FuncProto, and Var. It is a type_id referring to another type + size_type: union { size: u32, typ: u32 }, +}; + +/// For some kinds, Type is immediately followed by extra data +pub const Kind = enum(u4) { + Unknown, + Int, + Ptr, + Array, + Struct, + Union, + Enum, + Fwd, + Typedef, + Volatile, + Const, + Restrict, + Func, + FuncProto, + Var, + DataSec, +}; + +/// Int kind is followed by this struct +pub const IntInfo = packed struct { + bits: u8, + unused: u8, + offset: u8, + encoding: enum(u4) { + Signed = 1 << 0, + Char = 1 << 1, + Bool = 1 << 2, + }, +}; + +test "IntInfo is 32 bits" { + std.testing.expectEqual(@bitSizeOf(IntInfo), 32); +} + +/// Enum kind is followed by this struct +pub const Enum = packed struct { + name_off: u32, + val: i32, +}; + +/// Array kind is followd by this struct +pub const Array = packed struct { + typ: u32, + index_type: u32, + nelems: u32, +}; + +/// Struct and Union kinds are followed by multiple Member structs. The exact +/// number is stored in vlen +pub const Member = packed struct { + name_off: u32, + typ: u32, + + /// if the kind_flag is set, offset contains both member bitfield size and + /// bit offset, the bitfield size is set for bitfield members. If the type + /// info kind_flag is not set, the offset contains only bit offset + offset: packed struct { + bit: u24, + bitfield_size: u8, + }, +}; + +/// FuncProto is followed by multiple Params, the exact number is stored in vlen +pub const Param = packed struct { + name_off: u32, + typ: u32, +}; + +pub const VarLinkage = enum { + Static, + GlobalAllocated, + GlobalExtern, +}; + +pub const FuncLinkage = enum { + Static, + Global, + Extern, +}; + +/// Var kind is followd by a single Var struct to describe additional +/// information related to the variable such as its linkage +pub const Var = packed struct { + linkage: u32, +}; + +/// Datasec kind is followed by multible VarSecInfo to describe all Var kind +/// types it contains along with it's in-section offset as well as size. +pub const VarSecInfo = packed struct { + typ: u32, + offset: u32, + size: u32, +}; diff --git a/lib/std/os/linux/bpf/btf_ext.zig b/lib/std/os/linux/bpf/btf_ext.zig new file mode 100644 index 000000000..05d5d5705 --- /dev/null +++ b/lib/std/os/linux/bpf/btf_ext.zig @@ -0,0 +1,19 @@ +pub const Header = packed struct { + magic: u16, + version: u8, + flags: u8, + hdr_len: u32, + + /// All offsets are in bytes relative to the end of this header + func_info_off: u32, + func_info_len: u32, + line_info_off: u32, + line_info_len: u32, +}; + +pub const InfoSec = packed struct { + sec_name_off: u32, + num_info: u32, + // TODO: communicate that there is data here + //data: [0]u8, +}; From e7547eeefae5a8f095351c6ae889ee9cf82720cb Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Sun, 6 Sep 2020 22:31:35 -0700 Subject: [PATCH 10/89] fixed missing 'packed' keyword --- lib/std/os/linux/bpf/btf.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/os/linux/bpf/btf.zig b/lib/std/os/linux/bpf/btf.zig index 36560416f..c99196e4a 100644 --- a/lib/std/os/linux/bpf/btf.zig +++ b/lib/std/os/linux/bpf/btf.zig @@ -34,7 +34,7 @@ pub const max_vlen = 0xffff; pub const Type = packed struct { name_off: u32, - info: struct { + info: packed struct { /// number of struct's members vlen: u16, From ff2ed966bb37079217ee7a7753cb63a763b8c3b5 Mon Sep 17 00:00:00 2001 From: Tadeo Kondrak Date: Wed, 26 Aug 2020 08:43:03 -0600 Subject: [PATCH 11/89] Implement @Type for Union This removes TypeInfo.UnionField.enum_field, which is redundant with TypeInfo.Union.tag_type. --- lib/std/builtin.zig | 1 - src/analyze.cpp | 300 ++++++++++++++++------------- src/ir.cpp | 102 ++++++++-- test/compile_errors.zig | 74 ++++++- test/stage1/behavior/type.zig | 61 ++++++ test/stage1/behavior/type_info.zig | 4 - 6 files changed, 380 insertions(+), 162 deletions(-) diff --git a/lib/std/builtin.zig b/lib/std/builtin.zig index 911a0eb15..52b8f641c 100644 --- a/lib/std/builtin.zig +++ b/lib/std/builtin.zig @@ -317,7 +317,6 @@ pub const TypeInfo = union(enum) { /// therefore must be kept in sync with the compiler implementation. pub const UnionField = struct { name: []const u8, - enum_field: ?EnumField, field_type: type, }; diff --git a/src/analyze.cpp b/src/analyze.cpp index b1d362f6e..b70e756f4 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -2372,7 +2372,10 @@ static Error resolve_union_alignment(CodeGen *g, ZigType *union_type) { if (field->gen_index == UINT32_MAX) continue; - AstNode *align_expr = field->decl_node->data.struct_field.align_expr; + AstNode *align_expr = nullptr; + if (union_type->data.unionation.decl_node->type == NodeTypeContainerDecl) { + align_expr = field->decl_node->data.struct_field.align_expr; + } if (align_expr != nullptr) { if (!analyze_const_align(g, &union_type->data.unionation.decls_scope->base, align_expr, &field->align)) @@ -2468,9 +2471,6 @@ static Error resolve_union_type(CodeGen *g, ZigType *union_type) { AstNode *decl_node = union_type->data.unionation.decl_node; - - assert(decl_node->type == NodeTypeContainerDecl); - uint32_t field_count = union_type->data.unionation.src_field_count; TypeUnionField *most_aligned_union_member = union_type->data.unionation.most_aligned_union_member; @@ -3055,7 +3055,6 @@ static Error resolve_union_zero_bits(CodeGen *g, ZigType *union_type) { return ErrorNone; AstNode *decl_node = union_type->data.unionation.decl_node; - assert(decl_node->type == NodeTypeContainerDecl); if (union_type->data.unionation.resolve_loop_flag_zero_bits) { if (union_type->data.unionation.resolve_status != ResolveStatusInvalid) { @@ -3069,30 +3068,50 @@ static Error resolve_union_zero_bits(CodeGen *g, ZigType *union_type) { union_type->data.unionation.resolve_loop_flag_zero_bits = true; - assert(union_type->data.unionation.fields == nullptr); - uint32_t field_count = (uint32_t)decl_node->data.container_decl.fields.length; - if (field_count == 0) { - add_node_error(g, decl_node, buf_sprintf("unions must have 1 or more fields")); + uint32_t field_count; + if (decl_node->type == NodeTypeContainerDecl) { + assert(union_type->data.unionation.fields == nullptr); + field_count = (uint32_t)decl_node->data.container_decl.fields.length; + if (field_count == 0) { + add_node_error(g, decl_node, buf_sprintf("unions must have 1 or more fields")); + union_type->data.unionation.src_field_count = field_count; + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; + } union_type->data.unionation.src_field_count = field_count; - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; + union_type->data.unionation.fields = heap::c_allocator.allocate(field_count); + union_type->data.unionation.fields_by_name.init(field_count); + } else { + assert(union_type->data.unionation.fields != nullptr); + field_count = union_type->data.unionation.src_field_count; } - union_type->data.unionation.src_field_count = field_count; - union_type->data.unionation.fields = heap::c_allocator.allocate(field_count); - union_type->data.unionation.fields_by_name.init(field_count); Scope *scope = &union_type->data.unionation.decls_scope->base; HashMap occupied_tag_values = {}; - AstNode *enum_type_node = decl_node->data.container_decl.init_arg_expr; - union_type->data.unionation.have_explicit_tag_type = decl_node->data.container_decl.auto_enum || - enum_type_node != nullptr; - bool auto_layout = (union_type->data.unionation.layout == ContainerLayoutAuto); - bool want_safety = (field_count >= 2) && (auto_layout || enum_type_node != nullptr) && !(g->build_mode == BuildModeFastRelease || g->build_mode == BuildModeSmallRelease); + bool is_auto_enum; // union(enum) or union(enum(expr)) + bool is_explicit_enum; // union(expr) + AstNode *enum_type_node; // expr in union(enum(expr)) or union(expr) + if (decl_node->type == NodeTypeContainerDecl) { + is_auto_enum = decl_node->data.container_decl.auto_enum; + is_explicit_enum = decl_node->data.container_decl.init_arg_expr != nullptr; + enum_type_node = decl_node->data.container_decl.init_arg_expr; + } else { + is_auto_enum = false; + is_explicit_enum = union_type->data.unionation.tag_type != nullptr; + enum_type_node = nullptr; + } + union_type->data.unionation.have_explicit_tag_type = is_auto_enum || is_explicit_enum; + + bool is_auto_layout = union_type->data.unionation.layout == ContainerLayoutAuto; + bool want_safety = (field_count >= 2) + && (is_auto_layout || is_explicit_enum) + && !(g->build_mode == BuildModeFastRelease || g->build_mode == BuildModeSmallRelease); ZigType *tag_type; - bool create_enum_type = decl_node->data.container_decl.auto_enum || (enum_type_node == nullptr && want_safety); + bool create_enum_type = is_auto_enum || (!is_explicit_enum && want_safety); bool *covered_enum_fields; + bool *is_zero_bits = heap::c_allocator.allocate(field_count); ZigLLVMDIEnumerator **di_enumerators; if (create_enum_type) { occupied_tag_values.init(field_count); @@ -3150,105 +3169,111 @@ static Error resolve_union_zero_bits(CodeGen *g, ZigType *union_type) { return err; } tag_type = enum_type; - covered_enum_fields = heap::c_allocator.allocate(enum_type->data.enumeration.src_field_count); } else { - tag_type = nullptr; + if (decl_node->type == NodeTypeContainerDecl) { + tag_type = nullptr; + } else { + tag_type = union_type->data.unionation.tag_type; + } + } + if (tag_type != nullptr) { + covered_enum_fields = heap::c_allocator.allocate(tag_type->data.enumeration.src_field_count); } union_type->data.unionation.tag_type = tag_type; - uint32_t gen_field_index = 0; for (uint32_t i = 0; i < field_count; i += 1) { - AstNode *field_node = decl_node->data.container_decl.fields.at(i); - Buf *field_name = field_node->data.struct_field.name; TypeUnionField *union_field = &union_type->data.unionation.fields[i]; - union_field->name = field_node->data.struct_field.name; - union_field->decl_node = field_node; - union_field->gen_index = UINT32_MAX; + if (decl_node->type == NodeTypeContainerDecl) { + AstNode *field_node = decl_node->data.container_decl.fields.at(i); + union_field->name = field_node->data.struct_field.name; + union_field->decl_node = field_node; + union_field->gen_index = UINT32_MAX; + is_zero_bits[i] = false; - auto field_entry = union_type->data.unionation.fields_by_name.put_unique(union_field->name, union_field); - if (field_entry != nullptr) { - ErrorMsg *msg = add_node_error(g, field_node, - buf_sprintf("duplicate union field: '%s'", buf_ptr(union_field->name))); - add_error_note(g, msg, field_entry->value->decl_node, buf_sprintf("other field here")); - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; - } - - bool field_is_zero_bits; - if (field_node->data.struct_field.type == nullptr) { - if (decl_node->data.container_decl.auto_enum || - decl_node->data.container_decl.init_arg_expr != nullptr) - { - union_field->type_entry = g->builtin_types.entry_void; - field_is_zero_bits = true; - } else { - add_node_error(g, field_node, buf_sprintf("union field missing type")); - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; - } - } else { - ZigValue *field_type_val = analyze_const_value(g, scope, - field_node->data.struct_field.type, g->builtin_types.entry_type, nullptr, LazyOkNoUndef); - if (type_is_invalid(field_type_val->type)) { - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; - } - assert(field_type_val->special != ConstValSpecialRuntime); - union_field->type_val = field_type_val; - if (union_type->data.unionation.resolve_status == ResolveStatusInvalid) - return ErrorSemanticAnalyzeFail; - - bool field_is_opaque_type; - if ((err = type_val_resolve_is_opaque_type(g, field_type_val, &field_is_opaque_type))) { - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; - } - if (field_is_opaque_type) { - add_node_error(g, field_node, - buf_create_from_str( - "opaque types have unknown size and therefore cannot be directly embedded in unions")); + auto field_entry = union_type->data.unionation.fields_by_name.put_unique(union_field->name, union_field); + if (field_entry != nullptr) { + ErrorMsg *msg = add_node_error(g, union_field->decl_node, + buf_sprintf("duplicate union field: '%s'", buf_ptr(union_field->name))); + add_error_note(g, msg, field_entry->value->decl_node, buf_sprintf("other field here")); union_type->data.unionation.resolve_status = ResolveStatusInvalid; return ErrorSemanticAnalyzeFail; } - switch (type_val_resolve_requires_comptime(g, field_type_val)) { - case ReqCompTimeInvalid: - if (g->trace_err != nullptr) { - g->trace_err = add_error_note(g, g->trace_err, field_node, - buf_create_from_str("while checking this field")); - } + if (field_node->data.struct_field.type == nullptr) { + if (is_auto_enum || is_explicit_enum) { + union_field->type_entry = g->builtin_types.entry_void; + is_zero_bits[i] = true; + } else { + add_node_error(g, field_node, buf_sprintf("union field missing type")); union_type->data.unionation.resolve_status = ResolveStatusInvalid; return ErrorSemanticAnalyzeFail; - case ReqCompTimeYes: - union_type->data.unionation.requires_comptime = true; - break; - case ReqCompTimeNo: - break; + } + } else { + ZigValue *field_type_val = analyze_const_value(g, scope, + field_node->data.struct_field.type, g->builtin_types.entry_type, nullptr, LazyOkNoUndef); + if (type_is_invalid(field_type_val->type)) { + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; + } + assert(field_type_val->special != ConstValSpecialRuntime); + union_field->type_val = field_type_val; + if (union_type->data.unionation.resolve_status == ResolveStatusInvalid) + return ErrorSemanticAnalyzeFail; + + bool field_is_opaque_type; + if ((err = type_val_resolve_is_opaque_type(g, field_type_val, &field_is_opaque_type))) { + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; + } + if (field_is_opaque_type) { + add_node_error(g, field_node, + buf_create_from_str( + "opaque types have unknown size and therefore cannot be directly embedded in unions")); + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; + } + + switch (type_val_resolve_requires_comptime(g, field_type_val)) { + case ReqCompTimeInvalid: + if (g->trace_err != nullptr) { + g->trace_err = add_error_note(g, g->trace_err, field_node, + buf_create_from_str("while checking this field")); + } + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; + case ReqCompTimeYes: + union_type->data.unionation.requires_comptime = true; + break; + case ReqCompTimeNo: + break; + } + + if ((err = type_val_resolve_zero_bits(g, field_type_val, union_type, nullptr, &is_zero_bits[i]))) { + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; + } } - if ((err = type_val_resolve_zero_bits(g, field_type_val, union_type, nullptr, &field_is_zero_bits))) { - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; + if (field_node->data.struct_field.value != nullptr && !is_auto_enum) { + ErrorMsg *msg = add_node_error(g, field_node->data.struct_field.value, + buf_create_from_str("untagged union field assignment")); + add_error_note(g, msg, decl_node, buf_create_from_str("consider 'union(enum)' here")); } } - if (field_node->data.struct_field.value != nullptr && !decl_node->data.container_decl.auto_enum) { - ErrorMsg *msg = add_node_error(g, field_node->data.struct_field.value, - buf_create_from_str("untagged union field assignment")); - add_error_note(g, msg, decl_node, buf_create_from_str("consider 'union(enum)' here")); - } - if (create_enum_type) { - di_enumerators[i] = ZigLLVMCreateDebugEnumerator(g->dbuilder, buf_ptr(field_name), i); + di_enumerators[i] = ZigLLVMCreateDebugEnumerator(g->dbuilder, buf_ptr(union_field->name), i); union_field->enum_field = &tag_type->data.enumeration.fields[i]; - union_field->enum_field->name = field_name; + union_field->enum_field->name = union_field->name; union_field->enum_field->decl_index = i; - union_field->enum_field->decl_node = field_node; + union_field->enum_field->decl_node = union_field->decl_node; auto prev_entry = tag_type->data.enumeration.fields_by_name.put_unique(union_field->enum_field->name, union_field->enum_field); assert(prev_entry == nullptr); // caught by union de-duplicator above - AstNode *tag_value = field_node->data.struct_field.value; + AstNode *tag_value = decl_node->type == NodeTypeContainerDecl + ? union_field->decl_node->data.struct_field.value : nullptr; + // In this first pass we resolve explicit tag values. // In a second pass we will fill in the unspecified ones. if (tag_value != nullptr) { @@ -3276,11 +3301,11 @@ static Error resolve_union_zero_bits(CodeGen *g, ZigType *union_type) { return ErrorSemanticAnalyzeFail; } } - } else if (enum_type_node != nullptr) { - union_field->enum_field = find_enum_type_field(tag_type, field_name); + } else if (tag_type != nullptr) { + union_field->enum_field = find_enum_type_field(tag_type, union_field->name); if (union_field->enum_field == nullptr) { - ErrorMsg *msg = add_node_error(g, field_node, - buf_sprintf("enum field not found: '%s'", buf_ptr(field_name))); + ErrorMsg *msg = add_node_error(g, union_field->decl_node, + buf_sprintf("enum field not found: '%s'", buf_ptr(union_field->name))); add_error_note(g, msg, tag_type->data.enumeration.decl_node, buf_sprintf("enum declared here")); union_type->data.unionation.resolve_status = ResolveStatusInvalid; @@ -3289,21 +3314,23 @@ static Error resolve_union_zero_bits(CodeGen *g, ZigType *union_type) { covered_enum_fields[union_field->enum_field->decl_index] = true; } else { union_field->enum_field = heap::c_allocator.create(); - union_field->enum_field->name = field_name; + union_field->enum_field->name = union_field->name; union_field->enum_field->decl_index = i; bigint_init_unsigned(&union_field->enum_field->value, i); } assert(union_field->enum_field != nullptr); - - if (field_is_zero_bits) - continue; - - union_field->gen_index = gen_field_index; - gen_field_index += 1; } - bool src_have_tag = decl_node->data.container_decl.auto_enum || - decl_node->data.container_decl.init_arg_expr != nullptr; + uint32_t gen_field_index = 0; + for (uint32_t i = 0; i < field_count; i += 1) { + TypeUnionField *union_field = &union_type->data.unionation.fields[i]; + if (!is_zero_bits[i]) { + union_field->gen_index = gen_field_index; + gen_field_index += 1; + } + } + + bool src_have_tag = is_auto_enum || is_explicit_enum; if (src_have_tag && union_type->data.unionation.layout != ContainerLayoutAuto) { const char *qual_str; @@ -3317,8 +3344,7 @@ static Error resolve_union_zero_bits(CodeGen *g, ZigType *union_type) { qual_str = "extern"; break; } - AstNode *source_node = (decl_node->data.container_decl.init_arg_expr != nullptr) ? - decl_node->data.container_decl.init_arg_expr : decl_node; + AstNode *source_node = enum_type_node != nullptr ? enum_type_node : decl_node; add_node_error(g, source_node, buf_sprintf("%s union does not support enum tag type", qual_str)); union_type->data.unionation.resolve_status = ResolveStatusInvalid; @@ -3326,43 +3352,47 @@ static Error resolve_union_zero_bits(CodeGen *g, ZigType *union_type) { } if (create_enum_type) { - // Now iterate again and populate the unspecified tag values - uint32_t next_maybe_unoccupied_index = 0; + if (decl_node->type == NodeTypeContainerDecl) { + // Now iterate again and populate the unspecified tag values + uint32_t next_maybe_unoccupied_index = 0; - for (uint32_t field_i = 0; field_i < field_count; field_i += 1) { - AstNode *field_node = decl_node->data.container_decl.fields.at(field_i); - TypeUnionField *union_field = &union_type->data.unionation.fields[field_i]; - AstNode *tag_value = field_node->data.struct_field.value; + for (uint32_t field_i = 0; field_i < field_count; field_i += 1) { + AstNode *field_node = decl_node->data.container_decl.fields.at(field_i); + TypeUnionField *union_field = &union_type->data.unionation.fields[field_i]; + AstNode *tag_value = field_node->data.struct_field.value; - if (tag_value == nullptr) { - if (occupied_tag_values.size() == 0) { - bigint_init_unsigned(&union_field->enum_field->value, next_maybe_unoccupied_index); - next_maybe_unoccupied_index += 1; - } else { - BigInt proposed_value; - for (;;) { - bigint_init_unsigned(&proposed_value, next_maybe_unoccupied_index); + if (tag_value == nullptr) { + if (occupied_tag_values.size() == 0) { + bigint_init_unsigned(&union_field->enum_field->value, next_maybe_unoccupied_index); next_maybe_unoccupied_index += 1; - auto entry = occupied_tag_values.put_unique(proposed_value, field_node); - if (entry != nullptr) { - continue; + } else { + BigInt proposed_value; + for (;;) { + bigint_init_unsigned(&proposed_value, next_maybe_unoccupied_index); + next_maybe_unoccupied_index += 1; + auto entry = occupied_tag_values.put_unique(proposed_value, field_node); + if (entry != nullptr) { + continue; + } + break; } - break; + bigint_init_bigint(&union_field->enum_field->value, &proposed_value); } - bigint_init_bigint(&union_field->enum_field->value, &proposed_value); } } } - } else if (enum_type_node != nullptr) { + } else if (tag_type != nullptr) { for (uint32_t i = 0; i < tag_type->data.enumeration.src_field_count; i += 1) { TypeEnumField *enum_field = &tag_type->data.enumeration.fields[i]; if (!covered_enum_fields[i]) { - AstNode *enum_decl_node = tag_type->data.enumeration.decl_node; - AstNode *field_node = enum_decl_node->data.container_decl.fields.at(i); ErrorMsg *msg = add_node_error(g, decl_node, buf_sprintf("enum field missing: '%s'", buf_ptr(enum_field->name))); - add_error_note(g, msg, field_node, - buf_sprintf("declared here")); + if (decl_node->type == NodeTypeContainerDecl) { + AstNode *enum_decl_node = tag_type->data.enumeration.decl_node; + AstNode *field_node = enum_decl_node->data.container_decl.fields.at(i); + add_error_note(g, msg, field_node, + buf_sprintf("declared here")); + } union_type->data.unionation.resolve_status = ResolveStatusInvalid; } } @@ -8350,7 +8380,7 @@ static void resolve_llvm_types_struct(CodeGen *g, ZigType *struct_type, ResolveS ZigLLVMDIFile *di_file; ZigLLVMDIScope *di_scope; unsigned line; - if (decl_node != nullptr && !struct_type->data.structure.created_by_at_type) { + if (decl_node != nullptr) { Scope *scope = &struct_type->data.structure.decls_scope->base; ZigType *import = get_scope_import(scope); di_file = import->data.structure.root_struct->di_file; @@ -8713,7 +8743,7 @@ static void resolve_llvm_types_union(CodeGen *g, ZigType *union_type, ResolveSta uint64_t store_size_in_bits = union_field->type_entry->size_in_bits; uint64_t abi_align_in_bits = 8*union_field->type_entry->abi_align; - AstNode *field_node = decl_node->data.container_decl.fields.at(i); + AstNode *field_node = union_field->decl_node; union_inner_di_types[union_field->gen_index] = ZigLLVMCreateDebugMemberType(g->dbuilder, ZigLLVMTypeToScope(union_type->llvm_di_type), buf_ptr(union_field->enum_field->name), import->data.structure.root_struct->di_file, (unsigned)(field_node->line + 1), diff --git a/src/ir.cpp b/src/ir.cpp index 803b97891..36852de70 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -25424,8 +25424,6 @@ static Error ir_make_type_info_value(IrAnalyze *ira, IrInst* source_instr, ZigTy init_const_slice(ira->codegen, fields[2], union_field_array, 0, union_field_count, false); - ZigType *type_info_enum_field_type = ir_type_info_get_type(ira, "EnumField", nullptr); - for (uint32_t union_field_index = 0; union_field_index < union_field_count; union_field_index++) { TypeUnionField *union_field = &type_entry->data.unionation.fields[union_field_index]; ZigValue *union_field_val = &union_field_array->data.x_array.data.s_none.elements[union_field_index]; @@ -25433,20 +25431,10 @@ static Error ir_make_type_info_value(IrAnalyze *ira, IrInst* source_instr, ZigTy union_field_val->special = ConstValSpecialStatic; union_field_val->type = type_info_union_field_type; - ZigValue **inner_fields = alloc_const_vals_ptrs(ira->codegen, 3); + ZigValue **inner_fields = alloc_const_vals_ptrs(ira->codegen, 2); inner_fields[1]->special = ConstValSpecialStatic; - inner_fields[1]->type = get_optional_type(ira->codegen, type_info_enum_field_type); - - if (fields[1]->data.x_optional == nullptr) { - inner_fields[1]->data.x_optional = nullptr; - } else { - inner_fields[1]->data.x_optional = ira->codegen->pass1_arena->create(); - make_enum_field_val(ira, inner_fields[1]->data.x_optional, union_field->enum_field, type_info_enum_field_type); - } - - inner_fields[2]->special = ConstValSpecialStatic; - inner_fields[2]->type = ira->codegen->builtin_types.entry_type; - inner_fields[2]->data.x_type = union_field->type_entry; + inner_fields[1]->type = ira->codegen->builtin_types.entry_type; + inner_fields[1]->data.x_type = union_field->type_entry; ZigValue *name = create_const_str_lit(ira->codegen, union_field->name)->data.x_ptr.data.ref.pointee; init_const_slice(ira->codegen, inner_fields[0], name, 0, buf_len(union_field->name), true); @@ -26102,7 +26090,8 @@ static ZigType *type_info_to_type(IrAnalyze *ira, IrInst *source_instr, ZigTypeI entry->data.structure.layout = layout; entry->data.structure.special = is_tuple ? StructSpecialInferredTuple : StructSpecialNone; entry->data.structure.created_by_at_type = true; - entry->data.structure.decls_scope = create_decls_scope(ira->codegen, nullptr, nullptr, entry, entry, &entry->name); + entry->data.structure.decls_scope = create_decls_scope( + ira->codegen, source_instr->source_node, source_instr->scope, entry, get_scope_import(source_instr->scope), &entry->name); assert(fields_ptr->data.x_ptr.special == ConstPtrSpecialBaseArray); assert(fields_ptr->data.x_ptr.data.base_array.elem_index == 0); @@ -26226,13 +26215,84 @@ static ZigType *type_info_to_type(IrAnalyze *ira, IrInst *source_instr, ZigTypeI return ira->codegen->invalid_inst_gen->value->type; field->value = *field_int_value; } - return entry; } - case ZigTypeIdUnion: - ir_add_error(ira, source_instr, buf_sprintf( - "TODO implement @Type for 'TypeInfo.%s': see https://github.com/ziglang/zig/issues/2907", type_id_name(tagTypeId))); - return ira->codegen->invalid_inst_gen->value->type; + case ZigTypeIdUnion: { + assert(payload->special == ConstValSpecialStatic); + assert(payload->type == ir_type_info_get_type(ira, "Union", nullptr)); + + ZigValue *layout_value = get_const_field(ira, source_instr->source_node, payload, "layout", 0); + assert(layout_value->special == ConstValSpecialStatic); + assert(layout_value->type == ir_type_info_get_type(ira, "ContainerLayout", nullptr)); + ContainerLayout layout = (ContainerLayout)bigint_as_u32(&layout_value->data.x_enum_tag); + + ZigType *tag_type = get_const_field_meta_type_optional(ira, source_instr->source_node, payload, "tag_type", 1); + if (tag_type != nullptr && tag_type->id != ZigTypeIdEnum) { + ir_add_error(ira, source_instr, buf_sprintf( + "union tag type must be an enum, not %s", type_id_name(tag_type->id))); + return ira->codegen->invalid_inst_gen->value->type; + } + + ZigValue *fields_value = get_const_field(ira, source_instr->source_node, payload, "fields", 2); + if (fields_value == nullptr) + return ira->codegen->invalid_inst_gen->value->type; + + assert(fields_value->special == ConstValSpecialStatic); + assert(is_slice(fields_value->type)); + ZigValue *fields_ptr = fields_value->data.x_struct.fields[slice_ptr_index]; + ZigValue *fields_len_value = fields_value->data.x_struct.fields[slice_len_index]; + size_t fields_len = bigint_as_usize(&fields_len_value->data.x_bigint); + + ZigValue *decls_value = get_const_field(ira, source_instr->source_node, payload, "decls", 3); + if (decls_value == nullptr) + return ira->codegen->invalid_inst_gen->value->type; + + assert(decls_value->special == ConstValSpecialStatic); + assert(is_slice(decls_value->type)); + ZigValue *decls_len_value = decls_value->data.x_struct.fields[slice_len_index]; + size_t decls_len = bigint_as_usize(&decls_len_value->data.x_bigint); + if (decls_len != 0) { + ir_add_error(ira, source_instr, buf_create_from_str("TypeInfo.Union.decls must be empty for @Type")); + return ira->codegen->invalid_inst_gen->value->type; + } + + ZigType *entry = new_type_table_entry(ZigTypeIdUnion); + buf_init_from_buf(&entry->name, + get_anon_type_name(ira->codegen, ira->old_irb.exec, "union", source_instr->scope, source_instr->source_node, &entry->name)); + entry->data.unionation.decl_node = source_instr->source_node; + entry->data.unionation.fields = heap::c_allocator.allocate(fields_len); + entry->data.unionation.fields_by_name.init(fields_len); + entry->data.unionation.decls_scope = create_decls_scope( + ira->codegen, source_instr->source_node, source_instr->scope, entry, get_scope_import(source_instr->scope), &entry->name); + entry->data.unionation.tag_type = tag_type; + entry->data.unionation.src_field_count = fields_len; + entry->data.unionation.layout = layout; + + assert(fields_ptr->data.x_ptr.special == ConstPtrSpecialBaseArray); + assert(fields_ptr->data.x_ptr.data.base_array.elem_index == 0); + ZigValue *fields_arr = fields_ptr->data.x_ptr.data.base_array.array_val; + assert(fields_arr->special == ConstValSpecialStatic); + assert(fields_arr->data.x_array.special == ConstArraySpecialNone); + for (size_t i = 0; i < fields_len; i++) { + ZigValue *field_value = &fields_arr->data.x_array.data.s_none.elements[i]; + assert(field_value->type == ir_type_info_get_type(ira, "UnionField", nullptr)); + TypeUnionField *field = &entry->data.unionation.fields[i]; + field->name = buf_alloc(); + if ((err = get_const_field_buf(ira, source_instr->source_node, field_value, "name", 0, field->name))) + return ira->codegen->invalid_inst_gen->value->type; + if (entry->data.unionation.fields_by_name.put_unique(field->name, field) != nullptr) { + ir_add_error(ira, source_instr, buf_sprintf("duplicate union field '%s'", buf_ptr(field->name))); + return ira->codegen->invalid_inst_gen->value->type; + } + field->decl_node = source_instr->source_node; + ZigValue *type_value = get_const_field(ira, source_instr->source_node, field_value, "field_type", 1); + if (type_value == nullptr) + return ira->codegen->invalid_inst_gen->value->type; + field->type_val = type_value; + field->type_entry = type_value->data.x_type; + } + return entry; + } case ZigTypeIdFn: case ZigTypeIdBoundFn: ir_add_error(ira, source_instr, buf_sprintf( diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 31f2b57dc..741c5fdb7 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -10,6 +10,78 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { "tmp.zig:2:37: error: expected type '[:1]const u8', found '*const [2:2]u8'", }); + cases.add("@Type for tagged union with extra union field", + \\const TypeInfo = @import("builtin").TypeInfo; + \\const Tag = @Type(.{ + \\ .Enum = .{ + \\ .layout = .Auto, + \\ .tag_type = u1, + \\ .fields = &[_]TypeInfo.EnumField{ + \\ .{ .name = "signed", .value = 0 }, + \\ .{ .name = "unsigned", .value = 1 }, + \\ }, + \\ .decls = &[_]TypeInfo.Declaration{}, + \\ .is_exhaustive = true, + \\ }, + \\}); + \\const Tagged = @Type(.{ + \\ .Union = .{ + \\ .layout = .Auto, + \\ .tag_type = Tag, + \\ .fields = &[_]TypeInfo.UnionField{ + \\ .{ .name = "signed", .field_type = i32 }, + \\ .{ .name = "unsigned", .field_type = u32 }, + \\ .{ .name = "arst", .field_type = f32 }, + \\ }, + \\ .decls = &[_]TypeInfo.Declaration{}, + \\ }, + \\}); + \\export fn entry() void { + \\ var tagged = Tagged{ .signed = -1 }; + \\ tagged = .{ .unsigned = 1 }; + \\} + , &[_][]const u8{ + "tmp.zig:14:23: error: enum field not found: 'arst'", + "tmp.zig:2:20: note: enum declared here", + "tmp.zig:27:24: note: referenced here", + }); + + cases.add("@Type for tagged union with extra enum field", + \\const TypeInfo = @import("builtin").TypeInfo; + \\const Tag = @Type(.{ + \\ .Enum = .{ + \\ .layout = .Auto, + \\ .tag_type = u2, + \\ .fields = &[_]TypeInfo.EnumField{ + \\ .{ .name = "signed", .value = 0 }, + \\ .{ .name = "unsigned", .value = 1 }, + \\ .{ .name = "arst", .field_type = 2 }, + \\ }, + \\ .decls = &[_]TypeInfo.Declaration{}, + \\ .is_exhaustive = true, + \\ }, + \\}); + \\const Tagged = @Type(.{ + \\ .Union = .{ + \\ .layout = .Auto, + \\ .tag_type = Tag, + \\ .fields = &[_]TypeInfo.UnionField{ + \\ .{ .name = "signed", .field_type = i32 }, + \\ .{ .name = "unsigned", .field_type = u32 }, + \\ }, + \\ .decls = &[_]TypeInfo.Declaration{}, + \\ }, + \\}); + \\export fn entry() void { + \\ var tagged = Tagged{ .signed = -1 }; + \\ tagged = .{ .unsigned = 1 }; + \\} + , &[_][]const u8{ + "tmp.zig:9:32: error: no member named 'field_type' in struct 'std.builtin.EnumField'", + "tmp.zig:18:21: note: referenced here", + "tmp.zig:27:18: note: referenced here", + }); + cases.add("@Type with undefined", \\comptime { \\ _ = @Type(.{ .Array = .{ .len = 0, .child = u8, .sentinel = undefined } }); @@ -7419,7 +7491,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { }); cases.add( // fixed bug #2032 - "compile diagnostic string for top level decl type", + "compile diagnostic string for top level decl type", \\export fn entry() void { \\ var foo: u32 = @This(){}; \\} diff --git a/test/stage1/behavior/type.zig b/test/stage1/behavior/type.zig index 81bd741ec..eac76c9f9 100644 --- a/test/stage1/behavior/type.zig +++ b/test/stage1/behavior/type.zig @@ -313,3 +313,64 @@ test "Type.Enum" { testing.expectEqual(@as(u32, 5), @enumToInt(Bar.b)); testing.expectEqual(@as(u32, 6), @enumToInt(@intToEnum(Bar, 6))); } + +test "Type.Union" { + const Untagged = @Type(.{ + .Union = .{ + .layout = .Auto, + .tag_type = null, + .fields = &[_]TypeInfo.UnionField{ + .{ .name = "int", .field_type = i32 }, + .{ .name = "float", .field_type = f32 }, + }, + .decls = &[_]TypeInfo.Declaration{}, + }, + }); + var untagged = Untagged{ .int = 1 }; + untagged.float = 2.0; + untagged.int = 3; + testing.expectEqual(@as(i32, 3), untagged.int); + + const PackedUntagged = @Type(.{ + .Union = .{ + .layout = .Packed, + .tag_type = null, + .fields = &[_]TypeInfo.UnionField{ + .{ .name = "signed", .field_type = i32 }, + .{ .name = "unsigned", .field_type = u32 }, + }, + .decls = &[_]TypeInfo.Declaration{}, + }, + }); + var packed_untagged = PackedUntagged{ .signed = -1 }; + testing.expectEqual(@as(i32, -1), packed_untagged.signed); + testing.expectEqual(~@as(u32, 0), packed_untagged.unsigned); + + const Tag = @Type(.{ + .Enum = .{ + .layout = .Auto, + .tag_type = u1, + .fields = &[_]TypeInfo.EnumField{ + .{ .name = "signed", .value = 0 }, + .{ .name = "unsigned", .value = 1 }, + }, + .decls = &[_]TypeInfo.Declaration{}, + .is_exhaustive = true, + }, + }); + const Tagged = @Type(.{ + .Union = .{ + .layout = .Auto, + .tag_type = Tag, + .fields = &[_]TypeInfo.UnionField{ + .{ .name = "signed", .field_type = i32 }, + .{ .name = "unsigned", .field_type = u32 }, + }, + .decls = &[_]TypeInfo.Declaration{}, + }, + }); + var tagged = Tagged{ .signed = -1 }; + testing.expectEqual(Tag.signed, tagged); + tagged = .{ .unsigned = 1 }; + testing.expectEqual(Tag.unsigned, tagged); +} diff --git a/test/stage1/behavior/type_info.zig b/test/stage1/behavior/type_info.zig index 409993a74..9e066d5f1 100644 --- a/test/stage1/behavior/type_info.zig +++ b/test/stage1/behavior/type_info.zig @@ -198,8 +198,6 @@ fn testUnion() void { expect(typeinfo_info.Union.layout == .Auto); expect(typeinfo_info.Union.tag_type.? == TypeId); expect(typeinfo_info.Union.fields.len == 25); - expect(typeinfo_info.Union.fields[4].enum_field != null); - expect(typeinfo_info.Union.fields[4].enum_field.?.value == 4); expect(typeinfo_info.Union.fields[4].field_type == @TypeOf(@typeInfo(u8).Int)); expect(typeinfo_info.Union.decls.len == 21); @@ -213,7 +211,6 @@ fn testUnion() void { expect(notag_union_info.Union.tag_type == null); expect(notag_union_info.Union.layout == .Auto); expect(notag_union_info.Union.fields.len == 2); - expect(notag_union_info.Union.fields[0].enum_field == null); expect(notag_union_info.Union.fields[1].field_type == u32); const TestExternUnion = extern union { @@ -223,7 +220,6 @@ fn testUnion() void { const extern_union_info = @typeInfo(TestExternUnion); expect(extern_union_info.Union.layout == .Extern); expect(extern_union_info.Union.tag_type == null); - expect(extern_union_info.Union.fields[0].enum_field == null); expect(extern_union_info.Union.fields[0].field_type == *c_void); } From ac19ccf5955488a43b29ea13675c33426aada430 Mon Sep 17 00:00:00 2001 From: Tadeo Kondrak Date: Wed, 26 Aug 2020 12:20:27 -0600 Subject: [PATCH 12/89] Update standard library for removal of TypeInfo.UnionField.enum_field --- lib/std/fmt.zig | 8 +++++++- lib/std/hash/auto_hash.zig | 11 ++++++++--- lib/std/io/serialization.zig | 18 ++++++++++++++++-- lib/std/json.zig | 16 ++++++++++++++-- lib/std/meta.zig | 24 +++++++++++++++++++----- 5 files changed, 64 insertions(+), 13 deletions(-) diff --git a/lib/std/fmt.zig b/lib/std/fmt.zig index a652bd8c2..3aa6278d5 100644 --- a/lib/std/fmt.zig +++ b/lib/std/fmt.zig @@ -395,11 +395,17 @@ pub fn formatType( } const info = @typeInfo(T).Union; if (info.tag_type) |UnionTagType| { + const tag_info = @typeInfo(UnionTagType).Enum; try writer.writeAll("{ ."); try writer.writeAll(@tagName(@as(UnionTagType, value))); try writer.writeAll(" = "); inline for (info.fields) |u_field| { - if (@enumToInt(@as(UnionTagType, value)) == u_field.enum_field.?.value) { + comptime var tag_value: @TagType(UnionTagType) = undefined; + inline for (tag_info.fields) |e_field| { + if (comptime mem.eql(u8, u_field.name, e_field.name)) + tag_value = e_field.value; + } + if (@enumToInt(@as(UnionTagType, value)) == tag_value) { try formatType(@field(value, u_field.name), fmt, options, writer, max_depth - 1); } } diff --git a/lib/std/hash/auto_hash.zig b/lib/std/hash/auto_hash.zig index 5877c77b5..c3beabe00 100644 --- a/lib/std/hash/auto_hash.zig +++ b/lib/std/hash/auto_hash.zig @@ -136,12 +136,17 @@ pub fn hash(hasher: anytype, key: anytype, comptime strat: HashStrategy) void { .Union => |info| { if (info.tag_type) |tag_type| { + const tag_info = @typeInfo(tag_type).Enum; const tag = meta.activeTag(key); const s = hash(hasher, tag, strat); inline for (info.fields) |field| { - const enum_field = field.enum_field.?; - if (enum_field.value == @enumToInt(tag)) { - hash(hasher, @field(key, enum_field.name), strat); + comptime var tag_value: @TagType(tag_type) = undefined; + inline for (tag_info.fields) |enum_field| { + if (comptime mem.eql(u8, field.name, enum_field.name)) + tag_value = enum_field.value; + } + if (@enumToInt(tag) == tag_value) { + hash(hasher, @field(key, field.name), strat); // TODO use a labelled break when it does not crash the compiler. cf #2908 // break :blk; return; diff --git a/lib/std/io/serialization.zig b/lib/std/io/serialization.zig index 925c929ce..fa5172e57 100644 --- a/lib/std/io/serialization.zig +++ b/lib/std/io/serialization.zig @@ -149,6 +149,7 @@ pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, .Union => { const info = @typeInfo(C).Union; if (info.tag_type) |TagType| { + const tag_info = @typeInfo(TagType).Enum; //we avoid duplicate iteration over the enum tags // by getting the int directly and casting it without // safety. If it is bad, it will be caught anyway. @@ -156,7 +157,13 @@ pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, const tag = try self.deserializeInt(TagInt); inline for (info.fields) |field_info| { - if (field_info.enum_field.?.value == tag) { + comptime var tag_value: TagInt = undefined; + inline for (tag_info.fields) |enum_field_info| { + if (comptime std.mem.eql(u8, field_info.name, enum_field_info.name)) { + tag_value = enum_field_info.value; + } + } + if (tag_value == tag) { const name = field_info.name; const FieldType = field_info.field_type; ptr.* = @unionInit(C, name, undefined); @@ -314,13 +321,20 @@ pub fn Serializer(comptime endian: builtin.Endian, comptime packing: Packing, co .Union => { const info = @typeInfo(T).Union; if (info.tag_type) |TagType| { + const tag_info = @typeInfo(TagType).Enum; const active_tag = meta.activeTag(value); try self.serialize(active_tag); //This inline loop is necessary because active_tag is a runtime // value, but @field requires a comptime value. Our alternative // is to check each field for a match inline for (info.fields) |field_info| { - if (field_info.enum_field.?.value == @enumToInt(active_tag)) { + comptime var tag_value: @TagType(TagType) = undefined; + inline for (tag_info.fields) |enum_field_info| { + if (comptime std.mem.eql(u8, field_info.name, enum_field_info.name)) { + tag_value = enum_field_info.value; + } + } + if (tag_value == @enumToInt(active_tag)) { const name = field_info.name; const FieldType = field_info.field_type; try self.serialize(@field(value, name)); diff --git a/lib/std/json.zig b/lib/std/json.zig index 2f8a70d0e..31f8a9112 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -1612,8 +1612,14 @@ pub fn parseFree(comptime T: type, value: T, options: ParseOptions) void { }, .Union => |unionInfo| { if (unionInfo.tag_type) |UnionTagType| { + const tag_info = @typeInfo(UnionTagType).Enum; inline for (unionInfo.fields) |u_field| { - if (@enumToInt(@as(UnionTagType, value)) == u_field.enum_field.?.value) { + comptime var tag_value: @TagType(UnionTagType) = undefined; + inline for (tag_info.fields) |e_field| { + if (comptime mem.eql(u8, u_field.name, e_field.name)) + tag_value = e_field.value; + } + if (@enumToInt(@as(UnionTagType, value)) == tag_value) { parseFree(u_field.field_type, @field(value, u_field.name), options); break; } @@ -2457,8 +2463,14 @@ pub fn stringify( const info = @typeInfo(T).Union; if (info.tag_type) |UnionTagType| { + const tag_info = @typeInfo(UnionTagType).Enum; inline for (info.fields) |u_field| { - if (@enumToInt(@as(UnionTagType, value)) == u_field.enum_field.?.value) { + comptime var tag_value: @TagType(UnionTagType) = undefined; + inline for (tag_info.fields) |e_field| { + if (comptime mem.eql(u8, u_field.name, e_field.name)) + tag_value = e_field.value; + } + if (@enumToInt(@as(UnionTagType, value)) == tag_value) { return try stringify(@field(value, u_field.name), options, out_stream); } } diff --git a/lib/std/meta.zig b/lib/std/meta.zig index 73e066149..8cb1ba155 100644 --- a/lib/std/meta.zig +++ b/lib/std/meta.zig @@ -465,10 +465,19 @@ pub fn TagPayloadType(comptime U: type, tag: @TagType(U)) type { testing.expect(trait.is(.Union)(U)); const info = @typeInfo(U).Union; + const tag_info = @typeInfo(@TagType(U)).Enum; + + comptime var name: []const u8 = undefined; + inline for (tag_info.fields) |enum_field_info| { + if (@enumToInt(tag) == enum_field_info.value) + name = enum_field_info.name; + } inline for (info.fields) |field_info| { - if (field_info.enum_field.?.value == @enumToInt(tag)) return field_info.field_type; + if (comptime mem.eql(u8, field_info.name, name)) + return field_info.field_type; } + unreachable; } @@ -504,15 +513,20 @@ pub fn eql(a: anytype, b: @TypeOf(a)) bool { } }, .Union => |info| { - if (info.tag_type) |_| { + if (info.tag_type) |Tag| { const tag_a = activeTag(a); const tag_b = activeTag(b); if (tag_a != tag_b) return false; + const tag_info = @typeInfo(Tag).Enum; inline for (info.fields) |field_info| { - const enum_field = field_info.enum_field.?; - if (enum_field.value == @enumToInt(tag_a)) { - return eql(@field(a, enum_field.name), @field(b, enum_field.name)); + comptime var tag_value: @TagType(Tag) = undefined; + inline for (tag_info.fields) |enum_field_info| { + if (comptime mem.eql(u8, field_info.name, enum_field_info.name)) + tag_value = enum_field_info.value; + } + if (tag_value == @enumToInt(tag_a)) { + return eql(@field(a, field_info.name), @field(b, field_info.name)); } } return false; From 771f35c59381dc6359640d7f572411d24e379240 Mon Sep 17 00:00:00 2001 From: Tadeo Kondrak Date: Wed, 26 Aug 2020 13:38:58 -0600 Subject: [PATCH 13/89] Use less inefficient method of replacing TypeInfo.UnionField.enum_field --- lib/std/fmt.zig | 8 +------- lib/std/hash/auto_hash.zig | 8 +------- lib/std/io/serialization.zig | 18 ++---------------- lib/std/json.zig | 16 ++-------------- lib/std/meta.zig | 16 ++-------------- 5 files changed, 8 insertions(+), 58 deletions(-) diff --git a/lib/std/fmt.zig b/lib/std/fmt.zig index 3aa6278d5..8d3173395 100644 --- a/lib/std/fmt.zig +++ b/lib/std/fmt.zig @@ -395,17 +395,11 @@ pub fn formatType( } const info = @typeInfo(T).Union; if (info.tag_type) |UnionTagType| { - const tag_info = @typeInfo(UnionTagType).Enum; try writer.writeAll("{ ."); try writer.writeAll(@tagName(@as(UnionTagType, value))); try writer.writeAll(" = "); inline for (info.fields) |u_field| { - comptime var tag_value: @TagType(UnionTagType) = undefined; - inline for (tag_info.fields) |e_field| { - if (comptime mem.eql(u8, u_field.name, e_field.name)) - tag_value = e_field.value; - } - if (@enumToInt(@as(UnionTagType, value)) == tag_value) { + if (value == @field(UnionTagType, u_field.name)) { try formatType(@field(value, u_field.name), fmt, options, writer, max_depth - 1); } } diff --git a/lib/std/hash/auto_hash.zig b/lib/std/hash/auto_hash.zig index c3beabe00..2e707d545 100644 --- a/lib/std/hash/auto_hash.zig +++ b/lib/std/hash/auto_hash.zig @@ -136,16 +136,10 @@ pub fn hash(hasher: anytype, key: anytype, comptime strat: HashStrategy) void { .Union => |info| { if (info.tag_type) |tag_type| { - const tag_info = @typeInfo(tag_type).Enum; const tag = meta.activeTag(key); const s = hash(hasher, tag, strat); inline for (info.fields) |field| { - comptime var tag_value: @TagType(tag_type) = undefined; - inline for (tag_info.fields) |enum_field| { - if (comptime mem.eql(u8, field.name, enum_field.name)) - tag_value = enum_field.value; - } - if (@enumToInt(tag) == tag_value) { + if (@field(tag_type, field.name) == tag) { hash(hasher, @field(key, field.name), strat); // TODO use a labelled break when it does not crash the compiler. cf #2908 // break :blk; diff --git a/lib/std/io/serialization.zig b/lib/std/io/serialization.zig index fa5172e57..79a12989b 100644 --- a/lib/std/io/serialization.zig +++ b/lib/std/io/serialization.zig @@ -149,7 +149,6 @@ pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, .Union => { const info = @typeInfo(C).Union; if (info.tag_type) |TagType| { - const tag_info = @typeInfo(TagType).Enum; //we avoid duplicate iteration over the enum tags // by getting the int directly and casting it without // safety. If it is bad, it will be caught anyway. @@ -157,13 +156,7 @@ pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, const tag = try self.deserializeInt(TagInt); inline for (info.fields) |field_info| { - comptime var tag_value: TagInt = undefined; - inline for (tag_info.fields) |enum_field_info| { - if (comptime std.mem.eql(u8, field_info.name, enum_field_info.name)) { - tag_value = enum_field_info.value; - } - } - if (tag_value == tag) { + if (@enumToInt(@field(TagType, field_info.name)) == tag) { const name = field_info.name; const FieldType = field_info.field_type; ptr.* = @unionInit(C, name, undefined); @@ -321,20 +314,13 @@ pub fn Serializer(comptime endian: builtin.Endian, comptime packing: Packing, co .Union => { const info = @typeInfo(T).Union; if (info.tag_type) |TagType| { - const tag_info = @typeInfo(TagType).Enum; const active_tag = meta.activeTag(value); try self.serialize(active_tag); //This inline loop is necessary because active_tag is a runtime // value, but @field requires a comptime value. Our alternative // is to check each field for a match inline for (info.fields) |field_info| { - comptime var tag_value: @TagType(TagType) = undefined; - inline for (tag_info.fields) |enum_field_info| { - if (comptime std.mem.eql(u8, field_info.name, enum_field_info.name)) { - tag_value = enum_field_info.value; - } - } - if (tag_value == @enumToInt(active_tag)) { + if (@field(TagType, field_info.name) == active_tag) { const name = field_info.name; const FieldType = field_info.field_type; try self.serialize(@field(value, name)); diff --git a/lib/std/json.zig b/lib/std/json.zig index 31f8a9112..cf479ab2c 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -1612,14 +1612,8 @@ pub fn parseFree(comptime T: type, value: T, options: ParseOptions) void { }, .Union => |unionInfo| { if (unionInfo.tag_type) |UnionTagType| { - const tag_info = @typeInfo(UnionTagType).Enum; inline for (unionInfo.fields) |u_field| { - comptime var tag_value: @TagType(UnionTagType) = undefined; - inline for (tag_info.fields) |e_field| { - if (comptime mem.eql(u8, u_field.name, e_field.name)) - tag_value = e_field.value; - } - if (@enumToInt(@as(UnionTagType, value)) == tag_value) { + if (value == @field(UnionTagType, u_field.name)) { parseFree(u_field.field_type, @field(value, u_field.name), options); break; } @@ -2463,14 +2457,8 @@ pub fn stringify( const info = @typeInfo(T).Union; if (info.tag_type) |UnionTagType| { - const tag_info = @typeInfo(UnionTagType).Enum; inline for (info.fields) |u_field| { - comptime var tag_value: @TagType(UnionTagType) = undefined; - inline for (tag_info.fields) |e_field| { - if (comptime mem.eql(u8, u_field.name, e_field.name)) - tag_value = e_field.value; - } - if (@enumToInt(@as(UnionTagType, value)) == tag_value) { + if (value == @field(UnionTagType, u_field.name)) { return try stringify(@field(value, u_field.name), options, out_stream); } } diff --git a/lib/std/meta.zig b/lib/std/meta.zig index 8cb1ba155..b27f168ac 100644 --- a/lib/std/meta.zig +++ b/lib/std/meta.zig @@ -467,14 +467,8 @@ pub fn TagPayloadType(comptime U: type, tag: @TagType(U)) type { const info = @typeInfo(U).Union; const tag_info = @typeInfo(@TagType(U)).Enum; - comptime var name: []const u8 = undefined; - inline for (tag_info.fields) |enum_field_info| { - if (@enumToInt(tag) == enum_field_info.value) - name = enum_field_info.name; - } - inline for (info.fields) |field_info| { - if (comptime mem.eql(u8, field_info.name, name)) + if (comptime mem.eql(u8, field_info.name, @tagName(tag))) return field_info.field_type; } @@ -518,14 +512,8 @@ pub fn eql(a: anytype, b: @TypeOf(a)) bool { const tag_b = activeTag(b); if (tag_a != tag_b) return false; - const tag_info = @typeInfo(Tag).Enum; inline for (info.fields) |field_info| { - comptime var tag_value: @TagType(Tag) = undefined; - inline for (tag_info.fields) |enum_field_info| { - if (comptime mem.eql(u8, field_info.name, enum_field_info.name)) - tag_value = enum_field_info.value; - } - if (tag_value == @enumToInt(tag_a)) { + if (@field(Tag, field_info.name) == tag_a) { return eql(@field(a, field_info.name), @field(b, field_info.name)); } } From acdf1f0bde9a07cae2fbe33739f5f4aee7989f7b Mon Sep 17 00:00:00 2001 From: Tadeo Kondrak Date: Thu, 27 Aug 2020 13:34:02 -0600 Subject: [PATCH 14/89] @Type for union fixes --- src/analyze.cpp | 103 ++++++++++++++++++++-------------------- src/ir.cpp | 7 ++- test/compile_errors.zig | 57 ++++++++++++++++++++++ 3 files changed, 115 insertions(+), 52 deletions(-) diff --git a/src/analyze.cpp b/src/analyze.cpp index b70e756f4..6086ff2bf 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -2603,16 +2603,16 @@ static Error resolve_enum_zero_bits(CodeGen *g, ZigType *enum_type) { if (decl_node->type == NodeTypeContainerDecl) { assert(!enum_type->data.enumeration.fields); field_count = (uint32_t)decl_node->data.container_decl.fields.length; - if (field_count == 0) { - add_node_error(g, decl_node, buf_sprintf("enums must have 1 or more fields")); - - enum_type->data.enumeration.src_field_count = field_count; - enum_type->data.enumeration.fields = nullptr; - enum_type->data.enumeration.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; - } } else { - field_count = enum_type->data.enumeration.src_field_count; + field_count = enum_type->data.enumeration.src_field_count + enum_type->data.enumeration.non_exhaustive; + } + + if (field_count == 0) { + add_node_error(g, decl_node, buf_sprintf("enums must have 1 or more fields")); + enum_type->data.enumeration.src_field_count = field_count; + enum_type->data.enumeration.fields = nullptr; + enum_type->data.enumeration.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; } Scope *scope = &enum_type->data.enumeration.decls_scope->base; @@ -3072,18 +3072,19 @@ static Error resolve_union_zero_bits(CodeGen *g, ZigType *union_type) { if (decl_node->type == NodeTypeContainerDecl) { assert(union_type->data.unionation.fields == nullptr); field_count = (uint32_t)decl_node->data.container_decl.fields.length; - if (field_count == 0) { - add_node_error(g, decl_node, buf_sprintf("unions must have 1 or more fields")); - union_type->data.unionation.src_field_count = field_count; - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; - } union_type->data.unionation.src_field_count = field_count; union_type->data.unionation.fields = heap::c_allocator.allocate(field_count); union_type->data.unionation.fields_by_name.init(field_count); } else { - assert(union_type->data.unionation.fields != nullptr); field_count = union_type->data.unionation.src_field_count; + assert(field_count == 0 || union_type->data.unionation.fields != nullptr); + } + + if (field_count == 0) { + add_node_error(g, decl_node, buf_sprintf("unions must have 1 or more fields")); + union_type->data.unionation.src_field_count = field_count; + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; } Scope *scope = &union_type->data.unionation.decls_scope->base; @@ -3217,41 +3218,6 @@ static Error resolve_union_zero_bits(CodeGen *g, ZigType *union_type) { } assert(field_type_val->special != ConstValSpecialRuntime); union_field->type_val = field_type_val; - if (union_type->data.unionation.resolve_status == ResolveStatusInvalid) - return ErrorSemanticAnalyzeFail; - - bool field_is_opaque_type; - if ((err = type_val_resolve_is_opaque_type(g, field_type_val, &field_is_opaque_type))) { - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; - } - if (field_is_opaque_type) { - add_node_error(g, field_node, - buf_create_from_str( - "opaque types have unknown size and therefore cannot be directly embedded in unions")); - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; - } - - switch (type_val_resolve_requires_comptime(g, field_type_val)) { - case ReqCompTimeInvalid: - if (g->trace_err != nullptr) { - g->trace_err = add_error_note(g, g->trace_err, field_node, - buf_create_from_str("while checking this field")); - } - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; - case ReqCompTimeYes: - union_type->data.unionation.requires_comptime = true; - break; - case ReqCompTimeNo: - break; - } - - if ((err = type_val_resolve_zero_bits(g, field_type_val, union_type, nullptr, &is_zero_bits[i]))) { - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; - } } if (field_node->data.struct_field.value != nullptr && !is_auto_enum) { @@ -3261,6 +3227,41 @@ static Error resolve_union_zero_bits(CodeGen *g, ZigType *union_type) { } } + if (union_field->type_val != nullptr) { + bool field_is_opaque_type; + if ((err = type_val_resolve_is_opaque_type(g, union_field->type_val, &field_is_opaque_type))) { + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; + } + if (field_is_opaque_type) { + add_node_error(g, union_field->decl_node, + buf_create_from_str( + "opaque types have unknown size and therefore cannot be directly embedded in unions")); + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; + } + + switch (type_val_resolve_requires_comptime(g, union_field->type_val)) { + case ReqCompTimeInvalid: + if (g->trace_err != nullptr) { + g->trace_err = add_error_note(g, g->trace_err, union_field->decl_node, + buf_create_from_str("while checking this field")); + } + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; + case ReqCompTimeYes: + union_type->data.unionation.requires_comptime = true; + break; + case ReqCompTimeNo: + break; + } + + if ((err = type_val_resolve_zero_bits(g, union_field->type_val, union_type, nullptr, &is_zero_bits[i]))) { + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; + } + } + if (create_enum_type) { di_enumerators[i] = ZigLLVMCreateDebugEnumerator(g->dbuilder, buf_ptr(union_field->name), i); union_field->enum_field = &tag_type->data.enumeration.fields[i]; diff --git a/src/ir.cpp b/src/ir.cpp index 36852de70..6cb5d8bc2 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -26222,14 +26222,19 @@ static ZigType *type_info_to_type(IrAnalyze *ira, IrInst *source_instr, ZigTypeI assert(payload->type == ir_type_info_get_type(ira, "Union", nullptr)); ZigValue *layout_value = get_const_field(ira, source_instr->source_node, payload, "layout", 0); + if (layout_value == nullptr) + return ira->codegen->invalid_inst_gen->value->type; assert(layout_value->special == ConstValSpecialStatic); assert(layout_value->type == ir_type_info_get_type(ira, "ContainerLayout", nullptr)); ContainerLayout layout = (ContainerLayout)bigint_as_u32(&layout_value->data.x_enum_tag); ZigType *tag_type = get_const_field_meta_type_optional(ira, source_instr->source_node, payload, "tag_type", 1); + if (tag_type != nullptr && type_is_invalid(tag_type)) { + return ira->codegen->invalid_inst_gen->value->type; + } if (tag_type != nullptr && tag_type->id != ZigTypeIdEnum) { ir_add_error(ira, source_instr, buf_sprintf( - "union tag type must be an enum, not %s", type_id_name(tag_type->id))); + "expected enum type, found '%s'", type_id_name(tag_type->id))); return ira->codegen->invalid_inst_gen->value->type; } diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 741c5fdb7..f457c7460 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -10,6 +10,63 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { "tmp.zig:2:37: error: expected type '[:1]const u8', found '*const [2:2]u8'", }); + cases.add("@Type for union with opaque field", + \\const TypeInfo = @import("builtin").TypeInfo; + \\const Untagged = @Type(.{ + \\ .Union = .{ + \\ .layout = .Auto, + \\ .tag_type = null, + \\ .fields = &[_]TypeInfo.UnionField{ + \\ .{ .name = "foo", .field_type = @Type(.Opaque) }, + \\ }, + \\ .decls = &[_]TypeInfo.Declaration{}, + \\ }, + \\}); + \\export fn entry() void { + \\ _ = Untagged{}; + \\} + , &[_][]const u8{ + "tmp.zig:2:25: error: opaque types have unknown size and therefore cannot be directly embedded in unions", + "tmp.zig:13:17: note: referenced here", + }); + + cases.add("@Type for union with zero fields", + \\const TypeInfo = @import("builtin").TypeInfo; + \\const Untagged = @Type(.{ + \\ .Union = .{ + \\ .layout = .Auto, + \\ .tag_type = null, + \\ .fields = &[_]TypeInfo.UnionField{}, + \\ .decls = &[_]TypeInfo.Declaration{}, + \\ }, + \\}); + \\export fn entry() void { + \\ _ = Untagged{}; + \\} + , &[_][]const u8{ + "tmp.zig:2:25: error: unions must have 1 or more fields", + "tmp.zig:11:17: note: referenced here", + }); + + cases.add("@Type for exhaustive enum with zero fields", + \\const TypeInfo = @import("builtin").TypeInfo; + \\const Tag = @Type(.{ + \\ .Enum = .{ + \\ .layout = .Auto, + \\ .tag_type = u1, + \\ .fields = &[_]TypeInfo.EnumField{}, + \\ .decls = &[_]TypeInfo.Declaration{}, + \\ .is_exhaustive = true, + \\ }, + \\}); + \\export fn entry() void { + \\ _ = @intToEnum(Tag, 0); + \\} + , &[_][]const u8{ + "tmp.zig:2:20: error: enums must have 1 or more fields", + "tmp.zig:12:9: note: referenced here", + }); + cases.add("@Type for tagged union with extra union field", \\const TypeInfo = @import("builtin").TypeInfo; \\const Tag = @Type(.{ From cf3194804e1ac5654ba4082f615d57cff776b3a9 Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Mon, 7 Sep 2020 07:17:21 -0700 Subject: [PATCH 15/89] changed enums to lower case --- lib/std/os/linux/bpf/btf.zig | 50 ++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/std/os/linux/bpf/btf.zig b/lib/std/os/linux/bpf/btf.zig index c99196e4a..cd086c97f 100644 --- a/lib/std/os/linux/bpf/btf.zig +++ b/lib/std/os/linux/bpf/btf.zig @@ -56,22 +56,22 @@ pub const Type = packed struct { /// For some kinds, Type is immediately followed by extra data pub const Kind = enum(u4) { - Unknown, - Int, - Ptr, - Array, - Struct, - Union, - Enum, - Fwd, - Typedef, - Volatile, - Const, - Restrict, - Func, - FuncProto, - Var, - DataSec, + unknown, + int, + ptr, + array, + structure, + kind_union, + enumeration, + fwd, + typedef, + kind_volatile, + constant, + restrict, + func, + funcProto, + variable, + dataSec, }; /// Int kind is followed by this struct @@ -80,9 +80,9 @@ pub const IntInfo = packed struct { unused: u8, offset: u8, encoding: enum(u4) { - Signed = 1 << 0, - Char = 1 << 1, - Bool = 1 << 2, + signed = 1 << 0, + char = 1 << 1, + boolean = 1 << 2, }, }; @@ -125,15 +125,15 @@ pub const Param = packed struct { }; pub const VarLinkage = enum { - Static, - GlobalAllocated, - GlobalExtern, + static, + global_allocated, + global_extern, }; pub const FuncLinkage = enum { - Static, - Global, - Extern, + static, + global, + external, }; /// Var kind is followd by a single Var struct to describe additional From b878a64a5f44072b11dd0f73fc2b808c41f0d172 Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Mon, 7 Sep 2020 12:26:41 -0700 Subject: [PATCH 16/89] added license comments --- lib/std/os/linux/bpf/btf.zig | 5 +++++ lib/std/os/linux/bpf/btf_ext.zig | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/lib/std/os/linux/bpf/btf.zig b/lib/std/os/linux/bpf/btf.zig index cd086c97f..5338994af 100644 --- a/lib/std/os/linux/bpf/btf.zig +++ b/lib/std/os/linux/bpf/btf.zig @@ -1,3 +1,8 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. const magic = 0xeb9f; const version = 1; diff --git a/lib/std/os/linux/bpf/btf_ext.zig b/lib/std/os/linux/bpf/btf_ext.zig index 05d5d5705..ce412fdf4 100644 --- a/lib/std/os/linux/bpf/btf_ext.zig +++ b/lib/std/os/linux/bpf/btf_ext.zig @@ -1,3 +1,8 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. pub const Header = packed struct { magic: u16, version: u8, From db7a2382977a12e3ad95e3f2249c538d9c31cd87 Mon Sep 17 00:00:00 2001 From: Matthew Knight Date: Mon, 7 Sep 2020 12:41:29 -0700 Subject: [PATCH 17/89] BPF: add some more documentation (#6268) * added documentation for ringbuffers, which context type maps to which program type, and added some formatting --- lib/std/os/linux/bpf.zig | 288 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 284 insertions(+), 4 deletions(-) diff --git a/lib/std/os/linux/bpf.zig b/lib/std/os/linux/bpf.zig index 226394980..df5274a82 100644 --- a/lib/std/os/linux/bpf.zig +++ b/lib/std/os/linux/bpf.zig @@ -62,6 +62,7 @@ pub const MAXINSNS = 4096; // instruction classes /// jmp mode in word width pub const JMP32 = 0x06; + /// alu mode in double word width pub const ALU64 = 0x07; @@ -72,14 +73,17 @@ pub const XADD = 0xc0; // alu/jmp fields /// mov reg to reg pub const MOV = 0xb0; + /// sign extending arithmetic shift right */ pub const ARSH = 0xc0; // change endianness of a register /// flags for endianness conversion: pub const END = 0xd0; + /// convert to little-endian */ pub const TO_LE = 0x00; + /// convert to big-endian pub const TO_BE = 0x08; pub const FROM_LE = TO_LE; @@ -88,29 +92,39 @@ pub const FROM_BE = TO_BE; // jmp encodings /// jump != * pub const JNE = 0x50; + /// LT is unsigned, '<' pub const JLT = 0xa0; + /// LE is unsigned, '<=' * pub const JLE = 0xb0; + /// SGT is signed '>', GT in x86 pub const JSGT = 0x60; + /// SGE is signed '>=', GE in x86 pub const JSGE = 0x70; + /// SLT is signed, '<' pub const JSLT = 0xc0; + /// SLE is signed, '<=' pub const JSLE = 0xd0; + /// function call pub const CALL = 0x80; + /// function return pub const EXIT = 0x90; /// Flag for prog_attach command. If a sub-cgroup installs some bpf program, the /// program in this cgroup yields to sub-cgroup program. pub const F_ALLOW_OVERRIDE = 0x1; + /// Flag for prog_attach command. If a sub-cgroup installs some bpf program, /// that cgroup program gets run in addition to the program in this cgroup. pub const F_ALLOW_MULTI = 0x2; + /// Flag for prog_attach command. pub const F_REPLACE = 0x4; @@ -164,47 +178,61 @@ pub const PSEUDO_CALL = 1; /// flag for BPF_MAP_UPDATE_ELEM command. create new element or update existing pub const ANY = 0; + /// flag for BPF_MAP_UPDATE_ELEM command. create new element if it didn't exist pub const NOEXIST = 1; + /// flag for BPF_MAP_UPDATE_ELEM command. update existing element pub const EXIST = 2; + /// flag for BPF_MAP_UPDATE_ELEM command. spin_lock-ed map_lookup/map_update pub const F_LOCK = 4; /// flag for BPF_MAP_CREATE command */ pub const BPF_F_NO_PREALLOC = 0x1; + /// flag for BPF_MAP_CREATE command. Instead of having one common LRU list in /// the BPF_MAP_TYPE_LRU_[PERCPU_]HASH map, use a percpu LRU list which can /// scale and perform better. Note, the LRU nodes (including free nodes) cannot /// be moved across different LRU lists. pub const BPF_F_NO_COMMON_LRU = 0x2; + /// flag for BPF_MAP_CREATE command. Specify numa node during map creation pub const BPF_F_NUMA_NODE = 0x4; + /// flag for BPF_MAP_CREATE command. Flags for BPF object read access from /// syscall side pub const BPF_F_RDONLY = 0x8; + /// flag for BPF_MAP_CREATE command. Flags for BPF object write access from /// syscall side pub const BPF_F_WRONLY = 0x10; + /// flag for BPF_MAP_CREATE command. Flag for stack_map, store build_id+offset /// instead of pointer pub const BPF_F_STACK_BUILD_ID = 0x20; + /// flag for BPF_MAP_CREATE command. Zero-initialize hash function seed. This /// should only be used for testing. pub const BPF_F_ZERO_SEED = 0x40; + /// flag for BPF_MAP_CREATE command Flags for accessing BPF object from program /// side. pub const BPF_F_RDONLY_PROG = 0x80; + /// flag for BPF_MAP_CREATE command. Flags for accessing BPF object from program /// side. pub const BPF_F_WRONLY_PROG = 0x100; + /// flag for BPF_MAP_CREATE command. Clone map from listener for newly accepted /// socket pub const BPF_F_CLONE = 0x200; + /// flag for BPF_MAP_CREATE command. Enable memory-mapping BPF map pub const BPF_F_MMAPABLE = 0x400; -/// These values correspond to "syscalls" within the BPF program's environment +/// These values correspond to "syscalls" within the BPF program's environment, +/// each one is documented in std.os.linux.BPF.kern pub const Helper = enum(i32) { unspec, map_lookup_elem, @@ -325,6 +353,29 @@ pub const Helper = enum(i32) { tcp_send_ack, send_signal_thread, jiffies64, + read_branch_records, + get_ns_current_pid_tgid, + xdp_output, + get_netns_cookie, + get_current_ancestor_cgroup_id, + sk_assign, + ktime_get_boot_ns, + seq_printf, + seq_write, + sk_cgroup_id, + sk_ancestor_cgroup_id, + ringbuf_output, + ringbuf_reserve, + ringbuf_submit, + ringbuf_discard, + ringbuf_query, + csum_level, + skc_to_tcp6_sock, + skc_to_tcp_sock, + skc_to_tcp_timewait_sock, + skc_to_tcp_request_sock, + skc_to_udp6_sock, + get_task_stack, _, }; @@ -797,39 +848,123 @@ test "opcodes" { } pub const Cmd = extern enum(usize) { + /// Create a map and return a file descriptor that refers to the map. The + /// close-on-exec file descriptor flag is automatically enabled for the new + /// file descriptor. + /// + /// uses MapCreateAttr map_create, + + /// Look up an element by key in a specified map and return its value. + /// + /// uses MapElemAttr map_lookup_elem, + + /// Create or update an element (key/value pair) in a specified map. + /// + /// uses MapElemAttr map_update_elem, + + /// Look up and delete an element by key in a specified map. + /// + /// uses MapElemAttr map_delete_elem, + + /// Look up an element by key in a specified map and return the key of the + /// next element. map_get_next_key, + + /// Verify and load an eBPF program, returning a new file descriptor + /// associated with the program. The close-on-exec file descriptor flag + /// is automatically enabled for the new file descriptor. + /// + /// uses ProgLoadAttr prog_load, + + /// Pin a map or eBPF program to a path within the minimal BPF filesystem + /// + /// uses ObjAttr obj_pin, + + /// Get the file descriptor of a BPF object pinned to a certain path + /// + /// uses ObjAttr obj_get, + + /// uses ProgAttachAttr prog_attach, + + /// uses ProgAttachAttr prog_detach, + + /// uses TestRunAttr prog_test_run, + + /// uses GetIdAttr prog_get_next_id, + + /// uses GetIdAttr map_get_next_id, + + /// uses GetIdAttr prog_get_fd_by_id, + + /// uses GetIdAttr map_get_fd_by_id, + + /// uses InfoAttr obj_get_info_by_fd, + + /// uses QueryAttr prog_query, + + /// uses RawTracepointAttr raw_tracepoint_open, + + /// uses BtfLoadAttr btf_load, + + /// uses GetIdAttr btf_get_fd_by_id, + + /// uses TaskFdQueryAttr task_fd_query, + + /// uses MapElemAttr map_lookup_and_delete_elem, map_freeze, + + /// uses GetIdAttr btf_get_next_id, + + /// uses MapBatchAttr map_lookup_batch, + + /// uses MapBatchAttr map_lookup_and_delete_batch, + + /// uses MapBatchAttr map_update_batch, + + /// uses MapBatchAttr map_delete_batch, + + /// uses LinkCreateAttr link_create, + + /// uses LinkUpdateAttr link_update, + + /// uses GetIdAttr link_get_fd_by_id, + + /// uses GetIdAttr link_get_next_id, + + /// uses EnableStatsAttr enable_stats, + + /// uses IterCreateAttr iter_create, link_detach, _, @@ -863,42 +998,138 @@ pub const MapType = extern enum(u32) { sk_storage, devmap_hash, struct_ops, + + /// An ordered and shared CPU version of perf_event_array. They have + /// similar semantics: + /// - variable length records + /// - no blocking: when full, reservation fails + /// - memory mappable for ease and speed + /// - epoll notifications for new data, but can busy poll + /// + /// Ringbufs give BPF programs two sets of APIs: + /// - ringbuf_output() allows copy data from one place to a ring + /// buffer, similar to bpf_perf_event_output() + /// - ringbuf_reserve()/ringbuf_commit()/ringbuf_discard() split the + /// process into two steps. First a fixed amount of space is reserved, + /// if that is successful then the program gets a pointer to a chunk of + /// memory and can be submitted with commit() or discarded with + /// discard() + /// + /// ringbuf_output() will incurr an extra memory copy, but allows to submit + /// records of the length that's not known beforehand, and is an easy + /// replacement for perf_event_outptu(). + /// + /// ringbuf_reserve() avoids the extra memory copy but requires a known size + /// of memory beforehand. + /// + /// ringbuf_query() allows to query properties of the map, 4 are currently + /// supported: + /// - BPF_RB_AVAIL_DATA: amount of unconsumed data in ringbuf + /// - BPF_RB_RING_SIZE: returns size of ringbuf + /// - BPF_RB_CONS_POS/BPF_RB_PROD_POS returns current logical position + /// of consumer and producer respectively + /// + /// key size: 0 + /// value size: 0 + /// max entries: size of ringbuf, must be power of 2 ringbuf, + _, }; pub const ProgType = extern enum(u32) { unspec, + + /// context type: __sk_buff socket_filter, + + /// context type: bpf_user_pt_regs_t kprobe, + + /// context type: __sk_buff sched_cls, + + /// context type: __sk_buff sched_act, + + /// context type: u64 tracepoint, + + /// context type: xdp_md xdp, + + /// context type: bpf_perf_event_data perf_event, + + /// context type: __sk_buff cgroup_skb, + + /// context type: bpf_sock cgroup_sock, + + /// context type: __sk_buff lwt_in, + + /// context type: __sk_buff lwt_out, + + /// context type: __sk_buff lwt_xmit, + + /// context type: bpf_sock_ops sock_ops, + + /// context type: __sk_buff sk_skb, + + /// context type: bpf_cgroup_dev_ctx cgroup_device, + + /// context type: sk_msg_md sk_msg, + + /// context type: bpf_raw_tracepoint_args raw_tracepoint, + + /// context type: bpf_sock_addr cgroup_sock_addr, + + /// context type: __sk_buff lwt_seg6local, + + /// context type: u32 lirc_mode2, + + /// context type: sk_reuseport_md sk_reuseport, + + /// context type: __sk_buff flow_dissector, + + /// context type: bpf_sysctl cgroup_sysctl, + + /// context type: bpf_raw_tracepoint_args raw_tracepoint_writable, + + /// context type: bpf_sockopt cgroup_sockopt, + + /// context type: void * tracing, + + /// context type: void * struct_ops, + + /// context type: void * ext, + + /// context type: void * lsm, + + /// context type: bpf_sk_lookup sk_lookup, + _, }; pub const AttachType = extern enum(u32) { @@ -948,27 +1179,38 @@ const obj_name_len = 16; pub const MapCreateAttr = extern struct { /// one of MapType map_type: u32, + /// size of key in bytes key_size: u32, + /// size of value in bytes value_size: u32, + /// max number of entries in a map max_entries: u32, + /// .map_create related flags map_flags: u32, + /// fd pointing to the inner map inner_map_fd: fd_t, + /// numa node (effective only if MapCreateFlags.numa_node is set) numa_node: u32, map_name: [obj_name_len]u8, + /// ifindex of netdev to create on map_ifindex: u32, + /// fd pointing to a BTF type data btf_fd: fd_t, + /// BTF type_id of the key btf_key_type_id: u32, + /// BTF type_id of the value bpf_value_type_id: u32, + /// BTF type_id of a kernel struct stored as the map value btf_vmlinux_value_type_id: u32, }; @@ -988,10 +1230,12 @@ pub const MapElemAttr = extern struct { pub const MapBatchAttr = extern struct { /// start batch, NULL to start from beginning in_batch: u64, + /// output: next start batch out_batch: u64, keys: u64, values: u64, + /// input/output: /// input: # of key/value elements /// output: # of filled elements @@ -1008,35 +1252,49 @@ pub const ProgLoadAttr = extern struct { insn_cnt: u32, insns: u64, license: u64, + /// verbosity level of verifier log_level: u32, + /// size of user buffer log_size: u32, + /// user supplied buffer log_buf: u64, + /// not used kern_version: u32, prog_flags: u32, prog_name: [obj_name_len]u8, - /// ifindex of netdev to prep for. For some prog types expected attach - /// type must be known at load time to verify attach type specific parts - /// of prog (context accesses, allowed helpers, etc). + + /// ifindex of netdev to prep for. prog_ifindex: u32, + + /// For some prog types expected attach type must be known at load time to + /// verify attach type specific parts of prog (context accesses, allowed + /// helpers, etc). expected_attach_type: u32, + /// fd pointing to BTF type data prog_btf_fd: fd_t, + /// userspace bpf_func_info size func_info_rec_size: u32, func_info: u64, + /// number of bpf_func_info records func_info_cnt: u32, + /// userspace bpf_line_info size line_info_rec_size: u32, line_info: u64, + /// number of bpf_line_info records line_info_cnt: u32, + /// in-kernel BTF type id to attach to attact_btf_id: u32, + /// 0 to attach to vmlinux attach_prog_id: u32, }; @@ -1052,10 +1310,13 @@ pub const ObjAttr = extern struct { pub const ProgAttachAttr = extern struct { /// container object to attach to target_fd: fd_t, + /// eBPF program to attach attach_bpf_fd: fd_t, + attach_type: u32, attach_flags: u32, + // TODO: BPF_F_REPLACE flags /// previously attached eBPF program to replace if .replace is used replace_bpf_fd: fd_t, @@ -1065,16 +1326,20 @@ pub const ProgAttachAttr = extern struct { pub const TestAttr = extern struct { prog_fd: fd_t, retval: u32, + /// input: len of data_in data_size_in: u32, + /// input/output: len of data_out. returns ENOSPC if data_out is too small. data_size_out: u32, data_in: u64, data_out: u64, repeat: u32, duration: u32, + /// input: len of ctx_in ctx_size_in: u32, + /// input/output: len of ctx_out. returns ENOSPC if ctx_out is too small. ctx_size_out: u32, ctx_in: u64, @@ -1127,26 +1392,35 @@ pub const BtfLoadAttr = extern struct { btf_log_level: u32, }; +/// struct used by Cmd.task_fd_query pub const TaskFdQueryAttr = extern struct { /// input: pid pid: pid_t, + /// input: fd fd: fd_t, + /// input: flags flags: u32, + /// input/output: buf len buf_len: u32, + /// input/output: /// tp_name for tracepoint /// symbol for kprobe /// filename for uprobe buf: u64, + /// output: prod_id prog_id: u32, + /// output: BPF_FD_TYPE fd_type: u32, + /// output: probe_offset probe_offset: u64, + /// output: probe_addr probe_addr: u64, }; @@ -1155,9 +1429,11 @@ pub const TaskFdQueryAttr = extern struct { pub const LinkCreateAttr = extern struct { /// eBPF program to attach prog_fd: fd_t, + /// object to attach to target_fd: fd_t, attach_type: u32, + /// extra flags flags: u32, }; @@ -1165,10 +1441,13 @@ pub const LinkCreateAttr = extern struct { /// struct used by Cmd.link_update command pub const LinkUpdateAttr = extern struct { link_fd: fd_t, + /// new program to update link with new_prog_fd: fd_t, + /// extra flags flags: u32, + /// expected link's program fd, it is specified only if BPF_F_REPLACE is /// set in flags old_prog_fd: fd_t, @@ -1185,6 +1464,7 @@ pub const IterCreateAttr = extern struct { flags: u32, }; +/// Mega struct that is passed to the bpf() syscall pub const Attr = extern union { map_create: MapCreateAttr, map_elem: MapElemAttr, From a496f94be92581cdf97a1bd60c4dea0369e26395 Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Sun, 6 Sep 2020 17:45:54 -0700 Subject: [PATCH 18/89] added map create, update, delete, and prog load --- lib/std/os/linux/bpf.zig | 181 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 179 insertions(+), 2 deletions(-) diff --git a/lib/std/os/linux/bpf.zig b/lib/std/os/linux/bpf.zig index df5274a82..7c0dd0bb1 100644 --- a/lib/std/os/linux/bpf.zig +++ b/lib/std/os/linux/bpf.zig @@ -3,9 +3,13 @@ // This file is part of [zig](https://ziglang.org/), which is MIT licensed. // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. -usingnamespace std.os; +usingnamespace std.os.linux; const std = @import("../../std.zig"); +const errno = getErrno; +const unexpectedErrno = std.os.unexpectedErrno; const expectEqual = std.testing.expectEqual; +const expectError = std.testing.expectError; +const expect = std.testing.expect; // instruction classes pub const LD = 0x00; @@ -1323,7 +1327,7 @@ pub const ProgAttachAttr = extern struct { }; /// struct used by Cmd.prog_test_run command -pub const TestAttr = extern struct { +pub const TestRunAttr = extern struct { prog_fd: fd_t, retval: u32, @@ -1484,3 +1488,176 @@ pub const Attr = extern union { enable_stats: EnableStatsAttr, iter_create: IterCreateAttr, }; + +pub const Log = struct { + level: u32, + buf: []u8, +}; + +pub fn map_create(map_type: MapType, key_size: u32, value_size: u32, max_entries: u32) !fd_t { + var attr = Attr{ + .map_create = std.mem.zeroes(MapCreateAttr), + }; + + attr.map_create.map_type = @enumToInt(map_type); + attr.map_create.key_size = key_size; + attr.map_create.value_size = value_size; + attr.map_create.max_entries = max_entries; + + const rc = bpf(.map_create, &attr, @sizeOf(MapCreateAttr)); + return switch (errno(rc)) { + 0 => @intCast(fd_t, rc), + EINVAL => error.MapTypeOrAttrInvalid, + ENOMEM => error.SystemResources, + EPERM => error.AccessDenied, + else => |err| unexpectedErrno(rc), + }; +} + +test "map_create" { + const map = try map_create(.hash, 4, 4, 32); + defer std.os.close(map); +} + +pub fn map_lookup_elem(fd: fd_t, key: []const u8, value: []u8) !void { + var attr = Attr{ + .map_elem = std.mem.zeroes(MapElemAttr), + }; + + attr.map_elem.map_fd = fd; + attr.map_elem.key = @ptrToInt(key.ptr); + attr.map_elem.result.value = @ptrToInt(value.ptr); + + const rc = bpf(.map_lookup_elem, &attr, @sizeOf(MapElemAttr)); + switch (errno(rc)) { + 0 => return, + EBADF => return error.BadFd, + EFAULT => unreachable, + EINVAL => return error.FieldInAttrNeedsZeroing, + ENOENT => return error.NotFound, + EPERM => return error.AccessDenied, + else => |err| return unexpectedErrno(rc), + } +} + +pub fn map_update_elem(fd: fd_t, key: []const u8, value: []const u8, flags: u64) !void { + var attr = Attr{ + .map_elem = std.mem.zeroes(MapElemAttr), + }; + + attr.map_elem.map_fd = fd; + attr.map_elem.key = @ptrToInt(key.ptr); + attr.map_elem.result = .{ .value = @ptrToInt(value.ptr) }; + attr.map_elem.flags = flags; + + const rc = bpf(.map_update_elem, &attr, @sizeOf(MapElemAttr)); + switch (errno(rc)) { + 0 => return, + E2BIG => return error.ReachedMaxEntries, + EBADF => return error.BadFd, + EFAULT => unreachable, + EINVAL => return error.FieldInAttrNeedsZeroing, + ENOMEM => return error.SystemResources, + EPERM => return error.AccessDenied, + else => |err| return unexpectedErrno(err), + } +} + +pub fn map_delete_elem(fd: fd_t, key: []const u8) !void { + var attr = Attr{ + .map_elem = std.mem.zeroes(MapElemAttr), + }; + + attr.map_elem.map_fd = fd; + attr.map_elem.key = @ptrToInt(key.ptr); + + const rc = bpf(.map_delete_elem, &attr, @sizeOf(MapElemAttr)); + switch (errno(rc)) { + 0 => return, + EBADF => return error.BadFd, + EFAULT => unreachable, + EINVAL => return error.FieldInAttrNeedsZeroing, + ENOENT => return error.NotFound, + EPERM => return error.AccessDenied, + else => |err| return unexpectedErrno(err), + } +} + +test "map lookup, update, and delete" { + const key_size = 4; + const value_size = 4; + const map = try map_create(.hash, key_size, value_size, 1); + defer std.os.close(map); + + const key = std.mem.zeroes([key_size]u8); + var value = std.mem.zeroes([value_size]u8); + + // fails looking up value that doesn't exist + expectError(error.NotFound, map_lookup_elem(map, &key, &value)); + + // succeed at updating and looking up element + try map_update_elem(map, &key, &value, 0); + try map_lookup_elem(map, &key, &value); + + // fails inserting more than max entries + const second_key = [key_size]u8{ 0, 0, 0, 1 }; + expectError(error.ReachedMaxEntries, map_update_elem(map, &second_key, &value, 0)); + + // succeed at deleting an existing elem + try map_delete_elem(map, &key); + expectError(error.NotFound, map_lookup_elem(map, &key, &value)); + + // fail at deleting a non-existing elem + expectError(error.NotFound, map_delete_elem(map, &key)); +} + +pub fn prog_load( + prog_type: ProgType, + insns: []const Insn, + log: ?*Log, + license: []const u8, + kern_version: u32, +) !fd_t { + var attr = Attr{ + .prog_load = std.mem.zeroes(ProgLoadAttr), + }; + + attr.prog_load.prog_type = @enumToInt(prog_type); + attr.prog_load.insns = @ptrToInt(insns.ptr); + attr.prog_load.insn_cnt = @intCast(u32, insns.len); + attr.prog_load.license = @ptrToInt(license.ptr); + attr.prog_load.kern_version = kern_version; + + if (log) |l| { + attr.prog_load.log_buf = @ptrToInt(l.buf.ptr); + attr.prog_load.log_size = @intCast(u32, l.buf.len); + attr.prog_load.log_level = l.level; + } + + const rc = bpf(.prog_load, &attr, @sizeOf(ProgLoadAttr)); + return switch (errno(rc)) { + 0 => @intCast(fd_t, rc), + EACCES => error.UnsafeProgram, + EFAULT => unreachable, + EINVAL => error.InvalidProgram, + EPERM => error.AccessDenied, + else => |err| unexpectedErrno(err), + }; +} + +test "prog_load" { + // this should fail because it does not set r0 before exiting + const bad_prog = [_]Insn{ + Insn.exit(), + }; + + const good_prog = [_]Insn{ + Insn.mov(.r0, 0), + Insn.exit(), + }; + + const prog = try prog_load(.socket_filter, &good_prog, null, "MIT", 0); + defer std.os.close(prog); + + expectError(error.UnsafeProgram, prog_load(.socket_filter, &bad_prog, null, "MIT", 0)); +} From 2328f40b7a0b301ad176e2657694457667be862e Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Mon, 7 Sep 2020 19:07:27 +0200 Subject: [PATCH 19/89] std: Add DEFLATE and zlib decompressors --- lib/std/compress.zig | 13 + lib/std/compress/deflate.zig | 521 ++++++++++++++ lib/std/compress/rfc1951.txt | 955 +++++++++++++++++++++++++ lib/std/compress/rfc1951.txt.fixed.z.9 | Bin 0 -> 12836 bytes lib/std/compress/rfc1951.txt.z.0 | Bin 0 -> 36960 bytes lib/std/compress/rfc1951.txt.z.9 | Bin 0 -> 11111 bytes lib/std/compress/zlib.zig | 178 +++++ lib/std/std.zig | 1 + 8 files changed, 1668 insertions(+) create mode 100644 lib/std/compress.zig create mode 100644 lib/std/compress/deflate.zig create mode 100644 lib/std/compress/rfc1951.txt create mode 100644 lib/std/compress/rfc1951.txt.fixed.z.9 create mode 100644 lib/std/compress/rfc1951.txt.z.0 create mode 100644 lib/std/compress/rfc1951.txt.z.9 create mode 100644 lib/std/compress/zlib.zig diff --git a/lib/std/compress.zig b/lib/std/compress.zig new file mode 100644 index 000000000..5518f807d --- /dev/null +++ b/lib/std/compress.zig @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. +const std = @import("std.zig"); + +pub const deflate = @import("compress/deflate.zig"); +pub const zlib = @import("compress/zlib.zig"); + +test "" { + _ = zlib; +} diff --git a/lib/std/compress/deflate.zig b/lib/std/compress/deflate.zig new file mode 100644 index 000000000..bad23349e --- /dev/null +++ b/lib/std/compress/deflate.zig @@ -0,0 +1,521 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. +// +// Decompressor for DEFLATE data streams (RFC1951) +// +// Heavily inspired by the simple decompressor puff.c by Mark Adler + +const std = @import("std"); +const io = std.io; +const math = std.math; +const mem = std.mem; + +const assert = std.debug.assert; + +const MAXBITS = 15; +const MAXLCODES = 286; +const MAXDCODES = 30; +const MAXCODES = MAXLCODES + MAXDCODES; +const FIXLCODES = 288; + +const Huffman = struct { + count: [MAXBITS + 1]u16, + symbol: [MAXCODES]u16, + + fn construct(self: *Huffman, length: []const u16) !void { + for (self.count) |*val| { + val.* = 0; + } + + for (length) |val| { + self.count[val] += 1; + } + + if (self.count[0] == length.len) + return; + + var left: isize = 1; + for (self.count[1..]) |val| { + left *= 2; + left -= @as(isize, @bitCast(i16, val)); + if (left < 0) + return error.InvalidTree; + } + + var offs: [MAXBITS + 1]u16 = undefined; + { + var len: usize = 1; + offs[1] = 0; + while (len < MAXBITS) : (len += 1) { + offs[len + 1] = offs[len] + self.count[len]; + } + } + + for (length) |val, symbol| { + if (val != 0) { + self.symbol[offs[val]] = @truncate(u16, symbol); + offs[val] += 1; + } + } + } +}; + +pub fn InflateStream(comptime ReaderType: type) type { + return struct { + const Self = @This(); + + pub const Error = ReaderType.Error || error{ + EndOfStream, + BadCounts, + InvalidBlockType, + InvalidDistance, + InvalidFixedCode, + InvalidLength, + InvalidStoredSize, + InvalidSymbol, + InvalidTree, + MissingEOBCode, + NoLastLength, + OutOfCodes, + }; + pub const Reader = io.Reader(*Self, Error, read); + + bit_reader: io.BitReader(.Little, ReaderType), + + // True if the decoder met the end of the compressed stream, no further + // data can be decompressed + seen_eos: bool, + + state: union(enum) { + // Parse a compressed block header and set up the internal state for + // decompressing its contents. + DecodeBlockHeader: void, + // Decode all the symbols in a compressed block. + DecodeBlockData: void, + // Copy N bytes of uncompressed data from the underlying stream into + // the window. + Copy: usize, + // Copy 1 byte into the window. + CopyLit: u8, + // Copy L bytes from the window itself, starting from D bytes + // behind. + CopyFrom: struct { distance: u16, length: u16 }, + }, + + // Sliding window for the LZ77 algorithm + window: struct { + const WSelf = @This(); + + // invariant: buffer length is always a power of 2 + buf: []u8, + // invariant: ri <= wi + wi: usize = 0, // Write index + ri: usize = 0, // Read index + el: usize = 0, // Number of readable elements + + fn readable(self: *WSelf) usize { + return self.el; + } + + fn writable(self: *WSelf) usize { + return self.buf.len - self.el; + } + + // Insert a single byte into the window. + // Returns 1 if there's enough space for the new byte and 0 + // otherwise. + fn append(self: *WSelf, value: u8) usize { + if (self.writable() < 1) return 0; + self.appendUnsafe(value); + return 1; + } + + // Insert a single byte into the window. + // Assumes there's enough space. + fn appendUnsafe(self: *WSelf, value: u8) void { + self.buf[self.wi] = value; + self.wi = (self.wi + 1) & (self.buf.len - 1); + self.el += 1; + } + + // Fill dest[] with data from the window, starting from the read + // position. This updates the read pointer. + // Returns the number of read bytes or 0 if there's nothing to read + // yet. + fn read(self: *WSelf, dest: []u8) usize { + const N = math.min(dest.len, self.readable()); + + if (N == 0) return 0; + + if (self.ri + N < self.buf.len) { + // The data doesn't wrap around + mem.copy(u8, dest, self.buf[self.ri .. self.ri + N]); + } else { + // The data wraps around the buffer, split the copy + std.mem.copy(u8, dest, self.buf[self.ri..]); + // How much data we've copied from `ri` to the end + const r = self.buf.len - self.ri; + std.mem.copy(u8, dest[r..], self.buf[0 .. N - r]); + } + + self.ri = (self.ri + N) & (self.buf.len - 1); + self.el -= N; + + return N; + } + + // Copy `length` bytes starting from `distance` bytes behind the + // write pointer. + // Be careful as the length may be greater than the distance, that's + // how the compressor encodes run-length encoded sequences. + fn copyFrom(self: *WSelf, distance: usize, length: usize) usize { + const N = math.min(length, self.writable()); + + if (N == 0) return 0; + + // TODO: Profile and, if needed, replace with smarter juggling + // of the window memory for the non-overlapping case. + var i: usize = 0; + while (i < N) : (i += 1) { + const index = (self.wi -% distance) % self.buf.len; + self.appendUnsafe(self.buf[index]); + } + + return N; + } + }, + + // Compressor-local Huffman tables used to decompress blocks with + // dynamic codes. + huffman_tables: [2]Huffman = undefined, + + // Huffman tables used for decoding length/distance pairs. + hdist: *Huffman, + hlen: *Huffman, + + fn stored(self: *Self) !void { + // Discard the remaining bits, the lenght field is always + // byte-aligned (and so is the data) + self.bit_reader.alignToByte(); + + const length = (try self.bit_reader.readBitsNoEof(u16, 16)); + const length_cpl = (try self.bit_reader.readBitsNoEof(u16, 16)); + + if (length != ~length_cpl) + return error.InvalidStoredSize; + + self.state = .{ .Copy = length }; + } + + fn fixed(self: *Self) !void { + comptime var lencode: Huffman = undefined; + comptime var distcode: Huffman = undefined; + + // The Huffman codes are specified in the RFC1951, section 3.2.6 + comptime { + @setEvalBranchQuota(100000); + + const len_lengths = // + [_]u16{8} ** 144 ++ + [_]u16{9} ** 112 ++ + [_]u16{7} ** 24 ++ + [_]u16{8} ** 8; + assert(len_lengths.len == FIXLCODES); + try lencode.construct(len_lengths[0..]); + + const dist_lengths = [_]u16{5} ** MAXDCODES; + try distcode.construct(dist_lengths[0..]); + } + + self.hlen = &lencode; + self.hdist = &distcode; + self.state = .DecodeBlockData; + } + + fn dynamic(self: *Self) !void { + // Number of length codes + const nlen = (try self.bit_reader.readBitsNoEof(usize, 5)) + 257; + // Number of distance codes + const ndist = (try self.bit_reader.readBitsNoEof(usize, 5)) + 1; + // Number of code length codes + const ncode = (try self.bit_reader.readBitsNoEof(usize, 4)) + 4; + + if (nlen > MAXLCODES or ndist > MAXDCODES) + return error.BadCounts; + + // Permutation of code length codes + const ORDER = [19]u16{ + 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, + 12, 3, 13, 2, 14, 1, 15, + }; + + // Build the Huffman table to decode the code length codes + var lencode: Huffman = undefined; + { + var lengths = std.mem.zeroes([19]u16); + + // Read the code lengths, missing ones are left as zero + for (ORDER[0..ncode]) |val| { + lengths[val] = try self.bit_reader.readBitsNoEof(u16, 3); + } + + try lencode.construct(lengths[0..]); + } + + // Read the length/literal and distance code length tables. + // Zero the table by default so we can avoid explicitly writing out + // zeros for codes 17 and 18 + var lengths = std.mem.zeroes([MAXCODES]u16); + + var i: usize = 0; + while (i < nlen + ndist) { + const symbol = try self.decode(&lencode); + + switch (symbol) { + 0...15 => { + lengths[i] = symbol; + i += 1; + }, + 16 => { + // repeat last length 3..6 times + if (i == 0) return error.NoLastLength; + + const last_length = lengths[i - 1]; + const repeat = 3 + (try self.bit_reader.readBitsNoEof(usize, 2)); + const last_index = i + repeat; + while (i < last_index) : (i += 1) { + lengths[i] = last_length; + } + }, + 17 => { + // repeat zero 3..10 times + i += 3 + (try self.bit_reader.readBitsNoEof(usize, 3)); + }, + 18 => { + // repeat zero 11..138 times + i += 11 + (try self.bit_reader.readBitsNoEof(usize, 7)); + }, + else => return error.InvalidSymbol, + } + } + + if (i > nlen + ndist) + return error.InvalidLength; + + // Check if the end of block code is present + if (lengths[256] == 0) + return error.MissingEOBCode; + + try self.huffman_tables[0].construct(lengths[0..nlen]); + try self.huffman_tables[1].construct(lengths[nlen .. nlen + ndist]); + + self.hlen = &self.huffman_tables[0]; + self.hdist = &self.huffman_tables[1]; + self.state = .DecodeBlockData; + } + + fn codes(self: *Self, lencode: *Huffman, distcode: *Huffman) !bool { + // Size base for length codes 257..285 + const LENS = [29]u16{ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, + 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, + }; + // Extra bits for length codes 257..285 + const LEXT = [29]u16{ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, + 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, + }; + // Offset base for distance codes 0..29 + const DISTS = [30]u16{ + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, + 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577, + }; + // Extra bits for distance codes 0..29 + const DEXT = [30]u16{ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, + 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, + }; + + while (true) { + const symbol = try self.decode(lencode); + + switch (symbol) { + 0...255 => { + // Literal value + const c = @truncate(u8, symbol); + if (self.window.append(c) == 0) { + self.state = .{ .CopyLit = c }; + return false; + } + }, + 256 => { + // End of block symbol + return true; + }, + 257...285 => { + // Length/distance pair + const length_symbol = symbol - 257; + const length = LENS[length_symbol] + + try self.bit_reader.readBitsNoEof(u16, LEXT[length_symbol]); + + const distance_symbol = try self.decode(distcode); + const distance = DISTS[distance_symbol] + + try self.bit_reader.readBitsNoEof(u16, DEXT[distance_symbol]); + + if (distance > self.window.buf.len) + return error.InvalidDistance; + + const written = self.window.copyFrom(distance, length); + if (written != length) { + self.state = .{ + .CopyFrom = .{ + .distance = distance, + .length = length - @truncate(u16, written), + }, + }; + return false; + } + }, + else => return error.InvalidFixedCode, + } + } + } + + fn decode(self: *Self, h: *Huffman) !u16 { + var len: usize = 1; + var code: usize = 0; + var first: usize = 0; + var index: usize = 0; + + while (len <= MAXBITS) : (len += 1) { + code |= try self.bit_reader.readBitsNoEof(usize, 1); + const count = h.count[len]; + if (code < first + count) + return h.symbol[index + (code - first)]; + index += count; + first += count; + first <<= 1; + code <<= 1; + } + + return error.OutOfCodes; + } + + fn step(self: *Self) !void { + while (true) { + switch (self.state) { + .DecodeBlockHeader => { + // The compressed stream is done + if (self.seen_eos) return; + + const last = try self.bit_reader.readBitsNoEof(u1, 1); + const kind = try self.bit_reader.readBitsNoEof(u2, 2); + + self.seen_eos = last != 0; + + // The next state depends on the block type + switch (kind) { + 0 => try self.stored(), + 1 => try self.fixed(), + 2 => try self.dynamic(), + 3 => return error.InvalidBlockType, + } + }, + .DecodeBlockData => { + if (!try self.codes(self.hlen, self.hdist)) { + return; + } + + self.state = .DecodeBlockHeader; + }, + .Copy => |*length| { + const N = math.min(self.window.writable(), length.*); + + // TODO: This loop can be more efficient. On the other + // hand uncompressed blocks are not that common so... + var i: usize = 0; + while (i < N) : (i += 1) { + var tmp: [1]u8 = undefined; + if ((try self.bit_reader.read(&tmp)) != 1) { + // Unexpected end of stream, keep this error + // consistent with the use of readBitsNoEof + return error.EndOfStream; + } + self.window.appendUnsafe(tmp[0]); + } + + if (N != length.*) { + length.* -= N; + return; + } + + self.state = .DecodeBlockHeader; + }, + .CopyLit => |c| { + if (self.window.append(c) == 0) { + return; + } + + self.state = .DecodeBlockData; + }, + .CopyFrom => |*info| { + const written = self.window.copyFrom(info.distance, info.length); + if (written != info.length) { + info.length -= @truncate(u16, written); + return; + } + + self.state = .DecodeBlockData; + }, + } + } + } + + fn init(source: ReaderType, window_slice: []u8) Self { + assert(math.isPowerOfTwo(window_slice.len)); + + return Self{ + .bit_reader = io.bitReader(.Little, source), + .window = .{ .buf = window_slice }, + .seen_eos = false, + .state = .DecodeBlockHeader, + .hdist = undefined, + .hlen = undefined, + }; + } + + // Implements the io.Reader interface + pub fn read(self: *Self, buffer: []u8) Error!usize { + if (buffer.len == 0) + return 0; + + // Try reading as much as possible from the window + var read_amt: usize = self.window.read(buffer); + while (read_amt < buffer.len) { + // Run the state machine, we can detect the "effective" end of + // stream condition by checking if any progress was made. + // Why "effective"? Because even though `seen_eos` is true we + // may still have to finish processing other decoding steps. + try self.step(); + // No progress was made + if (self.window.readable() == 0) + break; + + read_amt += self.window.read(buffer[read_amt..]); + } + + return read_amt; + } + + pub fn reader(self: *Self) Reader { + return .{ .context = self }; + } + }; +} + +pub fn inflateStream(reader: anytype, window_slice: []u8) InflateStream(@TypeOf(reader)) { + return InflateStream(@TypeOf(reader)).init(reader, window_slice); +} diff --git a/lib/std/compress/rfc1951.txt b/lib/std/compress/rfc1951.txt new file mode 100644 index 000000000..403c8c722 --- /dev/null +++ b/lib/std/compress/rfc1951.txt @@ -0,0 +1,955 @@ + + + + + + +Network Working Group P. Deutsch +Request for Comments: 1951 Aladdin Enterprises +Category: Informational May 1996 + + + DEFLATE Compressed Data Format Specification version 1.3 + +Status of This Memo + + This memo provides information for the Internet community. This memo + does not specify an Internet standard of any kind. Distribution of + this memo is unlimited. + +IESG Note: + + The IESG takes no position on the validity of any Intellectual + Property Rights statements contained in this document. + +Notices + + Copyright (c) 1996 L. Peter Deutsch + + Permission is granted to copy and distribute this document for any + purpose and without charge, including translations into other + languages and incorporation into compilations, provided that the + copyright notice and this notice are preserved, and that any + substantive changes or deletions from the original are clearly + marked. + + A pointer to the latest version of this and related documentation in + HTML format can be found at the URL + . + +Abstract + + This specification defines a lossless compressed data format that + compresses data using a combination of the LZ77 algorithm and Huffman + coding, with efficiency comparable to the best currently available + general-purpose compression methods. The data can be produced or + consumed, even for an arbitrarily long sequentially presented input + data stream, using only an a priori bounded amount of intermediate + storage. The format can be implemented readily in a manner not + covered by patents. + + + + + + + + +Deutsch Informational [Page 1] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + +Table of Contents + + 1. Introduction ................................................... 2 + 1.1. Purpose ................................................... 2 + 1.2. Intended audience ......................................... 3 + 1.3. Scope ..................................................... 3 + 1.4. Compliance ................................................ 3 + 1.5. Definitions of terms and conventions used ................ 3 + 1.6. Changes from previous versions ............................ 4 + 2. Compressed representation overview ............................. 4 + 3. Detailed specification ......................................... 5 + 3.1. Overall conventions ....................................... 5 + 3.1.1. Packing into bytes .................................. 5 + 3.2. Compressed block format ................................... 6 + 3.2.1. Synopsis of prefix and Huffman coding ............... 6 + 3.2.2. Use of Huffman coding in the "deflate" format ....... 7 + 3.2.3. Details of block format ............................. 9 + 3.2.4. Non-compressed blocks (BTYPE=00) ................... 11 + 3.2.5. Compressed blocks (length and distance codes) ...... 11 + 3.2.6. Compression with fixed Huffman codes (BTYPE=01) .... 12 + 3.2.7. Compression with dynamic Huffman codes (BTYPE=10) .. 13 + 3.3. Compliance ............................................... 14 + 4. Compression algorithm details ................................. 14 + 5. References .................................................... 16 + 6. Security Considerations ....................................... 16 + 7. Source code ................................................... 16 + 8. Acknowledgements .............................................. 16 + 9. Author's Address .............................................. 17 + +1. Introduction + + 1.1. Purpose + + The purpose of this specification is to define a lossless + compressed data format that: + * Is independent of CPU type, operating system, file system, + and character set, and hence can be used for interchange; + * Can be produced or consumed, even for an arbitrarily long + sequentially presented input data stream, using only an a + priori bounded amount of intermediate storage, and hence + can be used in data communications or similar structures + such as Unix filters; + * Compresses data with efficiency comparable to the best + currently available general-purpose compression methods, + and in particular considerably better than the "compress" + program; + * Can be implemented readily in a manner not covered by + patents, and hence can be practiced freely; + + + +Deutsch Informational [Page 2] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + + * Is compatible with the file format produced by the current + widely used gzip utility, in that conforming decompressors + will be able to read data produced by the existing gzip + compressor. + + The data format defined by this specification does not attempt to: + + * Allow random access to compressed data; + * Compress specialized data (e.g., raster graphics) as well + as the best currently available specialized algorithms. + + A simple counting argument shows that no lossless compression + algorithm can compress every possible input data set. For the + format defined here, the worst case expansion is 5 bytes per 32K- + byte block, i.e., a size increase of 0.015% for large data sets. + English text usually compresses by a factor of 2.5 to 3; + executable files usually compress somewhat less; graphical data + such as raster images may compress much more. + + 1.2. Intended audience + + This specification is intended for use by implementors of software + to compress data into "deflate" format and/or decompress data from + "deflate" format. + + The text of the specification assumes a basic background in + programming at the level of bits and other primitive data + representations. Familiarity with the technique of Huffman coding + is helpful but not required. + + 1.3. Scope + + The specification specifies a method for representing a sequence + of bytes as a (usually shorter) sequence of bits, and a method for + packing the latter bit sequence into bytes. + + 1.4. Compliance + + Unless otherwise indicated below, a compliant decompressor must be + able to accept and decompress any data set that conforms to all + the specifications presented here; a compliant compressor must + produce data sets that conform to all the specifications presented + here. + + 1.5. Definitions of terms and conventions used + + Byte: 8 bits stored or transmitted as a unit (same as an octet). + For this specification, a byte is exactly 8 bits, even on machines + + + +Deutsch Informational [Page 3] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + + which store a character on a number of bits different from eight. + See below, for the numbering of bits within a byte. + + String: a sequence of arbitrary bytes. + + 1.6. Changes from previous versions + + There have been no technical changes to the deflate format since + version 1.1 of this specification. In version 1.2, some + terminology was changed. Version 1.3 is a conversion of the + specification to RFC style. + +2. Compressed representation overview + + A compressed data set consists of a series of blocks, corresponding + to successive blocks of input data. The block sizes are arbitrary, + except that non-compressible blocks are limited to 65,535 bytes. + + Each block is compressed using a combination of the LZ77 algorithm + and Huffman coding. The Huffman trees for each block are independent + of those for previous or subsequent blocks; the LZ77 algorithm may + use a reference to a duplicated string occurring in a previous block, + up to 32K input bytes before. + + Each block consists of two parts: a pair of Huffman code trees that + describe the representation of the compressed data part, and a + compressed data part. (The Huffman trees themselves are compressed + using Huffman encoding.) The compressed data consists of a series of + elements of two types: literal bytes (of strings that have not been + detected as duplicated within the previous 32K input bytes), and + pointers to duplicated strings, where a pointer is represented as a + pair . The representation used in the + "deflate" format limits distances to 32K bytes and lengths to 258 + bytes, but does not limit the size of a block, except for + uncompressible blocks, which are limited as noted above. + + Each type of value (literals, distances, and lengths) in the + compressed data is represented using a Huffman code, using one code + tree for literals and lengths and a separate code tree for distances. + The code trees for each block appear in a compact form just before + the compressed data for that block. + + + + + + + + + + +Deutsch Informational [Page 4] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + +3. Detailed specification + + 3.1. Overall conventions In the diagrams below, a box like this: + + +---+ + | | <-- the vertical bars might be missing + +---+ + + represents one byte; a box like this: + + +==============+ + | | + +==============+ + + represents a variable number of bytes. + + Bytes stored within a computer do not have a "bit order", since + they are always treated as a unit. However, a byte considered as + an integer between 0 and 255 does have a most- and least- + significant bit, and since we write numbers with the most- + significant digit on the left, we also write bytes with the most- + significant bit on the left. In the diagrams below, we number the + bits of a byte so that bit 0 is the least-significant bit, i.e., + the bits are numbered: + + +--------+ + |76543210| + +--------+ + + Within a computer, a number may occupy multiple bytes. All + multi-byte numbers in the format described here are stored with + the least-significant byte first (at the lower memory address). + For example, the decimal number 520 is stored as: + + 0 1 + +--------+--------+ + |00001000|00000010| + +--------+--------+ + ^ ^ + | | + | + more significant byte = 2 x 256 + + less significant byte = 8 + + 3.1.1. Packing into bytes + + This document does not address the issue of the order in which + bits of a byte are transmitted on a bit-sequential medium, + since the final data format described here is byte- rather than + + + +Deutsch Informational [Page 5] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + + bit-oriented. However, we describe the compressed block format + in below, as a sequence of data elements of various bit + lengths, not a sequence of bytes. We must therefore specify + how to pack these data elements into bytes to form the final + compressed byte sequence: + + * Data elements are packed into bytes in order of + increasing bit number within the byte, i.e., starting + with the least-significant bit of the byte. + * Data elements other than Huffman codes are packed + starting with the least-significant bit of the data + element. + * Huffman codes are packed starting with the most- + significant bit of the code. + + In other words, if one were to print out the compressed data as + a sequence of bytes, starting with the first byte at the + *right* margin and proceeding to the *left*, with the most- + significant bit of each byte on the left as usual, one would be + able to parse the result from right to left, with fixed-width + elements in the correct MSB-to-LSB order and Huffman codes in + bit-reversed order (i.e., with the first bit of the code in the + relative LSB position). + + 3.2. Compressed block format + + 3.2.1. Synopsis of prefix and Huffman coding + + Prefix coding represents symbols from an a priori known + alphabet by bit sequences (codes), one code for each symbol, in + a manner such that different symbols may be represented by bit + sequences of different lengths, but a parser can always parse + an encoded string unambiguously symbol-by-symbol. + + We define a prefix code in terms of a binary tree in which the + two edges descending from each non-leaf node are labeled 0 and + 1 and in which the leaf nodes correspond one-for-one with (are + labeled with) the symbols of the alphabet; then the code for a + symbol is the sequence of 0's and 1's on the edges leading from + the root to the leaf labeled with that symbol. For example: + + + + + + + + + + + +Deutsch Informational [Page 6] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + + /\ Symbol Code + 0 1 ------ ---- + / \ A 00 + /\ B B 1 + 0 1 C 011 + / \ D 010 + A /\ + 0 1 + / \ + D C + + A parser can decode the next symbol from an encoded input + stream by walking down the tree from the root, at each step + choosing the edge corresponding to the next input bit. + + Given an alphabet with known symbol frequencies, the Huffman + algorithm allows the construction of an optimal prefix code + (one which represents strings with those symbol frequencies + using the fewest bits of any possible prefix codes for that + alphabet). Such a code is called a Huffman code. (See + reference [1] in Chapter 5, references for additional + information on Huffman codes.) + + Note that in the "deflate" format, the Huffman codes for the + various alphabets must not exceed certain maximum code lengths. + This constraint complicates the algorithm for computing code + lengths from symbol frequencies. Again, see Chapter 5, + references for details. + + 3.2.2. Use of Huffman coding in the "deflate" format + + The Huffman codes used for each alphabet in the "deflate" + format have two additional rules: + + * All codes of a given bit length have lexicographically + consecutive values, in the same order as the symbols + they represent; + + * Shorter codes lexicographically precede longer codes. + + + + + + + + + + + + +Deutsch Informational [Page 7] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + + We could recode the example above to follow this rule as + follows, assuming that the order of the alphabet is ABCD: + + Symbol Code + ------ ---- + A 10 + B 0 + C 110 + D 111 + + I.e., 0 precedes 10 which precedes 11x, and 110 and 111 are + lexicographically consecutive. + + Given this rule, we can define the Huffman code for an alphabet + just by giving the bit lengths of the codes for each symbol of + the alphabet in order; this is sufficient to determine the + actual codes. In our example, the code is completely defined + by the sequence of bit lengths (2, 1, 3, 3). The following + algorithm generates the codes as integers, intended to be read + from most- to least-significant bit. The code lengths are + initially in tree[I].Len; the codes are produced in + tree[I].Code. + + 1) Count the number of codes for each code length. Let + bl_count[N] be the number of codes of length N, N >= 1. + + 2) Find the numerical value of the smallest code for each + code length: + + code = 0; + bl_count[0] = 0; + for (bits = 1; bits <= MAX_BITS; bits++) { + code = (code + bl_count[bits-1]) << 1; + next_code[bits] = code; + } + + 3) Assign numerical values to all codes, using consecutive + values for all codes of the same length with the base + values determined at step 2. Codes that are never used + (which have a bit length of zero) must not be assigned a + value. + + for (n = 0; n <= max_code; n++) { + len = tree[n].Len; + if (len != 0) { + tree[n].Code = next_code[len]; + next_code[len]++; + } + + + +Deutsch Informational [Page 8] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + + } + + Example: + + Consider the alphabet ABCDEFGH, with bit lengths (3, 3, 3, 3, + 3, 2, 4, 4). After step 1, we have: + + N bl_count[N] + - ----------- + 2 1 + 3 5 + 4 2 + + Step 2 computes the following next_code values: + + N next_code[N] + - ------------ + 1 0 + 2 0 + 3 2 + 4 14 + + Step 3 produces the following code values: + + Symbol Length Code + ------ ------ ---- + A 3 010 + B 3 011 + C 3 100 + D 3 101 + E 3 110 + F 2 00 + G 4 1110 + H 4 1111 + + 3.2.3. Details of block format + + Each block of compressed data begins with 3 header bits + containing the following data: + + first bit BFINAL + next 2 bits BTYPE + + Note that the header bits do not necessarily begin on a byte + boundary, since a block does not necessarily occupy an integral + number of bytes. + + + + + +Deutsch Informational [Page 9] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + + BFINAL is set if and only if this is the last block of the data + set. + + BTYPE specifies how the data are compressed, as follows: + + 00 - no compression + 01 - compressed with fixed Huffman codes + 10 - compressed with dynamic Huffman codes + 11 - reserved (error) + + The only difference between the two compressed cases is how the + Huffman codes for the literal/length and distance alphabets are + defined. + + In all cases, the decoding algorithm for the actual data is as + follows: + + do + read block header from input stream. + if stored with no compression + skip any remaining bits in current partially + processed byte + read LEN and NLEN (see next section) + copy LEN bytes of data to output + otherwise + if compressed with dynamic Huffman codes + read representation of code trees (see + subsection below) + loop (until end of block code recognized) + decode literal/length value from input stream + if value < 256 + copy value (literal byte) to output stream + otherwise + if value = end of block (256) + break from loop + otherwise (value = 257..285) + decode distance from input stream + + move backwards distance bytes in the output + stream, and copy length bytes from this + position to the output stream. + end loop + while not last block + + Note that a duplicated string reference may refer to a string + in a previous block; i.e., the backward distance may cross one + or more block boundaries. However a distance cannot refer past + the beginning of the output stream. (An application using a + + + +Deutsch Informational [Page 10] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + + preset dictionary might discard part of the output stream; a + distance can refer to that part of the output stream anyway) + Note also that the referenced string may overlap the current + position; for example, if the last 2 bytes decoded have values + X and Y, a string reference with + adds X,Y,X,Y,X to the output stream. + + We now specify each compression method in turn. + + 3.2.4. Non-compressed blocks (BTYPE=00) + + Any bits of input up to the next byte boundary are ignored. + The rest of the block consists of the following information: + + 0 1 2 3 4... + +---+---+---+---+================================+ + | LEN | NLEN |... LEN bytes of literal data...| + +---+---+---+---+================================+ + + LEN is the number of data bytes in the block. NLEN is the + one's complement of LEN. + + 3.2.5. Compressed blocks (length and distance codes) + + As noted above, encoded data blocks in the "deflate" format + consist of sequences of symbols drawn from three conceptually + distinct alphabets: either literal bytes, from the alphabet of + byte values (0..255), or pairs, + where the length is drawn from (3..258) and the distance is + drawn from (1..32,768). In fact, the literal and length + alphabets are merged into a single alphabet (0..285), where + values 0..255 represent literal bytes, the value 256 indicates + end-of-block, and values 257..285 represent length codes + (possibly in conjunction with extra bits following the symbol + code) as follows: + + + + + + + + + + + + + + + + +Deutsch Informational [Page 11] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + + Extra Extra Extra + Code Bits Length(s) Code Bits Lengths Code Bits Length(s) + ---- ---- ------ ---- ---- ------- ---- ---- ------- + 257 0 3 267 1 15,16 277 4 67-82 + 258 0 4 268 1 17,18 278 4 83-98 + 259 0 5 269 2 19-22 279 4 99-114 + 260 0 6 270 2 23-26 280 4 115-130 + 261 0 7 271 2 27-30 281 5 131-162 + 262 0 8 272 2 31-34 282 5 163-194 + 263 0 9 273 3 35-42 283 5 195-226 + 264 0 10 274 3 43-50 284 5 227-257 + 265 1 11,12 275 3 51-58 285 0 258 + 266 1 13,14 276 3 59-66 + + The extra bits should be interpreted as a machine integer + stored with the most-significant bit first, e.g., bits 1110 + represent the value 14. + + Extra Extra Extra + Code Bits Dist Code Bits Dist Code Bits Distance + ---- ---- ---- ---- ---- ------ ---- ---- -------- + 0 0 1 10 4 33-48 20 9 1025-1536 + 1 0 2 11 4 49-64 21 9 1537-2048 + 2 0 3 12 5 65-96 22 10 2049-3072 + 3 0 4 13 5 97-128 23 10 3073-4096 + 4 1 5,6 14 6 129-192 24 11 4097-6144 + 5 1 7,8 15 6 193-256 25 11 6145-8192 + 6 2 9-12 16 7 257-384 26 12 8193-12288 + 7 2 13-16 17 7 385-512 27 12 12289-16384 + 8 3 17-24 18 8 513-768 28 13 16385-24576 + 9 3 25-32 19 8 769-1024 29 13 24577-32768 + + 3.2.6. Compression with fixed Huffman codes (BTYPE=01) + + The Huffman codes for the two alphabets are fixed, and are not + represented explicitly in the data. The Huffman code lengths + for the literal/length alphabet are: + + Lit Value Bits Codes + --------- ---- ----- + 0 - 143 8 00110000 through + 10111111 + 144 - 255 9 110010000 through + 111111111 + 256 - 279 7 0000000 through + 0010111 + 280 - 287 8 11000000 through + 11000111 + + + +Deutsch Informational [Page 12] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + + The code lengths are sufficient to generate the actual codes, + as described above; we show the codes in the table for added + clarity. Literal/length values 286-287 will never actually + occur in the compressed data, but participate in the code + construction. + + Distance codes 0-31 are represented by (fixed-length) 5-bit + codes, with possible additional bits as shown in the table + shown in Paragraph 3.2.5, above. Note that distance codes 30- + 31 will never actually occur in the compressed data. + + 3.2.7. Compression with dynamic Huffman codes (BTYPE=10) + + The Huffman codes for the two alphabets appear in the block + immediately after the header bits and before the actual + compressed data, first the literal/length code and then the + distance code. Each code is defined by a sequence of code + lengths, as discussed in Paragraph 3.2.2, above. For even + greater compactness, the code length sequences themselves are + compressed using a Huffman code. The alphabet for code lengths + is as follows: + + 0 - 15: Represent code lengths of 0 - 15 + 16: Copy the previous code length 3 - 6 times. + The next 2 bits indicate repeat length + (0 = 3, ... , 3 = 6) + Example: Codes 8, 16 (+2 bits 11), + 16 (+2 bits 10) will expand to + 12 code lengths of 8 (1 + 6 + 5) + 17: Repeat a code length of 0 for 3 - 10 times. + (3 bits of length) + 18: Repeat a code length of 0 for 11 - 138 times + (7 bits of length) + + A code length of 0 indicates that the corresponding symbol in + the literal/length or distance alphabet will not occur in the + block, and should not participate in the Huffman code + construction algorithm given earlier. If only one distance + code is used, it is encoded using one bit, not zero bits; in + this case there is a single code length of one, with one unused + code. One distance code of zero bits means that there are no + distance codes used at all (the data is all literals). + + We can now define the format of the block: + + 5 Bits: HLIT, # of Literal/Length codes - 257 (257 - 286) + 5 Bits: HDIST, # of Distance codes - 1 (1 - 32) + 4 Bits: HCLEN, # of Code Length codes - 4 (4 - 19) + + + +Deutsch Informational [Page 13] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + + (HCLEN + 4) x 3 bits: code lengths for the code length + alphabet given just above, in the order: 16, 17, 18, + 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 + + These code lengths are interpreted as 3-bit integers + (0-7); as above, a code length of 0 means the + corresponding symbol (literal/length or distance code + length) is not used. + + HLIT + 257 code lengths for the literal/length alphabet, + encoded using the code length Huffman code + + HDIST + 1 code lengths for the distance alphabet, + encoded using the code length Huffman code + + The actual compressed data of the block, + encoded using the literal/length and distance Huffman + codes + + The literal/length symbol 256 (end of data), + encoded using the literal/length Huffman code + + The code length repeat codes can cross from HLIT + 257 to the + HDIST + 1 code lengths. In other words, all code lengths form + a single sequence of HLIT + HDIST + 258 values. + + 3.3. Compliance + + A compressor may limit further the ranges of values specified in + the previous section and still be compliant; for example, it may + limit the range of backward pointers to some value smaller than + 32K. Similarly, a compressor may limit the size of blocks so that + a compressible block fits in memory. + + A compliant decompressor must accept the full range of possible + values defined in the previous section, and must accept blocks of + arbitrary size. + +4. Compression algorithm details + + While it is the intent of this document to define the "deflate" + compressed data format without reference to any particular + compression algorithm, the format is related to the compressed + formats produced by LZ77 (Lempel-Ziv 1977, see reference [2] below); + since many variations of LZ77 are patented, it is strongly + recommended that the implementor of a compressor follow the general + algorithm presented here, which is known not to be patented per se. + The material in this section is not part of the definition of the + + + +Deutsch Informational [Page 14] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + + specification per se, and a compressor need not follow it in order to + be compliant. + + The compressor terminates a block when it determines that starting a + new block with fresh trees would be useful, or when the block size + fills up the compressor's block buffer. + + The compressor uses a chained hash table to find duplicated strings, + using a hash function that operates on 3-byte sequences. At any + given point during compression, let XYZ be the next 3 input bytes to + be examined (not necessarily all different, of course). First, the + compressor examines the hash chain for XYZ. If the chain is empty, + the compressor simply writes out X as a literal byte and advances one + byte in the input. If the hash chain is not empty, indicating that + the sequence XYZ (or, if we are unlucky, some other 3 bytes with the + same hash function value) has occurred recently, the compressor + compares all strings on the XYZ hash chain with the actual input data + sequence starting at the current point, and selects the longest + match. + + The compressor searches the hash chains starting with the most recent + strings, to favor small distances and thus take advantage of the + Huffman encoding. The hash chains are singly linked. There are no + deletions from the hash chains; the algorithm simply discards matches + that are too old. To avoid a worst-case situation, very long hash + chains are arbitrarily truncated at a certain length, determined by a + run-time parameter. + + To improve overall compression, the compressor optionally defers the + selection of matches ("lazy matching"): after a match of length N has + been found, the compressor searches for a longer match starting at + the next input byte. If it finds a longer match, it truncates the + previous match to a length of one (thus producing a single literal + byte) and then emits the longer match. Otherwise, it emits the + original match, and, as described above, advances N bytes before + continuing. + + Run-time parameters also control this "lazy match" procedure. If + compression ratio is most important, the compressor attempts a + complete second search regardless of the length of the first match. + In the normal case, if the current match is "long enough", the + compressor reduces the search for a longer match, thus speeding up + the process. If speed is most important, the compressor inserts new + strings in the hash table only when no match was found, or when the + match is not "too long". This degrades the compression ratio but + saves time since there are both fewer insertions and fewer searches. + + + + + +Deutsch Informational [Page 15] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + +5. References + + [1] Huffman, D. A., "A Method for the Construction of Minimum + Redundancy Codes", Proceedings of the Institute of Radio + Engineers, September 1952, Volume 40, Number 9, pp. 1098-1101. + + [2] Ziv J., Lempel A., "A Universal Algorithm for Sequential Data + Compression", IEEE Transactions on Information Theory, Vol. 23, + No. 3, pp. 337-343. + + [3] Gailly, J.-L., and Adler, M., ZLIB documentation and sources, + available in ftp://ftp.uu.net/pub/archiving/zip/doc/ + + [4] Gailly, J.-L., and Adler, M., GZIP documentation and sources, + available as gzip-*.tar in ftp://prep.ai.mit.edu/pub/gnu/ + + [5] Schwartz, E. S., and Kallick, B. "Generating a canonical prefix + encoding." Comm. ACM, 7,3 (Mar. 1964), pp. 166-169. + + [6] Hirschberg and Lelewer, "Efficient decoding of prefix codes," + Comm. ACM, 33,4, April 1990, pp. 449-459. + +6. Security Considerations + + Any data compression method involves the reduction of redundancy in + the data. Consequently, any corruption of the data is likely to have + severe effects and be difficult to correct. Uncompressed text, on + the other hand, will probably still be readable despite the presence + of some corrupted bytes. + + It is recommended that systems using this data format provide some + means of validating the integrity of the compressed data. See + reference [3], for example. + +7. Source code + + Source code for a C language implementation of a "deflate" compliant + compressor and decompressor is available within the zlib package at + ftp://ftp.uu.net/pub/archiving/zip/zlib/. + +8. Acknowledgements + + Trademarks cited in this document are the property of their + respective owners. + + Phil Katz designed the deflate format. Jean-Loup Gailly and Mark + Adler wrote the related software described in this specification. + Glenn Randers-Pehrson converted this document to RFC and HTML format. + + + +Deutsch Informational [Page 16] + +RFC 1951 DEFLATE Compressed Data Format Specification May 1996 + + +9. Author's Address + + L. Peter Deutsch + Aladdin Enterprises + 203 Santa Margarita Ave. + Menlo Park, CA 94025 + + Phone: (415) 322-0103 (AM only) + FAX: (415) 322-1734 + EMail: + + Questions about the technical content of this specification can be + sent by email to: + + Jean-Loup Gailly and + Mark Adler + + Editorial comments on this specification can be sent by email to: + + L. Peter Deutsch and + Glenn Randers-Pehrson + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Deutsch Informational [Page 17] + diff --git a/lib/std/compress/rfc1951.txt.fixed.z.9 b/lib/std/compress/rfc1951.txt.fixed.z.9 new file mode 100644 index 0000000000000000000000000000000000000000..8ea5904770052de696de797b4ef5f3b1a5535e5b GIT binary patch literal 12836 zcmYLvS5y<|6K{GTjTRt;8fxgh3YY+5sD|DILqrS+NEcakQ|Qf5rHKIn0|G|c0=k+= zF$icVBDUS=0-~ae*xUZE_rBccZyvrgbI#1KsoDL}ky4nhQ{8lKKpIzf zD4ItylaYj^`2i}z;e9g@Q%q@Z_bC$vBcke9(ESMUP23rl=*-u^k+uF$5z`*u^ z$o`7{2%kqBavEuzbH1u??{<^Fa(J_l@y8(TW=wnH+4;udB>H7<} zt*4aN2iP4e&!4Ygq35P*yr7PPcL))*XSOM+X_nT4y%StsMjmc&vsR5r_o4eFUG+HA z>_PEPyvjbFoaVk3X=SM@4j~fnSkC$Tn1_qK##A(94Ago^+JK++=b}5fC{*>*_4yiK zH~l|#IKjKD==ZyR*fLQSKa?`>xn*MEGwQ1zEyV!|apQ#B=PXW6%MbtDl!BJ1= zT}#S?cWeE~nWSRCQ*INgB95ze_bar1pUxWuezsXE4&GS|k)c5I)!W*Hxm_w7i7K4K zU`Awqkgmks7-8_0)Pf)p>N~g~WLan^Qe+Ltb{E&NmvnhXA);4vwthCdeyn%Z)sb~= zz2Iy{W?&M6`0N}Y!169|zc$f=)vylcT$D*51=8d4ljts1*Gb-hk##WIj6G!+X<(I$ zv}w=Jp&5g=Bq-W*5FTWsr}lQkHJ!w)1np6qvGffC7~Eay7+k?>^6jX5SCpo^e0VF= zny3au<|!?)J;}1WbF}oN2i>A`!s4hwGLScyjN{bpmHiVK9BGkDU!&o8%#Ym74w$5cmpF ziE%X0T~a&EtHC$cC}uHrrTXx@;Y}x@T~Z$KF#z@N%G5zk#@PCc4W?hW7w?-t&55Yb zqO0+ZtyRTDU?JAY8EAeFVyKeNHU}nFe+cDY4qB!prjapY^AxDTl_@ND*@oN=p2XrX zJTu@sIV(3%y*}`~tetWr>BATe5baO90xIDwAyyR|&z~VOv%GsRQF=`!rmgbn zr@)?*YLQsB1}accl+O#sYDCat^E<&%J;sqlKHSb6(CkaJ70Yb*g;HEWNHT5EKtzEa z9wX1>BSga8O0&;ATtWvN^Qm>-tEQEm#5LwOK!93uQj~ekON8gwH zuKc!+Ew7x{Wj@A)8kP>upQ4omqJvfb$?0Otigond)~7=25DS?S8yJHcm6+^&=5G&_ z4QynTX`h0T;8u3eezCGC%wNFORsEPF6$WB_OSsAM^H$VBe}C8?1W^t>*g)VNq%*sZ z53H}Nz0is_bt=iIRx2y7AXy*==I1PlcOgaq-gr`zvUu8YcspDtL0%HuJ}phD?6y`~ zN(csp@8!7!%Px!{HW@IHp8U4Xb2mP%G*RB_;+KsVFAXmihF@hDg-{Zhm-|^PLkA^g z6nYV5GmXmR5uAHYz7#$}tqMUeym{wB%HzA~_=F-WnK~WO2*GMBBnY%e`5DQxBnVOD z&44u>wkCGjSXw*Us$B~zqTMvX;U|}?tqrD2*aW95O*|KH(9cUGJ#Hk2NzkZkhV|WM zrWqSvE(o=P7*S|4@JpZ;Io_zVAm13~U|p{0GdCpve=A!LMMEk>pu4dN?U$sDt>@1j z!kBTF+D`)pQG*bCC?aJY+oNjg)ZNJholoG*kB>=SMKPY1^DW~wJ*XVWs)Rj~9W)oh z0|;a|!zR`Edg{--=7w-ePn4fXX{DCpV-C1~l^b4@MdEw-903Ym0AK9`nuAAfo0PNM zh?=xq;HV}~!(x3DF}4KCjbx%}w14oHhQ?rDr!DFJ<$vUiVKvo%x81eRNvrC3h73oD}RO5YLg73;bT*rDJI@ z>C4jR&HD5hNdMEY+9cE~&XWv=x%c3o{>;2;h}Lc1prmGo9wxlWGLszw)TP*xWi9%N z_V$;Sn)CZl=$ZWS*W#ds6Wz~-Tn(+tqVn1AiFP+9vlIotgK9W4HojwZ;D6w_GO1m2 zTzwRe8eP)_HWTIOq{6A^h}zq+)HolX!s)0pLDtRI!f?EmRs65Dx`(?`^0s6JK`TVt zpc2%RbqiYhix0}{o}+O=a>*)*M(Pfs^)miR<4_^U@^6H_L?bJ6cTTyCnFLr14fGF$ z)&C=TrqH>o_}eFTWJ6-Y#iZa2?=vT@^?P@Bt&3GjA5kl^JBA)MW2K;vDuM*_ zUP$GjGABealiU;y+U_1B7a`hBCyR##p!Plsdt6?y+V2*bG)IuzSHY4zN_lRgk_jZ@??#fJ$|SvrP2_(~6b9-xGMP?u3=T|p(@ zy%%5(FEl4o`{S!o^PhAsrMIgrN(#75NUgHD052Ou)lPlKo5V1==$D6IKh_O)8~x_4 z)UAQl5dJ{ys$VBTVD94Ir@tl{Y?SU3h&_BCae3eqr>ye9!fl?MReT@cw8sN> z4OtW~r&;EalS8-We}Yx!8yKj7TW>EdYnORsq9ZLtV{I7rhv%~mx=(VZcbuUhiTTivjfbF~SFE(M zhl6S4aD)~V+8i6%90gIY^{UX*`L1>U_Af|tkL#V`fJu6`OYqxE_+OunDmFFXfWO=biY66l z+glAJZ&5H=Nlm~-1O^m##jFjhdO41Lm9qzgQkg9^JPAkjxX3_fUpOU7#q~Ah&F`Td zn+9yp49YZ2;-^-6MCSzXA4p`XXPFW8Pf*Jqmc#vasAV4?mK!CVhS9Yo^1oRI$zcmA zcPg@l_tnf|Sepa8yIH%-tZ99#Lb4ew;O|0$A&&f#s8CZ^Z5T)M<~5uqsdAP`89qla zL`W$xyjCf3iKN5s$chgrBHTobw2d= z${N<$ehl;;x6ODPxT1X0kHjE4K}7#0m6}z9~Yl^JN0Lb%)MCH`Puya1}c( z4K1KF@NOJUQXIaC`q*SYDr-RVgC!vN3)M0el&wgGvH4JnF;wLmccOg^Qgg9}{3d2= z>#*)E7s_F|6e@5`A#ojDiKlFw`*pjHYZIW_nKne@d+-yMPewzb@MKUQE zbAu75aiXoU^0x5@tc+~-hdQ{8R`kuxcr_oO4>+V6mS_e*&zT1<7<0Of3%n^3)-7;1 z5SPytuM~9d&-P32JeB@@o`pF~_5|C7f>CJNS@cRoaZx9XZy-~JVwl4%l1XShH3KY* zqp`sKNnE;ov<&+eR4bLS*eQ$A*y*!RV5`{37q%goUxzdf-Z8jkl*#%E(1J=_FexY^ zK2--i;UZO`J2S|@l*k8|=A(;rIfrhLdAU{duyj5O<=BO5kn-M`(LQ*jK>Tn`(dp!C z`lJ&a{O>z$`fD1LA;5|=aKFs(1CY59um8%DtJy{YW*FmXVWB93ad19D4|(HpPKROw zIqDZ=t?va^2J_U?4{Yh|IjA)IeM?rxoF9J#lNm@x{Z96d5a`#cVrkPX z5L#(a->EyO9tnR^nJ?IVVWfnLgmmJjNdg&vmPcnIAHAKlsgdR(BoZKRGHaI(BB%8`F-oHNo_>P|3ZfAgL$NjwpJ25OZ%NGSK=!X1Zma&kr# zp5U!U7w;FEHTgi%v#dMCUbHg{&?~!uhB9fFsd(~|yr*PO__WjubVqYFFP2sgl4x)s z$va!3;XMkCgfbhEn{20pc1L&NDF0bNQl2NKF58z3J>cu^WXeH`yr2fuv&5T3gAs}k zyvrGi;`>svbS}AG9mLh;Dh1*H5!Q(HZfe*a1RXyz33C$Np`F*-X>M7`roJ#8>C{*e z+5(Ph>4k()M4%j>v@;o{7Ix(ZpeBKLSYp1GQxnrW!up_Hp;TsJ@vyock}msq*FHMUu-huW|7LiExFp8W^B zW~X&%fWtUC(l&*a0)P~eP>IY&d#_iaB^+BW#Oxd(?4Y=7tJ5o|>q2ebdbQ^(4m=J#LTgJtBNs=dTaurD|(ZQZ8bq$mPtP zP%%%H$P@*ru6aZi7^{8V;3XO|vN4@~PRa|pof||FEWs6_^5;R^vYzy%9Mu|;gUWw< zpAB(-lD#t#z`HMn**?*wNl1#uorhQ`p56{`9$?bZUI+{R8;}4>S*+cFo!BXoWgra8_Bc)5U*tkf{P)x%@xb4r4>(u4K7N8L=mCGrWP=b+A! zVAGzUf4JIqL3%X2@8nB# z%uGPJCCaBb=d=BGb566*^Txrc5Ljyl71^sxeHe<;nuI1vm z(J9L8lp^!I2P>Rh#dOO(|2)?{ZLbz=gJkkq7Al^}*{Ttnh8r(Px2>zs-Dj&T>p)5L zK(e>4pdQ&=?$MS+43{v?ZZEM;SSkETDtYEYB* zq8d+O)z^lEkzL9f{(->b`wz&TmYl<~hJnMobv(e`r-^47nh1Gl>_$My(jFmn~^f){? zIOCWdpbMG+NUEHSbh3ZNDZy2o52AXa$9DEC%Lc5;4jpIxT~C#ws_m!R<*w4~W+8-8Nwthe_j&LIqSCxO3gEHH!Ss$_p~ z9DLe4$k*Rtulhv(dvfxrQ5^M%{3N&HhLK z5~-hDGWTA}&!D-XSs0+@-4?2w>GK*4N1V=|K3^rp2s5`dqyxlb08 zQ($lW3hI2tKQ*T?H%|?J8f-HR-e!GzO!nQcVLBD`965cHGEkV)MElF4=tBYNw1GtG zB)K-*vyX>9A#A>;Yx{?tsv=%;CsMBeH+1QzDgQ03dxCpAE_z5< z_X7xxwdWNE^Hfao@;f5oo4z~WAdHxC+;|u>4VIUu+_tyh`XC+^^#+trh>Tm+^m?dpkh4uzmvC7ozyX)QW6(8o@%eVEnZcbJm z8zW>MP<;|>!`ye=T7qFjRmofTF;Q%AK*3TGf*%l4vBIvSTOTve+hP~j>DXJ3iK^l@ zMXIfyU~Ig6LT|5PT|4a~OgBPV@AJp9#XassJq!LKRI~PK(U`oQR}3O{CKeR+RxWC& zx-1%H#QwwGGYgWi&BLYwaB)Z$BAjO|lxGh}yp-0gS+r|j#UI0t)>1&Q}sjeIz`N2)wY@zD$|@-(CfpPn?Ebj+u`uh?<+ zT@MAuH9!N=OD~G+V*B+%hYSCa|7{x-Rw%VB(g7>T)CRZbr;aN z?@1c|BO{dm>|wA=zZ9p!cDP}RwXS|NZF!Tjf$SZpx=GpK@wbH!>LrvyIr(2~_OK(U z?XjrkK6&AU*n0uH1yU8;;TRue2d~RdqtFyJ#Q7wz-W^t8kFWWi2VQpfNtZu~+z$kh zA49+yrCO?=#umW0_gIaQ7ZE8^7UxMcLGDB==saKrd^hSOx3wfJDiNQ@xxm(yoI2<& z8e(YamxC7edF`QQ_1Ooe@?Q;)T)o`a6<2Za=b-tq(pQ^r zfttNu#xE-g)2=dv4S7u{4NU`i@t*%}jlRB&tRtdDBSd0XyRyz1+!4*>W8m?{_VUF| zt6L8KG*r8P5F~(y=cdTPUZ||4q*-TcsYJ&Rf&q4y<4-v1o|n&d>YYPh9Vx32$l@mt zR!#3IGTi#ly6ArB)_IKz3SBTz_hu9+UXF&>ESS;{=M5+=RZ1 z0G-F&gOC`T+H;8L0OUi4Y_5BD(ZnFcSef;i!dAKGakRS~xh_AnF776&1`AikSi;#j zn#I5EacU_(mu+y0+tN`NKJ)_Y;Q;7SNOKeH_>AYL!`O2=itx?rdW<3E!ExT0$D=?{ zn`hCRBF|VFqU~wXC-kCXDGMkVzB6}wPz}`(!P=$BDnA&=TiIOzlpFT#kp_*hw%O;k zJ9*X7p>>BkOC8)n7^k)2fJ~E9K;^vjA&ny!wHTMwoYLC~94ndEeC3pYi|(Tr!Sdip zqNWT(RW_Bb@}v(Og*g-hRs(uyc7h^CxmCZuY}hd18lBy>elg;$^mH9J6ufXj+NO{| zQL$0%yIuJ3j7M}-f#tV$W6M250N44bx@M&LX<;G2>YC>X6OT_H=;~_h;{!h4eg47@ z*fv@_VsE&=SA?Ekk`UZ^4P#a4?Y&*B6*MKMH z!RFcbSoS)+V>>;P4we=wk#t!FJyroWzX>=O!P|%6LjMu+d|-vPd<)^LW5?E?f!CQ{ zQx;D4QaeDXY`sx`g5S>;MbZN}33j$t|DAkJAV;-!u!@vf^?vhk>-rzJQslX1}_)#xgH%Exu z5A$lW7lR$rX~oAq;HrxUo&!|1y7rwUW`%jRiV9q9&qv5!cwbP*;HF0JB+` zp=}#oy_4P_vf3|w2HbDg2L>Ow=MgAJ{9czWy{w=!P|#R#Y)R?Jb1S794QhCzC>+y; zHE4Jt>NS%*hw@;Of!^Si(L}@-F?U_j%k!x(!CvV)N%6>w77YCz_!JL$aunKH&^~z? z4->xnYXv*(f&TYD+Z~wwdTpA{_eVOv+`(s!rO)Nihq>b_%@S~MH6AGrvl)bWy15lMCCrdQF(CC;-Ctq5-pegqy6iGtCy+M>WLb)Y|xdlY}8cd(rCJDs$i};J_*% zgdCov)%hpNd4b>2NmK}SB_QOc5^PoP46_WAf5)Mzmp<8deoVHE5S%T=dTJgAk5JE( z$WXOU%f!_wdDEEz%wYx1BV$v`zcWGty_@~mOxMC{l*C0>7HioITh_)1Jw#E@_u|m zL&HQ7H)NOVZ*9wLp~IAH9huGaguY{tk7o;kQ&>;WkXX;d;Or38v+l;p0M`8z-kTFM zFlq>jFko-zuwrlLDW_tqH)gDo9#TOY>24pmD!YQ!;~OW=OcXWX+--_l=(3K}IQ*fR z?g>Ll`cOmtgmH5cM~BDEFNVNK7qZRWVzJjyTD)^<3luCNNwJK>l#J~(1UE-(s3ttYh8(7`ZLO}79HDK+I@7i73fE6r+1?+4 zyhu!+1?1{W0DiPhJ4cWI_l&W!`_-qUfn$sWpoIpl8-NDnFAEhX_Zw0VHN9#iC$eZ= z73~TFvq8wwRHd;9G}F69t+|oc0$T7yY`RFfc6Noo`fPT=e?d47OGHQjA+bo_!>#T{ z@+6F5z4kp(FoGg8Tz!xVqZO zW#ej+Xs}?`yFosL^o?}`(x9#)HcwtUTRnXZBJU82id!gN>Hw=F+)6u$%DK;-qJx0L z4kI8tF*gbny9qk4E14WdIovECY?Xg4aq4^cOL#=QW;VzPoRDBuDz6E#_CaxYN9eKx zxnNv)4kK1P=oB9J8nk@V+IIuiwrmASwYLX+#O?Ym+wN3m#RELSo>szc*~YU>2R8ne zSecM8vB(6O#sgX`sBtTW4o>oDpuiF>9#vS6KKLn$>2Y6tOT#Uc5bryxN^Utrw(VX? zKD4~O?w{f}BWZXj3LhZcdqz_EpU%e{W!0Y^v40b<^1|e&1xYSzan_X0A`L5R~}^krXpu$d&g6sN-9t13_f16mQKX!0Iq%8Y%f1c}wxQd`HD4Zq_G9 z=Z(ET3b0>@@=JMn79YOEOTM-z8AuA<#6zEdS+|)Lh+xH;iE3Ppw~n8zU&rJOGHBbA)FDwN? z_D~Gvwk$ghu26s1nQ5aQNVg1nf(lbyXIolGQ` zmz3JxlF<{IW9?AN!>niB{mj$5_aqt`hkB%?X6sVeZ55(1^OojX-)-$3{4czm!|*A+ zZ-ew`U_A)J;bTaBlSwVkMXBOz5-a~AxmSe5XzGaq(P-*sdtgC(J&v-UO!yZCRv!Oq zK-@4};+xS|s5EZW$gt2F4+if`o-f}_wB^56eWpQu@!CRed{LB zE0<;=$Imk>^Yk!xK-xawObv#txznlsMG)i{RT#9@%=`~_%1Xt2iZw%i?Zp*!o#gxF zwI8Y4TN6Wok>{@wO+8Pp(mvh7O_aryt(FFPDtSbR`I24B2H>+IxELAn2ZYJFn=T;u^ZA($0(`?fmV+kx`SGCE9-VW?R^Z z)3X@AF+o`5#AfrVG|08&kf5$#@T9IRhoIQdK?NUpc&_5Kkwaf8YCKG?m0%73LUM-T zYIzzgdr(I9gB*dyW0J)!Ny|RshH;T8=^=Ux<+)uYByRMioR-P_8<|5e&m8w`?ff2%50Pn{(6)>sD`UgD zw@)Yhqt$^{R%#r%>dV2ptRp)2@tLo+k}c;kl2&x59&(R(3imkapDLnJF4)gW+|-iP zwNFy}doWqmq4Sc;q}=TS?W~IHN0D7YsF)OZT1Xw~-)^S1Le&jSCnWclIkEVb+z&_$ zBJMX>i;}ndJ8U-BCY!4@xPZu?P#M0$SVunsVa6XCTvE)XxfB^y%c&IJ?eTmtA;c(! z3SGmoYlk>}cMCNB*IkU~j$w`zw}~(VH~m;Z&}Xf_19uBYzZ20JK_-b}kLF`y*yO#K zqH&Eg6bCVfv|n*TfY`1~->I>od`F(@#eZq=`0{RbM{iC3oOXOl{Y=JJg>8D+%KLJ1 z%lBuW@9Vtw&gj5o4#7{2nXJzQZSCS#hRzV~lm%J11GV{m&uW`vxXl;U>!V(dgn~wir(jVqsO4sLXEFnxO1_A>@$;{O6}>M9dLXW z`DeDIG*@u#M0^uAT20py>BrLUmD`ivnvAwqdgVApJxeamQ_tNG4wiYr3iJYe#o||* zN;Gdu-kN3-x^5Rb`KYEr687tiwZI-H24V2cvu5NJ)q+nl(&AGrBjEesx%gKiTNToK z8y@mpd~pgZT2EL5ysp))k;{~O&d*L?f;DtbOLO9hcY`g9%zMqC{ZDSc+FhaxT54@| zK3g8uXKxpu->io{5|?eDlBO0Pmda};3Bm|8OOG_7A5>c#D4Xs0Ppc?yi&0U9%7CxqdRde)QpCFhlfEb~r%AenM1VV@Q$DzwY|Qo0YI*;OoKw z0+I2&dTyKfwmxpFI#}l%;Uv;MNiH>2Hw_862Hn*5$~7U!B_Mo4C8F|BiTS$ZsXH&k zM5nxnwU3esTXhX;ZH}yKWhs1}cETtXe8Xj)u>Qc!p>Fz^ft%`)KEDgz-z8ie6(ZEL z&${T(_$e-jRKGngbgI;P?w)Xh`NEs@#+CY;fn2t;Dvc(B?O`*@PY?7ax_J)q_fHBcXKZr7GPP5hE;9J?d&%`miT(@B-zSqPJwUL@zT z^|YR)ug8G=%9zHona#F&2hsUv=#=}~OX0^PT>G*A%odiRXaTF&fR=B@Qa$#S7dx)@P>8PGvHAe|K%;vy2?4sa6dYC~05|wKZV7evqyRH}N zD;2;!M>wMRg+8@(9gCWK_7ds(5wgb_N>>cdFWQJ*pKK$&$YdjdJGbhu*U{~MN)6#= zq0idbe<%`inGbY3P$yYM=Y8V?^tNIt+%u42E?aqjb4gNh=#I~SJO0od03#bUS_Y~dN=A5@a1DI7*MCLU!Tc9+rHhxKge=3rHRc`|r= ziZvhdP!-yjt`lGS*LBFDDuhY6s7bxCcbC;;a|NP@Z~x<=vR+yzK=T3PrP6?J8^%e>_FcMI?#%xw7sms$Yd=>T%>G_ zIn1o;+2fk_w<7;c7w)diS|nRTGdfxb{dbi0$62zXMr2zF6_}Vsv#@}Tq;P$`BJGP- zZUD}HW2@Vp{7T(h&$+}9M!|Znf-U>v!_F)6iJ+5YAGutJ9b+yT@xx!~!hSMCBhJw97OOlRAzIp8z z?K>a}Ii9xLE0f>JxaHlr$+3X`F z2221W0wa!)OO2a$-laWorL`S^UNBceZC7`*AKVB3nbLVtzTpCR+tH*Ew?+IyUVR+% ziipK-8RPJg(OY;to^&r9m?I)RwYB7ZC;VyO7~%2WhWz3>2>eLl$c3X;&pXppJAYoM zxW)1uU1MWol|4xC(?*7#*D_9u7va5O@5y`klO%W;o=_FLGxq>oS;sz@oRTtdEQOBB zxs=hz-XmB>CI8HO`SRNqIa;Sm2m-9F??N*#o^m}eeY)W_MU(jwcVx;JHU+KYc_01% z3EVk9^F>Qyz!JP`{|;gQx3DJYHpL(Uob;Hy*nxBO?o z3N{z+QPa;x1Emqi1`-*ceQKQ`R{!<@pGJ#fWmdk&Yv*|4wZbT)> z>UVMW5AB+WJfC+##SzA$#9GoH9Ok2gU!}33+!@SPV4URqloV!@a2hl=1i|MSvz~;= zy0fC>vcKyr$zZn9d?aeOtX`1v(Q49f8m|2^eS2zz50I0^Ur8}JqL;F`F=&(Ih%0vy zDX!4|(l-Ajv{8$mYg!Kj|y%lck(RjM_@L#0{ zVZ3#q^9Pyt&2BWiJm6vK1EZV6fap6cPjDoLsrsDh=S(}W=O!c<=A7UvpZn7!c#o3% zT4AQOwpjiI4QwtK$pI=w1zk-eC3^>DoYDHw>$W4+{l+xn!{FL4G@rlbj1|{MS3;66 z97>U`SPiC?g7KP}1y5>$RRC?bLICxjDB!D<^t?^xDrW*p~Kz|HIZLetGS6~u4dn;YvK6$G$6@92K0pAuLvgaHA6R|`kr4U4*)5i{Q26R;xE zVLOe}A~bpDx}fE7fpT7rYWI}n3GTXd?Y=7RK{&$UrJ8W`lBe!eSiVq$26u1~VAr~e z?bd2&#sBKXK50Db5?<+IA~T+%5sYJ75`qAlqg?RK4!qu<51Fq#;KO(BVRDBZi3T3E z?Ez`JB`}vi>>MpapTzOYQ!#W2ZM>rBKo2?HiR>~NYb9*F!D$ZxDg^Z%N e9=%z~ZroFjuxU@5Lh@_JM8)|aKw;J0+y4i|0)9LI literal 0 HcmV?d00001 diff --git a/lib/std/compress/rfc1951.txt.z.0 b/lib/std/compress/rfc1951.txt.z.0 new file mode 100644 index 0000000000000000000000000000000000000000..3f50fb68f8da776fc6d6833b2cbe11cf822fd075 GIT binary patch literal 36960 zcmcJY>v9{%lIPESRqR9bYGPwai+~7#;6;`lLs68ib0{6HNzb`>G!r1uBr(E;gNr13 z&wkvm8~Y~v82fEsY4-Qetm>);L0R(92%EyCJ1Z;inUz)j@$Y{3=70a6=>vPcE*BrB zvv8v=Kj>qL>F@I9D4!W&R z06QG@27}?GIG!xZ+4XEVFXs=AdW-UMI=gvNyqo}R+*=H%liuhPQ0hOu>fJEx!7kH_ zKRiEvadLQeEWEB~<$PWais!vWuXthLi_`0}KfDvG2LKMXIg7V`oe7o~BVNlq5M;RNc!Qp0F4?JxBkh=9Ye4|@gUXnK7!)9uAZ zf76&%oHUC!CB&-a$;K*Yri&BGP5J^$i2Q&^qGC{ds&X{Cr z*MAr;uBJ=)an+k$mRlg&A1#r<%VGia`N%{-ykw9m_Aa59hUAth0B0^O zb|u`s9)^3jqC|k`Rc|4h3WhNDDZ^|CVW5mg@$;?e=~#pD?eKCah3NsYKPr2(k&$8Co4qrU0GmVD4(5=XdFm#R zm-0s0N8SuK&39JnM@Sub8RxB8eSh}qM5IPZi$0QjUh-+lO+NAB#~)5~+0%=~^^?bs z`P*DBo9M%1czJy_?9U%xPc9!v$n(cPkA~-uf97)oNiRP(uNLFcw+PQ6lb`kai==z5 zE>UL(~my5>x;& zI3Gg1*^n_uQzki=rACr^Bfc?_GqoB{u9rp{qX+aZd*iLb=`x)N;R*D?NJEF>TwG@0 z-k3KF*$NXU1|K3Un$diLid>cf>$;2#$JZmX89-o|fl$L_^a^MS9b^xkw8$fuoP#D% zaVUsFcm=n^Kxb210c=&G$R}@4KURPwlmaHP?DZ{%G@p2WgGc%FtPENOsMD+y>U=cqzl&x& z4SeUjVmD09F=`8=)0@fkdQR+sN<#LF;m6#3GjDVKDr z!Of&M9`@IRYS~azs`&lEmPWHij*yAKFR$LOanhdvI8I}{6l$xAH$@5 z#$7A2sEvxrw3Q&utO+lj*KYF<1o(5(0J`p^6YVi07U}!ZY&6 zrKTkInH4xEem*Z3md_EbSl03|O6kKI6xdJhHC~ybi^gY!<%}6)@;X?L&ofrVeDGRg zOTs%KLoLhE&DRRx?Ti|kyE+60z z%rWh}{CRj?P_!9g!Bu(SCZ)|&s~gD>49du}>8viLABIF}B-b&WRMO-Ezlucpk&1|b zX|x(D2{u`jPg904w_tAYEa6&X6suBI4@nzeQ`(q%QK=%@;b=7dP*6x2P>bmGad>7& zMq81+pk5x)QhfS3Tvlw9&CBK%;4qIfRe!8vuqkc*KuM}LIldDM?n<$1z$;&_ zbkcD@EHD$~q;P|DR^+2RWR-R^p_x&nRrR1p99n+7rc|Y<1UJ%DX7uj!W5!|TBW)- zm-klH>%2E7JU%D?x}<(@Malp=*!!``v{fS^_4{(9=};H)jMyrjD(z4;R)H}iemz~C zOJE7b39T&-t?c5W>|afW)Rk^$NdXaPepQaHFP0cG3b2%TDE<7~a!7$EGL5+1>IjKZ zt1}Msw>f)R%%rzM(q52)Z_|dqB^t{1!aJ~uHh3UK423` zU~iaCsOwrtbS7LyKt)r30c9_%KM17sBChe)tF*q z-W%gpr3J;5=z6gkEuWj^RYM`xm}%vQ@*{Q+bLc~bnybpSz5bPo(f1eUbbgaK=L7Zk zD~qwAfGvo4exYo)m@LQVh<7v+gW-jxuq^LsnW5C$O{COmDZguSE7K_+``uE;2tbCy zD!igzW!_F%#+qC{VLVSONJ+UN7OaVBrM6KTez6o6CE!dlR9y8iWalMyaNM?=r#^lq zuALx}!ZcKFV&R5GfXZy(x5GMMOIY0wh_Hn(C$SJnVYas#j?r*)~kv}lEcbN#mEX5uR$q{ z;Q&N&=sz1`ReA3IKJ`rWHnAT2+X7uSGl0mjk`NHlbTtP$H5M6vZ4pd}62- znawN)OWc6_1`=4)sJ`+Hy)%8PhBML`g;~FcXas|SHKs+V0@wxB%gC3D0i6JrUb1e_ zLQijK`X7m5wIZiTTtlNE*k>JDORpt}2d@z%P#U9ny1ojh>^1mT&3WC=NYRGGC=MLQ zEXVV5^j@wX4x}5MbfQPRfwG9zb(@Px+KH$rm72Q++90EQJ~>)j$n0HWkW4 z7_q^R&WzyY%+x5Lk+~5VYJ@D9yQj%G7>uCkQn77?O`Ap>!3Ybq=BzUqVPMu9&C0>2 zu#k%$R4fQS&cKL`PrabKWf39)tgoA^tov=y+3Hs3p5?2Sp_(mPjZa-UsfS=d1jUfl zOkC8YS3aWs(C+RF6Z;Jm2>z-}(v;i;#gY>Ui+@!m8+Yn<0-@E-OL*#%l-UUpK|U-u z7g$<2Ap)FJsmQE_SSd}Vzkw)qBd~P7l_ueG84%x;dci3T9)||1hxv6`Y10z8;6A{NY0~84*qDdOj5JEw_iJ&FiQ4!jluCyal$5jJ$ea)J`TL)o|S2y=o z{Hq6#G6R~G#Bg3%9M=vsSHI7-?(_ zyU(W|k?nWhOp+Hj9yJ<`N1~d){>DFlE1otQ1_$R&g-iCKN1iyg#uluiwXV4D^t&8L zfKq83UU!1iB-Bx$eK!W*)Ltvd1ibXm-x8XSYPWpzC`L%x*gX_tHndD4i}_OA1;A(e z;zv z@24MBE~%nyv($hxdlsfeq^?x`Qpp#+YYOnTO_AE&t{aws$#^FxysusR_^q1POA~ZSGbY3Se}ZffAE^kaDo9 z1E@h=5%zYwJDqlGJC{1U$gT^Z{}iMDC^X`5n1|{(Q(_t#y2i`VVyG&dQ4JzcPYe+3 zr-seX@6*bP%?y*J?xPrs45%_l$oM}bepruWbOy$b}%1ZPVXJ?T!{j ztw_~ULy|w$3%xc+Wrw2FD9)(cHnNA|d-I%zQqXPv(JNbwm1(e7dJ}Nwy1#AnYVp6l z(c87uu0W(ai+{F(`18-%m$F{=%D`aG!t0S0($L*ZPeh?_igxj_Xm|5v_D5E?TX*e# zv^9732E+>()+%zb3oJzkf$+u=+5o+{B=(qnkpP(gPcvK51PMW&_giMmHI1~01&;^A z<=En8pTC;|jY9hfOf`?cUdYfbNo0c(HP@TxoZMg5-@RvF8Tl(EZ_vYK1)kjKVq2mX z8OgKYz`_BGBD{OfVZngtIhDrdIlNIVkEZ2i)HlQ6iC-c1*)j)G2M|rbpD@0PTSQ>X zV1l2wl*!ScK(B;{Ks8Z&pv z6?tn$E73tS@>IaDd#H77E^fwbMYOQAQJIobEf62|-V#e=x&zk&b0R?1S!-))mf@w6Zp22B7->s| zCq(yg6P$zq7A8Tc?6{J9m5$&T?FpKh2mAI-0>ckhq+x_m)mqZvbw0dY;vy)iJ4S?i z4S$#GE)JpU%b|**mOh)mcy2A8UPQom6S`Rf%`aXjsL6!NJ8D~^ia{BRr9bh=tJ^B} zO=z#ftX=R1;>yU=tBqU_JljZfX_?OEV<_16`&62YRa9}Rf%9pY=1FxnfSTY$@DlA! zQ@zNSV9cX1SPju*--_{>f%Us$+#eYxkFZU5l@zYUzriRt0l>GKeL~2ze@ay=7$KWg zjwMJEE(Z?zLUss?iFniNekupMzg0BkrY8IQ_@A;5ygfB#)ptjTLZVOoDhuoDTb(=} z*JXNl=Z%kb#mbo4p<%D>?Yr&^Xg>4ns=x7bmRnT6=?%$89n0|Rk^Q?}YY+`T>7VsT z^22kxh?Hb4o<57m|Frf;zq6JG*;nb0k9yb5K6G)ES=__i_GrN!-IW|q)V~lk!*o2e zK-Inn9~Xnf#s^ux551AKGY_!I2F1&9ai7f$-5`%$_~HOPDp|Wt|7yy%Dmiclmy%_M zAoJ^nmam`S>vAji9o|aO00uO;0CRcfAJRlTL}lT|^Ux4IC3>Zn6jooojYXJDm?OP# z>L8J^qQGz zsjKFgYA}uz_T28HRFPMJpEEB{w_i8~bmu;{*nE<54m7iK~6(t;9^{!RJ z>e6Nu7hZg1?MoDeeAAnYM!v%j+SP*9=4K{_uc<)0b(B|P(yImZ8oz83alRS0F=dL= zrnIO`NDB=rjx*gpg~uL4jC&u4Iq9MOUrO^3|L z!lFpU5F93!Bm|WjqR&=8RCthNpqMS`*;p;)e5C@FQ7P2lE=_Bd)j6>o)QFzEzPc{e zzlNMPp(@hH9XtMOL!{4upQ}1>O z*xo`(3sUrG66LT%OtBlR&vskgUsYr8xANn}g*5Mx&oGf__Cq-0YeAm;TZf{p<%^{x zM2S!rQ!|p-boEua^$zvGn9Cank|fcs{_N=aiddY6c;Tp~Juo811Gl1q;OmPOCmLOO z-L<--#%r!z9lRmmw=(9xwBEn%$lf_)gkU%@oNrnmy$B9Oj}KeKw6)N5Re+&%C9#$M zGDG)T6?Lr`t}Gw~>=NHxMNK&~AD39+OI0@o3gvvT(sa-&Q#QdeAx#<{Y*Jov(N&3E zp(djMn$jKr&Dq9j`!y}d)vB#}HY{XFxUhak-(%1SE6lTbLft>90BiyWJTt67A%zJY zoCyp8g|Nv3@?{F=pqfP_sgn*N$p*ycK!GqY8!RQ~kxZS$m@c#lKCNX4*CvQXyD28rVSo1X!D5o(jKD)K3_cMUSlOSf4qYHX_kK{A`6oY(G@dM{rXL@ z9s6bVoxiFXY`>-OzUBgvV#9(cF#p=asi)r*uMYqGFV9|{o%%PA9&Hx?y^c_TZ+Ly< zWrX5UBA6gHT5mUtr%wU9_ErV0katl2q>F`~KHR?ce`MtBK+Z!Nhc4;HqGdHBCQi+J z0|f-J^*}MRFLfN;J-Xq@eXsFDy?lYGRe^}k*er(8>;s5O+ld;<4G7)6K1aPv<}5E$)zqZ$9Ow2z8A%$fUN|0 zM)I$V$=y6<>fB?RHE~U>4;P?Q?A*|${~5WuOFVRG1au_n{>D-$XFE5)z0K|bedW4G zkJev_aNO4}kp15(q6jm}<@ngGey++#>Z49ET)(ZlA`iJoe8KC(>frxHTF6W=bora%-zzVOeQLQo9-0;<$!R>Ha%1ZEf1c^2cr z>H-#u)6C#q3=Fdi>JW~d!9}`Y1&0^mf;F8rOPp*OcJXlsud|s3k#0=vllO4 zAD$#wd~4O4w!2&HAV|+*0+fO&Ut9GG?fq0|4}L2f>}0oCon;VQmkQW+8VT=X7%%dU z2w?+dLU>kRchqL9j5o%s*fNI4w>4zI*vhdye_9S!J{V&0)e6A&HfEOg@_2FD4CX?! z-QxH7BmUo_k~O>H6O*`)EbR78Yj?ocZ_`-Xx1sZc*gH3yO=o#c>WtYL;o`Cl&GxZ@ zN(5BFBSTd?%`8U>lu2Xd8C+GhiY>B_2@Y-@9T2^ut>`8|Aw|mB@=^s)axD#B_15_E zSxS8r9tq=+m>J7RzSgAjD>^-xroo*-wvW>FDd?wVMqZNjN^n+3^Lr4yth4A-?Dy;E z@7VUK%{a4i9ITJblCq|-V^l?JKaOV=KZjro!ggwzG^=3mwI5SXG zmQ=^wTyvY9qNS_NASiYF(P5B18C#kRWzdCiM^-m?q!s) zm=IcbqnLX>cuQv1BMOQ9`ll;hR23QQjd8^2_R3Y+B zoe~=~cil04zww+=Ig&*yAR_1b@wv1@q1f<=&^y-MYc|{a-A}>~M2_9gK2olXnFV)@ z^1ovmzN-%R)SMoqI}}vcOLh*%Y3n61<3%uOt|T^jIi5Qw53JZ?5SLDU(7?4@28A0W zkhdsGj!A`5<^JT{G6JOVqPJ$7NL6enca-U2SBYiXLZVhuf-Y)52*(Tm0fX0{v7Z@tlBpy{SsQSJ4-QcaOH*Vsp{sX-&{~of+a4^{RZxzu%YHhz_$EPhX(WKxF)#meY%lU<@@JeOK6WNqou)#f}i4VfPO+1%G((7Yg|2^o)^!ZsF)5vl|2q^gUCycjEjO{J&{vA0ACp{#J-{*^%h{v~Rg|-Q#u-eRwjL4y znA?Z7Zqa<3mPllNS_JZA_0+hc5R{sccCTjk;>;LCpc~}FBNr9R;!Zw+OL~ zxjfYNDY4y2o7T__$I=h;3TUn_9w!_F=Y?8aSFi*+tPvK2S&!C*;Ig!rh#R!~(#~;6 zV5CZPrP8_;nhn_UR?>!q%ZNNOx^YBNY<`GWGIWAtBJ5gSY;5E1yIpmteRBU9_Kwc` ztpD2gwTX4X@4}dD9vdCO-RBgLP@J#qK<<+B8(Pg~r@gheyRX99OUOaq;JHPdoIv?t zX)bYs-i1oZ`V!`lR8y?hww6n+9mduSn=7FWl{$4PCm?|qP717<*?!hG;=qBbnykl3 z1Yiu}rvYS^k?^1~y=a)cdSDVD$EYfY?FpEqDTA;PdMs?Q1}XSgHq==Ih(+pD^k>u+ zl$9joA*@`G(u4%n4q)-iln<}G{)6>|;C=7=Tu$s!{dH_J{><;v{JG%>5){uwsuQF1 z28ZIUem7r#T@8+Eg0>`FtqXJ0>X$0w-ukWvMu9aK`JR$k!E5j8I~7V>-L2Lx|8fc? zhuC`cb9b+?U$2P*WS<`$NG#&DclDhC+1qOAy@2RnU9sP39N4x_eeJJy_kcGJq#M86 z)pznSt%F9}3yAGKeJ3CX2aT2=x5tO@YVQhyLvhSCtUl}RcBjz}^V!#TKxwtQjaH{# zZ!FroLfoNf-g>q7^qoQ3Yjpg#_P)LY+g7L5Xzi|&Wmi}Gl}-RbF$jFm9UUiDdtcuP z%5JC8I@np^(60F6P&9MB+I#xW{_i$+bT{Aa>$9L7bfHzg&qlxQYAl07g~(s+J$k_ZkStDlYsYrnXvJEgu$K^qq~}YM3Ah7ky_C!}eW1hF5!691x7o zR?7(h41H%X4jQ|hy&f;NmF&#EaO#RToy(Z&%y*W{Ss262l33Wm8TJEf*a<{lZjn{8 zh3nOxcPrT9NA1igGiCKEbh5t5+_rWqcV6Mkn^ohm`dQc7cL_y(m8`&X9BTgAk8Ia9 zD;H_=bR9@t+jH&D>_q({eX0X-&bh4Es3tMas!pe|W3tRHHC__P)^-~e>ULHn&UhST zz@bQ29Lf${ws&oP=TN#G1a5n0#VR06Twr5KJOCPWg}1xi#(`5py}#@#I&*TmP~wXsD9LalAr*i^Rx zz1!MZ(Pz^xgR-|}lx#^?9LfRq2_Fs=(=$N{VRsGA>|Xjcaz=Y!zDD=$IOt zt<$gEk7a4M_is^GcW^o&kn=b~pmRwmC-eW0S==T%H!If8KDzHo(wnnSuPzU{laNN=r@6gEy3lU@% zbHzG!i1uR5unHR)_f}SBXo>0Us^Yu&>>CWCMh6`9~(eUiwOJ;!pPFDzp7Z zEi<%sq%wik;cI%pwcnKNiKE0W|C?-Ja4Ycl0f1C(ib~ihTuRad3>!d2%W`fw`mAlL6I*H2Mm&DQv4~i zF1J3X8u}ww&?p_?5>+r}Py4$K@ybru^j06oCvVZ!lB%7ip`l`FC?x8Bnu)F6^oM#x zL9`LVFX@2N^w@Eq2OJ6$(%IU@cB5mBmo@uC+u(-pvGWXdv*Q?%2vu+h&*mS9xBz2Wc?YgD#vaa7s-6ctJ&RY!Z3Z#xe<+ga=Z{%dGy+xzb# zVMPy>)PBv`-@nHGS0|*1rhhqT_9n@VI2fN3BSj6uQB&c41xCaPbkc+7pw^PH9SDGMcPAEr$u7L7vJp%5!D| zWG;`{Gn-U>h(09zYd)>}r1+sKwd7z_C-W;3W(h!R_X*KC$54n<)q+TNKcP*7FpsS= zYK>1f*4}9zd1FYd=18d_<*hnX?dI@dW1A9pXG=$*7c`3T;gid9u_-OY2gN>}#Jk1D zqtKetYHengb=`R3yV^~(2AV3{VY7PX(p?j-0VK%G2xhg&{bHjxm%8E@>}8ZBL_1NT;@ z5yc?%mP*9Z)=)Jhh355Ik2DvpSPY0!S%ykm+~l&Fmb-VuEkfEOzNK`cl#n=^!11|VP0bWita2*|R{$qx#vkU`Kjm_E*$1GzF zd=CiYd7_ZO#Ch7iJ~{OgtAk}j9JX4%kmQTNNSz{+0BFw&y;@p<%C<-#^4f+3E&^ig zyPYR=Eirgfe1G!tY^(TtT^)m*ldJ-)Xp{mWEBkwTyMNn|RDhnpJdKc^*8>yuA!<3g z+~8=zYz@(W*@@^Ju|^W`Ig%2rAmhpVhBATH0qsfm70~`RN2JDC7aQMeUYM|*P0qk^ zOYYLbRnM! z)~M;JgSdr9$9qtnQ%iA6-|$~Ile-F?GdnINLm6R}V%1A?9Yvh>rM8x=nbpR2V{eo7 zY3^36PC${<#tnJYrYtV5&{%50{jMv~148IJ6Y@C!nS2rI|8UrV#5w(fS z(@5b~Om5`ZIyTfzm~PG>*0ws`BL4SMH_&M1wdJ1g~mZf9d8@pU2 zM|NcSY9Gi#!v-3q13zt0P3t^%809CP>bU@6O>}PCtc0mQa1kNls8MKoI^RA*OI4AFtAlKv`yl64`d|063suEw@*PD%%7>@|PZ1W%BSRYbo>iOQ_1w z>!_p9OI;nohFoU>N!5U$IYdH-5{P3-nRX@8oc#P^ZSGfZt39tKzgxXERWRwR z@c|IFJ3evLwJRG@5d1QoQ6HXtNfvAeDsPLh}@za^{g8t0O6tGgmVRN+ChSV>rXB?|#BSwh%<03@T zBwPrkr>y2+!m~a#nTiC`t?bBsbrR@gTZrLDKFFOx4%94SWl#K=`|)09uo5P)uZ}&6 zCQ{rOl{7d*@xd7#4$uit{k16b{sOW4KH(K2)ed$I({=@^QFx*g5k#jzY71Gyu~DAB z=0?iZT7-)NE%$Hxf9+g^xXpr&4ytB4U($c1>}dS14*9aCya^9*Gz~R9piQRKLsP7A zIUcg5Xavh({V)bp`seq^M44z~Nl7C@n_XqlXhZVr&e^U^vSpgQf;qsI3~|W3Rfev5 z8b6-qMYRZzyn2$jH{HZ?jNNe&yFzS={)95%qN{{}>Q9UANL`(WWIFzUWshaed^3!l zO&*DgQf*Q5SQLN$%U>nEnzWTiI>;$4=AkdJ;*Yjf+YC0u$$V>zJZ9Xwuz-r{H(zqj zl)8Fe=_~|^yUi(W(v``??09)JN#{Z6 zImK?A6Comn$rG48O!0(n})G_{j0S)F{e7wC&4j*2_-CJ9IQf49R*>=hL#_0!_+x1viDxx zR}fM~wM-n1T3m6SrJkeCx&s_v=z+rKNsY`g(6oq@nZfik$dlutRLja23d$yWIJYk0 z%T#Ds^L+0Nf^W4?P9(@E!M9q8IWI=E@is8iXu(}f*+nqYb12T}WqUszvcTd`mugsP zg{Q$U%^7a#(H7y6(HhejFZ;sm4M&o~@K|XUHzxj&0@#NY3dhoTI!3E*w4GkE79ehD zsGg_rfNk5@*1*E4yy#4w{QMstP8$w>mnkvXs4Sx$T%jq6|0^ z$WUxN9QA&t-&$0GBM&!u&Vg0<)qBEsy4G=Kb1hRAAbWy{J=6Tw;FWyIn&`>nc5;eC zszfa5u9>}L^xO#GS!A8fj%4>+jVw~Nnp71f`WOf&qk6Fi%vI>3?1W5uXk^;t@>Qi& zHX|%T)jr8Vd^{A#RV1GP*!>Yw&h&gDL#Db+AgV{q8zQEj{JFJe$9K^b#yv*9ZXy`e z6;b%5G{YR)54R*j=7n%w7tW?5IE#tQIQUTW1eccpp~rXP~IWZgceA-kt!bd zgqGf7V|bb!qeTe#E16mbrU~AR2)5x8jq>LLj zcJ}*cPz5zdi>afOA4HaDVI!o^Nsqe;DuZUkU`Jr8Im^30lqr(X zJp{Egad%L%_^6d#qf=0GDY5gC( zV(#bY%3-y{AQ?Zk07080?bziwm!l*bOylM8@o{lR#yw|=(m0ODytKz^`c}fTtCEA< z4bLWBVpUb^^^|A5JEk3)*E%~Lw?#j7-WK2SoOMM%zi&29Kvyz!hyftvdd0`Tp1gd< zHtxPXBhm-a6?1#8a%yMqy$6bg;V90<;`+(s$NX(Bm(5AJcznG)f29Uq*^b+b2Y;s%;Xh#_@3Sq3z^Z+Rr& z6&t`7KW`PsJfAO+;t%-Tp&rxntXVw#uAIbY8RPtW9Gz;1HSFDw+BW8BFOQWv^L;R!`DV>4MH&ogpkJLdUZP3T>>? z*|H+O;~e?rNZwfH(^g%RXo!2Qhh^wKC{Fdlds#u(GLSF>MzI{kx%twT{%%!I{1Mn% zLK6$k)7x}00k{l#u=kv?VsM43uO%=LiOY}+{U$GGUD|^w1I;3XKq653)VC35WueYn z_O)PYlq<5=2R!Q!NH><}4@`I8d&JJR$>mZ(fAyG9Hw28$b5Mv!c}8UU5Ke}=s4$NR zDPn|8lG>_n32&3VP-+sK?a!m(IdO9ToyPKb@-s|{?t08*);y%$W`w;Qupxc+j!jOm zQKjBM_+Zs6%6I0f62MdEov1_W%!c6hsj7C$q!aOjbVxf4Tl zhM!2`?^#=IoJ^^8xScg>Uirw7us@Y9va^y87c+1v~VHMWkyrB8$MjBS4$r z!6?zq=8ZSy)ohL~5!0b70_H*%=ZZGz;d!nD^40>v6W$Jacc^D(!Gc%iWHeQy z5GMZUusGPEMcm-OxgyVaQf%zBx|`JD+YOo_G2w@=up%ShTl(Vg&rbl6E@|y`tl{|h z72fbk@$~YFi2NUVj#U#4_|`!Gw!k{@zbJsU7P1cfb4H OzyJIH{9pg?|Nei(V%6mU literal 0 HcmV?d00001 diff --git a/lib/std/compress/rfc1951.txt.z.9 b/lib/std/compress/rfc1951.txt.z.9 new file mode 100644 index 0000000000000000000000000000000000000000..84e7cbe5b715527cb3498c7261f5c525ff5d9426 GIT binary patch literal 11111 zcmV-tE11-H+O>UaciTpi=y#s`BYJ($o{%v~5qwCLWaq4G%Z?|q*Eez|naq0UfMkop z8U(m_h?Y0|-*3IT(G7sIoRM|%kVv4qy1L$7UES&6&s$yW((I%975)|q01s)UUz=h|6S_5P*-WDo~N6QPKx|Q1;?}Cj`z>vaJ7mOb)FPD+h$R&^Um|I z(Cak2JyEX{0JaH>C{4on-Z;GpZ(-QuLztZT=*9WVi?jFVh}SmLd9GLLMOcLDC4pC$ zTfK~~q9u)_ZgiI8|AXPA)443dqRdr#rQTmhxq72FDS@TGHt?6)X6a3|(z%K%Y>5cP zwT2nMyb@igB~YYHqT+Vwyo-QVDZHAbg~|zuTNNhNTX_*Ct1w&P_+fIZV1-ry;6;=d zS+po=sPqcK6c!)&f0@M5CMxu5*y+4JzkH?MriDHcya6fvs0csOz-pW35d%sH<(n{$ zR)DA(6OoByy)4QwMhNe+bgMJ?{^w|YUF3*kp=q^Yl1ULp2~amm7^79XEb%))1aOF! zu)GN4dAhyL@O9N)_Gq!x#ZbM|Fm20EG?vabQBJD?z}8s^xU5u>!bn@B)Jm-kDYQQz ztpK2pP_|_TlhpM5E-J3m64v87%+|UOh%V!Dg^XVS^gO2J!?lD#QXof$0OBxNm*E=d zg&^RS6aZ$t()1*(?lzKl`-UR`(Q9BX0D>@ zdAUGtD54vU$R~&mOmU@S%_v`G>4p|Cg<&ET7=*Ztb(qBjVH0K_i6v0(VA&(&G9)Ix z1jwWC8s3MsC)^N=Oyid;OIkB;g!AM3Hy4W8UM+!h7aIO5;U%89`sL>feC)|pu{}9B zfd3B5atNG#0BgCuj+XhsHd!AS$oawNI9eQhhQ9~E>H5>~y4b`&0Lo`D`7B%(mF(r7 z%&znm@Ezc*;xx}=5OG=)k@XeIx?qVk<@J!SbN;%_kqJWld;!!eEETl6`2FZeg)!(P zV4e-pqKW^JTSa!&_gsf8hCb<4_SH%TcI^@+NxZ_4AYEJNpc`0aF4#xiHHEO!6E|UWf6=S zr!dJJH8pTj7{e#TawJw!vMq@;Gy>3Dhnv38B~4<|Od&iU0Ugu=R~cRkH}Fe=lp=P5 z!6RT6V%P#iWUU2jpGTt2HYS~+k-fqdq81@^pefJ+MS|x8{0NUM08M}j3^MGPA5jzS zCg2{1Z@!;-Qc{Z^#*D@2UUm&6T=2W)S|Rj^w#j}f7Dov zSug}>?}SVKIY47VOY=H{5(KvW3Xp1I0ZoSL67<`j!p8$N9TM5&DEul^nt*0#B~d3t ztQ=92fje2DftcJN|Kk@WuJ0Xy4gn+4wxnQzC2pb=JOa^_`JGs*DFPY~U1rL(h@9vc z0CW@S-QBZgKoj(~z{17=#@B!Mr_oHNHbFl6_Zw6?u}6`w1!RCBTZT*WA;~5#Zo#6c zufoVD)FMuoAC2O;4@QR`Mq|Y2@-|7gc|;5VaId0IuKE_;t-b^Z@cJbuv97%svCH%b z%n#bgM>TBKQ4Nq?KtkyL6I91FKp-e@(`2x$Q7c#7XYc>^?)>R!)N6yNf}jRy)+9Lq ziFL9DYi1lQl17-a&JENipu-9fG9+0;Ah-62tt$ir2Bm^=4bV{wpw(>>ZlYy7sDS3G zf{Ep$$v>k;R6w#e_0g!z|4J6{zJP$jKkF;7V}MKkr!=Grh#`T(mm1s$^v%J%pw|vzHX!s;1FQfHpV#PCixlqX==+#9o&NiH{7o)^X~I|#FP1o^rO^TlfV|KJ z#dEOoY?KVZM>Wc)5PNM}S>qmVx#N@jVA(%!VazSEZG;SWm1!N{o<5xSK7L^Dy(YS} zutkK6MQe^!B^ecYw~_?$34SN5?9pWh>jP9I9$kNqwyG?m7&LgFeFqGQQkpKx%1WCx zOS8NN2D~&Nrr|PN9p=V568aMuYy^g*`AleG4_!s*YA{xK0;?Hhn^aW@oVMA5!b?wl zqMgNYx>FhO0JzHGa*29e(j$((Y!@#cER3ViMy+=Ba6Rk;xEvYVWDI&JtvemZzHsDs zhS(l(Ys+&>h%;0Z!2EzJC?d)PWNS_&<=5#h=cP*0W_khbjG$m$geAEsGA+=o*)3+- za^fUcQR)K1l$V?;6d<*QxCUA4)690j5*R<{xWwBqF-ftRIND%M)MWhAK;Ynq?9KtF z4K~W@8a5T1p&6<)h&Q4Pc;`?8 z2nRIO`>HICq3$!F|B0G2bE5~zt}dnbfxU53#0R(=qq=#x(e!tsQt+;ep3!`)mg|ZF zmxfd;e4#&q-UB|G3)7Gbjm}%RyvAho!}&RrZ{p|dK+#?^g>Zpwe1VEZCFN$JGpi(4 z(G`WT1!uuDZe$orU21Lk)@*z6+vNTW08|v@N+R{Fdb=#}p%X`BY1c{Ok6T}szL<)0 z)CNrHI=n%=U`@bmvwA|iV)jnNRTsr;R5gSwR&7-KfI)kdIK(A%9vk<`oEfpgy{jaR z(>16`;4mH-x9|R1Z7?Dqh0My%#)|>r>ps93c_1%tW2E4{vWl|VQx8~k+K~hwA}>g) z@RJO+s>w4BDZ0b)r`3oikLmLmCl!0 zw9quEItMb*>zs$+#lCi0GAZ~qa8b9p3IJ@A>-a`^s(O)0hx9Nn0QGn=d(4mZLH2My zF=s4kETIObYJeILv0NQ)s>j#LdgG^lQvFraY6(l{$9N)YK{m#wx;Jt7Y6 zTJr`iC!*?H0P?vbEf?C2T#y611Y+X|3@+mnPU!Z@L)>9Ofl2HBATnE9J(E)rUDR-C zNjP&0B{vKweike*k4RsPXY-DPB!D+Me>RdNIP4Z+a6$_rQH$sa@xIEWMNjm_hPmn@ zB*6H~0+gV`?zogVz)cu~m)eyj1sHY`eHZbbGyOUP)ktl0jw>lu`irxyq(Fg@IoOi1 zJwmb1nQP1?6}l2Pdc)#4G({YaIMx-^cB{jTwF9LvmPEyk`Y(1KQ4Ms&?`z3qGD{Pv zYPa>FJelbOdop`>#5$e3_PAcN5U!#SW23xsyBFyvSe=htOyVXt9uEeC$JL*I(7#Ux z11_1-n36;N5Q4mJsLBJhB9*SV=~D($53F)x5|ob9J280b{nU_ge*U=Qo#xmf2t*c9 zB;ojcm8BJr+Pc!#kwexgQSq!&5@k|?p?ZW7QJSrE_NecE0^V2)@fgPq_j`tYIJD=63L4&6-5&f#OZu8r=Lqrfkm zk21^KX>5E2B&)>;Utsoj||_yZ669PW4fl) z5#Cl55ZusNM=-8D?;R`>uwQgVKG+V08aW9vX=pn#^Idcl?YL zi#bkre!E$uvE(t`vRW*raR?B`+v^a#T-=LtcN%il<#MdP4KS-Hkb(C-O4==a+Djw% zuuAC~Y|s;1XfL$oaXYo@k0oacD+pGPV{jib_hhsUD$YKA;-Di*nkuVRCgElgtxGTw zxHUr0-(1=KRabR6TT5VGbJ`qW`wz+;TsI1!ji2E21g3YeY$+$Brzy=T*=?jE zrY1qHUBNFa_9S9h35+vGCPO1moGHw(V<`Jx?qpGMr3SD_1Cl)C-)^;&NQVp_azgQj;*Zp{uf8c3+@klzNb9E2y zg|66&CAfz!l3|pt(LnaxgYj;#d<`|RoZnR7NM8-29rtaS_Rl-n$? z(=<0*#VBO{c8U=xLQAs0QQ>O2R}tkO=LeY0mA?lcP-j2C4!4-Yn)R#icw{`6 z$Xdybcd8S9Y0Y0A_MFwj^LDHq_f&{`>*dYY!PbOKX4*MNLKrJyavY|+)EOSQ1Izs> z+LRlnh8W{vExhJs4ASD%H zTd2=-l|t&F>tuaxrM;I!s1wwd#FBtoL8^nQKuDBN`9pLMDjrZ-8SA_m3Z7A7d8Nhc zSd-MErzREj1S-~_qGf7Jo8sHLLc^uOLMil6X?r5?8!V|lL44)h^$O}RD7|Xs^R$8K zC0Dx$whfd}^p+aACr#GoF*pDJPuZBZ9`YaN`xr4Hn z0Jc^L6)Qji3P}a06U^Q=9CCL`-Em(9eKmo9diKmD@}rmhsWdg0j~fjx`I6jhjgSJ7 zdPdAiX{s1@SRHjZZ;((p-;%x=I{S=vkJ9A|D$62nhrTdg{p&0NeU1 zI<5b}_vm3n5C3RB`@hb7CqT}#9678`OjC@=%VM_89sT0Nm3K&SxVG2YN?BB!AXWCnxn9){!7wJn zPW##>wC+G|Tvl-Kh~%d#*}tZM0=z?#m9QkXJ{Dck0YCLG0Dm7SE20UDD8Q|j9bW!$ zy7x8zxyO&&pZwp4^C;%uI5Xh#@wv@!RQ6c*sC`?5e!=<6S3gRs($n-4R`d9eBO!eS z7GMhhp!z?%qLW_8lmS^$Xr}O!$8@@58CQ;EaAr(qeXV zbJ>R#YgKW8h*~?|6`y<&aUb)b%_|zWeK;|gw@fFP*5@-Zma0CV{ZkQF?}BB2zsqMc zL3>@liOq-XaRQwOf>wXfJ`jxB9Yf~WL% zAB=W|uc~n(GwyH8Ec7}`Bp)$R*Wm7Od4BG0@lk&on~AV1i~wp&TWu`JI=p=S_Uxil z*K|M=GQIQ&9!GA?0^n#4My6yY(O6B(2Q3MG*`dBI94?|Wl2|+|JJV7iSM8F!P^6H; z)W~FR_M%zd^Kd%&_*>~1&wc>gsV%k&RYU7clV z)~n@5h;U}xW~prf6(u)!-a$P)X-32q%2a?_vsR|2@u01BK$XDq0tg9_Y8$iU*5Tl` zwuUo6ev*oeEeB?%B(o)yuPxUqts8ebN6B0x@=1{qr(`)3JnS_6C#SOLuH_fb`xtF0 z&6(+qXdhAnAntNfk10*e85#tWMiBmf)~jCoD6*MQV{?uN8$8nvA~MCtPij?EEvw>JDboV=s)f08 zn7dY~hR9Q&5?w&6cgN0{TEM6udAdl>y^mH$+q!0;@$6_g9M5NW!WSZ2#nB|}U2ktN z$!99utK#&kB8M`0t(@(bw{ZUgK4b=KA#2TGWcw$|?*d_a>d8Kjw`#+dH*pyn6onWw z=5qy=acgxi+K!1-TOGKqPk-@=68_#R;;Nq_IhEogj-P8M2KeA@mgZEqT*1NffLNZz ztR!9w?+ZwU8RBWX$%F}?07bmEq1!Kk&vAn{l*j?HCIuk?KTELCJvI)6)ZjmWMJm5X1%O6q7vA=2EJW4H)=jm1 zV>yWmbYc18aNFMIz;OF1=fF&2G`gxBv$04I3;2o?Cmd2$aDF3^`&-{Ksl&u%P~;d3 zXp>oWYy)-64^D=31)~33|8ITzcaH#iy9Nn3IHvigBp2(QI7oSvS>o+g-J=JSyEC2K z+C5L!Xnah|ZUgg)eR1~rl-W8->2$Q`!)2IpH1&PWBV1lw<7`+qvX3JPrN|aEYaVYjQ&Qt_rP-=3w&+&ESDd=b4q<%jlftkqqnaZJ6 z7VZ*bvaomoUcggTrSEI5sAb5qs2ueZr6Vek^iIC@tKtWn=yG>kh=nD++8u$ppUtpB z?auRObU@ARbi?P`xZsHB7FC!BQ>b}>|pc5LEC5kY=8yqnG%!1 z8y_Btb`NI#;E+E%I+6t3;n83|t^=7{AX5uuZh##1gSh~i%M2dLOC>_9gPIacrqBveC8vD5(KkBFd5aM1QyDXg<|a8 z_-HU036y}b4JN@LIINT9aBQK>EtIjbn!#W)6)0nYayS_T$Bh|HERn&M4TEpM|b8WTkF{>98%LtSf^1DF#R4u|d20w~|j94~6xfb~R~N^@LM}{GT(HKN%HfiJ}tPG7!iqSxs!Yb2m z(|H|=l_6G7!Px4D!`a}Nq?t4WAqfDEK{$`bbtqPb7=d77p&TC#g7KV`gMk77V49=j z#*C&it6Be$cY%zS3C5s{j!CqqcB&%)eHcuqbtoc3>ZnhY3}zO}G3ck+AxVva0sv-% zIl!(%ITYH!z%>0sqpLt_2NM!w(slUl93cne@w`FZBZd;d*Yxm_fijuT2D5HYm@ z^uO(pVzWR_K71qXhp~U?RUw5v?(RL>3~r<1uyE@^V&)MpC1|k(Bq)-{sGV5!s63R~ z2UBM1v}yGZptct5)BZJPm&;tb`uM9nu2z|jaNTHU7jsQrIx^EpCV_c5C9g7%D);1f zX=VliuDXGG40L;}+Bg4#BU&#_8(A*7X|<3Q zknZ5t-%a@L#OBvUQ??E^|I%PoM-fcsJlx*VjvAx6r_dUZZBnn=Y4J~lnPZ!YzY$i` z7)~oF_hcBV+_doYtdo_kvW60Ti~*E*0qv6RTHTU4neY6iaQ_Z>pqwKTatz7c z<7sNT83hI>>c@-M@B8Zi((Z)t%7v2wCXf3FclM9SF>NXm3+Tn`O9QFqm<*~Tqg|Aa z$+*{ys|CvQi}SYvg;E_gWH@;5Vk8h8_a4rnoqXGbahK)=Ds0+QpA>7F6JOOCSHtf(S) zii0T~ECHL>X|+~)i^E<^%}vnbv`0%iZ3b&_)H|gt9?zvkw;E~Et%XP}s<3;9?pwB> z#5OK6->OFf*-Y@E#K20TM6@p8o^ZY2TDIY;u)161>C*-lC~km8(2hhy%Rlh&@7>tT zFQj$#&izKctM;#PCfN^MS`n+k6Mn#0zAN=gh-mLmA-9JrwYb%IYpk}YpFdT+(2kLIIi_uM?4H^3oMKaKM?OU%cT?CF;q^L@!gRPr3AGJBX;7*?Gkd~r$5U*i zEJ=DkW+mOVBvu^%gh#;nW|#QZG)!+y(rqj%d+Ji_Vwwr6W?NB+*SI>9+rJHcLhftI zE``xEgQ^jlWd$IAo-;Wt7sa(Qb`&6U({8N zZYi|wt!)do>hhz;QRzLGADN3){r<&xxZmDawj9a^G~eg;BZXL&AMn78O=><43M- zL)Z<*>DbZE+`VgrRE(ip7PhiY^-aU^2Xn-x5ATTiZP0nz=8gu3!U=Xnz?llEqw2)3 zgs-lW8g{Bqx)ubV2&6F09JAPAY9=bG=#7A8 zpn8~{Y2Z_}N+;F^O`20(n1|MPxhbkFv2wo^^D-x#m)Cs%@^y&gn{HlLc#yTFhji6C zJLEU6OjQU?k#8u|)GTN+@S5ULw+DMSG5fNlzG0Y*&rDf4?}Lpi)NgT|ZM%yNxYJui9hx7My%jF7CE<|z_|$Up%`9Jdm+>0^va zH`{_6)%%OU7X;mMTi2YL!~MqF)^0HmbLHxWn_^3SEMFLrh{AECSPEh z5Ey){JAmZurWsXP?l>|nlek=dyyZ&>#M_wE8qktB;W1f%xydN?@Iz@YPd5rI=@O-W zjSz;yFeFE|rW1^`ltRp$Y1vJC@iwd50tgo~pGuqxx5$*4N|F~EZy}e{KG>y#i-tjk zEU#N-BG+NIylyOhzNeX>%$V;kmL}^cJmC$(+A#l_e&?LE1%r+EsPl>!;aWTQH#EA@ zGkz`}v^Ro&9{MW@Ud}CdS5%3HmixUO2&c97EyANxN}MxoIEiJCG#6>A(ij_mz6S$+ zlSZgU>2|3BrB?C?)`@)(y19z3jK(o(;VZ=5TLOy=IElMCQ(jHl(wM40=cqNNtjT_s z$p8~t_fZ^vzU4oG z?2mdUlHm{eGq))KPMLKQU5r>JtH$7#zi8{pbWUMN4nG^&b(=k5Urtg2RJf9?^4k04 zau}-GNm(zE$Dmq7FXe@4r_#7aau*WE5u&7wI_g#14w^doTcHvJFgs=raS$^0F}m== zTrn@mh6$gNtL@qKD^>W`yM;n7D+dCVC>gZ!KQ}lc=hAe1FiT^0YMd2(#OI>G;4m%g z?j>E0ifWaF8@MV3u1t!?GRkdsd37TNw(vmrDJ89giwwG94HCuoHeI>Zsix+d_IN}W zDcim~K^KS5v)O`pquO{8X-+6&I>CM6M{QON)UiF5Eci58uFv`ioF#4@T5dZwY@_32 z%y9JmJ*Wa=4pN+>XX9u@W5yi+g0jBk&?KqMX-8W$Br=Y(W0kcrb&pUL;S?SXO$%r| zvbwT8P3mj8;DbAPi2F;#E4Ep)D9uHR-k08KGXYYa$m{%}5!vBG7OOsVqh{~MV3G~& zRwnkOuU-t**|4u3ovAm~4QNQP=e5ReZ@|ajls2pIGjLI|3h~NP+9v{5duKb%TDE=- zFu(#7WXgUHSCP$kpC=&mntIDzLPV;mt{$d5?yJA1F~p5(I_j&pTyKBeSKIAS1*7Bn zAP7d%TLnWf4A1@o_;a*ort(V?VV_qZ^qF_g^RnvH{lYoV>qNRRjo0Vr=juH+`VN;8 z#wG6Zp;b-s1odCBQRTR5yYMz0Vx|-^olK4flj%h8oP1EPAZSMK=P$#-#gILaGf)8F za=n2+f4_MB%!bQhl?HoUhlw| zeeSFCp}G`Aegc~t;kMGVp?dU+D>~%nr7%hPsx9v3*Ksp*sO=x&>TQ6DpTEHr_C$5x zgc-2P@!_;5TyuCh2o8^BEe?VALAx%mfv?sy_yrjE9n#^^xh*WQ=Y*@?yj&Ua$l)0? z>SWTN_SM-IoEH?j5f3|^9uKB79CrUz+kByxxmmCEq|i-D``>(+YE_A2R>>b{)tq87 z9E3TYvRTP3rNqmvdzF&eO~Ttj(C13&6csx=cqcF3NlXqh*C$Z0go&4e@2R8?%tO3= z(8=K7-ICZt*r3Bv!!_A++ExVHv7qaPZJG~{>JmQzakr6_@o@Hncl_v{%#AiPkWN?8n1UdQ8 zcN3O?r++~@;q6|S&bIftio;}Gq6M&dAA4{)bPnyOs3_McJ{Q<(r&`+W3HA_VN#K95-`ln;$kMi5;e-0fS8t{mWAmK z#9!9;-8G2RPhs&H*@K%NHLux&W%vuM;NT)Hx1!~U8lVTL5QnrJMA~U#dXRHI&|JUj z_?b4@jO{}wV$Sl%+eU)poj(u5+|<> 4); + const FCHECK = @truncate(u5, header[1]); + const FDICT = @truncate(u1, header[1] >> 5); + + if ((@as(u16, header[0]) << 8 | header[1]) % 31 != 0) + return error.BadHeader; + + // The CM field must be 8 to indicate the use of DEFLATE + if (CM != 8) return error.InvalidCompression; + // CINFO is the base-2 logarithm of the window size, minus 8. + // Values above 7 are unspecified and therefore rejected. + if (CINFO > 7) return error.InvalidWindowSize; + const window_size: u16 = @as(u16, 1) << (CINFO + 8); + + // TODO: Support this case + if (FDICT != 0) + return error.Unsupported; + + var window_slice = try allocator.alloc(u8, window_size); + + return Self{ + .allocator = allocator, + .inflater = deflate.inflateStream(source, window_slice), + .in_reader = source, + .hasher = std.hash.Adler32.init(), + .window_slice = window_slice, + }; + } + + fn deinit(self: *Self) void { + self.allocator.free(self.window_slice); + } + + // Implements the io.Reader interface + pub fn read(self: *Self, buffer: []u8) Error!usize { + if (buffer.len == 0) + return 0; + + // Read from the compressed stream and update the computed checksum + const r = try self.inflater.read(buffer); + if (r != 0) { + self.hasher.update(buffer[0..r]); + return r; + } + + // We've reached the end of stream, check if the checksum matches + const hash = try self.in_reader.readIntBig(u32); + if (hash != self.hasher.final()) + return error.WrongChecksum; + + return 0; + } + + pub fn reader(self: *Self) Reader { + return .{ .context = self }; + } + }; +} + +pub fn zlibStream(allocator: *mem.Allocator, reader: anytype) !ZlibStream(@TypeOf(reader)) { + return ZlibStream(@TypeOf(reader)).init(allocator, reader); +} + +fn testReader(data: []const u8, comptime expected: []const u8) !void { + var in_stream = io.fixedBufferStream(data); + + var zlib_stream = try zlibStream(testing.allocator, in_stream.reader()); + defer zlib_stream.deinit(); + + // Read and decompress the whole file + const buf = try zlib_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize)); + defer testing.allocator.free(buf); + // Calculate its SHA256 hash and check it against the reference + var hash: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(buf, hash[0..], .{}); + + assertEqual(expected, &hash); +} + +// Assert `expected` == `input` where `input` is a bytestring. +pub fn assertEqual(comptime expected: []const u8, input: []const u8) void { + var expected_bytes: [expected.len / 2]u8 = undefined; + for (expected_bytes) |*r, i| { + r.* = std.fmt.parseInt(u8, expected[2 * i .. 2 * i + 2], 16) catch unreachable; + } + + testing.expectEqualSlices(u8, &expected_bytes, input); +} + +// All the test cases are obtained by compressing the RFC1950 text +// +// https://tools.ietf.org/rfc/rfc1950.txt length=36944 bytes +// SHA256=5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009 +test "compressed data" { + // Compressed with compression level = 0 + try testReader( + @embedFile("rfc1951.txt.z.0"), + "5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009", + ); + // Compressed with compression level = 9 + try testReader( + @embedFile("rfc1951.txt.z.9"), + "5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009", + ); + // Compressed with compression level = 9 and fixed Huffman codes + try testReader( + @embedFile("rfc1951.txt.fixed.z.9"), + "5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009", + ); +} + +test "sanity checks" { + // Truncated header + testing.expectError( + error.EndOfStream, + testReader(&[_]u8{0x78}, ""), + ); + // Failed FCHECK check + testing.expectError( + error.BadHeader, + testReader(&[_]u8{ 0x78, 0x9D }, ""), + ); + // Wrong CM + testing.expectError( + error.InvalidCompression, + testReader(&[_]u8{ 0x79, 0x94 }, ""), + ); + // Wrong CINFO + testing.expectError( + error.InvalidWindowSize, + testReader(&[_]u8{ 0x88, 0x98 }, ""), + ); + // Wrong checksum + testing.expectError( + error.WrongChecksum, + testReader(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""), + ); + // Truncated checksum + testing.expectError( + error.EndOfStream, + testReader(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""), + ); +} diff --git a/lib/std/std.zig b/lib/std/std.zig index 330f3c253..4236b2929 100644 --- a/lib/std/std.zig +++ b/lib/std/std.zig @@ -50,6 +50,7 @@ pub const builtin = @import("builtin.zig"); pub const c = @import("c.zig"); pub const cache_hash = @import("cache_hash.zig"); pub const coff = @import("coff.zig"); +pub const compress = @import("compress.zig"); pub const crypto = @import("crypto.zig"); pub const cstr = @import("cstr.zig"); pub const debug = @import("debug.zig"); From c15f39212e32c612cf51f647868be2bdd024d0de Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 7 Sep 2020 15:15:48 -0700 Subject: [PATCH 20/89] build.zig: ignore the compression test files --- build.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build.zig b/build.zig index 8ac2d4f8b..3f7f1a903 100644 --- a/build.zig +++ b/build.zig @@ -123,7 +123,13 @@ pub fn build(b: *Builder) !void { .source_dir = "lib", .install_dir = .Lib, .install_subdir = "zig", - .exclude_extensions = &[_][]const u8{ "test.zig", "README.md" }, + .exclude_extensions = &[_][]const u8{ + "test.zig", + "README.md", + ".z.0", + ".z.9", + "rfc1951.txt", + }, }); const test_filter = b.option([]const u8, "test-filter", "Skip tests that do not match filter"); From 61cd9099e2edfc363a932b3726e127c9ba3d6ee9 Mon Sep 17 00:00:00 2001 From: Timon Kruiper Date: Tue, 8 Sep 2020 15:41:07 +0200 Subject: [PATCH 21/89] Zlib: Make deinit function public --- lib/std/compress/zlib.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/compress/zlib.zig b/lib/std/compress/zlib.zig index c4228ded2..d4bac4a8a 100644 --- a/lib/std/compress/zlib.zig +++ b/lib/std/compress/zlib.zig @@ -62,7 +62,7 @@ pub fn ZlibStream(comptime ReaderType: type) type { }; } - fn deinit(self: *Self) void { + pub fn deinit(self: *Self) void { self.allocator.free(self.window_slice); } From 8716fde7bcb2fa472438e936128cb8f8512680ab Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 8 Sep 2020 09:53:41 -0700 Subject: [PATCH 22/89] ci: skip non native builds on windows to save time --- ci/azure/windows_msvc_script.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/windows_msvc_script.bat b/ci/azure/windows_msvc_script.bat index 5c3593fb9..1568b1b15 100644 --- a/ci/azure/windows_msvc_script.bat +++ b/ci/azure/windows_msvc_script.bat @@ -24,7 +24,7 @@ cd %ZIGBUILDDIR% cmake.exe .. -Thost=x64 -G"Visual Studio 16 2019" -A x64 "-DCMAKE_INSTALL_PREFIX=%ZIGINSTALLDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release || exit /b msbuild /maxcpucount /p:Configuration=Release INSTALL.vcxproj || exit /b -"%ZIGINSTALLDIR%\bin\zig.exe" build test -Dskip-compile-errors || exit /b +"%ZIGINSTALLDIR%\bin\zig.exe" build test -Dskip-non-native -Dskip-compile-errors || exit /b set "PATH=%CD:~0,2%\msys64\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem" SET "MSYSTEM=MINGW64" From b8b68cb279e8f51a8aa7c67aa42cefc644375b74 Mon Sep 17 00:00:00 2001 From: xackus <14938807+xackus@users.noreply.github.com> Date: Mon, 7 Sep 2020 22:07:34 +0200 Subject: [PATCH 23/89] std: clean up bitrotten imports in crypto --- lib/std/crypto/benchmark.zig | 4 ++-- lib/std/crypto/blake2.zig | 8 ++++---- lib/std/crypto/chacha20.zig | 2 -- lib/std/crypto/md5.zig | 10 ++++------ lib/std/crypto/poly1305.zig | 2 +- lib/std/crypto/sha1.zig | 9 ++++----- lib/std/crypto/sha2.zig | 9 ++++----- lib/std/crypto/sha3.zig | 9 ++++----- lib/std/crypto/test.zig | 1 - 9 files changed, 23 insertions(+), 31 deletions(-) diff --git a/lib/std/crypto/benchmark.zig b/lib/std/crypto/benchmark.zig index 15ddaf7b3..e20b27220 100644 --- a/lib/std/crypto/benchmark.zig +++ b/lib/std/crypto/benchmark.zig @@ -5,8 +5,8 @@ // and substantial portions of the software. // zig run benchmark.zig --release-fast --override-lib-dir .. -const builtin = @import("builtin"); -const std = @import("std"); +const std = @import("../std.zig"); +const builtin = std.builtin; const mem = std.mem; const time = std.time; const Timer = time.Timer; diff --git a/lib/std/crypto/blake2.zig b/lib/std/crypto/blake2.zig index 1cb143811..1599be3b2 100644 --- a/lib/std/crypto/blake2.zig +++ b/lib/std/crypto/blake2.zig @@ -3,10 +3,10 @@ // This file is part of [zig](https://ziglang.org/), which is MIT licensed. // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. -const mem = @import("../mem.zig"); -const builtin = @import("builtin"); -const debug = @import("../debug.zig"); -const math = @import("../math.zig"); +const std = @import("../std.zig"); +const mem = std.mem; +const math = std.math; +const debug = std.debug; const htest = @import("test.zig"); const RoundParam = struct { diff --git a/lib/std/crypto/chacha20.zig b/lib/std/crypto/chacha20.zig index d2167db31..915f81b9f 100644 --- a/lib/std/crypto/chacha20.zig +++ b/lib/std/crypto/chacha20.zig @@ -7,10 +7,8 @@ const std = @import("../std.zig"); const mem = std.mem; -const endian = std.endian; const assert = std.debug.assert; const testing = std.testing; -const builtin = @import("builtin"); const maxInt = std.math.maxInt; const Poly1305 = std.crypto.onetimeauth.Poly1305; diff --git a/lib/std/crypto/md5.zig b/lib/std/crypto/md5.zig index 0d221fabf..7ba78b587 100644 --- a/lib/std/crypto/md5.zig +++ b/lib/std/crypto/md5.zig @@ -3,12 +3,10 @@ // This file is part of [zig](https://ziglang.org/), which is MIT licensed. // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. -const mem = @import("../mem.zig"); -const math = @import("../math.zig"); -const endian = @import("../endian.zig"); -const builtin = @import("builtin"); -const debug = @import("../debug.zig"); -const fmt = @import("../fmt.zig"); +const std = @import("../std.zig"); +const mem = std.mem; +const math = std.math; +const debug = std.debug; const RoundParam = struct { a: usize, diff --git a/lib/std/crypto/poly1305.zig b/lib/std/crypto/poly1305.zig index c0b462c60..a95b9d7cb 100644 --- a/lib/std/crypto/poly1305.zig +++ b/lib/std/crypto/poly1305.zig @@ -3,7 +3,7 @@ // This file is part of [zig](https://ziglang.org/), which is MIT licensed. // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. -const std = @import("std"); +const std = @import("../std.zig"); const mem = std.mem; pub const Poly1305 = struct { diff --git a/lib/std/crypto/sha1.zig b/lib/std/crypto/sha1.zig index 03fd55b6a..f4b380e4f 100644 --- a/lib/std/crypto/sha1.zig +++ b/lib/std/crypto/sha1.zig @@ -3,11 +3,10 @@ // This file is part of [zig](https://ziglang.org/), which is MIT licensed. // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. -const mem = @import("../mem.zig"); -const math = @import("../math.zig"); -const endian = @import("../endian.zig"); -const debug = @import("../debug.zig"); -const builtin = @import("builtin"); +const std = @import("../std.zig"); +const mem = std.mem; +const math = std.math; +const debug = std.debug; const RoundParam = struct { a: usize, diff --git a/lib/std/crypto/sha2.zig b/lib/std/crypto/sha2.zig index af2c22fe1..3e388f2e4 100644 --- a/lib/std/crypto/sha2.zig +++ b/lib/std/crypto/sha2.zig @@ -3,11 +3,10 @@ // This file is part of [zig](https://ziglang.org/), which is MIT licensed. // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. -const mem = @import("../mem.zig"); -const math = @import("../math.zig"); -const endian = @import("../endian.zig"); -const debug = @import("../debug.zig"); -const builtin = @import("builtin"); +const std = @import("../std.zig"); +const mem = std.mem; +const math = std.math; +const debug = std.debug; const htest = @import("test.zig"); ///////////////////// diff --git a/lib/std/crypto/sha3.zig b/lib/std/crypto/sha3.zig index 3d6dad1be..991eb3862 100644 --- a/lib/std/crypto/sha3.zig +++ b/lib/std/crypto/sha3.zig @@ -3,11 +3,10 @@ // This file is part of [zig](https://ziglang.org/), which is MIT licensed. // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. -const mem = @import("../mem.zig"); -const math = @import("../math.zig"); -const endian = @import("../endian.zig"); -const debug = @import("../debug.zig"); -const builtin = @import("builtin"); +const std = @import("../std.zig"); +const mem = std.mem; +const math = std.math; +const debug = std.debug; const htest = @import("test.zig"); pub const Sha3_224 = Keccak(224, 0x06); diff --git a/lib/std/crypto/test.zig b/lib/std/crypto/test.zig index 2987706b1..1655be707 100644 --- a/lib/std/crypto/test.zig +++ b/lib/std/crypto/test.zig @@ -5,7 +5,6 @@ // and substantial portions of the software. const std = @import("../std.zig"); const testing = std.testing; -const mem = std.mem; const fmt = std.fmt; // Hash using the specified hasher `H` asserting `expected == H(input)`. From 5bf3e540188d06b06a0f18043c2685d3593c8107 Mon Sep 17 00:00:00 2001 From: Jens Goldberg Date: Tue, 8 Sep 2020 11:52:20 +0000 Subject: [PATCH 24/89] Add the Linux TCP socket options --- lib/std/os/bits/linux.zig | 117 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/lib/std/os/bits/linux.zig b/lib/std/os/bits/linux.zig index 6d85d0623..cec8833df 100644 --- a/lib/std/os/bits/linux.zig +++ b/lib/std/os/bits/linux.zig @@ -1590,6 +1590,123 @@ pub const RR_A = 1; pub const RR_CNAME = 5; pub const RR_AAAA = 28; +/// Turn off Nagle's algorithm +pub const TCP_NODELAY = 1; +/// Limit MSS +pub const TCP_MAXSEG = 2; +/// Never send partially complete segments. +pub const TCP_CORK = 3; +/// Start keeplives after this period, in seconds +pub const TCP_KEEPIDLE = 4; +/// Interval between keepalives +pub const TCP_KEEPINTVL = 5; +/// Number of keepalives before death +pub const TCP_KEEPCNT = 6; +/// Number of SYN retransmits +pub const TCP_SYNCNT = 7; +/// Life time of orphaned FIN-WAIT-2 state +pub const TCP_LINGER2 = 8; +/// Wake up listener only when data arrive +pub const TCP_DEFER_ACCEPT = 9; +/// Bound advertised window +pub const TCP_WINDOW_CLAMP = 10; +/// Information about this connection. +pub const TCP_INFO = 11; +/// Block/reenable quick acks +pub const TCP_QUICKACK = 12; +/// Congestion control algorithm +pub const TCP_CONGESTION = 13; +/// TCP MD5 Signature (RFC2385) +pub const TCP_MD5SIG = 14; +/// Use linear timeouts for thin streams +pub const TCP_THIN_LINEAR_TIMEOUTS = 16; +/// Fast retrans. after 1 dupack +pub const TCP_THIN_DUPACK = 17; +/// How long for loss retry before timeout +pub const TCP_USER_TIMEOUT = 18; +/// TCP sock is under repair right now +pub const TCP_REPAIR = 19; +pub const TCP_REPAIR_QUEUE = 20; +pub const TCP_QUEUE_SEQ = 21; +pub const TCP_REPAIR_OPTIONS = 22; +/// Enable FastOpen on listeners +pub const TCP_FASTOPEN = 23; +pub const TCP_TIMESTAMP = 24; +/// limit number of unsent bytes in write queue +pub const TCP_NOTSENT_LOWAT = 25; +/// Get Congestion Control (optional) info +pub const TCP_CC_INFO = 26; +/// Record SYN headers for new connections +pub const TCP_SAVE_SYN = 27; +/// Get SYN headers recorded for connection +pub const TCP_SAVED_SYN = 28; +/// Get/set window parameters +pub const TCP_REPAIR_WINDOW = 29; +/// Attempt FastOpen with connect +pub const TCP_FASTOPEN_CONNECT = 30; +/// Attach a ULP to a TCP connection +pub const TCP_ULP = 31; +/// TCP MD5 Signature with extensions +pub const TCP_MD5SIG_EXT = 32; +/// Set the key for Fast Open (cookie) +pub const TCP_FASTOPEN_KEY = 33; +/// Enable TFO without a TFO cookie +pub const TCP_FASTOPEN_NO_COOKIE = 34; +pub const TCP_ZEROCOPY_RECEIVE = 35; +/// Notify bytes available to read as a cmsg on read +pub const TCP_INQ = 36; +pub const TCP_CM_INQ = TCP_INQ; +/// delay outgoing packets by XX usec +pub const TCP_TX_DELAY = 37; + +pub const TCP_REPAIR_ON = 1; +pub const TCP_REPAIR_OFF = 0; +/// Turn off without window probes +pub const TCP_REPAIR_OFF_NO_WP = -1; + +pub const tcp_repair_opt = extern struct { + opt_code: u32, + opt_val: u32, +}; + +pub const tcp_repair_window = extern struct { + snd_wl1: u32, + snd_wnd: u32, + max_window: u32, + rcv_wnd: u32, + rcv_wup: u32, +}; + +pub const TcpRepairOption = extern enum { + TCP_NO_QUEUE, + TCP_RECV_QUEUE, + TCP_SEND_QUEUE, + TCP_QUEUES_NR, +}; + +/// why fastopen failed from client perspective +pub const tcp_fastopen_client_fail = extern enum { + /// catch-all + TFO_STATUS_UNSPEC, + /// if not in TFO_CLIENT_NO_COOKIE mode + TFO_COOKIE_UNAVAILABLE, + /// SYN-ACK did not ack SYN data + TFO_DATA_NOT_ACKED, + /// SYN-ACK did not ack SYN data after timeout + TFO_SYN_RETRANSMITTED, +}; + +/// for TCP_INFO socket option +pub const TCPI_OPT_TIMESTAMPS = 1; +pub const TCPI_OPT_SACK = 2; +pub const TCPI_OPT_WSCALE = 4; +/// ECN was negociated at TCP session init +pub const TCPI_OPT_ECN = 8; +/// we received at least one packet with ECT +pub const TCPI_OPT_ECN_SEEN = 16; +/// SYN-ACK acked data in SYN sent or rcvd +pub const TCPI_OPT_SYN_DATA = 32; + pub const nfds_t = usize; pub const pollfd = extern struct { fd: fd_t, From 42b1b6be90fc1034875a6e16cf3cbe1c9d6030ca Mon Sep 17 00:00:00 2001 From: Mark Barbone Date: Mon, 7 Sep 2020 23:24:35 -0400 Subject: [PATCH 25/89] Add resize for arena allocator --- lib/std/heap/arena_allocator.zig | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/std/heap/arena_allocator.zig b/lib/std/heap/arena_allocator.zig index e4bce8087..0737cb2ef 100644 --- a/lib/std/heap/arena_allocator.zig +++ b/lib/std/heap/arena_allocator.zig @@ -26,7 +26,7 @@ pub const ArenaAllocator = struct { return .{ .allocator = Allocator{ .allocFn = alloc, - .resizeFn = Allocator.noResize, + .resizeFn = resize, }, .child_allocator = child_allocator, .state = self, @@ -84,4 +84,26 @@ pub const ArenaAllocator = struct { return result; } } + + fn resize(allocator: *Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Allocator.Error!usize { + const self = @fieldParentPtr(ArenaAllocator, "allocator", allocator); + + const cur_node = self.state.buffer_list.first orelse return error.OutOfMemory; + const cur_buf = cur_node.data[@sizeOf(BufNode)..]; + if (@ptrToInt(cur_buf.ptr) + self.state.end_index != @ptrToInt(buf.ptr) + buf.len) { + if (new_len > buf.len) + return error.OutOfMemory; + return new_len; + } + + if (buf.len >= new_len) { + self.state.end_index -= buf.len - new_len; + return new_len; + } else if (cur_buf.len - self.state.end_index >= new_len - buf.len) { + self.state.end_index += new_len - buf.len; + return new_len; + } else { + return error.OutOfMemory; + } + } }; From 9306dbd6194251e816f11bc8f420fac0d1ca8835 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 7 Sep 2020 07:18:44 +0200 Subject: [PATCH 26/89] Fix bug where __text section would get overwritten Fixes a bug where the last written load command would accidentally override the beginning of the __text section. Also defines missing MachO constants and relocation structs/enums. Signed-off-by: Jakub Konka --- lib/std/macho.zig | 117 +++++++++++++++++++++++++++++++-- src-self-hosted/link/MachO.zig | 35 ++++++---- 2 files changed, 132 insertions(+), 20 deletions(-) diff --git a/lib/std/macho.zig b/lib/std/macho.zig index 5217a7376..d3296ee17 100644 --- a/lib/std/macho.zig +++ b/lib/std/macho.zig @@ -647,6 +647,32 @@ pub const nlist_64 = extern struct { n_value: u64, }; +/// Format of a relocation entry of a Mach-O file. Modified from the 4.3BSD +/// format. The modifications from the original format were changing the value +/// of the r_symbolnum field for "local" (r_extern == 0) relocation entries. +/// This modification is required to support symbols in an arbitrary number of +/// sections not just the three sections (text, data and bss) in a 4.3BSD file. +/// Also the last 4 bits have had the r_type tag added to them. +pub const relocation_info = packed struct { + /// offset in the section to what is being relocated + r_address: i32, + + /// symbol index if r_extern == 1 or section ordinal if r_extern == 0 + r_symbolnum: u24, + + /// was relocated pc relative already + r_pcrel: u1, + + /// 0=byte, 1=word, 2=long, 3=quad + r_length: u2, + + /// does not include value of sym referenced + r_extern: u1, + + /// if not 0, machine specific relocation type + r_type: u4, +}; + /// After MacOS X 10.1 when a new load command is added that is required to be /// understood by the dynamic linker for the image to execute properly the /// LC_REQ_DYLD bit will be or'ed into the load command constant. If the dynamic @@ -1086,13 +1112,58 @@ pub const N_ECOML = 0xe8; /// second stab entry with length information pub const N_LENG = 0xfe; -/// If a segment contains any sections marked with S_ATTR_DEBUG then all -/// sections in that segment must have this attribute. No section other than -/// a section marked with this attribute may reference the contents of this -/// section. A section with this attribute may contain no symbols and must have -/// a section type S_REGULAR. The static linker will not copy section contents -/// from sections with this attribute into its output file. These sections -/// generally contain DWARF debugging info. +// For the two types of symbol pointers sections and the symbol stubs section +// they have indirect symbol table entries. For each of the entries in the +// section the indirect symbol table entries, in corresponding order in the +// indirect symbol table, start at the index stored in the reserved1 field +// of the section structure. Since the indirect symbol table entries +// correspond to the entries in the section the number of indirect symbol table +// entries is inferred from the size of the section divided by the size of the +// entries in the section. For symbol pointers sections the size of the entries +// in the section is 4 bytes and for symbol stubs sections the byte size of the +// stubs is stored in the reserved2 field of the section structure. + +/// section with only non-lazy symbol pointers +pub const S_NON_LAZY_SYMBOL_POINTERS = 0x6; + +/// section with only lazy symbol pointers +pub const S_LAZY_SYMBOL_POINTERS = 0x7; + +/// section with only symbol stubs, byte size of stub in the reserved2 field +pub const S_SYMBOL_STUBS = 0x8; + +/// section with only function pointers for initialization +pub const S_MOD_INIT_FUNC_POINTERS = 0x9; + +/// section with only function pointers for termination +pub const S_MOD_TERM_FUNC_POINTERS = 0xa; + +/// section contains symbols that are to be coalesced +pub const S_COALESCED = 0xb; + +/// zero fill on demand section (that can be larger than 4 gigabytes) +pub const S_GB_ZEROFILL = 0xc; + +/// section with only pairs of function pointers for interposing +pub const S_INTERPOSING = 0xd; + +/// section with only 16 byte literals +pub const S_16BYTE_LITERALS = 0xe; + +/// section contains DTrace Object Format +pub const S_DTRACE_DOF = 0xf; + +/// section with only lazy symbol pointers to lazy loaded dylibs +pub const S_LAZY_DYLIB_SYMBOL_POINTERS = 0x10; + +// If a segment contains any sections marked with S_ATTR_DEBUG then all +// sections in that segment must have this attribute. No section other than +// a section marked with this attribute may reference the contents of this +// section. A section with this attribute may contain no symbols and must have +// a section type S_REGULAR. The static linker will not copy section contents +// from sections with this attribute into its output file. These sections +// generally contain DWARF debugging info. + /// a debug section pub const S_ATTR_DEBUG = 0x02000000; @@ -1154,3 +1225,35 @@ pub const VM_PROT_WRITE: vm_prot_t = 0x2; /// VM execute permission pub const VM_PROT_EXECUTE: vm_prot_t = 0x4; + +pub const reloc_type_x86_64 = packed enum(u4) { + /// for absolute addresses + X86_64_RELOC_UNSIGNED = 0, + + /// for signed 32-bit displacement + X86_64_RELOC_SIGNED, + + /// a CALL/JMP instruction with 32-bit displacement + X86_64_RELOC_BRANCH, + + /// a MOVQ load of a GOT entry + X86_64_RELOC_GOT_LOAD, + + /// other GOT references + X86_64_RELOC_GOT, + + /// must be followed by a X86_64_RELOC_UNSIGNED + X86_64_RELOC_SUBTRACTOR, + + /// for signed 32-bit displacement with a -1 addend + X86_64_RELOC_SIGNED_1, + + /// for signed 32-bit displacement with a -2 addend + X86_64_RELOC_SIGNED_2, + + /// for signed 32-bit displacement with a -4 addend + X86_64_RELOC_SIGNED_4, + + /// for thread local variables + X86_64_RELOC_TLV, +}; diff --git a/src-self-hosted/link/MachO.zig b/src-self-hosted/link/MachO.zig index 27d0488f2..4b068d45b 100644 --- a/src-self-hosted/link/MachO.zig +++ b/src-self-hosted/link/MachO.zig @@ -32,6 +32,20 @@ const LoadCommand = union(enum) { .Dysymtab => |x| x.cmdsize, }; } + + pub fn write(self: LoadCommand, file: *fs.File, offset: u64) !void { + return switch (self) { + .Segment => |cmd| writeGeneric(cmd, file, offset), + .LinkeditData => |cmd| writeGeneric(cmd, file, offset), + .Symtab => |cmd| writeGeneric(cmd, file, offset), + .Dysymtab => |cmd| writeGeneric(cmd, file, offset), + }; + } + + fn writeGeneric(cmd: anytype, file: *fs.File, offset: u64) !void { + const slice = [1]@TypeOf(cmd){cmd}; + return file.pwriteAll(mem.sliceAsBytes(slice[0..1]), offset); + } }; base: File, @@ -258,8 +272,7 @@ pub fn flush(self: *MachO, module: *Module) !void { var last_cmd_offset: usize = @sizeOf(macho.mach_header_64); for (self.load_commands.items) |cmd| { - const cmd_to_write = [1]@TypeOf(cmd){cmd}; - try self.base.file.?.pwriteAll(mem.sliceAsBytes(cmd_to_write[0..1]), last_cmd_offset); + try cmd.write(&self.base.file.?, last_cmd_offset); last_cmd_offset += cmd.cmdsize(); } const off = @sizeOf(macho.mach_header_64) + @sizeOf(macho.segment_command_64); @@ -346,19 +359,18 @@ pub fn updateDecl(self: *MachO, module: *Module, decl: *Module.Decl) !void { .n_desc = 0, .n_value = addr, }; - self.offset_table.items[decl.link.macho.offset_table_index.?] = addr; + // Since we updated the vaddr and the size, each corresponding export symbol also needs to be updated. + const decl_exports = module.decl_exports.get(decl) orelse &[0]*Module.Export{}; + try self.updateDeclExports(module, decl, decl_exports); try self.writeSymbol(decl.link.macho.symbol_table_index.?); const text_section = self.sections.items[self.text_section_index.?]; const section_offset = symbol.n_value - text_section.addr; const file_offset = text_section.offset + section_offset; log.debug("file_offset 0x{x}\n", .{file_offset}); - try self.base.file.?.pwriteAll(code, file_offset); - // Since we updated the vaddr and the size, each corresponding export symbol also needs to be updated. - const decl_exports = module.decl_exports.get(decl) orelse &[0]*Module.Export{}; - return self.updateDeclExports(module, decl, decl_exports); + try self.base.file.?.pwriteAll(code, file_offset); } pub fn updateDeclLineNumber(self: *MachO, module: *Module, decl: *const Module.Decl) !void {} @@ -374,7 +386,7 @@ pub fn updateDeclExports( if (decl.link.macho.symbol_table_index == null) return; - var decl_sym = self.symbol_table.items[decl.link.macho.symbol_table_index.?]; + const decl_sym = &self.symbol_table.items[decl.link.macho.symbol_table_index.?]; // TODO implement if (exports.len == 0) return; @@ -504,10 +516,7 @@ fn allocateTextBlock(self: *MachO, text_block: *TextBlock, new_block_size: u64, const text_capacity = self.allocatedSize(text_section.offset); const needed_size = (addr + new_block_size) - text_section.addr; log.debug("text capacity 0x{x}, needed size 0x{x}\n", .{ text_capacity, needed_size }); - - if (needed_size > text_capacity) { - // TODO handle growth - } + assert(needed_size <= text_capacity); // TODO handle growth self.last_text_block = text_block; text_section.size = needed_size; @@ -659,7 +668,7 @@ fn writeSymbol(self: *MachO, index: usize) !void { defer tracy.end(); const symtab = &self.load_commands.items[self.symtab_cmd_index.?].Symtab; - var sym = [1]macho.nlist_64{self.symbol_table.items[index]}; + const sym = [1]macho.nlist_64{self.symbol_table.items[index]}; const off = symtab.symoff + @sizeOf(macho.nlist_64) * index; log.debug("writing symbol {} at 0x{x}\n", .{ sym[0], off }); try self.base.file.?.pwriteAll(mem.sliceAsBytes(sym[0..1]), off); From edbfd04ec1fa11fde88166c4b906b7868cbbf218 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 8 Sep 2020 22:00:31 +0200 Subject: [PATCH 27/89] Do not pad out text blocks It seems MachO does not like padding between text block in __text section. Unlike in Elf, there is no size information in symbol struct `nlist_64`. Signed-off-by: Jakub Konka --- src-self-hosted/link/MachO.zig | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src-self-hosted/link/MachO.zig b/src-self-hosted/link/MachO.zig index 4b068d45b..13932e514 100644 --- a/src-self-hosted/link/MachO.zig +++ b/src-self-hosted/link/MachO.zig @@ -500,9 +500,8 @@ fn allocateTextBlock(self: *MachO, text_block: *TextBlock, new_block_size: u64, const addr = blk: { if (self.last_text_block) |last| { const last_symbol = self.symbol_table.items[last.symbol_table_index.?]; - const ideal_capacity = last.size * alloc_num / alloc_den; - const ideal_capacity_end_addr = last_symbol.n_value + ideal_capacity; - const new_start_addr = mem.alignForwardGeneric(u64, ideal_capacity_end_addr, alignment); + const end_addr = last_symbol.n_value + last.size; + const new_start_addr = mem.alignForwardGeneric(u64, end_addr, alignment); block_placement = last; break :blk new_start_addr; } else { From edc40157eb3b547f86185e2f85c08daf928eb9f1 Mon Sep 17 00:00:00 2001 From: Zachary Meadows Date: Tue, 8 Sep 2020 10:41:05 -0700 Subject: [PATCH 28/89] Switch type of HashMap's count from usize to u32 (#6262) --- lib/std/hash_map.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/hash_map.zig b/lib/std/hash_map.zig index 144a512ed..458706d71 100644 --- a/lib/std/hash_map.zig +++ b/lib/std/hash_map.zig @@ -113,7 +113,7 @@ pub fn HashMap( return self.unmanaged.clearAndFree(self.allocator); } - pub fn count(self: Self) usize { + pub fn count(self: Self) Size { return self.unmanaged.count(); } From 749417a1f3060f0695bbfe72d929f06b0be42535 Mon Sep 17 00:00:00 2001 From: Vexu Date: Wed, 9 Sep 2020 16:29:16 +0300 Subject: [PATCH 29/89] translate-c: check for builtin typedef macro identifiers Closes #6292 --- src-self-hosted/translate_c.zig | 2 +- test/translate_c.zig | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 68d1dabb0..a65c1b9fb 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -5881,7 +5881,7 @@ fn parseCPrimaryExpr(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!*ast.N }, .Identifier => { const mangled_name = scope.getAlias(slice); - return transCreateNodeIdentifier(c, mangled_name); + return transCreateNodeIdentifier(c, checkForBuiltinTypedef(mangled_name) orelse mangled_name); }, .LParen => { const inner_node = try parseCExpr(c, m, scope); diff --git a/test/translate_c.zig b/test/translate_c.zig index 41f1e7829..83fdda8d8 100644 --- a/test/translate_c.zig +++ b/test/translate_c.zig @@ -2761,12 +2761,15 @@ pub fn addCases(cases: *tests.TranslateCContext) void { cases.add("macro cast", \\#define FOO(bar) baz((void *)(baz)) \\#define BAR (void*) a + \\#define BAZ (uint32_t)(2) , &[_][]const u8{ \\pub inline fn FOO(bar: anytype) @TypeOf(baz((@import("std").meta.cast(?*c_void, baz)))) { \\ return baz((@import("std").meta.cast(?*c_void, baz))); \\} , \\pub const BAR = (@import("std").meta.cast(?*c_void, a)); + , + \\pub const BAZ = (@import("std").meta.cast(u32, 2)); }); cases.add("macro with cast to unsigned short, long, and long long", From 78baa16da020ec9ad741b8f3f9c6168ec77ef58d Mon Sep 17 00:00:00 2001 From: Literally Void Date: Wed, 9 Sep 2020 20:49:49 -0700 Subject: [PATCH 30/89] Fix issue #6303: iterating empty PriorityQueue crashes --- lib/std/priority_queue.zig | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/std/priority_queue.zig b/lib/std/priority_queue.zig index 1c0d230d4..b9be9b70b 100644 --- a/lib/std/priority_queue.zig +++ b/lib/std/priority_queue.zig @@ -195,7 +195,7 @@ pub fn PriorityQueue(comptime T: type) type { count: usize, pub fn next(it: *Iterator) ?T { - if (it.count > it.queue.len - 1) return null; + if (it.count >= it.queue.len) return null; const out = it.count; it.count += 1; return it.queue.items[out]; @@ -428,3 +428,12 @@ test "std.PriorityQueue: remove at index" { expectEqual(queue.remove(), 3); expectEqual(queue.removeOrNull(), null); } + +test "std.PriorityQueue: iterator while empty" { + var queue = PQ.init(testing.allocator, lessThan); + defer queue.deinit(); + + var it = queue.iterator(); + + expectEqual(it.next(), null); +} From 0833c8d06ba9a467bb8449cbca2ba6f43d218c32 Mon Sep 17 00:00:00 2001 From: Vexu Date: Thu, 10 Sep 2020 12:58:12 +0300 Subject: [PATCH 31/89] translate-c: support sizeof and _Alignof in macros Closes #6301 --- src-self-hosted/translate_c.zig | 38 +++++++++++++++++++++ test/stage1/behavior/translate_c_macros.h | 5 ++- test/stage1/behavior/translate_c_macros.zig | 8 ++++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index a65c1b9fb..cd42e8abe 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -5899,6 +5899,10 @@ fn parseCPrimaryExpr(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!*ast.N saw_l_paren = true; _ = m.next(); }, + // (type)sizeof(x) + .Keyword_sizeof, + // (type)alignof(x) + .Keyword_alignof, // (type)identifier .Identifier => {}, // (type)integer @@ -6309,6 +6313,40 @@ fn parseCPrefixOpExpr(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!*ast. node.rhs = try parseCPrefixOpExpr(c, m, scope); return &node.base; }, + .Keyword_sizeof => { + const inner = if (m.peek().? == .LParen) blk: { + _ = m.next(); + const inner = try parseCExpr(c, m, scope); + if (m.next().? != .RParen) { + try m.fail(c, "unable to translate C expr: expected ')'", .{}); + return error.ParseError; + } + break :blk inner; + } else try parseCPrefixOpExpr(c, m, scope); + + const builtin_call = try c.createBuiltinCall("@sizeOf", 1); + builtin_call.params()[0] = inner; + builtin_call.rparen_token = try appendToken(c, .RParen, ")"); + return &builtin_call.base; + }, + .Keyword_alignof => { + // TODO this won't work if using 's + // #define alignof _Alignof + if (m.next().? != .LParen) { + try m.fail(c, "unable to translate C expr: expected '('", .{}); + return error.ParseError; + } + const inner = try parseCExpr(c, m, scope); + if (m.next().? != .RParen) { + try m.fail(c, "unable to translate C expr: expected ')'", .{}); + return error.ParseError; + } + + const builtin_call = try c.createBuiltinCall("@alignOf", 1); + builtin_call.params()[0] = inner; + builtin_call.rparen_token = try appendToken(c, .RParen, ")"); + return &builtin_call.base; + }, else => { m.i -= 1; return try parseCSuffixOpExpr(c, m, scope); diff --git a/test/stage1/behavior/translate_c_macros.h b/test/stage1/behavior/translate_c_macros.h index abc6c1e3c..49806a524 100644 --- a/test/stage1/behavior/translate_c_macros.h +++ b/test/stage1/behavior/translate_c_macros.h @@ -6,4 +6,7 @@ typedef struct Color { unsigned char a; } Color; #define CLITERAL(type) (type) -#define LIGHTGRAY CLITERAL(Color){ 200, 200, 200, 255 } // Light Gray \ No newline at end of file +#define LIGHTGRAY CLITERAL(Color){ 200, 200, 200, 255 } // Light Gray + +#define MY_SIZEOF(x) ((int)sizeof(x)) +#define MY_SIZEOF2(x) ((int)sizeof x) diff --git a/test/stage1/behavior/translate_c_macros.zig b/test/stage1/behavior/translate_c_macros.zig index ea42016e9..2cfb2331f 100644 --- a/test/stage1/behavior/translate_c_macros.zig +++ b/test/stage1/behavior/translate_c_macros.zig @@ -1,12 +1,18 @@ const expect = @import("std").testing.expect; +const expectEqual = @import("std").testing.expectEqual; const h = @cImport(@cInclude("stage1/behavior/translate_c_macros.h")); test "initializer list expression" { - @import("std").testing.expectEqual(h.Color{ + expectEqual(h.Color{ .r = 200, .g = 200, .b = 200, .a = 255, }, h.LIGHTGRAY); } + +test "sizeof in macros" { + expectEqual(@as(c_int, @sizeOf(u32)), h.MY_SIZEOF(u32)); + expectEqual(@as(c_int, @sizeOf(u32)), h.MY_SIZEOF2(u32)); +} From 4170f3f77f23bca52a2a9b24d1a249fae35c04ac Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Thu, 10 Sep 2020 13:36:34 +0200 Subject: [PATCH 32/89] std: fix bitrot in process.posixGetUserInfo() --- lib/std/process.zig | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/std/process.zig b/lib/std/process.zig index 9cb571714..2813d8cba 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -593,8 +593,10 @@ pub fn getUserInfo(name: []const u8) !UserInfo { /// TODO this reads /etc/passwd. But sometimes the user/id mapping is in something else /// like NIS, AD, etc. See `man nss` or look at an strace for `id myuser`. pub fn posixGetUserInfo(name: []const u8) !UserInfo { - var reader = try io.Reader.open("/etc/passwd", null); - defer reader.close(); + const file = try std.fs.openFileAbsolute("/etc/passwd", .{}); + defer file.close(); + + const reader = file.reader(); const State = enum { Start, @@ -650,8 +652,8 @@ pub fn posixGetUserInfo(name: []const u8) !UserInfo { '0'...'9' => byte - '0', else => return error.CorruptPasswordFile, }; - if (@mulWithOverflow(u32, uid, 10, *uid)) return error.CorruptPasswordFile; - if (@addWithOverflow(u32, uid, digit, *uid)) return error.CorruptPasswordFile; + if (@mulWithOverflow(u32, uid, 10, &uid)) return error.CorruptPasswordFile; + if (@addWithOverflow(u32, uid, digit, &uid)) return error.CorruptPasswordFile; }, }, .ReadGroupId => switch (byte) { @@ -666,8 +668,8 @@ pub fn posixGetUserInfo(name: []const u8) !UserInfo { '0'...'9' => byte - '0', else => return error.CorruptPasswordFile, }; - if (@mulWithOverflow(u32, gid, 10, *gid)) return error.CorruptPasswordFile; - if (@addWithOverflow(u32, gid, digit, *gid)) return error.CorruptPasswordFile; + if (@mulWithOverflow(u32, gid, 10, &gid)) return error.CorruptPasswordFile; + if (@addWithOverflow(u32, gid, digit, &gid)) return error.CorruptPasswordFile; }, }, } From 486a774e5fa079c4caba334f311ee32b54272f1f Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Wed, 9 Sep 2020 19:51:59 +0200 Subject: [PATCH 33/89] std: add option to use single-threaded event loop std.event.Loop does not yet work in single threaded builds. However, using evented io on a single thread can be very convenient. This commit allows settind @import("root").event_loop_mode to .single_threaded in order to allow this without reimplementing the startup code in start.zig --- lib/std/event/loop.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index b34ad8c94..2600b337b 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -112,7 +112,8 @@ pub const Loop = struct { /// have the correct pointer value. /// https://github.com/ziglang/zig/issues/2761 and https://github.com/ziglang/zig/issues/2765 pub fn init(self: *Loop) !void { - if (builtin.single_threaded) { + if (builtin.single_threaded + or (@hasDecl(root, "event_loop_mode") and root.event_loop_mode == .single_threaded)) { return self.initSingleThreaded(); } else { return self.initMultiThreaded(); From bb9773f695ee1a783ebd347d301dcc089afb64ba Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Wed, 9 Sep 2020 20:36:06 +0200 Subject: [PATCH 34/89] std: fix errorset of std.os.seteuid/setegid --- lib/std/os.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 181bf4930..25c71e5d1 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -2515,9 +2515,9 @@ pub fn readlinkatZ(dirfd: fd_t, file_path: [*:0]const u8, out_buffer: []u8) Read pub const SetEidError = error{ InvalidUserId, PermissionDenied, -}; +} || UnexpectedError; -pub const SetIdError = error{ResourceLimitReached} || SetEidError || UnexpectedError; +pub const SetIdError = error{ResourceLimitReached} || SetEidError; pub fn setuid(uid: uid_t) SetIdError!void { switch (errno(system.setuid(uid))) { From 13b8c638346b82da4856e9c5d688feffa66a32f6 Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Wed, 9 Sep 2020 20:05:47 +0200 Subject: [PATCH 35/89] std: complete C definitions for uid/gid setters Also: - correct the definition of setgid - update the existing definitions to use the uid_t/gid_t types --- lib/std/c.zig | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index 1b3f403ab..bbc32cf35 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -132,8 +132,6 @@ pub usingnamespace switch (builtin.os.tag) { }, }; -pub extern "c" fn setreuid(ruid: c_uint, euid: c_uint) c_int; -pub extern "c" fn setregid(rgid: c_uint, egid: c_uint) c_int; pub extern "c" fn rmdir(path: [*:0]const u8) c_int; pub extern "c" fn getenv(name: [*:0]const u8) ?[*:0]u8; pub extern "c" fn sysctl(name: [*]const c_int, namelen: c_uint, oldp: ?*c_void, oldlenp: ?*usize, newp: ?*c_void, newlen: usize) c_int; @@ -237,8 +235,15 @@ pub usingnamespace switch (builtin.os.tag) { pub extern "c" fn kill(pid: pid_t, sig: c_int) c_int; pub extern "c" fn getdirentries(fd: fd_t, buf_ptr: [*]u8, nbytes: usize, basep: *i64) isize; -pub extern "c" fn setgid(ruid: c_uint, euid: c_uint) c_int; -pub extern "c" fn setuid(uid: c_uint) c_int; + +pub extern "c" fn setuid(uid: uid_t) c_int; +pub extern "c" fn setgid(gid: gid_t) c_int; +pub extern "c" fn seteuid(euid: uid_t) c_int; +pub extern "c" fn setegid(egid: gid_t) c_int; +pub extern "c" fn setreuid(ruid: uid_t, euid: uid_t) c_int; +pub extern "c" fn setregid(rgid: gid_t, egid: gid_t) c_int; +pub extern "c" fn setresuid(ruid: uid_t, euid: uid_t, suid: uid_t) c_int; +pub extern "c" fn setresgid(rgid: gid_t, egid: gid_t, sgid: gid_t) c_int; pub extern "c" fn aligned_alloc(alignment: usize, size: usize) ?*c_void; pub extern "c" fn malloc(usize) ?*c_void; From ea9b38c93c38e402c807b20802eb003bcc4ddc5b Mon Sep 17 00:00:00 2001 From: Christian Wesselhoeft Date: Thu, 10 Sep 2020 11:28:51 -0600 Subject: [PATCH 36/89] fs/file.zig: Update reader to use type alias This is a minor cosmetic change which updates `reader` and `inStream` to match `writer` and `outStream` below. --- lib/std/fs/file.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index ef1b501ec..db301a9a4 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -740,14 +740,16 @@ pub const File = struct { } pub const Reader = io.Reader(File, ReadError, read); + /// Deprecated: use `Reader` pub const InStream = Reader; - pub fn reader(file: File) io.Reader(File, ReadError, read) { + pub fn reader(file: File) Reader { return .{ .context = file }; } + /// Deprecated: use `reader` - pub fn inStream(file: File) io.InStream(File, ReadError, read) { + pub fn inStream(file: File) Reader { return .{ .context = file }; } From 800c5de2ae527cdefaa5567720dab2f899244897 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 10 Sep 2020 14:40:33 -0700 Subject: [PATCH 37/89] update the stack trace test case for lines added to start.zig --- lib/std/os/bits/linux/powerpc64.zig | 6 +++--- test/stack_traces.zig | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/std/os/bits/linux/powerpc64.zig b/lib/std/os/bits/linux/powerpc64.zig index a6884aa02..adc6c87c1 100644 --- a/lib/std/os/bits/linux/powerpc64.zig +++ b/lib/std/os/bits/linux/powerpc64.zig @@ -26,7 +26,7 @@ pub const SYS = extern enum(usize) { mknod = 14, chmod = 15, lchown = 16, - sys_break = 17, // sys_ prepended to avoid clashing with keyword + @"break" = 17, oldstat = 18, lseek = 19, getpid = 20, @@ -574,7 +574,7 @@ pub const vrregset = extern struct { _pad1: [2]u32, vscr_word_be: u32, vrsave: u32, - _pad2: [3]u32 + _pad2: [3]u32, }; pub const vrregset_t = vrregset; @@ -588,7 +588,7 @@ pub const mcontext_t = extern struct { gp_regs: gregset_t, fp_regs: fpregset_t, v_regs: *vrregset_t, - vmx_reserve: [34+34+32+1]i64, + vmx_reserve: [34 + 34 + 32 + 1]i64, }; pub const ucontext_t = extern struct { diff --git a/test/stack_traces.zig b/test/stack_traces.zig index 2ab022e6e..496be0513 100644 --- a/test/stack_traces.zig +++ b/test/stack_traces.zig @@ -282,10 +282,10 @@ pub fn addCases(cases: *tests.StackTracesContext) void { \\source.zig:10:8: [address] in main (test) \\ foo(); \\ ^ - \\start.zig:254:29: [address] in std.start.posixCallMainAndExit (test) + \\start.zig:269:29: [address] in std.start.posixCallMainAndExit (test) \\ return root.main(); \\ ^ - \\start.zig:128:5: [address] in std.start._start (test) + \\start.zig:143:5: [address] in std.start._start (test) \\ @call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{}); \\ ^ \\ @@ -294,7 +294,7 @@ pub fn addCases(cases: *tests.StackTracesContext) void { switch (std.Target.current.cpu.arch) { .aarch64 => "", // TODO disabled; results in segfault else => - \\start.zig:128:5: [address] in std.start._start (test) + \\start.zig:143:5: [address] in std.start._start (test) \\ @call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{}); \\ ^ \\ From c41cd3e13a51864728009ba73d9297259defb005 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 10 Sep 2020 18:53:20 -0700 Subject: [PATCH 38/89] std.fs.File: fix typo in writeFileAll --- lib/std/fs/file.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index db301a9a4..73babf5fa 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -728,7 +728,7 @@ pub const File = struct { } var i: usize = 0; while (i < trailers.len) { - while (amt >= headers[i].iov_len) { + while (amt >= trailers[i].iov_len) { amt -= trailers[i].iov_len; i += 1; if (i >= trailers.len) return; From 68bf29c31efb770d7e25d0922c9305673e606f36 Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Fri, 11 Sep 2020 01:59:06 +0200 Subject: [PATCH 39/89] std: allow overriding install dir of artifacts This is necessary when, for example, writing a PAM module which should be installed to lib/security/module_name.so. --- lib/std/build.zig | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/std/build.zig b/lib/std/build.zig index 1673737be..cb27e112a 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -1048,6 +1048,7 @@ pub const Builder = struct { .Bin => self.exe_dir, .Lib => self.lib_dir, .Header => self.h_dir, + .Custom => |path| fs.path.join(self.allocator, &[_][]const u8{ self.install_path, path }) catch unreachable, }; return fs.path.resolve( self.allocator, @@ -1212,6 +1213,8 @@ pub const LibExeObjStep = struct { is_linking_libc: bool = false, vcpkg_bin_path: ?[]const u8 = null, + /// This may be set in order to override the default install directory + override_dest_dir: ?InstallDir, installed_path: ?[]const u8, install_step: ?*InstallArtifactStep, @@ -1348,6 +1351,7 @@ pub const LibExeObjStep = struct { .rdynamic = false, .output_dir = null, .single_threaded = false, + .override_dest_dir = null, .installed_path = null, .install_step = null, }; @@ -2309,17 +2313,17 @@ pub const InstallArtifactStep = struct { .builder = builder, .step = Step.init(.InstallArtifact, builder.fmt("install {}", .{artifact.step.name}), builder.allocator, make), .artifact = artifact, - .dest_dir = switch (artifact.kind) { + .dest_dir = artifact.override_dest_dir orelse switch (artifact.kind) { .Obj => unreachable, .Test => unreachable, - .Exe => .Bin, - .Lib => .Lib, + .Exe => InstallDir{ .Bin = {} }, + .Lib => InstallDir{ .Lib = {} }, }, .pdb_dir = if (artifact.producesPdbFile()) blk: { if (artifact.kind == .Exe) { - break :blk InstallDir.Bin; + break :blk InstallDir{ .Bin = {} }; } else { - break :blk InstallDir.Lib; + break :blk InstallDir{ .Lib = {} }; } } else null, .h_dir = if (artifact.kind == .Lib and artifact.emit_h) .Header else null, @@ -2615,11 +2619,13 @@ const VcpkgRootStatus = enum { pub const VcpkgLinkage = std.builtin.LinkMode; -pub const InstallDir = enum { - Prefix, - Lib, - Bin, - Header, +pub const InstallDir = union(enum) { + Prefix: void, + Lib: void, + Bin: void, + Header: void, + /// A path relative to the prefix + Custom: []const u8, }; pub const InstalledFile = struct { From 0a76a9bd738471e2edddf71a822ff76c47bfd094 Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Fri, 11 Sep 2020 01:22:18 +0200 Subject: [PATCH 40/89] std, stage1: make shared library versioning optional This commit changes the behavior of stage1 to emit libfoo.so instead of libfoo.so.0.0.0 when none of the --ver-major, --ver-minor, or --ver-patch flags are set. It also makes it possible to create unversioned shared libraries using the zig build system, changing the version parameter of addSharedLibrary() to a tagged union. --- lib/std/build.zig | 98 +++++++++++++++++++++++++++++++---------------- src/all_types.hpp | 1 + src/codegen.cpp | 6 ++- src/codegen.hpp | 2 +- src/glibc.cpp | 2 +- src/main.cpp | 8 +++- src/target.cpp | 18 ++++++--- src/target.hpp | 2 +- 8 files changed, 92 insertions(+), 45 deletions(-) diff --git a/lib/std/build.zig b/lib/std/build.zig index 1673737be..8aa473f8b 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -258,9 +258,14 @@ pub const Builder = struct { })); } - pub fn addSharedLibrary(self: *Builder, name: []const u8, root_src: ?[]const u8, ver: Version) *LibExeObjStep { + pub fn addSharedLibrary( + self: *Builder, + name: []const u8, + root_src: ?[]const u8, + kind: LibExeObjStep.SharedLibKind, + ) *LibExeObjStep { const root_src_param = if (root_src) |p| @as(FileSource, .{ .path = p }) else null; - return LibExeObjStep.createSharedLibrary(self, name, root_src_param, ver); + return LibExeObjStep.createSharedLibrary(self, name, root_src_param, kind); } pub fn addStaticLibrary(self: *Builder, name: []const u8, root_src: ?[]const u8) *LibExeObjStep { @@ -338,11 +343,13 @@ pub const Builder = struct { return TranslateCStep.create(self, source); } - pub fn version(self: *const Builder, major: u32, minor: u32, patch: u32) Version { - return Version{ - .major = major, - .minor = minor, - .patch = patch, + pub fn version(self: *const Builder, major: u32, minor: u32, patch: u32) LibExeObjStep.SharedLibKind { + return .{ + .versioned = .{ + .major = major, + .minor = minor, + .patch = patch, + }, }; } @@ -1166,7 +1173,7 @@ pub const LibExeObjStep = struct { version_script: ?[]const u8 = null, out_filename: []const u8, is_dynamic: bool, - version: Version, + version: ?Version, build_mode: builtin.Mode, kind: Kind, major_only_filename: []const u8, @@ -1268,33 +1275,41 @@ pub const LibExeObjStep = struct { Test, }; - pub fn createSharedLibrary(builder: *Builder, name: []const u8, root_src: ?FileSource, ver: Version) *LibExeObjStep { + const SharedLibKind = union(enum) { + versioned: Version, + unversioned: void, + }; + + pub fn createSharedLibrary(builder: *Builder, name: []const u8, root_src: ?FileSource, kind: SharedLibKind) *LibExeObjStep { const self = builder.allocator.create(LibExeObjStep) catch unreachable; - self.* = initExtraArgs(builder, name, root_src, Kind.Lib, true, ver); + self.* = initExtraArgs(builder, name, root_src, Kind.Lib, true, switch (kind) { + .versioned => |ver| ver, + .unversioned => null, + }); return self; } pub fn createStaticLibrary(builder: *Builder, name: []const u8, root_src: ?FileSource) *LibExeObjStep { const self = builder.allocator.create(LibExeObjStep) catch unreachable; - self.* = initExtraArgs(builder, name, root_src, Kind.Lib, false, builder.version(0, 0, 0)); + self.* = initExtraArgs(builder, name, root_src, Kind.Lib, false, null); return self; } pub fn createObject(builder: *Builder, name: []const u8, root_src: ?FileSource) *LibExeObjStep { const self = builder.allocator.create(LibExeObjStep) catch unreachable; - self.* = initExtraArgs(builder, name, root_src, Kind.Obj, false, builder.version(0, 0, 0)); + self.* = initExtraArgs(builder, name, root_src, Kind.Obj, false, null); return self; } pub fn createExecutable(builder: *Builder, name: []const u8, root_src: ?FileSource, is_dynamic: bool) *LibExeObjStep { const self = builder.allocator.create(LibExeObjStep) catch unreachable; - self.* = initExtraArgs(builder, name, root_src, Kind.Exe, is_dynamic, builder.version(0, 0, 0)); + self.* = initExtraArgs(builder, name, root_src, Kind.Exe, is_dynamic, null); return self; } pub fn createTest(builder: *Builder, name: []const u8, root_src: FileSource) *LibExeObjStep { const self = builder.allocator.create(LibExeObjStep) catch unreachable; - self.* = initExtraArgs(builder, name, root_src, Kind.Test, false, builder.version(0, 0, 0)); + self.* = initExtraArgs(builder, name, root_src, Kind.Test, false, null); return self; } @@ -1304,7 +1319,7 @@ pub const LibExeObjStep = struct { root_src: ?FileSource, kind: Kind, is_dynamic: bool, - ver: Version, + ver: ?Version, ) LibExeObjStep { if (mem.indexOf(u8, name, "/") != null or mem.indexOf(u8, name, "\\") != null) { panic("invalid name: '{}'. It looks like a file path, but it is supposed to be the library or application name.", .{name}); @@ -1375,17 +1390,17 @@ pub const LibExeObjStep = struct { self.target.staticLibSuffix(), }); self.out_lib_filename = self.out_filename; - } else { + } else if (self.version) |version| { if (self.target.isDarwin()) { self.out_filename = self.builder.fmt("lib{}.{d}.{d}.{d}.dylib", .{ self.name, - self.version.major, - self.version.minor, - self.version.patch, + version.major, + version.minor, + version.patch, }); self.major_only_filename = self.builder.fmt("lib{}.{d}.dylib", .{ self.name, - self.version.major, + version.major, }); self.name_only_filename = self.builder.fmt("lib{}.dylib", .{self.name}); self.out_lib_filename = self.out_filename; @@ -1395,14 +1410,25 @@ pub const LibExeObjStep = struct { } else { self.out_filename = self.builder.fmt("lib{}.so.{d}.{d}.{d}", .{ self.name, - self.version.major, - self.version.minor, - self.version.patch, + version.major, + version.minor, + version.patch, }); - self.major_only_filename = self.builder.fmt("lib{}.so.{d}", .{ self.name, self.version.major }); + self.major_only_filename = self.builder.fmt("lib{}.so.{d}", .{ self.name, version.major }); self.name_only_filename = self.builder.fmt("lib{}.so", .{self.name}); self.out_lib_filename = self.out_filename; } + } else { + if (self.target.isDarwin()) { + self.out_filename = self.builder.fmt("lib{}.dylib", .{self.name}); + self.out_lib_filename = self.out_filename; + } else if (self.target.isWindows()) { + self.out_filename = self.builder.fmt("{}.dll", .{self.name}); + self.out_lib_filename = self.builder.fmt("{}.lib", .{self.name}); + } else { + self.out_filename = self.builder.fmt("lib{}.so", .{self.name}); + self.out_lib_filename = self.out_filename; + } } }, } @@ -2037,14 +2063,16 @@ pub const LibExeObjStep = struct { zig_args.append(self.name) catch unreachable; if (self.kind == Kind.Lib and self.is_dynamic) { - zig_args.append("--ver-major") catch unreachable; - zig_args.append(builder.fmt("{}", .{self.version.major})) catch unreachable; + if (self.version) |version| { + zig_args.append("--ver-major") catch unreachable; + zig_args.append(builder.fmt("{}", .{version.major})) catch unreachable; - zig_args.append("--ver-minor") catch unreachable; - zig_args.append(builder.fmt("{}", .{self.version.minor})) catch unreachable; + zig_args.append("--ver-minor") catch unreachable; + zig_args.append(builder.fmt("{}", .{version.minor})) catch unreachable; - zig_args.append("--ver-patch") catch unreachable; - zig_args.append(builder.fmt("{}", .{self.version.patch})) catch unreachable; + zig_args.append("--ver-patch") catch unreachable; + zig_args.append(builder.fmt("{}", .{version.patch})) catch unreachable; + } } if (self.is_dynamic) { try zig_args.append("-dynamic"); @@ -2285,7 +2313,7 @@ pub const LibExeObjStep = struct { } } - if (self.kind == Kind.Lib and self.is_dynamic and self.target.wantSharedLibSymLinks()) { + if (self.kind == Kind.Lib and self.is_dynamic and self.version != null and self.target.wantSharedLibSymLinks()) { try doAtomicSymLinks(builder.allocator, self.getOutputPath(), self.major_only_filename, self.name_only_filename); } } @@ -2329,8 +2357,10 @@ pub const InstallArtifactStep = struct { builder.pushInstalledFile(self.dest_dir, artifact.out_filename); if (self.artifact.isDynamicLibrary()) { - builder.pushInstalledFile(.Lib, artifact.major_only_filename); - builder.pushInstalledFile(.Lib, artifact.name_only_filename); + if (self.artifact.version != null) { + builder.pushInstalledFile(.Lib, artifact.major_only_filename); + builder.pushInstalledFile(.Lib, artifact.name_only_filename); + } if (self.artifact.target.isWindows()) { builder.pushInstalledFile(.Lib, artifact.out_lib_filename); } @@ -2350,7 +2380,7 @@ pub const InstallArtifactStep = struct { const full_dest_path = builder.getInstallPath(self.dest_dir, self.artifact.out_filename); try builder.updateFile(self.artifact.getOutputPath(), full_dest_path); - if (self.artifact.isDynamicLibrary() and self.artifact.target.wantSharedLibSymLinks()) { + if (self.artifact.isDynamicLibrary() and self.artifact.version != null and self.artifact.target.wantSharedLibSymLinks()) { try doAtomicSymLinks(builder.allocator, full_dest_path, self.artifact.major_only_filename, self.artifact.name_only_filename); } if (self.pdb_dir) |pdb_dir| { diff --git a/src/all_types.hpp b/src/all_types.hpp index 3fbcc8958..1fa04f2b7 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -2265,6 +2265,7 @@ struct CodeGen { Stage2LibCInstallation *libc; + bool is_versioned; size_t version_major; size_t version_minor; size_t version_patch; diff --git a/src/codegen.cpp b/src/codegen.cpp index 493bbbbae..b5c1ca3a4 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -90,7 +90,8 @@ void codegen_set_test_name_prefix(CodeGen *g, Buf *prefix) { g->test_name_prefix = prefix; } -void codegen_set_lib_version(CodeGen *g, size_t major, size_t minor, size_t patch) { +void codegen_set_lib_version(CodeGen *g, bool is_versioned, size_t major, size_t minor, size_t patch) { + g->is_versioned = is_versioned; g->version_major = major; g->version_minor = minor; g->version_patch = patch; @@ -10823,6 +10824,7 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { cache_bool(ch, g->emit_bin); cache_bool(ch, g->emit_llvm_ir); cache_bool(ch, g->emit_asm); + cache_bool(ch, g->is_versioned); cache_usize(ch, g->version_major); cache_usize(ch, g->version_minor); cache_usize(ch, g->version_patch); @@ -10893,7 +10895,7 @@ static void resolve_out_paths(CodeGen *g) { buf_resize(out_basename, 0); buf_append_str(out_basename, target_lib_file_prefix(g->zig_target)); buf_append_buf(out_basename, g->root_out_name); - buf_append_str(out_basename, target_lib_file_ext(g->zig_target, !g->is_dynamic, + buf_append_str(out_basename, target_lib_file_ext(g->zig_target, !g->is_dynamic, g->is_versioned, g->version_major, g->version_minor, g->version_patch)); break; } diff --git a/src/codegen.hpp b/src/codegen.hpp index 191da9a04..3139071d5 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -38,7 +38,7 @@ void codegen_set_rdynamic(CodeGen *g, bool rdynamic); void codegen_set_linker_script(CodeGen *g, const char *linker_script); void codegen_set_test_filter(CodeGen *g, Buf *filter); void codegen_set_test_name_prefix(CodeGen *g, Buf *prefix); -void codegen_set_lib_version(CodeGen *g, size_t major, size_t minor, size_t patch); +void codegen_set_lib_version(CodeGen *g, bool is_versioned, size_t major, size_t minor, size_t patch); void codegen_add_time_event(CodeGen *g, const char *name); void codegen_print_timing_report(CodeGen *g, FILE *f); void codegen_link(CodeGen *g); diff --git a/src/glibc.cpp b/src/glibc.cpp index 2456cab44..62f5604ba 100644 --- a/src/glibc.cpp +++ b/src/glibc.cpp @@ -335,7 +335,7 @@ Error glibc_build_dummies_and_maps(CodeGen *g, const ZigGLibCAbi *glibc_abi, con bool is_ld = (strcmp(lib->name, "ld") == 0); CodeGen *child_gen = create_child_codegen(g, zig_file_path, OutTypeLib, nullptr, lib->name, progress_node); - codegen_set_lib_version(child_gen, lib->sover, 0, 0); + codegen_set_lib_version(child_gen, true, lib->sover, 0, 0); child_gen->is_dynamic = true; child_gen->is_dummy_so = true; child_gen->version_script_path = map_file_path; diff --git a/src/main.cpp b/src/main.cpp index e2f6a82a1..348321598 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -416,6 +416,7 @@ static int main0(int argc, char **argv) { const char *test_filter = nullptr; const char *test_name_prefix = nullptr; bool test_evented_io = false; + bool is_versioned = false; size_t ver_major = 0; size_t ver_minor = 0; size_t ver_patch = 0; @@ -870,6 +871,7 @@ static int main0(int argc, char **argv) { fprintf(stderr, "expected linker arg after '%s'\n", buf_ptr(arg)); return EXIT_FAILURE; } + is_versioned = true; ver_major = atoi(buf_ptr(linker_args.at(i))); } else if (buf_eql_str(arg, "--minor-image-version")) { i += 1; @@ -877,6 +879,7 @@ static int main0(int argc, char **argv) { fprintf(stderr, "expected linker arg after '%s'\n", buf_ptr(arg)); return EXIT_FAILURE; } + is_versioned = true; ver_minor = atoi(buf_ptr(linker_args.at(i))); } else if (buf_eql_str(arg, "--stack")) { i += 1; @@ -1228,10 +1231,13 @@ static int main0(int argc, char **argv) { } else if (strcmp(arg, "--test-name-prefix") == 0) { test_name_prefix = argv[i]; } else if (strcmp(arg, "--ver-major") == 0) { + is_versioned = true; ver_major = atoi(argv[i]); } else if (strcmp(arg, "--ver-minor") == 0) { + is_versioned = true; ver_minor = atoi(argv[i]); } else if (strcmp(arg, "--ver-patch") == 0) { + is_versioned = true; ver_patch = atoi(argv[i]); } else if (strcmp(arg, "--test-cmd") == 0) { test_exec_args.append(argv[i]); @@ -1590,7 +1596,7 @@ static int main0(int argc, char **argv) { g->emit_llvm_ir = emit_llvm_ir; codegen_set_out_name(g, buf_out_name); - codegen_set_lib_version(g, ver_major, ver_minor, ver_patch); + codegen_set_lib_version(g, is_versioned, ver_major, ver_minor, ver_patch); g->want_single_threaded = want_single_threaded; codegen_set_linker_script(g, linker_script); g->version_script_path = version_script; diff --git a/src/target.cpp b/src/target.cpp index 6c9916ebd..dff134a01 100644 --- a/src/target.cpp +++ b/src/target.cpp @@ -779,7 +779,7 @@ const char *target_lib_file_prefix(const ZigTarget *target) { } } -const char *target_lib_file_ext(const ZigTarget *target, bool is_static, +const char *target_lib_file_ext(const ZigTarget *target, bool is_static, bool is_versioned, size_t version_major, size_t version_minor, size_t version_patch) { if (target_is_wasm(target)) { @@ -799,11 +799,19 @@ const char *target_lib_file_ext(const ZigTarget *target, bool is_static, if (is_static) { return ".a"; } else if (target_os_is_darwin(target->os)) { - return buf_ptr(buf_sprintf(".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".dylib", - version_major, version_minor, version_patch)); + if (is_versioned) { + return buf_ptr(buf_sprintf(".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".dylib", + version_major, version_minor, version_patch)); + } else { + return ".dylib"; + } } else { - return buf_ptr(buf_sprintf(".so.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize, - version_major, version_minor, version_patch)); + if (is_versioned) { + return buf_ptr(buf_sprintf(".so.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize, + version_major, version_minor, version_patch)); + } else { + return ".so"; + } } } } diff --git a/src/target.hpp b/src/target.hpp index 9a8e79de9..5e44301ff 100644 --- a/src/target.hpp +++ b/src/target.hpp @@ -87,7 +87,7 @@ const char *target_asm_file_ext(const ZigTarget *target); const char *target_llvm_ir_file_ext(const ZigTarget *target); const char *target_exe_file_ext(const ZigTarget *target); const char *target_lib_file_prefix(const ZigTarget *target); -const char *target_lib_file_ext(const ZigTarget *target, bool is_static, +const char *target_lib_file_ext(const ZigTarget *target, bool is_static, bool is_versioned, size_t version_major, size_t version_minor, size_t version_patch); bool target_can_exec(const ZigTarget *host_target, const ZigTarget *guest_target); From bee34f5d036a74e520f5174fcaa132e6fb4114c6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 11 Sep 2020 11:30:21 -0700 Subject: [PATCH 41/89] add a manifesto to the readme --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index e72b166c7..8031aa790 100644 --- a/README.md +++ b/README.md @@ -76,3 +76,21 @@ Hopefully this will be fixed upstream with LLVM 10.0.1. ##### Windows See https://github.com/ziglang/zig/wiki/Building-Zig-on-Windows + +## License + +The ultimate goal of the Zig project is to serve users. As a first-order +effect, this means users of the compiler, helping programmers to write better +code. Even more important, however, are the end users. + +Zig is intended to be used to help end users accomplish their goals. For +example, it would be inappropriate and offensive to use Zig to implement +[dark patterns](https://en.wikipedia.org/wiki/Dark_pattern) and it would be +shameful to utilize Zig to exploit people instead of benefit them. + +However, such problems are best solved with social norms, not with software +licenses. Any attempt to complicate the software license of Zig would risk +compromising the value Zig provides to users. + +Therefore, Zig is available under the MIT (Expat) License, and comes with a +humble request: use it to make software better serve the needs of end users. From 41f244bd2ff0a217a83a3d17fc82f4a773eca278 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 11 Sep 2020 11:00:13 +0200 Subject: [PATCH 42/89] std: Make the DEFLATE decompression routine 3x faster A profiler run showed that the main bottleneck was the naive decoding of the Huffman codes, replacing it with a nice trick borrowed by Zlib gave a substantial speedup. Replacing a `%` with a `and (mask-1)` gave another significant improvement (yay for low hanging fruits). A few numbers obtained by decompressing a 22M file: Before: ``` ./decompress 2,39s user 0,00s system 99% cpu 2,400 total ``` After: ``` ./decompress 0,79s user 0,00s system 99% cpu 0,798 total ```` --- lib/std/compress/deflate.zig | 184 ++++++++++++++++++++++++++++------- lib/std/compress/zlib.zig | 8 +- 2 files changed, 153 insertions(+), 39 deletions(-) diff --git a/lib/std/compress/deflate.zig b/lib/std/compress/deflate.zig index bad23349e..9fe96cacb 100644 --- a/lib/std/compress/deflate.zig +++ b/lib/std/compress/deflate.zig @@ -21,48 +21,121 @@ const MAXDCODES = 30; const MAXCODES = MAXLCODES + MAXDCODES; const FIXLCODES = 288; +// The maximum length of a Huffman code's prefix we can decode using the fast +// path. The factor 9 is inherited from Zlib, tweaking the value showed little +// or no changes in the profiler output. +const PREFIX_LUT_BITS = 9; + const Huffman = struct { + // Number of codes for each possible length count: [MAXBITS + 1]u16, + // Mapping between codes and symbols symbol: [MAXCODES]u16, - fn construct(self: *Huffman, length: []const u16) !void { + // The decoding process uses a trick explained by Mark Adler in [1]. + // We basically precompute for a fixed number of codes (0 <= x <= 2^N-1) + // the symbol and the effective code length we'd get if the decoder was run + // on the given N-bit sequence. + // A code with length 0 means the sequence is not a valid prefix for this + // canonical Huffman code and we have to decode it using a slower method. + // + // [1] https://github.com/madler/zlib/blob/v1.2.11/doc/algorithm.txt#L58 + prefix_lut: [1 << PREFIX_LUT_BITS]u16, + prefix_lut_len: [1 << PREFIX_LUT_BITS]u16, + // The following info refer to the codes of length PREFIX_LUT_BITS+1 and are + // used to bootstrap the bit-by-bit reading method if the fast-path fails. + last_code: u16, + last_index: u16, + + fn construct(self: *Huffman, code_length: []const u16) !void { for (self.count) |*val| { val.* = 0; } - for (length) |val| { - self.count[val] += 1; + for (code_length) |len| { + self.count[len] += 1; } - if (self.count[0] == length.len) + // All zero. + if (self.count[0] == code_length.len) return; var left: isize = 1; for (self.count[1..]) |val| { + // Each added bit doubles the amount of codes. left *= 2; + // Make sure the number of codes with this length isn't too high. left -= @as(isize, @bitCast(i16, val)); if (left < 0) return error.InvalidTree; } - var offs: [MAXBITS + 1]u16 = undefined; + // Compute the offset of the first symbol represented by a code of a + // given length in the symbol table, together with the first canonical + // Huffman code for that length. + var offset: [MAXBITS + 1]u16 = undefined; + var codes: [MAXBITS + 1]u16 = undefined; { + offset[1] = 0; + codes[1] = 0; var len: usize = 1; - offs[1] = 0; while (len < MAXBITS) : (len += 1) { - offs[len + 1] = offs[len] + self.count[len]; + offset[len + 1] = offset[len] + self.count[len]; + codes[len + 1] = (codes[len] + self.count[len]) << 1; } } - for (length) |val, symbol| { - if (val != 0) { - self.symbol[offs[val]] = @truncate(u16, symbol); - offs[val] += 1; + self.prefix_lut_len = mem.zeroes(@TypeOf(self.prefix_lut_len)); + + for (code_length) |len, symbol| { + if (len != 0) { + // Fill the symbol table. + // The symbols are assigned sequentially for each length. + self.symbol[offset[len]] = @truncate(u16, symbol); + // Track the last assigned offset + offset[len] += 1; + } + + if (len == 0 or len > PREFIX_LUT_BITS) + continue; + + // Given a Huffman code of length N we have to massage it so + // that it becomes an index in the lookup table. + // The bit order is reversed as the fast path reads the bit + // sequence MSB to LSB using an &, the order is flipped wrt the + // one obtained by reading bit-by-bit. + // The codes are prefix-free, if the prefix matches we can + // safely ignore the trail bits. We do so by replicating the + // symbol info for each combination of the trailing bits. + const bits_to_fill = @intCast(u5, PREFIX_LUT_BITS - len); + const rev_code = bitReverse(codes[len], len); + // Track the last used code, but only for lengths < PREFIX_LUT_BITS + codes[len] += 1; + + var j: usize = 0; + while (j < @as(usize, 1) << bits_to_fill) : (j += 1) { + const index = rev_code | (j << @intCast(u5, len)); + assert(self.prefix_lut_len[index] == 0); + self.prefix_lut[index] = @truncate(u16, symbol); + self.prefix_lut_len[index] = @truncate(u16, len); } } + + self.last_code = codes[PREFIX_LUT_BITS + 1]; + self.last_index = offset[PREFIX_LUT_BITS + 1] - self.count[PREFIX_LUT_BITS + 1]; } }; +// Reverse bit-by-bit a N-bit value +fn bitReverse(x: usize, N: usize) usize { + var tmp: usize = 0; + var i: usize = 0; + while (i < N) : (i += 1) { + tmp |= ((x >> @intCast(u5, i)) & 1) << @intCast(u5, N - i - 1); + } + return tmp; +} + pub fn InflateStream(comptime ReaderType: type) type { return struct { const Self = @This(); @@ -83,7 +156,7 @@ pub fn InflateStream(comptime ReaderType: type) type { }; pub const Reader = io.Reader(*Self, Error, read); - bit_reader: io.BitReader(.Little, ReaderType), + inner_reader: ReaderType, // True if the decoder met the end of the compressed stream, no further // data can be decompressed @@ -135,7 +208,7 @@ pub fn InflateStream(comptime ReaderType: type) type { // Insert a single byte into the window. // Assumes there's enough space. - fn appendUnsafe(self: *WSelf, value: u8) void { + inline fn appendUnsafe(self: *WSelf, value: u8) void { self.buf[self.wi] = value; self.wi = (self.wi + 1) & (self.buf.len - 1); self.el += 1; @@ -180,7 +253,7 @@ pub fn InflateStream(comptime ReaderType: type) type { // of the window memory for the non-overlapping case. var i: usize = 0; while (i < N) : (i += 1) { - const index = (self.wi -% distance) % self.buf.len; + const index = (self.wi -% distance) & (self.buf.len - 1); self.appendUnsafe(self.buf[index]); } @@ -196,13 +269,36 @@ pub fn InflateStream(comptime ReaderType: type) type { hdist: *Huffman, hlen: *Huffman, + // Temporary buffer for the bitstream, only bits 0..`bits_left` are + // considered valid. + bits: u32, + bits_left: usize, + + fn peekBits(self: *Self, bits: usize) !u32 { + while (self.bits_left < bits) { + const byte = try self.inner_reader.readByte(); + self.bits |= @as(u32, byte) << @intCast(u5, self.bits_left); + self.bits_left += 8; + } + return self.bits & ((@as(u32, 1) << @intCast(u5, bits)) - 1); + } + fn readBits(self: *Self, bits: usize) !u32 { + const val = self.peekBits(bits); + self.discardBits(bits); + return val; + } + fn discardBits(self: *Self, bits: usize) void { + self.bits >>= @intCast(u5, bits); + self.bits_left -= bits; + } + fn stored(self: *Self) !void { // Discard the remaining bits, the lenght field is always // byte-aligned (and so is the data) - self.bit_reader.alignToByte(); + self.discardBits(self.bits_left); - const length = (try self.bit_reader.readBitsNoEof(u16, 16)); - const length_cpl = (try self.bit_reader.readBitsNoEof(u16, 16)); + const length = try self.inner_reader.readIntLittle(u16); + const length_cpl = try self.inner_reader.readIntLittle(u16); if (length != ~length_cpl) return error.InvalidStoredSize; @@ -237,11 +333,11 @@ pub fn InflateStream(comptime ReaderType: type) type { fn dynamic(self: *Self) !void { // Number of length codes - const nlen = (try self.bit_reader.readBitsNoEof(usize, 5)) + 257; + const nlen = (try self.readBits(5)) + 257; // Number of distance codes - const ndist = (try self.bit_reader.readBitsNoEof(usize, 5)) + 1; + const ndist = (try self.readBits(5)) + 1; // Number of code length codes - const ncode = (try self.bit_reader.readBitsNoEof(usize, 4)) + 4; + const ncode = (try self.readBits(4)) + 4; if (nlen > MAXLCODES or ndist > MAXDCODES) return error.BadCounts; @@ -259,7 +355,7 @@ pub fn InflateStream(comptime ReaderType: type) type { // Read the code lengths, missing ones are left as zero for (ORDER[0..ncode]) |val| { - lengths[val] = try self.bit_reader.readBitsNoEof(u16, 3); + lengths[val] = @intCast(u16, try self.readBits(3)); } try lencode.construct(lengths[0..]); @@ -284,7 +380,7 @@ pub fn InflateStream(comptime ReaderType: type) type { if (i == 0) return error.NoLastLength; const last_length = lengths[i - 1]; - const repeat = 3 + (try self.bit_reader.readBitsNoEof(usize, 2)); + const repeat = 3 + (try self.readBits(2)); const last_index = i + repeat; while (i < last_index) : (i += 1) { lengths[i] = last_length; @@ -292,11 +388,11 @@ pub fn InflateStream(comptime ReaderType: type) type { }, 17 => { // repeat zero 3..10 times - i += 3 + (try self.bit_reader.readBitsNoEof(usize, 3)); + i += 3 + (try self.readBits(3)); }, 18 => { // repeat zero 11..138 times - i += 11 + (try self.bit_reader.readBitsNoEof(usize, 7)); + i += 11 + (try self.readBits(7)); }, else => return error.InvalidSymbol, } @@ -359,11 +455,11 @@ pub fn InflateStream(comptime ReaderType: type) type { // Length/distance pair const length_symbol = symbol - 257; const length = LENS[length_symbol] + - try self.bit_reader.readBitsNoEof(u16, LEXT[length_symbol]); + @intCast(u16, try self.readBits(LEXT[length_symbol])); const distance_symbol = try self.decode(distcode); const distance = DISTS[distance_symbol] + - try self.bit_reader.readBitsNoEof(u16, DEXT[distance_symbol]); + @intCast(u16, try self.readBits(DEXT[distance_symbol])); if (distance > self.window.buf.len) return error.InvalidDistance; @@ -385,13 +481,29 @@ pub fn InflateStream(comptime ReaderType: type) type { } fn decode(self: *Self, h: *Huffman) !u16 { - var len: usize = 1; - var code: usize = 0; - var first: usize = 0; - var index: usize = 0; + // Fast path, read some bits and hope they're prefixes of some code + const prefix = try self.peekBits(PREFIX_LUT_BITS); + if (h.prefix_lut_len[prefix] != 0) { + self.discardBits(h.prefix_lut_len[prefix]); + return h.prefix_lut[prefix]; + } + + // The sequence we've read is not a prefix of any code of length <= + // PREFIX_LUT_BITS, keep decoding it using a slower method + self.discardBits(PREFIX_LUT_BITS); + + // Speed up the decoding by starting from the first code length + // that's not covered by the table + var len: usize = PREFIX_LUT_BITS + 1; + var first: usize = h.last_code; + var index: usize = h.last_index; + + // Reverse the prefix so that the LSB becomes the MSB and make space + // for the next bit + var code = bitReverse(prefix, PREFIX_LUT_BITS + 1); while (len <= MAXBITS) : (len += 1) { - code |= try self.bit_reader.readBitsNoEof(usize, 1); + code |= try self.readBits(1); const count = h.count[len]; if (code < first + count) return h.symbol[index + (code - first)]; @@ -411,8 +523,8 @@ pub fn InflateStream(comptime ReaderType: type) type { // The compressed stream is done if (self.seen_eos) return; - const last = try self.bit_reader.readBitsNoEof(u1, 1); - const kind = try self.bit_reader.readBitsNoEof(u2, 2); + const last = @intCast(u1, try self.readBits(1)); + const kind = @intCast(u2, try self.readBits(2)); self.seen_eos = last != 0; @@ -439,7 +551,7 @@ pub fn InflateStream(comptime ReaderType: type) type { var i: usize = 0; while (i < N) : (i += 1) { var tmp: [1]u8 = undefined; - if ((try self.bit_reader.read(&tmp)) != 1) { + if ((try self.inner_reader.read(&tmp)) != 1) { // Unexpected end of stream, keep this error // consistent with the use of readBitsNoEof return error.EndOfStream; @@ -478,12 +590,14 @@ pub fn InflateStream(comptime ReaderType: type) type { assert(math.isPowerOfTwo(window_slice.len)); return Self{ - .bit_reader = io.bitReader(.Little, source), + .inner_reader = source, .window = .{ .buf = window_slice }, .seen_eos = false, .state = .DecodeBlockHeader, .hdist = undefined, .hlen = undefined, + .bits = 0, + .bits_left = 0, }; } diff --git a/lib/std/compress/zlib.zig b/lib/std/compress/zlib.zig index d4bac4a8a..acda7eade 100644 --- a/lib/std/compress/zlib.zig +++ b/lib/std/compress/zlib.zig @@ -138,10 +138,10 @@ test "compressed data" { "5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009", ); // Compressed with compression level = 9 and fixed Huffman codes - try testReader( - @embedFile("rfc1951.txt.fixed.z.9"), - "5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009", - ); + // try testReader( + // @embedFile("rfc1951.txt.fixed.z.9"), + // "5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009", + // ); } test "sanity checks" { From 4b851c7255d726f5743c42ae80a71ce7cf47c486 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 11 Sep 2020 21:04:08 +0200 Subject: [PATCH 43/89] Re-enable a compression test It somehow got commented out... --- lib/std/compress/zlib.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/std/compress/zlib.zig b/lib/std/compress/zlib.zig index acda7eade..d4bac4a8a 100644 --- a/lib/std/compress/zlib.zig +++ b/lib/std/compress/zlib.zig @@ -138,10 +138,10 @@ test "compressed data" { "5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009", ); // Compressed with compression level = 9 and fixed Huffman codes - // try testReader( - // @embedFile("rfc1951.txt.fixed.z.9"), - // "5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009", - // ); + try testReader( + @embedFile("rfc1951.txt.fixed.z.9"), + "5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009", + ); } test "sanity checks" { From 7b961a876b56d5926fe7ba6437a0459da9aa60bc Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Fri, 11 Sep 2020 01:34:10 +0200 Subject: [PATCH 44/89] std: add prctl definition for linux --- lib/std/os/bits/linux.zig | 1 + lib/std/os/bits/linux/prctl.zig | 158 ++++++++++++++++++++++++++++++++ lib/std/os/linux.zig | 4 + 3 files changed, 163 insertions(+) create mode 100644 lib/std/os/bits/linux/prctl.zig diff --git a/lib/std/os/bits/linux.zig b/lib/std/os/bits/linux.zig index f393f04ab..e8625179d 100644 --- a/lib/std/os/bits/linux.zig +++ b/lib/std/os/bits/linux.zig @@ -25,6 +25,7 @@ pub usingnamespace switch (builtin.arch) { }; pub usingnamespace @import("linux/netlink.zig"); +pub usingnamespace @import("linux/prctl.zig"); const is_mips = builtin.arch.isMIPS(); diff --git a/lib/std/os/bits/linux/prctl.zig b/lib/std/os/bits/linux/prctl.zig new file mode 100644 index 000000000..7fa9969af --- /dev/null +++ b/lib/std/os/bits/linux/prctl.zig @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +pub const PR_SET_PDEATHSIG = 1; +pub const PR_GET_PDEATHSIG = 2; + +pub const PR_GET_DUMPABLE = 3; +pub const PR_SET_DUMPABLE = 4; + +pub const PR_GET_UNALIGN = 5; +pub const PR_SET_UNALIGN = 6; +pub const PR_UNALIGN_NOPRINT = 1; +pub const PR_UNALIGN_SIGBUS = 2; + +pub const PR_GET_KEEPCAPS = 7; +pub const PR_SET_KEEPCAPS = 8; + +pub const PR_GET_FPEMU = 9; +pub const PR_SET_FPEMU = 10; +pub const PR_FPEMU_NOPRINT = 1; +pub const PR_FPEMU_SIGFPE = 2; + +pub const PR_GET_FPEXC = 11; +pub const PR_SET_FPEXC = 12; +pub const PR_FP_EXC_SW_ENABLE = 0x80; +pub const PR_FP_EXC_DIV = 0x010000; +pub const PR_FP_EXC_OVF = 0x020000; +pub const PR_FP_EXC_UND = 0x040000; +pub const PR_FP_EXC_RES = 0x080000; +pub const PR_FP_EXC_INV = 0x100000; +pub const PR_FP_EXC_DISABLED = 0; +pub const PR_FP_EXC_NONRECOV = 1; +pub const PR_FP_EXC_ASYNC = 2; +pub const PR_FP_EXC_PRECISE = 3; + +pub const PR_GET_TIMING = 13; +pub const PR_SET_TIMING = 14; +pub const PR_TIMING_STATISTICAL = 0; +pub const PR_TIMING_TIMESTAMP = 1; + +pub const PR_SET_NAME = 15; +pub const PR_GET_NAME = 16; + +pub const PR_GET_ENDIAN = 19; +pub const PR_SET_ENDIAN = 20; +pub const PR_ENDIAN_BIG = 0; +pub const PR_ENDIAN_LITTLE = 1; +pub const PR_ENDIAN_PPC_LITTLE = 2; + +pub const PR_GET_SECCOMP = 21; +pub const PR_SET_SECCOMP = 22; + +pub const PR_CAPBSET_READ = 23; +pub const PR_CAPBSET_DROP = 24; + +pub const PR_GET_TSC = 25; +pub const PR_SET_TSC = 26; +pub const PR_TSC_ENABLE = 1; +pub const PR_TSC_SIGSEGV = 2; + +pub const PR_GET_SECUREBITS = 27; +pub const PR_SET_SECUREBITS = 28; + +pub const PR_SET_TIMERSLACK = 29; +pub const PR_GET_TIMERSLACK = 30; + +pub const PR_TASK_PERF_EVENTS_DISABLE = 31; +pub const PR_TASK_PERF_EVENTS_ENABLE = 32; + +pub const PR_MCE_KILL = 33; +pub const PR_MCE_KILL_CLEAR = 0; +pub const PR_MCE_KILL_SET = 1; + +pub const PR_MCE_KILL_LATE = 0; +pub const PR_MCE_KILL_EARLY = 1; +pub const PR_MCE_KILL_DEFAULT = 2; + +pub const PR_MCE_KILL_GET = 34; + +pub const PR_SET_MM = 35; +pub const PR_SET_MM_START_CODE = 1; +pub const PR_SET_MM_END_CODE = 2; +pub const PR_SET_MM_START_DATA = 3; +pub const PR_SET_MM_END_DATA = 4; +pub const PR_SET_MM_START_STACK = 5; +pub const PR_SET_MM_START_BRK = 6; +pub const PR_SET_MM_BRK = 7; +pub const PR_SET_MM_ARG_START = 8; +pub const PR_SET_MM_ARG_END = 9; +pub const PR_SET_MM_ENV_START = 10; +pub const PR_SET_MM_ENV_END = 11; +pub const PR_SET_MM_AUXV = 12; +pub const PR_SET_MM_EXE_FILE = 13; +pub const PR_SET_MM_MAP = 14; +pub const PR_SET_MM_MAP_SIZE = 15; + +pub const prctl_mm_map = extern struct { + start_code: u64, + end_code: u64, + start_data: u64, + end_data: u64, + start_brk: u64, + brk: u64, + start_stack: u64, + arg_start: u64, + arg_end: u64, + env_start: u64, + env_end: u64, + auxv: *u64, + auxv_size: u32, + exe_fd: u32, +}; + +pub const PR_SET_PTRACER = 0x59616d61; +pub const PR_SET_PTRACER_ANY = std.math.maxInt(c_ulong); + +pub const PR_SET_CHILD_SUBREAPER = 36; +pub const PR_GET_CHILD_SUBREAPER = 37; + +pub const PR_SET_NO_NEW_PRIVS = 38; +pub const PR_GET_NO_NEW_PRIVS = 39; + +pub const PR_GET_TID_ADDRESS = 40; + +pub const PR_SET_THP_DISABLE = 41; +pub const PR_GET_THP_DISABLE = 42; + +pub const PR_MPX_ENABLE_MANAGEMENT = 43; +pub const PR_MPX_DISABLE_MANAGEMENT = 44; + +pub const PR_SET_FP_MODE = 45; +pub const PR_GET_FP_MODE = 46; +pub const PR_FP_MODE_FR = 1 << 0; +pub const PR_FP_MODE_FRE = 1 << 1; + +pub const PR_CAP_AMBIENT = 47; +pub const PR_CAP_AMBIENT_IS_SET = 1; +pub const PR_CAP_AMBIENT_RAISE = 2; +pub const PR_CAP_AMBIENT_LOWER = 3; +pub const PR_CAP_AMBIENT_CLEAR_ALL = 4; + +pub const PR_SVE_SET_VL = 50; +pub const PR_SVE_SET_VL_ONEXEC = 1 << 18; +pub const PR_SVE_GET_VL = 51; +pub const PR_SVE_VL_LEN_MASK = 0xffff; +pub const PR_SVE_VL_INHERIT = 1 << 17; + +pub const PR_GET_SPECULATION_CTRL = 52; +pub const PR_SET_SPECULATION_CTRL = 53; +pub const PR_SPEC_STORE_BYPASS = 0; +pub const PR_SPEC_NOT_AFFECTED = 0; +pub const PR_SPEC_PRCTL = 1 << 0; +pub const PR_SPEC_ENABLE = 1 << 1; +pub const PR_SPEC_DISABLE = 1 << 2; +pub const PR_SPEC_FORCE_DISABLE = 1 << 3; diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 25a0d6814..50d1e4ae7 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -1259,6 +1259,10 @@ pub fn fdatasync(fd: fd_t) usize { return syscall1(.fdatasync, @bitCast(usize, @as(isize, fd))); } +pub fn prctl(option: i32, arg2: usize, arg3: usize, arg4: usize, arg5: usize) usize { + return syscall5(.prctl, @bitCast(usize, @as(isize, option)), arg2, arg3, arg4, arg5); +} + test "" { if (builtin.os.tag == .linux) { _ = @import("linux/test.zig"); From 744b73ab46a75140537f9cf705939ecf0a2d229d Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Fri, 11 Sep 2020 14:10:55 +0200 Subject: [PATCH 45/89] std: add prctl wrapper to std.os --- lib/std/os.zig | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/lib/std/os.zig b/lib/std/os.zig index 25c71e5d1..91365c81d 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -5418,3 +5418,42 @@ pub fn fdatasync(fd: fd_t) SyncError!void { else => |err| return std.os.unexpectedErrno(err), } } + +pub const PrctlError = error{ + /// Can only occur with PR_SET_SECCOMP/SECCOMP_MODE_FILTER or + /// PR_SET_MM/PR_SET_MM_EXE_FILE + AccessDenied, + /// Can only occur with PR_SET_MM/PR_SET_MM_EXE_FILE + InvalidFileDescriptor, + InvalidAddress, + /// Can only occur with PR_SET_SPECULATION_CTRL, PR_MPX_ENABLE_MANAGEMENT, + /// or PR_MPX_DISABLE_MANAGEMENT + UnsupportedFeature, + /// Can only occur wih PR_SET_FP_MODE + OperationNotSupported, + PermissionDenied, +} || UnexpectedError; + +pub fn prctl(option: i32, args: anytype) PrctlError!u31 { + if (@typeInfo(@TypeOf(args)) != .Struct) + @compileError("Expected tuple or struct argument, found " ++ @typeName(@TypeOf(args))); + if (args.len > 4) + @compileError("prctl takes a maximum of 4 optional arguments"); + + var buf: [4]usize = undefined; + inline for (args) |arg, i| buf[i] = arg; + + const rc = system.prctl(option, buf[0], buf[1], buf[2], buf[3]); + switch (errno(rc)) { + 0 => return @intCast(u31, rc), + EACCES => return error.AccessDenied, + EBADF => return error.InvalidFileDescriptor, + EFAULT => return error.InvalidAddress, + EINVAL => unreachable, + ENODEV, ENXIO => return error.UnsupportedFeature, + EOPNOTSUPP => return error.OperationNotSupported, + EPERM, EBUSY => return error.PermissionDenied, + ERANGE => unreachable, + else => |err| return std.os.unexpectedErrno(err), + } +} From b6f4601545547fff7afa19cba0f26354d651e818 Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Fri, 11 Sep 2020 14:19:59 +0200 Subject: [PATCH 46/89] std: add securebits definitions for linux --- lib/std/os/bits/linux.zig | 1 + lib/std/os/bits/linux/securebits.zig | 41 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 lib/std/os/bits/linux/securebits.zig diff --git a/lib/std/os/bits/linux.zig b/lib/std/os/bits/linux.zig index e8625179d..df31bc32f 100644 --- a/lib/std/os/bits/linux.zig +++ b/lib/std/os/bits/linux.zig @@ -26,6 +26,7 @@ pub usingnamespace switch (builtin.arch) { pub usingnamespace @import("linux/netlink.zig"); pub usingnamespace @import("linux/prctl.zig"); +pub usingnamespace @import("linux/securebits.zig"); const is_mips = builtin.arch.isMIPS(); diff --git a/lib/std/os/bits/linux/securebits.zig b/lib/std/os/bits/linux/securebits.zig new file mode 100644 index 000000000..0086a694d --- /dev/null +++ b/lib/std/os/bits/linux/securebits.zig @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +fn issecure_mask(comptime x: comptime_int) comptime_int { + return 1 << x; +} + +pub const SECUREBITS_DEFAULT = 0x00000000; + +pub const SECURE_NOROOT = 0; +pub const SECURE_NOROOT_LOCKED = 1; + +pub const SECBIT_NOROOT = issecure_mask(SECURE_NOROOT); +pub const SECBIT_NOROOT_LOCKED = issecure_mask(SECURE_NOROOT_LOCKED); + +pub const SECURE_NO_SETUID_FIXUP = 2; +pub const SECURE_NO_SETUID_FIXUP_LOCKED = 3; + +pub const SECBIT_NO_SETUID_FIXUP = issecure_mask(SECURE_NO_SETUID_FIXUP); +pub const SECBIT_NO_SETUID_FIXUP_LOCKED = issecure_mask(SECURE_NO_SETUID_FIXUP_LOCKED); + +pub const SECURE_KEEP_CAPS = 4; +pub const SECURE_KEEP_CAPS_LOCKED = 5; + +pub const SECBIT_KEEP_CAPS = issecure_mask(SECURE_KEEP_CAPS); +pub const SECBIT_KEEP_CAPS_LOCKED = issecure_mask(SECURE_KEEP_CAPS_LOCKED); + +pub const SECURE_NO_CAP_AMBIENT_RAISE = 6; +pub const SECURE_NO_CAP_AMBIENT_RAISE_LOCKED = 7; + +pub const SECBIT_NO_CAP_AMBIENT_RAISE = issecure_mask(SECURE_NO_CAP_AMBIENT_RAISE); +pub const SECBIT_NO_CAP_AMBIENT_RAISE_LOCKED = issecure_mask(SECURE_NO_CAP_AMBIENT_RAISE_LOCKED); + +pub const SECURE_ALL_BITS = issecure_mask(SECURE_NOROOT) | + issecure_mask(SECURE_NO_SETUID_FIXUP) | + issecure_mask(SECURE_KEEP_CAPS) | + issecure_mask(SECURE_NO_CAP_AMBIENT_RAISE); +pub const SECURE_ALL_LOCKS = SECURE_ALL_BITS << 1; From 1078810cef4b346bdcd0ab0cab27dd997e68d206 Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Fri, 11 Sep 2020 16:32:24 +0200 Subject: [PATCH 47/89] std: add prctl definition when linking libc --- lib/std/c.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/std/c.zig b/lib/std/c.zig index bbc32cf35..aa50fff90 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -340,3 +340,5 @@ pub extern "c" fn sync() void; pub extern "c" fn syncfs(fd: c_int) c_int; pub extern "c" fn fsync(fd: c_int) c_int; pub extern "c" fn fdatasync(fd: c_int) c_int; + +pub extern "c" fn prctl(option: c_int, ...) c_int; From c2b02d01d5c0bb684565d5b23b34022b752e9507 Mon Sep 17 00:00:00 2001 From: Rob Napier Date: Fri, 11 Sep 2020 17:10:27 -0400 Subject: [PATCH 48/89] Add crypto.kdf.pbkdf2 --- lib/std/crypto.zig | 6 + lib/std/crypto/pbkdf2.zig | 227 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 lib/std/crypto/pbkdf2.zig diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig index 5de2f1389..2b4294282 100644 --- a/lib/std/crypto.zig +++ b/lib/std/crypto.zig @@ -35,6 +35,11 @@ pub const onetimeauth = struct { pub const Poly1305 = @import("crypto/poly1305.zig").Poly1305; }; +/// Key derivation functions +pub const kdf = struct { + pub const pbkdf2 = @import("crypto/pbkdf2.zig").pbkdf2; +}; + /// Core functions, that should rarely be used directly by applications. pub const core = struct { pub const aes = @import("crypto/aes.zig"); @@ -77,6 +82,7 @@ test "crypto" { _ = @import("crypto/gimli.zig"); _ = @import("crypto/hmac.zig"); _ = @import("crypto/md5.zig"); + _ = @import("crypto/pbkdf2.zig"); _ = @import("crypto/poly1305.zig"); _ = @import("crypto/sha1.zig"); _ = @import("crypto/sha2.zig"); diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig new file mode 100644 index 000000000..fe00f7ee0 --- /dev/null +++ b/lib/std/crypto/pbkdf2.zig @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +const std = @import("std"); +const crypto = std.crypto; +const debug = std.debug; +const assert = debug.assert; +const mem = std.mem; + +// RFC 2898 Section 5.2 +// +// FromSpec: +// +// PBKDF2 applies a pseudorandom function (see Appendix B.1 for an +// example) to derive keys. The length of the derived key is essentially +// unbounded. (However, the maximum effective search space for the +// derived key may be limited by the structure of the underlying +// pseudorandom function. See Appendix B.1 for further discussion.) +// PBKDF2 is recommended for new applications. +// +// PBKDF2 (P, S, c, dkLen) +// +// Options: PRF underlying pseudorandom function (hLen +// denotes the length in octets of the +// pseudorandom function output) +// +// Input: P password, an octet string +// S salt, an octet string +// c iteration count, a positive integer +// dkLen intended length in octets of the derived +// key, a positive integer, at most +// (2^32 - 1) * hLen +// +// Output: DK derived key, a dkLen-octet string + +// Based on Apple's CommonKeyDerivation, based originally on code by Damien Bergamini. + +pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: u32, comptime Hash: type) void { + assert(rounds >= 1); + + const dkLen = derivedKey.len; + const hLen = Hash.digest_length; + const Prf = crypto.auth.hmac.Hmac(Hash); + + // FromSpec: + // + // 1. If dkLen > (2^32 - 1) * hLen, output "derived key too long" and + // stop. + // + assert(dkLen > 0 and dkLen <= (1 << 32 - 1) * hLen); + + // FromSpec: + // + // 2. Let l be the number of hLen-octet blocks in the derived key, + // rounding up, and let r be the number of octets in the last + // block + // + const l = (dkLen + hLen - 1) / hLen; + var r = dkLen % hLen; + r = if (r != 0) r else hLen; + + // FromSpec: + // + // 3. For each block of the derived key apply the function F defined + // below to the password P, the salt S, the iteration count c, and + // the block index to compute the block: + // + // T_1 = F (P, S, c, 1) , + // T_2 = F (P, S, c, 2) , + // ... + // T_l = F (P, S, c, l) , + // + // where the function F is defined as the exclusive-or sum of the + // first c iterates of the underlying pseudorandom function PRF + // applied to the password P and the concatenation of the salt S + // and the block index i: + // + // F (P, S, c, i) = U_1 \xor U_2 \xor ... \xor U_c + // + // where + // + // U_1 = PRF (P, S || INT (i)) , + // U_2 = PRF (P, U_1) , + // ... + // U_c = PRF (P, U_{c-1}) . + // + // Here, INT (i) is a four-octet encoding of the integer i, most + // significant octet first. + // + // 4. Concatenate the blocks and extract the first dkLen octets to + // produce a derived key DK: + // + // DK = T_1 || T_2 || ... || T_l<0..r-1> + + var prevBlock: [hLen]u8 = undefined; + var newBlock: [hLen]u8 = undefined; + + var block: u32 = 0; // Spec limits to u32 + while (block < l) : (block += 1) { + + // U_1 = PRF (P, S || INT (i)) + const blockIndex = mem.toBytes(mem.nativeToBig(u32, block + 1)); // Block index starts at 0001 + var ctx = Prf.init(password); + ctx.update(salt); + ctx.update(blockIndex[0..]); + ctx.final(prevBlock[0..]); + + // Choose portion of DK to write into (T_n) and initialize + const offset = block * hLen; + const blockLen = if (block != l - 1) hLen else r; + var dkBlock = derivedKey[offset..(offset + blockLen)]; + mem.copy(u8, dkBlock[0..], prevBlock[0..dkBlock.len]); + + var i: u32 = 1; + while (i < rounds) : (i += 1) { + // U_c = PRF (P, U_{c-1}) + Prf.create(newBlock[0..], prevBlock[0..], password); + mem.copy(u8, prevBlock[0..], newBlock[0..]); + + // F (P, S, c, i) = U_1 \xor U_2 \xor ... \xor U_c + for (dkBlock) |_, j| { + dkBlock[j] ^= newBlock[j]; + } + } + } +} + +const htest = @import("test.zig"); + +// RFC 6070 PBKDF2 HMAC-SHA1 Test Vectors +test "RFC 6070 one iteration" { + const p = "password"; + const s = "salt"; + const c = 1; + const dkLen = 20; + + var derivedKey: [dkLen]u8 = undefined; + + pbkdf2(&derivedKey, p, s, c, crypto.hash.Sha1); + + const expected = "0c60c80f961f0e71f3a9b524af6012062fe037a6"; + + htest.assertEqual(expected, derivedKey[0..]); +} + +test "RFC 6070 two iterations" { + const p = "password"; + const s = "salt"; + const c = 2; + const dkLen = 20; + + var derivedKey: [dkLen]u8 = undefined; + + pbkdf2(&derivedKey, p, s, c, crypto.hash.Sha1); + + const expected = "ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957"; + + htest.assertEqual(expected, derivedKey[0..]); +} + +test "RFC 6070 4096 iterations" { + const p = "password"; + const s = "salt"; + const c = 4096; + const dkLen = 20; + + var derivedKey: [dkLen]u8 = undefined; + + pbkdf2(&derivedKey, p, s, c, crypto.hash.Sha1); + + const expected = "4b007901b765489abead49d926f721d065a429c1"; + + htest.assertEqual(expected, derivedKey[0..]); +} + +test "RFC 6070 16,777,216 iterations" { + // These iteration tests are slow so we always skip them. Results have been verified. + if (true) { + return error.SkipZigTest; + } + + const p = "password"; + const s = "salt"; + const c = 16777216; + const dkLen = 20; + + var derivedKey = [_]u8{0} ** dkLen; + + pbkdf2(&derivedKey, p, s, c, crypto.hash.Sha1); + + const expected = "eefe3d61cd4da4e4e9945b3d6ba2158c2634e984"; + + htest.assertEqual(expected, derivedKey[0..]); +} + +test "RFC 6070 multi-block salt and password" { + const p = "passwordPASSWORDpassword"; + const s = "saltSALTsaltSALTsaltSALTsaltSALTsalt"; + const c = 4096; + const dkLen = 25; + + var derivedKey: [dkLen]u8 = undefined; + + pbkdf2(&derivedKey, p, s, c, crypto.hash.Sha1); + + const expected = "3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038"; + + htest.assertEqual(expected, derivedKey[0..]); +} + +test "RFC 6070 embedded NUL" { + const p = "pass\x00word"; + const s = "sa\x00lt"; + const c = 4096; + const dkLen = 16; + + var derivedKey: [dkLen]u8 = undefined; + + pbkdf2(&derivedKey, p, s, c, crypto.hash.Sha1); + + const expected = "56fa6aa75548099dcc37d7f03425e0c3"; + + htest.assertEqual(expected, derivedKey[0..]); +} From 9fe4c89230df2d78c8bf37b4b1d7a9bedb92677b Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 11 Sep 2020 22:17:08 +0200 Subject: [PATCH 49/89] std: Add a gzip decoder --- build.zig | 1 + lib/std/compress.zig | 2 + lib/std/compress/gzip.zig | 248 ++++++++++++++++++++++++++++++++ lib/std/compress/rfc1952.txt.gz | Bin 0 -> 8059 bytes 4 files changed, 251 insertions(+) create mode 100644 lib/std/compress/gzip.zig create mode 100644 lib/std/compress/rfc1952.txt.gz diff --git a/build.zig b/build.zig index 3f7f1a903..a6a2d8737 100644 --- a/build.zig +++ b/build.zig @@ -128,6 +128,7 @@ pub fn build(b: *Builder) !void { "README.md", ".z.0", ".z.9", + ".gz", "rfc1951.txt", }, }); diff --git a/lib/std/compress.zig b/lib/std/compress.zig index 5518f807d..95f496021 100644 --- a/lib/std/compress.zig +++ b/lib/std/compress.zig @@ -6,8 +6,10 @@ const std = @import("std.zig"); pub const deflate = @import("compress/deflate.zig"); +pub const gzip = @import("compress/gzip.zig"); pub const zlib = @import("compress/zlib.zig"); test "" { + _ = gzip; _ = zlib; } diff --git a/lib/std/compress/gzip.zig b/lib/std/compress/gzip.zig new file mode 100644 index 000000000..aad173139 --- /dev/null +++ b/lib/std/compress/gzip.zig @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. +// +// Decompressor for GZIP data streams (RFC1952) + +const std = @import("std"); +const io = std.io; +const fs = std.fs; +const testing = std.testing; +const mem = std.mem; +const deflate = std.compress.deflate; + +// Flags for the FLG field in the header +const FTEXT = 1 << 0; +const FHCRC = 1 << 1; +const FEXTRA = 1 << 2; +const FNAME = 1 << 3; +const FCOMMENT = 1 << 4; + +pub fn GzipStream(comptime ReaderType: type) type { + return struct { + const Self = @This(); + + pub const Error = ReaderType.Error || + deflate.InflateStream(ReaderType).Error || + error{ CorruptedData, WrongChecksum }; + pub const Reader = io.Reader(*Self, Error, read); + + allocator: *mem.Allocator, + inflater: deflate.InflateStream(ReaderType), + in_reader: ReaderType, + hasher: std.hash.Crc32, + window_slice: []u8, + read_amt: usize, + + info: struct { + filename: ?[]const u8, + comment: ?[]const u8, + modification_time: u32, + }, + + fn init(allocator: *mem.Allocator, source: ReaderType) !Self { + // gzip header format is specified in RFC1952 + const header = try source.readBytesNoEof(10); + + // Check the ID1/ID2 fields + if (header[0] != 0x1f or header[1] != 0x8b) + return error.BadHeader; + + const CM = header[2]; + // The CM field must be 8 to indicate the use of DEFLATE + if (CM != 8) return error.InvalidCompression; + // Flags + const FLG = header[3]; + // Modification time, as a Unix timestamp. + // If zero there's no timestamp available. + const MTIME = mem.readIntLittle(u32, header[4..8]); + // Extra flags + const XFL = header[8]; + // Operating system where the compression took place + const OS = header[9]; + + if (FLG & FEXTRA != 0) { + // Skip the extra data, we could read and expose it to the user + // if somebody needs it. + const len = try source.readIntLittle(u16); + try source.skipBytes(len, .{}); + } + + var filename: ?[]const u8 = null; + if (FLG & FNAME != 0) { + filename = try source.readUntilDelimiterAlloc( + allocator, + 0, + std.math.maxInt(usize), + ); + } + errdefer if (filename) |p| allocator.free(p); + + var comment: ?[]const u8 = null; + if (FLG & FCOMMENT != 0) { + comment = try source.readUntilDelimiterAlloc( + allocator, + 0, + std.math.maxInt(usize), + ); + } + errdefer if (comment) |p| allocator.free(p); + + if (FLG & FHCRC != 0) { + // TODO: Evaluate and check the header checksum. The stdlib has + // no CRC16 yet :( + _ = try source.readIntLittle(u16); + } + + // The RFC doesn't say anything about the DEFLATE window size to be + // used, default to 32K. + var window_slice = try allocator.alloc(u8, 32 * 1024); + + return Self{ + .allocator = allocator, + .inflater = deflate.inflateStream(source, window_slice), + .in_reader = source, + .hasher = std.hash.Crc32.init(), + .window_slice = window_slice, + .info = .{ + .filename = filename, + .comment = comment, + .modification_time = MTIME, + }, + .read_amt = 0, + }; + } + + pub fn deinit(self: *Self) void { + self.allocator.free(self.window_slice); + if (self.info.filename) |filename| + self.allocator.free(filename); + if (self.info.comment) |comment| + self.allocator.free(comment); + } + + // Implements the io.Reader interface + pub fn read(self: *Self, buffer: []u8) Error!usize { + if (buffer.len == 0) + return 0; + + // Read from the compressed stream and update the computed checksum + const r = try self.inflater.read(buffer); + if (r != 0) { + self.hasher.update(buffer[0..r]); + self.read_amt += r; + return r; + } + + // We've reached the end of stream, check if the checksum matches + const hash = try self.in_reader.readIntLittle(u32); + if (hash != self.hasher.final()) + return error.WrongChecksum; + + // The ISIZE field is the size of the uncompressed input modulo 2^32 + const input_size = try self.in_reader.readIntLittle(u32); + if (self.read_amt & 0xffffffff != input_size) + return error.CorruptedData; + + return 0; + } + + pub fn reader(self: *Self) Reader { + return .{ .context = self }; + } + }; +} + +pub fn gzipStream(allocator: *mem.Allocator, reader: anytype) !GzipStream(@TypeOf(reader)) { + return GzipStream(@TypeOf(reader)).init(allocator, reader); +} + +fn testReader(data: []const u8, comptime expected: []const u8) !void { + var in_stream = io.fixedBufferStream(data); + + var gzip_stream = try gzipStream(testing.allocator, in_stream.reader()); + defer gzip_stream.deinit(); + + // Read and decompress the whole file + const buf = try gzip_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize)); + defer testing.allocator.free(buf); + // Calculate its SHA256 hash and check it against the reference + var hash: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(buf, hash[0..], .{}); + + assertEqual(expected, &hash); +} + +// Assert `expected` == `input` where `input` is a bytestring. +pub fn assertEqual(comptime expected: []const u8, input: []const u8) void { + var expected_bytes: [expected.len / 2]u8 = undefined; + for (expected_bytes) |*r, i| { + r.* = std.fmt.parseInt(u8, expected[2 * i .. 2 * i + 2], 16) catch unreachable; + } + + testing.expectEqualSlices(u8, &expected_bytes, input); +} + +// All the test cases are obtained by compressing the RFC1952 text +// +// https://tools.ietf.org/rfc/rfc1952.txt length=25037 bytes +// SHA256=164ef0897b4cbec63abf1b57f069f3599bd0fb7c72c2a4dee21bd7e03ec9af67 +test "compressed data" { + try testReader( + @embedFile("rfc1952.txt.gz"), + "164ef0897b4cbec63abf1b57f069f3599bd0fb7c72c2a4dee21bd7e03ec9af67", + ); +} + +test "sanity checks" { + // Truncated header + testing.expectError( + error.EndOfStream, + testReader(&[_]u8{ 0x1f, 0x8B }, ""), + ); + // Wrong CM + testing.expectError( + error.InvalidCompression, + testReader(&[_]u8{ + 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, + }, ""), + ); + // Wrong checksum + testing.expectError( + error.WrongChecksum, + testReader(&[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, + }, ""), + ); + // Truncated checksum + testing.expectError( + error.EndOfStream, + testReader(&[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, + }, ""), + ); + // Wrong initial size + testing.expectError( + error.CorruptedData, + testReader(&[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, + }, ""), + ); + // Truncated initial size field + testing.expectError( + error.EndOfStream, + testReader(&[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, + }, ""), + ); +} diff --git a/lib/std/compress/rfc1952.txt.gz b/lib/std/compress/rfc1952.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..be43b90a7917a993933c3266db50882f77a5662e GIT binary patch literal 8059 zcmV->AB5l^iwFogG^;TH19E0#F*!9dE_8Tw0JU6gbK6Fe{;s+oF{=+3GA1FCl5AOy zbE}mtM=43xT5^)woU;W2LlPwr;NnFuIq!eJr@Lo>0SL)fR$;{vC184bdiuS4W@Cds zXS%pavn%y8{ud>4^)5@xmHOJxxu@RfvdDwQ#s~ecQs;%5rI|WRmrI=#`Mw&z+?_ly z{vh_lFiO+LzC27JcYW}Sx!H9yzPyRi^4C;T%|L0xrlOgs+TFlbH);&FvZX$wMg74KdDFMg`b3e7SjBFa;LC0AppFI z@*<0-B~O*k2uxAogP&y*N6VRe;l6(jLnI$K6L?*@R)GatBwsz@=@iYOJT(2_zoC*&Of z`UGWFW>}==_iv(Nk(Stxg`ds!0Eh;084|}AfS$*^eYBUDBn3G#0ucMjy!7Xw7lFW# z6aX{XX?_#CyNb-%f#nDwx&U(l2*C)dB_~V>2E{1b-!qMoI@j5?4hQBj#+yat<&?Og zh^{pupAa@!VyI&+D9^HV$s3qrnusKYAO^AavzQ?){p^Zaf&>oR9ub!jF=-HxC#ki( zkG*Hy2#ZY9!>}T)T{pq`;qvsvN>KpLosvwI7$gfe#415QCRh;H| ztn-}r1mw&KbHDH{mQ2$qmn~VYFnkI{Ky1MX-(ch7J5|y!))Nr2*otkXFobJpK8AA_ z#8Ch`g;=?t1b5gV9bAzll{^IK6%ZieGDZdbla(t%T)+|#1DQD>DFlU_QF`W>lQZr%J98vWOE2%H*!+W4=H4@ZaV281`~h?Mw| zAb-(dhe(Jy-lw0hoh!~__?y}iY7 zcw{>6*z68Kd!2wLk7iEw(f|U5KIj>=8Ag8fR&@b_fL!RH%*cO&ipgP3GXLutK#fNR z5Cr5PxJuF+Fy-91SC2uc0t5j$D52}KZ*p}IhNS=h7@!vbv|2$MMYsFvN9`xW6ABUD zVc+9B0H6OJt*o`|m_d!l2GE`h=)xyUqYghDf?na*Y#RaEXlZPt#cR!LeSR`-*8Q_S zO;g122N+559~}3=jDNt1*mvWKZ`Cn5jiIJ>k^4y|=KP~7?pAOz$zk@{{O5N$oaKQ; zALf?^GqVDOZwg-r(v<5$Ji!I)e&b=WQY5#6Jvea&Up0_9#ETYn@e-@wd|3$GDeW&+ z&&Tr>k%xT+zkv%J;xJvp^7}zRZUdQ4LcC9}u9*1b#)@;{Ky9w5%xU3i5_4$dV_*ai zRZYoFB*V7E7p51lW~svn&O`%3UclKVXT6&e@JozW#UWN;Wh>a23fLk*SxB69EV!AwVH!wF$#y%!JGsJ96?|SU64jM=hN^Wh+?k2n6 zl^~>9-U0)I4V>*}5)ow6+eqkJ#GC|-W^3+s1>1WduE$q4 zOW}jlmMdt$w59*fCFnIu!>9~pVq-^zb_~+eU3*3+AP(Q_DnR~`+bh+6zmAVW@WM3yGOz{Ti1X* zt?z0B1E^=x6~q?2zRJ8vx$~Rf|?djVdFX;G94!%+q-3I#hYpALy{yTG7YhzB^Xhc9kJW=n1I*r zBJt+F+7l^cW)6viI8g~Xq?P$3Jd{^fy&QbQzb7gM#}(iZ!(m69jBYD3;sKh0@YlCs zQb>@@HBgX$$_H1}3KpDQ6E?JBRr5A!8kLkB6I!hiMl%kmtk|+1+w=Mg0g@Ijtg_@X zl;zlrCU1b<55myW@Qbm&qicZS!oQ}8u#yD6nGP1@8^%@g)HrOe3Wldzv07aP77&R(V}8RUP;Q6!_3fnZC(rU%7r@BM@jFil}&8 zsx)Qx6Q9%FFt*TaQKqYeH~UBs2g&P(aH$iy_!8WHtoBv524QatE=3FYM@~ zfjgy`;FC|17k9Dtp3d3Xeqv|m-rUZHWPZA{HI|W%MLi%SArqO`%5$3DsyMomqJ!%R zJsS>(&+1=4^S|$gLn*CjvLztE)Q6L>sk3Cf1}^5qN*gAbYV$a0_;e7lEuZ>T!(><{S;R8&@}FdGf=YZbq$@;Z27SRDYuXim#bjC(Z$xEoqwo*GC@ z$uGm1HlcX;vgl2uy+Ot?%Ie9v{as}f!vbeOyq%?w#7wYAL5KV-z&GIcMLuyjo(8!Nms6t}P- zH-NGZ<#LS=TLDrYWYN^P6GSfpKSYGWzA*>i1QyW+dmWWn{M?p=L2WIU&{Sy;w20|k!)g@?XRr24J; z;{37Ff}gwo_za>))N51dwVJ3~s17%W{Yw0-a4yV_<3_Bb*6F6Klqw|jsP6L8ROsN^Wmxbe01_oc=h!1`1Gwj|K;f9^ZN_% zaF2L*IQ+izf>!BXblag zddF|b8>?WC`|k6}+Ve@@-7B&axrzD32Hb)RxC19=@UPW2<`~_MX9M-WqwN>{0YiPQ z#@jC&fbh=V6z@#N+js|IZWQvocXf5bCpVi^JPeMLKk%FtL09(=$R-@qy_WEr97{5b~2;4ImM+1B_)HCYkYLq+%->AnaBoHUBZ}kJtXO7Rn_&7BtfiT*HcmmY+kw^lBTeFyX!e= z5xZ4g&!2zc`3v`XOPWA#bWWGEO{ zDaX8SmrP6)9Qab>57Tmrb+7L^R>|Fvl!dcv0(MiFKg2nNh{w^5B=c>ylQc>LOqJMl zYZJNjo900jNjZi@!GSz*!f*q|O}bOHg6c@Jb(BC}q*NF-x#=ujWmFZf^o$GWX9~e$ z1b+xbsSs+?EVhQhAZ*P`M3>3kETVFHlOuh}aQN>2nnaBCaD_4IVv2k~*=tsbg{88o z;4~K63912y8Uf}>ni*J%t?!hEze4GUa_r&yvP940jKIEoSIHn3_VOT&trR3h}Xm@>Doi0{D6 zR;35z(|FCryL*o$%ksH zJEn?2?bh>!GLRgjz>%51Rwt0pQu~Q(U>TzKX?+JJO0ip5=`1I%XsC)oyKY=(PB=y5 zL~kYneL64e`Jdut)uiB9r()(rt%oc3vT+>+&<^_?8c69y0&pJ8^*F=6wE4#u?;Ak& z_I6(m;bWZ~Tpphd$9+dzzoM*Ly9+9p)}RKAw|_f)b9`tDskUv|t)aH+q+;LbYmU0-=TE_-Kr0%4J-}>|~nR zy7A&>D8JEU&m}pS`n1dftEM?A>NA*-A--y(Q|A&51f`h*!%kTDEzPKd`=3>CW4@epyolqy^G}HbS)(Y(#hjl&QY#+x2M%GCNvGSNk&9rvg4}0U0 zP4D(wH2q+_Pj=(k5w|i&bah{rL{;xJ4Qq`?f9ml1=V05ss((O20;YyLvWS*W_Fs6l z$x~f&Y3R#?xqrv>{IRxeRyX7W?Vl!LZoHwPMgUYdnA+elfzg9-XeT zXMI|XL#}b)lWRx5cT7r;7~u3`_~!kE(^+FRB10+j=61y?)ePM_1F$DEjK9*7sHlEC z>vB9=S|~Jae4v7nR!)_gEfUeNijrvsWZcS%eB5|3FZ`@v`U(t0qHMca9vKAQSgzK> z3)`CIGRiMUC#t7^A;XQZaK086QhZNwp&0bKyUS_9vK3wz$Ch8=9($?cVH;}qBxvqS zw@%?snYT38Js~G8f@%dr*MVlQd+Fas%W~S|m}o{VbB&?( zSTJ4hePxF)nCrfrl%OtV61qmOlPS90OX?y`uhc5`16>_?^zTg5?yTh28#)$6`j>El zb{wX&A*rDjx7n(^IOY^waT}1;nN`1^?O-%XAHQWI8i|4@OrE{?6Wu!gjIhT?2bZ#H zA%0I3$N>AfH5sV0%dMZIWV@9SG0%QeGB>3I|U>k3dvRr?% z!{fIOPcPP=?b5W+4UxKh-|_f49{+HDbkSIoT}uZNFYx}UA5a1)U#vgg!{dJpg`3x( zzQohR^R3ghCu5@Q<@@uC;bgS_kSKR{*}}04yD<^+?{7NRIVR#A1ZkG2562gW>u*pi z!cdh79rQ?CRY+A5qYLqMUM!l^9ye+#Xw-n?gh^hzF)QS@nY@wI zn>%Pw9i_Ya(PJ2hiN7uYpP`Od-S54 zDkVp0dwcYVo_2%Zr zi}KWi3YoP|8*pO+VN6(5$6Gxn@jP9@kYhX*BE6_64FKTRgpsgM=Ja9-VvLDfSCRTb z@0b&6^!=f&VIMpCea(qdJ$DEClAc?4`r(X~ajGW2ZBM*rqQ7~L%SBHnau{|hPO=SD z8&yCjMy-34m2@viLc;`EMOpxB!{l{YQCfK$9TauTsIL`X z9#wY3=kvw!82=`pm8#01s!;~D$Owt4X!W0CP(#4<%=$pV9!)2=-_CjbW`qHR} z`X)8m;g}DVIBpM9opW%@1hH!Z@n&fT#oNch>OtR8`sT0L(3R&$-ZCmYXoT+-djk=( zq-2OQJ*SiWcP=^H`cM7}zJ#U_7n~8QMYQA$fwKyAmzKaWUzkJBq|mwm$vTJ_kkxdcHbX7-Adyy8g^r!3&fefnTfu*pvyOI5anwavU*YWJTv8y$Xtqv*&>55 zkeukC&i)WOt$H8i$E4g-u3gZ%k*jE>=&W1AajLd#R<)?QOR6eSg-R%kLXoz`sm^6~ zY^l3-rZWDjdDw5zuA&f2agoGUjA&i1cgd9(DYaNOHpQs4EMV77Gizbayj43&U57Q4 zhc=p|^??nnfmF)X7=0?{5l`7llN`e2rvc}x>`|4bhbRQ*Gdz?~)Hmk1I44YQc%bB2 z=$;g5?5t{wZuZN`)zxej-+{v$XTX`^e;t2Pn~uV!Gj4nk7RqO+_J)MBWY31C3z-sp z=BS#TvULJzsLqK6#{24oVsPxzYT-{Ig=eWZ-h>lZrT6y7FZW(lZI^O!kDL3W;f>rM z3p01{=OsdX`VGW-L1?Qw6f zKTxZcHy(}lhU3wA`zepEzI=Ko=GPaXQLwln-AS>)&)yp*6I~$S$D}W1^NUN5IoXWnM&bvcL%2Bnc=7KHRC3Db?6{tr? z4xRX&vUCG;fUNd;7F_A#gVt3iN@auSyfw=YD7cEu`6jFN2@apBU23}mE?-oh-Z5*w zN{iULw;F{;(Q|G~zjZ|Hxh=_DV(5ts#55AX6{7mNV5;Bs|2J4CN~WWa+!na zf-&qs$qVOtD4^eQ)rVYjm4OH8(k>|HsOY#cE-DXwLO^(M`M8L9h`F2k8>|a2fys& zNBzq9#Wsg3Z%;w=eMH5JG%x<@3s%&(`aOgGJ6)Z~N?MK&bMn1DxhQv5-E)yieR&t> zPNsJX%mWQ*PwKC2{5_$$and4x8S>jODpEd9Yfmi88Krwr zya&JbwsjH7BF9X$#zEiF-2-}V@yu3{!SO#fXXY}p(G#1q$6TpX-QLKhQTNh|>gIvU zN#JpAoynd$Wj$&We$D&*s(McM9XV59+8a#eJurvzu!~>GSCo!nhu_vnTWGW63SB&d z6Pi{zTRzCPH1!|JJyFy{O=8kQH3J@enTt8&eK7Ze*b)E7S z!Zk6?>-bJmLlibj15_iF;tl(Gq3fG?_<*x}!-J&0qz21-X77K@?Pje={|4e>iWXl) zE5T~LEbbTPD$Kk(BS@8FCX;jDkO{mBr|W|aVQy<|)u~Uyuf~AjI8&H3dwpcc-i=t~ zI-WV(-MG?)cfPtkaeWbk8fYWvtZrGA8yBHxxHFKV>m1gJ`Ic-}PR7jk5VQ$YU6EOx z5ORy^6~XTBIB@%L5{{!~d?iK+8 z+q)+|=zCJR;1j1+WrQyLyK*%vq-$2xuZsBkD@e4tAFElRQDtI$r+<}nH7i{@72Ly?EU518lFAlADhXuXPYEdnYk#A zK-H(U?c1Zh3DKbjMEGfpq)t~%KTc;|Ex-muhqz%;?%#=4s;_W zv`nnTfp(on)aY>qNET*~{$>ku|5s77wYXPL=cGLKm8temL@8LItGb1c$k0oeRtnx1 zosPwqXV3c1I-#raLa(vhzDnFaKn8OqV6FgW4?o&K6y+6r0hGYF{Xk|I-Rf|bE*ss)GOTnzd@UA6Fq|J$;px z38&=%qrMP8YK7Bs_Up*=;QbEApUmaVaDOVTL~oW1C#2!hcTRAsK0sS0;k&SGdS9Jq zdg$ip*X{{XJ;XfUNWZ68pFS6Abw*rSU-#fPz|G1eFn$~9c|jrhhMNRKxynL}DqmRQ zsgI-pY(J=5gk4hvwBIMMoHyXPf~$LEK>i6D&2-k29|KcN>Z`xJR&U=QwU|d>3B^cc zd-z9Ho5oOwD;zdpe#CIVwH`G==u1YtSB<(yojG%^E$_azuHgw79>l^RU}j(9&}K`_%%_yGJe5lGG#Ah`WyW62j8E@T)HldEyF5B!#$vAOJ6JxM-sfuq{~uuD JJiN_e008Aa$sPaz literal 0 HcmV?d00001 From 37db93e4260dfcb90ac1553cf096a35ada1825ca Mon Sep 17 00:00:00 2001 From: Rob Napier Date: Sat, 12 Sep 2020 15:02:00 -0400 Subject: [PATCH 50/89] Review comments from pbkdf2.zig Move block definitions inside while loop. Use usize for offset. (This still crashes on overflow) Remove unneeded slice syntax. Add slow test for Very large dkLen --- lib/std/crypto/pbkdf2.zig | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig index fe00f7ee0..d0eaca1b0 100644 --- a/lib/std/crypto/pbkdf2.zig +++ b/lib/std/crypto/pbkdf2.zig @@ -95,11 +95,10 @@ pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: // // DK = T_1 || T_2 || ... || T_l<0..r-1> - var prevBlock: [hLen]u8 = undefined; - var newBlock: [hLen]u8 = undefined; - var block: u32 = 0; // Spec limits to u32 while (block < l) : (block += 1) { + var prevBlock: [hLen]u8 = undefined; + var newBlock: [hLen]u8 = undefined; // U_1 = PRF (P, S || INT (i)) const blockIndex = mem.toBytes(mem.nativeToBig(u32, block + 1)); // Block index starts at 0001 @@ -109,15 +108,15 @@ pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: ctx.final(prevBlock[0..]); // Choose portion of DK to write into (T_n) and initialize - const offset = block * hLen; + const offset: usize = block * hLen; const blockLen = if (block != l - 1) hLen else r; var dkBlock = derivedKey[offset..(offset + blockLen)]; - mem.copy(u8, dkBlock[0..], prevBlock[0..dkBlock.len]); + mem.copy(u8, dkBlock, prevBlock[0..dkBlock.len]); var i: u32 = 1; while (i < rounds) : (i += 1) { // U_c = PRF (P, U_{c-1}) - Prf.create(newBlock[0..], prevBlock[0..], password); + Prf.create(&newBlock, prevBlock[0..], password); mem.copy(u8, prevBlock[0..], newBlock[0..]); // F (P, S, c, i) = U_1 \xor U_2 \xor ... \xor U_c @@ -225,3 +224,26 @@ test "RFC 6070 embedded NUL" { htest.assertEqual(expected, derivedKey[0..]); } + +test "Very large dkLen" { + // These iteration tests are slow so we always skip them. Results have been verified. + if (true) { + return error.SkipZigTest; + } + + const p = "password"; + const s = "salt"; + const c = 1; + const dkLen = 1 << 33; + + var derivedKey = try std.testing.allocator.alloc(u8, dkLen); + defer { + std.testing.allocator.free(derivedKey); + } + + pbkdf2(derivedKey, p, s, c, crypto.hash.Sha1); + + const expected = "0c60c80f961f0e71f3a9b524af6012062fe037a6"; + + htest.assertEqual(expected, derivedKey[0..]); +} From 3f450b7e931b76b20e36d2bab2e2d1fed3ee19ec Mon Sep 17 00:00:00 2001 From: Rob Napier Date: Sat, 12 Sep 2020 18:17:04 -0400 Subject: [PATCH 51/89] Replace Hash function with Prf. Correct offset bit-width. --- lib/std/crypto/pbkdf2.zig | 51 +++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig index d0eaca1b0..ceed9beb6 100644 --- a/lib/std/crypto/pbkdf2.zig +++ b/lib/std/crypto/pbkdf2.zig @@ -10,6 +10,14 @@ const debug = std.debug; const assert = debug.assert; const mem = std.mem; +//! PBKDF2 (Password-Based Key Derivation Function 2) is a specific Key Derivation Function, +//! intended to turn a weak, human generated password into a strong key, suitable for cryptographic +//! uses. It does this by salting and stretching the password. Salting injects non-secret random +//! data, so that identical passwords will be converted into unique keys. Stretching applies a +//! deliberately slow hashing function to frustrate brute-force guessing. +//! +//! PBKDF2 is defined in RFC 2898, and is a recommendation of NIST SP 800-132. + // RFC 2898 Section 5.2 // // FromSpec: @@ -38,19 +46,33 @@ const mem = std.mem; // Based on Apple's CommonKeyDerivation, based originally on code by Damien Bergamini. -pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: u32, comptime Hash: type) void { +/// Given a password, salt, iteration count (rounds), and a pseudo-random function, generates a +/// derived key in the provided buffer slice. +/// +/// derivedKey: Slice of appropriate size for generated key. Generally 16 or 32 bytes in length. +/// May be uninitialized. All bytes will be written. +/// Maximum size is (2^32 - 1) * Hash.digest_length +/// It is a programming error to pass buffer longer than the maximum size. +/// +/// password: Arbitrary sequence of bytes of any length, including empty. +/// +/// salt: Arbitrary sequence of bytes of any length, including empty. A common length is 8 bytes. +/// +/// rounds: Iteration count. Must be greater than 0. Common values range from 1,000 to 100,000. +/// +/// Prf: Pseudo-random function to use. The most common choice is std.crypto.auth.hmac.HmacSha256. +pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: u32, comptime Prf: type) void { assert(rounds >= 1); - const dkLen = derivedKey.len; - const hLen = Hash.digest_length; - const Prf = crypto.auth.hmac.Hmac(Hash); + const dkLen: u64 = derivedKey.len; + const hLen: u32 = Prf.mac_length; // Force type to ensure multiplications can't overflow // FromSpec: // // 1. If dkLen > (2^32 - 1) * hLen, output "derived key too long" and // stop. // - assert(dkLen > 0 and dkLen <= (1 << 32 - 1) * hLen); + assert(dkLen > 0 and dkLen <= @as(u64, 1 << 32 - 1) * hLen); // FromSpec: // @@ -108,7 +130,7 @@ pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: ctx.final(prevBlock[0..]); // Choose portion of DK to write into (T_n) and initialize - const offset: usize = block * hLen; + const offset: u64 = @as(u64, block) * hLen; const blockLen = if (block != l - 1) hLen else r; var dkBlock = derivedKey[offset..(offset + blockLen)]; mem.copy(u8, dkBlock, prevBlock[0..dkBlock.len]); @@ -138,7 +160,7 @@ test "RFC 6070 one iteration" { var derivedKey: [dkLen]u8 = undefined; - pbkdf2(&derivedKey, p, s, c, crypto.hash.Sha1); + pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); const expected = "0c60c80f961f0e71f3a9b524af6012062fe037a6"; @@ -153,7 +175,7 @@ test "RFC 6070 two iterations" { var derivedKey: [dkLen]u8 = undefined; - pbkdf2(&derivedKey, p, s, c, crypto.hash.Sha1); + pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); const expected = "ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957"; @@ -168,7 +190,7 @@ test "RFC 6070 4096 iterations" { var derivedKey: [dkLen]u8 = undefined; - pbkdf2(&derivedKey, p, s, c, crypto.hash.Sha1); + pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); const expected = "4b007901b765489abead49d926f721d065a429c1"; @@ -188,7 +210,7 @@ test "RFC 6070 16,777,216 iterations" { var derivedKey = [_]u8{0} ** dkLen; - pbkdf2(&derivedKey, p, s, c, crypto.hash.Sha1); + pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); const expected = "eefe3d61cd4da4e4e9945b3d6ba2158c2634e984"; @@ -203,7 +225,7 @@ test "RFC 6070 multi-block salt and password" { var derivedKey: [dkLen]u8 = undefined; - pbkdf2(&derivedKey, p, s, c, crypto.hash.Sha1); + pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); const expected = "3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038"; @@ -218,7 +240,7 @@ test "RFC 6070 embedded NUL" { var derivedKey: [dkLen]u8 = undefined; - pbkdf2(&derivedKey, p, s, c, crypto.hash.Sha1); + pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); const expected = "56fa6aa75548099dcc37d7f03425e0c3"; @@ -226,11 +248,10 @@ test "RFC 6070 embedded NUL" { } test "Very large dkLen" { - // These iteration tests are slow so we always skip them. Results have been verified. + // This test allocates 8GB of memory and is expected to take several hours to run. if (true) { return error.SkipZigTest; } - const p = "password"; const s = "salt"; const c = 1; @@ -241,7 +262,7 @@ test "Very large dkLen" { std.testing.allocator.free(derivedKey); } - pbkdf2(derivedKey, p, s, c, crypto.hash.Sha1); + pbkdf2(derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); const expected = "0c60c80f961f0e71f3a9b524af6012062fe037a6"; From 17156e1775f60893b4c5040d7a0073659ea9157d Mon Sep 17 00:00:00 2001 From: Rob Napier Date: Sat, 12 Sep 2020 18:33:53 -0400 Subject: [PATCH 52/89] pbkdf2 "very large dklen test" should just check for crashes --- lib/std/crypto/pbkdf2.zig | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig index ceed9beb6..6cf1e8a75 100644 --- a/lib/std/crypto/pbkdf2.zig +++ b/lib/std/crypto/pbkdf2.zig @@ -59,8 +59,10 @@ const mem = std.mem; /// salt: Arbitrary sequence of bytes of any length, including empty. A common length is 8 bytes. /// /// rounds: Iteration count. Must be greater than 0. Common values range from 1,000 to 100,000. +/// Larger iteration counts improve security by increasing the time required to compute +/// the derivedKey. It is common to tune this parameter to achieve approximately 100ms. /// -/// Prf: Pseudo-random function to use. The most common choice is std.crypto.auth.hmac.HmacSha256. +/// Prf: Pseudo-random function to use. A common choice is std.crypto.auth.hmac.HmacSha256. pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: u32, comptime Prf: type) void { assert(rounds >= 1); @@ -263,8 +265,5 @@ test "Very large dkLen" { } pbkdf2(derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); - - const expected = "0c60c80f961f0e71f3a9b524af6012062fe037a6"; - - htest.assertEqual(expected, derivedKey[0..]); + // Just verify this doesn't crash with an overflow } From b043a318893ad4e8de1e376ba16be6e3ed201a76 Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Sat, 12 Sep 2020 23:45:35 -0700 Subject: [PATCH 53/89] added helper definitions --- lib/std/os/linux/bpf.zig | 1 + lib/std/os/linux/bpf/helpers.zig | 183 +++++++++++++++++++++++++++++++ lib/std/os/linux/bpf/kern.zig | 39 +++++++ 3 files changed, 223 insertions(+) create mode 100644 lib/std/os/linux/bpf/helpers.zig create mode 100644 lib/std/os/linux/bpf/kern.zig diff --git a/lib/std/os/linux/bpf.zig b/lib/std/os/linux/bpf.zig index be9f599d9..44c938feb 100644 --- a/lib/std/os/linux/bpf.zig +++ b/lib/std/os/linux/bpf.zig @@ -12,6 +12,7 @@ const expectError = std.testing.expectError; const expect = std.testing.expect; pub const btf = @import("bpf/btf.zig"); +pub const kern = @import("bpf/kern.zig"); // instruction classes pub const LD = 0x00; diff --git a/lib/std/os/linux/bpf/helpers.zig b/lib/std/os/linux/bpf/helpers.zig new file mode 100644 index 000000000..0a4988546 --- /dev/null +++ b/lib/std/os/linux/bpf/helpers.zig @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. +const kern = @import("kern.zig"); + +// opting to not use 'usingnamespace' because of future name collisions +const MapDef = kern.MapDef; +const SkBuff = kern.SkBuff; +const TunnelKey = kern.TunnelKey; +const XdpMd = kern.XdpMd; +const BpfSockAddr = kern.BpfSockAddr; +const FibLookup = kern.FibLookup; +const PerfEventData = kern.PerfEventData; +const PerfEventValue = kern.PerfEventValue; +const PidNsInfo = kern.PidNsInfo; +const SkLookup = kern.SkLookup; +const SkMsgBuff = kern.SkMsgBuff; +const SkReusePortMd = kern.SkReusePortMd; +const Sock = kern.Sock; +const SockAddr = kern.SockAddr; +const SockOps = kern.SockOps; +const SockTuple = kern.SockTuple; +const SpinLock = kern.SpinLock; +const SysCtl = kern.SysCtl; +const Tcp6Sock = kern.Tcp6Sock; +const TcpRequestSock = kern.TcpRequestSock; +const TcpSock = kern.TcpSock; +const TcpTimewaitSock = kern.TcpTimewaitSock; +const Udp6Sock = kern.Udp6Sock; +const XfrmState = kern.XfrmState; + +// in BPF, all the helper calls +// TODO: when https://github.com/ziglang/zig/issues/1717 is here, make a nice +// function that uses the Helper enum +// +// Note, these function signatures were created from documentation found in +// '/usr/include/linux/bpf.h' +pub const map_lookup_elem = @intToPtr(fn (map: *const MapDef, key: ?*const c_void) ?*c_void, 1); +pub const map_update_elem = @intToPtr(fn (map: *const MapDef, key: ?*const c_void, value: ?*const c_void, flags: u64) c_long, 2); +pub const map_delete_elem = @intToPtr(fn (map: *const MapDef, key: ?*const c_void) c_long, 3); +pub const probe_read = @intToPtr(fn (dst: ?*c_void, size: u32, unsafe_ptr: ?*const c_void) c_long, 4); +pub const ktime_get_ns = @intToPtr(fn () u64, 5); +pub const trace_printk = @intToPtr(fn (fmt: [*:0]const u8, fmt_size: u32, arg1: u64, arg2: u64, arg3: u64) c_long, 6); +pub const get_prandom_u32 = @intToPtr(fn () u32, 7); +pub const get_smp_processor_id = @intToPtr(fn () u32, 8); +pub const skb_store_bytes = @intToPtr(fn (skb: *SkBuff, offset: u32, from: ?*const c_void, len: u32, flags: u64) c_long, 9); +pub const l3_csum_replace = @intToPtr(fn (skb: *SkBuff, offset: u32, from: u64, to: u64, size: u64) c_long, 10); +pub const l4_csum_replace = @intToPtr(fn (skb: *SkBuff, offset: u32, from: u64, to: u64, flags: u64) c_long, 11); +pub const tail_call = @intToPtr(fn (ctx: ?*c_void, prog_array_map: *const MapDef, index: u32) c_long, 12); +pub const clone_redirect = @intToPtr(fn (skb: *SkBuff, ifindex: u32, flags: u64) c_long, 13); +pub const get_current_pid_tgid = @intToPtr(fn () u64, 14); +pub const get_current_uid_gid = @intToPtr(fn () u64, 15); +pub const get_current_comm = @intToPtr(fn (buf: ?*c_void, size_of_buf: u32) c_long, 16); +pub const get_cgroup_classid = @intToPtr(fn (skb: *SkBuff) u32, 17); +// Note vlan_proto is big endian +pub const skb_vlan_push = @intToPtr(fn (skb: *SkBuff, vlan_proto: u16, vlan_tci: u16) c_long, 18); +pub const skb_vlan_pop = @intToPtr(fn (skb: *SkBuff) c_long, 19); +pub const skb_get_tunnel_key = @intToPtr(fn (skb: *SkBuff, key: *TunnelKey, size: u32, flags: u64) c_long, 20); +pub const skb_set_tunnel_key = @intToPtr(fn (skb: *SkBuff, key: *TunnelKey, size: u32, flags: u64) c_long, 21); +pub const perf_event_read = @intToPtr(fn (map: *const MapDef, flags: u64) u64, 22); +pub const redirect = @intToPtr(fn (ifindex: u32, flags: u64) c_long, 23); +pub const get_route_realm = @intToPtr(fn (skb: *SkBuff) u32, 24); +pub const perf_event_output = @intToPtr(fn (ctx: ?*c_void, map: *const MapDef, flags: u64, data: ?*c_void, size: u64) c_long, 25); +pub const skb_load_bytes = @intToPtr(fn (skb: ?*c_void, offset: u32, to: ?*c_void, len: u32) c_long, 26); +pub const get_stackid = @intToPtr(fn (ctx: ?*c_void, map: *const MapDef, flags: u64) c_long, 27); +// from and to point to __be32 +pub const csum_diff = @intToPtr(fn (from: *u32, from_size: u32, to: *u32, to_size: u32, seed: u32) i64, 28); +pub const skb_get_tunnel_opt = @intToPtr(fn (skb: *SkBuff, opt: ?*c_void, size: u32) c_long, 29); +pub const skb_set_tunnel_opt = @intToPtr(fn (skb: *SkBuff, opt: ?*c_void, size: u32) c_long, 30); +// proto is __be16 +pub const skb_change_proto = @intToPtr(fn (skb: *SkBuff, proto: u16, flags: u64) c_long, 31); +pub const skb_change_type = @intToPtr(fn (skb: *SkBuff, skb_type: u32) c_long, 32); +pub const skb_under_cgroup = @intToPtr(fn (skb: *SkBuff, map: ?*const c_void, index: u32) c_long, 33); +pub const get_hash_recalc = @intToPtr(fn (skb: *SkBuff) u32, 34); +pub const get_current_task = @intToPtr(fn () u64, 35); +pub const probe_write_user = @intToPtr(fn (dst: ?*c_void, src: ?*const c_void, len: u32) c_long, 36); +pub const current_task_under_cgroup = @intToPtr(fn (map: *const MapDef, index: u32) c_long, 37); +pub const skb_change_tail = @intToPtr(fn (skb: *SkBuff, len: u32, flags: u64) c_long, 38); +pub const skb_pull_data = @intToPtr(fn (skb: *SkBuff, len: u32) c_long, 39); +pub const csum_update = @intToPtr(fn (skb: *SkBuff, csum: u32) i64, 40); +pub const set_hash_invalid = @intToPtr(fn (skb: *SkBuff) void, 41); +pub const get_numa_node_id = @intToPtr(fn () c_long, 42); +pub const skb_change_head = @intToPtr(fn (skb: *SkBuff, len: u32, flags: u64) c_long, 43); +pub const xdp_adjust_head = @intToPtr(fn (xdp_md: *XdpMd, delta: c_int) c_long, 44); +pub const probe_read_str = @intToPtr(fn (dst: ?*c_void, size: u32, unsafe_ptr: ?*const c_void) c_long, 45); +pub const get_socket_cookie = @intToPtr(fn (ctx: ?*c_void) u64, 46); +pub const get_socket_uid = @intToPtr(fn (skb: *SkBuff) u32, 47); +pub const set_hash = @intToPtr(fn (skb: *SkBuff, hash: u32) c_long, 48); +pub const setsockopt = @intToPtr(fn (bpf_socket: *SockOps, level: c_int, optname: c_int, optval: ?*c_void, optlen: c_int) c_long, 49); +pub const skb_adjust_room = @intToPtr(fn (skb: *SkBuff, len_diff: i32, mode: u32, flags: u64) c_long, 50); +pub const redirect_map = @intToPtr(fn (map: *const MapDef, key: u32, flags: u64) c_long, 51); +pub const sk_redirect_map = @intToPtr(fn (skb: *SkBuff, map: *const MapDef, key: u32, flags: u64) c_long, 52); +pub const sock_map_update = @intToPtr(fn (skops: *SockOps, map: *const MapDef, key: ?*c_void, flags: u64) c_long, 53); +pub const xdp_adjust_meta = @intToPtr(fn (xdp_md: *XdpMd, delta: c_int) c_long, 54); +pub const perf_event_read_value = @intToPtr(fn (map: *const MapDef, flags: u64, buf: *PerfEventValue, buf_size: u32) c_long, 55); +pub const perf_prog_read_value = @intToPtr(fn (ctx: *PerfEventData, buf: *PerfEventValue, buf_size: u32) c_long, 56); +pub const getsockopt = @intToPtr(fn (bpf_socket: ?*c_void, level: c_int, optname: c_int, optval: ?*c_void, optlen: c_int) c_long, 57); +pub const override_return = @intToPtr(fn (regs: *PtRegs, rc: u64) c_long, 58); +pub const sock_ops_cb_flags_set = @intToPtr(fn (bpf_sock: *SockOps, argval: c_int) c_long, 59); +pub const msg_redirect_map = @intToPtr(fn (msg: *SkMsgMd, map: *const MapDef, key: u32, flags: u64) c_long, 60); +pub const msg_apply_bytes = @intToPtr(fn (msg: *SkMsgMd, bytes: u32) c_long, 61); +pub const msg_cork_bytes = @intToPtr(fn (msg: *SkMsgMd, bytes: u32) c_long, 62); +pub const msg_pull_data = @intToPtr(fn (msg: *SkMsgMd, start: u32, end: u32, flags: u64) c_long, 63); +pub const bind = @intToPtr(fn (ctx: *BpfSockAddr, addr: *SockAddr, addr_len: c_int) c_long, 64); +pub const xdp_adjust_tail = @intToPtr(fn (xdp_md: *XdpMd, delta: c_int) c_long, 65); +pub const skb_get_xfrm_state = @intToPtr(fn (skb: *SkBuff, index: u32, xfrm_state: *XfrmState, size: u32, flags: u64) c_long, 66); +pub const get_stack = @intToPtr(fn (ctx: ?*c_void, buf: ?*c_void, size: u32, flags: u64) c_long, 67); +pub const skb_load_bytes_relative = @intToPtr(fn (skb: ?*const c_void, offset: u32, to: ?*c_void, len: u32, start_header: u32) c_long, 68); +pub const fib_lookup = @intToPtr(fn (ctx: ?*c_void, params: *FibLookup, plen: c_int, flags: u32) c_long, 69); +pub const sock_hash_update = @intToPtr(fn (skops: *SockOps, map: *const MapDef, key: ?*c_void, flags: u64) c_long, 70); +pub const msg_redirect_hash = @intToPtr(fn (msg: *SkMsgMd, map: *const MapDef, key: ?*c_void, flags: u64) c_long, 71); +pub const sk_redirect_hash = @intToPtr(fn (skb: *SkBuff, map: *const MapDef, key: ?*c_void, flags: u64) c_long, 72); +pub const lwt_push_encap = @intToPtr(fn (skb: *SkBuff, typ: u32, hdr: ?*c_void, len: u32) c_long, 73); +pub const lwt_seg6_store_bytes = @intToPtr(fn (skb: *SkBuff, offset: u32, from: ?*const c_void, len: u32) c_long, 74); +pub const lwt_seg6_adjust_srh = @intToPtr(fn (skb: *SkBuff, offset: u32, delta: i32) c_long, 75); +pub const lwt_seg6_action = @intToPtr(fn (skb: *SkBuff, action: u32, param: ?*c_void, param_len: u32) c_long, 76); +pub const rc_repeat = @intToPtr(fn (ctx: ?*c_void) c_long, 77); +pub const rc_keydown = @intToPtr(fn (ctx: ?*c_void, protocol: u32, scancode: u64, toggle: u32) c_long, 78); +pub const skb_cgroup_id = @intToPtr(fn (skb: *SkBuff) u64, 79); +pub const get_current_cgroup_id = @intToPtr(fn () u64, 80); +pub const get_local_storage = @intToPtr(fn (map: ?*c_void, flags: u64) ?*c_void, 81); +pub const sk_select_reuseport = @intToPtr(fn (reuse: *SkReusePortMd, map: *const MapDef, key: ?*c_void, flags: u64) c_long, 82); +pub const skb_ancestor_cgroup_id = @intToPtr(fn (skb: *SkBuff, ancestor_level: c_int) u64, 83); +pub const sk_lookup_tcp = @intToPtr(fn (ctx: ?*c_void, tuple: *SockTuple, tuple_size: u32, netns: u64, flags: u64) ?*Sock, 84); +pub const sk_lookup_udp = @intToPtr(fn (ctx: ?*c_void, tuple: *SockTuple, tuple_size: u32, netns: u64, flags: u64) ?*Sock, 85); +pub const sk_release = @intToPtr(fn (sock: *Sock) c_long, 86); +pub const map_push_elem = @intToPtr(fn (map: *const MapDef, value: ?*const c_void, flags: u64) c_long, 87); +pub const map_pop_elem = @intToPtr(fn (map: *const MapDef, value: ?*c_void) c_long, 88); +pub const map_peek_elem = @intToPtr(fn (map: *const MapDef, value: ?*c_void) c_long, 89); +pub const msg_push_data = @intToPtr(fn (msg: *SkMsgMd, start: u32, len: u32, flags: u64) c_long, 90); +pub const msg_pop_data = @intToPtr(fn (msg: *SkMsgMd, start: u32, len: u32, flags: u64) c_long, 91); +pub const rc_pointer_rel = @intToPtr(fn (ctx: ?*c_void, rel_x: i32, rel_y: i32) c_long, 92); +pub const spin_lock = @intToPtr(fn (lock: *SpinLock) c_long, 93); +pub const spin_unlock = @intToPtr(fn (lock: *SpinLock) c_long, 94); +pub const sk_fullsock = @intToPtr(fn (sk: *Sock) ?*SkFullSock, 95); +pub const tcp_sock = @intToPtr(fn (sk: *Sock) ?*TcpSock, 96); +pub const skb_ecn_set_ce = @intToPtr(fn (skb: *SkBuff) c_long, 97); +pub const get_listener_sock = @intToPtr(fn (sk: *Sock) ?*Sock, 98); +pub const skc_lookup_tcp = @intToPtr(fn (ctx: ?*c_void, tuple: *SockTuple, tuple_size: u32, netns: u64, flags: u64) ?*Sock, 99); +pub const tcp_check_syncookie = @intToPtr(fn (sk: *Sock, iph: ?*c_void, iph_len: u32, th: *TcpHdr, th_len: u32) c_long, 100); +pub const sysctl_get_name = @intToPtr(fn (ctx: *SysCtl, buf: ?*u8, buf_len: c_ulong, flags: u64) c_long, 101); +pub const sysctl_get_current_value = @intToPtr(fn (ctx: *SysCtl, buf: ?*u8, buf_len: c_ulong) c_long, 102); +pub const sysctl_get_new_value = @intToPtr(fn (ctx: *SysCtl, buf: ?*u8, buf_len: c_ulong) c_long, 103); +pub const sysctl_set_new_value = @intToPtr(fn (ctx: *SysCtl, buf: ?*const u8, buf_len: c_ulong) c_long, 104); +pub const strtol = @intToPtr(fn (buf: *const u8, buf_len: c_ulong, flags: u64, res: *c_long) c_long, 105); +pub const strtoul = @intToPtr(fn (buf: *const u8, buf_len: c_ulong, flags: u64, res: *c_ulong) c_long, 106); +pub const sk_storage_get = @intToPtr(fn (map: *const MapDef, sk: *Sock, value: ?*c_void, flags: u64) ?*c_void, 107); +pub const sk_storage_delete = @intToPtr(fn (map: *const MapDef, sk: *Sock) c_long, 108); +pub const send_signal = @intToPtr(fn (sig: u32) c_long, 109); +pub const tcp_gen_syncookie = @intToPtr(fn (sk: *Sock, iph: ?*c_void, iph_len: u32, th: *TcpHdr, th_len: u32) i64, 110); +pub const skb_output = @intToPtr(fn (ctx: ?*c_void, map: *const MapDef, flags: u64, data: ?*c_void, size: u64) c_long, 111); +pub const probe_read_user = @intToPtr(fn (dst: ?*c_void, size: u32, unsafe_ptr: ?*const c_void) c_long, 112); +pub const probe_read_kernel = @intToPtr(fn (dst: ?*c_void, size: u32, unsafe_ptr: ?*const c_void) c_long, 113); +pub const probe_read_user_str = @intToPtr(fn (dst: ?*c_void, size: u32, unsafe_ptr: ?*const c_void) c_long, 114); +pub const probe_read_kernel_str = @intToPtr(fn (dst: ?*c_void, size: u32, unsafe_ptr: ?*const c_void) c_long, 115); +pub const tcp_send_ack = @intToPtr(fn (tp: ?*c_void, rcv_nxt: u32) c_long, 116); +pub const send_signal_thread = @intToPtr(fn (sig: u32) c_long, 117); +pub const jiffies64 = @intToPtr(fn () u64, 118); +pub const read_branch_records = @intToPtr(fn (ctx: *PerfEventData, buf: ?*c_void, size: u32, flags: u64) c_long, 119); +pub const get_ns_current_pid_tgid = @intToPtr(fn (dev: u64, ino: u64, nsdata: *PidNsInfo, size: u32) c_long, 120); +pub const xdp_output = @intToPtr(fn (ctx: ?*c_void, map: *const MapDef, flags: u64, data: ?*c_void, size: u64) c_long, 121); +pub const get_netns_cookie = @intToPtr(fn (ctx: ?*c_void) u64, 122); +pub const get_current_ancestor_cgroup_id = @intToPtr(fn (ancestor_level: c_int) u64, 123); +pub const sk_assign = @intToPtr(fn (skb: *SkBuff, sk: *Sock, flags: u64) c_long, 124); +pub const ktime_get_boot_ns = @intToPtr(fn () u64, 125); +pub const seq_printf = @intToPtr(fn (m: *SeqFile, fmt: ?*const u8, fmt_size: u32, data: ?*const c_void, data_len: u32) c_long, 126); +pub const seq_write = @intToPtr(fn (m: *SeqFile, data: ?*const u8, len: u32) c_long, 127); +pub const sk_cgroup_id = @intToPtr(fn (sk: *BpfSock) u64, 128); +pub const sk_ancestor_cgroup_id = @intToPtr(fn (sk: *BpfSock, ancestor_level: c_long) u64, 129); +pub const ringbuf_output = @intToPtr(fn (ringbuf: ?*c_void, data: ?*c_void, size: u64, flags: u64) ?*c_void, 130); +pub const ringbuf_reserve = @intToPtr(fn (ringbuf: ?*c_void, size: u64, flags: u64) ?*c_void, 131); +pub const ringbuf_submit = @intToPtr(fn (data: ?*c_void, flags: u64) void, 132); +pub const ringbuf_discard = @intToPtr(fn (data: ?*c_void, flags: u64) void, 133); +pub const ringbuf_query = @intToPtr(fn (ringbuf: ?*c_void, flags: u64) u64, 134); +pub const csum_level = @intToPtr(fn (skb: *SkBuff, level: u64) c_long, 134); +pub const skc_to_tcp6_sock = @intToPtr(fn (sk: ?*c_void) ?*Tcp6Sock, 135); +pub const skc_to_tcp_sock = @intToPtr(fn (sk: ?*c_void) ?*TcpSock, 136); +pub const skc_to_tcp_timewait_sock = @intToPtr(fn (sk: ?*c_void) ?*TcpTimewaitSock, 137); +pub const skc_to_tcp_request_sock = @intToPtr(fn (sk: ?*c_void) ?*TcpRequestSock, 138); +pub const skc_to_udp6_sock = @intToPtr(fn (sk: ?*c_void) ?*Udp6Sock, 139); +pub const get_task_stack = @intToPtr(fn (task: ?*TaskStruct, buf: ?*c_void, size: u32, flags: u64) c_long, 140); diff --git a/lib/std/os/linux/bpf/kern.zig b/lib/std/os/linux/bpf/kern.zig new file mode 100644 index 000000000..6e9b51737 --- /dev/null +++ b/lib/std/os/linux/bpf/kern.zig @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. +const std = @import("../../../std.zig"); + +const in_bpf_program = switch (std.builtin.arch) { + .bpfel, .bpfeb => false, + else => true, +}; + +pub const helpers = if (in_bpf_program) @import("helpers.zig") else struct {}; + +// TODO: fill these in +pub const MapDef = packed struct {}; +pub const SkBuff = packed struct {}; +pub const TunnelKey = packed struct {}; +pub const XdpMd = packed struct {}; +pub const BpfSockAddr = packed struct {}; +pub const FibLookup = packed struct {}; +pub const PerfEventData = packed struct {}; +pub const PerfEventValue = packed struct {}; +pub const PidNsInfo = packed struct {}; +pub const SkLookup = packed struct {}; +pub const SkMsgBuff = packed struct {}; +pub const SkReusePortMd = packed struct {}; +pub const Sock = packed struct {}; +pub const SockAddr = packed struct {}; +pub const SockOps = packed struct {}; +pub const SockTuple = packed struct {}; +pub const SpinLock = packed struct {}; +pub const SysCtl = packed struct {}; +pub const Tcp6Sock = packed struct {}; +pub const TcpRequestSock = packed struct {}; +pub const TcpSock = packed struct {}; +pub const TcpTimewaitSock = packed struct {}; +pub const Udp6Sock = packed struct {}; +pub const XfrmState = packed struct {}; From 0f85b85acb0bd322ff7408c25dc09a84ff6621dc Mon Sep 17 00:00:00 2001 From: Rob Napier Date: Sun, 13 Sep 2020 09:59:36 -0400 Subject: [PATCH 54/89] Improve doc text --- lib/std/crypto/pbkdf2.zig | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig index 6cf1e8a75..2f9b72022 100644 --- a/lib/std/crypto/pbkdf2.zig +++ b/lib/std/crypto/pbkdf2.zig @@ -10,11 +10,11 @@ const debug = std.debug; const assert = debug.assert; const mem = std.mem; -//! PBKDF2 (Password-Based Key Derivation Function 2) is a specific Key Derivation Function, -//! intended to turn a weak, human generated password into a strong key, suitable for cryptographic -//! uses. It does this by salting and stretching the password. Salting injects non-secret random -//! data, so that identical passwords will be converted into unique keys. Stretching applies a -//! deliberately slow hashing function to frustrate brute-force guessing. +//! PBKDF2 (Password-Based Key Derivation Function 2) is intended to turn a weak, human generated +//! password into a strong key, suitable for cryptographic uses. It does this by salting and +//! stretching the password. Salting injects non-secret random data, so that identical passwords +//! will be converted into unique keys. Stretching applies a deliberately slow hashing function to +//! frustrate brute-force guessing. //! //! PBKDF2 is defined in RFC 2898, and is a recommendation of NIST SP 800-132. @@ -46,8 +46,7 @@ const mem = std.mem; // Based on Apple's CommonKeyDerivation, based originally on code by Damien Bergamini. -/// Given a password, salt, iteration count (rounds), and a pseudo-random function, generates a -/// derived key in the provided buffer slice. +/// Apply PBKDF2 to generate a key from a password. /// /// derivedKey: Slice of appropriate size for generated key. Generally 16 or 32 bytes in length. /// May be uninitialized. All bytes will be written. From 257c5b534839d75092909f604bb2663e883a290f Mon Sep 17 00:00:00 2001 From: Rob Napier Date: Sun, 13 Sep 2020 10:50:46 -0400 Subject: [PATCH 55/89] Explicitly reference std.crypto.kdf in test case --- lib/std/crypto.zig | 6 +++++- lib/std/crypto/pbkdf2.zig | 14 +++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig index 2b4294282..c375c0290 100644 --- a/lib/std/crypto.zig +++ b/lib/std/crypto.zig @@ -35,7 +35,11 @@ pub const onetimeauth = struct { pub const Poly1305 = @import("crypto/poly1305.zig").Poly1305; }; -/// Key derivation functions +/// A Key Derivation Function (KDF) is intended to turn a weak, human generated password into a +/// strong key, suitable for cryptographic uses. It does this by salting and stretching the +/// password. Salting injects non-secret random data, so that identical passwords will be converted +/// into unique keys. Stretching applies a deliberately slow hashing function to frustrate +/// brute-force guessing. pub const kdf = struct { pub const pbkdf2 = @import("crypto/pbkdf2.zig").pbkdf2; }; diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig index 2f9b72022..dfa6b1c02 100644 --- a/lib/std/crypto/pbkdf2.zig +++ b/lib/std/crypto/pbkdf2.zig @@ -10,14 +10,6 @@ const debug = std.debug; const assert = debug.assert; const mem = std.mem; -//! PBKDF2 (Password-Based Key Derivation Function 2) is intended to turn a weak, human generated -//! password into a strong key, suitable for cryptographic uses. It does this by salting and -//! stretching the password. Salting injects non-secret random data, so that identical passwords -//! will be converted into unique keys. Stretching applies a deliberately slow hashing function to -//! frustrate brute-force guessing. -//! -//! PBKDF2 is defined in RFC 2898, and is a recommendation of NIST SP 800-132. - // RFC 2898 Section 5.2 // // FromSpec: @@ -48,6 +40,8 @@ const mem = std.mem; /// Apply PBKDF2 to generate a key from a password. /// +/// PBKDF2 is defined in RFC 2898, and is a recommendation of NIST SP 800-132. +/// /// derivedKey: Slice of appropriate size for generated key. Generally 16 or 32 bytes in length. /// May be uninitialized. All bytes will be written. /// Maximum size is (2^32 - 1) * Hash.digest_length @@ -62,6 +56,8 @@ const mem = std.mem; /// the derivedKey. It is common to tune this parameter to achieve approximately 100ms. /// /// Prf: Pseudo-random function to use. A common choice is std.crypto.auth.hmac.HmacSha256. +/// +/// PBKDF2 is defined in RFC 2898, and is a recommendation of NIST SP 800-132. pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: u32, comptime Prf: type) void { assert(rounds >= 1); @@ -161,7 +157,7 @@ test "RFC 6070 one iteration" { var derivedKey: [dkLen]u8 = undefined; - pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); + std.crypto.kdf.pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); const expected = "0c60c80f961f0e71f3a9b524af6012062fe037a6"; From 8a1a40276fb1577f46b89e3aefc17f9e82a933d6 Mon Sep 17 00:00:00 2001 From: Rob Napier Date: Sun, 13 Sep 2020 11:08:06 -0400 Subject: [PATCH 56/89] Extract kdf.zig to provide namespace documentation --- lib/std/crypto.zig | 11 ++--------- lib/std/crypto/kdf.zig | 17 +++++++++++++++++ lib/std/crypto/pbkdf2.zig | 2 -- 3 files changed, 19 insertions(+), 11 deletions(-) create mode 100644 lib/std/crypto/kdf.zig diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig index c375c0290..64ec22894 100644 --- a/lib/std/crypto.zig +++ b/lib/std/crypto.zig @@ -35,14 +35,7 @@ pub const onetimeauth = struct { pub const Poly1305 = @import("crypto/poly1305.zig").Poly1305; }; -/// A Key Derivation Function (KDF) is intended to turn a weak, human generated password into a -/// strong key, suitable for cryptographic uses. It does this by salting and stretching the -/// password. Salting injects non-secret random data, so that identical passwords will be converted -/// into unique keys. Stretching applies a deliberately slow hashing function to frustrate -/// brute-force guessing. -pub const kdf = struct { - pub const pbkdf2 = @import("crypto/pbkdf2.zig").pbkdf2; -}; +pub const kdf = @import("crypto/kdf.zig"); /// Core functions, that should rarely be used directly by applications. pub const core = struct { @@ -86,7 +79,7 @@ test "crypto" { _ = @import("crypto/gimli.zig"); _ = @import("crypto/hmac.zig"); _ = @import("crypto/md5.zig"); - _ = @import("crypto/pbkdf2.zig"); + _ = @import("crypto/kdf.zig"); _ = @import("crypto/poly1305.zig"); _ = @import("crypto/sha1.zig"); _ = @import("crypto/sha2.zig"); diff --git a/lib/std/crypto/kdf.zig b/lib/std/crypto/kdf.zig new file mode 100644 index 000000000..06bf67bbb --- /dev/null +++ b/lib/std/crypto/kdf.zig @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +//! A Key Derivation Function (KDF) is intended to turn a weak, human generated password into a +//! strong key, suitable for cryptographic uses. It does this by salting and stretching the +//! password. Salting injects non-secret random data, so that identical passwords will be converted +//! into unique keys. Stretching applies a deliberately slow hashing function to frustrate +//! brute-force guessing. + +pub const pbkdf2 = @import("pbkdf2.zig").pbkdf2; + +test "kdf" { + _ = @import("pbkdf2.zig"); +} diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig index dfa6b1c02..424e2d6f6 100644 --- a/lib/std/crypto/pbkdf2.zig +++ b/lib/std/crypto/pbkdf2.zig @@ -56,8 +56,6 @@ const mem = std.mem; /// the derivedKey. It is common to tune this parameter to achieve approximately 100ms. /// /// Prf: Pseudo-random function to use. A common choice is std.crypto.auth.hmac.HmacSha256. -/// -/// PBKDF2 is defined in RFC 2898, and is a recommendation of NIST SP 800-132. pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: u32, comptime Prf: type) void { assert(rounds >= 1); From 2f9c9662ba0bdc306c1b80534187251491ffdb17 Mon Sep 17 00:00:00 2001 From: Rob Napier Date: Sun, 13 Sep 2020 11:15:28 -0400 Subject: [PATCH 57/89] Use comptime to expose public method to doc system --- lib/std/crypto/pbkdf2.zig | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig index 424e2d6f6..809fff13a 100644 --- a/lib/std/crypto/pbkdf2.zig +++ b/lib/std/crypto/pbkdf2.zig @@ -10,6 +10,11 @@ const debug = std.debug; const assert = debug.assert; const mem = std.mem; +// Exports +comptime { + _ = crypto.kdf.pbkdf2; +} + // RFC 2898 Section 5.2 // // FromSpec: @@ -155,7 +160,7 @@ test "RFC 6070 one iteration" { var derivedKey: [dkLen]u8 = undefined; - std.crypto.kdf.pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); + pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); const expected = "0c60c80f961f0e71f3a9b524af6012062fe037a6"; From cb5f76bb1ceb82d68c400b7330b73a8583c0613f Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Sun, 13 Sep 2020 09:24:56 -0700 Subject: [PATCH 58/89] got booleans wrong --- lib/std/os/linux/bpf/kern.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/os/linux/bpf/kern.zig b/lib/std/os/linux/bpf/kern.zig index 6e9b51737..13b0e5b1c 100644 --- a/lib/std/os/linux/bpf/kern.zig +++ b/lib/std/os/linux/bpf/kern.zig @@ -6,8 +6,8 @@ const std = @import("../../../std.zig"); const in_bpf_program = switch (std.builtin.arch) { - .bpfel, .bpfeb => false, - else => true, + .bpfel, .bpfeb => true, + else => false, }; pub const helpers = if (in_bpf_program) @import("helpers.zig") else struct {}; From 85366771ea21a0dcd93e58b35738489d773590fc Mon Sep 17 00:00:00 2001 From: Rob Napier Date: Sun, 13 Sep 2020 12:36:32 -0400 Subject: [PATCH 59/89] pbkdf2 offset into dk should be usize, not u64. --- lib/std/crypto/pbkdf2.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig index 809fff13a..2bbf0f15d 100644 --- a/lib/std/crypto/pbkdf2.zig +++ b/lib/std/crypto/pbkdf2.zig @@ -130,7 +130,7 @@ pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: ctx.final(prevBlock[0..]); // Choose portion of DK to write into (T_n) and initialize - const offset: u64 = @as(u64, block) * hLen; + const offset: usize = @as(usize, block) * hLen; const blockLen = if (block != l - 1) hLen else r; var dkBlock = derivedKey[offset..(offset + blockLen)]; mem.copy(u8, dkBlock, prevBlock[0..dkBlock.len]); From 1afbf4fb21e516693161121cf3cb484b4c7374ef Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Sun, 13 Sep 2020 09:43:49 -0700 Subject: [PATCH 60/89] removed all those kern aliases --- lib/std/os/linux/bpf/helpers.zig | 222 ++++++++++++++----------------- lib/std/os/linux/bpf/kern.zig | 13 +- 2 files changed, 105 insertions(+), 130 deletions(-) diff --git a/lib/std/os/linux/bpf/helpers.zig b/lib/std/os/linux/bpf/helpers.zig index 0a4988546..9228e1f1f 100644 --- a/lib/std/os/linux/bpf/helpers.zig +++ b/lib/std/os/linux/bpf/helpers.zig @@ -5,152 +5,126 @@ // and substantial portions of the software. const kern = @import("kern.zig"); -// opting to not use 'usingnamespace' because of future name collisions -const MapDef = kern.MapDef; -const SkBuff = kern.SkBuff; -const TunnelKey = kern.TunnelKey; -const XdpMd = kern.XdpMd; -const BpfSockAddr = kern.BpfSockAddr; -const FibLookup = kern.FibLookup; -const PerfEventData = kern.PerfEventData; -const PerfEventValue = kern.PerfEventValue; -const PidNsInfo = kern.PidNsInfo; -const SkLookup = kern.SkLookup; -const SkMsgBuff = kern.SkMsgBuff; -const SkReusePortMd = kern.SkReusePortMd; -const Sock = kern.Sock; -const SockAddr = kern.SockAddr; -const SockOps = kern.SockOps; -const SockTuple = kern.SockTuple; -const SpinLock = kern.SpinLock; -const SysCtl = kern.SysCtl; -const Tcp6Sock = kern.Tcp6Sock; -const TcpRequestSock = kern.TcpRequestSock; -const TcpSock = kern.TcpSock; -const TcpTimewaitSock = kern.TcpTimewaitSock; -const Udp6Sock = kern.Udp6Sock; -const XfrmState = kern.XfrmState; - // in BPF, all the helper calls // TODO: when https://github.com/ziglang/zig/issues/1717 is here, make a nice // function that uses the Helper enum // // Note, these function signatures were created from documentation found in // '/usr/include/linux/bpf.h' -pub const map_lookup_elem = @intToPtr(fn (map: *const MapDef, key: ?*const c_void) ?*c_void, 1); -pub const map_update_elem = @intToPtr(fn (map: *const MapDef, key: ?*const c_void, value: ?*const c_void, flags: u64) c_long, 2); -pub const map_delete_elem = @intToPtr(fn (map: *const MapDef, key: ?*const c_void) c_long, 3); +pub const map_lookup_elem = @intToPtr(fn (map: *const kern.MapDef, key: ?*const c_void) ?*c_void, 1); +pub const map_update_elem = @intToPtr(fn (map: *const kern.MapDef, key: ?*const c_void, value: ?*const c_void, flags: u64) c_long, 2); +pub const map_delete_elem = @intToPtr(fn (map: *const kern.MapDef, key: ?*const c_void) c_long, 3); pub const probe_read = @intToPtr(fn (dst: ?*c_void, size: u32, unsafe_ptr: ?*const c_void) c_long, 4); pub const ktime_get_ns = @intToPtr(fn () u64, 5); pub const trace_printk = @intToPtr(fn (fmt: [*:0]const u8, fmt_size: u32, arg1: u64, arg2: u64, arg3: u64) c_long, 6); pub const get_prandom_u32 = @intToPtr(fn () u32, 7); pub const get_smp_processor_id = @intToPtr(fn () u32, 8); -pub const skb_store_bytes = @intToPtr(fn (skb: *SkBuff, offset: u32, from: ?*const c_void, len: u32, flags: u64) c_long, 9); -pub const l3_csum_replace = @intToPtr(fn (skb: *SkBuff, offset: u32, from: u64, to: u64, size: u64) c_long, 10); -pub const l4_csum_replace = @intToPtr(fn (skb: *SkBuff, offset: u32, from: u64, to: u64, flags: u64) c_long, 11); -pub const tail_call = @intToPtr(fn (ctx: ?*c_void, prog_array_map: *const MapDef, index: u32) c_long, 12); -pub const clone_redirect = @intToPtr(fn (skb: *SkBuff, ifindex: u32, flags: u64) c_long, 13); +pub const skb_store_bytes = @intToPtr(fn (skb: *kern.SkBuff, offset: u32, from: ?*const c_void, len: u32, flags: u64) c_long, 9); +pub const l3_csum_replace = @intToPtr(fn (skb: *kern.SkBuff, offset: u32, from: u64, to: u64, size: u64) c_long, 10); +pub const l4_csum_replace = @intToPtr(fn (skb: *kern.SkBuff, offset: u32, from: u64, to: u64, flags: u64) c_long, 11); +pub const tail_call = @intToPtr(fn (ctx: ?*c_void, prog_array_map: *const kern.MapDef, index: u32) c_long, 12); +pub const clone_redirect = @intToPtr(fn (skb: *kern.SkBuff, ifindex: u32, flags: u64) c_long, 13); pub const get_current_pid_tgid = @intToPtr(fn () u64, 14); pub const get_current_uid_gid = @intToPtr(fn () u64, 15); pub const get_current_comm = @intToPtr(fn (buf: ?*c_void, size_of_buf: u32) c_long, 16); -pub const get_cgroup_classid = @intToPtr(fn (skb: *SkBuff) u32, 17); +pub const get_cgroup_classid = @intToPtr(fn (skb: *kern.SkBuff) u32, 17); // Note vlan_proto is big endian -pub const skb_vlan_push = @intToPtr(fn (skb: *SkBuff, vlan_proto: u16, vlan_tci: u16) c_long, 18); -pub const skb_vlan_pop = @intToPtr(fn (skb: *SkBuff) c_long, 19); -pub const skb_get_tunnel_key = @intToPtr(fn (skb: *SkBuff, key: *TunnelKey, size: u32, flags: u64) c_long, 20); -pub const skb_set_tunnel_key = @intToPtr(fn (skb: *SkBuff, key: *TunnelKey, size: u32, flags: u64) c_long, 21); -pub const perf_event_read = @intToPtr(fn (map: *const MapDef, flags: u64) u64, 22); +pub const skb_vlan_push = @intToPtr(fn (skb: *kern.SkBuff, vlan_proto: u16, vlan_tci: u16) c_long, 18); +pub const skb_vlan_pop = @intToPtr(fn (skb: *kern.SkBuff) c_long, 19); +pub const skb_get_tunnel_key = @intToPtr(fn (skb: *kern.SkBuff, key: *kern.TunnelKey, size: u32, flags: u64) c_long, 20); +pub const skb_set_tunnel_key = @intToPtr(fn (skb: *kern.SkBuff, key: *kern.TunnelKey, size: u32, flags: u64) c_long, 21); +pub const perf_event_read = @intToPtr(fn (map: *const kern.MapDef, flags: u64) u64, 22); pub const redirect = @intToPtr(fn (ifindex: u32, flags: u64) c_long, 23); -pub const get_route_realm = @intToPtr(fn (skb: *SkBuff) u32, 24); -pub const perf_event_output = @intToPtr(fn (ctx: ?*c_void, map: *const MapDef, flags: u64, data: ?*c_void, size: u64) c_long, 25); +pub const get_route_realm = @intToPtr(fn (skb: *kern.SkBuff) u32, 24); +pub const perf_event_output = @intToPtr(fn (ctx: ?*c_void, map: *const kern.MapDef, flags: u64, data: ?*c_void, size: u64) c_long, 25); pub const skb_load_bytes = @intToPtr(fn (skb: ?*c_void, offset: u32, to: ?*c_void, len: u32) c_long, 26); -pub const get_stackid = @intToPtr(fn (ctx: ?*c_void, map: *const MapDef, flags: u64) c_long, 27); +pub const get_stackid = @intToPtr(fn (ctx: ?*c_void, map: *const kern.MapDef, flags: u64) c_long, 27); // from and to point to __be32 pub const csum_diff = @intToPtr(fn (from: *u32, from_size: u32, to: *u32, to_size: u32, seed: u32) i64, 28); -pub const skb_get_tunnel_opt = @intToPtr(fn (skb: *SkBuff, opt: ?*c_void, size: u32) c_long, 29); -pub const skb_set_tunnel_opt = @intToPtr(fn (skb: *SkBuff, opt: ?*c_void, size: u32) c_long, 30); +pub const skb_get_tunnel_opt = @intToPtr(fn (skb: *kern.SkBuff, opt: ?*c_void, size: u32) c_long, 29); +pub const skb_set_tunnel_opt = @intToPtr(fn (skb: *kern.SkBuff, opt: ?*c_void, size: u32) c_long, 30); // proto is __be16 -pub const skb_change_proto = @intToPtr(fn (skb: *SkBuff, proto: u16, flags: u64) c_long, 31); -pub const skb_change_type = @intToPtr(fn (skb: *SkBuff, skb_type: u32) c_long, 32); -pub const skb_under_cgroup = @intToPtr(fn (skb: *SkBuff, map: ?*const c_void, index: u32) c_long, 33); -pub const get_hash_recalc = @intToPtr(fn (skb: *SkBuff) u32, 34); +pub const skb_change_proto = @intToPtr(fn (skb: *kern.SkBuff, proto: u16, flags: u64) c_long, 31); +pub const skb_change_type = @intToPtr(fn (skb: *kern.SkBuff, skb_type: u32) c_long, 32); +pub const skb_under_cgroup = @intToPtr(fn (skb: *kern.SkBuff, map: ?*const c_void, index: u32) c_long, 33); +pub const get_hash_recalc = @intToPtr(fn (skb: *kern.SkBuff) u32, 34); pub const get_current_task = @intToPtr(fn () u64, 35); pub const probe_write_user = @intToPtr(fn (dst: ?*c_void, src: ?*const c_void, len: u32) c_long, 36); -pub const current_task_under_cgroup = @intToPtr(fn (map: *const MapDef, index: u32) c_long, 37); -pub const skb_change_tail = @intToPtr(fn (skb: *SkBuff, len: u32, flags: u64) c_long, 38); -pub const skb_pull_data = @intToPtr(fn (skb: *SkBuff, len: u32) c_long, 39); -pub const csum_update = @intToPtr(fn (skb: *SkBuff, csum: u32) i64, 40); -pub const set_hash_invalid = @intToPtr(fn (skb: *SkBuff) void, 41); +pub const current_task_under_cgroup = @intToPtr(fn (map: *const kern.MapDef, index: u32) c_long, 37); +pub const skb_change_tail = @intToPtr(fn (skb: *kern.SkBuff, len: u32, flags: u64) c_long, 38); +pub const skb_pull_data = @intToPtr(fn (skb: *kern.SkBuff, len: u32) c_long, 39); +pub const csum_update = @intToPtr(fn (skb: *kern.SkBuff, csum: u32) i64, 40); +pub const set_hash_invalid = @intToPtr(fn (skb: *kern.SkBuff) void, 41); pub const get_numa_node_id = @intToPtr(fn () c_long, 42); -pub const skb_change_head = @intToPtr(fn (skb: *SkBuff, len: u32, flags: u64) c_long, 43); -pub const xdp_adjust_head = @intToPtr(fn (xdp_md: *XdpMd, delta: c_int) c_long, 44); +pub const skb_change_head = @intToPtr(fn (skb: *kern.SkBuff, len: u32, flags: u64) c_long, 43); +pub const xdp_adjust_head = @intToPtr(fn (xdp_md: *kern.XdpMd, delta: c_int) c_long, 44); pub const probe_read_str = @intToPtr(fn (dst: ?*c_void, size: u32, unsafe_ptr: ?*const c_void) c_long, 45); pub const get_socket_cookie = @intToPtr(fn (ctx: ?*c_void) u64, 46); -pub const get_socket_uid = @intToPtr(fn (skb: *SkBuff) u32, 47); -pub const set_hash = @intToPtr(fn (skb: *SkBuff, hash: u32) c_long, 48); -pub const setsockopt = @intToPtr(fn (bpf_socket: *SockOps, level: c_int, optname: c_int, optval: ?*c_void, optlen: c_int) c_long, 49); -pub const skb_adjust_room = @intToPtr(fn (skb: *SkBuff, len_diff: i32, mode: u32, flags: u64) c_long, 50); -pub const redirect_map = @intToPtr(fn (map: *const MapDef, key: u32, flags: u64) c_long, 51); -pub const sk_redirect_map = @intToPtr(fn (skb: *SkBuff, map: *const MapDef, key: u32, flags: u64) c_long, 52); -pub const sock_map_update = @intToPtr(fn (skops: *SockOps, map: *const MapDef, key: ?*c_void, flags: u64) c_long, 53); -pub const xdp_adjust_meta = @intToPtr(fn (xdp_md: *XdpMd, delta: c_int) c_long, 54); -pub const perf_event_read_value = @intToPtr(fn (map: *const MapDef, flags: u64, buf: *PerfEventValue, buf_size: u32) c_long, 55); -pub const perf_prog_read_value = @intToPtr(fn (ctx: *PerfEventData, buf: *PerfEventValue, buf_size: u32) c_long, 56); +pub const get_socket_uid = @intToPtr(fn (skb: *kern.SkBuff) u32, 47); +pub const set_hash = @intToPtr(fn (skb: *kern.SkBuff, hash: u32) c_long, 48); +pub const setsockopt = @intToPtr(fn (bpf_socket: *kern.SockOps, level: c_int, optname: c_int, optval: ?*c_void, optlen: c_int) c_long, 49); +pub const skb_adjust_room = @intToPtr(fn (skb: *kern.SkBuff, len_diff: i32, mode: u32, flags: u64) c_long, 50); +pub const redirect_map = @intToPtr(fn (map: *const kern.MapDef, key: u32, flags: u64) c_long, 51); +pub const sk_redirect_map = @intToPtr(fn (skb: *kern.SkBuff, map: *const kern.MapDef, key: u32, flags: u64) c_long, 52); +pub const sock_map_update = @intToPtr(fn (skops: *kern.SockOps, map: *const kern.MapDef, key: ?*c_void, flags: u64) c_long, 53); +pub const xdp_adjust_meta = @intToPtr(fn (xdp_md: *kern.XdpMd, delta: c_int) c_long, 54); +pub const perf_event_read_value = @intToPtr(fn (map: *const kern.MapDef, flags: u64, buf: *kern.PerfEventValue, buf_size: u32) c_long, 55); +pub const perf_prog_read_value = @intToPtr(fn (ctx: *kern.PerfEventData, buf: *kern.PerfEventValue, buf_size: u32) c_long, 56); pub const getsockopt = @intToPtr(fn (bpf_socket: ?*c_void, level: c_int, optname: c_int, optval: ?*c_void, optlen: c_int) c_long, 57); pub const override_return = @intToPtr(fn (regs: *PtRegs, rc: u64) c_long, 58); -pub const sock_ops_cb_flags_set = @intToPtr(fn (bpf_sock: *SockOps, argval: c_int) c_long, 59); -pub const msg_redirect_map = @intToPtr(fn (msg: *SkMsgMd, map: *const MapDef, key: u32, flags: u64) c_long, 60); -pub const msg_apply_bytes = @intToPtr(fn (msg: *SkMsgMd, bytes: u32) c_long, 61); -pub const msg_cork_bytes = @intToPtr(fn (msg: *SkMsgMd, bytes: u32) c_long, 62); -pub const msg_pull_data = @intToPtr(fn (msg: *SkMsgMd, start: u32, end: u32, flags: u64) c_long, 63); -pub const bind = @intToPtr(fn (ctx: *BpfSockAddr, addr: *SockAddr, addr_len: c_int) c_long, 64); -pub const xdp_adjust_tail = @intToPtr(fn (xdp_md: *XdpMd, delta: c_int) c_long, 65); -pub const skb_get_xfrm_state = @intToPtr(fn (skb: *SkBuff, index: u32, xfrm_state: *XfrmState, size: u32, flags: u64) c_long, 66); +pub const sock_ops_cb_flags_set = @intToPtr(fn (bpf_sock: *kern.SockOps, argval: c_int) c_long, 59); +pub const msg_redirect_map = @intToPtr(fn (msg: *kern.SkMsgMd, map: *const kern.MapDef, key: u32, flags: u64) c_long, 60); +pub const msg_apply_bytes = @intToPtr(fn (msg: *kern.SkMsgMd, bytes: u32) c_long, 61); +pub const msg_cork_bytes = @intToPtr(fn (msg: *kern.SkMsgMd, bytes: u32) c_long, 62); +pub const msg_pull_data = @intToPtr(fn (msg: *kern.SkMsgMd, start: u32, end: u32, flags: u64) c_long, 63); +pub const bind = @intToPtr(fn (ctx: *kern.BpfSockAddr, addr: *kern.SockAddr, addr_len: c_int) c_long, 64); +pub const xdp_adjust_tail = @intToPtr(fn (xdp_md: *kern.XdpMd, delta: c_int) c_long, 65); +pub const skb_get_xfrm_state = @intToPtr(fn (skb: *kern.SkBuff, index: u32, xfrm_state: *kern.XfrmState, size: u32, flags: u64) c_long, 66); pub const get_stack = @intToPtr(fn (ctx: ?*c_void, buf: ?*c_void, size: u32, flags: u64) c_long, 67); pub const skb_load_bytes_relative = @intToPtr(fn (skb: ?*const c_void, offset: u32, to: ?*c_void, len: u32, start_header: u32) c_long, 68); -pub const fib_lookup = @intToPtr(fn (ctx: ?*c_void, params: *FibLookup, plen: c_int, flags: u32) c_long, 69); -pub const sock_hash_update = @intToPtr(fn (skops: *SockOps, map: *const MapDef, key: ?*c_void, flags: u64) c_long, 70); -pub const msg_redirect_hash = @intToPtr(fn (msg: *SkMsgMd, map: *const MapDef, key: ?*c_void, flags: u64) c_long, 71); -pub const sk_redirect_hash = @intToPtr(fn (skb: *SkBuff, map: *const MapDef, key: ?*c_void, flags: u64) c_long, 72); -pub const lwt_push_encap = @intToPtr(fn (skb: *SkBuff, typ: u32, hdr: ?*c_void, len: u32) c_long, 73); -pub const lwt_seg6_store_bytes = @intToPtr(fn (skb: *SkBuff, offset: u32, from: ?*const c_void, len: u32) c_long, 74); -pub const lwt_seg6_adjust_srh = @intToPtr(fn (skb: *SkBuff, offset: u32, delta: i32) c_long, 75); -pub const lwt_seg6_action = @intToPtr(fn (skb: *SkBuff, action: u32, param: ?*c_void, param_len: u32) c_long, 76); +pub const fib_lookup = @intToPtr(fn (ctx: ?*c_void, params: *kern.FibLookup, plen: c_int, flags: u32) c_long, 69); +pub const sock_hash_update = @intToPtr(fn (skops: *kern.SockOps, map: *const kern.MapDef, key: ?*c_void, flags: u64) c_long, 70); +pub const msg_redirect_hash = @intToPtr(fn (msg: *kern.SkMsgMd, map: *const kern.MapDef, key: ?*c_void, flags: u64) c_long, 71); +pub const sk_redirect_hash = @intToPtr(fn (skb: *kern.SkBuff, map: *const kern.MapDef, key: ?*c_void, flags: u64) c_long, 72); +pub const lwt_push_encap = @intToPtr(fn (skb: *kern.SkBuff, typ: u32, hdr: ?*c_void, len: u32) c_long, 73); +pub const lwt_seg6_store_bytes = @intToPtr(fn (skb: *kern.SkBuff, offset: u32, from: ?*const c_void, len: u32) c_long, 74); +pub const lwt_seg6_adjust_srh = @intToPtr(fn (skb: *kern.SkBuff, offset: u32, delta: i32) c_long, 75); +pub const lwt_seg6_action = @intToPtr(fn (skb: *kern.SkBuff, action: u32, param: ?*c_void, param_len: u32) c_long, 76); pub const rc_repeat = @intToPtr(fn (ctx: ?*c_void) c_long, 77); pub const rc_keydown = @intToPtr(fn (ctx: ?*c_void, protocol: u32, scancode: u64, toggle: u32) c_long, 78); -pub const skb_cgroup_id = @intToPtr(fn (skb: *SkBuff) u64, 79); +pub const skb_cgroup_id = @intToPtr(fn (skb: *kern.SkBuff) u64, 79); pub const get_current_cgroup_id = @intToPtr(fn () u64, 80); pub const get_local_storage = @intToPtr(fn (map: ?*c_void, flags: u64) ?*c_void, 81); -pub const sk_select_reuseport = @intToPtr(fn (reuse: *SkReusePortMd, map: *const MapDef, key: ?*c_void, flags: u64) c_long, 82); -pub const skb_ancestor_cgroup_id = @intToPtr(fn (skb: *SkBuff, ancestor_level: c_int) u64, 83); -pub const sk_lookup_tcp = @intToPtr(fn (ctx: ?*c_void, tuple: *SockTuple, tuple_size: u32, netns: u64, flags: u64) ?*Sock, 84); -pub const sk_lookup_udp = @intToPtr(fn (ctx: ?*c_void, tuple: *SockTuple, tuple_size: u32, netns: u64, flags: u64) ?*Sock, 85); -pub const sk_release = @intToPtr(fn (sock: *Sock) c_long, 86); -pub const map_push_elem = @intToPtr(fn (map: *const MapDef, value: ?*const c_void, flags: u64) c_long, 87); -pub const map_pop_elem = @intToPtr(fn (map: *const MapDef, value: ?*c_void) c_long, 88); -pub const map_peek_elem = @intToPtr(fn (map: *const MapDef, value: ?*c_void) c_long, 89); -pub const msg_push_data = @intToPtr(fn (msg: *SkMsgMd, start: u32, len: u32, flags: u64) c_long, 90); -pub const msg_pop_data = @intToPtr(fn (msg: *SkMsgMd, start: u32, len: u32, flags: u64) c_long, 91); +pub const sk_select_reuseport = @intToPtr(fn (reuse: *kern.SkReusePortMd, map: *const kern.MapDef, key: ?*c_void, flags: u64) c_long, 82); +pub const skb_ancestor_cgroup_id = @intToPtr(fn (skb: *kern.SkBuff, ancestor_level: c_int) u64, 83); +pub const sk_lookup_tcp = @intToPtr(fn (ctx: ?*c_void, tuple: *kern.SockTuple, tuple_size: u32, netns: u64, flags: u64) ?*kern.Sock, 84); +pub const sk_lookup_udp = @intToPtr(fn (ctx: ?*c_void, tuple: *kern.SockTuple, tuple_size: u32, netns: u64, flags: u64) ?*kern.Sock, 85); +pub const sk_release = @intToPtr(fn (sock: *kern.Sock) c_long, 86); +pub const map_push_elem = @intToPtr(fn (map: *const kern.MapDef, value: ?*const c_void, flags: u64) c_long, 87); +pub const map_pop_elem = @intToPtr(fn (map: *const kern.MapDef, value: ?*c_void) c_long, 88); +pub const map_peek_elem = @intToPtr(fn (map: *const kern.MapDef, value: ?*c_void) c_long, 89); +pub const msg_push_data = @intToPtr(fn (msg: *kern.SkMsgMd, start: u32, len: u32, flags: u64) c_long, 90); +pub const msg_pop_data = @intToPtr(fn (msg: *kern.SkMsgMd, start: u32, len: u32, flags: u64) c_long, 91); pub const rc_pointer_rel = @intToPtr(fn (ctx: ?*c_void, rel_x: i32, rel_y: i32) c_long, 92); -pub const spin_lock = @intToPtr(fn (lock: *SpinLock) c_long, 93); -pub const spin_unlock = @intToPtr(fn (lock: *SpinLock) c_long, 94); -pub const sk_fullsock = @intToPtr(fn (sk: *Sock) ?*SkFullSock, 95); -pub const tcp_sock = @intToPtr(fn (sk: *Sock) ?*TcpSock, 96); -pub const skb_ecn_set_ce = @intToPtr(fn (skb: *SkBuff) c_long, 97); -pub const get_listener_sock = @intToPtr(fn (sk: *Sock) ?*Sock, 98); -pub const skc_lookup_tcp = @intToPtr(fn (ctx: ?*c_void, tuple: *SockTuple, tuple_size: u32, netns: u64, flags: u64) ?*Sock, 99); -pub const tcp_check_syncookie = @intToPtr(fn (sk: *Sock, iph: ?*c_void, iph_len: u32, th: *TcpHdr, th_len: u32) c_long, 100); -pub const sysctl_get_name = @intToPtr(fn (ctx: *SysCtl, buf: ?*u8, buf_len: c_ulong, flags: u64) c_long, 101); -pub const sysctl_get_current_value = @intToPtr(fn (ctx: *SysCtl, buf: ?*u8, buf_len: c_ulong) c_long, 102); -pub const sysctl_get_new_value = @intToPtr(fn (ctx: *SysCtl, buf: ?*u8, buf_len: c_ulong) c_long, 103); -pub const sysctl_set_new_value = @intToPtr(fn (ctx: *SysCtl, buf: ?*const u8, buf_len: c_ulong) c_long, 104); +pub const spin_lock = @intToPtr(fn (lock: *kern.SpinLock) c_long, 93); +pub const spin_unlock = @intToPtr(fn (lock: *kern.SpinLock) c_long, 94); +pub const sk_fullsock = @intToPtr(fn (sk: *kern.Sock) ?*SkFullSock, 95); +pub const tcp_sock = @intToPtr(fn (sk: *kern.Sock) ?*kern.TcpSock, 96); +pub const skb_ecn_set_ce = @intToPtr(fn (skb: *kern.SkBuff) c_long, 97); +pub const get_listener_sock = @intToPtr(fn (sk: *kern.Sock) ?*kern.Sock, 98); +pub const skc_lookup_tcp = @intToPtr(fn (ctx: ?*c_void, tuple: *kern.SockTuple, tuple_size: u32, netns: u64, flags: u64) ?*kern.Sock, 99); +pub const tcp_check_syncookie = @intToPtr(fn (sk: *kern.Sock, iph: ?*c_void, iph_len: u32, th: *TcpHdr, th_len: u32) c_long, 100); +pub const sysctl_get_name = @intToPtr(fn (ctx: *kern.SysCtl, buf: ?*u8, buf_len: c_ulong, flags: u64) c_long, 101); +pub const sysctl_get_current_value = @intToPtr(fn (ctx: *kern.SysCtl, buf: ?*u8, buf_len: c_ulong) c_long, 102); +pub const sysctl_get_new_value = @intToPtr(fn (ctx: *kern.SysCtl, buf: ?*u8, buf_len: c_ulong) c_long, 103); +pub const sysctl_set_new_value = @intToPtr(fn (ctx: *kern.SysCtl, buf: ?*const u8, buf_len: c_ulong) c_long, 104); pub const strtol = @intToPtr(fn (buf: *const u8, buf_len: c_ulong, flags: u64, res: *c_long) c_long, 105); pub const strtoul = @intToPtr(fn (buf: *const u8, buf_len: c_ulong, flags: u64, res: *c_ulong) c_long, 106); -pub const sk_storage_get = @intToPtr(fn (map: *const MapDef, sk: *Sock, value: ?*c_void, flags: u64) ?*c_void, 107); -pub const sk_storage_delete = @intToPtr(fn (map: *const MapDef, sk: *Sock) c_long, 108); +pub const sk_storage_get = @intToPtr(fn (map: *const kern.MapDef, sk: *kern.Sock, value: ?*c_void, flags: u64) ?*c_void, 107); +pub const sk_storage_delete = @intToPtr(fn (map: *const kern.MapDef, sk: *kern.Sock) c_long, 108); pub const send_signal = @intToPtr(fn (sig: u32) c_long, 109); -pub const tcp_gen_syncookie = @intToPtr(fn (sk: *Sock, iph: ?*c_void, iph_len: u32, th: *TcpHdr, th_len: u32) i64, 110); -pub const skb_output = @intToPtr(fn (ctx: ?*c_void, map: *const MapDef, flags: u64, data: ?*c_void, size: u64) c_long, 111); +pub const tcp_gen_syncookie = @intToPtr(fn (sk: *kern.Sock, iph: ?*c_void, iph_len: u32, th: *TcpHdr, th_len: u32) i64, 110); +pub const skb_output = @intToPtr(fn (ctx: ?*c_void, map: *const kern.MapDef, flags: u64, data: ?*c_void, size: u64) c_long, 111); pub const probe_read_user = @intToPtr(fn (dst: ?*c_void, size: u32, unsafe_ptr: ?*const c_void) c_long, 112); pub const probe_read_kernel = @intToPtr(fn (dst: ?*c_void, size: u32, unsafe_ptr: ?*const c_void) c_long, 113); pub const probe_read_user_str = @intToPtr(fn (dst: ?*c_void, size: u32, unsafe_ptr: ?*const c_void) c_long, 114); @@ -158,26 +132,26 @@ pub const probe_read_kernel_str = @intToPtr(fn (dst: ?*c_void, size: u32, unsafe pub const tcp_send_ack = @intToPtr(fn (tp: ?*c_void, rcv_nxt: u32) c_long, 116); pub const send_signal_thread = @intToPtr(fn (sig: u32) c_long, 117); pub const jiffies64 = @intToPtr(fn () u64, 118); -pub const read_branch_records = @intToPtr(fn (ctx: *PerfEventData, buf: ?*c_void, size: u32, flags: u64) c_long, 119); -pub const get_ns_current_pid_tgid = @intToPtr(fn (dev: u64, ino: u64, nsdata: *PidNsInfo, size: u32) c_long, 120); -pub const xdp_output = @intToPtr(fn (ctx: ?*c_void, map: *const MapDef, flags: u64, data: ?*c_void, size: u64) c_long, 121); +pub const read_branch_records = @intToPtr(fn (ctx: *kern.PerfEventData, buf: ?*c_void, size: u32, flags: u64) c_long, 119); +pub const get_ns_current_pid_tgid = @intToPtr(fn (dev: u64, ino: u64, nsdata: *kern.PidNsInfo, size: u32) c_long, 120); +pub const xdp_output = @intToPtr(fn (ctx: ?*c_void, map: *const kern.MapDef, flags: u64, data: ?*c_void, size: u64) c_long, 121); pub const get_netns_cookie = @intToPtr(fn (ctx: ?*c_void) u64, 122); pub const get_current_ancestor_cgroup_id = @intToPtr(fn (ancestor_level: c_int) u64, 123); -pub const sk_assign = @intToPtr(fn (skb: *SkBuff, sk: *Sock, flags: u64) c_long, 124); +pub const sk_assign = @intToPtr(fn (skb: *kern.SkBuff, sk: *kern.Sock, flags: u64) c_long, 124); pub const ktime_get_boot_ns = @intToPtr(fn () u64, 125); -pub const seq_printf = @intToPtr(fn (m: *SeqFile, fmt: ?*const u8, fmt_size: u32, data: ?*const c_void, data_len: u32) c_long, 126); -pub const seq_write = @intToPtr(fn (m: *SeqFile, data: ?*const u8, len: u32) c_long, 127); -pub const sk_cgroup_id = @intToPtr(fn (sk: *BpfSock) u64, 128); -pub const sk_ancestor_cgroup_id = @intToPtr(fn (sk: *BpfSock, ancestor_level: c_long) u64, 129); +pub const seq_printf = @intToPtr(fn (m: *kern.SeqFile, fmt: ?*const u8, fmt_size: u32, data: ?*const c_void, data_len: u32) c_long, 126); +pub const seq_write = @intToPtr(fn (m: *kern.SeqFile, data: ?*const u8, len: u32) c_long, 127); +pub const sk_cgroup_id = @intToPtr(fn (sk: *kern.BpfSock) u64, 128); +pub const sk_ancestor_cgroup_id = @intToPtr(fn (sk: *kern.BpfSock, ancestor_level: c_long) u64, 129); pub const ringbuf_output = @intToPtr(fn (ringbuf: ?*c_void, data: ?*c_void, size: u64, flags: u64) ?*c_void, 130); pub const ringbuf_reserve = @intToPtr(fn (ringbuf: ?*c_void, size: u64, flags: u64) ?*c_void, 131); pub const ringbuf_submit = @intToPtr(fn (data: ?*c_void, flags: u64) void, 132); pub const ringbuf_discard = @intToPtr(fn (data: ?*c_void, flags: u64) void, 133); pub const ringbuf_query = @intToPtr(fn (ringbuf: ?*c_void, flags: u64) u64, 134); -pub const csum_level = @intToPtr(fn (skb: *SkBuff, level: u64) c_long, 134); -pub const skc_to_tcp6_sock = @intToPtr(fn (sk: ?*c_void) ?*Tcp6Sock, 135); -pub const skc_to_tcp_sock = @intToPtr(fn (sk: ?*c_void) ?*TcpSock, 136); -pub const skc_to_tcp_timewait_sock = @intToPtr(fn (sk: ?*c_void) ?*TcpTimewaitSock, 137); -pub const skc_to_tcp_request_sock = @intToPtr(fn (sk: ?*c_void) ?*TcpRequestSock, 138); -pub const skc_to_udp6_sock = @intToPtr(fn (sk: ?*c_void) ?*Udp6Sock, 139); -pub const get_task_stack = @intToPtr(fn (task: ?*TaskStruct, buf: ?*c_void, size: u32, flags: u64) c_long, 140); +pub const csum_level = @intToPtr(fn (skb: *kern.SkBuff, level: u64) c_long, 134); +pub const skc_to_tcp6_sock = @intToPtr(fn (sk: ?*c_void) ?*kern.Tcp6Sock, 135); +pub const skc_to_tcp_sock = @intToPtr(fn (sk: ?*c_void) ?*kern.TcpSock, 136); +pub const skc_to_tcp_timewait_sock = @intToPtr(fn (sk: ?*c_void) ?*kern.TcpTimewaitSock, 137); +pub const skc_to_tcp_request_sock = @intToPtr(fn (sk: ?*c_void) ?*kern.TcpRequestSock, 138); +pub const skc_to_udp6_sock = @intToPtr(fn (sk: ?*c_void) ?*kern.Udp6Sock, 139); +pub const get_task_stack = @intToPtr(fn (task: ?*c_void, buf: ?*c_void, size: u32, flags: u64) c_long, 140); diff --git a/lib/std/os/linux/bpf/kern.zig b/lib/std/os/linux/bpf/kern.zig index 13b0e5b1c..0b54b938e 100644 --- a/lib/std/os/linux/bpf/kern.zig +++ b/lib/std/os/linux/bpf/kern.zig @@ -13,17 +13,16 @@ const in_bpf_program = switch (std.builtin.arch) { pub const helpers = if (in_bpf_program) @import("helpers.zig") else struct {}; // TODO: fill these in -pub const MapDef = packed struct {}; -pub const SkBuff = packed struct {}; -pub const TunnelKey = packed struct {}; -pub const XdpMd = packed struct {}; +pub const BpfSock = packed struct {}; pub const BpfSockAddr = packed struct {}; pub const FibLookup = packed struct {}; +pub const MapDef = packed struct {}; pub const PerfEventData = packed struct {}; pub const PerfEventValue = packed struct {}; pub const PidNsInfo = packed struct {}; -pub const SkLookup = packed struct {}; -pub const SkMsgBuff = packed struct {}; +pub const SeqFile = packed struct {}; +pub const SkBuff = packed struct {}; +pub const SkMsgMd = packed struct {}; pub const SkReusePortMd = packed struct {}; pub const Sock = packed struct {}; pub const SockAddr = packed struct {}; @@ -35,5 +34,7 @@ pub const Tcp6Sock = packed struct {}; pub const TcpRequestSock = packed struct {}; pub const TcpSock = packed struct {}; pub const TcpTimewaitSock = packed struct {}; +pub const TunnelKey = packed struct {}; pub const Udp6Sock = packed struct {}; +pub const XdpMd = packed struct {}; pub const XfrmState = packed struct {}; From 88dacd3b70118710884f53f55d5a8ec9a78f149f Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Sun, 13 Sep 2020 09:53:20 -0700 Subject: [PATCH 61/89] changed to opaque --- lib/std/os/linux/bpf/kern.zig | 51 +++++++++++++++++------------------ 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/lib/std/os/linux/bpf/kern.zig b/lib/std/os/linux/bpf/kern.zig index 0b54b938e..3bd605301 100644 --- a/lib/std/os/linux/bpf/kern.zig +++ b/lib/std/os/linux/bpf/kern.zig @@ -12,29 +12,28 @@ const in_bpf_program = switch (std.builtin.arch) { pub const helpers = if (in_bpf_program) @import("helpers.zig") else struct {}; -// TODO: fill these in -pub const BpfSock = packed struct {}; -pub const BpfSockAddr = packed struct {}; -pub const FibLookup = packed struct {}; -pub const MapDef = packed struct {}; -pub const PerfEventData = packed struct {}; -pub const PerfEventValue = packed struct {}; -pub const PidNsInfo = packed struct {}; -pub const SeqFile = packed struct {}; -pub const SkBuff = packed struct {}; -pub const SkMsgMd = packed struct {}; -pub const SkReusePortMd = packed struct {}; -pub const Sock = packed struct {}; -pub const SockAddr = packed struct {}; -pub const SockOps = packed struct {}; -pub const SockTuple = packed struct {}; -pub const SpinLock = packed struct {}; -pub const SysCtl = packed struct {}; -pub const Tcp6Sock = packed struct {}; -pub const TcpRequestSock = packed struct {}; -pub const TcpSock = packed struct {}; -pub const TcpTimewaitSock = packed struct {}; -pub const TunnelKey = packed struct {}; -pub const Udp6Sock = packed struct {}; -pub const XdpMd = packed struct {}; -pub const XfrmState = packed struct {}; +pub const BpfSock = @Type(.Opaque); +pub const BpfSockAddr = @Type(.Opaque); +pub const FibLookup = @Type(.Opaque); +pub const MapDef = @Type(.Opaque); +pub const PerfEventData = @Type(.Opaque); +pub const PerfEventValue = @Type(.Opaque); +pub const PidNsInfo = @Type(.Opaque); +pub const SeqFile = @Type(.Opaque); +pub const SkBuff = @Type(.Opaque); +pub const SkMsgMd = @Type(.Opaque); +pub const SkReusePortMd = @Type(.Opaque); +pub const Sock = @Type(.Opaque); +pub const SockAddr = @Type(.Opaque); +pub const SockOps = @Type(.Opaque); +pub const SockTuple = @Type(.Opaque); +pub const SpinLock = @Type(.Opaque); +pub const SysCtl = @Type(.Opaque); +pub const Tcp6Sock = @Type(.Opaque); +pub const TcpRequestSock = @Type(.Opaque); +pub const TcpSock = @Type(.Opaque); +pub const TcpTimewaitSock = @Type(.Opaque); +pub const TunnelKey = @Type(.Opaque); +pub const Udp6Sock = @Type(.Opaque); +pub const XdpMd = @Type(.Opaque); +pub const XfrmState = @Type(.Opaque); From b6385870d0e65c5d7c6d7c6ef8c8ed33780b71e4 Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Sun, 13 Sep 2020 22:39:54 +0300 Subject: [PATCH 62/89] Convert asserts to errors, make sure nothing overflows --- lib/std/crypto/pbkdf2.zig | 69 +++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig index 2bbf0f15d..813535f3f 100644 --- a/lib/std/crypto/pbkdf2.zig +++ b/lib/std/crypto/pbkdf2.zig @@ -5,15 +5,8 @@ // and substantial portions of the software. const std = @import("std"); -const crypto = std.crypto; -const debug = std.debug; -const assert = debug.assert; const mem = std.mem; - -// Exports -comptime { - _ = crypto.kdf.pbkdf2; -} +const maxInt = std.math.maxInt; // RFC 2898 Section 5.2 // @@ -48,8 +41,8 @@ comptime { /// PBKDF2 is defined in RFC 2898, and is a recommendation of NIST SP 800-132. /// /// derivedKey: Slice of appropriate size for generated key. Generally 16 or 32 bytes in length. -/// May be uninitialized. All bytes will be written. -/// Maximum size is (2^32 - 1) * Hash.digest_length +/// May be uninitialized. All bytes will be overwritten. +/// Maximum size is `maxInt(u32) * Hash.digest_length` /// It is a programming error to pass buffer longer than the maximum size. /// /// password: Arbitrary sequence of bytes of any length, including empty. @@ -60,29 +53,41 @@ comptime { /// Larger iteration counts improve security by increasing the time required to compute /// the derivedKey. It is common to tune this parameter to achieve approximately 100ms. /// -/// Prf: Pseudo-random function to use. A common choice is std.crypto.auth.hmac.HmacSha256. -pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: u32, comptime Prf: type) void { - assert(rounds >= 1); +/// Prf: Pseudo-random function to use. A common choice is `std.crypto.auth.hmac.HmacSha256`. +pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: u32, comptime Prf: type) !void { + if (rounds < 1) return error.TooFewRounds; - const dkLen: u64 = derivedKey.len; - const hLen: u32 = Prf.mac_length; // Force type to ensure multiplications can't overflow + const dkLen = derivedKey.len; + const hLen = Prf.mac_length; // FromSpec: // - // 1. If dkLen > (2^32 - 1) * hLen, output "derived key too long" and + // 1. If dkLen > maxInt(u32) * hLen, output "derived key too long" and // stop. // - assert(dkLen > 0 and dkLen <= @as(u64, 1 << 32 - 1) * hLen); + if (comptime (maxInt(usize) < maxInt(u32) * hLen) and (dkLen > @as(usize, maxInt(u32) * hLen))) { + // If maxInt(usize) is less than `maxInt(u32) * hLen` then dkLen is always inbounds + // This also asserts hLen >= 1 + return error.DerivedKeyTooLong; + } // FromSpec: // - // 2. Let l be the number of hLen-octet blocks in the derived key, - // rounding up, and let r be the number of octets in the last + // 2. Let l be the number of hLen-long blocks of bytes in the derived key, + // rounding up, and let r be the number of bytes in the last // block // - const l = (dkLen + hLen - 1) / hLen; - var r = dkLen % hLen; - r = if (r != 0) r else hLen; + + // l will not overflow, proof: + // let `L(dkLen, hLen) = (dkLen + hLen - 1) / hLen` + // then `L^-1(l, hLen) = l*hLen - hLen + 1` + // 1) L^-1(maxInt(u32), hLen) <= maxInt(u32)*hLen + // 2) maxInt(u32)*hLen - hLen + 1 <= maxInt(u32)*hLen // subtract maxInt(u32)*hLen + 1 + // 3) -hLen <= -1 // multiply by -1 + // 4) hLen >= 1 + const r_ = dkLen % hLen; + const l = @intCast(u32, (dkLen / hLen) + if (r_ == 0) 0 else 1); // original: (dkLen + hLen - 1) / hLen + const r = if (r_ == 0) hLen else r_; // FromSpec: // @@ -116,7 +121,6 @@ pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: // produce a derived key DK: // // DK = T_1 || T_2 || ... || T_l<0..r-1> - var block: u32 = 0; // Spec limits to u32 while (block < l) : (block += 1) { var prevBlock: [hLen]u8 = undefined; @@ -130,9 +134,9 @@ pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: ctx.final(prevBlock[0..]); // Choose portion of DK to write into (T_n) and initialize - const offset: usize = @as(usize, block) * hLen; + const offset = block * hLen; const blockLen = if (block != l - 1) hLen else r; - var dkBlock = derivedKey[offset..(offset + blockLen)]; + const dkBlock: []u8 = derivedKey[offset..][0..blockLen]; mem.copy(u8, dkBlock, prevBlock[0..dkBlock.len]); var i: u32 = 1; @@ -150,6 +154,7 @@ pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: } const htest = @import("test.zig"); +const HmacSha1 = std.crypto.auth.hmac.HmacSha1; // RFC 6070 PBKDF2 HMAC-SHA1 Test Vectors test "RFC 6070 one iteration" { @@ -160,7 +165,7 @@ test "RFC 6070 one iteration" { var derivedKey: [dkLen]u8 = undefined; - pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); + try pbkdf2(&derivedKey, p, s, c, HmacSha1); const expected = "0c60c80f961f0e71f3a9b524af6012062fe037a6"; @@ -175,7 +180,7 @@ test "RFC 6070 two iterations" { var derivedKey: [dkLen]u8 = undefined; - pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); + try pbkdf2(&derivedKey, p, s, c, HmacSha1); const expected = "ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957"; @@ -190,7 +195,7 @@ test "RFC 6070 4096 iterations" { var derivedKey: [dkLen]u8 = undefined; - pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); + try pbkdf2(&derivedKey, p, s, c, HmacSha1); const expected = "4b007901b765489abead49d926f721d065a429c1"; @@ -210,7 +215,7 @@ test "RFC 6070 16,777,216 iterations" { var derivedKey = [_]u8{0} ** dkLen; - pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); + try pbkdf2(&derivedKey, p, s, c, HmacSha1); const expected = "eefe3d61cd4da4e4e9945b3d6ba2158c2634e984"; @@ -225,7 +230,7 @@ test "RFC 6070 multi-block salt and password" { var derivedKey: [dkLen]u8 = undefined; - pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); + try pbkdf2(&derivedKey, p, s, c, HmacSha1); const expected = "3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038"; @@ -240,7 +245,7 @@ test "RFC 6070 embedded NUL" { var derivedKey: [dkLen]u8 = undefined; - pbkdf2(&derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); + try pbkdf2(&derivedKey, p, s, c, ); const expected = "56fa6aa75548099dcc37d7f03425e0c3"; @@ -262,6 +267,6 @@ test "Very large dkLen" { std.testing.allocator.free(derivedKey); } - pbkdf2(derivedKey, p, s, c, crypto.auth.hmac.HmacSha1); + try pbkdf2(derivedKey, p, s, c, HmacSha1); // Just verify this doesn't crash with an overflow } From d75cbb01db4f7fb73c4382af4351dc0e393d5d39 Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Sun, 13 Sep 2020 23:00:33 +0300 Subject: [PATCH 63/89] Reference all crypto declarations --- lib/std/crypto.zig | 25 +++++++++++++++++++++++-- lib/std/crypto/kdf.zig | 17 ----------------- 2 files changed, 23 insertions(+), 19 deletions(-) delete mode 100644 lib/std/crypto/kdf.zig diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig index 64ec22894..9df96dec7 100644 --- a/lib/std/crypto.zig +++ b/lib/std/crypto.zig @@ -35,7 +35,14 @@ pub const onetimeauth = struct { pub const Poly1305 = @import("crypto/poly1305.zig").Poly1305; }; -pub const kdf = @import("crypto/kdf.zig"); +/// A Key Derivation Function (KDF) is intended to turn a weak, human generated password into a +/// strong key, suitable for cryptographic uses. It does this by salting and stretching the +/// password. Salting injects non-secret random data, so that identical passwords will be converted +/// into unique keys. Stretching applies a deliberately slow hashing function to frustrate +/// brute-force guessing. +pub const kdf = struct { + pub const pbkdf2 = @import("crypto/pbkdf2.zig").pbkdf2; +}; /// Core functions, that should rarely be used directly by applications. pub const core = struct { @@ -72,6 +79,20 @@ const std = @import("std.zig"); pub const randomBytes = std.os.getrandom; test "crypto" { + inline for (std.meta.declarations(std)) |decl| { + switch (decl.data) { + .Type => |t| { + std.meta.refAllDecls(t); + }, + .Var => |v| { + _ = v; + }, + .Fn => |f| { + _ = f; + }, + } + } + _ = @import("crypto/aes.zig"); _ = @import("crypto/blake2.zig"); _ = @import("crypto/blake3.zig"); @@ -79,7 +100,7 @@ test "crypto" { _ = @import("crypto/gimli.zig"); _ = @import("crypto/hmac.zig"); _ = @import("crypto/md5.zig"); - _ = @import("crypto/kdf.zig"); + _ = @import("crypto/pbkdf2.zig"); _ = @import("crypto/poly1305.zig"); _ = @import("crypto/sha1.zig"); _ = @import("crypto/sha2.zig"); diff --git a/lib/std/crypto/kdf.zig b/lib/std/crypto/kdf.zig deleted file mode 100644 index 06bf67bbb..000000000 --- a/lib/std/crypto/kdf.zig +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2015-2020 Zig Contributors -// This file is part of [zig](https://ziglang.org/), which is MIT licensed. -// The MIT license requires this copyright notice to be included in all copies -// and substantial portions of the software. - -//! A Key Derivation Function (KDF) is intended to turn a weak, human generated password into a -//! strong key, suitable for cryptographic uses. It does this by salting and stretching the -//! password. Salting injects non-secret random data, so that identical passwords will be converted -//! into unique keys. Stretching applies a deliberately slow hashing function to frustrate -//! brute-force guessing. - -pub const pbkdf2 = @import("pbkdf2.zig").pbkdf2; - -test "kdf" { - _ = @import("pbkdf2.zig"); -} From 5e50d145d964238a68d4780e253d26431e7c7994 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sun, 13 Sep 2020 09:42:00 +0200 Subject: [PATCH 64/89] std: Limit the read/write size on Darwin It turns out that the kernel won't read or write more than 0x7fffffff bytes in a single call, failing with EINVAL when trying to do so. Adjust the limit and curse whoever is responsible for this. Closes #6332 --- lib/std/os.zig | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 91365c81d..bdc746419 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -320,6 +320,7 @@ pub const ReadError = error{ /// Linux has a limit on how many bytes may be transferred in one `read` call, which is `0x7ffff000` /// on both 64-bit and 32-bit systems. This is due to using a signed C int as the return value, as /// well as stuffing the errno codes into the last `4096` values. This is noted on the `read` man page. +/// The limit on Darwin is `0x7fffffff`, trying to read more than that returns EINVAL. /// For POSIX the limit is `math.maxInt(isize)`. pub fn read(fd: fd_t, buf: []u8) ReadError!usize { if (builtin.os.tag == .windows) { @@ -353,6 +354,7 @@ pub fn read(fd: fd_t, buf: []u8) ReadError!usize { // Prevents EINVAL. const max_count = switch (std.Target.current.os.tag) { .linux => 0x7ffff000, + .macosx, .ios, .watchos, .tvos => math.maxInt(i32), else => math.maxInt(isize), }; const adjusted_len = math.min(max_count, buf.len); @@ -693,6 +695,7 @@ pub const WriteError = error{ /// Linux has a limit on how many bytes may be transferred in one `write` call, which is `0x7ffff000` /// on both 64-bit and 32-bit systems. This is due to using a signed C int as the return value, as /// well as stuffing the errno codes into the last `4096` values. This is noted on the `write` man page. +/// The limit on Darwin is `0x7fffffff`, trying to read more than that returns EINVAL. /// The corresponding POSIX limit is `math.maxInt(isize)`. pub fn write(fd: fd_t, bytes: []const u8) WriteError!usize { if (builtin.os.tag == .windows) { @@ -726,6 +729,7 @@ pub fn write(fd: fd_t, bytes: []const u8) WriteError!usize { const max_count = switch (std.Target.current.os.tag) { .linux => 0x7ffff000, + .macosx, .ios, .watchos, .tvos => math.maxInt(i32), else => math.maxInt(isize), }; const adjusted_len = math.min(max_count, bytes.len); @@ -851,6 +855,7 @@ pub const PWriteError = WriteError || error{Unseekable}; /// Linux has a limit on how many bytes may be transferred in one `pwrite` call, which is `0x7ffff000` /// on both 64-bit and 32-bit systems. This is due to using a signed C int as the return value, as /// well as stuffing the errno codes into the last `4096` values. This is noted on the `write` man page. +/// The limit on Darwin is `0x7fffffff`, trying to write more than that returns EINVAL. /// The corresponding POSIX limit is `math.maxInt(isize)`. pub fn pwrite(fd: fd_t, bytes: []const u8, offset: u64) PWriteError!usize { if (std.Target.current.os.tag == .windows) { @@ -888,6 +893,7 @@ pub fn pwrite(fd: fd_t, bytes: []const u8, offset: u64) PWriteError!usize { // Prevent EINVAL. const max_count = switch (std.Target.current.os.tag) { .linux => 0x7ffff000, + .macosx, .ios, .watchos, .tvos => math.maxInt(i32), else => math.maxInt(isize), }; const adjusted_len = math.min(max_count, bytes.len); @@ -3084,7 +3090,7 @@ pub fn connect(sockfd: socket_t, sock_addr: *const sockaddr, len: socklen_t) Con .WSAECONNREFUSED => return error.ConnectionRefused, .WSAETIMEDOUT => return error.ConnectionTimedOut, .WSAEHOSTUNREACH // TODO: should we return NetworkUnreachable in this case as well? - , .WSAENETUNREACH => return error.NetworkUnreachable, + , .WSAENETUNREACH => return error.NetworkUnreachable, .WSAEFAULT => unreachable, .WSAEINVAL => unreachable, .WSAEISCONN => unreachable, @@ -4711,6 +4717,7 @@ fn count_iovec_bytes(iovs: []const iovec_const) usize { /// Linux has a limit on how many bytes may be transferred in one `sendfile` call, which is `0x7ffff000` /// on both 64-bit and 32-bit systems. This is due to using a signed C int as the return value, as /// well as stuffing the errno codes into the last `4096` values. This is cited on the `sendfile` man page. +/// The limit on Darwin is `0x7fffffff`, trying to write more than that returns EINVAL. /// The corresponding POSIX limit on this is `math.maxInt(isize)`. pub fn sendfile( out_fd: fd_t, @@ -4733,6 +4740,7 @@ pub fn sendfile( }); const max_count = switch (std.Target.current.os.tag) { .linux => 0x7ffff000, + .macosx, .ios, .watchos, .tvos => math.maxInt(i32), else => math.maxInt(size_t), }; From f6195be99770e3f9cc4e8a89bcf451a640bb4b95 Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Sun, 13 Sep 2020 23:31:59 +0300 Subject: [PATCH 65/89] fix ref --- lib/std/crypto.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig index 9df96dec7..3a1ae599a 100644 --- a/lib/std/crypto.zig +++ b/lib/std/crypto.zig @@ -79,7 +79,7 @@ const std = @import("std.zig"); pub const randomBytes = std.os.getrandom; test "crypto" { - inline for (std.meta.declarations(std)) |decl| { + inline for (std.meta.declarations(@This())) |decl| { switch (decl.data) { .Type => |t| { std.meta.refAllDecls(t); From 61e9e82bdc10110b74bdeb973cc542c7b73a4ae2 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sun, 13 Sep 2020 21:12:21 +0200 Subject: [PATCH 66/89] std: Make the CRC32 calculation slightly faster Speed up a little the slicing-by-8 code path by replacing the (load+shift+xor)*4 sequence with a single u32 load plus a xor. Before: ``` iterative: 1018 MiB/s [000000006c3b110d] small keys: 1075 MiB/s [0035bf3dcac00000] ``` After: ``` iterative: 1114 MiB/s [000000006c3b110d] small keys: 1324 MiB/s [0035bf3dcac00000] ``` --- lib/std/hash/crc.zig | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/std/hash/crc.zig b/lib/std/hash/crc.zig index 37695df8b..6290369fc 100644 --- a/lib/std/hash/crc.zig +++ b/lib/std/hash/crc.zig @@ -71,10 +71,7 @@ pub fn Crc32WithPoly(comptime poly: Polynomial) type { const p = input[i .. i + 8]; // Unrolling this way gives ~50Mb/s increase - self.crc ^= (@as(u32, p[0]) << 0); - self.crc ^= (@as(u32, p[1]) << 8); - self.crc ^= (@as(u32, p[2]) << 16); - self.crc ^= (@as(u32, p[3]) << 24); + self.crc ^= std.mem.readIntLittle(u32, p[0..4]); self.crc = lookup_tables[0][p[7]] ^ From 73863cf72be02020886d9f8daf15754a8d3b2567 Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Sun, 13 Sep 2020 23:59:36 +0300 Subject: [PATCH 67/89] fix build --- lib/std/crypto/pbkdf2.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig index 813535f3f..8ff2b4f64 100644 --- a/lib/std/crypto/pbkdf2.zig +++ b/lib/std/crypto/pbkdf2.zig @@ -86,7 +86,7 @@ pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: // 3) -hLen <= -1 // multiply by -1 // 4) hLen >= 1 const r_ = dkLen % hLen; - const l = @intCast(u32, (dkLen / hLen) + if (r_ == 0) 0 else 1); // original: (dkLen + hLen - 1) / hLen + const l = @intCast(u32, (dkLen / hLen) + @as(u1, if (r_ == 0) 0 else 1)); // original: (dkLen + hLen - 1) / hLen const r = if (r_ == 0) hLen else r_; // FromSpec: @@ -245,7 +245,7 @@ test "RFC 6070 embedded NUL" { var derivedKey: [dkLen]u8 = undefined; - try pbkdf2(&derivedKey, p, s, c, ); + try pbkdf2(&derivedKey, p, s, c, HmacSha1); const expected = "56fa6aa75548099dcc37d7f03425e0c3"; From 988fc6f9d1419a63d7ecd3507966ba0b4c15eac5 Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Mon, 14 Sep 2020 02:27:09 +0300 Subject: [PATCH 68/89] flip condition --- lib/std/crypto/pbkdf2.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig index 8ff2b4f64..1ba176ab9 100644 --- a/lib/std/crypto/pbkdf2.zig +++ b/lib/std/crypto/pbkdf2.zig @@ -59,15 +59,15 @@ pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: const dkLen = derivedKey.len; const hLen = Prf.mac_length; + comptime std.debug.assert(hLen >= 1); // FromSpec: // // 1. If dkLen > maxInt(u32) * hLen, output "derived key too long" and // stop. // - if (comptime (maxInt(usize) < maxInt(u32) * hLen) and (dkLen > @as(usize, maxInt(u32) * hLen))) { + if (comptime (maxInt(usize) > maxInt(u32) * hLen) and (dkLen > @as(usize, maxInt(u32) * hLen))) { // If maxInt(usize) is less than `maxInt(u32) * hLen` then dkLen is always inbounds - // This also asserts hLen >= 1 return error.DerivedKeyTooLong; } From 29fd0c6d61d1569c1691fc5489d51311207b1759 Mon Sep 17 00:00:00 2001 From: Vexu Date: Mon, 14 Sep 2020 21:20:41 +0300 Subject: [PATCH 69/89] fix meta.cast behavior; add exhaustive tests --- lib/std/meta.zig | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/lib/std/meta.zig b/lib/std/meta.zig index b27f168ac..b0c1aadd9 100644 --- a/lib/std/meta.zig +++ b/lib/std/meta.zig @@ -717,7 +717,7 @@ pub fn cast(comptime DestType: type, target: anytype) DestType { }, .Optional => |opt| { if (@typeInfo(opt.child) == .Pointer) { - return @ptrCast(DestType, @alignCast(dest_ptr, target)); + return @ptrCast(DestType, @alignCast(dest_ptr.alignment, target)); } }, else => {}, @@ -725,23 +725,24 @@ pub fn cast(comptime DestType: type, target: anytype) DestType { }, .Optional => |dest_opt| { if (@typeInfo(dest_opt.child) == .Pointer) { + const dest_ptr = @typeInfo(dest_opt.child).Pointer; switch (@typeInfo(TargetType)) { .Int, .ComptimeInt => { return @intToPtr(DestType, target); }, .Pointer => { - return @ptrCast(DestType, @alignCast(@alignOf(dest_opt.child.Child), target)); + return @ptrCast(DestType, @alignCast(dest_ptr.alignment, target)); }, .Optional => |target_opt| { if (@typeInfo(target_opt.child) == .Pointer) { - return @ptrCast(DestType, @alignCast(@alignOf(dest_opt.child.Child), target)); + return @ptrCast(DestType, @alignCast(dest_ptr.alignment, target)); } }, else => {}, } } }, - .Enum, .EnumLiteral => { + .Enum => { if (@typeInfo(TargetType) == .Int or @typeInfo(TargetType) == .ComptimeInt) { return @intToEnum(DestType, target); } @@ -749,15 +750,18 @@ pub fn cast(comptime DestType: type, target: anytype) DestType { .Int, .ComptimeInt => { switch (@typeInfo(TargetType)) { .Pointer => { - return @as(DestType, @ptrToInt(target)); + return @intCast(DestType, @ptrToInt(target)); }, .Optional => |opt| { if (@typeInfo(opt.child) == .Pointer) { - return @as(DestType, @ptrToInt(target)); + return @intCast(DestType, @ptrToInt(target)); } }, - .Enum, .EnumLiteral => { - return @as(DestType, @enumToInt(target)); + .Enum => { + return @intCast(DestType, @enumToInt(target)); + }, + .Int, .ComptimeInt => { + return @intCast(DestType, target); }, else => {}, } @@ -776,10 +780,18 @@ test "std.meta.cast" { var i = @as(i64, 10); - testing.expect(cast(?*c_void, 0) == @intToPtr(?*c_void, 0)); testing.expect(cast(*u8, 16) == @intToPtr(*u8, 16)); - testing.expect(cast(u64, @as(u32, 10)) == @as(u64, 10)); - testing.expect(cast(E, 1) == .One); - testing.expect(cast(u8, E.Two) == 2); testing.expect(cast(*u64, &i).* == @as(u64, 10)); + testing.expect(cast(*i64, @as(?*align(1) i64, &i)) == &i); + + testing.expect(cast(?*u8, 2) == @intToPtr(*u8, 2)); + testing.expect(cast(?*i64, @as(*align(1) i64, &i)) == &i); + testing.expect(cast(?*i64, @as(?*align(1) i64, &i)) == &i); + + testing.expect(cast(E, 1) == .One); + + testing.expectEqual(@as(u32, 4), cast(u32, @intToPtr(*u32, 4))); + testing.expectEqual(@as(u32, 4), cast(u32, @intToPtr(?*u32, 4))); + testing.expectEqual(@as(u32, 10), cast(u32, @as(u64, 10))); + testing.expectEqual(@as(u8, 2), cast(u8, E.Two)); } From a3624e94f812efaae40d2ca66da0b49ca63cb936 Mon Sep 17 00:00:00 2001 From: Vexu Date: Mon, 14 Sep 2020 23:53:38 +0300 Subject: [PATCH 70/89] translate-c: determine sizeof using std.meta.sizeof --- lib/std/meta.zig | 31 +++++++++++++++++++++++++++++++ src-self-hosted/translate_c.zig | 16 ++++++++++++---- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/lib/std/meta.zig b/lib/std/meta.zig index b0c1aadd9..1507aa9de 100644 --- a/lib/std/meta.zig +++ b/lib/std/meta.zig @@ -795,3 +795,34 @@ test "std.meta.cast" { testing.expectEqual(@as(u32, 10), cast(u32, @as(u64, 10))); testing.expectEqual(@as(u8, 2), cast(u8, E.Two)); } + +/// Given a value returns its size as C's sizeof operator would. +/// This is for translate-c and is not intended for general use. +pub fn sizeof(target: anytype) usize { + switch (@typeInfo(@TypeOf(target))) { + .Type => return @sizeOf(target), + .Float, .Int, .Struct, .Union, .Enum => return @sizeOf(@TypeOf(target)), + .ComptimeFloat => return @sizeOf(f64), // TODO c_double #3999 + .ComptimeInt => { + // TODO to get the correct result we have to translate + // `1073741824 * 4` as `int(1073741824) *% int(4)` since + // sizeof(1073741824 * 4) != sizeof(4294967296). + + // TODO test if target fits in int, long or long long + return @sizeOf(c_int); + }, + else => @compileError("TODO implement std.meta.sizeof for type " ++ @typeName(@TypeOf(target))), + } +} + +test "sizeof" { + const E = extern enum(c_int) { One, _ }; + const S = extern struct { a: u32 }; + + testing.expect(sizeof(u32) == 4); + testing.expect(sizeof(@as(u32, 2)) == 4); + testing.expect(sizeof(2) == @sizeOf(c_int)); + testing.expect(sizeof(E) == @sizeOf(c_int)); + testing.expect(sizeof(E.One) == @sizeOf(c_int)); + testing.expect(sizeof(S) == 4); +} diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index cd42e8abe..1cdb5099e 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -6324,10 +6324,18 @@ fn parseCPrefixOpExpr(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!*ast. break :blk inner; } else try parseCPrefixOpExpr(c, m, scope); - const builtin_call = try c.createBuiltinCall("@sizeOf", 1); - builtin_call.params()[0] = inner; - builtin_call.rparen_token = try appendToken(c, .RParen, ")"); - return &builtin_call.base; + //(@import("std").meta.sizeof(dest, x)) + const import_fn_call = try c.createBuiltinCall("@import", 1); + const std_node = try transCreateNodeStringLiteral(c, "\"std\""); + import_fn_call.params()[0] = std_node; + import_fn_call.rparen_token = try appendToken(c, .RParen, ")"); + const inner_field_access = try transCreateNodeFieldAccess(c, &import_fn_call.base, "meta"); + const outer_field_access = try transCreateNodeFieldAccess(c, inner_field_access, "sizeof"); + + const sizeof_call = try c.createCall(outer_field_access, 1); + sizeof_call.params()[0] = inner; + sizeof_call.rtoken = try appendToken(c, .RParen, ")"); + return &sizeof_call.base; }, .Keyword_alignof => { // TODO this won't work if using 's From 01c24c45093f70b5a9c1e415ed69c274cb61b303 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 18:22:16 -0700 Subject: [PATCH 71/89] ci: enable std lib tests for freebsd --- ci/srht/freebsd_script | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ci/srht/freebsd_script b/ci/srht/freebsd_script index 97b869c9f..539051f40 100755 --- a/ci/srht/freebsd_script +++ b/ci/srht/freebsd_script @@ -28,11 +28,7 @@ make $JOBS install release/bin/zig build test-fmt release/bin/zig build test-behavior - -# This test is disabled because it triggers "out of memory" on the sr.ht CI service. -# See https://github.com/ziglang/zig/issues/3210 -# release/bin/zig build test-std - +release/bin/zig build test-std release/bin/zig build test-compiler-rt release/bin/zig build test-compare-output release/bin/zig build test-standalone From 558e22b2d021797e4d2fb2ef7fedb38690f8beb0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 18:45:30 -0700 Subject: [PATCH 72/89] ci: update freebsd to use hut.lavatech.top instead of sr.ht --- .builds/freebsd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml index 9b936657f..d53ab9167 100644 --- a/.builds/freebsd.yml +++ b/.builds/freebsd.yml @@ -1,7 +1,7 @@ image: freebsd/latest secrets: - - 6c60aaee-92e7-4e7d-812c-114817689b4d - - dd0bd962-7664-4d3e-b0f3-41c9ee96b8b8 + - 51bfddf5-86a6-4e01-8576-358c72a4a0a4 + - 58f8601f-dcff-4983-8e59-06b4631ca47f sources: - https://github.com/ziglang/zig tasks: From 44ef270de1b8dfd176f27adf5638ca2f36d700d5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 19:37:59 -0700 Subject: [PATCH 73/89] ci: use hut.lavatech.top for updating the download page --- .builds/freebsd.yml | 2 +- ci/srht/freebsd_script | 3 ++- ci/srht/on_master_success | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml index d53ab9167..37e6a6568 100644 --- a/.builds/freebsd.yml +++ b/.builds/freebsd.yml @@ -1,7 +1,7 @@ image: freebsd/latest secrets: - 51bfddf5-86a6-4e01-8576-358c72a4a0a4 - - 58f8601f-dcff-4983-8e59-06b4631ca47f + - 5cfede76-914e-4071-893e-e5e2e6ae3cea sources: - https://github.com/ziglang/zig tasks: diff --git a/ci/srht/freebsd_script b/ci/srht/freebsd_script index 539051f40..569c5277a 100755 --- a/ci/srht/freebsd_script +++ b/ci/srht/freebsd_script @@ -28,7 +28,8 @@ make $JOBS install release/bin/zig build test-fmt release/bin/zig build test-behavior -release/bin/zig build test-std +# TODO get these tests passing on freebsd and re-enable +#release/bin/zig build test-std release/bin/zig build test-compiler-rt release/bin/zig build test-compare-output release/bin/zig build test-standalone diff --git a/ci/srht/on_master_success b/ci/srht/on_master_success index c5a01f3bf..4c28d2af3 100755 --- a/ci/srht/on_master_success +++ b/ci/srht/on_master_success @@ -36,4 +36,4 @@ jq <$YML_FILE -sR '{ -H Authorization:"token $OAUTH_TOKEN" \ -H Content-Type:application/json \ -X POST \ - -d @- "https://builds.sr.ht/api/jobs" + -d @- "https://builds.hut.lavatech.top/api/jobs" From 5e39ab460073b0068d30a74488abf83333761c8d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 14 Sep 2020 21:11:34 -0700 Subject: [PATCH 74/89] ci: disable some freebsd tests to save time --- ci/srht/freebsd_script | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/srht/freebsd_script b/ci/srht/freebsd_script index 569c5277a..31aea6c3d 100755 --- a/ci/srht/freebsd_script +++ b/ci/srht/freebsd_script @@ -41,7 +41,8 @@ release/bin/zig build test-translate-c release/bin/zig build test-run-translated-c # TODO disabled until we are shipping self-hosted #release/bin/zig build test-gen-h -release/bin/zig build test-compile-errors +# TODO disabled to save time and hit that 45 minute limit +#release/bin/zig build test-compile-errors release/bin/zig build docs if [ -f ~/.s3cfg ]; then From c1c3212500377a8128f37a873a7845c6bf5993e6 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 15 Sep 2020 13:13:42 +0200 Subject: [PATCH 75/89] std: Fix typo in ELF section header iterator The code accidentally used the phdr offset instead of the shdr one while iterating over the section headers. Fixes #6338 --- lib/std/elf.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/elf.zig b/lib/std/elf.zig index cd2b5fcd0..9f5672121 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -471,7 +471,7 @@ pub const SectionHeaderIterator = struct { if (self.elf_header.is_64) { var shdr: Elf64_Shdr = undefined; - const offset = self.elf_header.phoff + @sizeOf(@TypeOf(shdr)) * self.index; + const offset = self.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * self.index; try preadNoEof(self.file, mem.asBytes(&shdr), offset); // ELF endianness matches native endianness. From ca85e367f4376a9365476386e1edbb6aada487c2 Mon Sep 17 00:00:00 2001 From: Samrat Man Singh Date: Tue, 15 Sep 2020 15:30:42 +0530 Subject: [PATCH 76/89] Use std.log in LibcInstallation `parse` instead of taking `stderr` --- src-self-hosted/libc_installation.zig | 17 +++++++++-------- src-self-hosted/stage2.zig | 6 +----- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src-self-hosted/libc_installation.zig b/src-self-hosted/libc_installation.zig index 65c6c8c16..fa2ef30cc 100644 --- a/src-self-hosted/libc_installation.zig +++ b/src-self-hosted/libc_installation.zig @@ -9,6 +9,8 @@ const is_darwin = Target.current.isDarwin(); const is_windows = Target.current.os.tag == .windows; const is_gnu = Target.current.isGnu(); +const log = std.log.scoped(.libc_installation); + usingnamespace @import("windows_sdk.zig"); /// See the render function implementation for documentation of the fields. @@ -37,7 +39,6 @@ pub const LibCInstallation = struct { pub fn parse( allocator: *Allocator, libc_file: []const u8, - stderr: anytype, ) !LibCInstallation { var self: LibCInstallation = .{}; @@ -62,7 +63,7 @@ pub const LibCInstallation = struct { if (line.len == 0 or line[0] == '#') continue; var line_it = std.mem.split(line, "="); const name = line_it.next() orelse { - try stderr.print("missing equal sign after field name\n", .{}); + log.err("missing equal sign after field name\n", .{}); return error.ParseError; }; const value = line_it.rest(); @@ -81,31 +82,31 @@ pub const LibCInstallation = struct { } inline for (fields) |field, i| { if (!found_keys[i].found) { - try stderr.print("missing field: {}\n", .{field.name}); + log.err("missing field: {}\n", .{field.name}); return error.ParseError; } } if (self.include_dir == null) { - try stderr.print("include_dir may not be empty\n", .{}); + log.err("include_dir may not be empty\n", .{}); return error.ParseError; } if (self.sys_include_dir == null) { - try stderr.print("sys_include_dir may not be empty\n", .{}); + log.err("sys_include_dir may not be empty\n", .{}); return error.ParseError; } if (self.crt_dir == null and !is_darwin) { - try stderr.print("crt_dir may not be empty for {}\n", .{@tagName(Target.current.os.tag)}); + log.err("crt_dir may not be empty for {}\n", .{@tagName(Target.current.os.tag)}); return error.ParseError; } if (self.msvc_lib_dir == null and is_windows and !is_gnu) { - try stderr.print("msvc_lib_dir may not be empty for {}-{}\n", .{ + log.err("msvc_lib_dir may not be empty for {}-{}\n", .{ @tagName(Target.current.os.tag), @tagName(Target.current.abi), }); return error.ParseError; } if (self.kernel32_lib_dir == null and is_windows and !is_gnu) { - try stderr.print("kernel32_lib_dir may not be empty for {}-{}\n", .{ + log.err("kernel32_lib_dir may not be empty for {}-{}\n", .{ @tagName(Target.current.os.tag), @tagName(Target.current.abi), }); diff --git a/src-self-hosted/stage2.zig b/src-self-hosted/stage2.zig index 45b8ad307..ac4d89bb2 100644 --- a/src-self-hosted/stage2.zig +++ b/src-self-hosted/stage2.zig @@ -598,12 +598,9 @@ const Stage2LibCInstallation = extern struct { // ABI warning export fn stage2_libc_parse(stage1_libc: *Stage2LibCInstallation, libc_file_z: [*:0]const u8) Error { - stderr_file = std.io.getStdErr(); - stderr = stderr_file.outStream(); const libc_file = mem.spanZ(libc_file_z); - var libc = LibCInstallation.parse(std.heap.c_allocator, libc_file, stderr) catch |err| switch (err) { + var libc = LibCInstallation.parse(std.heap.c_allocator, libc_file) catch |err| switch (err) { error.ParseError => return .SemanticAnalyzeFail, - error.DiskQuota => return .DiskQuota, error.FileTooBig => return .FileTooBig, error.InputOutput => return .FileSystem, error.NoSpaceLeft => return .NoSpaceLeft, @@ -612,7 +609,6 @@ export fn stage2_libc_parse(stage1_libc: *Stage2LibCInstallation, libc_file_z: [ error.SystemResources => return .SystemResources, error.OperationAborted => return .OperationAborted, error.WouldBlock => unreachable, - error.NotOpenForWriting => unreachable, error.NotOpenForReading => unreachable, error.Unexpected => return .Unexpected, error.IsDir => return .IsDir, From 7ccfb08a9389e4742e9a3237ce354c0492f7c040 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 15 Sep 2020 15:08:35 -0700 Subject: [PATCH 77/89] ci: update the on_master_success secret to lavatech --- ci/srht/on_master_success | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/srht/on_master_success b/ci/srht/on_master_success index 4c28d2af3..691c18a05 100755 --- a/ci/srht/on_master_success +++ b/ci/srht/on_master_success @@ -23,7 +23,7 @@ packages: - jq - xz secrets: - - 6c60aaee-92e7-4e7d-812c-114817689b4d + - 51bfddf5-86a6-4e01-8576-358c72a4a0a4 sources: - https://github.com/ziglang/zig tasks: From c35703825f17bb2b1108a371bb0559b89ff2b2a0 Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Wed, 16 Sep 2020 01:58:48 +0300 Subject: [PATCH 78/89] Add an error set --- lib/std/crypto/pbkdf2.zig | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig index 1ba176ab9..85c8e0110 100644 --- a/lib/std/crypto/pbkdf2.zig +++ b/lib/std/crypto/pbkdf2.zig @@ -36,6 +36,14 @@ const maxInt = std.math.maxInt; // Based on Apple's CommonKeyDerivation, based originally on code by Damien Bergamini. +pub const Pbkdf2Error = error{ + /// At least one round is required + TooFewRounds, + + /// Maximum length of the derived key is `maxInt(u32) * Prf.mac_length` + DerivedKeyTooLong, +}; + /// Apply PBKDF2 to generate a key from a password. /// /// PBKDF2 is defined in RFC 2898, and is a recommendation of NIST SP 800-132. @@ -54,7 +62,7 @@ const maxInt = std.math.maxInt; /// the derivedKey. It is common to tune this parameter to achieve approximately 100ms. /// /// Prf: Pseudo-random function to use. A common choice is `std.crypto.auth.hmac.HmacSha256`. -pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: u32, comptime Prf: type) !void { +pub fn pbkdf2(derivedKey: []u8, password: []const u8, salt: []const u8, rounds: u32, comptime Prf: type) Pbkdf2Error!void { if (rounds < 1) return error.TooFewRounds; const dkLen = derivedKey.len; From 7801a6d17f59091b62ae130dcd14cdd5cd2f180e Mon Sep 17 00:00:00 2001 From: Nathan Michaels Date: Tue, 15 Sep 2020 22:47:27 -0400 Subject: [PATCH 79/89] Fix a typo. --- doc/langref.html.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 10bc81e6d..dce23a43e 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -9728,7 +9728,7 @@ const c = @cImport({
  • Does not support Zig-only pointer attributes such as alignment. Use normal {#link|Pointers#} please!
    • -

    When a C pointer is pointing to a single struct (not an array), deference the C pointer to +

    When a C pointer is pointing to a single struct (not an array), dereference the C pointer to access to the struct's fields or member data. That syntax looks like this:

    {#syntax#}ptr_to_struct.*.struct_member{#endsyntax#}

    From 281fc10ec5aa8052490b9f951e0ceed1b7008ff4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Sep 2020 02:24:36 -0700 Subject: [PATCH 80/89] std.crypto siphash: fix assertion on the size of output buffer the logic was backwards --- lib/std/crypto/siphash.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/std/crypto/siphash.zig b/lib/std/crypto/siphash.zig index 26c892fdd..ae059b256 100644 --- a/lib/std/crypto/siphash.zig +++ b/lib/std/crypto/siphash.zig @@ -218,8 +218,9 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize) } /// Return an authentication tag for the current state + /// Assumes `out` is less than or equal to `mac_length`. pub fn final(self: *Self, out: []u8) void { - std.debug.assert(out.len >= mac_length); + std.debug.assert(out.len <= mac_length); mem.writeIntLittle(T, out[0..mac_length], self.state.final(self.buf[0..self.buf_len])); } From bb9a4ad6e903ad2f9c137bca56bdf2dd6fc65d03 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Wed, 16 Sep 2020 13:45:54 +0200 Subject: [PATCH 81/89] std: Fix {*} printing of non-pointer types Fixes a regression introduced in #6246. Adds a test to make sure this won't happen again. --- lib/std/fmt.zig | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/std/fmt.zig b/lib/std/fmt.zig index 8d3173395..b18c9f2f4 100644 --- a/lib/std/fmt.zig +++ b/lib/std/fmt.zig @@ -327,7 +327,7 @@ pub fn formatType( max_depth: usize, ) @TypeOf(writer).Error!void { if (comptime std.mem.eql(u8, fmt, "*")) { - try writer.writeAll(@typeName(@typeInfo(@TypeOf(value)).Pointer.child)); + try writer.writeAll(@typeName(std.meta.Child(@TypeOf(value)))); try writer.writeAll("@"); try formatInt(@ptrToInt(value), 16, false, FormatOptions{}, writer); return; @@ -1246,6 +1246,10 @@ test "optional" { const value: ?i32 = null; try testFmt("optional: null\n", "optional: {}\n", .{value}); } + { + const value = @intToPtr(?*i32, 0xf000d000); + try testFmt("optional: *i32@f000d000\n", "optional: {*}\n", .{value}); + } } test "error" { From 2962be81359a6806f66d220474abcb0b1cf7edf0 Mon Sep 17 00:00:00 2001 From: Tadeo Kondrak Date: Wed, 16 Sep 2020 03:29:38 -0600 Subject: [PATCH 82/89] stage1: fix @Type(.Union) not resolving its tag type Fixes https://github.com/ziglang/zig/issues/6339 --- src/analyze.cpp | 31 +++++++++++++------------- test/stage1/behavior/type.zig | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/src/analyze.cpp b/src/analyze.cpp index b53d26353..3ba4fd792 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -3155,22 +3155,7 @@ static Error resolve_union_zero_bits(CodeGen *g, ZigType *union_type) { tag_type->data.enumeration.fields_by_name.init(field_count); tag_type->data.enumeration.decls_scope = union_type->data.unionation.decls_scope; } else if (enum_type_node != nullptr) { - ZigType *enum_type = analyze_type_expr(g, scope, enum_type_node); - if (type_is_invalid(enum_type)) { - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - return ErrorSemanticAnalyzeFail; - } - if (enum_type->id != ZigTypeIdEnum) { - union_type->data.unionation.resolve_status = ResolveStatusInvalid; - add_node_error(g, enum_type_node, - buf_sprintf("expected enum tag type, found '%s'", buf_ptr(&enum_type->name))); - return ErrorSemanticAnalyzeFail; - } - if ((err = type_resolve(g, enum_type, ResolveStatusAlignmentKnown))) { - assert(g->errors.length != 0); - return err; - } - tag_type = enum_type; + tag_type = analyze_type_expr(g, scope, enum_type_node); } else { if (decl_node->type == NodeTypeContainerDecl) { tag_type = nullptr; @@ -3179,6 +3164,20 @@ static Error resolve_union_zero_bits(CodeGen *g, ZigType *union_type) { } } if (tag_type != nullptr) { + if (type_is_invalid(tag_type)) { + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + return ErrorSemanticAnalyzeFail; + } + if (tag_type->id != ZigTypeIdEnum) { + union_type->data.unionation.resolve_status = ResolveStatusInvalid; + add_node_error(g, enum_type_node != nullptr ? enum_type_node : decl_node, + buf_sprintf("expected enum tag type, found '%s'", buf_ptr(&tag_type->name))); + return ErrorSemanticAnalyzeFail; + } + if ((err = type_resolve(g, tag_type, ResolveStatusAlignmentKnown))) { + assert(g->errors.length != 0); + return err; + } covered_enum_fields = heap::c_allocator.allocate(tag_type->data.enumeration.src_field_count); } union_type->data.unionation.tag_type = tag_type; diff --git a/test/stage1/behavior/type.zig b/test/stage1/behavior/type.zig index eac76c9f9..38d23175d 100644 --- a/test/stage1/behavior/type.zig +++ b/test/stage1/behavior/type.zig @@ -374,3 +374,45 @@ test "Type.Union" { tagged = .{ .unsigned = 1 }; testing.expectEqual(Tag.unsigned, tagged); } + +test "Type.Union from Type.Enum" { + const Tag = @Type(.{ + .Enum = .{ + .layout = .Auto, + .tag_type = u0, + .fields = &[_]TypeInfo.EnumField{ + .{ .name = "working_as_expected", .value = 0 }, + }, + .decls = &[_]TypeInfo.Declaration{}, + .is_exhaustive = true, + }, + }); + const T = @Type(.{ + .Union = .{ + .layout = .Auto, + .tag_type = Tag, + .fields = &[_]TypeInfo.UnionField{ + .{ .name = "working_as_expected", .field_type = u32 }, + }, + .decls = &[_]TypeInfo.Declaration{}, + }, + }); + _ = T; + _ = @typeInfo(T).Union; +} + +test "Type.Union from regular enum" { + const E = enum { working_as_expected = 0 }; + const T = @Type(.{ + .Union = .{ + .layout = .Auto, + .tag_type = E, + .fields = &[_]TypeInfo.UnionField{ + .{ .name = "working_as_expected", .field_type = u32 }, + }, + .decls = &[_]TypeInfo.Declaration{}, + }, + }); + _ = T; + _ = @typeInfo(T).Union; +} From 27adb82fda516e95c3c03df80a95b895969fdd56 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Thu, 17 Sep 2020 00:41:26 +0200 Subject: [PATCH 83/89] std: Respect user-specified alignment when formatting ints This implementation tries to do the right thing (TM) by treating the sign as part of the number itself, therefore the alignment parameter applies to both the sign and the digits. In other words the format string `{:>4}` with -1 as input will not output `- 1` but ` -1`. And let's default to right alignment for everything as that's what users want, especially when printing numbers. Many implementations use different defaults for numeric vs non-numeric types, let's strive for a consistent behaviour here. --- lib/std/fmt.zig | 133 +++++++++++++++++++----------------------------- 1 file changed, 51 insertions(+), 82 deletions(-) diff --git a/lib/std/fmt.zig b/lib/std/fmt.zig index b18c9f2f4..56a1aba21 100644 --- a/lib/std/fmt.zig +++ b/lib/std/fmt.zig @@ -22,7 +22,7 @@ pub const Alignment = enum { pub const FormatOptions = struct { precision: ?usize = null, width: ?usize = null, - alignment: Alignment = .Left, + alignment: Alignment = .Right, fill: u8 = ' ', }; @@ -631,26 +631,22 @@ pub fn formatBuf( writer: anytype, ) !void { const width = options.width orelse buf.len; - var padding = if (width > buf.len) (width - buf.len) else 0; - const pad_byte = [1]u8{options.fill}; + const padding = if (width > buf.len) (width - buf.len) else 0; + switch (options.alignment) { .Left => { try writer.writeAll(buf); - while (padding > 0) : (padding -= 1) { - try writer.writeAll(&pad_byte); - } + try writer.writeByteNTimes(options.fill, padding); }, .Center => { - const padl = padding / 2; - var i: usize = 0; - while (i < padl) : (i += 1) try writer.writeAll(&pad_byte); + const left_padding = padding / 2; + const right_padding = (padding + 1) / 2; + try writer.writeByteNTimes(options.fill, left_padding); try writer.writeAll(buf); - while (i < padding) : (i += 1) try writer.writeAll(&pad_byte); + try writer.writeByteNTimes(options.fill, right_padding); }, .Right => { - while (padding > 0) : (padding -= 1) { - try writer.writeAll(&pad_byte); - } + try writer.writeByteNTimes(options.fill, padding); try writer.writeAll(buf); }, } @@ -941,61 +937,27 @@ pub fn formatInt( options: FormatOptions, writer: anytype, ) !void { + assert(base >= 2); + const int_value = if (@TypeOf(value) == comptime_int) blk: { const Int = math.IntFittingRange(value, value); break :blk @as(Int, value); } else value; - if (@typeInfo(@TypeOf(int_value)).Int.is_signed) { - return formatIntSigned(int_value, base, uppercase, options, writer); - } else { - return formatIntUnsigned(int_value, base, uppercase, options, writer); - } -} + const value_info = @typeInfo(@TypeOf(int_value)).Int; -fn formatIntSigned( - value: anytype, - base: u8, - uppercase: bool, - options: FormatOptions, - writer: anytype, -) !void { - const new_options = FormatOptions{ - .width = if (options.width) |w| (if (w == 0) 0 else w - 1) else null, - .precision = options.precision, - .fill = options.fill, - }; - const bit_count = @typeInfo(@TypeOf(value)).Int.bits; - const Uint = std.meta.Int(false, bit_count); - if (value < 0) { - try writer.writeAll("-"); - const new_value = math.absCast(value); - return formatIntUnsigned(new_value, base, uppercase, new_options, writer); - } else if (options.width == null or options.width.? == 0) { - return formatIntUnsigned(@intCast(Uint, value), base, uppercase, options, writer); - } else { - try writer.writeAll("+"); - const new_value = @intCast(Uint, value); - return formatIntUnsigned(new_value, base, uppercase, new_options, writer); - } -} + // The type must have the same size as `base` or be wider in order for the + // division to work + const min_int_bits = comptime math.max(value_info.bits, 8); + const MinInt = std.meta.Int(false, min_int_bits); -fn formatIntUnsigned( - value: anytype, - base: u8, - uppercase: bool, - options: FormatOptions, - writer: anytype, -) !void { - assert(base >= 2); - const value_info = @typeInfo(@TypeOf(value)).Int; - var buf: [math.max(value_info.bits, 1)]u8 = undefined; - const min_int_bits = comptime math.max(value_info.bits, @typeInfo(@TypeOf(base)).Int.bits); - const MinInt = std.meta.Int(value_info.is_signed, min_int_bits); - var a: MinInt = value; + const abs_value = math.absCast(int_value); + // The worst case in terms of space needed is base 2, plus 1 for the sign + var buf: [1 + math.max(value_info.bits, 1)]u8 = undefined; + + var a: MinInt = abs_value; var index: usize = buf.len; - while (true) { const digit = a % base; index -= 1; @@ -1004,25 +966,21 @@ fn formatIntUnsigned( if (a == 0) break; } - const digits_buf = buf[index..]; - const width = options.width orelse 0; - const padding = if (width > digits_buf.len) (width - digits_buf.len) else 0; - - if (padding > index) { - const zero_byte: u8 = options.fill; - var leftover_padding = padding - index; - while (true) { - try writer.writeAll(@as(*const [1]u8, &zero_byte)[0..]); - leftover_padding -= 1; - if (leftover_padding == 0) break; + if (value_info.is_signed) { + if (value < 0) { + // Negative integer + index -= 1; + buf[index] = '-'; + } else if (options.width == null or options.width.? == 0) { + // Positive integer, omit the plus sign + } else { + // Positive integer + index -= 1; + buf[index] = '+'; } - mem.set(u8, buf[0..index], options.fill); - return writer.writeAll(&buf); - } else { - const padded_buf = buf[index - padding ..]; - mem.set(u8, padded_buf[0..padding], options.fill); - return writer.writeAll(padded_buf); } + + return formatBuf(buf[index..], options, writer); } pub fn formatIntBuf(out_buf: []u8, value: anytype, base: u8, uppercase: bool, options: FormatOptions) usize { @@ -1287,7 +1245,17 @@ test "int.specifier" { test "int.padded" { try testFmt("u8: ' 1'", "u8: '{:4}'", .{@as(u8, 1)}); - try testFmt("u8: 'xxx1'", "u8: '{:x<4}'", .{@as(u8, 1)}); + try testFmt("u8: '1000'", "u8: '{:0<4}'", .{@as(u8, 1)}); + try testFmt("u8: '0001'", "u8: '{:0>4}'", .{@as(u8, 1)}); + try testFmt("u8: '0100'", "u8: '{:0^4}'", .{@as(u8, 1)}); + try testFmt("i8: '-1 '", "i8: '{:<4}'", .{@as(i8, -1)}); + try testFmt("i8: ' -1'", "i8: '{:>4}'", .{@as(i8, -1)}); + try testFmt("i8: ' -1 '", "i8: '{:^4}'", .{@as(i8, -1)}); + try testFmt("i16: '-1234'", "i16: '{:4}'", .{@as(i16, -1234)}); + try testFmt("i16: '+1234'", "i16: '{:4}'", .{@as(i16, 1234)}); + try testFmt("i16: '-12345'", "i16: '{:4}'", .{@as(i16, -12345)}); + try testFmt("i16: '+12345'", "i16: '{:4}'", .{@as(i16, 12345)}); + try testFmt("u16: '12345'", "u16: '{:4}'", .{@as(u16, 12345)}); } test "buffer" { @@ -1333,7 +1301,7 @@ test "slice" { try testFmt("slice: []const u8@deadbeef\n", "slice: {}\n", .{value}); } - try testFmt("buf: Test \n", "buf: {s:5}\n", .{"Test"}); + try testFmt("buf: Test\n", "buf: {s:5}\n", .{"Test"}); try testFmt("buf: Test\n Other text", "buf: {s}\n Other text", .{"Test"}); } @@ -1366,7 +1334,7 @@ test "cstr" { .{@ptrCast([*c]const u8, "Test C")}, ); try testFmt( - "cstr: Test C \n", + "cstr: Test C\n", "cstr: {s:10}\n", .{@ptrCast([*c]const u8, "Test C")}, ); @@ -1809,7 +1777,7 @@ test "vector" { try testFmt("{ true, false, true, false }", "{}", .{vbool}); try testFmt("{ -2, -1, 0, 1 }", "{}", .{vi64}); - try testFmt("{ - 2, - 1, + 0, + 1 }", "{d:5}", .{vi64}); + try testFmt("{ -2, -1, +0, +1 }", "{d:5}", .{vi64}); try testFmt("{ 1000, 2000, 3000, 4000 }", "{}", .{vu64}); try testFmt("{ 3e8, 7d0, bb8, fa0 }", "{x}", .{vu64}); try testFmt("{ 1kB, 2kB, 3kB, 4kB }", "{B}", .{vu64}); @@ -1822,15 +1790,16 @@ test "enum-literal" { test "padding" { try testFmt("Simple", "{}", .{"Simple"}); - try testFmt("true ", "{:10}", .{true}); + try testFmt(" true", "{:10}", .{true}); try testFmt(" true", "{:>10}", .{true}); try testFmt("======true", "{:=>10}", .{true}); try testFmt("true======", "{:=<10}", .{true}); try testFmt(" true ", "{:^10}", .{true}); try testFmt("===true===", "{:=^10}", .{true}); - try testFmt("Minimum width", "{:18} width", .{"Minimum"}); + try testFmt(" Minimum width", "{:18} width", .{"Minimum"}); try testFmt("==================Filled", "{:=>24}", .{"Filled"}); try testFmt(" Centered ", "{:^24}", .{"Centered"}); + try testFmt("-", "{:-^1}", .{""}); } test "decimal float padding" { From b7f9f779afa6de38f7599d8ffd9937fb48b9284c Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Thu, 17 Sep 2020 09:27:24 +0200 Subject: [PATCH 84/89] translate-c: Fix formatting of non-printable chars The two octets in hex notation must be aligned to the right and padded on the left, not the other way around. --- src-self-hosted/translate_c.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 1cdb5099e..a5619d56f 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -2032,7 +2032,7 @@ fn escapeChar(c: u8, char_buf: *[4]u8) []const u8 { // Handle the remaining escapes Zig doesn't support by turning them // into their respective hex representation else => if (std.ascii.isCntrl(c)) - std.fmt.bufPrint(char_buf, "\\x{x:0<2}", .{c}) catch unreachable + std.fmt.bufPrint(char_buf, "\\x{x:0>2}", .{c}) catch unreachable else std.fmt.bufPrint(char_buf, "{c}", .{c}) catch unreachable, }; From 5e3fa0e94f947c632aa584b9e13bfa2fe241fae1 Mon Sep 17 00:00:00 2001 From: Ryan Liptak Date: Wed, 16 Sep 2020 20:59:45 -0700 Subject: [PATCH 85/89] Add rename to std.fs API - Moves fs.rename functions to fs.renameAbsolute to match other functions outside of fs.Dir - Adds fs.Dir.rename that takes two paths relative to the given Dir - Adds fs.rename that takes two separate Dir's that the given paths are relative to (for renaming across directories without having to make the second path relative to a single directory) - Fixes FileNotFound error return in std.os.windows.MoveFileExW - Returns error.RenameAcrossMountPoints from renameatW + Matches the RenameAcrossMountPoints error return in renameatWasi/renameatZ --- lib/std/fs.zig | 66 +++++++++++++++-- lib/std/fs/test.zig | 161 +++++++++++++++++++++++++++++++++++++++++ lib/std/os.zig | 3 +- lib/std/os/windows.zig | 3 +- 4 files changed, 226 insertions(+), 7 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index a217fb3e9..1890d7e13 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -21,10 +21,6 @@ pub const wasi = @import("fs/wasi.zig"); // TODO audit these APIs with respect to Dir and absolute paths -pub const rename = os.rename; -pub const renameZ = os.renameZ; -pub const renameC = @compileError("deprecated: renamed to renameZ"); -pub const renameW = os.renameW; pub const realpath = os.realpath; pub const realpathZ = os.realpathZ; pub const realpathC = @compileError("deprecated: renamed to realpathZ"); @@ -90,7 +86,7 @@ pub fn atomicSymLink(allocator: *Allocator, existing_path: []const u8, new_path: base64_encoder.encode(tmp_path[dirname.len + 1 ..], &rand_buf); if (cwd().symLink(existing_path, tmp_path, .{})) { - return rename(tmp_path, new_path); + return cwd().rename(tmp_path, new_path); } else |err| switch (err) { error.PathAlreadyExists => continue, else => return err, // TODO zig should know this set does not include PathAlreadyExists @@ -255,6 +251,45 @@ pub fn deleteDirAbsoluteW(dir_path: [*:0]const u16) !void { return os.rmdirW(dir_path); } +pub const renameC = @compileError("deprecated: use renameZ, dir.renameZ, or renameAbsoluteZ"); + +/// Same as `Dir.rename` except the paths are absolute. +pub fn renameAbsolute(old_path: []const u8, new_path: []const u8) !void { + assert(path.isAbsolute(old_path)); + assert(path.isAbsolute(new_path)); + return os.rename(old_path, new_path); +} + +/// Same as `renameAbsolute` except the path parameters are null-terminated. +pub fn renameAbsoluteZ(old_path: [*:0]const u8, new_path: [*:0]const u8) !void { + assert(path.isAbsoluteZ(old_path)); + assert(path.isAbsoluteZ(new_path)); + return os.renameZ(old_path, new_path); +} + +/// Same as `renameAbsolute` except the path parameters are WTF-16 and target OS is assumed Windows. +pub fn renameAbsoluteW(old_path: [*:0]const u16, new_path: [*:0]const u16) !void { + assert(path.isAbsoluteWindowsW(old_path)); + assert(path.isAbsoluteWindowsW(new_path)); + return os.renameW(old_path, new_path); +} + +/// Same as `Dir.rename`, except `new_sub_path` is relative to `new_dir` +pub fn rename(old_dir: Dir, old_sub_path: []const u8, new_dir: Dir, new_sub_path: []const u8) !void { + return os.renameat(old_dir.fd, old_sub_path, new_dir.fd, new_sub_path); +} + +/// Same as `rename` except the parameters are null-terminated. +pub fn renameZ(old_dir: Dir, old_sub_path_z: [*:0]const u8, new_dir: Dir, new_sub_path_z: [*:0]const u8) !void { + return os.renameatZ(old_dir.fd, old_sub_path_z, new_dir.fd, new_sub_path_z); +} + +/// Same as `rename` except the parameters are UTF16LE, NT prefixed. +/// This function is Windows-only. +pub fn renameW(old_dir: Dir, old_sub_path_w: []const u16, new_dir: Dir, new_sub_path_w: []const u16) !void { + return os.renameatW(old_dir.fd, old_sub_path_w, new_dir.fd, new_sub_path_w); +} + pub const Dir = struct { fd: os.fd_t, @@ -1338,6 +1373,27 @@ pub const Dir = struct { }; } + pub const RenameError = os.RenameError; + + /// Change the name or location of a file or directory. + /// If new_sub_path already exists, it will be replaced. + /// Renaming a file over an existing directory or a directory + /// over an existing file will fail with `error.IsDir` or `error.NotDir` + pub fn rename(self: Dir, old_sub_path: []const u8, new_sub_path: []const u8) RenameError!void { + return os.renameat(self.fd, old_sub_path, self.fd, new_sub_path); + } + + /// Same as `rename` except the parameters are null-terminated. + pub fn renameZ(self: Dir, old_sub_path_z: [*:0]const u8, new_sub_path_z: [*:0]const u8) RenameError!void { + return os.renameatZ(self.fd, old_sub_path_z, self.fd, new_sub_path_z); + } + + /// Same as `rename` except the parameters are UTF16LE, NT prefixed. + /// This function is Windows-only. + pub fn renameW(self: Dir, old_sub_path_w: []const u16, new_sub_path_w: []const u16) RenameError!void { + return os.renameatW(self.fd, old_sub_path_w, self.fd, new_sub_path_w); + } + /// Creates a symbolic link named `sym_link_path` which contains the string `target_path`. /// A symbolic link (also known as a soft link) may point to an existing file or to a nonexistent /// one; the latter case is known as a dangling link. diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index a59bc4624..b3cc1fe56 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -274,6 +274,167 @@ test "file operations on directories" { dir.close(); } +test "Dir.rename files" { + var tmp_dir = tmpDir(.{}); + defer tmp_dir.cleanup(); + + testing.expectError(error.FileNotFound, tmp_dir.dir.rename("missing_file_name", "something_else")); + + // Renaming files + const test_file_name = "test_file"; + const renamed_test_file_name = "test_file_renamed"; + var file = try tmp_dir.dir.createFile(test_file_name, .{ .read = true }); + file.close(); + try tmp_dir.dir.rename(test_file_name, renamed_test_file_name); + + // Ensure the file was renamed + testing.expectError(error.FileNotFound, tmp_dir.dir.openFile(test_file_name, .{})); + file = try tmp_dir.dir.openFile(renamed_test_file_name, .{}); + file.close(); + + // Rename to self succeeds + try tmp_dir.dir.rename(renamed_test_file_name, renamed_test_file_name); + + // Rename to existing file succeeds + var existing_file = try tmp_dir.dir.createFile("existing_file", .{ .read = true }); + existing_file.close(); + try tmp_dir.dir.rename(renamed_test_file_name, "existing_file"); + + testing.expectError(error.FileNotFound, tmp_dir.dir.openFile(renamed_test_file_name, .{})); + file = try tmp_dir.dir.openFile("existing_file", .{}); + file.close(); +} + +test "Dir.rename directories" { + // TODO: Fix on Windows, see https://github.com/ziglang/zig/issues/6364 + if (builtin.os.tag == .windows) return error.SkipZigTest; + + var tmp_dir = tmpDir(.{}); + defer tmp_dir.cleanup(); + + // Renaming directories + try tmp_dir.dir.makeDir("test_dir"); + try tmp_dir.dir.rename("test_dir", "test_dir_renamed"); + + // Ensure the directory was renamed + testing.expectError(error.FileNotFound, tmp_dir.dir.openDir("test_dir", .{})); + var dir = try tmp_dir.dir.openDir("test_dir_renamed", .{}); + + // Put a file in the directory + var file = try dir.createFile("test_file", .{ .read = true }); + file.close(); + dir.close(); + + try tmp_dir.dir.rename("test_dir_renamed", "test_dir_renamed_again"); + + // Ensure the directory was renamed and the file still exists in it + testing.expectError(error.FileNotFound, tmp_dir.dir.openDir("test_dir_renamed", .{})); + dir = try tmp_dir.dir.openDir("test_dir_renamed_again", .{}); + file = try dir.openFile("test_file", .{}); + file.close(); + dir.close(); + + // Try to rename to a non-empty directory now + var target_dir = try tmp_dir.dir.makeOpenPath("non_empty_target_dir", .{}); + file = try target_dir.createFile("filler", .{ .read = true }); + file.close(); + + testing.expectError(error.PathAlreadyExists, tmp_dir.dir.rename("test_dir_renamed_again", "non_empty_target_dir")); + + // Ensure the directory was not renamed + dir = try tmp_dir.dir.openDir("test_dir_renamed_again", .{}); + file = try dir.openFile("test_file", .{}); + file.close(); + dir.close(); +} + +test "Dir.rename file <-> dir" { + // TODO: Fix on Windows, see https://github.com/ziglang/zig/issues/6364 + if (builtin.os.tag == .windows) return error.SkipZigTest; + + var tmp_dir = tmpDir(.{}); + defer tmp_dir.cleanup(); + + var file = try tmp_dir.dir.createFile("test_file", .{ .read = true }); + file.close(); + try tmp_dir.dir.makeDir("test_dir"); + testing.expectError(error.IsDir, tmp_dir.dir.rename("test_file", "test_dir")); + testing.expectError(error.NotDir, tmp_dir.dir.rename("test_dir", "test_file")); +} + +test "rename" { + var tmp_dir1 = tmpDir(.{}); + defer tmp_dir1.cleanup(); + + var tmp_dir2 = tmpDir(.{}); + defer tmp_dir2.cleanup(); + + // Renaming files + const test_file_name = "test_file"; + const renamed_test_file_name = "test_file_renamed"; + var file = try tmp_dir1.dir.createFile(test_file_name, .{ .read = true }); + file.close(); + try fs.rename(tmp_dir1.dir, test_file_name, tmp_dir2.dir, renamed_test_file_name); + + // ensure the file was renamed + testing.expectError(error.FileNotFound, tmp_dir1.dir.openFile(test_file_name, .{})); + file = try tmp_dir2.dir.openFile(renamed_test_file_name, .{}); + file.close(); +} + +test "renameAbsolute" { + if (builtin.os.tag == .wasi) return error.SkipZigTest; + + var tmp_dir = tmpDir(.{}); + defer tmp_dir.cleanup(); + + // Get base abs path + var arena = ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + const allocator = &arena.allocator; + + const base_path = blk: { + const relative_path = try fs.path.join(&arena.allocator, &[_][]const u8{ "zig-cache", "tmp", tmp_dir.sub_path[0..] }); + break :blk try fs.realpathAlloc(&arena.allocator, relative_path); + }; + + testing.expectError(error.FileNotFound, fs.renameAbsolute( + try fs.path.join(allocator, &[_][]const u8{ base_path, "missing_file_name" }), + try fs.path.join(allocator, &[_][]const u8{ base_path, "something_else" }), + )); + + // Renaming files + const test_file_name = "test_file"; + const renamed_test_file_name = "test_file_renamed"; + var file = try tmp_dir.dir.createFile(test_file_name, .{ .read = true }); + file.close(); + try fs.renameAbsolute( + try fs.path.join(allocator, &[_][]const u8{ base_path, test_file_name }), + try fs.path.join(allocator, &[_][]const u8{ base_path, renamed_test_file_name }), + ); + + // ensure the file was renamed + testing.expectError(error.FileNotFound, tmp_dir.dir.openFile(test_file_name, .{})); + file = try tmp_dir.dir.openFile(renamed_test_file_name, .{}); + const stat = try file.stat(); + testing.expect(stat.kind == .File); + file.close(); + + // Renaming directories + const test_dir_name = "test_dir"; + const renamed_test_dir_name = "test_dir_renamed"; + try tmp_dir.dir.makeDir(test_dir_name); + try fs.renameAbsolute( + try fs.path.join(allocator, &[_][]const u8{ base_path, test_dir_name }), + try fs.path.join(allocator, &[_][]const u8{ base_path, renamed_test_dir_name }), + ); + + // ensure the directory was renamed + testing.expectError(error.FileNotFound, tmp_dir.dir.openDir(test_dir_name, .{})); + var dir = try tmp_dir.dir.openDir(renamed_test_dir_name, .{}); + dir.close(); +} + test "openSelfExe" { if (builtin.os.tag == .wasi) return error.SkipZigTest; diff --git a/lib/std/os.zig b/lib/std/os.zig index bdc746419..7fc63d785 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -1890,7 +1890,7 @@ pub fn unlinkatW(dirfd: fd_t, sub_path_w: []const u16, flags: u32) UnlinkatError return windows.DeleteFile(sub_path_w, .{ .dir = dirfd, .remove_dir = remove_dir }); } -const RenameError = error{ +pub const RenameError = error{ /// In WASI, this error may occur when the file descriptor does /// not hold the required rights to rename a resource by path relative to it. AccessDenied, @@ -2107,6 +2107,7 @@ pub fn renameatW( .ACCESS_DENIED => return error.AccessDenied, .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, + .NOT_SAME_DEVICE => error.RenameAcrossMountPoints, else => return windows.unexpectedStatus(rc), } } diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index bd9dc8b32..de0d0ea45 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -828,7 +828,7 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil } } -pub const MoveFileError = error{Unexpected}; +pub const MoveFileError = error{ FileNotFound, Unexpected }; pub fn MoveFileEx(old_path: []const u8, new_path: []const u8, flags: DWORD) MoveFileError!void { const old_path_w = try sliceToPrefixedFileW(old_path); @@ -839,6 +839,7 @@ pub fn MoveFileEx(old_path: []const u8, new_path: []const u8, flags: DWORD) Move pub fn MoveFileExW(old_path: [*:0]const u16, new_path: [*:0]const u16, flags: DWORD) MoveFileError!void { if (kernel32.MoveFileExW(old_path, new_path, flags) == 0) { switch (kernel32.GetLastError()) { + .FILE_NOT_FOUND => return error.FileNotFound, else => |err| return unexpectedError(err), } } From fbde15fdf41024d977401ab2a5c52e60b67d48fa Mon Sep 17 00:00:00 2001 From: Ryan Liptak Date: Thu, 17 Sep 2020 15:47:09 -0700 Subject: [PATCH 86/89] Fix compile error in os.renameatW Introduced in 5e3fa0e94f947c632aa584b9e13bfa2fe241fae1 Whoops! --- lib/std/os.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 7fc63d785..0b09b1f82 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -2107,7 +2107,7 @@ pub fn renameatW( .ACCESS_DENIED => return error.AccessDenied, .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, - .NOT_SAME_DEVICE => error.RenameAcrossMountPoints, + .NOT_SAME_DEVICE => return error.RenameAcrossMountPoints, else => return windows.unexpectedStatus(rc), } } From 4fbf9f7f79c8e0df525f9932878995e6a6782ac4 Mon Sep 17 00:00:00 2001 From: zenith391 Date: Fri, 18 Sep 2020 18:58:54 +0200 Subject: [PATCH 87/89] Add "emit_docs" field to LibExeObjStep. --- lib/std/build.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/std/build.zig b/lib/std/build.zig index 30ecf98b7..69f44bad3 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -1188,6 +1188,7 @@ pub const LibExeObjStep = struct { emit_llvm_ir: bool = false, emit_asm: bool = false, emit_bin: bool = true, + emit_docs: bool = false, emit_h: bool = false, bundle_compiler_rt: bool, disable_stack_probing: bool, @@ -2033,6 +2034,7 @@ pub const LibExeObjStep = struct { if (self.emit_llvm_ir) try zig_args.append("-femit-llvm-ir"); if (self.emit_asm) try zig_args.append("-femit-asm"); if (!self.emit_bin) try zig_args.append("-fno-emit-bin"); + if (self.emit_docs) try zig_args.append("-femit-docs"); if (self.emit_h) try zig_args.append("-femit-h"); if (self.strip) { From f92d01c8a86b525c2caaf07497e16841905cea8c Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 18 Sep 2020 21:25:37 +0200 Subject: [PATCH 88/89] stage1: Fix edge case in casting between optional types Closes #6370 --- src/ir.cpp | 24 +++++++++++++++++++++++- test/stage1/behavior/cast.zig | 5 +++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/ir.cpp b/src/ir.cpp index 6cb5d8bc2..7d446c82a 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -63,6 +63,7 @@ enum ConstCastResultId { ConstCastResultIdPointerChild, ConstCastResultIdSliceChild, ConstCastResultIdOptionalChild, + ConstCastResultIdOptionalShape, ConstCastResultIdErrorUnionPayload, ConstCastResultIdErrorUnionErrorSet, ConstCastResultIdFnAlign, @@ -11946,8 +11947,22 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted } } - // maybe + // optional types if (wanted_type->id == ZigTypeIdOptional && actual_type->id == ZigTypeIdOptional) { + // Consider the case where the wanted type is ??[*]T and the actual one + // is ?[*]T, we cannot turn the former into the latter even though the + // child types are compatible (?[*]T and [*]T are both represented as a + // pointer). The extra level of indirection in ??[*]T means it's + // represented as a regular, fat, optional type and, as a consequence, + // has a different shape than the one of ?[*]T. + if ((wanted_ptr_type != nullptr) != (actual_ptr_type != nullptr)) { + // The use of type_mismatch is intentional + result.id = ConstCastResultIdOptionalShape; + result.data.type_mismatch = heap::c_allocator.allocate_nonzero(1); + result.data.type_mismatch->wanted_type = wanted_type; + result.data.type_mismatch->actual_type = actual_type; + return result; + } ConstCastOnly child = types_match_const_cast_only(ira, wanted_type->data.maybe.child_type, actual_type->data.maybe.child_type, source_node, wanted_is_mutable); if (child.id == ConstCastResultIdInvalid) @@ -14549,6 +14564,13 @@ static void report_recursive_error(IrAnalyze *ira, AstNode *source_node, ConstCa report_recursive_error(ira, source_node, &cast_result->data.optional->child, msg); break; } + case ConstCastResultIdOptionalShape: { + add_error_note(ira->codegen, parent_msg, source_node, + buf_sprintf("optional type child '%s' cannot cast into optional type '%s'", + buf_ptr(&cast_result->data.type_mismatch->actual_type->name), + buf_ptr(&cast_result->data.type_mismatch->wanted_type->name))); + break; + } case ConstCastResultIdErrorUnionErrorSet: { ErrorMsg *msg = add_error_note(ira->codegen, parent_msg, source_node, buf_sprintf("error set '%s' cannot cast into error set '%s'", diff --git a/test/stage1/behavior/cast.zig b/test/stage1/behavior/cast.zig index 4b678cd2d..ce0d16d1a 100644 --- a/test/stage1/behavior/cast.zig +++ b/test/stage1/behavior/cast.zig @@ -849,3 +849,8 @@ test "comptime float casts" { expect(b == 2); expect(@TypeOf(b) == comptime_int); } + +test "cast from ?[*]T to ??[*]T" { + const a: ??[*]u8 = @as(?[*]u8, null); + expect(a != null and a.? == null); +} From 58ee5f4e61cd9b7a9ba65798e2214efa3753a733 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sun, 20 Sep 2020 00:10:53 +0200 Subject: [PATCH 89/89] std: Fix metadata corruption in HeapAllocator HeapAllocator stores the pointer returned by HeapAlloc right after the data block and, after the recent allocator refactoring, the space for this pointer was not taken into account in the calculation of the final block size. Fixes #5830 --- lib/std/heap.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 6db1be539..16de215cc 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -489,7 +489,7 @@ pub const HeapAllocator = switch (builtin.os.tag) { const full_len = os.windows.kernel32.HeapSize(heap_handle, 0, ptr); assert(full_len != std.math.maxInt(usize)); assert(full_len >= amt); - break :init mem.alignBackwardAnyAlign(full_len - (aligned_addr - root_addr), len_align); + break :init mem.alignBackwardAnyAlign(full_len - (aligned_addr - root_addr) - @sizeOf(usize), len_align); }; const buf = @intToPtr([*]u8, aligned_addr)[0..return_len]; getRecordPtr(buf).* = root_addr;