From 2d4b95900e7d7273dbd3ce6b7b9a33d2a19779da Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 4 Sep 2018 18:15:27 -0400 Subject: [PATCH 01/46] stage1: import blake2b implementation from master branch of blake2 reference implementation 320c325437539ae91091ce62efec1913cd8093c2 --- CMakeLists.txt | 1 + src/blake2.h | 196 ++++++++++++++++++ src/blake2b.cpp | 539 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 736 insertions(+) create mode 100644 src/blake2.h create mode 100644 src/blake2b.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c3ee37ea8..fbf5b03ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -407,6 +407,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/ast_render.cpp" "${CMAKE_SOURCE_DIR}/src/bigfloat.cpp" "${CMAKE_SOURCE_DIR}/src/bigint.cpp" + "${CMAKE_SOURCE_DIR}/src/blake2b.cpp" "${CMAKE_SOURCE_DIR}/src/buffer.cpp" "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" diff --git a/src/blake2.h b/src/blake2.h new file mode 100644 index 000000000..6420c5367 --- /dev/null +++ b/src/blake2.h @@ -0,0 +1,196 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#ifndef BLAKE2_H +#define BLAKE2_H + +#include +#include + +#if defined(_MSC_VER) +#define BLAKE2_PACKED(x) __pragma(pack(push, 1)) x __pragma(pack(pop)) +#else +#define BLAKE2_PACKED(x) x __attribute__((packed)) +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + + enum blake2s_constant + { + BLAKE2S_BLOCKBYTES = 64, + BLAKE2S_OUTBYTES = 32, + BLAKE2S_KEYBYTES = 32, + BLAKE2S_SALTBYTES = 8, + BLAKE2S_PERSONALBYTES = 8 + }; + + enum blake2b_constant + { + BLAKE2B_BLOCKBYTES = 128, + BLAKE2B_OUTBYTES = 64, + BLAKE2B_KEYBYTES = 64, + BLAKE2B_SALTBYTES = 16, + BLAKE2B_PERSONALBYTES = 16 + }; + + typedef struct blake2s_state__ + { + uint32_t h[8]; + uint32_t t[2]; + uint32_t f[2]; + uint8_t buf[BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2s_state; + + typedef struct blake2b_state__ + { + uint64_t h[8]; + uint64_t t[2]; + uint64_t f[2]; + uint8_t buf[BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2b_state; + + typedef struct blake2sp_state__ + { + blake2s_state S[8][1]; + blake2s_state R[1]; + uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2sp_state; + + typedef struct blake2bp_state__ + { + blake2b_state S[4][1]; + blake2b_state R[1]; + uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2bp_state; + + + BLAKE2_PACKED(struct blake2s_param__ + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint16_t xof_length; /* 14 */ + uint8_t node_depth; /* 15 */ + uint8_t inner_length; /* 16 */ + /* uint8_t reserved[0]; */ + uint8_t salt[BLAKE2S_SALTBYTES]; /* 24 */ + uint8_t personal[BLAKE2S_PERSONALBYTES]; /* 32 */ + }); + + typedef struct blake2s_param__ blake2s_param; + + BLAKE2_PACKED(struct blake2b_param__ + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint32_t xof_length; /* 16 */ + uint8_t node_depth; /* 17 */ + uint8_t inner_length; /* 18 */ + uint8_t reserved[14]; /* 32 */ + uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ + uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ + }); + + typedef struct blake2b_param__ blake2b_param; + + typedef struct blake2xs_state__ + { + blake2s_state S[1]; + blake2s_param P[1]; + } blake2xs_state; + + typedef struct blake2xb_state__ + { + blake2b_state S[1]; + blake2b_param P[1]; + } blake2xb_state; + + /* Padded structs result in a compile-time error */ + enum { + BLAKE2_DUMMY_1 = 1/(sizeof(blake2s_param) == BLAKE2S_OUTBYTES), + BLAKE2_DUMMY_2 = 1/(sizeof(blake2b_param) == BLAKE2B_OUTBYTES) + }; + + /* Streaming API */ + int blake2s_init( blake2s_state *S, size_t outlen ); + int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); + int blake2s_update( blake2s_state *S, const void *in, size_t inlen ); + int blake2s_final( blake2s_state *S, void *out, size_t outlen ); + + int blake2b_init( blake2b_state *S, size_t outlen ); + int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); + int blake2b_update( blake2b_state *S, const void *in, size_t inlen ); + int blake2b_final( blake2b_state *S, void *out, size_t outlen ); + + int blake2sp_init( blake2sp_state *S, size_t outlen ); + int blake2sp_init_key( blake2sp_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2sp_update( blake2sp_state *S, const void *in, size_t inlen ); + int blake2sp_final( blake2sp_state *S, void *out, size_t outlen ); + + int blake2bp_init( blake2bp_state *S, size_t outlen ); + int blake2bp_init_key( blake2bp_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2bp_update( blake2bp_state *S, const void *in, size_t inlen ); + int blake2bp_final( blake2bp_state *S, void *out, size_t outlen ); + + /* Variable output length API */ + int blake2xs_init( blake2xs_state *S, const size_t outlen ); + int blake2xs_init_key( blake2xs_state *S, const size_t outlen, const void *key, size_t keylen ); + int blake2xs_update( blake2xs_state *S, const void *in, size_t inlen ); + int blake2xs_final(blake2xs_state *S, void *out, size_t outlen); + + int blake2xb_init( blake2xb_state *S, const size_t outlen ); + int blake2xb_init_key( blake2xb_state *S, const size_t outlen, const void *key, size_t keylen ); + int blake2xb_update( blake2xb_state *S, const void *in, size_t inlen ); + int blake2xb_final(blake2xb_state *S, void *out, size_t outlen); + + /* Simple API */ + int blake2s( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + int blake2sp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2bp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + int blake2xs( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2xb( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + /* This is simply an alias for blake2b */ + int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + +#if defined(__cplusplus) +} +#endif + +#endif + diff --git a/src/blake2b.cpp b/src/blake2b.cpp new file mode 100644 index 000000000..600f951b9 --- /dev/null +++ b/src/blake2b.cpp @@ -0,0 +1,539 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ + +#include +#include +#include + +#include "blake2.h" +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#ifndef BLAKE2_IMPL_H +#define BLAKE2_IMPL_H + +#include +#include + +#if !defined(__cplusplus) && (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L) + #if defined(_MSC_VER) + #define BLAKE2_INLINE __inline + #elif defined(__GNUC__) + #define BLAKE2_INLINE __inline__ + #else + #define BLAKE2_INLINE + #endif +#else + #define BLAKE2_INLINE inline +#endif + +static BLAKE2_INLINE uint32_t load32( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint32_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint32_t )( p[0] ) << 0) | + (( uint32_t )( p[1] ) << 8) | + (( uint32_t )( p[2] ) << 16) | + (( uint32_t )( p[3] ) << 24) ; +#endif +} + +static BLAKE2_INLINE uint64_t load64( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint64_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint64_t )( p[0] ) << 0) | + (( uint64_t )( p[1] ) << 8) | + (( uint64_t )( p[2] ) << 16) | + (( uint64_t )( p[3] ) << 24) | + (( uint64_t )( p[4] ) << 32) | + (( uint64_t )( p[5] ) << 40) | + (( uint64_t )( p[6] ) << 48) | + (( uint64_t )( p[7] ) << 56) ; +#endif +} + +static BLAKE2_INLINE uint16_t load16( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint16_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return ( uint16_t )((( uint32_t )( p[0] ) << 0) | + (( uint32_t )( p[1] ) << 8)); +#endif +} + +static BLAKE2_INLINE void store16( void *dst, uint16_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +#endif +} + +static BLAKE2_INLINE void store32( void *dst, uint32_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); +#endif +} + +static BLAKE2_INLINE void store64( void *dst, uint64_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); + p[4] = (uint8_t)(w >> 32); + p[5] = (uint8_t)(w >> 40); + p[6] = (uint8_t)(w >> 48); + p[7] = (uint8_t)(w >> 56); +#endif +} + +static BLAKE2_INLINE uint64_t load48( const void *src ) +{ + const uint8_t *p = ( const uint8_t * )src; + return (( uint64_t )( p[0] ) << 0) | + (( uint64_t )( p[1] ) << 8) | + (( uint64_t )( p[2] ) << 16) | + (( uint64_t )( p[3] ) << 24) | + (( uint64_t )( p[4] ) << 32) | + (( uint64_t )( p[5] ) << 40) ; +} + +static BLAKE2_INLINE void store48( void *dst, uint64_t w ) +{ + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); + p[4] = (uint8_t)(w >> 32); + p[5] = (uint8_t)(w >> 40); +} + +static BLAKE2_INLINE uint32_t rotr32( const uint32_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 32 - c ) ); +} + +static BLAKE2_INLINE uint64_t rotr64( const uint64_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 64 - c ) ); +} + +/* prevents compiler optimizing out memset() */ +static BLAKE2_INLINE void secure_zero_memory(void *v, size_t n) +{ + static void *(*const volatile memset_v)(void *, int, size_t) = &memset; + memset_v(v, 0, n); +} + +#endif + +static const uint64_t blake2b_IV[8] = +{ + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL +}; + +static const uint8_t blake2b_sigma[12][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } +}; + + +static void blake2b_set_lastnode( blake2b_state *S ) +{ + S->f[1] = (uint64_t)-1; +} + +/* Some helper functions, not necessarily useful */ +static int blake2b_is_lastblock( const blake2b_state *S ) +{ + return S->f[0] != 0; +} + +static void blake2b_set_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_set_lastnode( S ); + + S->f[0] = (uint64_t)-1; +} + +static void blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) +{ + S->t[0] += inc; + S->t[1] += ( S->t[0] < inc ); +} + +static void blake2b_init0( blake2b_state *S ) +{ + size_t i; + memset( S, 0, sizeof( blake2b_state ) ); + + for( i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; +} + +/* init xors IV with input parameter block */ +int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) +{ + const uint8_t *p = ( const uint8_t * )( P ); + size_t i; + + blake2b_init0( S ); + + /* IV XOR ParamBlock */ + for( i = 0; i < 8; ++i ) + S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); + + S->outlen = P->digest_length; + return 0; +} + + + +int blake2b_init( blake2b_state *S, size_t outlen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + P->digest_length = (uint8_t)outlen; + P->key_length = 0; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store32( &P->node_offset, 0 ); + store32( &P->xof_length, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + return blake2b_init_param( S, P ); +} + + +int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; + + P->digest_length = (uint8_t)outlen; + P->key_length = (uint8_t)keylen; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store32( &P->node_offset, 0 ); + store32( &P->xof_length, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + + if( blake2b_init_param( S, P ) < 0 ) return -1; + + { + uint8_t block[BLAKE2B_BLOCKBYTES]; + memset( block, 0, BLAKE2B_BLOCKBYTES ); + memcpy( block, key, keylen ); + blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); + secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ + } + return 0; +} + +#define G(r,i,a,b,c,d) \ + do { \ + a = a + b + m[blake2b_sigma[r][2*i+0]]; \ + d = rotr64(d ^ a, 32); \ + c = c + d; \ + b = rotr64(b ^ c, 24); \ + a = a + b + m[blake2b_sigma[r][2*i+1]]; \ + d = rotr64(d ^ a, 16); \ + c = c + d; \ + b = rotr64(b ^ c, 63); \ + } while(0) + +#define ROUND(r) \ + do { \ + G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ + G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ + G(r,2,v[ 2],v[ 6],v[10],v[14]); \ + G(r,3,v[ 3],v[ 7],v[11],v[15]); \ + G(r,4,v[ 0],v[ 5],v[10],v[15]); \ + G(r,5,v[ 1],v[ 6],v[11],v[12]); \ + G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ + G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ + } while(0) + +static void blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) +{ + uint64_t m[16]; + uint64_t v[16]; + size_t i; + + for( i = 0; i < 16; ++i ) { + m[i] = load64( block + i * sizeof( m[i] ) ); + } + + for( i = 0; i < 8; ++i ) { + v[i] = S->h[i]; + } + + v[ 8] = blake2b_IV[0]; + v[ 9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = blake2b_IV[4] ^ S->t[0]; + v[13] = blake2b_IV[5] ^ S->t[1]; + v[14] = blake2b_IV[6] ^ S->f[0]; + v[15] = blake2b_IV[7] ^ S->f[1]; + + ROUND( 0 ); + ROUND( 1 ); + ROUND( 2 ); + ROUND( 3 ); + ROUND( 4 ); + ROUND( 5 ); + ROUND( 6 ); + ROUND( 7 ); + ROUND( 8 ); + ROUND( 9 ); + ROUND( 10 ); + ROUND( 11 ); + + for( i = 0; i < 8; ++i ) { + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + } +} + +#undef G +#undef ROUND + +int blake2b_update( blake2b_state *S, const void *pin, size_t inlen ) +{ + const unsigned char * in = (const unsigned char *)pin; + if( inlen > 0 ) + { + size_t left = S->buflen; + size_t fill = BLAKE2B_BLOCKBYTES - left; + if( inlen > fill ) + { + S->buflen = 0; + memcpy( S->buf + left, in, fill ); /* Fill buffer */ + blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); + blake2b_compress( S, S->buf ); /* Compress */ + in += fill; inlen -= fill; + while(inlen > BLAKE2B_BLOCKBYTES) { + blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); + blake2b_compress( S, in ); + in += BLAKE2B_BLOCKBYTES; + inlen -= BLAKE2B_BLOCKBYTES; + } + } + memcpy( S->buf + S->buflen, in, inlen ); + S->buflen += inlen; + } + return 0; +} + +int blake2b_final( blake2b_state *S, void *out, size_t outlen ) +{ + uint8_t buffer[BLAKE2B_OUTBYTES] = {0}; + size_t i; + + if( out == NULL || outlen < S->outlen ) + return -1; + + if( blake2b_is_lastblock( S ) ) + return -1; + + blake2b_increment_counter( S, S->buflen ); + blake2b_set_lastblock( S ); + memset( S->buf + S->buflen, 0, BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ + blake2b_compress( S, S->buf ); + + for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ + store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); + + memcpy( out, buffer, S->outlen ); + secure_zero_memory(buffer, sizeof(buffer)); + return 0; +} + +/* inlen, at least, should be uint64_t. Others can be size_t. */ +int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) +{ + blake2b_state S[1]; + + /* Verify parameters */ + if ( NULL == in && inlen > 0 ) return -1; + + if ( NULL == out ) return -1; + + if( NULL == key && keylen > 0 ) return -1; + + if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; + + if( keylen > BLAKE2B_KEYBYTES ) return -1; + + if( keylen > 0 ) + { + if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; + } + else + { + if( blake2b_init( S, outlen ) < 0 ) return -1; + } + + blake2b_update( S, ( const uint8_t * )in, inlen ); + blake2b_final( S, out, outlen ); + return 0; +} + +int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) { + return blake2b(out, outlen, in, inlen, key, keylen); +} + +#if defined(SUPERCOP) +int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) +{ + return blake2b( out, BLAKE2B_OUTBYTES, in, inlen, NULL, 0 ); +} +#endif + +#if defined(BLAKE2B_SELFTEST) +#include +#include "blake2-kat.h" +int main( void ) +{ + uint8_t key[BLAKE2B_KEYBYTES]; + uint8_t buf[BLAKE2_KAT_LENGTH]; + size_t i, step; + + for( i = 0; i < BLAKE2B_KEYBYTES; ++i ) + key[i] = ( uint8_t )i; + + for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) + buf[i] = ( uint8_t )i; + + /* Test simple API */ + for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) + { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b( hash, BLAKE2B_OUTBYTES, buf, i, key, BLAKE2B_KEYBYTES ); + + if( 0 != memcmp( hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES ) ) + { + goto fail; + } + } + + /* Test streaming API */ + for(step = 1; step < BLAKE2B_BLOCKBYTES; ++step) { + for (i = 0; i < BLAKE2_KAT_LENGTH; ++i) { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b_state S; + uint8_t * p = buf; + size_t mlen = i; + int err = 0; + + if( (err = blake2b_init_key(&S, BLAKE2B_OUTBYTES, key, BLAKE2B_KEYBYTES)) < 0 ) { + goto fail; + } + + while (mlen >= step) { + if ( (err = blake2b_update(&S, p, step)) < 0 ) { + goto fail; + } + mlen -= step; + p += step; + } + if ( (err = blake2b_update(&S, p, mlen)) < 0) { + goto fail; + } + if ( (err = blake2b_final(&S, hash, BLAKE2B_OUTBYTES)) < 0) { + goto fail; + } + + if (0 != memcmp(hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES)) { + goto fail; + } + } + } + + puts( "ok" ); + return 0; +fail: + puts("error"); + return -1; +} +#endif + From 97c9f61db47ddba43a0db562e13773514875fa6a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Sep 2018 00:34:46 -0400 Subject: [PATCH 02/46] start creating a hash of input parameters See #1416 --- src/all_types.hpp | 30 ++++---- src/codegen.cpp | 175 ++++++++++++++++++++++++++++++++++++++++++---- src/codegen.hpp | 1 - src/link.cpp | 2 - src/main.cpp | 1 - 5 files changed, 179 insertions(+), 30 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index dbaa3b546..0b0de5879 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1543,23 +1543,17 @@ struct LinkLib { bool provided_explicitly; }; +// When adding fields, check if they should be added to the hash computation in build_with_cache struct CodeGen { + //////////////////////////// Runtime State LLVMModuleRef module; ZigList errors; LLVMBuilderRef builder; ZigLLVMDIBuilder *dbuilder; ZigLLVMDICompileUnit *compile_unit; ZigLLVMDIFile *compile_unit_file; - - ZigList link_libs_list; LinkLib *libc_link_lib; - // add -framework [name] args to linker - ZigList darwin_frameworks; - // add -rpath [name] args to linker - ZigList rpath_list; - - // reminder: hash tables must be initialized before use HashMap import_table; HashMap builtin_fn_table; @@ -1575,7 +1569,6 @@ struct CodeGen { HashMap string_literals_table; HashMap type_info_cache; - ZigList import_queue; size_t import_queue_index; ZigList resolve_queue; @@ -1620,7 +1613,20 @@ struct CodeGen { ZigType *entry_promise; } builtin_types; + //////////////////////////// Participates in Input Parameter Cache Hash + ZigList link_libs_list; + // add -framework [name] args to linker + ZigList darwin_frameworks; + // add -rpath [name] args to linker + ZigList rpath_list; + EmitFileType emit_file_type; + BuildMode build_mode; + OutType out_type; + + + //////////////////////////// Unsorted + ZigTarget zig_target; LLVMTargetDataRef target_data_ref; unsigned pointer_size_bytes; @@ -1647,7 +1653,6 @@ struct CodeGen { Buf *ar_path; ZigWindowsSDK *win_sdk; Buf triple_str; - BuildMode build_mode; bool is_test_build; bool have_err_ret_tracing; uint32_t target_os_index; @@ -1657,13 +1662,13 @@ struct CodeGen { LLVMTargetMachineRef target_machine; ZigLLVMDIFile *dummy_di_file; bool is_native_target; - PackageTableEntry *root_package; + PackageTableEntry *root_package; // participates in cache hash PackageTableEntry *std_package; PackageTableEntry *panic_package; PackageTableEntry *test_runner_package; PackageTableEntry *compile_var_package; ImportTableEntry *compile_var_import; - Buf *root_out_name; + Buf *root_out_name; // participates in cache hash bool windows_subsystem_windows; bool windows_subsystem_console; Buf *mmacosx_version_min; @@ -1676,7 +1681,6 @@ struct CodeGen { size_t fn_defs_index; ZigList global_vars; - OutType out_type; ZigFn *cur_fn; ZigFn *main_fn; ZigFn *panic_fn; diff --git a/src/codegen.cpp b/src/codegen.cpp index 0300ccca9..e258f8d60 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -19,6 +19,7 @@ #include "target.hpp" #include "util.hpp" #include "zig_llvm.h" +#include "blake2.h" #include #include @@ -183,10 +184,6 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out return g; } -void codegen_destroy(CodeGen *codegen) { - LLVMDisposeTargetMachine(codegen->target_machine); -} - void codegen_set_output_h_path(CodeGen *g, Buf *h_path) { g->out_h_path = h_path; } @@ -7112,23 +7109,30 @@ static void create_test_compile_var_and_add_test_runner(CodeGen *g) { g->test_runner_import = add_special_code(g, g->test_runner_package, "test_runner.zig"); } -static void gen_root_source(CodeGen *g) { +static Buf *get_resolved_root_src_path(CodeGen *g) { + // TODO memoize if (buf_len(&g->root_package->root_src_path) == 0) - return; + return nullptr; - codegen_add_time_event(g, "Semantic Analysis"); - - Buf *rel_full_path = buf_alloc(); - os_path_join(&g->root_package->root_src_dir, &g->root_package->root_src_path, rel_full_path); + Buf rel_full_path = BUF_INIT; + os_path_join(&g->root_package->root_src_dir, &g->root_package->root_src_path, &rel_full_path); Buf *resolved_path = buf_alloc(); - Buf *resolve_paths[] = {rel_full_path}; + Buf *resolve_paths[] = {&rel_full_path}; *resolved_path = os_path_resolve(resolve_paths, 1); + return resolved_path; +} + +static void gen_root_source(CodeGen *g) { + Buf *resolved_path = get_resolved_root_src_path(g); + if (resolved_path == nullptr) + return; + Buf *source_code = buf_alloc(); int err; - if ((err = os_fetch_file_path(rel_full_path, source_code, true))) { - fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(rel_full_path), err_str(err)); + if ((err = os_fetch_file_path(resolved_path, source_code, true))) { + fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(resolved_path), err_str(err)); exit(1); } @@ -7671,10 +7675,155 @@ void codegen_add_time_event(CodeGen *g, const char *name) { g->timing_events.append({os_get_time(), name}); } +static void add_cache_str(blake2b_state *blake, const char *ptr) { + assert(ptr != nullptr); + // + 1 to include the null byte + blake2b_update(blake, ptr, strlen(ptr) + 1); +} + +static void add_cache_int(blake2b_state *blake, int x) { + // + 1 to include the null byte + uint8_t buf[sizeof(int) + 1]; + memcpy(buf, &x, sizeof(int)); + buf[sizeof(int)] = 0; + blake2b_update(blake, buf, sizeof(int) + 1); +} + +static void add_cache_buf(blake2b_state *blake, Buf *buf) { + assert(buf != nullptr); + // + 1 to include the null byte + blake2b_update(blake, buf_ptr(buf), buf_len(buf) + 1); +} + +static void add_cache_buf_opt(blake2b_state *blake, Buf *buf) { + if (buf == nullptr) { + add_cache_str(blake, ""); + add_cache_str(blake, ""); + } else { + add_cache_buf(blake, buf); + } +} + +static void add_cache_list_of_link_lib(blake2b_state *blake, LinkLib **ptr, size_t len) { + for (size_t i = 0; i < len; i += 1) { + LinkLib *lib = ptr[i]; + if (lib->provided_explicitly) { + add_cache_buf(blake, lib->name); + } + } + add_cache_str(blake, ""); +} + +static void add_cache_list_of_buf(blake2b_state *blake, Buf **ptr, size_t len) { + for (size_t i = 0; i < len; i += 1) { + Buf *buf = ptr[i]; + add_cache_buf(blake, buf); + } + add_cache_str(blake, ""); +} + +//static void add_cache_file(CodeGen *g, blake2b_state *blake, Buf *resolved_path) { +// assert(file_name != nullptr); +// g->cache_files.append(resolved_path); +//} +// +//static void add_cache_file_opt(CodeGen *g, blake2b_state *blake, Buf *resolved_path) { +// if (resolved_path == nullptr) { +// add_cache_str(blake, ""); +// add_cache_str(blake, ""); +// } else { +// add_cache_file(g, blake, resolved_path); +// } +//} + +// Ported from std/base64.zig +static uint8_t base64_fs_alphabet[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"; +static void base64_encode(Slice dest, Slice source) { + size_t dest_len = ((source.len + 2) / 3) * 4; + assert(dest.len == dest_len); + + size_t i = 0; + size_t out_index = 0; + for (; i + 2 < source.len; i += 3) { + dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i + 1] & 0xf) << 2) | ((source.ptr[i + 2] & 0xc0) >> 6)]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[source.ptr[i + 2] & 0x3f]; + out_index += 1; + } + + // Assert that we never need pad characters. + assert(i == source.len); + //if (i < source.len) { + // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; + // out_index += 1; + + // if (i + 1 == source.len) { + // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] & 0x3) << 4]; + // out_index += 1; + + // dest.ptr[out_index] = encoder.pad_char; + // out_index += 1; + // } else { + // dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; + // out_index += 1; + + // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i + 1] & 0xf) << 2]; + // out_index += 1; + // } + + // dest.ptr[out_index] = encoder.pad_char; + // out_index += 1; + //} +} + +// Called before init() +static bool build_with_cache(CodeGen *g) { + blake2b_state blake; + int rc = blake2b_init(&blake, 48); + assert(rc == 0); + + // TODO zig exe & dynamic libraries + + add_cache_buf(&blake, g->root_out_name); + add_cache_buf_opt(&blake, get_resolved_root_src_path(g)); // Root source file + add_cache_list_of_link_lib(&blake, g->link_libs_list.items, g->link_libs_list.length); + add_cache_list_of_buf(&blake, g->darwin_frameworks.items, g->darwin_frameworks.length); + add_cache_list_of_buf(&blake, g->rpath_list.items, g->rpath_list.length); + add_cache_int(&blake, g->emit_file_type); + add_cache_int(&blake, g->build_mode); + add_cache_int(&blake, g->out_type); + // TODO the rest of the struct CodeGen fields + + uint8_t bin_digest[48]; + rc = blake2b_final(&blake, bin_digest, 48); + assert(rc == 0); + + Buf b64_digest = BUF_INIT; + buf_resize(&b64_digest, 64); + base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); + + fprintf(stderr, "input params hash: %s\n", buf_ptr(&b64_digest)); + // TODO next look for a manifest file that has all the files from the input parameters + // use that to construct the real hash, which looks up the output directory and another manifest file + + return false; +} + void codegen_build(CodeGen *g) { assert(g->out_type != OutTypeUnknown); + if (build_with_cache(g)) + return; init(g); + codegen_add_time_event(g, "Semantic Analysis"); + gen_global_asm(g); gen_root_source(g); do_code_gen(g); diff --git a/src/codegen.hpp b/src/codegen.hpp index 6297c4611..55b38a005 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -16,7 +16,6 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, Buf *zig_lib_dir); -void codegen_destroy(CodeGen *codegen); void codegen_set_clang_argv(CodeGen *codegen, const char **args, size_t len); void codegen_set_llvm_argv(CodeGen *codegen, const char **args, size_t len); diff --git a/src/link.cpp b/src/link.cpp index f44e8b105..78ad204fe 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -69,8 +69,6 @@ static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) os_path_join(&parent_gen->cache_dir, o_out_name, output_path); codegen_link(child_gen, buf_ptr(output_path)); - codegen_destroy(child_gen); - return output_path; } diff --git a/src/main.cpp b/src/main.cpp index 4394a1d9a..eed31438d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -461,7 +461,6 @@ int main(int argc, char **argv) { g->root_package->package_table.put(buf_create_from_str("@build"), build_pkg); codegen_build(g); codegen_link(g, buf_ptr(path_to_build_exe)); - codegen_destroy(g); Termination term; os_spawn_process(buf_ptr(path_to_build_exe), args, &term); From b4d5d4d1748d25efa6197914a36e276e93509f57 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Sep 2018 23:39:14 -0400 Subject: [PATCH 03/46] assume evenly divided base64 --- src/codegen.cpp | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index e258f8d60..f4823f104 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7760,27 +7760,6 @@ static void base64_encode(Slice dest, Slice source) { // Assert that we never need pad characters. assert(i == source.len); - //if (i < source.len) { - // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; - // out_index += 1; - - // if (i + 1 == source.len) { - // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] & 0x3) << 4]; - // out_index += 1; - - // dest.ptr[out_index] = encoder.pad_char; - // out_index += 1; - // } else { - // dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; - // out_index += 1; - - // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i + 1] & 0xf) << 2]; - // out_index += 1; - // } - - // dest.ptr[out_index] = encoder.pad_char; - // out_index += 1; - //} } // Called before init() From 173fc842c4eb1da6ca07a4ab6026c4c62bc8c09b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 9 Sep 2018 18:07:11 -0400 Subject: [PATCH 04/46] basic compiler id hash working --- CMakeLists.txt | 1 + src/buffer.hpp | 4 + src/cache_hash.cpp | 407 +++++++++++++++++++++++++++++++++++++++++++++ src/cache_hash.hpp | 56 +++++++ src/codegen.cpp | 152 ++++------------- src/error.cpp | 2 + src/error.hpp | 2 + src/main.cpp | 61 +++++++ src/os.cpp | 314 ++++++++++++++++++++++++++-------- src/os.hpp | 158 ++++++++++-------- src/util.cpp | 39 +++++ src/util.hpp | 12 ++ 12 files changed, 951 insertions(+), 257 deletions(-) create mode 100644 src/cache_hash.cpp create mode 100644 src/cache_hash.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fbf5b03ea..615128159 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -409,6 +409,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/bigint.cpp" "${CMAKE_SOURCE_DIR}/src/blake2b.cpp" "${CMAKE_SOURCE_DIR}/src/buffer.cpp" + "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" "${CMAKE_SOURCE_DIR}/src/errmsg.cpp" diff --git a/src/buffer.hpp b/src/buffer.hpp index 501e44b5a..8155df87a 100644 --- a/src/buffer.hpp +++ b/src/buffer.hpp @@ -78,6 +78,10 @@ static inline Buf *buf_create_from_mem(const char *ptr, size_t len) { return buf; } +static inline Buf *buf_create_from_slice(Slice slice) { + return buf_create_from_mem((const char *)slice.ptr, slice.len); +} + static inline Buf *buf_create_from_str(const char *str) { return buf_create_from_mem(str, strlen(str)); } diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp new file mode 100644 index 000000000..5447a5f3f --- /dev/null +++ b/src/cache_hash.cpp @@ -0,0 +1,407 @@ +/* + * Copyright (c) 2018 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#include "cache_hash.hpp" +#include "buffer.hpp" +#include "os.hpp" + +#include + +void cache_init(CacheHash *ch, Buf *manifest_dir) { + int rc = blake2b_init(&ch->blake, 48); + assert(rc == 0); + ch->files = {}; + ch->manifest_dir = manifest_dir; + ch->manifest_file_path = nullptr; + ch->manifest_dirty = false; +} + +void cache_str(CacheHash *ch, const char *ptr) { + assert(ch->manifest_file_path == nullptr); + assert(ptr != nullptr); + // + 1 to include the null byte + blake2b_update(&ch->blake, ptr, strlen(ptr) + 1); +} + +void cache_int(CacheHash *ch, int x) { + assert(ch->manifest_file_path == nullptr); + // + 1 to include the null byte + uint8_t buf[sizeof(int) + 1]; + memcpy(buf, &x, sizeof(int)); + buf[sizeof(int)] = 0; + blake2b_update(&ch->blake, buf, sizeof(int) + 1); +} + +void cache_buf(CacheHash *ch, Buf *buf) { + assert(ch->manifest_file_path == nullptr); + assert(buf != nullptr); + // + 1 to include the null byte + blake2b_update(&ch->blake, buf_ptr(buf), buf_len(buf) + 1); +} + +void cache_buf_opt(CacheHash *ch, Buf *buf) { + assert(ch->manifest_file_path == nullptr); + if (buf == nullptr) { + cache_str(ch, ""); + cache_str(ch, ""); + } else { + cache_buf(ch, buf); + } +} + +void cache_list_of_link_lib(CacheHash *ch, LinkLib **ptr, size_t len) { + assert(ch->manifest_file_path == nullptr); + for (size_t i = 0; i < len; i += 1) { + LinkLib *lib = ptr[i]; + if (lib->provided_explicitly) { + cache_buf(ch, lib->name); + } + } + cache_str(ch, ""); +} + +void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len) { + assert(ch->manifest_file_path == nullptr); + for (size_t i = 0; i < len; i += 1) { + Buf *buf = ptr[i]; + cache_buf(ch, buf); + } + cache_str(ch, ""); +} + +void cache_file(CacheHash *ch, Buf *file_path) { + assert(ch->manifest_file_path == nullptr); + assert(file_path != nullptr); + Buf *resolved_path = buf_alloc(); + *resolved_path = os_path_resolve(&file_path, 1); + CacheHashFile *chf = ch->files.add_one(); + chf->path = resolved_path; + cache_buf(ch, resolved_path); +} + +void cache_file_opt(CacheHash *ch, Buf *file_path) { + assert(ch->manifest_file_path == nullptr); + if (file_path == nullptr) { + cache_str(ch, ""); + cache_str(ch, ""); + } else { + cache_file(ch, file_path); + } +} + +// Ported from std/base64.zig +static uint8_t base64_fs_alphabet[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; +static void base64_encode(Slice dest, Slice source) { + size_t dest_len = ((source.len + 2) / 3) * 4; + assert(dest.len == dest_len); + + size_t i = 0; + size_t out_index = 0; + for (; i + 2 < source.len; i += 3) { + dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i + 1] & 0xf) << 2) | ((source.ptr[i + 2] & 0xc0) >> 6)]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[source.ptr[i + 2] & 0x3f]; + out_index += 1; + } + + // Assert that we never need pad characters. + assert(i == source.len); +} + +// Ported from std/base64.zig +static Error base64_decode(Slice dest, Slice source) { + assert(source.len % 4 == 0); + assert(dest.len == (source.len / 4) * 3); + + // In Zig this is comptime computed. In C++ it's not worth it to do that. + uint8_t char_to_index[256]; + bool char_in_alphabet[256] = {0}; + for (size_t i = 0; i < 64; i += 1) { + uint8_t c = base64_fs_alphabet[i]; + assert(!char_in_alphabet[c]); + char_in_alphabet[c] = true; + char_to_index[c] = i; + } + + size_t src_cursor = 0; + size_t dest_cursor = 0; + + for (;src_cursor < source.len; src_cursor += 4) { + if (!char_in_alphabet[source.ptr[src_cursor + 0]]) return ErrorInvalidFormat; + if (!char_in_alphabet[source.ptr[src_cursor + 1]]) return ErrorInvalidFormat; + if (!char_in_alphabet[source.ptr[src_cursor + 2]]) return ErrorInvalidFormat; + if (!char_in_alphabet[source.ptr[src_cursor + 3]]) return ErrorInvalidFormat; + dest.ptr[dest_cursor + 0] = (char_to_index[source.ptr[src_cursor + 0]] << 2) | (char_to_index[source.ptr[src_cursor + 1]] >> 4); + dest.ptr[dest_cursor + 1] = (char_to_index[source.ptr[src_cursor + 1]] << 4) | (char_to_index[source.ptr[src_cursor + 2]] >> 2); + dest.ptr[dest_cursor + 2] = (char_to_index[source.ptr[src_cursor + 2]] << 6) | (char_to_index[source.ptr[src_cursor + 3]]); + dest_cursor += 3; + } + + assert(src_cursor == source.len); + assert(dest_cursor == dest.len); + return ErrorNone; +} + +static Error hash_file(uint8_t *digest, OsFile handle) { + Error err; + + blake2b_state blake; + int rc = blake2b_init(&blake, 48); + assert(rc == 0); + + for (;;) { + uint8_t buf[4096]; + size_t amt = 4096; + if ((err = os_file_read(handle, buf, &amt))) + return err; + if (amt == 0) { + rc = blake2b_final(&blake, digest, 48); + assert(rc == 0); + return ErrorNone; + } + blake2b_update(&blake, buf, amt); + } +} + +static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf) { + Error err; + + assert(chf->path != nullptr); + + OsFile this_file; + if ((err = os_file_open_r(chf->path, &this_file))) + return err; + + if ((err = os_file_mtime(this_file, &chf->mtime))) { + os_file_close(this_file); + return err; + } + + if ((err = hash_file(chf->bin_digest, this_file))) { + os_file_close(this_file); + return err; + } + os_file_close(this_file); + + blake2b_update(&ch->blake, chf->bin_digest, 48); + + return ErrorNone; +} + +Error cache_hit(CacheHash *ch, Buf *out_digest) { + Error err; + + uint8_t bin_digest[48]; + int rc = blake2b_final(&ch->blake, bin_digest, 48); + assert(rc == 0); + + Buf b64_digest = BUF_INIT; + buf_resize(&b64_digest, 64); + base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); + + rc = blake2b_init(&ch->blake, 48); + assert(rc == 0); + blake2b_update(&ch->blake, bin_digest, 48); + + ch->manifest_file_path = buf_alloc(); + os_path_join(ch->manifest_dir, &b64_digest, ch->manifest_file_path); + + buf_append_str(ch->manifest_file_path, ".txt"); + + if ((err = os_file_open_lock_rw(ch->manifest_file_path, &ch->manifest_file))) + return err; + + Buf line_buf = BUF_INIT; + buf_resize(&line_buf, 512); + if ((err = os_file_read_all(ch->manifest_file, &line_buf))) { + os_file_close(ch->manifest_file); + return err; + } + + size_t input_file_count = ch->files.length; + bool any_file_changed = false; + size_t file_i = 0; + SplitIterator line_it = memSplit(buf_to_slice(&line_buf), str("\n")); + for (;; file_i += 1) { + Optional> opt_line = SplitIterator_next(&line_it); + if (!opt_line.is_some) + break; + + CacheHashFile *chf; + if (file_i < input_file_count) { + chf = &ch->files.at(file_i); + } else if (any_file_changed) { + // cache miss. + // keep the the manifest file open with the rw lock + // reset the hash + rc = blake2b_init(&ch->blake, 48); + assert(rc == 0); + blake2b_update(&ch->blake, bin_digest, 48); + ch->files.resize(input_file_count); + // bring the hash up to the input file hashes + for (file_i = 0; file_i < input_file_count; file_i += 1) { + blake2b_update(&ch->blake, ch->files.at(file_i).bin_digest, 48); + } + // caller can notice that out_digest is unmodified. + return ErrorNone; + } else { + chf = ch->files.add_one(); + chf->path = nullptr; + } + + SplitIterator it = memSplit(opt_line.value, str(" ")); + + Optional> opt_mtime_sec = SplitIterator_next(&it); + if (!opt_mtime_sec.is_some) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + chf->mtime.sec = strtoull((const char *)opt_mtime_sec.value.ptr, nullptr, 10); + + Optional> opt_mtime_nsec = SplitIterator_next(&it); + if (!opt_mtime_nsec.is_some) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + chf->mtime.nsec = strtoull((const char *)opt_mtime_nsec.value.ptr, nullptr, 10); + + Optional> opt_digest = SplitIterator_next(&it); + if (!opt_digest.is_some) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + if ((err = base64_decode({chf->bin_digest, 48}, opt_digest.value))) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + + Optional> opt_file_path = SplitIterator_next(&it); + if (!opt_file_path.is_some) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + Buf *this_path = buf_create_from_slice(opt_file_path.value); + if (chf->path != nullptr && !buf_eql_buf(this_path, chf->path)) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + chf->path = this_path; + + // if the mtime matches we can trust the digest + OsFile this_file; + if ((err = os_file_open_r(chf->path, &this_file))) { + os_file_close(ch->manifest_file); + return err; + } + OsTimeStamp actual_mtime; + if ((err = os_file_mtime(this_file, &actual_mtime))) { + os_file_close(this_file); + os_file_close(ch->manifest_file); + return err; + } + if (chf->mtime.sec == actual_mtime.sec && chf->mtime.nsec == actual_mtime.nsec) { + os_file_close(this_file); + } else { + // we have to recompute the digest. + // later we'll rewrite the manifest with the new mtime/digest values + ch->manifest_dirty = true; + chf->mtime = actual_mtime; + + uint8_t actual_digest[48]; + if ((err = hash_file(actual_digest, this_file))) { + os_file_close(this_file); + os_file_close(ch->manifest_file); + return err; + } + os_file_close(this_file); + if (memcmp(chf->bin_digest, actual_digest, 48) != 0) { + memcpy(chf->bin_digest, actual_digest, 48); + // keep going until we have the input file digests + any_file_changed = true; + } + } + if (!any_file_changed) { + blake2b_update(&ch->blake, chf->bin_digest, 48); + } + } + if (file_i < input_file_count) { + // manifest file is empty or missing entries, so this is a cache miss + ch->manifest_dirty = true; + for (; file_i < input_file_count; file_i += 1) { + CacheHashFile *chf = &ch->files.at(file_i); + if ((err = populate_file_hash(ch, chf))) { + os_file_close(ch->manifest_file); + return err; + } + } + return ErrorNone; + } + // Cache Hit + return cache_final(ch, out_digest); +} + +Error cache_add_file(CacheHash *ch, Buf *path) { + Error err; + + assert(ch->manifest_file_path != nullptr); + CacheHashFile *chf = ch->files.add_one(); + chf->path = path; + if ((err = populate_file_hash(ch, chf))) { + os_file_close(ch->manifest_file); + return err; + } + + return ErrorNone; +} + +static Error write_manifest_file(CacheHash *ch) { + Error err; + Buf contents = BUF_INIT; + buf_resize(&contents, 0); + uint8_t encoded_digest[65]; + encoded_digest[64] = 0; + for (size_t i = 0; i < ch->files.length; i += 1) { + CacheHashFile *chf = &ch->files.at(i); + base64_encode({encoded_digest, 64}, {chf->bin_digest, 48}); + buf_appendf(&contents, "%" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %s %s\n", + chf->mtime.sec, chf->mtime.nsec, encoded_digest, buf_ptr(chf->path)); + } + fprintf(stderr, "overwrite with\n%s\n", buf_ptr(&contents)); + if ((err = os_file_overwrite(ch->manifest_file, &contents))) + return err; + + return ErrorNone; +} + +Error cache_final(CacheHash *ch, Buf *out_digest) { + Error err; + + assert(ch->manifest_file_path != nullptr); + + if (ch->manifest_dirty) { + if ((err = write_manifest_file(ch))) { + fprintf(stderr, "Warning: Unable to write cache file '%s': %s\n", + buf_ptr(ch->manifest_file_path), err_str(err)); + } + } + os_file_close(ch->manifest_file); + + uint8_t bin_digest[48]; + int rc = blake2b_final(&ch->blake, bin_digest, 48); + assert(rc == 0); + buf_resize(out_digest, 64); + base64_encode(buf_to_slice(out_digest), {bin_digest, 48}); + + return ErrorNone; +} diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp new file mode 100644 index 000000000..77b22a1e4 --- /dev/null +++ b/src/cache_hash.hpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2018 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#ifndef ZIG_CACHE_HASH_HPP +#define ZIG_CACHE_HASH_HPP + +#include "all_types.hpp" +#include "blake2.h" +#include "os.hpp" + +struct CacheHashFile { + Buf *path; + OsTimeStamp mtime; + uint8_t bin_digest[48]; +}; + +struct CacheHash { + blake2b_state blake; + ZigList files; + Buf *manifest_dir; + Buf *manifest_file_path; + OsFile manifest_file; + bool manifest_dirty; +}; + +// Always call this first to set up. +void cache_init(CacheHash *ch, Buf *manifest_dir); + +// Next, use the hash population functions to add the initial parameters. +void cache_str(CacheHash *ch, const char *ptr); +void cache_int(CacheHash *ch, int x); +void cache_buf(CacheHash *ch, Buf *buf); +void cache_buf_opt(CacheHash *ch, Buf *buf); +void cache_list_of_link_lib(CacheHash *ch, LinkLib **ptr, size_t len); +void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len); +void cache_file(CacheHash *ch, Buf *path); +void cache_file_opt(CacheHash *ch, Buf *path); + +// Then call cache_hit when you're ready to see if you can skip the next step. +// out_b64_digest will be left unchanged if it was a cache miss +Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest); + +// If you got a cache hit, the flow is done. No more functions to call. +// Next call this function for every file that is depended on. +Error ATTRIBUTE_MUST_USE cache_add_file(CacheHash *ch, Buf *path); + +// If you did not get a cache hit, use the hash population functions again +// and do all the actual work. When done use cache_final to save the cache +// for next time. +Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_digest); + +#endif diff --git a/src/codegen.cpp b/src/codegen.cpp index f4823f104..6c5648f62 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -19,7 +19,6 @@ #include "target.hpp" #include "util.hpp" #include "zig_llvm.h" -#include "blake2.h" #include #include @@ -7675,130 +7674,45 @@ void codegen_add_time_event(CodeGen *g, const char *name) { g->timing_events.append({os_get_time(), name}); } -static void add_cache_str(blake2b_state *blake, const char *ptr) { - assert(ptr != nullptr); - // + 1 to include the null byte - blake2b_update(blake, ptr, strlen(ptr) + 1); -} -static void add_cache_int(blake2b_state *blake, int x) { - // + 1 to include the null byte - uint8_t buf[sizeof(int) + 1]; - memcpy(buf, &x, sizeof(int)); - buf[sizeof(int)] = 0; - blake2b_update(blake, buf, sizeof(int) + 1); -} - -static void add_cache_buf(blake2b_state *blake, Buf *buf) { - assert(buf != nullptr); - // + 1 to include the null byte - blake2b_update(blake, buf_ptr(buf), buf_len(buf) + 1); -} - -static void add_cache_buf_opt(blake2b_state *blake, Buf *buf) { - if (buf == nullptr) { - add_cache_str(blake, ""); - add_cache_str(blake, ""); - } else { - add_cache_buf(blake, buf); - } -} - -static void add_cache_list_of_link_lib(blake2b_state *blake, LinkLib **ptr, size_t len) { - for (size_t i = 0; i < len; i += 1) { - LinkLib *lib = ptr[i]; - if (lib->provided_explicitly) { - add_cache_buf(blake, lib->name); - } - } - add_cache_str(blake, ""); -} - -static void add_cache_list_of_buf(blake2b_state *blake, Buf **ptr, size_t len) { - for (size_t i = 0; i < len; i += 1) { - Buf *buf = ptr[i]; - add_cache_buf(blake, buf); - } - add_cache_str(blake, ""); -} - -//static void add_cache_file(CodeGen *g, blake2b_state *blake, Buf *resolved_path) { -// assert(file_name != nullptr); -// g->cache_files.append(resolved_path); -//} +//// Called before init() +//static bool build_with_cache(CodeGen *g) { +// // TODO zig exe & dynamic libraries +// // should be in main.cpp I think. only needs to happen +// // once on startup. // -//static void add_cache_file_opt(CodeGen *g, blake2b_state *blake, Buf *resolved_path) { -// if (resolved_path == nullptr) { -// add_cache_str(blake, ""); -// add_cache_str(blake, ""); -// } else { -// add_cache_file(g, blake, resolved_path); -// } +// CacheHash comp; +// cache_init(&comp); +// +// add_cache_buf(&blake, g->root_out_name); +// add_cache_buf_opt(&blake, get_resolved_root_src_path(g)); // Root source file +// add_cache_list_of_link_lib(&blake, g->link_libs_list.items, g->link_libs_list.length); +// add_cache_list_of_buf(&blake, g->darwin_frameworks.items, g->darwin_frameworks.length); +// add_cache_list_of_buf(&blake, g->rpath_list.items, g->rpath_list.length); +// add_cache_int(&blake, g->emit_file_type); +// add_cache_int(&blake, g->build_mode); +// add_cache_int(&blake, g->out_type); +// // TODO the rest of the struct CodeGen fields +// +// uint8_t bin_digest[48]; +// rc = blake2b_final(&blake, bin_digest, 48); +// assert(rc == 0); +// +// Buf b64_digest = BUF_INIT; +// buf_resize(&b64_digest, 64); +// base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); +// +// fprintf(stderr, "input params hash: %s\n", buf_ptr(&b64_digest)); +// // TODO next look for a manifest file that has all the files from the input parameters +// // use that to construct the real hash, which looks up the output directory and another manifest file +// +// return false; //} -// Ported from std/base64.zig -static uint8_t base64_fs_alphabet[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"; -static void base64_encode(Slice dest, Slice source) { - size_t dest_len = ((source.len + 2) / 3) * 4; - assert(dest.len == dest_len); - - size_t i = 0; - size_t out_index = 0; - for (; i + 2 < source.len; i += 3) { - dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; - out_index += 1; - - dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; - out_index += 1; - - dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i + 1] & 0xf) << 2) | ((source.ptr[i + 2] & 0xc0) >> 6)]; - out_index += 1; - - dest.ptr[out_index] = base64_fs_alphabet[source.ptr[i + 2] & 0x3f]; - out_index += 1; - } - - // Assert that we never need pad characters. - assert(i == source.len); -} - -// Called before init() -static bool build_with_cache(CodeGen *g) { - blake2b_state blake; - int rc = blake2b_init(&blake, 48); - assert(rc == 0); - - // TODO zig exe & dynamic libraries - - add_cache_buf(&blake, g->root_out_name); - add_cache_buf_opt(&blake, get_resolved_root_src_path(g)); // Root source file - add_cache_list_of_link_lib(&blake, g->link_libs_list.items, g->link_libs_list.length); - add_cache_list_of_buf(&blake, g->darwin_frameworks.items, g->darwin_frameworks.length); - add_cache_list_of_buf(&blake, g->rpath_list.items, g->rpath_list.length); - add_cache_int(&blake, g->emit_file_type); - add_cache_int(&blake, g->build_mode); - add_cache_int(&blake, g->out_type); - // TODO the rest of the struct CodeGen fields - - uint8_t bin_digest[48]; - rc = blake2b_final(&blake, bin_digest, 48); - assert(rc == 0); - - Buf b64_digest = BUF_INIT; - buf_resize(&b64_digest, 64); - base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); - - fprintf(stderr, "input params hash: %s\n", buf_ptr(&b64_digest)); - // TODO next look for a manifest file that has all the files from the input parameters - // use that to construct the real hash, which looks up the output directory and another manifest file - - return false; -} - void codegen_build(CodeGen *g) { assert(g->out_type != OutTypeUnknown); - if (build_with_cache(g)) - return; + //if (build_with_cache(g)) + // return; init(g); codegen_add_time_event(g, "Semantic Analysis"); diff --git a/src/error.cpp b/src/error.cpp index 8303a80a1..d9da6e6c5 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -27,6 +27,8 @@ const char *err_str(int err) { case ErrorNegativeDenominator: return "negative denominator"; case ErrorShiftedOutOneBits: return "exact shift shifted out one bits"; case ErrorCCompileErrors: return "C compile errors"; + case ErrorEndOfFile: return "end of file"; + case ErrorIsDir: return "is directory"; } return "(invalid error)"; } diff --git a/src/error.hpp b/src/error.hpp index e3b87fc6b..c99c6e6b1 100644 --- a/src/error.hpp +++ b/src/error.hpp @@ -27,6 +27,8 @@ enum Error { ErrorNegativeDenominator, ErrorShiftedOutOneBits, ErrorCCompileErrors, + ErrorEndOfFile, + ErrorIsDir, }; const char *err_str(int err); diff --git a/src/main.cpp b/src/main.cpp index eed31438d..172fe04da 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -13,6 +13,7 @@ #include "link.hpp" #include "os.hpp" #include "target.hpp" +#include "cache_hash.hpp" #include @@ -270,6 +271,66 @@ int main(int argc, char **argv) { return 0; } + if (argc == 2 && strcmp(argv[1], "TEST") == 0) { + Error err; + Buf app_data_dir = BUF_INIT; + if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { + fprintf(stderr, "get app dir: %s\n", err_str(err)); + return 1; + } + Buf *stage1_dir = buf_alloc(); + os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); + + if ((err = os_make_path(manifest_dir))) { + fprintf(stderr, "make path: %s\n", err_str(err)); + return 1; + } + CacheHash cache_hash; + CacheHash *ch = &cache_hash; + cache_init(ch, manifest_dir); + Buf self_exe_path = BUF_INIT; + if ((err = os_self_exe_path(&self_exe_path))) { + fprintf(stderr, "self exe path: %s\n", err_str(err)); + return 1; + } + + cache_file(ch, &self_exe_path); + + Buf exe_digest = BUF_INIT; + buf_resize(&exe_digest, 0); + if ((err = cache_hit(ch, &exe_digest))) { + fprintf(stderr, "cache hit error: %s\n", err_str(err)); + return 1; + } + if (buf_len(&exe_digest) != 0) { + fprintf(stderr, "cache hit: %s\n", buf_ptr(&exe_digest)); + return 0; + } + fprintf(stderr, "cache miss\n"); + ZigList lib_paths = {}; + if ((err = os_self_exe_shared_libs(lib_paths))) { + fprintf(stderr, "finding out shared libs: %s\n", err_str(err)); + return 1; + } + for (size_t i = 0; i < lib_paths.length; i += 1) { + Buf *lib_path = lib_paths.at(i); + if ((err = cache_add_file(ch, lib_path))) { + fprintf(stderr, "cache add file %s: %s", buf_ptr(lib_path), err_str(err)); + return 1; + } + } + if ((err = cache_final(ch, &exe_digest))) { + fprintf(stderr, "final: %s\n", err_str(err)); + return 1; + } + + + fprintf(stderr, "computed2: %s\n", buf_ptr(&exe_digest)); + return 0; + } + os_init(); char *arg0 = argv[0]; diff --git a/src/os.cpp b/src/os.cpp index 0e62d84a4..d6e17ca5a 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -40,6 +40,10 @@ typedef SSIZE_T ssize_t; #endif +#if defined(ZIG_OS_LINUX) +#include +#endif + #if defined(__MACH__) #include @@ -57,54 +61,6 @@ static clock_serv_t cclock; #include #include -// Ported from std/mem.zig. -// Coordinate struct fields with memSplit function -struct SplitIterator { - size_t index; - Slice buffer; - Slice split_bytes; -}; - -// Ported from std/mem.zig. -static bool SplitIterator_isSplitByte(SplitIterator *self, uint8_t byte) { - for (size_t i = 0; i < self->split_bytes.len; i += 1) { - if (byte == self->split_bytes.ptr[i]) { - return true; - } - } - return false; -} - -// Ported from std/mem.zig. -static Optional> SplitIterator_next(SplitIterator *self) { - // move to beginning of token - while (self->index < self->buffer.len && - SplitIterator_isSplitByte(self, self->buffer.ptr[self->index])) - { - self->index += 1; - } - size_t start = self->index; - if (start == self->buffer.len) { - return {}; - } - - // move to end of token - while (self->index < self->buffer.len && - !SplitIterator_isSplitByte(self, self->buffer.ptr[self->index])) - { - self->index += 1; - } - size_t end = self->index; - - return Optional>::some(self->buffer.slice(start, end)); -} - -// Ported from std/mem.zig -static SplitIterator memSplit(Slice buffer, Slice split_bytes) { - return SplitIterator{0, buffer, split_bytes}; -} - - #if defined(ZIG_OS_POSIX) static void populate_termination(Termination *term, int status) { if (WIFEXITED(status)) { @@ -1368,16 +1324,16 @@ double os_get_time(void) { #endif } -int os_make_path(Buf *path) { +Error os_make_path(Buf *path) { Buf resolved_path = os_path_resolve(&path, 1); size_t end_index = buf_len(&resolved_path); - int err; + Error err; while (true) { if ((err = os_make_dir(buf_slice(&resolved_path, 0, end_index)))) { if (err == ErrorPathAlreadyExists) { if (end_index == buf_len(&resolved_path)) - return 0; + return ErrorNone; } else if (err == ErrorFileNotFound) { // march end_index backward until next path component while (true) { @@ -1391,7 +1347,7 @@ int os_make_path(Buf *path) { } } if (end_index == buf_len(&resolved_path)) - return 0; + return ErrorNone; // march end_index forward until next path component while (true) { end_index += 1; @@ -1399,10 +1355,10 @@ int os_make_path(Buf *path) { break; } } - return 0; + return ErrorNone; } -int os_make_dir(Buf *path) { +Error os_make_dir(Buf *path) { #if defined(ZIG_OS_WINDOWS) if (!CreateDirectory(buf_ptr(path), NULL)) { if (GetLastError() == ERROR_ALREADY_EXISTS) @@ -1413,7 +1369,7 @@ int os_make_dir(Buf *path) { return ErrorAccess; return ErrorUnexpected; } - return 0; + return ErrorNone; #else if (mkdir(buf_ptr(path), 0755) == -1) { if (errno == EEXIST) @@ -1424,7 +1380,7 @@ int os_make_dir(Buf *path) { return ErrorAccess; return ErrorUnexpected; } - return 0; + return ErrorNone; #endif } @@ -1447,7 +1403,7 @@ int os_init(void) { return 0; } -int os_self_exe_path(Buf *out_path) { +Error os_self_exe_path(Buf *out_path) { #if defined(ZIG_OS_WINDOWS) buf_resize(out_path, 256); for (;;) { @@ -1480,27 +1436,21 @@ int os_self_exe_path(Buf *out_path) { char *real_path = realpath(buf_ptr(tmp), buf_ptr(out_path)); if (!real_path) { buf_init_from_buf(out_path, tmp); - return 0; + return ErrorNone; } // Resize out_path for the correct length. buf_resize(out_path, strlen(buf_ptr(out_path))); - return 0; + return ErrorNone; #elif defined(ZIG_OS_LINUX) - buf_resize(out_path, 256); - for (;;) { - ssize_t amt = readlink("/proc/self/exe", buf_ptr(out_path), buf_len(out_path)); - if (amt == -1) { - return ErrorUnexpected; - } - if (amt == (ssize_t)buf_len(out_path)) { - buf_resize(out_path, buf_len(out_path) * 2); - continue; - } - buf_resize(out_path, amt); - return 0; + buf_resize(out_path, PATH_MAX); + ssize_t amt = readlink("/proc/self/exe", buf_ptr(out_path), buf_len(out_path)); + if (amt == -1) { + return ErrorUnexpected; } + buf_resize(out_path, amt); + return ErrorNone; #endif return ErrorFileNotFound; } @@ -1685,3 +1635,225 @@ int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf* output_buf, ZigLLVM_ArchTy return ErrorFileNotFound; #endif } + +// Ported from std/os/get_app_data_dir.zig +Error os_get_app_data_dir(Buf *out_path, const char *appname) { +#if defined(ZIG_OS_WINDOWS) +#error "Unimplemented" +#elif defined(ZIG_OS_DARWIN) + const char *home_dir = getenv("HOME"); + if (home_dir == nullptr) { + // TODO use /etc/passwd + return ErrorFileNotFound; + } + buf_resize(out_path, 0); + buf_appendf(out_path, "%s/Library/Application Support/%s", home_dir, appname); + return ErrorNone; +#elif defined(ZIG_OS_LINUX) + const char *home_dir = getenv("HOME"); + if (home_dir == nullptr) { + // TODO use /etc/passwd + return ErrorFileNotFound; + } + buf_resize(out_path, 0); + buf_appendf(out_path, "%s/.local/share/%s", home_dir, appname); + return ErrorNone; +#endif +} + + +#if defined(ZIG_OS_LINUX) +static int self_exe_shared_libs_callback(struct dl_phdr_info *info, size_t size, void *data) { + ZigList *libs = reinterpret_cast< ZigList *>(data); + if (info->dlpi_name[0] == '/') { + libs->append(buf_create_from_str(info->dlpi_name)); + } + return 0; +} +#endif + +Error os_self_exe_shared_libs(ZigList &paths) { +#if defined(ZIG_OS_LINUX) + paths.resize(0); + dl_iterate_phdr(self_exe_shared_libs_callback, &paths); + return ErrorNone; +#else +#error "unimplemented" +#endif +} + +Error os_file_open_r(Buf *full_path, OsFile *out_file) { +#if defined(ZIG_OS_WINDOWS) +#error "unimplemented" +#else + for (;;) { + int fd = open(buf_ptr(full_path), O_RDONLY|O_CLOEXEC); + if (fd == -1) { + switch (errno) { + case EINTR: + continue; + case EINVAL: + zig_unreachable(); + case EFAULT: + zig_unreachable(); + case EACCES: + return ErrorAccess; + case EISDIR: + return ErrorIsDir; + case ENOENT: + return ErrorFileNotFound; + default: + return ErrorFileSystem; + } + } + *out_file = fd; + return ErrorNone; + } +#endif +} + +Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) { +#if defined(ZIG_OS_WINDOWS) +#error "unimplemented" +#else + int fd; + for (;;) { + fd = open(buf_ptr(full_path), O_RDWR|O_CLOEXEC|O_CREAT, 0666); + if (fd == -1) { + switch (errno) { + case EINTR: + continue; + case EINVAL: + zig_unreachable(); + case EFAULT: + zig_unreachable(); + case EACCES: + return ErrorAccess; + case EISDIR: + return ErrorIsDir; + case ENOENT: + return ErrorFileNotFound; + default: + return ErrorFileSystem; + } + } + break; + } + for (;;) { + struct flock lock; + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + if (fcntl(fd, F_SETLKW, &lock) == -1) { + switch (errno) { + case EINTR: + continue; + case EBADF: + zig_unreachable(); + case EFAULT: + zig_unreachable(); + case EINVAL: + zig_unreachable(); + default: + close(fd); + return ErrorFileSystem; + } + } + break; + } + *out_file = fd; + return ErrorNone; +#endif +} + +Error os_file_mtime(OsFile file, OsTimeStamp *mtime) { +#if defined(ZIG_OS_WINDOWS) +#error unimplemented +#else + struct stat statbuf; + if (fstat(file, &statbuf) == -1) + return ErrorFileSystem; + + mtime->sec = statbuf.st_mtim.tv_sec; + mtime->nsec = statbuf.st_mtim.tv_nsec; + return ErrorNone; +#endif +} + +Error os_file_read(OsFile file, void *ptr, size_t *len) { +#if defined(ZIG_OS_WINDOWS) +#error unimplemented +#else + for (;;) { + ssize_t rc = read(file, ptr, *len); + if (rc == -1) { + switch (errno) { + case EINTR: + continue; + case EBADF: + zig_unreachable(); + case EFAULT: + zig_unreachable(); + case EISDIR: + zig_unreachable(); + default: + return ErrorFileSystem; + } + } + *len = rc; + return ErrorNone; + } +#endif +} + +Error os_file_read_all(OsFile file, Buf *contents) { + Error err; + size_t index = 0; + for (;;) { + size_t amt = buf_len(contents) - index; + + if (amt < 512) { + buf_resize(contents, buf_len(contents) + 512); + amt += 512; + } + + if ((err = os_file_read(file, buf_ptr(contents) + index, &amt))) + return err; + + if (amt == 0) { + buf_resize(contents, index); + return ErrorNone; + } + + index += amt; + } +} + +Error os_file_overwrite(OsFile file, Buf *contents) { +#if defined(ZIG_OS_WINDOWS) +#error unimplemented +#else + if (lseek(file, 0, SEEK_SET) == -1) + return ErrorFileSystem; + for (;;) { + if (write(file, buf_ptr(contents), buf_len(contents)) == -1) { + switch (errno) { + case EINTR: + continue; + case EINVAL: + zig_unreachable(); + case EBADF: + zig_unreachable(); + default: + return ErrorFileSystem; + } + } + return ErrorNone; + } +#endif +} + +void os_file_close(OsFile file) { + close(file); +} diff --git a/src/os.hpp b/src/os.hpp index a44fa8160..c64eccf8d 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -13,77 +13,11 @@ #include "error.hpp" #include "zig_llvm.h" #include "windows_sdk.h" +#include "result.hpp" #include #include -enum TermColor { - TermColorRed, - TermColorGreen, - TermColorCyan, - TermColorWhite, - TermColorBold, - TermColorReset, -}; - -enum TerminationId { - TerminationIdClean, - TerminationIdSignaled, - TerminationIdStopped, - TerminationIdUnknown, -}; - -struct Termination { - TerminationId how; - int code; -}; - -int os_init(void); - -void os_spawn_process(const char *exe, ZigList &args, Termination *term); -int os_exec_process(const char *exe, ZigList &args, - Termination *term, Buf *out_stderr, Buf *out_stdout); - -void os_path_dirname(Buf *full_path, Buf *out_dirname); -void os_path_split(Buf *full_path, Buf *out_dirname, Buf *out_basename); -void os_path_extname(Buf *full_path, Buf *out_basename, Buf *out_extname); -void os_path_join(Buf *dirname, Buf *basename, Buf *out_full_path); -int os_path_real(Buf *rel_path, Buf *out_abs_path); -Buf os_path_resolve(Buf **paths_ptr, size_t paths_len); -bool os_path_is_absolute(Buf *path); - -int os_get_global_cache_directory(Buf *out_tmp_path); - -int os_make_path(Buf *path); -int os_make_dir(Buf *path); - -void os_write_file(Buf *full_path, Buf *contents); -int os_copy_file(Buf *src_path, Buf *dest_path); - -int os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); -int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); - -int os_get_cwd(Buf *out_cwd); - -bool os_stderr_tty(void); -void os_stderr_set_color(TermColor color); - -int os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path); -int os_delete_file(Buf *path); - -int os_file_exists(Buf *full_path, bool *result); - -int os_rename(Buf *src_path, Buf *dest_path); -double os_get_time(void); - -bool os_is_sep(uint8_t c); - -int os_self_exe_path(Buf *out_path); - -int os_get_win32_ucrt_include_path(ZigWindowsSDK *sdk, Buf *output_buf); -int os_get_win32_ucrt_lib_path(ZigWindowsSDK *sdk, Buf *output_buf, ZigLLVM_ArchType platform_type); -int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf *output_buf, ZigLLVM_ArchType platform_type); - #if defined(__APPLE__) #define ZIG_OS_DARWIN #elif defined(_WIN32) @@ -116,4 +50,94 @@ int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf *output_buf, ZigLLVM_ArchTy #define ZIG_OS_SEP_CHAR '/' #endif +enum TermColor { + TermColorRed, + TermColorGreen, + TermColorCyan, + TermColorWhite, + TermColorBold, + TermColorReset, +}; + +enum TerminationId { + TerminationIdClean, + TerminationIdSignaled, + TerminationIdStopped, + TerminationIdUnknown, +}; + +struct Termination { + TerminationId how; + int code; +}; + +#if defined(ZIG_OS_WINDOWS) +#define OsFile (void *) +#else +#define OsFile int +#endif + +struct OsTimeStamp { + uint64_t sec; + uint64_t nsec; +}; + +int os_init(void); + +void os_spawn_process(const char *exe, ZigList &args, Termination *term); +int os_exec_process(const char *exe, ZigList &args, + Termination *term, Buf *out_stderr, Buf *out_stdout); + +void os_path_dirname(Buf *full_path, Buf *out_dirname); +void os_path_split(Buf *full_path, Buf *out_dirname, Buf *out_basename); +void os_path_extname(Buf *full_path, Buf *out_basename, Buf *out_extname); +void os_path_join(Buf *dirname, Buf *basename, Buf *out_full_path); +int os_path_real(Buf *rel_path, Buf *out_abs_path); +Buf os_path_resolve(Buf **paths_ptr, size_t paths_len); +bool os_path_is_absolute(Buf *path); + +int os_get_global_cache_directory(Buf *out_tmp_path); + +Error ATTRIBUTE_MUST_USE os_make_path(Buf *path); +Error ATTRIBUTE_MUST_USE os_make_dir(Buf *path); + +Error ATTRIBUTE_MUST_USE os_file_open_r(Buf *full_path, OsFile *out_file); +Error ATTRIBUTE_MUST_USE os_file_open_lock_rw(Buf *full_path, OsFile *out_file); +Error ATTRIBUTE_MUST_USE os_file_mtime(OsFile file, OsTimeStamp *mtime); +Error ATTRIBUTE_MUST_USE os_file_read(OsFile file, void *ptr, size_t *len); +Error ATTRIBUTE_MUST_USE os_file_read_all(OsFile file, Buf *contents); +Error ATTRIBUTE_MUST_USE os_file_overwrite(OsFile file, Buf *contents); +void os_file_close(OsFile file); + +void os_write_file(Buf *full_path, Buf *contents); +int os_copy_file(Buf *src_path, Buf *dest_path); + +int os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); +int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); + +int os_get_cwd(Buf *out_cwd); + +bool os_stderr_tty(void); +void os_stderr_set_color(TermColor color); + +int os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path); +int os_delete_file(Buf *path); + +int os_file_exists(Buf *full_path, bool *result); + +int os_rename(Buf *src_path, Buf *dest_path); +double os_get_time(void); + +bool os_is_sep(uint8_t c); + +Error ATTRIBUTE_MUST_USE os_self_exe_path(Buf *out_path); + +Error ATTRIBUTE_MUST_USE os_get_app_data_dir(Buf *out_path, const char *appname); + +int os_get_win32_ucrt_include_path(ZigWindowsSDK *sdk, Buf *output_buf); +int os_get_win32_ucrt_lib_path(ZigWindowsSDK *sdk, Buf *output_buf, ZigLLVM_ArchType platform_type); +int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf *output_buf, ZigLLVM_ArchType platform_type); + +Error ATTRIBUTE_MUST_USE os_self_exe_shared_libs(ZigList &paths); + #endif diff --git a/src/util.cpp b/src/util.cpp index cafd2c3d8..060d7f8fb 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -43,3 +43,42 @@ uint32_t ptr_hash(const void *ptr) { bool ptr_eq(const void *a, const void *b) { return a == b; } + +// Ported from std/mem.zig. +bool SplitIterator_isSplitByte(SplitIterator *self, uint8_t byte) { + for (size_t i = 0; i < self->split_bytes.len; i += 1) { + if (byte == self->split_bytes.ptr[i]) { + return true; + } + } + return false; +} + +// Ported from std/mem.zig. +Optional> SplitIterator_next(SplitIterator *self) { + // move to beginning of token + while (self->index < self->buffer.len && + SplitIterator_isSplitByte(self, self->buffer.ptr[self->index])) + { + self->index += 1; + } + size_t start = self->index; + if (start == self->buffer.len) { + return {}; + } + + // move to end of token + while (self->index < self->buffer.len && + !SplitIterator_isSplitByte(self, self->buffer.ptr[self->index])) + { + self->index += 1; + } + size_t end = self->index; + + return Optional>::some(self->buffer.slice(start, end)); +} + +// Ported from std/mem.zig +SplitIterator memSplit(Slice buffer, Slice split_bytes) { + return SplitIterator{0, buffer, split_bytes}; +} diff --git a/src/util.hpp b/src/util.hpp index 1e02e3043..f18c369fb 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -254,4 +254,16 @@ static inline void memCopy(Slice dest, Slice src) { memcpy(dest.ptr, src.ptr, src.len * sizeof(T)); } +// Ported from std/mem.zig. +// Coordinate struct fields with memSplit function +struct SplitIterator { + size_t index; + Slice buffer; + Slice split_bytes; +}; + +bool SplitIterator_isSplitByte(SplitIterator *self, uint8_t byte); +Optional> SplitIterator_next(SplitIterator *self); +SplitIterator memSplit(Slice buffer, Slice split_bytes); + #endif From c0bdcc7417a23d338dc1df6bd74b30fe6e3d1a1a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 9 Sep 2018 23:58:52 -0400 Subject: [PATCH 05/46] `zig id` command --- src/cache_hash.cpp | 1 - src/main.cpp | 113 +++++++++++++++++++++++---------------------- src/os.cpp | 6 +-- 3 files changed, 60 insertions(+), 60 deletions(-) diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index 5447a5f3f..ff530ca65 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -377,7 +377,6 @@ static Error write_manifest_file(CacheHash *ch) { buf_appendf(&contents, "%" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %s %s\n", chf->mtime.sec, chf->mtime.nsec, encoded_digest, buf_ptr(chf->path)); } - fprintf(stderr, "overwrite with\n%s\n", buf_ptr(&contents)); if ((err = os_file_overwrite(ch->manifest_file, &contents))) return err; diff --git a/src/main.cpp b/src/main.cpp index 172fe04da..c385a1c88 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -25,6 +25,7 @@ static int usage(const char *arg0) { " build-lib [source] create library from source or object files\n" " build-obj [source] create object from source or assembly\n" " builtin show the source code of that @import(\"builtin\")\n" + " id print the base64-encoded compiler id\n" " run [source] create executable and run immediately\n" " translate-c [source] convert c code to zig code\n" " targets list available compilation targets\n" @@ -257,6 +258,55 @@ static void add_package(CodeGen *g, CliPkg *cli_pkg, PackageTableEntry *pkg) { } } +static Buf saved_compiler_id = BUF_INIT; +static Error get_compiler_id(Buf **result) { + if (saved_compiler_id.list.length != 0) { + *result = &saved_compiler_id; + return ErrorNone; + } + + Error err; + Buf app_data_dir = BUF_INIT; + if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) + return err; + Buf *stage1_dir = buf_alloc(); + os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); + + if ((err = os_make_path(manifest_dir))) + return err; + CacheHash cache_hash; + CacheHash *ch = &cache_hash; + cache_init(ch, manifest_dir); + Buf self_exe_path = BUF_INIT; + if ((err = os_self_exe_path(&self_exe_path))) + return err; + + cache_file(ch, &self_exe_path); + + buf_resize(&saved_compiler_id, 0); + if ((err = cache_hit(ch, &saved_compiler_id))) + return err; + if (buf_len(&saved_compiler_id) != 0) { + *result = &saved_compiler_id; + return ErrorNone; + } + ZigList lib_paths = {}; + if ((err = os_self_exe_shared_libs(lib_paths))) + return err; + for (size_t i = 0; i < lib_paths.length; i += 1) { + Buf *lib_path = lib_paths.at(i); + if ((err = cache_add_file(ch, lib_path))) + return err; + } + if ((err = cache_final(ch, &saved_compiler_id))) + return err; + + *result = &saved_compiler_id; + return ErrorNone; +} + int main(int argc, char **argv) { if (argc == 2 && strcmp(argv[1], "BUILD_INFO") == 0) { printf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", @@ -271,64 +321,15 @@ int main(int argc, char **argv) { return 0; } - if (argc == 2 && strcmp(argv[1], "TEST") == 0) { + if (argc == 2 && strcmp(argv[1], "id") == 0) { Error err; - Buf app_data_dir = BUF_INIT; - if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { - fprintf(stderr, "get app dir: %s\n", err_str(err)); - return 1; + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) { + fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); + return EXIT_FAILURE; } - Buf *stage1_dir = buf_alloc(); - os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); - Buf *manifest_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); - - if ((err = os_make_path(manifest_dir))) { - fprintf(stderr, "make path: %s\n", err_str(err)); - return 1; - } - CacheHash cache_hash; - CacheHash *ch = &cache_hash; - cache_init(ch, manifest_dir); - Buf self_exe_path = BUF_INIT; - if ((err = os_self_exe_path(&self_exe_path))) { - fprintf(stderr, "self exe path: %s\n", err_str(err)); - return 1; - } - - cache_file(ch, &self_exe_path); - - Buf exe_digest = BUF_INIT; - buf_resize(&exe_digest, 0); - if ((err = cache_hit(ch, &exe_digest))) { - fprintf(stderr, "cache hit error: %s\n", err_str(err)); - return 1; - } - if (buf_len(&exe_digest) != 0) { - fprintf(stderr, "cache hit: %s\n", buf_ptr(&exe_digest)); - return 0; - } - fprintf(stderr, "cache miss\n"); - ZigList lib_paths = {}; - if ((err = os_self_exe_shared_libs(lib_paths))) { - fprintf(stderr, "finding out shared libs: %s\n", err_str(err)); - return 1; - } - for (size_t i = 0; i < lib_paths.length; i += 1) { - Buf *lib_path = lib_paths.at(i); - if ((err = cache_add_file(ch, lib_path))) { - fprintf(stderr, "cache add file %s: %s", buf_ptr(lib_path), err_str(err)); - return 1; - } - } - if ((err = cache_final(ch, &exe_digest))) { - fprintf(stderr, "final: %s\n", err_str(err)); - return 1; - } - - - fprintf(stderr, "computed2: %s\n", buf_ptr(&exe_digest)); - return 0; + printf("%s\n", buf_ptr(compiler_id)); + return EXIT_SUCCESS; } os_init(); diff --git a/src/os.cpp b/src/os.cpp index d6e17ca5a..4ee461818 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -1813,9 +1813,9 @@ Error os_file_read_all(OsFile file, Buf *contents) { for (;;) { size_t amt = buf_len(contents) - index; - if (amt < 512) { - buf_resize(contents, buf_len(contents) + 512); - amt += 512; + if (amt < 4096) { + buf_resize(contents, buf_len(contents) + (4096 - amt)); + amt = buf_len(contents) - index; } if ((err = os_file_read(file, buf_ptr(contents) + index, &amt))) From fbe5737c84c783cd31e6e2d595fc47eb782c5e3c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 10 Sep 2018 09:46:15 -0400 Subject: [PATCH 06/46] stage1: always optimize blake and softfloat even in debug mode --- CMakeLists.txt | 14 +++++-- src/all_types.hpp | 4 ++ src/cache_hash.cpp | 4 ++ src/cache_hash.hpp | 3 +- src/codegen.cpp | 91 ++++++++++++++++++++++++++++------------------ src/codegen.hpp | 2 +- src/link.cpp | 2 +- src/main.cpp | 21 ++++++++--- 8 files changed, 94 insertions(+), 47 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 615128159..8339be71b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -390,7 +390,7 @@ if(MSVC) ) else() set_target_properties(embedded_softfloat PROPERTIES - COMPILE_FLAGS "-std=c99" + COMPILE_FLAGS "-std=c99 -O3" ) endif() target_include_directories(embedded_softfloat PUBLIC @@ -407,10 +407,9 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/ast_render.cpp" "${CMAKE_SOURCE_DIR}/src/bigfloat.cpp" "${CMAKE_SOURCE_DIR}/src/bigint.cpp" - "${CMAKE_SOURCE_DIR}/src/blake2b.cpp" "${CMAKE_SOURCE_DIR}/src/buffer.cpp" - "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp" + "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" "${CMAKE_SOURCE_DIR}/src/errmsg.cpp" "${CMAKE_SOURCE_DIR}/src/error.cpp" @@ -426,6 +425,9 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/util.cpp" "${CMAKE_SOURCE_DIR}/src/translate_c.cpp" ) +set(ZIG_SOURCES_O3 + "${CMAKE_SOURCE_DIR}/src/blake2b.cpp" +) set(ZIG_CPP_SOURCES "${CMAKE_SOURCE_DIR}/src/zig_llvm.cpp" "${CMAKE_SOURCE_DIR}/src/windows_sdk.cpp" @@ -813,6 +815,11 @@ set_target_properties(zig_cpp PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} ) +add_library(zig_O3 STATIC ${ZIG_SOURCES_O3}) +set_target_properties(zig_O3 PROPERTIES + COMPILE_FLAGS "${EXE_CFLAGS} -O3" +) + add_executable(zig ${ZIG_SOURCES}) set_target_properties(zig PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} @@ -821,6 +828,7 @@ set_target_properties(zig PROPERTIES target_link_libraries(zig LINK_PUBLIC zig_cpp + zig_O3 ${SOFTFLOAT_LIBRARIES} ${CLANG_LIBRARIES} ${LLD_LIBRARIES} diff --git a/src/all_types.hpp b/src/all_types.hpp index 0b0de5879..2dfbcaaa6 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -10,6 +10,7 @@ #include "list.hpp" #include "buffer.hpp" +#include "cache_hash.hpp" #include "zig_llvm.h" #include "hash_map.hpp" #include "errmsg.hpp" @@ -1613,7 +1614,10 @@ struct CodeGen { ZigType *entry_promise; } builtin_types; + CacheHash cache_hash; + //////////////////////////// Participates in Input Parameter Cache Hash + Buf *compiler_id; ZigList link_libs_list; // add -framework [name] args to linker ZigList darwin_frameworks; diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index ff530ca65..a0c43bc81 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -6,6 +6,7 @@ */ #include "cache_hash.hpp" +#include "all_types.hpp" #include "buffer.hpp" #include "os.hpp" @@ -219,6 +220,9 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { buf_append_str(ch->manifest_file_path, ".txt"); + if ((err = os_make_path(ch->manifest_dir))) + return err; + if ((err = os_file_open_lock_rw(ch->manifest_file_path, &ch->manifest_file))) return err; diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp index 77b22a1e4..3c2d77abc 100644 --- a/src/cache_hash.hpp +++ b/src/cache_hash.hpp @@ -8,10 +8,11 @@ #ifndef ZIG_CACHE_HASH_HPP #define ZIG_CACHE_HASH_HPP -#include "all_types.hpp" #include "blake2.h" #include "os.hpp" +struct LinkLib; + struct CacheHashFile { Buf *path; OsTimeStamp mtime; diff --git a/src/codegen.cpp b/src/codegen.cpp index 6c5648f62..a18b26bb1 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -88,12 +88,13 @@ static const char *symbols_that_llvm_depends_on[] = { }; CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir) + Buf *zig_lib_dir, Buf *compiler_id) { CodeGen *g = allocate(1); codegen_add_time_event(g, "Initialize"); + g->compiler_id = compiler_id; g->zig_lib_dir = zig_lib_dir; g->zig_std_dir = buf_alloc(); @@ -7675,44 +7676,62 @@ void codegen_add_time_event(CodeGen *g, const char *name) { } -//// Called before init() -//static bool build_with_cache(CodeGen *g) { -// // TODO zig exe & dynamic libraries -// // should be in main.cpp I think. only needs to happen -// // once on startup. -// -// CacheHash comp; -// cache_init(&comp); -// -// add_cache_buf(&blake, g->root_out_name); -// add_cache_buf_opt(&blake, get_resolved_root_src_path(g)); // Root source file -// add_cache_list_of_link_lib(&blake, g->link_libs_list.items, g->link_libs_list.length); -// add_cache_list_of_buf(&blake, g->darwin_frameworks.items, g->darwin_frameworks.length); -// add_cache_list_of_buf(&blake, g->rpath_list.items, g->rpath_list.length); -// add_cache_int(&blake, g->emit_file_type); -// add_cache_int(&blake, g->build_mode); -// add_cache_int(&blake, g->out_type); -// // TODO the rest of the struct CodeGen fields -// -// uint8_t bin_digest[48]; -// rc = blake2b_final(&blake, bin_digest, 48); -// assert(rc == 0); -// -// Buf b64_digest = BUF_INIT; -// buf_resize(&b64_digest, 64); -// base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); -// -// fprintf(stderr, "input params hash: %s\n", buf_ptr(&b64_digest)); -// // TODO next look for a manifest file that has all the files from the input parameters -// // use that to construct the real hash, which looks up the output directory and another manifest file -// -// return false; -//} +// Called before init() +static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { + Error err; + + CacheHash *ch = &g->cache_hash; + cache_init(ch, manifest_dir); + + cache_buf(ch, g->compiler_id); + cache_buf(ch, g->root_out_name); + cache_file(ch, get_resolved_root_src_path(g)); // Root source file + cache_list_of_link_lib(ch, g->link_libs_list.items, g->link_libs_list.length); + cache_list_of_buf(ch, g->darwin_frameworks.items, g->darwin_frameworks.length); + cache_list_of_buf(ch, g->rpath_list.items, g->rpath_list.length); + cache_int(ch, g->emit_file_type); + cache_int(ch, g->build_mode); + cache_int(ch, g->out_type); + // TODO the rest of the struct CodeGen fields + + buf_resize(digest, 0); + if ((err = cache_hit(ch, digest))) + return err; + + return ErrorNone; +} void codegen_build(CodeGen *g) { + Error err; assert(g->out_type != OutTypeUnknown); - //if (build_with_cache(g)) - // return; + + Buf app_data_dir = BUF_INIT; + if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { + fprintf(stderr, "Unable to get app data dir: %s\n", err_str(err)); + exit(1); + } + Buf *stage1_dir = buf_alloc(); + os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); + + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("build"), manifest_dir); + + Buf digest = BUF_INIT; + if ((err = check_cache(g, manifest_dir, &digest))) { + fprintf(stderr, "Unable to check cache: %s\n", err_str(err)); + exit(1); + } + if (buf_len(&digest) != 0) { + Buf *artifact_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + + Buf *this_artifact_dir = buf_alloc(); + os_path_join(artifact_dir, &digest, this_artifact_dir); + + fprintf(stderr, "copy artifacts from %s\n", buf_ptr(this_artifact_dir)); + return; + } + init(g); codegen_add_time_event(g, "Semantic Analysis"); diff --git a/src/codegen.hpp b/src/codegen.hpp index 55b38a005..649cf0685 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -15,7 +15,7 @@ #include CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir); + Buf *zig_lib_dir, Buf *compiler_id); void codegen_set_clang_argv(CodeGen *codegen, const char **args, size_t len); void codegen_set_llvm_argv(CodeGen *codegen, const char **args, size_t len); diff --git a/src/link.cpp b/src/link.cpp index 78ad204fe..c0eb25bd5 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -34,7 +34,7 @@ static const char *get_libc_static_file(CodeGen *g, const char *file) { static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) { ZigTarget *child_target = parent_gen->is_native_target ? nullptr : &parent_gen->zig_target; CodeGen *child_gen = codegen_create(full_path, child_target, OutTypeObj, parent_gen->build_mode, - parent_gen->zig_lib_dir); + parent_gen->zig_lib_dir, parent_gen->compiler_id); child_gen->want_h_file = false; child_gen->verbose_tokenize = parent_gen->verbose_tokenize; diff --git a/src/main.cpp b/src/main.cpp index c385a1c88..1520ae7d6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -274,8 +274,6 @@ static Error get_compiler_id(Buf **result) { Buf *manifest_dir = buf_alloc(); os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); - if ((err = os_make_path(manifest_dir))) - return err; CacheHash cache_hash; CacheHash *ch = &cache_hash; cache_init(ch, manifest_dir); @@ -432,8 +430,14 @@ int main(int argc, char **argv) { Buf *build_runner_path = buf_alloc(); os_path_join(special_dir, buf_create_from_str("build_runner.zig"), build_runner_path); + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) { + fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); + return EXIT_FAILURE; + } - CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf); + CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf, + compiler_id); codegen_set_out_name(g, buf_create_from_str("build")); Buf *build_file_buf = buf_create_from_str(build_file); @@ -796,7 +800,7 @@ int main(int argc, char **argv) { switch (cmd) { case CmdBuiltin: { Buf *zig_lib_dir_buf = resolve_zig_lib_dir(); - CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, zig_lib_dir_buf); + CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, zig_lib_dir_buf, nullptr); Buf *builtin_source = codegen_generate_builtin_source(g); if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) { fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout))); @@ -871,7 +875,14 @@ int main(int argc, char **argv) { Buf *zig_lib_dir_buf = resolve_zig_lib_dir(); - CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf); + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) { + fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); + return EXIT_FAILURE; + } + + CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf, + compiler_id); codegen_set_out_name(g, buf_out_name); codegen_set_lib_version(g, ver_major, ver_minor, ver_patch); codegen_set_is_test(g, cmd == CmdTest); From 32be6e9b2a9e6de501aadbe271c554a4682a10f8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 10 Sep 2018 13:46:23 -0400 Subject: [PATCH 07/46] caching is working * add almost all the input parameter state to the hash - missing items are the detected MSVC installation on Windows and detected libc installation on POSIX - also missing are C files and .h files that libclang finds * artifacts are created in global cache directory instead of zig-cache. - exception: builtin.zig is still in zig-cache * zig run uses the new cache correctly * zig run uses execv on posix systems --- src/all_types.hpp | 295 ++++++++++++++++++++++----------------------- src/cache_hash.cpp | 70 +++++++++-- src/cache_hash.hpp | 27 +++-- src/codegen.cpp | 199 +++++++++++++++++++++--------- src/codegen.hpp | 4 +- src/error.cpp | 1 + src/error.hpp | 1 + src/ir.cpp | 11 +- src/link.cpp | 74 +++--------- src/link.hpp | 17 --- src/main.cpp | 64 ++++------ src/os.cpp | 54 +++------ src/os.hpp | 3 +- 13 files changed, 434 insertions(+), 386 deletions(-) delete mode 100644 src/link.hpp diff --git a/src/all_types.hpp b/src/all_types.hpp index 2dfbcaaa6..6d11244a2 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1554,6 +1554,40 @@ struct CodeGen { ZigLLVMDICompileUnit *compile_unit; ZigLLVMDIFile *compile_unit_file; LinkLib *libc_link_lib; + LLVMTargetDataRef target_data_ref; + LLVMTargetMachineRef target_machine; + ZigLLVMDIFile *dummy_di_file; + LLVMValueRef cur_ret_ptr; + LLVMValueRef cur_fn_val; + LLVMValueRef cur_err_ret_trace_val_arg; + LLVMValueRef cur_err_ret_trace_val_stack; + LLVMValueRef memcpy_fn_val; + LLVMValueRef memset_fn_val; + LLVMValueRef trap_fn_val; + LLVMValueRef return_address_fn_val; + LLVMValueRef frame_address_fn_val; + LLVMValueRef coro_destroy_fn_val; + LLVMValueRef coro_id_fn_val; + LLVMValueRef coro_alloc_fn_val; + LLVMValueRef coro_size_fn_val; + LLVMValueRef coro_begin_fn_val; + LLVMValueRef coro_suspend_fn_val; + LLVMValueRef coro_end_fn_val; + LLVMValueRef coro_free_fn_val; + LLVMValueRef coro_resume_fn_val; + LLVMValueRef coro_save_fn_val; + LLVMValueRef coro_promise_fn_val; + LLVMValueRef coro_alloc_helper_fn_val; + LLVMValueRef coro_frame_fn_val; + LLVMValueRef merge_err_ret_traces_fn_val; + LLVMValueRef add_error_return_trace_addr_fn_val; + LLVMValueRef stacksave_fn_val; + LLVMValueRef stackrestore_fn_val; + LLVMValueRef write_register_fn_val; + LLVMValueRef sp_md_node; + LLVMValueRef err_name_table; + LLVMValueRef safety_crash_err_fn; + LLVMValueRef return_err_fn; // reminder: hash tables must be initialized before use HashMap import_table; @@ -1576,8 +1610,23 @@ struct CodeGen { size_t resolve_queue_index; ZigList use_queue; size_t use_queue_index; + ZigList timing_events; + ZigList error_di_types; + ZigList tld_ref_source_node_stack; + ZigList inline_fns; + ZigList test_fns; + ZigList err_enumerators; + ZigList errors_by_index; + size_t largest_err_name_len; - uint32_t next_unresolved_index; + PackageTableEntry *std_package; + PackageTableEntry *panic_package; + PackageTableEntry *test_runner_package; + PackageTableEntry *compile_var_package; + ImportTableEntry *compile_var_import; + ImportTableEntry *root_import; + ImportTableEntry *bootstrap_import; + ImportTableEntry *test_runner_import; struct { ZigType *entry_bool; @@ -1613,30 +1662,45 @@ struct CodeGen { ZigType *entry_arg_tuple; ZigType *entry_promise; } builtin_types; + ZigType *align_amt_type; + ZigType *stack_trace_type; + ZigType *ptr_to_stack_trace_type; + ZigType *err_tag_type; + ZigType *test_fn_type; + + Buf triple_str; + Buf global_asm; + Buf *out_h_path; + Buf cache_dir; + Buf artifact_dir; + Buf output_file_path; + Buf o_file_output_path; + Buf *wanted_output_file_path; + + IrInstruction *invalid_instruction; + + ConstExprValue const_void_val; + ConstExprValue panic_msg_vals[PanicMsgIdCount]; + + // The function definitions this module includes. + ZigList fn_defs; + size_t fn_defs_index; + ZigList global_vars; + + ZigFn *cur_fn; + ZigFn *main_fn; + ZigFn *panic_fn; + AstNode *root_export_decl; CacheHash cache_hash; - - //////////////////////////// Participates in Input Parameter Cache Hash - Buf *compiler_id; - ZigList link_libs_list; - // add -framework [name] args to linker - ZigList darwin_frameworks; - // add -rpath [name] args to linker - ZigList rpath_list; - - EmitFileType emit_file_type; - BuildMode build_mode; - OutType out_type; - - - //////////////////////////// Unsorted - - ZigTarget zig_target; - LLVMTargetDataRef target_data_ref; + ErrColor err_color; + uint32_t next_unresolved_index; unsigned pointer_size_bytes; + uint32_t target_os_index; + uint32_t target_arch_index; + uint32_t target_environ_index; + uint32_t target_oformat_index; bool is_big_endian; - bool is_static; - bool strip_debug_symbols; bool want_h_file; bool have_pub_main; bool have_c_main; @@ -1644,6 +1708,64 @@ struct CodeGen { bool have_winmain_crt_startup; bool have_dllmain_crt_startup; bool have_pub_panic; + bool have_err_ret_tracing; + bool c_want_stdint; + bool c_want_stdbool; + bool verbose_tokenize; + bool verbose_ast; + bool verbose_link; + bool verbose_ir; + bool verbose_llvm_ir; + bool verbose_cimport; + bool error_during_imports; + bool generate_error_name_table; + + //////////////////////////// Participates in Input Parameter Cache Hash + ZigList link_libs_list; + // add -framework [name] args to linker + ZigList darwin_frameworks; + // add -rpath [name] args to linker + ZigList rpath_list; + ZigList forbidden_libs; + ZigList link_objects; + ZigList assembly_files; + ZigList lib_dirs; + + Buf *compiler_id; + size_t version_major; + size_t version_minor; + size_t version_patch; + const char *linker_script; + + EmitFileType emit_file_type; + BuildMode build_mode; + OutType out_type; + ZigTarget zig_target; + bool is_static; + bool strip_debug_symbols; + bool is_test_build; + bool is_native_target; + bool windows_subsystem_windows; + bool windows_subsystem_console; + bool linker_rdynamic; + bool no_rosegment_workaround; + bool each_lib_rpath; + + Buf *mmacosx_version_min; + Buf *mios_version_min; + Buf *root_out_name; + Buf *test_filter; + Buf *test_name_prefix; + PackageTableEntry *root_package; + + const char **llvm_argv; + size_t llvm_argv_len; + + const char **clang_argv; + size_t clang_argv_len; + + //////////////////////////// Unsorted + Buf *libc_lib_dir; Buf *libc_static_lib_dir; Buf *libc_include_dir; @@ -1654,138 +1776,7 @@ struct CodeGen { Buf *zig_c_headers_dir; Buf *zig_std_special_dir; Buf *dynamic_linker; - Buf *ar_path; ZigWindowsSDK *win_sdk; - Buf triple_str; - bool is_test_build; - bool have_err_ret_tracing; - uint32_t target_os_index; - uint32_t target_arch_index; - uint32_t target_environ_index; - uint32_t target_oformat_index; - LLVMTargetMachineRef target_machine; - ZigLLVMDIFile *dummy_di_file; - bool is_native_target; - PackageTableEntry *root_package; // participates in cache hash - PackageTableEntry *std_package; - PackageTableEntry *panic_package; - PackageTableEntry *test_runner_package; - PackageTableEntry *compile_var_package; - ImportTableEntry *compile_var_import; - Buf *root_out_name; // participates in cache hash - bool windows_subsystem_windows; - bool windows_subsystem_console; - Buf *mmacosx_version_min; - Buf *mios_version_min; - bool linker_rdynamic; - const char *linker_script; - - // The function definitions this module includes. - ZigList fn_defs; - size_t fn_defs_index; - ZigList global_vars; - - ZigFn *cur_fn; - ZigFn *main_fn; - ZigFn *panic_fn; - LLVMValueRef cur_ret_ptr; - LLVMValueRef cur_fn_val; - LLVMValueRef cur_err_ret_trace_val_arg; - LLVMValueRef cur_err_ret_trace_val_stack; - bool c_want_stdint; - bool c_want_stdbool; - AstNode *root_export_decl; - size_t version_major; - size_t version_minor; - size_t version_patch; - bool verbose_tokenize; - bool verbose_ast; - bool verbose_link; - bool verbose_ir; - bool verbose_llvm_ir; - bool verbose_cimport; - ErrColor err_color; - ImportTableEntry *root_import; - ImportTableEntry *bootstrap_import; - ImportTableEntry *test_runner_import; - LLVMValueRef memcpy_fn_val; - LLVMValueRef memset_fn_val; - LLVMValueRef trap_fn_val; - LLVMValueRef return_address_fn_val; - LLVMValueRef frame_address_fn_val; - LLVMValueRef coro_destroy_fn_val; - LLVMValueRef coro_id_fn_val; - LLVMValueRef coro_alloc_fn_val; - LLVMValueRef coro_size_fn_val; - LLVMValueRef coro_begin_fn_val; - LLVMValueRef coro_suspend_fn_val; - LLVMValueRef coro_end_fn_val; - LLVMValueRef coro_free_fn_val; - LLVMValueRef coro_resume_fn_val; - LLVMValueRef coro_save_fn_val; - LLVMValueRef coro_promise_fn_val; - LLVMValueRef coro_alloc_helper_fn_val; - LLVMValueRef coro_frame_fn_val; - LLVMValueRef merge_err_ret_traces_fn_val; - LLVMValueRef add_error_return_trace_addr_fn_val; - LLVMValueRef stacksave_fn_val; - LLVMValueRef stackrestore_fn_val; - LLVMValueRef write_register_fn_val; - bool error_during_imports; - - LLVMValueRef sp_md_node; - - const char **clang_argv; - size_t clang_argv_len; - ZigList lib_dirs; - - const char **llvm_argv; - size_t llvm_argv_len; - - ZigList test_fns; - ZigType *test_fn_type; - - bool each_lib_rpath; - - ZigType *err_tag_type; - ZigList err_enumerators; - ZigList errors_by_index; - bool generate_error_name_table; - LLVMValueRef err_name_table; - size_t largest_err_name_len; - LLVMValueRef safety_crash_err_fn; - - LLVMValueRef return_err_fn; - - IrInstruction *invalid_instruction; - ConstExprValue const_void_val; - - ConstExprValue panic_msg_vals[PanicMsgIdCount]; - - Buf global_asm; - ZigList link_objects; - ZigList assembly_files; - - Buf *test_filter; - Buf *test_name_prefix; - - ZigList timing_events; - - Buf cache_dir; - Buf *out_h_path; - - ZigList inline_fns; - ZigList tld_ref_source_node_stack; - - ZigType *align_amt_type; - ZigType *stack_trace_type; - ZigType *ptr_to_stack_trace_type; - - ZigList error_di_types; - - ZigList forbidden_libs; - - bool no_rosegment_workaround; }; enum VarLinkage { diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index a0c43bc81..b6aebdac6 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -37,6 +37,20 @@ void cache_int(CacheHash *ch, int x) { blake2b_update(&ch->blake, buf, sizeof(int) + 1); } +void cache_usize(CacheHash *ch, size_t x) { + assert(ch->manifest_file_path == nullptr); + // + 1 to include the null byte + uint8_t buf[sizeof(size_t) + 1]; + memcpy(buf, &x, sizeof(size_t)); + buf[sizeof(size_t)] = 0; + blake2b_update(&ch->blake, buf, sizeof(size_t) + 1); +} + +void cache_bool(CacheHash *ch, bool x) { + assert(ch->manifest_file_path == nullptr); + blake2b_update(&ch->blake, &x, 1); +} + void cache_buf(CacheHash *ch, Buf *buf) { assert(ch->manifest_file_path == nullptr); assert(buf != nullptr); @@ -74,6 +88,26 @@ void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len) { cache_str(ch, ""); } +void cache_list_of_file(CacheHash *ch, Buf **ptr, size_t len) { + assert(ch->manifest_file_path == nullptr); + + for (size_t i = 0; i < len; i += 1) { + Buf *buf = ptr[i]; + cache_file(ch, buf); + } + cache_str(ch, ""); +} + +void cache_list_of_str(CacheHash *ch, const char **ptr, size_t len) { + assert(ch->manifest_file_path == nullptr); + + for (size_t i = 0; i < len; i += 1) { + const char *s = ptr[i]; + cache_str(ch, s); + } + cache_str(ch, ""); +} + void cache_file(CacheHash *ch, Buf *file_path) { assert(ch->manifest_file_path == nullptr); assert(file_path != nullptr); @@ -154,9 +188,13 @@ static Error base64_decode(Slice dest, Slice source) { return ErrorNone; } -static Error hash_file(uint8_t *digest, OsFile handle) { +static Error hash_file(uint8_t *digest, OsFile handle, Buf *contents) { Error err; + if (contents) { + buf_resize(contents, 0); + } + blake2b_state blake; int rc = blake2b_init(&blake, 48); assert(rc == 0); @@ -172,10 +210,13 @@ static Error hash_file(uint8_t *digest, OsFile handle) { return ErrorNone; } blake2b_update(&blake, buf, amt); + if (contents) { + buf_append_mem(contents, (char*)buf, amt); + } } } -static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf) { +static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf, Buf *contents) { Error err; assert(chf->path != nullptr); @@ -189,7 +230,7 @@ static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf) { return err; } - if ((err = hash_file(chf->bin_digest, this_file))) { + if ((err = hash_file(chf->bin_digest, this_file, contents))) { os_file_close(this_file); return err; } @@ -323,7 +364,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { chf->mtime = actual_mtime; uint8_t actual_digest[48]; - if ((err = hash_file(actual_digest, this_file))) { + if ((err = hash_file(actual_digest, this_file, nullptr))) { os_file_close(this_file); os_file_close(ch->manifest_file); return err; @@ -344,7 +385,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { ch->manifest_dirty = true; for (; file_i < input_file_count; file_i += 1) { CacheHashFile *chf = &ch->files.at(file_i); - if ((err = populate_file_hash(ch, chf))) { + if ((err = populate_file_hash(ch, chf, nullptr))) { os_file_close(ch->manifest_file); return err; } @@ -355,13 +396,13 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { return cache_final(ch, out_digest); } -Error cache_add_file(CacheHash *ch, Buf *path) { +Error cache_add_file_fetch(CacheHash *ch, Buf *resolved_path, Buf *contents) { Error err; assert(ch->manifest_file_path != nullptr); CacheHashFile *chf = ch->files.add_one(); - chf->path = path; - if ((err = populate_file_hash(ch, chf))) { + chf->path = resolved_path; + if ((err = populate_file_hash(ch, chf, contents))) { os_file_close(ch->manifest_file); return err; } @@ -369,6 +410,12 @@ Error cache_add_file(CacheHash *ch, Buf *path) { return ErrorNone; } +Error cache_add_file(CacheHash *ch, Buf *path) { + Buf *resolved_path = buf_alloc(); + *resolved_path = os_path_resolve(&path, 1); + return cache_add_file_fetch(ch, resolved_path, nullptr); +} + static Error write_manifest_file(CacheHash *ch) { Error err; Buf contents = BUF_INIT; @@ -398,7 +445,8 @@ Error cache_final(CacheHash *ch, Buf *out_digest) { buf_ptr(ch->manifest_file_path), err_str(err)); } } - os_file_close(ch->manifest_file); + // We don't close the manifest file yet, because we want to + // keep it locked until the API user is done using it. uint8_t bin_digest[48]; int rc = blake2b_final(&ch->blake, bin_digest, 48); @@ -408,3 +456,7 @@ Error cache_final(CacheHash *ch, Buf *out_digest) { return ErrorNone; } + +void cache_release(CacheHash *ch) { + os_file_close(ch->manifest_file); +} diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp index 3c2d77abc..ede7344c7 100644 --- a/src/cache_hash.hpp +++ b/src/cache_hash.hpp @@ -17,6 +17,7 @@ struct CacheHashFile { Buf *path; OsTimeStamp mtime; uint8_t bin_digest[48]; + Buf *contents; }; struct CacheHash { @@ -34,24 +35,36 @@ void cache_init(CacheHash *ch, Buf *manifest_dir); // Next, use the hash population functions to add the initial parameters. void cache_str(CacheHash *ch, const char *ptr); void cache_int(CacheHash *ch, int x); +void cache_bool(CacheHash *ch, bool x); +void cache_usize(CacheHash *ch, size_t x); void cache_buf(CacheHash *ch, Buf *buf); void cache_buf_opt(CacheHash *ch, Buf *buf); void cache_list_of_link_lib(CacheHash *ch, LinkLib **ptr, size_t len); void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len); +void cache_list_of_file(CacheHash *ch, Buf **ptr, size_t len); +void cache_list_of_str(CacheHash *ch, const char **ptr, size_t len); void cache_file(CacheHash *ch, Buf *path); void cache_file_opt(CacheHash *ch, Buf *path); // Then call cache_hit when you're ready to see if you can skip the next step. -// out_b64_digest will be left unchanged if it was a cache miss +// out_b64_digest will be left unchanged if it was a cache miss. +// If you got a cache hit, the next step is cache_release. +// From this point on, there is a lock on the input params. Release +// the lock with cache_release. Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest); -// If you got a cache hit, the flow is done. No more functions to call. -// Next call this function for every file that is depended on. +// If you did not get a cache hit, call this function for every file +// that is depended on, and then finish with cache_final. Error ATTRIBUTE_MUST_USE cache_add_file(CacheHash *ch, Buf *path); -// If you did not get a cache hit, use the hash population functions again -// and do all the actual work. When done use cache_final to save the cache -// for next time. -Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_digest); +// This variant of cache_add_file returns the file contents. +// Also the file path argument must be already resolved. +Error ATTRIBUTE_MUST_USE cache_add_file_fetch(CacheHash *ch, Buf *resolved_path, Buf *contents); + +// out_b64_digest will be the same thing that cache_hit returns if you got a cache hit +Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_b64_digest); + +// Until this function is called, no one will be able to get a lock on your input params. +void cache_release(CacheHash *ch); #endif diff --git a/src/codegen.cpp b/src/codegen.cpp index a18b26bb1..1f53bb4c9 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -13,7 +13,6 @@ #include "error.hpp" #include "hash_map.hpp" #include "ir.hpp" -#include "link.hpp" #include "os.hpp" #include "translate_c.hpp" #include "target.hpp" @@ -188,6 +187,10 @@ void codegen_set_output_h_path(CodeGen *g, Buf *h_path) { g->out_h_path = h_path; } +void codegen_set_output_path(CodeGen *g, Buf *path) { + g->wanted_output_file_path = path; +} + void codegen_set_clang_argv(CodeGen *g, const char **args, size_t len) { g->clang_argv = args; g->clang_argv_len = len; @@ -5756,8 +5759,6 @@ static void validate_inline_fns(CodeGen *g) { static void do_code_gen(CodeGen *g) { assert(!g->errors.length); - codegen_add_time_event(g, "Code Generation"); - { // create debug type for error sets assert(g->err_enumerators.length == g->errors_by_index.length); @@ -6068,38 +6069,10 @@ static void do_code_gen(CodeGen *g) { codegen_add_time_event(g, "LLVM Emit Output"); - char *err_msg = nullptr; - Buf *o_basename = buf_create_from_buf(g->root_out_name); - - switch (g->emit_file_type) { - case EmitFileTypeBinary: - { - const char *o_ext = target_o_file_ext(&g->zig_target); - buf_append_str(o_basename, o_ext); - break; - } - case EmitFileTypeAssembly: - { - const char *asm_ext = target_asm_file_ext(&g->zig_target); - buf_append_str(o_basename, asm_ext); - break; - } - case EmitFileTypeLLVMIr: - { - const char *llvm_ir_ext = target_llvm_ir_file_ext(&g->zig_target); - buf_append_str(o_basename, llvm_ir_ext); - break; - } - default: - zig_unreachable(); - } - - Buf *output_path = buf_alloc(); - os_path_join(&g->cache_dir, o_basename, output_path); - ensure_cache_dir(g); - bool is_small = g->build_mode == BuildModeSmallRelease; + Buf *output_path = &g->o_file_output_path; + char *err_msg = nullptr; switch (g->emit_file_type) { case EmitFileTypeBinary: if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path), @@ -6118,7 +6091,6 @@ static void do_code_gen(CodeGen *g) { zig_panic("unable to write assembly file %s: %s", buf_ptr(output_path), err_msg); } validate_inline_fns(g); - g->link_objects.append(output_path); break; case EmitFileTypeLLVMIr: @@ -6128,7 +6100,6 @@ static void do_code_gen(CodeGen *g) { zig_panic("unable to write llvm-ir file %s: %s", buf_ptr(output_path), err_msg); } validate_inline_fns(g); - g->link_objects.append(output_path); break; default: @@ -7035,8 +7006,8 @@ static ImportTableEntry *add_special_code(CodeGen *g, PackageTableEntry *package Buf *resolved_path = buf_alloc(); *resolved_path = os_path_resolve(resolve_paths, 1); Buf *import_code = buf_alloc(); - int err; - if ((err = os_fetch_file_path(resolved_path, import_code, false))) { + Error err; + if ((err = cache_add_file_fetch(&g->cache_hash, resolved_path, import_code))) { zig_panic("unable to open '%s': %s\n", buf_ptr(&path_to_code_src), err_str(err)); } @@ -7131,6 +7102,8 @@ static void gen_root_source(CodeGen *g) { Buf *source_code = buf_alloc(); int err; + // No need for using the caching system for this file fetch because it is handled + // separately. if ((err = os_fetch_file_path(resolved_path, source_code, true))) { fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(resolved_path), err_str(err)); exit(1); @@ -7197,6 +7170,8 @@ static void gen_global_asm(CodeGen *g) { int err; for (size_t i = 0; i < g->assembly_files.length; i += 1) { Buf *asm_file = g->assembly_files.at(i); + // No need to use the caching system for these fetches because they + // are handled separately. if ((err = os_fetch_file_path(asm_file, &contents, false))) { zig_panic("Unable to read %s: %s", buf_ptr(asm_file), err_str(err)); } @@ -7460,14 +7435,9 @@ static Buf *preprocessor_mangle(Buf *src) { } static void gen_h_file(CodeGen *g) { - if (!g->want_h_file) - return; - GenH gen_h_data = {0}; GenH *gen_h = &gen_h_data; - codegen_add_time_event(g, "Generate .h"); - assert(!g->is_test_build); if (!g->out_h_path) { @@ -7675,6 +7645,24 @@ void codegen_add_time_event(CodeGen *g, const char *name) { g->timing_events.append({os_get_time(), name}); } +static void add_cache_pkg(CodeGen *g, CacheHash *ch, PackageTableEntry *pkg) { + if (buf_len(&pkg->root_src_path) == 0) + return; + + Buf *rel_full_path = buf_alloc(); + os_path_join(&pkg->root_src_dir, &pkg->root_src_path, rel_full_path); + cache_file(ch, rel_full_path); + + auto it = pkg->package_table.entry_iterator(); + for (;;) { + auto *entry = it.next(); + if (!entry) + break; + + cache_buf(ch, entry->key); + add_cache_pkg(g, ch, entry->value); + } +} // Called before init() static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { @@ -7683,16 +7671,46 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { CacheHash *ch = &g->cache_hash; cache_init(ch, manifest_dir); + add_cache_pkg(g, ch, g->root_package); + if (g->linker_script != nullptr) { + cache_file(ch, buf_create_from_str(g->linker_script)); + } cache_buf(ch, g->compiler_id); cache_buf(ch, g->root_out_name); - cache_file(ch, get_resolved_root_src_path(g)); // Root source file cache_list_of_link_lib(ch, g->link_libs_list.items, g->link_libs_list.length); cache_list_of_buf(ch, g->darwin_frameworks.items, g->darwin_frameworks.length); cache_list_of_buf(ch, g->rpath_list.items, g->rpath_list.length); + cache_list_of_buf(ch, g->forbidden_libs.items, g->forbidden_libs.length); + cache_list_of_file(ch, g->link_objects.items, g->link_objects.length); + cache_list_of_file(ch, g->assembly_files.items, g->assembly_files.length); cache_int(ch, g->emit_file_type); cache_int(ch, g->build_mode); cache_int(ch, g->out_type); - // TODO the rest of the struct CodeGen fields + cache_int(ch, g->zig_target.arch.arch); + cache_int(ch, g->zig_target.arch.sub_arch); + cache_int(ch, g->zig_target.vendor); + cache_int(ch, g->zig_target.os); + cache_int(ch, g->zig_target.env_type); + cache_int(ch, g->zig_target.oformat); + cache_bool(ch, g->is_static); + cache_bool(ch, g->strip_debug_symbols); + cache_bool(ch, g->is_test_build); + cache_bool(ch, g->is_native_target); + cache_bool(ch, g->windows_subsystem_windows); + cache_bool(ch, g->windows_subsystem_console); + cache_bool(ch, g->linker_rdynamic); + cache_bool(ch, g->no_rosegment_workaround); + cache_bool(ch, g->each_lib_rpath); + cache_buf_opt(ch, g->mmacosx_version_min); + cache_buf_opt(ch, g->mios_version_min); + cache_usize(ch, g->version_major); + cache_usize(ch, g->version_minor); + cache_usize(ch, g->version_patch); + cache_buf_opt(ch, g->test_filter); + cache_buf_opt(ch, g->test_name_prefix); + cache_list_of_str(ch, g->llvm_argv, g->llvm_argv_len); + cache_list_of_str(ch, g->clang_argv, g->clang_argv_len); + cache_list_of_str(ch, g->lib_dirs.items, g->lib_dirs.length); buf_resize(digest, 0); if ((err = cache_hit(ch, digest))) @@ -7701,10 +7719,53 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { return ErrorNone; } -void codegen_build(CodeGen *g) { +static void resolve_out_paths(CodeGen *g) { + Buf *o_basename = buf_create_from_buf(g->root_out_name); + + switch (g->emit_file_type) { + case EmitFileTypeBinary: + { + const char *o_ext = target_o_file_ext(&g->zig_target); + buf_append_str(o_basename, o_ext); + break; + } + case EmitFileTypeAssembly: + { + const char *asm_ext = target_asm_file_ext(&g->zig_target); + buf_append_str(o_basename, asm_ext); + break; + } + case EmitFileTypeLLVMIr: + { + const char *llvm_ir_ext = target_llvm_ir_file_ext(&g->zig_target); + buf_append_str(o_basename, llvm_ir_ext); + break; + } + default: + zig_unreachable(); + } + + os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path); + + if (g->out_type == OutTypeObj) { + buf_init_from_buf(&g->output_file_path, &g->o_file_output_path); + } else if (g->out_type == OutTypeExe) { + assert(g->root_out_name); + + Buf basename = BUF_INIT; + buf_init_from_buf(&basename, g->root_out_name); + buf_append_str(&basename, target_exe_file_ext(&g->zig_target)); + os_path_join(&g->artifact_dir, &basename, &g->output_file_path); + } +} + + +void codegen_build_and_link(CodeGen *g) { Error err; assert(g->out_type != OutTypeUnknown); + codegen_add_time_event(g, "Check Cache"); + Buf app_data_dir = BUF_INIT; if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { fprintf(stderr, "Unable to get app data dir: %s\n", err_str(err)); @@ -7721,25 +7782,45 @@ void codegen_build(CodeGen *g) { fprintf(stderr, "Unable to check cache: %s\n", err_str(err)); exit(1); } + + Buf *artifact_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + if (buf_len(&digest) != 0) { - Buf *artifact_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + os_path_join(artifact_dir, &digest, &g->artifact_dir); + resolve_out_paths(g); + } else { + init(g); - Buf *this_artifact_dir = buf_alloc(); - os_path_join(artifact_dir, &digest, this_artifact_dir); + codegen_add_time_event(g, "Semantic Analysis"); - fprintf(stderr, "copy artifacts from %s\n", buf_ptr(this_artifact_dir)); - return; + gen_global_asm(g); + gen_root_source(g); + + if ((err = cache_final(&g->cache_hash, &digest))) { + fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); + exit(1); + } + os_path_join(artifact_dir, &digest, &g->artifact_dir); + if ((err = os_make_path(&g->artifact_dir))) { + fprintf(stderr, "Unable to create artifact directory: %s\n", err_str(err)); + exit(1); + } + resolve_out_paths(g); + + codegen_add_time_event(g, "Code Generation"); + do_code_gen(g); + if (g->want_h_file) { + codegen_add_time_event(g, "Generate .h"); + gen_h_file(g); + } + if (g->out_type != OutTypeObj) { + codegen_link(g); + } } + // TODO hard link output_file_path to wanted_output_file_path - init(g); - - codegen_add_time_event(g, "Semantic Analysis"); - - gen_global_asm(g); - gen_root_source(g); - do_code_gen(g); - gen_h_file(g); + codegen_add_time_event(g, "Done"); } PackageTableEntry *codegen_create_package(CodeGen *g, const char *root_src_dir, const char *root_src_path) { diff --git a/src/codegen.hpp b/src/codegen.hpp index 649cf0685..1e7fafa28 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -48,9 +48,11 @@ void codegen_set_test_name_prefix(CodeGen *g, Buf *prefix); void codegen_set_lib_version(CodeGen *g, size_t major, size_t minor, size_t patch); void codegen_set_cache_dir(CodeGen *g, Buf cache_dir); void codegen_set_output_h_path(CodeGen *g, Buf *h_path); +void codegen_set_output_path(CodeGen *g, Buf *path); void codegen_add_time_event(CodeGen *g, const char *name); void codegen_print_timing_report(CodeGen *g, FILE *f); -void codegen_build(CodeGen *g); +void codegen_link(CodeGen *g); +void codegen_build_and_link(CodeGen *g); PackageTableEntry *codegen_create_package(CodeGen *g, const char *root_src_dir, const char *root_src_path); void codegen_add_assembly(CodeGen *g, Buf *path); diff --git a/src/error.cpp b/src/error.cpp index d9da6e6c5..d503eaa18 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -29,6 +29,7 @@ const char *err_str(int err) { case ErrorCCompileErrors: return "C compile errors"; case ErrorEndOfFile: return "end of file"; case ErrorIsDir: return "is directory"; + case ErrorUnsupportedOperatingSystem: return "unsupported operating system"; } return "(invalid error)"; } diff --git a/src/error.hpp b/src/error.hpp index c99c6e6b1..8c1f8b9aa 100644 --- a/src/error.hpp +++ b/src/error.hpp @@ -29,6 +29,7 @@ enum Error { ErrorCCompileErrors, ErrorEndOfFile, ErrorIsDir, + ErrorUnsupportedOperatingSystem, }; const char *err_str(int err); diff --git a/src/ir.cpp b/src/ir.cpp index 0269c29b1..22d9a9bc4 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -16232,6 +16232,8 @@ static ZigType *ir_analyze_instruction_union_tag(IrAnalyze *ira, IrInstructionUn } static ZigType *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructionImport *import_instruction) { + Error err; + IrInstruction *name_value = import_instruction->name->other; Buf *import_target_str = ir_resolve_str(ira, name_value); if (!import_target_str) @@ -16275,8 +16277,7 @@ static ZigType *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructionImpor return ira->codegen->builtin_types.entry_namespace; } - int err; - if ((err = os_fetch_file_path(resolved_path, import_code, true))) { + if ((err = cache_add_file_fetch(&ira->codegen->cache_hash, resolved_path, import_code))) { if (err == ErrorFileNotFound) { ir_add_error_node(ira, source_node, buf_sprintf("unable to find '%s'", buf_ptr(import_target_path))); @@ -16287,6 +16288,7 @@ static ZigType *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructionImpor return ira->codegen->builtin_types.entry_invalid; } } + ImportTableEntry *target_import = add_source_file(ira->codegen, target_package, resolved_path, import_code); scan_import(ira->codegen, target_import); @@ -18106,7 +18108,7 @@ static ZigType *ir_analyze_instruction_embed_file(IrAnalyze *ira, IrInstructionE // load from file system into const expr Buf *file_contents = buf_alloc(); int err; - if ((err = os_fetch_file_path(&file_path, file_contents, false))) { + if ((err = cache_add_file_fetch(&ira->codegen->cache_hash, &file_path, file_contents))) { if (err == ErrorFileNotFound) { ir_add_error(ira, instruction->name, buf_sprintf("unable to find '%s'", buf_ptr(&file_path))); return ira->codegen->builtin_types.entry_invalid; @@ -18116,9 +18118,6 @@ static ZigType *ir_analyze_instruction_embed_file(IrAnalyze *ira, IrInstructionE } } - // TODO add dependency on the file we embedded so that we know if it changes - // we'll have to invalidate the cache - ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); init_const_str_lit(ira->codegen, out_val, file_contents); diff --git a/src/link.cpp b/src/link.cpp index c0eb25bd5..8c250fbbe 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -5,7 +5,6 @@ * See http://opensource.org/licenses/MIT */ -#include "link.hpp" #include "os.hpp" #include "config.h" #include "codegen.hpp" @@ -13,7 +12,6 @@ struct LinkJob { CodeGen *codegen; - Buf out_file; ZigList args; bool link_in_crt; HashMap rpath_table; @@ -62,14 +60,8 @@ static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) new_link_lib->provided_explicitly = link_lib->provided_explicitly; } - codegen_build(child_gen); - const char *o_ext = target_o_file_ext(&child_gen->zig_target); - Buf *o_out_name = buf_sprintf("%s%s", oname, o_ext); - Buf *output_path = buf_alloc(); - os_path_join(&parent_gen->cache_dir, o_out_name, output_path); - codegen_link(child_gen, buf_ptr(output_path)); - - return output_path; + codegen_build_and_link(child_gen); + return &child_gen->output_file_path; } static Buf *build_o(CodeGen *parent_gen, const char *oname) { @@ -237,15 +229,15 @@ static void construct_linker_job_elf(LinkJob *lj) { } else if (shared) { lj->args.append("-shared"); - if (buf_len(&lj->out_file) == 0) { - buf_appendf(&lj->out_file, "lib%s.so.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize "", + if (buf_len(&g->output_file_path) == 0) { + buf_appendf(&g->output_file_path, "lib%s.so.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize "", buf_ptr(g->root_out_name), g->version_major, g->version_minor, g->version_patch); } soname = buf_sprintf("lib%s.so.%" ZIG_PRI_usize "", buf_ptr(g->root_out_name), g->version_major); } lj->args.append("-o"); - lj->args.append(buf_ptr(&lj->out_file)); + lj->args.append(buf_ptr(&g->output_file_path)); if (lj->link_in_crt) { const char *crt1o; @@ -397,7 +389,7 @@ static void construct_linker_job_wasm(LinkJob *lj) { lj->args.append("--relocatable"); // So lld doesn't look for _start. lj->args.append("-o"); - lj->args.append(buf_ptr(&lj->out_file)); + lj->args.append(buf_ptr(&g->output_file_path)); // .o files for (size_t i = 0; i < g->link_objects.length; i += 1) { @@ -478,7 +470,7 @@ static void construct_linker_job_coff(LinkJob *lj) { // } //} - lj->args.append(buf_ptr(buf_sprintf("-OUT:%s", buf_ptr(&lj->out_file)))); + lj->args.append(buf_ptr(buf_sprintf("-OUT:%s", buf_ptr(&g->output_file_path)))); if (g->libc_link_lib != nullptr) { lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->msvc_lib_dir)))); @@ -585,11 +577,11 @@ static void construct_linker_job_coff(LinkJob *lj) { buf_appendf(def_contents, "\n"); Buf *def_path = buf_alloc(); - os_path_join(&g->cache_dir, buf_sprintf("%s.def", buf_ptr(link_lib->name)), def_path); + os_path_join(&g->artifact_dir, buf_sprintf("%s.def", buf_ptr(link_lib->name)), def_path); os_write_file(def_path, def_contents); Buf *generated_lib_path = buf_alloc(); - os_path_join(&g->cache_dir, buf_sprintf("%s.lib", buf_ptr(link_lib->name)), generated_lib_path); + os_path_join(&g->artifact_dir, buf_sprintf("%s.lib", buf_ptr(link_lib->name)), generated_lib_path); gen_lib_args.resize(0); gen_lib_args.append("link"); @@ -797,8 +789,8 @@ static void construct_linker_job_macho(LinkJob *lj) { //lj->args.append("-install_name"); //lj->args.append(buf_ptr(dylib_install_name)); - if (buf_len(&lj->out_file) == 0) { - buf_appendf(&lj->out_file, "lib%s.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".dylib", + if (buf_len(&g->output_file_path) == 0) { + buf_appendf(&g->output_file_path, "lib%s.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".dylib", buf_ptr(g->root_out_name), g->version_major, g->version_minor, g->version_patch); } } @@ -832,13 +824,13 @@ static void construct_linker_job_macho(LinkJob *lj) { } lj->args.append("-o"); - lj->args.append(buf_ptr(&lj->out_file)); + lj->args.append(buf_ptr(&g->output_file_path)); for (size_t i = 0; i < g->rpath_list.length; i += 1) { Buf *rpath = g->rpath_list.at(i); add_rpath(lj, rpath); } - add_rpath(lj, &lj->out_file); + add_rpath(lj, &g->output_file_path); if (shared) { lj->args.append("-headerpad_max_install_names"); @@ -942,7 +934,8 @@ static void construct_linker_job(LinkJob *lj) { } } -void codegen_link(CodeGen *g, const char *out_file) { +void codegen_link(CodeGen *g) { + assert(g->out_type != OutTypeObj); codegen_add_time_event(g, "Build Dependencies"); LinkJob lj = {0}; @@ -953,11 +946,6 @@ void codegen_link(CodeGen *g, const char *out_file) { lj.rpath_table.init(4); lj.codegen = g; - if (out_file) { - buf_init_from_str(&lj.out_file, out_file); - } else { - buf_resize(&lj.out_file, 0); - } if (g->verbose_llvm_ir) { fprintf(stderr, "\nOptimization:\n"); @@ -966,35 +954,9 @@ void codegen_link(CodeGen *g, const char *out_file) { LLVMDumpModule(g->module); } - bool override_out_file = (buf_len(&lj.out_file) != 0); - if (!override_out_file) { - assert(g->root_out_name); - - buf_init_from_buf(&lj.out_file, g->root_out_name); - if (g->out_type == OutTypeExe) { - buf_append_str(&lj.out_file, target_exe_file_ext(&g->zig_target)); - } - } - - if (g->out_type == OutTypeObj) { - if (override_out_file) { - assert(g->link_objects.length == 1); - Buf *o_file_path = g->link_objects.at(0); - int err; - if ((err = os_rename(o_file_path, &lj.out_file))) { - zig_panic("unable to rename object file %s into final output %s: %s", buf_ptr(o_file_path), buf_ptr(&lj.out_file), err_str(err)); - } - } - return; - } - if (g->out_type == OutTypeLib && g->is_static) { - // invoke `ar` - // example: - // # static link into libfoo.a - // ar rcs libfoo.a foo1.o foo2.o - zig_panic("TODO invoke ar"); - return; + fprintf(stderr, "Zig does not yet support creating static libraries\nSee https://github.com/ziglang/zig/issues/1493\n"); + exit(1); } lj.link_in_crt = (g->libc_link_lib != nullptr && g->out_type == OutTypeExe); @@ -1017,6 +979,4 @@ void codegen_link(CodeGen *g, const char *out_file) { fprintf(stderr, "%s\n", buf_ptr(&diag)); exit(1); } - - codegen_add_time_event(g, "Done"); } diff --git a/src/link.hpp b/src/link.hpp deleted file mode 100644 index 9f978c28d..000000000 --- a/src/link.hpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2015 Andrew Kelley - * - * This file is part of zig, which is MIT licensed. - * See http://opensource.org/licenses/MIT - */ - -#ifndef ZIG_LINK_HPP -#define ZIG_LINK_HPP - -#include "all_types.hpp" - -void codegen_link(CodeGen *g, const char *out_file); - - -#endif - diff --git a/src/main.cpp b/src/main.cpp index 1520ae7d6..ff2c061a8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -10,7 +10,6 @@ #include "codegen.hpp" #include "config.h" #include "error.hpp" -#include "link.hpp" #include "os.hpp" #include "target.hpp" #include "cache_hash.hpp" @@ -385,8 +384,7 @@ int main(int argc, char **argv) { CliPkg *cur_pkg = allocate(1); BuildMode build_mode = BuildModeDebug; ZigList test_exec_args = {0}; - int comptime_args_end = 0; - int runtime_args_start = argc; + int runtime_args_start = -1; bool no_rosegment_workaround = false; if (argc >= 2 && strcmp(argv[1], "build") == 0) { @@ -454,8 +452,6 @@ int main(int argc, char **argv) { full_cache_dir = os_path_resolve(&cache_dir_buf, 1); } - Buf *path_to_build_exe = buf_alloc(); - os_path_join(&full_cache_dir, buf_create_from_str("build"), path_to_build_exe); codegen_set_cache_dir(g, full_cache_dir); args.items[1] = buf_ptr(&build_file_dirname); @@ -525,14 +521,13 @@ int main(int argc, char **argv) { PackageTableEntry *build_pkg = codegen_create_package(g, buf_ptr(&build_file_dirname), buf_ptr(&build_file_basename)); g->root_package->package_table.put(buf_create_from_str("@build"), build_pkg); - codegen_build(g); - codegen_link(g, buf_ptr(path_to_build_exe)); + codegen_build_and_link(g); Termination term; - os_spawn_process(buf_ptr(path_to_build_exe), args, &term); + os_spawn_process(buf_ptr(&g->output_file_path), args, &term); if (term.how != TerminationIdClean || term.code != 0) { fprintf(stderr, "\nBuild failed. The following command failed:\n"); - fprintf(stderr, "%s", buf_ptr(path_to_build_exe)); + fprintf(stderr, "%s", buf_ptr(&g->output_file_path)); for (size_t i = 0; i < args.length; i += 1) { fprintf(stderr, " %s", args.at(i)); } @@ -541,15 +536,11 @@ int main(int argc, char **argv) { return (term.how == TerminationIdClean) ? term.code : -1; } - for (int i = 1; i < argc; i += 1, comptime_args_end += 1) { + for (int i = 1; i < argc; i += 1) { char *arg = argv[i]; if (arg[0] == '-') { - if (strcmp(arg, "--") == 0) { - // ignore -- from both compile and runtime arg sets - runtime_args_start = i + 1; - break; - } else if (strcmp(arg, "--release-fast") == 0) { + if (strcmp(arg, "--release-fast") == 0) { build_mode = BuildModeFastRelease; } else if (strcmp(arg, "--release-safe") == 0) { build_mode = BuildModeSafeRelease; @@ -746,6 +737,10 @@ int main(int argc, char **argv) { case CmdTest: if (!in_file) { in_file = arg; + if (cmd == CmdRun) { + runtime_args_start = i + 1; + break; // rest of the args are for the program + } } else { fprintf(stderr, "Unexpected extra parameter: %s\n", arg); return usage(arg0); @@ -856,19 +851,10 @@ int main(int argc, char **argv) { Buf *zig_root_source_file = (cmd == CmdTranslateC) ? nullptr : in_file_buf; Buf full_cache_dir = BUF_INIT; - Buf *run_exec_path = buf_alloc(); - if (cmd == CmdRun) { - if (buf_out_name == nullptr) { - buf_out_name = buf_create_from_str("run"); - } - - Buf *global_cache_dir = buf_alloc(); - os_get_global_cache_directory(global_cache_dir); - os_path_join(global_cache_dir, buf_out_name, run_exec_path); - full_cache_dir = os_path_resolve(&global_cache_dir, 1); - - out_file = buf_ptr(run_exec_path); - } else { + if (cmd == CmdRun && buf_out_name == nullptr) { + buf_out_name = buf_create_from_str("run"); + } + { Buf *resolve_paths = buf_create_from_str((cache_dir == nullptr) ? default_zig_cache_name : cache_dir); full_cache_dir = os_path_resolve(&resolve_paths, 1); } @@ -957,6 +943,8 @@ int main(int argc, char **argv) { codegen_set_test_name_prefix(g, buf_create_from_str(test_name_prefix)); } + if (out_file) + codegen_set_output_path(g, buf_create_from_str(out_file)); if (out_file_h) codegen_set_output_h_path(g, buf_create_from_str(out_file_h)); @@ -976,8 +964,7 @@ int main(int argc, char **argv) { if (cmd == CmdBuild || cmd == CmdRun) { codegen_set_emit_file_type(g, emit_file_type); - codegen_build(g); - codegen_link(g, out_file); + codegen_build_and_link(g); if (timing_info) codegen_print_timing_report(g, stdout); @@ -987,8 +974,14 @@ int main(int argc, char **argv) { args.append(argv[i]); } + const char *exec_path = buf_ptr(&g->output_file_path); + args.append(nullptr); + + os_execv(exec_path, args.items); + + args.pop(); Termination term; - os_spawn_process(buf_ptr(run_exec_path), args, &term); + os_spawn_process(exec_path, args, &term); return term.code; } @@ -1005,11 +998,8 @@ int main(int argc, char **argv) { ZigTarget native; get_native_target(&native); - ZigTarget *non_null_target = target ? target : &native; - - Buf *test_exe_name = buf_sprintf("test%s", target_exe_file_ext(non_null_target)); - Buf *test_exe_path = buf_alloc(); - os_path_join(&full_cache_dir, test_exe_name, test_exe_path); + codegen_build_and_link(g); + Buf *test_exe_path = &g->output_file_path; for (size_t i = 0; i < test_exec_args.length; i += 1) { if (test_exec_args.items[i] == nullptr) { @@ -1017,8 +1007,6 @@ int main(int argc, char **argv) { } } - codegen_build(g); - codegen_link(g, buf_ptr(test_exe_path)); if (!target_can_exec(&native, target)) { fprintf(stderr, "Created %s but skipping execution because it is non-native.\n", diff --git a/src/os.cpp b/src/os.cpp index 4ee461818..8502e7271 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -972,6 +972,22 @@ static int os_exec_process_windows(const char *exe, ZigList &args, } #endif +Error os_execv(const char *exe, const char **argv) { +#if defined(ZIG_OS_WINDOWS) + return ErrorUnsupportedOperatingSystem; +#else + execv(exe, (char *const *)argv); + switch (errno) { + case ENOMEM: + return ErrorSystemResources; + case EIO: + return ErrorFileSystem; + default: + return ErrorUnexpected; + } +#endif +} + int os_exec_process(const char *exe, ZigList &args, Termination *term, Buf *out_stderr, Buf *out_stdout) { @@ -1238,44 +1254,6 @@ int os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path) { #endif } -#if defined(ZIG_OS_POSIX) -int os_get_global_cache_directory(Buf *out_tmp_path) { - const char *tmp_dir = getenv("TMPDIR"); - if (!tmp_dir) { - tmp_dir = P_tmpdir; - } - - Buf *tmp_dir_buf = buf_create_from_str(tmp_dir); - Buf *cache_dirname_buf = buf_create_from_str("zig-cache"); - - buf_resize(out_tmp_path, 0); - os_path_join(tmp_dir_buf, cache_dirname_buf, out_tmp_path); - - buf_deinit(tmp_dir_buf); - buf_deinit(cache_dirname_buf); - return 0; -} -#endif - -#if defined(ZIG_OS_WINDOWS) -int os_get_global_cache_directory(Buf *out_tmp_path) { - char tmp_dir[MAX_PATH + 1]; - if (GetTempPath(MAX_PATH, tmp_dir) == 0) { - zig_panic("GetTempPath failed"); - } - - Buf *tmp_dir_buf = buf_create_from_str(tmp_dir); - Buf *cache_dirname_buf = buf_create_from_str("zig-cache"); - - buf_resize(out_tmp_path, 0); - os_path_join(tmp_dir_buf, cache_dirname_buf, out_tmp_path); - - buf_deinit(tmp_dir_buf); - buf_deinit(cache_dirname_buf); - return 0; -} -#endif - int os_delete_file(Buf *path) { if (remove(buf_ptr(path))) { return ErrorFileSystem; diff --git a/src/os.hpp b/src/os.hpp index c64eccf8d..fc2a34326 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -87,6 +87,7 @@ int os_init(void); void os_spawn_process(const char *exe, ZigList &args, Termination *term); int os_exec_process(const char *exe, ZigList &args, Termination *term, Buf *out_stderr, Buf *out_stdout); +Error os_execv(const char *exe, const char **argv); void os_path_dirname(Buf *full_path, Buf *out_dirname); void os_path_split(Buf *full_path, Buf *out_dirname, Buf *out_basename); @@ -96,8 +97,6 @@ int os_path_real(Buf *rel_path, Buf *out_abs_path); Buf os_path_resolve(Buf **paths_ptr, size_t paths_len); bool os_path_is_absolute(Buf *path); -int os_get_global_cache_directory(Buf *out_tmp_path); - Error ATTRIBUTE_MUST_USE os_make_path(Buf *path); Error ATTRIBUTE_MUST_USE os_make_dir(Buf *path); From 5ee5933ade09c535bd1806d91cb606f49d07acea Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 10 Sep 2018 17:30:45 -0400 Subject: [PATCH 08/46] stage1 caching: zig no longer uses zig-cache --- CMakeLists.txt | 1 + src/all_types.hpp | 2 - src/cache_hash.cpp | 7 +++ src/codegen.cpp | 110 ++++++++++++++++++++++++++++++--------------- src/codegen.hpp | 3 +- src/compiler.cpp | 66 +++++++++++++++++++++++++++ src/compiler.hpp | 17 +++++++ src/link.cpp | 4 +- src/main.cpp | 78 ++------------------------------ src/os.cpp | 22 ++++----- src/os.hpp | 6 +-- 11 files changed, 187 insertions(+), 129 deletions(-) create mode 100644 src/compiler.cpp create mode 100644 src/compiler.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8339be71b..11bb31892 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -411,6 +411,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp" "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" + "${CMAKE_SOURCE_DIR}/src/compiler.cpp" "${CMAKE_SOURCE_DIR}/src/errmsg.cpp" "${CMAKE_SOURCE_DIR}/src/error.cpp" "${CMAKE_SOURCE_DIR}/src/ir.cpp" diff --git a/src/all_types.hpp b/src/all_types.hpp index 6d11244a2..4a1516207 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1671,7 +1671,6 @@ struct CodeGen { Buf triple_str; Buf global_asm; Buf *out_h_path; - Buf cache_dir; Buf artifact_dir; Buf output_file_path; Buf o_file_output_path; @@ -1731,7 +1730,6 @@ struct CodeGen { ZigList assembly_files; ZigList lib_dirs; - Buf *compiler_id; size_t version_major; size_t version_minor; size_t version_patch; diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index b6aebdac6..b30294631 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -248,6 +248,12 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { int rc = blake2b_final(&ch->blake, bin_digest, 48); assert(rc == 0); + if (ch->files.length == 0) { + buf_resize(out_digest, 64); + base64_encode(buf_to_slice(out_digest), {bin_digest, 48}); + return ErrorNone; + } + Buf b64_digest = BUF_INIT; buf_resize(&b64_digest, 64); base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); @@ -458,5 +464,6 @@ Error cache_final(CacheHash *ch, Buf *out_digest) { } void cache_release(CacheHash *ch) { + assert(ch->manifest_file_path != nullptr); os_file_close(ch->manifest_file); } diff --git a/src/codegen.cpp b/src/codegen.cpp index 1f53bb4c9..578fb314a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -8,6 +8,7 @@ #include "analyze.hpp" #include "ast_render.hpp" #include "codegen.hpp" +#include "compiler.hpp" #include "config.h" #include "errmsg.hpp" #include "error.hpp" @@ -87,13 +88,12 @@ static const char *symbols_that_llvm_depends_on[] = { }; CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir, Buf *compiler_id) + Buf *zig_lib_dir) { CodeGen *g = allocate(1); codegen_add_time_event(g, "Initialize"); - g->compiler_id = compiler_id; g->zig_lib_dir = zig_lib_dir; g->zig_std_dir = buf_alloc(); @@ -243,10 +243,6 @@ void codegen_set_out_name(CodeGen *g, Buf *out_name) { g->root_out_name = out_name; } -void codegen_set_cache_dir(CodeGen *g, Buf cache_dir) { - g->cache_dir = cache_dir; -} - void codegen_set_libc_lib_dir(CodeGen *g, Buf *libc_lib_dir) { g->libc_lib_dir = libc_lib_dir; } @@ -5728,13 +5724,6 @@ static LLVMValueRef build_alloca(CodeGen *g, ZigType *type_entry, const char *na return result; } -static void ensure_cache_dir(CodeGen *g) { - int err; - if ((err = os_make_path(&g->cache_dir))) { - zig_panic("unable to make cache dir: %s", err_str(err)); - } -} - static void report_errors_and_maybe_exit(CodeGen *g) { if (g->errors.length != 0) { for (size_t i = 0; i < g->errors.length; i += 1) { @@ -6824,36 +6813,84 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { return contents; } -static void define_builtin_compile_vars(CodeGen *g) { +static Error define_builtin_compile_vars(CodeGen *g) { if (g->std_package == nullptr) - return; + return ErrorNone; + + Error err; + + Buf *manifest_dir = buf_alloc(); + os_path_join(get_stage1_cache_path(), buf_create_from_str("builtin"), manifest_dir); + + CacheHash cache_hash; + cache_init(&cache_hash, manifest_dir); + + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) + return err; + + // Only a few things affect builtin.zig + cache_buf(&cache_hash, compiler_id); + cache_int(&cache_hash, g->build_mode); + cache_bool(&cache_hash, g->is_test_build); + cache_int(&cache_hash, g->zig_target.arch.arch); + cache_int(&cache_hash, g->zig_target.arch.sub_arch); + cache_int(&cache_hash, g->zig_target.vendor); + cache_int(&cache_hash, g->zig_target.os); + cache_int(&cache_hash, g->zig_target.env_type); + cache_int(&cache_hash, g->zig_target.oformat); + cache_bool(&cache_hash, g->have_err_ret_tracing); + cache_bool(&cache_hash, g->libc_link_lib != nullptr); + + Buf digest = BUF_INIT; + buf_resize(&digest, 0); + if ((err = cache_hit(&cache_hash, &digest))) + return err; + + // We should always get a cache hit because there are no + // files in the input hash. + assert(buf_len(&digest) != 0); + + Buf *this_dir = buf_alloc(); + os_path_join(manifest_dir, &digest, this_dir); + + if ((err = os_make_path(this_dir))) + return err; const char *builtin_zig_basename = "builtin.zig"; Buf *builtin_zig_path = buf_alloc(); - os_path_join(&g->cache_dir, buf_create_from_str(builtin_zig_basename), builtin_zig_path); + os_path_join(this_dir, buf_create_from_str(builtin_zig_basename), builtin_zig_path); - Buf *contents = codegen_generate_builtin_source(g); - ensure_cache_dir(g); - os_write_file(builtin_zig_path, contents); - - Buf *resolved_path = buf_alloc(); - Buf *resolve_paths[] = {builtin_zig_path}; - *resolved_path = os_path_resolve(resolve_paths, 1); + bool hit; + if ((err = os_file_exists(builtin_zig_path, &hit))) + return err; + Buf *contents; + if (hit) { + contents = buf_alloc(); + if ((err = os_fetch_file_path(builtin_zig_path, contents, false))) { + fprintf(stderr, "Unable to open '%s': %s\n", buf_ptr(builtin_zig_path), err_str(err)); + exit(1); + } + } else { + contents = codegen_generate_builtin_source(g); + os_write_file(builtin_zig_path, contents); + } assert(g->root_package); assert(g->std_package); - g->compile_var_package = new_package(buf_ptr(&g->cache_dir), builtin_zig_basename); + g->compile_var_package = new_package(buf_ptr(this_dir), builtin_zig_basename); g->root_package->package_table.put(buf_create_from_str("builtin"), g->compile_var_package); g->std_package->package_table.put(buf_create_from_str("builtin"), g->compile_var_package); - g->compile_var_import = add_source_file(g, g->compile_var_package, resolved_path, contents); + g->compile_var_import = add_source_file(g, g->compile_var_package, builtin_zig_path, contents); scan_import(g, g->compile_var_import); + + return ErrorNone; } static void init(CodeGen *g) { if (g->module) return; - if (g->llvm_argv_len > 0) { const char **args = allocate_nonzero(g->llvm_argv_len + 2); args[0] = "zig (LLVM option parsing)"; @@ -6960,7 +6997,11 @@ static void init(CodeGen *g) { g->have_err_ret_tracing = g->build_mode != BuildModeFastRelease && g->build_mode != BuildModeSmallRelease; define_builtin_fns(g); - define_builtin_compile_vars(g); + Error err; + if ((err = define_builtin_compile_vars(g))) { + fprintf(stderr, "Unable to create builtin.zig: %s\n", err_str(err)); + exit(1); + } } void codegen_translate_c(CodeGen *g, Buf *full_path) { @@ -7668,6 +7709,10 @@ static void add_cache_pkg(CodeGen *g, CacheHash *ch, PackageTableEntry *pkg) { static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { Error err; + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) + return err; + CacheHash *ch = &g->cache_hash; cache_init(ch, manifest_dir); @@ -7675,7 +7720,7 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { if (g->linker_script != nullptr) { cache_file(ch, buf_create_from_str(g->linker_script)); } - cache_buf(ch, g->compiler_id); + cache_buf(ch, compiler_id); cache_buf(ch, g->root_out_name); cache_list_of_link_lib(ch, g->link_libs_list.items, g->link_libs_list.length); cache_list_of_buf(ch, g->darwin_frameworks.items, g->darwin_frameworks.length); @@ -7766,13 +7811,7 @@ void codegen_build_and_link(CodeGen *g) { codegen_add_time_event(g, "Check Cache"); - Buf app_data_dir = BUF_INIT; - if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { - fprintf(stderr, "Unable to get app data dir: %s\n", err_str(err)); - exit(1); - } - Buf *stage1_dir = buf_alloc(); - os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); + Buf *stage1_dir = get_stage1_cache_path(); Buf *manifest_dir = buf_alloc(); os_path_join(stage1_dir, buf_create_from_str("build"), manifest_dir); @@ -7820,6 +7859,7 @@ void codegen_build_and_link(CodeGen *g) { } // TODO hard link output_file_path to wanted_output_file_path + cache_release(&g->cache_hash); codegen_add_time_event(g, "Done"); } diff --git a/src/codegen.hpp b/src/codegen.hpp index 1e7fafa28..203e2d2b9 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -15,7 +15,7 @@ #include CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir, Buf *compiler_id); + Buf *zig_lib_dir); void codegen_set_clang_argv(CodeGen *codegen, const char **args, size_t len); void codegen_set_llvm_argv(CodeGen *codegen, const char **args, size_t len); @@ -46,7 +46,6 @@ void codegen_set_linker_script(CodeGen *g, const char *linker_script); void codegen_set_test_filter(CodeGen *g, Buf *filter); void codegen_set_test_name_prefix(CodeGen *g, Buf *prefix); void codegen_set_lib_version(CodeGen *g, size_t major, size_t minor, size_t patch); -void codegen_set_cache_dir(CodeGen *g, Buf cache_dir); void codegen_set_output_h_path(CodeGen *g, Buf *h_path); void codegen_set_output_path(CodeGen *g, Buf *path); void codegen_add_time_event(CodeGen *g, const char *name); diff --git a/src/compiler.cpp b/src/compiler.cpp new file mode 100644 index 000000000..dd02b541d --- /dev/null +++ b/src/compiler.cpp @@ -0,0 +1,66 @@ +#include "cache_hash.hpp" + +#include + +static Buf saved_compiler_id = BUF_INIT; +static Buf saved_app_data_dir = BUF_INIT; +static Buf saved_stage1_path = BUF_INIT; + +Buf *get_stage1_cache_path() { + if (saved_stage1_path.list.length != 0) { + return &saved_stage1_path; + } + Error err; + if ((err = os_get_app_data_dir(&saved_app_data_dir, "zig"))) { + fprintf(stderr, "Unable to get app data dir: %s\n", err_str(err)); + exit(1); + } + os_path_join(&saved_app_data_dir, buf_create_from_str("stage1"), &saved_stage1_path); + return &saved_stage1_path; +} + +Error get_compiler_id(Buf **result) { + if (saved_compiler_id.list.length != 0) { + *result = &saved_compiler_id; + return ErrorNone; + } + + Error err; + Buf *stage1_dir = get_stage1_cache_path(); + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); + + CacheHash cache_hash; + CacheHash *ch = &cache_hash; + cache_init(ch, manifest_dir); + Buf self_exe_path = BUF_INIT; + if ((err = os_self_exe_path(&self_exe_path))) + return err; + + cache_file(ch, &self_exe_path); + + buf_resize(&saved_compiler_id, 0); + if ((err = cache_hit(ch, &saved_compiler_id))) + return err; + if (buf_len(&saved_compiler_id) != 0) { + cache_release(ch); + *result = &saved_compiler_id; + return ErrorNone; + } + ZigList lib_paths = {}; + if ((err = os_self_exe_shared_libs(lib_paths))) + return err; + for (size_t i = 0; i < lib_paths.length; i += 1) { + Buf *lib_path = lib_paths.at(i); + if ((err = cache_add_file(ch, lib_path))) + return err; + } + if ((err = cache_final(ch, &saved_compiler_id))) + return err; + + cache_release(ch); + + *result = &saved_compiler_id; + return ErrorNone; +} + diff --git a/src/compiler.hpp b/src/compiler.hpp new file mode 100644 index 000000000..b95e4ceda --- /dev/null +++ b/src/compiler.hpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2018 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#ifndef ZIG_COMPILER_HPP +#define ZIG_COMPILER_HPP + +#include "buffer.hpp" +#include "error.hpp" + +Buf *get_stage1_cache_path(); +Error get_compiler_id(Buf **result); + +#endif diff --git a/src/link.cpp b/src/link.cpp index 8c250fbbe..8d7b8b4d5 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -32,7 +32,7 @@ static const char *get_libc_static_file(CodeGen *g, const char *file) { static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) { ZigTarget *child_target = parent_gen->is_native_target ? nullptr : &parent_gen->zig_target; CodeGen *child_gen = codegen_create(full_path, child_target, OutTypeObj, parent_gen->build_mode, - parent_gen->zig_lib_dir, parent_gen->compiler_id); + parent_gen->zig_lib_dir); child_gen->want_h_file = false; child_gen->verbose_tokenize = parent_gen->verbose_tokenize; @@ -42,8 +42,6 @@ static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) child_gen->verbose_llvm_ir = parent_gen->verbose_llvm_ir; child_gen->verbose_cimport = parent_gen->verbose_cimport; - codegen_set_cache_dir(child_gen, parent_gen->cache_dir); - codegen_set_strip(child_gen, parent_gen->strip_debug_symbols); codegen_set_is_static(child_gen, parent_gen->is_static); diff --git a/src/main.cpp b/src/main.cpp index ff2c061a8..10b789a6e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -8,11 +8,11 @@ #include "ast_render.hpp" #include "buffer.hpp" #include "codegen.hpp" +#include "compiler.hpp" #include "config.h" #include "error.hpp" #include "os.hpp" #include "target.hpp" -#include "cache_hash.hpp" #include @@ -257,53 +257,6 @@ static void add_package(CodeGen *g, CliPkg *cli_pkg, PackageTableEntry *pkg) { } } -static Buf saved_compiler_id = BUF_INIT; -static Error get_compiler_id(Buf **result) { - if (saved_compiler_id.list.length != 0) { - *result = &saved_compiler_id; - return ErrorNone; - } - - Error err; - Buf app_data_dir = BUF_INIT; - if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) - return err; - Buf *stage1_dir = buf_alloc(); - os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); - Buf *manifest_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); - - CacheHash cache_hash; - CacheHash *ch = &cache_hash; - cache_init(ch, manifest_dir); - Buf self_exe_path = BUF_INIT; - if ((err = os_self_exe_path(&self_exe_path))) - return err; - - cache_file(ch, &self_exe_path); - - buf_resize(&saved_compiler_id, 0); - if ((err = cache_hit(ch, &saved_compiler_id))) - return err; - if (buf_len(&saved_compiler_id) != 0) { - *result = &saved_compiler_id; - return ErrorNone; - } - ZigList lib_paths = {}; - if ((err = os_self_exe_shared_libs(lib_paths))) - return err; - for (size_t i = 0; i < lib_paths.length; i += 1) { - Buf *lib_path = lib_paths.at(i); - if ((err = cache_add_file(ch, lib_path))) - return err; - } - if ((err = cache_final(ch, &saved_compiler_id))) - return err; - - *result = &saved_compiler_id; - return ErrorNone; -} - int main(int argc, char **argv) { if (argc == 2 && strcmp(argv[1], "BUILD_INFO") == 0) { printf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", @@ -428,14 +381,7 @@ int main(int argc, char **argv) { Buf *build_runner_path = buf_alloc(); os_path_join(special_dir, buf_create_from_str("build_runner.zig"), build_runner_path); - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) { - fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); - return EXIT_FAILURE; - } - - CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf, - compiler_id); + CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf); codegen_set_out_name(g, buf_create_from_str("build")); Buf *build_file_buf = buf_create_from_str(build_file); @@ -452,8 +398,6 @@ int main(int argc, char **argv) { full_cache_dir = os_path_resolve(&cache_dir_buf, 1); } - codegen_set_cache_dir(g, full_cache_dir); - args.items[1] = buf_ptr(&build_file_dirname); args.items[2] = buf_ptr(&full_cache_dir); @@ -795,7 +739,7 @@ int main(int argc, char **argv) { switch (cmd) { case CmdBuiltin: { Buf *zig_lib_dir_buf = resolve_zig_lib_dir(); - CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, zig_lib_dir_buf, nullptr); + CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, zig_lib_dir_buf); Buf *builtin_source = codegen_generate_builtin_source(g); if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) { fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout))); @@ -850,30 +794,16 @@ int main(int argc, char **argv) { Buf *zig_root_source_file = (cmd == CmdTranslateC) ? nullptr : in_file_buf; - Buf full_cache_dir = BUF_INIT; if (cmd == CmdRun && buf_out_name == nullptr) { buf_out_name = buf_create_from_str("run"); } - { - Buf *resolve_paths = buf_create_from_str((cache_dir == nullptr) ? default_zig_cache_name : cache_dir); - full_cache_dir = os_path_resolve(&resolve_paths, 1); - } - Buf *zig_lib_dir_buf = resolve_zig_lib_dir(); - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) { - fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); - return EXIT_FAILURE; - } - - CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf, - compiler_id); + CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf); codegen_set_out_name(g, buf_out_name); codegen_set_lib_version(g, ver_major, ver_minor, ver_patch); codegen_set_is_test(g, cmd == CmdTest); codegen_set_linker_script(g, linker_script); - codegen_set_cache_dir(g, full_cache_dir); if (each_lib_rpath) codegen_set_each_lib_rpath(g, each_lib_rpath); diff --git a/src/os.cpp b/src/os.cpp index 8502e7271..3950711c5 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -721,7 +721,7 @@ Buf os_path_resolve(Buf **paths_ptr, size_t paths_len) { #endif } -int os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { +Error os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { static const ssize_t buf_size = 0x2000; buf_resize(out_buf, buf_size); ssize_t actual_buf_len = 0; @@ -757,7 +757,7 @@ int os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { if (amt_read != buf_size) { if (feof(f)) { buf_resize(out_buf, actual_buf_len); - return 0; + return ErrorNone; } else { return ErrorFileSystem; } @@ -769,13 +769,13 @@ int os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { zig_unreachable(); } -int os_file_exists(Buf *full_path, bool *result) { +Error os_file_exists(Buf *full_path, bool *result) { #if defined(ZIG_OS_WINDOWS) *result = GetFileAttributes(buf_ptr(full_path)) != INVALID_FILE_ATTRIBUTES; - return 0; + return ErrorNone; #else *result = access(buf_ptr(full_path), F_OK) != -1; - return 0; + return ErrorNone; #endif } @@ -834,13 +834,15 @@ static int os_exec_process_posix(const char *exe, ZigList &args, FILE *stdout_f = fdopen(stdout_pipe[0], "rb"); FILE *stderr_f = fdopen(stderr_pipe[0], "rb"); - os_fetch_file(stdout_f, out_stdout, false); - os_fetch_file(stderr_f, out_stderr, false); + Error err1 = os_fetch_file(stdout_f, out_stdout, false); + Error err2 = os_fetch_file(stderr_f, out_stderr, false); fclose(stdout_f); fclose(stderr_f); - return 0; + if (err1) return err1; + if (err2) return err2; + return ErrorNone; } } #endif @@ -1064,7 +1066,7 @@ int os_copy_file(Buf *src_path, Buf *dest_path) { } } -int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang) { +Error os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang) { FILE *f = fopen(buf_ptr(full_path), "rb"); if (!f) { switch (errno) { @@ -1083,7 +1085,7 @@ int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang) { return ErrorFileSystem; } } - int result = os_fetch_file(f, out_contents, skip_shebang); + Error result = os_fetch_file(f, out_contents, skip_shebang); fclose(f); return result; } diff --git a/src/os.hpp b/src/os.hpp index fc2a34326..ac422fbd2 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -111,8 +111,8 @@ void os_file_close(OsFile file); void os_write_file(Buf *full_path, Buf *contents); int os_copy_file(Buf *src_path, Buf *dest_path); -int os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); -int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); +Error ATTRIBUTE_MUST_USE os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); +Error ATTRIBUTE_MUST_USE os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); int os_get_cwd(Buf *out_cwd); @@ -122,7 +122,7 @@ void os_stderr_set_color(TermColor color); int os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path); int os_delete_file(Buf *path); -int os_file_exists(Buf *full_path, bool *result); +Error ATTRIBUTE_MUST_USE os_file_exists(Buf *full_path, bool *result); int os_rename(Buf *src_path, Buf *dest_path); double os_get_time(void); From 67735c6f1557092efe6e8c1712445c30655fe283 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 11 Sep 2018 00:32:40 -0400 Subject: [PATCH 09/46] ability to disable cache. off by default except for... ...zig run, zig build, compiler_rt.a, and builtin.a --- src/all_types.hpp | 1 + src/analyze.cpp | 8 ++++ src/analyze.hpp | 3 ++ src/cache_hash.cpp | 8 ++++ src/cache_hash.hpp | 5 +++ src/codegen.cpp | 105 ++++++++++++++++++++++++++++++++------------- src/ir.cpp | 4 +- src/link.cpp | 1 + src/main.cpp | 50 +++++++++++++++++++-- src/target.cpp | 16 +++++++ src/target.hpp | 1 + 11 files changed, 167 insertions(+), 35 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index 4a1516207..cdc290268 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1718,6 +1718,7 @@ struct CodeGen { bool verbose_cimport; bool error_during_imports; bool generate_error_name_table; + bool enable_cache; //////////////////////////// Participates in Input Parameter Cache Hash ZigList link_libs_list; diff --git a/src/analyze.cpp b/src/analyze.cpp index aa4fda762..7fe7a4824 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -6367,3 +6367,11 @@ not_integer: } return nullptr; } + +Error file_fetch(CodeGen *g, Buf *resolved_path, Buf *contents) { + if (g->enable_cache) { + return cache_add_file_fetch(&g->cache_hash, resolved_path, contents); + } else { + return os_fetch_file_path(resolved_path, contents, false); + } +} diff --git a/src/analyze.hpp b/src/analyze.hpp index 41cc50916..9f036c409 100644 --- a/src/analyze.hpp +++ b/src/analyze.hpp @@ -210,4 +210,7 @@ ZigType *get_primitive_type(CodeGen *g, Buf *name); bool calling_convention_allows_zig_types(CallingConvention cc); const char *calling_convention_name(CallingConvention cc); + +Error ATTRIBUTE_MUST_USE file_fetch(CodeGen *g, Buf *resolved_path, Buf *contents); + #endif diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index b30294631..65e3e62ae 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -467,3 +467,11 @@ void cache_release(CacheHash *ch) { assert(ch->manifest_file_path != nullptr); os_file_close(ch->manifest_file); } + +Buf *get_random_basename() { + Buf *result = buf_alloc(); + for (size_t i = 0; i < 16; i += 1) { + buf_append_char(result, base64_fs_alphabet[rand() % 64]); + } + return result; +} diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp index ede7344c7..6acd805be 100644 --- a/src/cache_hash.hpp +++ b/src/cache_hash.hpp @@ -67,4 +67,9 @@ Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_b64_digest); // Until this function is called, no one will be able to get a lock on your input params. void cache_release(CacheHash *ch); + + +// Completely independent function. Just returns a random filename safe basename. +Buf *get_random_basename(); + #endif diff --git a/src/codegen.cpp b/src/codegen.cpp index 578fb314a..1e0fd7fec 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7048,7 +7048,7 @@ static ImportTableEntry *add_special_code(CodeGen *g, PackageTableEntry *package *resolved_path = os_path_resolve(resolve_paths, 1); Buf *import_code = buf_alloc(); Error err; - if ((err = cache_add_file_fetch(&g->cache_hash, resolved_path, import_code))) { + if ((err = file_fetch(g, resolved_path, import_code))) { zig_panic("unable to open '%s': %s\n", buf_ptr(&path_to_code_src), err_str(err)); } @@ -7480,10 +7480,7 @@ static void gen_h_file(CodeGen *g) { GenH *gen_h = &gen_h_data; assert(!g->is_test_build); - - if (!g->out_h_path) { - g->out_h_path = buf_sprintf("%s.h", buf_ptr(g->root_out_name)); - } + assert(g->out_h_path != nullptr); FILE *out_h = fopen(buf_ptr(g->out_h_path), "wb"); if (!out_h) @@ -7790,17 +7787,56 @@ static void resolve_out_paths(CodeGen *g) { zig_unreachable(); } - os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path); + if (g->enable_cache || g->out_type != OutTypeObj) { + os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path); + } else { + buf_init_from_buf(&g->o_file_output_path, o_basename); + } if (g->out_type == OutTypeObj) { buf_init_from_buf(&g->output_file_path, &g->o_file_output_path); } else if (g->out_type == OutTypeExe) { - assert(g->root_out_name); + if (!g->enable_cache && g->wanted_output_file_path != nullptr) { + buf_init_from_buf(&g->output_file_path, g->wanted_output_file_path); + } else { + assert(g->root_out_name); - Buf basename = BUF_INIT; - buf_init_from_buf(&basename, g->root_out_name); - buf_append_str(&basename, target_exe_file_ext(&g->zig_target)); - os_path_join(&g->artifact_dir, &basename, &g->output_file_path); + Buf basename = BUF_INIT; + buf_init_from_buf(&basename, g->root_out_name); + buf_append_str(&basename, target_exe_file_ext(&g->zig_target)); + if (g->enable_cache) { + os_path_join(&g->artifact_dir, &basename, &g->output_file_path); + } else { + buf_init_from_buf(&g->output_file_path, &basename); + } + } + } else if (g->out_type == OutTypeLib) { + if (!g->enable_cache && g->wanted_output_file_path != nullptr) { + buf_init_from_buf(&g->output_file_path, g->wanted_output_file_path); + } else { + Buf basename = BUF_INIT; + buf_init_from_buf(&basename, g->root_out_name); + buf_append_str(&basename, target_lib_file_ext(&g->zig_target, g->is_static, + g->version_major, g->version_minor, g->version_patch)); + if (g->enable_cache) { + os_path_join(&g->artifact_dir, &basename, &g->output_file_path); + } else { + buf_init_from_buf(&g->output_file_path, &basename); + } + } + } else { + zig_unreachable(); + } + + if (g->want_h_file && !g->out_h_path) { + assert(g->root_out_name); + Buf *h_basename = buf_sprintf("%s.h", buf_ptr(g->root_out_name)); + if (g->enable_cache) { + g->out_h_path = buf_alloc(); + os_path_join(&g->artifact_dir, h_basename, g->out_h_path); + } else { + g->out_h_path = h_basename; + } } } @@ -7809,23 +7845,26 @@ void codegen_build_and_link(CodeGen *g) { Error err; assert(g->out_type != OutTypeUnknown); - codegen_add_time_event(g, "Check Cache"); - Buf *stage1_dir = get_stage1_cache_path(); - - Buf *manifest_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("build"), manifest_dir); - + Buf *artifact_dir = buf_alloc(); Buf digest = BUF_INIT; - if ((err = check_cache(g, manifest_dir, &digest))) { - fprintf(stderr, "Unable to check cache: %s\n", err_str(err)); - exit(1); + if (g->enable_cache) { + codegen_add_time_event(g, "Check Cache"); + + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("build"), manifest_dir); + + if ((err = check_cache(g, manifest_dir, &digest))) { + fprintf(stderr, "Unable to check cache: %s\n", err_str(err)); + exit(1); + } + + os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + } else { + os_path_join(stage1_dir, buf_create_from_str("tmp"), artifact_dir); } - Buf *artifact_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); - - if (buf_len(&digest) != 0) { + if (g->enable_cache && buf_len(&digest) != 0) { os_path_join(artifact_dir, &digest, &g->artifact_dir); resolve_out_paths(g); } else { @@ -7836,11 +7875,16 @@ void codegen_build_and_link(CodeGen *g) { gen_global_asm(g); gen_root_source(g); - if ((err = cache_final(&g->cache_hash, &digest))) { - fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); - exit(1); + if (g->enable_cache) { + if ((err = cache_final(&g->cache_hash, &digest))) { + fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); + exit(1); + } + os_path_join(artifact_dir, &digest, &g->artifact_dir); + } else { + Buf *tmp_basename = get_random_basename(); + os_path_join(artifact_dir, tmp_basename, &g->artifact_dir); } - os_path_join(artifact_dir, &digest, &g->artifact_dir); if ((err = os_make_path(&g->artifact_dir))) { fprintf(stderr, "Unable to create artifact directory: %s\n", err_str(err)); exit(1); @@ -7857,9 +7901,10 @@ void codegen_build_and_link(CodeGen *g) { codegen_link(g); } } - // TODO hard link output_file_path to wanted_output_file_path - cache_release(&g->cache_hash); + if (g->enable_cache) { + cache_release(&g->cache_hash); + } codegen_add_time_event(g, "Done"); } diff --git a/src/ir.cpp b/src/ir.cpp index 22d9a9bc4..bfe21f974 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -16277,7 +16277,7 @@ static ZigType *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructionImpor return ira->codegen->builtin_types.entry_namespace; } - if ((err = cache_add_file_fetch(&ira->codegen->cache_hash, resolved_path, import_code))) { + if ((err = file_fetch(ira->codegen, resolved_path, import_code))) { if (err == ErrorFileNotFound) { ir_add_error_node(ira, source_node, buf_sprintf("unable to find '%s'", buf_ptr(import_target_path))); @@ -18108,7 +18108,7 @@ static ZigType *ir_analyze_instruction_embed_file(IrAnalyze *ira, IrInstructionE // load from file system into const expr Buf *file_contents = buf_alloc(); int err; - if ((err = cache_add_file_fetch(&ira->codegen->cache_hash, &file_path, file_contents))) { + if ((err = file_fetch(ira->codegen, &file_path, file_contents))) { if (err == ErrorFileNotFound) { ir_add_error(ira, instruction->name, buf_sprintf("unable to find '%s'", buf_ptr(&file_path))); return ira->codegen->builtin_types.entry_invalid; diff --git a/src/link.cpp b/src/link.cpp index 8d7b8b4d5..aa0edde61 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -58,6 +58,7 @@ static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) new_link_lib->provided_explicitly = link_lib->provided_explicitly; } + child_gen->enable_cache = true; codegen_build_and_link(child_gen); return &child_gen->output_file_path; } diff --git a/src/main.cpp b/src/main.cpp index 10b789a6e..e4019f10d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -34,6 +34,7 @@ static int usage(const char *arg0) { "Compile Options:\n" " --assembly [source] add assembly file to build\n" " --cache-dir [path] override the cache directory\n" + " --cache [auto|off|on] build to the global cache and print output path to stdout\n" " --color [auto|off|on] enable or disable colored error messages\n" " --emit [asm|bin|llvm-ir] emit a specific file format as compilation output\n" " --enable-timing-info print timing diagnostics\n" @@ -257,6 +258,24 @@ static void add_package(CodeGen *g, CliPkg *cli_pkg, PackageTableEntry *pkg) { } } +enum CacheOpt { + CacheOptAuto, + CacheOptOn, + CacheOptOff, +}; + +static bool get_cache_opt(CacheOpt opt, bool default_value) { + switch (opt) { + case CacheOptAuto: + return default_value; + case CacheOptOn: + return true; + case CacheOptOff: + return false; + } + zig_unreachable(); +} + int main(int argc, char **argv) { if (argc == 2 && strcmp(argv[1], "BUILD_INFO") == 0) { printf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", @@ -301,6 +320,7 @@ int main(int argc, char **argv) { bool verbose_llvm_ir = false; bool verbose_cimport = false; ErrColor color = ErrColorAuto; + CacheOpt enable_cache = CacheOptAuto; const char *libc_lib_dir = nullptr; const char *libc_static_lib_dir = nullptr; const char *libc_include_dir = nullptr; @@ -465,6 +485,7 @@ int main(int argc, char **argv) { PackageTableEntry *build_pkg = codegen_create_package(g, buf_ptr(&build_file_dirname), buf_ptr(&build_file_basename)); g->root_package->package_table.put(buf_create_from_str("@build"), build_pkg); + g->enable_cache = get_cache_opt(enable_cache, true); codegen_build_and_link(g); Termination term; @@ -562,6 +583,17 @@ int main(int argc, char **argv) { fprintf(stderr, "--color options are 'auto', 'on', or 'off'\n"); return usage(arg0); } + } else if (strcmp(arg, "--cache") == 0) { + if (strcmp(argv[i], "auto") == 0) { + enable_cache = CacheOptAuto; + } else if (strcmp(argv[i], "on") == 0) { + enable_cache = CacheOptOn; + } else if (strcmp(argv[i], "off") == 0) { + enable_cache = CacheOptOff; + } else { + fprintf(stderr, "--cache options are 'auto', 'on', or 'off'\n"); + return usage(arg0); + } } else if (strcmp(arg, "--emit") == 0) { if (strcmp(argv[i], "asm") == 0) { emit_file_type = EmitFileTypeAssembly; @@ -894,6 +926,7 @@ int main(int argc, char **argv) { if (cmd == CmdBuild || cmd == CmdRun) { codegen_set_emit_file_type(g, emit_file_type); + g->enable_cache = get_cache_opt(enable_cache, cmd == CmdRun); codegen_build_and_link(g); if (timing_info) codegen_print_timing_report(g, stdout); @@ -913,9 +946,17 @@ int main(int argc, char **argv) { Termination term; os_spawn_process(exec_path, args, &term); return term.code; + } else if (cmd == CmdBuild) { + if (g->enable_cache) { + printf("%s\n", buf_ptr(&g->output_file_path)); + if (g->out_h_path != nullptr) { + printf("%s\n", buf_ptr(g->out_h_path)); + } + } + return EXIT_SUCCESS; + } else { + zig_unreachable(); } - - return EXIT_SUCCESS; } else if (cmd == CmdTranslateC) { codegen_translate_c(g, in_file_buf); ast_render(g, stdout, g->root_import->root, 4); @@ -928,8 +969,11 @@ int main(int argc, char **argv) { ZigTarget native; get_native_target(&native); + g->enable_cache = get_cache_opt(enable_cache, false); codegen_build_and_link(g); - Buf *test_exe_path = &g->output_file_path; + Buf *test_exe_path_unresolved = &g->output_file_path; + Buf *test_exe_path = buf_alloc(); + *test_exe_path = os_path_resolve(&test_exe_path_unresolved, 1); for (size_t i = 0; i < test_exec_args.length; i += 1) { if (test_exec_args.items[i] == nullptr) { diff --git a/src/target.cpp b/src/target.cpp index 91d36c510..f657af8af 100644 --- a/src/target.cpp +++ b/src/target.cpp @@ -813,6 +813,22 @@ const char *target_exe_file_ext(ZigTarget *target) { } } +const char *target_lib_file_ext(ZigTarget *target, bool is_static, size_t version_major, size_t version_minor, size_t version_patch) { + if (target->os == OsWindows) { + if (is_static) { + return ".lib"; + } else { + return ".dll"; + } + } else { + if (is_static) { + return ".a"; + } else { + return buf_ptr(buf_sprintf(".so.%zu", version_major)); + } + } +} + enum FloatAbi { FloatAbiHard, FloatAbiSoft, diff --git a/src/target.hpp b/src/target.hpp index 5a118f6d8..c8658bf35 100644 --- a/src/target.hpp +++ b/src/target.hpp @@ -113,6 +113,7 @@ const char *target_o_file_ext(ZigTarget *target); const char *target_asm_file_ext(ZigTarget *target); const char *target_llvm_ir_file_ext(ZigTarget *target); const char *target_exe_file_ext(ZigTarget *target); +const char *target_lib_file_ext(ZigTarget *target, bool is_static, size_t version_major, size_t version_minor, size_t version_patch); Buf *target_dynamic_linker(ZigTarget *target); From c4f96ea745ae2aa56ec5cb2c871e5b3d86b04c8f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 11 Sep 2018 11:52:16 -0400 Subject: [PATCH 10/46] disable stage2 tests on all targets See #1364 --- build.zig | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build.zig b/build.zig index 6584e5ab1..d4a159e5e 100644 --- a/build.zig +++ b/build.zig @@ -76,7 +76,11 @@ pub fn build(b: *Builder) !void { const test_stage2_step = b.step("test-stage2", "Run the stage2 compiler tests"); test_stage2_step.dependOn(&test_stage2.step); - test_step.dependOn(test_stage2_step); + + // TODO see https://github.com/ziglang/zig/issues/1364 + if (false) { + test_step.dependOn(test_stage2_step); + } const all_modes = []builtin.Mode{ builtin.Mode.Debug, From dd1338b0e6280b10b9b62ca73bf9ece34bd8524e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 11 Sep 2018 12:57:53 -0400 Subject: [PATCH 11/46] fix incorrect union const value generation closes #1381 The union was generated as a 3 byte struct when it needed to be 4 bytes so that the packed struct bitcast could work correctly. Now it recognizes this situation and adds padding bytes to become the correct size so that it can fit into an array. --- src/codegen.cpp | 15 ++++++++++++--- test/behavior.zig | 1 + test/cases/bugs/1381.zig | 21 +++++++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 test/cases/bugs/1381.zig diff --git a/src/codegen.cpp b/src/codegen.cpp index 71caf18e2..2548b3fa8 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3218,7 +3218,8 @@ static LLVMValueRef ir_render_decl_var(CodeGen *g, IrExecutable *executable, assert(var->value->type == init_value->value.type); ZigType *var_ptr_type = get_pointer_to_type_extra(g, var->value->type, false, false, PtrLenSingle, var->align_bytes, 0, 0); - gen_assign_raw(g, var->value_ref, var_ptr_type, ir_llvm_value(g, init_value)); + LLVMValueRef llvm_init_val = ir_llvm_value(g, init_value); + gen_assign_raw(g, var->value_ref, var_ptr_type, llvm_init_val); } else { bool want_safe = ir_want_runtime_safety(g, &decl_var_instruction->base); if (want_safe) { @@ -5863,12 +5864,20 @@ static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const c LLVMValueRef tag_value = bigint_to_llvm_const(type_entry->data.unionation.tag_type->type_ref, &const_val->data.x_union.tag); - LLVMValueRef fields[2]; + LLVMValueRef fields[3]; fields[type_entry->data.unionation.gen_union_index] = union_value_ref; fields[type_entry->data.unionation.gen_tag_index] = tag_value; if (make_unnamed_struct) { - return LLVMConstStruct(fields, 2, false); + LLVMValueRef result = LLVMConstStruct(fields, 2, false); + size_t expected_sz = LLVMStoreSizeOfType(g->target_data_ref, type_entry->type_ref); + size_t actual_sz = LLVMStoreSizeOfType(g->target_data_ref, LLVMTypeOf(result)); + if (actual_sz < expected_sz) { + unsigned pad_sz = expected_sz - actual_sz; + fields[2] = LLVMGetUndef(LLVMArrayType(LLVMInt8Type(), pad_sz)); + result = LLVMConstStruct(fields, 3, false); + } + return result; } else { return LLVMConstNamedStruct(type_entry->type_ref, fields, 2); } diff --git a/test/behavior.zig b/test/behavior.zig index 25997a2a4..cae8c4b0f 100644 --- a/test/behavior.zig +++ b/test/behavior.zig @@ -10,6 +10,7 @@ comptime { _ = @import("cases/bool.zig"); _ = @import("cases/bugs/1111.zig"); _ = @import("cases/bugs/1277.zig"); + _ = @import("cases/bugs/1381.zig"); _ = @import("cases/bugs/1421.zig"); _ = @import("cases/bugs/394.zig"); _ = @import("cases/bugs/655.zig"); diff --git a/test/cases/bugs/1381.zig b/test/cases/bugs/1381.zig new file mode 100644 index 000000000..2d452da15 --- /dev/null +++ b/test/cases/bugs/1381.zig @@ -0,0 +1,21 @@ +const std = @import("std"); + +const B = union(enum) { + D: u8, + E: u16, +}; + +const A = union(enum) { + B: B, + C: u8, +}; + +test "union that needs padding bytes inside an array" { + var as = []A{ + A{ .B = B{ .D = 1 } }, + A{ .B = B{ .D = 1 } }, + }; + + const a = as[0].B; + std.debug.assertOrPanic(a.D == 1); +} From 7dd3c3814de0caf808bc112aa07044cdd8bba135 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 11 Sep 2018 15:16:50 -0400 Subject: [PATCH 12/46] fix incorrect error union const value generation closes #1442 zig needed to insert explicit padding into this structure before it got bitcasted. --- src/analyze.cpp | 15 +++++++++++++-- src/codegen.cpp | 38 +++++++++++++++++++++++++++++--------- std/os/index.zig | 36 +++++++++++++++++++----------------- test/behavior.zig | 1 + test/cases/bugs/1442.zig | 11 +++++++++++ 5 files changed, 73 insertions(+), 28 deletions(-) create mode 100644 test/cases/bugs/1442.zig diff --git a/src/analyze.cpp b/src/analyze.cpp index e0539df10..07c7b94e2 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -5882,12 +5882,23 @@ void render_const_value(CodeGen *g, Buf *buf, ConstExprValue *const_val) { } case ZigTypeIdErrorUnion: { - buf_appendf(buf, "(error union %s constant)", buf_ptr(&type_entry->name)); + buf_appendf(buf, "%s(", buf_ptr(&type_entry->name)); + if (const_val->data.x_err_union.err == nullptr) { + render_const_value(g, buf, const_val->data.x_err_union.payload); + } else { + buf_appendf(buf, "%s.%s", buf_ptr(&type_entry->data.error_union.err_set_type->name), + buf_ptr(&const_val->data.x_err_union.err->name)); + } + buf_appendf(buf, ")"); return; } case ZigTypeIdUnion: { - buf_appendf(buf, "(union %s constant)", buf_ptr(&type_entry->name)); + uint64_t tag = bigint_as_unsigned(&const_val->data.x_union.tag); + TypeUnionField *field = &type_entry->data.unionation.fields[tag]; + buf_appendf(buf, "%s { .%s = ", buf_ptr(&type_entry->name), buf_ptr(field->name)); + render_const_value(g, buf, const_val->data.x_union.payload); + buf_append_str(buf, "}"); return; } case ZigTypeIdErrorSet: diff --git a/src/codegen.cpp b/src/codegen.cpp index 2548b3fa8..5b6c53e8f 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5870,13 +5870,17 @@ static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const c if (make_unnamed_struct) { LLVMValueRef result = LLVMConstStruct(fields, 2, false); - size_t expected_sz = LLVMStoreSizeOfType(g->target_data_ref, type_entry->type_ref); - size_t actual_sz = LLVMStoreSizeOfType(g->target_data_ref, LLVMTypeOf(result)); - if (actual_sz < expected_sz) { - unsigned pad_sz = expected_sz - actual_sz; + uint64_t last_field_offset = LLVMOffsetOfElement(g->target_data_ref, LLVMTypeOf(result), 1); + uint64_t end_offset = last_field_offset + + LLVMStoreSizeOfType(g->target_data_ref, LLVMTypeOf(fields[1])); + uint64_t expected_sz = LLVMStoreSizeOfType(g->target_data_ref, type_entry->type_ref); + unsigned pad_sz = expected_sz - end_offset; + if (pad_sz != 0) { fields[2] = LLVMGetUndef(LLVMArrayType(LLVMInt8Type(), pad_sz)); result = LLVMConstStruct(fields, 3, false); } + uint64_t actual_sz = LLVMStoreSizeOfType(g->target_data_ref, LLVMTypeOf(result)); + assert(actual_sz == expected_sz); return result; } else { return LLVMConstNamedStruct(type_entry->type_ref, fields, 2); @@ -5917,13 +5921,29 @@ static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const c err_payload_value = gen_const_val(g, payload_val, ""); make_unnamed_struct = is_llvm_value_unnamed_type(payload_val->type, err_payload_value); } - LLVMValueRef fields[] = { - err_tag_value, - err_payload_value, - }; if (make_unnamed_struct) { - return LLVMConstStruct(fields, 2, false); + uint64_t payload_off = LLVMOffsetOfElement(g->target_data_ref, type_entry->type_ref, 1); + uint64_t err_sz = LLVMStoreSizeOfType(g->target_data_ref, LLVMTypeOf(err_tag_value)); + unsigned pad_sz = payload_off - err_sz; + if (pad_sz == 0) { + LLVMValueRef fields[] = { + err_tag_value, + err_payload_value, + }; + return LLVMConstStruct(fields, 2, false); + } else { + LLVMValueRef fields[] = { + err_tag_value, + LLVMGetUndef(LLVMArrayType(LLVMInt8Type(), pad_sz)), + err_payload_value, + }; + return LLVMConstStruct(fields, 3, false); + } } else { + LLVMValueRef fields[] = { + err_tag_value, + err_payload_value, + }; return LLVMConstNamedStruct(type_entry->type_ref, fields, 2); } } diff --git a/std/os/index.zig b/std/os/index.zig index 3ab73a79a..8e92fc23c 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -343,23 +343,25 @@ pub fn posixWrite(fd: i32, bytes: []const u8) !void { const amt_to_write = math.min(bytes.len - index, usize(max_bytes_len)); const rc = posix.write(fd, bytes.ptr + index, amt_to_write); const write_err = posix.getErrno(rc); - if (write_err > 0) { - return switch (write_err) { - posix.EINTR => continue, - posix.EINVAL, posix.EFAULT => unreachable, - posix.EAGAIN => PosixWriteError.WouldBlock, - posix.EBADF => PosixWriteError.FileClosed, - posix.EDESTADDRREQ => PosixWriteError.DestinationAddressRequired, - posix.EDQUOT => PosixWriteError.DiskQuota, - posix.EFBIG => PosixWriteError.FileTooBig, - posix.EIO => PosixWriteError.InputOutput, - posix.ENOSPC => PosixWriteError.NoSpaceLeft, - posix.EPERM => PosixWriteError.AccessDenied, - posix.EPIPE => PosixWriteError.BrokenPipe, - else => unexpectedErrorPosix(write_err), - }; + switch (write_err) { + 0 => { + index += rc; + continue; + }, + posix.EINTR => continue, + posix.EINVAL => unreachable, + posix.EFAULT => unreachable, + posix.EAGAIN => return PosixWriteError.WouldBlock, + posix.EBADF => return PosixWriteError.FileClosed, + posix.EDESTADDRREQ => return PosixWriteError.DestinationAddressRequired, + posix.EDQUOT => return PosixWriteError.DiskQuota, + posix.EFBIG => return PosixWriteError.FileTooBig, + posix.EIO => return PosixWriteError.InputOutput, + posix.ENOSPC => return PosixWriteError.NoSpaceLeft, + posix.EPERM => return PosixWriteError.AccessDenied, + posix.EPIPE => return PosixWriteError.BrokenPipe, + else => return unexpectedErrorPosix(write_err), } - index += rc; } } @@ -1614,7 +1616,7 @@ pub const Dir = struct { return null; } const name_utf16le = mem.toSlice(u16, self.handle.find_file_data.cFileName[0..].ptr); - if (mem.eql(u16, name_utf16le, []u16{'.'}) or mem.eql(u16, name_utf16le, []u16{'.', '.'})) + if (mem.eql(u16, name_utf16le, []u16{'.'}) or mem.eql(u16, name_utf16le, []u16{ '.', '.' })) continue; // Trust that Windows gives us valid UTF-16LE const name_utf8_len = std.unicode.utf16leToUtf8(self.handle.name_data[0..], name_utf16le) catch unreachable; diff --git a/test/behavior.zig b/test/behavior.zig index cae8c4b0f..869024f29 100644 --- a/test/behavior.zig +++ b/test/behavior.zig @@ -12,6 +12,7 @@ comptime { _ = @import("cases/bugs/1277.zig"); _ = @import("cases/bugs/1381.zig"); _ = @import("cases/bugs/1421.zig"); + _ = @import("cases/bugs/1442.zig"); _ = @import("cases/bugs/394.zig"); _ = @import("cases/bugs/655.zig"); _ = @import("cases/bugs/656.zig"); diff --git a/test/cases/bugs/1442.zig b/test/cases/bugs/1442.zig new file mode 100644 index 000000000..d7057d9ed --- /dev/null +++ b/test/cases/bugs/1442.zig @@ -0,0 +1,11 @@ +const std = @import("std"); + +const Union = union(enum) { + Text: []const u8, + Color: u32, +}; + +test "const error union field alignment" { + var union_or_err: error!Union = Union{ .Color = 1234 }; + std.debug.assertOrPanic((union_or_err catch unreachable).Color == 1234); +} From 15c67d2d50aae11dd425ad2629ebc9770fb95f30 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 11 Sep 2018 16:52:50 -0400 Subject: [PATCH 13/46] fix docgen tests --- doc/docgen.zig | 7 +++++++ src/codegen.cpp | 2 ++ 2 files changed, 9 insertions(+) diff --git a/doc/docgen.zig b/doc/docgen.zig index c1158dc03..5d77a1ea1 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -11,6 +11,7 @@ const max_doc_file_size = 10 * 1024 * 1024; const exe_ext = std.build.Target(std.build.Target.Native).exeFileExt(); const obj_ext = std.build.Target(std.build.Target.Native).oFileExt(); const tmp_dir_name = "docgen_tmp"; +const test_out_path = tmp_dir_name ++ os.path.sep_str ++ "test" ++ exe_ext; pub fn main() !void { var direct_allocator = std.heap.DirectAllocator.init(); @@ -821,6 +822,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var zig_exe, "test", tmp_source_file_name, + "--output", + test_out_path, }); try out.print("
$ zig test {}.zig", code.name);
                         switch (code.mode) {
@@ -863,6 +866,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
                             "--color",
                             "on",
                             tmp_source_file_name,
+                            "--output",
+                            test_out_path,
                         });
                         try out.print("
$ zig test {}.zig", code.name);
                         switch (code.mode) {
@@ -918,6 +923,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
                             zig_exe,
                             "test",
                             tmp_source_file_name,
+                            "--output",
+                            test_out_path,
                         });
                         switch (code.mode) {
                             builtin.Mode.Debug => {},
diff --git a/src/codegen.cpp b/src/codegen.cpp
index d1239eb23..af42f324b 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -8122,6 +8122,8 @@ static void resolve_out_paths(CodeGen *g) {
 
     if (g->enable_cache || g->out_type != OutTypeObj) {
         os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path);
+    } else if (g->wanted_output_file_path != nullptr && g->out_type == OutTypeObj) {
+        buf_init_from_buf(&g->o_file_output_path, g->wanted_output_file_path);
     } else {
         buf_init_from_buf(&g->o_file_output_path, o_basename);
     }

From 9227315bf24238c05c0c01a9f516449aa8525e41 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 17:23:36 -0400
Subject: [PATCH 14/46] zig build: better placement of test exe artifact

---
 std/build.zig | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/std/build.zig b/std/build.zig
index 4e323eaf7..0ef6109bc 100644
--- a/std/build.zig
+++ b/std/build.zig
@@ -1641,6 +1641,7 @@ pub const TestStep = struct {
     lib_paths: ArrayList([]const u8),
     object_files: ArrayList([]const u8),
     no_rosegment: bool,
+    output_path: ?[]const u8,
 
     pub fn init(builder: *Builder, root_src: []const u8) TestStep {
         const step_name = builder.fmt("test {}", root_src);
@@ -1659,6 +1660,7 @@ pub const TestStep = struct {
             .lib_paths = ArrayList([]const u8).init(builder.allocator),
             .object_files = ArrayList([]const u8).init(builder.allocator),
             .no_rosegment = false,
+            .output_path = null,
         };
     }
 
@@ -1682,6 +1684,24 @@ pub const TestStep = struct {
         self.build_mode = mode;
     }
 
+    pub fn setOutputPath(self: *TestStep, file_path: []const u8) void {
+        self.output_path = file_path;
+
+        // catch a common mistake
+        if (mem.eql(u8, self.builder.pathFromRoot(file_path), self.builder.pathFromRoot("."))) {
+            debug.panic("setOutputPath wants a file path, not a directory\n");
+        }
+    }
+
+    pub fn getOutputPath(self: *TestStep) []const u8 {
+        if (self.output_path) |output_path| {
+            return output_path;
+        } else {
+            const basename = self.builder.fmt("test{}", self.target.exeFileExt());
+            return os.path.join(self.builder.allocator, self.builder.cache_root, basename) catch unreachable;
+        }
+    }
+
     pub fn linkSystemLibrary(self: *TestStep, name: []const u8) void {
         self.link_libs.put(name) catch unreachable;
     }
@@ -1746,6 +1766,10 @@ pub const TestStep = struct {
             builtin.Mode.ReleaseSmall => try zig_args.append("--release-small"),
         }
 
+        const output_path = builder.pathFromRoot(self.getOutputPath());
+        try zig_args.append("--output");
+        try zig_args.append(output_path);
+
         switch (self.target) {
             Target.Native => {},
             Target.Cross => |cross_target| {

From a1132ffe0ff965ab48c160a658f11bae345bf2d2 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 17:29:18 -0400
Subject: [PATCH 15/46] stage1: build blake code with -std=c99

---
 CMakeLists.txt                 | 13 +++++++------
 src/{blake2b.cpp => blake2b.c} |  0
 2 files changed, 7 insertions(+), 6 deletions(-)
 rename src/{blake2b.cpp => blake2b.c} (100%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 11bb31892..ee9f9c281 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -426,8 +426,8 @@ set(ZIG_SOURCES
     "${CMAKE_SOURCE_DIR}/src/util.cpp"
     "${CMAKE_SOURCE_DIR}/src/translate_c.cpp"
 )
-set(ZIG_SOURCES_O3
-    "${CMAKE_SOURCE_DIR}/src/blake2b.cpp"
+set(BLAKE_SOURCES
+    "${CMAKE_SOURCE_DIR}/src/blake2b.c"
 )
 set(ZIG_CPP_SOURCES
     "${CMAKE_SOURCE_DIR}/src/zig_llvm.cpp"
@@ -795,6 +795,7 @@ else()
   set(EXE_CFLAGS "${EXE_CFLAGS} -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D_GNU_SOURCE -fno-exceptions -fno-rtti  -Werror=strict-prototypes -Werror=old-style-definition -Werror=type-limits -Wno-missing-braces")
 endif()
 
+set(BLAKE_CFLAGS "-std=c99")
 
 set(EXE_LDFLAGS " ")
 if(MINGW)
@@ -816,9 +817,9 @@ set_target_properties(zig_cpp PROPERTIES
     COMPILE_FLAGS ${EXE_CFLAGS}
 )
 
-add_library(zig_O3 STATIC ${ZIG_SOURCES_O3})
-set_target_properties(zig_O3 PROPERTIES
-    COMPILE_FLAGS "${EXE_CFLAGS} -O3"
+add_library(embedded_blake STATIC ${BLAKE_SOURCES})
+set_target_properties(embedded_blake PROPERTIES
+    COMPILE_FLAGS "${BLAKE_CFLAGS} -O3"
 )
 
 add_executable(zig ${ZIG_SOURCES})
@@ -829,7 +830,7 @@ set_target_properties(zig PROPERTIES
 
 target_link_libraries(zig LINK_PUBLIC
     zig_cpp
-    zig_O3
+    embedded_blake
     ${SOFTFLOAT_LIBRARIES}
     ${CLANG_LIBRARIES}
     ${LLD_LIBRARIES}
diff --git a/src/blake2b.cpp b/src/blake2b.c
similarity index 100%
rename from src/blake2b.cpp
rename to src/blake2b.c

From 1a4dcf10fecf3fed7db0573fc859718e05e93f11 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 17:42:03 -0400
Subject: [PATCH 16/46] darwin fixups

---
 src/os.cpp | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/os.cpp b/src/os.cpp
index f3569ca3b..9970c04c7 100644
--- a/src/os.cpp
+++ b/src/os.cpp
@@ -1657,6 +1657,14 @@ Error os_self_exe_shared_libs(ZigList &paths) {
     paths.resize(0);
     dl_iterate_phdr(self_exe_shared_libs_callback, &paths);
     return ErrorNone;
+#elif defined(ZIG_OS_DARWIN)
+    paths.resize(0);
+    uint32_t img_count = _dyld_image_count();
+    for (uint32_t i = 0; i != img_count; i += 1) {
+        const char *name = _dyld_get_image_name(i);
+        paths.append(buf_create_from_str(name));
+    }
+    return ErrorNone;
 #else
 #error "unimplemented"
 #endif
@@ -1750,7 +1758,7 @@ Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) {
 Error os_file_mtime(OsFile file, OsTimeStamp *mtime) {
 #if defined(ZIG_OS_WINDOWS)
 #error unimplemented
-#else
+#elif defined(ZIG_OS_LINUX)
     struct stat statbuf;
     if (fstat(file, &statbuf) == -1)
         return ErrorFileSystem;
@@ -1758,6 +1766,16 @@ Error os_file_mtime(OsFile file, OsTimeStamp *mtime) {
     mtime->sec = statbuf.st_mtim.tv_sec;
     mtime->nsec = statbuf.st_mtim.tv_nsec;
     return ErrorNone;
+#elif defined(ZIG_OS_DARWIN)
+    struct stat statbuf;
+    if (fstat(file, &statbuf) == -1)
+        return ErrorFileSystem;
+
+    mtime->sec = statbuf.st_mtimespec.tv_sec;
+    mtime->nsec = statbuf.st_mtimespec.tv_nsec;
+    return ErrorNone;
+#else
+#error unimplemented
 #endif
 }
 

From 04dc5cdacaa953ca5365245f32158db70213b90e Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 18:15:08 -0400
Subject: [PATCH 17/46] zig build: make the cache root dir before building

---
 std/build.zig | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/std/build.zig b/std/build.zig
index 0ef6109bc..5800e69c6 100644
--- a/std/build.zig
+++ b/std/build.zig
@@ -232,6 +232,8 @@ pub const Builder = struct {
     }
 
     pub fn make(self: *Builder, step_names: []const []const u8) !void {
+        try self.makePath(self.cache_root);
+
         var wanted_steps = ArrayList(*Step).init(self.allocator);
         defer wanted_steps.deinit();
 

From 6b7f3d01ae3ea2c6535b72339f1023ab426c40fd Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 20:53:28 -0400
Subject: [PATCH 18/46] ci: build zig in release mode

It makes sense to test release mode, plus we're up against the
time limits of CI, so this should make room.
---
 ci/travis_linux_script | 2 +-
 ci/travis_osx_script   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/travis_linux_script b/ci/travis_linux_script
index f2f4bd95d..f823429a4 100755
--- a/ci/travis_linux_script
+++ b/ci/travis_linux_script
@@ -8,7 +8,7 @@ export CXX=clang++-6.0
 echo $PATH
 mkdir build
 cd build
-cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd)
+cmake .. -DCMAKE_BUILD_TYPE=Release
 make -j2 install
 ./zig build --build-file ../build.zig test
 
diff --git a/ci/travis_osx_script b/ci/travis_osx_script
index 56cf45cd5..7fa54d26f 100755
--- a/ci/travis_osx_script
+++ b/ci/travis_osx_script
@@ -5,7 +5,7 @@ set -e
 
 mkdir build
 cd build
-cmake .. -DCMAKE_PREFIX_PATH=/usr/local/opt/llvm@6/ -DCMAKE_INSTALL_PREFIX=$(pwd)
+cmake .. -DCMAKE_PREFIX_PATH=/usr/local/opt/llvm@6/ -DCMAKE_BUILD_TYPE=Release
 make VERBOSE=1
 make install
 

From 7e9f25dd18b7da750e547de94a779e3c14c07d94 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 20:54:39 -0400
Subject: [PATCH 19/46] stage1: clean up timing report in test mode

---
 src/codegen.cpp | 7 +++++--
 src/main.cpp    | 8 +++++---
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/codegen.cpp b/src/codegen.cpp
index af42f324b..96e979f9d 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -6388,9 +6388,9 @@ static void do_code_gen(CodeGen *g) {
     char *error = nullptr;
     LLVMVerifyModule(g->module, LLVMAbortProcessAction, &error);
 #endif
+}
 
-    codegen_add_time_event(g, "LLVM Emit Output");
-
+static void zig_llvm_emit_output(CodeGen *g) {
     bool is_small = g->build_mode == BuildModeSmallRelease;
 
     Buf *output_path = &g->o_file_output_path;
@@ -8228,6 +8228,9 @@ void codegen_build_and_link(CodeGen *g) {
 
         codegen_add_time_event(g, "Code Generation");
         do_code_gen(g);
+        codegen_add_time_event(g, "LLVM Emit Output");
+        zig_llvm_emit_output(g);
+
         if (g->want_h_file) {
             codegen_add_time_event(g, "Generate .h");
             gen_h_file(g);
diff --git a/src/main.cpp b/src/main.cpp
index e4019f10d..1751fabfe 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -971,6 +971,11 @@ int main(int argc, char **argv) {
 
                 g->enable_cache = get_cache_opt(enable_cache, false);
                 codegen_build_and_link(g);
+
+                if (timing_info) {
+                    codegen_print_timing_report(g, stdout);
+                }
+
                 Buf *test_exe_path_unresolved = &g->output_file_path;
                 Buf *test_exe_path = buf_alloc();
                 *test_exe_path = os_path_resolve(&test_exe_path_unresolved, 1);
@@ -981,7 +986,6 @@ int main(int argc, char **argv) {
                     }
                 }
 
-
                 if (!target_can_exec(&native, target)) {
                     fprintf(stderr, "Created %s but skipping execution because it is non-native.\n",
                             buf_ptr(test_exe_path));
@@ -1003,8 +1007,6 @@ int main(int argc, char **argv) {
                 if (term.how != TerminationIdClean || term.code != 0) {
                     fprintf(stderr, "\nTests failed. Use the following command to reproduce the failure:\n");
                     fprintf(stderr, "%s\n", buf_ptr(test_exe_path));
-                } else if (timing_info) {
-                    codegen_print_timing_report(g, stdout);
                 }
                 return (term.how == TerminationIdClean) ? term.code : -1;
             } else {

From ee263a15cc8fb52c2ac6053a29168fb15089839b Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 22:25:52 -0400
Subject: [PATCH 20/46] bring back zig-cache

we need somewhere to put .o files and leave them while the user
executes their program, so that stack traces on MacOS can find
the .o files and get at the DWARF info.

if we try to clean up old global tmp dir files, first of all that's
a hard and complicated problem, and secondly it's not clear how
that is better than dumping the .o file inside zig-cache locally.
---
 src/all_types.hpp  |  1 +
 src/cache_hash.cpp |  8 --------
 src/cache_hash.hpp |  4 ----
 src/codegen.cpp    |  5 +----
 src/main.cpp       | 13 +++++--------
 5 files changed, 7 insertions(+), 24 deletions(-)

diff --git a/src/all_types.hpp b/src/all_types.hpp
index f9c50eeaa..3dfbc6ba7 100644
--- a/src/all_types.hpp
+++ b/src/all_types.hpp
@@ -1682,6 +1682,7 @@ struct CodeGen {
     Buf output_file_path;
     Buf o_file_output_path;
     Buf *wanted_output_file_path;
+    Buf cache_dir;
 
     IrInstruction *invalid_instruction;
 
diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp
index 65e3e62ae..b30294631 100644
--- a/src/cache_hash.cpp
+++ b/src/cache_hash.cpp
@@ -467,11 +467,3 @@ void cache_release(CacheHash *ch) {
     assert(ch->manifest_file_path != nullptr);
     os_file_close(ch->manifest_file);
 }
-
-Buf *get_random_basename() {
-    Buf *result = buf_alloc();
-    for (size_t i = 0; i < 16; i += 1) {
-        buf_append_char(result, base64_fs_alphabet[rand() % 64]);
-    }
-    return result;
-}
diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp
index 6acd805be..db1c42ec0 100644
--- a/src/cache_hash.hpp
+++ b/src/cache_hash.hpp
@@ -68,8 +68,4 @@ Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_b64_digest);
 void cache_release(CacheHash *ch);
 
 
-
-// Completely independent function. Just returns a random filename safe basename.
-Buf *get_random_basename();
-
 #endif
diff --git a/src/codegen.cpp b/src/codegen.cpp
index 96e979f9d..aeb2b6edc 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -8195,8 +8195,6 @@ void codegen_build_and_link(CodeGen *g) {
         }
 
         os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir);
-    } else {
-        os_path_join(stage1_dir, buf_create_from_str("tmp"), artifact_dir);
     }
 
     if (g->enable_cache && buf_len(&digest) != 0) {
@@ -8217,8 +8215,7 @@ void codegen_build_and_link(CodeGen *g) {
             }
             os_path_join(artifact_dir, &digest, &g->artifact_dir);
         } else {
-            Buf *tmp_basename = get_random_basename(); 
-            os_path_join(artifact_dir, tmp_basename, &g->artifact_dir);
+            buf_init_from_buf(&g->artifact_dir, &g->cache_dir);
         }
         if ((err = os_make_path(&g->artifact_dir))) {
             fprintf(stderr, "Unable to create artifact directory: %s\n", err_str(err));
diff --git a/src/main.cpp b/src/main.cpp
index 1751fabfe..f18dd949b 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -353,7 +353,7 @@ int main(int argc, char **argv) {
     size_t ver_minor = 0;
     size_t ver_patch = 0;
     bool timing_info = false;
-    const char *cache_dir = nullptr;
+    const char *cache_dir = default_zig_cache_name;
     CliPkg *cur_pkg = allocate(1);
     BuildMode build_mode = BuildModeDebug;
     ZigList test_exec_args = {0};
@@ -402,6 +402,7 @@ int main(int argc, char **argv) {
         os_path_join(special_dir, buf_create_from_str("build_runner.zig"), build_runner_path);
 
         CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf);
+        buf_init_from_str(&g->cache_dir, cache_dir);
         codegen_set_out_name(g, buf_create_from_str("build"));
 
         Buf *build_file_buf = buf_create_from_str(build_file);
@@ -410,13 +411,8 @@ int main(int argc, char **argv) {
         Buf build_file_dirname = BUF_INIT;
         os_path_split(&build_file_abs, &build_file_dirname, &build_file_basename);
 
-        Buf full_cache_dir = BUF_INIT;
-        if (cache_dir == nullptr) {
-            os_path_join(&build_file_dirname, buf_create_from_str(default_zig_cache_name), &full_cache_dir);
-        } else {
-            Buf *cache_dir_buf = buf_create_from_str(cache_dir);
-            full_cache_dir = os_path_resolve(&cache_dir_buf, 1);
-        }
+        Buf *cache_dir_buf = buf_create_from_str(cache_dir);
+        Buf full_cache_dir = os_path_resolve(&cache_dir_buf, 1);
 
         args.items[1] = buf_ptr(&build_file_dirname);
         args.items[2] = buf_ptr(&full_cache_dir);
@@ -832,6 +828,7 @@ int main(int argc, char **argv) {
             Buf *zig_lib_dir_buf = resolve_zig_lib_dir();
 
             CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf);
+            buf_init_from_str(&g->cache_dir, cache_dir);
             codegen_set_out_name(g, buf_out_name);
             codegen_set_lib_version(g, ver_major, ver_minor, ver_patch);
             codegen_set_is_test(g, cmd == CmdTest);

From 014cc60a72ac2d64935bf424459b5fa13e281ed9 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 22:46:22 -0400
Subject: [PATCH 21/46] rename --enable-timing-info to -ftime-report to match
 clang

and have it print llvm's internal timing info
---
 src/all_types.hpp | 1 +
 src/codegen.cpp   | 9 ++++++---
 src/main.cpp      | 6 ++++--
 src/zig_llvm.cpp  | 9 ++++++++-
 src/zig_llvm.h    | 3 ++-
 5 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/src/all_types.hpp b/src/all_types.hpp
index 3dfbc6ba7..8aa2718d3 100644
--- a/src/all_types.hpp
+++ b/src/all_types.hpp
@@ -1727,6 +1727,7 @@ struct CodeGen {
     bool error_during_imports;
     bool generate_error_name_table;
     bool enable_cache;
+    bool enable_time_report;
 
     //////////////////////////// Participates in Input Parameter Cache Hash
     ZigList link_libs_list;
diff --git a/src/codegen.cpp b/src/codegen.cpp
index aeb2b6edc..6f21ceeca 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -6398,7 +6398,8 @@ static void zig_llvm_emit_output(CodeGen *g) {
     switch (g->emit_file_type) {
         case EmitFileTypeBinary:
             if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path),
-                        ZigLLVM_EmitBinary, &err_msg, g->build_mode == BuildModeDebug, is_small))
+                        ZigLLVM_EmitBinary, &err_msg, g->build_mode == BuildModeDebug, is_small,
+                        g->enable_time_report))
             {
                 zig_panic("unable to write object file %s: %s", buf_ptr(output_path), err_msg);
             }
@@ -6408,7 +6409,8 @@ static void zig_llvm_emit_output(CodeGen *g) {
 
         case EmitFileTypeAssembly:
             if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path),
-                        ZigLLVM_EmitAssembly, &err_msg, g->build_mode == BuildModeDebug, is_small))
+                        ZigLLVM_EmitAssembly, &err_msg, g->build_mode == BuildModeDebug, is_small,
+                        g->enable_time_report))
             {
                 zig_panic("unable to write assembly file %s: %s", buf_ptr(output_path), err_msg);
             }
@@ -6417,7 +6419,8 @@ static void zig_llvm_emit_output(CodeGen *g) {
 
         case EmitFileTypeLLVMIr:
             if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path),
-                        ZigLLVM_EmitLLVMIr, &err_msg, g->build_mode == BuildModeDebug, is_small))
+                        ZigLLVM_EmitLLVMIr, &err_msg, g->build_mode == BuildModeDebug, is_small,
+                        g->enable_time_report))
             {
                 zig_panic("unable to write llvm-ir file %s: %s", buf_ptr(output_path), err_msg);
             }
diff --git a/src/main.cpp b/src/main.cpp
index f18dd949b..fdc34a83a 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -37,7 +37,7 @@ static int usage(const char *arg0) {
         "  --cache [auto|off|on]        build to the global cache and print output path to stdout\n"
         "  --color [auto|off|on]        enable or disable colored error messages\n"
         "  --emit [asm|bin|llvm-ir]     emit a specific file format as compilation output\n"
-        "  --enable-timing-info         print timing diagnostics\n"
+        "  -ftime-report                print timing diagnostics\n"
         "  --libc-include-dir [path]    directory where libc stdlib.h resides\n"
         "  --name [name]                override output name\n"
         "  --output [file]              override destination path\n"
@@ -402,6 +402,7 @@ int main(int argc, char **argv) {
         os_path_join(special_dir, buf_create_from_str("build_runner.zig"), build_runner_path);
 
         CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf);
+        g->enable_time_report = timing_info;
         buf_init_from_str(&g->cache_dir, cache_dir);
         codegen_set_out_name(g, buf_create_from_str("build"));
 
@@ -533,7 +534,7 @@ int main(int argc, char **argv) {
                 no_rosegment_workaround = true;
             } else if (strcmp(arg, "--each-lib-rpath") == 0) {
                 each_lib_rpath = true;
-            } else if (strcmp(arg, "--enable-timing-info") == 0) {
+            } else if (strcmp(arg, "-ftime-report") == 0) {
                 timing_info = true;
             } else if (strcmp(arg, "--test-cmd-bin") == 0) {
                 test_exec_args.append(nullptr);
@@ -828,6 +829,7 @@ int main(int argc, char **argv) {
             Buf *zig_lib_dir_buf = resolve_zig_lib_dir();
 
             CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf);
+            g->enable_time_report = timing_info;
             buf_init_from_str(&g->cache_dir, cache_dir);
             codegen_set_out_name(g, buf_out_name);
             codegen_set_lib_version(g, ver_major, ver_minor, ver_patch);
diff --git a/src/zig_llvm.cpp b/src/zig_llvm.cpp
index a43d2d182..61287f620 100644
--- a/src/zig_llvm.cpp
+++ b/src/zig_llvm.cpp
@@ -30,6 +30,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -81,8 +82,11 @@ static const bool assertions_on = false;
 #endif
 
 bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMModuleRef module_ref,
-        const char *filename, ZigLLVM_EmitOutputType output_type, char **error_message, bool is_debug, bool is_small)
+        const char *filename, ZigLLVM_EmitOutputType output_type, char **error_message, bool is_debug,
+        bool is_small, bool time_report)
 {
+    TimePassesIsEnabled = time_report;
+
     std::error_code EC;
     raw_fd_ostream dest(filename, EC, sys::fs::F_None);
     if (EC) {
@@ -182,6 +186,9 @@ bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMM
         }
     }
 
+    if (time_report) {
+        TimerGroup::printAll(errs());
+    }
     return false;
 }
 
diff --git a/src/zig_llvm.h b/src/zig_llvm.h
index 63d69bd23..5cdc6cc04 100644
--- a/src/zig_llvm.h
+++ b/src/zig_llvm.h
@@ -55,7 +55,8 @@ enum ZigLLVM_EmitOutputType {
 };
 
 ZIG_EXTERN_C bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMModuleRef module_ref,
-        const char *filename, enum ZigLLVM_EmitOutputType output_type, char **error_message, bool is_debug, bool is_small);
+        const char *filename, enum ZigLLVM_EmitOutputType output_type, char **error_message, bool is_debug,
+        bool is_small, bool time_report);
 
 ZIG_EXTERN_C LLVMTypeRef ZigLLVMTokenTypeInContext(LLVMContextRef context_ref);
 

From ff0b7fe29a49a5619b3c2f7fee0baccb979caf2d Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 22:57:39 -0400
Subject: [PATCH 22/46] error messages for attempted cache when zig cannot
 perfectly do it

---
 src/analyze.cpp | 6 ++++++
 src/ir.cpp      | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/src/analyze.cpp b/src/analyze.cpp
index bae4b15f8..59be76518 100644
--- a/src/analyze.cpp
+++ b/src/analyze.cpp
@@ -6289,6 +6289,12 @@ LinkLib *add_link_lib(CodeGen *g, Buf *name) {
     if (is_libc && g->libc_link_lib != nullptr)
         return g->libc_link_lib;
 
+    if (g->enable_cache && is_libc && g->zig_target.os != OsMacOSX && g->zig_target.os != OsIOS) {
+        fprintf(stderr, "TODO linking against libc is currently incompatible with `--cache on`.\n"
+        "Zig is not yet capable of determining whether the libc installation has changed on subsequent builds.\n");
+        exit(1);
+    }
+
     for (size_t i = 0; i < g->link_libs_list.length; i += 1) {
         LinkLib *existing_lib = g->link_libs_list.at(i);
         if (buf_eql_buf(existing_lib->name, name)) {
diff --git a/src/ir.cpp b/src/ir.cpp
index 858a6df33..030f6627a 100644
--- a/src/ir.cpp
+++ b/src/ir.cpp
@@ -17961,6 +17961,12 @@ static ZigType *ir_analyze_instruction_type_name(IrAnalyze *ira, IrInstructionTy
 }
 
 static ZigType *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstructionCImport *instruction) {
+    if (ira->codegen->enable_cache) {
+        ir_add_error(ira, &instruction->base,
+            buf_sprintf("TODO @cImport is incompatible with --cache on. The cache system currently is unable to detect subsequent changes in .h files."));
+        return ira->codegen->builtin_types.entry_invalid;
+    }
+
     AstNode *node = instruction->base.source_node;
     assert(node->type == NodeTypeFnCallExpr);
     AstNode *block_node = node->data.fn_call_expr.params.at(0);

From 54f7d58722e16e47e3781f7a6bd7c80ee4e22520 Mon Sep 17 00:00:00 2001
From: emekoi 
Date: Wed, 12 Sep 2018 07:01:48 -0500
Subject: [PATCH 23/46] fixed WriteFile segfault

---
 std/os/windows/util.zig | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/std/os/windows/util.zig b/std/os/windows/util.zig
index a489cca16..2a1a82a96 100644
--- a/std/os/windows/util.zig
+++ b/std/os/windows/util.zig
@@ -52,7 +52,8 @@ pub const WriteError = error{
 };
 
 pub fn windowsWrite(handle: windows.HANDLE, bytes: []const u8) WriteError!void {
-    if (windows.WriteFile(handle, bytes.ptr, @intCast(u32, bytes.len), null, null) == 0) {
+    var bytes_written: windows.DWORD = undefined;
+    if (windows.WriteFile(handle, bytes.ptr, @intCast(u32, bytes.len), &bytes_written, null) == 0) {
         const err = windows.GetLastError();
         return switch (err) {
             windows.ERROR.INVALID_USER_BUFFER => WriteError.SystemResources,

From 1caa48c2df63d61b8d2501b3e3ee7c85ee8d89cb Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 12 Sep 2018 11:33:26 -0400
Subject: [PATCH 24/46] windows os.cpp implementations

---
 src/error.cpp |   2 +
 src/error.hpp |   2 +
 src/os.cpp    | 207 +++++++++++++++++++++++++++++++++++++++++++++++---
 src/os.hpp    |   2 +-
 4 files changed, 203 insertions(+), 10 deletions(-)

diff --git a/src/error.cpp b/src/error.cpp
index d503eaa18..7a9bd963b 100644
--- a/src/error.cpp
+++ b/src/error.cpp
@@ -30,6 +30,8 @@ const char *err_str(int err) {
         case ErrorEndOfFile: return "end of file";
         case ErrorIsDir: return "is directory";
         case ErrorUnsupportedOperatingSystem: return "unsupported operating system";
+        case ErrorSharingViolation: return "sharing violation";
+        case ErrorPipeBusy: return "pipe busy";
     }
     return "(invalid error)";
 }
diff --git a/src/error.hpp b/src/error.hpp
index 8c1f8b9aa..0996a41d5 100644
--- a/src/error.hpp
+++ b/src/error.hpp
@@ -30,6 +30,8 @@ enum Error {
     ErrorEndOfFile,
     ErrorIsDir,
     ErrorUnsupportedOperatingSystem,
+    ErrorSharingViolation,
+    ErrorPipeBusy,
 };
 
 const char *err_str(int err);
diff --git a/src/os.cpp b/src/os.cpp
index 9970c04c7..6e46b96e4 100644
--- a/src/os.cpp
+++ b/src/os.cpp
@@ -24,6 +24,7 @@
 #endif
 
 #include 
+#include 
 #include 
 #include 
 
@@ -1393,7 +1394,7 @@ Error os_self_exe_path(Buf *out_path) {
         }
         if (copied_amt < buf_len(out_path)) {
             buf_resize(out_path, copied_amt);
-            return 0;
+            return ErrorNone;
         }
         buf_resize(out_path, buf_len(out_path) * 2);
     }
@@ -1616,10 +1617,118 @@ int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf* output_buf, ZigLLVM_ArchTy
 #endif
 }
 
-// Ported from std/os/get_app_data_dir.zig
+#if defined(ZIG_OS_WINDOWS)
+// Ported from std/unicode.zig
+struct Utf16LeIterator {
+    uint8_t *bytes;
+    size_t i;
+};
+
+// Ported from std/unicode.zig
+static Utf16LeIterator Utf16LeIterator_init(WCHAR *ptr) {
+    return {(uint8_t*)ptr, 0};
+}
+
+// Ported from std/unicode.zig
+static Optional Utf16LeIterator_nextCodepoint(Utf16LeIterator *it) {
+    if (it->bytes[it->i] == 0 && it->bytes[it->i + 1] == 0)
+        return {};
+    uint32_t c0 = ((uint32_t)it->bytes[it->i]) | (((uint32_t)it->bytes[it->i + 1]) << 8);
+    if (c0 & ~((uint32_t)0x03ff) == 0xd800) {
+        // surrogate pair
+        it->i += 2;
+        assert(it->bytes[it->i] != 0 || it->bytes[it->i + 1] != 0);
+        uint32_t c1 = ((uint32_t)it->bytes[it->i]) | (((uint32_t)it->bytes[it->i + 1]) << 8);
+        assert(c1 & ~((uint32_t)0x03ff) == 0xdc00);
+        it->i += 2;
+        return Optional::some(0x10000 + (((c0 & 0x03ff) << 10) | (c1 & 0x03ff)));
+    } else {
+        assert(c0 & ~((uint32_t)0x03ff) != 0xdc00);
+        it->i += 2;
+        return Optional::some(c0);
+    }
+}
+
+// Ported from std/unicode.zig
+static uint8_t utf8CodepointSequenceLength(uint32_t c) {
+    if (c < 0x80) return 1;
+    if (c < 0x800) return 2;
+    if (c < 0x10000) return 3;
+    if (c < 0x110000) return 4;
+    zig_unreachable();
+}
+
+// Ported from std/unicode.zig
+static size_t utf8Encode(uint32_t c, Slice out) {
+    size_t length = utf8CodepointSequenceLength(c);
+    assert(out.len >= length);
+    switch (length) {
+        // The pattern for each is the same
+        // - Increasing the initial shift by 6 each time
+        // - Each time after the first shorten the shifted
+        //   value to a max of 0b111111 (63)
+        case 1:
+            out.ptr[0] = c; // Can just do 0 + codepoint for initial range
+            break;
+        case 2:
+            out.ptr[0] = 0b11000000 | (c >> 6);
+            out.ptr[1] = 0b10000000 | (c & 0b111111);
+            break;
+        case 3:
+            assert(!(0xd800 <= c && c <= 0xdfff));
+            out.ptr[0] = 0b11100000 | (c >> 12);
+            out.ptr[1] = 0b10000000 | ((c >> 6) & 0b111111);
+            out.ptr[2] = 0b10000000 | (c & 0b111111);
+            break;
+        case 4:
+            out.ptr[0] = 0b11110000 | (c >> 18);
+            out.ptr[1] = 0b10000000 | ((c >> 12) & 0b111111);
+            out.ptr[2] = 0b10000000 | ((c >> 6) & 0b111111);
+            out.ptr[3] = 0b10000000 | (c & 0b111111);
+            break;
+        default:
+            zig_unreachable();
+    }
+    return length;
+}
+
+// Ported from std.unicode.utf16leToUtf8Alloc
+static void utf16le_ptr_to_utf8(Buf *out, WCHAR *utf16le) {
+    // optimistically guess that it will all be ascii.
+    buf_resize(out, 0);
+    size_t out_index = 0;
+    Utf16LeIterator it = Utf16LeIterator_init(utf16le);
+    for (;;) {
+        Optional opt_codepoint = Utf16LeIterator_nextCodepoint(&it);
+        if (!opt_codepoint.is_some) break;
+        uint32_t codepoint = opt_codepoint.value;
+
+        size_t utf8_len = utf8CodepointSequenceLength(codepoint);
+        buf_resize(out, buf_len(out) + utf8_len);
+        utf8Encode(codepoint, {(uint8_t*)buf_ptr(out)+out_index, buf_len(out)-out_index});
+        out_index += utf8_len;
+    }
+}
+#endif
+
+// Ported from std.os.getAppDataDir
 Error os_get_app_data_dir(Buf *out_path, const char *appname) {
 #if defined(ZIG_OS_WINDOWS)
-#error "Unimplemented"
+    Error err;
+    WCHAR *dir_path_ptr;
+    switch (SHGetKnownFolderPath(FOLDERID_LocalAppData, KF_FLAG_CREATE, nullptr, &dir_path_ptr)) {
+        case S_OK:
+            // defer os.windows.CoTaskMemFree(@ptrCast(*c_void, dir_path_ptr));
+            utf16le_ptr_to_utf8(out_path, dir_path_ptr);
+            CoTaskMemFree(dir_path_ptr);
+            buf_appendf(out_path, "\\%s", appname);
+            return ErrorNone;
+        case E_OUTOFMEMORY:
+            return ErrorNoMem;
+        default:
+            return ErrorUnexpected;
+    }
+    zig_unreachable();
 #elif defined(ZIG_OS_DARWIN)
     const char *home_dir = getenv("HOME");
     if (home_dir == nullptr) {
@@ -1665,14 +1774,44 @@ Error os_self_exe_shared_libs(ZigList &paths) {
         paths.append(buf_create_from_str(name));
     }
     return ErrorNone;
+#elif defined(ZIG_OS_WINDOWS)
+    // zig is built statically on windows, so we can return an empty list
+    paths.resize(0);
+    return ErrorNone;
 #else
-#error "unimplemented"
+#error unimplemented
 #endif
 }
 
 Error os_file_open_r(Buf *full_path, OsFile *out_file) {
 #if defined(ZIG_OS_WINDOWS)
-#error "unimplemented"
+    // TODO use CreateFileW
+    HANDLE result = CreateFileA(buf_ptr(full_path), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
+
+    if (result == INVALID_HANDLE_VALUE) {
+        DWORD err = GetLastError();
+        switch (err) {
+            case ERROR_SHARING_VIOLATION:
+                return ErrorSharingViolation;
+            case ERROR_ALREADY_EXISTS:
+                return ErrorPathAlreadyExists;
+            case ERROR_FILE_EXISTS:
+                return ErrorPathAlreadyExists;
+            case ERROR_FILE_NOT_FOUND:
+                return ErrorFileNotFound;
+            case ERROR_PATH_NOT_FOUND:
+                return ErrorFileNotFound;
+            case ERROR_ACCESS_DENIED:
+                return ErrorAccess;
+            case ERROR_PIPE_BUSY:
+                return ErrorPipeBusy;
+            default:
+                return ErrorUnexpected;
+        }
+    }
+
+    *out_file = result;
+    return ErrorNone;
 #else
     for (;;) {
         int fd = open(buf_ptr(full_path), O_RDONLY|O_CLOEXEC);
@@ -1702,7 +1841,36 @@ Error os_file_open_r(Buf *full_path, OsFile *out_file) {
 
 Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) {
 #if defined(ZIG_OS_WINDOWS)
-#error "unimplemented"
+    for (;;) {
+        HANDLE result = CreateFileA(buf_ptr(full_path), GENERIC_READ | GENERIC_WRITE,
+            0, nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
+
+        if (result == INVALID_HANDLE_VALUE) {
+            DWORD err = GetLastError();
+            switch (err) {
+                case ERROR_SHARING_VIOLATION:
+                    // TODO wait for the lock instead of sleeping
+                    Sleep(10);
+                    continue;
+                case ERROR_ALREADY_EXISTS:
+                    return ErrorPathAlreadyExists;
+                case ERROR_FILE_EXISTS:
+                    return ErrorPathAlreadyExists;
+                case ERROR_FILE_NOT_FOUND:
+                    return ErrorFileNotFound;
+                case ERROR_PATH_NOT_FOUND:
+                    return ErrorFileNotFound;
+                case ERROR_ACCESS_DENIED:
+                    return ErrorAccess;
+                case ERROR_PIPE_BUSY:
+                    return ErrorPipeBusy;
+                default:
+                    return ErrorUnexpected;
+            }
+        }
+        *out_file = result;
+        return ErrorNone;
+    }
 #else
     int fd;
     for (;;) {
@@ -1757,7 +1925,12 @@ Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) {
 
 Error os_file_mtime(OsFile file, OsTimeStamp *mtime) {
 #if defined(ZIG_OS_WINDOWS)
-#error unimplemented
+    FILETIME last_write_time;
+    if (!GetFileTime(file, nullptr, nullptr, &last_write_time))
+        return ErrorUnexpected;
+    mtime->sec = last_write_time.dwLowDateTime | (last_write_time.dwHighDateTime << 32);
+    mtime->nsec = 0;
+    return ErrorNone;
 #elif defined(ZIG_OS_LINUX)
     struct stat statbuf;
     if (fstat(file, &statbuf) == -1)
@@ -1781,7 +1954,11 @@ Error os_file_mtime(OsFile file, OsTimeStamp *mtime) {
 
 Error os_file_read(OsFile file, void *ptr, size_t *len) {
 #if defined(ZIG_OS_WINDOWS)
-#error unimplemented
+    DWORD amt_read;
+    if (ReadFile(file, ptr, *len, &amt_read, nullptr) == 0)
+        return ErrorUnexpected;
+    *len = amt_read;
+    return ErrorNone;
 #else
     for (;;) {
         ssize_t rc = read(file, ptr, *len);
@@ -1830,10 +2007,18 @@ Error os_file_read_all(OsFile file, Buf *contents) {
 
 Error os_file_overwrite(OsFile file, Buf *contents) {
 #if defined(ZIG_OS_WINDOWS)
-#error unimplemented
+    if (SetFilePointer(file, 0, nullptr, FILE_BEGIN) == INVALID_SET_FILE_POINTER)
+        return ErrorFileSystem;
+    if (!SetEndOfFile(file))
+        return ErrorFileSystem;
+    if (!WriteFile(file, buf_ptr(contents), buf_len(contents), nullptr, nullptr))
+        return ErrorFileSystem;
+    return ErrorNone;
 #else
     if (lseek(file, 0, SEEK_SET) == -1)
         return ErrorFileSystem;
+    if (ftruncate(file, 0) == -1)
+        return ErrorFileSystem;
     for (;;) {
         if (write(file, buf_ptr(contents), buf_len(contents)) == -1) {
             switch (errno) {
@@ -1853,5 +2038,9 @@ Error os_file_overwrite(OsFile file, Buf *contents) {
 }
 
 void os_file_close(OsFile file) {
+#if defined(ZIG_OS_WINDOWS)
+    CloseHandle(file);
+#else
     close(file);
+#endif
 }
diff --git a/src/os.hpp b/src/os.hpp
index ac422fbd2..1054bf24a 100644
--- a/src/os.hpp
+++ b/src/os.hpp
@@ -72,7 +72,7 @@ struct Termination {
 };
 
 #if defined(ZIG_OS_WINDOWS)
-#define OsFile (void *)
+#define OsFile void *
 #else
 #define OsFile int
 #endif

From 3a49d115cf38154f0094d9615334529890059006 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 12 Sep 2018 11:49:46 -0400
Subject: [PATCH 25/46] fix zig build cache dir path

---
 build.zig    |  3 ++-
 src/main.cpp | 16 +++++++++++-----
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/build.zig b/build.zig
index d4a159e5e..e88637713 100644
--- a/build.zig
+++ b/build.zig
@@ -16,11 +16,12 @@ pub fn build(b: *Builder) !void {
     var docgen_exe = b.addExecutable("docgen", "doc/docgen.zig");
 
     const rel_zig_exe = try os.path.relative(b.allocator, b.build_root, b.zig_exe);
+    const langref_out_path = os.path.join(b.allocator, b.cache_root, "langref.html") catch unreachable;
     var docgen_cmd = b.addCommand(null, b.env_map, [][]const u8{
         docgen_exe.getOutputPath(),
         rel_zig_exe,
         "doc" ++ os.path.sep_str ++ "langref.html.in",
-        os.path.join(b.allocator, b.cache_root, "langref.html") catch unreachable,
+        langref_out_path,
     });
     docgen_cmd.step.dependOn(&docgen_exe.step);
 
diff --git a/src/main.cpp b/src/main.cpp
index fdc34a83a..bf6c14297 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -353,7 +353,7 @@ int main(int argc, char **argv) {
     size_t ver_minor = 0;
     size_t ver_patch = 0;
     bool timing_info = false;
-    const char *cache_dir = default_zig_cache_name;
+    const char *cache_dir = nullptr;
     CliPkg *cur_pkg = allocate(1);
     BuildMode build_mode = BuildModeDebug;
     ZigList test_exec_args = {0};
@@ -403,7 +403,7 @@ int main(int argc, char **argv) {
 
         CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf);
         g->enable_time_report = timing_info;
-        buf_init_from_str(&g->cache_dir, cache_dir);
+        buf_init_from_str(&g->cache_dir, cache_dir ? cache_dir : default_zig_cache_name);
         codegen_set_out_name(g, buf_create_from_str("build"));
 
         Buf *build_file_buf = buf_create_from_str(build_file);
@@ -412,8 +412,14 @@ int main(int argc, char **argv) {
         Buf build_file_dirname = BUF_INIT;
         os_path_split(&build_file_abs, &build_file_dirname, &build_file_basename);
 
-        Buf *cache_dir_buf = buf_create_from_str(cache_dir);
-        Buf full_cache_dir = os_path_resolve(&cache_dir_buf, 1);
+
+        Buf full_cache_dir = BUF_INIT;
+        if (cache_dir == nullptr) {
+            os_path_join(&build_file_dirname, buf_create_from_str(default_zig_cache_name), &full_cache_dir);
+        } else {
+            Buf *cache_dir_buf = buf_create_from_str(cache_dir);
+            full_cache_dir = os_path_resolve(&cache_dir_buf, 1);
+        }
 
         args.items[1] = buf_ptr(&build_file_dirname);
         args.items[2] = buf_ptr(&full_cache_dir);
@@ -830,7 +836,7 @@ int main(int argc, char **argv) {
 
             CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf);
             g->enable_time_report = timing_info;
-            buf_init_from_str(&g->cache_dir, cache_dir);
+            buf_init_from_str(&g->cache_dir, cache_dir ? cache_dir : default_zig_cache_name);
             codegen_set_out_name(g, buf_out_name);
             codegen_set_lib_version(g, ver_major, ver_minor, ver_patch);
             codegen_set_is_test(g, cmd == CmdTest);

From 178d69191ba008dffd70d2854df09cec556b59dd Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 12 Sep 2018 13:55:02 -0400
Subject: [PATCH 26/46] windows: std.fs functions support concurrent ops

when reading and writing the same file descriptors
---
 src-self-hosted/main.zig |  2 +-
 std/event/fs.zig         | 75 +++++++++++++++++++---------------------
 std/event/loop.zig       | 46 ++++++++++++++++--------
 3 files changed, 67 insertions(+), 56 deletions(-)

diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig
index 6a450030c..6aa5f067e 100644
--- a/src-self-hosted/main.zig
+++ b/src-self-hosted/main.zig
@@ -737,7 +737,7 @@ async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8) FmtError!void {
         file_path,
         max_src_size,
     )) catch |err| switch (err) {
-        error.IsDir => {
+        error.IsDir, error.AccessDenied => {
             // TODO make event based (and dir.next())
             var dir = try std.os.Dir.open(fmt.loop.allocator, file_path);
             defer dir.close();
diff --git a/std/event/fs.zig b/std/event/fs.zig
index 5e7e24ff4..c13a940a3 100644
--- a/std/event/fs.zig
+++ b/std/event/fs.zig
@@ -109,30 +109,28 @@ pub async fn pwriteWindows(loop: *Loop, fd: os.FileHandle, data: []const u8, off
         .base = Loop.ResumeNode{
             .id = Loop.ResumeNode.Id.Basic,
             .handle = @handle(),
+            .overlapped = windows.OVERLAPPED{
+                .Internal = 0,
+                .InternalHigh = 0,
+                .Offset = @truncate(u32, offset),
+                .OffsetHigh = @truncate(u32, offset >> 32),
+                .hEvent = null,
+            },
         },
     };
-    const completion_key = @ptrToInt(&resume_node.base);
-    // TODO support concurrent async ops on the file handle
-    // we can do this by ignoring completion key and using @fieldParentPtr with the *Overlapped
-    _ = try os.windowsCreateIoCompletionPort(fd, loop.os_data.io_port, completion_key, undefined);
-    var overlapped = windows.OVERLAPPED{
-        .Internal = 0,
-        .InternalHigh = 0,
-        .Offset = @truncate(u32, offset),
-        .OffsetHigh = @truncate(u32, offset >> 32),
-        .hEvent = null,
-    };
+    // TODO only call create io completion port once per fd
+    _ = try os.windowsCreateIoCompletionPort(fd, loop.os_data.io_port, undefined, undefined);
     loop.beginOneEvent();
     errdefer loop.finishOneEvent();
 
     errdefer {
-        _ = windows.CancelIoEx(fd, &overlapped);
+        _ = windows.CancelIoEx(fd, &resume_node.base.overlapped);
     }
     suspend {
-        _ = windows.WriteFile(fd, data.ptr, @intCast(windows.DWORD, data.len), null, &overlapped);
+        _ = windows.WriteFile(fd, data.ptr, @intCast(windows.DWORD, data.len), null, &resume_node.base.overlapped);
     }
     var bytes_transferred: windows.DWORD = undefined;
-    if (windows.GetOverlappedResult(fd, &overlapped, &bytes_transferred, windows.FALSE) == 0) {
+    if (windows.GetOverlappedResult(fd, &resume_node.base.overlapped, &bytes_transferred, windows.FALSE) == 0) {
         const err = windows.GetLastError();
         return switch (err) {
             windows.ERROR.IO_PENDING => unreachable,
@@ -243,30 +241,28 @@ pub async fn preadWindows(loop: *Loop, fd: os.FileHandle, data: []u8, offset: u6
         .base = Loop.ResumeNode{
             .id = Loop.ResumeNode.Id.Basic,
             .handle = @handle(),
+            .overlapped = windows.OVERLAPPED{
+                .Internal = 0,
+                .InternalHigh = 0,
+                .Offset = @truncate(u32, offset),
+                .OffsetHigh = @truncate(u32, offset >> 32),
+                .hEvent = null,
+            },
         },
     };
-    const completion_key = @ptrToInt(&resume_node.base);
-    // TODO support concurrent async ops on the file handle
-    // we can do this by ignoring completion key and using @fieldParentPtr with the *Overlapped
-    _ = try os.windowsCreateIoCompletionPort(fd, loop.os_data.io_port, completion_key, undefined);
-    var overlapped = windows.OVERLAPPED{
-        .Internal = 0,
-        .InternalHigh = 0,
-        .Offset = @truncate(u32, offset),
-        .OffsetHigh = @truncate(u32, offset >> 32),
-        .hEvent = null,
-    };
+    // TODO only call create io completion port once per fd
+    _ = try os.windowsCreateIoCompletionPort(fd, loop.os_data.io_port, undefined, undefined);
     loop.beginOneEvent();
     errdefer loop.finishOneEvent();
 
     errdefer {
-        _ = windows.CancelIoEx(fd, &overlapped);
+        _ = windows.CancelIoEx(fd, &resume_node.base.overlapped);
     }
     suspend {
-        _ = windows.ReadFile(fd, data.ptr, @intCast(windows.DWORD, data.len), null, &overlapped);
+        _ = windows.ReadFile(fd, data.ptr, @intCast(windows.DWORD, data.len), null, &resume_node.base.overlapped);
     }
     var bytes_transferred: windows.DWORD = undefined;
-    if (windows.GetOverlappedResult(fd, &overlapped, &bytes_transferred, windows.FALSE) == 0) {
+    if (windows.GetOverlappedResult(fd, &resume_node.base.overlapped, &bytes_transferred, windows.FALSE) == 0) {
         const err = windows.GetLastError();
         return switch (err) {
             windows.ERROR.IO_PENDING => unreachable,
@@ -1074,23 +1070,22 @@ pub fn Watch(comptime V: type) type {
                 .base = Loop.ResumeNode{
                     .id = Loop.ResumeNode.Id.Basic,
                     .handle = @handle(),
+                    .overlapped = windows.OVERLAPPED{
+                        .Internal = 0,
+                        .InternalHigh = 0,
+                        .Offset = 0,
+                        .OffsetHigh = 0,
+                        .hEvent = null,
+                    },
                 },
             };
-            const completion_key = @ptrToInt(&resume_node.base);
-            var overlapped = windows.OVERLAPPED{
-                .Internal = 0,
-                .InternalHigh = 0,
-                .Offset = 0,
-                .OffsetHigh = 0,
-                .hEvent = null,
-            };
             var event_buf: [4096]u8 align(@alignOf(windows.FILE_NOTIFY_INFORMATION)) = undefined;
 
             // TODO handle this error not in the channel but in the setup
             _ = os.windowsCreateIoCompletionPort(
                 dir_handle,
                 self.channel.loop.os_data.io_port,
-                completion_key,
+                undefined,
                 undefined,
             ) catch |err| {
                 await (async self.channel.put(err) catch unreachable);
@@ -1103,7 +1098,7 @@ pub fn Watch(comptime V: type) type {
                     self.channel.loop.beginOneEvent();
                     errdefer self.channel.loop.finishOneEvent();
                     errdefer {
-                        _ = windows.CancelIoEx(dir_handle, &overlapped);
+                        _ = windows.CancelIoEx(dir_handle, &resume_node.base.overlapped);
                     }
                     suspend {
                         _ = windows.ReadDirectoryChangesW(
@@ -1116,13 +1111,13 @@ pub fn Watch(comptime V: type) type {
                                 windows.FILE_NOTIFY_CHANGE_LAST_WRITE | windows.FILE_NOTIFY_CHANGE_LAST_ACCESS |
                                 windows.FILE_NOTIFY_CHANGE_CREATION | windows.FILE_NOTIFY_CHANGE_SECURITY,
                             null, // number of bytes transferred (unused for async)
-                            &overlapped,
+                            &resume_node.base.overlapped,
                             null, // completion routine - unused because we use IOCP
                         );
                     }
                 }
                 var bytes_transferred: windows.DWORD = undefined;
-                if (windows.GetOverlappedResult(dir_handle, &overlapped, &bytes_transferred, windows.FALSE) == 0) {
+                if (windows.GetOverlappedResult(dir_handle, &resume_node.base.overlapped, &bytes_transferred, windows.FALSE) == 0) {
                     const errno = windows.GetLastError();
                     const err = switch (errno) {
                         else => os.unexpectedErrorWindows(errno),
diff --git a/std/event/loop.zig b/std/event/loop.zig
index 733112549..6cd22ff69 100644
--- a/std/event/loop.zig
+++ b/std/event/loop.zig
@@ -27,6 +27,19 @@ pub const Loop = struct {
     pub const ResumeNode = struct {
         id: Id,
         handle: promise,
+        overlapped: Overlapped,
+
+        const overlapped_init = switch (builtin.os) {
+            builtin.Os.windows => windows.OVERLAPPED{
+                .Internal = 0,
+                .InternalHigh = 0,
+                .Offset = 0,
+                .OffsetHigh = 0,
+                .hEvent = null,
+            },
+            else => {},
+        };
+        const Overlapped = @typeOf(overlapped_init);
 
         pub const Id = enum {
             Basic,
@@ -101,6 +114,7 @@ pub const Loop = struct {
             .final_resume_node = ResumeNode{
                 .id = ResumeNode.Id.Stop,
                 .handle = undefined,
+                .overlapped = ResumeNode.overlapped_init,
             },
         };
         const extra_thread_count = thread_count - 1;
@@ -153,6 +167,7 @@ pub const Loop = struct {
                             .base = ResumeNode{
                                 .id = ResumeNode.Id.EventFd,
                                 .handle = undefined,
+                                .overlapped = ResumeNode.overlapped_init,
                             },
                             .eventfd = try os.linuxEventFd(1, posix.EFD_CLOEXEC | posix.EFD_NONBLOCK),
                             .epoll_op = posix.EPOLL_CTL_ADD,
@@ -225,6 +240,7 @@ pub const Loop = struct {
                             .base = ResumeNode{
                                 .id = ResumeNode.Id.EventFd,
                                 .handle = undefined,
+                                .overlapped = ResumeNode.overlapped_init,
                             },
                             // this one is for sending events
                             .kevent = posix.Kevent{
@@ -311,6 +327,7 @@ pub const Loop = struct {
                             .base = ResumeNode{
                                 .id = ResumeNode.Id.EventFd,
                                 .handle = undefined,
+                                .overlapped = ResumeNode.overlapped_init,
                             },
                             // this one is for sending events
                             .completion_key = @ptrToInt(&eventfd_node.data.base),
@@ -325,8 +342,8 @@ pub const Loop = struct {
                     var i: usize = 0;
                     while (i < extra_thread_index) : (i += 1) {
                         while (true) {
-                            const overlapped = @intToPtr(?*windows.OVERLAPPED, 0x1);
-                            os.windowsPostQueuedCompletionStatus(self.os_data.io_port, undefined, @ptrToInt(&self.final_resume_node), overlapped) catch continue;
+                            const overlapped = &self.final_resume_node.overlapped;
+                            os.windowsPostQueuedCompletionStatus(self.os_data.io_port, undefined, undefined, overlapped) catch continue;
                             break;
                         }
                     }
@@ -413,6 +430,7 @@ pub const Loop = struct {
             .base = ResumeNode{
                 .id = ResumeNode.Id.Basic,
                 .handle = @handle(),
+                .overlapped = ResumeNode.overlapped_init,
             },
             .kev = undefined,
         };
@@ -489,15 +507,11 @@ pub const Loop = struct {
                     };
                 },
                 builtin.Os.windows => {
-                    // this value is never dereferenced but we need it to be non-null so that
-                    // the consumer code can decide whether to read the completion key.
-                    // it has to do this for normal I/O, so we match that behavior here.
-                    const overlapped = @intToPtr(?*windows.OVERLAPPED, 0x1);
                     os.windowsPostQueuedCompletionStatus(
                         self.os_data.io_port,
                         undefined,
-                        eventfd_node.completion_key,
-                        overlapped,
+                        undefined,
+                        &eventfd_node.base.overlapped,
                     ) catch {
                         self.next_tick_queue.unget(next_tick_node);
                         self.available_eventfd_resume_nodes.push(resume_stack_node);
@@ -606,8 +620,8 @@ pub const Loop = struct {
                     var i: usize = 0;
                     while (i < self.extra_threads.len + 1) : (i += 1) {
                         while (true) {
-                            const overlapped = @intToPtr(?*windows.OVERLAPPED, 0x1);
-                            os.windowsPostQueuedCompletionStatus(self.os_data.io_port, undefined, @ptrToInt(&self.final_resume_node), overlapped) catch continue;
+                            const overlapped = &self.final_resume_node.overlapped;
+                            os.windowsPostQueuedCompletionStatus(self.os_data.io_port, undefined, undefined, overlapped) catch continue;
                             break;
                         }
                     }
@@ -680,17 +694,19 @@ pub const Loop = struct {
                 },
                 builtin.Os.windows => {
                     var completion_key: usize = undefined;
-                    while (true) {
+                    const overlapped = while (true) {
                         var nbytes: windows.DWORD = undefined;
                         var overlapped: ?*windows.OVERLAPPED = undefined;
-                        switch (os.windowsGetQueuedCompletionStatus(self.os_data.io_port, &nbytes, &completion_key, &overlapped, windows.INFINITE)) {
+                        switch (os.windowsGetQueuedCompletionStatus(self.os_data.io_port, &nbytes, &completion_key,
+                            &overlapped, windows.INFINITE))
+                        {
                             os.WindowsWaitResult.Aborted => return,
                             os.WindowsWaitResult.Normal => {},
                             os.WindowsWaitResult.Cancelled => continue,
                         }
-                        if (overlapped != null) break;
-                    }
-                    const resume_node = @intToPtr(*ResumeNode, completion_key);
+                        if (overlapped) |o| break o;
+                    } else unreachable; // TODO else unreachable should not be necessary
+                    const resume_node = @fieldParentPtr(ResumeNode, "overlapped", overlapped);
                     const handle = resume_node.handle;
                     const resume_node_id = resume_node.id;
                     switch (resume_node_id) {

From a75753338657b85588fe6c54a40fecac0584b336 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 12 Sep 2018 14:26:21 -0400
Subject: [PATCH 27/46] fix zig fmt on windows

closes #1069
---
 std/coff.zig                |  52 ++++++------
 std/crypto/x25519.zig       |   2 +-
 std/debug/index.zig         |  23 +++---
 std/event/fs.zig            |  15 ++--
 std/event/loop.zig          |  10 +--
 std/event/tcp.zig           |   1 +
 std/os/child_process.zig    |  12 ++-
 std/os/windows/kernel32.zig |   1 -
 std/os/windows/util.zig     |   4 +-
 std/pdb.zig                 | 157 ++++++++++++++++++++++--------------
 10 files changed, 159 insertions(+), 118 deletions(-)

diff --git a/std/coff.zig b/std/coff.zig
index 379fd1af4..bf5a93ef3 100644
--- a/std/coff.zig
+++ b/std/coff.zig
@@ -8,9 +8,9 @@ const ArrayList = std.ArrayList;
 
 // CoffHeader.machine values
 // see https://msdn.microsoft.com/en-us/library/windows/desktop/ms680313(v=vs.85).aspx
-const IMAGE_FILE_MACHINE_I386   = 0x014c;
-const IMAGE_FILE_MACHINE_IA64   = 0x0200;
-const IMAGE_FILE_MACHINE_AMD64  = 0x8664;
+const IMAGE_FILE_MACHINE_I386 = 0x014c;
+const IMAGE_FILE_MACHINE_IA64 = 0x0200;
+const IMAGE_FILE_MACHINE_AMD64 = 0x8664;
 
 // OptionalHeader.magic values
 // see https://msdn.microsoft.com/en-us/library/windows/desktop/ms680339(v=vs.85).aspx
@@ -20,7 +20,7 @@ const IMAGE_NT_OPTIONAL_HDR64_MAGIC = 0x20b;
 const IMAGE_NUMBEROF_DIRECTORY_ENTRIES = 16;
 const DEBUG_DIRECTORY = 6;
 
-pub const CoffError = error {
+pub const CoffError = error{
     InvalidPEMagic,
     InvalidPEHeader,
     InvalidMachine,
@@ -56,24 +56,21 @@ pub const Coff = struct {
 
         var pe_header_magic: [4]u8 = undefined;
         try in.readNoEof(pe_header_magic[0..]);
-        if (!mem.eql(u8, pe_header_magic, []u8{'P', 'E', 0, 0}))
+        if (!mem.eql(u8, pe_header_magic, []u8{ 'P', 'E', 0, 0 }))
             return error.InvalidPEHeader;
 
-        self.coff_header = CoffHeader {
+        self.coff_header = CoffHeader{
             .machine = try in.readIntLe(u16),
-            .number_of_sections = try in.readIntLe(u16),           
-            .timedate_stamp = try in.readIntLe(u32),           
-            .pointer_to_symbol_table = try in.readIntLe(u32),           
-            .number_of_symbols = try in.readIntLe(u32),           
-            .size_of_optional_header = try in.readIntLe(u16),           
-            .characteristics = try in.readIntLe(u16),           
+            .number_of_sections = try in.readIntLe(u16),
+            .timedate_stamp = try in.readIntLe(u32),
+            .pointer_to_symbol_table = try in.readIntLe(u32),
+            .number_of_symbols = try in.readIntLe(u32),
+            .size_of_optional_header = try in.readIntLe(u16),
+            .characteristics = try in.readIntLe(u16),
         };
 
         switch (self.coff_header.machine) {
-            IMAGE_FILE_MACHINE_I386,
-            IMAGE_FILE_MACHINE_AMD64,
-            IMAGE_FILE_MACHINE_IA64
-                => {},
+            IMAGE_FILE_MACHINE_I386, IMAGE_FILE_MACHINE_AMD64, IMAGE_FILE_MACHINE_IA64 => {},
             else => return error.InvalidMachine,
         }
 
@@ -89,11 +86,9 @@ pub const Coff = struct {
         var skip_size: u16 = undefined;
         if (self.pe_header.magic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) {
             skip_size = 2 * @sizeOf(u8) + 8 * @sizeOf(u16) + 18 * @sizeOf(u32);
-        }
-        else if (self.pe_header.magic == IMAGE_NT_OPTIONAL_HDR64_MAGIC) {
+        } else if (self.pe_header.magic == IMAGE_NT_OPTIONAL_HDR64_MAGIC) {
             skip_size = 2 * @sizeOf(u8) + 8 * @sizeOf(u16) + 12 * @sizeOf(u32) + 5 * @sizeOf(u64);
-        }
-        else
+        } else
             return error.InvalidPEMagic;
 
         try self.in_file.seekForward(skip_size);
@@ -103,7 +98,7 @@ pub const Coff = struct {
             return error.InvalidPEHeader;
 
         for (self.pe_header.data_directory) |*data_dir| {
-            data_dir.* = OptionalHeader.DataDirectory {
+            data_dir.* = OptionalHeader.DataDirectory{
                 .virtual_address = try in.readIntLe(u32),
                 .size = try in.readIntLe(u32),
             };
@@ -114,7 +109,7 @@ pub const Coff = struct {
         try self.loadSections();
         const header = (self.getSection(".rdata") orelse return error.MissingCoffSection).header;
 
-        // The linker puts a chunk that contains the .pdb path right after the 
+        // The linker puts a chunk that contains the .pdb path right after the
         // debug_directory.
         const debug_dir = &self.pe_header.data_directory[DEBUG_DIRECTORY];
         const file_offset = debug_dir.virtual_address - header.virtual_address + header.pointer_to_raw_data;
@@ -159,10 +154,10 @@ pub const Coff = struct {
         var i: u16 = 0;
         while (i < self.coff_header.number_of_sections) : (i += 1) {
             try in.readNoEof(name[0..]);
-            try self.sections.append(Section {
-                .header = SectionHeader {
+            try self.sections.append(Section{
+                .header = SectionHeader{
                     .name = name,
-                    .misc = SectionHeader.Misc { .physical_address = try in.readIntLe(u32) },
+                    .misc = SectionHeader.Misc{ .physical_address = try in.readIntLe(u32) },
                     .virtual_address = try in.readIntLe(u32),
                     .size_of_raw_data = try in.readIntLe(u32),
                     .pointer_to_raw_data = try in.readIntLe(u32),
@@ -184,7 +179,6 @@ pub const Coff = struct {
         }
         return null;
     }
-
 };
 
 const CoffHeader = struct {
@@ -194,13 +188,13 @@ const CoffHeader = struct {
     pointer_to_symbol_table: u32,
     number_of_symbols: u32,
     size_of_optional_header: u16,
-    characteristics: u16
+    characteristics: u16,
 };
 
 const OptionalHeader = struct {
     const DataDirectory = struct {
         virtual_address: u32,
-        size: u32
+        size: u32,
     };
 
     magic: u16,
@@ -214,7 +208,7 @@ pub const Section = struct {
 const SectionHeader = struct {
     const Misc = union {
         physical_address: u32,
-        virtual_size: u32
+        virtual_size: u32,
     };
 
     name: [8]u8,
diff --git a/std/crypto/x25519.zig b/std/crypto/x25519.zig
index 41b2ff095..bf78511f9 100644
--- a/std/crypto/x25519.zig
+++ b/std/crypto/x25519.zig
@@ -115,7 +115,7 @@ pub const X25519 = struct {
         return !zerocmp(u8, out);
     }
 
-    pub fn createPublicKey(public_key: [] u8, private_key: []const u8) bool {
+    pub fn createPublicKey(public_key: []u8, private_key: []const u8) bool {
         var base_point = []u8{9} ++ []u8{0} ** 31;
         return create(public_key, private_key, base_point);
     }
diff --git a/std/debug/index.zig b/std/debug/index.zig
index 8db7c75d2..14e8b9197 100644
--- a/std/debug/index.zig
+++ b/std/debug/index.zig
@@ -242,9 +242,12 @@ pub fn writeCurrentStackTrace(out_stream: var, debug_info: *DebugInfo, tty_color
     }
 }
 
-pub fn writeCurrentStackTraceWindows(out_stream: var, debug_info: *DebugInfo,
-    tty_color: bool, start_addr: ?usize) !void
-{
+pub fn writeCurrentStackTraceWindows(
+    out_stream: var,
+    debug_info: *DebugInfo,
+    tty_color: bool,
+    start_addr: ?usize,
+) !void {
     var addr_buf: [1024]usize = undefined;
     const casted_len = @intCast(u32, addr_buf.len); // TODO shouldn't need this cast
     const n = windows.RtlCaptureStackBackTrace(0, casted_len, @ptrCast(**c_void, &addr_buf), null);
@@ -391,7 +394,7 @@ fn printSourceAtAddressWindows(di: *DebugInfo, out_stream: var, relocated_addres
             break :subsections null;
         }
     };
-    
+
     if (tty_color) {
         setTtyColor(TtyColor.White);
         if (opt_line_info) |li| {
@@ -438,7 +441,7 @@ fn printSourceAtAddressWindows(di: *DebugInfo, out_stream: var, relocated_addres
     }
 }
 
-const TtyColor = enum{
+const TtyColor = enum {
     Red,
     Green,
     Cyan,
@@ -465,18 +468,16 @@ fn setTtyColor(tty_color: TtyColor) void {
     // TODO handle errors
     switch (tty_color) {
         TtyColor.Red => {
-            _ = windows.SetConsoleTextAttribute(stderr_file.handle, windows.FOREGROUND_RED|windows.FOREGROUND_INTENSITY);
+            _ = windows.SetConsoleTextAttribute(stderr_file.handle, windows.FOREGROUND_RED | windows.FOREGROUND_INTENSITY);
         },
         TtyColor.Green => {
-            _ = windows.SetConsoleTextAttribute(stderr_file.handle, windows.FOREGROUND_GREEN|windows.FOREGROUND_INTENSITY);
+            _ = windows.SetConsoleTextAttribute(stderr_file.handle, windows.FOREGROUND_GREEN | windows.FOREGROUND_INTENSITY);
         },
         TtyColor.Cyan => {
-            _ = windows.SetConsoleTextAttribute(stderr_file.handle,
-                windows.FOREGROUND_GREEN|windows.FOREGROUND_BLUE|windows.FOREGROUND_INTENSITY);
+            _ = windows.SetConsoleTextAttribute(stderr_file.handle, windows.FOREGROUND_GREEN | windows.FOREGROUND_BLUE | windows.FOREGROUND_INTENSITY);
         },
         TtyColor.White, TtyColor.Bold => {
-            _ = windows.SetConsoleTextAttribute(stderr_file.handle,
-                windows.FOREGROUND_RED|windows.FOREGROUND_GREEN|windows.FOREGROUND_BLUE|windows.FOREGROUND_INTENSITY);
+            _ = windows.SetConsoleTextAttribute(stderr_file.handle, windows.FOREGROUND_RED | windows.FOREGROUND_GREEN | windows.FOREGROUND_BLUE | windows.FOREGROUND_INTENSITY);
         },
         TtyColor.Dim => {
             _ = windows.SetConsoleTextAttribute(stderr_file.handle, windows.FOREGROUND_INTENSITY);
diff --git a/std/event/fs.zig b/std/event/fs.zig
index c13a940a3..563547121 100644
--- a/std/event/fs.zig
+++ b/std/event/fs.zig
@@ -119,7 +119,7 @@ pub async fn pwriteWindows(loop: *Loop, fd: os.FileHandle, data: []const u8, off
         },
     };
     // TODO only call create io completion port once per fd
-    _ = try os.windowsCreateIoCompletionPort(fd, loop.os_data.io_port, undefined, undefined);
+    _ = windows.CreateIoCompletionPort(fd, loop.os_data.io_port, undefined, undefined);
     loop.beginOneEvent();
     errdefer loop.finishOneEvent();
 
@@ -251,7 +251,7 @@ pub async fn preadWindows(loop: *Loop, fd: os.FileHandle, data: []u8, offset: u6
         },
     };
     // TODO only call create io completion port once per fd
-    _ = try os.windowsCreateIoCompletionPort(fd, loop.os_data.io_port, undefined, undefined);
+    _ = windows.CreateIoCompletionPort(fd, loop.os_data.io_port, undefined, undefined);
     loop.beginOneEvent();
     errdefer loop.finishOneEvent();
 
@@ -264,12 +264,13 @@ pub async fn preadWindows(loop: *Loop, fd: os.FileHandle, data: []u8, offset: u6
     var bytes_transferred: windows.DWORD = undefined;
     if (windows.GetOverlappedResult(fd, &resume_node.base.overlapped, &bytes_transferred, windows.FALSE) == 0) {
         const err = windows.GetLastError();
-        return switch (err) {
+        switch (err) {
             windows.ERROR.IO_PENDING => unreachable,
-            windows.ERROR.OPERATION_ABORTED => error.OperationAborted,
-            windows.ERROR.BROKEN_PIPE => error.BrokenPipe,
-            else => os.unexpectedErrorWindows(err),
-        };
+            windows.ERROR.OPERATION_ABORTED => return error.OperationAborted,
+            windows.ERROR.BROKEN_PIPE => return error.BrokenPipe,
+            windows.ERROR.HANDLE_EOF => return usize(bytes_transferred),
+            else => return os.unexpectedErrorWindows(err),
+        }
     }
     return usize(bytes_transferred);
 }
diff --git a/std/event/loop.zig b/std/event/loop.zig
index 6cd22ff69..e87e92804 100644
--- a/std/event/loop.zig
+++ b/std/event/loop.zig
@@ -29,7 +29,7 @@ pub const Loop = struct {
         handle: promise,
         overlapped: Overlapped,
 
-        const overlapped_init = switch (builtin.os) {
+        pub const overlapped_init = switch (builtin.os) {
             builtin.Os.windows => windows.OVERLAPPED{
                 .Internal = 0,
                 .InternalHigh = 0,
@@ -39,7 +39,7 @@ pub const Loop = struct {
             },
             else => {},
         };
-        const Overlapped = @typeOf(overlapped_init);
+        pub const Overlapped = @typeOf(overlapped_init);
 
         pub const Id = enum {
             Basic,
@@ -415,6 +415,7 @@ pub const Loop = struct {
                 .base = ResumeNode{
                     .id = ResumeNode.Id.Basic,
                     .handle = @handle(),
+                    .overlapped = ResumeNode.overlapped_init,
                 },
             };
             try self.linuxAddFd(fd, &resume_node.base, flags);
@@ -697,11 +698,10 @@ pub const Loop = struct {
                     const overlapped = while (true) {
                         var nbytes: windows.DWORD = undefined;
                         var overlapped: ?*windows.OVERLAPPED = undefined;
-                        switch (os.windowsGetQueuedCompletionStatus(self.os_data.io_port, &nbytes, &completion_key,
-                            &overlapped, windows.INFINITE))
-                        {
+                        switch (os.windowsGetQueuedCompletionStatus(self.os_data.io_port, &nbytes, &completion_key, &overlapped, windows.INFINITE)) {
                             os.WindowsWaitResult.Aborted => return,
                             os.WindowsWaitResult.Normal => {},
+                            os.WindowsWaitResult.EOF => {},
                             os.WindowsWaitResult.Cancelled => continue,
                         }
                         if (overlapped) |o| break o;
diff --git a/std/event/tcp.zig b/std/event/tcp.zig
index 491acab39..69b143dda 100644
--- a/std/event/tcp.zig
+++ b/std/event/tcp.zig
@@ -32,6 +32,7 @@ pub const Server = struct {
             .listen_resume_node = event.Loop.ResumeNode{
                 .id = event.Loop.ResumeNode.Id.Basic,
                 .handle = undefined,
+                .overlapped = event.Loop.ResumeNode.overlapped_init,
             },
         };
     }
diff --git a/std/os/child_process.zig b/std/os/child_process.zig
index a8f307f3b..8b76a3cb9 100644
--- a/std/os/child_process.zig
+++ b/std/os/child_process.zig
@@ -658,8 +658,16 @@ fn windowsCreateProcess(app_name: [*]u16, cmd_line: [*]u16, envp_ptr: ?[*]u16, c
     // environment variables to programs that were not, which seems unlikely.
     // More investigation is needed.
     if (windows.CreateProcessW(
-        app_name, cmd_line, null, null, windows.TRUE, windows.CREATE_UNICODE_ENVIRONMENT,
-        @ptrCast(?*c_void, envp_ptr), cwd_ptr, lpStartupInfo, lpProcessInformation,
+        app_name,
+        cmd_line,
+        null,
+        null,
+        windows.TRUE,
+        windows.CREATE_UNICODE_ENVIRONMENT,
+        @ptrCast(?*c_void, envp_ptr),
+        cwd_ptr,
+        lpStartupInfo,
+        lpProcessInformation,
     ) == 0) {
         const err = windows.GetLastError();
         switch (err) {
diff --git a/std/os/windows/kernel32.zig b/std/os/windows/kernel32.zig
index 1a7b28eac..94b339fa6 100644
--- a/std/os/windows/kernel32.zig
+++ b/std/os/windows/kernel32.zig
@@ -206,7 +206,6 @@ pub const FILE_NOTIFY_CHANGE_DIR_NAME = 2;
 pub const FILE_NOTIFY_CHANGE_FILE_NAME = 1;
 pub const FILE_NOTIFY_CHANGE_ATTRIBUTES = 4;
 
-
 pub const CONSOLE_SCREEN_BUFFER_INFO = extern struct {
     dwSize: COORD,
     dwCursorPosition: COORD,
diff --git a/std/os/windows/util.zig b/std/os/windows/util.zig
index 2a1a82a96..6534d16f5 100644
--- a/std/os/windows/util.zig
+++ b/std/os/windows/util.zig
@@ -223,7 +223,7 @@ pub fn windowsFindFirstFile(
     dir_path: []const u8,
     find_file_data: *windows.WIN32_FIND_DATAW,
 ) !windows.HANDLE {
-    const dir_path_w = try sliceToPrefixedSuffixedFileW(dir_path, []u16{'\\', '*', 0});
+    const dir_path_w = try sliceToPrefixedSuffixedFileW(dir_path, []u16{ '\\', '*', 0 });
     const handle = windows.FindFirstFileW(&dir_path_w, find_file_data);
 
     if (handle == windows.INVALID_HANDLE_VALUE) {
@@ -278,6 +278,7 @@ pub const WindowsWaitResult = enum {
     Normal,
     Aborted,
     Cancelled,
+    EOF,
 };
 
 pub fn windowsGetQueuedCompletionStatus(completion_port: windows.HANDLE, bytes_transferred_count: *windows.DWORD, lpCompletionKey: *usize, lpOverlapped: *?*windows.OVERLAPPED, dwMilliseconds: windows.DWORD) WindowsWaitResult {
@@ -286,6 +287,7 @@ pub fn windowsGetQueuedCompletionStatus(completion_port: windows.HANDLE, bytes_t
         switch (err) {
             windows.ERROR.ABANDONED_WAIT_0 => return WindowsWaitResult.Aborted,
             windows.ERROR.OPERATION_ABORTED => return WindowsWaitResult.Cancelled,
+            windows.ERROR.HANDLE_EOF => return WindowsWaitResult.EOF,
             else => {
                 if (std.debug.runtime_safety) {
                     std.debug.panic("unexpected error: {}\n", err);
diff --git a/std/pdb.zig b/std/pdb.zig
index 907eddff0..3bb6c9d58 100644
--- a/std/pdb.zig
+++ b/std/pdb.zig
@@ -64,19 +64,35 @@ pub const ModInfo = packed struct {
 };
 
 pub const SectionMapHeader = packed struct {
-  Count: u16,    /// Number of segment descriptors
-  LogCount: u16, /// Number of logical segment descriptors
+    /// Number of segment descriptors
+    Count: u16,
+
+    /// Number of logical segment descriptors
+    LogCount: u16,
 };
 
 pub const SectionMapEntry = packed struct {
-  Flags: u16 ,         /// See the SectionMapEntryFlags enum below.
-  Ovl: u16 ,           /// Logical overlay number
-  Group: u16 ,         /// Group index into descriptor array.
-  Frame: u16 ,
-  SectionName: u16 ,   /// Byte index of segment / group name in string table, or 0xFFFF.
-  ClassName: u16 ,     /// Byte index of class in string table, or 0xFFFF.
-  Offset: u32 ,        /// Byte offset of the logical segment within physical segment.  If group is set in flags, this is the offset of the group.
-  SectionLength: u32 , /// Byte count of the segment or group.
+    /// See the SectionMapEntryFlags enum below.
+    Flags: u16,
+
+    /// Logical overlay number
+    Ovl: u16,
+
+    /// Group index into descriptor array.
+    Group: u16,
+    Frame: u16,
+
+    /// Byte index of segment / group name in string table, or 0xFFFF.
+    SectionName: u16,
+
+    /// Byte index of class in string table, or 0xFFFF.
+    ClassName: u16,
+
+    /// Byte offset of the logical segment within physical segment.  If group is set in flags, this is the offset of the group.
+    Offset: u32,
+
+    /// Byte count of the segment or group.
+    SectionLength: u32,
 };
 
 pub const StreamType = enum(u16) {
@@ -290,13 +306,13 @@ pub const SymbolKind = packed enum(u16) {
 pub const TypeIndex = u32;
 
 pub const ProcSym = packed struct {
-    Parent: u32 ,
-    End: u32 ,
-    Next: u32 ,
-    CodeSize: u32 ,
-    DbgStart: u32 ,
-    DbgEnd: u32 ,
-    FunctionType: TypeIndex ,
+    Parent: u32,
+    End: u32,
+    Next: u32,
+    CodeSize: u32,
+    DbgStart: u32,
+    DbgEnd: u32,
+    FunctionType: TypeIndex,
     CodeOffset: u32,
     Segment: u16,
     Flags: ProcSymFlags,
@@ -315,25 +331,34 @@ pub const ProcSymFlags = packed struct {
     HasOptimizedDebugInfo: bool,
 };
 
-pub const SectionContrSubstreamVersion  = enum(u32) {
-  Ver60 = 0xeffe0000 + 19970605,
-  V2 = 0xeffe0000 + 20140516
+pub const SectionContrSubstreamVersion = enum(u32) {
+    Ver60 = 0xeffe0000 + 19970605,
+    V2 = 0xeffe0000 + 20140516,
 };
 
 pub const RecordPrefix = packed struct {
-    RecordLen: u16, /// Record length, starting from &RecordKind.
-    RecordKind: SymbolKind, /// Record kind enum (SymRecordKind or TypeRecordKind)
+    /// Record length, starting from &RecordKind.
+    RecordLen: u16,
+
+    /// Record kind enum (SymRecordKind or TypeRecordKind)
+    RecordKind: SymbolKind,
 };
 
 pub const LineFragmentHeader = packed struct {
-    RelocOffset: u32, /// Code offset of line contribution.
-    RelocSegment: u16, /// Code segment of line contribution.
+    /// Code offset of line contribution.
+    RelocOffset: u32,
+
+    /// Code segment of line contribution.
+    RelocSegment: u16,
     Flags: LineFlags,
-    CodeSize: u32, /// Code size of this line contribution.
+
+    /// Code size of this line contribution.
+    CodeSize: u32,
 };
 
 pub const LineFlags = packed struct {
-    LF_HaveColumns: bool, /// CV_LINES_HAVE_COLUMNS
+    /// CV_LINES_HAVE_COLUMNS
+    LF_HaveColumns: bool,
     unused: u15,
 };
 
@@ -348,12 +373,14 @@ pub const LineBlockFragmentHeader = packed struct {
     /// table of the actual name.
     NameIndex: u32,
     NumLines: u32,
-    BlockSize: u32, /// code size of block, in bytes
+
+    /// code size of block, in bytes
+    BlockSize: u32,
 };
 
-
 pub const LineNumberEntry = packed struct {
-    Offset: u32, /// Offset to start of code bytes for line number
+    /// Offset to start of code bytes for line number
+    Offset: u32,
     Flags: u32,
 
     /// TODO runtime crash when I make the actual type of Flags this
@@ -371,42 +398,53 @@ pub const ColumnNumberEntry = packed struct {
 
 /// Checksum bytes follow.
 pub const FileChecksumEntryHeader = packed struct {
-    FileNameOffset: u32, /// Byte offset of filename in global string table.
-    ChecksumSize: u8, /// Number of bytes of checksum.
-    ChecksumKind: u8, /// FileChecksumKind
+    /// Byte offset of filename in global string table.
+    FileNameOffset: u32,
+
+    /// Number of bytes of checksum.
+    ChecksumSize: u8,
+
+    /// FileChecksumKind
+    ChecksumKind: u8,
 };
 
 pub const DebugSubsectionKind = packed enum(u32) {
-  None = 0,
-  Symbols = 0xf1,
-  Lines = 0xf2,
-  StringTable = 0xf3,
-  FileChecksums = 0xf4,
-  FrameData = 0xf5,
-  InlineeLines = 0xf6,
-  CrossScopeImports = 0xf7,
-  CrossScopeExports = 0xf8,
+    None = 0,
+    Symbols = 0xf1,
+    Lines = 0xf2,
+    StringTable = 0xf3,
+    FileChecksums = 0xf4,
+    FrameData = 0xf5,
+    InlineeLines = 0xf6,
+    CrossScopeImports = 0xf7,
+    CrossScopeExports = 0xf8,
 
-  // These appear to relate to .Net assembly info.
-  ILLines = 0xf9,
-  FuncMDTokenMap = 0xfa,
-  TypeMDTokenMap = 0xfb,
-  MergedAssemblyInput = 0xfc,
+    // These appear to relate to .Net assembly info.
+    ILLines = 0xf9,
+    FuncMDTokenMap = 0xfa,
+    TypeMDTokenMap = 0xfb,
+    MergedAssemblyInput = 0xfc,
 
-  CoffSymbolRVA = 0xfd,
+    CoffSymbolRVA = 0xfd,
 };
 
-
 pub const DebugSubsectionHeader = packed struct {
-    Kind: DebugSubsectionKind, /// codeview::DebugSubsectionKind enum
-    Length: u32, /// number of bytes occupied by this record.
+    /// codeview::DebugSubsectionKind enum
+    Kind: DebugSubsectionKind,
+
+    /// number of bytes occupied by this record.
+    Length: u32,
 };
 
-
 pub const PDBStringTableHeader = packed struct {
-    Signature: u32, /// PDBStringTableSignature
-    HashVersion: u32, /// 1 or 2
-    ByteSize: u32, /// Number of bytes of names buffer.
+    /// PDBStringTableSignature
+    Signature: u32,
+
+    /// 1 or 2
+    HashVersion: u32,
+
+    /// Number of bytes of names buffer.
+    ByteSize: u32,
 };
 
 pub const Pdb = struct {
@@ -456,7 +494,7 @@ const Msf = struct {
         switch (superblock.BlockSize) {
             // llvm only supports 4096 but we can handle any of these values
             512, 1024, 2048, 4096 => {},
-            else => return error.InvalidDebugInfo
+            else => return error.InvalidDebugInfo,
         }
 
         if (superblock.NumBlocks * superblock.BlockSize != try file.getEndPos())
@@ -536,7 +574,6 @@ const SuperBlock = packed struct {
     /// The number of ulittle32_t’s in this array is given by
     /// ceil(NumDirectoryBytes / BlockSize).
     BlockMapAddr: u32,
-
 };
 
 const MsfStream = struct {
@@ -552,14 +589,12 @@ const MsfStream = struct {
     pub const Stream = io.InStream(Error);
 
     fn init(block_size: u32, block_count: u32, pos: usize, file: os.File, allocator: *mem.Allocator) !MsfStream {
-        var stream = MsfStream {
+        var stream = MsfStream{
             .in_file = file,
             .pos = 0,
             .blocks = try allocator.alloc(u32, block_count),
             .block_size = block_size,
-            .stream = Stream {
-                .readFn = readFn,
-            },
+            .stream = Stream{ .readFn = readFn },
         };
 
         var file_stream = io.FileInStream.init(file);
@@ -597,7 +632,7 @@ const MsfStream = struct {
 
         var size: usize = 0;
         for (buffer) |*byte| {
-            byte.* = try in.readByte();           
+            byte.* = try in.readByte();
 
             offset += 1;
             size += 1;

From 869475c110c583e641c26f9dfe670132d2e0901e Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 12 Sep 2018 14:50:26 -0400
Subject: [PATCH 28/46] ci: skip release-small tests to save time

we keep hitting the limit on how long CI tests take to run.
---
 build.zig                    | 27 ++++++++++++++++++++-------
 ci/appveyor/build_script.bat |  2 +-
 ci/travis_linux_script       |  2 +-
 ci/travis_osx_script         |  2 +-
 4 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/build.zig b/build.zig
index e88637713..0bb4808dc 100644
--- a/build.zig
+++ b/build.zig
@@ -62,6 +62,9 @@ pub fn build(b: *Builder) !void {
     b.default_step.dependOn(&exe.step);
 
     const skip_release = b.option(bool, "skip-release", "Main test suite skips release builds") orelse false;
+    const skip_release_small = b.option(bool, "skip-release-small", "Main test suite skips release-small builds") orelse skip_release;
+    const skip_release_fast = b.option(bool, "skip-release-fast", "Main test suite skips release-fast builds") orelse skip_release;
+    const skip_release_safe = b.option(bool, "skip-release-safe", "Main test suite skips release-safe builds") orelse skip_release;
     const skip_self_hosted = b.option(bool, "skip-self-hosted", "Main test suite skips building self hosted compiler") orelse false;
     if (!skip_self_hosted) {
         test_step.dependOn(&exe.step);
@@ -83,13 +86,23 @@ pub fn build(b: *Builder) !void {
         test_step.dependOn(test_stage2_step);
     }
 
-    const all_modes = []builtin.Mode{
-        builtin.Mode.Debug,
-        builtin.Mode.ReleaseSafe,
-        builtin.Mode.ReleaseFast,
-        builtin.Mode.ReleaseSmall,
-    };
-    const modes = if (skip_release) []builtin.Mode{builtin.Mode.Debug} else all_modes;
+    var chosen_modes: [4]builtin.Mode = undefined;
+    var chosen_mode_index: usize = 0;
+    chosen_modes[chosen_mode_index] = builtin.Mode.Debug;
+    chosen_mode_index += 1;
+    if (!skip_release_safe) {
+        chosen_modes[chosen_mode_index] = builtin.Mode.ReleaseSafe;
+        chosen_mode_index += 1;
+    }
+    if (!skip_release_fast) {
+        chosen_modes[chosen_mode_index] = builtin.Mode.ReleaseFast;
+        chosen_mode_index += 1;
+    }
+    if (!skip_release_small) {
+        chosen_modes[chosen_mode_index] = builtin.Mode.ReleaseSmall;
+        chosen_mode_index += 1;
+    }
+    const modes = chosen_modes[0..chosen_mode_index];
 
     test_step.dependOn(tests.addPkgTests(b, test_filter, "test/behavior.zig", "behavior", "Run the behavior tests", modes));
 
diff --git a/ci/appveyor/build_script.bat b/ci/appveyor/build_script.bat
index 9aee7a7bf..f2d217894 100644
--- a/ci/appveyor/build_script.bat
+++ b/ci/appveyor/build_script.bat
@@ -23,4 +23,4 @@ cd %ZIGBUILDDIR%
 cmake.exe .. -Thost=x64 -G"Visual Studio 14 2015 Win64" "-DCMAKE_INSTALL_PREFIX=%ZIGBUILDDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release || exit /b
 msbuild /p:Configuration=Release INSTALL.vcxproj || exit /b
 
-bin\zig.exe build --build-file ..\build.zig test || exit /b
+bin\zig.exe build --build-file ..\build.zig test -Dskip-release-small || exit /b
diff --git a/ci/travis_linux_script b/ci/travis_linux_script
index f823429a4..8c8eafe1d 100755
--- a/ci/travis_linux_script
+++ b/ci/travis_linux_script
@@ -10,7 +10,7 @@ mkdir build
 cd build
 cmake .. -DCMAKE_BUILD_TYPE=Release
 make -j2 install
-./zig build --build-file ../build.zig test
+./zig build --build-file ../build.zig test -Dskip-release-small
 
 if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then
   mkdir $TRAVIS_BUILD_DIR/artifacts
diff --git a/ci/travis_osx_script b/ci/travis_osx_script
index 7fa54d26f..1c9d80998 100755
--- a/ci/travis_osx_script
+++ b/ci/travis_osx_script
@@ -9,4 +9,4 @@ cmake .. -DCMAKE_PREFIX_PATH=/usr/local/opt/llvm@6/ -DCMAKE_BUILD_TYPE=Release
 make VERBOSE=1
 make install
 
-./zig build --build-file ../build.zig test
+./zig build --build-file ../build.zig test -Dskip-release-small

From 0dbbc91bc92fa23f7290e8bee2d9114b2c41d218 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 12 Sep 2018 17:26:51 -0400
Subject: [PATCH 29/46] docs: langref is now javascript-free

---
 doc/docgen.zig      | 226 +++++++++++++++++++++++++++++++++++++++++++-
 doc/langref.html.in | 184 ++++++++----------------------------
 2 files changed, 263 insertions(+), 147 deletions(-)

diff --git a/doc/docgen.zig b/doc/docgen.zig
index 5d77a1ea1..0143de5b8 100644
--- a/doc/docgen.zig
+++ b/doc/docgen.zig
@@ -571,6 +571,11 @@ fn escapeHtml(allocator: *mem.Allocator, input: []const u8) ![]u8 {
 
     var buf_adapter = io.BufferOutStream.init(&buf);
     var out = &buf_adapter.stream;
+    try writeEscaped(out, input);
+    return buf.toOwnedSlice();
+}
+
+fn writeEscaped(out: var, input: []const u8) !void {
     for (input) |c| {
         try switch (c) {
             '&' => out.write("&"),
@@ -580,7 +585,6 @@ fn escapeHtml(allocator: *mem.Allocator, input: []const u8) ![]u8 {
             else => out.writeByte(c),
         };
     }
-    return buf.toOwnedSlice();
 }
 
 //#define VT_RED "\x1b[31;1m"
@@ -687,6 +691,223 @@ fn termColor(allocator: *mem.Allocator, input: []const u8) ![]u8 {
     return buf.toOwnedSlice();
 }
 
+const builtin_types = [][]const u8{
+    "f16", "f32", "f64", "f128", "c_longdouble", "c_short",
+    "c_ushort", "c_int", "c_uint", "c_long", "c_ulong", "c_longlong",
+    "c_ulonglong", "c_char", "c_void", "void", "bool", "isize",
+    "usize", "noreturn", "type", "error", "comptime_int", "comptime_float",
+};
+
+fn isType(name: []const u8) bool {
+    for (builtin_types) |t| {
+        if (mem.eql(u8, t, name))
+            return true;
+    }
+    return false;
+}
+
+fn tokenizeAndPrint(allocator: *mem.Allocator, out: var, src: []const u8) !void {
+    try out.write("
");
+    var tokenizer = std.zig.Tokenizer.init(src);
+    var index: usize = 0;
+    var next_tok_is_fn = false;
+    while (true) {
+        const prev_tok_was_fn = next_tok_is_fn;
+        next_tok_is_fn = false;
+
+        const token = tokenizer.next();
+        try writeEscaped(out, src[index..token.start]);
+        switch (token.id) {
+            std.zig.Token.Id.Eof => break,
+
+            std.zig.Token.Id.Keyword_align,
+            std.zig.Token.Id.Keyword_and,
+            std.zig.Token.Id.Keyword_asm,
+            std.zig.Token.Id.Keyword_async,
+            std.zig.Token.Id.Keyword_await,
+            std.zig.Token.Id.Keyword_break,
+            std.zig.Token.Id.Keyword_cancel,
+            std.zig.Token.Id.Keyword_catch,
+            std.zig.Token.Id.Keyword_comptime,
+            std.zig.Token.Id.Keyword_const,
+            std.zig.Token.Id.Keyword_continue,
+            std.zig.Token.Id.Keyword_defer,
+            std.zig.Token.Id.Keyword_else,
+            std.zig.Token.Id.Keyword_enum,
+            std.zig.Token.Id.Keyword_errdefer,
+            std.zig.Token.Id.Keyword_error,
+            std.zig.Token.Id.Keyword_export,
+            std.zig.Token.Id.Keyword_extern,
+            std.zig.Token.Id.Keyword_for,
+            std.zig.Token.Id.Keyword_if,
+            std.zig.Token.Id.Keyword_inline,
+            std.zig.Token.Id.Keyword_nakedcc,
+            std.zig.Token.Id.Keyword_noalias,
+            std.zig.Token.Id.Keyword_or,
+            std.zig.Token.Id.Keyword_orelse,
+            std.zig.Token.Id.Keyword_packed,
+            std.zig.Token.Id.Keyword_promise,
+            std.zig.Token.Id.Keyword_pub,
+            std.zig.Token.Id.Keyword_resume,
+            std.zig.Token.Id.Keyword_return,
+            std.zig.Token.Id.Keyword_section,
+            std.zig.Token.Id.Keyword_stdcallcc,
+            std.zig.Token.Id.Keyword_struct,
+            std.zig.Token.Id.Keyword_suspend,
+            std.zig.Token.Id.Keyword_switch,
+            std.zig.Token.Id.Keyword_test,
+            std.zig.Token.Id.Keyword_try,
+            std.zig.Token.Id.Keyword_union,
+            std.zig.Token.Id.Keyword_unreachable,
+            std.zig.Token.Id.Keyword_use,
+            std.zig.Token.Id.Keyword_var,
+            std.zig.Token.Id.Keyword_volatile,
+            std.zig.Token.Id.Keyword_while,
+            => {
+                try out.write("");
+                try writeEscaped(out, src[token.start..token.end]);
+                try out.write("");
+            },
+
+            std.zig.Token.Id.Keyword_fn => {
+                try out.write("");
+                try writeEscaped(out, src[token.start..token.end]);
+                try out.write("");
+                next_tok_is_fn = true;
+            },
+
+            std.zig.Token.Id.Keyword_undefined,
+            std.zig.Token.Id.Keyword_null,
+            std.zig.Token.Id.Keyword_true,
+            std.zig.Token.Id.Keyword_false,
+            std.zig.Token.Id.Keyword_this,
+            => {
+                try out.write("");
+                try writeEscaped(out, src[token.start..token.end]);
+                try out.write("");
+            },
+
+            std.zig.Token.Id.StringLiteral,
+            std.zig.Token.Id.MultilineStringLiteralLine,
+            std.zig.Token.Id.CharLiteral,
+            => {
+                try out.write("");
+                try writeEscaped(out, src[token.start..token.end]);
+                try out.write("");
+            },
+
+            std.zig.Token.Id.Builtin => {
+                try out.write("");
+                try writeEscaped(out, src[token.start..token.end]);
+                try out.write("");
+            },
+
+            std.zig.Token.Id.LineComment,
+            std.zig.Token.Id.DocComment,
+            => {
+                try out.write("");
+                try writeEscaped(out, src[token.start..token.end]);
+                try out.write("");
+            },
+
+            std.zig.Token.Id.Identifier => {
+                if (prev_tok_was_fn) {
+                    try out.write("");
+                    try writeEscaped(out, src[token.start..token.end]);
+                    try out.write("");
+                } else {
+                    const is_int = blk: {
+                        if (src[token.start] != 'i' and src[token.start] != 'u')
+                            break :blk false;
+                        var i = token.start + 1;
+                        if (i == token.end)
+                            break :blk false;
+                        while (i != token.end) : (i += 1) {
+                            if (src[i] < '0' or src[i] > '9')
+                                break :blk false;
+                        }
+                        break :blk true;
+                    };
+                    if (is_int or isType(src[token.start..token.end])) {
+                        try out.write("");
+                        try writeEscaped(out, src[token.start..token.end]);
+                        try out.write("");
+                    } else {
+                        try writeEscaped(out, src[token.start..token.end]);
+                    }
+                }
+            },
+
+            std.zig.Token.Id.IntegerLiteral,
+            std.zig.Token.Id.FloatLiteral,
+            => {
+                try out.write("");
+                try writeEscaped(out, src[token.start..token.end]);
+                try out.write("");
+            },
+
+            std.zig.Token.Id.Bang,
+            std.zig.Token.Id.Pipe,
+            std.zig.Token.Id.PipePipe,
+            std.zig.Token.Id.PipeEqual,
+            std.zig.Token.Id.Equal,
+            std.zig.Token.Id.EqualEqual,
+            std.zig.Token.Id.EqualAngleBracketRight,
+            std.zig.Token.Id.BangEqual,
+            std.zig.Token.Id.LParen,
+            std.zig.Token.Id.RParen,
+            std.zig.Token.Id.Semicolon,
+            std.zig.Token.Id.Percent,
+            std.zig.Token.Id.PercentEqual,
+            std.zig.Token.Id.LBrace,
+            std.zig.Token.Id.RBrace,
+            std.zig.Token.Id.LBracket,
+            std.zig.Token.Id.RBracket,
+            std.zig.Token.Id.Period,
+            std.zig.Token.Id.Ellipsis2,
+            std.zig.Token.Id.Ellipsis3,
+            std.zig.Token.Id.Caret,
+            std.zig.Token.Id.CaretEqual,
+            std.zig.Token.Id.Plus,
+            std.zig.Token.Id.PlusPlus,
+            std.zig.Token.Id.PlusEqual,
+            std.zig.Token.Id.PlusPercent,
+            std.zig.Token.Id.PlusPercentEqual,
+            std.zig.Token.Id.Minus,
+            std.zig.Token.Id.MinusEqual,
+            std.zig.Token.Id.MinusPercent,
+            std.zig.Token.Id.MinusPercentEqual,
+            std.zig.Token.Id.Asterisk,
+            std.zig.Token.Id.AsteriskEqual,
+            std.zig.Token.Id.AsteriskAsterisk,
+            std.zig.Token.Id.AsteriskPercent,
+            std.zig.Token.Id.AsteriskPercentEqual,
+            std.zig.Token.Id.Arrow,
+            std.zig.Token.Id.Colon,
+            std.zig.Token.Id.Slash,
+            std.zig.Token.Id.SlashEqual,
+            std.zig.Token.Id.Comma,
+            std.zig.Token.Id.Ampersand,
+            std.zig.Token.Id.AmpersandEqual,
+            std.zig.Token.Id.QuestionMark,
+            std.zig.Token.Id.AngleBracketLeft,
+            std.zig.Token.Id.AngleBracketLeftEqual,
+            std.zig.Token.Id.AngleBracketAngleBracketLeft,
+            std.zig.Token.Id.AngleBracketAngleBracketLeftEqual,
+            std.zig.Token.Id.AngleBracketRight,
+            std.zig.Token.Id.AngleBracketRightEqual,
+            std.zig.Token.Id.AngleBracketAngleBracketRight,
+            std.zig.Token.Id.AngleBracketAngleBracketRightEqual,
+            std.zig.Token.Id.Tilde,
+            std.zig.Token.Id.BracketStarBracket,
+            std.zig.Token.Id.Invalid,
+            => try writeEscaped(out, src[token.start..token.end]),
+        }
+        index = token.end;
+    }
+    try out.write("
");
+}
+
 fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var, zig_exe: []const u8) !void {
     var code_progress_index: usize = 0;
 
@@ -732,11 +953,10 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
 
                 const raw_source = tokenizer.buffer[code.source_token.start..code.source_token.end];
                 const trimmed_raw_source = mem.trim(u8, raw_source, " \n");
-                const escaped_source = try escapeHtml(allocator, trimmed_raw_source);
                 if (!code.is_inline) {
                     try out.print("
{}.zig
", code.name);
                 }
-                try out.print("
{}
", escaped_source);
+                try tokenizeAndPrint(allocator, out, trimmed_raw_source);
                 const name_plus_ext = try std.fmt.allocPrint(allocator, "{}.zig", code.name);
                 const tmp_source_file_name = try os.path.join(allocator, tmp_dir_name, name_plus_ext);
                 try io.writeFile(tmp_source_file_name, trimmed_raw_source);
diff --git a/doc/langref.html.in b/doc/langref.html.in
index 8fa80996a..b13427da9 100644
--- a/doc/langref.html.in
+++ b/doc/langref.html.in
@@ -5,9 +5,6 @@
     
     Documentation - The Zig Programming Language
     
-