From 2d4b95900e7d7273dbd3ce6b7b9a33d2a19779da Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 4 Sep 2018 18:15:27 -0400 Subject: [PATCH 01/20] stage1: import blake2b implementation from master branch of blake2 reference implementation 320c325437539ae91091ce62efec1913cd8093c2 --- CMakeLists.txt | 1 + src/blake2.h | 196 ++++++++++++++++++ src/blake2b.cpp | 539 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 736 insertions(+) create mode 100644 src/blake2.h create mode 100644 src/blake2b.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c3ee37ea89cc57becc8e8ab4a89636431191f89f..fbf5b03eac0543ab9cf705577597ca37075cfc69 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -407,6 +407,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/ast_render.cpp" "${CMAKE_SOURCE_DIR}/src/bigfloat.cpp" "${CMAKE_SOURCE_DIR}/src/bigint.cpp" + "${CMAKE_SOURCE_DIR}/src/blake2b.cpp" "${CMAKE_SOURCE_DIR}/src/buffer.cpp" "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" diff --git a/src/blake2.h b/src/blake2.h new file mode 100644 index 0000000000000000000000000000000000000000..6420c5367a2ebb09dd055381cebf5f2d20c82951 --- /dev/null +++ b/src/blake2.h @@ -0,0 +1,196 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#ifndef BLAKE2_H +#define BLAKE2_H + +#include +#include + +#if defined(_MSC_VER) +#define BLAKE2_PACKED(x) __pragma(pack(push, 1)) x __pragma(pack(pop)) +#else +#define BLAKE2_PACKED(x) x __attribute__((packed)) +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + + enum blake2s_constant + { + BLAKE2S_BLOCKBYTES = 64, + BLAKE2S_OUTBYTES = 32, + BLAKE2S_KEYBYTES = 32, + BLAKE2S_SALTBYTES = 8, + BLAKE2S_PERSONALBYTES = 8 + }; + + enum blake2b_constant + { + BLAKE2B_BLOCKBYTES = 128, + BLAKE2B_OUTBYTES = 64, + BLAKE2B_KEYBYTES = 64, + BLAKE2B_SALTBYTES = 16, + BLAKE2B_PERSONALBYTES = 16 + }; + + typedef struct blake2s_state__ + { + uint32_t h[8]; + uint32_t t[2]; + uint32_t f[2]; + uint8_t buf[BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2s_state; + + typedef struct blake2b_state__ + { + uint64_t h[8]; + uint64_t t[2]; + uint64_t f[2]; + uint8_t buf[BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2b_state; + + typedef struct blake2sp_state__ + { + blake2s_state S[8][1]; + blake2s_state R[1]; + uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2sp_state; + + typedef struct blake2bp_state__ + { + blake2b_state S[4][1]; + blake2b_state R[1]; + uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2bp_state; + + + BLAKE2_PACKED(struct blake2s_param__ + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint16_t xof_length; /* 14 */ + uint8_t node_depth; /* 15 */ + uint8_t inner_length; /* 16 */ + /* uint8_t reserved[0]; */ + uint8_t salt[BLAKE2S_SALTBYTES]; /* 24 */ + uint8_t personal[BLAKE2S_PERSONALBYTES]; /* 32 */ + }); + + typedef struct blake2s_param__ blake2s_param; + + BLAKE2_PACKED(struct blake2b_param__ + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint32_t xof_length; /* 16 */ + uint8_t node_depth; /* 17 */ + uint8_t inner_length; /* 18 */ + uint8_t reserved[14]; /* 32 */ + uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ + uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ + }); + + typedef struct blake2b_param__ blake2b_param; + + typedef struct blake2xs_state__ + { + blake2s_state S[1]; + blake2s_param P[1]; + } blake2xs_state; + + typedef struct blake2xb_state__ + { + blake2b_state S[1]; + blake2b_param P[1]; + } blake2xb_state; + + /* Padded structs result in a compile-time error */ + enum { + BLAKE2_DUMMY_1 = 1/(sizeof(blake2s_param) == BLAKE2S_OUTBYTES), + BLAKE2_DUMMY_2 = 1/(sizeof(blake2b_param) == BLAKE2B_OUTBYTES) + }; + + /* Streaming API */ + int blake2s_init( blake2s_state *S, size_t outlen ); + int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); + int blake2s_update( blake2s_state *S, const void *in, size_t inlen ); + int blake2s_final( blake2s_state *S, void *out, size_t outlen ); + + int blake2b_init( blake2b_state *S, size_t outlen ); + int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); + int blake2b_update( blake2b_state *S, const void *in, size_t inlen ); + int blake2b_final( blake2b_state *S, void *out, size_t outlen ); + + int blake2sp_init( blake2sp_state *S, size_t outlen ); + int blake2sp_init_key( blake2sp_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2sp_update( blake2sp_state *S, const void *in, size_t inlen ); + int blake2sp_final( blake2sp_state *S, void *out, size_t outlen ); + + int blake2bp_init( blake2bp_state *S, size_t outlen ); + int blake2bp_init_key( blake2bp_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2bp_update( blake2bp_state *S, const void *in, size_t inlen ); + int blake2bp_final( blake2bp_state *S, void *out, size_t outlen ); + + /* Variable output length API */ + int blake2xs_init( blake2xs_state *S, const size_t outlen ); + int blake2xs_init_key( blake2xs_state *S, const size_t outlen, const void *key, size_t keylen ); + int blake2xs_update( blake2xs_state *S, const void *in, size_t inlen ); + int blake2xs_final(blake2xs_state *S, void *out, size_t outlen); + + int blake2xb_init( blake2xb_state *S, const size_t outlen ); + int blake2xb_init_key( blake2xb_state *S, const size_t outlen, const void *key, size_t keylen ); + int blake2xb_update( blake2xb_state *S, const void *in, size_t inlen ); + int blake2xb_final(blake2xb_state *S, void *out, size_t outlen); + + /* Simple API */ + int blake2s( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + int blake2sp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2bp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + int blake2xs( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2xb( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + /* This is simply an alias for blake2b */ + int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + +#if defined(__cplusplus) +} +#endif + +#endif + diff --git a/src/blake2b.cpp b/src/blake2b.cpp new file mode 100644 index 0000000000000000000000000000000000000000..600f951b9b5e8afe5c59bd55153b0f040c63822e --- /dev/null +++ b/src/blake2b.cpp @@ -0,0 +1,539 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ + +#include +#include +#include + +#include "blake2.h" +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#ifndef BLAKE2_IMPL_H +#define BLAKE2_IMPL_H + +#include +#include + +#if !defined(__cplusplus) && (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L) + #if defined(_MSC_VER) + #define BLAKE2_INLINE __inline + #elif defined(__GNUC__) + #define BLAKE2_INLINE __inline__ + #else + #define BLAKE2_INLINE + #endif +#else + #define BLAKE2_INLINE inline +#endif + +static BLAKE2_INLINE uint32_t load32( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint32_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint32_t )( p[0] ) << 0) | + (( uint32_t )( p[1] ) << 8) | + (( uint32_t )( p[2] ) << 16) | + (( uint32_t )( p[3] ) << 24) ; +#endif +} + +static BLAKE2_INLINE uint64_t load64( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint64_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint64_t )( p[0] ) << 0) | + (( uint64_t )( p[1] ) << 8) | + (( uint64_t )( p[2] ) << 16) | + (( uint64_t )( p[3] ) << 24) | + (( uint64_t )( p[4] ) << 32) | + (( uint64_t )( p[5] ) << 40) | + (( uint64_t )( p[6] ) << 48) | + (( uint64_t )( p[7] ) << 56) ; +#endif +} + +static BLAKE2_INLINE uint16_t load16( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint16_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return ( uint16_t )((( uint32_t )( p[0] ) << 0) | + (( uint32_t )( p[1] ) << 8)); +#endif +} + +static BLAKE2_INLINE void store16( void *dst, uint16_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +#endif +} + +static BLAKE2_INLINE void store32( void *dst, uint32_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); +#endif +} + +static BLAKE2_INLINE void store64( void *dst, uint64_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); + p[4] = (uint8_t)(w >> 32); + p[5] = (uint8_t)(w >> 40); + p[6] = (uint8_t)(w >> 48); + p[7] = (uint8_t)(w >> 56); +#endif +} + +static BLAKE2_INLINE uint64_t load48( const void *src ) +{ + const uint8_t *p = ( const uint8_t * )src; + return (( uint64_t )( p[0] ) << 0) | + (( uint64_t )( p[1] ) << 8) | + (( uint64_t )( p[2] ) << 16) | + (( uint64_t )( p[3] ) << 24) | + (( uint64_t )( p[4] ) << 32) | + (( uint64_t )( p[5] ) << 40) ; +} + +static BLAKE2_INLINE void store48( void *dst, uint64_t w ) +{ + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); + p[4] = (uint8_t)(w >> 32); + p[5] = (uint8_t)(w >> 40); +} + +static BLAKE2_INLINE uint32_t rotr32( const uint32_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 32 - c ) ); +} + +static BLAKE2_INLINE uint64_t rotr64( const uint64_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 64 - c ) ); +} + +/* prevents compiler optimizing out memset() */ +static BLAKE2_INLINE void secure_zero_memory(void *v, size_t n) +{ + static void *(*const volatile memset_v)(void *, int, size_t) = &memset; + memset_v(v, 0, n); +} + +#endif + +static const uint64_t blake2b_IV[8] = +{ + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL +}; + +static const uint8_t blake2b_sigma[12][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } +}; + + +static void blake2b_set_lastnode( blake2b_state *S ) +{ + S->f[1] = (uint64_t)-1; +} + +/* Some helper functions, not necessarily useful */ +static int blake2b_is_lastblock( const blake2b_state *S ) +{ + return S->f[0] != 0; +} + +static void blake2b_set_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_set_lastnode( S ); + + S->f[0] = (uint64_t)-1; +} + +static void blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) +{ + S->t[0] += inc; + S->t[1] += ( S->t[0] < inc ); +} + +static void blake2b_init0( blake2b_state *S ) +{ + size_t i; + memset( S, 0, sizeof( blake2b_state ) ); + + for( i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; +} + +/* init xors IV with input parameter block */ +int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) +{ + const uint8_t *p = ( const uint8_t * )( P ); + size_t i; + + blake2b_init0( S ); + + /* IV XOR ParamBlock */ + for( i = 0; i < 8; ++i ) + S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); + + S->outlen = P->digest_length; + return 0; +} + + + +int blake2b_init( blake2b_state *S, size_t outlen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + P->digest_length = (uint8_t)outlen; + P->key_length = 0; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store32( &P->node_offset, 0 ); + store32( &P->xof_length, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + return blake2b_init_param( S, P ); +} + + +int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; + + P->digest_length = (uint8_t)outlen; + P->key_length = (uint8_t)keylen; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store32( &P->node_offset, 0 ); + store32( &P->xof_length, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + + if( blake2b_init_param( S, P ) < 0 ) return -1; + + { + uint8_t block[BLAKE2B_BLOCKBYTES]; + memset( block, 0, BLAKE2B_BLOCKBYTES ); + memcpy( block, key, keylen ); + blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); + secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ + } + return 0; +} + +#define G(r,i,a,b,c,d) \ + do { \ + a = a + b + m[blake2b_sigma[r][2*i+0]]; \ + d = rotr64(d ^ a, 32); \ + c = c + d; \ + b = rotr64(b ^ c, 24); \ + a = a + b + m[blake2b_sigma[r][2*i+1]]; \ + d = rotr64(d ^ a, 16); \ + c = c + d; \ + b = rotr64(b ^ c, 63); \ + } while(0) + +#define ROUND(r) \ + do { \ + G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ + G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ + G(r,2,v[ 2],v[ 6],v[10],v[14]); \ + G(r,3,v[ 3],v[ 7],v[11],v[15]); \ + G(r,4,v[ 0],v[ 5],v[10],v[15]); \ + G(r,5,v[ 1],v[ 6],v[11],v[12]); \ + G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ + G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ + } while(0) + +static void blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) +{ + uint64_t m[16]; + uint64_t v[16]; + size_t i; + + for( i = 0; i < 16; ++i ) { + m[i] = load64( block + i * sizeof( m[i] ) ); + } + + for( i = 0; i < 8; ++i ) { + v[i] = S->h[i]; + } + + v[ 8] = blake2b_IV[0]; + v[ 9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = blake2b_IV[4] ^ S->t[0]; + v[13] = blake2b_IV[5] ^ S->t[1]; + v[14] = blake2b_IV[6] ^ S->f[0]; + v[15] = blake2b_IV[7] ^ S->f[1]; + + ROUND( 0 ); + ROUND( 1 ); + ROUND( 2 ); + ROUND( 3 ); + ROUND( 4 ); + ROUND( 5 ); + ROUND( 6 ); + ROUND( 7 ); + ROUND( 8 ); + ROUND( 9 ); + ROUND( 10 ); + ROUND( 11 ); + + for( i = 0; i < 8; ++i ) { + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + } +} + +#undef G +#undef ROUND + +int blake2b_update( blake2b_state *S, const void *pin, size_t inlen ) +{ + const unsigned char * in = (const unsigned char *)pin; + if( inlen > 0 ) + { + size_t left = S->buflen; + size_t fill = BLAKE2B_BLOCKBYTES - left; + if( inlen > fill ) + { + S->buflen = 0; + memcpy( S->buf + left, in, fill ); /* Fill buffer */ + blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); + blake2b_compress( S, S->buf ); /* Compress */ + in += fill; inlen -= fill; + while(inlen > BLAKE2B_BLOCKBYTES) { + blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); + blake2b_compress( S, in ); + in += BLAKE2B_BLOCKBYTES; + inlen -= BLAKE2B_BLOCKBYTES; + } + } + memcpy( S->buf + S->buflen, in, inlen ); + S->buflen += inlen; + } + return 0; +} + +int blake2b_final( blake2b_state *S, void *out, size_t outlen ) +{ + uint8_t buffer[BLAKE2B_OUTBYTES] = {0}; + size_t i; + + if( out == NULL || outlen < S->outlen ) + return -1; + + if( blake2b_is_lastblock( S ) ) + return -1; + + blake2b_increment_counter( S, S->buflen ); + blake2b_set_lastblock( S ); + memset( S->buf + S->buflen, 0, BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ + blake2b_compress( S, S->buf ); + + for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ + store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); + + memcpy( out, buffer, S->outlen ); + secure_zero_memory(buffer, sizeof(buffer)); + return 0; +} + +/* inlen, at least, should be uint64_t. Others can be size_t. */ +int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) +{ + blake2b_state S[1]; + + /* Verify parameters */ + if ( NULL == in && inlen > 0 ) return -1; + + if ( NULL == out ) return -1; + + if( NULL == key && keylen > 0 ) return -1; + + if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; + + if( keylen > BLAKE2B_KEYBYTES ) return -1; + + if( keylen > 0 ) + { + if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; + } + else + { + if( blake2b_init( S, outlen ) < 0 ) return -1; + } + + blake2b_update( S, ( const uint8_t * )in, inlen ); + blake2b_final( S, out, outlen ); + return 0; +} + +int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) { + return blake2b(out, outlen, in, inlen, key, keylen); +} + +#if defined(SUPERCOP) +int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) +{ + return blake2b( out, BLAKE2B_OUTBYTES, in, inlen, NULL, 0 ); +} +#endif + +#if defined(BLAKE2B_SELFTEST) +#include +#include "blake2-kat.h" +int main( void ) +{ + uint8_t key[BLAKE2B_KEYBYTES]; + uint8_t buf[BLAKE2_KAT_LENGTH]; + size_t i, step; + + for( i = 0; i < BLAKE2B_KEYBYTES; ++i ) + key[i] = ( uint8_t )i; + + for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) + buf[i] = ( uint8_t )i; + + /* Test simple API */ + for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) + { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b( hash, BLAKE2B_OUTBYTES, buf, i, key, BLAKE2B_KEYBYTES ); + + if( 0 != memcmp( hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES ) ) + { + goto fail; + } + } + + /* Test streaming API */ + for(step = 1; step < BLAKE2B_BLOCKBYTES; ++step) { + for (i = 0; i < BLAKE2_KAT_LENGTH; ++i) { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b_state S; + uint8_t * p = buf; + size_t mlen = i; + int err = 0; + + if( (err = blake2b_init_key(&S, BLAKE2B_OUTBYTES, key, BLAKE2B_KEYBYTES)) < 0 ) { + goto fail; + } + + while (mlen >= step) { + if ( (err = blake2b_update(&S, p, step)) < 0 ) { + goto fail; + } + mlen -= step; + p += step; + } + if ( (err = blake2b_update(&S, p, mlen)) < 0) { + goto fail; + } + if ( (err = blake2b_final(&S, hash, BLAKE2B_OUTBYTES)) < 0) { + goto fail; + } + + if (0 != memcmp(hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES)) { + goto fail; + } + } + } + + puts( "ok" ); + return 0; +fail: + puts("error"); + return -1; +} +#endif + -- 2.54.0 From 97c9f61db47ddba43a0db562e13773514875fa6a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Sep 2018 00:34:46 -0400 Subject: [PATCH 02/20] start creating a hash of input parameters See #1416 --- src/all_types.hpp | 30 ++++---- src/codegen.cpp | 175 ++++++++++++++++++++++++++++++++++++++++++---- src/codegen.hpp | 1 - src/link.cpp | 2 - src/main.cpp | 1 - 5 files changed, 179 insertions(+), 30 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index dbaa3b5467e4343c719d149b96e8b0a17abd8088..0b0de587916b888843151392c4354c1cac967e29 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1543,23 +1543,17 @@ struct LinkLib { bool provided_explicitly; }; +// When adding fields, check if they should be added to the hash computation in build_with_cache struct CodeGen { + //////////////////////////// Runtime State LLVMModuleRef module; ZigList errors; LLVMBuilderRef builder; ZigLLVMDIBuilder *dbuilder; ZigLLVMDICompileUnit *compile_unit; ZigLLVMDIFile *compile_unit_file; - - ZigList link_libs_list; LinkLib *libc_link_lib; - // add -framework [name] args to linker - ZigList darwin_frameworks; - // add -rpath [name] args to linker - ZigList rpath_list; - - // reminder: hash tables must be initialized before use HashMap import_table; HashMap builtin_fn_table; @@ -1575,7 +1569,6 @@ struct CodeGen { HashMap string_literals_table; HashMap type_info_cache; - ZigList import_queue; size_t import_queue_index; ZigList resolve_queue; @@ -1620,7 +1613,20 @@ struct CodeGen { ZigType *entry_promise; } builtin_types; + //////////////////////////// Participates in Input Parameter Cache Hash + ZigList link_libs_list; + // add -framework [name] args to linker + ZigList darwin_frameworks; + // add -rpath [name] args to linker + ZigList rpath_list; + EmitFileType emit_file_type; + BuildMode build_mode; + OutType out_type; + + + //////////////////////////// Unsorted + ZigTarget zig_target; LLVMTargetDataRef target_data_ref; unsigned pointer_size_bytes; @@ -1647,7 +1653,6 @@ struct CodeGen { Buf *ar_path; ZigWindowsSDK *win_sdk; Buf triple_str; - BuildMode build_mode; bool is_test_build; bool have_err_ret_tracing; uint32_t target_os_index; @@ -1657,13 +1662,13 @@ struct CodeGen { LLVMTargetMachineRef target_machine; ZigLLVMDIFile *dummy_di_file; bool is_native_target; - PackageTableEntry *root_package; + PackageTableEntry *root_package; // participates in cache hash PackageTableEntry *std_package; PackageTableEntry *panic_package; PackageTableEntry *test_runner_package; PackageTableEntry *compile_var_package; ImportTableEntry *compile_var_import; - Buf *root_out_name; + Buf *root_out_name; // participates in cache hash bool windows_subsystem_windows; bool windows_subsystem_console; Buf *mmacosx_version_min; @@ -1676,7 +1681,6 @@ struct CodeGen { size_t fn_defs_index; ZigList global_vars; - OutType out_type; ZigFn *cur_fn; ZigFn *main_fn; ZigFn *panic_fn; diff --git a/src/codegen.cpp b/src/codegen.cpp index 0300ccca993947b210ffedb3d1a6ca669616c755..e258f8d6093b56a0c31d83d85c4b63e2338afbb4 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -19,6 +19,7 @@ #include "target.hpp" #include "util.hpp" #include "zig_llvm.h" +#include "blake2.h" #include #include @@ -183,10 +184,6 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out return g; } -void codegen_destroy(CodeGen *codegen) { - LLVMDisposeTargetMachine(codegen->target_machine); -} - void codegen_set_output_h_path(CodeGen *g, Buf *h_path) { g->out_h_path = h_path; } @@ -7112,23 +7109,30 @@ static void create_test_compile_var_and_add_test_runner(CodeGen *g) { g->test_runner_import = add_special_code(g, g->test_runner_package, "test_runner.zig"); } -static void gen_root_source(CodeGen *g) { +static Buf *get_resolved_root_src_path(CodeGen *g) { + // TODO memoize if (buf_len(&g->root_package->root_src_path) == 0) - return; + return nullptr; - codegen_add_time_event(g, "Semantic Analysis"); - - Buf *rel_full_path = buf_alloc(); - os_path_join(&g->root_package->root_src_dir, &g->root_package->root_src_path, rel_full_path); + Buf rel_full_path = BUF_INIT; + os_path_join(&g->root_package->root_src_dir, &g->root_package->root_src_path, &rel_full_path); Buf *resolved_path = buf_alloc(); - Buf *resolve_paths[] = {rel_full_path}; + Buf *resolve_paths[] = {&rel_full_path}; *resolved_path = os_path_resolve(resolve_paths, 1); + return resolved_path; +} + +static void gen_root_source(CodeGen *g) { + Buf *resolved_path = get_resolved_root_src_path(g); + if (resolved_path == nullptr) + return; + Buf *source_code = buf_alloc(); int err; - if ((err = os_fetch_file_path(rel_full_path, source_code, true))) { - fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(rel_full_path), err_str(err)); + if ((err = os_fetch_file_path(resolved_path, source_code, true))) { + fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(resolved_path), err_str(err)); exit(1); } @@ -7671,10 +7675,155 @@ void codegen_add_time_event(CodeGen *g, const char *name) { g->timing_events.append({os_get_time(), name}); } +static void add_cache_str(blake2b_state *blake, const char *ptr) { + assert(ptr != nullptr); + // + 1 to include the null byte + blake2b_update(blake, ptr, strlen(ptr) + 1); +} + +static void add_cache_int(blake2b_state *blake, int x) { + // + 1 to include the null byte + uint8_t buf[sizeof(int) + 1]; + memcpy(buf, &x, sizeof(int)); + buf[sizeof(int)] = 0; + blake2b_update(blake, buf, sizeof(int) + 1); +} + +static void add_cache_buf(blake2b_state *blake, Buf *buf) { + assert(buf != nullptr); + // + 1 to include the null byte + blake2b_update(blake, buf_ptr(buf), buf_len(buf) + 1); +} + +static void add_cache_buf_opt(blake2b_state *blake, Buf *buf) { + if (buf == nullptr) { + add_cache_str(blake, ""); + add_cache_str(blake, ""); + } else { + add_cache_buf(blake, buf); + } +} + +static void add_cache_list_of_link_lib(blake2b_state *blake, LinkLib **ptr, size_t len) { + for (size_t i = 0; i < len; i += 1) { + LinkLib *lib = ptr[i]; + if (lib->provided_explicitly) { + add_cache_buf(blake, lib->name); + } + } + add_cache_str(blake, ""); +} + +static void add_cache_list_of_buf(blake2b_state *blake, Buf **ptr, size_t len) { + for (size_t i = 0; i < len; i += 1) { + Buf *buf = ptr[i]; + add_cache_buf(blake, buf); + } + add_cache_str(blake, ""); +} + +//static void add_cache_file(CodeGen *g, blake2b_state *blake, Buf *resolved_path) { +// assert(file_name != nullptr); +// g->cache_files.append(resolved_path); +//} +// +//static void add_cache_file_opt(CodeGen *g, blake2b_state *blake, Buf *resolved_path) { +// if (resolved_path == nullptr) { +// add_cache_str(blake, ""); +// add_cache_str(blake, ""); +// } else { +// add_cache_file(g, blake, resolved_path); +// } +//} + +// Ported from std/base64.zig +static uint8_t base64_fs_alphabet[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"; +static void base64_encode(Slice dest, Slice source) { + size_t dest_len = ((source.len + 2) / 3) * 4; + assert(dest.len == dest_len); + + size_t i = 0; + size_t out_index = 0; + for (; i + 2 < source.len; i += 3) { + dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i + 1] & 0xf) << 2) | ((source.ptr[i + 2] & 0xc0) >> 6)]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[source.ptr[i + 2] & 0x3f]; + out_index += 1; + } + + // Assert that we never need pad characters. + assert(i == source.len); + //if (i < source.len) { + // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; + // out_index += 1; + + // if (i + 1 == source.len) { + // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] & 0x3) << 4]; + // out_index += 1; + + // dest.ptr[out_index] = encoder.pad_char; + // out_index += 1; + // } else { + // dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; + // out_index += 1; + + // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i + 1] & 0xf) << 2]; + // out_index += 1; + // } + + // dest.ptr[out_index] = encoder.pad_char; + // out_index += 1; + //} +} + +// Called before init() +static bool build_with_cache(CodeGen *g) { + blake2b_state blake; + int rc = blake2b_init(&blake, 48); + assert(rc == 0); + + // TODO zig exe & dynamic libraries + + add_cache_buf(&blake, g->root_out_name); + add_cache_buf_opt(&blake, get_resolved_root_src_path(g)); // Root source file + add_cache_list_of_link_lib(&blake, g->link_libs_list.items, g->link_libs_list.length); + add_cache_list_of_buf(&blake, g->darwin_frameworks.items, g->darwin_frameworks.length); + add_cache_list_of_buf(&blake, g->rpath_list.items, g->rpath_list.length); + add_cache_int(&blake, g->emit_file_type); + add_cache_int(&blake, g->build_mode); + add_cache_int(&blake, g->out_type); + // TODO the rest of the struct CodeGen fields + + uint8_t bin_digest[48]; + rc = blake2b_final(&blake, bin_digest, 48); + assert(rc == 0); + + Buf b64_digest = BUF_INIT; + buf_resize(&b64_digest, 64); + base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); + + fprintf(stderr, "input params hash: %s\n", buf_ptr(&b64_digest)); + // TODO next look for a manifest file that has all the files from the input parameters + // use that to construct the real hash, which looks up the output directory and another manifest file + + return false; +} + void codegen_build(CodeGen *g) { assert(g->out_type != OutTypeUnknown); + if (build_with_cache(g)) + return; init(g); + codegen_add_time_event(g, "Semantic Analysis"); + gen_global_asm(g); gen_root_source(g); do_code_gen(g); diff --git a/src/codegen.hpp b/src/codegen.hpp index 6297c4611b1a5189b6f5c9d88a531ceb80832928..55b38a00509d989491162672979812337533389a 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -16,7 +16,6 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, Buf *zig_lib_dir); -void codegen_destroy(CodeGen *codegen); void codegen_set_clang_argv(CodeGen *codegen, const char **args, size_t len); void codegen_set_llvm_argv(CodeGen *codegen, const char **args, size_t len); diff --git a/src/link.cpp b/src/link.cpp index f44e8b105ed9a1dd36c88f2097670ca6adbc5bdd..78ad204fe93657f68945a2007415d911b79c4bf6 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -69,8 +69,6 @@ static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) os_path_join(&parent_gen->cache_dir, o_out_name, output_path); codegen_link(child_gen, buf_ptr(output_path)); - codegen_destroy(child_gen); - return output_path; } diff --git a/src/main.cpp b/src/main.cpp index 4394a1d9ae59f381eeab8494ee4fb5b9604d7b6d..eed31438d9a46cbd3f7f118065bf176176dd4b20 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -461,7 +461,6 @@ int main(int argc, char **argv) { g->root_package->package_table.put(buf_create_from_str("@build"), build_pkg); codegen_build(g); codegen_link(g, buf_ptr(path_to_build_exe)); - codegen_destroy(g); Termination term; os_spawn_process(buf_ptr(path_to_build_exe), args, &term); -- 2.54.0 From b4d5d4d1748d25efa6197914a36e276e93509f57 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Sep 2018 23:39:14 -0400 Subject: [PATCH 03/20] assume evenly divided base64 --- src/codegen.cpp | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index e258f8d6093b56a0c31d83d85c4b63e2338afbb4..f4823f1048b784270a41f02bd9b0bcf5b9528032 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7760,27 +7760,6 @@ static void base64_encode(Slice dest, Slice source) { // Assert that we never need pad characters. assert(i == source.len); - //if (i < source.len) { - // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; - // out_index += 1; - - // if (i + 1 == source.len) { - // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] & 0x3) << 4]; - // out_index += 1; - - // dest.ptr[out_index] = encoder.pad_char; - // out_index += 1; - // } else { - // dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; - // out_index += 1; - - // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i + 1] & 0xf) << 2]; - // out_index += 1; - // } - - // dest.ptr[out_index] = encoder.pad_char; - // out_index += 1; - //} } // Called before init() -- 2.54.0 From 173fc842c4eb1da6ca07a4ab6026c4c62bc8c09b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 9 Sep 2018 18:07:11 -0400 Subject: [PATCH 04/20] basic compiler id hash working --- CMakeLists.txt | 1 + src/buffer.hpp | 4 + src/cache_hash.cpp | 407 +++++++++++++++++++++++++++++++++++++++++++++ src/cache_hash.hpp | 56 +++++++ src/codegen.cpp | 152 ++++------------- src/error.cpp | 2 + src/error.hpp | 2 + src/main.cpp | 61 +++++++ src/os.cpp | 314 ++++++++++++++++++++++++++-------- src/os.hpp | 92 ++++++---- src/util.cpp | 39 +++++ src/util.hpp | 12 ++ 12 files changed, 918 insertions(+), 224 deletions(-) create mode 100644 src/cache_hash.cpp create mode 100644 src/cache_hash.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fbf5b03eac0543ab9cf705577597ca37075cfc69..615128159f5725cbe093488c39c9acb9bcef4704 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -409,6 +409,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/bigint.cpp" "${CMAKE_SOURCE_DIR}/src/blake2b.cpp" "${CMAKE_SOURCE_DIR}/src/buffer.cpp" + "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" "${CMAKE_SOURCE_DIR}/src/errmsg.cpp" diff --git a/src/buffer.hpp b/src/buffer.hpp index 501e44b5acc1ede86487a765c74aa2a5a3d8256d..8155df87a151d4b6930919d166cdc3c34cd207b8 100644 --- a/src/buffer.hpp +++ b/src/buffer.hpp @@ -78,6 +78,10 @@ static inline Buf *buf_create_from_mem(const char *ptr, size_t len) { return buf; } +static inline Buf *buf_create_from_slice(Slice slice) { + return buf_create_from_mem((const char *)slice.ptr, slice.len); +} + static inline Buf *buf_create_from_str(const char *str) { return buf_create_from_mem(str, strlen(str)); } diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5447a5f3f2ad8758cf2e74e220ef8432e60c58f1 --- /dev/null +++ b/src/cache_hash.cpp @@ -0,0 +1,407 @@ +/* + * Copyright (c) 2018 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#include "cache_hash.hpp" +#include "buffer.hpp" +#include "os.hpp" + +#include + +void cache_init(CacheHash *ch, Buf *manifest_dir) { + int rc = blake2b_init(&ch->blake, 48); + assert(rc == 0); + ch->files = {}; + ch->manifest_dir = manifest_dir; + ch->manifest_file_path = nullptr; + ch->manifest_dirty = false; +} + +void cache_str(CacheHash *ch, const char *ptr) { + assert(ch->manifest_file_path == nullptr); + assert(ptr != nullptr); + // + 1 to include the null byte + blake2b_update(&ch->blake, ptr, strlen(ptr) + 1); +} + +void cache_int(CacheHash *ch, int x) { + assert(ch->manifest_file_path == nullptr); + // + 1 to include the null byte + uint8_t buf[sizeof(int) + 1]; + memcpy(buf, &x, sizeof(int)); + buf[sizeof(int)] = 0; + blake2b_update(&ch->blake, buf, sizeof(int) + 1); +} + +void cache_buf(CacheHash *ch, Buf *buf) { + assert(ch->manifest_file_path == nullptr); + assert(buf != nullptr); + // + 1 to include the null byte + blake2b_update(&ch->blake, buf_ptr(buf), buf_len(buf) + 1); +} + +void cache_buf_opt(CacheHash *ch, Buf *buf) { + assert(ch->manifest_file_path == nullptr); + if (buf == nullptr) { + cache_str(ch, ""); + cache_str(ch, ""); + } else { + cache_buf(ch, buf); + } +} + +void cache_list_of_link_lib(CacheHash *ch, LinkLib **ptr, size_t len) { + assert(ch->manifest_file_path == nullptr); + for (size_t i = 0; i < len; i += 1) { + LinkLib *lib = ptr[i]; + if (lib->provided_explicitly) { + cache_buf(ch, lib->name); + } + } + cache_str(ch, ""); +} + +void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len) { + assert(ch->manifest_file_path == nullptr); + for (size_t i = 0; i < len; i += 1) { + Buf *buf = ptr[i]; + cache_buf(ch, buf); + } + cache_str(ch, ""); +} + +void cache_file(CacheHash *ch, Buf *file_path) { + assert(ch->manifest_file_path == nullptr); + assert(file_path != nullptr); + Buf *resolved_path = buf_alloc(); + *resolved_path = os_path_resolve(&file_path, 1); + CacheHashFile *chf = ch->files.add_one(); + chf->path = resolved_path; + cache_buf(ch, resolved_path); +} + +void cache_file_opt(CacheHash *ch, Buf *file_path) { + assert(ch->manifest_file_path == nullptr); + if (file_path == nullptr) { + cache_str(ch, ""); + cache_str(ch, ""); + } else { + cache_file(ch, file_path); + } +} + +// Ported from std/base64.zig +static uint8_t base64_fs_alphabet[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; +static void base64_encode(Slice dest, Slice source) { + size_t dest_len = ((source.len + 2) / 3) * 4; + assert(dest.len == dest_len); + + size_t i = 0; + size_t out_index = 0; + for (; i + 2 < source.len; i += 3) { + dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i + 1] & 0xf) << 2) | ((source.ptr[i + 2] & 0xc0) >> 6)]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[source.ptr[i + 2] & 0x3f]; + out_index += 1; + } + + // Assert that we never need pad characters. + assert(i == source.len); +} + +// Ported from std/base64.zig +static Error base64_decode(Slice dest, Slice source) { + assert(source.len % 4 == 0); + assert(dest.len == (source.len / 4) * 3); + + // In Zig this is comptime computed. In C++ it's not worth it to do that. + uint8_t char_to_index[256]; + bool char_in_alphabet[256] = {0}; + for (size_t i = 0; i < 64; i += 1) { + uint8_t c = base64_fs_alphabet[i]; + assert(!char_in_alphabet[c]); + char_in_alphabet[c] = true; + char_to_index[c] = i; + } + + size_t src_cursor = 0; + size_t dest_cursor = 0; + + for (;src_cursor < source.len; src_cursor += 4) { + if (!char_in_alphabet[source.ptr[src_cursor + 0]]) return ErrorInvalidFormat; + if (!char_in_alphabet[source.ptr[src_cursor + 1]]) return ErrorInvalidFormat; + if (!char_in_alphabet[source.ptr[src_cursor + 2]]) return ErrorInvalidFormat; + if (!char_in_alphabet[source.ptr[src_cursor + 3]]) return ErrorInvalidFormat; + dest.ptr[dest_cursor + 0] = (char_to_index[source.ptr[src_cursor + 0]] << 2) | (char_to_index[source.ptr[src_cursor + 1]] >> 4); + dest.ptr[dest_cursor + 1] = (char_to_index[source.ptr[src_cursor + 1]] << 4) | (char_to_index[source.ptr[src_cursor + 2]] >> 2); + dest.ptr[dest_cursor + 2] = (char_to_index[source.ptr[src_cursor + 2]] << 6) | (char_to_index[source.ptr[src_cursor + 3]]); + dest_cursor += 3; + } + + assert(src_cursor == source.len); + assert(dest_cursor == dest.len); + return ErrorNone; +} + +static Error hash_file(uint8_t *digest, OsFile handle) { + Error err; + + blake2b_state blake; + int rc = blake2b_init(&blake, 48); + assert(rc == 0); + + for (;;) { + uint8_t buf[4096]; + size_t amt = 4096; + if ((err = os_file_read(handle, buf, &amt))) + return err; + if (amt == 0) { + rc = blake2b_final(&blake, digest, 48); + assert(rc == 0); + return ErrorNone; + } + blake2b_update(&blake, buf, amt); + } +} + +static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf) { + Error err; + + assert(chf->path != nullptr); + + OsFile this_file; + if ((err = os_file_open_r(chf->path, &this_file))) + return err; + + if ((err = os_file_mtime(this_file, &chf->mtime))) { + os_file_close(this_file); + return err; + } + + if ((err = hash_file(chf->bin_digest, this_file))) { + os_file_close(this_file); + return err; + } + os_file_close(this_file); + + blake2b_update(&ch->blake, chf->bin_digest, 48); + + return ErrorNone; +} + +Error cache_hit(CacheHash *ch, Buf *out_digest) { + Error err; + + uint8_t bin_digest[48]; + int rc = blake2b_final(&ch->blake, bin_digest, 48); + assert(rc == 0); + + Buf b64_digest = BUF_INIT; + buf_resize(&b64_digest, 64); + base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); + + rc = blake2b_init(&ch->blake, 48); + assert(rc == 0); + blake2b_update(&ch->blake, bin_digest, 48); + + ch->manifest_file_path = buf_alloc(); + os_path_join(ch->manifest_dir, &b64_digest, ch->manifest_file_path); + + buf_append_str(ch->manifest_file_path, ".txt"); + + if ((err = os_file_open_lock_rw(ch->manifest_file_path, &ch->manifest_file))) + return err; + + Buf line_buf = BUF_INIT; + buf_resize(&line_buf, 512); + if ((err = os_file_read_all(ch->manifest_file, &line_buf))) { + os_file_close(ch->manifest_file); + return err; + } + + size_t input_file_count = ch->files.length; + bool any_file_changed = false; + size_t file_i = 0; + SplitIterator line_it = memSplit(buf_to_slice(&line_buf), str("\n")); + for (;; file_i += 1) { + Optional> opt_line = SplitIterator_next(&line_it); + if (!opt_line.is_some) + break; + + CacheHashFile *chf; + if (file_i < input_file_count) { + chf = &ch->files.at(file_i); + } else if (any_file_changed) { + // cache miss. + // keep the the manifest file open with the rw lock + // reset the hash + rc = blake2b_init(&ch->blake, 48); + assert(rc == 0); + blake2b_update(&ch->blake, bin_digest, 48); + ch->files.resize(input_file_count); + // bring the hash up to the input file hashes + for (file_i = 0; file_i < input_file_count; file_i += 1) { + blake2b_update(&ch->blake, ch->files.at(file_i).bin_digest, 48); + } + // caller can notice that out_digest is unmodified. + return ErrorNone; + } else { + chf = ch->files.add_one(); + chf->path = nullptr; + } + + SplitIterator it = memSplit(opt_line.value, str(" ")); + + Optional> opt_mtime_sec = SplitIterator_next(&it); + if (!opt_mtime_sec.is_some) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + chf->mtime.sec = strtoull((const char *)opt_mtime_sec.value.ptr, nullptr, 10); + + Optional> opt_mtime_nsec = SplitIterator_next(&it); + if (!opt_mtime_nsec.is_some) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + chf->mtime.nsec = strtoull((const char *)opt_mtime_nsec.value.ptr, nullptr, 10); + + Optional> opt_digest = SplitIterator_next(&it); + if (!opt_digest.is_some) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + if ((err = base64_decode({chf->bin_digest, 48}, opt_digest.value))) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + + Optional> opt_file_path = SplitIterator_next(&it); + if (!opt_file_path.is_some) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + Buf *this_path = buf_create_from_slice(opt_file_path.value); + if (chf->path != nullptr && !buf_eql_buf(this_path, chf->path)) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + chf->path = this_path; + + // if the mtime matches we can trust the digest + OsFile this_file; + if ((err = os_file_open_r(chf->path, &this_file))) { + os_file_close(ch->manifest_file); + return err; + } + OsTimeStamp actual_mtime; + if ((err = os_file_mtime(this_file, &actual_mtime))) { + os_file_close(this_file); + os_file_close(ch->manifest_file); + return err; + } + if (chf->mtime.sec == actual_mtime.sec && chf->mtime.nsec == actual_mtime.nsec) { + os_file_close(this_file); + } else { + // we have to recompute the digest. + // later we'll rewrite the manifest with the new mtime/digest values + ch->manifest_dirty = true; + chf->mtime = actual_mtime; + + uint8_t actual_digest[48]; + if ((err = hash_file(actual_digest, this_file))) { + os_file_close(this_file); + os_file_close(ch->manifest_file); + return err; + } + os_file_close(this_file); + if (memcmp(chf->bin_digest, actual_digest, 48) != 0) { + memcpy(chf->bin_digest, actual_digest, 48); + // keep going until we have the input file digests + any_file_changed = true; + } + } + if (!any_file_changed) { + blake2b_update(&ch->blake, chf->bin_digest, 48); + } + } + if (file_i < input_file_count) { + // manifest file is empty or missing entries, so this is a cache miss + ch->manifest_dirty = true; + for (; file_i < input_file_count; file_i += 1) { + CacheHashFile *chf = &ch->files.at(file_i); + if ((err = populate_file_hash(ch, chf))) { + os_file_close(ch->manifest_file); + return err; + } + } + return ErrorNone; + } + // Cache Hit + return cache_final(ch, out_digest); +} + +Error cache_add_file(CacheHash *ch, Buf *path) { + Error err; + + assert(ch->manifest_file_path != nullptr); + CacheHashFile *chf = ch->files.add_one(); + chf->path = path; + if ((err = populate_file_hash(ch, chf))) { + os_file_close(ch->manifest_file); + return err; + } + + return ErrorNone; +} + +static Error write_manifest_file(CacheHash *ch) { + Error err; + Buf contents = BUF_INIT; + buf_resize(&contents, 0); + uint8_t encoded_digest[65]; + encoded_digest[64] = 0; + for (size_t i = 0; i < ch->files.length; i += 1) { + CacheHashFile *chf = &ch->files.at(i); + base64_encode({encoded_digest, 64}, {chf->bin_digest, 48}); + buf_appendf(&contents, "%" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %s %s\n", + chf->mtime.sec, chf->mtime.nsec, encoded_digest, buf_ptr(chf->path)); + } + fprintf(stderr, "overwrite with\n%s\n", buf_ptr(&contents)); + if ((err = os_file_overwrite(ch->manifest_file, &contents))) + return err; + + return ErrorNone; +} + +Error cache_final(CacheHash *ch, Buf *out_digest) { + Error err; + + assert(ch->manifest_file_path != nullptr); + + if (ch->manifest_dirty) { + if ((err = write_manifest_file(ch))) { + fprintf(stderr, "Warning: Unable to write cache file '%s': %s\n", + buf_ptr(ch->manifest_file_path), err_str(err)); + } + } + os_file_close(ch->manifest_file); + + uint8_t bin_digest[48]; + int rc = blake2b_final(&ch->blake, bin_digest, 48); + assert(rc == 0); + buf_resize(out_digest, 64); + base64_encode(buf_to_slice(out_digest), {bin_digest, 48}); + + return ErrorNone; +} diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp new file mode 100644 index 0000000000000000000000000000000000000000..77b22a1e41bbb903b3061cb1a2ee423694c387e2 --- /dev/null +++ b/src/cache_hash.hpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2018 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#ifndef ZIG_CACHE_HASH_HPP +#define ZIG_CACHE_HASH_HPP + +#include "all_types.hpp" +#include "blake2.h" +#include "os.hpp" + +struct CacheHashFile { + Buf *path; + OsTimeStamp mtime; + uint8_t bin_digest[48]; +}; + +struct CacheHash { + blake2b_state blake; + ZigList files; + Buf *manifest_dir; + Buf *manifest_file_path; + OsFile manifest_file; + bool manifest_dirty; +}; + +// Always call this first to set up. +void cache_init(CacheHash *ch, Buf *manifest_dir); + +// Next, use the hash population functions to add the initial parameters. +void cache_str(CacheHash *ch, const char *ptr); +void cache_int(CacheHash *ch, int x); +void cache_buf(CacheHash *ch, Buf *buf); +void cache_buf_opt(CacheHash *ch, Buf *buf); +void cache_list_of_link_lib(CacheHash *ch, LinkLib **ptr, size_t len); +void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len); +void cache_file(CacheHash *ch, Buf *path); +void cache_file_opt(CacheHash *ch, Buf *path); + +// Then call cache_hit when you're ready to see if you can skip the next step. +// out_b64_digest will be left unchanged if it was a cache miss +Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest); + +// If you got a cache hit, the flow is done. No more functions to call. +// Next call this function for every file that is depended on. +Error ATTRIBUTE_MUST_USE cache_add_file(CacheHash *ch, Buf *path); + +// If you did not get a cache hit, use the hash population functions again +// and do all the actual work. When done use cache_final to save the cache +// for next time. +Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_digest); + +#endif diff --git a/src/codegen.cpp b/src/codegen.cpp index f4823f1048b784270a41f02bd9b0bcf5b9528032..6c5648f6261c123c3bf422e9614ea3ab77ac07c1 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -19,7 +19,6 @@ #include "target.hpp" #include "util.hpp" #include "zig_llvm.h" -#include "blake2.h" #include #include @@ -7675,130 +7674,45 @@ void codegen_add_time_event(CodeGen *g, const char *name) { g->timing_events.append({os_get_time(), name}); } -static void add_cache_str(blake2b_state *blake, const char *ptr) { - assert(ptr != nullptr); - // + 1 to include the null byte - blake2b_update(blake, ptr, strlen(ptr) + 1); -} -static void add_cache_int(blake2b_state *blake, int x) { - // + 1 to include the null byte - uint8_t buf[sizeof(int) + 1]; - memcpy(buf, &x, sizeof(int)); - buf[sizeof(int)] = 0; - blake2b_update(blake, buf, sizeof(int) + 1); -} - -static void add_cache_buf(blake2b_state *blake, Buf *buf) { - assert(buf != nullptr); - // + 1 to include the null byte - blake2b_update(blake, buf_ptr(buf), buf_len(buf) + 1); -} - -static void add_cache_buf_opt(blake2b_state *blake, Buf *buf) { - if (buf == nullptr) { - add_cache_str(blake, ""); - add_cache_str(blake, ""); - } else { - add_cache_buf(blake, buf); - } -} - -static void add_cache_list_of_link_lib(blake2b_state *blake, LinkLib **ptr, size_t len) { - for (size_t i = 0; i < len; i += 1) { - LinkLib *lib = ptr[i]; - if (lib->provided_explicitly) { - add_cache_buf(blake, lib->name); - } - } - add_cache_str(blake, ""); -} - -static void add_cache_list_of_buf(blake2b_state *blake, Buf **ptr, size_t len) { - for (size_t i = 0; i < len; i += 1) { - Buf *buf = ptr[i]; - add_cache_buf(blake, buf); - } - add_cache_str(blake, ""); -} - -//static void add_cache_file(CodeGen *g, blake2b_state *blake, Buf *resolved_path) { -// assert(file_name != nullptr); -// g->cache_files.append(resolved_path); -//} +//// Called before init() +//static bool build_with_cache(CodeGen *g) { +// // TODO zig exe & dynamic libraries +// // should be in main.cpp I think. only needs to happen +// // once on startup. // -//static void add_cache_file_opt(CodeGen *g, blake2b_state *blake, Buf *resolved_path) { -// if (resolved_path == nullptr) { -// add_cache_str(blake, ""); -// add_cache_str(blake, ""); -// } else { -// add_cache_file(g, blake, resolved_path); -// } +// CacheHash comp; +// cache_init(&comp); +// +// add_cache_buf(&blake, g->root_out_name); +// add_cache_buf_opt(&blake, get_resolved_root_src_path(g)); // Root source file +// add_cache_list_of_link_lib(&blake, g->link_libs_list.items, g->link_libs_list.length); +// add_cache_list_of_buf(&blake, g->darwin_frameworks.items, g->darwin_frameworks.length); +// add_cache_list_of_buf(&blake, g->rpath_list.items, g->rpath_list.length); +// add_cache_int(&blake, g->emit_file_type); +// add_cache_int(&blake, g->build_mode); +// add_cache_int(&blake, g->out_type); +// // TODO the rest of the struct CodeGen fields +// +// uint8_t bin_digest[48]; +// rc = blake2b_final(&blake, bin_digest, 48); +// assert(rc == 0); +// +// Buf b64_digest = BUF_INIT; +// buf_resize(&b64_digest, 64); +// base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); +// +// fprintf(stderr, "input params hash: %s\n", buf_ptr(&b64_digest)); +// // TODO next look for a manifest file that has all the files from the input parameters +// // use that to construct the real hash, which looks up the output directory and another manifest file +// +// return false; //} -// Ported from std/base64.zig -static uint8_t base64_fs_alphabet[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"; -static void base64_encode(Slice dest, Slice source) { - size_t dest_len = ((source.len + 2) / 3) * 4; - assert(dest.len == dest_len); - - size_t i = 0; - size_t out_index = 0; - for (; i + 2 < source.len; i += 3) { - dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; - out_index += 1; - - dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; - out_index += 1; - - dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i + 1] & 0xf) << 2) | ((source.ptr[i + 2] & 0xc0) >> 6)]; - out_index += 1; - - dest.ptr[out_index] = base64_fs_alphabet[source.ptr[i + 2] & 0x3f]; - out_index += 1; - } - - // Assert that we never need pad characters. - assert(i == source.len); -} - -// Called before init() -static bool build_with_cache(CodeGen *g) { - blake2b_state blake; - int rc = blake2b_init(&blake, 48); - assert(rc == 0); - - // TODO zig exe & dynamic libraries - - add_cache_buf(&blake, g->root_out_name); - add_cache_buf_opt(&blake, get_resolved_root_src_path(g)); // Root source file - add_cache_list_of_link_lib(&blake, g->link_libs_list.items, g->link_libs_list.length); - add_cache_list_of_buf(&blake, g->darwin_frameworks.items, g->darwin_frameworks.length); - add_cache_list_of_buf(&blake, g->rpath_list.items, g->rpath_list.length); - add_cache_int(&blake, g->emit_file_type); - add_cache_int(&blake, g->build_mode); - add_cache_int(&blake, g->out_type); - // TODO the rest of the struct CodeGen fields - - uint8_t bin_digest[48]; - rc = blake2b_final(&blake, bin_digest, 48); - assert(rc == 0); - - Buf b64_digest = BUF_INIT; - buf_resize(&b64_digest, 64); - base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); - - fprintf(stderr, "input params hash: %s\n", buf_ptr(&b64_digest)); - // TODO next look for a manifest file that has all the files from the input parameters - // use that to construct the real hash, which looks up the output directory and another manifest file - - return false; -} - void codegen_build(CodeGen *g) { assert(g->out_type != OutTypeUnknown); - if (build_with_cache(g)) - return; + //if (build_with_cache(g)) + // return; init(g); codegen_add_time_event(g, "Semantic Analysis"); diff --git a/src/error.cpp b/src/error.cpp index 8303a80a1de39ac8171f41e85e14c0d974c4391e..d9da6e6c52f248032ab4635b03224ce973172c20 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -27,6 +27,8 @@ const char *err_str(int err) { case ErrorNegativeDenominator: return "negative denominator"; case ErrorShiftedOutOneBits: return "exact shift shifted out one bits"; case ErrorCCompileErrors: return "C compile errors"; + case ErrorEndOfFile: return "end of file"; + case ErrorIsDir: return "is directory"; } return "(invalid error)"; } diff --git a/src/error.hpp b/src/error.hpp index e3b87fc6b85f331e28bbc68c66afdfce69d55f3f..c99c6e6b12d73043e24cb3e5eac748e3b41d93d0 100644 --- a/src/error.hpp +++ b/src/error.hpp @@ -27,6 +27,8 @@ enum Error { ErrorNegativeDenominator, ErrorShiftedOutOneBits, ErrorCCompileErrors, + ErrorEndOfFile, + ErrorIsDir, }; const char *err_str(int err); diff --git a/src/main.cpp b/src/main.cpp index eed31438d9a46cbd3f7f118065bf176176dd4b20..172fe04da9513841d291019e345a07bed4a90174 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -13,6 +13,7 @@ #include "link.hpp" #include "os.hpp" #include "target.hpp" +#include "cache_hash.hpp" #include @@ -270,6 +271,66 @@ int main(int argc, char **argv) { return 0; } + if (argc == 2 && strcmp(argv[1], "TEST") == 0) { + Error err; + Buf app_data_dir = BUF_INIT; + if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { + fprintf(stderr, "get app dir: %s\n", err_str(err)); + return 1; + } + Buf *stage1_dir = buf_alloc(); + os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); + + if ((err = os_make_path(manifest_dir))) { + fprintf(stderr, "make path: %s\n", err_str(err)); + return 1; + } + CacheHash cache_hash; + CacheHash *ch = &cache_hash; + cache_init(ch, manifest_dir); + Buf self_exe_path = BUF_INIT; + if ((err = os_self_exe_path(&self_exe_path))) { + fprintf(stderr, "self exe path: %s\n", err_str(err)); + return 1; + } + + cache_file(ch, &self_exe_path); + + Buf exe_digest = BUF_INIT; + buf_resize(&exe_digest, 0); + if ((err = cache_hit(ch, &exe_digest))) { + fprintf(stderr, "cache hit error: %s\n", err_str(err)); + return 1; + } + if (buf_len(&exe_digest) != 0) { + fprintf(stderr, "cache hit: %s\n", buf_ptr(&exe_digest)); + return 0; + } + fprintf(stderr, "cache miss\n"); + ZigList lib_paths = {}; + if ((err = os_self_exe_shared_libs(lib_paths))) { + fprintf(stderr, "finding out shared libs: %s\n", err_str(err)); + return 1; + } + for (size_t i = 0; i < lib_paths.length; i += 1) { + Buf *lib_path = lib_paths.at(i); + if ((err = cache_add_file(ch, lib_path))) { + fprintf(stderr, "cache add file %s: %s", buf_ptr(lib_path), err_str(err)); + return 1; + } + } + if ((err = cache_final(ch, &exe_digest))) { + fprintf(stderr, "final: %s\n", err_str(err)); + return 1; + } + + + fprintf(stderr, "computed2: %s\n", buf_ptr(&exe_digest)); + return 0; + } + os_init(); char *arg0 = argv[0]; diff --git a/src/os.cpp b/src/os.cpp index 0e62d84a48ca27ecf4ee0450242a032fce9b7b05..d6e17ca5a1521c4eff2d3f608ad2f15a29c281bd 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -40,6 +40,10 @@ typedef SSIZE_T ssize_t; #endif +#if defined(ZIG_OS_LINUX) +#include +#endif + #if defined(__MACH__) #include @@ -57,54 +61,6 @@ static clock_serv_t cclock; #include #include -// Ported from std/mem.zig. -// Coordinate struct fields with memSplit function -struct SplitIterator { - size_t index; - Slice buffer; - Slice split_bytes; -}; - -// Ported from std/mem.zig. -static bool SplitIterator_isSplitByte(SplitIterator *self, uint8_t byte) { - for (size_t i = 0; i < self->split_bytes.len; i += 1) { - if (byte == self->split_bytes.ptr[i]) { - return true; - } - } - return false; -} - -// Ported from std/mem.zig. -static Optional> SplitIterator_next(SplitIterator *self) { - // move to beginning of token - while (self->index < self->buffer.len && - SplitIterator_isSplitByte(self, self->buffer.ptr[self->index])) - { - self->index += 1; - } - size_t start = self->index; - if (start == self->buffer.len) { - return {}; - } - - // move to end of token - while (self->index < self->buffer.len && - !SplitIterator_isSplitByte(self, self->buffer.ptr[self->index])) - { - self->index += 1; - } - size_t end = self->index; - - return Optional>::some(self->buffer.slice(start, end)); -} - -// Ported from std/mem.zig -static SplitIterator memSplit(Slice buffer, Slice split_bytes) { - return SplitIterator{0, buffer, split_bytes}; -} - - #if defined(ZIG_OS_POSIX) static void populate_termination(Termination *term, int status) { if (WIFEXITED(status)) { @@ -1368,16 +1324,16 @@ double os_get_time(void) { #endif } -int os_make_path(Buf *path) { +Error os_make_path(Buf *path) { Buf resolved_path = os_path_resolve(&path, 1); size_t end_index = buf_len(&resolved_path); - int err; + Error err; while (true) { if ((err = os_make_dir(buf_slice(&resolved_path, 0, end_index)))) { if (err == ErrorPathAlreadyExists) { if (end_index == buf_len(&resolved_path)) - return 0; + return ErrorNone; } else if (err == ErrorFileNotFound) { // march end_index backward until next path component while (true) { @@ -1391,7 +1347,7 @@ int os_make_path(Buf *path) { } } if (end_index == buf_len(&resolved_path)) - return 0; + return ErrorNone; // march end_index forward until next path component while (true) { end_index += 1; @@ -1399,10 +1355,10 @@ int os_make_path(Buf *path) { break; } } - return 0; + return ErrorNone; } -int os_make_dir(Buf *path) { +Error os_make_dir(Buf *path) { #if defined(ZIG_OS_WINDOWS) if (!CreateDirectory(buf_ptr(path), NULL)) { if (GetLastError() == ERROR_ALREADY_EXISTS) @@ -1413,7 +1369,7 @@ int os_make_dir(Buf *path) { return ErrorAccess; return ErrorUnexpected; } - return 0; + return ErrorNone; #else if (mkdir(buf_ptr(path), 0755) == -1) { if (errno == EEXIST) @@ -1424,7 +1380,7 @@ int os_make_dir(Buf *path) { return ErrorAccess; return ErrorUnexpected; } - return 0; + return ErrorNone; #endif } @@ -1447,7 +1403,7 @@ int os_init(void) { return 0; } -int os_self_exe_path(Buf *out_path) { +Error os_self_exe_path(Buf *out_path) { #if defined(ZIG_OS_WINDOWS) buf_resize(out_path, 256); for (;;) { @@ -1480,27 +1436,21 @@ int os_self_exe_path(Buf *out_path) { char *real_path = realpath(buf_ptr(tmp), buf_ptr(out_path)); if (!real_path) { buf_init_from_buf(out_path, tmp); - return 0; + return ErrorNone; } // Resize out_path for the correct length. buf_resize(out_path, strlen(buf_ptr(out_path))); - return 0; + return ErrorNone; #elif defined(ZIG_OS_LINUX) - buf_resize(out_path, 256); - for (;;) { - ssize_t amt = readlink("/proc/self/exe", buf_ptr(out_path), buf_len(out_path)); - if (amt == -1) { - return ErrorUnexpected; - } - if (amt == (ssize_t)buf_len(out_path)) { - buf_resize(out_path, buf_len(out_path) * 2); - continue; - } - buf_resize(out_path, amt); - return 0; + buf_resize(out_path, PATH_MAX); + ssize_t amt = readlink("/proc/self/exe", buf_ptr(out_path), buf_len(out_path)); + if (amt == -1) { + return ErrorUnexpected; } + buf_resize(out_path, amt); + return ErrorNone; #endif return ErrorFileNotFound; } @@ -1685,3 +1635,225 @@ int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf* output_buf, ZigLLVM_ArchTy return ErrorFileNotFound; #endif } + +// Ported from std/os/get_app_data_dir.zig +Error os_get_app_data_dir(Buf *out_path, const char *appname) { +#if defined(ZIG_OS_WINDOWS) +#error "Unimplemented" +#elif defined(ZIG_OS_DARWIN) + const char *home_dir = getenv("HOME"); + if (home_dir == nullptr) { + // TODO use /etc/passwd + return ErrorFileNotFound; + } + buf_resize(out_path, 0); + buf_appendf(out_path, "%s/Library/Application Support/%s", home_dir, appname); + return ErrorNone; +#elif defined(ZIG_OS_LINUX) + const char *home_dir = getenv("HOME"); + if (home_dir == nullptr) { + // TODO use /etc/passwd + return ErrorFileNotFound; + } + buf_resize(out_path, 0); + buf_appendf(out_path, "%s/.local/share/%s", home_dir, appname); + return ErrorNone; +#endif +} + + +#if defined(ZIG_OS_LINUX) +static int self_exe_shared_libs_callback(struct dl_phdr_info *info, size_t size, void *data) { + ZigList *libs = reinterpret_cast< ZigList *>(data); + if (info->dlpi_name[0] == '/') { + libs->append(buf_create_from_str(info->dlpi_name)); + } + return 0; +} +#endif + +Error os_self_exe_shared_libs(ZigList &paths) { +#if defined(ZIG_OS_LINUX) + paths.resize(0); + dl_iterate_phdr(self_exe_shared_libs_callback, &paths); + return ErrorNone; +#else +#error "unimplemented" +#endif +} + +Error os_file_open_r(Buf *full_path, OsFile *out_file) { +#if defined(ZIG_OS_WINDOWS) +#error "unimplemented" +#else + for (;;) { + int fd = open(buf_ptr(full_path), O_RDONLY|O_CLOEXEC); + if (fd == -1) { + switch (errno) { + case EINTR: + continue; + case EINVAL: + zig_unreachable(); + case EFAULT: + zig_unreachable(); + case EACCES: + return ErrorAccess; + case EISDIR: + return ErrorIsDir; + case ENOENT: + return ErrorFileNotFound; + default: + return ErrorFileSystem; + } + } + *out_file = fd; + return ErrorNone; + } +#endif +} + +Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) { +#if defined(ZIG_OS_WINDOWS) +#error "unimplemented" +#else + int fd; + for (;;) { + fd = open(buf_ptr(full_path), O_RDWR|O_CLOEXEC|O_CREAT, 0666); + if (fd == -1) { + switch (errno) { + case EINTR: + continue; + case EINVAL: + zig_unreachable(); + case EFAULT: + zig_unreachable(); + case EACCES: + return ErrorAccess; + case EISDIR: + return ErrorIsDir; + case ENOENT: + return ErrorFileNotFound; + default: + return ErrorFileSystem; + } + } + break; + } + for (;;) { + struct flock lock; + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + if (fcntl(fd, F_SETLKW, &lock) == -1) { + switch (errno) { + case EINTR: + continue; + case EBADF: + zig_unreachable(); + case EFAULT: + zig_unreachable(); + case EINVAL: + zig_unreachable(); + default: + close(fd); + return ErrorFileSystem; + } + } + break; + } + *out_file = fd; + return ErrorNone; +#endif +} + +Error os_file_mtime(OsFile file, OsTimeStamp *mtime) { +#if defined(ZIG_OS_WINDOWS) +#error unimplemented +#else + struct stat statbuf; + if (fstat(file, &statbuf) == -1) + return ErrorFileSystem; + + mtime->sec = statbuf.st_mtim.tv_sec; + mtime->nsec = statbuf.st_mtim.tv_nsec; + return ErrorNone; +#endif +} + +Error os_file_read(OsFile file, void *ptr, size_t *len) { +#if defined(ZIG_OS_WINDOWS) +#error unimplemented +#else + for (;;) { + ssize_t rc = read(file, ptr, *len); + if (rc == -1) { + switch (errno) { + case EINTR: + continue; + case EBADF: + zig_unreachable(); + case EFAULT: + zig_unreachable(); + case EISDIR: + zig_unreachable(); + default: + return ErrorFileSystem; + } + } + *len = rc; + return ErrorNone; + } +#endif +} + +Error os_file_read_all(OsFile file, Buf *contents) { + Error err; + size_t index = 0; + for (;;) { + size_t amt = buf_len(contents) - index; + + if (amt < 512) { + buf_resize(contents, buf_len(contents) + 512); + amt += 512; + } + + if ((err = os_file_read(file, buf_ptr(contents) + index, &amt))) + return err; + + if (amt == 0) { + buf_resize(contents, index); + return ErrorNone; + } + + index += amt; + } +} + +Error os_file_overwrite(OsFile file, Buf *contents) { +#if defined(ZIG_OS_WINDOWS) +#error unimplemented +#else + if (lseek(file, 0, SEEK_SET) == -1) + return ErrorFileSystem; + for (;;) { + if (write(file, buf_ptr(contents), buf_len(contents)) == -1) { + switch (errno) { + case EINTR: + continue; + case EINVAL: + zig_unreachable(); + case EBADF: + zig_unreachable(); + default: + return ErrorFileSystem; + } + } + return ErrorNone; + } +#endif +} + +void os_file_close(OsFile file) { + close(file); +} diff --git a/src/os.hpp b/src/os.hpp index a44fa8160ef3a4edb5c4dcc4c8bb5b261a2ca37a..c64eccf8d44af408a121a7da9ad3c74689ded1c8 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -13,10 +13,43 @@ #include "error.hpp" #include "zig_llvm.h" #include "windows_sdk.h" +#include "result.hpp" #include #include +#if defined(__APPLE__) +#define ZIG_OS_DARWIN +#elif defined(_WIN32) +#define ZIG_OS_WINDOWS +#elif defined(__linux__) +#define ZIG_OS_LINUX +#else +#define ZIG_OS_UNKNOWN +#endif + +#if defined(__x86_64__) +#define ZIG_ARCH_X86_64 +#else +#define ZIG_ARCH_UNKNOWN +#endif + +#if defined(ZIG_OS_WINDOWS) +#define ZIG_PRI_usize "I64u" +#define ZIG_PRI_u64 "I64u" +#define ZIG_PRI_llu "I64u" +#define ZIG_PRI_x64 "I64x" +#define OS_SEP "\\" +#define ZIG_OS_SEP_CHAR '\\' +#else +#define ZIG_PRI_usize "zu" +#define ZIG_PRI_u64 PRIu64 +#define ZIG_PRI_llu "llu" +#define ZIG_PRI_x64 PRIx64 +#define OS_SEP "/" +#define ZIG_OS_SEP_CHAR '/' +#endif + enum TermColor { TermColorRed, TermColorGreen, @@ -38,6 +71,17 @@ struct Termination { int code; }; +#if defined(ZIG_OS_WINDOWS) +#define OsFile (void *) +#else +#define OsFile int +#endif + +struct OsTimeStamp { + uint64_t sec; + uint64_t nsec; +}; + int os_init(void); void os_spawn_process(const char *exe, ZigList &args, Termination *term); @@ -54,8 +98,16 @@ bool os_path_is_absolute(Buf *path); int os_get_global_cache_directory(Buf *out_tmp_path); -int os_make_path(Buf *path); -int os_make_dir(Buf *path); +Error ATTRIBUTE_MUST_USE os_make_path(Buf *path); +Error ATTRIBUTE_MUST_USE os_make_dir(Buf *path); + +Error ATTRIBUTE_MUST_USE os_file_open_r(Buf *full_path, OsFile *out_file); +Error ATTRIBUTE_MUST_USE os_file_open_lock_rw(Buf *full_path, OsFile *out_file); +Error ATTRIBUTE_MUST_USE os_file_mtime(OsFile file, OsTimeStamp *mtime); +Error ATTRIBUTE_MUST_USE os_file_read(OsFile file, void *ptr, size_t *len); +Error ATTRIBUTE_MUST_USE os_file_read_all(OsFile file, Buf *contents); +Error ATTRIBUTE_MUST_USE os_file_overwrite(OsFile file, Buf *contents); +void os_file_close(OsFile file); void os_write_file(Buf *full_path, Buf *contents); int os_copy_file(Buf *src_path, Buf *dest_path); @@ -78,42 +130,14 @@ double os_get_time(void); bool os_is_sep(uint8_t c); -int os_self_exe_path(Buf *out_path); +Error ATTRIBUTE_MUST_USE os_self_exe_path(Buf *out_path); + +Error ATTRIBUTE_MUST_USE os_get_app_data_dir(Buf *out_path, const char *appname); int os_get_win32_ucrt_include_path(ZigWindowsSDK *sdk, Buf *output_buf); int os_get_win32_ucrt_lib_path(ZigWindowsSDK *sdk, Buf *output_buf, ZigLLVM_ArchType platform_type); int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf *output_buf, ZigLLVM_ArchType platform_type); -#if defined(__APPLE__) -#define ZIG_OS_DARWIN -#elif defined(_WIN32) -#define ZIG_OS_WINDOWS -#elif defined(__linux__) -#define ZIG_OS_LINUX -#else -#define ZIG_OS_UNKNOWN -#endif - -#if defined(__x86_64__) -#define ZIG_ARCH_X86_64 -#else -#define ZIG_ARCH_UNKNOWN -#endif - -#if defined(ZIG_OS_WINDOWS) -#define ZIG_PRI_usize "I64u" -#define ZIG_PRI_u64 "I64u" -#define ZIG_PRI_llu "I64u" -#define ZIG_PRI_x64 "I64x" -#define OS_SEP "\\" -#define ZIG_OS_SEP_CHAR '\\' -#else -#define ZIG_PRI_usize "zu" -#define ZIG_PRI_u64 PRIu64 -#define ZIG_PRI_llu "llu" -#define ZIG_PRI_x64 PRIx64 -#define OS_SEP "/" -#define ZIG_OS_SEP_CHAR '/' -#endif +Error ATTRIBUTE_MUST_USE os_self_exe_shared_libs(ZigList &paths); #endif diff --git a/src/util.cpp b/src/util.cpp index cafd2c3d857ba6170d42b4ac021768ae3aa0a5a7..060d7f8fb5c2e58802b3471fc08d60ddd4d108c7 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -43,3 +43,42 @@ uint32_t ptr_hash(const void *ptr) { bool ptr_eq(const void *a, const void *b) { return a == b; } + +// Ported from std/mem.zig. +bool SplitIterator_isSplitByte(SplitIterator *self, uint8_t byte) { + for (size_t i = 0; i < self->split_bytes.len; i += 1) { + if (byte == self->split_bytes.ptr[i]) { + return true; + } + } + return false; +} + +// Ported from std/mem.zig. +Optional> SplitIterator_next(SplitIterator *self) { + // move to beginning of token + while (self->index < self->buffer.len && + SplitIterator_isSplitByte(self, self->buffer.ptr[self->index])) + { + self->index += 1; + } + size_t start = self->index; + if (start == self->buffer.len) { + return {}; + } + + // move to end of token + while (self->index < self->buffer.len && + !SplitIterator_isSplitByte(self, self->buffer.ptr[self->index])) + { + self->index += 1; + } + size_t end = self->index; + + return Optional>::some(self->buffer.slice(start, end)); +} + +// Ported from std/mem.zig +SplitIterator memSplit(Slice buffer, Slice split_bytes) { + return SplitIterator{0, buffer, split_bytes}; +} diff --git a/src/util.hpp b/src/util.hpp index 1e02e3043c78a84dde791819a8584f4e8be91f03..f18c369fb5b4d61653a9e1a0dd312a09ec63df1a 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -254,4 +254,16 @@ static inline void memCopy(Slice dest, Slice src) { memcpy(dest.ptr, src.ptr, src.len * sizeof(T)); } +// Ported from std/mem.zig. +// Coordinate struct fields with memSplit function +struct SplitIterator { + size_t index; + Slice buffer; + Slice split_bytes; +}; + +bool SplitIterator_isSplitByte(SplitIterator *self, uint8_t byte); +Optional> SplitIterator_next(SplitIterator *self); +SplitIterator memSplit(Slice buffer, Slice split_bytes); + #endif -- 2.54.0 From c0bdcc7417a23d338dc1df6bd74b30fe6e3d1a1a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 9 Sep 2018 23:58:52 -0400 Subject: [PATCH 05/20] `zig id` command --- src/cache_hash.cpp | 1 - src/main.cpp | 113 +++++++++++++++++++++++---------------------- src/os.cpp | 6 +-- 3 files changed, 60 insertions(+), 60 deletions(-) diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index 5447a5f3f2ad8758cf2e74e220ef8432e60c58f1..ff530ca65c53267ef7a23aad3fd45d4896cd4f63 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -377,7 +377,6 @@ static Error write_manifest_file(CacheHash *ch) { buf_appendf(&contents, "%" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %s %s\n", chf->mtime.sec, chf->mtime.nsec, encoded_digest, buf_ptr(chf->path)); } - fprintf(stderr, "overwrite with\n%s\n", buf_ptr(&contents)); if ((err = os_file_overwrite(ch->manifest_file, &contents))) return err; diff --git a/src/main.cpp b/src/main.cpp index 172fe04da9513841d291019e345a07bed4a90174..c385a1c88e329be6845006c35e805a0e792db144 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -25,6 +25,7 @@ static int usage(const char *arg0) { " build-lib [source] create library from source or object files\n" " build-obj [source] create object from source or assembly\n" " builtin show the source code of that @import(\"builtin\")\n" + " id print the base64-encoded compiler id\n" " run [source] create executable and run immediately\n" " translate-c [source] convert c code to zig code\n" " targets list available compilation targets\n" @@ -257,6 +258,55 @@ static void add_package(CodeGen *g, CliPkg *cli_pkg, PackageTableEntry *pkg) { } } +static Buf saved_compiler_id = BUF_INIT; +static Error get_compiler_id(Buf **result) { + if (saved_compiler_id.list.length != 0) { + *result = &saved_compiler_id; + return ErrorNone; + } + + Error err; + Buf app_data_dir = BUF_INIT; + if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) + return err; + Buf *stage1_dir = buf_alloc(); + os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); + + if ((err = os_make_path(manifest_dir))) + return err; + CacheHash cache_hash; + CacheHash *ch = &cache_hash; + cache_init(ch, manifest_dir); + Buf self_exe_path = BUF_INIT; + if ((err = os_self_exe_path(&self_exe_path))) + return err; + + cache_file(ch, &self_exe_path); + + buf_resize(&saved_compiler_id, 0); + if ((err = cache_hit(ch, &saved_compiler_id))) + return err; + if (buf_len(&saved_compiler_id) != 0) { + *result = &saved_compiler_id; + return ErrorNone; + } + ZigList lib_paths = {}; + if ((err = os_self_exe_shared_libs(lib_paths))) + return err; + for (size_t i = 0; i < lib_paths.length; i += 1) { + Buf *lib_path = lib_paths.at(i); + if ((err = cache_add_file(ch, lib_path))) + return err; + } + if ((err = cache_final(ch, &saved_compiler_id))) + return err; + + *result = &saved_compiler_id; + return ErrorNone; +} + int main(int argc, char **argv) { if (argc == 2 && strcmp(argv[1], "BUILD_INFO") == 0) { printf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", @@ -271,64 +321,15 @@ int main(int argc, char **argv) { return 0; } - if (argc == 2 && strcmp(argv[1], "TEST") == 0) { + if (argc == 2 && strcmp(argv[1], "id") == 0) { Error err; - Buf app_data_dir = BUF_INIT; - if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { - fprintf(stderr, "get app dir: %s\n", err_str(err)); - return 1; + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) { + fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); + return EXIT_FAILURE; } - Buf *stage1_dir = buf_alloc(); - os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); - Buf *manifest_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); - - if ((err = os_make_path(manifest_dir))) { - fprintf(stderr, "make path: %s\n", err_str(err)); - return 1; - } - CacheHash cache_hash; - CacheHash *ch = &cache_hash; - cache_init(ch, manifest_dir); - Buf self_exe_path = BUF_INIT; - if ((err = os_self_exe_path(&self_exe_path))) { - fprintf(stderr, "self exe path: %s\n", err_str(err)); - return 1; - } - - cache_file(ch, &self_exe_path); - - Buf exe_digest = BUF_INIT; - buf_resize(&exe_digest, 0); - if ((err = cache_hit(ch, &exe_digest))) { - fprintf(stderr, "cache hit error: %s\n", err_str(err)); - return 1; - } - if (buf_len(&exe_digest) != 0) { - fprintf(stderr, "cache hit: %s\n", buf_ptr(&exe_digest)); - return 0; - } - fprintf(stderr, "cache miss\n"); - ZigList lib_paths = {}; - if ((err = os_self_exe_shared_libs(lib_paths))) { - fprintf(stderr, "finding out shared libs: %s\n", err_str(err)); - return 1; - } - for (size_t i = 0; i < lib_paths.length; i += 1) { - Buf *lib_path = lib_paths.at(i); - if ((err = cache_add_file(ch, lib_path))) { - fprintf(stderr, "cache add file %s: %s", buf_ptr(lib_path), err_str(err)); - return 1; - } - } - if ((err = cache_final(ch, &exe_digest))) { - fprintf(stderr, "final: %s\n", err_str(err)); - return 1; - } - - - fprintf(stderr, "computed2: %s\n", buf_ptr(&exe_digest)); - return 0; + printf("%s\n", buf_ptr(compiler_id)); + return EXIT_SUCCESS; } os_init(); diff --git a/src/os.cpp b/src/os.cpp index d6e17ca5a1521c4eff2d3f608ad2f15a29c281bd..4ee461818401376460f0554267a401677574a84b 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -1813,9 +1813,9 @@ Error os_file_read_all(OsFile file, Buf *contents) { for (;;) { size_t amt = buf_len(contents) - index; - if (amt < 512) { - buf_resize(contents, buf_len(contents) + 512); - amt += 512; + if (amt < 4096) { + buf_resize(contents, buf_len(contents) + (4096 - amt)); + amt = buf_len(contents) - index; } if ((err = os_file_read(file, buf_ptr(contents) + index, &amt))) -- 2.54.0 From fbe5737c84c783cd31e6e2d595fc47eb782c5e3c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 10 Sep 2018 09:46:15 -0400 Subject: [PATCH 06/20] stage1: always optimize blake and softfloat even in debug mode --- CMakeLists.txt | 14 +++++-- src/all_types.hpp | 4 ++ src/cache_hash.cpp | 4 ++ src/cache_hash.hpp | 3 +- src/codegen.cpp | 91 ++++++++++++++++++++++++++++------------------ src/codegen.hpp | 2 +- src/link.cpp | 2 +- src/main.cpp | 21 ++++++++--- 8 files changed, 94 insertions(+), 47 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 615128159f5725cbe093488c39c9acb9bcef4704..8339be71b9b1f9b9b3a2b2b5e2db2c4e2f6278cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -390,7 +390,7 @@ if(MSVC) ) else() set_target_properties(embedded_softfloat PROPERTIES - COMPILE_FLAGS "-std=c99" + COMPILE_FLAGS "-std=c99 -O3" ) endif() target_include_directories(embedded_softfloat PUBLIC @@ -407,10 +407,9 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/ast_render.cpp" "${CMAKE_SOURCE_DIR}/src/bigfloat.cpp" "${CMAKE_SOURCE_DIR}/src/bigint.cpp" - "${CMAKE_SOURCE_DIR}/src/blake2b.cpp" "${CMAKE_SOURCE_DIR}/src/buffer.cpp" - "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp" + "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" "${CMAKE_SOURCE_DIR}/src/errmsg.cpp" "${CMAKE_SOURCE_DIR}/src/error.cpp" @@ -426,6 +425,9 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/util.cpp" "${CMAKE_SOURCE_DIR}/src/translate_c.cpp" ) +set(ZIG_SOURCES_O3 + "${CMAKE_SOURCE_DIR}/src/blake2b.cpp" +) set(ZIG_CPP_SOURCES "${CMAKE_SOURCE_DIR}/src/zig_llvm.cpp" "${CMAKE_SOURCE_DIR}/src/windows_sdk.cpp" @@ -813,6 +815,11 @@ set_target_properties(zig_cpp PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} ) +add_library(zig_O3 STATIC ${ZIG_SOURCES_O3}) +set_target_properties(zig_O3 PROPERTIES + COMPILE_FLAGS "${EXE_CFLAGS} -O3" +) + add_executable(zig ${ZIG_SOURCES}) set_target_properties(zig PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} @@ -821,6 +828,7 @@ set_target_properties(zig PROPERTIES target_link_libraries(zig LINK_PUBLIC zig_cpp + zig_O3 ${SOFTFLOAT_LIBRARIES} ${CLANG_LIBRARIES} ${LLD_LIBRARIES} diff --git a/src/all_types.hpp b/src/all_types.hpp index 0b0de587916b888843151392c4354c1cac967e29..2dfbcaaa623ccc7134b1dafc30ce17394669b697 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -10,6 +10,7 @@ #include "list.hpp" #include "buffer.hpp" +#include "cache_hash.hpp" #include "zig_llvm.h" #include "hash_map.hpp" #include "errmsg.hpp" @@ -1613,7 +1614,10 @@ struct CodeGen { ZigType *entry_promise; } builtin_types; + CacheHash cache_hash; + //////////////////////////// Participates in Input Parameter Cache Hash + Buf *compiler_id; ZigList link_libs_list; // add -framework [name] args to linker ZigList darwin_frameworks; diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index ff530ca65c53267ef7a23aad3fd45d4896cd4f63..a0c43bc8100ee0b885d3e30d7c94f527e2d23fcb 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -6,6 +6,7 @@ */ #include "cache_hash.hpp" +#include "all_types.hpp" #include "buffer.hpp" #include "os.hpp" @@ -219,6 +220,9 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { buf_append_str(ch->manifest_file_path, ".txt"); + if ((err = os_make_path(ch->manifest_dir))) + return err; + if ((err = os_file_open_lock_rw(ch->manifest_file_path, &ch->manifest_file))) return err; diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp index 77b22a1e41bbb903b3061cb1a2ee423694c387e2..3c2d77abc0199999d3cdb36a1c8366cd966d1363 100644 --- a/src/cache_hash.hpp +++ b/src/cache_hash.hpp @@ -8,10 +8,11 @@ #ifndef ZIG_CACHE_HASH_HPP #define ZIG_CACHE_HASH_HPP -#include "all_types.hpp" #include "blake2.h" #include "os.hpp" +struct LinkLib; + struct CacheHashFile { Buf *path; OsTimeStamp mtime; diff --git a/src/codegen.cpp b/src/codegen.cpp index 6c5648f6261c123c3bf422e9614ea3ab77ac07c1..a18b26bb18307ea3b021647c5e4be58cf6e50dac 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -88,12 +88,13 @@ static const char *symbols_that_llvm_depends_on[] = { }; CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir) + Buf *zig_lib_dir, Buf *compiler_id) { CodeGen *g = allocate(1); codegen_add_time_event(g, "Initialize"); + g->compiler_id = compiler_id; g->zig_lib_dir = zig_lib_dir; g->zig_std_dir = buf_alloc(); @@ -7675,44 +7676,62 @@ void codegen_add_time_event(CodeGen *g, const char *name) { } -//// Called before init() -//static bool build_with_cache(CodeGen *g) { -// // TODO zig exe & dynamic libraries -// // should be in main.cpp I think. only needs to happen -// // once on startup. -// -// CacheHash comp; -// cache_init(&comp); -// -// add_cache_buf(&blake, g->root_out_name); -// add_cache_buf_opt(&blake, get_resolved_root_src_path(g)); // Root source file -// add_cache_list_of_link_lib(&blake, g->link_libs_list.items, g->link_libs_list.length); -// add_cache_list_of_buf(&blake, g->darwin_frameworks.items, g->darwin_frameworks.length); -// add_cache_list_of_buf(&blake, g->rpath_list.items, g->rpath_list.length); -// add_cache_int(&blake, g->emit_file_type); -// add_cache_int(&blake, g->build_mode); -// add_cache_int(&blake, g->out_type); -// // TODO the rest of the struct CodeGen fields -// -// uint8_t bin_digest[48]; -// rc = blake2b_final(&blake, bin_digest, 48); -// assert(rc == 0); -// -// Buf b64_digest = BUF_INIT; -// buf_resize(&b64_digest, 64); -// base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); -// -// fprintf(stderr, "input params hash: %s\n", buf_ptr(&b64_digest)); -// // TODO next look for a manifest file that has all the files from the input parameters -// // use that to construct the real hash, which looks up the output directory and another manifest file -// -// return false; -//} +// Called before init() +static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { + Error err; + + CacheHash *ch = &g->cache_hash; + cache_init(ch, manifest_dir); + + cache_buf(ch, g->compiler_id); + cache_buf(ch, g->root_out_name); + cache_file(ch, get_resolved_root_src_path(g)); // Root source file + cache_list_of_link_lib(ch, g->link_libs_list.items, g->link_libs_list.length); + cache_list_of_buf(ch, g->darwin_frameworks.items, g->darwin_frameworks.length); + cache_list_of_buf(ch, g->rpath_list.items, g->rpath_list.length); + cache_int(ch, g->emit_file_type); + cache_int(ch, g->build_mode); + cache_int(ch, g->out_type); + // TODO the rest of the struct CodeGen fields + + buf_resize(digest, 0); + if ((err = cache_hit(ch, digest))) + return err; + + return ErrorNone; +} void codegen_build(CodeGen *g) { + Error err; assert(g->out_type != OutTypeUnknown); - //if (build_with_cache(g)) - // return; + + Buf app_data_dir = BUF_INIT; + if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { + fprintf(stderr, "Unable to get app data dir: %s\n", err_str(err)); + exit(1); + } + Buf *stage1_dir = buf_alloc(); + os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); + + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("build"), manifest_dir); + + Buf digest = BUF_INIT; + if ((err = check_cache(g, manifest_dir, &digest))) { + fprintf(stderr, "Unable to check cache: %s\n", err_str(err)); + exit(1); + } + if (buf_len(&digest) != 0) { + Buf *artifact_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + + Buf *this_artifact_dir = buf_alloc(); + os_path_join(artifact_dir, &digest, this_artifact_dir); + + fprintf(stderr, "copy artifacts from %s\n", buf_ptr(this_artifact_dir)); + return; + } + init(g); codegen_add_time_event(g, "Semantic Analysis"); diff --git a/src/codegen.hpp b/src/codegen.hpp index 55b38a00509d989491162672979812337533389a..649cf06856e6ff02f46a5c26cec626f259b6cdfc 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -15,7 +15,7 @@ #include CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir); + Buf *zig_lib_dir, Buf *compiler_id); void codegen_set_clang_argv(CodeGen *codegen, const char **args, size_t len); void codegen_set_llvm_argv(CodeGen *codegen, const char **args, size_t len); diff --git a/src/link.cpp b/src/link.cpp index 78ad204fe93657f68945a2007415d911b79c4bf6..c0eb25bd55141f70808688955c210f65cc54c993 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -34,7 +34,7 @@ static const char *get_libc_static_file(CodeGen *g, const char *file) { static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) { ZigTarget *child_target = parent_gen->is_native_target ? nullptr : &parent_gen->zig_target; CodeGen *child_gen = codegen_create(full_path, child_target, OutTypeObj, parent_gen->build_mode, - parent_gen->zig_lib_dir); + parent_gen->zig_lib_dir, parent_gen->compiler_id); child_gen->want_h_file = false; child_gen->verbose_tokenize = parent_gen->verbose_tokenize; diff --git a/src/main.cpp b/src/main.cpp index c385a1c88e329be6845006c35e805a0e792db144..1520ae7d67ad81df5ccb0cfdbe8b8a088818fc36 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -274,8 +274,6 @@ static Error get_compiler_id(Buf **result) { Buf *manifest_dir = buf_alloc(); os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); - if ((err = os_make_path(manifest_dir))) - return err; CacheHash cache_hash; CacheHash *ch = &cache_hash; cache_init(ch, manifest_dir); @@ -432,8 +430,14 @@ int main(int argc, char **argv) { Buf *build_runner_path = buf_alloc(); os_path_join(special_dir, buf_create_from_str("build_runner.zig"), build_runner_path); + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) { + fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); + return EXIT_FAILURE; + } - CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf); + CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf, + compiler_id); codegen_set_out_name(g, buf_create_from_str("build")); Buf *build_file_buf = buf_create_from_str(build_file); @@ -796,7 +800,7 @@ int main(int argc, char **argv) { switch (cmd) { case CmdBuiltin: { Buf *zig_lib_dir_buf = resolve_zig_lib_dir(); - CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, zig_lib_dir_buf); + CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, zig_lib_dir_buf, nullptr); Buf *builtin_source = codegen_generate_builtin_source(g); if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) { fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout))); @@ -871,7 +875,14 @@ int main(int argc, char **argv) { Buf *zig_lib_dir_buf = resolve_zig_lib_dir(); - CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf); + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) { + fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); + return EXIT_FAILURE; + } + + CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf, + compiler_id); codegen_set_out_name(g, buf_out_name); codegen_set_lib_version(g, ver_major, ver_minor, ver_patch); codegen_set_is_test(g, cmd == CmdTest); -- 2.54.0 From 32be6e9b2a9e6de501aadbe271c554a4682a10f8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 10 Sep 2018 13:46:23 -0400 Subject: [PATCH 07/20] caching is working * add almost all the input parameter state to the hash - missing items are the detected MSVC installation on Windows and detected libc installation on POSIX - also missing are C files and .h files that libclang finds * artifacts are created in global cache directory instead of zig-cache. - exception: builtin.zig is still in zig-cache * zig run uses the new cache correctly * zig run uses execv on posix systems --- src/all_types.hpp | 275 ++++++++++++++++++++++----------------------- src/cache_hash.cpp | 86 +++++++++++--- src/cache_hash.hpp | 27 +++-- src/codegen.cpp | 199 ++++++++++++++++++++++---------- src/codegen.hpp | 4 +- src/error.cpp | 1 + src/error.hpp | 1 + src/ir.cpp | 11 +- src/link.cpp | 74 +++--------- src/link.hpp | 17 --- src/main.cpp | 64 +++++------ src/os.cpp | 54 +++------ src/os.hpp | 3 +- 13 files changed, 432 insertions(+), 384 deletions(-) delete mode 100644 src/link.hpp diff --git a/src/all_types.hpp b/src/all_types.hpp index 2dfbcaaa623ccc7134b1dafc30ce17394669b697..6d11244a2526f0c167147bd79dbfeaf85bfa5daa 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1554,6 +1554,40 @@ struct CodeGen { ZigLLVMDICompileUnit *compile_unit; ZigLLVMDIFile *compile_unit_file; LinkLib *libc_link_lib; + LLVMTargetDataRef target_data_ref; + LLVMTargetMachineRef target_machine; + ZigLLVMDIFile *dummy_di_file; + LLVMValueRef cur_ret_ptr; + LLVMValueRef cur_fn_val; + LLVMValueRef cur_err_ret_trace_val_arg; + LLVMValueRef cur_err_ret_trace_val_stack; + LLVMValueRef memcpy_fn_val; + LLVMValueRef memset_fn_val; + LLVMValueRef trap_fn_val; + LLVMValueRef return_address_fn_val; + LLVMValueRef frame_address_fn_val; + LLVMValueRef coro_destroy_fn_val; + LLVMValueRef coro_id_fn_val; + LLVMValueRef coro_alloc_fn_val; + LLVMValueRef coro_size_fn_val; + LLVMValueRef coro_begin_fn_val; + LLVMValueRef coro_suspend_fn_val; + LLVMValueRef coro_end_fn_val; + LLVMValueRef coro_free_fn_val; + LLVMValueRef coro_resume_fn_val; + LLVMValueRef coro_save_fn_val; + LLVMValueRef coro_promise_fn_val; + LLVMValueRef coro_alloc_helper_fn_val; + LLVMValueRef coro_frame_fn_val; + LLVMValueRef merge_err_ret_traces_fn_val; + LLVMValueRef add_error_return_trace_addr_fn_val; + LLVMValueRef stacksave_fn_val; + LLVMValueRef stackrestore_fn_val; + LLVMValueRef write_register_fn_val; + LLVMValueRef sp_md_node; + LLVMValueRef err_name_table; + LLVMValueRef safety_crash_err_fn; + LLVMValueRef return_err_fn; // reminder: hash tables must be initialized before use HashMap import_table; @@ -1576,8 +1610,23 @@ struct CodeGen { size_t resolve_queue_index; ZigList use_queue; size_t use_queue_index; + ZigList timing_events; + ZigList error_di_types; + ZigList tld_ref_source_node_stack; + ZigList inline_fns; + ZigList test_fns; + ZigList err_enumerators; + ZigList errors_by_index; + size_t largest_err_name_len; - uint32_t next_unresolved_index; + PackageTableEntry *std_package; + PackageTableEntry *panic_package; + PackageTableEntry *test_runner_package; + PackageTableEntry *compile_var_package; + ImportTableEntry *compile_var_import; + ImportTableEntry *root_import; + ImportTableEntry *bootstrap_import; + ImportTableEntry *test_runner_import; struct { ZigType *entry_bool; @@ -1613,30 +1662,45 @@ struct CodeGen { ZigType *entry_arg_tuple; ZigType *entry_promise; } builtin_types; + ZigType *align_amt_type; + ZigType *stack_trace_type; + ZigType *ptr_to_stack_trace_type; + ZigType *err_tag_type; + ZigType *test_fn_type; + + Buf triple_str; + Buf global_asm; + Buf *out_h_path; + Buf cache_dir; + Buf artifact_dir; + Buf output_file_path; + Buf o_file_output_path; + Buf *wanted_output_file_path; + + IrInstruction *invalid_instruction; + + ConstExprValue const_void_val; + ConstExprValue panic_msg_vals[PanicMsgIdCount]; + + // The function definitions this module includes. + ZigList fn_defs; + size_t fn_defs_index; + ZigList global_vars; + + ZigFn *cur_fn; + ZigFn *main_fn; + ZigFn *panic_fn; + AstNode *root_export_decl; CacheHash cache_hash; - - //////////////////////////// Participates in Input Parameter Cache Hash - Buf *compiler_id; - ZigList link_libs_list; - // add -framework [name] args to linker - ZigList darwin_frameworks; - // add -rpath [name] args to linker - ZigList rpath_list; - - EmitFileType emit_file_type; - BuildMode build_mode; - OutType out_type; - - - //////////////////////////// Unsorted - - ZigTarget zig_target; - LLVMTargetDataRef target_data_ref; + ErrColor err_color; + uint32_t next_unresolved_index; unsigned pointer_size_bytes; + uint32_t target_os_index; + uint32_t target_arch_index; + uint32_t target_environ_index; + uint32_t target_oformat_index; bool is_big_endian; - bool is_static; - bool strip_debug_symbols; bool want_h_file; bool have_pub_main; bool have_c_main; @@ -1644,148 +1708,75 @@ struct CodeGen { bool have_winmain_crt_startup; bool have_dllmain_crt_startup; bool have_pub_panic; - Buf *libc_lib_dir; - Buf *libc_static_lib_dir; - Buf *libc_include_dir; - Buf *msvc_lib_dir; - Buf *kernel32_lib_dir; - Buf *zig_lib_dir; - Buf *zig_std_dir; - Buf *zig_c_headers_dir; - Buf *zig_std_special_dir; - Buf *dynamic_linker; - Buf *ar_path; - ZigWindowsSDK *win_sdk; - Buf triple_str; - bool is_test_build; bool have_err_ret_tracing; - uint32_t target_os_index; - uint32_t target_arch_index; - uint32_t target_environ_index; - uint32_t target_oformat_index; - LLVMTargetMachineRef target_machine; - ZigLLVMDIFile *dummy_di_file; - bool is_native_target; - PackageTableEntry *root_package; // participates in cache hash - PackageTableEntry *std_package; - PackageTableEntry *panic_package; - PackageTableEntry *test_runner_package; - PackageTableEntry *compile_var_package; - ImportTableEntry *compile_var_import; - Buf *root_out_name; // participates in cache hash - bool windows_subsystem_windows; - bool windows_subsystem_console; - Buf *mmacosx_version_min; - Buf *mios_version_min; - bool linker_rdynamic; - const char *linker_script; - - // The function definitions this module includes. - ZigList fn_defs; - size_t fn_defs_index; - ZigList global_vars; - - ZigFn *cur_fn; - ZigFn *main_fn; - ZigFn *panic_fn; - LLVMValueRef cur_ret_ptr; - LLVMValueRef cur_fn_val; - LLVMValueRef cur_err_ret_trace_val_arg; - LLVMValueRef cur_err_ret_trace_val_stack; bool c_want_stdint; bool c_want_stdbool; - AstNode *root_export_decl; - size_t version_major; - size_t version_minor; - size_t version_patch; bool verbose_tokenize; bool verbose_ast; bool verbose_link; bool verbose_ir; bool verbose_llvm_ir; bool verbose_cimport; - ErrColor err_color; - ImportTableEntry *root_import; - ImportTableEntry *bootstrap_import; - ImportTableEntry *test_runner_import; - LLVMValueRef memcpy_fn_val; - LLVMValueRef memset_fn_val; - LLVMValueRef trap_fn_val; - LLVMValueRef return_address_fn_val; - LLVMValueRef frame_address_fn_val; - LLVMValueRef coro_destroy_fn_val; - LLVMValueRef coro_id_fn_val; - LLVMValueRef coro_alloc_fn_val; - LLVMValueRef coro_size_fn_val; - LLVMValueRef coro_begin_fn_val; - LLVMValueRef coro_suspend_fn_val; - LLVMValueRef coro_end_fn_val; - LLVMValueRef coro_free_fn_val; - LLVMValueRef coro_resume_fn_val; - LLVMValueRef coro_save_fn_val; - LLVMValueRef coro_promise_fn_val; - LLVMValueRef coro_alloc_helper_fn_val; - LLVMValueRef coro_frame_fn_val; - LLVMValueRef merge_err_ret_traces_fn_val; - LLVMValueRef add_error_return_trace_addr_fn_val; - LLVMValueRef stacksave_fn_val; - LLVMValueRef stackrestore_fn_val; - LLVMValueRef write_register_fn_val; bool error_during_imports; - - LLVMValueRef sp_md_node; - - const char **clang_argv; - size_t clang_argv_len; - ZigList lib_dirs; - - const char **llvm_argv; - size_t llvm_argv_len; - - ZigList test_fns; - ZigType *test_fn_type; - - bool each_lib_rpath; - - ZigType *err_tag_type; - ZigList err_enumerators; - ZigList errors_by_index; bool generate_error_name_table; - LLVMValueRef err_name_table; - size_t largest_err_name_len; - LLVMValueRef safety_crash_err_fn; - LLVMValueRef return_err_fn; - - IrInstruction *invalid_instruction; - ConstExprValue const_void_val; - - ConstExprValue panic_msg_vals[PanicMsgIdCount]; - - Buf global_asm; + //////////////////////////// Participates in Input Parameter Cache Hash + ZigList link_libs_list; + // add -framework [name] args to linker + ZigList darwin_frameworks; + // add -rpath [name] args to linker + ZigList rpath_list; + ZigList forbidden_libs; ZigList link_objects; ZigList assembly_files; + ZigList lib_dirs; + Buf *compiler_id; + size_t version_major; + size_t version_minor; + size_t version_patch; + const char *linker_script; + + EmitFileType emit_file_type; + BuildMode build_mode; + OutType out_type; + ZigTarget zig_target; + bool is_static; + bool strip_debug_symbols; + bool is_test_build; + bool is_native_target; + bool windows_subsystem_windows; + bool windows_subsystem_console; + bool linker_rdynamic; + bool no_rosegment_workaround; + bool each_lib_rpath; + + Buf *mmacosx_version_min; + Buf *mios_version_min; + Buf *root_out_name; Buf *test_filter; Buf *test_name_prefix; + PackageTableEntry *root_package; - ZigList timing_events; + const char **llvm_argv; + size_t llvm_argv_len; - Buf cache_dir; - Buf *out_h_path; + const char **clang_argv; + size_t clang_argv_len; - ZigList inline_fns; - ZigList tld_ref_source_node_stack; + //////////////////////////// Unsorted - ZigType *align_amt_type; - ZigType *stack_trace_type; - ZigType *ptr_to_stack_trace_type; - - ZigList error_di_types; - - ZigList forbidden_libs; - - bool no_rosegment_workaround; + Buf *libc_lib_dir; + Buf *libc_static_lib_dir; + Buf *libc_include_dir; + Buf *msvc_lib_dir; + Buf *kernel32_lib_dir; + Buf *zig_lib_dir; + Buf *zig_std_dir; + Buf *zig_c_headers_dir; + Buf *zig_std_special_dir; + Buf *dynamic_linker; + ZigWindowsSDK *win_sdk; }; enum VarLinkage { diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index a0c43bc8100ee0b885d3e30d7c94f527e2d23fcb..b6aebdac671bdae04ab00a3f98128e19b6e79b7d 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -37,6 +37,20 @@ void cache_int(CacheHash *ch, int x) { blake2b_update(&ch->blake, buf, sizeof(int) + 1); } +void cache_usize(CacheHash *ch, size_t x) { + assert(ch->manifest_file_path == nullptr); + // + 1 to include the null byte + uint8_t buf[sizeof(size_t) + 1]; + memcpy(buf, &x, sizeof(size_t)); + buf[sizeof(size_t)] = 0; + blake2b_update(&ch->blake, buf, sizeof(size_t) + 1); +} + +void cache_bool(CacheHash *ch, bool x) { + assert(ch->manifest_file_path == nullptr); + blake2b_update(&ch->blake, &x, 1); +} + void cache_buf(CacheHash *ch, Buf *buf) { assert(ch->manifest_file_path == nullptr); assert(buf != nullptr); @@ -74,6 +88,26 @@ void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len) { cache_str(ch, ""); } +void cache_list_of_file(CacheHash *ch, Buf **ptr, size_t len) { + assert(ch->manifest_file_path == nullptr); + + for (size_t i = 0; i < len; i += 1) { + Buf *buf = ptr[i]; + cache_file(ch, buf); + } + cache_str(ch, ""); +} + +void cache_list_of_str(CacheHash *ch, const char **ptr, size_t len) { + assert(ch->manifest_file_path == nullptr); + + for (size_t i = 0; i < len; i += 1) { + const char *s = ptr[i]; + cache_str(ch, s); + } + cache_str(ch, ""); +} + void cache_file(CacheHash *ch, Buf *file_path) { assert(ch->manifest_file_path == nullptr); assert(file_path != nullptr); @@ -154,9 +188,13 @@ static Error base64_decode(Slice dest, Slice source) { return ErrorNone; } -static Error hash_file(uint8_t *digest, OsFile handle) { +static Error hash_file(uint8_t *digest, OsFile handle, Buf *contents) { Error err; + if (contents) { + buf_resize(contents, 0); + } + blake2b_state blake; int rc = blake2b_init(&blake, 48); assert(rc == 0); @@ -172,10 +210,13 @@ static Error hash_file(uint8_t *digest, OsFile handle) { return ErrorNone; } blake2b_update(&blake, buf, amt); + if (contents) { + buf_append_mem(contents, (char*)buf, amt); + } } } -static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf) { +static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf, Buf *contents) { Error err; assert(chf->path != nullptr); @@ -189,7 +230,7 @@ static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf) { return err; } - if ((err = hash_file(chf->bin_digest, this_file))) { + if ((err = hash_file(chf->bin_digest, this_file, contents))) { os_file_close(this_file); return err; } @@ -323,7 +364,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { chf->mtime = actual_mtime; uint8_t actual_digest[48]; - if ((err = hash_file(actual_digest, this_file))) { + if ((err = hash_file(actual_digest, this_file, nullptr))) { os_file_close(this_file); os_file_close(ch->manifest_file); return err; @@ -344,7 +385,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { ch->manifest_dirty = true; for (; file_i < input_file_count; file_i += 1) { CacheHashFile *chf = &ch->files.at(file_i); - if ((err = populate_file_hash(ch, chf))) { + if ((err = populate_file_hash(ch, chf, nullptr))) { os_file_close(ch->manifest_file); return err; } @@ -355,18 +396,24 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { return cache_final(ch, out_digest); } +Error cache_add_file_fetch(CacheHash *ch, Buf *resolved_path, Buf *contents) { + Error err; + + assert(ch->manifest_file_path != nullptr); + CacheHashFile *chf = ch->files.add_one(); + chf->path = resolved_path; + if ((err = populate_file_hash(ch, chf, contents))) { + os_file_close(ch->manifest_file); + return err; + } + + return ErrorNone; +} + Error cache_add_file(CacheHash *ch, Buf *path) { - Error err; - - assert(ch->manifest_file_path != nullptr); - CacheHashFile *chf = ch->files.add_one(); - chf->path = path; - if ((err = populate_file_hash(ch, chf))) { - os_file_close(ch->manifest_file); - return err; - } - - return ErrorNone; + Buf *resolved_path = buf_alloc(); + *resolved_path = os_path_resolve(&path, 1); + return cache_add_file_fetch(ch, resolved_path, nullptr); } static Error write_manifest_file(CacheHash *ch) { @@ -398,7 +445,8 @@ Error cache_final(CacheHash *ch, Buf *out_digest) { buf_ptr(ch->manifest_file_path), err_str(err)); } } - os_file_close(ch->manifest_file); + // We don't close the manifest file yet, because we want to + // keep it locked until the API user is done using it. uint8_t bin_digest[48]; int rc = blake2b_final(&ch->blake, bin_digest, 48); @@ -408,3 +456,7 @@ Error cache_final(CacheHash *ch, Buf *out_digest) { return ErrorNone; } + +void cache_release(CacheHash *ch) { + os_file_close(ch->manifest_file); +} diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp index 3c2d77abc0199999d3cdb36a1c8366cd966d1363..ede7344c7531fcdde22cb715bc63e234313902c2 100644 --- a/src/cache_hash.hpp +++ b/src/cache_hash.hpp @@ -17,6 +17,7 @@ struct CacheHashFile { Buf *path; OsTimeStamp mtime; uint8_t bin_digest[48]; + Buf *contents; }; struct CacheHash { @@ -34,24 +35,36 @@ void cache_init(CacheHash *ch, Buf *manifest_dir); // Next, use the hash population functions to add the initial parameters. void cache_str(CacheHash *ch, const char *ptr); void cache_int(CacheHash *ch, int x); +void cache_bool(CacheHash *ch, bool x); +void cache_usize(CacheHash *ch, size_t x); void cache_buf(CacheHash *ch, Buf *buf); void cache_buf_opt(CacheHash *ch, Buf *buf); void cache_list_of_link_lib(CacheHash *ch, LinkLib **ptr, size_t len); void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len); +void cache_list_of_file(CacheHash *ch, Buf **ptr, size_t len); +void cache_list_of_str(CacheHash *ch, const char **ptr, size_t len); void cache_file(CacheHash *ch, Buf *path); void cache_file_opt(CacheHash *ch, Buf *path); // Then call cache_hit when you're ready to see if you can skip the next step. -// out_b64_digest will be left unchanged if it was a cache miss +// out_b64_digest will be left unchanged if it was a cache miss. +// If you got a cache hit, the next step is cache_release. +// From this point on, there is a lock on the input params. Release +// the lock with cache_release. Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest); -// If you got a cache hit, the flow is done. No more functions to call. -// Next call this function for every file that is depended on. +// If you did not get a cache hit, call this function for every file +// that is depended on, and then finish with cache_final. Error ATTRIBUTE_MUST_USE cache_add_file(CacheHash *ch, Buf *path); -// If you did not get a cache hit, use the hash population functions again -// and do all the actual work. When done use cache_final to save the cache -// for next time. -Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_digest); +// This variant of cache_add_file returns the file contents. +// Also the file path argument must be already resolved. +Error ATTRIBUTE_MUST_USE cache_add_file_fetch(CacheHash *ch, Buf *resolved_path, Buf *contents); + +// out_b64_digest will be the same thing that cache_hit returns if you got a cache hit +Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_b64_digest); + +// Until this function is called, no one will be able to get a lock on your input params. +void cache_release(CacheHash *ch); #endif diff --git a/src/codegen.cpp b/src/codegen.cpp index a18b26bb18307ea3b021647c5e4be58cf6e50dac..1f53bb4c9d9124a495d444ab7cbbdf4dec510e11 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -13,7 +13,6 @@ #include "error.hpp" #include "hash_map.hpp" #include "ir.hpp" -#include "link.hpp" #include "os.hpp" #include "translate_c.hpp" #include "target.hpp" @@ -188,6 +187,10 @@ void codegen_set_output_h_path(CodeGen *g, Buf *h_path) { g->out_h_path = h_path; } +void codegen_set_output_path(CodeGen *g, Buf *path) { + g->wanted_output_file_path = path; +} + void codegen_set_clang_argv(CodeGen *g, const char **args, size_t len) { g->clang_argv = args; g->clang_argv_len = len; @@ -5756,8 +5759,6 @@ static void validate_inline_fns(CodeGen *g) { static void do_code_gen(CodeGen *g) { assert(!g->errors.length); - codegen_add_time_event(g, "Code Generation"); - { // create debug type for error sets assert(g->err_enumerators.length == g->errors_by_index.length); @@ -6068,38 +6069,10 @@ static void do_code_gen(CodeGen *g) { codegen_add_time_event(g, "LLVM Emit Output"); - char *err_msg = nullptr; - Buf *o_basename = buf_create_from_buf(g->root_out_name); - - switch (g->emit_file_type) { - case EmitFileTypeBinary: - { - const char *o_ext = target_o_file_ext(&g->zig_target); - buf_append_str(o_basename, o_ext); - break; - } - case EmitFileTypeAssembly: - { - const char *asm_ext = target_asm_file_ext(&g->zig_target); - buf_append_str(o_basename, asm_ext); - break; - } - case EmitFileTypeLLVMIr: - { - const char *llvm_ir_ext = target_llvm_ir_file_ext(&g->zig_target); - buf_append_str(o_basename, llvm_ir_ext); - break; - } - default: - zig_unreachable(); - } - - Buf *output_path = buf_alloc(); - os_path_join(&g->cache_dir, o_basename, output_path); - ensure_cache_dir(g); - bool is_small = g->build_mode == BuildModeSmallRelease; + Buf *output_path = &g->o_file_output_path; + char *err_msg = nullptr; switch (g->emit_file_type) { case EmitFileTypeBinary: if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path), @@ -6118,7 +6091,6 @@ static void do_code_gen(CodeGen *g) { zig_panic("unable to write assembly file %s: %s", buf_ptr(output_path), err_msg); } validate_inline_fns(g); - g->link_objects.append(output_path); break; case EmitFileTypeLLVMIr: @@ -6128,7 +6100,6 @@ static void do_code_gen(CodeGen *g) { zig_panic("unable to write llvm-ir file %s: %s", buf_ptr(output_path), err_msg); } validate_inline_fns(g); - g->link_objects.append(output_path); break; default: @@ -7035,8 +7006,8 @@ static ImportTableEntry *add_special_code(CodeGen *g, PackageTableEntry *package Buf *resolved_path = buf_alloc(); *resolved_path = os_path_resolve(resolve_paths, 1); Buf *import_code = buf_alloc(); - int err; - if ((err = os_fetch_file_path(resolved_path, import_code, false))) { + Error err; + if ((err = cache_add_file_fetch(&g->cache_hash, resolved_path, import_code))) { zig_panic("unable to open '%s': %s\n", buf_ptr(&path_to_code_src), err_str(err)); } @@ -7131,6 +7102,8 @@ static void gen_root_source(CodeGen *g) { Buf *source_code = buf_alloc(); int err; + // No need for using the caching system for this file fetch because it is handled + // separately. if ((err = os_fetch_file_path(resolved_path, source_code, true))) { fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(resolved_path), err_str(err)); exit(1); @@ -7197,6 +7170,8 @@ static void gen_global_asm(CodeGen *g) { int err; for (size_t i = 0; i < g->assembly_files.length; i += 1) { Buf *asm_file = g->assembly_files.at(i); + // No need to use the caching system for these fetches because they + // are handled separately. if ((err = os_fetch_file_path(asm_file, &contents, false))) { zig_panic("Unable to read %s: %s", buf_ptr(asm_file), err_str(err)); } @@ -7460,14 +7435,9 @@ static Buf *preprocessor_mangle(Buf *src) { } static void gen_h_file(CodeGen *g) { - if (!g->want_h_file) - return; - GenH gen_h_data = {0}; GenH *gen_h = &gen_h_data; - codegen_add_time_event(g, "Generate .h"); - assert(!g->is_test_build); if (!g->out_h_path) { @@ -7675,6 +7645,24 @@ void codegen_add_time_event(CodeGen *g, const char *name) { g->timing_events.append({os_get_time(), name}); } +static void add_cache_pkg(CodeGen *g, CacheHash *ch, PackageTableEntry *pkg) { + if (buf_len(&pkg->root_src_path) == 0) + return; + + Buf *rel_full_path = buf_alloc(); + os_path_join(&pkg->root_src_dir, &pkg->root_src_path, rel_full_path); + cache_file(ch, rel_full_path); + + auto it = pkg->package_table.entry_iterator(); + for (;;) { + auto *entry = it.next(); + if (!entry) + break; + + cache_buf(ch, entry->key); + add_cache_pkg(g, ch, entry->value); + } +} // Called before init() static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { @@ -7683,16 +7671,46 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { CacheHash *ch = &g->cache_hash; cache_init(ch, manifest_dir); + add_cache_pkg(g, ch, g->root_package); + if (g->linker_script != nullptr) { + cache_file(ch, buf_create_from_str(g->linker_script)); + } cache_buf(ch, g->compiler_id); cache_buf(ch, g->root_out_name); - cache_file(ch, get_resolved_root_src_path(g)); // Root source file cache_list_of_link_lib(ch, g->link_libs_list.items, g->link_libs_list.length); cache_list_of_buf(ch, g->darwin_frameworks.items, g->darwin_frameworks.length); cache_list_of_buf(ch, g->rpath_list.items, g->rpath_list.length); + cache_list_of_buf(ch, g->forbidden_libs.items, g->forbidden_libs.length); + cache_list_of_file(ch, g->link_objects.items, g->link_objects.length); + cache_list_of_file(ch, g->assembly_files.items, g->assembly_files.length); cache_int(ch, g->emit_file_type); cache_int(ch, g->build_mode); cache_int(ch, g->out_type); - // TODO the rest of the struct CodeGen fields + cache_int(ch, g->zig_target.arch.arch); + cache_int(ch, g->zig_target.arch.sub_arch); + cache_int(ch, g->zig_target.vendor); + cache_int(ch, g->zig_target.os); + cache_int(ch, g->zig_target.env_type); + cache_int(ch, g->zig_target.oformat); + cache_bool(ch, g->is_static); + cache_bool(ch, g->strip_debug_symbols); + cache_bool(ch, g->is_test_build); + cache_bool(ch, g->is_native_target); + cache_bool(ch, g->windows_subsystem_windows); + cache_bool(ch, g->windows_subsystem_console); + cache_bool(ch, g->linker_rdynamic); + cache_bool(ch, g->no_rosegment_workaround); + cache_bool(ch, g->each_lib_rpath); + cache_buf_opt(ch, g->mmacosx_version_min); + cache_buf_opt(ch, g->mios_version_min); + cache_usize(ch, g->version_major); + cache_usize(ch, g->version_minor); + cache_usize(ch, g->version_patch); + cache_buf_opt(ch, g->test_filter); + cache_buf_opt(ch, g->test_name_prefix); + cache_list_of_str(ch, g->llvm_argv, g->llvm_argv_len); + cache_list_of_str(ch, g->clang_argv, g->clang_argv_len); + cache_list_of_str(ch, g->lib_dirs.items, g->lib_dirs.length); buf_resize(digest, 0); if ((err = cache_hit(ch, digest))) @@ -7701,10 +7719,53 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { return ErrorNone; } -void codegen_build(CodeGen *g) { +static void resolve_out_paths(CodeGen *g) { + Buf *o_basename = buf_create_from_buf(g->root_out_name); + + switch (g->emit_file_type) { + case EmitFileTypeBinary: + { + const char *o_ext = target_o_file_ext(&g->zig_target); + buf_append_str(o_basename, o_ext); + break; + } + case EmitFileTypeAssembly: + { + const char *asm_ext = target_asm_file_ext(&g->zig_target); + buf_append_str(o_basename, asm_ext); + break; + } + case EmitFileTypeLLVMIr: + { + const char *llvm_ir_ext = target_llvm_ir_file_ext(&g->zig_target); + buf_append_str(o_basename, llvm_ir_ext); + break; + } + default: + zig_unreachable(); + } + + os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path); + + if (g->out_type == OutTypeObj) { + buf_init_from_buf(&g->output_file_path, &g->o_file_output_path); + } else if (g->out_type == OutTypeExe) { + assert(g->root_out_name); + + Buf basename = BUF_INIT; + buf_init_from_buf(&basename, g->root_out_name); + buf_append_str(&basename, target_exe_file_ext(&g->zig_target)); + os_path_join(&g->artifact_dir, &basename, &g->output_file_path); + } +} + + +void codegen_build_and_link(CodeGen *g) { Error err; assert(g->out_type != OutTypeUnknown); + codegen_add_time_event(g, "Check Cache"); + Buf app_data_dir = BUF_INIT; if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { fprintf(stderr, "Unable to get app data dir: %s\n", err_str(err)); @@ -7721,25 +7782,45 @@ void codegen_build(CodeGen *g) { fprintf(stderr, "Unable to check cache: %s\n", err_str(err)); exit(1); } + + Buf *artifact_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + if (buf_len(&digest) != 0) { - Buf *artifact_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + os_path_join(artifact_dir, &digest, &g->artifact_dir); + resolve_out_paths(g); + } else { + init(g); - Buf *this_artifact_dir = buf_alloc(); - os_path_join(artifact_dir, &digest, this_artifact_dir); + codegen_add_time_event(g, "Semantic Analysis"); - fprintf(stderr, "copy artifacts from %s\n", buf_ptr(this_artifact_dir)); - return; + gen_global_asm(g); + gen_root_source(g); + + if ((err = cache_final(&g->cache_hash, &digest))) { + fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); + exit(1); + } + os_path_join(artifact_dir, &digest, &g->artifact_dir); + if ((err = os_make_path(&g->artifact_dir))) { + fprintf(stderr, "Unable to create artifact directory: %s\n", err_str(err)); + exit(1); + } + resolve_out_paths(g); + + codegen_add_time_event(g, "Code Generation"); + do_code_gen(g); + if (g->want_h_file) { + codegen_add_time_event(g, "Generate .h"); + gen_h_file(g); + } + if (g->out_type != OutTypeObj) { + codegen_link(g); + } } + // TODO hard link output_file_path to wanted_output_file_path - init(g); - - codegen_add_time_event(g, "Semantic Analysis"); - - gen_global_asm(g); - gen_root_source(g); - do_code_gen(g); - gen_h_file(g); + codegen_add_time_event(g, "Done"); } PackageTableEntry *codegen_create_package(CodeGen *g, const char *root_src_dir, const char *root_src_path) { diff --git a/src/codegen.hpp b/src/codegen.hpp index 649cf06856e6ff02f46a5c26cec626f259b6cdfc..1e7fafa28af69958f60e8de2181d27465fda06b4 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -48,9 +48,11 @@ void codegen_set_test_name_prefix(CodeGen *g, Buf *prefix); void codegen_set_lib_version(CodeGen *g, size_t major, size_t minor, size_t patch); void codegen_set_cache_dir(CodeGen *g, Buf cache_dir); void codegen_set_output_h_path(CodeGen *g, Buf *h_path); +void codegen_set_output_path(CodeGen *g, Buf *path); void codegen_add_time_event(CodeGen *g, const char *name); void codegen_print_timing_report(CodeGen *g, FILE *f); -void codegen_build(CodeGen *g); +void codegen_link(CodeGen *g); +void codegen_build_and_link(CodeGen *g); PackageTableEntry *codegen_create_package(CodeGen *g, const char *root_src_dir, const char *root_src_path); void codegen_add_assembly(CodeGen *g, Buf *path); diff --git a/src/error.cpp b/src/error.cpp index d9da6e6c52f248032ab4635b03224ce973172c20..d503eaa18b5a37af668cc129933dc24b6f44e080 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -29,6 +29,7 @@ const char *err_str(int err) { case ErrorCCompileErrors: return "C compile errors"; case ErrorEndOfFile: return "end of file"; case ErrorIsDir: return "is directory"; + case ErrorUnsupportedOperatingSystem: return "unsupported operating system"; } return "(invalid error)"; } diff --git a/src/error.hpp b/src/error.hpp index c99c6e6b12d73043e24cb3e5eac748e3b41d93d0..8c1f8b9aa585ba876161f204988adbdbb66e93d9 100644 --- a/src/error.hpp +++ b/src/error.hpp @@ -29,6 +29,7 @@ enum Error { ErrorCCompileErrors, ErrorEndOfFile, ErrorIsDir, + ErrorUnsupportedOperatingSystem, }; const char *err_str(int err); diff --git a/src/ir.cpp b/src/ir.cpp index 0269c29b1aa1459ce03d972aa8981109cea3ee81..22d9a9bc49e822cdaa1d0cb7943830a850f72809 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -16232,6 +16232,8 @@ static ZigType *ir_analyze_instruction_union_tag(IrAnalyze *ira, IrInstructionUn } static ZigType *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructionImport *import_instruction) { + Error err; + IrInstruction *name_value = import_instruction->name->other; Buf *import_target_str = ir_resolve_str(ira, name_value); if (!import_target_str) @@ -16275,8 +16277,7 @@ static ZigType *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructionImpor return ira->codegen->builtin_types.entry_namespace; } - int err; - if ((err = os_fetch_file_path(resolved_path, import_code, true))) { + if ((err = cache_add_file_fetch(&ira->codegen->cache_hash, resolved_path, import_code))) { if (err == ErrorFileNotFound) { ir_add_error_node(ira, source_node, buf_sprintf("unable to find '%s'", buf_ptr(import_target_path))); @@ -16287,6 +16288,7 @@ static ZigType *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructionImpor return ira->codegen->builtin_types.entry_invalid; } } + ImportTableEntry *target_import = add_source_file(ira->codegen, target_package, resolved_path, import_code); scan_import(ira->codegen, target_import); @@ -18106,7 +18108,7 @@ static ZigType *ir_analyze_instruction_embed_file(IrAnalyze *ira, IrInstructionE // load from file system into const expr Buf *file_contents = buf_alloc(); int err; - if ((err = os_fetch_file_path(&file_path, file_contents, false))) { + if ((err = cache_add_file_fetch(&ira->codegen->cache_hash, &file_path, file_contents))) { if (err == ErrorFileNotFound) { ir_add_error(ira, instruction->name, buf_sprintf("unable to find '%s'", buf_ptr(&file_path))); return ira->codegen->builtin_types.entry_invalid; @@ -18116,9 +18118,6 @@ static ZigType *ir_analyze_instruction_embed_file(IrAnalyze *ira, IrInstructionE } } - // TODO add dependency on the file we embedded so that we know if it changes - // we'll have to invalidate the cache - ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); init_const_str_lit(ira->codegen, out_val, file_contents); diff --git a/src/link.cpp b/src/link.cpp index c0eb25bd55141f70808688955c210f65cc54c993..8c250fbbe11e5ac84aa2109ba2032299dfee26aa 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -5,7 +5,6 @@ * See http://opensource.org/licenses/MIT */ -#include "link.hpp" #include "os.hpp" #include "config.h" #include "codegen.hpp" @@ -13,7 +12,6 @@ struct LinkJob { CodeGen *codegen; - Buf out_file; ZigList args; bool link_in_crt; HashMap rpath_table; @@ -62,14 +60,8 @@ static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) new_link_lib->provided_explicitly = link_lib->provided_explicitly; } - codegen_build(child_gen); - const char *o_ext = target_o_file_ext(&child_gen->zig_target); - Buf *o_out_name = buf_sprintf("%s%s", oname, o_ext); - Buf *output_path = buf_alloc(); - os_path_join(&parent_gen->cache_dir, o_out_name, output_path); - codegen_link(child_gen, buf_ptr(output_path)); - - return output_path; + codegen_build_and_link(child_gen); + return &child_gen->output_file_path; } static Buf *build_o(CodeGen *parent_gen, const char *oname) { @@ -237,15 +229,15 @@ static void construct_linker_job_elf(LinkJob *lj) { } else if (shared) { lj->args.append("-shared"); - if (buf_len(&lj->out_file) == 0) { - buf_appendf(&lj->out_file, "lib%s.so.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize "", + if (buf_len(&g->output_file_path) == 0) { + buf_appendf(&g->output_file_path, "lib%s.so.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize "", buf_ptr(g->root_out_name), g->version_major, g->version_minor, g->version_patch); } soname = buf_sprintf("lib%s.so.%" ZIG_PRI_usize "", buf_ptr(g->root_out_name), g->version_major); } lj->args.append("-o"); - lj->args.append(buf_ptr(&lj->out_file)); + lj->args.append(buf_ptr(&g->output_file_path)); if (lj->link_in_crt) { const char *crt1o; @@ -397,7 +389,7 @@ static void construct_linker_job_wasm(LinkJob *lj) { lj->args.append("--relocatable"); // So lld doesn't look for _start. lj->args.append("-o"); - lj->args.append(buf_ptr(&lj->out_file)); + lj->args.append(buf_ptr(&g->output_file_path)); // .o files for (size_t i = 0; i < g->link_objects.length; i += 1) { @@ -478,7 +470,7 @@ static void construct_linker_job_coff(LinkJob *lj) { // } //} - lj->args.append(buf_ptr(buf_sprintf("-OUT:%s", buf_ptr(&lj->out_file)))); + lj->args.append(buf_ptr(buf_sprintf("-OUT:%s", buf_ptr(&g->output_file_path)))); if (g->libc_link_lib != nullptr) { lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->msvc_lib_dir)))); @@ -585,11 +577,11 @@ static void construct_linker_job_coff(LinkJob *lj) { buf_appendf(def_contents, "\n"); Buf *def_path = buf_alloc(); - os_path_join(&g->cache_dir, buf_sprintf("%s.def", buf_ptr(link_lib->name)), def_path); + os_path_join(&g->artifact_dir, buf_sprintf("%s.def", buf_ptr(link_lib->name)), def_path); os_write_file(def_path, def_contents); Buf *generated_lib_path = buf_alloc(); - os_path_join(&g->cache_dir, buf_sprintf("%s.lib", buf_ptr(link_lib->name)), generated_lib_path); + os_path_join(&g->artifact_dir, buf_sprintf("%s.lib", buf_ptr(link_lib->name)), generated_lib_path); gen_lib_args.resize(0); gen_lib_args.append("link"); @@ -797,8 +789,8 @@ static void construct_linker_job_macho(LinkJob *lj) { //lj->args.append("-install_name"); //lj->args.append(buf_ptr(dylib_install_name)); - if (buf_len(&lj->out_file) == 0) { - buf_appendf(&lj->out_file, "lib%s.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".dylib", + if (buf_len(&g->output_file_path) == 0) { + buf_appendf(&g->output_file_path, "lib%s.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".dylib", buf_ptr(g->root_out_name), g->version_major, g->version_minor, g->version_patch); } } @@ -832,13 +824,13 @@ static void construct_linker_job_macho(LinkJob *lj) { } lj->args.append("-o"); - lj->args.append(buf_ptr(&lj->out_file)); + lj->args.append(buf_ptr(&g->output_file_path)); for (size_t i = 0; i < g->rpath_list.length; i += 1) { Buf *rpath = g->rpath_list.at(i); add_rpath(lj, rpath); } - add_rpath(lj, &lj->out_file); + add_rpath(lj, &g->output_file_path); if (shared) { lj->args.append("-headerpad_max_install_names"); @@ -942,7 +934,8 @@ static void construct_linker_job(LinkJob *lj) { } } -void codegen_link(CodeGen *g, const char *out_file) { +void codegen_link(CodeGen *g) { + assert(g->out_type != OutTypeObj); codegen_add_time_event(g, "Build Dependencies"); LinkJob lj = {0}; @@ -953,11 +946,6 @@ void codegen_link(CodeGen *g, const char *out_file) { lj.rpath_table.init(4); lj.codegen = g; - if (out_file) { - buf_init_from_str(&lj.out_file, out_file); - } else { - buf_resize(&lj.out_file, 0); - } if (g->verbose_llvm_ir) { fprintf(stderr, "\nOptimization:\n"); @@ -966,35 +954,9 @@ void codegen_link(CodeGen *g, const char *out_file) { LLVMDumpModule(g->module); } - bool override_out_file = (buf_len(&lj.out_file) != 0); - if (!override_out_file) { - assert(g->root_out_name); - - buf_init_from_buf(&lj.out_file, g->root_out_name); - if (g->out_type == OutTypeExe) { - buf_append_str(&lj.out_file, target_exe_file_ext(&g->zig_target)); - } - } - - if (g->out_type == OutTypeObj) { - if (override_out_file) { - assert(g->link_objects.length == 1); - Buf *o_file_path = g->link_objects.at(0); - int err; - if ((err = os_rename(o_file_path, &lj.out_file))) { - zig_panic("unable to rename object file %s into final output %s: %s", buf_ptr(o_file_path), buf_ptr(&lj.out_file), err_str(err)); - } - } - return; - } - if (g->out_type == OutTypeLib && g->is_static) { - // invoke `ar` - // example: - // # static link into libfoo.a - // ar rcs libfoo.a foo1.o foo2.o - zig_panic("TODO invoke ar"); - return; + fprintf(stderr, "Zig does not yet support creating static libraries\nSee https://github.com/ziglang/zig/issues/1493\n"); + exit(1); } lj.link_in_crt = (g->libc_link_lib != nullptr && g->out_type == OutTypeExe); @@ -1017,6 +979,4 @@ void codegen_link(CodeGen *g, const char *out_file) { fprintf(stderr, "%s\n", buf_ptr(&diag)); exit(1); } - - codegen_add_time_event(g, "Done"); } diff --git a/src/link.hpp b/src/link.hpp deleted file mode 100644 index 9f978c28d9e975f5db771b57162b920ab375f5ce..0000000000000000000000000000000000000000 --- a/src/link.hpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2015 Andrew Kelley - * - * This file is part of zig, which is MIT licensed. - * See http://opensource.org/licenses/MIT - */ - -#ifndef ZIG_LINK_HPP -#define ZIG_LINK_HPP - -#include "all_types.hpp" - -void codegen_link(CodeGen *g, const char *out_file); - - -#endif - diff --git a/src/main.cpp b/src/main.cpp index 1520ae7d67ad81df5ccb0cfdbe8b8a088818fc36..ff2c061a833b7e162adfa747e010fb486e3036dd 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -10,7 +10,6 @@ #include "codegen.hpp" #include "config.h" #include "error.hpp" -#include "link.hpp" #include "os.hpp" #include "target.hpp" #include "cache_hash.hpp" @@ -385,8 +384,7 @@ int main(int argc, char **argv) { CliPkg *cur_pkg = allocate(1); BuildMode build_mode = BuildModeDebug; ZigList test_exec_args = {0}; - int comptime_args_end = 0; - int runtime_args_start = argc; + int runtime_args_start = -1; bool no_rosegment_workaround = false; if (argc >= 2 && strcmp(argv[1], "build") == 0) { @@ -454,8 +452,6 @@ int main(int argc, char **argv) { full_cache_dir = os_path_resolve(&cache_dir_buf, 1); } - Buf *path_to_build_exe = buf_alloc(); - os_path_join(&full_cache_dir, buf_create_from_str("build"), path_to_build_exe); codegen_set_cache_dir(g, full_cache_dir); args.items[1] = buf_ptr(&build_file_dirname); @@ -525,14 +521,13 @@ int main(int argc, char **argv) { PackageTableEntry *build_pkg = codegen_create_package(g, buf_ptr(&build_file_dirname), buf_ptr(&build_file_basename)); g->root_package->package_table.put(buf_create_from_str("@build"), build_pkg); - codegen_build(g); - codegen_link(g, buf_ptr(path_to_build_exe)); + codegen_build_and_link(g); Termination term; - os_spawn_process(buf_ptr(path_to_build_exe), args, &term); + os_spawn_process(buf_ptr(&g->output_file_path), args, &term); if (term.how != TerminationIdClean || term.code != 0) { fprintf(stderr, "\nBuild failed. The following command failed:\n"); - fprintf(stderr, "%s", buf_ptr(path_to_build_exe)); + fprintf(stderr, "%s", buf_ptr(&g->output_file_path)); for (size_t i = 0; i < args.length; i += 1) { fprintf(stderr, " %s", args.at(i)); } @@ -541,15 +536,11 @@ int main(int argc, char **argv) { return (term.how == TerminationIdClean) ? term.code : -1; } - for (int i = 1; i < argc; i += 1, comptime_args_end += 1) { + for (int i = 1; i < argc; i += 1) { char *arg = argv[i]; if (arg[0] == '-') { - if (strcmp(arg, "--") == 0) { - // ignore -- from both compile and runtime arg sets - runtime_args_start = i + 1; - break; - } else if (strcmp(arg, "--release-fast") == 0) { + if (strcmp(arg, "--release-fast") == 0) { build_mode = BuildModeFastRelease; } else if (strcmp(arg, "--release-safe") == 0) { build_mode = BuildModeSafeRelease; @@ -746,6 +737,10 @@ int main(int argc, char **argv) { case CmdTest: if (!in_file) { in_file = arg; + if (cmd == CmdRun) { + runtime_args_start = i + 1; + break; // rest of the args are for the program + } } else { fprintf(stderr, "Unexpected extra parameter: %s\n", arg); return usage(arg0); @@ -856,19 +851,10 @@ int main(int argc, char **argv) { Buf *zig_root_source_file = (cmd == CmdTranslateC) ? nullptr : in_file_buf; Buf full_cache_dir = BUF_INIT; - Buf *run_exec_path = buf_alloc(); - if (cmd == CmdRun) { - if (buf_out_name == nullptr) { - buf_out_name = buf_create_from_str("run"); - } - - Buf *global_cache_dir = buf_alloc(); - os_get_global_cache_directory(global_cache_dir); - os_path_join(global_cache_dir, buf_out_name, run_exec_path); - full_cache_dir = os_path_resolve(&global_cache_dir, 1); - - out_file = buf_ptr(run_exec_path); - } else { + if (cmd == CmdRun && buf_out_name == nullptr) { + buf_out_name = buf_create_from_str("run"); + } + { Buf *resolve_paths = buf_create_from_str((cache_dir == nullptr) ? default_zig_cache_name : cache_dir); full_cache_dir = os_path_resolve(&resolve_paths, 1); } @@ -957,6 +943,8 @@ int main(int argc, char **argv) { codegen_set_test_name_prefix(g, buf_create_from_str(test_name_prefix)); } + if (out_file) + codegen_set_output_path(g, buf_create_from_str(out_file)); if (out_file_h) codegen_set_output_h_path(g, buf_create_from_str(out_file_h)); @@ -976,8 +964,7 @@ int main(int argc, char **argv) { if (cmd == CmdBuild || cmd == CmdRun) { codegen_set_emit_file_type(g, emit_file_type); - codegen_build(g); - codegen_link(g, out_file); + codegen_build_and_link(g); if (timing_info) codegen_print_timing_report(g, stdout); @@ -987,8 +974,14 @@ int main(int argc, char **argv) { args.append(argv[i]); } + const char *exec_path = buf_ptr(&g->output_file_path); + args.append(nullptr); + + os_execv(exec_path, args.items); + + args.pop(); Termination term; - os_spawn_process(buf_ptr(run_exec_path), args, &term); + os_spawn_process(exec_path, args, &term); return term.code; } @@ -1005,11 +998,8 @@ int main(int argc, char **argv) { ZigTarget native; get_native_target(&native); - ZigTarget *non_null_target = target ? target : &native; - - Buf *test_exe_name = buf_sprintf("test%s", target_exe_file_ext(non_null_target)); - Buf *test_exe_path = buf_alloc(); - os_path_join(&full_cache_dir, test_exe_name, test_exe_path); + codegen_build_and_link(g); + Buf *test_exe_path = &g->output_file_path; for (size_t i = 0; i < test_exec_args.length; i += 1) { if (test_exec_args.items[i] == nullptr) { @@ -1017,8 +1007,6 @@ int main(int argc, char **argv) { } } - codegen_build(g); - codegen_link(g, buf_ptr(test_exe_path)); if (!target_can_exec(&native, target)) { fprintf(stderr, "Created %s but skipping execution because it is non-native.\n", diff --git a/src/os.cpp b/src/os.cpp index 4ee461818401376460f0554267a401677574a84b..8502e72715bee7a94e66c603a3833df4d6bdb50e 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -972,6 +972,22 @@ static int os_exec_process_windows(const char *exe, ZigList &args, } #endif +Error os_execv(const char *exe, const char **argv) { +#if defined(ZIG_OS_WINDOWS) + return ErrorUnsupportedOperatingSystem; +#else + execv(exe, (char *const *)argv); + switch (errno) { + case ENOMEM: + return ErrorSystemResources; + case EIO: + return ErrorFileSystem; + default: + return ErrorUnexpected; + } +#endif +} + int os_exec_process(const char *exe, ZigList &args, Termination *term, Buf *out_stderr, Buf *out_stdout) { @@ -1238,44 +1254,6 @@ int os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path) { #endif } -#if defined(ZIG_OS_POSIX) -int os_get_global_cache_directory(Buf *out_tmp_path) { - const char *tmp_dir = getenv("TMPDIR"); - if (!tmp_dir) { - tmp_dir = P_tmpdir; - } - - Buf *tmp_dir_buf = buf_create_from_str(tmp_dir); - Buf *cache_dirname_buf = buf_create_from_str("zig-cache"); - - buf_resize(out_tmp_path, 0); - os_path_join(tmp_dir_buf, cache_dirname_buf, out_tmp_path); - - buf_deinit(tmp_dir_buf); - buf_deinit(cache_dirname_buf); - return 0; -} -#endif - -#if defined(ZIG_OS_WINDOWS) -int os_get_global_cache_directory(Buf *out_tmp_path) { - char tmp_dir[MAX_PATH + 1]; - if (GetTempPath(MAX_PATH, tmp_dir) == 0) { - zig_panic("GetTempPath failed"); - } - - Buf *tmp_dir_buf = buf_create_from_str(tmp_dir); - Buf *cache_dirname_buf = buf_create_from_str("zig-cache"); - - buf_resize(out_tmp_path, 0); - os_path_join(tmp_dir_buf, cache_dirname_buf, out_tmp_path); - - buf_deinit(tmp_dir_buf); - buf_deinit(cache_dirname_buf); - return 0; -} -#endif - int os_delete_file(Buf *path) { if (remove(buf_ptr(path))) { return ErrorFileSystem; diff --git a/src/os.hpp b/src/os.hpp index c64eccf8d44af408a121a7da9ad3c74689ded1c8..fc2a34326f79258bead84cf214c464f84e6b800b 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -87,6 +87,7 @@ int os_init(void); void os_spawn_process(const char *exe, ZigList &args, Termination *term); int os_exec_process(const char *exe, ZigList &args, Termination *term, Buf *out_stderr, Buf *out_stdout); +Error os_execv(const char *exe, const char **argv); void os_path_dirname(Buf *full_path, Buf *out_dirname); void os_path_split(Buf *full_path, Buf *out_dirname, Buf *out_basename); @@ -96,8 +97,6 @@ int os_path_real(Buf *rel_path, Buf *out_abs_path); Buf os_path_resolve(Buf **paths_ptr, size_t paths_len); bool os_path_is_absolute(Buf *path); -int os_get_global_cache_directory(Buf *out_tmp_path); - Error ATTRIBUTE_MUST_USE os_make_path(Buf *path); Error ATTRIBUTE_MUST_USE os_make_dir(Buf *path); -- 2.54.0 From 5ee5933ade09c535bd1806d91cb606f49d07acea Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 10 Sep 2018 17:30:45 -0400 Subject: [PATCH 08/20] stage1 caching: zig no longer uses zig-cache --- CMakeLists.txt | 1 + src/all_types.hpp | 2 - src/cache_hash.cpp | 7 +++ src/codegen.cpp | 110 ++++++++++++++++++++++++++++++--------------- src/codegen.hpp | 3 +- src/compiler.cpp | 66 +++++++++++++++++++++++++++ src/compiler.hpp | 17 +++++++ src/link.cpp | 4 +- src/main.cpp | 78 ++------------------------------ src/os.cpp | 22 ++++----- src/os.hpp | 6 +-- 11 files changed, 187 insertions(+), 129 deletions(-) create mode 100644 src/compiler.cpp create mode 100644 src/compiler.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8339be71b9b1f9b9b3a2b2b5e2db2c4e2f6278cb..11bb31892ca4e5b3946bc125d37bd90ad189769a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -411,6 +411,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp" "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" + "${CMAKE_SOURCE_DIR}/src/compiler.cpp" "${CMAKE_SOURCE_DIR}/src/errmsg.cpp" "${CMAKE_SOURCE_DIR}/src/error.cpp" "${CMAKE_SOURCE_DIR}/src/ir.cpp" diff --git a/src/all_types.hpp b/src/all_types.hpp index 6d11244a2526f0c167147bd79dbfeaf85bfa5daa..4a15162076401d493dea5efafb67d57846888efd 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1671,7 +1671,6 @@ struct CodeGen { Buf triple_str; Buf global_asm; Buf *out_h_path; - Buf cache_dir; Buf artifact_dir; Buf output_file_path; Buf o_file_output_path; @@ -1731,7 +1730,6 @@ struct CodeGen { ZigList assembly_files; ZigList lib_dirs; - Buf *compiler_id; size_t version_major; size_t version_minor; size_t version_patch; diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index b6aebdac671bdae04ab00a3f98128e19b6e79b7d..b302946310c7145b60100290d6f35ceebfbc2be7 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -248,6 +248,12 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { int rc = blake2b_final(&ch->blake, bin_digest, 48); assert(rc == 0); + if (ch->files.length == 0) { + buf_resize(out_digest, 64); + base64_encode(buf_to_slice(out_digest), {bin_digest, 48}); + return ErrorNone; + } + Buf b64_digest = BUF_INIT; buf_resize(&b64_digest, 64); base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); @@ -458,5 +464,6 @@ Error cache_final(CacheHash *ch, Buf *out_digest) { } void cache_release(CacheHash *ch) { + assert(ch->manifest_file_path != nullptr); os_file_close(ch->manifest_file); } diff --git a/src/codegen.cpp b/src/codegen.cpp index 1f53bb4c9d9124a495d444ab7cbbdf4dec510e11..578fb314a8888d425e54f12c4d0b9721c099ec10 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -8,6 +8,7 @@ #include "analyze.hpp" #include "ast_render.hpp" #include "codegen.hpp" +#include "compiler.hpp" #include "config.h" #include "errmsg.hpp" #include "error.hpp" @@ -87,13 +88,12 @@ static const char *symbols_that_llvm_depends_on[] = { }; CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir, Buf *compiler_id) + Buf *zig_lib_dir) { CodeGen *g = allocate(1); codegen_add_time_event(g, "Initialize"); - g->compiler_id = compiler_id; g->zig_lib_dir = zig_lib_dir; g->zig_std_dir = buf_alloc(); @@ -243,10 +243,6 @@ void codegen_set_out_name(CodeGen *g, Buf *out_name) { g->root_out_name = out_name; } -void codegen_set_cache_dir(CodeGen *g, Buf cache_dir) { - g->cache_dir = cache_dir; -} - void codegen_set_libc_lib_dir(CodeGen *g, Buf *libc_lib_dir) { g->libc_lib_dir = libc_lib_dir; } @@ -5728,13 +5724,6 @@ static LLVMValueRef build_alloca(CodeGen *g, ZigType *type_entry, const char *na return result; } -static void ensure_cache_dir(CodeGen *g) { - int err; - if ((err = os_make_path(&g->cache_dir))) { - zig_panic("unable to make cache dir: %s", err_str(err)); - } -} - static void report_errors_and_maybe_exit(CodeGen *g) { if (g->errors.length != 0) { for (size_t i = 0; i < g->errors.length; i += 1) { @@ -6824,36 +6813,84 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { return contents; } -static void define_builtin_compile_vars(CodeGen *g) { +static Error define_builtin_compile_vars(CodeGen *g) { if (g->std_package == nullptr) - return; + return ErrorNone; + + Error err; + + Buf *manifest_dir = buf_alloc(); + os_path_join(get_stage1_cache_path(), buf_create_from_str("builtin"), manifest_dir); + + CacheHash cache_hash; + cache_init(&cache_hash, manifest_dir); + + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) + return err; + + // Only a few things affect builtin.zig + cache_buf(&cache_hash, compiler_id); + cache_int(&cache_hash, g->build_mode); + cache_bool(&cache_hash, g->is_test_build); + cache_int(&cache_hash, g->zig_target.arch.arch); + cache_int(&cache_hash, g->zig_target.arch.sub_arch); + cache_int(&cache_hash, g->zig_target.vendor); + cache_int(&cache_hash, g->zig_target.os); + cache_int(&cache_hash, g->zig_target.env_type); + cache_int(&cache_hash, g->zig_target.oformat); + cache_bool(&cache_hash, g->have_err_ret_tracing); + cache_bool(&cache_hash, g->libc_link_lib != nullptr); + + Buf digest = BUF_INIT; + buf_resize(&digest, 0); + if ((err = cache_hit(&cache_hash, &digest))) + return err; + + // We should always get a cache hit because there are no + // files in the input hash. + assert(buf_len(&digest) != 0); + + Buf *this_dir = buf_alloc(); + os_path_join(manifest_dir, &digest, this_dir); + + if ((err = os_make_path(this_dir))) + return err; const char *builtin_zig_basename = "builtin.zig"; Buf *builtin_zig_path = buf_alloc(); - os_path_join(&g->cache_dir, buf_create_from_str(builtin_zig_basename), builtin_zig_path); + os_path_join(this_dir, buf_create_from_str(builtin_zig_basename), builtin_zig_path); - Buf *contents = codegen_generate_builtin_source(g); - ensure_cache_dir(g); - os_write_file(builtin_zig_path, contents); - - Buf *resolved_path = buf_alloc(); - Buf *resolve_paths[] = {builtin_zig_path}; - *resolved_path = os_path_resolve(resolve_paths, 1); + bool hit; + if ((err = os_file_exists(builtin_zig_path, &hit))) + return err; + Buf *contents; + if (hit) { + contents = buf_alloc(); + if ((err = os_fetch_file_path(builtin_zig_path, contents, false))) { + fprintf(stderr, "Unable to open '%s': %s\n", buf_ptr(builtin_zig_path), err_str(err)); + exit(1); + } + } else { + contents = codegen_generate_builtin_source(g); + os_write_file(builtin_zig_path, contents); + } assert(g->root_package); assert(g->std_package); - g->compile_var_package = new_package(buf_ptr(&g->cache_dir), builtin_zig_basename); + g->compile_var_package = new_package(buf_ptr(this_dir), builtin_zig_basename); g->root_package->package_table.put(buf_create_from_str("builtin"), g->compile_var_package); g->std_package->package_table.put(buf_create_from_str("builtin"), g->compile_var_package); - g->compile_var_import = add_source_file(g, g->compile_var_package, resolved_path, contents); + g->compile_var_import = add_source_file(g, g->compile_var_package, builtin_zig_path, contents); scan_import(g, g->compile_var_import); + + return ErrorNone; } static void init(CodeGen *g) { if (g->module) return; - if (g->llvm_argv_len > 0) { const char **args = allocate_nonzero(g->llvm_argv_len + 2); args[0] = "zig (LLVM option parsing)"; @@ -6960,7 +6997,11 @@ static void init(CodeGen *g) { g->have_err_ret_tracing = g->build_mode != BuildModeFastRelease && g->build_mode != BuildModeSmallRelease; define_builtin_fns(g); - define_builtin_compile_vars(g); + Error err; + if ((err = define_builtin_compile_vars(g))) { + fprintf(stderr, "Unable to create builtin.zig: %s\n", err_str(err)); + exit(1); + } } void codegen_translate_c(CodeGen *g, Buf *full_path) { @@ -7668,6 +7709,10 @@ static void add_cache_pkg(CodeGen *g, CacheHash *ch, PackageTableEntry *pkg) { static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { Error err; + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) + return err; + CacheHash *ch = &g->cache_hash; cache_init(ch, manifest_dir); @@ -7675,7 +7720,7 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { if (g->linker_script != nullptr) { cache_file(ch, buf_create_from_str(g->linker_script)); } - cache_buf(ch, g->compiler_id); + cache_buf(ch, compiler_id); cache_buf(ch, g->root_out_name); cache_list_of_link_lib(ch, g->link_libs_list.items, g->link_libs_list.length); cache_list_of_buf(ch, g->darwin_frameworks.items, g->darwin_frameworks.length); @@ -7766,13 +7811,7 @@ void codegen_build_and_link(CodeGen *g) { codegen_add_time_event(g, "Check Cache"); - Buf app_data_dir = BUF_INIT; - if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { - fprintf(stderr, "Unable to get app data dir: %s\n", err_str(err)); - exit(1); - } - Buf *stage1_dir = buf_alloc(); - os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); + Buf *stage1_dir = get_stage1_cache_path(); Buf *manifest_dir = buf_alloc(); os_path_join(stage1_dir, buf_create_from_str("build"), manifest_dir); @@ -7820,6 +7859,7 @@ void codegen_build_and_link(CodeGen *g) { } // TODO hard link output_file_path to wanted_output_file_path + cache_release(&g->cache_hash); codegen_add_time_event(g, "Done"); } diff --git a/src/codegen.hpp b/src/codegen.hpp index 1e7fafa28af69958f60e8de2181d27465fda06b4..203e2d2b94f4b085b332e3b086a27747eb208c18 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -15,7 +15,7 @@ #include CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir, Buf *compiler_id); + Buf *zig_lib_dir); void codegen_set_clang_argv(CodeGen *codegen, const char **args, size_t len); void codegen_set_llvm_argv(CodeGen *codegen, const char **args, size_t len); @@ -46,7 +46,6 @@ void codegen_set_linker_script(CodeGen *g, const char *linker_script); void codegen_set_test_filter(CodeGen *g, Buf *filter); void codegen_set_test_name_prefix(CodeGen *g, Buf *prefix); void codegen_set_lib_version(CodeGen *g, size_t major, size_t minor, size_t patch); -void codegen_set_cache_dir(CodeGen *g, Buf cache_dir); void codegen_set_output_h_path(CodeGen *g, Buf *h_path); void codegen_set_output_path(CodeGen *g, Buf *path); void codegen_add_time_event(CodeGen *g, const char *name); diff --git a/src/compiler.cpp b/src/compiler.cpp new file mode 100644 index 0000000000000000000000000000000000000000..dd02b541dd09d15f6551b778d206ba65b4533d05 --- /dev/null +++ b/src/compiler.cpp @@ -0,0 +1,66 @@ +#include "cache_hash.hpp" + +#include + +static Buf saved_compiler_id = BUF_INIT; +static Buf saved_app_data_dir = BUF_INIT; +static Buf saved_stage1_path = BUF_INIT; + +Buf *get_stage1_cache_path() { + if (saved_stage1_path.list.length != 0) { + return &saved_stage1_path; + } + Error err; + if ((err = os_get_app_data_dir(&saved_app_data_dir, "zig"))) { + fprintf(stderr, "Unable to get app data dir: %s\n", err_str(err)); + exit(1); + } + os_path_join(&saved_app_data_dir, buf_create_from_str("stage1"), &saved_stage1_path); + return &saved_stage1_path; +} + +Error get_compiler_id(Buf **result) { + if (saved_compiler_id.list.length != 0) { + *result = &saved_compiler_id; + return ErrorNone; + } + + Error err; + Buf *stage1_dir = get_stage1_cache_path(); + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); + + CacheHash cache_hash; + CacheHash *ch = &cache_hash; + cache_init(ch, manifest_dir); + Buf self_exe_path = BUF_INIT; + if ((err = os_self_exe_path(&self_exe_path))) + return err; + + cache_file(ch, &self_exe_path); + + buf_resize(&saved_compiler_id, 0); + if ((err = cache_hit(ch, &saved_compiler_id))) + return err; + if (buf_len(&saved_compiler_id) != 0) { + cache_release(ch); + *result = &saved_compiler_id; + return ErrorNone; + } + ZigList lib_paths = {}; + if ((err = os_self_exe_shared_libs(lib_paths))) + return err; + for (size_t i = 0; i < lib_paths.length; i += 1) { + Buf *lib_path = lib_paths.at(i); + if ((err = cache_add_file(ch, lib_path))) + return err; + } + if ((err = cache_final(ch, &saved_compiler_id))) + return err; + + cache_release(ch); + + *result = &saved_compiler_id; + return ErrorNone; +} + diff --git a/src/compiler.hpp b/src/compiler.hpp new file mode 100644 index 0000000000000000000000000000000000000000..b95e4ceda7badc034c4bf252e49cd69ad22b2a0d --- /dev/null +++ b/src/compiler.hpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2018 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#ifndef ZIG_COMPILER_HPP +#define ZIG_COMPILER_HPP + +#include "buffer.hpp" +#include "error.hpp" + +Buf *get_stage1_cache_path(); +Error get_compiler_id(Buf **result); + +#endif diff --git a/src/link.cpp b/src/link.cpp index 8c250fbbe11e5ac84aa2109ba2032299dfee26aa..8d7b8b4d5fdf794ab32f3f664057fe142eb07025 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -32,7 +32,7 @@ static const char *get_libc_static_file(CodeGen *g, const char *file) { static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) { ZigTarget *child_target = parent_gen->is_native_target ? nullptr : &parent_gen->zig_target; CodeGen *child_gen = codegen_create(full_path, child_target, OutTypeObj, parent_gen->build_mode, - parent_gen->zig_lib_dir, parent_gen->compiler_id); + parent_gen->zig_lib_dir); child_gen->want_h_file = false; child_gen->verbose_tokenize = parent_gen->verbose_tokenize; @@ -42,8 +42,6 @@ static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) child_gen->verbose_llvm_ir = parent_gen->verbose_llvm_ir; child_gen->verbose_cimport = parent_gen->verbose_cimport; - codegen_set_cache_dir(child_gen, parent_gen->cache_dir); - codegen_set_strip(child_gen, parent_gen->strip_debug_symbols); codegen_set_is_static(child_gen, parent_gen->is_static); diff --git a/src/main.cpp b/src/main.cpp index ff2c061a833b7e162adfa747e010fb486e3036dd..10b789a6e1b9cfec953fa37aab8d128706e9aeec 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -8,11 +8,11 @@ #include "ast_render.hpp" #include "buffer.hpp" #include "codegen.hpp" +#include "compiler.hpp" #include "config.h" #include "error.hpp" #include "os.hpp" #include "target.hpp" -#include "cache_hash.hpp" #include @@ -257,53 +257,6 @@ static void add_package(CodeGen *g, CliPkg *cli_pkg, PackageTableEntry *pkg) { } } -static Buf saved_compiler_id = BUF_INIT; -static Error get_compiler_id(Buf **result) { - if (saved_compiler_id.list.length != 0) { - *result = &saved_compiler_id; - return ErrorNone; - } - - Error err; - Buf app_data_dir = BUF_INIT; - if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) - return err; - Buf *stage1_dir = buf_alloc(); - os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); - Buf *manifest_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); - - CacheHash cache_hash; - CacheHash *ch = &cache_hash; - cache_init(ch, manifest_dir); - Buf self_exe_path = BUF_INIT; - if ((err = os_self_exe_path(&self_exe_path))) - return err; - - cache_file(ch, &self_exe_path); - - buf_resize(&saved_compiler_id, 0); - if ((err = cache_hit(ch, &saved_compiler_id))) - return err; - if (buf_len(&saved_compiler_id) != 0) { - *result = &saved_compiler_id; - return ErrorNone; - } - ZigList lib_paths = {}; - if ((err = os_self_exe_shared_libs(lib_paths))) - return err; - for (size_t i = 0; i < lib_paths.length; i += 1) { - Buf *lib_path = lib_paths.at(i); - if ((err = cache_add_file(ch, lib_path))) - return err; - } - if ((err = cache_final(ch, &saved_compiler_id))) - return err; - - *result = &saved_compiler_id; - return ErrorNone; -} - int main(int argc, char **argv) { if (argc == 2 && strcmp(argv[1], "BUILD_INFO") == 0) { printf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", @@ -428,14 +381,7 @@ int main(int argc, char **argv) { Buf *build_runner_path = buf_alloc(); os_path_join(special_dir, buf_create_from_str("build_runner.zig"), build_runner_path); - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) { - fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); - return EXIT_FAILURE; - } - - CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf, - compiler_id); + CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf); codegen_set_out_name(g, buf_create_from_str("build")); Buf *build_file_buf = buf_create_from_str(build_file); @@ -452,8 +398,6 @@ int main(int argc, char **argv) { full_cache_dir = os_path_resolve(&cache_dir_buf, 1); } - codegen_set_cache_dir(g, full_cache_dir); - args.items[1] = buf_ptr(&build_file_dirname); args.items[2] = buf_ptr(&full_cache_dir); @@ -795,7 +739,7 @@ int main(int argc, char **argv) { switch (cmd) { case CmdBuiltin: { Buf *zig_lib_dir_buf = resolve_zig_lib_dir(); - CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, zig_lib_dir_buf, nullptr); + CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, zig_lib_dir_buf); Buf *builtin_source = codegen_generate_builtin_source(g); if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) { fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout))); @@ -850,30 +794,16 @@ int main(int argc, char **argv) { Buf *zig_root_source_file = (cmd == CmdTranslateC) ? nullptr : in_file_buf; - Buf full_cache_dir = BUF_INIT; if (cmd == CmdRun && buf_out_name == nullptr) { buf_out_name = buf_create_from_str("run"); } - { - Buf *resolve_paths = buf_create_from_str((cache_dir == nullptr) ? default_zig_cache_name : cache_dir); - full_cache_dir = os_path_resolve(&resolve_paths, 1); - } - Buf *zig_lib_dir_buf = resolve_zig_lib_dir(); - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) { - fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); - return EXIT_FAILURE; - } - - CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf, - compiler_id); + CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf); codegen_set_out_name(g, buf_out_name); codegen_set_lib_version(g, ver_major, ver_minor, ver_patch); codegen_set_is_test(g, cmd == CmdTest); codegen_set_linker_script(g, linker_script); - codegen_set_cache_dir(g, full_cache_dir); if (each_lib_rpath) codegen_set_each_lib_rpath(g, each_lib_rpath); diff --git a/src/os.cpp b/src/os.cpp index 8502e72715bee7a94e66c603a3833df4d6bdb50e..3950711c568b35b351a74783a7e2fcd46efb8234 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -721,7 +721,7 @@ Buf os_path_resolve(Buf **paths_ptr, size_t paths_len) { #endif } -int os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { +Error os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { static const ssize_t buf_size = 0x2000; buf_resize(out_buf, buf_size); ssize_t actual_buf_len = 0; @@ -757,7 +757,7 @@ int os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { if (amt_read != buf_size) { if (feof(f)) { buf_resize(out_buf, actual_buf_len); - return 0; + return ErrorNone; } else { return ErrorFileSystem; } @@ -769,13 +769,13 @@ int os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { zig_unreachable(); } -int os_file_exists(Buf *full_path, bool *result) { +Error os_file_exists(Buf *full_path, bool *result) { #if defined(ZIG_OS_WINDOWS) *result = GetFileAttributes(buf_ptr(full_path)) != INVALID_FILE_ATTRIBUTES; - return 0; + return ErrorNone; #else *result = access(buf_ptr(full_path), F_OK) != -1; - return 0; + return ErrorNone; #endif } @@ -834,13 +834,15 @@ static int os_exec_process_posix(const char *exe, ZigList &args, FILE *stdout_f = fdopen(stdout_pipe[0], "rb"); FILE *stderr_f = fdopen(stderr_pipe[0], "rb"); - os_fetch_file(stdout_f, out_stdout, false); - os_fetch_file(stderr_f, out_stderr, false); + Error err1 = os_fetch_file(stdout_f, out_stdout, false); + Error err2 = os_fetch_file(stderr_f, out_stderr, false); fclose(stdout_f); fclose(stderr_f); - return 0; + if (err1) return err1; + if (err2) return err2; + return ErrorNone; } } #endif @@ -1064,7 +1066,7 @@ int os_copy_file(Buf *src_path, Buf *dest_path) { } } -int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang) { +Error os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang) { FILE *f = fopen(buf_ptr(full_path), "rb"); if (!f) { switch (errno) { @@ -1083,7 +1085,7 @@ int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang) { return ErrorFileSystem; } } - int result = os_fetch_file(f, out_contents, skip_shebang); + Error result = os_fetch_file(f, out_contents, skip_shebang); fclose(f); return result; } diff --git a/src/os.hpp b/src/os.hpp index fc2a34326f79258bead84cf214c464f84e6b800b..ac422fbd21bc901e63334893253196c7e00f3b3c 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -111,8 +111,8 @@ void os_file_close(OsFile file); void os_write_file(Buf *full_path, Buf *contents); int os_copy_file(Buf *src_path, Buf *dest_path); -int os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); -int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); +Error ATTRIBUTE_MUST_USE os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); +Error ATTRIBUTE_MUST_USE os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); int os_get_cwd(Buf *out_cwd); @@ -122,7 +122,7 @@ void os_stderr_set_color(TermColor color); int os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path); int os_delete_file(Buf *path); -int os_file_exists(Buf *full_path, bool *result); +Error ATTRIBUTE_MUST_USE os_file_exists(Buf *full_path, bool *result); int os_rename(Buf *src_path, Buf *dest_path); double os_get_time(void); -- 2.54.0 From 67735c6f1557092efe6e8c1712445c30655fe283 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 11 Sep 2018 00:32:40 -0400 Subject: [PATCH 09/20] ability to disable cache. off by default except for... ...zig run, zig build, compiler_rt.a, and builtin.a --- src/all_types.hpp | 1 + src/analyze.cpp | 8 ++++ src/analyze.hpp | 3 ++ src/cache_hash.cpp | 8 ++++ src/cache_hash.hpp | 5 +++ src/codegen.cpp | 107 ++++++++++++++++++++++++++++++++------------- src/ir.cpp | 4 +- src/link.cpp | 1 + src/main.cpp | 50 +++++++++++++++++++-- src/target.cpp | 16 +++++++ src/target.hpp | 1 + 11 files changed, 168 insertions(+), 36 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index 4a15162076401d493dea5efafb67d57846888efd..cdc290268a337b9afb8c33832d4256c74dc08ce4 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1718,6 +1718,7 @@ struct CodeGen { bool verbose_cimport; bool error_during_imports; bool generate_error_name_table; + bool enable_cache; //////////////////////////// Participates in Input Parameter Cache Hash ZigList link_libs_list; diff --git a/src/analyze.cpp b/src/analyze.cpp index aa4fda76242e59a99a9e8f63ff2be6967bb5f529..7fe7a48245a2b180cea239845c28dd863bce22a4 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -6367,3 +6367,11 @@ not_integer: } return nullptr; } + +Error file_fetch(CodeGen *g, Buf *resolved_path, Buf *contents) { + if (g->enable_cache) { + return cache_add_file_fetch(&g->cache_hash, resolved_path, contents); + } else { + return os_fetch_file_path(resolved_path, contents, false); + } +} diff --git a/src/analyze.hpp b/src/analyze.hpp index 41cc50916eac81e690343b6dbf960335f9b796ec..9f036c409d73d927d4556bb8e85c81c9193f09b3 100644 --- a/src/analyze.hpp +++ b/src/analyze.hpp @@ -210,4 +210,7 @@ ZigType *get_primitive_type(CodeGen *g, Buf *name); bool calling_convention_allows_zig_types(CallingConvention cc); const char *calling_convention_name(CallingConvention cc); + +Error ATTRIBUTE_MUST_USE file_fetch(CodeGen *g, Buf *resolved_path, Buf *contents); + #endif diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index b302946310c7145b60100290d6f35ceebfbc2be7..65e3e62ae0c6abcb9529b06e8b47b2484422548c 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -467,3 +467,11 @@ void cache_release(CacheHash *ch) { assert(ch->manifest_file_path != nullptr); os_file_close(ch->manifest_file); } + +Buf *get_random_basename() { + Buf *result = buf_alloc(); + for (size_t i = 0; i < 16; i += 1) { + buf_append_char(result, base64_fs_alphabet[rand() % 64]); + } + return result; +} diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp index ede7344c7531fcdde22cb715bc63e234313902c2..6acd805be65e89807115666dfa358df0ca18fda0 100644 --- a/src/cache_hash.hpp +++ b/src/cache_hash.hpp @@ -67,4 +67,9 @@ Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_b64_digest); // Until this function is called, no one will be able to get a lock on your input params. void cache_release(CacheHash *ch); + + +// Completely independent function. Just returns a random filename safe basename. +Buf *get_random_basename(); + #endif diff --git a/src/codegen.cpp b/src/codegen.cpp index 578fb314a8888d425e54f12c4d0b9721c099ec10..1e0fd7fec3b79f5b5021e3bd56b82880c6cf42f4 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7048,7 +7048,7 @@ static ImportTableEntry *add_special_code(CodeGen *g, PackageTableEntry *package *resolved_path = os_path_resolve(resolve_paths, 1); Buf *import_code = buf_alloc(); Error err; - if ((err = cache_add_file_fetch(&g->cache_hash, resolved_path, import_code))) { + if ((err = file_fetch(g, resolved_path, import_code))) { zig_panic("unable to open '%s': %s\n", buf_ptr(&path_to_code_src), err_str(err)); } @@ -7480,10 +7480,7 @@ static void gen_h_file(CodeGen *g) { GenH *gen_h = &gen_h_data; assert(!g->is_test_build); - - if (!g->out_h_path) { - g->out_h_path = buf_sprintf("%s.h", buf_ptr(g->root_out_name)); - } + assert(g->out_h_path != nullptr); FILE *out_h = fopen(buf_ptr(g->out_h_path), "wb"); if (!out_h) @@ -7790,17 +7787,56 @@ static void resolve_out_paths(CodeGen *g) { zig_unreachable(); } - os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path); + if (g->enable_cache || g->out_type != OutTypeObj) { + os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path); + } else { + buf_init_from_buf(&g->o_file_output_path, o_basename); + } if (g->out_type == OutTypeObj) { buf_init_from_buf(&g->output_file_path, &g->o_file_output_path); } else if (g->out_type == OutTypeExe) { + if (!g->enable_cache && g->wanted_output_file_path != nullptr) { + buf_init_from_buf(&g->output_file_path, g->wanted_output_file_path); + } else { + assert(g->root_out_name); + + Buf basename = BUF_INIT; + buf_init_from_buf(&basename, g->root_out_name); + buf_append_str(&basename, target_exe_file_ext(&g->zig_target)); + if (g->enable_cache) { + os_path_join(&g->artifact_dir, &basename, &g->output_file_path); + } else { + buf_init_from_buf(&g->output_file_path, &basename); + } + } + } else if (g->out_type == OutTypeLib) { + if (!g->enable_cache && g->wanted_output_file_path != nullptr) { + buf_init_from_buf(&g->output_file_path, g->wanted_output_file_path); + } else { + Buf basename = BUF_INIT; + buf_init_from_buf(&basename, g->root_out_name); + buf_append_str(&basename, target_lib_file_ext(&g->zig_target, g->is_static, + g->version_major, g->version_minor, g->version_patch)); + if (g->enable_cache) { + os_path_join(&g->artifact_dir, &basename, &g->output_file_path); + } else { + buf_init_from_buf(&g->output_file_path, &basename); + } + } + } else { + zig_unreachable(); + } + + if (g->want_h_file && !g->out_h_path) { assert(g->root_out_name); - - Buf basename = BUF_INIT; - buf_init_from_buf(&basename, g->root_out_name); - buf_append_str(&basename, target_exe_file_ext(&g->zig_target)); - os_path_join(&g->artifact_dir, &basename, &g->output_file_path); + Buf *h_basename = buf_sprintf("%s.h", buf_ptr(g->root_out_name)); + if (g->enable_cache) { + g->out_h_path = buf_alloc(); + os_path_join(&g->artifact_dir, h_basename, g->out_h_path); + } else { + g->out_h_path = h_basename; + } } } @@ -7809,23 +7845,26 @@ void codegen_build_and_link(CodeGen *g) { Error err; assert(g->out_type != OutTypeUnknown); - codegen_add_time_event(g, "Check Cache"); - Buf *stage1_dir = get_stage1_cache_path(); - - Buf *manifest_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("build"), manifest_dir); - - Buf digest = BUF_INIT; - if ((err = check_cache(g, manifest_dir, &digest))) { - fprintf(stderr, "Unable to check cache: %s\n", err_str(err)); - exit(1); - } - Buf *artifact_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + Buf digest = BUF_INIT; + if (g->enable_cache) { + codegen_add_time_event(g, "Check Cache"); - if (buf_len(&digest) != 0) { + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("build"), manifest_dir); + + if ((err = check_cache(g, manifest_dir, &digest))) { + fprintf(stderr, "Unable to check cache: %s\n", err_str(err)); + exit(1); + } + + os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + } else { + os_path_join(stage1_dir, buf_create_from_str("tmp"), artifact_dir); + } + + if (g->enable_cache && buf_len(&digest) != 0) { os_path_join(artifact_dir, &digest, &g->artifact_dir); resolve_out_paths(g); } else { @@ -7836,11 +7875,16 @@ void codegen_build_and_link(CodeGen *g) { gen_global_asm(g); gen_root_source(g); - if ((err = cache_final(&g->cache_hash, &digest))) { - fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); - exit(1); + if (g->enable_cache) { + if ((err = cache_final(&g->cache_hash, &digest))) { + fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); + exit(1); + } + os_path_join(artifact_dir, &digest, &g->artifact_dir); + } else { + Buf *tmp_basename = get_random_basename(); + os_path_join(artifact_dir, tmp_basename, &g->artifact_dir); } - os_path_join(artifact_dir, &digest, &g->artifact_dir); if ((err = os_make_path(&g->artifact_dir))) { fprintf(stderr, "Unable to create artifact directory: %s\n", err_str(err)); exit(1); @@ -7857,9 +7901,10 @@ void codegen_build_and_link(CodeGen *g) { codegen_link(g); } } - // TODO hard link output_file_path to wanted_output_file_path - cache_release(&g->cache_hash); + if (g->enable_cache) { + cache_release(&g->cache_hash); + } codegen_add_time_event(g, "Done"); } diff --git a/src/ir.cpp b/src/ir.cpp index 22d9a9bc49e822cdaa1d0cb7943830a850f72809..bfe21f974d72ed8ae7d7ea2b3ec396f5fd56aeb1 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -16277,7 +16277,7 @@ static ZigType *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructionImpor return ira->codegen->builtin_types.entry_namespace; } - if ((err = cache_add_file_fetch(&ira->codegen->cache_hash, resolved_path, import_code))) { + if ((err = file_fetch(ira->codegen, resolved_path, import_code))) { if (err == ErrorFileNotFound) { ir_add_error_node(ira, source_node, buf_sprintf("unable to find '%s'", buf_ptr(import_target_path))); @@ -18108,7 +18108,7 @@ static ZigType *ir_analyze_instruction_embed_file(IrAnalyze *ira, IrInstructionE // load from file system into const expr Buf *file_contents = buf_alloc(); int err; - if ((err = cache_add_file_fetch(&ira->codegen->cache_hash, &file_path, file_contents))) { + if ((err = file_fetch(ira->codegen, &file_path, file_contents))) { if (err == ErrorFileNotFound) { ir_add_error(ira, instruction->name, buf_sprintf("unable to find '%s'", buf_ptr(&file_path))); return ira->codegen->builtin_types.entry_invalid; diff --git a/src/link.cpp b/src/link.cpp index 8d7b8b4d5fdf794ab32f3f664057fe142eb07025..aa0edde61b7a9c5ff7b148990a04e6279453d911 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -58,6 +58,7 @@ static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) new_link_lib->provided_explicitly = link_lib->provided_explicitly; } + child_gen->enable_cache = true; codegen_build_and_link(child_gen); return &child_gen->output_file_path; } diff --git a/src/main.cpp b/src/main.cpp index 10b789a6e1b9cfec953fa37aab8d128706e9aeec..e4019f10df50258c5e15ee4351a5114725f4acaa 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -34,6 +34,7 @@ static int usage(const char *arg0) { "Compile Options:\n" " --assembly [source] add assembly file to build\n" " --cache-dir [path] override the cache directory\n" + " --cache [auto|off|on] build to the global cache and print output path to stdout\n" " --color [auto|off|on] enable or disable colored error messages\n" " --emit [asm|bin|llvm-ir] emit a specific file format as compilation output\n" " --enable-timing-info print timing diagnostics\n" @@ -257,6 +258,24 @@ static void add_package(CodeGen *g, CliPkg *cli_pkg, PackageTableEntry *pkg) { } } +enum CacheOpt { + CacheOptAuto, + CacheOptOn, + CacheOptOff, +}; + +static bool get_cache_opt(CacheOpt opt, bool default_value) { + switch (opt) { + case CacheOptAuto: + return default_value; + case CacheOptOn: + return true; + case CacheOptOff: + return false; + } + zig_unreachable(); +} + int main(int argc, char **argv) { if (argc == 2 && strcmp(argv[1], "BUILD_INFO") == 0) { printf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", @@ -301,6 +320,7 @@ int main(int argc, char **argv) { bool verbose_llvm_ir = false; bool verbose_cimport = false; ErrColor color = ErrColorAuto; + CacheOpt enable_cache = CacheOptAuto; const char *libc_lib_dir = nullptr; const char *libc_static_lib_dir = nullptr; const char *libc_include_dir = nullptr; @@ -465,6 +485,7 @@ int main(int argc, char **argv) { PackageTableEntry *build_pkg = codegen_create_package(g, buf_ptr(&build_file_dirname), buf_ptr(&build_file_basename)); g->root_package->package_table.put(buf_create_from_str("@build"), build_pkg); + g->enable_cache = get_cache_opt(enable_cache, true); codegen_build_and_link(g); Termination term; @@ -562,6 +583,17 @@ int main(int argc, char **argv) { fprintf(stderr, "--color options are 'auto', 'on', or 'off'\n"); return usage(arg0); } + } else if (strcmp(arg, "--cache") == 0) { + if (strcmp(argv[i], "auto") == 0) { + enable_cache = CacheOptAuto; + } else if (strcmp(argv[i], "on") == 0) { + enable_cache = CacheOptOn; + } else if (strcmp(argv[i], "off") == 0) { + enable_cache = CacheOptOff; + } else { + fprintf(stderr, "--cache options are 'auto', 'on', or 'off'\n"); + return usage(arg0); + } } else if (strcmp(arg, "--emit") == 0) { if (strcmp(argv[i], "asm") == 0) { emit_file_type = EmitFileTypeAssembly; @@ -894,6 +926,7 @@ int main(int argc, char **argv) { if (cmd == CmdBuild || cmd == CmdRun) { codegen_set_emit_file_type(g, emit_file_type); + g->enable_cache = get_cache_opt(enable_cache, cmd == CmdRun); codegen_build_and_link(g); if (timing_info) codegen_print_timing_report(g, stdout); @@ -913,9 +946,17 @@ int main(int argc, char **argv) { Termination term; os_spawn_process(exec_path, args, &term); return term.code; + } else if (cmd == CmdBuild) { + if (g->enable_cache) { + printf("%s\n", buf_ptr(&g->output_file_path)); + if (g->out_h_path != nullptr) { + printf("%s\n", buf_ptr(g->out_h_path)); + } + } + return EXIT_SUCCESS; + } else { + zig_unreachable(); } - - return EXIT_SUCCESS; } else if (cmd == CmdTranslateC) { codegen_translate_c(g, in_file_buf); ast_render(g, stdout, g->root_import->root, 4); @@ -928,8 +969,11 @@ int main(int argc, char **argv) { ZigTarget native; get_native_target(&native); + g->enable_cache = get_cache_opt(enable_cache, false); codegen_build_and_link(g); - Buf *test_exe_path = &g->output_file_path; + Buf *test_exe_path_unresolved = &g->output_file_path; + Buf *test_exe_path = buf_alloc(); + *test_exe_path = os_path_resolve(&test_exe_path_unresolved, 1); for (size_t i = 0; i < test_exec_args.length; i += 1) { if (test_exec_args.items[i] == nullptr) { diff --git a/src/target.cpp b/src/target.cpp index 91d36c5109e0e070252d73056992c49221d65c0a..f657af8af3d0e32563c00b5216d5d795d0dfcbef 100644 --- a/src/target.cpp +++ b/src/target.cpp @@ -813,6 +813,22 @@ const char *target_exe_file_ext(ZigTarget *target) { } } +const char *target_lib_file_ext(ZigTarget *target, bool is_static, size_t version_major, size_t version_minor, size_t version_patch) { + if (target->os == OsWindows) { + if (is_static) { + return ".lib"; + } else { + return ".dll"; + } + } else { + if (is_static) { + return ".a"; + } else { + return buf_ptr(buf_sprintf(".so.%zu", version_major)); + } + } +} + enum FloatAbi { FloatAbiHard, FloatAbiSoft, diff --git a/src/target.hpp b/src/target.hpp index 5a118f6d8d3f1c7a3162a8c64c520ff180139883..c8658bf35258254d7c2cfac7a361d786d65b0649 100644 --- a/src/target.hpp +++ b/src/target.hpp @@ -113,6 +113,7 @@ const char *target_o_file_ext(ZigTarget *target); const char *target_asm_file_ext(ZigTarget *target); const char *target_llvm_ir_file_ext(ZigTarget *target); const char *target_exe_file_ext(ZigTarget *target); +const char *target_lib_file_ext(ZigTarget *target, bool is_static, size_t version_major, size_t version_minor, size_t version_patch); Buf *target_dynamic_linker(ZigTarget *target); -- 2.54.0 From 15c67d2d50aae11dd425ad2629ebc9770fb95f30 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 11 Sep 2018 16:52:50 -0400 Subject: [PATCH 10/20] fix docgen tests --- doc/docgen.zig | 7 +++++++ src/codegen.cpp | 2 ++ 2 files changed, 9 insertions(+) diff --git a/doc/docgen.zig b/doc/docgen.zig index c1158dc03f57fa3a27e6c4205c5424bef0d15b54..5d77a1ea14b4ac8dd418e9c4d1ab7a8573310f4e 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -11,6 +11,7 @@ const max_doc_file_size = 10 * 1024 * 1024; const exe_ext = std.build.Target(std.build.Target.Native).exeFileExt(); const obj_ext = std.build.Target(std.build.Target.Native).oFileExt(); const tmp_dir_name = "docgen_tmp"; +const test_out_path = tmp_dir_name ++ os.path.sep_str ++ "test" ++ exe_ext; pub fn main() !void { var direct_allocator = std.heap.DirectAllocator.init(); @@ -821,6 +822,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var zig_exe, "test", tmp_source_file_name, + "--output", + test_out_path, }); try out.print("
$ zig test {}.zig", code.name);
                         switch (code.mode) {
@@ -863,6 +866,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
                             "--color",
                             "on",
                             tmp_source_file_name,
+                            "--output",
+                            test_out_path,
                         });
                         try out.print("
$ zig test {}.zig", code.name);
                         switch (code.mode) {
@@ -918,6 +923,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
                             zig_exe,
                             "test",
                             tmp_source_file_name,
+                            "--output",
+                            test_out_path,
                         });
                         switch (code.mode) {
                             builtin.Mode.Debug => {},
diff --git a/src/codegen.cpp b/src/codegen.cpp
index d1239eb23f40d172f9e5c3929bf7e77df84404df..af42f324b425a53bbf3b30c9463c0b51e1cac31e 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -8122,6 +8122,8 @@ static void resolve_out_paths(CodeGen *g) {
 
     if (g->enable_cache || g->out_type != OutTypeObj) {
         os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path);
+    } else if (g->wanted_output_file_path != nullptr && g->out_type == OutTypeObj) {
+        buf_init_from_buf(&g->o_file_output_path, g->wanted_output_file_path);
     } else {
         buf_init_from_buf(&g->o_file_output_path, o_basename);
     }
-- 
2.54.0


From 9227315bf24238c05c0c01a9f516449aa8525e41 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 17:23:36 -0400
Subject: [PATCH 11/20] zig build: better placement of test exe artifact

---
 std/build.zig | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/std/build.zig b/std/build.zig
index 4e323eaf7be1e9803a879daa21ba1e63b46297b1..0ef6109bc846d7f4a415c43d53119f4306a67256 100644
--- a/std/build.zig
+++ b/std/build.zig
@@ -1641,6 +1641,7 @@ pub const TestStep = struct {
     lib_paths: ArrayList([]const u8),
     object_files: ArrayList([]const u8),
     no_rosegment: bool,
+    output_path: ?[]const u8,
 
     pub fn init(builder: *Builder, root_src: []const u8) TestStep {
         const step_name = builder.fmt("test {}", root_src);
@@ -1659,6 +1660,7 @@ pub const TestStep = struct {
             .lib_paths = ArrayList([]const u8).init(builder.allocator),
             .object_files = ArrayList([]const u8).init(builder.allocator),
             .no_rosegment = false,
+            .output_path = null,
         };
     }
 
@@ -1682,6 +1684,24 @@ pub const TestStep = struct {
         self.build_mode = mode;
     }
 
+    pub fn setOutputPath(self: *TestStep, file_path: []const u8) void {
+        self.output_path = file_path;
+
+        // catch a common mistake
+        if (mem.eql(u8, self.builder.pathFromRoot(file_path), self.builder.pathFromRoot("."))) {
+            debug.panic("setOutputPath wants a file path, not a directory\n");
+        }
+    }
+
+    pub fn getOutputPath(self: *TestStep) []const u8 {
+        if (self.output_path) |output_path| {
+            return output_path;
+        } else {
+            const basename = self.builder.fmt("test{}", self.target.exeFileExt());
+            return os.path.join(self.builder.allocator, self.builder.cache_root, basename) catch unreachable;
+        }
+    }
+
     pub fn linkSystemLibrary(self: *TestStep, name: []const u8) void {
         self.link_libs.put(name) catch unreachable;
     }
@@ -1746,6 +1766,10 @@ pub const TestStep = struct {
             builtin.Mode.ReleaseSmall => try zig_args.append("--release-small"),
         }
 
+        const output_path = builder.pathFromRoot(self.getOutputPath());
+        try zig_args.append("--output");
+        try zig_args.append(output_path);
+
         switch (self.target) {
             Target.Native => {},
             Target.Cross => |cross_target| {
-- 
2.54.0


From a1132ffe0ff965ab48c160a658f11bae345bf2d2 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 17:29:18 -0400
Subject: [PATCH 12/20] stage1: build blake code with -std=c99

---
 CMakeLists.txt                 | 13 +++++++------
 src/{blake2b.cpp => blake2b.c} |  0
 2 files changed, 7 insertions(+), 6 deletions(-)
 rename src/{blake2b.cpp => blake2b.c} (100%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 11bb31892ca4e5b3946bc125d37bd90ad189769a..ee9f9c2817ad1479c611caa215a20c67e5d38bdb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -426,8 +426,8 @@ set(ZIG_SOURCES
     "${CMAKE_SOURCE_DIR}/src/util.cpp"
     "${CMAKE_SOURCE_DIR}/src/translate_c.cpp"
 )
-set(ZIG_SOURCES_O3
-    "${CMAKE_SOURCE_DIR}/src/blake2b.cpp"
+set(BLAKE_SOURCES
+    "${CMAKE_SOURCE_DIR}/src/blake2b.c"
 )
 set(ZIG_CPP_SOURCES
     "${CMAKE_SOURCE_DIR}/src/zig_llvm.cpp"
@@ -795,6 +795,7 @@ else()
   set(EXE_CFLAGS "${EXE_CFLAGS} -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D_GNU_SOURCE -fno-exceptions -fno-rtti  -Werror=strict-prototypes -Werror=old-style-definition -Werror=type-limits -Wno-missing-braces")
 endif()
 
+set(BLAKE_CFLAGS "-std=c99")
 
 set(EXE_LDFLAGS " ")
 if(MINGW)
@@ -816,9 +817,9 @@ set_target_properties(zig_cpp PROPERTIES
     COMPILE_FLAGS ${EXE_CFLAGS}
 )
 
-add_library(zig_O3 STATIC ${ZIG_SOURCES_O3})
-set_target_properties(zig_O3 PROPERTIES
-    COMPILE_FLAGS "${EXE_CFLAGS} -O3"
+add_library(embedded_blake STATIC ${BLAKE_SOURCES})
+set_target_properties(embedded_blake PROPERTIES
+    COMPILE_FLAGS "${BLAKE_CFLAGS} -O3"
 )
 
 add_executable(zig ${ZIG_SOURCES})
@@ -829,7 +830,7 @@ set_target_properties(zig PROPERTIES
 
 target_link_libraries(zig LINK_PUBLIC
     zig_cpp
-    zig_O3
+    embedded_blake
     ${SOFTFLOAT_LIBRARIES}
     ${CLANG_LIBRARIES}
     ${LLD_LIBRARIES}
diff --git a/src/blake2b.cpp b/src/blake2b.c
similarity index 100%
rename from src/blake2b.cpp
rename to src/blake2b.c
-- 
2.54.0


From 1a4dcf10fecf3fed7db0573fc859718e05e93f11 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 17:42:03 -0400
Subject: [PATCH 13/20] darwin fixups

---
 src/os.cpp | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/os.cpp b/src/os.cpp
index f3569ca3bbe164b7e453d5c7640ef63b80569231..9970c04c72aef9df7e503e33eb6b74042310c3b8 100644
--- a/src/os.cpp
+++ b/src/os.cpp
@@ -1657,6 +1657,14 @@ Error os_self_exe_shared_libs(ZigList &paths) {
     paths.resize(0);
     dl_iterate_phdr(self_exe_shared_libs_callback, &paths);
     return ErrorNone;
+#elif defined(ZIG_OS_DARWIN)
+    paths.resize(0);
+    uint32_t img_count = _dyld_image_count();
+    for (uint32_t i = 0; i != img_count; i += 1) {
+        const char *name = _dyld_get_image_name(i);
+        paths.append(buf_create_from_str(name));
+    }
+    return ErrorNone;
 #else
 #error "unimplemented"
 #endif
@@ -1750,7 +1758,7 @@ Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) {
 Error os_file_mtime(OsFile file, OsTimeStamp *mtime) {
 #if defined(ZIG_OS_WINDOWS)
 #error unimplemented
-#else
+#elif defined(ZIG_OS_LINUX)
     struct stat statbuf;
     if (fstat(file, &statbuf) == -1)
         return ErrorFileSystem;
@@ -1758,6 +1766,16 @@ Error os_file_mtime(OsFile file, OsTimeStamp *mtime) {
     mtime->sec = statbuf.st_mtim.tv_sec;
     mtime->nsec = statbuf.st_mtim.tv_nsec;
     return ErrorNone;
+#elif defined(ZIG_OS_DARWIN)
+    struct stat statbuf;
+    if (fstat(file, &statbuf) == -1)
+        return ErrorFileSystem;
+
+    mtime->sec = statbuf.st_mtimespec.tv_sec;
+    mtime->nsec = statbuf.st_mtimespec.tv_nsec;
+    return ErrorNone;
+#else
+#error unimplemented
 #endif
 }
 
-- 
2.54.0


From 04dc5cdacaa953ca5365245f32158db70213b90e Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 18:15:08 -0400
Subject: [PATCH 14/20] zig build: make the cache root dir before building

---
 std/build.zig | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/std/build.zig b/std/build.zig
index 0ef6109bc846d7f4a415c43d53119f4306a67256..5800e69c62c27c4aef78db61821ccb441032ed09 100644
--- a/std/build.zig
+++ b/std/build.zig
@@ -232,6 +232,8 @@ pub const Builder = struct {
     }
 
     pub fn make(self: *Builder, step_names: []const []const u8) !void {
+        try self.makePath(self.cache_root);
+
         var wanted_steps = ArrayList(*Step).init(self.allocator);
         defer wanted_steps.deinit();
 
-- 
2.54.0


From 7e9f25dd18b7da750e547de94a779e3c14c07d94 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 20:54:39 -0400
Subject: [PATCH 15/20] stage1: clean up timing report in test mode

---
 src/codegen.cpp | 7 +++++--
 src/main.cpp    | 8 +++++---
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/codegen.cpp b/src/codegen.cpp
index af42f324b425a53bbf3b30c9463c0b51e1cac31e..96e979f9d364019d7aab994fbb7352c2eeb6b8ac 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -6388,9 +6388,9 @@ static void do_code_gen(CodeGen *g) {
     char *error = nullptr;
     LLVMVerifyModule(g->module, LLVMAbortProcessAction, &error);
 #endif
+}
 
-    codegen_add_time_event(g, "LLVM Emit Output");
-
+static void zig_llvm_emit_output(CodeGen *g) {
     bool is_small = g->build_mode == BuildModeSmallRelease;
 
     Buf *output_path = &g->o_file_output_path;
@@ -8228,6 +8228,9 @@ void codegen_build_and_link(CodeGen *g) {
 
         codegen_add_time_event(g, "Code Generation");
         do_code_gen(g);
+        codegen_add_time_event(g, "LLVM Emit Output");
+        zig_llvm_emit_output(g);
+
         if (g->want_h_file) {
             codegen_add_time_event(g, "Generate .h");
             gen_h_file(g);
diff --git a/src/main.cpp b/src/main.cpp
index e4019f10df50258c5e15ee4351a5114725f4acaa..1751fabfe5400afdcfaf2cc9dd48dbe13215d057 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -971,6 +971,11 @@ int main(int argc, char **argv) {
 
                 g->enable_cache = get_cache_opt(enable_cache, false);
                 codegen_build_and_link(g);
+
+                if (timing_info) {
+                    codegen_print_timing_report(g, stdout);
+                }
+
                 Buf *test_exe_path_unresolved = &g->output_file_path;
                 Buf *test_exe_path = buf_alloc();
                 *test_exe_path = os_path_resolve(&test_exe_path_unresolved, 1);
@@ -981,7 +986,6 @@ int main(int argc, char **argv) {
                     }
                 }
 
-
                 if (!target_can_exec(&native, target)) {
                     fprintf(stderr, "Created %s but skipping execution because it is non-native.\n",
                             buf_ptr(test_exe_path));
@@ -1003,8 +1007,6 @@ int main(int argc, char **argv) {
                 if (term.how != TerminationIdClean || term.code != 0) {
                     fprintf(stderr, "\nTests failed. Use the following command to reproduce the failure:\n");
                     fprintf(stderr, "%s\n", buf_ptr(test_exe_path));
-                } else if (timing_info) {
-                    codegen_print_timing_report(g, stdout);
                 }
                 return (term.how == TerminationIdClean) ? term.code : -1;
             } else {
-- 
2.54.0


From ee263a15cc8fb52c2ac6053a29168fb15089839b Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 22:25:52 -0400
Subject: [PATCH 16/20] bring back zig-cache

we need somewhere to put .o files and leave them while the user
executes their program, so that stack traces on MacOS can find
the .o files and get at the DWARF info.

if we try to clean up old global tmp dir files, first of all that's
a hard and complicated problem, and secondly it's not clear how
that is better than dumping the .o file inside zig-cache locally.
---
 src/all_types.hpp  |  1 +
 src/cache_hash.cpp |  8 --------
 src/cache_hash.hpp |  4 ----
 src/codegen.cpp    |  5 +----
 src/main.cpp       | 13 +++++--------
 5 files changed, 7 insertions(+), 24 deletions(-)

diff --git a/src/all_types.hpp b/src/all_types.hpp
index f9c50eeaac806619a7b7563ae69ca0d3310fe422..3dfbc6ba7f0520e7bcb46322553e37eabe34bce2 100644
--- a/src/all_types.hpp
+++ b/src/all_types.hpp
@@ -1682,6 +1682,7 @@ struct CodeGen {
     Buf output_file_path;
     Buf o_file_output_path;
     Buf *wanted_output_file_path;
+    Buf cache_dir;
 
     IrInstruction *invalid_instruction;
 
diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp
index 65e3e62ae0c6abcb9529b06e8b47b2484422548c..b302946310c7145b60100290d6f35ceebfbc2be7 100644
--- a/src/cache_hash.cpp
+++ b/src/cache_hash.cpp
@@ -467,11 +467,3 @@ void cache_release(CacheHash *ch) {
     assert(ch->manifest_file_path != nullptr);
     os_file_close(ch->manifest_file);
 }
-
-Buf *get_random_basename() {
-    Buf *result = buf_alloc();
-    for (size_t i = 0; i < 16; i += 1) {
-        buf_append_char(result, base64_fs_alphabet[rand() % 64]);
-    }
-    return result;
-}
diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp
index 6acd805be65e89807115666dfa358df0ca18fda0..db1c42ec03f3087c59af45b2b67d2d8c7a79feb4 100644
--- a/src/cache_hash.hpp
+++ b/src/cache_hash.hpp
@@ -68,8 +68,4 @@ Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_b64_digest);
 void cache_release(CacheHash *ch);
 
 
-
-// Completely independent function. Just returns a random filename safe basename.
-Buf *get_random_basename();
-
 #endif
diff --git a/src/codegen.cpp b/src/codegen.cpp
index 96e979f9d364019d7aab994fbb7352c2eeb6b8ac..aeb2b6edc4de2fb795fb3c5cc067e1a9c3511bea 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -8195,8 +8195,6 @@ void codegen_build_and_link(CodeGen *g) {
         }
 
         os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir);
-    } else {
-        os_path_join(stage1_dir, buf_create_from_str("tmp"), artifact_dir);
     }
 
     if (g->enable_cache && buf_len(&digest) != 0) {
@@ -8217,8 +8215,7 @@ void codegen_build_and_link(CodeGen *g) {
             }
             os_path_join(artifact_dir, &digest, &g->artifact_dir);
         } else {
-            Buf *tmp_basename = get_random_basename(); 
-            os_path_join(artifact_dir, tmp_basename, &g->artifact_dir);
+            buf_init_from_buf(&g->artifact_dir, &g->cache_dir);
         }
         if ((err = os_make_path(&g->artifact_dir))) {
             fprintf(stderr, "Unable to create artifact directory: %s\n", err_str(err));
diff --git a/src/main.cpp b/src/main.cpp
index 1751fabfe5400afdcfaf2cc9dd48dbe13215d057..f18dd949b0d799492f71f7ad7441eb7824cd1d76 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -353,7 +353,7 @@ int main(int argc, char **argv) {
     size_t ver_minor = 0;
     size_t ver_patch = 0;
     bool timing_info = false;
-    const char *cache_dir = nullptr;
+    const char *cache_dir = default_zig_cache_name;
     CliPkg *cur_pkg = allocate(1);
     BuildMode build_mode = BuildModeDebug;
     ZigList test_exec_args = {0};
@@ -402,6 +402,7 @@ int main(int argc, char **argv) {
         os_path_join(special_dir, buf_create_from_str("build_runner.zig"), build_runner_path);
 
         CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf);
+        buf_init_from_str(&g->cache_dir, cache_dir);
         codegen_set_out_name(g, buf_create_from_str("build"));
 
         Buf *build_file_buf = buf_create_from_str(build_file);
@@ -410,13 +411,8 @@ int main(int argc, char **argv) {
         Buf build_file_dirname = BUF_INIT;
         os_path_split(&build_file_abs, &build_file_dirname, &build_file_basename);
 
-        Buf full_cache_dir = BUF_INIT;
-        if (cache_dir == nullptr) {
-            os_path_join(&build_file_dirname, buf_create_from_str(default_zig_cache_name), &full_cache_dir);
-        } else {
-            Buf *cache_dir_buf = buf_create_from_str(cache_dir);
-            full_cache_dir = os_path_resolve(&cache_dir_buf, 1);
-        }
+        Buf *cache_dir_buf = buf_create_from_str(cache_dir);
+        Buf full_cache_dir = os_path_resolve(&cache_dir_buf, 1);
 
         args.items[1] = buf_ptr(&build_file_dirname);
         args.items[2] = buf_ptr(&full_cache_dir);
@@ -832,6 +828,7 @@ int main(int argc, char **argv) {
             Buf *zig_lib_dir_buf = resolve_zig_lib_dir();
 
             CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf);
+            buf_init_from_str(&g->cache_dir, cache_dir);
             codegen_set_out_name(g, buf_out_name);
             codegen_set_lib_version(g, ver_major, ver_minor, ver_patch);
             codegen_set_is_test(g, cmd == CmdTest);
-- 
2.54.0


From 014cc60a72ac2d64935bf424459b5fa13e281ed9 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 22:46:22 -0400
Subject: [PATCH 17/20] rename --enable-timing-info to -ftime-report to match
 clang

and have it print llvm's internal timing info
---
 src/all_types.hpp | 1 +
 src/codegen.cpp   | 9 ++++++---
 src/main.cpp      | 6 ++++--
 src/zig_llvm.cpp  | 9 ++++++++-
 src/zig_llvm.h    | 3 ++-
 5 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/src/all_types.hpp b/src/all_types.hpp
index 3dfbc6ba7f0520e7bcb46322553e37eabe34bce2..8aa2718d303fe56cf700f0a62f8b1a3a6d9c64de 100644
--- a/src/all_types.hpp
+++ b/src/all_types.hpp
@@ -1727,6 +1727,7 @@ struct CodeGen {
     bool error_during_imports;
     bool generate_error_name_table;
     bool enable_cache;
+    bool enable_time_report;
 
     //////////////////////////// Participates in Input Parameter Cache Hash
     ZigList link_libs_list;
diff --git a/src/codegen.cpp b/src/codegen.cpp
index aeb2b6edc4de2fb795fb3c5cc067e1a9c3511bea..6f21ceecabeeed13a43b20a5b74cf2f319e74b52 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -6398,7 +6398,8 @@ static void zig_llvm_emit_output(CodeGen *g) {
     switch (g->emit_file_type) {
         case EmitFileTypeBinary:
             if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path),
-                        ZigLLVM_EmitBinary, &err_msg, g->build_mode == BuildModeDebug, is_small))
+                        ZigLLVM_EmitBinary, &err_msg, g->build_mode == BuildModeDebug, is_small,
+                        g->enable_time_report))
             {
                 zig_panic("unable to write object file %s: %s", buf_ptr(output_path), err_msg);
             }
@@ -6408,7 +6409,8 @@ static void zig_llvm_emit_output(CodeGen *g) {
 
         case EmitFileTypeAssembly:
             if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path),
-                        ZigLLVM_EmitAssembly, &err_msg, g->build_mode == BuildModeDebug, is_small))
+                        ZigLLVM_EmitAssembly, &err_msg, g->build_mode == BuildModeDebug, is_small,
+                        g->enable_time_report))
             {
                 zig_panic("unable to write assembly file %s: %s", buf_ptr(output_path), err_msg);
             }
@@ -6417,7 +6419,8 @@ static void zig_llvm_emit_output(CodeGen *g) {
 
         case EmitFileTypeLLVMIr:
             if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path),
-                        ZigLLVM_EmitLLVMIr, &err_msg, g->build_mode == BuildModeDebug, is_small))
+                        ZigLLVM_EmitLLVMIr, &err_msg, g->build_mode == BuildModeDebug, is_small,
+                        g->enable_time_report))
             {
                 zig_panic("unable to write llvm-ir file %s: %s", buf_ptr(output_path), err_msg);
             }
diff --git a/src/main.cpp b/src/main.cpp
index f18dd949b0d799492f71f7ad7441eb7824cd1d76..fdc34a83ae4a8463a3f15b13b445d06582856042 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -37,7 +37,7 @@ static int usage(const char *arg0) {
         "  --cache [auto|off|on]        build to the global cache and print output path to stdout\n"
         "  --color [auto|off|on]        enable or disable colored error messages\n"
         "  --emit [asm|bin|llvm-ir]     emit a specific file format as compilation output\n"
-        "  --enable-timing-info         print timing diagnostics\n"
+        "  -ftime-report                print timing diagnostics\n"
         "  --libc-include-dir [path]    directory where libc stdlib.h resides\n"
         "  --name [name]                override output name\n"
         "  --output [file]              override destination path\n"
@@ -402,6 +402,7 @@ int main(int argc, char **argv) {
         os_path_join(special_dir, buf_create_from_str("build_runner.zig"), build_runner_path);
 
         CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf);
+        g->enable_time_report = timing_info;
         buf_init_from_str(&g->cache_dir, cache_dir);
         codegen_set_out_name(g, buf_create_from_str("build"));
 
@@ -533,7 +534,7 @@ int main(int argc, char **argv) {
                 no_rosegment_workaround = true;
             } else if (strcmp(arg, "--each-lib-rpath") == 0) {
                 each_lib_rpath = true;
-            } else if (strcmp(arg, "--enable-timing-info") == 0) {
+            } else if (strcmp(arg, "-ftime-report") == 0) {
                 timing_info = true;
             } else if (strcmp(arg, "--test-cmd-bin") == 0) {
                 test_exec_args.append(nullptr);
@@ -828,6 +829,7 @@ int main(int argc, char **argv) {
             Buf *zig_lib_dir_buf = resolve_zig_lib_dir();
 
             CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf);
+            g->enable_time_report = timing_info;
             buf_init_from_str(&g->cache_dir, cache_dir);
             codegen_set_out_name(g, buf_out_name);
             codegen_set_lib_version(g, ver_major, ver_minor, ver_patch);
diff --git a/src/zig_llvm.cpp b/src/zig_llvm.cpp
index a43d2d182c0360eff512a95d757bbb3f86ab9399..61287f620cf638aaf14d47f59856bc579a87d866 100644
--- a/src/zig_llvm.cpp
+++ b/src/zig_llvm.cpp
@@ -30,6 +30,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -81,8 +82,11 @@ static const bool assertions_on = false;
 #endif
 
 bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMModuleRef module_ref,
-        const char *filename, ZigLLVM_EmitOutputType output_type, char **error_message, bool is_debug, bool is_small)
+        const char *filename, ZigLLVM_EmitOutputType output_type, char **error_message, bool is_debug,
+        bool is_small, bool time_report)
 {
+    TimePassesIsEnabled = time_report;
+
     std::error_code EC;
     raw_fd_ostream dest(filename, EC, sys::fs::F_None);
     if (EC) {
@@ -182,6 +186,9 @@ bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMM
         }
     }
 
+    if (time_report) {
+        TimerGroup::printAll(errs());
+    }
     return false;
 }
 
diff --git a/src/zig_llvm.h b/src/zig_llvm.h
index 63d69bd23ed8309c2fd5e930c2be27b95ce695ff..5cdc6cc0410ba738a9d42c146b52f06c95e8d086 100644
--- a/src/zig_llvm.h
+++ b/src/zig_llvm.h
@@ -55,7 +55,8 @@ enum ZigLLVM_EmitOutputType {
 };
 
 ZIG_EXTERN_C bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMModuleRef module_ref,
-        const char *filename, enum ZigLLVM_EmitOutputType output_type, char **error_message, bool is_debug, bool is_small);
+        const char *filename, enum ZigLLVM_EmitOutputType output_type, char **error_message, bool is_debug,
+        bool is_small, bool time_report);
 
 ZIG_EXTERN_C LLVMTypeRef ZigLLVMTokenTypeInContext(LLVMContextRef context_ref);
 
-- 
2.54.0


From ff0b7fe29a49a5619b3c2f7fee0baccb979caf2d Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 22:57:39 -0400
Subject: [PATCH 18/20] error messages for attempted cache when zig cannot
 perfectly do it

---
 src/analyze.cpp | 6 ++++++
 src/ir.cpp      | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/src/analyze.cpp b/src/analyze.cpp
index bae4b15f81c80e678aef2d7b09165501c03beddc..59be76518b9323e2a012dc004a38abd6a9db5187 100644
--- a/src/analyze.cpp
+++ b/src/analyze.cpp
@@ -6289,6 +6289,12 @@ LinkLib *add_link_lib(CodeGen *g, Buf *name) {
     if (is_libc && g->libc_link_lib != nullptr)
         return g->libc_link_lib;
 
+    if (g->enable_cache && is_libc && g->zig_target.os != OsMacOSX && g->zig_target.os != OsIOS) {
+        fprintf(stderr, "TODO linking against libc is currently incompatible with `--cache on`.\n"
+        "Zig is not yet capable of determining whether the libc installation has changed on subsequent builds.\n");
+        exit(1);
+    }
+
     for (size_t i = 0; i < g->link_libs_list.length; i += 1) {
         LinkLib *existing_lib = g->link_libs_list.at(i);
         if (buf_eql_buf(existing_lib->name, name)) {
diff --git a/src/ir.cpp b/src/ir.cpp
index 858a6df33c30f313f7cae31df66cddaec306b214..030f6627ad917dad72c6f79b046ad554d0dd8506 100644
--- a/src/ir.cpp
+++ b/src/ir.cpp
@@ -17961,6 +17961,12 @@ static ZigType *ir_analyze_instruction_type_name(IrAnalyze *ira, IrInstructionTy
 }
 
 static ZigType *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstructionCImport *instruction) {
+    if (ira->codegen->enable_cache) {
+        ir_add_error(ira, &instruction->base,
+            buf_sprintf("TODO @cImport is incompatible with --cache on. The cache system currently is unable to detect subsequent changes in .h files."));
+        return ira->codegen->builtin_types.entry_invalid;
+    }
+
     AstNode *node = instruction->base.source_node;
     assert(node->type == NodeTypeFnCallExpr);
     AstNode *block_node = node->data.fn_call_expr.params.at(0);
-- 
2.54.0


From 1caa48c2df63d61b8d2501b3e3ee7c85ee8d89cb Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 12 Sep 2018 11:33:26 -0400
Subject: [PATCH 19/20] windows os.cpp implementations

---
 src/error.cpp |   2 +
 src/error.hpp |   2 +
 src/os.cpp    | 207 +++++++++++++++++++++++++++++++++++++++++++++++---
 src/os.hpp    |   2 +-
 4 files changed, 203 insertions(+), 10 deletions(-)

diff --git a/src/error.cpp b/src/error.cpp
index d503eaa18b5a37af668cc129933dc24b6f44e080..7a9bd963bb7c3a134224b2d7a0fae89a32e26aa1 100644
--- a/src/error.cpp
+++ b/src/error.cpp
@@ -30,6 +30,8 @@ const char *err_str(int err) {
         case ErrorEndOfFile: return "end of file";
         case ErrorIsDir: return "is directory";
         case ErrorUnsupportedOperatingSystem: return "unsupported operating system";
+        case ErrorSharingViolation: return "sharing violation";
+        case ErrorPipeBusy: return "pipe busy";
     }
     return "(invalid error)";
 }
diff --git a/src/error.hpp b/src/error.hpp
index 8c1f8b9aa585ba876161f204988adbdbb66e93d9..0996a41d58b184ddbe9804cb17e69e097444a077 100644
--- a/src/error.hpp
+++ b/src/error.hpp
@@ -30,6 +30,8 @@ enum Error {
     ErrorEndOfFile,
     ErrorIsDir,
     ErrorUnsupportedOperatingSystem,
+    ErrorSharingViolation,
+    ErrorPipeBusy,
 };
 
 const char *err_str(int err);
diff --git a/src/os.cpp b/src/os.cpp
index 9970c04c72aef9df7e503e33eb6b74042310c3b8..6e46b96e41fd8eb05b3566a6146a23f13427b908 100644
--- a/src/os.cpp
+++ b/src/os.cpp
@@ -24,6 +24,7 @@
 #endif
 
 #include 
+#include 
 #include 
 #include 
 
@@ -1393,7 +1394,7 @@ Error os_self_exe_path(Buf *out_path) {
         }
         if (copied_amt < buf_len(out_path)) {
             buf_resize(out_path, copied_amt);
-            return 0;
+            return ErrorNone;
         }
         buf_resize(out_path, buf_len(out_path) * 2);
     }
@@ -1616,10 +1617,118 @@ int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf* output_buf, ZigLLVM_ArchTy
 #endif
 }
 
-// Ported from std/os/get_app_data_dir.zig
+#if defined(ZIG_OS_WINDOWS)
+// Ported from std/unicode.zig
+struct Utf16LeIterator {
+    uint8_t *bytes;
+    size_t i;
+};
+
+// Ported from std/unicode.zig
+static Utf16LeIterator Utf16LeIterator_init(WCHAR *ptr) {
+    return {(uint8_t*)ptr, 0};
+}
+
+// Ported from std/unicode.zig
+static Optional Utf16LeIterator_nextCodepoint(Utf16LeIterator *it) {
+    if (it->bytes[it->i] == 0 && it->bytes[it->i + 1] == 0)
+        return {};
+    uint32_t c0 = ((uint32_t)it->bytes[it->i]) | (((uint32_t)it->bytes[it->i + 1]) << 8);
+    if (c0 & ~((uint32_t)0x03ff) == 0xd800) {
+        // surrogate pair
+        it->i += 2;
+        assert(it->bytes[it->i] != 0 || it->bytes[it->i + 1] != 0);
+        uint32_t c1 = ((uint32_t)it->bytes[it->i]) | (((uint32_t)it->bytes[it->i + 1]) << 8);
+        assert(c1 & ~((uint32_t)0x03ff) == 0xdc00);
+        it->i += 2;
+        return Optional::some(0x10000 + (((c0 & 0x03ff) << 10) | (c1 & 0x03ff)));
+    } else {
+        assert(c0 & ~((uint32_t)0x03ff) != 0xdc00);
+        it->i += 2;
+        return Optional::some(c0);
+    }
+}
+
+// Ported from std/unicode.zig
+static uint8_t utf8CodepointSequenceLength(uint32_t c) {
+    if (c < 0x80) return 1;
+    if (c < 0x800) return 2;
+    if (c < 0x10000) return 3;
+    if (c < 0x110000) return 4;
+    zig_unreachable();
+}
+
+// Ported from std/unicode.zig
+static size_t utf8Encode(uint32_t c, Slice out) {
+    size_t length = utf8CodepointSequenceLength(c);
+    assert(out.len >= length);
+    switch (length) {
+        // The pattern for each is the same
+        // - Increasing the initial shift by 6 each time
+        // - Each time after the first shorten the shifted
+        //   value to a max of 0b111111 (63)
+        case 1:
+            out.ptr[0] = c; // Can just do 0 + codepoint for initial range
+            break;
+        case 2:
+            out.ptr[0] = 0b11000000 | (c >> 6);
+            out.ptr[1] = 0b10000000 | (c & 0b111111);
+            break;
+        case 3:
+            assert(!(0xd800 <= c && c <= 0xdfff));
+            out.ptr[0] = 0b11100000 | (c >> 12);
+            out.ptr[1] = 0b10000000 | ((c >> 6) & 0b111111);
+            out.ptr[2] = 0b10000000 | (c & 0b111111);
+            break;
+        case 4:
+            out.ptr[0] = 0b11110000 | (c >> 18);
+            out.ptr[1] = 0b10000000 | ((c >> 12) & 0b111111);
+            out.ptr[2] = 0b10000000 | ((c >> 6) & 0b111111);
+            out.ptr[3] = 0b10000000 | (c & 0b111111);
+            break;
+        default:
+            zig_unreachable();
+    }
+    return length;
+}
+
+// Ported from std.unicode.utf16leToUtf8Alloc
+static void utf16le_ptr_to_utf8(Buf *out, WCHAR *utf16le) {
+    // optimistically guess that it will all be ascii.
+    buf_resize(out, 0);
+    size_t out_index = 0;
+    Utf16LeIterator it = Utf16LeIterator_init(utf16le);
+    for (;;) {
+        Optional opt_codepoint = Utf16LeIterator_nextCodepoint(&it);
+        if (!opt_codepoint.is_some) break;
+        uint32_t codepoint = opt_codepoint.value;
+
+        size_t utf8_len = utf8CodepointSequenceLength(codepoint);
+        buf_resize(out, buf_len(out) + utf8_len);
+        utf8Encode(codepoint, {(uint8_t*)buf_ptr(out)+out_index, buf_len(out)-out_index});
+        out_index += utf8_len;
+    }
+}
+#endif
+
+// Ported from std.os.getAppDataDir
 Error os_get_app_data_dir(Buf *out_path, const char *appname) {
 #if defined(ZIG_OS_WINDOWS)
-#error "Unimplemented"
+    Error err;
+    WCHAR *dir_path_ptr;
+    switch (SHGetKnownFolderPath(FOLDERID_LocalAppData, KF_FLAG_CREATE, nullptr, &dir_path_ptr)) {
+        case S_OK:
+            // defer os.windows.CoTaskMemFree(@ptrCast(*c_void, dir_path_ptr));
+            utf16le_ptr_to_utf8(out_path, dir_path_ptr);
+            CoTaskMemFree(dir_path_ptr);
+            buf_appendf(out_path, "\\%s", appname);
+            return ErrorNone;
+        case E_OUTOFMEMORY:
+            return ErrorNoMem;
+        default:
+            return ErrorUnexpected;
+    }
+    zig_unreachable();
 #elif defined(ZIG_OS_DARWIN)
     const char *home_dir = getenv("HOME");
     if (home_dir == nullptr) {
@@ -1665,14 +1774,44 @@ Error os_self_exe_shared_libs(ZigList &paths) {
         paths.append(buf_create_from_str(name));
     }
     return ErrorNone;
+#elif defined(ZIG_OS_WINDOWS)
+    // zig is built statically on windows, so we can return an empty list
+    paths.resize(0);
+    return ErrorNone;
 #else
-#error "unimplemented"
+#error unimplemented
 #endif
 }
 
 Error os_file_open_r(Buf *full_path, OsFile *out_file) {
 #if defined(ZIG_OS_WINDOWS)
-#error "unimplemented"
+    // TODO use CreateFileW
+    HANDLE result = CreateFileA(buf_ptr(full_path), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
+
+    if (result == INVALID_HANDLE_VALUE) {
+        DWORD err = GetLastError();
+        switch (err) {
+            case ERROR_SHARING_VIOLATION:
+                return ErrorSharingViolation;
+            case ERROR_ALREADY_EXISTS:
+                return ErrorPathAlreadyExists;
+            case ERROR_FILE_EXISTS:
+                return ErrorPathAlreadyExists;
+            case ERROR_FILE_NOT_FOUND:
+                return ErrorFileNotFound;
+            case ERROR_PATH_NOT_FOUND:
+                return ErrorFileNotFound;
+            case ERROR_ACCESS_DENIED:
+                return ErrorAccess;
+            case ERROR_PIPE_BUSY:
+                return ErrorPipeBusy;
+            default:
+                return ErrorUnexpected;
+        }
+    }
+
+    *out_file = result;
+    return ErrorNone;
 #else
     for (;;) {
         int fd = open(buf_ptr(full_path), O_RDONLY|O_CLOEXEC);
@@ -1702,7 +1841,36 @@ Error os_file_open_r(Buf *full_path, OsFile *out_file) {
 
 Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) {
 #if defined(ZIG_OS_WINDOWS)
-#error "unimplemented"
+    for (;;) {
+        HANDLE result = CreateFileA(buf_ptr(full_path), GENERIC_READ | GENERIC_WRITE,
+            0, nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
+
+        if (result == INVALID_HANDLE_VALUE) {
+            DWORD err = GetLastError();
+            switch (err) {
+                case ERROR_SHARING_VIOLATION:
+                    // TODO wait for the lock instead of sleeping
+                    Sleep(10);
+                    continue;
+                case ERROR_ALREADY_EXISTS:
+                    return ErrorPathAlreadyExists;
+                case ERROR_FILE_EXISTS:
+                    return ErrorPathAlreadyExists;
+                case ERROR_FILE_NOT_FOUND:
+                    return ErrorFileNotFound;
+                case ERROR_PATH_NOT_FOUND:
+                    return ErrorFileNotFound;
+                case ERROR_ACCESS_DENIED:
+                    return ErrorAccess;
+                case ERROR_PIPE_BUSY:
+                    return ErrorPipeBusy;
+                default:
+                    return ErrorUnexpected;
+            }
+        }
+        *out_file = result;
+        return ErrorNone;
+    }
 #else
     int fd;
     for (;;) {
@@ -1757,7 +1925,12 @@ Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) {
 
 Error os_file_mtime(OsFile file, OsTimeStamp *mtime) {
 #if defined(ZIG_OS_WINDOWS)
-#error unimplemented
+    FILETIME last_write_time;
+    if (!GetFileTime(file, nullptr, nullptr, &last_write_time))
+        return ErrorUnexpected;
+    mtime->sec = last_write_time.dwLowDateTime | (last_write_time.dwHighDateTime << 32);
+    mtime->nsec = 0;
+    return ErrorNone;
 #elif defined(ZIG_OS_LINUX)
     struct stat statbuf;
     if (fstat(file, &statbuf) == -1)
@@ -1781,7 +1954,11 @@ Error os_file_mtime(OsFile file, OsTimeStamp *mtime) {
 
 Error os_file_read(OsFile file, void *ptr, size_t *len) {
 #if defined(ZIG_OS_WINDOWS)
-#error unimplemented
+    DWORD amt_read;
+    if (ReadFile(file, ptr, *len, &amt_read, nullptr) == 0)
+        return ErrorUnexpected;
+    *len = amt_read;
+    return ErrorNone;
 #else
     for (;;) {
         ssize_t rc = read(file, ptr, *len);
@@ -1830,10 +2007,18 @@ Error os_file_read_all(OsFile file, Buf *contents) {
 
 Error os_file_overwrite(OsFile file, Buf *contents) {
 #if defined(ZIG_OS_WINDOWS)
-#error unimplemented
+    if (SetFilePointer(file, 0, nullptr, FILE_BEGIN) == INVALID_SET_FILE_POINTER)
+        return ErrorFileSystem;
+    if (!SetEndOfFile(file))
+        return ErrorFileSystem;
+    if (!WriteFile(file, buf_ptr(contents), buf_len(contents), nullptr, nullptr))
+        return ErrorFileSystem;
+    return ErrorNone;
 #else
     if (lseek(file, 0, SEEK_SET) == -1)
         return ErrorFileSystem;
+    if (ftruncate(file, 0) == -1)
+        return ErrorFileSystem;
     for (;;) {
         if (write(file, buf_ptr(contents), buf_len(contents)) == -1) {
             switch (errno) {
@@ -1853,5 +2038,9 @@ Error os_file_overwrite(OsFile file, Buf *contents) {
 }
 
 void os_file_close(OsFile file) {
+#if defined(ZIG_OS_WINDOWS)
+    CloseHandle(file);
+#else
     close(file);
+#endif
 }
diff --git a/src/os.hpp b/src/os.hpp
index ac422fbd21bc901e63334893253196c7e00f3b3c..1054bf24a77ebc612ddadec764582ced6d0b28b1 100644
--- a/src/os.hpp
+++ b/src/os.hpp
@@ -72,7 +72,7 @@ struct Termination {
 };
 
 #if defined(ZIG_OS_WINDOWS)
-#define OsFile (void *)
+#define OsFile void *
 #else
 #define OsFile int
 #endif
-- 
2.54.0


From 3a49d115cf38154f0094d9615334529890059006 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 12 Sep 2018 11:49:46 -0400
Subject: [PATCH 20/20] fix zig build cache dir path

---
 build.zig    |  3 ++-
 src/main.cpp | 16 +++++++++++-----
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/build.zig b/build.zig
index d4a159e5e6eb9781b231dcb2efb146e714deaacc..e886377139fbc760416b12e0521ec3241a4daf93 100644
--- a/build.zig
+++ b/build.zig
@@ -16,11 +16,12 @@ pub fn build(b: *Builder) !void {
     var docgen_exe = b.addExecutable("docgen", "doc/docgen.zig");
 
     const rel_zig_exe = try os.path.relative(b.allocator, b.build_root, b.zig_exe);
+    const langref_out_path = os.path.join(b.allocator, b.cache_root, "langref.html") catch unreachable;
     var docgen_cmd = b.addCommand(null, b.env_map, [][]const u8{
         docgen_exe.getOutputPath(),
         rel_zig_exe,
         "doc" ++ os.path.sep_str ++ "langref.html.in",
-        os.path.join(b.allocator, b.cache_root, "langref.html") catch unreachable,
+        langref_out_path,
     });
     docgen_cmd.step.dependOn(&docgen_exe.step);
 
diff --git a/src/main.cpp b/src/main.cpp
index fdc34a83ae4a8463a3f15b13b445d06582856042..bf6c142975f4e1291b3d0cb99d04c1082d8bfba3 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -353,7 +353,7 @@ int main(int argc, char **argv) {
     size_t ver_minor = 0;
     size_t ver_patch = 0;
     bool timing_info = false;
-    const char *cache_dir = default_zig_cache_name;
+    const char *cache_dir = nullptr;
     CliPkg *cur_pkg = allocate(1);
     BuildMode build_mode = BuildModeDebug;
     ZigList test_exec_args = {0};
@@ -403,7 +403,7 @@ int main(int argc, char **argv) {
 
         CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf);
         g->enable_time_report = timing_info;
-        buf_init_from_str(&g->cache_dir, cache_dir);
+        buf_init_from_str(&g->cache_dir, cache_dir ? cache_dir : default_zig_cache_name);
         codegen_set_out_name(g, buf_create_from_str("build"));
 
         Buf *build_file_buf = buf_create_from_str(build_file);
@@ -412,8 +412,14 @@ int main(int argc, char **argv) {
         Buf build_file_dirname = BUF_INIT;
         os_path_split(&build_file_abs, &build_file_dirname, &build_file_basename);
 
-        Buf *cache_dir_buf = buf_create_from_str(cache_dir);
-        Buf full_cache_dir = os_path_resolve(&cache_dir_buf, 1);
+
+        Buf full_cache_dir = BUF_INIT;
+        if (cache_dir == nullptr) {
+            os_path_join(&build_file_dirname, buf_create_from_str(default_zig_cache_name), &full_cache_dir);
+        } else {
+            Buf *cache_dir_buf = buf_create_from_str(cache_dir);
+            full_cache_dir = os_path_resolve(&cache_dir_buf, 1);
+        }
 
         args.items[1] = buf_ptr(&build_file_dirname);
         args.items[2] = buf_ptr(&full_cache_dir);
@@ -830,7 +836,7 @@ int main(int argc, char **argv) {
 
             CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf);
             g->enable_time_report = timing_info;
-            buf_init_from_str(&g->cache_dir, cache_dir);
+            buf_init_from_str(&g->cache_dir, cache_dir ? cache_dir : default_zig_cache_name);
             codegen_set_out_name(g, buf_out_name);
             codegen_set_lib_version(g, ver_major, ver_minor, ver_patch);
             codegen_set_is_test(g, cmd == CmdTest);
-- 
2.54.0