authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-03-11 12:47:55-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-03-11 12:47:55-04:00
log80e5af2be21427b8590c31e21c8e6b4cae1b7a6e
tree318a78764265ac2c97287fe0eda33e57a1d46cf8
parent9efa18f687a8c05f6651df6a7455a39c3d42d212
parent3ff0e8bd96bc6bf1d8eb4985c6d56766dab578f2
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #2049 from ziglang/problematic-mtime-detection

stage1 caching system: detect problematic mtimes

10 files changed, 189 insertions(+), 85 deletions(-)

doc/docgen.zig-6
...@@ -1088,8 +1088,6 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var...@@ -1088,8 +1088,6 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
1088 tmp_source_file_name,1088 tmp_source_file_name,
1089 "--output-dir",1089 "--output-dir",
1090 tmp_dir_name,1090 tmp_dir_name,
1091 "--cache",
1092 "off",
1093 });1091 });
1094 try out.print("<pre><code class=\"shell\">$ zig test {}.zig", code.name);1092 try out.print("<pre><code class=\"shell\">$ zig test {}.zig", code.name);
1095 switch (code.mode) {1093 switch (code.mode) {
...@@ -1127,8 +1125,6 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var...@@ -1127,8 +1125,6 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
1127 tmp_source_file_name,1125 tmp_source_file_name,
1128 "--output-dir",1126 "--output-dir",
1129 tmp_dir_name,1127 tmp_dir_name,
1130 "--cache",
1131 "off",
1132 });1128 });
1133 try out.print("<pre><code class=\"shell\">$ zig test {}.zig", code.name);1129 try out.print("<pre><code class=\"shell\">$ zig test {}.zig", code.name);
1134 switch (code.mode) {1130 switch (code.mode) {
...@@ -1186,8 +1182,6 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var...@@ -1186,8 +1182,6 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
1186 tmp_source_file_name,1182 tmp_source_file_name,
1187 "--output-dir",1183 "--output-dir",
1188 tmp_dir_name,1184 tmp_dir_name,
1189 "--cache",
1190 "off",
1191 });1185 });
1192 switch (code.mode) {1186 switch (code.mode) {
1193 builtin.Mode.Debug => {},1187 builtin.Mode.Debug => {},
src/cache_hash.cpp+71-26
...@@ -158,8 +158,10 @@ static void base64_encode(Slice<uint8_t> dest, Slice<uint8_t> source) {...@@ -158,8 +158,10 @@ static void base64_encode(Slice<uint8_t> dest, Slice<uint8_t> source) {
158158
159// Ported from std/base64.zig159// Ported from std/base64.zig
160static Error base64_decode(Slice<uint8_t> dest, Slice<uint8_t> source) {160static Error base64_decode(Slice<uint8_t> dest, Slice<uint8_t> source) {
161 assert(source.len % 4 == 0);161 if (source.len % 4 != 0)
162 assert(dest.len == (source.len / 4) * 3);162 return ErrorInvalidFormat;
163 if (dest.len != (source.len / 4) * 3)
164 return ErrorInvalidFormat;
163165
164 // In Zig this is comptime computed. In C++ it's not worth it to do that.166 // In Zig this is comptime computed. In C++ it's not worth it to do that.
165 uint8_t char_to_index[256];167 uint8_t char_to_index[256];
...@@ -218,15 +220,41 @@ static Error hash_file(uint8_t *digest, OsFile handle, Buf *contents) {...@@ -218,15 +220,41 @@ static Error hash_file(uint8_t *digest, OsFile handle, Buf *contents) {
218 }220 }
219}221}
220222
223// If the wall clock time, rounded to the same precision as the
224// mtime, is equal to the mtime, then we cannot rely on this mtime
225// yet. We will instead save an mtime value that indicates the hash
226// must be unconditionally computed.
227static bool is_problematic_timestamp(const OsTimeStamp *fs_clock) {
228 OsTimeStamp wall_clock = os_timestamp_calendar();
229 // First make all the least significant zero bits in the fs_clock, also zero bits in the wall clock.
230 if (fs_clock->nsec == 0) {
231 wall_clock.nsec = 0;
232 if (fs_clock->sec == 0) {
233 wall_clock.sec = 0;
234 } else {
235 wall_clock.sec &= (-1ull) << ctzll(fs_clock->sec);
236 }
237 } else {
238 wall_clock.nsec &= (-1ull) << ctzll(fs_clock->nsec);
239 }
240 return wall_clock.nsec == fs_clock->nsec && wall_clock.sec == fs_clock->sec;
241}
242
221static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf, Buf *contents) {243static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf, Buf *contents) {
222 Error err;244 Error err;
223245
224 assert(chf->path != nullptr);246 assert(chf->path != nullptr);
225247
226 OsFile this_file;248 OsFile this_file;
227 if ((err = os_file_open_r(chf->path, &this_file, &chf->mtime)))249 if ((err = os_file_open_r(chf->path, &this_file, &chf->attr)))
228 return err;250 return err;
229251
252 if (is_problematic_timestamp(&chf->attr.mtime)) {
253 chf->attr.mtime.sec = 0;
254 chf->attr.mtime.nsec = 0;
255 chf->attr.inode = 0;
256 }
257
230 if ((err = hash_file(chf->bin_digest, this_file, contents))) {258 if ((err = hash_file(chf->bin_digest, this_file, contents))) {
231 os_file_close(this_file);259 os_file_close(this_file);
232 return err;260 return err;
...@@ -278,6 +306,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {...@@ -278,6 +306,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {
278306
279 size_t input_file_count = ch->files.length;307 size_t input_file_count = ch->files.length;
280 bool any_file_changed = false;308 bool any_file_changed = false;
309 Error return_code = ErrorNone;
281 size_t file_i = 0;310 size_t file_i = 0;
282 SplitIterator line_it = memSplit(buf_to_slice(&line_buf), str("\n"));311 SplitIterator line_it = memSplit(buf_to_slice(&line_buf), str("\n"));
283 for (;; file_i += 1) {312 for (;; file_i += 1) {
...@@ -299,7 +328,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {...@@ -299,7 +328,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {
299 blake2b_update(&ch->blake, ch->files.at(file_i).bin_digest, 48);328 blake2b_update(&ch->blake, ch->files.at(file_i).bin_digest, 48);
300 }329 }
301 // caller can notice that out_digest is unmodified.330 // caller can notice that out_digest is unmodified.
302 return ErrorNone;331 return return_code;
303 } else if (!opt_line.is_some) {332 } else if (!opt_line.is_some) {
304 break;333 break;
305 } else {334 } else {
...@@ -312,57 +341,73 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {...@@ -312,57 +341,73 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {
312341
313 SplitIterator it = memSplit(opt_line.value, str(" "));342 SplitIterator it = memSplit(opt_line.value, str(" "));
314343
344 Optional<Slice<uint8_t>> opt_inode = SplitIterator_next(&it);
345 if (!opt_inode.is_some) {
346 return_code = ErrorInvalidFormat;
347 break;
348 }
349 chf->attr.inode = strtoull((const char *)opt_inode.value.ptr, nullptr, 10);
350
315 Optional<Slice<uint8_t>> opt_mtime_sec = SplitIterator_next(&it);351 Optional<Slice<uint8_t>> opt_mtime_sec = SplitIterator_next(&it);
316 if (!opt_mtime_sec.is_some) {352 if (!opt_mtime_sec.is_some) {
317 os_file_close(ch->manifest_file);353 return_code = ErrorInvalidFormat;
318 return ErrorInvalidFormat;354 break;
319 }355 }
320 chf->mtime.sec = strtoull((const char *)opt_mtime_sec.value.ptr, nullptr, 10);356 chf->attr.mtime.sec = strtoull((const char *)opt_mtime_sec.value.ptr, nullptr, 10);
321357
322 Optional<Slice<uint8_t>> opt_mtime_nsec = SplitIterator_next(&it);358 Optional<Slice<uint8_t>> opt_mtime_nsec = SplitIterator_next(&it);
323 if (!opt_mtime_nsec.is_some) {359 if (!opt_mtime_nsec.is_some) {
324 os_file_close(ch->manifest_file);360 return_code = ErrorInvalidFormat;
325 return ErrorInvalidFormat;361 break;
326 }362 }
327 chf->mtime.nsec = strtoull((const char *)opt_mtime_nsec.value.ptr, nullptr, 10);363 chf->attr.mtime.nsec = strtoull((const char *)opt_mtime_nsec.value.ptr, nullptr, 10);
328364
329 Optional<Slice<uint8_t>> opt_digest = SplitIterator_next(&it);365 Optional<Slice<uint8_t>> opt_digest = SplitIterator_next(&it);
330 if (!opt_digest.is_some) {366 if (!opt_digest.is_some) {
331 os_file_close(ch->manifest_file);367 return_code = ErrorInvalidFormat;
332 return ErrorInvalidFormat;368 break;
333 }369 }
334 if ((err = base64_decode({chf->bin_digest, 48}, opt_digest.value))) {370 if ((err = base64_decode({chf->bin_digest, 48}, opt_digest.value))) {
335 os_file_close(ch->manifest_file);371 return_code = ErrorInvalidFormat;
336 return ErrorInvalidFormat;372 break;
337 }373 }
338374
339 Slice<uint8_t> file_path = SplitIterator_rest(&it);375 Slice<uint8_t> file_path = SplitIterator_rest(&it);
340 if (file_path.len == 0) {376 if (file_path.len == 0) {
341 os_file_close(ch->manifest_file);377 return_code = ErrorInvalidFormat;
342 return ErrorInvalidFormat;378 break;
343 }379 }
344 Buf *this_path = buf_create_from_slice(file_path);380 Buf *this_path = buf_create_from_slice(file_path);
345 if (chf->path != nullptr && !buf_eql_buf(this_path, chf->path)) {381 if (chf->path != nullptr && !buf_eql_buf(this_path, chf->path)) {
346 os_file_close(ch->manifest_file);382 return_code = ErrorInvalidFormat;
347 return ErrorInvalidFormat;383 break;
348 }384 }
349 chf->path = this_path;385 chf->path = this_path;
350386
351 // if the mtime matches we can trust the digest387 // if the mtime matches we can trust the digest
352 OsFile this_file;388 OsFile this_file;
353 OsTimeStamp actual_mtime;389 OsFileAttr actual_attr;
354 if ((err = os_file_open_r(chf->path, &this_file, &actual_mtime))) {390 if ((err = os_file_open_r(chf->path, &this_file, &actual_attr))) {
355 fprintf(stderr, "Unable to open %s\n: %s", buf_ptr(chf->path), err_str(err));391 fprintf(stderr, "Unable to open %s\n: %s", buf_ptr(chf->path), err_str(err));
356 os_file_close(ch->manifest_file);392 os_file_close(ch->manifest_file);
357 return ErrorCacheUnavailable;393 return ErrorCacheUnavailable;
358 }394 }
359 if (chf->mtime.sec == actual_mtime.sec && chf->mtime.nsec == actual_mtime.nsec) {395 if (chf->attr.mtime.sec == actual_attr.mtime.sec &&
396 chf->attr.mtime.nsec == actual_attr.mtime.nsec &&
397 chf->attr.inode == actual_attr.inode)
398 {
360 os_file_close(this_file);399 os_file_close(this_file);
361 } else {400 } else {
362 // we have to recompute the digest.401 // we have to recompute the digest.
363 // later we'll rewrite the manifest with the new mtime/digest values402 // later we'll rewrite the manifest with the new mtime/digest values
364 ch->manifest_dirty = true;403 ch->manifest_dirty = true;
365 chf->mtime = actual_mtime;404 chf->attr = actual_attr;
405
406 if (is_problematic_timestamp(&actual_attr.mtime)) {
407 chf->attr.mtime.sec = 0;
408 chf->attr.mtime.nsec = 0;
409 chf->attr.inode = 0;
410 }
366411
367 uint8_t actual_digest[48];412 uint8_t actual_digest[48];
368 if ((err = hash_file(actual_digest, this_file, nullptr))) {413 if ((err = hash_file(actual_digest, this_file, nullptr))) {
...@@ -381,7 +426,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {...@@ -381,7 +426,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {
381 blake2b_update(&ch->blake, chf->bin_digest, 48);426 blake2b_update(&ch->blake, chf->bin_digest, 48);
382 }427 }
383 }428 }
384 if (file_i < input_file_count || file_i == 0) {429 if (file_i < input_file_count || file_i == 0 || return_code != ErrorNone) {
385 // manifest file is empty or missing entries, so this is a cache miss430 // manifest file is empty or missing entries, so this is a cache miss
386 ch->manifest_dirty = true;431 ch->manifest_dirty = true;
387 for (; file_i < input_file_count; file_i += 1) {432 for (; file_i < input_file_count; file_i += 1) {
...@@ -392,7 +437,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {...@@ -392,7 +437,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {
392 return ErrorCacheUnavailable;437 return ErrorCacheUnavailable;
393 }438 }
394 }439 }
395 return ErrorNone;440 return return_code;
396 }441 }
397 // Cache Hit442 // Cache Hit
398 return cache_final(ch, out_digest);443 return cache_final(ch, out_digest);
...@@ -499,8 +544,8 @@ static Error write_manifest_file(CacheHash *ch) {...@@ -499,8 +544,8 @@ static Error write_manifest_file(CacheHash *ch) {
499 for (size_t i = 0; i < ch->files.length; i += 1) {544 for (size_t i = 0; i < ch->files.length; i += 1) {
500 CacheHashFile *chf = &ch->files.at(i);545 CacheHashFile *chf = &ch->files.at(i);
501 base64_encode({encoded_digest, 64}, {chf->bin_digest, 48});546 base64_encode({encoded_digest, 64}, {chf->bin_digest, 48});
502 buf_appendf(&contents, "%" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %s %s\n",547 buf_appendf(&contents, "%" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %s %s\n",
503 chf->mtime.sec, chf->mtime.nsec, encoded_digest, buf_ptr(chf->path));548 chf->attr.inode, chf->attr.mtime.sec, chf->attr.mtime.nsec, encoded_digest, buf_ptr(chf->path));
504 }549 }
505 if ((err = os_file_overwrite(ch->manifest_file, &contents)))550 if ((err = os_file_overwrite(ch->manifest_file, &contents)))
506 return err;551 return err;
src/cache_hash.hpp+3-1
...@@ -15,7 +15,7 @@ struct LinkLib;...@@ -15,7 +15,7 @@ struct LinkLib;
1515
16struct CacheHashFile {16struct CacheHashFile {
17 Buf *path;17 Buf *path;
18 OsTimeStamp mtime;18 OsFileAttr attr;
19 uint8_t bin_digest[48];19 uint8_t bin_digest[48];
20 Buf *contents;20 Buf *contents;
21};21};
...@@ -57,6 +57,8 @@ void cache_file_opt(CacheHash *ch, Buf *path);...@@ -57,6 +57,8 @@ void cache_file_opt(CacheHash *ch, Buf *path);
57// added any files before calling cache_hit. CacheHash::b64_digest becomes57// added any files before calling cache_hit. CacheHash::b64_digest becomes
58// available for use after this call, even in the case of a miss, and it58// available for use after this call, even in the case of a miss, and it
59// is a hash of the input parameters only.59// is a hash of the input parameters only.
60// If this function returns ErrorInvalidFormat, that error may be treated
61// as a cache miss.
60Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest);62Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest);
6163
62// If you did not get a cache hit, call this function for every file64// If you did not get a cache hit, call this function for every file
src/codegen.cpp+20-10
...@@ -7724,8 +7724,11 @@ static Error define_builtin_compile_vars(CodeGen *g) {...@@ -7724,8 +7724,11 @@ static Error define_builtin_compile_vars(CodeGen *g) {
77247724
7725 Buf digest = BUF_INIT;7725 Buf digest = BUF_INIT;
7726 buf_resize(&digest, 0);7726 buf_resize(&digest, 0);
7727 if ((err = cache_hit(&cache_hash, &digest)))7727 if ((err = cache_hit(&cache_hash, &digest))) {
7728 return err;7728 // Treat an invalid format error as a cache miss.
7729 if (err != ErrorInvalidFormat)
7730 return err;
7731 }
77297732
7730 // We should always get a cache hit because there are no7733 // We should always get a cache hit because there are no
7731 // files in the input hash.7734 // files in the input hash.
...@@ -8342,12 +8345,14 @@ static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) {...@@ -8342,12 +8345,14 @@ static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) {
8342 Buf digest = BUF_INIT;8345 Buf digest = BUF_INIT;
8343 buf_resize(&digest, 0);8346 buf_resize(&digest, 0);
8344 if ((err = cache_hit(cache_hash, &digest))) {8347 if ((err = cache_hit(cache_hash, &digest))) {
8345 if (err == ErrorCacheUnavailable) {8348 if (err != ErrorInvalidFormat) {
8346 // already printed error8349 if (err == ErrorCacheUnavailable) {
8347 } else {8350 // already printed error
8348 fprintf(stderr, "unable to check cache when compiling C object: %s\n", err_str(err));8351 } else {
8352 fprintf(stderr, "unable to check cache when compiling C object: %s\n", err_str(err));
8353 }
8354 exit(1);
8349 }8355 }
8350 exit(1);
8351 }8356 }
8352 bool is_cache_miss = (buf_len(&digest) == 0);8357 bool is_cache_miss = (buf_len(&digest) == 0);
8353 if (is_cache_miss) {8358 if (is_cache_miss) {
...@@ -8993,7 +8998,10 @@ void codegen_print_timing_report(CodeGen *g, FILE *f) {...@@ -8993,7 +8998,10 @@ void codegen_print_timing_report(CodeGen *g, FILE *f) {
8993}8998}
89948999
8995void codegen_add_time_event(CodeGen *g, const char *name) {9000void codegen_add_time_event(CodeGen *g, const char *name) {
8996 g->timing_events.append({os_get_time(), name});9001 OsTimeStamp timestamp = os_timestamp_monotonic();
9002 double seconds = (double)timestamp.sec;
9003 seconds += ((double)timestamp.nsec) / 1000000000.0;
9004 g->timing_events.append({seconds, name});
8997}9005}
89989006
8999static void add_cache_pkg(CodeGen *g, CacheHash *ch, ZigPackage *pkg) {9007static void add_cache_pkg(CodeGen *g, CacheHash *ch, ZigPackage *pkg) {
...@@ -9090,8 +9098,10 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) {...@@ -9090,8 +9098,10 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) {
9090 cache_list_of_file(ch, g->link_objects.items, g->link_objects.length);9098 cache_list_of_file(ch, g->link_objects.items, g->link_objects.length);
90919099
9092 buf_resize(digest, 0);9100 buf_resize(digest, 0);
9093 if ((err = cache_hit(ch, digest)))9101 if ((err = cache_hit(ch, digest))) {
9094 return err;9102 if (err != ErrorInvalidFormat)
9103 return err;
9104 }
90959105
9096 if (ch->manifest_file_path != nullptr) {9106 if (ch->manifest_file_path != nullptr) {
9097 g->caches_to_release.append(ch);9107 g->caches_to_release.append(ch);
src/compiler.cpp+4-2
...@@ -75,8 +75,10 @@ Error get_compiler_id(Buf **result) {...@@ -75,8 +75,10 @@ Error get_compiler_id(Buf **result) {
75 cache_file(ch, &self_exe_path);75 cache_file(ch, &self_exe_path);
7676
77 buf_resize(&saved_compiler_id, 0);77 buf_resize(&saved_compiler_id, 0);
78 if ((err = cache_hit(ch, &saved_compiler_id)))78 if ((err = cache_hit(ch, &saved_compiler_id))) {
79 return err;79 if (err != ErrorInvalidFormat)
80 return err;
81 }
80 if (buf_len(&saved_compiler_id) != 0) {82 if (buf_len(&saved_compiler_id) != 0) {
81 cache_release(ch);83 cache_release(ch);
82 *result = &saved_compiler_id;84 *result = &saved_compiler_id;
src/ir.cpp+4-2
...@@ -18732,8 +18732,10 @@ static IrInstruction *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstruct...@@ -18732,8 +18732,10 @@ static IrInstruction *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstruct
18732 Buf tmp_c_file_digest = BUF_INIT;18732 Buf tmp_c_file_digest = BUF_INIT;
18733 buf_resize(&tmp_c_file_digest, 0);18733 buf_resize(&tmp_c_file_digest, 0);
18734 if ((err = cache_hit(cache_hash, &tmp_c_file_digest))) {18734 if ((err = cache_hit(cache_hash, &tmp_c_file_digest))) {
18735 ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to check cache: %s", err_str(err)));18735 if (err != ErrorInvalidFormat) {
18736 return ira->codegen->invalid_instruction;18736 ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to check cache: %s", err_str(err)));
18737 return ira->codegen->invalid_instruction;
18738 }
18737 }18739 }
18738 ira->codegen->caches_to_release.append(cache_hash);18740 ira->codegen->caches_to_release.append(cache_hash);
1873918741
src/os.cpp+61-29
...@@ -70,9 +70,10 @@ typedef SSIZE_T ssize_t;...@@ -70,9 +70,10 @@ typedef SSIZE_T ssize_t;
70#endif70#endif
7171
72#if defined(ZIG_OS_WINDOWS)72#if defined(ZIG_OS_WINDOWS)
73static double win32_time_resolution;73static uint64_t windows_perf_freq;
74#elif defined(__MACH__)74#elif defined(__MACH__)
75static clock_serv_t cclock;75static clock_serv_t macos_calendar_clock;
76static clock_serv_t macos_monotonic_clock;
76#endif77#endif
7778
78#include <stdlib.h>79#include <stdlib.h>
...@@ -1233,28 +1234,60 @@ Error os_rename(Buf *src_path, Buf *dest_path) {...@@ -1233,28 +1234,60 @@ Error os_rename(Buf *src_path, Buf *dest_path) {
1233 return ErrorNone;1234 return ErrorNone;
1234}1235}
12351236
1236double os_get_time(void) {
1237#if defined(ZIG_OS_WINDOWS)1237#if defined(ZIG_OS_WINDOWS)
1238 unsigned __int64 time;1238static void windows_filetime_to_os_timestamp(FILETIME *ft, OsTimeStamp *mtime) {
1239 QueryPerformanceCounter((LARGE_INTEGER*) &time);1239 mtime->sec = (((ULONGLONG) ft->dwHighDateTime) << 32) + ft->dwLowDateTime;
1240 return time * win32_time_resolution;1240 mtime->nsec = 0;
1241}
1242#endif
1243
1244OsTimeStamp os_timestamp_calendar(void) {
1245 OsTimeStamp result;
1246#if defined(ZIG_OS_WINDOWS)
1247 FILETIME ft;
1248 GetSystemTimeAsFileTime(&ft);
1249 windows_filetime_to_os_timestamp(&ft, &result);
1241#elif defined(__MACH__)1250#elif defined(__MACH__)
1242 mach_timespec_t mts;1251 mach_timespec_t mts;
12431252
1244 kern_return_t err = clock_get_time(cclock, &mts);1253 kern_return_t err = clock_get_time(macos_calendar_clock, &mts);
1245 assert(!err);1254 assert(!err);
12461255
1247 double seconds = (double)mts.tv_sec;1256 result.sec = mts.tv_sec;
1248 seconds += ((double)mts.tv_nsec) / 1000000000.0;1257 result.nsec = mts.tv_nsec;
1258#else
1259 struct timespec tms;
1260 clock_gettime(CLOCK_REALTIME, &tms);
1261
1262 result.sec = tms.tv_sec;
1263 result.nsec = tms.tv_nsec;
1264#endif
1265 return result;
1266}
12491267
1250 return seconds;1268OsTimeStamp os_timestamp_monotonic(void) {
1269 OsTimeStamp result;
1270#if defined(ZIG_OS_WINDOWS)
1271 uint64_t counts;
1272 QueryPerformanceCounter((LARGE_INTEGER*)&counts);
1273 result.sec = counts / windows_perf_freq;
1274 result.nsec = (counts % windows_perf_freq) * 1000000000u / windows_perf_freq;
1275#elif defined(__MACH__)
1276 mach_timespec_t mts;
1277
1278 kern_return_t err = clock_get_time(macos_monotonic_clock, &mts);
1279 assert(!err);
1280
1281 result.sec = mts.tv_sec;
1282 result.nsec = mts.tv_nsec;
1251#else1283#else
1252 struct timespec tms;1284 struct timespec tms;
1253 clock_gettime(CLOCK_MONOTONIC, &tms);1285 clock_gettime(CLOCK_MONOTONIC, &tms);
1254 double seconds = (double)tms.tv_sec;1286
1255 seconds += ((double)tms.tv_nsec) / 1000000000.0;1287 result.sec = tms.tv_sec;
1256 return seconds;1288 result.nsec = tms.tv_nsec;
1257#endif1289#endif
1290 return result;
1258}1291}
12591292
1260Error os_make_path(Buf *path) {1293Error os_make_path(Buf *path) {
...@@ -1352,14 +1385,12 @@ int os_init(void) {...@@ -1352,14 +1385,12 @@ int os_init(void) {
1352#if defined(ZIG_OS_WINDOWS)1385#if defined(ZIG_OS_WINDOWS)
1353 _setmode(fileno(stdout), _O_BINARY);1386 _setmode(fileno(stdout), _O_BINARY);
1354 _setmode(fileno(stderr), _O_BINARY);1387 _setmode(fileno(stderr), _O_BINARY);
1355 unsigned __int64 frequency;1388 if (!QueryPerformanceFrequency((LARGE_INTEGER*)&windows_perf_freq)) {
1356 if (QueryPerformanceFrequency((LARGE_INTEGER*) &frequency)) {
1357 win32_time_resolution = 1.0 / (double) frequency;
1358 } else {
1359 return ErrorSystemResources;1389 return ErrorSystemResources;
1360 }1390 }
1361#elif defined(__MACH__)1391#elif defined(__MACH__)
1362 host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &cclock);1392 host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &macos_monotonic_clock);
1393 host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &macos_calendar_clock);
1363#endif1394#endif
1364 return 0;1395 return 0;
1365}1396}
...@@ -1780,7 +1811,7 @@ Error os_self_exe_shared_libs(ZigList<Buf *> &paths) {...@@ -1780,7 +1811,7 @@ Error os_self_exe_shared_libs(ZigList<Buf *> &paths) {
1780#endif1811#endif
1781}1812}
17821813
1783Error os_file_open_r(Buf *full_path, OsFile *out_file, OsTimeStamp *mtime) {1814Error os_file_open_r(Buf *full_path, OsFile *out_file, OsFileAttr *attr) {
1784#if defined(ZIG_OS_WINDOWS)1815#if defined(ZIG_OS_WINDOWS)
1785 // TODO use CreateFileW1816 // TODO use CreateFileW
1786 HANDLE result = CreateFileA(buf_ptr(full_path), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);1817 HANDLE result = CreateFileA(buf_ptr(full_path), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
...@@ -1808,14 +1839,14 @@ Error os_file_open_r(Buf *full_path, OsFile *out_file, OsTimeStamp *mtime) {...@@ -1808,14 +1839,14 @@ Error os_file_open_r(Buf *full_path, OsFile *out_file, OsTimeStamp *mtime) {
1808 }1839 }
1809 *out_file = result;1840 *out_file = result;
18101841
1811 if (mtime != nullptr) {1842 if (attr != nullptr) {
1812 FILETIME last_write_time;1843 BY_HANDLE_FILE_INFORMATION file_info;
1813 if (!GetFileTime(result, nullptr, nullptr, &last_write_time)) {1844 if (!GetFileInformationByHandle(result, &file_info)) {
1814 CloseHandle(result);1845 CloseHandle(result);
1815 return ErrorUnexpected;1846 return ErrorUnexpected;
1816 }1847 }
1817 mtime->sec = (((ULONGLONG) last_write_time.dwHighDateTime) << 32) + last_write_time.dwLowDateTime;1848 windows_filetime_to_os_timestamp(&file_info.ftLastWriteTime, &attr->mtime);
1818 mtime->nsec = 0;1849 attr->inode = (((uint64_t)file_info.nFileIndexHigh) << 32) | file_info.nFileIndexLow;
1819 }1850 }
18201851
1821 return ErrorNone;1852 return ErrorNone;
...@@ -1851,13 +1882,14 @@ Error os_file_open_r(Buf *full_path, OsFile *out_file, OsTimeStamp *mtime) {...@@ -1851,13 +1882,14 @@ Error os_file_open_r(Buf *full_path, OsFile *out_file, OsTimeStamp *mtime) {
1851 }1882 }
1852 *out_file = fd;1883 *out_file = fd;
18531884
1854 if (mtime != nullptr) {1885 if (attr != nullptr) {
1886 attr->inode = statbuf.st_ino;
1855#if defined(ZIG_OS_DARWIN)1887#if defined(ZIG_OS_DARWIN)
1856 mtime->sec = statbuf.st_mtimespec.tv_sec;1888 attr->mtime.sec = statbuf.st_mtimespec.tv_sec;
1857 mtime->nsec = statbuf.st_mtimespec.tv_nsec;1889 attr->mtime.nsec = statbuf.st_mtimespec.tv_nsec;
1858#else1890#else
1859 mtime->sec = statbuf.st_mtim.tv_sec;1891 attr->mtime.sec = statbuf.st_mtim.tv_sec;
1860 mtime->nsec = statbuf.st_mtim.tv_nsec;1892 attr->mtime.nsec = statbuf.st_mtim.tv_nsec;
1861#endif1893#endif
1862 }1894 }
1863 return ErrorNone;1895 return ErrorNone;
src/os.hpp+8-2
...@@ -85,6 +85,11 @@ struct OsTimeStamp {...@@ -85,6 +85,11 @@ struct OsTimeStamp {
85 uint64_t nsec;85 uint64_t nsec;
86};86};
8787
88struct OsFileAttr {
89 OsTimeStamp mtime;
90 uint64_t inode;
91};
92
88int os_init(void);93int os_init(void);
8994
90void os_spawn_process(const char *exe, ZigList<const char *> &args, Termination *term);95void os_spawn_process(const char *exe, ZigList<const char *> &args, Termination *term);
...@@ -103,7 +108,7 @@ bool os_path_is_absolute(Buf *path);...@@ -103,7 +108,7 @@ bool os_path_is_absolute(Buf *path);
103Error ATTRIBUTE_MUST_USE os_make_path(Buf *path);108Error ATTRIBUTE_MUST_USE os_make_path(Buf *path);
104Error ATTRIBUTE_MUST_USE os_make_dir(Buf *path);109Error ATTRIBUTE_MUST_USE os_make_dir(Buf *path);
105110
106Error ATTRIBUTE_MUST_USE os_file_open_r(Buf *full_path, OsFile *out_file, OsTimeStamp *mtime);111Error ATTRIBUTE_MUST_USE os_file_open_r(Buf *full_path, OsFile *out_file, OsFileAttr *attr);
107Error ATTRIBUTE_MUST_USE os_file_open_lock_rw(Buf *full_path, OsFile *out_file);112Error ATTRIBUTE_MUST_USE os_file_open_lock_rw(Buf *full_path, OsFile *out_file);
108Error ATTRIBUTE_MUST_USE os_file_read(OsFile file, void *ptr, size_t *len);113Error ATTRIBUTE_MUST_USE os_file_read(OsFile file, void *ptr, size_t *len);
109Error ATTRIBUTE_MUST_USE os_file_read_all(OsFile file, Buf *contents);114Error ATTRIBUTE_MUST_USE os_file_read_all(OsFile file, Buf *contents);
...@@ -126,7 +131,8 @@ Error os_delete_file(Buf *path);...@@ -126,7 +131,8 @@ Error os_delete_file(Buf *path);
126Error ATTRIBUTE_MUST_USE os_file_exists(Buf *full_path, bool *result);131Error ATTRIBUTE_MUST_USE os_file_exists(Buf *full_path, bool *result);
127132
128Error os_rename(Buf *src_path, Buf *dest_path);133Error os_rename(Buf *src_path, Buf *dest_path);
129double os_get_time(void);134OsTimeStamp os_timestamp_monotonic(void);
135OsTimeStamp os_timestamp_calendar(void);
130136
131bool os_is_sep(uint8_t c);137bool os_is_sep(uint8_t c);
132138
src/util.hpp+17
...@@ -63,10 +63,27 @@ static inline int clzll(unsigned long long mask) {...@@ -63,10 +63,27 @@ static inline int clzll(unsigned long long mask) {
63 return 63 - lz;63 return 63 - lz;
64#endif64#endif
65}65}
66static inline int ctzll(unsigned long long mask) {
67 unsigned long result;
68#if defined(_WIN64)
69 if (_BitScanForward64(&result, mask))
70 return result;
71 zig_unreachable();
72#else
73 if (_BitScanForward(&result, mask & 0xffffffff))
74 return result;
75 }
76 if (_BitScanForward(&result, mask >> 32))
77 return 32 + result;
78 zig_unreachable();
79#endif
80}
66#else81#else
67#define clzll(x) __builtin_clzll(x)82#define clzll(x) __builtin_clzll(x)
83#define ctzll(x) __builtin_ctzll(x)
68#endif84#endif
6985
86
70template<typename T>87template<typename T>
71ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) {88ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) {
72#ifndef NDEBUG89#ifndef NDEBUG
test/tests.zig+1-7
...@@ -158,7 +158,6 @@ pub fn addGenHTests(b: *build.Builder, test_filter: ?[]const u8) *build.Step {...@@ -158,7 +158,6 @@ pub fn addGenHTests(b: *build.Builder, test_filter: ?[]const u8) *build.Step {
158 .step = b.step("test-gen-h", "Run the C header file generation tests"),158 .step = b.step("test-gen-h", "Run the C header file generation tests"),
159 .test_index = 0,159 .test_index = 0,
160 .test_filter = test_filter,160 .test_filter = test_filter,
161 .counter = 0,
162 };161 };
163162
164 gen_h.addCases(cases);163 gen_h.addCases(cases);
...@@ -1105,7 +1104,6 @@ pub const GenHContext = struct {...@@ -1105,7 +1104,6 @@ pub const GenHContext = struct {
1105 step: *build.Step,1104 step: *build.Step,
1106 test_index: usize,1105 test_index: usize,
1107 test_filter: ?[]const u8,1106 test_filter: ?[]const u8,
1108 counter: usize,
11091107
1110 const TestCase = struct {1108 const TestCase = struct {
1111 name: []const u8,1109 name: []const u8,
...@@ -1208,11 +1206,7 @@ pub const GenHContext = struct {...@@ -1208,11 +1206,7 @@ pub const GenHContext = struct {
1208 }1206 }
12091207
1210 pub fn add(self: *GenHContext, name: []const u8, source: []const u8, expected_lines: ...) void {1208 pub fn add(self: *GenHContext, name: []const u8, source: []const u8, expected_lines: ...) void {
1211 // MacOS appears to not be returning nanoseconds in fstat mtime,1209 const tc = self.create("test.zig", name, source, expected_lines);
1212 // which causes fast test executions to think the file contents are unchanged.
1213 const modified_name = self.b.fmt("test-{}.zig", self.counter);
1214 self.counter += 1;
1215 const tc = self.create(modified_name, name, source, expected_lines);
1216 self.addCase(tc);1210 self.addCase(tc);
1217 }1211 }
12181212