authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-12-30 20:49:02-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-02 13:16:17-07:00
log0ad2a99675d331c686847a7b2a84feddfcce6573
treeba1b08642950fd481e95bb0cd198ada0176850f5
parentdbd0a2c35d590752dfa586a816f2870e4bcb3200

stage2: CacheMode.whole: trigger loading zig source files

Previously the code asserted source files were already loaded, but this is not the case when cached ZIR is loaded. Now it will trigger .zig source code to be loaded for the purposes of hashing the source for `CacheMode.whole`. This additionally refactors stat_size, stat_inode, and stat_mtime fields into using the `Cache.File.Stat` struct.

3 files changed, 121 insertions(+), 73 deletions(-)

src/Compilation.zig+2-2
......@@ -421,7 +421,7 @@ pub const AllErrors = struct {
421421 const module_note = module_err_msg.notes[i];
422422 const source = try module_note.src_loc.file_scope.getSource(module.gpa);
423423 const byte_offset = try module_note.src_loc.byteOffset(module.gpa);
424 const loc = std.zig.findLineColumn(source, byte_offset);
424 const loc = std.zig.findLineColumn(source.bytes, byte_offset);
425425 const file_path = try module_note.src_loc.file_scope.fullPath(allocator);
426426 note.* = .{
427427 .src = .{
......@@ -444,7 +444,7 @@ pub const AllErrors = struct {
444444 }
445445 const source = try module_err_msg.src_loc.file_scope.getSource(module.gpa);
446446 const byte_offset = try module_err_msg.src_loc.byteOffset(module.gpa);
447 const loc = std.zig.findLineColumn(source, byte_offset);
447 const loc = std.zig.findLineColumn(source.bytes, byte_offset);
448448 const file_path = try module_err_msg.src_loc.file_scope.fullPath(allocator);
449449 try errors.append(.{
450450 .src = .{
src/Module.zig+101-55
......@@ -1463,11 +1463,7 @@ pub const File = struct {
14631463 /// Whether this is populated depends on `source_loaded`.
14641464 source: [:0]const u8,
14651465 /// Whether this is populated depends on `status`.
1466 stat_size: u64,
1467 /// Whether this is populated depends on `status`.
1468 stat_inode: std.fs.File.INode,
1469 /// Whether this is populated depends on `status`.
1470 stat_mtime: i128,
1466 stat: Cache.File.Stat,
14711467 /// Whether this is populated or not depends on `tree_loaded`.
14721468 tree: Ast,
14731469 /// Whether this is populated or not depends on `zir_loaded`.
......@@ -1535,8 +1531,16 @@ pub const File = struct {
15351531 file.* = undefined;
15361532 }
15371533
1538 pub fn getSource(file: *File, gpa: Allocator) ![:0]const u8 {
1539 if (file.source_loaded) return file.source;
1534 pub const Source = struct {
1535 bytes: [:0]const u8,
1536 stat: Cache.File.Stat,
1537 };
1538
1539 pub fn getSource(file: *File, gpa: Allocator) !Source {
1540 if (file.source_loaded) return Source{
1541 .bytes = file.source,
1542 .stat = file.stat,
1543 };
15401544
15411545 const root_dir_path = file.pkg.root_src_directory.path orelse ".";
15421546 log.debug("File.getSource, not cached. pkgdir={s} sub_file_path={s}", .{
......@@ -1565,14 +1569,21 @@ pub const File = struct {
15651569
15661570 file.source = source;
15671571 file.source_loaded = true;
1568 return source;
1572 return Source{
1573 .bytes = source,
1574 .stat = .{
1575 .size = stat.size,
1576 .inode = stat.inode,
1577 .mtime = stat.mtime,
1578 },
1579 };
15691580 }
15701581
15711582 pub fn getTree(file: *File, gpa: Allocator) !*const Ast {
15721583 if (file.tree_loaded) return &file.tree;
15731584
15741585 const source = try file.getSource(gpa);
1575 file.tree = try std.zig.parse(gpa, source);
1586 file.tree = try std.zig.parse(gpa, source.bytes);
15761587 file.tree_loaded = true;
15771588 return &file.tree;
15781589 }
......@@ -1631,9 +1642,7 @@ pub const EmbedFile = struct {
16311642 /// Memory is stored in gpa, owned by EmbedFile.
16321643 sub_file_path: []const u8,
16331644 bytes: [:0]const u8,
1634 stat_size: u64,
1635 stat_inode: std.fs.File.INode,
1636 stat_mtime: i128,
1645 stat: Cache.File.Stat,
16371646 /// Package that this file is a part of, managed externally.
16381647 pkg: *Package,
16391648 /// The Decl that was created from the `@embedFile` to own this resource.
......@@ -2704,9 +2713,11 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
27042713 keep_zir = true;
27052714 file.zir = zir;
27062715 file.zir_loaded = true;
2707 file.stat_size = header.stat_size;
2708 file.stat_inode = header.stat_inode;
2709 file.stat_mtime = header.stat_mtime;
2716 file.stat = .{
2717 .size = header.stat_size,
2718 .inode = header.stat_inode,
2719 .mtime = header.stat_mtime,
2720 };
27102721 file.status = .success_zir;
27112722 log.debug("AstGen cached success: {s}", .{file.sub_file_path});
27122723
......@@ -2724,9 +2735,9 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
27242735 },
27252736 .parse_failure, .astgen_failure, .success_zir => {
27262737 const unchanged_metadata =
2727 stat.size == file.stat_size and
2728 stat.mtime == file.stat_mtime and
2729 stat.inode == file.stat_inode;
2738 stat.size == file.stat.size and
2739 stat.mtime == file.stat.mtime and
2740 stat.inode == file.stat.inode;
27302741
27312742 if (unchanged_metadata) {
27322743 log.debug("unmodified metadata of file: {s}", .{file.sub_file_path});
......@@ -2787,9 +2798,11 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
27872798 if (amt != stat.size)
27882799 return error.UnexpectedEndOfFile;
27892800
2790 file.stat_size = stat.size;
2791 file.stat_inode = stat.inode;
2792 file.stat_mtime = stat.mtime;
2801 file.stat = .{
2802 .size = stat.size,
2803 .inode = stat.inode,
2804 .mtime = stat.mtime,
2805 };
27932806 file.source = source;
27942807 file.source_loaded = true;
27952808
......@@ -3069,9 +3082,11 @@ pub fn populateBuiltinFile(mod: *Module) !void {
30693082
30703083 try writeBuiltinFile(file, builtin_pkg);
30713084 } else {
3072 file.stat_size = stat.size;
3073 file.stat_inode = stat.inode;
3074 file.stat_mtime = stat.mtime;
3085 file.stat = .{
3086 .size = stat.size,
3087 .inode = stat.inode,
3088 .mtime = stat.mtime,
3089 };
30753090 }
30763091 } else |err| switch (err) {
30773092 error.BadPathName => unreachable, // it's always "builtin.zig"
......@@ -3099,9 +3114,11 @@ pub fn writeBuiltinFile(file: *File, builtin_pkg: *Package) !void {
30993114 try af.file.writeAll(file.source);
31003115 try af.finish();
31013116
3102 file.stat_size = file.source.len;
3103 file.stat_inode = 0; // dummy value
3104 file.stat_mtime = 0; // dummy value
3117 file.stat = .{
3118 .size = file.source.len,
3119 .inode = 0, // dummy value
3120 .mtime = 0, // dummy value
3121 };
31053122}
31063123
31073124pub fn mapOldZirToNew(
......@@ -3382,16 +3399,16 @@ pub fn semaFile(mod: *Module, file: *File) SemaError!void {
33823399 }
33833400
33843401 if (mod.comp.whole_cache_manifest) |man| {
3385 assert(file.source_loaded);
3402 const source = file.getSource(gpa) catch |err| {
3403 try reportRetryableFileError(mod, file, "unable to load source: {s}", .{@errorName(err)});
3404 return error.AnalysisFail;
3405 };
33863406 const resolved_path = try file.pkg.root_src_directory.join(gpa, &.{
33873407 file.sub_file_path,
33883408 });
33893409 errdefer gpa.free(resolved_path);
3390 try man.addFilePostContents(resolved_path, file.source, .{
3391 .size = file.stat_size,
3392 .inode = file.stat_inode,
3393 .mtime = file.stat_mtime,
3394 });
3410
3411 try man.addFilePostContents(resolved_path, source.bytes, source.stat);
33953412 }
33963413 } else {
33973414 new_decl.analysis = .file_failure;
......@@ -3723,9 +3740,7 @@ pub fn importPkg(mod: *Module, pkg: *Package) !ImportFileResult {
37233740 .source_loaded = false,
37243741 .tree_loaded = false,
37253742 .zir_loaded = false,
3726 .stat_size = undefined,
3727 .stat_inode = undefined,
3728 .stat_mtime = undefined,
3743 .stat = undefined,
37293744 .tree = undefined,
37303745 .zir = undefined,
37313746 .status = .never_loaded,
......@@ -3793,9 +3808,7 @@ pub fn importFile(
37933808 .source_loaded = false,
37943809 .tree_loaded = false,
37953810 .zir_loaded = false,
3796 .stat_size = undefined,
3797 .stat_inode = undefined,
3798 .stat_mtime = undefined,
3811 .stat = undefined,
37993812 .tree = undefined,
38003813 .zir = undefined,
38013814 .status = .never_loaded,
......@@ -3840,8 +3853,13 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb
38403853 var file = try cur_file.pkg.root_src_directory.handle.openFile(sub_file_path, .{});
38413854 defer file.close();
38423855
3843 const stat = try file.stat();
3844 const size_usize = try std.math.cast(usize, stat.size);
3856 const actual_stat = try file.stat();
3857 const stat: Cache.File.Stat = .{
3858 .size = actual_stat.size,
3859 .inode = actual_stat.inode,
3860 .mtime = actual_stat.mtime,
3861 };
3862 const size_usize = try std.math.cast(usize, actual_stat.size);
38453863 const bytes = try file.readToEndAllocOptions(gpa, std.math.maxInt(u32), size_usize, 1, 0);
38463864 errdefer gpa.free(bytes);
38473865
......@@ -3852,11 +3870,7 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb
38523870 if (mod.comp.whole_cache_manifest) |man| {
38533871 const copied_resolved_path = try gpa.dupe(u8, resolved_path);
38543872 errdefer gpa.free(copied_resolved_path);
3855 try man.addFilePostContents(copied_resolved_path, bytes, .{
3856 .size = stat.size,
3857 .inode = stat.inode,
3858 .mtime = stat.mtime,
3859 });
3873 try man.addFilePostContents(copied_resolved_path, bytes, stat);
38603874 }
38613875
38623876 keep_resolved_path = true; // It's now owned by embed_table.
......@@ -3864,9 +3878,7 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb
38643878 new_file.* = .{
38653879 .sub_file_path = sub_file_path,
38663880 .bytes = bytes,
3867 .stat_size = stat.size,
3868 .stat_inode = stat.inode,
3869 .stat_mtime = stat.mtime,
3881 .stat = stat,
38703882 .pkg = cur_file.pkg,
38713883 .owner_decl = undefined, // Set by Sema immediately after this function returns.
38723884 };
......@@ -3880,9 +3892,9 @@ pub fn detectEmbedFileUpdate(mod: *Module, embed_file: *EmbedFile) !void {
38803892 const stat = try file.stat();
38813893
38823894 const unchanged_metadata =
3883 stat.size == embed_file.stat_size and
3884 stat.mtime == embed_file.stat_mtime and
3885 stat.inode == embed_file.stat_inode;
3895 stat.size == embed_file.stat.size and
3896 stat.mtime == embed_file.stat.mtime and
3897 stat.inode == embed_file.stat.inode;
38863898
38873899 if (unchanged_metadata) return;
38883900
......@@ -3891,9 +3903,11 @@ pub fn detectEmbedFileUpdate(mod: *Module, embed_file: *EmbedFile) !void {
38913903 const bytes = try file.readToEndAllocOptions(gpa, std.math.maxInt(u32), size_usize, 1, 0);
38923904 gpa.free(embed_file.bytes);
38933905 embed_file.bytes = bytes;
3894 embed_file.stat_size = stat.size;
3895 embed_file.stat_mtime = stat.mtime;
3896 embed_file.stat_inode = stat.inode;
3906 embed_file.stat = .{
3907 .size = stat.size,
3908 .mtime = stat.mtime,
3909 .inode = stat.inode,
3910 };
38973911
38983912 mod.comp.mutex.lock();
38993913 defer mod.comp.mutex.unlock();
......@@ -5024,3 +5038,35 @@ pub fn linkerUpdateDecl(mod: *Module, decl: *Decl) !void {
50245038 },
50255039 };
50265040}
5041
5042fn reportRetryableFileError(
5043 mod: *Module,
5044 file: *File,
5045 comptime format: []const u8,
5046 args: anytype,
5047) error{OutOfMemory}!void {
5048 file.status = .retryable_failure;
5049
5050 const err_msg = try ErrorMsg.create(
5051 mod.gpa,
5052 .{
5053 .file_scope = file,
5054 .parent_decl_node = 0,
5055 .lazy = .entire_file,
5056 },
5057 format,
5058 args,
5059 );
5060 errdefer err_msg.destroy(mod.gpa);
5061
5062 mod.comp.mutex.lock();
5063 defer mod.comp.mutex.unlock();
5064
5065 const gop = try mod.failed_files.getOrPut(mod.gpa, file);
5066 if (gop.found_existing) {
5067 if (gop.value_ptr.*) |old_err_msg| {
5068 old_err_msg.destroy(mod.gpa);
5069 }
5070 }
5071 gop.value_ptr.* = err_msg;
5072}
src/main.zig+18-16
......@@ -3664,9 +3664,7 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void
36643664 .zir_loaded = false,
36653665 .sub_file_path = "<stdin>",
36663666 .source = source_code,
3667 .stat_size = undefined,
3668 .stat_inode = undefined,
3669 .stat_mtime = undefined,
3667 .stat = undefined,
36703668 .tree = tree,
36713669 .tree_loaded = true,
36723670 .zir = undefined,
......@@ -3860,9 +3858,11 @@ fn fmtPathFile(
38603858 .zir_loaded = false,
38613859 .sub_file_path = file_path,
38623860 .source = source_code,
3863 .stat_size = stat.size,
3864 .stat_inode = stat.inode,
3865 .stat_mtime = stat.mtime,
3861 .stat = .{
3862 .size = stat.size,
3863 .inode = stat.inode,
3864 .mtime = stat.mtime,
3865 },
38663866 .tree = tree,
38673867 .tree_loaded = true,
38683868 .zir = undefined,
......@@ -4458,9 +4458,7 @@ pub fn cmdAstCheck(
44584458 .zir_loaded = false,
44594459 .sub_file_path = undefined,
44604460 .source = undefined,
4461 .stat_size = undefined,
4462 .stat_inode = undefined,
4463 .stat_mtime = undefined,
4461 .stat = undefined,
44644462 .tree = undefined,
44654463 .zir = undefined,
44664464 .pkg = undefined,
......@@ -4485,9 +4483,11 @@ pub fn cmdAstCheck(
44854483 file.sub_file_path = file_name;
44864484 file.source = source;
44874485 file.source_loaded = true;
4488 file.stat_size = stat.size;
4489 file.stat_inode = stat.inode;
4490 file.stat_mtime = stat.mtime;
4486 file.stat = .{
4487 .size = stat.size,
4488 .inode = stat.inode,
4489 .mtime = stat.mtime,
4490 };
44914491 } else {
44924492 const stdin = io.getStdIn();
44934493 const source = readSourceFileToEndAlloc(arena, &stdin, null) catch |err| {
......@@ -4496,7 +4496,7 @@ pub fn cmdAstCheck(
44964496 file.sub_file_path = "<stdin>";
44974497 file.source = source;
44984498 file.source_loaded = true;
4499 file.stat_size = source.len;
4499 file.stat.size = source.len;
45004500 }
45014501
45024502 file.pkg = try Package.create(gpa, null, file.sub_file_path);
......@@ -4609,9 +4609,11 @@ pub fn cmdChangelist(
46094609 .zir_loaded = false,
46104610 .sub_file_path = old_source_file,
46114611 .source = undefined,
4612 .stat_size = stat.size,
4613 .stat_inode = stat.inode,
4614 .stat_mtime = stat.mtime,
4612 .stat = .{
4613 .size = stat.size,
4614 .inode = stat.inode,
4615 .mtime = stat.mtime,
4616 },
46154617 .tree = undefined,
46164618 .zir = undefined,
46174619 .pkg = undefined,