authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-24 01:14:09-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-10-24 01:14:09-04:00
logb798aaf49937568541e8d89eb0be0058871a969f
tree7671d5a76b00d7d11bed51cac88950fb5ed27324
parent794dc694b140908a9affc5b449cda09bbe971cfe
parentd4911794ae8f43745b2d98a725acf440954f8d90
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17687

frontend: rework `@embedFile` for incremental compilation

10 files changed, 215 insertions(+), 253 deletions(-)

src/Compilation.zig+26-29
...@@ -267,10 +267,6 @@ const Job = union(enum) {...@@ -267,10 +267,6 @@ const Job = union(enum) {
267 /// It may have already be analyzed, or it may have been determined267 /// It may have already be analyzed, or it may have been determined
268 /// to be outdated; in this case perform semantic analysis again.268 /// to be outdated; in this case perform semantic analysis again.
269 analyze_decl: Module.Decl.Index,269 analyze_decl: Module.Decl.Index,
270 /// The file that was loaded with `@embedFile` has changed on disk
271 /// and has been re-loaded into memory. All Decls that depend on it
272 /// need to be re-analyzed.
273 update_embed_file: *Module.EmbedFile,
274 /// The source file containing the Decl has been updated, and so the270 /// The source file containing the Decl has been updated, and so the
275 /// Decl may need its line number information updated in the debug info.271 /// Decl may need its line number information updated in the debug info.
276 update_line_number: Module.Decl.Index,272 update_line_number: Module.Decl.Index,
...@@ -3374,9 +3370,6 @@ pub fn performAllTheWork(...@@ -3374,9 +3370,6 @@ pub fn performAllTheWork(
3374 var win32_resource_prog_node = main_progress_node.start("Compile Win32 Resources", comp.rc_source_files.len);3370 var win32_resource_prog_node = main_progress_node.start("Compile Win32 Resources", comp.rc_source_files.len);
3375 defer win32_resource_prog_node.end();3371 defer win32_resource_prog_node.end();
33763372
3377 var embed_file_prog_node = main_progress_node.start("Detect @embedFile updates", comp.embed_file_work_queue.count);
3378 defer embed_file_prog_node.end();
3379
3380 comp.work_queue_wait_group.reset();3373 comp.work_queue_wait_group.reset();
3381 defer comp.work_queue_wait_group.wait();3374 defer comp.work_queue_wait_group.wait();
33823375
...@@ -3412,7 +3405,7 @@ pub fn performAllTheWork(...@@ -3412,7 +3405,7 @@ pub fn performAllTheWork(
3412 while (comp.embed_file_work_queue.readItem()) |embed_file| {3405 while (comp.embed_file_work_queue.readItem()) |embed_file| {
3413 comp.astgen_wait_group.start();3406 comp.astgen_wait_group.start();
3414 try comp.thread_pool.spawn(workerCheckEmbedFile, .{3407 try comp.thread_pool.spawn(workerCheckEmbedFile, .{
3415 comp, embed_file, &embed_file_prog_node, &comp.astgen_wait_group,3408 comp, embed_file, &comp.astgen_wait_group,
3416 });3409 });
3417 }3410 }
34183411
...@@ -3602,16 +3595,6 @@ fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !v...@@ -3602,16 +3595,6 @@ fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !v
3602 try module.ensureFuncBodyAnalysisQueued(decl.val.toIntern());3595 try module.ensureFuncBodyAnalysisQueued(decl.val.toIntern());
3603 }3596 }
3604 },3597 },
3605 .update_embed_file => |embed_file| {
3606 const named_frame = tracy.namedFrame("update_embed_file");
3607 defer named_frame.end();
3608
3609 const module = comp.bin_file.options.module.?;
3610 module.updateEmbedFile(embed_file) catch |err| switch (err) {
3611 error.OutOfMemory => return error.OutOfMemory,
3612 error.AnalysisFail => return,
3613 };
3614 },
3615 .update_line_number => |decl_index| {3598 .update_line_number => |decl_index| {
3616 const named_frame = tracy.namedFrame("update_line_number");3599 const named_frame = tracy.namedFrame("update_line_number");
3617 defer named_frame.end();3600 defer named_frame.end();
...@@ -3921,17 +3904,11 @@ fn workerUpdateBuiltinZigFile(...@@ -3921,17 +3904,11 @@ fn workerUpdateBuiltinZigFile(
3921fn workerCheckEmbedFile(3904fn workerCheckEmbedFile(
3922 comp: *Compilation,3905 comp: *Compilation,
3923 embed_file: *Module.EmbedFile,3906 embed_file: *Module.EmbedFile,
3924 prog_node: *std.Progress.Node,
3925 wg: *WaitGroup,3907 wg: *WaitGroup,
3926) void {3908) void {
3927 defer wg.finish();3909 defer wg.finish();
39283910
3929 var child_prog_node = prog_node.start(embed_file.sub_file_path, 0);3911 comp.detectEmbedFileUpdate(embed_file) catch |err| {
3930 child_prog_node.activate();
3931 defer child_prog_node.end();
3932
3933 const mod = comp.bin_file.options.module.?;
3934 mod.detectEmbedFileUpdate(embed_file) catch |err| {
3935 comp.reportRetryableEmbedFileError(embed_file, err) catch |oom| switch (oom) {3912 comp.reportRetryableEmbedFileError(embed_file, err) catch |oom| switch (oom) {
3936 // Swallowing this error is OK because it's implied to be OOM when3913 // Swallowing this error is OK because it's implied to be OOM when
3937 // there is a missing `failed_embed_files` error message.3914 // there is a missing `failed_embed_files` error message.
...@@ -3941,6 +3918,25 @@ fn workerCheckEmbedFile(...@@ -3941,6 +3918,25 @@ fn workerCheckEmbedFile(
3941 };3918 };
3942}3919}
39433920
3921fn detectEmbedFileUpdate(comp: *Compilation, embed_file: *Module.EmbedFile) !void {
3922 const mod = comp.bin_file.options.module.?;
3923 const ip = &mod.intern_pool;
3924 const sub_file_path = ip.stringToSlice(embed_file.sub_file_path);
3925 var file = try embed_file.owner.root.openFile(sub_file_path, .{});
3926 defer file.close();
3927
3928 const stat = try file.stat();
3929
3930 const unchanged_metadata =
3931 stat.size == embed_file.stat.size and
3932 stat.mtime == embed_file.stat.mtime and
3933 stat.inode == embed_file.stat.inode;
3934
3935 if (unchanged_metadata) return;
3936
3937 @panic("TODO: handle embed file incremental update");
3938}
3939
3944pub fn obtainCObjectCacheManifest(comp: *const Compilation) Cache.Manifest {3940pub fn obtainCObjectCacheManifest(comp: *const Compilation) Cache.Manifest {
3945 var man = comp.cache_parent.obtain();3941 var man = comp.cache_parent.obtain();
39463942
...@@ -4298,11 +4294,12 @@ fn reportRetryableEmbedFileError(...@@ -4298,11 +4294,12 @@ fn reportRetryableEmbedFileError(
4298) error{OutOfMemory}!void {4294) error{OutOfMemory}!void {
4299 const mod = comp.bin_file.options.module.?;4295 const mod = comp.bin_file.options.module.?;
4300 const gpa = mod.gpa;4296 const gpa = mod.gpa;
43014297 const src_loc = embed_file.src_loc;
4302 const src_loc: Module.SrcLoc = mod.declPtr(embed_file.owner_decl).srcLoc(mod);4298 const ip = &mod.intern_pool;
4303
4304 const err_msg = try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{4299 const err_msg = try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{
4305 embed_file.mod.root, embed_file.sub_file_path, @errorName(err),4300 embed_file.owner.root,
4301 ip.stringToSlice(embed_file.sub_file_path),
4302 @errorName(err),
4306 });4303 });
43074304
4308 errdefer err_msg.destroy(gpa);4305 errdefer err_msg.destroy(gpa);
src/InternPool.zig+39-12
...@@ -5059,13 +5059,13 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) Allocator.Error!Index {...@@ -5059,13 +5059,13 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) Allocator.Error!Index {
5059 assert(child == .u8_type);5059 assert(child == .u8_type);
5060 if (bytes.len != len) {5060 if (bytes.len != len) {
5061 assert(bytes.len == len_including_sentinel);5061 assert(bytes.len == len_including_sentinel);
5062 assert(bytes[@as(usize, @intCast(len))] == ip.indexToKey(sentinel).int.storage.u64);5062 assert(bytes[@intCast(len)] == ip.indexToKey(sentinel).int.storage.u64);
5063 }5063 }
5064 },5064 },
5065 .elems => |elems| {5065 .elems => |elems| {
5066 if (elems.len != len) {5066 if (elems.len != len) {
5067 assert(elems.len == len_including_sentinel);5067 assert(elems.len == len_including_sentinel);
5068 assert(elems[@as(usize, @intCast(len))] == sentinel);5068 assert(elems[@intCast(len)] == sentinel);
5069 }5069 }
5070 },5070 },
5071 .repeated_elem => |elem| {5071 .repeated_elem => |elem| {
...@@ -5168,7 +5168,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) Allocator.Error!Index {...@@ -5168,7 +5168,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) Allocator.Error!Index {
51685168
5169 if (child == .u8_type) bytes: {5169 if (child == .u8_type) bytes: {
5170 const string_bytes_index = ip.string_bytes.items.len;5170 const string_bytes_index = ip.string_bytes.items.len;
5171 try ip.string_bytes.ensureUnusedCapacity(gpa, @as(usize, @intCast(len_including_sentinel + 1)));5171 try ip.string_bytes.ensureUnusedCapacity(gpa, @intCast(len_including_sentinel + 1));
5172 try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Bytes).Struct.fields.len);5172 try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Bytes).Struct.fields.len);
5173 switch (aggregate.storage) {5173 switch (aggregate.storage) {
5174 .bytes => |bytes| ip.string_bytes.appendSliceAssumeCapacity(bytes[0..@intCast(len)]),5174 .bytes => |bytes| ip.string_bytes.appendSliceAssumeCapacity(bytes[0..@intCast(len)]),
...@@ -5178,15 +5178,15 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) Allocator.Error!Index {...@@ -5178,15 +5178,15 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) Allocator.Error!Index {
5178 break :bytes;5178 break :bytes;
5179 },5179 },
5180 .int => |int| ip.string_bytes.appendAssumeCapacity(5180 .int => |int| ip.string_bytes.appendAssumeCapacity(
5181 @as(u8, @intCast(int.storage.u64)),5181 @intCast(int.storage.u64),
5182 ),5182 ),
5183 else => unreachable,5183 else => unreachable,
5184 },5184 },
5185 .repeated_elem => |elem| switch (ip.indexToKey(elem)) {5185 .repeated_elem => |elem| switch (ip.indexToKey(elem)) {
5186 .undef => break :bytes,5186 .undef => break :bytes,
5187 .int => |int| @memset(5187 .int => |int| @memset(
5188 ip.string_bytes.addManyAsSliceAssumeCapacity(@as(usize, @intCast(len))),5188 ip.string_bytes.addManyAsSliceAssumeCapacity(@intCast(len)),
5189 @as(u8, @intCast(int.storage.u64)),5189 @intCast(int.storage.u64),
5190 ),5190 ),
5191 else => unreachable,5191 else => unreachable,
5192 },5192 },
...@@ -5194,12 +5194,12 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) Allocator.Error!Index {...@@ -5194,12 +5194,12 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) Allocator.Error!Index {
5194 const has_internal_null =5194 const has_internal_null =
5195 std.mem.indexOfScalar(u8, ip.string_bytes.items[string_bytes_index..], 0) != null;5195 std.mem.indexOfScalar(u8, ip.string_bytes.items[string_bytes_index..], 0) != null;
5196 if (sentinel != .none) ip.string_bytes.appendAssumeCapacity(5196 if (sentinel != .none) ip.string_bytes.appendAssumeCapacity(
5197 @as(u8, @intCast(ip.indexToKey(sentinel).int.storage.u64)),5197 @intCast(ip.indexToKey(sentinel).int.storage.u64),
5198 );5198 );
5199 const string = if (has_internal_null)5199 const string: String = if (has_internal_null)
5200 @as(String, @enumFromInt(string_bytes_index))5200 @enumFromInt(string_bytes_index)
5201 else5201 else
5202 (try ip.getOrPutTrailingString(gpa, @as(usize, @intCast(len_including_sentinel)))).toString();5202 (try ip.getOrPutTrailingString(gpa, @intCast(len_including_sentinel))).toString();
5203 ip.items.appendAssumeCapacity(.{5203 ip.items.appendAssumeCapacity(.{
5204 .tag = .bytes,5204 .tag = .bytes,
5205 .data = ip.addExtraAssumeCapacity(Bytes{5205 .data = ip.addExtraAssumeCapacity(Bytes{
...@@ -7557,7 +7557,7 @@ pub fn getOrPutStringFmt(...@@ -7557,7 +7557,7 @@ pub fn getOrPutStringFmt(
7557 args: anytype,7557 args: anytype,
7558) Allocator.Error!NullTerminatedString {7558) Allocator.Error!NullTerminatedString {
7559 // ensure that references to string_bytes in args do not get invalidated7559 // ensure that references to string_bytes in args do not get invalidated
7560 const len = @as(usize, @intCast(std.fmt.count(format, args) + 1));7560 const len: usize = @intCast(std.fmt.count(format, args) + 1);
7561 try ip.string_bytes.ensureUnusedCapacity(gpa, len);7561 try ip.string_bytes.ensureUnusedCapacity(gpa, len);
7562 ip.string_bytes.writer(undefined).print(format, args) catch unreachable;7562 ip.string_bytes.writer(undefined).print(format, args) catch unreachable;
7563 ip.string_bytes.appendAssumeCapacity(0);7563 ip.string_bytes.appendAssumeCapacity(0);
...@@ -7581,7 +7581,7 @@ pub fn getOrPutTrailingString(...@@ -7581,7 +7581,7 @@ pub fn getOrPutTrailingString(
7581 len: usize,7581 len: usize,
7582) Allocator.Error!NullTerminatedString {7582) Allocator.Error!NullTerminatedString {
7583 const string_bytes = &ip.string_bytes;7583 const string_bytes = &ip.string_bytes;
7584 const str_index = @as(u32, @intCast(string_bytes.items.len - len));7584 const str_index: u32 = @intCast(string_bytes.items.len - len);
7585 if (len > 0 and string_bytes.getLast() == 0) {7585 if (len > 0 and string_bytes.getLast() == 0) {
7586 _ = string_bytes.pop();7586 _ = string_bytes.pop();
7587 } else {7587 } else {
...@@ -7603,6 +7603,33 @@ pub fn getOrPutTrailingString(...@@ -7603,6 +7603,33 @@ pub fn getOrPutTrailingString(
7603 }7603 }
7604}7604}
76057605
7606/// Uses the last len bytes of ip.string_bytes as the key.
7607pub fn getTrailingAggregate(
7608 ip: *InternPool,
7609 gpa: Allocator,
7610 ty: Index,
7611 len: usize,
7612) Allocator.Error!Index {
7613 try ip.items.ensureUnusedCapacity(gpa, 1);
7614 try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Bytes).Struct.fields.len);
7615 const str: String = @enumFromInt(@intFromEnum(try getOrPutTrailingString(ip, gpa, len)));
7616 const adapter: KeyAdapter = .{ .intern_pool = ip };
7617 const gop = try ip.map.getOrPutAdapted(gpa, Key{ .aggregate = .{
7618 .ty = ty,
7619 .storage = .{ .bytes = ip.string_bytes.items[@intFromEnum(str)..] },
7620 } }, adapter);
7621 if (gop.found_existing) return @enumFromInt(gop.index);
7622
7623 ip.items.appendAssumeCapacity(.{
7624 .tag = .bytes,
7625 .data = ip.addExtraAssumeCapacity(Bytes{
7626 .ty = ty,
7627 .bytes = str,
7628 }),
7629 });
7630 return @enumFromInt(ip.items.len - 1);
7631}
7632
7606pub fn getString(ip: *InternPool, s: []const u8) OptionalNullTerminatedString {7633pub fn getString(ip: *InternPool, s: []const u8) OptionalNullTerminatedString {
7607 if (ip.string_table.getKeyAdapted(s, std.hash_map.StringIndexAdapter{7634 if (ip.string_table.getKeyAdapted(s, std.hash_map.StringIndexAdapter{
7608 .bytes = &ip.string_bytes,7635 .bytes = &ip.string_bytes,
src/Module.zig+66-96
...@@ -1214,26 +1214,14 @@ pub const File = struct {...@@ -1214,26 +1214,14 @@ pub const File = struct {
1214 }1214 }
1215};1215};
12161216
1217/// Represents the contents of a file loaded with `@embedFile`.
1218pub const EmbedFile = struct {1217pub const EmbedFile = struct {
1219 /// Relative to the owning package's root_src_dir.1218 /// Relative to the owning module's root directory.
1220 /// Memory is stored in gpa, owned by EmbedFile.1219 sub_file_path: InternPool.NullTerminatedString,
1221 sub_file_path: []const u8,1220 /// Module that this file is a part of, managed externally.
1222 bytes: [:0]const u8,1221 owner: *Package.Module,
1223 stat: Cache.File.Stat,1222 stat: Cache.File.Stat,
1224 /// Package that this file is a part of, managed externally.1223 val: InternPool.Index,
1225 mod: *Package.Module,1224 src_loc: SrcLoc,
1226 /// The Decl that was created from the `@embedFile` to own this resource.
1227 /// This is how zig knows what other Decl objects to invalidate if the file
1228 /// changes on disk.
1229 owner_decl: Decl.Index,
1230
1231 fn destroy(embed_file: *EmbedFile, mod: *Module) void {
1232 const gpa = mod.gpa;
1233 gpa.free(embed_file.sub_file_path);
1234 gpa.free(embed_file.bytes);
1235 gpa.destroy(embed_file);
1236 }
1237};1225};
12381226
1239/// This struct holds data necessary to construct API-facing `AllErrors.Message`.1227/// This struct holds data necessary to construct API-facing `AllErrors.Message`.
...@@ -2532,7 +2520,8 @@ pub fn deinit(mod: *Module) void {...@@ -2532,7 +2520,8 @@ pub fn deinit(mod: *Module) void {
2532 var it = mod.embed_table.iterator();2520 var it = mod.embed_table.iterator();
2533 while (it.next()) |entry| {2521 while (it.next()) |entry| {
2534 gpa.free(entry.key_ptr.*);2522 gpa.free(entry.key_ptr.*);
2535 entry.value_ptr.*.destroy(mod);2523 const ef: *EmbedFile = entry.value_ptr.*;
2524 gpa.destroy(ef);
2536 }2525 }
2537 mod.embed_table.deinit(gpa);2526 mod.embed_table.deinit(gpa);
2538 }2527 }
...@@ -3543,35 +3532,6 @@ pub fn ensureFuncBodyAnalysisQueued(mod: *Module, func_index: InternPool.Index)...@@ -3543,35 +3532,6 @@ pub fn ensureFuncBodyAnalysisQueued(mod: *Module, func_index: InternPool.Index)
3543 func.analysis(ip).state = .queued;3532 func.analysis(ip).state = .queued;
3544}3533}
35453534
3546pub fn updateEmbedFile(mod: *Module, embed_file: *EmbedFile) SemaError!void {
3547 const tracy = trace(@src());
3548 defer tracy.end();
3549
3550 // TODO we can potentially relax this if we store some more information along
3551 // with decl dependency edges
3552 const owner_decl = mod.declPtr(embed_file.owner_decl);
3553 for (owner_decl.dependants.keys()) |dep_index| {
3554 const dep = mod.declPtr(dep_index);
3555 switch (dep.analysis) {
3556 .unreferenced => unreachable,
3557 .in_progress => continue, // already doing analysis, ok
3558 .outdated => continue, // already queued for update
3559
3560 .file_failure,
3561 .dependency_failure,
3562 .sema_failure,
3563 .sema_failure_retryable,
3564 .liveness_failure,
3565 .codegen_failure,
3566 .codegen_failure_retryable,
3567 .complete,
3568 => if (dep.generation != mod.generation) {
3569 try mod.markOutdatedDecl(dep_index);
3570 },
3571 }
3572 }
3573}
3574
3575/// https://github.com/ziglang/zig/issues/143073535/// https://github.com/ziglang/zig/issues/14307
3576pub fn semaPkg(mod: *Module, pkg: *Package.Module) !void {3536pub fn semaPkg(mod: *Module, pkg: *Package.Module) !void {
3577 const file = (try mod.importPkg(pkg)).file;3537 const file = (try mod.importPkg(pkg)).file;
...@@ -4153,7 +4113,12 @@ pub fn importFile(...@@ -4153,7 +4113,12 @@ pub fn importFile(
4153 };4113 };
4154}4114}
41554115
4156pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*EmbedFile {4116pub fn embedFile(
4117 mod: *Module,
4118 cur_file: *File,
4119 import_string: []const u8,
4120 src_loc: SrcLoc,
4121) !InternPool.Index {
4157 const gpa = mod.gpa;4122 const gpa = mod.gpa;
41584123
4159 if (cur_file.mod.deps.get(import_string)) |pkg| {4124 if (cur_file.mod.deps.get(import_string)) |pkg| {
...@@ -4166,13 +4131,17 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb...@@ -4166,13 +4131,17 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb
4166 defer if (!keep_resolved_path) gpa.free(resolved_path);4131 defer if (!keep_resolved_path) gpa.free(resolved_path);
41674132
4168 const gop = try mod.embed_table.getOrPut(gpa, resolved_path);4133 const gop = try mod.embed_table.getOrPut(gpa, resolved_path);
4169 errdefer assert(mod.embed_table.remove(resolved_path));4134 errdefer {
4170 if (gop.found_existing) return gop.value_ptr.*;4135 assert(mod.embed_table.remove(resolved_path));
4136 keep_resolved_path = false;
4137 }
4138 if (gop.found_existing) return gop.value_ptr.*.val;
4139 keep_resolved_path = true;
41714140
4172 const sub_file_path = try gpa.dupe(u8, pkg.root_src_path);4141 const sub_file_path = try gpa.dupe(u8, pkg.root_src_path);
4173 errdefer gpa.free(sub_file_path);4142 errdefer gpa.free(sub_file_path);
41744143
4175 return newEmbedFile(mod, pkg, sub_file_path, resolved_path, &keep_resolved_path, gop);4144 return newEmbedFile(mod, pkg, sub_file_path, resolved_path, gop, src_loc);
4176 }4145 }
41774146
4178 // The resolved path is used as the key in the table, to detect if a file4147 // The resolved path is used as the key in the table, to detect if a file
...@@ -4189,8 +4158,12 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb...@@ -4189,8 +4158,12 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb
4189 defer if (!keep_resolved_path) gpa.free(resolved_path);4158 defer if (!keep_resolved_path) gpa.free(resolved_path);
41904159
4191 const gop = try mod.embed_table.getOrPut(gpa, resolved_path);4160 const gop = try mod.embed_table.getOrPut(gpa, resolved_path);
4192 errdefer assert(mod.embed_table.remove(resolved_path));4161 errdefer {
4193 if (gop.found_existing) return gop.value_ptr.*;4162 assert(mod.embed_table.remove(resolved_path));
4163 keep_resolved_path = false;
4164 }
4165 if (gop.found_existing) return gop.value_ptr.*.val;
4166 keep_resolved_path = true;
41944167
4195 const resolved_root_path = try std.fs.path.resolve(gpa, &.{4168 const resolved_root_path = try std.fs.path.resolve(gpa, &.{
4196 cur_file.mod.root.root_dir.path orelse ".",4169 cur_file.mod.root.root_dir.path orelse ".",
...@@ -4213,7 +4186,7 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb...@@ -4213,7 +4186,7 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb
4213 };4186 };
4214 errdefer gpa.free(sub_file_path);4187 errdefer gpa.free(sub_file_path);
42154188
4216 return newEmbedFile(mod, cur_file.mod, sub_file_path, resolved_path, &keep_resolved_path, gop);4189 return newEmbedFile(mod, cur_file.mod, sub_file_path, resolved_path, gop, src_loc);
4217}4190}
42184191
4219/// https://github.com/ziglang/zig/issues/143074192/// https://github.com/ziglang/zig/issues/14307
...@@ -4222,9 +4195,9 @@ fn newEmbedFile(...@@ -4222,9 +4195,9 @@ fn newEmbedFile(
4222 pkg: *Package.Module,4195 pkg: *Package.Module,
4223 sub_file_path: []const u8,4196 sub_file_path: []const u8,
4224 resolved_path: []const u8,4197 resolved_path: []const u8,
4225 keep_resolved_path: *bool,
4226 gop: std.StringHashMapUnmanaged(*EmbedFile).GetOrPutResult,4198 gop: std.StringHashMapUnmanaged(*EmbedFile).GetOrPutResult,
4227) !*EmbedFile {4199 src_loc: SrcLoc,
4200) !InternPool.Index {
4228 const gpa = mod.gpa;4201 const gpa = mod.gpa;
42294202
4230 const new_file = try gpa.create(EmbedFile);4203 const new_file = try gpa.create(EmbedFile);
...@@ -4239,57 +4212,54 @@ fn newEmbedFile(...@@ -4239,57 +4212,54 @@ fn newEmbedFile(
4239 .inode = actual_stat.inode,4212 .inode = actual_stat.inode,
4240 .mtime = actual_stat.mtime,4213 .mtime = actual_stat.mtime,
4241 };4214 };
4242 const size_usize = std.math.cast(usize, actual_stat.size) orelse return error.Overflow;4215 const size = std.math.cast(usize, actual_stat.size) orelse return error.Overflow;
4243 const bytes = try file.readToEndAllocOptions(gpa, std.math.maxInt(u32), size_usize, 1, 0);4216 const ip = &mod.intern_pool;
4244 errdefer gpa.free(bytes);4217
4218 const ptr = try ip.string_bytes.addManyAsSlice(gpa, size);
4219 const actual_read = try file.readAll(ptr);
4220 if (actual_read != size) return error.UnexpectedEndOfFile;
42454221
4246 if (mod.comp.whole_cache_manifest) |whole_cache_manifest| {4222 if (mod.comp.whole_cache_manifest) |whole_cache_manifest| {
4247 const copied_resolved_path = try gpa.dupe(u8, resolved_path);4223 const copied_resolved_path = try gpa.dupe(u8, resolved_path);
4248 errdefer gpa.free(copied_resolved_path);4224 errdefer gpa.free(copied_resolved_path);
4249 mod.comp.whole_cache_manifest_mutex.lock();4225 mod.comp.whole_cache_manifest_mutex.lock();
4250 defer mod.comp.whole_cache_manifest_mutex.unlock();4226 defer mod.comp.whole_cache_manifest_mutex.unlock();
4251 try whole_cache_manifest.addFilePostContents(copied_resolved_path, bytes, stat);4227 try whole_cache_manifest.addFilePostContents(copied_resolved_path, ptr, stat);
4252 }4228 }
42534229
4254 keep_resolved_path.* = true; // It's now owned by embed_table.4230 const array_ty = try ip.get(gpa, .{ .array_type = .{
4231 .len = size,
4232 .sentinel = .zero_u8,
4233 .child = .u8_type,
4234 } });
4235 const array_val = try ip.getTrailingAggregate(gpa, array_ty, size);
4236
4237 const ptr_ty = (try mod.ptrType(.{
4238 .child = array_ty,
4239 .flags = .{
4240 .alignment = .none,
4241 .is_const = true,
4242 .address_space = .generic,
4243 },
4244 })).toIntern();
4245
4246 const ptr_val = try ip.get(gpa, .{ .ptr = .{
4247 .ty = ptr_ty,
4248 .addr = .{ .anon_decl = .{
4249 .val = array_val,
4250 .orig_ty = ptr_ty,
4251 } },
4252 } });
4253
4255 gop.value_ptr.* = new_file;4254 gop.value_ptr.* = new_file;
4256 new_file.* = .{4255 new_file.* = .{
4257 .sub_file_path = sub_file_path,4256 .sub_file_path = try ip.getOrPutString(gpa, sub_file_path),
4258 .bytes = bytes,4257 .owner = pkg,
4259 .stat = stat,4258 .stat = stat,
4260 .mod = pkg,4259 .val = ptr_val,
4261 .owner_decl = undefined, // Set by Sema immediately after this function returns.4260 .src_loc = src_loc,
4262 };
4263 return new_file;
4264}
4265
4266pub fn detectEmbedFileUpdate(mod: *Module, embed_file: *EmbedFile) !void {
4267 var file = try embed_file.mod.root.openFile(embed_file.sub_file_path, .{});
4268 defer file.close();
4269
4270 const stat = try file.stat();
4271
4272 const unchanged_metadata =
4273 stat.size == embed_file.stat.size and
4274 stat.mtime == embed_file.stat.mtime and
4275 stat.inode == embed_file.stat.inode;
4276
4277 if (unchanged_metadata) return;
4278
4279 const gpa = mod.gpa;
4280 const size_usize = std.math.cast(usize, stat.size) orelse return error.Overflow;
4281 const bytes = try file.readToEndAllocOptions(gpa, std.math.maxInt(u32), size_usize, 1, 0);
4282 gpa.free(embed_file.bytes);
4283 embed_file.bytes = bytes;
4284 embed_file.stat = .{
4285 .size = stat.size,
4286 .mtime = stat.mtime,
4287 .inode = stat.inode,
4288 };4261 };
42894262 return ptr_val;
4290 mod.comp.mutex.lock();
4291 defer mod.comp.mutex.unlock();
4292 try mod.comp.work_queue.writeItem(.{ .update_embed_file = embed_file });
4293}4263}
42944264
4295pub fn scanNamespace(4265pub fn scanNamespace(
src/Sema.zig+6-28
...@@ -3739,7 +3739,7 @@ fn resolveComptimeKnownAllocValue(sema: *Sema, block: *Block, alloc: Air.Inst.Re...@@ -3739,7 +3739,7 @@ fn resolveComptimeKnownAllocValue(sema: *Sema, block: *Block, alloc: Air.Inst.Re
3739 // The simple strategy failed: we must create a mutable comptime alloc and3739 // The simple strategy failed: we must create a mutable comptime alloc and
3740 // perform all of the runtime store operations at comptime.3740 // perform all of the runtime store operations at comptime.
37413741
3742 var anon_decl = try block.startAnonDecl();3742 var anon_decl = try block.startAnonDecl(); // TODO: comptime value mutation without Decl
3743 defer anon_decl.deinit();3743 defer anon_decl.deinit();
3744 const decl_index = try anon_decl.finish(elem_ty, try mod.undefValue(elem_ty), ptr_info.flags.alignment);3744 const decl_index = try anon_decl.finish(elem_ty, try mod.undefValue(elem_ty), ptr_info.flags.alignment);
37453745
...@@ -5454,7 +5454,7 @@ fn storeToInferredAllocComptime(...@@ -5454,7 +5454,7 @@ fn storeToInferredAllocComptime(
5454 // The alloc will turn into a Decl.5454 // The alloc will turn into a Decl.
5455 if (try sema.resolveMaybeUndefValAllowVariables(operand)) |operand_val| store: {5455 if (try sema.resolveMaybeUndefValAllowVariables(operand)) |operand_val| store: {
5456 if (operand_val.getVariable(sema.mod) != null) break :store;5456 if (operand_val.getVariable(sema.mod) != null) break :store;
5457 var anon_decl = try block.startAnonDecl();5457 var anon_decl = try block.startAnonDecl(); // TODO: comptime value mutation without Decl
5458 defer anon_decl.deinit();5458 defer anon_decl.deinit();
5459 iac.decl_index = try anon_decl.finish(operand_ty, operand_val, iac.alignment);5459 iac.decl_index = try anon_decl.finish(operand_ty, operand_val, iac.alignment);
5460 try sema.comptime_mutable_decls.append(iac.decl_index);5460 try sema.comptime_mutable_decls.append(iac.decl_index);
...@@ -6113,7 +6113,7 @@ fn zirExportValue(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError...@@ -6113,7 +6113,7 @@ fn zirExportValue(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
6113 else => |e| return e,6113 else => |e| return e,
6114 };6114 };
6115 const decl_index = if (operand.val.getFunction(sema.mod)) |function| function.owner_decl else blk: {6115 const decl_index = if (operand.val.getFunction(sema.mod)) |function| function.owner_decl else blk: {
6116 var anon_decl = try block.startAnonDecl();6116 var anon_decl = try block.startAnonDecl(); // TODO: export value without Decl
6117 defer anon_decl.deinit();6117 defer anon_decl.deinit();
6118 break :blk try anon_decl.finish(operand.ty, operand.val, .none);6118 break :blk try anon_decl.finish(operand.ty, operand.val, .none);
6119 };6119 };
...@@ -13155,7 +13155,8 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -13155,7 +13155,8 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
13155 return sema.fail(block, operand_src, "file path name cannot be empty", .{});13155 return sema.fail(block, operand_src, "file path name cannot be empty", .{});
13156 }13156 }
1315713157
13158 const embed_file = mod.embedFile(block.getFileScope(mod), name) catch |err| switch (err) {13158 const src_loc = operand_src.toSrcLoc(mod.declPtr(block.src_decl), mod);
13159 const val = mod.embedFile(block.getFileScope(mod), name, src_loc) catch |err| switch (err) {
13159 error.ImportOutsideModulePath => {13160 error.ImportOutsideModulePath => {
13160 return sema.fail(block, operand_src, "embed of file outside package path: '{s}'", .{name});13161 return sema.fail(block, operand_src, "embed of file outside package path: '{s}'", .{name});
13161 },13162 },
...@@ -13166,30 +13167,7 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -13166,30 +13167,7 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
13166 },13167 },
13167 };13168 };
1316813169
13169 var anon_decl = try block.startAnonDecl();13170 return Air.internedToRef(val);
13170 defer anon_decl.deinit();
13171
13172 // TODO instead of using `.bytes`, create a new value tag for pointing at
13173 // a `*Module.EmbedFile`. The purpose of this would be:
13174 // - If only the length is read and the bytes are not inspected by comptime code,
13175 // there can be an optimization where the codegen backend does a copy_file_range
13176 // into the final binary, and never loads the data into memory.
13177 // - When a Decl is destroyed, it can free the `*Module.EmbedFile`.
13178 const ty = try mod.arrayType(.{
13179 .len = embed_file.bytes.len,
13180 .sentinel = .zero_u8,
13181 .child = .u8_type,
13182 });
13183 embed_file.owner_decl = try anon_decl.finish(
13184 ty,
13185 (try mod.intern(.{ .aggregate = .{
13186 .ty = ty.toIntern(),
13187 .storage = .{ .bytes = embed_file.bytes },
13188 } })).toValue(),
13189 .none, // default alignment
13190 );
13191
13192 return sema.analyzeDeclRef(embed_file.owner_decl);
13193}13171}
1319413172
13195fn zirRetErrValueCode(sema: *Sema, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {13173fn zirRetErrValueCode(sema: *Sema, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
src/main.zig-10
...@@ -607,16 +607,6 @@ const usage_build_generic =...@@ -607,16 +607,6 @@ const usage_build_generic =
607 \\607 \\
608;608;
609609
610const repl_help =
611 \\Commands:
612 \\ update Detect changes to source files and update output files.
613 \\ run Execute the output file, if it is an executable or test.
614 \\ update-and-run Perform an `update` followed by `run`.
615 \\ help Print this text
616 \\ exit Quit this repl
617 \\
618;
619
620const SOName = union(enum) {610const SOName = union(enum) {
621 no,611 no,
622 yes_default_value,612 yes_default_value,
test/behavior.zig+1-1
...@@ -55,7 +55,6 @@ test {...@@ -55,7 +55,6 @@ test {
55 _ = @import("behavior/bugs/3384.zig");55 _ = @import("behavior/bugs/3384.zig");
56 _ = @import("behavior/bugs/3586.zig");56 _ = @import("behavior/bugs/3586.zig");
57 _ = @import("behavior/bugs/3742.zig");57 _ = @import("behavior/bugs/3742.zig");
58 _ = @import("behavior/bugs/3779.zig");
59 _ = @import("behavior/bugs/4328.zig");58 _ = @import("behavior/bugs/4328.zig");
60 _ = @import("behavior/bugs/4560.zig");59 _ = @import("behavior/bugs/4560.zig");
61 _ = @import("behavior/bugs/4769_a.zig");60 _ = @import("behavior/bugs/4769_a.zig");
...@@ -209,6 +208,7 @@ test {...@@ -209,6 +208,7 @@ test {
209 _ = @import("behavior/slice.zig");208 _ = @import("behavior/slice.zig");
210 _ = @import("behavior/slice_sentinel_comptime.zig");209 _ = @import("behavior/slice_sentinel_comptime.zig");
211 _ = @import("behavior/src.zig");210 _ = @import("behavior/src.zig");
211 _ = @import("behavior/string_literals.zig");
212 _ = @import("behavior/struct.zig");212 _ = @import("behavior/struct.zig");
213 _ = @import("behavior/struct_contains_null_ptr_itself.zig");213 _ = @import("behavior/struct_contains_null_ptr_itself.zig");
214 _ = @import("behavior/struct_contains_slice_of_itself.zig");214 _ = @import("behavior/struct_contains_slice_of_itself.zig");
test/behavior/bugs/3779.zig deleted-76
...@@ -1,76 +0,0 @@
1const std = @import("std");
2const builtin = @import("builtin");
3
4const TestEnum = enum { TestEnumValue };
5const tag_name = @tagName(TestEnum.TestEnumValue);
6const ptr_tag_name: [*:0]const u8 = tag_name;
7
8test "@tagName() returns a string literal" {
9 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
11
12 try std.testing.expect(*const [13:0]u8 == @TypeOf(tag_name));
13 try std.testing.expect(std.mem.eql(u8, "TestEnumValue", tag_name));
14 try std.testing.expect(std.mem.eql(u8, "TestEnumValue", ptr_tag_name[0..tag_name.len]));
15}
16
17const TestError = error{TestErrorCode};
18const error_name = @errorName(TestError.TestErrorCode);
19const ptr_error_name: [*:0]const u8 = error_name;
20
21test "@errorName() returns a string literal" {
22 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
23 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
24
25 try std.testing.expect(*const [13:0]u8 == @TypeOf(error_name));
26 try std.testing.expect(std.mem.eql(u8, "TestErrorCode", error_name));
27 try std.testing.expect(std.mem.eql(u8, "TestErrorCode", ptr_error_name[0..error_name.len]));
28}
29
30const TestType = struct {};
31const type_name = @typeName(TestType);
32const ptr_type_name: [*:0]const u8 = type_name;
33
34test "@typeName() returns a string literal" {
35 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
36 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
37
38 try std.testing.expect(*const [type_name.len:0]u8 == @TypeOf(type_name));
39 try std.testing.expect(std.mem.eql(u8, "behavior.bugs.3779.TestType", type_name));
40 try std.testing.expect(std.mem.eql(u8, "behavior.bugs.3779.TestType", ptr_type_name[0..type_name.len]));
41}
42
43const actual_contents = @embedFile("3779_file_to_embed.txt");
44const ptr_actual_contents: [*:0]const u8 = actual_contents;
45const expected_contents = "hello zig\n";
46
47test "@embedFile() returns a string literal" {
48 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
49 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
50
51 try std.testing.expect(*const [expected_contents.len:0]u8 == @TypeOf(actual_contents));
52 try std.testing.expect(std.mem.eql(u8, expected_contents, actual_contents));
53 try std.testing.expect(std.mem.eql(u8, expected_contents, actual_contents));
54 try std.testing.expect(std.mem.eql(u8, expected_contents, ptr_actual_contents[0..actual_contents.len]));
55}
56
57fn testFnForSrc() std.builtin.SourceLocation {
58 return @src();
59}
60
61test "@src() returns a struct containing 0-terminated string slices" {
62 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
63 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
64
65 const src = testFnForSrc();
66 try std.testing.expect([:0]const u8 == @TypeOf(src.file));
67 try std.testing.expect(std.mem.endsWith(u8, src.file, "3779.zig"));
68 try std.testing.expect([:0]const u8 == @TypeOf(src.fn_name));
69 try std.testing.expect(std.mem.endsWith(u8, src.fn_name, "testFnForSrc"));
70
71 const ptr_src_file: [*:0]const u8 = src.file;
72 _ = ptr_src_file; // unused
73
74 const ptr_src_fn_name: [*:0]const u8 = src.fn_name;
75 _ = ptr_src_fn_name; // unused
76}
test/behavior/bugs/3779_file_to_embed.txt deleted-1
...@@ -1 +0,0 @@
1hello zig
test/behavior/file_to_embed.txt created+1
...@@ -0,0 +1 @@
1hello zig
test/behavior/string_literals.zig created+76
...@@ -0,0 +1,76 @@
1const std = @import("std");
2const builtin = @import("builtin");
3
4const TestEnum = enum { TestEnumValue };
5const tag_name = @tagName(TestEnum.TestEnumValue);
6const ptr_tag_name: [*:0]const u8 = tag_name;
7
8test "@tagName() returns a string literal" {
9 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
11
12 try std.testing.expect(*const [13:0]u8 == @TypeOf(tag_name));
13 try std.testing.expect(std.mem.eql(u8, "TestEnumValue", tag_name));
14 try std.testing.expect(std.mem.eql(u8, "TestEnumValue", ptr_tag_name[0..tag_name.len]));
15}
16
17const TestError = error{TestErrorCode};
18const error_name = @errorName(TestError.TestErrorCode);
19const ptr_error_name: [*:0]const u8 = error_name;
20
21test "@errorName() returns a string literal" {
22 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
23 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
24
25 try std.testing.expect(*const [13:0]u8 == @TypeOf(error_name));
26 try std.testing.expect(std.mem.eql(u8, "TestErrorCode", error_name));
27 try std.testing.expect(std.mem.eql(u8, "TestErrorCode", ptr_error_name[0..error_name.len]));
28}
29
30const TestType = struct {};
31const type_name = @typeName(TestType);
32const ptr_type_name: [*:0]const u8 = type_name;
33
34test "@typeName() returns a string literal" {
35 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
36 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
37
38 try std.testing.expect(*const [type_name.len:0]u8 == @TypeOf(type_name));
39 try std.testing.expect(std.mem.eql(u8, "behavior.string_literals.TestType", type_name));
40 try std.testing.expect(std.mem.eql(u8, "behavior.string_literals.TestType", ptr_type_name[0..type_name.len]));
41}
42
43const actual_contents = @embedFile("file_to_embed.txt");
44const ptr_actual_contents: [*:0]const u8 = actual_contents;
45const expected_contents = "hello zig\n";
46
47test "@embedFile() returns a string literal" {
48 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
49 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
50
51 try std.testing.expect(*const [expected_contents.len:0]u8 == @TypeOf(actual_contents));
52 try std.testing.expect(std.mem.eql(u8, expected_contents, actual_contents));
53 try std.testing.expect(std.mem.eql(u8, expected_contents, actual_contents));
54 try std.testing.expect(std.mem.eql(u8, expected_contents, ptr_actual_contents[0..actual_contents.len]));
55}
56
57fn testFnForSrc() std.builtin.SourceLocation {
58 return @src();
59}
60
61test "@src() returns a struct containing 0-terminated string slices" {
62 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
63 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
64
65 const src = testFnForSrc();
66 try std.testing.expect([:0]const u8 == @TypeOf(src.file));
67 try std.testing.expect(std.mem.endsWith(u8, src.file, "string_literals.zig"));
68 try std.testing.expect([:0]const u8 == @TypeOf(src.fn_name));
69 try std.testing.expect(std.mem.endsWith(u8, src.fn_name, "testFnForSrc"));
70
71 const ptr_src_file: [*:0]const u8 = src.file;
72 _ = ptr_src_file; // unused
73
74 const ptr_src_fn_name: [*:0]const u8 = src.fn_name;
75 _ = ptr_src_fn_name; // unused
76}