authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-12-18 18:28:23-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-01-15 15:11:36-08:00
log766284fec8184e7765b9a7900aea1a7494214153
tree85ff71a3e4d28f684f108e3e50d0a2b4ef0b3c8c
parent3cb00c5bcd3dec191bea0cbab9fcac77c0e3a910

wasm linker: fix data segments memory flow

Recognize three distinct phases: * before prelink ("object phase") * after prelink, before flush ("zcu phase") * during flush ("flush phase") With this setup, we create data structures during the object phase, then mutate them during the zcu phase, and then further mutate them during the flush phase. In order to make the flush phase repeatable, the data structures are copied just before starting the flush phase. Further Zcu updates occur against the non-copied data structures. What's not implemented is frontend garbage collection, in which case some more changes will be needed in this linker logic to achieve a valid state with data invariants intact.

2 files changed, 77 insertions(+), 100 deletions(-)

src/link/Wasm.zig+58-79
......@@ -189,7 +189,10 @@ debug_sections: DebugSections = .{},
189189
190190flush_buffer: Flush = .{},
191191
192missing_exports_init: []String = &.{},
192/// Empty until `prelink`. There it is populated based on object files.
193/// Next, it is copied into `Flush.missing_exports` just before `flush`
194/// and that data is used during `flush`.
195missing_exports: std.AutoArrayHashMapUnmanaged(String, void) = .empty,
193196entry_resolution: FunctionImport.Resolution = .unresolved,
194197
195198/// Empty when outputting an object.
......@@ -206,13 +209,7 @@ functions: std.AutoArrayHashMapUnmanaged(FunctionImport.Resolution, void) = .emp
206209/// Tracks the value at the end of prelink, at which point `functions`
207210/// contains only object file functions, and nothing from the Zcu yet.
208211functions_end_prelink: u32 = 0,
209/// Immutable after prelink. The undefined functions coming only from all object files.
210/// The Zcu must satisfy these.
211function_imports_init_keys: []String = &.{},
212function_imports_init_vals: []FunctionImportId = &.{},
213/// Initialized as copy of `function_imports_init_keys` and
214/// `function_import_init_vals`; entries are deleted as they are satisfied by
215/// the Zcu.
212/// Entries are deleted as they are satisfied by the Zcu.
216213function_imports: std.AutoArrayHashMapUnmanaged(String, FunctionImportId) = .empty,
217214
218215/// Ordered list of non-import globals that will appear in the final binary.
......@@ -221,8 +218,6 @@ globals: std.AutoArrayHashMapUnmanaged(GlobalImport.Resolution, void) = .empty,
221218/// Tracks the value at the end of prelink, at which point `globals`
222219/// contains only object file globals, and nothing from the Zcu yet.
223220globals_end_prelink: u32 = 0,
224global_imports_init_keys: []String = &.{},
225global_imports_init_vals: []GlobalImportId = &.{},
226221global_imports: std.AutoArrayHashMapUnmanaged(String, GlobalImportId) = .empty,
227222
228223/// Ordered list of non-import tables that will appear in the final binary.
......@@ -233,11 +228,10 @@ table_imports: std.AutoArrayHashMapUnmanaged(String, TableImport.Index) = .empty
233228/// Ordered list of data segments that will appear in the final binary.
234229/// When sorted, to-be-merged segments will be made adjacent.
235230/// Values are offset relative to segment start.
236data_segments: std.AutoArrayHashMapUnmanaged(Wasm.DataSegment.Id, u32) = .empty,
231data_segments: std.AutoArrayHashMapUnmanaged(Wasm.DataSegment.Id, void) = .empty,
237232
238233error_name_table_ref_count: u32 = 0,
239234
240any_exports_updated: bool = true,
241235/// Set to true if any `GLOBAL_INDEX` relocation is encountered with
242236/// `SymbolFlags.tls` set to true. This is for objects only; final
243237/// value must be this OR'd with the same logic for zig functions
......@@ -313,7 +307,7 @@ pub const GlobalExport = extern struct {
313307 global_index: GlobalIndex,
314308};
315309
316/// 0. Index into `function_imports`
310/// 0. Index into `Flush.function_imports`
317311/// 1. Index into `functions`.
318312///
319313/// Note that function_imports indexes are subject to swap removals during
......@@ -529,13 +523,6 @@ pub const SymbolFlags = packed struct(u32) {
529523 return flags.exported;
530524 }
531525
532 pub fn requiresImport(flags: SymbolFlags, is_data: bool) bool {
533 if (is_data) return false;
534 if (!flags.undefined) return false;
535 if (flags.binding == .weak) return false;
536 return true;
537 }
538
539526 /// Returns the name as how it will be output into the final object
540527 /// file or binary. When `merge` is true, this will return the
541528 /// short name. i.e. ".rodata". When false, it returns the entire name instead.
......@@ -2268,6 +2255,8 @@ pub fn deinit(wasm: *Wasm) void {
22682255
22692256 wasm.params_scratch.deinit(gpa);
22702257 wasm.returns_scratch.deinit(gpa);
2258
2259 wasm.missing_exports.deinit(gpa);
22712260}
22722261
22732262pub fn updateFunc(wasm: *Wasm, pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, liveness: Liveness) !void {
......@@ -2306,28 +2295,27 @@ pub fn updateNav(wasm: *Wasm, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index
23062295 const gpa = comp.gpa;
23072296 const is_obj = comp.config.output_mode == .Obj;
23082297
2309 const is_extern, const nav_init = switch (ip.indexToKey(nav.status.resolved.val)) {
2310 .func => return,
2311 .@"extern" => .{ true, .none },
2312 .variable => |variable| .{ false, variable.init },
2313 else => .{ false, nav.status.resolved.val },
2298 const nav_init = switch (ip.indexToKey(nav.status.resolved.val)) {
2299 .func => return, // global const which is a function alias
2300 .@"extern" => {
2301 if (is_obj) {
2302 assert(!wasm.navs_obj.contains(nav_index));
2303 } else {
2304 assert(!wasm.navs_exe.contains(nav_index));
2305 }
2306 try wasm.imports.put(gpa, nav_index, {});
2307 return;
2308 },
2309 .variable => |variable| variable.init,
2310 else => nav.status.resolved.val,
23142311 };
2315 if (is_extern) {
2316 try wasm.imports.put(gpa, nav_index, {});
2317 if (is_obj) {
2318 if (wasm.navs_obj.swapRemove(nav_index)) @panic("TODO reclaim resources");
2319 } else {
2320 if (wasm.navs_exe.swapRemove(nav_index)) @panic("TODO reclaim resources");
2321 }
2322 return;
2323 }
2324 _ = wasm.imports.swapRemove(nav_index);
2312 assert(!wasm.imports.contains(nav_index));
23252313
23262314 if (nav_init != .none and !Value.fromInterned(nav_init).typeOf(zcu).hasRuntimeBits(zcu)) {
23272315 if (is_obj) {
2328 if (wasm.navs_obj.swapRemove(nav_index)) @panic("TODO reclaim resources");
2316 assert(!wasm.navs_obj.contains(nav_index));
23292317 } else {
2330 if (wasm.navs_exe.swapRemove(nav_index)) @panic("TODO reclaim resources");
2318 assert(!wasm.navs_exe.contains(nav_index));
23312319 }
23322320 return;
23332321 }
......@@ -2339,9 +2327,7 @@ pub fn updateNav(wasm: *Wasm, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index
23392327 if (is_obj) {
23402328 const gop = try wasm.navs_obj.getOrPut(gpa, nav_index);
23412329 gop.value_ptr.* = zcu_data;
2342 wasm.data_segments.putAssumeCapacity(.pack(wasm, .{
2343 .nav_obj = @enumFromInt(gop.index),
2344 }), @as(u32, undefined));
2330 wasm.data_segments.putAssumeCapacity(.pack(wasm, .{ .nav_obj = @enumFromInt(gop.index) }), {});
23452331 }
23462332
23472333 assert(zcu_data.relocs.len == 0);
......@@ -2351,9 +2337,7 @@ pub fn updateNav(wasm: *Wasm, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index
23512337 .code = zcu_data.code,
23522338 .count = 0,
23532339 };
2354 wasm.data_segments.putAssumeCapacity(.pack(wasm, .{
2355 .nav_exe = @enumFromInt(gop.index),
2356 }), @as(u32, undefined));
2340 wasm.data_segments.putAssumeCapacity(.pack(wasm, .{ .nav_exe = @enumFromInt(gop.index) }), {});
23572341}
23582342
23592343pub fn updateLineNumber(wasm: *Wasm, pt: Zcu.PerThread, ti_id: InternPool.TrackedInst.Index) !void {
......@@ -2380,7 +2364,6 @@ pub fn deleteExport(
23802364 },
23812365 .uav => |uav_index| assert(wasm.uav_exports.swapRemove(.{ .uav_index = uav_index, .name = export_name })),
23822366 }
2383 wasm.any_exports_updated = true;
23842367}
23852368
23862369pub fn updateExports(
......@@ -2409,7 +2392,6 @@ pub fn updateExports(
24092392 .uav => |uav_index| try wasm.uav_exports.put(gpa, .{ .uav_index = uav_index, .name = name }, export_idx),
24102393 }
24112394 }
2412 wasm.any_exports_updated = true;
24132395}
24142396
24152397pub fn loadInput(wasm: *Wasm, input: link.Input) !void {
......@@ -2464,32 +2446,28 @@ pub fn prelink(wasm: *Wasm, prog_node: std.Progress.Node) link.File.FlushError!v
24642446 const gpa = comp.gpa;
24652447 const rdynamic = comp.config.rdynamic;
24662448
2467 {
2468 var missing_exports: std.AutoArrayHashMapUnmanaged(String, void) = .empty;
2469 defer missing_exports.deinit(gpa);
2470 for (wasm.export_symbol_names) |exp_name| {
2471 const exp_name_interned = try wasm.internString(exp_name);
2472 if (wasm.object_function_imports.getPtr(exp_name_interned)) |import| {
2473 if (import.resolution != .unresolved) {
2474 import.flags.exported = true;
2475 continue;
2476 }
2449 assert(wasm.missing_exports.entries.len == 0);
2450 for (wasm.export_symbol_names) |exp_name| {
2451 const exp_name_interned = try wasm.internString(exp_name);
2452 if (wasm.object_function_imports.getPtr(exp_name_interned)) |import| {
2453 if (import.resolution != .unresolved) {
2454 import.flags.exported = true;
2455 continue;
24772456 }
2478 if (wasm.object_global_imports.getPtr(exp_name_interned)) |import| {
2479 if (import.resolution != .unresolved) {
2480 import.flags.exported = true;
2481 continue;
2482 }
2457 }
2458 if (wasm.object_global_imports.getPtr(exp_name_interned)) |import| {
2459 if (import.resolution != .unresolved) {
2460 import.flags.exported = true;
2461 continue;
24832462 }
2484 if (wasm.object_table_imports.getPtr(exp_name_interned)) |import| {
2485 if (import.resolution != .unresolved) {
2486 import.flags.exported = true;
2487 continue;
2488 }
2463 }
2464 if (wasm.object_table_imports.getPtr(exp_name_interned)) |import| {
2465 if (import.resolution != .unresolved) {
2466 import.flags.exported = true;
2467 continue;
24892468 }
2490 try missing_exports.put(gpa, exp_name_interned, {});
24912469 }
2492 wasm.missing_exports_init = try gpa.dupe(String, missing_exports.keys());
2470 try wasm.missing_exports.put(gpa, exp_name_interned, {});
24932471 }
24942472
24952473 if (wasm.entry_name.unwrap()) |entry_name| {
......@@ -2515,8 +2493,6 @@ pub fn prelink(wasm: *Wasm, prog_node: std.Progress.Node) link.File.FlushError!v
25152493 }
25162494 }
25172495 wasm.functions_end_prelink = @intCast(wasm.functions.entries.len);
2518 wasm.function_imports_init_keys = try gpa.dupe(String, wasm.function_imports.keys());
2519 wasm.function_imports_init_vals = try gpa.dupe(FunctionImportId, wasm.function_imports.values());
25202496 wasm.function_exports_len = @intCast(wasm.function_exports.items.len);
25212497
25222498 for (wasm.object_global_imports.keys(), wasm.object_global_imports.values(), 0..) |name, *import, i| {
......@@ -2525,8 +2501,6 @@ pub fn prelink(wasm: *Wasm, prog_node: std.Progress.Node) link.File.FlushError!v
25252501 }
25262502 }
25272503 wasm.globals_end_prelink = @intCast(wasm.globals.entries.len);
2528 wasm.global_imports_init_keys = try gpa.dupe(String, wasm.global_imports.keys());
2529 wasm.global_imports_init_vals = try gpa.dupe(GlobalImportId, wasm.global_imports.values());
25302504 wasm.global_exports_len = @intCast(wasm.global_exports.items.len);
25312505
25322506 for (wasm.object_table_imports.keys(), wasm.object_table_imports.values(), 0..) |name, *import, i| {
......@@ -2692,6 +2666,7 @@ pub fn flushModule(
26922666 const comp = wasm.base.comp;
26932667 const use_lld = build_options.have_llvm and comp.config.use_lld;
26942668 const diags = &comp.link_diags;
2669 const gpa = comp.gpa;
26952670
26962671 if (wasm.llvm_object) |llvm_object| {
26972672 try wasm.base.emitLlvmObject(arena, llvm_object, prog_node);
......@@ -2728,6 +2703,10 @@ pub fn flushModule(
27282703 defer wasm.data_segments.shrinkRetainingCapacity(data_segments_end_zcu);
27292704
27302705 wasm.flush_buffer.clear();
2706 try wasm.flush_buffer.missing_exports.reinit(gpa, wasm.missing_exports.keys(), &.{});
2707 try wasm.flush_buffer.data_segments.reinit(gpa, wasm.data_segments.keys(), &.{});
2708 try wasm.flush_buffer.function_imports.reinit(gpa, wasm.function_imports.keys(), wasm.function_imports.values());
2709 try wasm.flush_buffer.global_imports.reinit(gpa, wasm.global_imports.keys(), wasm.global_imports.values());
27312710
27322711 return wasm.flush_buffer.finish(wasm) catch |err| switch (err) {
27332712 error.OutOfMemory => return error.OutOfMemory,
......@@ -3330,7 +3309,7 @@ pub fn refUavObj(wasm: *Wasm, pt: Zcu.PerThread, ip_index: InternPool.Index) !Ua
33303309 const gop = try wasm.uavs_obj.getOrPut(gpa, ip_index);
33313310 if (!gop.found_existing) gop.value_ptr.* = try lowerZcuData(wasm, pt, ip_index);
33323311 const uav_index: UavsObjIndex = @enumFromInt(gop.index);
3333 try wasm.data_segments.put(gpa, .pack(wasm, .{ .uav_obj = uav_index }), @as(u32, undefined));
3312 try wasm.data_segments.put(gpa, .pack(wasm, .{ .uav_obj = uav_index }), {});
33343313 return uav_index;
33353314}
33363315
......@@ -3349,34 +3328,34 @@ pub fn refUavExe(wasm: *Wasm, pt: Zcu.PerThread, ip_index: InternPool.Index) !Ua
33493328 };
33503329 }
33513330 const uav_index: UavsExeIndex = @enumFromInt(gop.index);
3352 try wasm.data_segments.put(gpa, .pack(wasm, .{ .uav_exe = uav_index }), @as(u32, undefined));
3331 try wasm.data_segments.put(gpa, .pack(wasm, .{ .uav_exe = uav_index }), {});
33533332 return uav_index;
33543333}
33553334
3356/// Asserts it is called after `Wasm.data_segments` is fully populated and sorted.
3335/// Asserts it is called after `Flush.data_segments` is fully populated and sorted.
33573336pub fn uavAddr(wasm: *Wasm, uav_index: UavsExeIndex) u32 {
33583337 assert(wasm.flush_buffer.memory_layout_finished);
33593338 const comp = wasm.base.comp;
33603339 assert(comp.config.output_mode != .Obj);
33613340 const ds_id: DataSegment.Id = .pack(wasm, .{ .uav_exe = uav_index });
3362 return wasm.data_segments.get(ds_id).?;
3341 return wasm.flush_buffer.data_segments.get(ds_id).?;
33633342}
33643343
3365/// Asserts it is called after `Wasm.data_segments` is fully populated and sorted.
3344/// Asserts it is called after `Flush.data_segments` is fully populated and sorted.
33663345pub fn navAddr(wasm: *Wasm, nav_index: InternPool.Nav.Index) u32 {
33673346 assert(wasm.flush_buffer.memory_layout_finished);
33683347 const comp = wasm.base.comp;
33693348 assert(comp.config.output_mode != .Obj);
33703349 const ds_id: DataSegment.Id = .pack(wasm, .{ .nav_exe = @enumFromInt(wasm.navs_exe.getIndex(nav_index).?) });
3371 return wasm.data_segments.get(ds_id).?;
3350 return wasm.flush_buffer.data_segments.get(ds_id).?;
33723351}
33733352
3374/// Asserts it is called after `Wasm.data_segments` is fully populated and sorted.
3353/// Asserts it is called after `Flush.data_segments` is fully populated and sorted.
33753354pub fn errorNameTableAddr(wasm: *Wasm) u32 {
33763355 assert(wasm.flush_buffer.memory_layout_finished);
33773356 const comp = wasm.base.comp;
33783357 assert(comp.config.output_mode != .Obj);
3379 return wasm.data_segments.get(.__zig_error_name_table).?;
3358 return wasm.flush_buffer.data_segments.get(.__zig_error_name_table).?;
33803359}
33813360
33823361fn convertZcuFnType(
src/link/Wasm/Flush.zig+19-21
......@@ -19,12 +19,15 @@ const leb = std.leb;
1919const log = std.log.scoped(.link);
2020const assert = std.debug.assert;
2121
22data_segments: std.AutoArrayHashMapUnmanaged(Wasm.DataSegment.Id, u32) = .empty,
2223/// Each time a `data_segment` offset equals zero it indicates a new group, and
2324/// the next element in this array will contain the total merged segment size.
2425data_segment_groups: std.ArrayListUnmanaged(u32) = .empty,
2526
2627binary_bytes: std.ArrayListUnmanaged(u8) = .empty,
2728missing_exports: std.AutoArrayHashMapUnmanaged(String, void) = .empty,
29function_imports: std.AutoArrayHashMapUnmanaged(String, Wasm.FunctionImportId) = .empty,
30global_imports: std.AutoArrayHashMapUnmanaged(String, Wasm.GlobalImportId) = .empty,
2831
2932indirect_function_table: std.AutoArrayHashMapUnmanaged(Wasm.OutputFunctionIndex, u32) = .empty,
3033
......@@ -39,8 +42,12 @@ pub fn clear(f: *Flush) void {
3942}
4043
4144pub fn deinit(f: *Flush, gpa: Allocator) void {
42 f.binary_bytes.deinit(gpa);
45 f.data_segments.deinit(gpa);
4346 f.data_segment_groups.deinit(gpa);
47 f.binary_bytes.deinit(gpa);
48 f.missing_exports.deinit(gpa);
49 f.function_imports.deinit(gpa);
50 f.global_imports.deinit(gpa);
4451 f.indirect_function_table.deinit(gpa);
4552 f.* = undefined;
4653}
......@@ -58,18 +65,9 @@ pub fn finish(f: *Flush, wasm: *Wasm) !void {
5865 const zcu = wasm.base.comp.zcu.?;
5966 const ip: *const InternPool = &zcu.intern_pool; // No mutations allowed!
6067
61 if (wasm.any_exports_updated) {
62 wasm.any_exports_updated = false;
63
64 wasm.function_exports.shrinkRetainingCapacity(wasm.function_exports_len);
65 wasm.global_exports.shrinkRetainingCapacity(wasm.global_exports_len);
66
68 {
6769 const entry_name = if (wasm.entry_resolution.isNavOrUnresolved(wasm)) wasm.entry_name else .none;
6870
69 try f.missing_exports.reinit(gpa, wasm.missing_exports_init, &.{});
70 try wasm.function_imports.reinit(gpa, wasm.function_imports_init_keys, wasm.function_imports_init_vals);
71 try wasm.global_imports.reinit(gpa, wasm.global_imports_init_keys, wasm.global_imports_init_vals);
72
7371 for (wasm.nav_exports.keys()) |*nav_export| {
7472 if (ip.isFunctionType(ip.getNav(nav_export.nav_index).typeOf(ip))) {
7573 log.debug("flush export '{s}' nav={d}", .{ nav_export.name.slice(wasm), nav_export.nav_index });
......@@ -134,17 +132,17 @@ pub fn finish(f: *Flush, wasm: *Wasm) !void {
134132
135133 // Merge and order the data segments. Depends on garbage collection so that
136134 // unused segments can be omitted.
137 try wasm.data_segments.ensureUnusedCapacity(gpa, wasm.object_data_segments.items.len + 1);
135 try f.data_segments.ensureUnusedCapacity(gpa, wasm.object_data_segments.items.len + 1);
138136 for (wasm.object_data_segments.items, 0..) |*ds, i| {
139137 if (!ds.flags.alive) continue;
140138 const data_segment_index: Wasm.ObjectDataSegmentIndex = @enumFromInt(i);
141139 any_passive_inits = any_passive_inits or ds.flags.is_passive or (import_memory and !wasm.isBss(ds.name));
142 wasm.data_segments.putAssumeCapacityNoClobber(.pack(wasm, .{
140 f.data_segments.putAssumeCapacityNoClobber(.pack(wasm, .{
143141 .object = data_segment_index,
144142 }), @as(u32, undefined));
145143 }
146144 if (wasm.error_name_table_ref_count > 0) {
147 wasm.data_segments.putAssumeCapacity(.__zig_error_name_table, @as(u32, undefined));
145 f.data_segments.putAssumeCapacity(.__zig_error_name_table, @as(u32, undefined));
148146 }
149147
150148 try wasm.functions.ensureUnusedCapacity(gpa, 3);
......@@ -223,9 +221,9 @@ pub fn finish(f: *Flush, wasm: *Wasm) !void {
223221 return @intFromEnum(lhs_segment) < @intFromEnum(rhs_segment);
224222 }
225223 };
226 wasm.data_segments.sortUnstable(@as(Sort, .{
224 f.data_segments.sortUnstable(@as(Sort, .{
227225 .wasm = wasm,
228 .segments = wasm.data_segments.keys(),
226 .segments = f.data_segments.keys(),
229227 }));
230228
231229 const page_size = std.wasm.page_size; // 64kb
......@@ -260,8 +258,8 @@ pub fn finish(f: *Flush, wasm: *Wasm) !void {
260258 virtual_addrs.stack_pointer = @intCast(memory_ptr);
261259 }
262260
263 const segment_ids = wasm.data_segments.keys();
264 const segment_offsets = wasm.data_segments.values();
261 const segment_ids = f.data_segments.keys();
262 const segment_offsets = f.data_segments.values();
265263 assert(f.data_segment_groups.items.len == 0);
266264 {
267265 var seen_tls: enum { before, during, after } = .before;
......@@ -703,11 +701,11 @@ pub fn finish(f: *Flush, wasm: *Wasm) !void {
703701 // try wasm.emitCodeRelocations(binary_bytes, code_index, symbol_table);
704702 //}
705703 //if (data_section_index) |data_index| {
706 // if (wasm.data_segments.count() > 0)
704 // if (f.data_segments.count() > 0)
707705 // try wasm.emitDataRelocations(binary_bytes, data_index, symbol_table);
708706 //}
709707 } else if (comp.config.debug_format != .strip) {
710 try emitNameSection(wasm, &wasm.data_segments, binary_bytes);
708 try emitNameSection(wasm, &f.data_segments, binary_bytes);
711709 }
712710
713711 if (comp.config.debug_format != .strip) {
......@@ -993,7 +991,7 @@ fn emitProducerSection(gpa: Allocator, binary_bytes: *std.ArrayListUnmanaged(u8)
993991// var count: u32 = 0;
994992// // for each atom, we calculate the uleb size and append that
995993// var size_offset: u32 = 5; // account for code section size leb128
996// for (wasm.data_segments.values()) |segment_index| {
994// for (f.data_segments.values()) |segment_index| {
997995// var atom: *Atom = wasm.atoms.get(segment_index).?.ptr(wasm);
998996// while (true) {
999997// size_offset += getUleb128Size(atom.code.len);