authorgravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2022-05-25 22:35:11+02:00
committergravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2022-06-24 08:12:17+02:00
logcb28fc2e63dea2902fda21b7738aa93eaf4a2ea0
tree4dcbaa9735c384a3a6012ba8c33f35bfdf979489
parent0606fbbc4bb7a006e2dcbf151c9505c920b735be

wasm-linker: Resolve symbols from archives

Lazily load object files by default, and only load the object file when an unresolved symbol has been found within an archive.

2 files changed, 135 insertions(+), 15 deletions(-)

src/link/Wasm.zig+105-12
......@@ -29,6 +29,7 @@ const Air = @import("../Air.zig");
2929const Liveness = @import("../Liveness.zig");
3030const Symbol = @import("Wasm/Symbol.zig");
3131const Object = @import("Wasm/Object.zig");
32const Archive = @import("Wasm/Archive.zig");
3233const types = @import("Wasm/types.zig");
3334
3435pub const base_tag = link.File.Tag.wasm;
......@@ -125,6 +126,10 @@ function_table: std.AutoHashMapUnmanaged(SymbolLoc, u32) = .{},
125126
126127/// All object files and their data which are linked into the final binary
127128objects: std.ArrayListUnmanaged(Object) = .{},
129/// All archive files that are lazy loaded.
130/// e.g. when an undefined symbol references a symbol from the archive.
131archives: std.ArrayListUnmanaged(Archive) = .{},
132
128133/// A map of global names (read: offset into string table) to their symbol location
129134globals: std.AutoHashMapUnmanaged(u32, SymbolLoc) = .{},
130135/// Maps discarded symbols and their positions to the location of the symbol
......@@ -133,6 +138,8 @@ discarded: std.AutoHashMapUnmanaged(SymbolLoc, SymbolLoc) = .{},
133138/// List of all symbol locations which have been resolved by the linker and will be emit
134139/// into the final binary.
135140resolved_symbols: std.AutoArrayHashMapUnmanaged(SymbolLoc, void) = .{},
141/// Symbols that remain undefined after symbol resolution.
142undefs: std.StringArrayHashMapUnmanaged(SymbolLoc) = .{},
136143/// Maps a symbol's location to an atom. This can be used to find meta
137144/// data of a symbol, such as its size, or its offset to perform a relocation.
138145/// Undefined (and synthetic) symbols do not have an Atom and therefore cannot be mapped.
......@@ -359,6 +366,7 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Wasm {
359366fn parseInputFiles(self: *Wasm, files: []const []const u8) !void {
360367 for (files) |path| {
361368 if (try self.parseObjectFile(path)) continue;
369 if (try self.parseArchive(path, false)) continue; // load archives lazily
362370 log.warn("Unexpected file format at path: '{s}'", .{path});
363371 }
364372}
......@@ -371,10 +379,7 @@ fn parseObjectFile(self: *Wasm, path: []const u8) !bool {
371379 errdefer file.close();
372380
373381 var object = Object.create(self.base.allocator, file, path) catch |err| switch (err) {
374 error.InvalidMagicByte, error.NotObjectFile => {
375 log.warn("Self hosted linker does not support non-object file parsing: {s}", .{@errorName(err)});
376 return false;
377 },
382 error.InvalidMagicByte, error.NotObjectFile => return false,
378383 else => |e| return e,
379384 };
380385 errdefer object.deinit(self.base.allocator);
......@@ -382,6 +387,56 @@ fn parseObjectFile(self: *Wasm, path: []const u8) !bool {
382387 return true;
383388}
384389
390/// Parses an archive file and will then parse each object file
391/// that was found in the archive file.
392/// Returns false when the file is not an archive file.
393/// May return an error instead when parsing failed.
394///
395/// When `force_load` is `true`, it will for link all object files in the archive.
396/// When false, it will only link with object files that contain symbols that
397/// are referenced by other object files or Zig code.
398fn parseArchive(self: *Wasm, path: []const u8, force_load: bool) !bool {
399 const file = try fs.cwd().openFile(path, .{});
400 errdefer file.close();
401
402 var archive: Archive = .{
403 .file = file,
404 .name = path,
405 };
406 archive.parse(self.base.allocator) catch |err| switch (err) {
407 error.EndOfStream, error.NotArchive => {
408 archive.deinit(self.base.allocator);
409 return false;
410 },
411 else => |e| return e,
412 };
413
414 if (!force_load) {
415 errdefer archive.deinit(self.base.allocator);
416 try self.archives.append(self.base.allocator, archive);
417 return true;
418 }
419 defer archive.deinit(self.base.allocator);
420
421 // In this case we must force link all embedded object files within the archive
422 // We loop over all symbols, and then group them by offset as the offset
423 // notates where the object file starts.
424 var offsets = std.AutoArrayHashMap(u32, void).init(self.base.allocator);
425 defer offsets.deinit();
426 for (archive.toc.values()) |symbol_offsets| {
427 for (symbol_offsets.items) |sym_offset| {
428 try offsets.put(sym_offset, {});
429 }
430 }
431
432 for (offsets.keys()) |file_offset| {
433 const object = try self.objects.addOne(self.base.allocator);
434 object.* = try archive.parseObject(self.base.allocator, file_offset);
435 }
436
437 return true;
438}
439
385440fn resolveSymbolsInObject(self: *Wasm, object_index: u16) !void {
386441 const object: Object = self.objects.items[object_index];
387442 log.debug("Resolving symbols in object: '{s}'", .{object.name});
......@@ -414,6 +469,10 @@ fn resolveSymbolsInObject(self: *Wasm, object_index: u16) !void {
414469 if (!maybe_existing.found_existing) {
415470 maybe_existing.value_ptr.* = location;
416471 try self.resolved_symbols.putNoClobber(self.base.allocator, location, {});
472
473 if (symbol.isUndefined()) {
474 try self.undefs.putNoClobber(self.base.allocator, sym_name, location);
475 }
417476 continue;
418477 }
419478
......@@ -456,6 +515,42 @@ fn resolveSymbolsInObject(self: *Wasm, object_index: u16) !void {
456515 try self.globals.put(self.base.allocator, sym_name_index, location);
457516 try self.resolved_symbols.put(self.base.allocator, location, {});
458517 assert(self.resolved_symbols.swapRemove(existing_loc));
518 if (existing_sym.isUndefined()) {
519 // ensure order remains intact in case we later
520 // resolve symbols again in a loop
521 assert(self.undefs.orderedRemove(sym_name));
522 }
523 }
524}
525
526fn resolveSymbolsInArchives(self: *Wasm) !void {
527 if (self.archives.items.len == 0) return;
528
529 log.debug("Resolving symbols in archives", .{});
530 var index: u32 = 0;
531 undef_loop: while (index < self.undefs.count()) {
532 const undef_sym_loc = self.undefs.values()[index];
533 const sym_name = undef_sym_loc.getName(self);
534
535 for (self.archives.items) |archive| {
536 const offset = archive.toc.get(sym_name) orelse {
537 // symbol does not exist in this archive
538 continue;
539 };
540
541 // Symbol is found in unparsed object file within current archive.
542 // Parse object and and resolve symbols again before we check remaining
543 // undefined symbols.
544 const object_file_index = @intCast(u16, self.objects.items.len);
545 const object = try self.objects.addOne(self.base.allocator);
546 object.* = try archive.parseObject(self.base.allocator, offset.items[0]);
547 try self.resolveSymbolsInObject(object_file_index);
548
549 // continue loop for any remaining undefined symbols that still exist
550 // after resolving last object file
551 continue :undef_loop;
552 }
553 index += 1;
459554 }
460555}
461556
......@@ -789,6 +884,7 @@ pub fn getGlobalSymbol(self: *Wasm, name: []const u8) !u32 {
789884 self.symbols.items[sym_index] = symbol;
790885 gop.value_ptr.* = .{ .index = sym_index, .file = null };
791886 try self.resolved_symbols.put(self.base.allocator, gop.value_ptr.*, {});
887 try self.undefs.putNoClobber(self.base.allocator, name, gop.value_ptr.*);
792888 return sym_index;
793889}
794890
......@@ -1017,6 +1113,7 @@ pub fn addOrUpdateImport(
10171113 const loc: SymbolLoc = .{ .file = null, .index = symbol_index };
10181114 global_gop.value_ptr.* = loc;
10191115 try self.resolved_symbols.put(self.base.allocator, loc, {});
1116 try self.undefs.putNoClobber(self.base.allocator, name, loc);
10201117 }
10211118
10221119 if (type_index) |ty_index| {
......@@ -1298,7 +1395,7 @@ fn mergeTypes(self: *Wasm) !void {
12981395 // type inserted. If we do this for the same function multiple times,
12991396 // it will be overwritten with the incorrect type.
13001397 var dirty = std.AutoHashMap(u32, void).init(self.base.allocator);
1301 try dirty.ensureUnusedCapacity(@intCast(u32, self.functions.count()) + self.imported_functions_count);
1398 try dirty.ensureUnusedCapacity(@intCast(u32, self.functions.count()));
13021399 defer dirty.deinit();
13031400
13041401 for (self.resolved_symbols.keys()) |sym_loc| {
......@@ -1313,22 +1410,17 @@ fn mergeTypes(self: *Wasm) !void {
13131410 continue;
13141411 }
13151412
1316 if (dirty.contains(symbol.index)) {
1317 continue; // We already added the type of this symbol
1318 }
1319
13201413 if (symbol.isUndefined()) {
13211414 log.debug("Adding type from extern function '{s}'", .{sym_loc.getName(self)});
13221415 const import: *types.Import = self.imports.getPtr(sym_loc).?;
13231416 const original_type = object.func_types[import.kind.function];
13241417 import.kind.function = try self.putOrGetFuncType(original_type);
1325 } else {
1418 } else if (!dirty.contains(symbol.index)) {
13261419 log.debug("Adding type from function '{s}'", .{sym_loc.getName(self)});
13271420 const func = &self.functions.values()[symbol.index - self.imported_functions_count];
13281421 func.type_index = try self.putOrGetFuncType(object.func_types[func.type_index]);
1422 dirty.putAssumeCapacityNoClobber(symbol.index, {});
13291423 }
1330
1331 dirty.putAssumeCapacityNoClobber(symbol.index, {});
13321424 }
13331425 log.debug("Completed merging and deduplicating types. Total count: ({d})", .{self.func_types.items.len});
13341426}
......@@ -1747,6 +1839,7 @@ pub fn flushModule(self: *Wasm, comp: *Compilation, prog_node: *std.Progress.Nod
17471839 while (object_index < self.objects.items.len) : (object_index += 1) {
17481840 try self.resolveSymbolsInObject(object_index);
17491841 }
1842 try self.resolveSymbolsInArchives();
17501843
17511844 // When we finish/error we reset the state of the linker
17521845 // So we can rebuild the binary file on each incremental update
src/link/Wasm/Archive.zig+30-3
......@@ -113,8 +113,6 @@ pub fn parse(self: *Archive, allocator: Allocator) !void {
113113 return error.NotArchive;
114114 }
115115
116 log.debug("parsing archive '{s}' at '{s}'", .{ std.mem.sliceTo(&self.header.ar_name, 0), self.name });
117
118116 try self.parseTableOfContents(allocator, reader);
119117}
120118
......@@ -175,6 +173,35 @@ fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !
175173 gop.value_ptr.* = .{};
176174 }
177175 try gop.value_ptr.append(allocator, symbol_positions[gop.index]);
178 log.debug(" parsed symbol '{s}' for position {d}", .{ string, symbol_positions[gop.index] });
179176 }
180177}
178
179/// From a given file offset, starts reading for a file header.
180/// When found, parses the object file into an `Object` and returns it.
181pub fn parseObject(self: Archive, allocator: Allocator, file_offset: u32) !Object {
182 try self.file.seekTo(file_offset);
183 const reader = self.file.reader();
184 const header = try reader.readStruct(ar_hdr);
185
186 if (!mem.eql(u8, &header.ar_fmag, ARFMAG)) {
187 log.err("invalid header delimiter: expected '{s}', found '{s}'", .{ ARFMAG, header.ar_fmag });
188 return error.MalformedArchive;
189 }
190
191 const object_name = try parseName(allocator, header, reader);
192 defer allocator.free(object_name);
193
194 const name = name: {
195 if (object_name.len == 0) {
196 break :name try std.fmt.allocPrint(allocator, "{s}.o", .{self.name});
197 }
198 const base_path = std.fs.path.dirname(self.name);
199 break :name try std.fmt.allocPrint(allocator, "{s}/{s}.o", .{ base_path, object_name });
200 };
201
202 log.debug(" parsing object file '{s}' from archive\n", .{name});
203 const object_file = try std.fs.cwd().openFile(name, .{});
204 errdefer object_file.close();
205
206 return Object.create(allocator, object_file, name);
207}