authorgravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2022-05-25 22:35:11+02:00
committergravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2022-06-24 08:12:17+02:00
logcb28fc2e63dea2902fda21b7738aa93eaf4a2ea0
tree4dcbaa9735c384a3a6012ba8c33f35bfdf979489
parent0606fbbc4bb7a006e2dcbf151c9505c920b735be

wasm-linker: Resolve symbols from archives

Lazily load object files by default, and only load the object file when an unresolved symbol has been found within an archive.

2 files changed, 135 insertions(+), 15 deletions(-)

src/link/Wasm.zig+105-12
...@@ -29,6 +29,7 @@ const Air = @import("../Air.zig");...@@ -29,6 +29,7 @@ const Air = @import("../Air.zig");
29const Liveness = @import("../Liveness.zig");29const Liveness = @import("../Liveness.zig");
30const Symbol = @import("Wasm/Symbol.zig");30const Symbol = @import("Wasm/Symbol.zig");
31const Object = @import("Wasm/Object.zig");31const Object = @import("Wasm/Object.zig");
32const Archive = @import("Wasm/Archive.zig");
32const types = @import("Wasm/types.zig");33const types = @import("Wasm/types.zig");
3334
34pub const base_tag = link.File.Tag.wasm;35pub const base_tag = link.File.Tag.wasm;
...@@ -125,6 +126,10 @@ function_table: std.AutoHashMapUnmanaged(SymbolLoc, u32) = .{},...@@ -125,6 +126,10 @@ function_table: std.AutoHashMapUnmanaged(SymbolLoc, u32) = .{},
125126
126/// All object files and their data which are linked into the final binary127/// All object files and their data which are linked into the final binary
127objects: std.ArrayListUnmanaged(Object) = .{},128objects: std.ArrayListUnmanaged(Object) = .{},
129/// All archive files that are lazy loaded.
130/// e.g. when an undefined symbol references a symbol from the archive.
131archives: std.ArrayListUnmanaged(Archive) = .{},
132
128/// A map of global names (read: offset into string table) to their symbol location133/// A map of global names (read: offset into string table) to their symbol location
129globals: std.AutoHashMapUnmanaged(u32, SymbolLoc) = .{},134globals: std.AutoHashMapUnmanaged(u32, SymbolLoc) = .{},
130/// Maps discarded symbols and their positions to the location of the symbol135/// Maps discarded symbols and their positions to the location of the symbol
...@@ -133,6 +138,8 @@ discarded: std.AutoHashMapUnmanaged(SymbolLoc, SymbolLoc) = .{},...@@ -133,6 +138,8 @@ discarded: std.AutoHashMapUnmanaged(SymbolLoc, SymbolLoc) = .{},
133/// List of all symbol locations which have been resolved by the linker and will be emit138/// List of all symbol locations which have been resolved by the linker and will be emit
134/// into the final binary.139/// into the final binary.
135resolved_symbols: std.AutoArrayHashMapUnmanaged(SymbolLoc, void) = .{},140resolved_symbols: std.AutoArrayHashMapUnmanaged(SymbolLoc, void) = .{},
141/// Symbols that remain undefined after symbol resolution.
142undefs: std.StringArrayHashMapUnmanaged(SymbolLoc) = .{},
136/// Maps a symbol's location to an atom. This can be used to find meta143/// Maps a symbol's location to an atom. This can be used to find meta
137/// data of a symbol, such as its size, or its offset to perform a relocation.144/// data of a symbol, such as its size, or its offset to perform a relocation.
138/// Undefined (and synthetic) symbols do not have an Atom and therefore cannot be mapped.145/// Undefined (and synthetic) symbols do not have an Atom and therefore cannot be mapped.
...@@ -359,6 +366,7 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Wasm {...@@ -359,6 +366,7 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Wasm {
359fn parseInputFiles(self: *Wasm, files: []const []const u8) !void {366fn parseInputFiles(self: *Wasm, files: []const []const u8) !void {
360 for (files) |path| {367 for (files) |path| {
361 if (try self.parseObjectFile(path)) continue;368 if (try self.parseObjectFile(path)) continue;
369 if (try self.parseArchive(path, false)) continue; // load archives lazily
362 log.warn("Unexpected file format at path: '{s}'", .{path});370 log.warn("Unexpected file format at path: '{s}'", .{path});
363 }371 }
364}372}
...@@ -371,10 +379,7 @@ fn parseObjectFile(self: *Wasm, path: []const u8) !bool {...@@ -371,10 +379,7 @@ fn parseObjectFile(self: *Wasm, path: []const u8) !bool {
371 errdefer file.close();379 errdefer file.close();
372380
373 var object = Object.create(self.base.allocator, file, path) catch |err| switch (err) {381 var object = Object.create(self.base.allocator, file, path) catch |err| switch (err) {
374 error.InvalidMagicByte, error.NotObjectFile => {382 error.InvalidMagicByte, error.NotObjectFile => return false,
375 log.warn("Self hosted linker does not support non-object file parsing: {s}", .{@errorName(err)});
376 return false;
377 },
378 else => |e| return e,383 else => |e| return e,
379 };384 };
380 errdefer object.deinit(self.base.allocator);385 errdefer object.deinit(self.base.allocator);
...@@ -382,6 +387,56 @@ fn parseObjectFile(self: *Wasm, path: []const u8) !bool {...@@ -382,6 +387,56 @@ fn parseObjectFile(self: *Wasm, path: []const u8) !bool {
382 return true;387 return true;
383}388}
384389
390/// Parses an archive file and will then parse each object file
391/// that was found in the archive file.
392/// Returns false when the file is not an archive file.
393/// May return an error instead when parsing failed.
394///
395/// When `force_load` is `true`, it will for link all object files in the archive.
396/// When false, it will only link with object files that contain symbols that
397/// are referenced by other object files or Zig code.
398fn parseArchive(self: *Wasm, path: []const u8, force_load: bool) !bool {
399 const file = try fs.cwd().openFile(path, .{});
400 errdefer file.close();
401
402 var archive: Archive = .{
403 .file = file,
404 .name = path,
405 };
406 archive.parse(self.base.allocator) catch |err| switch (err) {
407 error.EndOfStream, error.NotArchive => {
408 archive.deinit(self.base.allocator);
409 return false;
410 },
411 else => |e| return e,
412 };
413
414 if (!force_load) {
415 errdefer archive.deinit(self.base.allocator);
416 try self.archives.append(self.base.allocator, archive);
417 return true;
418 }
419 defer archive.deinit(self.base.allocator);
420
421 // In this case we must force link all embedded object files within the archive
422 // We loop over all symbols, and then group them by offset as the offset
423 // notates where the object file starts.
424 var offsets = std.AutoArrayHashMap(u32, void).init(self.base.allocator);
425 defer offsets.deinit();
426 for (archive.toc.values()) |symbol_offsets| {
427 for (symbol_offsets.items) |sym_offset| {
428 try offsets.put(sym_offset, {});
429 }
430 }
431
432 for (offsets.keys()) |file_offset| {
433 const object = try self.objects.addOne(self.base.allocator);
434 object.* = try archive.parseObject(self.base.allocator, file_offset);
435 }
436
437 return true;
438}
439
385fn resolveSymbolsInObject(self: *Wasm, object_index: u16) !void {440fn resolveSymbolsInObject(self: *Wasm, object_index: u16) !void {
386 const object: Object = self.objects.items[object_index];441 const object: Object = self.objects.items[object_index];
387 log.debug("Resolving symbols in object: '{s}'", .{object.name});442 log.debug("Resolving symbols in object: '{s}'", .{object.name});
...@@ -414,6 +469,10 @@ fn resolveSymbolsInObject(self: *Wasm, object_index: u16) !void {...@@ -414,6 +469,10 @@ fn resolveSymbolsInObject(self: *Wasm, object_index: u16) !void {
414 if (!maybe_existing.found_existing) {469 if (!maybe_existing.found_existing) {
415 maybe_existing.value_ptr.* = location;470 maybe_existing.value_ptr.* = location;
416 try self.resolved_symbols.putNoClobber(self.base.allocator, location, {});471 try self.resolved_symbols.putNoClobber(self.base.allocator, location, {});
472
473 if (symbol.isUndefined()) {
474 try self.undefs.putNoClobber(self.base.allocator, sym_name, location);
475 }
417 continue;476 continue;
418 }477 }
419478
...@@ -456,6 +515,42 @@ fn resolveSymbolsInObject(self: *Wasm, object_index: u16) !void {...@@ -456,6 +515,42 @@ fn resolveSymbolsInObject(self: *Wasm, object_index: u16) !void {
456 try self.globals.put(self.base.allocator, sym_name_index, location);515 try self.globals.put(self.base.allocator, sym_name_index, location);
457 try self.resolved_symbols.put(self.base.allocator, location, {});516 try self.resolved_symbols.put(self.base.allocator, location, {});
458 assert(self.resolved_symbols.swapRemove(existing_loc));517 assert(self.resolved_symbols.swapRemove(existing_loc));
518 if (existing_sym.isUndefined()) {
519 // ensure order remains intact in case we later
520 // resolve symbols again in a loop
521 assert(self.undefs.orderedRemove(sym_name));
522 }
523 }
524}
525
526fn resolveSymbolsInArchives(self: *Wasm) !void {
527 if (self.archives.items.len == 0) return;
528
529 log.debug("Resolving symbols in archives", .{});
530 var index: u32 = 0;
531 undef_loop: while (index < self.undefs.count()) {
532 const undef_sym_loc = self.undefs.values()[index];
533 const sym_name = undef_sym_loc.getName(self);
534
535 for (self.archives.items) |archive| {
536 const offset = archive.toc.get(sym_name) orelse {
537 // symbol does not exist in this archive
538 continue;
539 };
540
541 // Symbol is found in unparsed object file within current archive.
542 // Parse object and and resolve symbols again before we check remaining
543 // undefined symbols.
544 const object_file_index = @intCast(u16, self.objects.items.len);
545 const object = try self.objects.addOne(self.base.allocator);
546 object.* = try archive.parseObject(self.base.allocator, offset.items[0]);
547 try self.resolveSymbolsInObject(object_file_index);
548
549 // continue loop for any remaining undefined symbols that still exist
550 // after resolving last object file
551 continue :undef_loop;
552 }
553 index += 1;
459 }554 }
460}555}
461556
...@@ -789,6 +884,7 @@ pub fn getGlobalSymbol(self: *Wasm, name: []const u8) !u32 {...@@ -789,6 +884,7 @@ pub fn getGlobalSymbol(self: *Wasm, name: []const u8) !u32 {
789 self.symbols.items[sym_index] = symbol;884 self.symbols.items[sym_index] = symbol;
790 gop.value_ptr.* = .{ .index = sym_index, .file = null };885 gop.value_ptr.* = .{ .index = sym_index, .file = null };
791 try self.resolved_symbols.put(self.base.allocator, gop.value_ptr.*, {});886 try self.resolved_symbols.put(self.base.allocator, gop.value_ptr.*, {});
887 try self.undefs.putNoClobber(self.base.allocator, name, gop.value_ptr.*);
792 return sym_index;888 return sym_index;
793}889}
794890
...@@ -1017,6 +1113,7 @@ pub fn addOrUpdateImport(...@@ -1017,6 +1113,7 @@ pub fn addOrUpdateImport(
1017 const loc: SymbolLoc = .{ .file = null, .index = symbol_index };1113 const loc: SymbolLoc = .{ .file = null, .index = symbol_index };
1018 global_gop.value_ptr.* = loc;1114 global_gop.value_ptr.* = loc;
1019 try self.resolved_symbols.put(self.base.allocator, loc, {});1115 try self.resolved_symbols.put(self.base.allocator, loc, {});
1116 try self.undefs.putNoClobber(self.base.allocator, name, loc);
1020 }1117 }
10211118
1022 if (type_index) |ty_index| {1119 if (type_index) |ty_index| {
...@@ -1298,7 +1395,7 @@ fn mergeTypes(self: *Wasm) !void {...@@ -1298,7 +1395,7 @@ fn mergeTypes(self: *Wasm) !void {
1298 // type inserted. If we do this for the same function multiple times,1395 // type inserted. If we do this for the same function multiple times,
1299 // it will be overwritten with the incorrect type.1396 // it will be overwritten with the incorrect type.
1300 var dirty = std.AutoHashMap(u32, void).init(self.base.allocator);1397 var dirty = std.AutoHashMap(u32, void).init(self.base.allocator);
1301 try dirty.ensureUnusedCapacity(@intCast(u32, self.functions.count()) + self.imported_functions_count);1398 try dirty.ensureUnusedCapacity(@intCast(u32, self.functions.count()));
1302 defer dirty.deinit();1399 defer dirty.deinit();
13031400
1304 for (self.resolved_symbols.keys()) |sym_loc| {1401 for (self.resolved_symbols.keys()) |sym_loc| {
...@@ -1313,22 +1410,17 @@ fn mergeTypes(self: *Wasm) !void {...@@ -1313,22 +1410,17 @@ fn mergeTypes(self: *Wasm) !void {
1313 continue;1410 continue;
1314 }1411 }
13151412
1316 if (dirty.contains(symbol.index)) {
1317 continue; // We already added the type of this symbol
1318 }
1319
1320 if (symbol.isUndefined()) {1413 if (symbol.isUndefined()) {
1321 log.debug("Adding type from extern function '{s}'", .{sym_loc.getName(self)});1414 log.debug("Adding type from extern function '{s}'", .{sym_loc.getName(self)});
1322 const import: *types.Import = self.imports.getPtr(sym_loc).?;1415 const import: *types.Import = self.imports.getPtr(sym_loc).?;
1323 const original_type = object.func_types[import.kind.function];1416 const original_type = object.func_types[import.kind.function];
1324 import.kind.function = try self.putOrGetFuncType(original_type);1417 import.kind.function = try self.putOrGetFuncType(original_type);
1325 } else {1418 } else if (!dirty.contains(symbol.index)) {
1326 log.debug("Adding type from function '{s}'", .{sym_loc.getName(self)});1419 log.debug("Adding type from function '{s}'", .{sym_loc.getName(self)});
1327 const func = &self.functions.values()[symbol.index - self.imported_functions_count];1420 const func = &self.functions.values()[symbol.index - self.imported_functions_count];
1328 func.type_index = try self.putOrGetFuncType(object.func_types[func.type_index]);1421 func.type_index = try self.putOrGetFuncType(object.func_types[func.type_index]);
1422 dirty.putAssumeCapacityNoClobber(symbol.index, {});
1329 }1423 }
1330
1331 dirty.putAssumeCapacityNoClobber(symbol.index, {});
1332 }1424 }
1333 log.debug("Completed merging and deduplicating types. Total count: ({d})", .{self.func_types.items.len});1425 log.debug("Completed merging and deduplicating types. Total count: ({d})", .{self.func_types.items.len});
1334}1426}
...@@ -1747,6 +1839,7 @@ pub fn flushModule(self: *Wasm, comp: *Compilation, prog_node: *std.Progress.Nod...@@ -1747,6 +1839,7 @@ pub fn flushModule(self: *Wasm, comp: *Compilation, prog_node: *std.Progress.Nod
1747 while (object_index < self.objects.items.len) : (object_index += 1) {1839 while (object_index < self.objects.items.len) : (object_index += 1) {
1748 try self.resolveSymbolsInObject(object_index);1840 try self.resolveSymbolsInObject(object_index);
1749 }1841 }
1842 try self.resolveSymbolsInArchives();
17501843
1751 // When we finish/error we reset the state of the linker1844 // When we finish/error we reset the state of the linker
1752 // So we can rebuild the binary file on each incremental update1845 // So we can rebuild the binary file on each incremental update
src/link/Wasm/Archive.zig+30-3
...@@ -113,8 +113,6 @@ pub fn parse(self: *Archive, allocator: Allocator) !void {...@@ -113,8 +113,6 @@ pub fn parse(self: *Archive, allocator: Allocator) !void {
113 return error.NotArchive;113 return error.NotArchive;
114 }114 }
115115
116 log.debug("parsing archive '{s}' at '{s}'", .{ std.mem.sliceTo(&self.header.ar_name, 0), self.name });
117
118 try self.parseTableOfContents(allocator, reader);116 try self.parseTableOfContents(allocator, reader);
119}117}
120118
...@@ -175,6 +173,35 @@ fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !...@@ -175,6 +173,35 @@ fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !
175 gop.value_ptr.* = .{};173 gop.value_ptr.* = .{};
176 }174 }
177 try gop.value_ptr.append(allocator, symbol_positions[gop.index]);175 try gop.value_ptr.append(allocator, symbol_positions[gop.index]);
178 log.debug(" parsed symbol '{s}' for position {d}", .{ string, symbol_positions[gop.index] });
179 }176 }
180}177}
178
179/// From a given file offset, starts reading for a file header.
180/// When found, parses the object file into an `Object` and returns it.
181pub fn parseObject(self: Archive, allocator: Allocator, file_offset: u32) !Object {
182 try self.file.seekTo(file_offset);
183 const reader = self.file.reader();
184 const header = try reader.readStruct(ar_hdr);
185
186 if (!mem.eql(u8, &header.ar_fmag, ARFMAG)) {
187 log.err("invalid header delimiter: expected '{s}', found '{s}'", .{ ARFMAG, header.ar_fmag });
188 return error.MalformedArchive;
189 }
190
191 const object_name = try parseName(allocator, header, reader);
192 defer allocator.free(object_name);
193
194 const name = name: {
195 if (object_name.len == 0) {
196 break :name try std.fmt.allocPrint(allocator, "{s}.o", .{self.name});
197 }
198 const base_path = std.fs.path.dirname(self.name);
199 break :name try std.fmt.allocPrint(allocator, "{s}/{s}.o", .{ base_path, object_name });
200 };
201
202 log.debug(" parsing object file '{s}' from archive\n", .{name});
203 const object_file = try std.fs.cwd().openFile(name, .{});
204 errdefer object_file.close();
205
206 return Object.create(allocator, object_file, name);
207}