authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-08-22 08:38:41+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-08-22 08:38:41+02:00
log6c020cdb767192757b6c4b43e2f14c5394760431
tree130b47a125baeff98040a4eb819d90d4075086cf
parent9cf667a21bfef3660183ed67baacb5b32c0f06e2
parentd4c56804dfd3f53de1f0950fac9478b096a2a0b4
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #12557 from Luukdegram/wasm-archive

wasm-linker: Improve archive linking

8 files changed, 139 insertions(+), 63 deletions(-)

lib/std/build.zig+2-3
......@@ -1898,11 +1898,10 @@ pub const LibExeObjStep = struct {
18981898 pub fn runEmulatable(exe: *LibExeObjStep) *EmulatableRunStep {
18991899 assert(exe.kind == .exe or exe.kind == .test_exe);
19001900
1901 const run_step = EmulatableRunStep.create(exe.builder.fmt("run {s}", .{exe.step.name}), exe);
1901 const run_step = EmulatableRunStep.create(exe.builder, exe.builder.fmt("run {s}", .{exe.step.name}), exe);
19021902 if (exe.vcpkg_bin_path) |path| {
1903 run_step.addPathDir(path);
1903 RunStep.addPathDirInternal(&run_step.step, exe.builder, path);
19041904 }
1905
19061905 return run_step;
19071906 }
19081907
lib/std/build/RunStep.zig+1-1
......@@ -101,7 +101,7 @@ pub fn addPathDir(self: *RunStep, search_path: []const u8) void {
101101}
102102
103103/// For internal use only, users of `RunStep` should use `addPathDir` directly.
104fn addPathDirInternal(step: *Step, builder: *Builder, search_path: []const u8) void {
104pub fn addPathDirInternal(step: *Step, builder: *Builder, search_path: []const u8) void {
105105 const env_map = getEnvMapInternal(step, builder.allocator);
106106
107107 const key = "PATH";
src/link/Wasm.zig+8-5
......@@ -378,7 +378,7 @@ fn parseObjectFile(self: *Wasm, path: []const u8) !bool {
378378 const file = try fs.cwd().openFile(path, .{});
379379 errdefer file.close();
380380
381 var object = Object.create(self.base.allocator, file, path) catch |err| switch (err) {
381 var object = Object.create(self.base.allocator, file, path, null) catch |err| switch (err) {
382382 error.InvalidMagicByte, error.NotObjectFile => return false,
383383 else => |e| return e,
384384 };
......@@ -595,8 +595,8 @@ fn resolveSymbolsInArchives(self: *Wasm) !void {
595595 // Parse object and and resolve symbols again before we check remaining
596596 // undefined symbols.
597597 const object_file_index = @intCast(u16, self.objects.items.len);
598 const object = try self.objects.addOne(self.base.allocator);
599 object.* = try archive.parseObject(self.base.allocator, offset.items[0]);
598 var object = try archive.parseObject(self.base.allocator, offset.items[0]);
599 try self.objects.append(self.base.allocator, object);
600600 try self.resolveSymbolsInObject(object_file_index);
601601
602602 // continue loop for any remaining undefined symbols that still exist
......@@ -860,7 +860,8 @@ fn getGlobalType(self: *const Wasm, loc: SymbolLoc) wasm.GlobalType {
860860 if (is_undefined) {
861861 return obj.findImport(.global, symbol.index).kind.global;
862862 }
863 return obj.globals[symbol.index].global_type;
863 const import_global_count = obj.importedCountByKind(.global);
864 return obj.globals[symbol.index - import_global_count].global_type;
864865 }
865866 if (is_undefined) {
866867 return self.imports.get(loc).?.kind.global;
......@@ -880,7 +881,9 @@ fn getFunctionSignature(self: *const Wasm, loc: SymbolLoc) wasm.Type {
880881 const ty_index = obj.findImport(.function, symbol.index).kind.function;
881882 return obj.func_types[ty_index];
882883 }
883 return obj.func_types[obj.functions[symbol.index].type_index];
884 const import_function_count = obj.importedCountByKind(.function);
885 const type_index = obj.functions[symbol.index - import_function_count].type_index;
886 return obj.func_types[type_index];
884887 }
885888 if (is_undefined) {
886889 const ty_index = self.imports.get(loc).?.kind.function;
src/link/Wasm/Archive.zig+69-52
......@@ -15,6 +15,12 @@ name: []const u8,
1515
1616header: ar_hdr = undefined,
1717
18/// A list of long file names, delimited by a LF character (0x0a).
19/// This is stored as a single slice of bytes, as the header-names
20/// point to the character index of a file name, rather than the index
21/// in the list.
22long_file_names: []const u8 = undefined,
23
1824/// Parsed table of contents.
1925/// Each symbol name points to a list of all definition
2026/// sites within the current static archive.
......@@ -53,32 +59,33 @@ const ar_hdr = extern struct {
5359 /// Always contains ARFMAG.
5460 ar_fmag: [2]u8,
5561
56 const NameOrLength = union(enum) {
57 Name: []const u8,
58 Length: u32,
62 const NameOrIndex = union(enum) {
63 name: []const u8,
64 index: u32,
5965 };
60 fn nameOrLength(self: ar_hdr) !NameOrLength {
61 const value = getValue(&self.ar_name);
66
67 fn nameOrIndex(archive: ar_hdr) !NameOrIndex {
68 const value = getValue(&archive.ar_name);
6269 const slash_index = mem.indexOfScalar(u8, value, '/') orelse return error.MalformedArchive;
6370 const len = value.len;
6471 if (slash_index == len - 1) {
6572 // Name stored directly
66 return NameOrLength{ .Name = value };
73 return NameOrIndex{ .name = value };
6774 } else {
6875 // Name follows the header directly and its length is encoded in
6976 // the name field.
70 const length = try std.fmt.parseInt(u32, value[slash_index + 1 ..], 10);
71 return NameOrLength{ .Length = length };
77 const index = try std.fmt.parseInt(u32, value[slash_index + 1 ..], 10);
78 return NameOrIndex{ .index = index };
7279 }
7380 }
7481
75 fn date(self: ar_hdr) !u64 {
76 const value = getValue(&self.ar_date);
82 fn date(archive: ar_hdr) !u64 {
83 const value = getValue(&archive.ar_date);
7784 return std.fmt.parseInt(u64, value, 10);
7885 }
7986
80 fn size(self: ar_hdr) !u32 {
81 const value = getValue(&self.ar_size);
87 fn size(archive: ar_hdr) !u32 {
88 const value = getValue(&archive.ar_size);
8289 return std.fmt.parseInt(u32, value, 10);
8390 }
8491
......@@ -87,18 +94,19 @@ const ar_hdr = extern struct {
8794 }
8895};
8996
90pub fn deinit(self: *Archive, allocator: Allocator) void {
91 for (self.toc.keys()) |*key| {
97pub fn deinit(archive: *Archive, allocator: Allocator) void {
98 for (archive.toc.keys()) |*key| {
9299 allocator.free(key.*);
93100 }
94 for (self.toc.values()) |*value| {
101 for (archive.toc.values()) |*value| {
95102 value.deinit(allocator);
96103 }
97 self.toc.deinit(allocator);
104 archive.toc.deinit(allocator);
105 allocator.free(archive.long_file_names);
98106}
99107
100pub fn parse(self: *Archive, allocator: Allocator) !void {
101 const reader = self.file.reader();
108pub fn parse(archive: *Archive, allocator: Allocator) !void {
109 const reader = archive.file.reader();
102110
103111 const magic = try reader.readBytesNoEof(SARMAG);
104112 if (!mem.eql(u8, &magic, ARMAG)) {
......@@ -106,38 +114,31 @@ pub fn parse(self: *Archive, allocator: Allocator) !void {
106114 return error.NotArchive;
107115 }
108116
109 self.header = try reader.readStruct(ar_hdr);
110 if (!mem.eql(u8, &self.header.ar_fmag, ARFMAG)) {
111 log.debug("invalid header delimiter: expected '{s}', found '{s}'", .{ ARFMAG, self.header.ar_fmag });
117 archive.header = try reader.readStruct(ar_hdr);
118 if (!mem.eql(u8, &archive.header.ar_fmag, ARFMAG)) {
119 log.debug("invalid header delimiter: expected '{s}', found '{s}'", .{ ARFMAG, archive.header.ar_fmag });
112120 return error.NotArchive;
113121 }
114122
115 try self.parseTableOfContents(allocator, reader);
123 try archive.parseTableOfContents(allocator, reader);
124 try archive.parseNameTable(allocator, reader);
116125}
117126
118fn parseName(allocator: Allocator, header: ar_hdr, reader: anytype) ![]u8 {
119 const name_or_length = try header.nameOrLength();
120 var name: []u8 = undefined;
121 switch (name_or_length) {
122 .Name => |n| {
123 name = try allocator.dupe(u8, n);
124 },
125 .Length => |len| {
126 var n = try allocator.alloc(u8, len);
127 defer allocator.free(n);
128 try reader.readNoEof(n);
129 const actual_len = mem.indexOfScalar(u8, n, @as(u8, 0)) orelse n.len;
130 name = try allocator.dupe(u8, n[0..actual_len]);
127fn parseName(archive: *const Archive, header: ar_hdr) ![]const u8 {
128 const name_or_index = try header.nameOrIndex();
129 switch (name_or_index) {
130 .name => |name| return name,
131 .index => |index| {
132 const name = mem.sliceTo(archive.long_file_names[index..], 0x0a);
133 return mem.trimRight(u8, name, "/");
131134 },
132135 }
133 return name;
134136}
135137
136fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !void {
137 log.debug("parsing table of contents for archive file '{s}'", .{self.name});
138fn parseTableOfContents(archive: *Archive, allocator: Allocator, reader: anytype) !void {
138139 // size field can have extra spaces padded in front as well as the end,
139140 // so we trim those first before parsing the ASCII value.
140 const size_trimmed = std.mem.trim(u8, &self.header.ar_size, " ");
141 const size_trimmed = mem.trim(u8, &archive.header.ar_size, " ");
141142 const sym_tab_size = try std.fmt.parseInt(u32, size_trimmed, 10);
142143
143144 const num_symbols = try reader.readIntBig(u32);
......@@ -157,7 +158,7 @@ fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !
157158
158159 var i: usize = 0;
159160 while (i < sym_tab.len) {
160 const string = std.mem.sliceTo(sym_tab[i..], 0);
161 const string = mem.sliceTo(sym_tab[i..], 0);
161162 if (string.len == 0) {
162163 i += 1;
163164 continue;
......@@ -165,7 +166,7 @@ fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !
165166 i += string.len;
166167 const name = try allocator.dupe(u8, string);
167168 errdefer allocator.free(name);
168 const gop = try self.toc.getOrPut(allocator, name);
169 const gop = try archive.toc.getOrPut(allocator, name);
169170 if (gop.found_existing) {
170171 allocator.free(name);
171172 } else {
......@@ -175,33 +176,49 @@ fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !
175176 }
176177}
177178
179fn parseNameTable(archive: *Archive, allocator: Allocator, reader: anytype) !void {
180 const header: ar_hdr = try reader.readStruct(ar_hdr);
181 if (!mem.eql(u8, &header.ar_fmag, ARFMAG)) {
182 log.err("invalid header delimiter: expected '{s}', found '{s}'", .{ ARFMAG, header.ar_fmag });
183 return error.MalformedArchive;
184 }
185 if (!mem.eql(u8, header.ar_name[0..2], "//")) {
186 log.err("invalid archive. Long name table missing", .{});
187 return error.MalformedArchive;
188 }
189 const table_size = try header.size();
190 const long_file_names = try allocator.alloc(u8, table_size);
191 errdefer allocator.free(long_file_names);
192 try reader.readNoEof(long_file_names);
193 archive.long_file_names = long_file_names;
194}
195
178196/// From a given file offset, starts reading for a file header.
179197/// When found, parses the object file into an `Object` and returns it.
180pub fn parseObject(self: Archive, allocator: Allocator, file_offset: u32) !Object {
181 try self.file.seekTo(file_offset);
182 const reader = self.file.reader();
198pub fn parseObject(archive: Archive, allocator: Allocator, file_offset: u32) !Object {
199 try archive.file.seekTo(file_offset);
200 const reader = archive.file.reader();
183201 const header = try reader.readStruct(ar_hdr);
184 const current_offset = try self.file.getPos();
185 try self.file.seekTo(0);
202 const current_offset = try archive.file.getPos();
203 try archive.file.seekTo(0);
186204
187205 if (!mem.eql(u8, &header.ar_fmag, ARFMAG)) {
188206 log.err("invalid header delimiter: expected '{s}', found '{s}'", .{ ARFMAG, header.ar_fmag });
189207 return error.MalformedArchive;
190208 }
191209
192 const object_name = try parseName(allocator, header, reader);
193 defer allocator.free(object_name);
194
210 const object_name = try archive.parseName(header);
195211 const name = name: {
196212 var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
197 const path = try std.os.realpath(self.name, &buffer);
213 const path = try std.os.realpath(archive.name, &buffer);
198214 break :name try std.fmt.allocPrint(allocator, "{s}({s})", .{ path, object_name });
199215 };
200216 defer allocator.free(name);
201217
202 const object_file = try std.fs.cwd().openFile(self.name, .{});
218 const object_file = try std.fs.cwd().openFile(archive.name, .{});
203219 errdefer object_file.close();
204220
221 const object_file_size = try header.size();
205222 try object_file.seekTo(current_offset);
206 return Object.create(allocator, object_file, name);
223 return Object.create(allocator, object_file, name, object_file_size);
207224}
src/link/Wasm/Object.zig+21-2
......@@ -105,14 +105,33 @@ pub const InitError = error{NotObjectFile} || ParseError || std.fs.File.ReadErro
105105
106106/// Initializes a new `Object` from a wasm object file.
107107/// This also parses and verifies the object file.
108pub fn create(gpa: Allocator, file: std.fs.File, name: []const u8) InitError!Object {
108/// When a max size is given, will only parse up to the given size,
109/// else will read until the end of the file.
110pub fn create(gpa: Allocator, file: std.fs.File, name: []const u8, maybe_max_size: ?usize) InitError!Object {
109111 var object: Object = .{
110112 .file = file,
111113 .name = try gpa.dupe(u8, name),
112114 };
113115
114116 var is_object_file: bool = false;
115 try object.parse(gpa, file.reader(), &is_object_file);
117 const size = maybe_max_size orelse size: {
118 errdefer gpa.free(object.name);
119 const stat = try file.stat();
120 break :size @intCast(usize, stat.size);
121 };
122
123 const file_contents = try gpa.alloc(u8, size);
124 defer gpa.free(file_contents);
125 var file_reader = file.reader();
126 var read: usize = 0;
127 while (read < size) {
128 const n = try file_reader.read(file_contents[read..]);
129 std.debug.assert(n != 0);
130 read += n;
131 }
132 var fbs = std.io.fixedBufferStream(file_contents);
133
134 try object.parse(gpa, fbs.reader(), &is_object_file);
116135 errdefer object.deinit(gpa);
117136 if (!is_object_file) return error.NotObjectFile;
118137
test/link.zig+5
......@@ -47,6 +47,11 @@ fn addWasmCases(cases: *tests.StandaloneContext) void {
4747 .build_modes = true,
4848 .requires_stage2 = true,
4949 });
50
51 cases.addBuildFile("test/link/wasm/archive/build.zig", .{
52 .build_modes = true,
53 .requires_stage2 = true,
54 });
5055}
5156
5257fn addMachOCases(cases: *tests.StandaloneContext) void {
test/link/wasm/archive/build.zig created+27
......@@ -0,0 +1,27 @@
1const std = @import("std");
2const Builder = std.build.Builder;
3
4pub fn build(b: *Builder) void {
5 const mode = b.standardReleaseOptions();
6
7 const test_step = b.step("test", "Test");
8 test_step.dependOn(b.getInstallStep());
9
10 // The code in question will pull-in compiler-rt,
11 // and therefore link with its archive file.
12 const lib = b.addSharedLibrary("main", "main.zig", .unversioned);
13 lib.setBuildMode(mode);
14 lib.setTarget(.{ .cpu_arch = .wasm32, .os_tag = .freestanding });
15 lib.use_llvm = false;
16 lib.use_stage1 = false;
17 lib.use_lld = false;
18
19 const check = lib.checkObject(.wasm);
20 check.checkStart("Section import");
21 check.checkNext("entries 1"); // __truncsfhf2 should have been resolved, so only 1 import (compiler-rt's memcpy).
22
23 check.checkStart("Section custom");
24 check.checkNext("name __truncsfhf2"); // Ensure it was imported and resolved
25
26 test_step.dependOn(&check.step);
27}
test/link/wasm/archive/main.zig created+6
......@@ -0,0 +1,6 @@
1export fn foo() void {
2 var a: f16 = 2.2;
3 // this will pull-in compiler-rt
4 var b = @trunc(a);
5 _ = b;
6}