authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-11-01 00:34:10-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-11-01 14:41:56-07:00
logaaf54ce6a7e5c69c92fe30e9ca0e8ffb7765f824
tree836ddeee83fad8412302651caaaa9426a2c6f544
parentd30e287543ff525cf62ecf7f4cfcd20fee23a8ee

link.File.Wasm.Archive: simplify

Don't use the reader interface Avoid unnecessary heap allocations At first I started working on incorporating the Archive fields into the Wasm data model, however, I realized a better strategy: simply omit Archive data from the serialized linker state. Those files can be trivially reparsed on next compiler process start. If they haven't changed, great. Otherwise if they have, the prelink phase needs to be restarted anyway.

2 files changed, 89 insertions(+), 106 deletions(-)

src/link/Wasm.zig+21-14
......@@ -1,39 +1,40 @@
11const Wasm = @This();
2const build_options = @import("build_options");
23
3const std = @import("std");
4const builtin = @import("builtin");
5const native_endian = builtin.cpu.arch.endian();
46
7const std = @import("std");
8const Allocator = std.mem.Allocator;
9const Cache = std.Build.Cache;
10const Path = Cache.Path;
511const assert = std.debug.assert;
6const build_options = @import("build_options");
7const builtin = @import("builtin");
8const codegen = @import("../codegen.zig");
9const dev = @import("../dev.zig");
1012const fs = std.fs;
13const gc_log = std.log.scoped(.gc);
1114const leb = std.leb;
12const link = @import("../link.zig");
13const lldMain = @import("../main.zig").lldMain;
1415const log = std.log.scoped(.link);
15const gc_log = std.log.scoped(.gc);
1616const mem = std.mem;
17const trace = @import("../tracy.zig").trace;
18const wasi_libc = @import("../wasi_libc.zig");
1917
2018const Air = @import("../Air.zig");
21const Allocator = std.mem.Allocator;
2219const Archive = @import("Wasm/Archive.zig");
23const Cache = std.Build.Cache;
24const Path = Cache.Path;
2520const CodeGen = @import("../arch/wasm/CodeGen.zig");
2621const Compilation = @import("../Compilation.zig");
2722const Dwarf = @import("Dwarf.zig");
2823const InternPool = @import("../InternPool.zig");
2924const Liveness = @import("../Liveness.zig");
3025const LlvmObject = @import("../codegen/llvm.zig").Object;
31const Zcu = @import("../Zcu.zig");
3226const Object = @import("Wasm/Object.zig");
3327const Symbol = @import("Wasm/Symbol.zig");
3428const Type = @import("../Type.zig");
3529const Value = @import("../Value.zig");
30const Zcu = @import("../Zcu.zig");
3631const ZigObject = @import("Wasm/ZigObject.zig");
32const codegen = @import("../codegen.zig");
33const dev = @import("../dev.zig");
34const link = @import("../link.zig");
35const lldMain = @import("../main.zig").lldMain;
36const trace = @import("../tracy.zig").trace;
37const wasi_libc = @import("../wasi_libc.zig");
3738
3839base: link.File,
3940/// Null-terminated strings, indexes have type String and string_table provides
......@@ -141,6 +142,9 @@ function_table: std.AutoHashMapUnmanaged(SymbolLoc, u32) = .empty,
141142
142143/// All archive files that are lazy loaded.
143144/// e.g. when an undefined symbol references a symbol from the archive.
145/// None of this data is serialized to disk because it is trivially reloaded
146/// from unchanged archive files on the next start of the compiler process,
147/// or if those files have changed, the prelink phase needs to be restarted.
144148lazy_archives: std.ArrayListUnmanaged(LazyArchive) = .empty,
145149
146150/// A map of global names to their symbol location
......@@ -283,12 +287,15 @@ pub const OptionalObjectId = enum(u16) {
283287 }
284288};
285289
290/// None of this data is serialized since it can be re-loaded from disk, or if
291/// it has been changed, the data must be discarded.
286292const LazyArchive = struct {
287293 path: Path,
288294 file_contents: []const u8,
289295 archive: Archive,
290296
291297 fn deinit(la: *LazyArchive, gpa: Allocator) void {
298 la.archive.deinit(gpa);
292299 gpa.free(la.path.sub_path);
293300 gpa.free(la.file_contents);
294301 la.* = undefined;
src/link/Wasm/Archive.zig+68-92
......@@ -2,25 +2,25 @@
22/// This is stored as a single slice of bytes, as the header-names
33/// point to the character index of a file name, rather than the index
44/// in the list.
5long_file_names: []const u8,
5/// Points into `file_contents`.
6long_file_names: RelativeSlice,
67
78/// Parsed table of contents.
89/// Each symbol name points to a list of all definition
910/// sites within the current static archive.
1011toc: Toc,
1112
13/// Key points into `LazyArchive` `file_contents`.
14/// Value is allocated with gpa.
1215const Toc = std.StringArrayHashMapUnmanaged(std.ArrayListUnmanaged(u32));
1316
14// Archive files start with the ARMAG identifying string. Then follows a
15// `struct Header', and as many bytes of member file data as its `size'
16// member indicates, for each member file.
17/// String that begins an archive file.
18const ARMAG: *const [SARMAG:0]u8 = "!<arch>\n";
19/// Size of that string.
20const SARMAG: u4 = 8;
17const ARMAG = std.elf.ARMAG;
18const ARFMAG = std.elf.ARFMAG;
2119
22/// String in fmag at the end of each header.
23const ARFMAG: *const [2:0]u8 = "`\n";
20const RelativeSlice = struct {
21 off: u32,
22 len: u32,
23};
2424
2525const Header = extern struct {
2626 /// Member file name, sometimes / terminated.
......@@ -70,130 +70,106 @@ const Header = extern struct {
7070
7171pub fn deinit(archive: *Archive, gpa: Allocator) void {
7272 deinitToc(gpa, &archive.toc);
73 gpa.free(archive.long_file_names);
7473 archive.* = undefined;
7574}
7675
7776fn deinitToc(gpa: Allocator, toc: *Toc) void {
78 for (toc.keys()) |key| gpa.free(key);
7977 for (toc.values()) |*value| value.deinit(gpa);
8078 toc.deinit(gpa);
8179}
8280
8381pub fn parse(gpa: Allocator, file_contents: []const u8) !Archive {
84 var fbs = std.io.fixedBufferStream(file_contents);
85 const reader = fbs.reader();
82 var pos: usize = 0;
8683
87 const magic = try reader.readBytesNoEof(SARMAG);
88 if (!mem.eql(u8, &magic, ARMAG)) return error.BadArchiveMagic;
84 if (!mem.eql(u8, file_contents[0..ARMAG.len], ARMAG)) return error.BadArchiveMagic;
85 pos += ARMAG.len;
8986
90 const header = try reader.readStruct(Header);
87 const header = mem.bytesAsValue(Header, file_contents[pos..][0..@sizeOf(Header)]);
9188 if (!mem.eql(u8, &header.fmag, ARFMAG)) return error.BadHeaderDelimiter;
89 pos += @sizeOf(Header);
9290
93 var toc = try parseTableOfContents(gpa, header, reader);
94 errdefer deinitToc(gpa, &toc);
95
96 const long_file_names = try parseNameTable(gpa, reader);
97 errdefer gpa.free(long_file_names);
98
99 return .{
100 .toc = toc,
101 .long_file_names = long_file_names,
102 };
103}
104
105fn parseName(archive: *const Archive, header: Header) ![]const u8 {
106 const name_or_index = try header.nameOrIndex();
107 switch (name_or_index) {
108 .name => |name| return name,
109 .index => |index| {
110 const name = mem.sliceTo(archive.long_file_names[index..], 0x0a);
111 return mem.trimRight(u8, name, "/");
112 },
113 }
114}
115
116fn parseTableOfContents(gpa: Allocator, header: Header, reader: anytype) !Toc {
117 // size field can have extra spaces padded in front as well as the end,
118 // so we trim those first before parsing the ASCII value.
91 // The size field can have extra spaces padded in front as well as
92 // the end, so we trim those first before parsing the ASCII value.
11993 const size_trimmed = mem.trim(u8, &header.size, " ");
12094 const sym_tab_size = try std.fmt.parseInt(u32, size_trimmed, 10);
12195
122 const num_symbols = try reader.readInt(u32, .big);
123 const symbol_positions = try gpa.alloc(u32, num_symbols);
124 defer gpa.free(symbol_positions);
125 for (symbol_positions) |*index| {
126 index.* = try reader.readInt(u32, .big);
127 }
96 const num_symbols = mem.readInt(u32, file_contents[pos..][0..4], .big);
97 pos += 4;
12898
129 const sym_tab = try gpa.alloc(u8, sym_tab_size - 4 - (4 * num_symbols));
130 defer gpa.free(sym_tab);
99 const symbol_positions_size = @sizeOf(u32) * num_symbols;
100 const symbol_positions_be = mem.bytesAsSlice(u32, file_contents[pos..][0..symbol_positions_size]);
101 pos += symbol_positions_size;
131102
132 reader.readNoEof(sym_tab) catch return error.IncompleteSymbolTable;
103 const sym_tab = file_contents[pos..][0 .. sym_tab_size - 4 - symbol_positions_size];
104 pos += sym_tab.len;
133105
134106 var toc: Toc = .empty;
135107 errdefer deinitToc(gpa, &toc);
136108
137 var i: usize = 0;
138 var pos: usize = 0;
139 while (i < num_symbols) : (i += 1) {
140 const string = mem.sliceTo(sym_tab[pos..], 0);
141 pos += string.len + 1;
142 if (string.len == 0) continue;
109 var sym_tab_pos: usize = 0;
110 for (0..num_symbols) |i| {
111 const name = mem.sliceTo(sym_tab[sym_tab_pos..], 0);
112 sym_tab_pos += name.len + 1;
113 if (name.len == 0) continue;
143114
144 const name = try gpa.dupe(u8, string);
145 errdefer gpa.free(name);
146115 const gop = try toc.getOrPut(gpa, name);
147 if (gop.found_existing) {
148 gpa.free(name);
149 } else {
150 gop.value_ptr.* = .{};
151 }
152 try gop.value_ptr.append(gpa, symbol_positions[i]);
116 if (!gop.found_existing) gop.value_ptr.* = .empty;
117 try gop.value_ptr.append(gpa, switch (native_endian) {
118 .big => symbol_positions_be[i],
119 .little => @byteSwap(symbol_positions_be[i]),
120 });
153121 }
154122
155 return toc;
156}
123 const long_file_names: RelativeSlice = s: {
124 const sub_header = mem.bytesAsValue(Header, file_contents[pos..][0..@sizeOf(Header)]);
125 pos += @sizeOf(Header);
157126
158fn parseNameTable(gpa: Allocator, reader: anytype) ![]const u8 {
159 const header: Header = try reader.readStruct(Header);
160 if (!mem.eql(u8, &header.fmag, ARFMAG)) {
161 return error.InvalidHeaderDelimiter;
162 }
163 if (!mem.eql(u8, header.name[0..2], "//")) {
164 return error.MissingTableName;
165 }
166 const table_size = try header.parsedSize();
167 const long_file_names = try gpa.alloc(u8, table_size);
168 errdefer gpa.free(long_file_names);
169 try reader.readNoEof(long_file_names);
127 if (!mem.eql(u8, &header.fmag, ARFMAG)) return error.BadHeaderDelimiter;
128 if (!mem.eql(u8, sub_header.name[0..2], "//")) return error.MissingTableName;
129 const table_size = try sub_header.parsedSize();
130
131 break :s .{
132 .off = @intCast(pos),
133 .len = table_size,
134 };
135 };
170136
171 return long_file_names;
137 return .{
138 .toc = toc,
139 .long_file_names = long_file_names,
140 };
172141}
173142
174143/// From a given file offset, starts reading for a file header.
175144/// When found, parses the object file into an `Object` and returns it.
176145pub fn parseObject(archive: Archive, wasm: *Wasm, file_contents: []const u8, path: Path) !Object {
177 var fbs = std.io.fixedBufferStream(file_contents);
178 const header = try fbs.reader().readStruct(Header);
146 const header = mem.bytesAsValue(Header, file_contents[0..@sizeOf(Header)]);
147 if (!mem.eql(u8, &header.fmag, ARFMAG)) return error.BadHeaderDelimiter;
179148
180 if (!mem.eql(u8, &header.fmag, ARFMAG)) return error.BadArchiveHeaderDelimiter;
149 const name_or_index = try header.nameOrIndex();
150 const object_name = switch (name_or_index) {
151 .name => |name| name,
152 .index => |index| n: {
153 const long_file_names = file_contents[archive.long_file_names.off..][0..archive.long_file_names.len];
154 const name = mem.sliceTo(long_file_names[index..], 0x0a);
155 break :n mem.trimRight(u8, name, "/");
156 },
157 };
181158
182 const object_name = try archive.parseName(header);
183159 const object_file_size = try header.parsedSize();
184160
185161 return Object.create(wasm, file_contents[@sizeOf(Header)..][0..object_file_size], path, object_name);
186162}
187163
164const Archive = @This();
165
166const builtin = @import("builtin");
167const native_endian = builtin.cpu.arch.endian();
168
188169const std = @import("std");
189const assert = std.debug.assert;
190const fs = std.fs;
191const log = std.log.scoped(.archive);
192170const mem = std.mem;
171const Allocator = std.mem.Allocator;
193172const Path = std.Build.Cache.Path;
194173
195const Allocator = mem.Allocator;
196const Object = @import("Object.zig");
197174const Wasm = @import("../Wasm.zig");
198
199const Archive = @This();
175const Object = @import("Object.zig");