Commit aaf54ce6a7e5c69c92fe30e9ca0e8ffb7765f824 | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-11-01 00:34:10-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-11-01 14:41:56-07:00
log	aaf54ce6a7e5c69c92fe30e9ca0e8ffb7765f824
tree	836ddeee83fad8412302651caaaa9426a2c6f544
parent	d30e287543ff525cf62ecf7f4cfcd20fee23a8ee

link.File.Wasm.Archive: simplify

Don't use the reader interface
Avoid unnecessary heap allocations

At first I started working on incorporating the Archive fields into the
Wasm data model, however, I realized a better strategy: simply omit
Archive data from the serialized linker state. Those files can be
trivially reparsed on next compiler process start. If they haven't
changed, great. Otherwise if they have, the prelink phase needs to be
restarted anyway.

2 files changed, 89 insertions(+), 106 deletions(-)

src
- link
  - Wasm.zig
  - Wasm
    Archive.zig

src/link/Wasm.zig+21-14

 @@ -1,39 +1,40 @@
 const Wasm = @This();
+const build_options = @import("build_options");
-const std = @import("std");
+const builtin = @import("builtin");
+const native_endian = builtin.cpu.arch.endian();
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const Cache = std.Build.Cache;
+const Path = Cache.Path;
 const assert = std.debug.assert;
-const build_options = @import("build_options");
-const builtin = @import("builtin");
-const codegen = @import("../codegen.zig");
-const dev = @import("../dev.zig");
 const fs = std.fs;
+const gc_log = std.log.scoped(.gc);
 const leb = std.leb;
-const link = @import("../link.zig");
-const lldMain = @import("../main.zig").lldMain;
 const log = std.log.scoped(.link);
-const gc_log = std.log.scoped(.gc);
 const mem = std.mem;
-const trace = @import("../tracy.zig").trace;
-const wasi_libc = @import("../wasi_libc.zig");
 const Air = @import("../Air.zig");
-const Allocator = std.mem.Allocator;
 const Archive = @import("Wasm/Archive.zig");
-const Cache = std.Build.Cache;
-const Path = Cache.Path;
 const CodeGen = @import("../arch/wasm/CodeGen.zig");
 const Compilation = @import("../Compilation.zig");
 const Dwarf = @import("Dwarf.zig");
 const InternPool = @import("../InternPool.zig");
 const Liveness = @import("../Liveness.zig");
 const LlvmObject = @import("../codegen/llvm.zig").Object;
-const Zcu = @import("../Zcu.zig");
 const Object = @import("Wasm/Object.zig");
 const Symbol = @import("Wasm/Symbol.zig");
 const Type = @import("../Type.zig");
 const Value = @import("../Value.zig");
+const Zcu = @import("../Zcu.zig");
 const ZigObject = @import("Wasm/ZigObject.zig");
+const codegen = @import("../codegen.zig");
+const dev = @import("../dev.zig");
+const link = @import("../link.zig");
+const lldMain = @import("../main.zig").lldMain;
+const trace = @import("../tracy.zig").trace;
+const wasi_libc = @import("../wasi_libc.zig");
 base: link.File,
 /// Null-terminated strings, indexes have type String and string_table provides
 @@ -141,6 +142,9 @@ function_table: std.AutoHashMapUnmanaged(SymbolLoc, u32) = .empty,
 /// All archive files that are lazy loaded.
 /// e.g. when an undefined symbol references a symbol from the archive.
+/// None of this data is serialized to disk because it is trivially reloaded
+/// from unchanged archive files on the next start of the compiler process,
+/// or if those files have changed, the prelink phase needs to be restarted.
 lazy_archives: std.ArrayListUnmanaged(LazyArchive) = .empty,
 /// A map of global names to their symbol location
 @@ -283,12 +287,15 @@ pub const OptionalObjectId = enum(u16) {
     }
 };
+/// None of this data is serialized since it can be re-loaded from disk, or if
+/// it has been changed, the data must be discarded.
 const LazyArchive = struct {
     path: Path,
     file_contents: []const u8,
     archive: Archive,
     fn deinit(la: *LazyArchive, gpa: Allocator) void {
+        la.archive.deinit(gpa);
         gpa.free(la.path.sub_path);
         gpa.free(la.file_contents);
         la.* = undefined;

src/link/Wasm/Archive.zig+68-92

...	@@ -2,25 +2,25 @@	...	@@ -2,25 +2,25 @@
2	/// This is stored as a single slice of bytes, as the header-names	2	/// This is stored as a single slice of bytes, as the header-names
3	/// point to the character index of a file name, rather than the index	3	/// point to the character index of a file name, rather than the index
4	/// in the list.	4	/// in the list.
5	long_file_names: []const u8,	5	/// Points into `file_contents`.
		6	long_file_names: RelativeSlice,
6		7
7	/// Parsed table of contents.	8	/// Parsed table of contents.
8	/// Each symbol name points to a list of all definition	9	/// Each symbol name points to a list of all definition
9	/// sites within the current static archive.	10	/// sites within the current static archive.
10	toc: Toc,	11	toc: Toc,
11		12
		13	/// Key points into `LazyArchive` `file_contents`.
		14	/// Value is allocated with gpa.
12	const Toc = std.StringArrayHashMapUnmanaged(std.ArrayListUnmanaged(u32));	15	const Toc = std.StringArrayHashMapUnmanaged(std.ArrayListUnmanaged(u32));
13		16
14	// Archive files start with the ARMAG identifying string. Then follows a	17	const ARMAG = std.elf.ARMAG;
15	// `struct Header', and as many bytes of member file data as its `size'	18	const ARFMAG = std.elf.ARFMAG;
16	// member indicates, for each member file.
17	/// String that begins an archive file.
18	const ARMAG: *const [SARMAG:0]u8 = "!<arch>\n";
19	/// Size of that string.
20	const SARMAG: u4 = 8;
21		19
22	/// String in fmag at the end of each header.	20	const RelativeSlice = struct {
23	const ARFMAG: *const [2:0]u8 = "`\n";	21	off: u32,
		22	len: u32,
		23	};
24		24
25	const Header = extern struct {	25	const Header = extern struct {
26	/// Member file name, sometimes / terminated.	26	/// Member file name, sometimes / terminated.
...	@@ -70,130 +70,106 @@ const Header = extern struct {	...	@@ -70,130 +70,106 @@ const Header = extern struct {
70		70
71	pub fn deinit(archive: *Archive, gpa: Allocator) void {	71	pub fn deinit(archive: *Archive, gpa: Allocator) void {
72	deinitToc(gpa, &archive.toc);	72	deinitToc(gpa, &archive.toc);
73	gpa.free(archive.long_file_names);
74	archive.* = undefined;	73	archive.* = undefined;
75	}	74	}
76		75
77	fn deinitToc(gpa: Allocator, toc: *Toc) void {	76	fn deinitToc(gpa: Allocator, toc: *Toc) void {
78	for (toc.keys()) \|key\| gpa.free(key);
79	for (toc.values()) \|*value\| value.deinit(gpa);	77	for (toc.values()) \|*value\| value.deinit(gpa);
80	toc.deinit(gpa);	78	toc.deinit(gpa);
81	}	79	}
82		80
83	pub fn parse(gpa: Allocator, file_contents: []const u8) !Archive {	81	pub fn parse(gpa: Allocator, file_contents: []const u8) !Archive {
84	var fbs = std.io.fixedBufferStream(file_contents);	82	var pos: usize = 0;
85	const reader = fbs.reader();
86		83
87	const magic = try reader.readBytesNoEof(SARMAG);	84	if (!mem.eql(u8, file_contents[0..ARMAG.len], ARMAG)) return error.BadArchiveMagic;
88	if (!mem.eql(u8, &magic, ARMAG)) return error.BadArchiveMagic;	85	pos += ARMAG.len;
89		86
90	const header = try reader.readStruct(Header);	87	const header = mem.bytesAsValue(Header, file_contents[pos..][0..@sizeOf(Header)]);
91	if (!mem.eql(u8, &header.fmag, ARFMAG)) return error.BadHeaderDelimiter;	88	if (!mem.eql(u8, &header.fmag, ARFMAG)) return error.BadHeaderDelimiter;
		89	pos += @sizeOf(Header);
92		90
93	var toc = try parseTableOfContents(gpa, header, reader);	91	// The size field can have extra spaces padded in front as well as
94	errdefer deinitToc(gpa, &toc);	92	// the end, so we trim those first before parsing the ASCII value.
95
96	const long_file_names = try parseNameTable(gpa, reader);
97	errdefer gpa.free(long_file_names);
98
99	return .{
100	.toc = toc,
101	.long_file_names = long_file_names,
102	};
103	}
104
105	fn parseName(archive: *const Archive, header: Header) ![]const u8 {
106	const name_or_index = try header.nameOrIndex();
107	switch (name_or_index) {
108	.name => \|name\| return name,
109	.index => \|index\| {
110	const name = mem.sliceTo(archive.long_file_names[index..], 0x0a);
111	return mem.trimRight(u8, name, "/");
112	},
113	}
114	}
115
116	fn parseTableOfContents(gpa: Allocator, header: Header, reader: anytype) !Toc {
117	// size field can have extra spaces padded in front as well as the end,
118	// so we trim those first before parsing the ASCII value.
119	const size_trimmed = mem.trim(u8, &header.size, " ");	93	const size_trimmed = mem.trim(u8, &header.size, " ");
120	const sym_tab_size = try std.fmt.parseInt(u32, size_trimmed, 10);	94	const sym_tab_size = try std.fmt.parseInt(u32, size_trimmed, 10);
121		95
122	const num_symbols = try reader.readInt(u32, .big);	96	const num_symbols = mem.readInt(u32, file_contents[pos..][0..4], .big);
123	const symbol_positions = try gpa.alloc(u32, num_symbols);	97	pos += 4;
124	defer gpa.free(symbol_positions);
125	for (symbol_positions) \|*index\| {
126	index.* = try reader.readInt(u32, .big);
127	}
128		98
129	const sym_tab = try gpa.alloc(u8, sym_tab_size - 4 - (4 * num_symbols));	99	const symbol_positions_size = @sizeOf(u32) * num_symbols;
130	defer gpa.free(sym_tab);	100	const symbol_positions_be = mem.bytesAsSlice(u32, file_contents[pos..][0..symbol_positions_size]);
		101	pos += symbol_positions_size;
131		102
132	reader.readNoEof(sym_tab) catch return error.IncompleteSymbolTable;	103	const sym_tab = file_contents[pos..][0 .. sym_tab_size - 4 - symbol_positions_size];
		104	pos += sym_tab.len;
133		105
134	var toc: Toc = .empty;	106	var toc: Toc = .empty;
135	errdefer deinitToc(gpa, &toc);	107	errdefer deinitToc(gpa, &toc);
136		108
137	var i: usize = 0;	109	var sym_tab_pos: usize = 0;
138	var pos: usize = 0;	110	for (0..num_symbols) \|i\| {
139	while (i < num_symbols) : (i += 1) {	111	const name = mem.sliceTo(sym_tab[sym_tab_pos..], 0);
140	const string = mem.sliceTo(sym_tab[pos..], 0);	112	sym_tab_pos += name.len + 1;
141	pos += string.len + 1;	113	if (name.len == 0) continue;
142	if (string.len == 0) continue;
143		114
144	const name = try gpa.dupe(u8, string);
145	errdefer gpa.free(name);
146	const gop = try toc.getOrPut(gpa, name);	115	const gop = try toc.getOrPut(gpa, name);
147	if (gop.found_existing) {	116	if (!gop.found_existing) gop.value_ptr.* = .empty;
148	gpa.free(name);	117	try gop.value_ptr.append(gpa, switch (native_endian) {
149	} else {	118	.big => symbol_positions_be[i],
150	gop.value_ptr.* = .{};	119	.little => @byteSwap(symbol_positions_be[i]),
151	}	120	});
152	try gop.value_ptr.append(gpa, symbol_positions[i]);
153	}	121	}
154		122
155	return toc;	123	const long_file_names: RelativeSlice = s: {
156	}	124	const sub_header = mem.bytesAsValue(Header, file_contents[pos..][0..@sizeOf(Header)]);
		125	pos += @sizeOf(Header);
157		126
158	fn parseNameTable(gpa: Allocator, reader: anytype) ![]const u8 {	127	if (!mem.eql(u8, &header.fmag, ARFMAG)) return error.BadHeaderDelimiter;
159	const header: Header = try reader.readStruct(Header);	128	if (!mem.eql(u8, sub_header.name[0..2], "//")) return error.MissingTableName;
160	if (!mem.eql(u8, &header.fmag, ARFMAG)) {	129	const table_size = try sub_header.parsedSize();
161	return error.InvalidHeaderDelimiter;	130
162	}	131	break :s .{
163	if (!mem.eql(u8, header.name[0..2], "//")) {	132	.off = @intCast(pos),
164	return error.MissingTableName;	133	.len = table_size,
165	}	134	};
166	const table_size = try header.parsedSize();	135	};
167	const long_file_names = try gpa.alloc(u8, table_size);
168	errdefer gpa.free(long_file_names);
169	try reader.readNoEof(long_file_names);
170		136
171	return long_file_names;	137	return .{
		138	.toc = toc,
		139	.long_file_names = long_file_names,
		140	};
172	}	141	}
173		142
174	/// From a given file offset, starts reading for a file header.	143	/// From a given file offset, starts reading for a file header.
175	/// When found, parses the object file into an `Object` and returns it.	144	/// When found, parses the object file into an `Object` and returns it.
176	pub fn parseObject(archive: Archive, wasm: *Wasm, file_contents: []const u8, path: Path) !Object {	145	pub fn parseObject(archive: Archive, wasm: *Wasm, file_contents: []const u8, path: Path) !Object {
177	var fbs = std.io.fixedBufferStream(file_contents);	146	const header = mem.bytesAsValue(Header, file_contents[0..@sizeOf(Header)]);
178	const header = try fbs.reader().readStruct(Header);	147	if (!mem.eql(u8, &header.fmag, ARFMAG)) return error.BadHeaderDelimiter;
179		148
180	if (!mem.eql(u8, &header.fmag, ARFMAG)) return error.BadArchiveHeaderDelimiter;	149	const name_or_index = try header.nameOrIndex();
		150	const object_name = switch (name_or_index) {
		151	.name => \|name\| name,
		152	.index => \|index\| n: {
		153	const long_file_names = file_contents[archive.long_file_names.off..][0..archive.long_file_names.len];
		154	const name = mem.sliceTo(long_file_names[index..], 0x0a);
		155	break :n mem.trimRight(u8, name, "/");
		156	},
		157	};
181		158
182	const object_name = try archive.parseName(header);
183	const object_file_size = try header.parsedSize();	159	const object_file_size = try header.parsedSize();
184		160
185	return Object.create(wasm, file_contents[@sizeOf(Header)..][0..object_file_size], path, object_name);	161	return Object.create(wasm, file_contents[@sizeOf(Header)..][0..object_file_size], path, object_name);
186	}	162	}
187		163
		164	const Archive = @This();
		165
		166	const builtin = @import("builtin");
		167	const native_endian = builtin.cpu.arch.endian();
		168
188	const std = @import("std");	169	const std = @import("std");
189	const assert = std.debug.assert;
190	const fs = std.fs;
191	const log = std.log.scoped(.archive);
192	const mem = std.mem;	170	const mem = std.mem;
		171	const Allocator = std.mem.Allocator;
193	const Path = std.Build.Cache.Path;	172	const Path = std.Build.Cache.Path;
194		173
195	const Allocator = mem.Allocator;
196	const Object = @import("Object.zig");
197	const Wasm = @import("../Wasm.zig");	174	const Wasm = @import("../Wasm.zig");
198		175	const Object = @import("Object.zig");
199	const Archive = @This();