authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-09-13 19:05:22+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-09-13 21:51:43+02:00
logdbde746f9d95f2dbc8872e2d6283c2db64ac7519
tree6c22465f27b7b22c2b3447e668603e1d95aae126
parent5eef7577d1a49006d69d5067af0dd87be272db52

elf: parse archives

5 files changed, 312 insertions(+), 17 deletions(-)

src/link/Elf.zig+134-14
...@@ -1052,13 +1052,6 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node...@@ -1052,13 +1052,6 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
1052 const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.1052 const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
1053 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});1053 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
10541054
1055 const compiler_rt_path: ?[]const u8 = blk: {
1056 if (comp.compiler_rt_lib) |x| break :blk x.full_object_path;
1057 if (comp.compiler_rt_obj) |x| break :blk x.full_object_path;
1058 break :blk null;
1059 };
1060 _ = compiler_rt_path;
1061
1062 // Here we will parse input positional and library files (if referenced).1055 // Here we will parse input positional and library files (if referenced).
1063 // This will roughly match in any linker backend we support.1056 // This will roughly match in any linker backend we support.
1064 var positionals = std.ArrayList(Compilation.LinkObject).init(arena);1057 var positionals = std.ArrayList(Compilation.LinkObject).init(arena);
...@@ -1084,6 +1077,15 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node...@@ -1084,6 +1077,15 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
1084 try positionals.append(.{ .path = key.status.success.object_path });1077 try positionals.append(.{ .path = key.status.success.object_path });
1085 }1078 }
10861079
1080 const compiler_rt_path: ?[]const u8 = blk: {
1081 if (comp.compiler_rt_lib) |x| break :blk x.full_object_path;
1082 if (comp.compiler_rt_obj) |x| break :blk x.full_object_path;
1083 break :blk null;
1084 };
1085 if (compiler_rt_path) |path| {
1086 try positionals.append(.{ .path = path });
1087 }
1088
1087 for (positionals.items) |obj| {1089 for (positionals.items) |obj| {
1088 const in_file = try std.fs.cwd().openFile(obj.path, .{});1090 const in_file = try std.fs.cwd().openFile(obj.path, .{});
1089 defer in_file.close();1091 defer in_file.close();
...@@ -1140,7 +1142,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node...@@ -1140,7 +1142,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
1140 // input Object files.1142 // input Object files.
1141 // Any qualifing unresolved symbol will be upgraded to an absolute, weak1143 // Any qualifing unresolved symbol will be upgraded to an absolute, weak
1142 // symbol for potential resolution at load-time.1144 // symbol for potential resolution at load-time.
1143 self.resolveSymbols();1145 try self.resolveSymbols();
1144 self.markImportsExports();1146 self.markImportsExports();
1145 self.claimUnresolved();1147 self.claimUnresolved();
11461148
...@@ -1403,6 +1405,7 @@ const ParseError = error{...@@ -1403,6 +1405,7 @@ const ParseError = error{
1403 EndOfStream,1405 EndOfStream,
1404 FileSystem,1406 FileSystem,
1405 NotSupported,1407 NotSupported,
1408 InvalidCharacter,
1406} || std.os.SeekError || std.fs.File.OpenError || std.fs.File.ReadError;1409} || std.os.SeekError || std.fs.File.OpenError || std.fs.File.ReadError;
14071410
1408fn parsePositional(1411fn parsePositional(
...@@ -1414,10 +1417,32 @@ fn parsePositional(...@@ -1414,10 +1417,32 @@ fn parsePositional(
1414) ParseError!void {1417) ParseError!void {
1415 const tracy = trace(@src());1418 const tracy = trace(@src());
1416 defer tracy.end();1419 defer tracy.end();
1417 _ = must_link;
14181420
1419 if (Object.isObject(in_file)) {1421 if (Object.isObject(in_file)) {
1420 try self.parseObject(in_file, path, ctx);1422 try self.parseObject(in_file, path, ctx);
1423 } else {
1424 try self.parseLibrary(in_file, path, .{
1425 .path = null,
1426 .needed = false,
1427 .weak = false,
1428 }, must_link, ctx);
1429 }
1430}
1431
1432fn parseLibrary(
1433 self: *Elf,
1434 in_file: std.fs.File,
1435 path: []const u8,
1436 lib: link.SystemLib,
1437 must_link: bool,
1438 ctx: *ParseErrorCtx,
1439) ParseError!void {
1440 const tracy = trace(@src());
1441 defer tracy.end();
1442 _ = lib;
1443
1444 if (Archive.isArchive(in_file)) {
1445 try self.parseArchive(in_file, path, must_link, ctx);
1421 } else return error.UnknownFileType;1446 } else return error.UnknownFileType;
1422}1447}
14231448
...@@ -1442,15 +1467,109 @@ fn parseObject(self: *Elf, in_file: std.fs.File, path: []const u8, ctx: *ParseEr...@@ -1442,15 +1467,109 @@ fn parseObject(self: *Elf, in_file: std.fs.File, path: []const u8, ctx: *ParseEr
1442 if (ctx.detected_cpu_arch != self.base.options.target.cpu.arch) return error.InvalidCpuArch;1467 if (ctx.detected_cpu_arch != self.base.options.target.cpu.arch) return error.InvalidCpuArch;
1443}1468}
14441469
1445fn resolveSymbols(self: *Elf) void {1470fn parseArchive(
1446 if (self.zig_module_index) |index| {1471 self: *Elf,
1447 const zig_module = self.file(index).?.zig_module;1472 in_file: std.fs.File,
1448 zig_module.resolveSymbols(self);1473 path: []const u8,
1474 must_link: bool,
1475 ctx: *ParseErrorCtx,
1476) ParseError!void {
1477 const tracy = trace(@src());
1478 defer tracy.end();
1479
1480 const gpa = self.base.allocator;
1481 const data = try in_file.readToEndAlloc(gpa, std.math.maxInt(u32));
1482 var archive = Archive{ .path = path, .data = data };
1483 defer archive.deinit(gpa);
1484 try archive.parse(self);
1485
1486 for (archive.objects.items) |extracted| {
1487 const index = @as(File.Index, @intCast(try self.files.addOne(gpa)));
1488 self.files.set(index, .{ .object = extracted });
1489 const object = &self.files.items(.data)[index].object;
1490 object.index = index;
1491 object.alive = must_link;
1492 try object.parse(self);
1493 try self.objects.append(gpa, index);
1494
1495 ctx.detected_cpu_arch = object.header.?.e_machine.toTargetCpuArch().?;
1496 if (ctx.detected_cpu_arch != self.base.options.target.cpu.arch) return error.InvalidCpuArch;
1497 }
1498}
1499
1500/// When resolving symbols, we approach the problem similarly to `mold`.
1501/// 1. Resolve symbols across all objects (including those preemptively extracted archives).
1502/// 2. Resolve symbols across all shared objects.
1503/// 3. Mark live objects (see `Elf.markLive`)
1504/// 4. Reset state of all resolved globals since we will redo this bit on the pruned set.
1505/// 5. Remove references to dead objects/shared objects
1506/// 6. Re-run symbol resolution on pruned objects and shared objects sets.
1507fn resolveSymbols(self: *Elf) error{Overflow}!void {
1508 // Resolve symbols in the ZigModule. For now, we assume that it's always live.
1509 if (self.zig_module_index) |index| self.file(index).?.resolveSymbols(self);
1510 // Resolve symbols on the set of all objects and shared objects (even if some are unneeded).
1511 for (self.objects.items) |index| self.file(index).?.resolveSymbols(self);
1512
1513 // Mark live objects.
1514 self.markLive();
1515
1516 // Reset state of all globals after marking live objects.
1517 if (self.zig_module_index) |index| self.file(index).?.resetGlobals(self);
1518 for (self.objects.items) |index| self.file(index).?.resetGlobals(self);
1519
1520 // Prune dead objects and shared objects.
1521 var i: usize = 0;
1522 while (i < self.objects.items.len) {
1523 const index = self.objects.items[i];
1524 if (!self.file(index).?.isAlive()) {
1525 _ = self.objects.orderedRemove(i);
1526 } else i += 1;
1449 }1527 }
14501528
1529 // Dedup comdat groups.
1451 for (self.objects.items) |index| {1530 for (self.objects.items) |index| {
1452 const object = self.file(index).?.object;1531 const object = self.file(index).?.object;
1453 object.resolveSymbols(self);1532 for (object.comdat_groups.items) |cg_index| {
1533 const cg = self.comdatGroup(cg_index);
1534 const cg_owner = self.comdatGroupOwner(cg.owner);
1535 const owner_file_index = if (self.file(cg_owner.file)) |file_ptr|
1536 file_ptr.object.index
1537 else
1538 std.math.maxInt(File.Index);
1539 cg_owner.file = @min(owner_file_index, index);
1540 }
1541 }
1542
1543 for (self.objects.items) |index| {
1544 const object = self.file(index).?.object;
1545 for (object.comdat_groups.items) |cg_index| {
1546 const cg = self.comdatGroup(cg_index);
1547 const cg_owner = self.comdatGroupOwner(cg.owner);
1548 if (cg_owner.file != index) {
1549 for (try object.comdatGroupMembers(cg.shndx)) |shndx| {
1550 const atom_index = object.atoms.items[shndx];
1551 if (self.atom(atom_index)) |atom_ptr| {
1552 atom_ptr.alive = false;
1553 // atom_ptr.markFdesDead(self);
1554 }
1555 }
1556 }
1557 }
1558 }
1559
1560 // Re-resolve the symbols.
1561 if (self.zig_module_index) |index| self.file(index).?.resolveSymbols(self);
1562 for (self.objects.items) |index| self.file(index).?.resolveSymbols(self);
1563}
1564
1565/// Traverses all objects and shared objects marking any object referenced by
1566/// a live object/shared object as alive itself.
1567/// This routine will prune unneeded objects extracted from archives and
1568/// unneeded shared objects.
1569fn markLive(self: *Elf) void {
1570 for (self.objects.items) |index| {
1571 const file_ptr = self.file(index).?;
1572 if (file_ptr.isAlive()) file_ptr.markLive(self);
1454 }1573 }
1455}1574}
14561575
...@@ -4059,6 +4178,7 @@ const synthetic_sections = @import("Elf/synthetic_sections.zig");...@@ -4059,6 +4178,7 @@ const synthetic_sections = @import("Elf/synthetic_sections.zig");
40594178
4060const Air = @import("../Air.zig");4179const Air = @import("../Air.zig");
4061const Allocator = std.mem.Allocator;4180const Allocator = std.mem.Allocator;
4181const Archive = @import("Elf/Archive.zig");
4062pub const Atom = @import("Elf/Atom.zig");4182pub const Atom = @import("Elf/Atom.zig");
4063const Cache = std.Build.Cache;4183const Cache = std.Build.Cache;
4064const Compilation = @import("../Compilation.zig");4184const Compilation = @import("../Compilation.zig");
src/link/Elf/Archive.zig created+153
...@@ -0,0 +1,153 @@
1path: []const u8,
2data: []const u8,
3
4objects: std.ArrayListUnmanaged(Object) = .{},
5strtab: []const u8 = &[0]u8{},
6
7// Archive files start with the ARMAG identifying string. Then follows a
8// `struct ar_hdr', and as many bytes of member file data as its `ar_size'
9// member indicates, for each member file.
10/// String that begins an archive file.
11pub const ARMAG: *const [SARMAG:0]u8 = "!<arch>\n";
12/// Size of that string.
13pub const SARMAG: u4 = 8;
14
15/// String in ar_fmag at the end of each header.
16const ARFMAG: *const [2:0]u8 = "`\n";
17
18const SYM64NAME: *const [7:0]u8 = "/SYM64/";
19
20const ar_hdr = extern struct {
21 /// Member file name, sometimes / terminated.
22 ar_name: [16]u8,
23
24 /// File date, decimal seconds since Epoch.
25 ar_date: [12]u8,
26
27 /// User ID, in ASCII format.
28 ar_uid: [6]u8,
29
30 /// Group ID, in ASCII format.
31 ar_gid: [6]u8,
32
33 /// File mode, in ASCII octal.
34 ar_mode: [8]u8,
35
36 /// File size, in ASCII decimal.
37 ar_size: [10]u8,
38
39 /// Always contains ARFMAG.
40 ar_fmag: [2]u8,
41
42 fn date(self: ar_hdr) !u64 {
43 const value = getValue(&self.ar_date);
44 return std.fmt.parseInt(u64, value, 10);
45 }
46
47 fn size(self: ar_hdr) !u32 {
48 const value = getValue(&self.ar_size);
49 return std.fmt.parseInt(u32, value, 10);
50 }
51
52 fn getValue(raw: []const u8) []const u8 {
53 return mem.trimRight(u8, raw, &[_]u8{@as(u8, 0x20)});
54 }
55
56 fn isStrtab(self: ar_hdr) bool {
57 return mem.eql(u8, getValue(&self.ar_name), "//");
58 }
59
60 fn isSymtab(self: ar_hdr) bool {
61 return mem.eql(u8, getValue(&self.ar_name), "/");
62 }
63};
64
65pub fn isArchive(file: std.fs.File) bool {
66 const reader = file.reader();
67 const magic = reader.readBytesNoEof(Archive.SARMAG) catch return false;
68 defer file.seekTo(0) catch {};
69 if (!mem.eql(u8, &magic, ARMAG)) return false;
70 return true;
71}
72
73pub fn deinit(self: *Archive, allocator: Allocator) void {
74 allocator.free(self.data);
75 self.objects.deinit(allocator);
76}
77
78pub fn parse(self: *Archive, elf_file: *Elf) !void {
79 const gpa = elf_file.base.allocator;
80
81 var stream = std.io.fixedBufferStream(self.data);
82 const reader = stream.reader();
83 _ = try reader.readBytesNoEof(SARMAG);
84
85 while (true) {
86 if (stream.pos % 2 != 0) {
87 stream.pos += 1;
88 }
89
90 const hdr = reader.readStruct(ar_hdr) catch break;
91
92 if (!mem.eql(u8, &hdr.ar_fmag, ARFMAG)) {
93 // TODO convert into an error
94 log.debug(
95 "{s}: invalid header delimiter: expected '{s}', found '{s}'",
96 .{ self.path, std.fmt.fmtSliceEscapeLower(ARFMAG), std.fmt.fmtSliceEscapeLower(&hdr.ar_fmag) },
97 );
98 return;
99 }
100
101 const size = try hdr.size();
102 defer {
103 _ = stream.seekBy(size) catch {};
104 }
105
106 if (hdr.isSymtab()) continue;
107 if (hdr.isStrtab()) {
108 self.strtab = self.data[stream.pos..][0..size];
109 continue;
110 }
111
112 const name = ar_hdr.getValue(&hdr.ar_name);
113
114 if (mem.eql(u8, name, "__.SYMDEF") or mem.eql(u8, name, "__.SYMDEF SORTED")) continue;
115
116 const object_name = blk: {
117 if (name[0] == '/') {
118 const off = try std.fmt.parseInt(u32, name[1..], 10);
119 break :blk self.getString(off);
120 }
121 break :blk name;
122 };
123
124 const object = Object{
125 .archive = self.path,
126 .path = try gpa.dupe(u8, object_name[0 .. object_name.len - 1]), // To account for trailing '/'
127 .data = try gpa.dupe(u8, self.data[stream.pos..][0..size]),
128 .index = undefined,
129 .alive = false,
130 };
131
132 log.debug("extracting object '{s}' from archive '{s}'", .{ object.path, self.path });
133
134 try self.objects.append(gpa, object);
135 }
136}
137
138fn getString(self: Archive, off: u32) []const u8 {
139 assert(off < self.strtab.len);
140 return mem.sliceTo(@as([*:'\n']const u8, @ptrCast(self.strtab.ptr + off)), 0);
141}
142
143const std = @import("std");
144const assert = std.debug.assert;
145const elf = std.elf;
146const fs = std.fs;
147const log = std.log.scoped(.link);
148const mem = std.mem;
149
150const Allocator = mem.Allocator;
151const Archive = @This();
152const Elf = @import("../Elf.zig");
153const Object = @import("Object.zig");
src/link/Elf/Object.zig+3-3
...@@ -485,9 +485,9 @@ pub fn claimUnresolved(self: *Object, elf_file: *Elf) void {...@@ -485,9 +485,9 @@ pub fn claimUnresolved(self: *Object, elf_file: *Elf) void {
485pub fn resetGlobals(self: *Object, elf_file: *Elf) void {485pub fn resetGlobals(self: *Object, elf_file: *Elf) void {
486 for (self.globals()) |index| {486 for (self.globals()) |index| {
487 const global = elf_file.symbol(index);487 const global = elf_file.symbol(index);
488 const name = global.name;488 const off = global.name_offset;
489 global.* = .{};489 global.* = .{};
490 global.name = name;490 global.name_offset = off;
491 }491 }
492}492}
493493
...@@ -499,7 +499,7 @@ pub fn markLive(self: *Object, elf_file: *Elf) void {...@@ -499,7 +499,7 @@ pub fn markLive(self: *Object, elf_file: *Elf) void {
499 if (sym.st_bind() == elf.STB_WEAK) continue;499 if (sym.st_bind() == elf.STB_WEAK) continue;
500500
501 const global = elf_file.symbol(index);501 const global = elf_file.symbol(index);
502 const file = global.getFile(elf_file) orelse continue;502 const file = global.file(elf_file) orelse continue;
503 const should_keep = sym.st_shndx == elf.SHN_UNDEF or503 const should_keep = sym.st_shndx == elf.SHN_UNDEF or
504 (sym.st_shndx == elf.SHN_COMMON and global.elfSym(elf_file).st_shndx != elf.SHN_COMMON);504 (sym.st_shndx == elf.SHN_COMMON and global.elfSym(elf_file).st_shndx != elf.SHN_COMMON);
505 if (should_keep and !file.isAlive()) {505 if (should_keep and !file.isAlive()) {
src/link/Elf/ZigModule.zig+9
...@@ -148,6 +148,15 @@ pub fn scanRelocs(self: *ZigModule, elf_file: *Elf, undefs: anytype) !void {...@@ -148,6 +148,15 @@ pub fn scanRelocs(self: *ZigModule, elf_file: *Elf, undefs: anytype) !void {
148 }148 }
149}149}
150150
151pub fn resetGlobals(self: *ZigModule, elf_file: *Elf) void {
152 for (self.globals()) |index| {
153 const global = elf_file.symbol(index);
154 const off = global.name_offset;
155 global.* = .{};
156 global.name_offset = off;
157 }
158}
159
151pub fn updateSymtabSize(self: *ZigModule, elf_file: *Elf) void {160pub fn updateSymtabSize(self: *ZigModule, elf_file: *Elf) void {
152 for (self.locals()) |local_index| {161 for (self.locals()) |local_index| {
153 const local = elf_file.symbol(local_index);162 const local = elf_file.symbol(local_index);
src/link/Elf/file.zig+13
...@@ -62,6 +62,19 @@ pub const File = union(enum) {...@@ -62,6 +62,19 @@ pub const File = union(enum) {
62 return (@as(u32, base) << 24) + file.index();62 return (@as(u32, base) << 24) + file.index();
63 }63 }
6464
65 pub fn resolveSymbols(file: File, elf_file: *Elf) void {
66 switch (file) {
67 inline else => |x| x.resolveSymbols(elf_file),
68 }
69 }
70
71 pub fn resetGlobals(file: File, elf_file: *Elf) void {
72 switch (file) {
73 .linker_defined => unreachable,
74 inline else => |x| x.resetGlobals(elf_file),
75 }
76 }
77
65 pub fn setAlive(file: File) void {78 pub fn setAlive(file: File) void {
66 switch (file) {79 switch (file) {
67 .zig_module, .linker_defined => {},80 .zig_module, .linker_defined => {},