authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2020-10-08 17:52:08+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2020-10-08 19:53:03+02:00
log5f86505cf79a0ce75e1a02602ae0e9c845024982
tree951e93be9825ff007af21949516263f874d28b01
parentea44d12d1be8eb17a1555f6ab794621da0212171
signature Commit is signed but in an unrecognized format.

Fix ULEB128 encoding of trie

Use algorithm described in official Apple `ld64` implementation. Link: https://opensource.apple.com/source/ld64/ld64-123.2.1/src/abstraction/MachOTrie.hpp Signed-off-by: Jakub Konka <kubkon@jakubkonka.com>

2 files changed, 84 insertions(+), 62 deletions(-)

src/link/MachO.zig+1-1
......@@ -1415,7 +1415,7 @@ fn writeExportTrie(self: *MachO) !void {
14151415 assert(symbol.n_value >= text_segment.vmaddr);
14161416 try trie.put(self.base.allocator, .{
14171417 .name = name,
1418 .offset = symbol.n_value - text_segment.vmaddr,
1418 .vmaddr_offset = symbol.n_value - text_segment.vmaddr,
14191419 .export_flags = 0, // TODO workout creation of export flags
14201420 });
14211421 }
src/link/MachO/Trie.zig+83-61
......@@ -39,7 +39,7 @@ const Allocator = mem.Allocator;
3939
4040pub const Symbol = struct {
4141 name: []const u8,
42 offset: u64,
42 vmaddr_offset: u64,
4343 export_flags: u64,
4444};
4545
......@@ -58,7 +58,8 @@ const Edge = struct {
5858
5959const Node = struct {
6060 export_flags: ?u64 = null,
61 offset: ?u64 = null,
61 vmaddr_offset: ?u64 = null,
62 trie_offset: usize = 0,
6263 edges: std.ArrayListUnmanaged(Edge) = .{},
6364
6465 fn deinit(self: *Node, alloc: *Allocator) void {
......@@ -111,8 +112,8 @@ const Node = struct {
111112 return node;
112113 }
113114
114 fn writeULEB128Mem(self: Node, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) Trie.WriteError!void {
115 if (self.offset) |offset| {
115 fn writeULEB128Mem(self: Node, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) !void {
116 if (self.vmaddr_offset) |offset| {
116117 // Terminal node info: encode export flags and vmaddr offset of this symbol.
117118 var info_buf_len: usize = 0;
118119 var info_buf: [@sizeOf(u64) * 2]u8 = undefined;
......@@ -134,34 +135,53 @@ const Node = struct {
134135 // Write number of edges (max legal number of edges is 256).
135136 try buffer.append(alloc, @intCast(u8, self.edges.items.len));
136137
137 var node_offset_info: [std.math.maxInt(u8)]u64 = undefined;
138 for (self.edges.items) |edge, i| {
139 // Write edges labels leaving out space in-between to later populate
140 // with offsets to each node.
141 try buffer.ensureCapacity(alloc, buffer.items.len + edge.label.len + 1 + @sizeOf(u64)); // +1 to account for null-byte
138 for (self.edges.items) |edge| {
139 // Write edges labels.
140 try buffer.ensureCapacity(alloc, buffer.items.len + edge.label.len + 1); // +1 to account for null-byte
142141 buffer.appendSliceAssumeCapacity(edge.label);
143142 buffer.appendAssumeCapacity(0);
144 node_offset_info[i] = buffer.items.len;
145 const padding = [_]u8{0} ** @sizeOf(u64);
146 buffer.appendSliceAssumeCapacity(padding[0..]);
143
144 var buf: [@sizeOf(u64)]u8 = undefined;
145 const buf_len = try leb.writeULEB128Mem(buf[0..], edge.to.trie_offset);
146 try buffer.appendSlice(alloc, buf[0..buf_len]);
147147 }
148 }
149
150 const UpdateResult = struct {
151 node_size: usize,
152 updated: bool,
153 };
148154
149 for (self.edges.items) |edge, i| {
150 const offset = buffer.items.len;
151 try edge.to.writeULEB128Mem(alloc, buffer);
152 // We can now populate the offset to the node pointed by this edge.
153 // TODO this is not the approach taken by `ld64` which does several iterations
154 // to close the gap between the space encoding the offset to the node pointed
155 // by this edge. However, it seems that as long as we are contiguous, the padding
156 // introduced here should not influence the performance of `dyld`. I'm leaving
157 // this TODO here though as a reminder to re-investigate in the future and especially
158 // when we start working on dylibs in case `dyld` refuses to cooperate and/or the
159 // performance is noticably sufferring.
160 // Link to official impl: https://opensource.apple.com/source/ld64/ld64-123.2.1/src/abstraction/MachOTrie.hpp
161 var offset_buf: [@sizeOf(u64)]u8 = undefined;
162 const offset_buf_len = try leb.writeULEB128Mem(offset_buf[0..], offset);
163 mem.copy(u8, buffer.items[node_offset_info[i]..], offset_buf[0..offset_buf_len]);
155 fn updateOffset(self: *Node, offset: usize) UpdateResult {
156 var node_size: usize = 0;
157 if (self.vmaddr_offset) |vmaddr| {
158 node_size += sizeULEB128Mem(self.export_flags.?);
159 node_size += sizeULEB128Mem(vmaddr);
160 node_size += sizeULEB128Mem(node_size);
161 } else {
162 node_size += 1; // 0x0 for non-terminal nodes
164163 }
164 node_size += 1; // 1 byte for edge count
165
166 for (self.edges.items) |edge| {
167 node_size += edge.label.len + 1 + sizeULEB128Mem(edge.to.trie_offset);
168 }
169
170 const updated = offset != self.trie_offset;
171 self.trie_offset = offset;
172
173 return .{ .node_size = node_size, .updated = updated };
174 }
175
176 fn sizeULEB128Mem(value: u64) usize {
177 var res: usize = 0;
178 var v = value;
179 while (true) {
180 v = v >> 7;
181 res += 1;
182 if (v == 0) break;
183 }
184 return res;
165185 }
166186};
167187
......@@ -172,15 +192,38 @@ root: Node,
172192/// certain circumstances.
173193pub fn put(self: *Trie, alloc: *Allocator, symbol: Symbol) !void {
174194 const node = try self.root.put(alloc, symbol.name);
175 node.offset = symbol.offset;
195 node.vmaddr_offset = symbol.vmaddr_offset;
176196 node.export_flags = symbol.export_flags;
177197}
178198
179pub const WriteError = error{ OutOfMemory, NoSpaceLeft };
180
181199/// Write the trie to a buffer ULEB128 encoded.
182pub fn writeULEB128Mem(self: Trie, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) WriteError!void {
183 return self.root.writeULEB128Mem(alloc, buffer);
200pub fn writeULEB128Mem(self: *Trie, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) !void {
201 var ordered_nodes: std.ArrayListUnmanaged(*Node) = .{};
202 defer ordered_nodes.deinit(alloc);
203
204 try walkInOrder(&self.root, alloc, &ordered_nodes);
205
206 var more: bool = true;
207 while (more) {
208 var offset: usize = 0;
209 more = false;
210 for (ordered_nodes.items) |node| {
211 const res = node.updateOffset(offset);
212 offset += res.node_size;
213 if (res.updated) more = true;
214 }
215 }
216
217 for (ordered_nodes.items) |node| {
218 try node.writeULEB128Mem(alloc, buffer);
219 }
220}
221
222fn walkInOrder(node: *Node, alloc: *Allocator, list: *std.ArrayListUnmanaged(*Node)) error{OutOfMemory}!void {
223 try list.append(alloc, node);
224 for (node.edges.items) |*edge| {
225 try walkInOrder(edge.to, alloc, list);
226 }
184227}
185228
186229pub fn deinit(self: *Trie, alloc: *Allocator) void {
......@@ -200,7 +243,7 @@ test "Trie basic" {
200243 // root --- _st ---> node
201244 try trie.put(gpa, .{
202245 .name = "_st",
203 .offset = 0,
246 .vmaddr_offset = 0,
204247 .export_flags = 0,
205248 });
206249 testing.expect(trie.root.edges.items.len == 1);
......@@ -210,7 +253,7 @@ test "Trie basic" {
210253 // root --- _st ---> node --- art ---> node
211254 try trie.put(gpa, .{
212255 .name = "_start",
213 .offset = 0,
256 .vmaddr_offset = 0,
214257 .export_flags = 0,
215258 });
216259 testing.expect(trie.root.edges.items.len == 1);
......@@ -226,7 +269,7 @@ test "Trie basic" {
226269 // | --- main ---> node
227270 try trie.put(gpa, .{
228271 .name = "_main",
229 .offset = 0,
272 .vmaddr_offset = 0,
230273 .export_flags = 0,
231274 });
232275 testing.expect(trie.root.edges.items.len == 1);
......@@ -251,12 +294,12 @@ test "Trie.writeULEB128Mem" {
251294
252295 try trie.put(gpa, .{
253296 .name = "__mh_execute_header",
254 .offset = 0,
297 .vmaddr_offset = 0,
255298 .export_flags = 0,
256299 });
257300 try trie.put(gpa, .{
258301 .name = "_main",
259 .offset = 0x1000,
302 .vmaddr_offset = 0x1000,
260303 .export_flags = 0,
261304 });
262305
......@@ -270,14 +313,7 @@ test "Trie.writeULEB128Mem" {
270313 0x1,
271314 0x5f,
272315 0x0,
273 0xc,
274 0x0,
275 0x0,
276 0x0,
277 0x0,
278 0x0,
279 0x0,
280 0x0,
316 0x5,
281317 0x0,
282318 0x2,
283319 0x5f,
......@@ -299,27 +335,13 @@ test "Trie.writeULEB128Mem" {
299335 0x65,
300336 0x72,
301337 0x0,
302 0x36,
303 0x0,
304 0x0,
305 0x0,
306 0x0,
307 0x0,
308 0x0,
309 0x0,
338 0x21,
310339 0x6d,
311340 0x61,
312341 0x69,
313342 0x6e,
314343 0x0,
315 0x3a,
316 0x0,
317 0x0,
318 0x0,
319 0x0,
320 0x0,
321 0x0,
322 0x0,
344 0x25,
323345 0x2,
324346 0x0,
325347 0x0,