authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-07-09 10:15:37+01:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2026-07-09 15:41:19-04:00
log0fe0d69c63888d92a959ff20ad032348b5a2bd53
tree175d12e4bd287461db710c5feb418d9c1b63d33f
parentebaa801cdcaf3fd480b4fe6f87136d8ef3a5a0d7

Elf2: make binaries more reproducible

Binaries emitted by the Elf2 linker are currently non-reproducible for several reasons (these will be fixed, don't worry!). This solves the most serious reproducibility issue, which is that we were processing UAVs and lazy code/data as "idle tasks", but because generating those symbols resizes `MappedFile` nodes and adds relocations, the output file contents could differ wildly between runs. We instead need to run these operations deterministically, during `updateNav` etc.

1 files changed, 97 insertions(+), 72 deletions(-)

src/link/Elf2.zig+97-72
......@@ -4796,7 +4796,7 @@ fn uavMapIndex(
47964796 if (!uav_gop.found_existing) {
47974797 const shndx: Section.Index = .data_rel_ro; // TODO: it would be better to use `.rodata` if the UAV value doesn't have relocs
47984798 const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{
4799 .moved = true, // see assert at end of `flushUav`
4799 .moved = true, // see assert at end of `genUav`
48004800 .alignment = resolved_align.toStdMem(),
48014801 });
48024802 var name_buf: [32]u8 = undefined;
......@@ -7005,22 +7005,27 @@ fn updateNavInner(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index)
70057005 // called to apply the NAV's new relocations.
70067006 try ni.moved(gpa, &elf.mf);
70077007
7008 var nw: MappedFile.Node.Writer = undefined;
7009 ni.writer(&elf.mf, gpa, &nw);
7010 defer nw.deinit();
7011 codegen.generateSymbol(
7012 &elf.base,
7013 pt,
7014 .fromInterned(nav.resolved.?.value),
7015 &nw.interface,
7016 .{ .atom_index = Node.toAtom(ni) },
7017 ) catch |err| switch (err) {
7018 error.WriteFailed => return nw.err.?,
7019 else => |e| return e,
7020 };
7021 switch (elf.symPtr(nmi.symbol(elf).index())) {
7022 inline else => |sym| elf.targetStore(&sym.size, @intCast(nw.interface.end)),
7008 {
7009 var nw: MappedFile.Node.Writer = undefined;
7010 ni.writer(&elf.mf, gpa, &nw);
7011 defer nw.deinit();
7012 codegen.generateSymbol(
7013 &elf.base,
7014 pt,
7015 .fromInterned(nav.resolved.?.value),
7016 &nw.interface,
7017 .{ .atom_index = Node.toAtom(ni) },
7018 ) catch |err| switch (err) {
7019 error.WriteFailed => return nw.err.?,
7020 else => |e| return e,
7021 };
7022 switch (elf.symPtr(nmi.symbol(elf).index())) {
7023 inline else => |sym| elf.targetStore(&sym.size, @intCast(nw.interface.end)),
7024 }
70237025 }
7026
7027 // The NAV's node is done---now generate any UAVs or lazy code/data which the NAV needs.
7028 try elf.genPending(pt);
70247029}
70257030
70267031pub fn updateFunc(
......@@ -7056,29 +7061,34 @@ fn updateFuncInner(
70567061 // called to apply the NAV's new relocations.
70577062 try ni.moved(gpa, &elf.mf);
70587063
7059 var nw: MappedFile.Node.Writer = undefined;
7060 ni.writer(&elf.mf, gpa, &nw);
7061 defer nw.deinit();
7062 codegen.emitFunction(
7063 &elf.base,
7064 pt,
7065 func_index,
7066 Node.toAtom(ni),
7067 mir,
7068 &nw.interface,
7069 .none,
7070 ) catch |err| switch (err) {
7071 error.WriteFailed => return nw.err.?,
7072 else => |e| return e,
7073 };
7074 switch (elf.symPtr(nmi.symbol(elf).index())) {
7075 inline else => |sym| elf.targetStore(&sym.size, @intCast(nw.interface.end)),
7064 {
7065 var nw: MappedFile.Node.Writer = undefined;
7066 ni.writer(&elf.mf, gpa, &nw);
7067 defer nw.deinit();
7068 codegen.emitFunction(
7069 &elf.base,
7070 pt,
7071 func_index,
7072 Node.toAtom(ni),
7073 mir,
7074 &nw.interface,
7075 .none,
7076 ) catch |err| switch (err) {
7077 error.WriteFailed => return nw.err.?,
7078 else => |e| return e,
7079 };
7080 switch (elf.symPtr(nmi.symbol(elf).index())) {
7081 inline else => |sym| elf.targetStore(&sym.size, @intCast(nw.interface.end)),
7082 }
70767083 }
7084
7085 // The NAV's node is done---now generate any UAVs or lazy code/data which the NAV needs.
7086 try elf.genPending(pt);
70777087}
70787088
70797089pub fn updateErrorData(elf: *Elf, pt: Zcu.PerThread) link.Error!void {
70807090 const diags = &elf.base.comp.link_diags;
7081 elf.flushLazy(pt, .{
7091 elf.genLazy(pt, .{
70827092 .kind = .const_data,
70837093 .index = @intCast(elf.lazy.getPtr(.const_data).map.getIndex(.anyerror_type) orelse return),
70847094 }) catch |err| switch (err) {
......@@ -7183,40 +7193,13 @@ fn updateDynamicTextrel(elf: *Elf) Error!void {
71837193pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) link.Error!bool {
71847194 const comp = elf.base.comp;
71857195 const diags = &comp.link_diags;
7196
7197 assert(elf.pending_uavs.items.len == 0);
7198 for (&elf.lazy.values) |*lazy| {
7199 assert(lazy.pending_index == lazy.map.count());
7200 }
7201
71867202 task: {
7187 while (elf.pending_uavs.pop()) |umi| {
7188 const sub_prog_node = elf.idleProgNode(tid, elf.const_prog_node, .{ .uav = umi });
7189 defer sub_prog_node.end();
7190 elf.flushUav(.{ .zcu = comp.zcu.?, .tid = tid }, umi) catch |err| switch (err) {
7191 error.MappedFileIo => return diags.fail("failed to write output file: {t}", .{elf.mf.io_err.?}),
7192 else => |e| return e,
7193 };
7194 break :task;
7195 }
7196 var lazy_it = elf.lazy.iterator();
7197 while (lazy_it.next()) |lazy| if (lazy.value.pending_index < lazy.value.map.count()) {
7198 const pt: Zcu.PerThread = .{ .zcu = comp.zcu.?, .tid = tid };
7199 const lmr: Node.LazyMapRef = .{ .kind = lazy.key, .index = lazy.value.pending_index };
7200 lazy.value.pending_index += 1;
7201 const kind = switch (lmr.kind) {
7202 .code => "code",
7203 .const_data => "data",
7204 };
7205 var name: [std.Progress.Node.max_name_len]u8 = undefined;
7206 const sub_prog_node = elf.synth_prog_node.start(
7207 std.fmt.bufPrint(&name, "lazy {s} for {f}", .{
7208 kind,
7209 Type.fromInterned(lmr.lazySymbol(elf).ty).fmt(pt),
7210 }) catch &name,
7211 0,
7212 );
7213 defer sub_prog_node.end();
7214 elf.flushLazy(pt, lmr) catch |err| switch (err) {
7215 error.MappedFileIo => return diags.fail("failed to write output file: {t}", .{elf.mf.io_err.?}),
7216 else => |e| return e,
7217 };
7218 break :task;
7219 };
72207203 if (elf.input_section_pending_index < elf.input_sections.items.len) {
72217204 const isi: InputSection.Index = @enumFromInt(elf.input_section_pending_index);
72227205 elf.input_section_pending_index += 1;
......@@ -7315,8 +7298,6 @@ pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) link.Error!bool {
73157298 } else elf.mf.update_prog_node.completeOne();
73167299 }
73177300 }
7318 if (elf.pending_uavs.items.len > 0) return true;
7319 for (&elf.lazy.values) |lazy| if (lazy.map.count() > lazy.pending_index) return true;
73207301 if (elf.input_sections.items.len > elf.input_section_pending_index) return true;
73217302 if (elf.changed_symtab_index.count() > 0) return true;
73227303 if (elf.mf.updates.items.len > 0) return true;
......@@ -7351,7 +7332,43 @@ fn idleProgNode(
73517332 }, 0);
73527333}
73537334
7354fn flushUav(
7335fn genPending(elf: *Elf, pt: Zcu.PerThread) Error!void {
7336 const zcu = elf.base.comp.zcu.?;
7337 pending: while (true) {
7338 if (elf.pending_uavs.pop()) |umi| {
7339 var prog_name_buf: [std.Progress.Node.max_name_len]u8 = undefined;
7340 const prog_name = std.mem.print(&prog_name_buf, "{f}", .{
7341 Value.fromInterned(umi.uavValue(elf)).fmtValue(pt),
7342 }) catch &prog_name_buf;
7343 const prog_node = elf.const_prog_node.start(prog_name, 0);
7344 defer prog_node.end();
7345 try elf.genUav(pt, umi);
7346 continue :pending;
7347 }
7348 var lazy_it = elf.lazy.iterator();
7349 while (lazy_it.next()) |lazy| if (lazy.value.pending_index < lazy.value.map.count()) {
7350 const lmr: Node.LazyMapRef = .{ .kind = lazy.key, .index = lazy.value.pending_index };
7351 lazy.value.pending_index += 1;
7352 const lazy_ty: Type = .fromInterned(lmr.lazySymbol(elf).ty);
7353 var prog_name_buf: [std.Progress.Node.max_name_len]u8 = undefined;
7354 const prog_name: []const u8 = switch (lazy_ty.zigTypeTag(zcu)) {
7355 .@"enum" => std.mem.print(&prog_name_buf, "@tagName({f})", .{lazy_ty.fmt(pt)}) catch &prog_name_buf,
7356 .error_set => switch (lmr.kind) {
7357 .code => std.mem.print(&prog_name_buf, "@errorCast({f})", .{lazy_ty.fmt(pt)}) catch &prog_name_buf,
7358 .const_data => "@errorName",
7359 },
7360 else => unreachable,
7361 };
7362 const prog_node = elf.synth_prog_node.start(prog_name, 0);
7363 defer prog_node.end();
7364 try elf.genLazy(pt, lmr);
7365 continue :pending;
7366 };
7367 break;
7368 }
7369}
7370
7371fn genUav(
73557372 elf: *Elf,
73567373 pt: Zcu.PerThread,
73577374 umi: Node.UavMapIndex,
......@@ -7380,11 +7397,11 @@ fn flushUav(
73807397 inline else => |sym| elf.targetStore(&sym.size, @intCast(nw.interface.end)),
73817398 }
73827399 // The UAV should already be considered to have moved, because it is created as moved and
7383 // pending calls to `flushUav` always happen before pending calls to `flushMoved`.
7400 // pending calls to `genUav` always happen before pending calls to `flushMoved`.
73847401 assert(ni.hasMoved(&elf.mf));
73857402}
73867403
7387fn flushLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void {
7404fn genLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void {
73887405 const zcu = pt.zcu;
73897406 const gpa = zcu.gpa;
73907407
......@@ -7497,6 +7514,10 @@ fn flushFileOffset(elf: *Elf, ni: MappedFile.Node.Index) void {
74977514fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void {
74987515 const trace = tracy.trace(@src());
74997516 defer trace.end();
7517
7518 elf.mf.nodes_lock.lock();
7519 defer elf.mf.nodes_lock.unlock();
7520
75007521 switch (elf.getNode(ni)) {
75017522 .file => unreachable,
75027523 .ehdr, .shdr => elf.flushFileOffset(ni),
......@@ -7662,6 +7683,10 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void
76627683fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void {
76637684 const trace = tracy.trace(@src());
76647685 defer trace.end();
7686
7687 elf.mf.nodes_lock.lock();
7688 defer elf.mf.nodes_lock.unlock();
7689
76657690 _, const size = ni.location(&elf.mf).resolve(&elf.mf);
76667691 switch (elf.getNode(ni)) {
76677692 .file => {},