From f5f0d95f0e882ce43146b269f994c6c242f9ed3b Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Sun, 2 Apr 2023 20:36:41 -0400
Subject: [PATCH 01/13] x86_64: implement a more generic assembler for inline
 assembly

---
 src/arch/x86_64/CodeGen.zig | 199 ++++++++++++++++++++++++------------
 1 file changed, 135 insertions(+), 64 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index c7dfa2c6a9e312b1d7a1ccfaf8d443d531057670..8309fe7335e218bc3d31f18c3476675b06451a73 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -28,10 +28,11 @@ const Type = @import("../../type.zig").Type;
 const TypedValue = @import("../../TypedValue.zig");
 const Value = @import("../../value.zig").Value;
 
-const bits = @import("bits.zig");
 const abi = @import("abi.zig");
-const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
+const bits = @import("bits.zig");
+const encoder = @import("encoder.zig");
 const errUnionErrorOffset = codegen.errUnionErrorOffset;
+const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
 
 const Condition = bits.Condition;
 const Immediate = bits.Immediate;
@@ -6570,7 +6571,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
                 return self.fail("unrecognized constraint: '{s}'", .{constraint});
             args.putAssumeCapacity(name, mcv);
             switch (mcv) {
-                .register => |reg| _ = self.register_manager.lockRegAssumeUnused(reg),
+                .register => |reg| _ = if (RegisterManager.indexOfRegIntoTracked(reg)) |_|
+                    self.register_manager.lockRegAssumeUnused(reg),
                 else => {},
             }
             if (output == .none) result = mcv;
@@ -6609,70 +6611,139 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
         }
 
         const asm_source = mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
-        var line_it = mem.tokenize(u8, asm_source, "\n\r");
+        var line_it = mem.tokenize(u8, asm_source, "\n\r;");
         while (line_it.next()) |line| {
             var mnem_it = mem.tokenize(u8, line, " \t");
-            const mnem = mnem_it.next() orelse continue;
-            if (mem.startsWith(u8, mnem, "#")) continue;
-            var arg_it = mem.tokenize(u8, mnem_it.rest(), ", ");
-            if (std.ascii.eqlIgnoreCase(mnem, "syscall")) {
-                if (arg_it.next()) |trailing| if (!mem.startsWith(u8, trailing, "#"))
-                    return self.fail("Too many operands: '{s}'", .{line});
-                try self.asmOpOnly(.syscall);
-            } else if (std.ascii.eqlIgnoreCase(mnem, "push")) {
-                const src = arg_it.next() orelse
-                    return self.fail("Not enough operands: '{s}'", .{line});
-                if (arg_it.next()) |trailing| if (!mem.startsWith(u8, trailing, "#"))
-                    return self.fail("Too many operands: '{s}'", .{line});
-                if (mem.startsWith(u8, src, "$")) {
-                    const imm = std.fmt.parseInt(u32, src["$".len..], 0) catch
-                        return self.fail("Invalid immediate: '{s}'", .{src});
-                    try self.asmImmediate(.push, Immediate.u(imm));
-                } else if (mem.startsWith(u8, src, "%%")) {
-                    const reg = parseRegName(src["%%".len..]) orelse
-                        return self.fail("Invalid register: '{s}'", .{src});
-                    try self.asmRegister(.push, reg);
-                } else return self.fail("Unsupported operand: '{s}'", .{src});
-            } else if (std.ascii.eqlIgnoreCase(mnem, "pop")) {
-                const dst = arg_it.next() orelse
-                    return self.fail("Not enough operands: '{s}'", .{line});
-                if (arg_it.next()) |trailing| if (!mem.startsWith(u8, trailing, "#"))
-                    return self.fail("Too many operands: '{s}'", .{line});
-                if (mem.startsWith(u8, dst, "%%")) {
-                    const reg = parseRegName(dst["%%".len..]) orelse
-                        return self.fail("Invalid register: '{s}'", .{dst});
-                    try self.asmRegister(.pop, reg);
-                } else return self.fail("Unsupported operand: '{s}'", .{dst});
-            } else if (std.ascii.eqlIgnoreCase(mnem, "movq")) {
-                const src = arg_it.next() orelse
-                    return self.fail("Not enough operands: '{s}'", .{line});
-                const dst = arg_it.next() orelse
-                    return self.fail("Not enough operands: '{s}'", .{line});
-                if (arg_it.next()) |trailing| if (!mem.startsWith(u8, trailing, "#"))
-                    return self.fail("Too many operands: '{s}'", .{line});
-                if (mem.startsWith(u8, src, "%%")) {
-                    const colon = mem.indexOfScalarPos(u8, src, "%%".len + 2, ':');
-                    const src_reg = parseRegName(src["%%".len .. colon orelse src.len]) orelse
-                        return self.fail("Invalid register: '{s}'", .{src});
+            const mnem_str = mnem_it.next() orelse continue;
+            if (mem.startsWith(u8, mnem_str, "#")) continue;
+
+            const mnem_size: ?Memory.PtrSize = if (mem.endsWith(u8, mnem_str, "b"))
+                .byte
+            else if (mem.endsWith(u8, mnem_str, "w"))
+                .word
+            else if (mem.endsWith(u8, mnem_str, "l"))
+                .dword
+            else if (mem.endsWith(u8, mnem_str, "q"))
+                .qword
+            else
+                null;
+            const mnem = std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse
+                (if (mnem_size) |_|
+                std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])
+            else
+                null) orelse return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});
+
+            var op_it = mem.tokenize(u8, mnem_it.rest(), ",");
+            var ops = [1]encoder.Instruction.Operand{.none} ** 4;
+            for (&ops) |*op| {
+                const op_str = mem.trim(u8, op_it.next() orelse break, " \t");
+                if (mem.startsWith(u8, op_str, "#")) break;
+                if (mem.startsWith(u8, op_str, "%%")) {
+                    const colon = mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':');
+                    const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse
+                        return self.fail("Invalid register: '{s}'", .{op_str});
                     if (colon) |colon_pos| {
-                        const src_disp = std.fmt.parseInt(i32, src[colon_pos + 1 ..], 0) catch
-                            return self.fail("Invalid immediate: '{s}'", .{src});
-                        if (mem.startsWith(u8, dst, "%[") and mem.endsWith(u8, dst, "]")) {
-                            switch (args.get(dst["%[".len .. dst.len - "]".len]) orelse
-                                return self.fail("no matching constraint for: '{s}'", .{dst})) {
-                                .register => |dst_reg| try self.asmRegisterMemory(
-                                    .mov,
-                                    dst_reg,
-                                    Memory.sib(.qword, .{ .base = src_reg, .disp = src_disp }),
-                                ),
-                                else => return self.fail("Invalid constraint: '{s}'", .{dst}),
-                            }
-                        } else return self.fail("Unsupported operand: '{s}'", .{dst});
-                    } else return self.fail("Unsupported operand: '{s}'", .{src});
-                }
-            } else {
-                return self.fail("Unsupported instruction: '{s}'", .{mnem});
-            }
+                        const disp = std.fmt.parseInt(i32, op_str[colon_pos + 1 ..], 0) catch
+                            return self.fail("Invalid displacement: '{s}'", .{op_str});
+                        op.* = .{ .mem = Memory.sib(
+                            mnem_size orelse return self.fail("Unknown size: '{s}'", .{op_str}),
+                            .{ .base = reg, .disp = disp },
+                        ) };
+                    } else {
+                        if (mnem_size) |size| if (reg.bitSize() != size.bitSize())
+                            return self.fail("Invalid register size: '{s}'", .{op_str});
+                        op.* = .{ .reg = reg };
+                    }
+                } else if (mem.startsWith(u8, op_str, "%[") and mem.endsWith(u8, op_str, "]")) {
+                    switch (args.get(op_str["%[".len .. op_str.len - "]".len]) orelse
+                        return self.fail("No matching constraint: '{s}'", .{op_str})) {
+                        .register => |reg| op.* = .{ .reg = reg },
+                        else => return self.fail("Invalid constraint: '{s}'", .{op_str}),
+                    }
+                } else if (mem.startsWith(u8, op_str, "$")) {
+                    if (std.fmt.parseInt(i32, op_str["$".len..], 0)) |s| {
+                        if (mnem_size) |size| {
+                            const max = @as(u64, std.math.maxInt(u64)) >>
+                                @intCast(u6, 64 - (size.bitSize() - 1));
+                            if ((if (s < 0) ~s else s) > max)
+                                return self.fail("Invalid immediate size: '{s}'", .{op_str});
+                        }
+                        op.* = .{ .imm = Immediate.s(s) };
+                    } else |_| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) |u| {
+                        if (mnem_size) |size| {
+                            const max = @as(u64, std.math.maxInt(u64)) >>
+                                @intCast(u6, 64 - size.bitSize());
+                            if (u > max)
+                                return self.fail("Invalid immediate size: '{s}'", .{op_str});
+                        }
+                        op.* = .{ .imm = Immediate.u(u) };
+                    } else |_| return self.fail("Invalid immediate: '{s}'", .{op_str});
+                } else return self.fail("Invalid operand: '{s}'", .{op_str});
+            } else if (op_it.next()) |op_str| return self.fail("Extra operand: '{s}'", .{op_str});
+
+            (switch (ops[0]) {
+                .none => self.asmOpOnly(mnem),
+                .reg => |reg0| switch (ops[1]) {
+                    .none => self.asmRegister(mnem, reg0),
+                    .reg => |reg1| switch (ops[2]) {
+                        .none => self.asmRegisterRegister(mnem, reg1, reg0),
+                        .reg => |reg2| switch (ops[3]) {
+                            .none => self.asmRegisterRegisterRegister(mnem, reg2, reg1, reg0),
+                            else => error.InvalidInstruction,
+                        },
+                        .mem => |mem2| switch (ops[3]) {
+                            .none => self.asmMemoryRegisterRegister(mnem, mem2, reg1, reg0),
+                            else => error.InvalidInstruction,
+                        },
+                        else => error.InvalidInstruction,
+                    },
+                    .mem => |mem1| switch (ops[2]) {
+                        .none => self.asmMemoryRegister(mnem, mem1, reg0),
+                        else => error.InvalidInstruction,
+                    },
+                    else => error.InvalidInstruction,
+                },
+                .mem => |mem0| switch (ops[1]) {
+                    .none => self.asmMemory(mnem, mem0),
+                    .reg => |reg1| switch (ops[2]) {
+                        .none => self.asmRegisterMemory(mnem, reg1, mem0),
+                        else => error.InvalidInstruction,
+                    },
+                    else => error.InvalidInstruction,
+                },
+                .imm => |imm0| switch (ops[1]) {
+                    .none => self.asmImmediate(mnem, imm0),
+                    .reg => |reg1| switch (ops[2]) {
+                        .none => self.asmRegisterImmediate(mnem, reg1, imm0),
+                        .reg => |reg2| switch (ops[3]) {
+                            .none => self.asmRegisterRegisterImmediate(mnem, reg2, reg1, imm0),
+                            else => error.InvalidInstruction,
+                        },
+                        .mem => |mem2| switch (ops[3]) {
+                            .none => self.asmMemoryRegisterImmediate(mnem, mem2, reg1, imm0),
+                            else => error.InvalidInstruction,
+                        },
+                        else => error.InvalidInstruction,
+                    },
+                    .mem => |mem1| switch (ops[2]) {
+                        .none => self.asmMemoryImmediate(mnem, mem1, imm0),
+                        else => error.InvalidInstruction,
+                    },
+                    else => error.InvalidInstruction,
+                },
+            }) catch |err| switch (err) {
+                error.InvalidInstruction => return self.fail(
+                    "Invalid instruction: '{s} {s} {s} {s} {s}'",
+                    .{
+                        @tagName(mnem),
+                        @tagName(ops[0]),
+                        @tagName(ops[1]),
+                        @tagName(ops[2]),
+                        @tagName(ops[3]),
+                    },
+                ),
+                else => |e| return e,
+            };
         }
     }
 
-- 
2.54.0


From 272acb7ee5368a32d9f4bc559e93d96759b80173 Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Sun, 2 Apr 2023 21:39:20 -0400
Subject: [PATCH 02/13] x86_64: implement storing large immediates

---
 src/arch/x86_64/CodeGen.zig | 241 +++++++++++++++---------------------
 test/behavior/bugs/3007.zig |   1 -
 test/behavior/eval.zig      |   3 -
 3 files changed, 98 insertions(+), 147 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 8309fe7335e218bc3d31f18c3476675b06451a73..f3bd6a9e8a92f7106f68a55f54978e9dbbd6f7c3 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -3669,52 +3669,67 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
                 .none => unreachable,
                 .dead => unreachable,
                 .unreach => unreachable,
-                .eflags => |cc| {
-                    try self.asmSetccMemory(Memory.sib(
-                        Memory.PtrSize.fromSize(abi_size),
-                        .{ .base = reg.to64() },
-                    ), cc);
+                .eflags => |cc| try self.asmSetccMemory(
+                    Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
+                    cc,
+                ),
+                .undef => if (self.wantSafety()) switch (abi_size) {
+                    1 => try self.store(ptr, .{ .immediate = 0xaa }, ptr_ty, value_ty),
+                    2 => try self.store(ptr, .{ .immediate = 0xaaaa }, ptr_ty, value_ty),
+                    4 => try self.store(ptr, .{ .immediate = 0xaaaaaaaa }, ptr_ty, value_ty),
+                    8 => try self.store(ptr, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, ptr_ty, value_ty),
+                    else => try self.genInlineMemset(
+                        ptr,
+                        .{ .immediate = 0xaa },
+                        .{ .immediate = abi_size },
+                        .{},
+                    ),
                 },
-                .undef => {
-                    if (!self.wantSafety()) return; // The already existing value will do just fine.
-                    switch (abi_size) {
-                        1 => try self.store(ptr, .{ .immediate = 0xaa }, ptr_ty, value_ty),
-                        2 => try self.store(ptr, .{ .immediate = 0xaaaa }, ptr_ty, value_ty),
-                        4 => try self.store(ptr, .{ .immediate = 0xaaaaaaaa }, ptr_ty, value_ty),
-                        8 => try self.store(ptr, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, ptr_ty, value_ty),
-                        else => try self.genInlineMemset(ptr, .{ .immediate = 0xaa }, .{ .immediate = abi_size }, .{}),
-                    }
-                },
-                .immediate => |imm| {
-                    switch (abi_size) {
-                        1, 2, 4 => {
-                            const immediate = if (value_ty.isSignedInt())
-                                Immediate.s(@intCast(i32, @bitCast(i64, imm)))
-                            else
-                                Immediate.u(@truncate(u32, imm));
-                            try self.asmMemoryImmediate(.mov, Memory.sib(
-                                Memory.PtrSize.fromSize(abi_size),
-                                .{ .base = reg.to64() },
-                            ), immediate);
-                        },
-                        8 => {
-                            // TODO: optimization: if the imm is only using the lower
-                            // 4 bytes and can be sign extended we can use a normal mov
-                            // with indirect addressing (mov [reg64], imm32).
-
-                            // movabs does not support indirect register addressing
-                            // so we need an extra register and an extra mov.
-                            const tmp_reg = try self.copyToTmpRegister(value_ty, value);
-                            return self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
-                        },
-                        else => {
-                            return self.fail("TODO implement set pointee with immediate of ABI size {d}", .{abi_size});
-                        },
-                    }
-                },
-                .register => |src_reg| {
-                    try self.genInlineMemcpyRegisterRegister(value_ty, reg, src_reg, 0);
+                .immediate => |imm| switch (self.regBitSize(value_ty)) {
+                    8 => try self.asmMemoryImmediate(
+                        .mov,
+                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
+                        if (math.cast(i8, @bitCast(i64, imm))) |small|
+                            Immediate.s(small)
+                        else
+                            Immediate.u(@intCast(u8, imm)),
+                    ),
+                    16 => try self.asmMemoryImmediate(
+                        .mov,
+                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
+                        if (math.cast(i16, @bitCast(i64, imm))) |small|
+                            Immediate.s(small)
+                        else
+                            Immediate.u(@intCast(u16, imm)),
+                    ),
+                    32 => try self.asmMemoryImmediate(
+                        .mov,
+                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
+                        if (math.cast(i32, @bitCast(i64, imm))) |small|
+                            Immediate.s(small)
+                        else
+                            Immediate.u(@intCast(u32, imm)),
+                    ),
+                    64 => if (math.cast(i32, @bitCast(i64, imm))) |small|
+                        try self.asmMemoryImmediate(
+                            .mov,
+                            Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
+                            Immediate.s(small),
+                        )
+                    else
+                        try self.asmMemoryRegister(
+                            .mov,
+                            Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
+                            registerAlias(try self.copyToTmpRegister(value_ty, value), abi_size),
+                        ),
+                    else => unreachable,
                 },
+                .register => |src_reg| try self.genInlineMemcpyRegisterRegister(
+                    value_ty,
+                    reg,
+                    src_reg,
+                    0,
+                ),
                 .register_overflow => |ro| {
                     const ro_reg_lock = self.register_manager.lockReg(ro.reg);
                     defer if (ro_reg_lock) |lock| self.register_manager.unlockReg(lock);
@@ -3765,8 +3780,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
             defer self.register_manager.unlockReg(addr_reg_lock);
 
             try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr);
-
-            // To get the actual address of the value we want to modify we have to go through the GOT
+            // Load the pointer, which is stored in memory
             try self.asmRegisterMemory(
                 .mov,
                 addr_reg.to64(),
@@ -3774,62 +3788,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
             );
 
             const new_ptr = MCValue{ .register = addr_reg.to64() };
-
-            switch (value) {
-                .immediate => |imm| {
-                    if (abi_size > 8) {
-                        return self.fail("TODO saving imm to memory for abi_size {}", .{abi_size});
-                    }
-
-                    if (abi_size == 8) {
-                        // TODO
-                        const top_bits: u32 = @intCast(u32, imm >> 32);
-                        const can_extend = if (value_ty.isUnsignedInt())
-                            (top_bits == 0) and (imm & 0x8000_0000) == 0
-                        else
-                            top_bits == 0xffff_ffff;
-
-                        if (!can_extend) {
-                            return self.fail("TODO imm64 would get incorrectly sign extended", .{});
-                        }
-                    }
-                    try self.asmMemoryImmediate(
-                        .mov,
-                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg.to64() }),
-                        Immediate.u(@intCast(u32, imm)),
-                    );
-                },
-                .register => {
-                    return self.store(new_ptr, value, ptr_ty, value_ty);
-                },
-                .linker_load, .memory => {
-                    if (abi_size <= 8) {
-                        const tmp_reg = try self.register_manager.allocReg(null, gp);
-                        const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
-                        defer self.register_manager.unlockReg(tmp_reg_lock);
-
-                        try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);
-                        try self.asmRegisterMemory(
-                            .mov,
-                            tmp_reg,
-                            Memory.sib(.qword, .{ .base = tmp_reg }),
-                        );
-
-                        return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
-                    }
-
-                    try self.genInlineMemcpy(new_ptr, value, .{ .immediate = abi_size }, .{});
-                },
-                .stack_offset => {
-                    if (abi_size <= 8) {
-                        const tmp_reg = try self.copyToTmpRegister(value_ty, value);
-                        return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
-                    }
-
-                    try self.genInlineMemcpy(new_ptr, value, .{ .immediate = abi_size }, .{});
-                },
-                else => return self.fail("TODO implement storing {} to MCValue.memory", .{value}),
-            }
+            return self.store(new_ptr, value, ptr_ty, value_ty);
         },
     }
 }
@@ -4887,41 +4846,39 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
                         registerAlias(src_reg, abi_size),
                     ),
                 },
-                .immediate => |imm| {
-                    switch (self.regBitSize(ty)) {
-                        8 => try self.asmRegisterImmediate(
-                            mir_tag,
-                            dst_alias,
-                            if (math.cast(i8, @bitCast(i64, imm))) |small|
-                                Immediate.s(small)
-                            else
-                                Immediate.u(@intCast(u8, imm)),
-                        ),
-                        16 => try self.asmRegisterImmediate(
-                            mir_tag,
-                            dst_alias,
-                            if (math.cast(i16, @bitCast(i64, imm))) |small|
-                                Immediate.s(small)
-                            else
-                                Immediate.u(@intCast(u16, imm)),
-                        ),
-                        32 => try self.asmRegisterImmediate(
-                            mir_tag,
-                            dst_alias,
-                            if (math.cast(i32, @bitCast(i64, imm))) |small|
-                                Immediate.s(small)
-                            else
-                                Immediate.u(@intCast(u32, imm)),
-                        ),
-                        64 => if (math.cast(i32, @bitCast(i64, imm))) |small|
-                            try self.asmRegisterImmediate(mir_tag, dst_alias, Immediate.s(small))
+                .immediate => |imm| switch (self.regBitSize(ty)) {
+                    8 => try self.asmRegisterImmediate(
+                        mir_tag,
+                        dst_alias,
+                        if (math.cast(i8, @bitCast(i64, imm))) |small|
+                            Immediate.s(small)
                         else
-                            try self.asmRegisterRegister(mir_tag, dst_alias, registerAlias(
-                                try self.copyToTmpRegister(ty, src_mcv),
-                                abi_size,
-                            )),
-                        else => unreachable,
-                    }
+                            Immediate.u(@intCast(u8, imm)),
+                    ),
+                    16 => try self.asmRegisterImmediate(
+                        mir_tag,
+                        dst_alias,
+                        if (math.cast(i16, @bitCast(i64, imm))) |small|
+                            Immediate.s(small)
+                        else
+                            Immediate.u(@intCast(u16, imm)),
+                    ),
+                    32 => try self.asmRegisterImmediate(
+                        mir_tag,
+                        dst_alias,
+                        if (math.cast(i32, @bitCast(i64, imm))) |small|
+                            Immediate.s(small)
+                        else
+                            Immediate.u(@intCast(u32, imm)),
+                    ),
+                    64 => if (math.cast(i32, @bitCast(i64, imm))) |small|
+                        try self.asmRegisterImmediate(mir_tag, dst_alias, Immediate.s(small))
+                    else
+                        try self.asmRegisterRegister(mir_tag, dst_alias, registerAlias(
+                            try self.copyToTmpRegister(ty, src_mcv),
+                            abi_size,
+                        )),
+                    else => unreachable,
                 },
                 .memory, .linker_load, .eflags => {
                     assert(abi_size <= 8);
@@ -4931,13 +4888,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
                     const reg = try self.copyToTmpRegister(ty, src_mcv);
                     return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = reg });
                 },
-                .stack_offset => |off| {
-                    try self.asmRegisterMemory(
-                        mir_tag,
-                        registerAlias(dst_reg, abi_size),
-                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
-                    );
-                },
+                .stack_offset => |off| try self.asmRegisterMemory(
+                    mir_tag,
+                    registerAlias(dst_reg, abi_size),
+                    Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
+                ),
             }
         },
         .memory, .linker_load, .stack_offset => {
diff --git a/test/behavior/bugs/3007.zig b/test/behavior/bugs/3007.zig
index df6a63b6403c9426f72c92b1405c9996a39e46e9..3ae1562e9bf6934447718d719f54ed2ef02d9bc7 100644
--- a/test/behavior/bugs/3007.zig
+++ b/test/behavior/bugs/3007.zig
@@ -21,7 +21,6 @@ fn get_foo() Foo.FooError!*Foo {
 test "fixed" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     default_foo = get_foo() catch null; // This Line
diff --git a/test/behavior/eval.zig b/test/behavior/eval.zig
index 7fa80665057def34a2c44cd6e88fa8ef898bbf9b..214f2beb07f7aa4f7976e2b91fa2ebdebfd7dab0 100644
--- a/test/behavior/eval.zig
+++ b/test/behavior/eval.zig
@@ -967,7 +967,6 @@ test "closure capture type of runtime-known parameter" {
 }
 
 test "comptime break passing through runtime condition converted to runtime break" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const S = struct {
@@ -999,7 +998,6 @@ test "comptime break passing through runtime condition converted to runtime brea
 }
 
 test "comptime break to outer loop passing through runtime condition converted to runtime break" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1218,7 +1216,6 @@ test "storing an array of type in a field" {
 }
 
 test "pass pointer to field of comptime-only type as a runtime parameter" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
-- 
2.54.0


From 1980f5479b2139112ff67de5d5f2e6af09fb215e Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Sun, 2 Apr 2023 22:25:35 -0400
Subject: [PATCH 03/13] x86_64: implement store to immediate address

---
 src/arch/x86_64/CodeGen.zig | 32 ++++++++++++--------------------
 test/behavior/inttoptr.zig  |  1 -
 2 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index f3bd6a9e8a92f7106f68a55f54978e9dbbd6f7c3..44362f9bc05d7fe7a2989ece5bfe9e31c1ab8579 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -3651,10 +3651,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
         .dead => unreachable,
         .eflags => unreachable,
         .register_overflow => unreachable,
-        .immediate => |imm| {
-            try self.setRegOrMem(value_ty, .{ .memory = imm }, value);
-        },
-        .stack_offset => {
+        .immediate, .stack_offset => {
             const reg = try self.copyToTmpRegister(ptr_ty, ptr);
             try self.store(.{ .register = reg }, value, ptr_ty, value_ty);
         },
@@ -3748,23 +3745,18 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
                         -@intCast(i32, overflow_bit_offset),
                     );
                 },
-                .linker_load,
-                .memory,
-                .stack_offset,
-                => {
-                    if (abi_size <= 8) {
-                        const tmp_reg = try self.copyToTmpRegister(value_ty, value);
-                        return self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
-                    }
-
-                    try self.genInlineMemcpy(.{ .stack_offset = 0 }, value, .{ .immediate = abi_size }, .{
-                        .source_stack_base = .rbp,
-                        .dest_stack_base = reg.to64(),
-                    });
-                },
+                .linker_load, .memory, .stack_offset => if (abi_size <= 8) {
+                    const tmp_reg = try self.copyToTmpRegister(value_ty, value);
+                    try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
+                } else try self.genInlineMemcpy(
+                    .{ .stack_offset = 0 },
+                    value,
+                    .{ .immediate = abi_size },
+                    .{ .source_stack_base = .rbp, .dest_stack_base = reg.to64() },
+                ),
                 .ptr_stack_offset => {
                     const tmp_reg = try self.copyToTmpRegister(value_ty, value);
-                    return self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
+                    try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
                 },
             }
         },
@@ -3788,7 +3780,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
             );
 
             const new_ptr = MCValue{ .register = addr_reg.to64() };
-            return self.store(new_ptr, value, ptr_ty, value_ty);
+            try self.store(new_ptr, value, ptr_ty, value_ty);
         },
     }
 }
diff --git a/test/behavior/inttoptr.zig b/test/behavior/inttoptr.zig
index e1281c406d2c1290b4f894fbb9b8e02af4871fd0..29a263b2ce87c88bef599b466aa85da30f8d7cb2 100644
--- a/test/behavior/inttoptr.zig
+++ b/test/behavior/inttoptr.zig
@@ -11,7 +11,6 @@ fn addressToFunction() void {
 }
 
 test "mutate through ptr initialized with constant intToPtr value" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-- 
2.54.0


From fde1ec5d0e57471b43bab226a78da2eb61e6bb66 Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Sun, 2 Apr 2023 22:25:53 -0400
Subject: [PATCH 04/13] x86_64: remove returns from naked functions

---
 lib/std/os/linux/x86.zig    | 16 ++++++++++++----
 lib/std/os/linux/x86_64.zig |  7 +++++--
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/lib/std/os/linux/x86.zig b/lib/std/os/linux/x86.zig
index 6ad3c9fa31f65cc08fd0317a66e1f27818e00e2f..2e67fa6b5ba2e5e57cc2ac08bf6356972363b5ff 100644
--- a/lib/std/os/linux/x86.zig
+++ b/lib/std/os/linux/x86.zig
@@ -125,36 +125,44 @@ pub extern fn clone(func: CloneFn, stack: usize, flags: u32, arg: usize, ptid: *
 
 pub fn restore() callconv(.Naked) void {
     switch (@import("builtin").zig_backend) {
-        .stage2_c => return asm volatile (
+        .stage2_c => asm volatile (
             \\ movl %[number], %%eax
             \\ int $0x80
+            \\ ret
             :
             : [number] "i" (@enumToInt(SYS.sigreturn)),
             : "memory"
         ),
-        else => return asm volatile ("int $0x80"
+        else => asm volatile (
+            \\ int $0x80
+            \\ ret
             :
             : [number] "{eax}" (@enumToInt(SYS.sigreturn)),
             : "memory"
         ),
     }
+    unreachable;
 }
 
 pub fn restore_rt() callconv(.Naked) void {
     switch (@import("builtin").zig_backend) {
-        .stage2_c => return asm volatile (
+        .stage2_c => asm volatile (
             \\ movl %[number], %%eax
             \\ int $0x80
+            \\ ret
             :
             : [number] "i" (@enumToInt(SYS.rt_sigreturn)),
             : "memory"
         ),
-        else => return asm volatile ("int $0x80"
+        else => asm volatile (
+            \\ int $0x80
+            \\ ret
             :
             : [number] "{eax}" (@enumToInt(SYS.rt_sigreturn)),
             : "memory"
         ),
     }
+    unreachable;
 }
 
 pub const O = struct {
diff --git a/lib/std/os/linux/x86_64.zig b/lib/std/os/linux/x86_64.zig
index bc1bccf53fc067b532e6a2b6b81454ab21fc7ea6..09047bda83e9ea64643c6b160e655804915515a3 100644
--- a/lib/std/os/linux/x86_64.zig
+++ b/lib/std/os/linux/x86_64.zig
@@ -109,7 +109,7 @@ pub const restore = restore_rt;
 
 pub fn restore_rt() callconv(.Naked) void {
     switch (@import("builtin").zig_backend) {
-        .stage2_c => return asm volatile (
+        .stage2_c => asm volatile (
             \\ movl %[number], %%eax
             \\ syscall
             \\ retq
@@ -117,12 +117,15 @@ pub fn restore_rt() callconv(.Naked) void {
             : [number] "i" (@enumToInt(SYS.rt_sigreturn)),
             : "rcx", "r11", "memory"
         ),
-        else => return asm volatile ("syscall"
+        else => asm volatile (
+            \\ syscall
+            \\ retq
             :
             : [number] "{rax}" (@enumToInt(SYS.rt_sigreturn)),
             : "rcx", "r11", "memory"
         ),
     }
+    unreachable;
 }
 
 pub const mode_t = usize;
-- 
2.54.0


From cd24ab7f6e56c8365896a9b2dfc506a787346ecb Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 01:34:05 -0400
Subject: [PATCH 05/13] x86_64: canonicalise loops

---
 src/arch/x86_64/CodeGen.zig | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 44362f9bc05d7fe7a2989ece5bfe9e31c1ab8579..a2d0d4da410f8a4e1efc633e197d11ef0642bbcd 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -6132,7 +6132,28 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
     const loop = self.air.extraData(Air.Block, ty_pl.payload);
     const body = self.air.extra[loop.end..][0..loop.data.body_len];
     const jmp_target = @intCast(u32, self.mir_instructions.len);
-    try self.genBody(body);
+
+    {
+        try self.branch_stack.append(.{});
+        errdefer _ = self.branch_stack.pop();
+
+        try self.genBody(body);
+    }
+
+    var branch = self.branch_stack.pop();
+    defer branch.deinit(self.gpa);
+
+    log.debug("airLoop: %{d}", .{inst});
+    log.debug("Upper branches:", .{});
+    for (self.branch_stack.items) |bs| {
+        log.debug("{}", .{bs.fmtDebug()});
+    }
+    log.debug("Loop branch: {}", .{branch.fmtDebug()});
+
+    var dummy_branch = Branch{};
+    defer dummy_branch.deinit(self.gpa);
+    try self.canonicaliseBranches(true, &dummy_branch, &branch, true, false);
+
     _ = try self.asmJmpReloc(jmp_target);
     return self.finishAirBookkeeping();
 }
-- 
2.54.0


From f0d13489f819ecd1c0d1c23914127c225f507241 Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 04:55:15 -0400
Subject: [PATCH 06/13] Elf: add program headers for the program header table

---
 lib/std/start.zig                             |   1 -
 src/link/Elf.zig                              | 305 +++++++++++-------
 .../hello_world_with_updates.0.zig            |   2 +-
 .../hello_world_with_updates.0.zig            |   2 +-
 .../hello_world_with_updates.0.zig            |   2 +-
 .../hello_world_with_updates.0.zig            |   2 +-
 6 files changed, 185 insertions(+), 129 deletions(-)

diff --git a/lib/std/start.zig b/lib/std/start.zig
index b3a064c7f01eb9364245488b8b4a47dd9f42ec7b..e79dd1b9a72e3ddc675c135a712d0111179c5d8e 100644
--- a/lib/std/start.zig
+++ b/lib/std/start.zig
@@ -19,7 +19,6 @@ const start_sym_name = if (native_arch.isMIPS()) "__start" else "_start";
 // self-hosted is capable enough to handle all of the real start.zig logic.
 pub const simplified_logic =
     builtin.zig_backend == .stage2_wasm or
-    builtin.zig_backend == .stage2_x86_64 or
     builtin.zig_backend == .stage2_x86 or
     builtin.zig_backend == .stage2_aarch64 or
     builtin.zig_backend == .stage2_arm or
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 3b7c2efa0e8a5aaa8be6a05a975499eb87030b43..8079b091669136fcc86fe676b1a2ed1fc44dff65 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -106,7 +106,12 @@ shdr_table_offset: ?u64 = null,
 /// Stored in native-endian format, depending on target endianness needs to be bswapped on read/write.
 /// Same order as in the file.
 program_headers: std.ArrayListUnmanaged(elf.Elf64_Phdr) = .{},
-phdr_table_offset: ?u64 = null,
+/// The index into the program headers of the PT_PHDR program header
+phdr_table_index: ?u16 = null,
+/// The index into the program headers of the PT_LOAD program header containing the phdr
+/// Most linkers would merge this with phdr_load_ro_index,
+/// but hot swap means we can't ensure they are consecutive.
+phdr_table_load_index: ?u16 = null,
 /// The index into the program headers of a PT_LOAD program header with Read and Execute flags
 phdr_load_re_index: ?u16 = null,
 /// The index into the program headers of the global offset table.
@@ -386,6 +391,17 @@ fn detectAllocCollision(self: *Elf, start: u64, size: u64) ?u64 {
 
     const end = start + padToIdeal(size);
 
+    if (self.phdr_table_index) |index| {
+        const off = self.program_headers.items[index].p_offset;
+        const phdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Phdr) else @sizeOf(elf.Elf64_Phdr);
+        const tight_size = self.program_headers.items.len * phdr_size;
+        const increased_size = padToIdeal(tight_size);
+        const test_end = off + increased_size;
+        if (end > off and start < test_end) {
+            return test_end;
+        }
+    }
+
     if (self.shdr_table_offset) |off| {
         const shdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Shdr) else @sizeOf(elf.Elf64_Shdr);
         const tight_size = self.sections.slice().len * shdr_size;
@@ -396,16 +412,6 @@ fn detectAllocCollision(self: *Elf, start: u64, size: u64) ?u64 {
         }
     }
 
-    if (self.phdr_table_offset) |off| {
-        const phdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Phdr) else @sizeOf(elf.Elf64_Phdr);
-        const tight_size = self.sections.slice().len * phdr_size;
-        const increased_size = padToIdeal(tight_size);
-        const test_end = off + increased_size;
-        if (end > off and start < test_end) {
-            return test_end;
-        }
-    }
-
     for (self.sections.items(.shdr)) |section| {
         const increased_size = padToIdeal(section.sh_size);
         const test_end = section.sh_offset + increased_size;
@@ -427,10 +433,11 @@ pub fn allocatedSize(self: *Elf, start: u64) u64 {
     if (start == 0)
         return 0;
     var min_pos: u64 = std.math.maxInt(u64);
-    if (self.shdr_table_offset) |off| {
+    if (self.phdr_table_index) |index| {
+        const off = self.program_headers.items[index].p_offset;
         if (off > start and off < min_pos) min_pos = off;
     }
-    if (self.phdr_table_offset) |off| {
+    if (self.shdr_table_offset) |off| {
         if (off > start and off < min_pos) min_pos = off;
     }
     for (self.sections.items(.shdr)) |section| {
@@ -462,96 +469,146 @@ pub fn populateMissingMetadata(self: *Elf) !void {
     };
     const ptr_size: u8 = self.ptrWidthBytes();
 
-    if (self.phdr_load_re_index == null) {
-        self.phdr_load_re_index = @intCast(u16, self.program_headers.items.len);
-        const file_size = self.base.options.program_code_size_hint;
-        const p_align = self.page_size;
-        const off = self.findFreeSpace(file_size, p_align);
-        log.debug("found PT_LOAD RE free space 0x{x} to 0x{x}", .{ off, off + file_size });
-        const entry_addr: u64 = self.entry_addr orelse if (self.base.options.target.cpu.arch == .spu_2) @as(u64, 0) else default_entry_addr;
-        try self.program_headers.append(gpa, .{
-            .p_type = elf.PT_LOAD,
-            .p_offset = off,
-            .p_filesz = file_size,
-            .p_vaddr = entry_addr,
-            .p_paddr = entry_addr,
-            .p_memsz = file_size,
-            .p_align = p_align,
-            .p_flags = elf.PF_X | elf.PF_R | elf.PF_W,
-        });
-        self.entry_addr = null;
-        self.phdr_table_dirty = true;
-    }
+    { // Program Headers
+        var new_phdr_table_index: ?u16 = null;
+        if (self.phdr_table_index == null) {
+            new_phdr_table_index = @intCast(u16, self.program_headers.items.len);
+            const phalign: u16 = switch (self.ptr_width) {
+                .p32 => @alignOf(elf.Elf32_Phdr),
+                .p64 => @alignOf(elf.Elf64_Phdr),
+            };
+            try self.program_headers.append(gpa, .{
+                .p_type = elf.PT_PHDR,
+                .p_offset = undefined,
+                .p_filesz = undefined,
+                .p_vaddr = undefined,
+                .p_paddr = undefined,
+                .p_memsz = undefined,
+                .p_align = phalign,
+                .p_flags = elf.PF_R,
+            });
+            self.phdr_table_dirty = true;
+        }
 
-    if (self.phdr_got_index == null) {
-        self.phdr_got_index = @intCast(u16, self.program_headers.items.len);
-        const file_size = @as(u64, ptr_size) * self.base.options.symbol_count_hint;
-        // We really only need ptr alignment but since we are using PROGBITS, linux requires
-        // page align.
-        const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
-        const off = self.findFreeSpace(file_size, p_align);
-        log.debug("found PT_LOAD GOT free space 0x{x} to 0x{x}", .{ off, off + file_size });
-        // TODO instead of hard coding the vaddr, make a function to find a vaddr to put things at.
-        // we'll need to re-use that function anyway, in case the GOT grows and overlaps something
-        // else in virtual memory.
-        const got_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x4000000 else 0x8000;
-        try self.program_headers.append(gpa, .{
-            .p_type = elf.PT_LOAD,
-            .p_offset = off,
-            .p_filesz = file_size,
-            .p_vaddr = got_addr,
-            .p_paddr = got_addr,
-            .p_memsz = file_size,
-            .p_align = p_align,
-            .p_flags = elf.PF_R | elf.PF_W,
-        });
-        self.phdr_table_dirty = true;
-    }
+        if (self.phdr_table_load_index == null) {
+            self.phdr_table_load_index = @intCast(u16, self.program_headers.items.len);
+            // TODO Same as for GOT
+            const phdr_addr: u64 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x1000000 else 0x1000;
+            try self.program_headers.append(gpa, .{
+                .p_type = elf.PT_LOAD,
+                .p_offset = undefined,
+                .p_filesz = undefined,
+                .p_vaddr = phdr_addr,
+                .p_paddr = phdr_addr,
+                .p_memsz = undefined,
+                .p_align = self.page_size,
+                .p_flags = elf.PF_R,
+            });
+            self.phdr_table_dirty = true;
+        }
 
-    if (self.phdr_load_ro_index == null) {
-        self.phdr_load_ro_index = @intCast(u16, self.program_headers.items.len);
-        // TODO Find a hint about how much data need to be in rodata ?
-        const file_size = 1024;
-        // Same reason as for GOT
-        const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
-        const off = self.findFreeSpace(file_size, p_align);
-        log.debug("found PT_LOAD RO free space 0x{x} to 0x{x}", .{ off, off + file_size });
-        // TODO Same as for GOT
-        const rodata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0xc000000 else 0xa000;
-        try self.program_headers.append(gpa, .{
-            .p_type = elf.PT_LOAD,
-            .p_offset = off,
-            .p_filesz = file_size,
-            .p_vaddr = rodata_addr,
-            .p_paddr = rodata_addr,
-            .p_memsz = file_size,
-            .p_align = p_align,
-            .p_flags = elf.PF_R | elf.PF_W,
-        });
-        self.phdr_table_dirty = true;
-    }
+        if (self.phdr_load_re_index == null) {
+            self.phdr_load_re_index = @intCast(u16, self.program_headers.items.len);
+            const file_size = self.base.options.program_code_size_hint;
+            const p_align = self.page_size;
+            const off = self.findFreeSpace(file_size, p_align);
+            log.debug("found PT_LOAD RE free space 0x{x} to 0x{x}", .{ off, off + file_size });
+            const entry_addr: u64 = self.entry_addr orelse if (self.base.options.target.cpu.arch == .spu_2) @as(u64, 0) else default_entry_addr;
+            try self.program_headers.append(gpa, .{
+                .p_type = elf.PT_LOAD,
+                .p_offset = off,
+                .p_filesz = file_size,
+                .p_vaddr = entry_addr,
+                .p_paddr = entry_addr,
+                .p_memsz = file_size,
+                .p_align = p_align,
+                .p_flags = elf.PF_X | elf.PF_R | elf.PF_W,
+            });
+            self.entry_addr = null;
+            self.phdr_table_dirty = true;
+        }
 
-    if (self.phdr_load_rw_index == null) {
-        self.phdr_load_rw_index = @intCast(u16, self.program_headers.items.len);
-        // TODO Find a hint about how much data need to be in data ?
-        const file_size = 1024;
-        // Same reason as for GOT
-        const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
-        const off = self.findFreeSpace(file_size, p_align);
-        log.debug("found PT_LOAD RW free space 0x{x} to 0x{x}", .{ off, off + file_size });
-        // TODO Same as for GOT
-        const rwdata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x10000000 else 0xc000;
-        try self.program_headers.append(gpa, .{
-            .p_type = elf.PT_LOAD,
-            .p_offset = off,
-            .p_filesz = file_size,
-            .p_vaddr = rwdata_addr,
-            .p_paddr = rwdata_addr,
-            .p_memsz = file_size,
-            .p_align = p_align,
-            .p_flags = elf.PF_R | elf.PF_W,
-        });
-        self.phdr_table_dirty = true;
+        if (self.phdr_got_index == null) {
+            self.phdr_got_index = @intCast(u16, self.program_headers.items.len);
+            const file_size = @as(u64, ptr_size) * self.base.options.symbol_count_hint;
+            // We really only need ptr alignment but since we are using PROGBITS, linux requires
+            // page align.
+            const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
+            const off = self.findFreeSpace(file_size, p_align);
+            log.debug("found PT_LOAD GOT free space 0x{x} to 0x{x}", .{ off, off + file_size });
+            // TODO instead of hard coding the vaddr, make a function to find a vaddr to put things at.
+            // we'll need to re-use that function anyway, in case the GOT grows and overlaps something
+            // else in virtual memory.
+            const got_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x4000000 else 0x8000;
+            try self.program_headers.append(gpa, .{
+                .p_type = elf.PT_LOAD,
+                .p_offset = off,
+                .p_filesz = file_size,
+                .p_vaddr = got_addr,
+                .p_paddr = got_addr,
+                .p_memsz = file_size,
+                .p_align = p_align,
+                .p_flags = elf.PF_R | elf.PF_W,
+            });
+            self.phdr_table_dirty = true;
+        }
+
+        if (self.phdr_load_ro_index == null) {
+            self.phdr_load_ro_index = @intCast(u16, self.program_headers.items.len);
+            // TODO Find a hint about how much data need to be in rodata ?
+            const file_size = 1024;
+            // Same reason as for GOT
+            const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
+            const off = self.findFreeSpace(file_size, p_align);
+            log.debug("found PT_LOAD RO free space 0x{x} to 0x{x}", .{ off, off + file_size });
+            // TODO Same as for GOT
+            const rodata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0xc000000 else 0xa000;
+            try self.program_headers.append(gpa, .{
+                .p_type = elf.PT_LOAD,
+                .p_offset = off,
+                .p_filesz = file_size,
+                .p_vaddr = rodata_addr,
+                .p_paddr = rodata_addr,
+                .p_memsz = file_size,
+                .p_align = p_align,
+                .p_flags = elf.PF_R | elf.PF_W,
+            });
+            self.phdr_table_dirty = true;
+        }
+
+        if (self.phdr_load_rw_index == null) {
+            self.phdr_load_rw_index = @intCast(u16, self.program_headers.items.len);
+            // TODO Find a hint about how much data need to be in data ?
+            const file_size = 1024;
+            // Same reason as for GOT
+            const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
+            const off = self.findFreeSpace(file_size, p_align);
+            log.debug("found PT_LOAD RW free space 0x{x} to 0x{x}", .{ off, off + file_size });
+            // TODO Same as for GOT
+            const rwdata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x10000000 else 0xc000;
+            try self.program_headers.append(gpa, .{
+                .p_type = elf.PT_LOAD,
+                .p_offset = off,
+                .p_filesz = file_size,
+                .p_vaddr = rwdata_addr,
+                .p_paddr = rwdata_addr,
+                .p_memsz = file_size,
+                .p_align = p_align,
+                .p_flags = elf.PF_R | elf.PF_W,
+            });
+            self.phdr_table_dirty = true;
+        }
+
+        if (new_phdr_table_index) |index| {
+            const phsize: u64 = switch (self.ptr_width) {
+                .p32 => @sizeOf(elf.Elf32_Phdr),
+                .p64 => @sizeOf(elf.Elf64_Phdr),
+            };
+            const phdr_table = &self.program_headers.items[index];
+            phdr_table.p_offset = self.findFreeSpace(self.program_headers.items.len * phsize, @intCast(u32, phdr_table.p_align));
+            self.phdr_table_index = index;
+            self.phdr_table_dirty = true;
+        }
     }
 
     if (self.shstrtab_index == null) {
@@ -849,19 +906,6 @@ pub fn populateMissingMetadata(self: *Elf) !void {
         self.shdr_table_dirty = true;
     }
 
-    const phsize: u64 = switch (self.ptr_width) {
-        .p32 => @sizeOf(elf.Elf32_Phdr),
-        .p64 => @sizeOf(elf.Elf64_Phdr),
-    };
-    const phalign: u16 = switch (self.ptr_width) {
-        .p32 => @alignOf(elf.Elf32_Phdr),
-        .p64 => @alignOf(elf.Elf64_Phdr),
-    };
-    if (self.phdr_table_offset == null) {
-        self.phdr_table_offset = self.findFreeSpace(self.program_headers.items.len * phsize, phalign);
-        self.phdr_table_dirty = true;
-    }
-
     {
         // Iterate over symbols, populating free_list and last_text_block.
         if (self.local_symbols.items.len != 1) {
@@ -1132,18 +1176,30 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
             .p32 => @sizeOf(elf.Elf32_Phdr),
             .p64 => @sizeOf(elf.Elf64_Phdr),
         };
-        const phalign: u16 = switch (self.ptr_width) {
-            .p32 => @alignOf(elf.Elf32_Phdr),
-            .p64 => @alignOf(elf.Elf64_Phdr),
-        };
-        const allocated_size = self.allocatedSize(self.phdr_table_offset.?);
+
+        const phdr_table_index = self.phdr_table_index.?;
+        const phdr_table = &self.program_headers.items[phdr_table_index];
+        const phdr_table_load = &self.program_headers.items[self.phdr_table_load_index.?];
+
+        const allocated_size = self.allocatedSize(phdr_table.p_offset);
         const needed_size = self.program_headers.items.len * phsize;
 
         if (needed_size > allocated_size) {
-            self.phdr_table_offset = null; // free the space
-            self.phdr_table_offset = self.findFreeSpace(needed_size, phalign);
+            self.phdr_table_index = null; // free the space
+            defer self.phdr_table_index = phdr_table_index;
+            phdr_table.p_offset = self.findFreeSpace(needed_size, @intCast(u32, phdr_table.p_align));
         }
 
+        phdr_table_load.p_offset = mem.alignBackwardGeneric(u64, phdr_table.p_offset, phdr_table_load.p_align);
+        const load_align_offset = phdr_table.p_offset - phdr_table_load.p_offset;
+        phdr_table_load.p_filesz = load_align_offset + needed_size;
+        phdr_table_load.p_memsz = load_align_offset + needed_size;
+
+        phdr_table.p_filesz = needed_size;
+        phdr_table.p_vaddr = phdr_table_load.p_vaddr + load_align_offset;
+        phdr_table.p_paddr = phdr_table_load.p_paddr + load_align_offset;
+        phdr_table.p_memsz = needed_size;
+
         switch (self.ptr_width) {
             .p32 => {
                 const buf = try gpa.alloc(elf.Elf32_Phdr, self.program_headers.items.len);
@@ -1155,7 +1211,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
                         mem.byteSwapAllFields(elf.Elf32_Phdr, phdr);
                     }
                 }
-                try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), self.phdr_table_offset.?);
+                try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), phdr_table.p_offset);
             },
             .p64 => {
                 const buf = try gpa.alloc(elf.Elf64_Phdr, self.program_headers.items.len);
@@ -1167,7 +1223,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
                         mem.byteSwapAllFields(elf.Elf64_Phdr, phdr);
                     }
                 }
-                try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), self.phdr_table_offset.?);
+                try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), phdr_table.p_offset);
             },
         }
         self.phdr_table_dirty = false;
@@ -1992,13 +2048,14 @@ fn writeElfHeader(self: *Elf) !void {
 
     const e_entry = if (elf_type == .REL) 0 else self.entry_addr.?;
 
+    const phdr_table_offset = self.program_headers.items[self.phdr_table_index.?].p_offset;
     switch (self.ptr_width) {
         .p32 => {
             mem.writeInt(u32, hdr_buf[index..][0..4], @intCast(u32, e_entry), endian);
             index += 4;
 
             // e_phoff
-            mem.writeInt(u32, hdr_buf[index..][0..4], @intCast(u32, self.phdr_table_offset.?), endian);
+            mem.writeInt(u32, hdr_buf[index..][0..4], @intCast(u32, phdr_table_offset), endian);
             index += 4;
 
             // e_shoff
@@ -2011,7 +2068,7 @@ fn writeElfHeader(self: *Elf) !void {
             index += 8;
 
             // e_phoff
-            mem.writeInt(u64, hdr_buf[index..][0..8], self.phdr_table_offset.?, endian);
+            mem.writeInt(u64, hdr_buf[index..][0..8], phdr_table_offset, endian);
             index += 8;
 
             // e_shoff
diff --git a/test/cases/aarch64-macos/hello_world_with_updates.0.zig b/test/cases/aarch64-macos/hello_world_with_updates.0.zig
index 90aa14ab65a5217950729681a26d1c75fdabd722..9f516ff1390f02aafc6203c8625793f8e58acdaf 100644
--- a/test/cases/aarch64-macos/hello_world_with_updates.0.zig
+++ b/test/cases/aarch64-macos/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=aarch64-macos
 //
-// :110:9: error: root struct of file 'tmp' has no member named 'main'
+// :109:9: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-linux/hello_world_with_updates.0.zig b/test/cases/x86_64-linux/hello_world_with_updates.0.zig
index 16c280774ea3615a19277a04ddfc4f6d5662ecb0..40abdd6c1fa4217b99f077793ecbb56bfa3f89a0 100644
--- a/test/cases/x86_64-linux/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-linux/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=x86_64-linux
 //
-// :110:9: error: root struct of file 'tmp' has no member named 'main'
+// :109:9: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-macos/hello_world_with_updates.0.zig b/test/cases/x86_64-macos/hello_world_with_updates.0.zig
index d91bd9dc9191538f63a619b9771d8b576cfbbe6d..e0680c81d7420441fabd29498bf9258a5c16f291 100644
--- a/test/cases/x86_64-macos/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-macos/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=x86_64-macos
 //
-// :110:9: error: root struct of file 'tmp' has no member named 'main'
+// :109:9: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-windows/hello_world_with_updates.0.zig b/test/cases/x86_64-windows/hello_world_with_updates.0.zig
index 0c4d2c2bc26b39dc0db27e7e3e76539b237979cf..04e1d4cfad0ac9045fb033b9f550e2d5fb1e64c8 100644
--- a/test/cases/x86_64-windows/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-windows/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=x86_64-windows
 //
-// :131:9: error: root struct of file 'tmp' has no member named 'main'
+// :130:9: error: root struct of file 'tmp' has no member named 'main'
-- 
2.54.0


From 562170681a3d0c8502892f8975f369c4b269fe1a Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 06:02:38 -0400
Subject: [PATCH 07/13] link: cleanup lazy symbols

We now only update one lazy symbol in flushModule.
Updating the rest from updateDecl is TBD.
---
 src/Module.zig     |   2 +-
 src/link.zig       |  26 ++++++------
 src/link/Coff.zig  |  99 +++++++++++++++++++++----------------------
 src/link/Elf.zig   |  92 +++++++++++++++++++++-------------------
 src/link/MachO.zig | 102 ++++++++++++++++++++++-----------------------
 5 files changed, 158 insertions(+), 163 deletions(-)

diff --git a/src/Module.zig b/src/Module.zig
index 9dab3edccdfeee40f8b5010b55f91fb2a603090f..70732369ca9f3cad48d908bc7a02cee405615b47 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -555,7 +555,7 @@ pub const Decl = struct {
         _,
 
         pub fn init(oi: ?Index) OptionalIndex {
-            return oi orelse .none;
+            return @intToEnum(OptionalIndex, @enumToInt(oi orelse return .none));
         }
 
         pub fn unwrap(oi: OptionalIndex) ?Index {
diff --git a/src/link.zig b/src/link.zig
index 7ca94f4f6d813749f22fb8941e4015444309c2a0..45873fec26d4d80bd8cde723d8a6ce31593ea3d5 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -1106,23 +1106,21 @@ pub const File = struct {
     };
 
     pub const LazySymbol = struct {
-        kind: enum { code, const_data },
+        pub const Kind = enum { code, const_data };
+
+        kind: Kind,
         ty: Type,
 
-        pub const Context = struct {
-            mod: *Module,
+        pub fn initDecl(kind: Kind, decl: Module.Decl.OptionalIndex, mod: *Module) LazySymbol {
+            return .{ .kind = kind, .ty = if (decl.unwrap()) |decl_index|
+                mod.declPtr(decl_index).val.castTag(.ty).?.data
+            else
+                Type.anyerror };
+        }
 
-            pub fn hash(ctx: @This(), sym: LazySymbol) u32 {
-                var hasher = std.hash.Wyhash.init(0);
-                std.hash.autoHash(&hasher, sym.kind);
-                sym.ty.hashWithHasher(&hasher, ctx.mod);
-                return @truncate(u32, hasher.final());
-            }
-
-            pub fn eql(ctx: @This(), lhs: LazySymbol, rhs: LazySymbol, _: usize) bool {
-                return lhs.kind == rhs.kind and lhs.ty.eql(rhs.ty, ctx.mod);
-            }
-        };
+        pub fn getDecl(self: LazySymbol) Module.Decl.OptionalIndex {
+            return Module.Decl.OptionalIndex.init(self.ty.getOwnerDeclOrNull());
+        }
     };
 
     pub const C = @import("link/C.zig");
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 3a94feb841c5460ee2a5b5a39abd18e957c9d452..825afff36d12336b89402c89292859583ff253ea 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -145,16 +145,11 @@ const Section = struct {
     free_list: std.ArrayListUnmanaged(Atom.Index) = .{},
 };
 
-const LazySymbolTable = std.ArrayHashMapUnmanaged(
-    link.File.LazySymbol,
-    LazySymbolMetadata,
-    link.File.LazySymbol.Context,
-    true,
-);
+const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex, LazySymbolMetadata);
 
 const LazySymbolMetadata = struct {
-    atom: Atom.Index,
-    section: u16,
+    text_atom: ?Atom.Index = null,
+    rdata_atom: ?Atom.Index = null,
     alignment: u32,
 };
 
@@ -1176,10 +1171,28 @@ pub fn updateDecl(self: *Coff, module: *Module, decl_index: Module.Decl.Index) !
     return self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-fn updateLazySymbol(
+fn updateLazySymbol(self: *Coff, decl: Module.Decl.OptionalIndex, metadata: LazySymbolMetadata) !void {
+    const mod = self.base.options.module.?;
+    if (metadata.text_atom) |atom| try self.updateLazySymbolAtom(
+        link.File.LazySymbol.initDecl(.code, decl, mod),
+        atom,
+        self.text_section_index.?,
+        metadata.alignment,
+    );
+    if (metadata.rdata_atom) |atom| try self.updateLazySymbolAtom(
+        link.File.LazySymbol.initDecl(.const_data, decl, mod),
+        atom,
+        self.rdata_section_index.?,
+        metadata.alignment,
+    );
+}
+
+fn updateLazySymbolAtom(
     self: *Coff,
-    lazy_sym: link.File.LazySymbol,
-    lazy_metadata: LazySymbolMetadata,
+    sym: link.File.LazySymbol,
+    atom_index: Atom.Index,
+    section_index: u16,
+    required_alignment: u32,
 ) !void {
     const gpa = self.base.allocator;
     const mod = self.base.options.module.?;
@@ -1188,16 +1201,15 @@ fn updateLazySymbol(
     defer code_buffer.deinit();
 
     const name = try std.fmt.allocPrint(gpa, "__lazy_{s}_{}", .{
-        @tagName(lazy_sym.kind),
-        lazy_sym.ty.fmt(mod),
+        @tagName(sym.kind),
+        sym.ty.fmt(mod),
     });
     defer gpa.free(name);
 
-    const atom_index = lazy_metadata.atom;
     const atom = self.getAtomPtr(atom_index);
     const local_sym_index = atom.getSymbolIndex().?;
 
-    const src = if (lazy_sym.ty.getOwnerDeclOrNull()) |owner_decl|
+    const src = if (sym.ty.getOwnerDeclOrNull()) |owner_decl|
         mod.declPtr(owner_decl).srcLoc()
     else
         Module.SrcLoc{
@@ -1205,14 +1217,9 @@ fn updateLazySymbol(
             .parent_decl_node = undefined,
             .lazy = .unneeded,
         };
-    const res = try codegen.generateLazySymbol(
-        &self.base,
-        src,
-        lazy_sym,
-        &code_buffer,
-        .none,
-        .{ .parent_atom_index = local_sym_index },
-    );
+    const res = try codegen.generateLazySymbol(&self.base, src, sym, &code_buffer, .none, .{
+        .parent_atom_index = local_sym_index,
+    });
     const code = switch (res) {
         .ok => code_buffer.items,
         .fail => |em| {
@@ -1221,11 +1228,10 @@ fn updateLazySymbol(
         },
     };
 
-    const required_alignment = lazy_metadata.alignment;
     const code_len = @intCast(u32, code.len);
     const symbol = atom.getSymbolPtr(self);
     try self.setSymbolName(symbol, name);
-    symbol.section_number = @intToEnum(coff.SectionNumber, lazy_metadata.section + 1);
+    symbol.section_number = @intToEnum(coff.SectionNumber, section_index + 1);
     symbol.type = .{ .complex_type = .NULL, .base_type = .NULL };
 
     const vaddr = try self.allocateAtom(atom_index, code_len, required_alignment);
@@ -1250,24 +1256,18 @@ fn updateLazySymbol(
 
 pub fn getOrCreateAtomForLazySymbol(
     self: *Coff,
-    lazy_sym: link.File.LazySymbol,
+    sym: link.File.LazySymbol,
     alignment: u32,
 ) !Atom.Index {
-    const gop = try self.lazy_syms.getOrPutContext(self.base.allocator, lazy_sym, .{
-        .mod = self.base.options.module.?,
-    });
+    const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
     errdefer _ = self.lazy_syms.pop();
-    if (!gop.found_existing) {
-        gop.value_ptr.* = .{
-            .atom = try self.createAtom(),
-            .section = switch (lazy_sym.kind) {
-                .code => self.text_section_index.?,
-                .const_data => self.rdata_section_index.?,
-            },
-            .alignment = alignment,
-        };
-    }
-    return gop.value_ptr.atom;
+    if (!gop.found_existing) gop.value_ptr.* = .{ .alignment = alignment };
+    const atom = switch (sym.kind) {
+        .code => &gop.value_ptr.text_atom,
+        .const_data => &gop.value_ptr.rdata_atom,
+    };
+    if (atom.* == null) atom.* = try self.createAtom();
+    return atom.*.?;
 }
 
 pub fn getOrCreateAtomForDecl(self: *Coff, decl_index: Module.Decl.Index) !Atom.Index {
@@ -1600,17 +1600,13 @@ pub fn flushModule(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod
     sub_prog_node.activate();
     defer sub_prog_node.end();
 
-    {
-        var lazy_it = self.lazy_syms.iterator();
-        while (lazy_it.next()) |lazy_entry| {
-            self.updateLazySymbol(
-                lazy_entry.key_ptr.*,
-                lazy_entry.value_ptr.*,
-            ) catch |err| switch (err) {
-                error.CodegenFail => return error.FlushFailure,
-                else => |e| return e,
-            };
-        }
+    // Most lazy symbols can be updated when the corresponding decl is,
+    // so we only have to worry about the one without an associated decl.
+    if (self.lazy_syms.get(.none)) |metadata| {
+        self.updateLazySymbol(.none, metadata) catch |err| switch (err) {
+            error.CodegenFail => return error.FlushFailure,
+            else => |e| return e,
+        };
     }
 
     const gpa = self.base.allocator;
@@ -2489,6 +2485,7 @@ const Module = @import("../Module.zig");
 const Object = @import("Coff/Object.zig");
 const Relocation = @import("Coff/Relocation.zig");
 const StringTable = @import("strtab.zig").StringTable;
+const Type = @import("../type.zig").Type;
 const TypedValue = @import("../TypedValue.zig");
 
 pub const base_tag: link.File.Tag = .coff;
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 8079b091669136fcc86fe676b1a2ed1fc44dff65..ec05f5895a3aba97ddae4b0736caffa7cbd71560 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -64,8 +64,8 @@ const Section = struct {
 };
 
 const LazySymbolMetadata = struct {
-    atom: Atom.Index,
-    shdr: u16,
+    text_atom: ?Atom.Index = null,
+    rodata_atom: ?Atom.Index = null,
     alignment: u32,
 };
 
@@ -208,7 +208,7 @@ relocs: RelocTable = .{},
 
 const RelocTable = std.AutoHashMapUnmanaged(Atom.Index, std.ArrayListUnmanaged(Atom.Reloc));
 const UnnamedConstTable = std.AutoHashMapUnmanaged(Module.Decl.Index, std.ArrayListUnmanaged(Atom.Index));
-const LazySymbolTable = std.ArrayHashMapUnmanaged(File.LazySymbol, LazySymbolMetadata, File.LazySymbol.Context, true);
+const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex, LazySymbolMetadata);
 
 /// When allocating, the ideal_capacity is calculated by
 /// actual_capacity + (actual_capacity / ideal_factor)
@@ -1065,17 +1065,13 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
     sub_prog_node.activate();
     defer sub_prog_node.end();
 
-    {
-        var lazy_it = self.lazy_syms.iterator();
-        while (lazy_it.next()) |lazy_entry| {
-            self.updateLazySymbol(
-                lazy_entry.key_ptr.*,
-                lazy_entry.value_ptr.*,
-            ) catch |err| switch (err) {
-                error.CodegenFail => return error.FlushFailure,
-                else => |e| return e,
-            };
-        }
+    // Most lazy symbols can be updated when the corresponding decl is,
+    // so we only have to worry about the one without an associated decl.
+    if (self.lazy_syms.get(.none)) |metadata| {
+        self.updateLazySymbol(.none, metadata) catch |err| switch (err) {
+            error.CodegenFail => return error.FlushFailure,
+            else => |e| return e,
+        };
     }
 
     // TODO This linker code currently assumes there is only 1 compilation unit and it
@@ -2424,22 +2420,16 @@ pub fn freeDecl(self: *Elf, decl_index: Module.Decl.Index) void {
     }
 }
 
-pub fn getOrCreateAtomForLazySymbol(self: *Elf, lazy_sym: File.LazySymbol, alignment: u32) !Atom.Index {
-    const gop = try self.lazy_syms.getOrPutContext(self.base.allocator, lazy_sym, .{
-        .mod = self.base.options.module.?,
-    });
+pub fn getOrCreateAtomForLazySymbol(self: *Elf, sym: File.LazySymbol, alignment: u32) !Atom.Index {
+    const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
     errdefer _ = self.lazy_syms.pop();
-    if (!gop.found_existing) {
-        gop.value_ptr.* = .{
-            .atom = try self.createAtom(),
-            .shdr = switch (lazy_sym.kind) {
-                .code => self.text_section_index.?,
-                .const_data => self.rodata_section_index.?,
-            },
-            .alignment = alignment,
-        };
-    }
-    return gop.value_ptr.atom;
+    if (!gop.found_existing) gop.value_ptr.* = .{ .alignment = alignment };
+    const atom = switch (sym.kind) {
+        .code => &gop.value_ptr.text_atom,
+        .const_data => &gop.value_ptr.rodata_atom,
+    };
+    if (atom.* == null) atom.* = try self.createAtom();
+    return atom.*.?;
 }
 
 pub fn getOrCreateAtomForDecl(self: *Elf, decl_index: Module.Decl.Index) !Atom.Index {
@@ -2708,7 +2698,29 @@ pub fn updateDecl(self: *Elf, module: *Module, decl_index: Module.Decl.Index) !v
     return self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-fn updateLazySymbol(self: *Elf, lazy_sym: File.LazySymbol, lazy_metadata: LazySymbolMetadata) !void {
+fn updateLazySymbol(self: *Elf, decl: Module.Decl.OptionalIndex, metadata: LazySymbolMetadata) !void {
+    const mod = self.base.options.module.?;
+    if (metadata.text_atom) |atom| try self.updateLazySymbolAtom(
+        File.LazySymbol.initDecl(.code, decl, mod),
+        atom,
+        self.text_section_index.?,
+        metadata.alignment,
+    );
+    if (metadata.rodata_atom) |atom| try self.updateLazySymbolAtom(
+        File.LazySymbol.initDecl(.const_data, decl, mod),
+        atom,
+        self.rodata_section_index.?,
+        metadata.alignment,
+    );
+}
+
+fn updateLazySymbolAtom(
+    self: *Elf,
+    sym: File.LazySymbol,
+    atom_index: Atom.Index,
+    shdr_index: u16,
+    required_alignment: u32,
+) !void {
     const gpa = self.base.allocator;
     const mod = self.base.options.module.?;
 
@@ -2717,19 +2729,18 @@ fn updateLazySymbol(self: *Elf, lazy_sym: File.LazySymbol, lazy_metadata: LazySy
 
     const name_str_index = blk: {
         const name = try std.fmt.allocPrint(gpa, "__lazy_{s}_{}", .{
-            @tagName(lazy_sym.kind),
-            lazy_sym.ty.fmt(mod),
+            @tagName(sym.kind),
+            sym.ty.fmt(mod),
         });
         defer gpa.free(name);
         break :blk try self.shstrtab.insert(gpa, name);
     };
     const name = self.shstrtab.get(name_str_index).?;
 
-    const atom_index = lazy_metadata.atom;
     const atom = self.getAtom(atom_index);
     const local_sym_index = atom.getSymbolIndex().?;
 
-    const src = if (lazy_sym.ty.getOwnerDeclOrNull()) |owner_decl|
+    const src = if (sym.ty.getOwnerDeclOrNull()) |owner_decl|
         mod.declPtr(owner_decl).srcLoc()
     else
         Module.SrcLoc{
@@ -2737,14 +2748,9 @@ fn updateLazySymbol(self: *Elf, lazy_sym: File.LazySymbol, lazy_metadata: LazySy
             .parent_decl_node = undefined,
             .lazy = .unneeded,
         };
-    const res = try codegen.generateLazySymbol(
-        &self.base,
-        src,
-        lazy_sym,
-        &code_buffer,
-        .none,
-        .{ .parent_atom_index = local_sym_index },
-    );
+    const res = try codegen.generateLazySymbol(&self.base, src, sym, &code_buffer, .none, .{
+        .parent_atom_index = local_sym_index,
+    });
     const code = switch (res) {
         .ok => code_buffer.items,
         .fail => |em| {
@@ -2753,7 +2759,6 @@ fn updateLazySymbol(self: *Elf, lazy_sym: File.LazySymbol, lazy_metadata: LazySy
         },
     };
 
-    const shdr_index = lazy_metadata.shdr;
     const phdr_index = self.sections.items(.phdr_index)[shdr_index];
     const local_sym = atom.getSymbolPtr(self);
     local_sym.* = .{
@@ -2764,7 +2769,6 @@ fn updateLazySymbol(self: *Elf, lazy_sym: File.LazySymbol, lazy_metadata: LazySy
         .st_value = 0,
         .st_size = 0,
     };
-    const required_alignment = lazy_metadata.alignment;
     const vaddr = try self.allocateAtom(atom_index, code.len, required_alignment);
     errdefer self.freeAtom(atom_index);
     log.debug("allocated text block for {s} at 0x{x}", .{ name, vaddr });
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 2e1076cdf6247551045402bfb5108dfc3a54ea78..b0a05e4050757a19e386a67347f813bbe2a057c7 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -232,16 +232,11 @@ const is_hot_update_compatible = switch (builtin.target.os.tag) {
     else => false,
 };
 
-const LazySymbolTable = std.ArrayHashMapUnmanaged(
-    link.File.LazySymbol,
-    LazySymbolMetadata,
-    link.File.LazySymbol.Context,
-    true,
-);
+const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex, LazySymbolMetadata);
 
 const LazySymbolMetadata = struct {
-    atom: Atom.Index,
-    section: u8,
+    text_atom: ?Atom.Index = null,
+    data_const_atom: ?Atom.Index = null,
     alignment: u32,
 };
 
@@ -513,17 +508,13 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
     sub_prog_node.activate();
     defer sub_prog_node.end();
 
-    {
-        var lazy_it = self.lazy_syms.iterator();
-        while (lazy_it.next()) |lazy_entry| {
-            self.updateLazySymbol(
-                lazy_entry.key_ptr.*,
-                lazy_entry.value_ptr.*,
-            ) catch |err| switch (err) {
-                error.CodegenFail => return error.FlushFailure,
-                else => |e| return e,
-            };
-        }
+    // Most lazy symbols can be updated when the corresponding decl is,
+    // so we only have to worry about the one without an associated decl.
+    if (self.lazy_syms.get(.none)) |metadata| {
+        self.updateLazySymbol(.none, metadata) catch |err| switch (err) {
+            error.CodegenFail => return error.FlushFailure,
+            else => |e| return e,
+        };
     }
 
     const module = self.base.options.module orelse return error.LinkingWithoutZigSourceUnimplemented;
@@ -2309,7 +2300,29 @@ pub fn updateDecl(self: *MachO, module: *Module, decl_index: Module.Decl.Index)
     try self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-fn updateLazySymbol(self: *MachO, lazy_sym: File.LazySymbol, lazy_metadata: LazySymbolMetadata) !void {
+fn updateLazySymbol(self: *MachO, decl: Module.Decl.OptionalIndex, metadata: LazySymbolMetadata) !void {
+    const mod = self.base.options.module.?;
+    if (metadata.text_atom) |atom| try self.updateLazySymbolAtom(
+        File.LazySymbol.initDecl(.code, decl, mod),
+        atom,
+        self.text_section_index.?,
+        metadata.alignment,
+    );
+    if (metadata.data_const_atom) |atom| try self.updateLazySymbolAtom(
+        File.LazySymbol.initDecl(.const_data, decl, mod),
+        atom,
+        self.data_const_section_index.?,
+        metadata.alignment,
+    );
+}
+
+fn updateLazySymbolAtom(
+    self: *MachO,
+    sym: File.LazySymbol,
+    atom_index: Atom.Index,
+    section_index: u8,
+    required_alignment: u32,
+) !void {
     const gpa = self.base.allocator;
     const mod = self.base.options.module.?;
 
@@ -2318,19 +2331,18 @@ fn updateLazySymbol(self: *MachO, lazy_sym: File.LazySymbol, lazy_metadata: Lazy
 
     const name_str_index = blk: {
         const name = try std.fmt.allocPrint(gpa, "___lazy_{s}_{}", .{
-            @tagName(lazy_sym.kind),
-            lazy_sym.ty.fmt(mod),
+            @tagName(sym.kind),
+            sym.ty.fmt(mod),
         });
         defer gpa.free(name);
         break :blk try self.strtab.insert(gpa, name);
     };
     const name = self.strtab.get(name_str_index).?;
 
-    const atom_index = lazy_metadata.atom;
     const atom = self.getAtomPtr(atom_index);
     const local_sym_index = atom.getSymbolIndex().?;
 
-    const src = if (lazy_sym.ty.getOwnerDeclOrNull()) |owner_decl|
+    const src = if (sym.ty.getOwnerDeclOrNull()) |owner_decl|
         mod.declPtr(owner_decl).srcLoc()
     else
         Module.SrcLoc{
@@ -2338,14 +2350,9 @@ fn updateLazySymbol(self: *MachO, lazy_sym: File.LazySymbol, lazy_metadata: Lazy
             .parent_decl_node = undefined,
             .lazy = .unneeded,
         };
-    const res = try codegen.generateLazySymbol(
-        &self.base,
-        src,
-        lazy_sym,
-        &code_buffer,
-        .none,
-        .{ .parent_atom_index = local_sym_index },
-    );
+    const res = try codegen.generateLazySymbol(&self.base, src, sym, &code_buffer, .none, .{
+        .parent_atom_index = local_sym_index,
+    });
     const code = switch (res) {
         .ok => code_buffer.items,
         .fail => |em| {
@@ -2354,11 +2361,10 @@ fn updateLazySymbol(self: *MachO, lazy_sym: File.LazySymbol, lazy_metadata: Lazy
         },
     };
 
-    const required_alignment = lazy_metadata.alignment;
     const symbol = atom.getSymbolPtr(self);
     symbol.n_strx = name_str_index;
     symbol.n_type = macho.N_SECT;
-    symbol.n_sect = lazy_metadata.section + 1;
+    symbol.n_sect = section_index + 1;
     symbol.n_desc = 0;
 
     const vaddr = try self.allocateAtom(atom_index, code.len, required_alignment);
@@ -2381,26 +2387,16 @@ fn updateLazySymbol(self: *MachO, lazy_sym: File.LazySymbol, lazy_metadata: Lazy
     try self.writeAtom(atom_index, code);
 }
 
-pub fn getOrCreateAtomForLazySymbol(
-    self: *MachO,
-    lazy_sym: File.LazySymbol,
-    alignment: u32,
-) !Atom.Index {
-    const gop = try self.lazy_syms.getOrPutContext(self.base.allocator, lazy_sym, .{
-        .mod = self.base.options.module.?,
-    });
+pub fn getOrCreateAtomForLazySymbol(self: *MachO, sym: File.LazySymbol, alignment: u32) !Atom.Index {
+    const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
     errdefer _ = self.lazy_syms.pop();
-    if (!gop.found_existing) {
-        gop.value_ptr.* = .{
-            .atom = try self.createAtom(),
-            .section = switch (lazy_sym.kind) {
-                .code => self.text_section_index.?,
-                .const_data => self.data_const_section_index.?,
-            },
-            .alignment = alignment,
-        };
-    }
-    return gop.value_ptr.atom;
+    if (!gop.found_existing) gop.value_ptr.* = .{ .alignment = alignment };
+    const atom = switch (sym.kind) {
+        .code => &gop.value_ptr.text_atom,
+        .const_data => &gop.value_ptr.data_const_atom,
+    };
+    if (atom.* == null) atom.* = try self.createAtom();
+    return atom.*.?;
 }
 
 pub fn getOrCreateAtomForDecl(self: *MachO, decl_index: Module.Decl.Index) !Atom.Index {
-- 
2.54.0


From 5900dc05804a4089eb7d1ccd8a36b4ade16c02e4 Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 04:55:27 -0400
Subject: [PATCH 08/13] x86_64: fix typos

---
 src/arch/x86_64/CodeGen.zig                  | 4 ++--
 src/print_air.zig                            | 1 +
 test/behavior/bugs/12776.zig                 | 1 -
 test/behavior/bugs/1442.zig                  | 1 -
 test/behavior/error.zig                      | 4 ----
 test/behavior/struct.zig                     | 3 ---
 test/behavior/switch_prong_err_enum.zig      | 1 -
 test/behavior/switch_prong_implicit_cast.zig | 1 -
 8 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index a2d0d4da410f8a4e1efc633e197d11ef0642bbcd..748f996032d1c077505d125586bab7e69baff81d 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -2528,7 +2528,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
         const dst_lock = self.register_manager.lockReg(dst_reg);
         defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
-        const pl_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
+        const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
         const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
         try self.asmRegisterMemory(
             .lea,
@@ -5602,7 +5602,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
 
         const rhs_mcv = try self.resolveInst(bin_op.rhs);
         const rhs_lock = switch (rhs_mcv) {
-            .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
+            .register => |reg| self.register_manager.lockReg(reg),
             else => null,
         };
         defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
diff --git a/src/print_air.zig b/src/print_air.zig
index 803a0f2886bc4065a036e5137b89793874db9bed..2493d234ebf4c3aff46b34f1e2bd25e285e8b817 100644
--- a/src/print_air.zig
+++ b/src/print_air.zig
@@ -804,6 +804,7 @@ const Writer = struct {
         var case_i: u32 = 0;
 
         try w.writeOperand(s, inst, 0, pl_op.operand);
+        if (w.skip_body) return s.writeAll(", ...");
         const old_indent = w.indent;
         w.indent += 2;
 
diff --git a/test/behavior/bugs/12776.zig b/test/behavior/bugs/12776.zig
index f95709a2a457b169a179a5178653dfdc5e797ddb..1611fe27eeaee032de5618d8e27cf54f5cbb051e 100644
--- a/test/behavior/bugs/12776.zig
+++ b/test/behavior/bugs/12776.zig
@@ -31,7 +31,6 @@ const CPU = packed struct {
 test {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
 
     var ram = try RAM.new();
diff --git a/test/behavior/bugs/1442.zig b/test/behavior/bugs/1442.zig
index 5f788f6cef6b7f199a326acd94c40488f7c929fe..f9bef6ec504e465809f7bb7a39724c6d92be0f1a 100644
--- a/test/behavior/bugs/1442.zig
+++ b/test/behavior/bugs/1442.zig
@@ -9,7 +9,6 @@ const Union = union(enum) {
 test "const error union field alignment" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     var union_or_err: anyerror!Union = Union{ .Color = 1234 };
     try std.testing.expect((union_or_err catch unreachable).Color == 1234);
diff --git a/test/behavior/error.zig b/test/behavior/error.zig
index b1f77f33a8cbf67201d5e61c80d3db5885fd8b86..0cd9be05ca409308e3d09f2d1d998db6adcbc0e9 100644
--- a/test/behavior/error.zig
+++ b/test/behavior/error.zig
@@ -370,7 +370,6 @@ fn intLiteral(str: []const u8) !?i64 {
 }
 
 test "nested error union function call in optional unwrap" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -499,7 +498,6 @@ test "function pointer with return type that is error union with payload which i
 }
 
 test "return result loc as peer result loc in inferred error set function" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -531,7 +529,6 @@ test "return result loc as peer result loc in inferred error set function" {
 }
 
 test "error payload type is correctly resolved" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -665,7 +662,6 @@ test "coerce error set to the current inferred error set" {
 }
 
 test "error union payload is properly aligned" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/struct.zig b/test/behavior/struct.zig
index 37b24a1ed5c89643bda42616cf13264a9858a7c8..1dcfce53dd4f6b4c238a5533139f83d33ddcc81b 100644
--- a/test/behavior/struct.zig
+++ b/test/behavior/struct.zig
@@ -1116,7 +1116,6 @@ test "for loop over pointers to struct, getting field from struct pointer" {
 }
 
 test "anon init through error unions and optionals" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1162,7 +1161,6 @@ test "anon init through optional" {
 }
 
 test "anon init through error union" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1182,7 +1180,6 @@ test "anon init through error union" {
 }
 
 test "typed init through error unions and optionals" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/switch_prong_err_enum.zig b/test/behavior/switch_prong_err_enum.zig
index ddcc76634e8b7a9c9710dd5f9dc7035ee0b639c5..15d366d04f1f20c632354f982f375eaeeebb53c9 100644
--- a/test/behavior/switch_prong_err_enum.zig
+++ b/test/behavior/switch_prong_err_enum.zig
@@ -23,7 +23,6 @@ fn doThing(form_id: u64) anyerror!FormValue {
 test "switch prong returns error enum" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     switch (doThing(17) catch unreachable) {
diff --git a/test/behavior/switch_prong_implicit_cast.zig b/test/behavior/switch_prong_implicit_cast.zig
index 26ec57676b0cfa8acccd2e511972b3afc5e69500..54107bb6bd1e9ff60b9ca58faffc3b2868674354 100644
--- a/test/behavior/switch_prong_implicit_cast.zig
+++ b/test/behavior/switch_prong_implicit_cast.zig
@@ -17,7 +17,6 @@ fn foo(id: u64) !FormValue {
 test "switch prong implicit cast" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const result = switch (foo(2) catch unreachable) {
-- 
2.54.0


From c0e9b849974cdbc93053919f54f0cf3830243572 Mon Sep 17 00:00:00 2001
From: Jakub Konka <kubkon@jakubkonka.com>
Date: Mon, 3 Apr 2023 18:30:48 +0200
Subject: [PATCH 09/13] elf: preallocate offsets for PT_PHDR and PT_LOAD
 (empty) segment

Otherwise, we will be using `undefined` as the offset to allocate
remaining phdrs and shdrs.
---
 src/link/Elf.zig | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index ec05f5895a3aba97ddae4b0736caffa7cbd71560..74a538b8cc626cfcc75e86ca3eb5a106d938a277 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -110,7 +110,7 @@ program_headers: std.ArrayListUnmanaged(elf.Elf64_Phdr) = .{},
 phdr_table_index: ?u16 = null,
 /// The index into the program headers of the PT_LOAD program header containing the phdr
 /// Most linkers would merge this with phdr_load_ro_index,
-/// but hot swap means we can't ensure they are consecutive.
+/// but incremental linking means we can't ensure they are consecutive.
 phdr_table_load_index: ?u16 = null,
 /// The index into the program headers of a PT_LOAD program header with Read and Execute flags
 phdr_load_re_index: ?u16 = null,
@@ -473,18 +473,19 @@ pub fn populateMissingMetadata(self: *Elf) !void {
         var new_phdr_table_index: ?u16 = null;
         if (self.phdr_table_index == null) {
             new_phdr_table_index = @intCast(u16, self.program_headers.items.len);
-            const phalign: u16 = switch (self.ptr_width) {
+            const p_align: u16 = switch (self.ptr_width) {
                 .p32 => @alignOf(elf.Elf32_Phdr),
                 .p64 => @alignOf(elf.Elf64_Phdr),
             };
+            const off = self.findFreeSpace(1, p_align);
             try self.program_headers.append(gpa, .{
                 .p_type = elf.PT_PHDR,
-                .p_offset = undefined,
-                .p_filesz = undefined,
-                .p_vaddr = undefined,
-                .p_paddr = undefined,
-                .p_memsz = undefined,
-                .p_align = phalign,
+                .p_offset = off,
+                .p_filesz = 1,
+                .p_vaddr = 0,
+                .p_paddr = 0,
+                .p_memsz = 0,
+                .p_align = p_align,
                 .p_flags = elf.PF_R,
             });
             self.phdr_table_dirty = true;
@@ -494,14 +495,18 @@ pub fn populateMissingMetadata(self: *Elf) !void {
             self.phdr_table_load_index = @intCast(u16, self.program_headers.items.len);
             // TODO Same as for GOT
             const phdr_addr: u64 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x1000000 else 0x1000;
+            const p_align = self.page_size;
+            const off = self.findFreeSpace(1, p_align);
+            const file_size: u32 = 1;
+            log.debug("found PT_LOAD free space 0x{x} to 0x{x}", .{ off, off + file_size });
             try self.program_headers.append(gpa, .{
                 .p_type = elf.PT_LOAD,
-                .p_offset = undefined,
-                .p_filesz = undefined,
+                .p_offset = off,
+                .p_filesz = file_size,
                 .p_vaddr = phdr_addr,
                 .p_paddr = phdr_addr,
-                .p_memsz = undefined,
-                .p_align = self.page_size,
+                .p_memsz = file_size,
+                .p_align = p_align,
                 .p_flags = elf.PF_R,
             });
             self.phdr_table_dirty = true;
-- 
2.54.0


From 9fd460821f992c51b35a54ba93562af93ac478f6 Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 13:55:20 -0400
Subject: [PATCH 10/13] elf: cleanup phdr tracking

Since one of the program header entries is now the program header table
itself, we can avoid tracking it explicitly and just track it as yet
another program segment.
---
 src/link/Elf.zig | 285 +++++++++++++++++++++--------------------------
 1 file changed, 128 insertions(+), 157 deletions(-)

diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 74a538b8cc626cfcc75e86ca3eb5a106d938a277..7a3a0b0fdddfbaf0fc49f0d237b17bb1c325d589 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -391,17 +391,6 @@ fn detectAllocCollision(self: *Elf, start: u64, size: u64) ?u64 {
 
     const end = start + padToIdeal(size);
 
-    if (self.phdr_table_index) |index| {
-        const off = self.program_headers.items[index].p_offset;
-        const phdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Phdr) else @sizeOf(elf.Elf64_Phdr);
-        const tight_size = self.program_headers.items.len * phdr_size;
-        const increased_size = padToIdeal(tight_size);
-        const test_end = off + increased_size;
-        if (end > off and start < test_end) {
-            return test_end;
-        }
-    }
-
     if (self.shdr_table_offset) |off| {
         const shdr_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Shdr) else @sizeOf(elf.Elf64_Shdr);
         const tight_size = self.sections.slice().len * shdr_size;
@@ -433,10 +422,6 @@ pub fn allocatedSize(self: *Elf, start: u64) u64 {
     if (start == 0)
         return 0;
     var min_pos: u64 = std.math.maxInt(u64);
-    if (self.phdr_table_index) |index| {
-        const off = self.program_headers.items[index].p_offset;
-        if (off > start and off < min_pos) min_pos = off;
-    }
     if (self.shdr_table_offset) |off| {
         if (off > start and off < min_pos) min_pos = off;
     }
@@ -469,151 +454,133 @@ pub fn populateMissingMetadata(self: *Elf) !void {
     };
     const ptr_size: u8 = self.ptrWidthBytes();
 
-    { // Program Headers
-        var new_phdr_table_index: ?u16 = null;
-        if (self.phdr_table_index == null) {
-            new_phdr_table_index = @intCast(u16, self.program_headers.items.len);
-            const p_align: u16 = switch (self.ptr_width) {
-                .p32 => @alignOf(elf.Elf32_Phdr),
-                .p64 => @alignOf(elf.Elf64_Phdr),
-            };
-            const off = self.findFreeSpace(1, p_align);
-            try self.program_headers.append(gpa, .{
-                .p_type = elf.PT_PHDR,
-                .p_offset = off,
-                .p_filesz = 1,
-                .p_vaddr = 0,
-                .p_paddr = 0,
-                .p_memsz = 0,
-                .p_align = p_align,
-                .p_flags = elf.PF_R,
-            });
-            self.phdr_table_dirty = true;
-        }
+    if (self.phdr_table_index == null) {
+        self.phdr_table_index = @intCast(u16, self.program_headers.items.len);
+        const p_align: u16 = switch (self.ptr_width) {
+            .p32 => @alignOf(elf.Elf32_Phdr),
+            .p64 => @alignOf(elf.Elf64_Phdr),
+        };
+        try self.program_headers.append(gpa, .{
+            .p_type = elf.PT_PHDR,
+            .p_offset = 0,
+            .p_filesz = 0,
+            .p_vaddr = 0,
+            .p_paddr = 0,
+            .p_memsz = 0,
+            .p_align = p_align,
+            .p_flags = elf.PF_R,
+        });
+        self.phdr_table_dirty = true;
+    }
 
-        if (self.phdr_table_load_index == null) {
-            self.phdr_table_load_index = @intCast(u16, self.program_headers.items.len);
-            // TODO Same as for GOT
-            const phdr_addr: u64 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x1000000 else 0x1000;
-            const p_align = self.page_size;
-            const off = self.findFreeSpace(1, p_align);
-            const file_size: u32 = 1;
-            log.debug("found PT_LOAD free space 0x{x} to 0x{x}", .{ off, off + file_size });
-            try self.program_headers.append(gpa, .{
-                .p_type = elf.PT_LOAD,
-                .p_offset = off,
-                .p_filesz = file_size,
-                .p_vaddr = phdr_addr,
-                .p_paddr = phdr_addr,
-                .p_memsz = file_size,
-                .p_align = p_align,
-                .p_flags = elf.PF_R,
-            });
-            self.phdr_table_dirty = true;
-        }
+    if (self.phdr_table_load_index == null) {
+        self.phdr_table_load_index = @intCast(u16, self.program_headers.items.len);
+        // TODO Same as for GOT
+        const phdr_addr: u64 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x1000000 else 0x1000;
+        const p_align = self.page_size;
+        try self.program_headers.append(gpa, .{
+            .p_type = elf.PT_LOAD,
+            .p_offset = 0,
+            .p_filesz = 0,
+            .p_vaddr = phdr_addr,
+            .p_paddr = phdr_addr,
+            .p_memsz = 0,
+            .p_align = p_align,
+            .p_flags = elf.PF_R,
+        });
+        self.phdr_table_dirty = true;
+    }
 
-        if (self.phdr_load_re_index == null) {
-            self.phdr_load_re_index = @intCast(u16, self.program_headers.items.len);
-            const file_size = self.base.options.program_code_size_hint;
-            const p_align = self.page_size;
-            const off = self.findFreeSpace(file_size, p_align);
-            log.debug("found PT_LOAD RE free space 0x{x} to 0x{x}", .{ off, off + file_size });
-            const entry_addr: u64 = self.entry_addr orelse if (self.base.options.target.cpu.arch == .spu_2) @as(u64, 0) else default_entry_addr;
-            try self.program_headers.append(gpa, .{
-                .p_type = elf.PT_LOAD,
-                .p_offset = off,
-                .p_filesz = file_size,
-                .p_vaddr = entry_addr,
-                .p_paddr = entry_addr,
-                .p_memsz = file_size,
-                .p_align = p_align,
-                .p_flags = elf.PF_X | elf.PF_R | elf.PF_W,
-            });
-            self.entry_addr = null;
-            self.phdr_table_dirty = true;
-        }
+    if (self.phdr_load_re_index == null) {
+        self.phdr_load_re_index = @intCast(u16, self.program_headers.items.len);
+        const file_size = self.base.options.program_code_size_hint;
+        const p_align = self.page_size;
+        const off = self.findFreeSpace(file_size, p_align);
+        log.debug("found PT_LOAD RE free space 0x{x} to 0x{x}", .{ off, off + file_size });
+        const entry_addr: u64 = self.entry_addr orelse if (self.base.options.target.cpu.arch == .spu_2) @as(u64, 0) else default_entry_addr;
+        try self.program_headers.append(gpa, .{
+            .p_type = elf.PT_LOAD,
+            .p_offset = off,
+            .p_filesz = file_size,
+            .p_vaddr = entry_addr,
+            .p_paddr = entry_addr,
+            .p_memsz = file_size,
+            .p_align = p_align,
+            .p_flags = elf.PF_X | elf.PF_R | elf.PF_W,
+        });
+        self.entry_addr = null;
+        self.phdr_table_dirty = true;
+    }
 
-        if (self.phdr_got_index == null) {
-            self.phdr_got_index = @intCast(u16, self.program_headers.items.len);
-            const file_size = @as(u64, ptr_size) * self.base.options.symbol_count_hint;
-            // We really only need ptr alignment but since we are using PROGBITS, linux requires
-            // page align.
-            const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
-            const off = self.findFreeSpace(file_size, p_align);
-            log.debug("found PT_LOAD GOT free space 0x{x} to 0x{x}", .{ off, off + file_size });
-            // TODO instead of hard coding the vaddr, make a function to find a vaddr to put things at.
-            // we'll need to re-use that function anyway, in case the GOT grows and overlaps something
-            // else in virtual memory.
-            const got_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x4000000 else 0x8000;
-            try self.program_headers.append(gpa, .{
-                .p_type = elf.PT_LOAD,
-                .p_offset = off,
-                .p_filesz = file_size,
-                .p_vaddr = got_addr,
-                .p_paddr = got_addr,
-                .p_memsz = file_size,
-                .p_align = p_align,
-                .p_flags = elf.PF_R | elf.PF_W,
-            });
-            self.phdr_table_dirty = true;
-        }
+    if (self.phdr_got_index == null) {
+        self.phdr_got_index = @intCast(u16, self.program_headers.items.len);
+        const file_size = @as(u64, ptr_size) * self.base.options.symbol_count_hint;
+        // We really only need ptr alignment but since we are using PROGBITS, linux requires
+        // page align.
+        const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
+        const off = self.findFreeSpace(file_size, p_align);
+        log.debug("found PT_LOAD GOT free space 0x{x} to 0x{x}", .{ off, off + file_size });
+        // TODO instead of hard coding the vaddr, make a function to find a vaddr to put things at.
+        // we'll need to re-use that function anyway, in case the GOT grows and overlaps something
+        // else in virtual memory.
+        const got_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x4000000 else 0x8000;
+        try self.program_headers.append(gpa, .{
+            .p_type = elf.PT_LOAD,
+            .p_offset = off,
+            .p_filesz = file_size,
+            .p_vaddr = got_addr,
+            .p_paddr = got_addr,
+            .p_memsz = file_size,
+            .p_align = p_align,
+            .p_flags = elf.PF_R | elf.PF_W,
+        });
+        self.phdr_table_dirty = true;
+    }
 
-        if (self.phdr_load_ro_index == null) {
-            self.phdr_load_ro_index = @intCast(u16, self.program_headers.items.len);
-            // TODO Find a hint about how much data need to be in rodata ?
-            const file_size = 1024;
-            // Same reason as for GOT
-            const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
-            const off = self.findFreeSpace(file_size, p_align);
-            log.debug("found PT_LOAD RO free space 0x{x} to 0x{x}", .{ off, off + file_size });
-            // TODO Same as for GOT
-            const rodata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0xc000000 else 0xa000;
-            try self.program_headers.append(gpa, .{
-                .p_type = elf.PT_LOAD,
-                .p_offset = off,
-                .p_filesz = file_size,
-                .p_vaddr = rodata_addr,
-                .p_paddr = rodata_addr,
-                .p_memsz = file_size,
-                .p_align = p_align,
-                .p_flags = elf.PF_R | elf.PF_W,
-            });
-            self.phdr_table_dirty = true;
-        }
+    if (self.phdr_load_ro_index == null) {
+        self.phdr_load_ro_index = @intCast(u16, self.program_headers.items.len);
+        // TODO Find a hint about how much data need to be in rodata ?
+        const file_size = 1024;
+        // Same reason as for GOT
+        const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
+        const off = self.findFreeSpace(file_size, p_align);
+        log.debug("found PT_LOAD RO free space 0x{x} to 0x{x}", .{ off, off + file_size });
+        // TODO Same as for GOT
+        const rodata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0xc000000 else 0xa000;
+        try self.program_headers.append(gpa, .{
+            .p_type = elf.PT_LOAD,
+            .p_offset = off,
+            .p_filesz = file_size,
+            .p_vaddr = rodata_addr,
+            .p_paddr = rodata_addr,
+            .p_memsz = file_size,
+            .p_align = p_align,
+            .p_flags = elf.PF_R | elf.PF_W,
+        });
+        self.phdr_table_dirty = true;
+    }
 
-        if (self.phdr_load_rw_index == null) {
-            self.phdr_load_rw_index = @intCast(u16, self.program_headers.items.len);
-            // TODO Find a hint about how much data need to be in data ?
-            const file_size = 1024;
-            // Same reason as for GOT
-            const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
-            const off = self.findFreeSpace(file_size, p_align);
-            log.debug("found PT_LOAD RW free space 0x{x} to 0x{x}", .{ off, off + file_size });
-            // TODO Same as for GOT
-            const rwdata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x10000000 else 0xc000;
-            try self.program_headers.append(gpa, .{
-                .p_type = elf.PT_LOAD,
-                .p_offset = off,
-                .p_filesz = file_size,
-                .p_vaddr = rwdata_addr,
-                .p_paddr = rwdata_addr,
-                .p_memsz = file_size,
-                .p_align = p_align,
-                .p_flags = elf.PF_R | elf.PF_W,
-            });
-            self.phdr_table_dirty = true;
-        }
-
-        if (new_phdr_table_index) |index| {
-            const phsize: u64 = switch (self.ptr_width) {
-                .p32 => @sizeOf(elf.Elf32_Phdr),
-                .p64 => @sizeOf(elf.Elf64_Phdr),
-            };
-            const phdr_table = &self.program_headers.items[index];
-            phdr_table.p_offset = self.findFreeSpace(self.program_headers.items.len * phsize, @intCast(u32, phdr_table.p_align));
-            self.phdr_table_index = index;
-            self.phdr_table_dirty = true;
-        }
+    if (self.phdr_load_rw_index == null) {
+        self.phdr_load_rw_index = @intCast(u16, self.program_headers.items.len);
+        // TODO Find a hint about how much data need to be in data ?
+        const file_size = 1024;
+        // Same reason as for GOT
+        const p_align = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
+        const off = self.findFreeSpace(file_size, p_align);
+        log.debug("found PT_LOAD RW free space 0x{x} to 0x{x}", .{ off, off + file_size });
+        // TODO Same as for GOT
+        const rwdata_addr: u32 = if (self.base.options.target.cpu.arch.ptrBitWidth() >= 32) 0x10000000 else 0xc000;
+        try self.program_headers.append(gpa, .{
+            .p_type = elf.PT_LOAD,
+            .p_offset = off,
+            .p_filesz = file_size,
+            .p_vaddr = rwdata_addr,
+            .p_paddr = rwdata_addr,
+            .p_memsz = file_size,
+            .p_align = p_align,
+            .p_flags = elf.PF_R | elf.PF_W,
+        });
+        self.phdr_table_dirty = true;
     }
 
     if (self.shstrtab_index == null) {
@@ -1186,8 +1153,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
         const needed_size = self.program_headers.items.len * phsize;
 
         if (needed_size > allocated_size) {
-            self.phdr_table_index = null; // free the space
-            defer self.phdr_table_index = phdr_table_index;
+            phdr_table.p_offset = 0; // free the space
             phdr_table.p_offset = self.findFreeSpace(needed_size, @intCast(u32, phdr_table.p_align));
         }
 
@@ -1227,6 +1193,11 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
                 try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), phdr_table.p_offset);
             },
         }
+
+        // We don't actually care if the phdr load section overlaps, only the phdr section matters.
+        phdr_table_load.p_offset = 0;
+        phdr_table_load.p_filesz = 0;
+
         self.phdr_table_dirty = false;
     }
 
-- 
2.54.0


From 821eb595f4bd65d003a2cb499c7953f4f57a4c4a Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 14:14:55 -0400
Subject: [PATCH 11/13] x86_64: implement cmp_lt_errors_len

---
 src/arch/x86_64/CodeGen.zig | 69 ++++++++++++++++++++++++++++++++-----
 src/codegen.zig             | 10 ++++--
 test/behavior/cast.zig      |  1 -
 3 files changed, 68 insertions(+), 12 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 748f996032d1c077505d125586bab7e69baff81d..8a3e0852a2e525e1f2a1cf98ae586a4bfd936707 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -5650,9 +5650,62 @@ fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {
 
 fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
-    const operand = try self.resolveInst(un_op);
-    _ = operand;
-    const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCmpLtErrorsLen for {}", .{self.target.cpu.arch});
+    const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
+        const addr_reg = try self.register_manager.allocReg(null, gp);
+        const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
+        defer self.register_manager.unlockReg(addr_lock);
+
+        if (self.bin_file.cast(link.File.Elf)) |elf_file| {
+            const atom_index = try elf_file.getOrCreateAtomForLazySymbol(
+                .{ .kind = .const_data, .ty = Type.anyerror },
+                4, // dword alignment
+            );
+            const got_addr = elf_file.getAtom(atom_index).getOffsetTableAddress(elf_file);
+            try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
+                .base = .ds,
+                .disp = @intCast(i32, got_addr),
+            }));
+        } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
+            const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
+                .{ .kind = .const_data, .ty = Type.anyerror },
+                4, // dword alignment
+            );
+            const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
+            try self.genSetReg(Type.usize, addr_reg, .{ .linker_load = .{
+                .type = .got,
+                .sym_index = sym_index,
+            } });
+        } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
+            const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
+                .{ .kind = .const_data, .ty = Type.anyerror },
+                4, // dword alignment
+            );
+            const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
+            try self.genSetReg(Type.usize, addr_reg, .{ .linker_load = .{
+                .type = .got,
+                .sym_index = sym_index,
+            } });
+        } else {
+            return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
+        }
+
+        try self.spillEflagsIfOccupied();
+        self.eflags_inst = inst;
+
+        const op_ty = self.air.typeOf(un_op);
+        const op_abi_size = @intCast(u32, op_ty.abiSize(self.target.*));
+        const op_mcv = try self.resolveInst(un_op);
+        const dst_reg = switch (op_mcv) {
+            .register => |reg| reg,
+            else => try self.copyToTmpRegister(op_ty, op_mcv),
+        };
+        try self.asmRegisterMemory(
+            .cmp,
+            registerAlias(dst_reg, op_abi_size),
+            Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = addr_reg }),
+        );
+        break :result .{ .eflags = .b };
+    };
     return self.finishAir(inst, result, .{ un_op, .none, .none });
 }
 
@@ -8025,15 +8078,15 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
         try self.truncateRegister(err_ty, err_reg.to32());
 
         try self.asmRegisterMemory(.mov, start_reg.to32(), Memory.sib(.dword, .{
-            .base = addr_reg.to64(),
-            .scale_index = .{ .scale = 4, .index = err_reg.to64() },
-            .disp = 0,
-        }));
-        try self.asmRegisterMemory(.mov, end_reg.to32(), Memory.sib(.dword, .{
             .base = addr_reg.to64(),
             .scale_index = .{ .scale = 4, .index = err_reg.to64() },
             .disp = 4,
         }));
+        try self.asmRegisterMemory(.mov, end_reg.to32(), Memory.sib(.dword, .{
+            .base = addr_reg.to64(),
+            .scale_index = .{ .scale = 4, .index = err_reg.to64() },
+            .disp = 8,
+        }));
         try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());
         try self.asmRegisterMemory(.lea, start_reg.to64(), Memory.sib(.byte, .{
             .base = addr_reg.to64(),
diff --git a/src/codegen.zig b/src/codegen.zig
index 67ada2fedd6e3eef2ef31f71213f2c97864f150f..57c33ad524c06024ee59cf3bab93f45dd258d256 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -124,13 +124,17 @@ pub fn generateLazySymbol(
 
     if (lazy_sym.kind == .const_data and lazy_sym.ty.isAnyError()) {
         const err_names = mod.error_name_list.items;
-        try code.resize(err_names.len * 4);
-        for (err_names, 0..) |err_name, index| {
-            mem.writeInt(u32, code.items[index * 4 ..][0..4], @intCast(u32, code.items.len), endian);
+        mem.writeInt(u32, try code.addManyAsArray(4), @intCast(u32, err_names.len), endian);
+        var offset = code.items.len;
+        try code.resize((1 + err_names.len + 1) * 4);
+        for (err_names) |err_name| {
+            mem.writeInt(u32, code.items[offset..][0..4], @intCast(u32, code.items.len), endian);
+            offset += 4;
             try code.ensureUnusedCapacity(err_name.len + 1);
             code.appendSliceAssumeCapacity(err_name);
             code.appendAssumeCapacity(0);
         }
+        mem.writeInt(u32, code.items[offset..][0..4], @intCast(u32, code.items.len), endian);
         return Result.ok;
     } else return .{ .fail = try ErrorMsg.create(
         bin_file.allocator,
diff --git a/test/behavior/cast.zig b/test/behavior/cast.zig
index 470b17e0b68e6c72c200a0fea956e126287bd21a..802751573094adb8eb132991f9ba3176e641fe98 100644
--- a/test/behavior/cast.zig
+++ b/test/behavior/cast.zig
@@ -402,7 +402,6 @@ test "expected [*c]const u8, found [*:0]const u8" {
 
 test "explicit cast from integer to error type" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-- 
2.54.0


From c91929090db337810aa3b55f9bd26f5fd0af4fff Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Mon, 3 Apr 2023 14:17:00 -0400
Subject: [PATCH 12/13] start: disable extra start logic on various x86_64
 subtargets

---
 lib/std/start.zig                                        | 1 +
 test/cases/aarch64-macos/hello_world_with_updates.0.zig  | 2 +-
 test/cases/x86_64-linux/hello_world_with_updates.0.zig   | 5 ++++-
 test/cases/x86_64-macos/hello_world_with_updates.0.zig   | 2 +-
 test/cases/x86_64-windows/hello_world_with_updates.0.zig | 4 +++-
 5 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/std/start.zig b/lib/std/start.zig
index e79dd1b9a72e3ddc675c135a712d0111179c5d8e..1b686e43f5cea25335f44a4427f4f2c9c946f050 100644
--- a/lib/std/start.zig
+++ b/lib/std/start.zig
@@ -19,6 +19,7 @@ const start_sym_name = if (native_arch.isMIPS()) "__start" else "_start";
 // self-hosted is capable enough to handle all of the real start.zig logic.
 pub const simplified_logic =
     builtin.zig_backend == .stage2_wasm or
+    (builtin.zig_backend == .stage2_x86_64 and (builtin.link_libc or builtin.os.tag == .plan9)) or
     builtin.zig_backend == .stage2_x86 or
     builtin.zig_backend == .stage2_aarch64 or
     builtin.zig_backend == .stage2_arm or
diff --git a/test/cases/aarch64-macos/hello_world_with_updates.0.zig b/test/cases/aarch64-macos/hello_world_with_updates.0.zig
index 9f516ff1390f02aafc6203c8625793f8e58acdaf..90aa14ab65a5217950729681a26d1c75fdabd722 100644
--- a/test/cases/aarch64-macos/hello_world_with_updates.0.zig
+++ b/test/cases/aarch64-macos/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=aarch64-macos
 //
-// :109:9: error: root struct of file 'tmp' has no member named 'main'
+// :110:9: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-linux/hello_world_with_updates.0.zig b/test/cases/x86_64-linux/hello_world_with_updates.0.zig
index 40abdd6c1fa4217b99f077793ecbb56bfa3f89a0..e498eacf930d597202dff035dd1a902309686de4 100644
--- a/test/cases/x86_64-linux/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-linux/hello_world_with_updates.0.zig
@@ -2,4 +2,7 @@
 // output_mode=Exe
 // target=x86_64-linux
 //
-// :109:9: error: root struct of file 'tmp' has no member named 'main'
+// :604:45: error: root struct of file 'tmp' has no member named 'main'
+// :553:12: note: called from here
+// :503:36: note: called from here
+// :466:17: note: called from here
diff --git a/test/cases/x86_64-macos/hello_world_with_updates.0.zig b/test/cases/x86_64-macos/hello_world_with_updates.0.zig
index e0680c81d7420441fabd29498bf9258a5c16f291..d91bd9dc9191538f63a619b9771d8b576cfbbe6d 100644
--- a/test/cases/x86_64-macos/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-macos/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=x86_64-macos
 //
-// :109:9: error: root struct of file 'tmp' has no member named 'main'
+// :110:9: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-windows/hello_world_with_updates.0.zig b/test/cases/x86_64-windows/hello_world_with_updates.0.zig
index 04e1d4cfad0ac9045fb033b9f550e2d5fb1e64c8..320057b86eaa9afac94bf38bafd37e63b02b266f 100644
--- a/test/cases/x86_64-windows/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-windows/hello_world_with_updates.0.zig
@@ -2,4 +2,6 @@
 // output_mode=Exe
 // target=x86_64-windows
 //
-// :130:9: error: root struct of file 'tmp' has no member named 'main'
+// :604:45: error: root struct of file 'tmp' has no member named 'main'
+// :553:12: note: called from here
+// :389:65: note: called from here
-- 
2.54.0


From 83b7dbe52f75161a2ac6f5d8f39b275fd9473c15 Mon Sep 17 00:00:00 2001
From: Jacob Young <jacobly0@users.noreply.github.com>
Date: Tue, 4 Apr 2023 11:39:02 -0400
Subject: [PATCH 13/13] cases: disable failing incremental tests

Enabling start.zig logic breaks incremental compilation
See #15174
---
 .../cases/aarch64-macos/hello_world_with_updates.0.zig |  2 +-
 test/cases/x86_64-linux/hello_world_with_updates.0.zig |  8 ++++----
 test/cases/x86_64-macos/hello_world_with_updates.0.zig |  2 +-
 .../x86_64-windows/hello_world_with_updates.0.zig      |  6 +++---
 test/src/Cases.zig                                     | 10 ++++++----
 5 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/test/cases/aarch64-macos/hello_world_with_updates.0.zig b/test/cases/aarch64-macos/hello_world_with_updates.0.zig
index 90aa14ab65a5217950729681a26d1c75fdabd722..f939a7fa712e2c175d5e1d120e2875f871b5a27c 100644
--- a/test/cases/aarch64-macos/hello_world_with_updates.0.zig
+++ b/test/cases/aarch64-macos/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=aarch64-macos
 //
-// :110:9: error: root struct of file 'tmp' has no member named 'main'
+// :?:?: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-linux/hello_world_with_updates.0.zig b/test/cases/x86_64-linux/hello_world_with_updates.0.zig
index e498eacf930d597202dff035dd1a902309686de4..c330234fdbb4abc98756b3205aa7e3d39d857ddc 100644
--- a/test/cases/x86_64-linux/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-linux/hello_world_with_updates.0.zig
@@ -2,7 +2,7 @@
 // output_mode=Exe
 // target=x86_64-linux
 //
-// :604:45: error: root struct of file 'tmp' has no member named 'main'
-// :553:12: note: called from here
-// :503:36: note: called from here
-// :466:17: note: called from here
+// :?:?: error: root struct of file 'tmp' has no member named 'main'
+// :?:?: note: called from here
+// :?:?: note: called from here
+// :?:?: note: called from here
diff --git a/test/cases/x86_64-macos/hello_world_with_updates.0.zig b/test/cases/x86_64-macos/hello_world_with_updates.0.zig
index d91bd9dc9191538f63a619b9771d8b576cfbbe6d..7fc13a4fa567ceacaf31b9e8f8916322ab987532 100644
--- a/test/cases/x86_64-macos/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-macos/hello_world_with_updates.0.zig
@@ -2,4 +2,4 @@
 // output_mode=Exe
 // target=x86_64-macos
 //
-// :110:9: error: root struct of file 'tmp' has no member named 'main'
+// :?:?: error: root struct of file 'tmp' has no member named 'main'
diff --git a/test/cases/x86_64-windows/hello_world_with_updates.0.zig b/test/cases/x86_64-windows/hello_world_with_updates.0.zig
index 320057b86eaa9afac94bf38bafd37e63b02b266f..d322de2d1b48b596a6e0f68078f178ad3014b5b4 100644
--- a/test/cases/x86_64-windows/hello_world_with_updates.0.zig
+++ b/test/cases/x86_64-windows/hello_world_with_updates.0.zig
@@ -2,6 +2,6 @@
 // output_mode=Exe
 // target=x86_64-windows
 //
-// :604:45: error: root struct of file 'tmp' has no member named 'main'
-// :553:12: note: called from here
-// :389:65: note: called from here
+// :?:?: error: root struct of file 'tmp' has no member named 'main'
+// :?:?: note: called from here
+// :?:?: note: called from here
diff --git a/test/src/Cases.zig b/test/src/Cases.zig
index c3a4c1df4737d56db37ccee3e1db90cddc87c93b..890f4f6f894a795e06fa8849baa55020266a4f56 100644
--- a/test/src/Cases.zig
+++ b/test/src/Cases.zig
@@ -1045,8 +1045,7 @@ pub fn main() !void {
     var ctx = Cases.init(gpa, arena);
 
     var test_it = TestIterator{ .filenames = filenames.items };
-    while (test_it.next()) |maybe_batch| {
-        const batch = maybe_batch orelse break;
+    while (try test_it.next()) |batch| {
         const strategy: TestStrategy = if (batch.len > 1) .incremental else .independent;
         var cases = std.ArrayList(usize).init(arena);
 
@@ -1084,6 +1083,11 @@ pub fn main() !void {
 
             for (cases.items) |case_index| {
                 const case = &ctx.cases.items[case_index];
+                if (strategy == .incremental and case.backend == .stage2 and case.target.getCpuArch() == .x86_64 and !case.link_libc and case.target.getOsTag() != .plan9) {
+                    // https://github.com/ziglang/zig/issues/15174
+                    continue;
+                }
+
                 switch (manifest.type) {
                     .compile => {
                         case.addCompile(src);
@@ -1115,8 +1119,6 @@ pub fn main() !void {
                 }
             }
         }
-    } else |err| {
-        return err;
     }
 
     return runCases(&ctx, zig_exe_path);
-- 
2.54.0