authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-02-21 23:31:55+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-02-22 21:56:34+01:00
log261711722175447d402632fd7f55b371bfc72df2
tree93a12c77f9d7929c1c3e08b031791f7b2093c5cf
parent87a0d754be9a26aeabb90f25d2971d263680fe9e

x64: fix lowering of error unions (we didn't pad to alignment)

* fix returning large values on stack from procedure calls - we need to explicitly specify source and dest base registers for `genSetStack` as well

2 files changed, 101 insertions(+), 56 deletions(-)

src/arch/x86_64/CodeGen.zig+70-46
......@@ -886,7 +886,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
886886 assert(reg.to64() == reg_mcv.register.to64());
887887 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
888888 try branch.inst_table.put(self.gpa, inst, stack_mcv);
889 try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, .rbp, reg_mcv);
889 try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{});
890890}
891891
892892/// Copies a value to a register without tracking the register. The register is not considered
......@@ -1196,8 +1196,8 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
11961196 const len_ty = self.air.typeOf(bin_op.rhs);
11971197
11981198 const stack_offset = @intCast(i32, try self.allocMem(inst, 16, 16));
1199 try self.genSetStack(ptr_ty, stack_offset, .rbp, ptr);
1200 try self.genSetStack(len_ty, stack_offset - 8, .rbp, len);
1199 try self.genSetStack(ptr_ty, stack_offset, ptr, .{});
1200 try self.genSetStack(len_ty, stack_offset - 8, len, .{});
12011201 const result = MCValue{ .stack_offset = stack_offset };
12021202
12031203 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
......@@ -1797,24 +1797,47 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
17971797/// T to E!T
17981798fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
17991799 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1800 const result: MCValue = if (self.liveness.isUnused(inst))
1801 .dead
1802 else
1803 return self.fail("TODO implement wrap errunion payload for {}", .{self.target.cpu.arch});
1804 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1800 if (self.liveness.isUnused(inst)) {
1801 return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
1802 }
1803 const error_union_ty = self.air.getRefType(ty_op.ty);
1804 const error_ty = error_union_ty.errorUnionSet();
1805 const payload_ty = error_union_ty.errorUnionPayload();
1806 const operand = try self.resolveInst(ty_op.operand);
1807 assert(payload_ty.hasRuntimeBits());
1808
1809 const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
1810 const abi_align = error_union_ty.abiAlignment(self.target.*);
1811 const err_abi_size = @intCast(u32, error_ty.abiSize(self.target.*));
1812 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
1813 try self.genSetStack(error_ty, stack_offset, .{ .immediate = 0 }, .{});
1814 try self.genSetStack(payload_ty, stack_offset - @intCast(i32, err_abi_size), operand, .{});
1815
1816 return self.finishAir(inst, .{ .stack_offset = stack_offset }, .{ ty_op.operand, .none, .none });
18051817}
18061818
18071819/// E to E!T
18081820fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
18091821 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1810 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1811 const error_union_ty = self.air.getRefType(ty_op.ty);
1812 const payload_ty = error_union_ty.errorUnionPayload();
1813 const mcv = try self.resolveInst(ty_op.operand);
1814 if (!payload_ty.hasRuntimeBits()) break :result mcv;
1815
1816 return self.fail("TODO implement wrap errunion error for non-empty payloads", .{});
1822 if (self.liveness.isUnused(inst)) {
1823 return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
1824 }
1825 const error_union_ty = self.air.getRefType(ty_op.ty);
1826 const error_ty = error_union_ty.errorUnionSet();
1827 const payload_ty = error_union_ty.errorUnionPayload();
1828 const err = try self.resolveInst(ty_op.operand);
1829 const result: MCValue = result: {
1830 if (!payload_ty.hasRuntimeBits()) break :result err;
1831
1832 const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
1833 const abi_align = error_union_ty.abiAlignment(self.target.*);
1834 const err_abi_size = @intCast(u32, error_ty.abiSize(self.target.*));
1835 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
1836 try self.genSetStack(error_ty, stack_offset, err, .{});
1837 try self.genSetStack(payload_ty, stack_offset - @intCast(i32, err_abi_size), .undef, .{});
1838 break :result MCValue{ .stack_offset = stack_offset };
18171839 };
1840
18181841 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
18191842}
18201843
......@@ -1974,7 +1997,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
19741997 @intCast(u32, array_ty.abiSize(self.target.*)),
19751998 array_ty.abiAlignment(self.target.*),
19761999 ));
1977 try self.genSetStack(array_ty, off, .rbp, array);
2000 try self.genSetStack(array_ty, off, array, .{});
19782001 break :inner off;
19792002 },
19802003 .stack_offset => |off| {
......@@ -2207,7 +2230,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
22072230 if (abi_size <= 8) {
22082231 const tmp_reg = try self.register_manager.allocReg(null);
22092232 try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
2210 return self.genSetStack(elem_ty, off, .rbp, MCValue{ .register = tmp_reg });
2233 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});
22112234 }
22122235
22132236 try self.genInlineMemcpy(off, elem_ty, ptr, .{});
......@@ -2305,7 +2328,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
23052328 try self.store(.{ .register = reg }, value, ptr_ty, value_ty);
23062329 },
23072330 .ptr_stack_offset => |off| {
2308 try self.genSetStack(value_ty, off, .rbp, value);
2331 try self.genSetStack(value_ty, off, value, .{});
23092332 },
23102333 .ptr_embedded_in_code => |off| {
23112334 try self.setRegOrMem(value_ty, .{ .embedded_in_code = off }, value);
......@@ -2779,7 +2802,7 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
27792802
27802803 switch (src_mcv) {
27812804 .none => unreachable,
2782 .undef => return self.genSetStack(dst_ty, off, .rbp, .undef),
2805 .undef => return self.genSetStack(dst_ty, off, .undef, .{}),
27832806 .dead, .unreach => unreachable,
27842807 .ptr_stack_offset => unreachable,
27852808 .ptr_embedded_in_code => unreachable,
......@@ -2922,7 +2945,7 @@ fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !
29222945 .stack_offset => |off| {
29232946 switch (src_mcv) {
29242947 .none => unreachable,
2925 .undef => return self.genSetStack(dst_ty, off, .rbp, .undef),
2948 .undef => return self.genSetStack(dst_ty, off, .undef, .{}),
29262949 .dead, .unreach => unreachable,
29272950 .ptr_stack_offset => unreachable,
29282951 .ptr_embedded_in_code => unreachable,
......@@ -2940,7 +2963,7 @@ fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !
29402963 .data = undefined,
29412964 });
29422965 // copy dst_reg back out
2943 return self.genSetStack(dst_ty, off, .rbp, MCValue{ .register = dst_reg });
2966 return self.genSetStack(dst_ty, off, MCValue{ .register = dst_reg }, .{});
29442967 },
29452968 .immediate => |imm| {
29462969 _ = imm;
......@@ -3279,7 +3302,10 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
32793302 }).encode(),
32803303 .data = undefined,
32813304 });
3282 try self.genSetStack(ret_ty, 0, reg, operand);
3305 try self.genSetStack(ret_ty, 0, operand, .{
3306 .source_stack_base = .rbp,
3307 .dest_stack_base = reg,
3308 });
32833309 },
32843310 else => {
32853311 try self.setRegOrMem(ret_ty, self.ret_mcv, operand);
......@@ -4179,7 +4205,7 @@ fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
41794205 .none => return,
41804206 .immediate => unreachable,
41814207 .register => |reg| return self.genSetReg(ty, reg, val),
4182 .stack_offset => |off| return self.genSetStack(ty, off, .rbp, val),
4208 .stack_offset => |off| return self.genSetStack(ty, off, val, .{}),
41834209 .memory => {
41844210 return self.fail("TODO implement setRegOrMem for memory", .{});
41854211 },
......@@ -4200,7 +4226,9 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
42004226 const reg = try self.copyToTmpRegister(ty, mcv);
42014227 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
42024228 }
4203 try self.genInlineMemset(stack_offset, .rsp, ty, .{ .immediate = 0xaa });
4229 try self.genInlineMemset(stack_offset, ty, .{ .immediate = 0xaa }, .{
4230 .dest_stack_base = .rsp,
4231 });
42044232 },
42054233 .compare_flags_unsigned,
42064234 .compare_flags_signed,
......@@ -4289,7 +4317,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
42894317 }
42904318}
42914319
4292fn genSetStack(self: *Self, ty: Type, stack_offset: i32, base_reg: Register, mcv: MCValue) InnerError!void {
4320fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: InlineMemcpyOpts) InnerError!void {
42934321 const abi_size = ty.abiSize(self.target.*);
42944322 switch (mcv) {
42954323 .dead => unreachable,
......@@ -4300,20 +4328,21 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, base_reg: Register, mcv
43004328 return; // The already existing value will do just fine.
43014329 // TODO Upgrade this to a memset call when we have that available.
43024330 switch (ty.abiSize(self.target.*)) {
4303 1 => return self.genSetStack(ty, stack_offset, base_reg, .{ .immediate = 0xaa }),
4304 2 => return self.genSetStack(ty, stack_offset, base_reg, .{ .immediate = 0xaaaa }),
4305 4 => return self.genSetStack(ty, stack_offset, base_reg, .{ .immediate = 0xaaaaaaaa }),
4306 8 => return self.genSetStack(ty, stack_offset, base_reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
4307 else => return self.genInlineMemset(stack_offset, base_reg, ty, .{ .immediate = 0xaa }),
4331 1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }, opts),
4332 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }, opts),
4333 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }, opts),
4334 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, opts),
4335 else => return self.genInlineMemset(stack_offset, ty, .{ .immediate = 0xaa }, opts),
43084336 }
43094337 },
43104338 .compare_flags_unsigned,
43114339 .compare_flags_signed,
43124340 => {
43134341 const reg = try self.copyToTmpRegister(ty, mcv);
4314 return self.genSetStack(ty, stack_offset, base_reg, .{ .register = reg });
4342 return self.genSetStack(ty, stack_offset, .{ .register = reg }, opts);
43154343 },
43164344 .immediate => |x_big| {
4345 const base_reg = opts.dest_stack_base orelse .rbp;
43174346 switch (abi_size) {
43184347 1, 2, 4 => {
43194348 const payload = try self.addExtra(Mir.ImmPair{
......@@ -4376,6 +4405,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, base_reg: Register, mcv
43764405 return self.fail("stack offset too large", .{});
43774406 }
43784407
4408 const base_reg = opts.dest_stack_base orelse .rbp;
43794409 const is_power_of_two = (abi_size % 2) == 0;
43804410 if (!is_power_of_two) {
43814411 self.register_manager.freezeRegs(&.{reg});
......@@ -4431,17 +4461,14 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, base_reg: Register, mcv
44314461 => {
44324462 if (abi_size <= 8) {
44334463 const reg = try self.copyToTmpRegister(ty, mcv);
4434 return self.genSetStack(ty, stack_offset, base_reg, MCValue{ .register = reg });
4464 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
44354465 }
44364466
4437 try self.genInlineMemcpy(stack_offset, ty, mcv, .{
4438 .source_stack_base = base_reg,
4439 .dest_stack_base = base_reg,
4440 });
4467 try self.genInlineMemcpy(stack_offset, ty, mcv, opts);
44414468 },
44424469 .ptr_stack_offset => {
44434470 const reg = try self.copyToTmpRegister(ty, mcv);
4444 return self.genSetStack(ty, stack_offset, base_reg, MCValue{ .register = reg });
4471 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
44454472 },
44464473 .stack_offset => |off| {
44474474 if (stack_offset == off) {
......@@ -4451,13 +4478,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, base_reg: Register, mcv
44514478
44524479 if (abi_size <= 8) {
44534480 const reg = try self.copyToTmpRegister(ty, mcv);
4454 return self.genSetStack(ty, stack_offset, base_reg, MCValue{ .register = reg });
4481 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
44554482 }
44564483
4457 try self.genInlineMemcpy(stack_offset, ty, mcv, .{
4458 .source_stack_base = base_reg,
4459 .dest_stack_base = base_reg,
4460 });
4484 try self.genInlineMemcpy(stack_offset, ty, mcv, opts);
44614485 },
44624486 }
44634487}
......@@ -4627,9 +4651,9 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts:
46274651fn genInlineMemset(
46284652 self: *Self,
46294653 stack_offset: i32,
4630 stack_register: Register,
46314654 ty: Type,
46324655 value: MCValue,
4656 opts: InlineMemcpyOpts,
46334657) InnerError!void {
46344658 try self.register_manager.getReg(.rax, null);
46354659
......@@ -4694,7 +4718,7 @@ fn genInlineMemset(
46944718 _ = try self.addInst(.{
46954719 .tag = .mov_mem_index_imm,
46964720 .ops = (Mir.Ops{
4697 .reg1 = stack_register.to64(),
4721 .reg1 = opts.dest_stack_base orelse .rbp,
46984722 }).encode(),
46994723 .data = .{ .payload = payload },
47004724 });
......@@ -5026,8 +5050,8 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
50265050 const array_len = array_ty.arrayLenIncludingSentinel();
50275051 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else blk: {
50285052 const stack_offset = @intCast(i32, try self.allocMem(inst, 16, 16));
5029 try self.genSetStack(ptr_ty, stack_offset, .rbp, ptr);
5030 try self.genSetStack(Type.initTag(.u64), stack_offset - 8, .rbp, .{ .immediate = array_len });
5053 try self.genSetStack(ptr_ty, stack_offset, ptr, .{});
5054 try self.genSetStack(Type.initTag(.u64), stack_offset - 8, .{ .immediate = array_len }, .{});
50315055 break :blk .{ .stack_offset = stack_offset };
50325056 };
50335057 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
src/codegen.zig+31-10
......@@ -486,20 +486,34 @@ pub fn generateSymbol(
486486 const payload_ty = typed_value.ty.errorUnionPayload();
487487 const is_payload = typed_value.val.errorUnionIsPayload();
488488
489 const error_val = if (!is_payload) typed_value.val else Value.initTag(.zero);
490 switch (try generateSymbol(bin_file, parent_atom_index, src_loc, .{
491 .ty = error_ty,
492 .val = error_val,
493 }, code, debug_output)) {
494 .appended => {},
495 .externally_managed => |external_slice| {
496 code.appendSliceAssumeCapacity(external_slice);
497 },
498 .fail => |em| return Result{ .fail = em },
489 const target = bin_file.options.target;
490 const abi_align = typed_value.ty.abiAlignment(target);
491
492 {
493 const error_val = if (!is_payload) typed_value.val else Value.initTag(.zero);
494 const begin = code.items.len;
495 switch (try generateSymbol(bin_file, parent_atom_index, src_loc, .{
496 .ty = error_ty,
497 .val = error_val,
498 }, code, debug_output)) {
499 .appended => {},
500 .externally_managed => |external_slice| {
501 code.appendSliceAssumeCapacity(external_slice);
502 },
503 .fail => |em| return Result{ .fail = em },
504 }
505 const unpadded_end = code.items.len - begin;
506 const padded_end = mem.alignForwardGeneric(u64, unpadded_end, abi_align);
507 const padding = try math.cast(usize, padded_end - unpadded_end);
508
509 if (padding > 0) {
510 try code.writer().writeByteNTimes(0, padding);
511 }
499512 }
500513
501514 if (payload_ty.hasRuntimeBits()) {
502515 const payload_val = if (typed_value.val.castTag(.eu_payload)) |val| val.data else Value.initTag(.undef);
516 const begin = code.items.len;
503517 switch (try generateSymbol(bin_file, parent_atom_index, src_loc, .{
504518 .ty = payload_ty,
505519 .val = payload_val,
......@@ -510,6 +524,13 @@ pub fn generateSymbol(
510524 },
511525 .fail => |em| return Result{ .fail = em },
512526 }
527 const unpadded_end = code.items.len - begin;
528 const padded_end = mem.alignForwardGeneric(u64, unpadded_end, abi_align);
529 const padding = try math.cast(usize, padded_end - unpadded_end);
530
531 if (padding > 0) {
532 try code.writer().writeByteNTimes(0, padding);
533 }
513534 }
514535
515536 return Result{ .appended = {} };