authorgravatar for joachim.schmidt557@outlook.comJoachim Schmidt <joachim.schmidt557@outlook.com> 2022-04-15 22:58:54+02:00
committergravatar for joachim.schmidt557@outlook.comJoachim Schmidt <joachim.schmidt557@outlook.com> 2022-04-19 22:37:56+02:00
logc78daeb642e742af3ac42bac0468776ccc4cd452
tree1f790dbc2142be7b550c488d53760217308f3c96
parent9c2cbe39c2caa9137e1123fcdf2f326282cad1b5
signaturelock-open Commit is signed but in an unrecognized format.

stage2 AArch64: add basic assertions to bits.zig for correct codegen

Includes many fixes of errors discovered by adding these assertions

3 files changed, 126 insertions(+), 69 deletions(-)

src/arch/aarch64/CodeGen.zig+47-29
......@@ -376,7 +376,7 @@ fn gen(self: *Self) !void {
376376 // mov fp, sp
377377 _ = try self.addInst(.{
378378 .tag = .mov_to_from_sp,
379 .data = .{ .rr = .{ .rd = .x29, .rn = .xzr } },
379 .data = .{ .rr = .{ .rd = .x29, .rn = .sp } },
380380 });
381381
382382 // sub sp, sp, #reloc
......@@ -421,7 +421,7 @@ fn gen(self: *Self) !void {
421421 if (math.cast(u12, stack_size)) |size| {
422422 self.mir_instructions.set(backpatch_reloc, .{
423423 .tag = .sub_immediate,
424 .data = .{ .rr_imm12_sh = .{ .rd = .xzr, .rn = .xzr, .imm12 = size } },
424 .data = .{ .rr_imm12_sh = .{ .rd = .sp, .rn = .sp, .imm12 = size } },
425425 });
426426 } else |_| {
427427 return self.failSymbol("TODO AArch64: allow larger stacks", .{});
......@@ -453,7 +453,7 @@ fn gen(self: *Self) !void {
453453 // add sp, sp, #stack_size
454454 _ = try self.addInst(.{
455455 .tag = .add_immediate,
456 .data = .{ .rr_imm12_sh = .{ .rd = .xzr, .rn = .xzr, .imm12 = @intCast(u12, stack_size) } },
456 .data = .{ .rr_imm12_sh = .{ .rd = .sp, .rn = .sp, .imm12 = @intCast(u12, stack_size) } },
457457 });
458458
459459 // <load other registers>
......@@ -882,7 +882,8 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void {
882882/// allocated. A second call to `copyToTmpRegister` may return the same register.
883883/// This can have a side effect of spilling instructions to the stack to free up a register.
884884fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
885 const reg = try self.register_manager.allocReg(null);
885 const raw_reg = try self.register_manager.allocReg(null);
886 const reg = registerAlias(raw_reg, ty.abiSize(self.target.*));
886887 try self.genSetReg(ty, reg, mcv);
887888 return reg;
888889}
......@@ -891,7 +892,9 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
891892/// `reg_owner` is the instruction that gets associated with the register in the register table.
892893/// This can have a side effect of spilling instructions to the stack to free up a register.
893894fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
894 const reg = try self.register_manager.allocReg(reg_owner);
895 const raw_reg = try self.register_manager.allocReg(reg_owner);
896 const ty = self.air.typeOfIndex(reg_owner);
897 const reg = registerAlias(raw_reg, ty.abiSize(self.target.*));
895898 try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv);
896899 return MCValue{ .register = reg };
897900}
......@@ -1003,7 +1006,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
10031006 break :blk op_reg;
10041007 }
10051008
1006 break :blk try self.register_manager.allocReg(null);
1009 const raw_reg = try self.register_manager.allocReg(null);
1010 break :blk raw_reg.to32();
10071011 };
10081012
10091013 _ = try self.addInst(.{
......@@ -1013,7 +1017,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
10131017 .rn = op_reg,
10141018 .imms = 0b000000,
10151019 .immr = 0b000000,
1016 .n = 0b1,
1020 .n = 0b0,
10171021 } },
10181022 });
10191023
......@@ -1035,7 +1039,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
10351039 break :blk op_reg;
10361040 }
10371041
1038 break :blk try self.register_manager.allocReg(null);
1042 const raw_reg = try self.register_manager.allocReg(null);
1043 break :blk registerAlias(raw_reg, operand_ty.abiSize(self.target.*));
10391044 };
10401045
10411046 _ = try self.addInst(.{
......@@ -1124,7 +1129,8 @@ fn binOpRegister(
11241129 break :inst Air.refToIndex(bin_op.lhs).?;
11251130 } else null;
11261131
1127 const reg = try self.register_manager.allocReg(track_inst);
1132 const raw_reg = try self.register_manager.allocReg(track_inst);
1133 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
11281134 self.register_manager.freezeRegs(&.{reg});
11291135
11301136 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
......@@ -1139,7 +1145,8 @@ fn binOpRegister(
11391145 break :inst Air.refToIndex(bin_op.rhs).?;
11401146 } else null;
11411147
1142 const reg = try self.register_manager.allocReg(track_inst);
1148 const raw_reg = try self.register_manager.allocReg(track_inst);
1149 const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
11431150 self.register_manager.freezeRegs(&.{reg});
11441151
11451152 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
......@@ -1156,7 +1163,8 @@ fn binOpRegister(
11561163 } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) {
11571164 break :blk rhs_reg;
11581165 } else {
1159 break :blk try self.register_manager.allocReg(inst);
1166 const raw_reg = try self.register_manager.allocReg(inst);
1167 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
11601168 }
11611169 } else try self.register_manager.allocReg(null);
11621170
......@@ -1276,7 +1284,8 @@ fn binOpImmediate(
12761284 ).?;
12771285 } else null;
12781286
1279 const reg = try self.register_manager.allocReg(track_inst);
1287 const raw_reg = try self.register_manager.allocReg(track_inst);
1288 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
12801289 self.register_manager.freezeRegs(&.{reg});
12811290
12821291 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
......@@ -1298,7 +1307,8 @@ fn binOpImmediate(
12981307 )) {
12991308 break :blk lhs_reg;
13001309 } else {
1301 break :blk try self.register_manager.allocReg(inst);
1310 const raw_reg = try self.register_manager.allocReg(inst);
1311 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
13021312 }
13031313 } else try self.register_manager.allocReg(null),
13041314 };
......@@ -1965,7 +1975,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
19651975 },
19661976 .stack_offset => |off| {
19671977 if (elem_size <= 8) {
1968 const tmp_reg = try self.register_manager.allocReg(null);
1978 const raw_tmp_reg = try self.register_manager.allocReg(null);
1979 const tmp_reg = registerAlias(raw_tmp_reg, elem_size);
19691980 self.register_manager.freezeRegs(&.{tmp_reg});
19701981 defer self.register_manager.unfreezeRegs(&.{tmp_reg});
19711982
......@@ -2001,12 +2012,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
20012012 .got_load,
20022013 .direct_load,
20032014 => {
2004 const reg = try self.register_manager.allocReg(null);
2005 self.register_manager.freezeRegs(&.{reg});
2006 defer self.register_manager.unfreezeRegs(&.{reg});
2007
2008 try self.genSetReg(ptr_ty, reg, ptr);
2009 try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
2015 const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr);
2016 try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty);
20102017 },
20112018 }
20122019}
......@@ -2091,6 +2098,7 @@ fn genInlineMemcpy(
20912098fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
20922099 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
20932100 const elem_ty = self.air.typeOfIndex(inst);
2101 const elem_size = elem_ty.abiSize(self.target.*);
20942102 const result: MCValue = result: {
20952103 if (!elem_ty.hasRuntimeBits())
20962104 break :result MCValue.none;
......@@ -2101,9 +2109,12 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
21012109 break :result MCValue.dead;
21022110
21032111 const dst_mcv: MCValue = blk: {
2104 if (self.reuseOperand(inst, ty_op.operand, 0, ptr)) {
2112 if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr)) {
21052113 // The MCValue that holds the pointer can be re-used as the value.
2106 break :blk ptr;
2114 break :blk switch (ptr) {
2115 .register => |r| MCValue{ .register = registerAlias(r, elem_size) },
2116 else => ptr,
2117 };
21072118 } else {
21082119 break :blk try self.allocRegOrMem(inst, true);
21092120 }
......@@ -2209,6 +2220,8 @@ fn genStrRegister(self: *Self, value_reg: Register, addr_reg: Register, abi_size
22092220}
22102221
22112222fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {
2223 const abi_size = value_ty.abiSize(self.target.*);
2224
22122225 switch (ptr) {
22132226 .none => unreachable,
22142227 .undef => unreachable,
......@@ -2226,14 +2239,14 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
22262239 self.register_manager.freezeRegs(&.{addr_reg});
22272240 defer self.register_manager.unfreezeRegs(&.{addr_reg});
22282241
2229 const abi_size = value_ty.abiSize(self.target.*);
22302242 switch (value) {
22312243 .register => |value_reg| {
22322244 try self.genStrRegister(value_reg, addr_reg, abi_size);
22332245 },
22342246 else => {
22352247 if (abi_size <= 8) {
2236 const tmp_reg = try self.register_manager.allocReg(null);
2248 const raw_tmp_reg = try self.register_manager.allocReg(null);
2249 const tmp_reg = registerAlias(raw_tmp_reg, abi_size);
22372250 self.register_manager.freezeRegs(&.{tmp_reg});
22382251 defer self.register_manager.unfreezeRegs(&.{tmp_reg});
22392252
......@@ -3522,8 +3535,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
35223535 .memory => |addr| {
35233536 // The value is in memory at a hard-coded address.
35243537 // If the type is a pointer, it means the pointer address is at this memory location.
3525 try self.genSetReg(ty, reg, .{ .immediate = addr });
3526 try self.genLdrRegister(reg, reg, ty.abiSize(self.target.*));
3538 try self.genSetReg(ty, reg.to64(), .{ .immediate = addr });
3539 try self.genLdrRegister(reg, reg.to64(), ty.abiSize(self.target.*));
35273540 },
35283541 .stack_offset => |off| {
35293542 const abi_size = ty.abiSize(self.target.*);
......@@ -3998,6 +4011,12 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
39984011 var nsaa: u32 = 0; // Next stacked argument address
39994012
40004013 for (param_types) |ty, i| {
4014 const param_size = @intCast(u32, ty.abiSize(self.target.*));
4015 if (param_size == 0) {
4016 result.args[i] = .{ .none = {} };
4017 continue;
4018 }
4019
40014020 // We round up NCRN only for non-Apple platforms which allow the 16-byte aligned
40024021 // values to spread across odd-numbered registers.
40034022 if (ty.abiAlignment(self.target.*) == 16 and !self.target.isDarwin()) {
......@@ -4005,10 +4024,9 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
40054024 ncrn += ncrn % 2;
40064025 }
40074026
4008 const param_size = @intCast(u32, ty.abiSize(self.target.*));
40094027 if (std.math.divCeil(u32, param_size, 8) catch unreachable <= 8 - ncrn) {
40104028 if (param_size <= 8) {
4011 result.args[i] = .{ .register = c_abi_int_param_regs[ncrn] };
4029 result.args[i] = .{ .register = registerAlias(c_abi_int_param_regs[ncrn], param_size) };
40124030 ncrn += 1;
40134031 } else {
40144032 return self.fail("TODO MCValues with multiple registers", .{});
......@@ -4045,7 +4063,7 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
40454063 .Unspecified, .C => {
40464064 const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
40474065 if (ret_ty_size <= 8) {
4048 result.return_value = .{ .register = c_abi_int_return_regs[0] };
4066 result.return_value = .{ .register = registerAlias(c_abi_int_return_regs[0], ret_ty_size) };
40494067 } else {
40504068 return self.fail("TODO support more return types for ARM backend", .{});
40514069 }
src/arch/aarch64/Emit.zig+35-9
......@@ -457,8 +457,13 @@ fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
457457 const rn = r_imm12_sh.rn;
458458 const imm12 = r_imm12_sh.imm12;
459459 const sh = r_imm12_sh.sh == 1;
460 const zr: Register = switch (rn.size()) {
461 32 => .wzr,
462 64 => .xzr,
463 else => unreachable,
464 };
460465
461 try emit.writeInstruction(Instruction.subs(.xzr, rn, imm12, sh));
466 try emit.writeInstruction(Instruction.subs(zr, rn, imm12, sh));
462467 },
463468 else => unreachable,
464469 }
......@@ -674,8 +679,13 @@ fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
674679 const rm = rr_imm6_shift.rm;
675680 const shift = rr_imm6_shift.shift;
676681 const imm6 = rr_imm6_shift.imm6;
682 const zr: Register = switch (rn.size()) {
683 32 => .wzr,
684 64 => .xzr,
685 else => unreachable,
686 };
677687
678 try emit.writeInstruction(Instruction.subsShiftedRegister(.xzr, rn, rm, shift, imm6));
688 try emit.writeInstruction(Instruction.subsShiftedRegister(zr, rn, rm, shift, imm6));
679689 },
680690 else => unreachable,
681691 }
......@@ -686,7 +696,12 @@ fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void {
686696 switch (tag) {
687697 .cset => {
688698 const r_cond = emit.mir.instructions.items(.data)[inst].r_cond;
689 try emit.writeInstruction(Instruction.csinc(r_cond.rd, .xzr, .xzr, r_cond.cond));
699 const zr: Register = switch (r_cond.rd.size()) {
700 32 => .wzr,
701 64 => .xzr,
702 else => unreachable,
703 };
704 try emit.writeInstruction(Instruction.csinc(r_cond.rd, zr, zr, r_cond.cond));
690705 },
691706 else => unreachable,
692707 }
......@@ -718,14 +733,14 @@ fn mirLoadMemoryPie(emit: *Emit, inst: Mir.Inst.Index) !void {
718733 // PC-relative displacement to the entry in memory.
719734 // adrp
720735 const offset = @intCast(u32, emit.code.items.len);
721 try emit.writeInstruction(Instruction.adrp(reg, 0));
736 try emit.writeInstruction(Instruction.adrp(reg.to64(), 0));
722737
723738 switch (tag) {
724739 .load_memory_got => {
725740 // ldr reg, reg, offset
726741 try emit.writeInstruction(Instruction.ldr(
727742 reg,
728 reg,
743 reg.to64(),
729744 Instruction.LoadStoreOffset.imm(0),
730745 ));
731746 },
......@@ -739,11 +754,11 @@ fn mirLoadMemoryPie(emit: *Emit, inst: Mir.Inst.Index) !void {
739754 // Note that this can potentially be optimised out by the codegen/linker if the
740755 // target address is appropriately aligned.
741756 // add reg, reg, offset
742 try emit.writeInstruction(Instruction.add(reg, reg, 0, false));
757 try emit.writeInstruction(Instruction.add(reg.to64(), reg.to64(), 0, false));
743758 // ldr reg, reg, offset
744759 try emit.writeInstruction(Instruction.ldr(
745760 reg,
746 reg,
761 reg.to64(),
747762 Instruction.LoadStoreOffset.imm(0),
748763 ));
749764 },
......@@ -905,7 +920,13 @@ fn mirMoveRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
905920 switch (tag) {
906921 .mov_register => {
907922 const rr = emit.mir.instructions.items(.data)[inst].rr;
908 try emit.writeInstruction(Instruction.orrShiftedRegister(rr.rd, .xzr, rr.rn, .lsl, 0));
923 const zr: Register = switch (rr.rd.size()) {
924 32 => .wzr,
925 64 => .xzr,
926 else => unreachable,
927 };
928
929 try emit.writeInstruction(Instruction.orrShiftedRegister(rr.rd, zr, rr.rn, .lsl, 0));
909930 },
910931 .mov_to_from_sp => {
911932 const rr = emit.mir.instructions.items(.data)[inst].rr;
......@@ -917,8 +938,13 @@ fn mirMoveRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
917938 const rm = rr_imm6_logical_shift.rm;
918939 const shift = rr_imm6_logical_shift.shift;
919940 const imm6 = rr_imm6_logical_shift.imm6;
941 const zr: Register = switch (rd.size()) {
942 32 => .wzr,
943 64 => .xzr,
944 else => unreachable,
945 };
920946
921 try emit.writeInstruction(Instruction.ornShiftedRegister(rd, .xzr, rm, shift, imm6));
947 try emit.writeInstruction(Instruction.ornShiftedRegister(rd, zr, rm, shift, imm6));
922948 },
923949 else => unreachable,
924950 }
src/arch/aarch64/bits.zig+44-31
......@@ -695,6 +695,9 @@ pub const Instruction = union(enum) {
695695 offset: LoadStoreOffset,
696696 variant: LoadStoreVariant,
697697 ) Instruction {
698 assert(rn.size() == 64);
699 assert(rn.id() != Register.xzr.id());
700
698701 const off = offset.toU12();
699702 const op1: u2 = blk: {
700703 switch (offset) {
......@@ -741,6 +744,9 @@ pub const Instruction = union(enum) {
741744 encoding: u2,
742745 load: bool,
743746 ) Instruction {
747 assert(rn.size() == 64);
748 assert(rn.id() != Register.xzr.id());
749
744750 switch (rt1.size()) {
745751 32 => {
746752 assert(-256 <= offset and offset <= 252);
......@@ -849,38 +855,26 @@ pub const Instruction = union(enum) {
849855 shift: LogicalShiftedRegisterShift,
850856 amount: u6,
851857 ) Instruction {
852 switch (rd.size()) {
853 32 => {
854 assert(amount < 32);
855 return Instruction{
856 .logical_shifted_register = .{
857 .rd = rd.enc(),
858 .rn = rn.enc(),
859 .imm6 = amount,
860 .rm = rm.enc(),
861 .n = n,
862 .shift = @enumToInt(shift),
863 .opc = opc,
864 .sf = 0b0,
865 },
866 };
867 },
868 64 => {
869 return Instruction{
870 .logical_shifted_register = .{
871 .rd = rd.enc(),
872 .rn = rn.enc(),
873 .imm6 = amount,
874 .rm = rm.enc(),
875 .n = n,
876 .shift = @enumToInt(shift),
877 .opc = opc,
878 .sf = 0b1,
879 },
880 };
858 assert(rd.size() == rn.size());
859 assert(rd.size() == rm.size());
860 if (rd.size() == 32) assert(amount < 32);
861
862 return Instruction{
863 .logical_shifted_register = .{
864 .rd = rd.enc(),
865 .rn = rn.enc(),
866 .imm6 = amount,
867 .rm = rm.enc(),
868 .n = n,
869 .shift = @enumToInt(shift),
870 .opc = opc,
871 .sf = switch (rd.size()) {
872 32 => 0b0,
873 64 => 0b1,
874 else => unreachable,
875 },
881876 },
882 else => unreachable, // unexpected register size
883 }
877 };
884878 }
885879
886880 fn addSubtractImmediate(
......@@ -891,6 +885,9 @@ pub const Instruction = union(enum) {
891885 imm12: u12,
892886 shift: bool,
893887 ) Instruction {
888 assert(rd.size() == rn.size());
889 assert(rn.id() != Register.xzr.id());
890
894891 return Instruction{
895892 .add_subtract_immediate = .{
896893 .rd = rd.enc(),
......@@ -916,6 +913,9 @@ pub const Instruction = union(enum) {
916913 immr: u6,
917914 n: u1,
918915 ) Instruction {
916 assert(rd.size() == rn.size());
917 assert(!(rd.size() == 32 and n == 1));
918
919919 return Instruction{
920920 .logical_immediate = .{
921921 .rd = rd.enc(),
......@@ -941,6 +941,8 @@ pub const Instruction = union(enum) {
941941 immr: u6,
942942 imms: u6,
943943 ) Instruction {
944 assert(rd.size() == rn.size());
945
944946 return Instruction{
945947 .bitfield = .{
946948 .rd = rd.enc(),
......@@ -969,6 +971,9 @@ pub const Instruction = union(enum) {
969971 rm: Register,
970972 imm6: u6,
971973 ) Instruction {
974 assert(rd.size() == rn.size());
975 assert(rd.size() == rm.size());
976
972977 return Instruction{
973978 .add_subtract_shifted_register = .{
974979 .rd = rd.enc(),
......@@ -994,6 +999,7 @@ pub const Instruction = union(enum) {
994999 offset: i21,
9951000 ) Instruction {
9961001 assert(offset & 0b11 == 0b00);
1002
9971003 return Instruction{
9981004 .conditional_branch = .{
9991005 .cond = @enumToInt(cond),
......@@ -1010,6 +1016,7 @@ pub const Instruction = union(enum) {
10101016 offset: i21,
10111017 ) Instruction {
10121018 assert(offset & 0b11 == 0b00);
1019
10131020 return Instruction{
10141021 .compare_and_branch = .{
10151022 .rt = rt.enc(),
......@@ -1033,6 +1040,9 @@ pub const Instruction = union(enum) {
10331040 rm: Register,
10341041 cond: Condition,
10351042 ) Instruction {
1043 assert(rd.size() == rn.size());
1044 assert(rd.size() == rm.size());
1045
10361046 return Instruction{
10371047 .conditional_select = .{
10381048 .rd = rd.enc(),
......@@ -1085,6 +1095,9 @@ pub const Instruction = union(enum) {
10851095 rn: Register,
10861096 rm: Register,
10871097 ) Instruction {
1098 assert(rd.size() == rn.size());
1099 assert(rd.size() == rm.size());
1100
10881101 return Instruction{
10891102 .data_processing_2_source = .{
10901103 .rd = rd.enc(),