authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-04 23:00:41+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-05 21:43:36+02:00
logd112cd52f36cbb00e18009417044ab1e4496dd80
treec9e34928674e6434c5a75171f6cb97d375293add
parentf4421c01e8a8793a7c02fce870bacbb710bb1e12

aarch64: fix mul_with_overflow for ints <= 32bits

4 files changed, 224 insertions(+), 37 deletions(-)

src/arch/aarch64/CodeGen.zig+46-32
......@@ -1294,29 +1294,23 @@ fn binOpRegister(
12941294 };
12951295 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
12961296
1297 const dest_reg: Register = reg: {
1298 const dest_reg = switch (mir_tag) {
1299 .cmp_shifted_register => undefined, // cmp has no destination register
1300 else => if (maybe_inst) |inst| blk: {
1301 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1302
1303 if (lhs_is_register and self.reuseOperand(inst, bin_op.lhs, 0, lhs)) {
1304 break :blk lhs_reg;
1305 } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) {
1306 break :blk rhs_reg;
1307 } else {
1308 const raw_reg = try self.register_manager.allocReg(inst);
1309 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1310 }
1311 } else blk: {
1312 const raw_reg = try self.register_manager.allocReg(null);
1297 const dest_reg = switch (mir_tag) {
1298 .cmp_shifted_register => undefined, // cmp has no destination register
1299 else => if (maybe_inst) |inst| blk: {
1300 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1301
1302 if (lhs_is_register and self.reuseOperand(inst, bin_op.lhs, 0, lhs)) {
1303 break :blk lhs_reg;
1304 } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) {
1305 break :blk rhs_reg;
1306 } else {
1307 const raw_reg = try self.register_manager.allocReg(inst);
13131308 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1314 },
1315 };
1316 break :reg switch (mir_tag) {
1317 .smull, .umull => dest_reg.to64(),
1318 else => dest_reg,
1319 };
1309 }
1310 } else blk: {
1311 const raw_reg = try self.register_manager.allocReg(null);
1312 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1313 },
13201314 };
13211315
13221316 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
......@@ -1341,9 +1335,7 @@ fn binOpRegister(
13411335 .shift = .lsl,
13421336 } },
13431337 .mul,
1344 .smulh,
13451338 .smull,
1346 .umulh,
13471339 .umull,
13481340 .lsl_register,
13491341 .asr_register,
......@@ -1932,16 +1924,38 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
19321924 self.register_manager.freezeRegs(&.{truncated_reg});
19331925 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
19341926
1935 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
1936 _ = try self.binOp(
1937 .cmp_eq,
1938 null,
1939 dest,
1940 .{ .register = truncated_reg },
1941 Type.usize,
1942 Type.usize,
1927 try self.truncRegister(
1928 dest_reg.to32(),
1929 truncated_reg.to32(),
1930 int_info.signedness,
1931 int_info.bits,
19431932 );
19441933
1934 switch (int_info.signedness) {
1935 .signed => {
1936 _ = try self.addInst(.{
1937 .tag = .cmp_extended_register,
1938 .data = .{ .rr_extend_shift = .{
1939 .rn = dest_reg.to64(),
1940 .rm = truncated_reg.to32(),
1941 .ext_type = .sxtw,
1942 .imm3 = 0,
1943 } },
1944 });
1945 },
1946 .unsigned => {
1947 _ = try self.addInst(.{
1948 .tag = .cmp_extended_register,
1949 .data = .{ .rr_extend_shift = .{
1950 .rn = dest_reg.to64(),
1951 .rm = truncated_reg.to32(),
1952 .ext_type = .uxtw,
1953 .imm3 = 0,
1954 } },
1955 });
1956 },
1957 }
1958
19451959 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
19461960 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{
19471961 .compare_flags_unsigned = .neq,
src/arch/aarch64/Emit.zig+51-4
......@@ -114,6 +114,12 @@ pub fn emitMir(
114114 .sub_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
115115 .subs_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
116116
117 .add_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
118 .adds_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
119 .sub_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
120 .subs_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
121 .cmp_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
122
117123 .cset => try emit.mirConditionalSelect(inst),
118124
119125 .dbg_line => try emit.mirDbgLine(inst),
......@@ -732,6 +738,47 @@ fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
732738 }
733739}
734740
741fn mirAddSubtractExtendedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
742 const tag = emit.mir.instructions.items(.tag)[inst];
743 switch (tag) {
744 .add_extended_register,
745 .adds_extended_register,
746 .sub_extended_register,
747 .subs_extended_register,
748 => {
749 const rrr_extend_shift = emit.mir.instructions.items(.data)[inst].rrr_extend_shift;
750 const rd = rrr_extend_shift.rd;
751 const rn = rrr_extend_shift.rn;
752 const rm = rrr_extend_shift.rm;
753 const ext_type = rrr_extend_shift.ext_type;
754 const imm3 = rrr_extend_shift.imm3;
755
756 switch (tag) {
757 .add_extended_register => try emit.writeInstruction(Instruction.addExtendedRegister(rd, rn, rm, ext_type, imm3)),
758 .adds_extended_register => try emit.writeInstruction(Instruction.addsExtendedRegister(rd, rn, rm, ext_type, imm3)),
759 .sub_extended_register => try emit.writeInstruction(Instruction.subExtendedRegister(rd, rn, rm, ext_type, imm3)),
760 .subs_extended_register => try emit.writeInstruction(Instruction.subsExtendedRegister(rd, rn, rm, ext_type, imm3)),
761 else => unreachable,
762 }
763 },
764 .cmp_extended_register => {
765 const rr_extend_shift = emit.mir.instructions.items(.data)[inst].rr_extend_shift;
766 const rn = rr_extend_shift.rn;
767 const rm = rr_extend_shift.rm;
768 const ext_type = rr_extend_shift.ext_type;
769 const imm3 = rr_extend_shift.imm3;
770 const zr: Register = switch (rn.size()) {
771 32 => .wzr,
772 64 => .xzr,
773 else => unreachable,
774 };
775
776 try emit.writeInstruction(Instruction.subsExtendedRegister(zr, rn, rm, ext_type, imm3));
777 },
778 else => unreachable,
779 }
780}
781
735782fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void {
736783 const tag = emit.mir.instructions.items(.tag)[inst];
737784 switch (tag) {
......@@ -1013,10 +1060,10 @@ fn mirDataProcessing3Source(emit: *Emit, inst: Mir.Inst.Index) !void {
10131060
10141061 switch (tag) {
10151062 .mul => try emit.writeInstruction(Instruction.mul(rrr.rd, rrr.rn, rrr.rm)),
1016 .smulh => try emit.writeInstruction(Instruction.smulh(rrr.rd, rrr.rn, rrr.rm)),
1017 .smull => try emit.writeInstruction(Instruction.smull(rrr.rd, rrr.rn, rrr.rm)),
1018 .umulh => try emit.writeInstruction(Instruction.umulh(rrr.rd, rrr.rn, rrr.rm)),
1019 .umull => try emit.writeInstruction(Instruction.umull(rrr.rd, rrr.rn, rrr.rm)),
1063 .smulh => try emit.writeInstruction(Instruction.smulh(rrr.rd.to64(), rrr.rn.to64(), rrr.rm.to64())),
1064 .smull => try emit.writeInstruction(Instruction.smull(rrr.rd.to64(), rrr.rn.to32(), rrr.rm.to32())),
1065 .umulh => try emit.writeInstruction(Instruction.umulh(rrr.rd.to64(), rrr.rn.to64(), rrr.rm.to64())),
1066 .umull => try emit.writeInstruction(Instruction.umull(rrr.rd.to64(), rrr.rn.to32(), rrr.rm.to32())),
10201067 else => unreachable,
10211068 }
10221069}
src/arch/aarch64/Mir.zig+29
......@@ -32,6 +32,10 @@ pub const Inst = struct {
3232 add_shifted_register,
3333 /// Add, update condition flags (shifted register)
3434 adds_shifted_register,
35 /// Add (extended register)
36 add_extended_register,
37 /// Add, update condition flags (extended register)
38 adds_extended_register,
3539 /// Bitwise AND (shifted register)
3640 and_shifted_register,
3741 /// Arithmetic Shift Right (immediate)
......@@ -56,6 +60,8 @@ pub const Inst = struct {
5660 cmp_immediate,
5761 /// Compare (shifted register)
5862 cmp_shifted_register,
63 /// Compare (extended register)
64 cmp_extended_register,
5965 /// Conditional set
6066 cset,
6167 /// Pseudo-instruction: End of prologue
......@@ -184,6 +190,10 @@ pub const Inst = struct {
184190 sub_shifted_register,
185191 /// Subtract, update condition flags (shifted register)
186192 subs_shifted_register,
193 /// Subtract (extended register)
194 sub_extended_register,
195 /// Subtract, update condition flags (extended register)
196 subs_extended_register,
187197 /// Supervisor Call
188198 svc,
189199 /// Test bits (immediate)
......@@ -300,6 +310,15 @@ pub const Inst = struct {
300310 imm6: u6,
301311 shift: bits.Instruction.AddSubtractShiftedRegisterShift,
302312 },
313 /// Two registers with sign-extension (extension type and 3-bit shift amount)
314 ///
315 /// Used by e.g. cmp_extended_register
316 rr_extend_shift: struct {
317 rn: Register,
318 rm: Register,
319 ext_type: bits.Instruction.AddSubtractExtendedRegisterOption,
320 imm3: u3,
321 },
303322 /// Two registers and a shift (logical instruction version)
304323 /// (shift type and 6-bit amount)
305324 ///
......@@ -356,6 +375,16 @@ pub const Inst = struct {
356375 imm6: u6,
357376 shift: bits.Instruction.AddSubtractShiftedRegisterShift,
358377 },
378 /// Three registers with sign-extension (extension type and 3-bit shift amount)
379 ///
380 /// Used by e.g. add_extended_register
381 rrr_extend_shift: struct {
382 rd: Register,
383 rn: Register,
384 rm: Register,
385 ext_type: bits.Instruction.AddSubtractExtendedRegisterOption,
386 imm3: u3,
387 },
359388 /// Three registers and a shift (logical instruction version)
360389 /// (shift type and 6-bit amount)
361390 ///
src/arch/aarch64/bits.zig+98-1
......@@ -330,6 +330,17 @@ pub const Instruction = union(enum) {
330330 op: u1,
331331 sf: u1,
332332 },
333 add_subtract_extended_register: packed struct {
334 rd: u5,
335 rn: u5,
336 imm3: u3,
337 option: u3,
338 rm: u5,
339 fixed: u8 = 0b01011_00_1,
340 s: u1,
341 op: u1,
342 sf: u1,
343 },
333344 conditional_branch: struct {
334345 cond: u4,
335346 o0: u1,
......@@ -495,6 +506,7 @@ pub const Instruction = union(enum) {
495506 .logical_immediate => |v| @bitCast(u32, v),
496507 .bitfield => |v| @bitCast(u32, v),
497508 .add_subtract_shifted_register => |v| @bitCast(u32, v),
509 .add_subtract_extended_register => |v| @bitCast(u32, v),
498510 // TODO once packed structs work, this can be refactored
499511 .conditional_branch => |v| @as(u32, v.cond) | (@as(u32, v.o0) << 4) | (@as(u32, v.imm19) << 5) | (@as(u32, v.o1) << 24) | (@as(u32, v.fixed) << 25),
500512 .compare_and_branch => |v| @as(u32, v.rt) | (@as(u32, v.imm19) << 5) | (@as(u32, v.op) << 24) | (@as(u32, v.fixed) << 25) | (@as(u32, v.sf) << 31),
......@@ -1006,6 +1018,44 @@ pub const Instruction = union(enum) {
10061018 };
10071019 }
10081020
1021 pub const AddSubtractExtendedRegisterOption = enum(u3) {
1022 uxtb,
1023 uxth,
1024 uxtw,
1025 uxtx, // serves also as lsl
1026 sxtb,
1027 sxth,
1028 sxtw,
1029 sxtx,
1030 };
1031
1032 fn addSubtractExtendedRegister(
1033 op: u1,
1034 s: u1,
1035 rd: Register,
1036 rn: Register,
1037 rm: Register,
1038 extend: AddSubtractExtendedRegisterOption,
1039 imm3: u3,
1040 ) Instruction {
1041 return Instruction{
1042 .add_subtract_extended_register = .{
1043 .rd = rd.enc(),
1044 .rn = rn.enc(),
1045 .imm3 = imm3,
1046 .option = @enumToInt(extend),
1047 .rm = rm.enc(),
1048 .s = s,
1049 .op = op,
1050 .sf = switch (rd.size()) {
1051 32 => 0b0,
1052 64 => 0b1,
1053 else => unreachable, // unexpected register size
1054 },
1055 },
1056 };
1057 }
1058
10091059 fn conditionalBranch(
10101060 o0: u1,
10111061 o1: u1,
......@@ -1524,6 +1574,48 @@ pub const Instruction = union(enum) {
15241574 return addSubtractShiftedRegister(0b1, 0b1, shift, rd, rn, rm, imm6);
15251575 }
15261576
1577 // Add/subtract (extended register)
1578
1579 pub fn addExtendedRegister(
1580 rd: Register,
1581 rn: Register,
1582 rm: Register,
1583 extend: AddSubtractExtendedRegisterOption,
1584 imm3: u3,
1585 ) Instruction {
1586 return addSubtractExtendedRegister(0b0, 0b0, rd, rn, rm, extend, imm3);
1587 }
1588
1589 pub fn addsExtendedRegister(
1590 rd: Register,
1591 rn: Register,
1592 rm: Register,
1593 extend: AddSubtractExtendedRegisterOption,
1594 imm3: u3,
1595 ) Instruction {
1596 return addSubtractExtendedRegister(0b0, 0b1, rd, rn, rm, extend, imm3);
1597 }
1598
1599 pub fn subExtendedRegister(
1600 rd: Register,
1601 rn: Register,
1602 rm: Register,
1603 extend: AddSubtractExtendedRegisterOption,
1604 imm3: u3,
1605 ) Instruction {
1606 return addSubtractExtendedRegister(0b1, 0b0, rd, rn, rm, extend, imm3);
1607 }
1608
1609 pub fn subsExtendedRegister(
1610 rd: Register,
1611 rn: Register,
1612 rm: Register,
1613 extend: AddSubtractExtendedRegisterOption,
1614 imm3: u3,
1615 ) Instruction {
1616 return addSubtractExtendedRegister(0b1, 0b1, rd, rn, rm, extend, imm3);
1617 }
1618
15271619 // Conditional branch
15281620
15291621 pub fn bCond(cond: Condition, offset: i21) Instruction {
......@@ -1565,11 +1657,12 @@ pub const Instruction = union(enum) {
15651657 }
15661658
15671659 pub fn smaddl(rd: Register, rn: Register, rm: Register, ra: Register) Instruction {
1660 assert(rd.size() == 64 and rn.size() == 32 and rm.size() == 32 and ra.size() == 64);
15681661 return dataProcessing3Source(0b00, 0b001, 0b0, rd, rn, rm, ra);
15691662 }
15701663
15711664 pub fn umaddl(rd: Register, rn: Register, rm: Register, ra: Register) Instruction {
1572 assert(rd.size() == 64);
1665 assert(rd.size() == 64 and rn.size() == 32 and rm.size() == 32 and ra.size() == 64);
15731666 return dataProcessing3Source(0b00, 0b101, 0b0, rd, rn, rm, ra);
15741667 }
15751668
......@@ -1837,6 +1930,10 @@ test "serialize instructions" {
18371930 .inst = Instruction.smulh(.x0, .x1, .x2),
18381931 .expected = 0b1_00_11011_0_10_00010_0_11111_00001_00000,
18391932 },
1933 .{ // adds x0, x1, x2, sxtx
1934 .inst = Instruction.addsExtendedRegister(.x0, .x1, .x2, .sxtx, 0),
1935 .expected = 0b1_0_1_01011_00_1_00010_111_000_00001_00000,
1936 },
18401937 };
18411938
18421939 for (testcases) |case| {