authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-06 07:28:44+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-05-06 07:28:44+02:00
logdf38dfa4d1c9028453f90c7e37dd6c06f829a995
tree1092812bea75587f79d933d1a6dd20a5d4731184
parent09d2b6c4e1b2edd1d011ef4f947cdd835313a2e5
parentc592f0ca21e8274681fc1dc3c4597ce96ab2096e
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11591 from ziglang/x64-overflow

3 files changed, 221 insertions(+), 79 deletions(-)

src/arch/x86_64/CodeGen.zig+220-77
......@@ -1086,29 +1086,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
10861086 // have to be removed. this only happens if the dst if not a power-of-two size.
10871087 const dst_bit_size = dst_ty.bitSize(self.target.*);
10881088 if (!math.isPowerOfTwo(dst_bit_size) or dst_bit_size < 8) {
1089 const max_reg_bit_width = Register.rax.size();
1090 const shift = @intCast(u6, max_reg_bit_width - dst_ty.bitSize(self.target.*));
1091 const mask = (~@as(u64, 0)) >> shift;
1092 try self.genBinMathOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .immediate = mask });
1093
1094 if (src_ty.intInfo(self.target.*).signedness == .signed) {
1095 _ = try self.addInst(.{
1096 .tag = .sal,
1097 .ops = (Mir.Ops{
1098 .reg1 = reg,
1099 .flags = 0b10,
1100 }).encode(),
1101 .data = .{ .imm = shift },
1102 });
1103 _ = try self.addInst(.{
1104 .tag = .sar,
1105 .ops = (Mir.Ops{
1106 .reg1 = reg,
1107 .flags = 0b10,
1108 }).encode(),
1109 .data = .{ .imm = shift },
1110 });
1111 }
1089 try self.truncateRegister(dst_ty, reg);
11121090 }
11131091
11141092 return self.finishAir(inst, .{ .register = reg }, .{ ty_op.operand, .none, .none });
......@@ -1414,23 +1392,27 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
14141392fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
14151393 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
14161394 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1395 const result = if (self.liveness.isUnused(inst)) .dead else result: {
1396 const ty = self.air.typeOf(bin_op.lhs);
14171397
1418 if (self.liveness.isUnused(inst)) {
1419 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1420 }
1398 switch (ty.zigTypeTag()) {
1399 .Vector => return self.fail("TODO implement add_with_overflow for Vector type", .{}),
1400 .Int => {
1401 const int_info = ty.intInfo(self.target.*);
14211402
1422 const ty = self.air.typeOf(bin_op.lhs);
1423 const signedness: std.builtin.Signedness = blk: {
1424 if (ty.zigTypeTag() != .Int) {
1425 return self.fail("TODO implement airAddWithOverflow for type {}", .{ty.fmtDebug()});
1426 }
1427 break :blk ty.intInfo(self.target.*).signedness;
1428 };
1403 if (int_info.bits > 64) {
1404 return self.fail("TODO implement add_with_overflow for Ints larger than 64bits", .{});
1405 }
14291406
1430 const partial = try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs);
1431 const result: MCValue = switch (signedness) {
1432 .signed => .{ .register_overflow_signed = partial.register },
1433 .unsigned => .{ .register_overflow_unsigned = partial.register },
1407 const partial = try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs);
1408 const result: MCValue = switch (int_info.signedness) {
1409 .signed => .{ .register_overflow_signed = partial.register },
1410 .unsigned => .{ .register_overflow_unsigned = partial.register },
1411 };
1412 break :result result;
1413 },
1414 else => unreachable,
1415 }
14341416 };
14351417
14361418 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
......@@ -1439,23 +1421,27 @@ fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
14391421fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
14401422 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
14411423 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1424 const result = if (self.liveness.isUnused(inst)) .dead else result: {
1425 const ty = self.air.typeOf(bin_op.lhs);
14421426
1443 if (self.liveness.isUnused(inst)) {
1444 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1445 }
1427 switch (ty.zigTypeTag()) {
1428 .Vector => return self.fail("TODO implement sub_with_overflow for Vector type", .{}),
1429 .Int => {
1430 const int_info = ty.intInfo(self.target.*);
14461431
1447 const ty = self.air.typeOf(bin_op.lhs);
1448 const signedness: std.builtin.Signedness = blk: {
1449 if (ty.zigTypeTag() != .Int) {
1450 return self.fail("TODO implement airSubWithOverflow for type {}", .{ty.fmtDebug()});
1451 }
1452 break :blk ty.intInfo(self.target.*).signedness;
1453 };
1432 if (int_info.bits > 64) {
1433 return self.fail("TODO implement sub_with_overflow for Ints larger than 64bits", .{});
1434 }
14541435
1455 const partial = try self.genSubOp(inst, bin_op.lhs, bin_op.rhs);
1456 const result: MCValue = switch (signedness) {
1457 .signed => .{ .register_overflow_signed = partial.register },
1458 .unsigned => .{ .register_overflow_unsigned = partial.register },
1436 const partial = try self.genSubOp(inst, bin_op.lhs, bin_op.rhs);
1437 const result: MCValue = switch (int_info.signedness) {
1438 .signed => .{ .register_overflow_signed = partial.register },
1439 .unsigned => .{ .register_overflow_unsigned = partial.register },
1440 };
1441 break :result result;
1442 },
1443 else => unreachable,
1444 }
14591445 };
14601446
14611447 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
......@@ -1466,30 +1452,149 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
14661452 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
14671453 const result = if (self.liveness.isUnused(inst)) .dead else result: {
14681454 const ty = self.air.typeOf(bin_op.lhs);
1469 const signedness: std.builtin.Signedness = blk: {
1470 if (ty.zigTypeTag() != .Int) {
1471 return self.fail("TODO implement airMulWithOverflow for type {}", .{ty.fmtDebug()});
1472 }
1473 break :blk ty.intInfo(self.target.*).signedness;
1474 };
14751455
1476 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1477 try self.register_manager.getReg(.rax, inst);
1478 try self.register_manager.getReg(.rdx, null);
1479 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1480 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1456 switch (ty.zigTypeTag()) {
1457 .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}),
1458 .Int => {
1459 try self.spillCompareFlagsIfOccupied();
1460 self.compare_flags_inst = null;
14811461
1482 const lhs = try self.resolveInst(bin_op.lhs);
1483 const rhs = try self.resolveInst(bin_op.rhs);
1462 const int_info = ty.intInfo(self.target.*);
14841463
1485 try self.genIntMulDivOpMir(switch (signedness) {
1486 .signed => .imul,
1487 .unsigned => .mul,
1488 }, ty, signedness, lhs, rhs);
1464 if (int_info.bits > 64) {
1465 return self.fail("TODO implement mul_with_overflow for Ints larger than 64bits", .{});
1466 }
14891467
1490 switch (signedness) {
1491 .signed => break :result MCValue{ .register_overflow_signed = .rax },
1492 .unsigned => break :result MCValue{ .register_overflow_unsigned = .rax },
1468 if (math.isPowerOfTwo(int_info.bits)) {
1469 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1470 try self.register_manager.getReg(.rax, inst);
1471 try self.register_manager.getReg(.rdx, null);
1472 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1473 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1474
1475 const lhs = try self.resolveInst(bin_op.lhs);
1476 const rhs = try self.resolveInst(bin_op.rhs);
1477
1478 try self.genIntMulDivOpMir(switch (int_info.signedness) {
1479 .signed => .imul,
1480 .unsigned => .mul,
1481 }, ty, int_info.signedness, lhs, rhs);
1482
1483 const result: MCValue = switch (int_info.signedness) {
1484 .signed => .{ .register_overflow_signed = .rax },
1485 .unsigned => .{ .register_overflow_unsigned = .rax },
1486 };
1487 break :result result;
1488 }
1489
1490 const dst_reg: Register = dst_reg: {
1491 switch (int_info.signedness) {
1492 .signed => {
1493 const lhs = try self.resolveInst(bin_op.lhs);
1494 const rhs = try self.resolveInst(bin_op.rhs);
1495
1496 rhs.freezeIfRegister(&self.register_manager);
1497 defer rhs.unfreezeIfRegister(&self.register_manager);
1498
1499 const dst_reg: Register = blk: {
1500 if (lhs.isRegister()) break :blk lhs.register;
1501 break :blk try self.copyToTmpRegister(ty, lhs);
1502 };
1503 self.register_manager.freezeRegs(&.{dst_reg});
1504
1505 const rhs_mcv = blk: {
1506 if (rhs.isRegister() or rhs.isMemory()) break :blk rhs;
1507 break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) };
1508 };
1509 rhs_mcv.freezeIfRegister(&self.register_manager);
1510 defer rhs_mcv.unfreezeIfRegister(&self.register_manager);
1511
1512 try self.genIntMulComplexOpMir(Type.isize, .{ .register = dst_reg }, rhs_mcv);
1513
1514 break :dst_reg dst_reg;
1515 },
1516 .unsigned => {
1517 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1518 try self.register_manager.getReg(.rax, null);
1519 try self.register_manager.getReg(.rdx, null);
1520 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1521 defer self.register_manager.unfreezeRegs(&.{.rdx});
1522
1523 const lhs = try self.resolveInst(bin_op.lhs);
1524 const rhs = try self.resolveInst(bin_op.rhs);
1525
1526 try self.genIntMulDivOpMir(.mul, ty, .unsigned, lhs, rhs);
1527
1528 break :dst_reg registerAlias(.rax, @intCast(u32, ty.abiSize(self.target.*)));
1529 },
1530 }
1531 };
1532 defer self.register_manager.unfreezeRegs(&.{dst_reg});
1533
1534 const tuple_ty = self.air.typeOfIndex(inst);
1535 const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
1536 const tuple_align = tuple_ty.abiAlignment(self.target.*);
1537 const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));
1538
1539 const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));
1540 const extended_ty = switch (int_info.signedness) {
1541 .signed => Type.isize,
1542 .unsigned => ty,
1543 };
1544
1545 const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null });
1546 self.register_manager.freezeRegs(&temp_regs);
1547 defer self.register_manager.unfreezeRegs(&temp_regs);
1548
1549 const overflow_reg = temp_regs[0];
1550 const flags: u2 = switch (int_info.signedness) {
1551 .signed => 0b00,
1552 .unsigned => 0b10,
1553 };
1554 _ = try self.addInst(.{
1555 .tag = .cond_set_byte_overflow,
1556 .ops = (Mir.Ops{
1557 .reg1 = overflow_reg.to8(),
1558 .flags = flags,
1559 }).encode(),
1560 .data = undefined,
1561 });
1562
1563 const scratch_reg = temp_regs[1];
1564 try self.genSetReg(extended_ty, scratch_reg, .{ .register = dst_reg });
1565 try self.truncateRegister(ty, scratch_reg);
1566 try self.genBinMathOpMir(
1567 .cmp,
1568 extended_ty,
1569 .{ .register = dst_reg },
1570 .{ .register = scratch_reg },
1571 );
1572
1573 const eq_reg = temp_regs[2];
1574 _ = try self.addInst(.{
1575 .tag = .cond_set_byte_eq_ne,
1576 .ops = (Mir.Ops{
1577 .reg1 = eq_reg.to8(),
1578 .flags = 0b00,
1579 }).encode(),
1580 .data = undefined,
1581 });
1582
1583 try self.genBinMathOpMir(
1584 .@"or",
1585 Type.u8,
1586 .{ .register = overflow_reg },
1587 .{ .register = eq_reg },
1588 );
1589
1590 try self.genSetStack(ty, stack_offset, .{ .register = scratch_reg }, .{});
1591 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{
1592 .register = overflow_reg.to8(),
1593 }, .{});
1594
1595 break :result MCValue{ .stack_offset = stack_offset };
1596 },
1597 else => unreachable,
14931598 }
14941599 };
14951600
......@@ -1633,7 +1738,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
16331738 }).encode(),
16341739 .data = undefined,
16351740 });
1636 try self.genBinMathOpMir(.add, Type.isize, .{ .register = divisor.to64() }, .{ .register = .rax });
1741 try self.genBinMathOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax });
16371742 return MCValue{ .register = divisor };
16381743}
16391744
......@@ -2207,8 +2312,8 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
22072312 }
22082313 // TODO we could allocate register here, but need to expect addr register and potentially
22092314 // offset register.
2210 try self.genBinMathOpMir(.add, slice_ptr_field_type, .{ .register = addr_reg.to64() }, .{
2211 .register = offset_reg.to64(),
2315 try self.genBinMathOpMir(.add, slice_ptr_field_type, .{ .register = addr_reg }, .{
2316 .register = offset_reg,
22122317 });
22132318 return MCValue{ .register = addr_reg.to64() };
22142319}
......@@ -2300,7 +2405,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
23002405 // TODO we could allocate register here, but need to expect addr register and potentially
23012406 // offset register.
23022407 const dst_mcv = try self.allocRegOrMem(inst, false);
2303 try self.genBinMathOpMir(.add, array_ty, .{ .register = addr_reg.to64() }, .{ .register = offset_reg.to64() });
2408 try self.genBinMathOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
23042409 try self.load(dst_mcv, .{ .register = addr_reg.to64() }, array_ty);
23052410 break :result dst_mcv;
23062411 };
......@@ -3163,8 +3268,8 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
31633268 _ = try self.addInst(.{
31643269 .tag = mir_tag,
31653270 .ops = (Mir.Ops{
3166 .reg1 = registerAlias(dst_reg, @divExact(src_reg.size(), 8)),
3167 .reg2 = src_reg,
3271 .reg1 = registerAlias(dst_reg, abi_size),
3272 .reg2 = registerAlias(src_reg, abi_size),
31683273 }).encode(),
31693274 .data = undefined,
31703275 });
......@@ -6516,3 +6621,41 @@ fn shiftRegister(self: *Self, reg: Register, shift: u8) !void {
65166621 });
65176622 }
65186623}
6624
6625/// Truncates the value in the register in place.
6626/// Clobbers any remaining bits.
6627fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
6628 const int_info = ty.intInfo(self.target.*);
6629 const max_reg_bit_width = Register.rax.size();
6630 switch (int_info.signedness) {
6631 .signed => {
6632 const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
6633 _ = try self.addInst(.{
6634 .tag = .sal,
6635 .ops = (Mir.Ops{
6636 .reg1 = reg.to64(),
6637 .flags = 0b10,
6638 }).encode(),
6639 .data = .{ .imm = shift },
6640 });
6641 _ = try self.addInst(.{
6642 .tag = .sar,
6643 .ops = (Mir.Ops{
6644 .reg1 = reg.to64(),
6645 .flags = 0b10,
6646 }).encode(),
6647 .data = .{ .imm = shift },
6648 });
6649 },
6650 .unsigned => {
6651 const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
6652 const mask = (~@as(u64, 0)) >> shift;
6653 if (int_info.bits <= 32) {
6654 try self.genBinMathOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .immediate = mask });
6655 } else {
6656 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
6657 try self.genBinMathOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .register = tmp_reg });
6658 }
6659 },
6660 }
6661}
src/arch/x86_64/Emit.zig+1-1
......@@ -1383,7 +1383,7 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
13831383 .sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b),
13841384 .xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33),
13851385 .@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23),
1386 .@"or" => OpCode.oneByte(if (is_one_byte) 0x0b else 0x0b),
1386 .@"or" => OpCode.oneByte(if (is_one_byte) 0x0a else 0x0b),
13871387 .sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b),
13881388 .cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b),
13891389 .mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b),
test/behavior/math.zig-1
......@@ -686,7 +686,6 @@ test "basic @mulWithOverflow" {
686686// TODO migrate to this for all backends once they handle more cases
687687test "extensive @mulWithOverflow" {
688688 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
689 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
690689 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
691690 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
692691