| ... | ... | @@ -3104,18 +3104,17 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void { |
| 3104 | 3104 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 3105 | 3105 | |
| 3106 | 3106 | const slice_ty = self.typeOfIndex(inst); |
| 3107 | | const ptr = try self.resolveInst(bin_op.lhs); |
| 3107 | const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod)); |
| 3108 | |
| 3108 | 3109 | const ptr_ty = self.typeOf(bin_op.lhs); |
| 3109 | | const len = try self.resolveInst(bin_op.rhs); |
| 3110 | | const len_ty = self.typeOf(bin_op.rhs); |
| 3110 | try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }); |
| 3111 | 3111 | |
| 3112 | | const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod)); |
| 3113 | | try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr); |
| 3112 | const len_ty = self.typeOf(bin_op.rhs); |
| 3114 | 3113 | try self.genSetMem( |
| 3115 | 3114 | .{ .frame = frame_index }, |
| 3116 | 3115 | @intCast(ptr_ty.abiSize(mod)), |
| 3117 | 3116 | len_ty, |
| 3118 | | len, |
| 3117 | .{ .air_ref = bin_op.rhs }, |
| 3119 | 3118 | ); |
| 3120 | 3119 | |
| 3121 | 3120 | const result = MCValue{ .load_frame = .{ .index = frame_index } }; |
| ... | ... | @@ -7099,28 +7098,26 @@ fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerErr |
| 7099 | 7098 | |
| 7100 | 7099 | fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void { |
| 7101 | 7100 | const mod = self.bin_file.options.module.?; |
| 7102 | | if (safety) { |
| 7103 | | // TODO if the value is undef, write 0xaa bytes to dest |
| 7104 | | } else { |
| 7105 | | // TODO if the value is undef, don't lower this instruction |
| 7106 | | } |
| 7101 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 7107 | 7102 | |
| 7108 | | try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); |
| 7109 | | const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx }); |
| 7110 | | defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); |
| 7103 | result: { |
| 7104 | if (!safety and (try self.resolveInst(bin_op.rhs)) == .undef) break :result; |
| 7111 | 7105 | |
| 7112 | | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 7113 | | const ptr_mcv = try self.resolveInst(bin_op.lhs); |
| 7114 | | const ptr_ty = self.typeOf(bin_op.lhs); |
| 7115 | | const src_mcv = try self.resolveInst(bin_op.rhs); |
| 7106 | try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); |
| 7107 | const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx }); |
| 7108 | defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); |
| 7116 | 7109 | |
| 7117 | | const ptr_info = ptr_ty.ptrInfo(mod); |
| 7118 | | if (ptr_info.flags.vector_index != .none or ptr_info.packed_offset.host_size > 0) { |
| 7119 | | try self.packedStore(ptr_ty, ptr_mcv, src_mcv); |
| 7120 | | } else { |
| 7121 | | try self.store(ptr_ty, ptr_mcv, src_mcv); |
| 7122 | | } |
| 7110 | const src_mcv = try self.resolveInst(bin_op.rhs); |
| 7111 | const ptr_mcv = try self.resolveInst(bin_op.lhs); |
| 7112 | const ptr_ty = self.typeOf(bin_op.lhs); |
| 7123 | 7113 | |
| 7114 | const ptr_info = ptr_ty.ptrInfo(mod); |
| 7115 | if (ptr_info.flags.vector_index != .none or ptr_info.packed_offset.host_size > 0) { |
| 7116 | try self.packedStore(ptr_ty, ptr_mcv, src_mcv); |
| 7117 | } else { |
| 7118 | try self.store(ptr_ty, ptr_mcv, src_mcv); |
| 7119 | } |
| 7120 | } |
| 7124 | 7121 | return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 7125 | 7122 | } |
| 7126 | 7123 | |
| ... | ... | @@ -11549,7 +11546,6 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index { |
| 11549 | 11546 | }, |
| 11550 | 11547 | else => return self.fail("TODO implement condbr when condition is {s}", .{@tagName(mcv)}), |
| 11551 | 11548 | } |
| 11552 | | return 0; // TODO |
| 11553 | 11549 | } |
| 11554 | 11550 | |
| 11555 | 11551 | fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { |
| ... | ... | @@ -12336,7 +12332,18 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { |
| 12336 | 12332 | // for the string, we still use the next u32 for the null terminator. |
| 12337 | 12333 | extra_i += clobber.len / 4 + 1; |
| 12338 | 12334 | |
| 12339 | | // TODO honor these |
| 12335 | if (std.mem.eql(u8, clobber, "") or std.mem.eql(u8, clobber, "memory")) { |
| 12336 | // ok, sure |
| 12337 | } else if (std.mem.eql(u8, clobber, "cc") or |
| 12338 | std.mem.eql(u8, clobber, "flags") or |
| 12339 | std.mem.eql(u8, clobber, "eflags") or |
| 12340 | std.mem.eql(u8, clobber, "rflags")) |
| 12341 | { |
| 12342 | try self.spillEflagsIfOccupied(); |
| 12343 | } else { |
| 12344 | try self.register_manager.getReg(parseRegName(clobber) orelse |
| 12345 | return self.fail("invalid clobber: '{s}'", .{clobber}), null); |
| 12346 | } |
| 12340 | 12347 | } |
| 12341 | 12348 | } |
| 12342 | 12349 | |
| ... | ... | @@ -13517,7 +13524,11 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal |
| 13517 | 13524 | }; |
| 13518 | 13525 | switch (src_mcv) { |
| 13519 | 13526 | .none, .unreach, .dead, .reserved_frame => unreachable, |
| 13520 | | .undef => {}, |
| 13527 | .undef => try self.genInlineMemset( |
| 13528 | dst_ptr_mcv, |
| 13529 | .{ .immediate = 0xaa }, |
| 13530 | .{ .immediate = abi_size }, |
| 13531 | ), |
| 13521 | 13532 | .immediate => |imm| switch (abi_size) { |
| 13522 | 13533 | 1, 2, 4 => { |
| 13523 | 13534 | const immediate = switch (if (ty.isAbiInt(mod)) |
| ... | ... | @@ -14596,128 +14607,129 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr |
| 14596 | 14607 | |
| 14597 | 14608 | fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void { |
| 14598 | 14609 | const mod = self.bin_file.options.module.?; |
| 14599 | | if (safety) { |
| 14600 | | // TODO if the value is undef, write 0xaa bytes to dest |
| 14601 | | } else { |
| 14602 | | // TODO if the value is undef, don't lower this instruction |
| 14603 | | } |
| 14604 | | |
| 14605 | 14610 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 14606 | 14611 | |
| 14607 | | try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); |
| 14608 | | const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx }); |
| 14609 | | defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); |
| 14612 | result: { |
| 14613 | if (!safety and (try self.resolveInst(bin_op.rhs)) == .undef) break :result; |
| 14610 | 14614 | |
| 14611 | | const dst_ptr = try self.resolveInst(bin_op.lhs); |
| 14612 | | const dst_ptr_ty = self.typeOf(bin_op.lhs); |
| 14613 | | const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) { |
| 14614 | | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 14615 | | else => null, |
| 14616 | | }; |
| 14617 | | defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock); |
| 14618 | | |
| 14619 | | const src_val = try self.resolveInst(bin_op.rhs); |
| 14620 | | const elem_ty = self.typeOf(bin_op.rhs); |
| 14621 | | const src_val_lock: ?RegisterLock = switch (src_val) { |
| 14622 | | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 14623 | | else => null, |
| 14624 | | }; |
| 14625 | | defer if (src_val_lock) |lock| self.register_manager.unlockReg(lock); |
| 14626 | | |
| 14627 | | const elem_abi_size: u31 = @intCast(elem_ty.abiSize(mod)); |
| 14615 | try self.spillRegisters(&.{ .rax, .rdi, .rsi, .rcx }); |
| 14616 | const reg_locks = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdi, .rsi, .rcx }); |
| 14617 | defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); |
| 14628 | 14618 | |
| 14629 | | if (elem_abi_size == 1) { |
| 14630 | | const ptr: MCValue = switch (dst_ptr_ty.ptrSize(mod)) { |
| 14631 | | // TODO: this only handles slices stored in the stack |
| 14632 | | .Slice => dst_ptr, |
| 14633 | | .One => dst_ptr, |
| 14634 | | .C, .Many => unreachable, |
| 14635 | | }; |
| 14636 | | const len: MCValue = switch (dst_ptr_ty.ptrSize(mod)) { |
| 14637 | | // TODO: this only handles slices stored in the stack |
| 14638 | | .Slice => dst_ptr.address().offset(8).deref(), |
| 14639 | | .One => .{ .immediate = dst_ptr_ty.childType(mod).arrayLen(mod) }, |
| 14640 | | .C, .Many => unreachable, |
| 14641 | | }; |
| 14642 | | const len_lock: ?RegisterLock = switch (len) { |
| 14619 | const dst_ptr = try self.resolveInst(bin_op.lhs); |
| 14620 | const dst_ptr_ty = self.typeOf(bin_op.lhs); |
| 14621 | const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) { |
| 14643 | 14622 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 14644 | 14623 | else => null, |
| 14645 | 14624 | }; |
| 14646 | | defer if (len_lock) |lock| self.register_manager.unlockReg(lock); |
| 14647 | | |
| 14648 | | try self.genInlineMemset(ptr, src_val, len); |
| 14649 | | return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 14650 | | } |
| 14625 | defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock); |
| 14651 | 14626 | |
| 14652 | | // Store the first element, and then rely on memcpy copying forwards. |
| 14653 | | // Length zero requires a runtime check - so we handle arrays specially |
| 14654 | | // here to elide it. |
| 14655 | | switch (dst_ptr_ty.ptrSize(mod)) { |
| 14656 | | .Slice => { |
| 14657 | | const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(mod); |
| 14658 | | |
| 14659 | | // TODO: this only handles slices stored in the stack |
| 14660 | | const ptr = dst_ptr; |
| 14661 | | const len = dst_ptr.address().offset(8).deref(); |
| 14662 | | |
| 14663 | | // Used to store the number of elements for comparison. |
| 14664 | | // After comparison, updated to store number of bytes needed to copy. |
| 14665 | | const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 14666 | | const len_mcv: MCValue = .{ .register = len_reg }; |
| 14667 | | const len_lock = self.register_manager.lockRegAssumeUnused(len_reg); |
| 14668 | | defer self.register_manager.unlockReg(len_lock); |
| 14669 | | |
| 14670 | | try self.genSetReg(len_reg, Type.usize, len); |
| 14671 | | try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg); |
| 14627 | const src_val = try self.resolveInst(bin_op.rhs); |
| 14628 | const elem_ty = self.typeOf(bin_op.rhs); |
| 14629 | const src_val_lock: ?RegisterLock = switch (src_val) { |
| 14630 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 14631 | else => null, |
| 14632 | }; |
| 14633 | defer if (src_val_lock) |lock| self.register_manager.unlockReg(lock); |
| 14672 | 14634 | |
| 14673 | | const skip_reloc = try self.asmJccReloc(.z, undefined); |
| 14674 | | try self.store(slice_ptr_ty, ptr, src_val); |
| 14635 | const elem_abi_size: u31 = @intCast(elem_ty.abiSize(mod)); |
| 14675 | 14636 | |
| 14676 | | const second_elem_ptr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 14677 | | const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; |
| 14678 | | const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); |
| 14679 | | defer self.register_manager.unlockReg(second_elem_ptr_lock); |
| 14637 | if (elem_abi_size == 1) { |
| 14638 | const ptr: MCValue = switch (dst_ptr_ty.ptrSize(mod)) { |
| 14639 | // TODO: this only handles slices stored in the stack |
| 14640 | .Slice => dst_ptr, |
| 14641 | .One => dst_ptr, |
| 14642 | .C, .Many => unreachable, |
| 14643 | }; |
| 14644 | const len: MCValue = switch (dst_ptr_ty.ptrSize(mod)) { |
| 14645 | // TODO: this only handles slices stored in the stack |
| 14646 | .Slice => dst_ptr.address().offset(8).deref(), |
| 14647 | .One => .{ .immediate = dst_ptr_ty.childType(mod).arrayLen(mod) }, |
| 14648 | .C, .Many => unreachable, |
| 14649 | }; |
| 14650 | const len_lock: ?RegisterLock = switch (len) { |
| 14651 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 14652 | else => null, |
| 14653 | }; |
| 14654 | defer if (len_lock) |lock| self.register_manager.unlockReg(lock); |
| 14680 | 14655 | |
| 14681 | | try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{ |
| 14682 | | .reg = try self.copyToTmpRegister(Type.usize, ptr), |
| 14683 | | .off = elem_abi_size, |
| 14684 | | } }); |
| 14656 | try self.genInlineMemset(ptr, src_val, len); |
| 14657 | break :result; |
| 14658 | } |
| 14685 | 14659 | |
| 14686 | | try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 }); |
| 14687 | | try self.asmRegisterRegisterImmediate( |
| 14688 | | .{ .i_, .mul }, |
| 14689 | | len_reg, |
| 14690 | | len_reg, |
| 14691 | | Immediate.s(elem_abi_size), |
| 14692 | | ); |
| 14693 | | try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv); |
| 14660 | // Store the first element, and then rely on memcpy copying forwards. |
| 14661 | // Length zero requires a runtime check - so we handle arrays specially |
| 14662 | // here to elide it. |
| 14663 | switch (dst_ptr_ty.ptrSize(mod)) { |
| 14664 | .Slice => { |
| 14665 | const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(mod); |
| 14666 | |
| 14667 | // TODO: this only handles slices stored in the stack |
| 14668 | const ptr = dst_ptr; |
| 14669 | const len = dst_ptr.address().offset(8).deref(); |
| 14670 | |
| 14671 | // Used to store the number of elements for comparison. |
| 14672 | // After comparison, updated to store number of bytes needed to copy. |
| 14673 | const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 14674 | const len_mcv: MCValue = .{ .register = len_reg }; |
| 14675 | const len_lock = self.register_manager.lockRegAssumeUnused(len_reg); |
| 14676 | defer self.register_manager.unlockReg(len_lock); |
| 14677 | |
| 14678 | try self.genSetReg(len_reg, Type.usize, len); |
| 14679 | try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg); |
| 14680 | |
| 14681 | const skip_reloc = try self.asmJccReloc(.z, undefined); |
| 14682 | try self.store(slice_ptr_ty, ptr, src_val); |
| 14683 | |
| 14684 | const second_elem_ptr_reg = |
| 14685 | try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 14686 | const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; |
| 14687 | const second_elem_ptr_lock = |
| 14688 | self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); |
| 14689 | defer self.register_manager.unlockReg(second_elem_ptr_lock); |
| 14690 | |
| 14691 | try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{ |
| 14692 | .reg = try self.copyToTmpRegister(Type.usize, ptr), |
| 14693 | .off = elem_abi_size, |
| 14694 | } }); |
| 14695 | |
| 14696 | try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 }); |
| 14697 | try self.asmRegisterRegisterImmediate( |
| 14698 | .{ .i_, .mul }, |
| 14699 | len_reg, |
| 14700 | len_reg, |
| 14701 | Immediate.s(elem_abi_size), |
| 14702 | ); |
| 14703 | try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv); |
| 14694 | 14704 | |
| 14695 | | try self.performReloc(skip_reloc); |
| 14696 | | }, |
| 14697 | | .One => { |
| 14698 | | const elem_ptr_ty = try mod.singleMutPtrType(elem_ty); |
| 14705 | try self.performReloc(skip_reloc); |
| 14706 | }, |
| 14707 | .One => { |
| 14708 | const elem_ptr_ty = try mod.singleMutPtrType(elem_ty); |
| 14699 | 14709 | |
| 14700 | | const len = dst_ptr_ty.childType(mod).arrayLen(mod); |
| 14710 | const len = dst_ptr_ty.childType(mod).arrayLen(mod); |
| 14701 | 14711 | |
| 14702 | | assert(len != 0); // prevented by Sema |
| 14703 | | try self.store(elem_ptr_ty, dst_ptr, src_val); |
| 14712 | assert(len != 0); // prevented by Sema |
| 14713 | try self.store(elem_ptr_ty, dst_ptr, src_val); |
| 14704 | 14714 | |
| 14705 | | const second_elem_ptr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 14706 | | const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; |
| 14707 | | const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); |
| 14708 | | defer self.register_manager.unlockReg(second_elem_ptr_lock); |
| 14715 | const second_elem_ptr_reg = |
| 14716 | try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 14717 | const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; |
| 14718 | const second_elem_ptr_lock = |
| 14719 | self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); |
| 14720 | defer self.register_manager.unlockReg(second_elem_ptr_lock); |
| 14709 | 14721 | |
| 14710 | | try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{ |
| 14711 | | .reg = try self.copyToTmpRegister(Type.usize, dst_ptr), |
| 14712 | | .off = elem_abi_size, |
| 14713 | | } }); |
| 14722 | try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{ |
| 14723 | .reg = try self.copyToTmpRegister(Type.usize, dst_ptr), |
| 14724 | .off = elem_abi_size, |
| 14725 | } }); |
| 14714 | 14726 | |
| 14715 | | const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) }; |
| 14716 | | try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy); |
| 14717 | | }, |
| 14718 | | .C, .Many => unreachable, |
| 14727 | const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) }; |
| 14728 | try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy); |
| 14729 | }, |
| 14730 | .C, .Many => unreachable, |
| 14731 | } |
| 14719 | 14732 | } |
| 14720 | | |
| 14721 | 14733 | return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 14722 | 14734 | } |
| 14723 | 14735 | |
Jacob Young <jacobly@ziglang.org> 2023-12-02 13:02:45-05:00
src