| ... | @@ -2050,13 +2050,10 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2050,13 +2050,10 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { |
| 2050 | registerAlias(src_reg, min_abi_size), | 2050 | registerAlias(src_reg, min_abi_size), |
| 2051 | ); | 2051 | ); |
| 2052 | }, | 2052 | }, |
| 2053 | .load_frame => |frame_addr| try self.asmRegisterMemory( | 2053 | .memory, .indirect, .load_frame => try self.asmRegisterMemory( |
| 2054 | tag, | 2054 | tag, |
| 2055 | dst_alias, | 2055 | dst_alias, |
| 2056 | Memory.sib(Memory.PtrSize.fromSize(min_abi_size), .{ | 2056 | src_mcv.mem(Memory.PtrSize.fromSize(min_abi_size)), |
| 2057 | .base = .{ .frame = frame_addr.index }, | | |
| 2058 | .disp = frame_addr.off, | | |
| 2059 | }), | | |
| 2060 | ), | 2057 | ), |
| 2061 | else => return self.fail("TODO airIntCast from {s} to {s}", .{ | 2058 | else => return self.fail("TODO airIntCast from {s} to {s}", .{ |
| 2062 | @tagName(src_mcv), | 2059 | @tagName(src_mcv), |
| ... | @@ -2738,19 +2735,9 @@ fn genIntMulDivOpMir( | ... | @@ -2738,19 +2735,9 @@ fn genIntMulDivOpMir( |
| 2738 | }; | 2735 | }; |
| 2739 | switch (mat_rhs) { | 2736 | switch (mat_rhs) { |
| 2740 | .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)), | 2737 | .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)), |
| 2741 | .indirect, .load_frame => try self.asmMemory( | 2738 | .memory, .indirect, .load_frame => try self.asmMemory( |
| 2742 | tag, | 2739 | tag, |
| 2743 | Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (mat_rhs) { | 2740 | mat_rhs.mem(Memory.PtrSize.fromSize(abi_size)), |
| 2744 | .indirect => |reg_off| .{ | | |
| 2745 | .base = .{ .reg = reg_off.reg }, | | |
| 2746 | .disp = reg_off.off, | | |
| 2747 | }, | | |
| 2748 | .load_frame => |frame_addr| .{ | | |
| 2749 | .base = .{ .frame = frame_addr.index }, | | |
| 2750 | .disp = frame_addr.off, | | |
| 2751 | }, | | |
| 2752 | else => unreachable, | | |
| 2753 | }), | | |
| 2754 | ), | 2741 | ), |
| 2755 | else => unreachable, | 2742 | else => unreachable, |
| 2756 | } | 2743 | } |
| ... | @@ -4628,9 +4615,6 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: | ... | @@ -4628,9 +4615,6 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: |
| 4628 | if (src_ty.zigTypeTag() == .Vector) { | 4615 | if (src_ty.zigTypeTag() == .Vector) { |
| 4629 | return self.fail("TODO implement genUnOp for {}", .{src_ty.fmt(self.bin_file.options.module.?)}); | 4616 | return self.fail("TODO implement genUnOp for {}", .{src_ty.fmt(self.bin_file.options.module.?)}); |
| 4630 | } | 4617 | } |
| 4631 | if (src_ty.abiSize(self.target.*) > 8) { | | |
| 4632 | return self.fail("TODO implement genUnOp for {}", .{src_ty.fmt(self.bin_file.options.module.?)}); | | |
| 4633 | } | | |
| 4634 | | 4618 | |
| 4635 | switch (src_mcv) { | 4619 | switch (src_mcv) { |
| 4636 | .eflags => |cc| switch (tag) { | 4620 | .eflags => |cc| switch (tag) { |
| ... | @@ -4646,13 +4630,13 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: | ... | @@ -4646,13 +4630,13 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: |
| 4646 | }; | 4630 | }; |
| 4647 | defer if (src_lock) |lock| self.register_manager.unlockReg(lock); | 4631 | defer if (src_lock) |lock| self.register_manager.unlockReg(lock); |
| 4648 | | 4632 | |
| 4649 | const dst_mcv: MCValue = if (maybe_inst) |inst| | 4633 | const dst_mcv: MCValue = dst: { |
| 4650 | if (self.reuseOperand(inst, src_air, 0, src_mcv)) | 4634 | if (maybe_inst) |inst| if (self.reuseOperand(inst, src_air, 0, src_mcv)) break :dst src_mcv; |
| 4651 | src_mcv | 4635 | |
| 4652 | else | 4636 | const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, maybe_inst, true); |
| 4653 | try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv) | 4637 | try self.genCopy(src_ty, dst_mcv, src_mcv); |
| 4654 | else | 4638 | break :dst dst_mcv; |
| 4655 | .{ .register = try self.copyToTmpRegister(src_ty, src_mcv) }; | 4639 | }; |
| 4656 | const dst_lock = switch (dst_mcv) { | 4640 | const dst_lock = switch (dst_mcv) { |
| 4657 | .register => |reg| self.register_manager.lockReg(reg), | 4641 | .register => |reg| self.register_manager.lockReg(reg), |
| 4658 | else => null, | 4642 | else => null, |
| ... | @@ -4661,19 +4645,33 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: | ... | @@ -4661,19 +4645,33 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: |
| 4661 | | 4645 | |
| 4662 | switch (tag) { | 4646 | switch (tag) { |
| 4663 | .not => { | 4647 | .not => { |
| | 4648 | const limb_abi_size = @intCast(u16, @min(src_ty.abiSize(self.target.*), 8)); |
| 4664 | const int_info = if (src_ty.tag() == .bool) | 4649 | const int_info = if (src_ty.tag() == .bool) |
| 4665 | std.builtin.Type.Int{ .signedness = .unsigned, .bits = 1 } | 4650 | std.builtin.Type.Int{ .signedness = .unsigned, .bits = 1 } |
| 4666 | else | 4651 | else |
| 4667 | src_ty.intInfo(self.target.*); | 4652 | src_ty.intInfo(self.target.*); |
| 4668 | const extra_bits = self.regExtraBits(src_ty); | 4653 | var byte_off: i32 = 0; |
| 4669 | if (int_info.signedness == .unsigned and extra_bits > 0) { | 4654 | while (byte_off * 8 < int_info.bits) : (byte_off += limb_abi_size) { |
| 4670 | const mask = (@as(u64, 1) << @intCast(u6, src_ty.bitSize(self.target.*))) - 1; | 4655 | var limb_pl = Type.Payload.Bits{ |
| 4671 | try self.genBinOpMir(.xor, src_ty, dst_mcv, .{ .immediate = mask }); | 4656 | .base = .{ .tag = switch (int_info.signedness) { |
| 4672 | } else try self.genUnOpMir(.not, src_ty, dst_mcv); | 4657 | .signed => .int_signed, |
| 4673 | }, | 4658 | .unsigned => .int_unsigned, |
| | 4659 | } }, |
| | 4660 | .data = @intCast(u16, @min(int_info.bits - byte_off * 8, limb_abi_size * 8)), |
| | 4661 | }; |
| | 4662 | const limb_ty = Type.initPayload(&limb_pl.base); |
| | 4663 | const limb_mcv = switch (byte_off) { |
| | 4664 | 0 => dst_mcv, |
| | 4665 | else => dst_mcv.address().offset(byte_off).deref(), |
| | 4666 | }; |
| 4674 | | 4667 | |
| | 4668 | if (limb_pl.base.tag == .int_unsigned and self.regExtraBits(limb_ty) > 0) { |
| | 4669 | const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_pl.data); |
| | 4670 | try self.genBinOpMir(.xor, limb_ty, limb_mcv, .{ .immediate = mask }); |
| | 4671 | } else try self.genUnOpMir(.not, limb_ty, limb_mcv); |
| | 4672 | } |
| | 4673 | }, |
| 4675 | .neg => try self.genUnOpMir(.neg, src_ty, dst_mcv), | 4674 | .neg => try self.genUnOpMir(.neg, src_ty, dst_mcv), |
| 4676 | | | |
| 4677 | else => unreachable, | 4675 | else => unreachable, |
| 4678 | } | 4676 | } |
| 4679 | return dst_mcv; | 4677 | return dst_mcv; |
| ... | @@ -4714,17 +4712,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue | ... | @@ -4714,17 +4712,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue |
| 4714 | }, | 4712 | }, |
| 4715 | .indirect, .load_frame => try self.asmMemory( | 4713 | .indirect, .load_frame => try self.asmMemory( |
| 4716 | mir_tag, | 4714 | mir_tag, |
| 4717 | Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (dst_mcv) { | 4715 | dst_mcv.mem(Memory.PtrSize.fromSize(abi_size)), |
| 4718 | .indirect => |reg_off| .{ | | |
| 4719 | .base = .{ .reg = reg_off.reg }, | | |
| 4720 | .disp = reg_off.off, | | |
| 4721 | }, | | |
| 4722 | .load_frame => |frame_addr| .{ | | |
| 4723 | .base = .{ .frame = frame_addr.index }, | | |
| 4724 | .disp = frame_addr.off, | | |
| 4725 | }, | | |
| 4726 | else => unreachable, | | |
| 4727 | }), | | |
| 4728 | ), | 4716 | ), |
| 4729 | } | 4717 | } |
| 4730 | } | 4718 | } |
| ... | @@ -8179,12 +8167,9 @@ fn atomicOp( | ... | @@ -8179,12 +8167,9 @@ fn atomicOp( |
| 8179 | registerAlias(val_reg, cmov_abi_size), | 8167 | registerAlias(val_reg, cmov_abi_size), |
| 8180 | cc, | 8168 | cc, |
| 8181 | ), | 8169 | ), |
| 8182 | .load_frame => |frame_addr| try self.asmCmovccRegisterMemory( | 8170 | .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory( |
| 8183 | registerAlias(tmp_reg, cmov_abi_size), | 8171 | registerAlias(tmp_reg, cmov_abi_size), |
| 8184 | Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{ | 8172 | val_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)), |
| 8185 | .base = .{ .frame = frame_addr.index }, | | |
| 8186 | .disp = frame_addr.off, | | |
| 8187 | }), | | |
| 8188 | cc, | 8173 | cc, |
| 8189 | ), | 8174 | ), |
| 8190 | else => { | 8175 | else => { |
| ... | @@ -8207,72 +8192,62 @@ fn atomicOp( | ... | @@ -8207,72 +8192,62 @@ fn atomicOp( |
| 8207 | } else { | 8192 | } else { |
| 8208 | try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{ | 8193 | try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{ |
| 8209 | .base = ptr_mem.sib.base, | 8194 | .base = ptr_mem.sib.base, |
| 8210 | .scale_index = ptr_mem.sib.scale_index, | 8195 | .scale_index = ptr_mem.scaleIndex(), |
| 8211 | .disp = ptr_mem.sib.disp + 0, | 8196 | .disp = ptr_mem.sib.disp + 0, |
| 8212 | })); | 8197 | })); |
| 8213 | try self.asmRegisterMemory(.mov, .rdx, Memory.sib(.qword, .{ | 8198 | try self.asmRegisterMemory(.mov, .rdx, Memory.sib(.qword, .{ |
| 8214 | .base = ptr_mem.sib.base, | 8199 | .base = ptr_mem.sib.base, |
| 8215 | .scale_index = ptr_mem.sib.scale_index, | 8200 | .scale_index = ptr_mem.scaleIndex(), |
| 8216 | .disp = ptr_mem.sib.disp + 8, | 8201 | .disp = ptr_mem.sib.disp + 8, |
| 8217 | })); | 8202 | })); |
| 8218 | const loop = @intCast(u32, self.mir_instructions.len); | 8203 | const loop = @intCast(u32, self.mir_instructions.len); |
| 8219 | switch (val_mcv) { | 8204 | const val_mem_mcv: MCValue = switch (val_mcv) { |
| 8220 | .load_frame => |frame_addr| { | 8205 | .memory, .indirect, .load_frame => val_mcv, |
| 8221 | const val_lo_mem = Memory.sib(.qword, .{ | 8206 | else => .{ .indirect = .{ |
| 8222 | .base = .{ .frame = frame_addr.index }, | 8207 | .reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()), |
| 8223 | .disp = frame_addr.off + 0, | 8208 | } }, |
| 8224 | }); | 8209 | }; |
| 8225 | const val_hi_mem = Memory.sib(.qword, .{ | 8210 | const val_lo_mem = val_mem_mcv.mem(.qword); |
| 8226 | .base = .{ .frame = frame_addr.index }, | 8211 | const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword); |
| 8227 | .disp = frame_addr.off + 8, | 8212 | if (rmw_op != std.builtin.AtomicRmwOp.Xchg) { |
| 8228 | }); | 8213 | try self.asmRegisterRegister(.mov, .rbx, .rax); |
| 8229 | | 8214 | try self.asmRegisterRegister(.mov, .rcx, .rdx); |
| 8230 | if (rmw_op != std.builtin.AtomicRmwOp.Xchg) { | | |
| 8231 | try self.asmRegisterRegister(.mov, .rbx, .rax); | | |
| 8232 | try self.asmRegisterRegister(.mov, .rcx, .rdx); | | |
| 8233 | } | | |
| 8234 | if (rmw_op) |op| switch (op) { | | |
| 8235 | .Xchg => { | | |
| 8236 | try self.asmRegisterMemory(.mov, .rbx, val_lo_mem); | | |
| 8237 | try self.asmRegisterMemory(.mov, .rcx, val_hi_mem); | | |
| 8238 | }, | | |
| 8239 | .Add => { | | |
| 8240 | try self.asmRegisterMemory(.add, .rbx, val_lo_mem); | | |
| 8241 | try self.asmRegisterMemory(.adc, .rcx, val_hi_mem); | | |
| 8242 | }, | | |
| 8243 | .Sub => { | | |
| 8244 | try self.asmRegisterMemory(.sub, .rbx, val_lo_mem); | | |
| 8245 | try self.asmRegisterMemory(.sbb, .rcx, val_hi_mem); | | |
| 8246 | }, | | |
| 8247 | .And => { | | |
| 8248 | try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem); | | |
| 8249 | try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem); | | |
| 8250 | }, | | |
| 8251 | .Nand => { | | |
| 8252 | try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem); | | |
| 8253 | try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem); | | |
| 8254 | try self.asmRegister(.not, .rbx); | | |
| 8255 | try self.asmRegister(.not, .rcx); | | |
| 8256 | }, | | |
| 8257 | .Or => { | | |
| 8258 | try self.asmRegisterMemory(.@"or", .rbx, val_lo_mem); | | |
| 8259 | try self.asmRegisterMemory(.@"or", .rcx, val_hi_mem); | | |
| 8260 | }, | | |
| 8261 | .Xor => { | | |
| 8262 | try self.asmRegisterMemory(.xor, .rbx, val_lo_mem); | | |
| 8263 | try self.asmRegisterMemory(.xor, .rcx, val_hi_mem); | | |
| 8264 | }, | | |
| 8265 | else => return self.fail( | | |
| 8266 | "TODO implement x86 atomic loop for large abi {s}", | | |
| 8267 | .{@tagName(op)}, | | |
| 8268 | ), | | |
| 8269 | }; | | |
| 8270 | }, | | |
| 8271 | else => return self.fail( | | |
| 8272 | "TODO implement x86 atomic loop for large abi {s}", | | |
| 8273 | .{@tagName(val_mcv)}, | | |
| 8274 | ), | | |
| 8275 | } | 8215 | } |
| | 8216 | if (rmw_op) |op| switch (op) { |
| | 8217 | .Xchg => { |
| | 8218 | try self.asmRegisterMemory(.mov, .rbx, val_lo_mem); |
| | 8219 | try self.asmRegisterMemory(.mov, .rcx, val_hi_mem); |
| | 8220 | }, |
| | 8221 | .Add => { |
| | 8222 | try self.asmRegisterMemory(.add, .rbx, val_lo_mem); |
| | 8223 | try self.asmRegisterMemory(.adc, .rcx, val_hi_mem); |
| | 8224 | }, |
| | 8225 | .Sub => { |
| | 8226 | try self.asmRegisterMemory(.sub, .rbx, val_lo_mem); |
| | 8227 | try self.asmRegisterMemory(.sbb, .rcx, val_hi_mem); |
| | 8228 | }, |
| | 8229 | .And => { |
| | 8230 | try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem); |
| | 8231 | try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem); |
| | 8232 | }, |
| | 8233 | .Nand => { |
| | 8234 | try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem); |
| | 8235 | try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem); |
| | 8236 | try self.asmRegister(.not, .rbx); |
| | 8237 | try self.asmRegister(.not, .rcx); |
| | 8238 | }, |
| | 8239 | .Or => { |
| | 8240 | try self.asmRegisterMemory(.@"or", .rbx, val_lo_mem); |
| | 8241 | try self.asmRegisterMemory(.@"or", .rcx, val_hi_mem); |
| | 8242 | }, |
| | 8243 | .Xor => { |
| | 8244 | try self.asmRegisterMemory(.xor, .rbx, val_lo_mem); |
| | 8245 | try self.asmRegisterMemory(.xor, .rcx, val_hi_mem); |
| | 8246 | }, |
| | 8247 | else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{ |
| | 8248 | val_ty.fmt(self.bin_file.options.module.?), @tagName(op), |
| | 8249 | }), |
| | 8250 | }; |
| 8276 | _ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{ | 8251 | _ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{ |
| 8277 | .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)), | 8252 | .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)), |
| 8278 | } }); | 8253 | } }); |
| ... | @@ -9177,15 +9152,16 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void { | ... | @@ -9177,15 +9152,16 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void { |
| 9177 | } | 9152 | } |
| 9178 | | 9153 | |
| 9179 | fn regBitSize(self: *Self, ty: Type) u64 { | 9154 | fn regBitSize(self: *Self, ty: Type) u64 { |
| | 9155 | const abi_size = ty.abiSize(self.target.*); |
| 9180 | return switch (ty.zigTypeTag()) { | 9156 | return switch (ty.zigTypeTag()) { |
| 9181 | else => switch (ty.abiSize(self.target.*)) { | 9157 | else => switch (abi_size) { |
| 9182 | 1 => 8, | 9158 | 1 => 8, |
| 9183 | 2 => 16, | 9159 | 2 => 16, |
| 9184 | 3...4 => 32, | 9160 | 3...4 => 32, |
| 9185 | 5...8 => 64, | 9161 | 5...8 => 64, |
| 9186 | else => unreachable, | 9162 | else => unreachable, |
| 9187 | }, | 9163 | }, |
| 9188 | .Float => switch (ty.abiSize(self.target.*)) { | 9164 | .Float => switch (abi_size) { |
| 9189 | 1...16 => 128, | 9165 | 1...16 => 128, |
| 9190 | 17...32 => 256, | 9166 | 17...32 => 256, |
| 9191 | else => unreachable, | 9167 | else => unreachable, |
| ... | @@ -9197,10 +9173,6 @@ fn regExtraBits(self: *Self, ty: Type) u64 { | ... | @@ -9197,10 +9173,6 @@ fn regExtraBits(self: *Self, ty: Type) u64 { |
| 9197 | return self.regBitSize(ty) - ty.bitSize(self.target.*); | 9173 | return self.regBitSize(ty) - ty.bitSize(self.target.*); |
| 9198 | } | 9174 | } |
| 9199 | | 9175 | |
| 9200 | fn hasAvxSupport(target: Target) bool { | | |
| 9201 | return Target.x86.featureSetHasAny(target.cpu.features, .{ .avx, .avx2 }); | | |
| 9202 | } | | |
| 9203 | | | |
| 9204 | fn getSymbolIndexForDecl(self: *Self, decl_index: Module.Decl.Index) !u32 { | 9176 | fn getSymbolIndexForDecl(self: *Self, decl_index: Module.Decl.Index) !u32 { |
| 9205 | if (self.bin_file.cast(link.File.MachO)) |macho_file| { | 9177 | if (self.bin_file.cast(link.File.MachO)) |macho_file| { |
| 9206 | const atom = try macho_file.getOrCreateAtomForDecl(decl_index); | 9178 | const atom = try macho_file.getOrCreateAtomForDecl(decl_index); |
Jacob Young <jacobly@ziglang.org> 2023-04-28 20:02:28-04:00
src