| ... | ... | @@ -2844,6 +2844,9 @@ fn genBinOp( |
| 2844 | 2844 | .cmp_gt, |
| 2845 | 2845 | .cmp_gte, |
| 2846 | 2846 | => { |
| 2847 | try func.truncateRegister(lhs_ty, lhs_reg); |
| 2848 | try func.truncateRegister(rhs_ty, rhs_reg); |
| 2849 | |
| 2847 | 2850 | _ = try func.addInst(.{ |
| 2848 | 2851 | .tag = .pseudo_compare, |
| 2849 | 2852 | .data = .{ |
| ... | ... | @@ -3925,10 +3928,6 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void { |
| 3925 | 3928 | const base_ptr_ty = func.typeOf(extra.lhs); |
| 3926 | 3929 | |
| 3927 | 3930 | if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) { |
| 3928 | | // break :result if (func.reuseOperand(inst, extra.lhs, 0, base_ptr_mcv)) |
| 3929 | | // base_ptr_mcv |
| 3930 | | // else |
| 3931 | | // try func.copyToNewRegister(inst, base_ptr_mcv); |
| 3932 | 3931 | @panic("audit"); |
| 3933 | 3932 | } |
| 3934 | 3933 | |
| ... | ... | @@ -3992,7 +3991,7 @@ fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void { |
| 3992 | 3991 | defer func.register_manager.unlockReg(result_lock); |
| 3993 | 3992 | |
| 3994 | 3993 | switch (frame_mcv) { |
| 3995 | | .load_frame => |frame_addr| { |
| 3994 | .load_frame => { |
| 3996 | 3995 | if (tag_abi_size <= 8) { |
| 3997 | 3996 | const off: i32 = if (layout.tag_align.compare(.lt, layout.payload_align)) |
| 3998 | 3997 | @intCast(layout.payload_size) |
| ... | ... | @@ -4002,7 +4001,7 @@ fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void { |
| 4002 | 4001 | try func.genCopy( |
| 4003 | 4002 | tag_ty, |
| 4004 | 4003 | .{ .register = result_reg }, |
| 4005 | | .{ .load_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off } }, |
| 4004 | frame_mcv.offset(off), |
| 4006 | 4005 | ); |
| 4007 | 4006 | } else { |
| 4008 | 4007 | return func.fail( |
| ... | ... | @@ -4081,7 +4080,37 @@ fn airCtz(func: *Func, inst: Air.Inst.Index) !void { |
| 4081 | 4080 | |
| 4082 | 4081 | fn airPopcount(func: *Func, inst: Air.Inst.Index) !void { |
| 4083 | 4082 | const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 4084 | | const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airPopcount for {}", .{func.target.cpu.arch}); |
| 4083 | const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { |
| 4084 | const pt = func.pt; |
| 4085 | |
| 4086 | const operand = try func.resolveInst(ty_op.operand); |
| 4087 | const src_ty = func.typeOf(ty_op.operand); |
| 4088 | const operand_reg, const operand_lock = try func.promoteReg(src_ty, operand); |
| 4089 | defer if (operand_lock) |lock| func.register_manager.unlockReg(lock); |
| 4090 | |
| 4091 | const dst_reg, const dst_lock = try func.allocReg(.int); |
| 4092 | defer func.register_manager.unlockReg(dst_lock); |
| 4093 | |
| 4094 | const bit_size = src_ty.bitSize(pt); |
| 4095 | switch (bit_size) { |
| 4096 | 32, 64 => {}, |
| 4097 | 1...31, 33...63 => try func.truncateRegister(src_ty, operand_reg), |
| 4098 | else => return func.fail("TODO: airPopcount > 64 bits", .{}), |
| 4099 | } |
| 4100 | |
| 4101 | _ = try func.addInst(.{ |
| 4102 | .tag = if (bit_size <= 32) .cpopw else .cpop, |
| 4103 | .data = .{ |
| 4104 | .r_type = .{ |
| 4105 | .rd = dst_reg, |
| 4106 | .rs1 = operand_reg, |
| 4107 | .rs2 = @enumFromInt(0b00010), // this is the cpop funct5 |
| 4108 | }, |
| 4109 | }, |
| 4110 | }); |
| 4111 | |
| 4112 | break :result .{ .register = dst_reg }; |
| 4113 | }; |
| 4085 | 4114 | return func.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 4086 | 4115 | } |
| 4087 | 4116 | |
| ... | ... | @@ -5630,7 +5659,6 @@ fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) ! |
| 5630 | 5659 | |
| 5631 | 5660 | fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void { |
| 5632 | 5661 | const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 5633 | | const condition = try func.resolveInst(pl_op.operand); |
| 5634 | 5662 | const condition_ty = func.typeOf(pl_op.operand); |
| 5635 | 5663 | const switch_br = func.air.extraData(Air.SwitchBr, pl_op.payload); |
| 5636 | 5664 | var extra_index: usize = switch_br.end; |
| ... | ... | @@ -5638,6 +5666,8 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void { |
| 5638 | 5666 | const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.data.cases_len + 1); |
| 5639 | 5667 | defer func.gpa.free(liveness.deaths); |
| 5640 | 5668 | |
| 5669 | const condition = try func.resolveInst(pl_op.operand); |
| 5670 | |
| 5641 | 5671 | // If the condition dies here in this switch instruction, process |
| 5642 | 5672 | // that death now instead of later as this has an effect on |
| 5643 | 5673 | // whether it needs to be spilled in the branches |
| ... | ... | @@ -5660,9 +5690,14 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void { |
| 5660 | 5690 | defer func.gpa.free(relocs); |
| 5661 | 5691 | |
| 5662 | 5692 | for (items, relocs, 0..) |item, *reloc, i| { |
| 5663 | | // switch branches must be comptime-known, so this is stored in an immediate |
| 5664 | 5693 | const item_mcv = try func.resolveInst(item); |
| 5665 | 5694 | |
| 5695 | const cond_lock = switch (condition) { |
| 5696 | .register => func.register_manager.lockRegAssumeUnused(condition.register), |
| 5697 | else => null, |
| 5698 | }; |
| 5699 | defer if (cond_lock) |lock| func.register_manager.unlockReg(lock); |
| 5700 | |
| 5666 | 5701 | const cmp_reg, const cmp_lock = try func.allocReg(.int); |
| 5667 | 5702 | defer func.register_manager.unlockReg(cmp_lock); |
| 5668 | 5703 | |
| ... | ... | @@ -7110,57 +7145,58 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void { |
| 7110 | 7145 | const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 7111 | 7146 | const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data; |
| 7112 | 7147 | |
| 7113 | | const op = extra.op(); |
| 7114 | | const order = extra.ordering(); |
| 7148 | const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { |
| 7149 | const op = extra.op(); |
| 7150 | const order = extra.ordering(); |
| 7115 | 7151 | |
| 7116 | | const ptr_ty = func.typeOf(pl_op.operand); |
| 7117 | | const ptr_mcv = try func.resolveInst(pl_op.operand); |
| 7152 | const ptr_ty = func.typeOf(pl_op.operand); |
| 7153 | const ptr_mcv = try func.resolveInst(pl_op.operand); |
| 7118 | 7154 | |
| 7119 | | const val_ty = func.typeOf(extra.operand); |
| 7120 | | const val_size = val_ty.abiSize(pt); |
| 7121 | | const val_mcv = try func.resolveInst(extra.operand); |
| 7155 | const val_ty = func.typeOf(extra.operand); |
| 7156 | const val_size = val_ty.abiSize(pt); |
| 7157 | const val_mcv = try func.resolveInst(extra.operand); |
| 7122 | 7158 | |
| 7123 | | if (!math.isPowerOfTwo(val_size)) |
| 7124 | | return func.fail("TODO: airAtomicRmw non-pow 2", .{}); |
| 7159 | if (!math.isPowerOfTwo(val_size)) |
| 7160 | return func.fail("TODO: airAtomicRmw non-pow 2", .{}); |
| 7125 | 7161 | |
| 7126 | | switch (val_ty.zigTypeTag(pt.zcu)) { |
| 7127 | | .Int => {}, |
| 7128 | | inline .Bool, .Float, .Enum, .Pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}), |
| 7129 | | else => unreachable, |
| 7130 | | } |
| 7162 | switch (val_ty.zigTypeTag(pt.zcu)) { |
| 7163 | .Int => {}, |
| 7164 | inline .Bool, .Float, .Enum, .Pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}), |
| 7165 | else => unreachable, |
| 7166 | } |
| 7131 | 7167 | |
| 7132 | | const method: enum { amo, loop } = switch (val_size) { |
| 7133 | | 1, 2 => .loop, |
| 7134 | | 4, 8 => .amo, |
| 7135 | | else => unreachable, |
| 7136 | | }; |
| 7168 | const method: enum { amo, loop } = switch (val_size) { |
| 7169 | 1, 2 => .loop, |
| 7170 | 4, 8 => .amo, |
| 7171 | else => unreachable, |
| 7172 | }; |
| 7137 | 7173 | |
| 7138 | | const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv); |
| 7139 | | defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock); |
| 7174 | const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv); |
| 7175 | defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock); |
| 7140 | 7176 | |
| 7141 | | const val_register, const val_lock = try func.promoteReg(val_ty, val_mcv); |
| 7142 | | defer if (val_lock) |lock| func.register_manager.unlockReg(lock); |
| 7177 | const val_register, const val_lock = try func.promoteReg(val_ty, val_mcv); |
| 7178 | defer if (val_lock) |lock| func.register_manager.unlockReg(lock); |
| 7143 | 7179 | |
| 7144 | | const result_mcv = try func.allocRegOrMem(val_ty, inst, true); |
| 7145 | | assert(result_mcv == .register); // should fit into 8 bytes |
| 7146 | | const result_reg = result_mcv.register; |
| 7180 | const result_mcv = try func.allocRegOrMem(val_ty, inst, true); |
| 7181 | assert(result_mcv == .register); // should fit into 8 bytes |
| 7182 | const result_reg = result_mcv.register; |
| 7147 | 7183 | |
| 7148 | | const aq, const rl = switch (order) { |
| 7149 | | .unordered => unreachable, |
| 7150 | | .monotonic => .{ false, false }, |
| 7151 | | .acquire => .{ true, false }, |
| 7152 | | .release => .{ false, true }, |
| 7153 | | .acq_rel => .{ true, true }, |
| 7154 | | .seq_cst => .{ true, true }, |
| 7155 | | }; |
| 7184 | const aq, const rl = switch (order) { |
| 7185 | .unordered => unreachable, |
| 7186 | .monotonic => .{ false, false }, |
| 7187 | .acquire => .{ true, false }, |
| 7188 | .release => .{ false, true }, |
| 7189 | .acq_rel => .{ true, true }, |
| 7190 | .seq_cst => .{ true, true }, |
| 7191 | }; |
| 7156 | 7192 | |
| 7157 | | switch (method) { |
| 7158 | | .amo => { |
| 7159 | | const is_d = val_ty.abiSize(pt) == 8; |
| 7160 | | const is_un = val_ty.isUnsignedInt(zcu); |
| 7193 | switch (method) { |
| 7194 | .amo => { |
| 7195 | const is_d = val_ty.abiSize(pt) == 8; |
| 7196 | const is_un = val_ty.isUnsignedInt(zcu); |
| 7161 | 7197 | |
| 7162 | | const mnem: Mnemonic = switch (op) { |
| 7163 | | // zig fmt: off |
| 7198 | const mnem: Mnemonic = switch (op) { |
| 7199 | // zig fmt: off |
| 7164 | 7200 | .Xchg => if (is_d) .amoswapd else .amoswapw, |
| 7165 | 7201 | .Add => if (is_d) .amoaddd else .amoaddw, |
| 7166 | 7202 | .And => if (is_d) .amoandd else .amoandw, |
| ... | ... | @@ -7170,82 +7206,79 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void { |
| 7170 | 7206 | .Min => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw, |
| 7171 | 7207 | else => return func.fail("TODO: airAtomicRmw amo {s}", .{@tagName(op)}), |
| 7172 | 7208 | // zig fmt: on |
| 7173 | | }; |
| 7209 | }; |
| 7174 | 7210 | |
| 7175 | | _ = try func.addInst(.{ |
| 7176 | | .tag = mnem, |
| 7177 | | .data = .{ .amo = .{ |
| 7178 | | .rd = result_reg, |
| 7179 | | .rs1 = ptr_register, |
| 7180 | | .rs2 = val_register, |
| 7181 | | .aq = if (aq) .aq else .none, |
| 7182 | | .rl = if (rl) .rl else .none, |
| 7183 | | } }, |
| 7184 | | }); |
| 7185 | | }, |
| 7186 | | .loop => { |
| 7187 | | // where we'll jump back when the sc fails |
| 7188 | | const jump_back = try func.addInst(.{ |
| 7189 | | .tag = .lrw, |
| 7190 | | .data = .{ .amo = .{ |
| 7191 | | .rd = result_reg, |
| 7192 | | .rs1 = ptr_register, |
| 7193 | | .rs2 = .zero, |
| 7194 | | .aq = if (aq) .aq else .none, |
| 7195 | | .rl = if (rl) .rl else .none, |
| 7196 | | } }, |
| 7197 | | }); |
| 7211 | _ = try func.addInst(.{ |
| 7212 | .tag = mnem, |
| 7213 | .data = .{ .amo = .{ |
| 7214 | .rd = result_reg, |
| 7215 | .rs1 = ptr_register, |
| 7216 | .rs2 = val_register, |
| 7217 | .aq = if (aq) .aq else .none, |
| 7218 | .rl = if (rl) .rl else .none, |
| 7219 | } }, |
| 7220 | }); |
| 7221 | }, |
| 7222 | .loop => { |
| 7223 | // where we'll jump back when the sc fails |
| 7224 | const jump_back = try func.addInst(.{ |
| 7225 | .tag = .lrw, |
| 7226 | .data = .{ .amo = .{ |
| 7227 | .rd = result_reg, |
| 7228 | .rs1 = ptr_register, |
| 7229 | .rs2 = .zero, |
| 7230 | .aq = if (aq) .aq else .none, |
| 7231 | .rl = if (rl) .rl else .none, |
| 7232 | } }, |
| 7233 | }); |
| 7198 | 7234 | |
| 7199 | | const after_reg, const after_lock = try func.allocReg(.int); |
| 7200 | | defer func.register_manager.unlockReg(after_lock); |
| 7235 | const after_reg, const after_lock = try func.allocReg(.int); |
| 7236 | defer func.register_manager.unlockReg(after_lock); |
| 7201 | 7237 | |
| 7202 | | switch (op) { |
| 7203 | | .Add => { |
| 7204 | | _ = try func.genBinOp( |
| 7205 | | .add, |
| 7206 | | .{ .register = result_reg }, |
| 7207 | | val_ty, |
| 7208 | | .{ .register = val_register }, |
| 7209 | | val_ty, |
| 7210 | | after_reg, |
| 7211 | | ); |
| 7212 | | }, |
| 7213 | | .Sub => { |
| 7214 | | _ = try func.genBinOp( |
| 7215 | | .sub, |
| 7216 | | .{ .register = result_reg }, |
| 7217 | | val_ty, |
| 7218 | | .{ .register = val_register }, |
| 7219 | | val_ty, |
| 7220 | | after_reg, |
| 7221 | | ); |
| 7222 | | }, |
| 7223 | | else => return func.fail("TODO: airAtomicRmw loop {s}", .{@tagName(op)}), |
| 7224 | | } |
| 7238 | switch (op) { |
| 7239 | .Add, .Sub => |tag| { |
| 7240 | _ = try func.genBinOp( |
| 7241 | switch (tag) { |
| 7242 | .Add => .add, |
| 7243 | .Sub => .sub, |
| 7244 | else => unreachable, |
| 7245 | }, |
| 7246 | .{ .register = result_reg }, |
| 7247 | val_ty, |
| 7248 | .{ .register = val_register }, |
| 7249 | val_ty, |
| 7250 | after_reg, |
| 7251 | ); |
| 7252 | }, |
| 7225 | 7253 | |
| 7226 | | _ = try func.addInst(.{ |
| 7227 | | .tag = .scw, |
| 7228 | | .data = .{ .amo = .{ |
| 7229 | | .rd = after_reg, |
| 7230 | | .rs1 = ptr_register, |
| 7231 | | .rs2 = after_reg, |
| 7232 | | .aq = if (aq) .aq else .none, |
| 7233 | | .rl = if (rl) .rl else .none, |
| 7234 | | } }, |
| 7235 | | }); |
| 7254 | else => return func.fail("TODO: airAtomicRmw loop {s}", .{@tagName(op)}), |
| 7255 | } |
| 7236 | 7256 | |
| 7237 | | _ = try func.addInst(.{ |
| 7238 | | .tag = .bne, |
| 7239 | | .data = .{ .b_type = .{ |
| 7240 | | .inst = jump_back, |
| 7241 | | .rs1 = after_reg, |
| 7242 | | .rs2 = .zero, |
| 7243 | | } }, |
| 7244 | | }); |
| 7245 | | }, |
| 7246 | | } |
| 7257 | _ = try func.addInst(.{ |
| 7258 | .tag = .scw, |
| 7259 | .data = .{ .amo = .{ |
| 7260 | .rd = after_reg, |
| 7261 | .rs1 = ptr_register, |
| 7262 | .rs2 = after_reg, |
| 7263 | .aq = if (aq) .aq else .none, |
| 7264 | .rl = if (rl) .rl else .none, |
| 7265 | } }, |
| 7266 | }); |
| 7267 | |
| 7268 | _ = try func.addInst(.{ |
| 7269 | .tag = .bne, |
| 7270 | .data = .{ .b_type = .{ |
| 7271 | .inst = jump_back, |
| 7272 | .rs1 = after_reg, |
| 7273 | .rs2 = .zero, |
| 7274 | } }, |
| 7275 | }); |
| 7276 | }, |
| 7277 | } |
| 7278 | break :result result_mcv; |
| 7279 | }; |
| 7247 | 7280 | |
| 7248 | | return func.finishAir(inst, result_mcv, .{ pl_op.operand, extra.operand, .none }); |
| 7281 | return func.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none }); |
| 7249 | 7282 | } |
| 7250 | 7283 | |
| 7251 | 7284 | fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void { |
David Rubin <david@vortan.dev> 2024-07-26 02:28:04-07:00
Commit is signed but in an unrecognized format.
src