authorgravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2024-07-26 02:28:04-07:00
committergravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2024-07-26 04:19:57-07:00
log046001a34a070dffa6230ccbd8f1571e0b1c1240
tree897d17068a42eb2da3a05ce4a5994e15a55c9c45
parent7ff5709e1b04de4c33988ce6a27bc593dcf20f63
signaturelock-open Commit is signed but in an unrecognized format.

riscv implement `@popCount`

4 files changed, 158 insertions(+), 121 deletions(-)

src/arch/riscv64/CodeGen.zig+152-119
...@@ -2844,6 +2844,9 @@ fn genBinOp(...@@ -2844,6 +2844,9 @@ fn genBinOp(
2844 .cmp_gt,2844 .cmp_gt,
2845 .cmp_gte,2845 .cmp_gte,
2846 => {2846 => {
2847 try func.truncateRegister(lhs_ty, lhs_reg);
2848 try func.truncateRegister(rhs_ty, rhs_reg);
2849
2847 _ = try func.addInst(.{2850 _ = try func.addInst(.{
2848 .tag = .pseudo_compare,2851 .tag = .pseudo_compare,
2849 .data = .{2852 .data = .{
...@@ -3925,10 +3928,6 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void {...@@ -3925,10 +3928,6 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void {
3925 const base_ptr_ty = func.typeOf(extra.lhs);3928 const base_ptr_ty = func.typeOf(extra.lhs);
39263929
3927 if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) {3930 if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) {
3928 // break :result if (func.reuseOperand(inst, extra.lhs, 0, base_ptr_mcv))
3929 // base_ptr_mcv
3930 // else
3931 // try func.copyToNewRegister(inst, base_ptr_mcv);
3932 @panic("audit");3931 @panic("audit");
3933 }3932 }
39343933
...@@ -3992,7 +3991,7 @@ fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void {...@@ -3992,7 +3991,7 @@ fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void {
3992 defer func.register_manager.unlockReg(result_lock);3991 defer func.register_manager.unlockReg(result_lock);
39933992
3994 switch (frame_mcv) {3993 switch (frame_mcv) {
3995 .load_frame => |frame_addr| {3994 .load_frame => {
3996 if (tag_abi_size <= 8) {3995 if (tag_abi_size <= 8) {
3997 const off: i32 = if (layout.tag_align.compare(.lt, layout.payload_align))3996 const off: i32 = if (layout.tag_align.compare(.lt, layout.payload_align))
3998 @intCast(layout.payload_size)3997 @intCast(layout.payload_size)
...@@ -4002,7 +4001,7 @@ fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void {...@@ -4002,7 +4001,7 @@ fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void {
4002 try func.genCopy(4001 try func.genCopy(
4003 tag_ty,4002 tag_ty,
4004 .{ .register = result_reg },4003 .{ .register = result_reg },
4005 .{ .load_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off } },4004 frame_mcv.offset(off),
4006 );4005 );
4007 } else {4006 } else {
4008 return func.fail(4007 return func.fail(
...@@ -4081,7 +4080,37 @@ fn airCtz(func: *Func, inst: Air.Inst.Index) !void {...@@ -4081,7 +4080,37 @@ fn airCtz(func: *Func, inst: Air.Inst.Index) !void {
40814080
4082fn airPopcount(func: *Func, inst: Air.Inst.Index) !void {4081fn airPopcount(func: *Func, inst: Air.Inst.Index) !void {
4083 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4082 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4084 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airPopcount for {}", .{func.target.cpu.arch});4083 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4084 const pt = func.pt;
4085
4086 const operand = try func.resolveInst(ty_op.operand);
4087 const src_ty = func.typeOf(ty_op.operand);
4088 const operand_reg, const operand_lock = try func.promoteReg(src_ty, operand);
4089 defer if (operand_lock) |lock| func.register_manager.unlockReg(lock);
4090
4091 const dst_reg, const dst_lock = try func.allocReg(.int);
4092 defer func.register_manager.unlockReg(dst_lock);
4093
4094 const bit_size = src_ty.bitSize(pt);
4095 switch (bit_size) {
4096 32, 64 => {},
4097 1...31, 33...63 => try func.truncateRegister(src_ty, operand_reg),
4098 else => return func.fail("TODO: airPopcount > 64 bits", .{}),
4099 }
4100
4101 _ = try func.addInst(.{
4102 .tag = if (bit_size <= 32) .cpopw else .cpop,
4103 .data = .{
4104 .r_type = .{
4105 .rd = dst_reg,
4106 .rs1 = operand_reg,
4107 .rs2 = @enumFromInt(0b00010), // this is the cpop funct5
4108 },
4109 },
4110 });
4111
4112 break :result .{ .register = dst_reg };
4113 };
4085 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });4114 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4086}4115}
40874116
...@@ -5630,7 +5659,6 @@ fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !...@@ -5630,7 +5659,6 @@ fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
56305659
5631fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {5660fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5632 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;5661 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5633 const condition = try func.resolveInst(pl_op.operand);
5634 const condition_ty = func.typeOf(pl_op.operand);5662 const condition_ty = func.typeOf(pl_op.operand);
5635 const switch_br = func.air.extraData(Air.SwitchBr, pl_op.payload);5663 const switch_br = func.air.extraData(Air.SwitchBr, pl_op.payload);
5636 var extra_index: usize = switch_br.end;5664 var extra_index: usize = switch_br.end;
...@@ -5638,6 +5666,8 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {...@@ -5638,6 +5666,8 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5638 const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.data.cases_len + 1);5666 const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.data.cases_len + 1);
5639 defer func.gpa.free(liveness.deaths);5667 defer func.gpa.free(liveness.deaths);
56405668
5669 const condition = try func.resolveInst(pl_op.operand);
5670
5641 // If the condition dies here in this switch instruction, process5671 // If the condition dies here in this switch instruction, process
5642 // that death now instead of later as this has an effect on5672 // that death now instead of later as this has an effect on
5643 // whether it needs to be spilled in the branches5673 // whether it needs to be spilled in the branches
...@@ -5660,9 +5690,14 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {...@@ -5660,9 +5690,14 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5660 defer func.gpa.free(relocs);5690 defer func.gpa.free(relocs);
56615691
5662 for (items, relocs, 0..) |item, *reloc, i| {5692 for (items, relocs, 0..) |item, *reloc, i| {
5663 // switch branches must be comptime-known, so this is stored in an immediate
5664 const item_mcv = try func.resolveInst(item);5693 const item_mcv = try func.resolveInst(item);
56655694
5695 const cond_lock = switch (condition) {
5696 .register => func.register_manager.lockRegAssumeUnused(condition.register),
5697 else => null,
5698 };
5699 defer if (cond_lock) |lock| func.register_manager.unlockReg(lock);
5700
5666 const cmp_reg, const cmp_lock = try func.allocReg(.int);5701 const cmp_reg, const cmp_lock = try func.allocReg(.int);
5667 defer func.register_manager.unlockReg(cmp_lock);5702 defer func.register_manager.unlockReg(cmp_lock);
56685703
...@@ -7110,57 +7145,58 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {...@@ -7110,57 +7145,58 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
7110 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;7145 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7111 const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data;7146 const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data;
71127147
7113 const op = extra.op();7148 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
7114 const order = extra.ordering();7149 const op = extra.op();
7150 const order = extra.ordering();
71157151
7116 const ptr_ty = func.typeOf(pl_op.operand);7152 const ptr_ty = func.typeOf(pl_op.operand);
7117 const ptr_mcv = try func.resolveInst(pl_op.operand);7153 const ptr_mcv = try func.resolveInst(pl_op.operand);
71187154
7119 const val_ty = func.typeOf(extra.operand);7155 const val_ty = func.typeOf(extra.operand);
7120 const val_size = val_ty.abiSize(pt);7156 const val_size = val_ty.abiSize(pt);
7121 const val_mcv = try func.resolveInst(extra.operand);7157 const val_mcv = try func.resolveInst(extra.operand);
71227158
7123 if (!math.isPowerOfTwo(val_size))7159 if (!math.isPowerOfTwo(val_size))
7124 return func.fail("TODO: airAtomicRmw non-pow 2", .{});7160 return func.fail("TODO: airAtomicRmw non-pow 2", .{});
71257161
7126 switch (val_ty.zigTypeTag(pt.zcu)) {7162 switch (val_ty.zigTypeTag(pt.zcu)) {
7127 .Int => {},7163 .Int => {},
7128 inline .Bool, .Float, .Enum, .Pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}),7164 inline .Bool, .Float, .Enum, .Pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}),
7129 else => unreachable,7165 else => unreachable,
7130 }7166 }
71317167
7132 const method: enum { amo, loop } = switch (val_size) {7168 const method: enum { amo, loop } = switch (val_size) {
7133 1, 2 => .loop,7169 1, 2 => .loop,
7134 4, 8 => .amo,7170 4, 8 => .amo,
7135 else => unreachable,7171 else => unreachable,
7136 };7172 };
71377173
7138 const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv);7174 const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv);
7139 defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock);7175 defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock);
71407176
7141 const val_register, const val_lock = try func.promoteReg(val_ty, val_mcv);7177 const val_register, const val_lock = try func.promoteReg(val_ty, val_mcv);
7142 defer if (val_lock) |lock| func.register_manager.unlockReg(lock);7178 defer if (val_lock) |lock| func.register_manager.unlockReg(lock);
71437179
7144 const result_mcv = try func.allocRegOrMem(val_ty, inst, true);7180 const result_mcv = try func.allocRegOrMem(val_ty, inst, true);
7145 assert(result_mcv == .register); // should fit into 8 bytes7181 assert(result_mcv == .register); // should fit into 8 bytes
7146 const result_reg = result_mcv.register;7182 const result_reg = result_mcv.register;
71477183
7148 const aq, const rl = switch (order) {7184 const aq, const rl = switch (order) {
7149 .unordered => unreachable,7185 .unordered => unreachable,
7150 .monotonic => .{ false, false },7186 .monotonic => .{ false, false },
7151 .acquire => .{ true, false },7187 .acquire => .{ true, false },
7152 .release => .{ false, true },7188 .release => .{ false, true },
7153 .acq_rel => .{ true, true },7189 .acq_rel => .{ true, true },
7154 .seq_cst => .{ true, true },7190 .seq_cst => .{ true, true },
7155 };7191 };
71567192
7157 switch (method) {7193 switch (method) {
7158 .amo => {7194 .amo => {
7159 const is_d = val_ty.abiSize(pt) == 8;7195 const is_d = val_ty.abiSize(pt) == 8;
7160 const is_un = val_ty.isUnsignedInt(zcu);7196 const is_un = val_ty.isUnsignedInt(zcu);
71617197
7162 const mnem: Mnemonic = switch (op) {7198 const mnem: Mnemonic = switch (op) {
7163 // zig fmt: off7199 // zig fmt: off
7164 .Xchg => if (is_d) .amoswapd else .amoswapw,7200 .Xchg => if (is_d) .amoswapd else .amoswapw,
7165 .Add => if (is_d) .amoaddd else .amoaddw,7201 .Add => if (is_d) .amoaddd else .amoaddw,
7166 .And => if (is_d) .amoandd else .amoandw,7202 .And => if (is_d) .amoandd else .amoandw,
...@@ -7170,82 +7206,79 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {...@@ -7170,82 +7206,79 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
7170 .Min => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw,7206 .Min => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw,
7171 else => return func.fail("TODO: airAtomicRmw amo {s}", .{@tagName(op)}),7207 else => return func.fail("TODO: airAtomicRmw amo {s}", .{@tagName(op)}),
7172 // zig fmt: on7208 // zig fmt: on
7173 };7209 };
71747210
7175 _ = try func.addInst(.{7211 _ = try func.addInst(.{
7176 .tag = mnem,7212 .tag = mnem,
7177 .data = .{ .amo = .{7213 .data = .{ .amo = .{
7178 .rd = result_reg,7214 .rd = result_reg,
7179 .rs1 = ptr_register,7215 .rs1 = ptr_register,
7180 .rs2 = val_register,7216 .rs2 = val_register,
7181 .aq = if (aq) .aq else .none,7217 .aq = if (aq) .aq else .none,
7182 .rl = if (rl) .rl else .none,7218 .rl = if (rl) .rl else .none,
7183 } },7219 } },
7184 });7220 });
7185 },7221 },
7186 .loop => {7222 .loop => {
7187 // where we'll jump back when the sc fails7223 // where we'll jump back when the sc fails
7188 const jump_back = try func.addInst(.{7224 const jump_back = try func.addInst(.{
7189 .tag = .lrw,7225 .tag = .lrw,
7190 .data = .{ .amo = .{7226 .data = .{ .amo = .{
7191 .rd = result_reg,7227 .rd = result_reg,
7192 .rs1 = ptr_register,7228 .rs1 = ptr_register,
7193 .rs2 = .zero,7229 .rs2 = .zero,
7194 .aq = if (aq) .aq else .none,7230 .aq = if (aq) .aq else .none,
7195 .rl = if (rl) .rl else .none,7231 .rl = if (rl) .rl else .none,
7196 } },7232 } },
7197 });7233 });
71987234
7199 const after_reg, const after_lock = try func.allocReg(.int);7235 const after_reg, const after_lock = try func.allocReg(.int);
7200 defer func.register_manager.unlockReg(after_lock);7236 defer func.register_manager.unlockReg(after_lock);
72017237
7202 switch (op) {7238 switch (op) {
7203 .Add => {7239 .Add, .Sub => |tag| {
7204 _ = try func.genBinOp(7240 _ = try func.genBinOp(
7205 .add,7241 switch (tag) {
7206 .{ .register = result_reg },7242 .Add => .add,
7207 val_ty,7243 .Sub => .sub,
7208 .{ .register = val_register },7244 else => unreachable,
7209 val_ty,7245 },
7210 after_reg,7246 .{ .register = result_reg },
7211 );7247 val_ty,
7212 },7248 .{ .register = val_register },
7213 .Sub => {7249 val_ty,
7214 _ = try func.genBinOp(7250 after_reg,
7215 .sub,7251 );
7216 .{ .register = result_reg },7252 },
7217 val_ty,
7218 .{ .register = val_register },
7219 val_ty,
7220 after_reg,
7221 );
7222 },
7223 else => return func.fail("TODO: airAtomicRmw loop {s}", .{@tagName(op)}),
7224 }
72257253
7226 _ = try func.addInst(.{7254 else => return func.fail("TODO: airAtomicRmw loop {s}", .{@tagName(op)}),
7227 .tag = .scw,7255 }
7228 .data = .{ .amo = .{
7229 .rd = after_reg,
7230 .rs1 = ptr_register,
7231 .rs2 = after_reg,
7232 .aq = if (aq) .aq else .none,
7233 .rl = if (rl) .rl else .none,
7234 } },
7235 });
72367256
7237 _ = try func.addInst(.{7257 _ = try func.addInst(.{
7238 .tag = .bne,7258 .tag = .scw,
7239 .data = .{ .b_type = .{7259 .data = .{ .amo = .{
7240 .inst = jump_back,7260 .rd = after_reg,
7241 .rs1 = after_reg,7261 .rs1 = ptr_register,
7242 .rs2 = .zero,7262 .rs2 = after_reg,
7243 } },7263 .aq = if (aq) .aq else .none,
7244 });7264 .rl = if (rl) .rl else .none,
7245 },7265 } },
7246 }7266 });
7267
7268 _ = try func.addInst(.{
7269 .tag = .bne,
7270 .data = .{ .b_type = .{
7271 .inst = jump_back,
7272 .rs1 = after_reg,
7273 .rs2 = .zero,
7274 } },
7275 });
7276 },
7277 }
7278 break :result result_mcv;
7279 };
72477280
7248 return func.finishAir(inst, result_mcv, .{ pl_op.operand, extra.operand, .none });7281 return func.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
7249}7282}
72507283
7251fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {7284fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
src/arch/riscv64/abi.zig+1-1
...@@ -164,7 +164,7 @@ pub fn classifySystem(ty: Type, pt: Zcu.PerThread) [8]SystemClass {...@@ -164,7 +164,7 @@ pub fn classifySystem(ty: Type, pt: Zcu.PerThread) [8]SystemClass {
164164
165 return memory_class;165 return memory_class;
166 },166 },
167 .Struct => {167 .Struct, .Union => {
168 const layout = ty.containerLayout(pt.zcu);168 const layout = ty.containerLayout(pt.zcu);
169 const ty_size = ty.abiSize(pt);169 const ty_size = ty.abiSize(pt);
170170
src/arch/riscv64/encoding.zig+3-1
...@@ -206,6 +206,7 @@ pub const Lir = struct {...@@ -206,6 +206,7 @@ pub const Lir = struct {
206 .srai => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .sh = .{ .typ = 0b010000, .funct3 = 0b101, .has_5 = true } } },206 .srai => .{ .opcode = .OP_IMM, .format = .I, .data = .{ .sh = .{ .typ = 0b010000, .funct3 = 0b101, .has_5 = true } } },
207207
208 .clz => .{ .opcode = .OP_IMM, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } },208 .clz => .{ .opcode = .OP_IMM, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } },
209 .cpop => .{ .opcode = .OP_IMM, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } },
209210
210 // OP_IMM_32211 // OP_IMM_32
211212
...@@ -213,7 +214,8 @@ pub const Lir = struct {...@@ -213,7 +214,8 @@ pub const Lir = struct {
213 .srliw => .{ .opcode = .OP_IMM_32, .format = .I, .data = .{ .sh = .{ .typ = 0b000000, .funct3 = 0b101, .has_5 = false } } },214 .srliw => .{ .opcode = .OP_IMM_32, .format = .I, .data = .{ .sh = .{ .typ = 0b000000, .funct3 = 0b101, .has_5 = false } } },
214 .sraiw => .{ .opcode = .OP_IMM_32, .format = .I, .data = .{ .sh = .{ .typ = 0b010000, .funct3 = 0b101, .has_5 = false } } },215 .sraiw => .{ .opcode = .OP_IMM_32, .format = .I, .data = .{ .sh = .{ .typ = 0b010000, .funct3 = 0b101, .has_5 = false } } },
215216
216 .clzw => .{ .opcode = .OP_IMM_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } },217 .clzw => .{ .opcode = .OP_IMM_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } },
218 .cpopw => .{ .opcode = .OP_IMM_32, .format = .R, .data = .{ .ff = .{ .funct3 = 0b001, .funct7 = 0b0110000 } } },
217219
218 // OP_32220 // OP_32
219221
src/arch/riscv64/mnem.zig+2
...@@ -174,6 +174,8 @@ pub const Mnemonic = enum(u16) {...@@ -174,6 +174,8 @@ pub const Mnemonic = enum(u16) {
174 // Zbb Extension Instructions174 // Zbb Extension Instructions
175 clz,175 clz,
176 clzw,176 clzw,
177 cpop,
178 cpopw,
177179
178 // A Extension Instructions180 // A Extension Instructions
179 fence,181 fence,