authorgravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2024-05-28 22:10:51-07:00
committergravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2024-07-14 23:02:29-07:00
logea084e9519a8e6f14d1bcb5f1fb2cddf842333b6
tree58ed4b874791547a05c37d128a41096e7aff39fc
parentd404d8a3637bc30dffc736e5fa1a68b8af0e19cb
signaturelock-open Commit is signed but in an unrecognized format.

riscv: `@atomicLoad` and `@atomicStore`

5 files changed, 221 insertions(+), 50 deletions(-)

src/arch/riscv64/CodeGen.zig+119-24
...@@ -2087,19 +2087,17 @@ fn airNot(func: *Func, inst: Air.Inst.Index) !void {...@@ -2087,19 +2087,17 @@ fn airNot(func: *Func, inst: Air.Inst.Index) !void {
2087 const operand = try func.resolveInst(ty_op.operand);2087 const operand = try func.resolveInst(ty_op.operand);
2088 const ty = func.typeOf(ty_op.operand);2088 const ty = func.typeOf(ty_op.operand);
20892089
2090 switch (ty.zigTypeTag(zcu)) {2090 const operand_reg, const operand_lock = try func.promoteReg(ty, operand);
2091 .Bool => {2091 defer if (operand_lock) |lock| func.register_manager.unlockReg(lock);
2092 const operand_reg = blk: {
2093 if (operand == .register) break :blk operand.register;
2094 break :blk try func.copyToTmpRegister(ty, operand);
2095 };
20962092
2097 const dst_reg: Register =2093 const dst_reg: Register =
2098 if (func.reuseOperand(inst, ty_op.operand, 0, operand) and operand == .register)2094 if (func.reuseOperand(inst, ty_op.operand, 0, operand) and operand == .register)
2099 operand.register2095 operand.register
2100 else2096 else
2101 (try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register;2097 (try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register;
21022098
2099 switch (ty.zigTypeTag(zcu)) {
2100 .Bool => {
2103 _ = try func.addInst(.{2101 _ = try func.addInst(.{
2104 .tag = .pseudo,2102 .tag = .pseudo,
2105 .ops = .pseudo_not,2103 .ops = .pseudo_not,
...@@ -2110,12 +2108,34 @@ fn airNot(func: *Func, inst: Air.Inst.Index) !void {...@@ -2110,12 +2108,34 @@ fn airNot(func: *Func, inst: Air.Inst.Index) !void {
2110 },2108 },
2111 },2109 },
2112 });2110 });
2111 },
2112 .Int => {
2113 const size = ty.bitSize(zcu);
2114 if (!math.isPowerOfTwo(size))
2115 return func.fail("TODO: airNot non-pow 2 int size", .{});
21132116
2114 break :result .{ .register = dst_reg };2117 switch (size) {
2118 32, 64 => {
2119 _ = try func.addInst(.{
2120 .tag = .xori,
2121 .ops = .rri,
2122 .data = .{
2123 .i_type = .{
2124 .rd = dst_reg,
2125 .rs1 = operand_reg,
2126 .imm12 = Immediate.s(-1),
2127 },
2128 },
2129 });
2130 },
2131 8, 16 => return func.fail("TODO: airNot 8 or 16, {}", .{size}),
2132 else => unreachable,
2133 }
2115 },2134 },
2116 .Int => return func.fail("TODO: airNot ints", .{}),
2117 else => unreachable,2135 else => unreachable,
2118 }2136 }
2137
2138 break :result .{ .register = dst_reg };
2119 };2139 };
2120 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });2140 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2121}2141}
...@@ -5600,17 +5620,24 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError!...@@ -5600,17 +5620,24 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
5600 const abi_size: u32 = @intCast(ty.abiSize(pt));5620 const abi_size: u32 = @intCast(ty.abiSize(pt));
56015621
5602 if (abi_size > 8) return std.debug.panic("tried to set reg with size {}", .{abi_size});5622 if (abi_size > 8) return std.debug.panic("tried to set reg with size {}", .{abi_size});
5603
5604 const dst_reg_class = reg.class();5623 const dst_reg_class = reg.class();
56055624
5606 switch (src_mcv) {5625 switch (src_mcv) {
5607 .dead => unreachable,5626 .unreach,
5608 .unreach, .none => return, // Nothing to do.5627 .none,
5628 .dead,
5629 => unreachable,
5609 .undef => {5630 .undef => {
5610 if (!func.wantSafety())5631 if (!func.wantSafety())
5611 return; // The already existing value will do just fine.5632 return;
5612 // Write the debug undefined value.5633
5613 return func.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa });5634 switch (abi_size) {
5635 1 => return func.genSetReg(ty, reg, .{ .immediate = 0xAA }),
5636 2 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAA }),
5637 3...4 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAA }),
5638 5...8 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAAAAAAAAAA }),
5639 else => unreachable,
5640 }
5614 },5641 },
5615 .immediate => |unsigned_x| {5642 .immediate => |unsigned_x| {
5616 assert(dst_reg_class == .int);5643 assert(dst_reg_class == .int);
...@@ -6047,14 +6074,82 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {...@@ -6047,14 +6074,82 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
6047}6074}
60486075
6049fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {6076fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
6050 _ = inst;6077 const zcu = func.bin_file.comp.module.?;
6051 return func.fail("TODO implement airAtomicLoad for {}", .{func.target.cpu.arch});6078 const atomic_load = func.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
6079 const order: std.builtin.AtomicOrder = atomic_load.order;
6080
6081 const ptr_ty = func.typeOf(atomic_load.ptr);
6082 const elem_ty = ptr_ty.childType(zcu);
6083 const ptr_mcv = try func.resolveInst(atomic_load.ptr);
6084
6085 const result_mcv = try func.allocRegOrMem(elem_ty, inst, true);
6086
6087 if (order == .seq_cst) {
6088 _ = try func.addInst(.{
6089 .tag = .fence,
6090 .ops = .fence,
6091 .data = .{
6092 .fence = .{
6093 .pred = .rw,
6094 .succ = .rw,
6095 },
6096 },
6097 });
6098 }
6099
6100 try func.load(result_mcv, ptr_mcv, ptr_ty);
6101
6102 switch (order) {
6103 // Don't guarnetee other memory operations to be ordered after the load.
6104 .unordered => {},
6105 .monotonic => {},
6106 // Make sure all previous reads happen before any reading or writing accurs.
6107 .seq_cst, .acquire => {
6108 _ = try func.addInst(.{
6109 .tag = .fence,
6110 .ops = .fence,
6111 .data = .{
6112 .fence = .{
6113 .pred = .r,
6114 .succ = .rw,
6115 },
6116 },
6117 });
6118 },
6119 else => unreachable,
6120 }
6121
6122 return func.finishAir(inst, result_mcv, .{ atomic_load.ptr, .none, .none });
6052}6123}
60536124
6054fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {6125fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
6055 _ = inst;6126 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6056 _ = order;6127
6057 return func.fail("TODO implement airAtomicStore for {}", .{func.target.cpu.arch});6128 const ptr_ty = func.typeOf(bin_op.lhs);
6129 const ptr_mcv = try func.resolveInst(bin_op.lhs);
6130
6131 const val_ty = func.typeOf(bin_op.rhs);
6132 const val_mcv = try func.resolveInst(bin_op.rhs);
6133
6134 switch (order) {
6135 .unordered, .monotonic => {},
6136 .release, .seq_cst => {
6137 _ = try func.addInst(.{
6138 .tag = .fence,
6139 .ops = .fence,
6140 .data = .{
6141 .fence = .{
6142 .pred = .rw,
6143 .succ = .w,
6144 },
6145 },
6146 });
6147 },
6148 else => unreachable,
6149 }
6150
6151 try func.store(ptr_mcv, val_mcv, ptr_ty, val_ty);
6152 return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
6058}6153}
60596154
6060fn airMemset(func: *Func, inst: Air.Inst.Index, safety: bool) !void {6155fn airMemset(func: *Func, inst: Air.Inst.Index, safety: bool) !void {
src/arch/riscv64/Encoding.zig+67-23
...@@ -2,25 +2,30 @@ mnemonic: Mnemonic,...@@ -2,25 +2,30 @@ mnemonic: Mnemonic,
2data: Data,2data: Data,
33
4const OpCode = enum(u7) {4const OpCode = enum(u7) {
5 OP = 0b0110011,5 LOAD = 0b0000011,
6 LOAD_FP = 0b0000111,
7 MISC_MEM = 0b0001111,
6 OP_IMM = 0b0010011,8 OP_IMM = 0b0010011,
9 AUIPC = 0b0010111,
7 OP_IMM_32 = 0b0011011,10 OP_IMM_32 = 0b0011011,
8 OP_32 = 0b0111011,
9
10 BRANCH = 0b1100011,
11 LOAD = 0b0000011,
12 STORE = 0b0100011,11 STORE = 0b0100011,
13 SYSTEM = 0b1110011,
14
15 OP_FP = 0b1010011,
16 LOAD_FP = 0b0000111,
17 STORE_FP = 0b0100111,12 STORE_FP = 0b0100111,
1813 AMO = 0b0101111,
19 JALR = 0b1100111,14 OP = 0b0110011,
20 AUIPC = 0b0010111,15 OP_32 = 0b0111011,
21 LUI = 0b0110111,16 LUI = 0b0110111,
17 MADD = 0b1000011,
18 MSUB = 0b1000111,
19 NMSUB = 0b1001011,
20 NMADD = 0b1001111,
21 OP_FP = 0b1010011,
22 OP_IMM_64 = 0b1011011,
23 BRANCH = 0b1100011,
24 JALR = 0b1100111,
22 JAL = 0b1101111,25 JAL = 0b1101111,
23 NONE = 0b0000000,26 SYSTEM = 0b1110011,
27 OP_64 = 0b1111011,
28 NONE = 0b00000000,
24};29};
2530
26const Fmt = enum(u2) {31const Fmt = enum(u2) {
...@@ -28,7 +33,9 @@ const Fmt = enum(u2) {...@@ -28,7 +33,9 @@ const Fmt = enum(u2) {
28 S = 0b00,33 S = 0b00,
29 /// 64-bit double-precision34 /// 64-bit double-precision
30 D = 0b01,35 D = 0b01,
31 _reserved = 0b10,36
37 // H = 0b10, unused in the G extension
38
32 /// 128-bit quad-precision39 /// 128-bit quad-precision
33 Q = 0b11,40 Q = 0b11,
34};41};
...@@ -192,6 +199,9 @@ pub const Mnemonic = enum {...@@ -192,6 +199,9 @@ pub const Mnemonic = enum {
192 fsgnjnd,199 fsgnjnd,
193 fsgnjxd,200 fsgnjxd,
194201
202 // MISC
203 fence,
204
195 pub fn encoding(mnem: Mnemonic) Enc {205 pub fn encoding(mnem: Mnemonic) Enc {
196 return switch (mnem) {206 return switch (mnem) {
197 // zig fmt: off207 // zig fmt: off
...@@ -366,6 +376,10 @@ pub const Mnemonic = enum {...@@ -366,6 +376,10 @@ pub const Mnemonic = enum {
366 376
367 .unimp => .{ .opcode = .NONE, .data = .{ .f = .{ .funct3 = 0b000 } } },377 .unimp => .{ .opcode = .NONE, .data = .{ .f = .{ .funct3 = 0b000 } } },
368378
379 // MISC_MEM
380
381 .fence => .{ .opcode = .MISC_MEM, .data = .{ .f = .{ .funct3 = 0b000 } } },
382
369383
370 // zig fmt: on384 // zig fmt: on
371 };385 };
...@@ -380,7 +394,7 @@ pub const InstEnc = enum {...@@ -380,7 +394,7 @@ pub const InstEnc = enum {
380 B,394 B,
381 U,395 U,
382 J,396 J,
383397 fence,
384 /// extras that have unusual op counts398 /// extras that have unusual op counts
385 system,399 system,
386400
...@@ -509,20 +523,24 @@ pub const InstEnc = enum {...@@ -509,20 +523,24 @@ pub const InstEnc = enum {
509 .ebreak,523 .ebreak,
510 .unimp,524 .unimp,
511 => .system,525 => .system,
526
527 .fence,
528 => .fence,
512 };529 };
513 }530 }
514531
515 pub fn opsList(enc: InstEnc) [4]std.meta.FieldEnum(Operand) {532 pub fn opsList(enc: InstEnc) [4]std.meta.FieldEnum(Operand) {
516 return switch (enc) {533 return switch (enc) {
517 // zig fmt: off534 // zig fmt: off
518 .R => .{ .reg, .reg, .reg, .none },535 .R => .{ .reg, .reg, .reg, .none },
519 .R4 => .{ .reg, .reg, .reg, .reg }, 536 .R4 => .{ .reg, .reg, .reg, .reg },
520 .I => .{ .reg, .reg, .imm, .none },537 .I => .{ .reg, .reg, .imm, .none },
521 .S => .{ .reg, .reg, .imm, .none },538 .S => .{ .reg, .reg, .imm, .none },
522 .B => .{ .reg, .reg, .imm, .none },539 .B => .{ .reg, .reg, .imm, .none },
523 .U => .{ .reg, .imm, .none, .none },540 .U => .{ .reg, .imm, .none, .none },
524 .J => .{ .reg, .imm, .none, .none },541 .J => .{ .reg, .imm, .none, .none },
525 .system => .{ .none, .none, .none, .none },542 .system => .{ .none, .none, .none, .none },
543 .fence => .{ .barrier, .barrier, .none, .none },
526 // zig fmt: on544 // zig fmt: on
527 };545 };
528 }546 }
...@@ -584,6 +602,15 @@ pub const Data = union(InstEnc) {...@@ -584,6 +602,15 @@ pub const Data = union(InstEnc) {
584 imm1_10: u10,602 imm1_10: u10,
585 imm20: u1,603 imm20: u1,
586 },604 },
605 fence: packed struct {
606 opcode: u7,
607 rd: u5 = 0,
608 funct3: u3,
609 rs1: u5 = 0,
610 succ: u4,
611 pred: u4,
612 _ignored: u4 = 0,
613 },
587 system: void,614 system: void,
588615
589 pub fn toU32(self: Data) u32 {616 pub fn toU32(self: Data) u32 {
...@@ -596,6 +623,7 @@ pub const Data = union(InstEnc) {...@@ -596,6 +623,7 @@ pub const Data = union(InstEnc) {
596 .B => |v| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.imm11)) << 7) + (@as(u32, @intCast(v.imm1_4)) << 8) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.rs2)) << 20) + (@as(u32, @intCast(v.imm5_10)) << 25) + (@as(u32, @intCast(v.imm12)) << 31),623 .B => |v| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.imm11)) << 7) + (@as(u32, @intCast(v.imm1_4)) << 8) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.rs2)) << 20) + (@as(u32, @intCast(v.imm5_10)) << 25) + (@as(u32, @intCast(v.imm12)) << 31),
597 .U => |v| @bitCast(v),624 .U => |v| @bitCast(v),
598 .J => |v| @bitCast(v),625 .J => |v| @bitCast(v),
626 .fence => |v| @bitCast(v),
599 .system => unreachable,627 .system => unreachable,
600 // zig fmt: on628 // zig fmt: on
601 };629 };
...@@ -748,6 +776,22 @@ pub const Data = union(InstEnc) {...@@ -748,6 +776,22 @@ pub const Data = union(InstEnc) {
748 },776 },
749 };777 };
750 },778 },
779 .fence => {
780 assert(ops.len == 2);
781
782 const succ = ops[0];
783 const pred = ops[1];
784
785 return .{
786 .fence = .{
787 .succ = @intFromEnum(succ.barrier),
788 .pred = @intFromEnum(pred.barrier),
789
790 .opcode = @intFromEnum(enc.opcode),
791 .funct3 = enc.data.f.funct3,
792 },
793 };
794 },
751795
752 else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}),796 else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}),
753 }797 }
src/arch/riscv64/Lower.zig+12-1
...@@ -378,7 +378,14 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -378,7 +378,14 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
378 const rr = inst.data.rr;378 const rr = inst.data.rr;
379 assert(rr.rs.class() == .int and rr.rd.class() == .int);379 assert(rr.rs.class() == .int and rr.rd.class() == .int);
380380
381 try lower.emit(.xori, &.{381 // mask out any other bits that aren't the boolean
382 try lower.emit(.andi, &.{
383 .{ .reg = rr.rs },
384 .{ .reg = rr.rs },
385 .{ .imm = Immediate.s(1) },
386 });
387
388 try lower.emit(.sltiu, &.{
382 .{ .reg = rr.rd },389 .{ .reg = rr.rd },
383 .{ .reg = rr.rs },390 .{ .reg = rr.rs },
384 .{ .imm = Immediate.s(1) },391 .{ .imm = Immediate.s(1) },
...@@ -447,6 +454,10 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -447,6 +454,10 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
447 .{ .reg = inst.data.r_type.rs1 },454 .{ .reg = inst.data.r_type.rs1 },
448 .{ .reg = inst.data.r_type.rs2 },455 .{ .reg = inst.data.r_type.rs2 },
449 },456 },
457 .fence => &.{
458 .{ .barrier = inst.data.fence.succ },
459 .{ .barrier = inst.data.fence.pred },
460 },
450 else => return lower.fail("TODO: generic lower ops {s}", .{@tagName(inst.ops)}),461 else => return lower.fail("TODO: generic lower ops {s}", .{@tagName(inst.ops)}),
451 });462 });
452}463}
src/arch/riscv64/Mir.zig+19
...@@ -31,6 +31,7 @@ pub const Inst = struct {...@@ -31,6 +31,7 @@ pub const Inst = struct {
31 @"and",31 @"and",
32 andi,32 andi,
3333
34 xori,
34 xor,35 xor,
35 @"or",36 @"or",
3637
...@@ -38,6 +39,8 @@ pub const Inst = struct {...@@ -38,6 +39,8 @@ pub const Inst = struct {
38 ecall,39 ecall,
39 unimp,40 unimp,
4041
42 fence,
43
41 add,44 add,
42 addw,45 addw,
43 sub,46 sub,
...@@ -246,6 +249,11 @@ pub const Inst = struct {...@@ -246,6 +249,11 @@ pub const Inst = struct {
246 atom_index: u32,249 atom_index: u32,
247 sym_index: u32,250 sym_index: u32,
248 },251 },
252
253 fence: struct {
254 pred: Barrier,
255 succ: Barrier,
256 },
249 };257 };
250258
251 pub const Ops = enum {259 pub const Ops = enum {
...@@ -326,10 +334,15 @@ pub const Inst = struct {...@@ -326,10 +334,15 @@ pub const Inst = struct {
326 pseudo_spill_regs,334 pseudo_spill_regs,
327335
328 pseudo_compare,336 pseudo_compare,
337
338 /// NOT operation on booleans. Does an `andi reg, reg, 1` to mask out any other bits from the boolean.
329 pseudo_not,339 pseudo_not,
330340
331 /// Generates an auipc + jalr pair, with a R_RISCV_CALL_PLT reloc341 /// Generates an auipc + jalr pair, with a R_RISCV_CALL_PLT reloc
332 pseudo_extern_fn_reloc,342 pseudo_extern_fn_reloc,
343
344 /// IORW, IORW
345 fence,
333 };346 };
334347
335 // Make sure we don't accidentally make instructions bigger than expected.348 // Make sure we don't accidentally make instructions bigger than expected.
...@@ -365,6 +378,12 @@ pub const FrameLoc = struct {...@@ -365,6 +378,12 @@ pub const FrameLoc = struct {
365 disp: i32,378 disp: i32,
366};379};
367380
381pub const Barrier = enum(u4) {
382 r = 0b0001,
383 w = 0b0010,
384 rw = 0b0011,
385};
386
368/// Returns the requested data, as well as the new index which is at the start of the387/// Returns the requested data, as well as the new index which is at the start of the
369/// trailers for the object.388/// trailers for the object.
370pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } {389pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } {
src/arch/riscv64/encoder.zig+4-2
...@@ -1,12 +1,13 @@...@@ -1,12 +1,13 @@
1pub const Instruction = struct {1pub const Instruction = struct {
2 encoding: Encoding,2 encoding: Encoding,
3 ops: [3]Operand = .{.none} ** 3,3 ops: [4]Operand = .{.none} ** 4,
44
5 pub const Operand = union(enum) {5 pub const Operand = union(enum) {
6 none,6 none,
7 reg: Register,7 reg: Register,
8 mem: Memory,8 mem: Memory,
9 imm: Immediate,9 imm: Immediate,
10 barrier: Mir.Barrier,
10 };11 };
1112
12 pub fn new(mnemonic: Encoding.Mnemonic, ops: []const Operand) !Instruction {13 pub fn new(mnemonic: Encoding.Mnemonic, ops: []const Operand) !Instruction {
...@@ -20,7 +21,7 @@ pub const Instruction = struct {...@@ -20,7 +21,7 @@ pub const Instruction = struct {
20 return error.InvalidInstruction;21 return error.InvalidInstruction;
21 };22 };
2223
23 var result_ops: [3]Operand = .{.none} ** 3;24 var result_ops: [4]Operand = .{.none} ** 4;
24 @memcpy(result_ops[0..ops.len], ops);25 @memcpy(result_ops[0..ops.len], ops);
2526
26 return .{27 return .{
...@@ -54,6 +55,7 @@ pub const Instruction = struct {...@@ -54,6 +55,7 @@ pub const Instruction = struct {
54 .reg => |reg| try writer.writeAll(@tagName(reg)),55 .reg => |reg| try writer.writeAll(@tagName(reg)),
55 .imm => |imm| try writer.print("{d}", .{imm.asSigned(64)}),56 .imm => |imm| try writer.print("{d}", .{imm.asSigned(64)}),
56 .mem => unreachable, // there is no "mem" operand in the actual instructions57 .mem => unreachable, // there is no "mem" operand in the actual instructions
58 .barrier => |barrier| try writer.writeAll(@tagName(barrier)),
57 }59 }
58 }60 }
59 }61 }