authorgravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2024-05-28 22:10:51-07:00
committergravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2024-07-14 23:02:29-07:00
logea084e9519a8e6f14d1bcb5f1fb2cddf842333b6
tree58ed4b874791547a05c37d128a41096e7aff39fc
parentd404d8a3637bc30dffc736e5fa1a68b8af0e19cb
signaturelock-open Commit is signed but in an unrecognized format.

riscv: `@atomicLoad` and `@atomicStore`

5 files changed, 221 insertions(+), 50 deletions(-)

src/arch/riscv64/CodeGen.zig+119-24
......@@ -2087,19 +2087,17 @@ fn airNot(func: *Func, inst: Air.Inst.Index) !void {
20872087 const operand = try func.resolveInst(ty_op.operand);
20882088 const ty = func.typeOf(ty_op.operand);
20892089
2090 switch (ty.zigTypeTag(zcu)) {
2091 .Bool => {
2092 const operand_reg = blk: {
2093 if (operand == .register) break :blk operand.register;
2094 break :blk try func.copyToTmpRegister(ty, operand);
2095 };
2090 const operand_reg, const operand_lock = try func.promoteReg(ty, operand);
2091 defer if (operand_lock) |lock| func.register_manager.unlockReg(lock);
20962092
2097 const dst_reg: Register =
2098 if (func.reuseOperand(inst, ty_op.operand, 0, operand) and operand == .register)
2099 operand.register
2100 else
2101 (try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register;
2093 const dst_reg: Register =
2094 if (func.reuseOperand(inst, ty_op.operand, 0, operand) and operand == .register)
2095 operand.register
2096 else
2097 (try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register;
21022098
2099 switch (ty.zigTypeTag(zcu)) {
2100 .Bool => {
21032101 _ = try func.addInst(.{
21042102 .tag = .pseudo,
21052103 .ops = .pseudo_not,
......@@ -2110,12 +2108,34 @@ fn airNot(func: *Func, inst: Air.Inst.Index) !void {
21102108 },
21112109 },
21122110 });
2111 },
2112 .Int => {
2113 const size = ty.bitSize(zcu);
2114 if (!math.isPowerOfTwo(size))
2115 return func.fail("TODO: airNot non-pow 2 int size", .{});
21132116
2114 break :result .{ .register = dst_reg };
2117 switch (size) {
2118 32, 64 => {
2119 _ = try func.addInst(.{
2120 .tag = .xori,
2121 .ops = .rri,
2122 .data = .{
2123 .i_type = .{
2124 .rd = dst_reg,
2125 .rs1 = operand_reg,
2126 .imm12 = Immediate.s(-1),
2127 },
2128 },
2129 });
2130 },
2131 8, 16 => return func.fail("TODO: airNot 8 or 16, {}", .{size}),
2132 else => unreachable,
2133 }
21152134 },
2116 .Int => return func.fail("TODO: airNot ints", .{}),
21172135 else => unreachable,
21182136 }
2137
2138 break :result .{ .register = dst_reg };
21192139 };
21202140 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
21212141}
......@@ -5600,17 +5620,24 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
56005620 const abi_size: u32 = @intCast(ty.abiSize(pt));
56015621
56025622 if (abi_size > 8) return std.debug.panic("tried to set reg with size {}", .{abi_size});
5603
56045623 const dst_reg_class = reg.class();
56055624
56065625 switch (src_mcv) {
5607 .dead => unreachable,
5608 .unreach, .none => return, // Nothing to do.
5626 .unreach,
5627 .none,
5628 .dead,
5629 => unreachable,
56095630 .undef => {
56105631 if (!func.wantSafety())
5611 return; // The already existing value will do just fine.
5612 // Write the debug undefined value.
5613 return func.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa });
5632 return;
5633
5634 switch (abi_size) {
5635 1 => return func.genSetReg(ty, reg, .{ .immediate = 0xAA }),
5636 2 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAA }),
5637 3...4 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAA }),
5638 5...8 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAAAAAAAAAA }),
5639 else => unreachable,
5640 }
56145641 },
56155642 .immediate => |unsigned_x| {
56165643 assert(dst_reg_class == .int);
......@@ -6047,14 +6074,82 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
60476074}
60486075
60496076fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
6050 _ = inst;
6051 return func.fail("TODO implement airAtomicLoad for {}", .{func.target.cpu.arch});
6077 const zcu = func.bin_file.comp.module.?;
6078 const atomic_load = func.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
6079 const order: std.builtin.AtomicOrder = atomic_load.order;
6080
6081 const ptr_ty = func.typeOf(atomic_load.ptr);
6082 const elem_ty = ptr_ty.childType(zcu);
6083 const ptr_mcv = try func.resolveInst(atomic_load.ptr);
6084
6085 const result_mcv = try func.allocRegOrMem(elem_ty, inst, true);
6086
6087 if (order == .seq_cst) {
6088 _ = try func.addInst(.{
6089 .tag = .fence,
6090 .ops = .fence,
6091 .data = .{
6092 .fence = .{
6093 .pred = .rw,
6094 .succ = .rw,
6095 },
6096 },
6097 });
6098 }
6099
6100 try func.load(result_mcv, ptr_mcv, ptr_ty);
6101
6102 switch (order) {
6103 // Don't guarnetee other memory operations to be ordered after the load.
6104 .unordered => {},
6105 .monotonic => {},
6106 // Make sure all previous reads happen before any reading or writing accurs.
6107 .seq_cst, .acquire => {
6108 _ = try func.addInst(.{
6109 .tag = .fence,
6110 .ops = .fence,
6111 .data = .{
6112 .fence = .{
6113 .pred = .r,
6114 .succ = .rw,
6115 },
6116 },
6117 });
6118 },
6119 else => unreachable,
6120 }
6121
6122 return func.finishAir(inst, result_mcv, .{ atomic_load.ptr, .none, .none });
60526123}
60536124
60546125fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
6055 _ = inst;
6056 _ = order;
6057 return func.fail("TODO implement airAtomicStore for {}", .{func.target.cpu.arch});
6126 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6127
6128 const ptr_ty = func.typeOf(bin_op.lhs);
6129 const ptr_mcv = try func.resolveInst(bin_op.lhs);
6130
6131 const val_ty = func.typeOf(bin_op.rhs);
6132 const val_mcv = try func.resolveInst(bin_op.rhs);
6133
6134 switch (order) {
6135 .unordered, .monotonic => {},
6136 .release, .seq_cst => {
6137 _ = try func.addInst(.{
6138 .tag = .fence,
6139 .ops = .fence,
6140 .data = .{
6141 .fence = .{
6142 .pred = .rw,
6143 .succ = .w,
6144 },
6145 },
6146 });
6147 },
6148 else => unreachable,
6149 }
6150
6151 try func.store(ptr_mcv, val_mcv, ptr_ty, val_ty);
6152 return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
60586153}
60596154
60606155fn airMemset(func: *Func, inst: Air.Inst.Index, safety: bool) !void {
src/arch/riscv64/Encoding.zig+67-23
......@@ -2,25 +2,30 @@ mnemonic: Mnemonic,
22data: Data,
33
44const OpCode = enum(u7) {
5 OP = 0b0110011,
5 LOAD = 0b0000011,
6 LOAD_FP = 0b0000111,
7 MISC_MEM = 0b0001111,
68 OP_IMM = 0b0010011,
9 AUIPC = 0b0010111,
710 OP_IMM_32 = 0b0011011,
8 OP_32 = 0b0111011,
9
10 BRANCH = 0b1100011,
11 LOAD = 0b0000011,
1211 STORE = 0b0100011,
13 SYSTEM = 0b1110011,
14
15 OP_FP = 0b1010011,
16 LOAD_FP = 0b0000111,
1712 STORE_FP = 0b0100111,
18
19 JALR = 0b1100111,
20 AUIPC = 0b0010111,
13 AMO = 0b0101111,
14 OP = 0b0110011,
15 OP_32 = 0b0111011,
2116 LUI = 0b0110111,
17 MADD = 0b1000011,
18 MSUB = 0b1000111,
19 NMSUB = 0b1001011,
20 NMADD = 0b1001111,
21 OP_FP = 0b1010011,
22 OP_IMM_64 = 0b1011011,
23 BRANCH = 0b1100011,
24 JALR = 0b1100111,
2225 JAL = 0b1101111,
23 NONE = 0b0000000,
26 SYSTEM = 0b1110011,
27 OP_64 = 0b1111011,
28 NONE = 0b00000000,
2429};
2530
2631const Fmt = enum(u2) {
......@@ -28,7 +33,9 @@ const Fmt = enum(u2) {
2833 S = 0b00,
2934 /// 64-bit double-precision
3035 D = 0b01,
31 _reserved = 0b10,
36
37 // H = 0b10, unused in the G extension
38
3239 /// 128-bit quad-precision
3340 Q = 0b11,
3441};
......@@ -192,6 +199,9 @@ pub const Mnemonic = enum {
192199 fsgnjnd,
193200 fsgnjxd,
194201
202 // MISC
203 fence,
204
195205 pub fn encoding(mnem: Mnemonic) Enc {
196206 return switch (mnem) {
197207 // zig fmt: off
......@@ -366,6 +376,10 @@ pub const Mnemonic = enum {
366376
367377 .unimp => .{ .opcode = .NONE, .data = .{ .f = .{ .funct3 = 0b000 } } },
368378
379 // MISC_MEM
380
381 .fence => .{ .opcode = .MISC_MEM, .data = .{ .f = .{ .funct3 = 0b000 } } },
382
369383
370384 // zig fmt: on
371385 };
......@@ -380,7 +394,7 @@ pub const InstEnc = enum {
380394 B,
381395 U,
382396 J,
383
397 fence,
384398 /// extras that have unusual op counts
385399 system,
386400
......@@ -509,20 +523,24 @@ pub const InstEnc = enum {
509523 .ebreak,
510524 .unimp,
511525 => .system,
526
527 .fence,
528 => .fence,
512529 };
513530 }
514531
515532 pub fn opsList(enc: InstEnc) [4]std.meta.FieldEnum(Operand) {
516533 return switch (enc) {
517534 // zig fmt: off
518 .R => .{ .reg, .reg, .reg, .none },
519 .R4 => .{ .reg, .reg, .reg, .reg },
520 .I => .{ .reg, .reg, .imm, .none },
521 .S => .{ .reg, .reg, .imm, .none },
522 .B => .{ .reg, .reg, .imm, .none },
523 .U => .{ .reg, .imm, .none, .none },
524 .J => .{ .reg, .imm, .none, .none },
525 .system => .{ .none, .none, .none, .none },
535 .R => .{ .reg, .reg, .reg, .none },
536 .R4 => .{ .reg, .reg, .reg, .reg },
537 .I => .{ .reg, .reg, .imm, .none },
538 .S => .{ .reg, .reg, .imm, .none },
539 .B => .{ .reg, .reg, .imm, .none },
540 .U => .{ .reg, .imm, .none, .none },
541 .J => .{ .reg, .imm, .none, .none },
542 .system => .{ .none, .none, .none, .none },
543 .fence => .{ .barrier, .barrier, .none, .none },
526544 // zig fmt: on
527545 };
528546 }
......@@ -584,6 +602,15 @@ pub const Data = union(InstEnc) {
584602 imm1_10: u10,
585603 imm20: u1,
586604 },
605 fence: packed struct {
606 opcode: u7,
607 rd: u5 = 0,
608 funct3: u3,
609 rs1: u5 = 0,
610 succ: u4,
611 pred: u4,
612 _ignored: u4 = 0,
613 },
587614 system: void,
588615
589616 pub fn toU32(self: Data) u32 {
......@@ -596,6 +623,7 @@ pub const Data = union(InstEnc) {
596623 .B => |v| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.imm11)) << 7) + (@as(u32, @intCast(v.imm1_4)) << 8) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.rs2)) << 20) + (@as(u32, @intCast(v.imm5_10)) << 25) + (@as(u32, @intCast(v.imm12)) << 31),
597624 .U => |v| @bitCast(v),
598625 .J => |v| @bitCast(v),
626 .fence => |v| @bitCast(v),
599627 .system => unreachable,
600628 // zig fmt: on
601629 };
......@@ -748,6 +776,22 @@ pub const Data = union(InstEnc) {
748776 },
749777 };
750778 },
779 .fence => {
780 assert(ops.len == 2);
781
782 const succ = ops[0];
783 const pred = ops[1];
784
785 return .{
786 .fence = .{
787 .succ = @intFromEnum(succ.barrier),
788 .pred = @intFromEnum(pred.barrier),
789
790 .opcode = @intFromEnum(enc.opcode),
791 .funct3 = enc.data.f.funct3,
792 },
793 };
794 },
751795
752796 else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}),
753797 }
src/arch/riscv64/Lower.zig+12-1
......@@ -378,7 +378,14 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
378378 const rr = inst.data.rr;
379379 assert(rr.rs.class() == .int and rr.rd.class() == .int);
380380
381 try lower.emit(.xori, &.{
381 // mask out any other bits that aren't the boolean
382 try lower.emit(.andi, &.{
383 .{ .reg = rr.rs },
384 .{ .reg = rr.rs },
385 .{ .imm = Immediate.s(1) },
386 });
387
388 try lower.emit(.sltiu, &.{
382389 .{ .reg = rr.rd },
383390 .{ .reg = rr.rs },
384391 .{ .imm = Immediate.s(1) },
......@@ -447,6 +454,10 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
447454 .{ .reg = inst.data.r_type.rs1 },
448455 .{ .reg = inst.data.r_type.rs2 },
449456 },
457 .fence => &.{
458 .{ .barrier = inst.data.fence.succ },
459 .{ .barrier = inst.data.fence.pred },
460 },
450461 else => return lower.fail("TODO: generic lower ops {s}", .{@tagName(inst.ops)}),
451462 });
452463}
src/arch/riscv64/Mir.zig+19
......@@ -31,6 +31,7 @@ pub const Inst = struct {
3131 @"and",
3232 andi,
3333
34 xori,
3435 xor,
3536 @"or",
3637
......@@ -38,6 +39,8 @@ pub const Inst = struct {
3839 ecall,
3940 unimp,
4041
42 fence,
43
4144 add,
4245 addw,
4346 sub,
......@@ -246,6 +249,11 @@ pub const Inst = struct {
246249 atom_index: u32,
247250 sym_index: u32,
248251 },
252
253 fence: struct {
254 pred: Barrier,
255 succ: Barrier,
256 },
249257 };
250258
251259 pub const Ops = enum {
......@@ -326,10 +334,15 @@ pub const Inst = struct {
326334 pseudo_spill_regs,
327335
328336 pseudo_compare,
337
338 /// NOT operation on booleans. Does an `andi reg, reg, 1` to mask out any other bits from the boolean.
329339 pseudo_not,
330340
331341 /// Generates an auipc + jalr pair, with a R_RISCV_CALL_PLT reloc
332342 pseudo_extern_fn_reloc,
343
344 /// IORW, IORW
345 fence,
333346 };
334347
335348 // Make sure we don't accidentally make instructions bigger than expected.
......@@ -365,6 +378,12 @@ pub const FrameLoc = struct {
365378 disp: i32,
366379};
367380
381pub const Barrier = enum(u4) {
382 r = 0b0001,
383 w = 0b0010,
384 rw = 0b0011,
385};
386
368387/// Returns the requested data, as well as the new index which is at the start of the
369388/// trailers for the object.
370389pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } {
src/arch/riscv64/encoder.zig+4-2
......@@ -1,12 +1,13 @@
11pub const Instruction = struct {
22 encoding: Encoding,
3 ops: [3]Operand = .{.none} ** 3,
3 ops: [4]Operand = .{.none} ** 4,
44
55 pub const Operand = union(enum) {
66 none,
77 reg: Register,
88 mem: Memory,
99 imm: Immediate,
10 barrier: Mir.Barrier,
1011 };
1112
1213 pub fn new(mnemonic: Encoding.Mnemonic, ops: []const Operand) !Instruction {
......@@ -20,7 +21,7 @@ pub const Instruction = struct {
2021 return error.InvalidInstruction;
2122 };
2223
23 var result_ops: [3]Operand = .{.none} ** 3;
24 var result_ops: [4]Operand = .{.none} ** 4;
2425 @memcpy(result_ops[0..ops.len], ops);
2526
2627 return .{
......@@ -54,6 +55,7 @@ pub const Instruction = struct {
5455 .reg => |reg| try writer.writeAll(@tagName(reg)),
5556 .imm => |imm| try writer.print("{d}", .{imm.asSigned(64)}),
5657 .mem => unreachable, // there is no "mem" operand in the actual instructions
58 .barrier => |barrier| try writer.writeAll(@tagName(barrier)),
5759 }
5860 }
5961 }