authorgravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2024-06-01 00:42:29-07:00
committergravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2024-07-14 23:02:33-07:00
log3e73f37d0a371276c534240ee2d26cbf4b3a0033
treeb0736a37d79066c8cb0649107fcb14db3ef260e7
parent7a02878f4e1ff18275e61db7450fb3e12ee1926e
signaturelock-open Commit is signed but in an unrecognized format.

riscv: implement `@fence`

6 files changed, 75 insertions(+), 36 deletions(-)

src/arch/riscv64/CodeGen.zig+32-10
...@@ -1318,7 +1318,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {...@@ -1318,7 +1318,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1318 .breakpoint => try func.airBreakpoint(),1318 .breakpoint => try func.airBreakpoint(),
1319 .ret_addr => try func.airRetAddr(inst),1319 .ret_addr => try func.airRetAddr(inst),
1320 .frame_addr => try func.airFrameAddress(inst),1320 .frame_addr => try func.airFrameAddress(inst),
1321 .fence => try func.airFence(),1321 .fence => try func.airFence(inst),
1322 .cond_br => try func.airCondBr(inst),1322 .cond_br => try func.airCondBr(inst),
1323 .dbg_stmt => try func.airDbgStmt(inst),1323 .dbg_stmt => try func.airDbgStmt(inst),
1324 .fptrunc => try func.airFptrunc(inst),1324 .fptrunc => try func.airFptrunc(inst),
...@@ -4238,9 +4238,28 @@ fn airFrameAddress(func: *Func, inst: Air.Inst.Index) !void {...@@ -4238,9 +4238,28 @@ fn airFrameAddress(func: *Func, inst: Air.Inst.Index) !void {
4238 return func.finishAir(inst, dst_mcv, .{ .none, .none, .none });4238 return func.finishAir(inst, dst_mcv, .{ .none, .none, .none });
4239}4239}
42404240
4241fn airFence(func: *Func) !void {4241fn airFence(func: *Func, inst: Air.Inst.Index) !void {
4242 return func.fail("TODO implement fence() for {}", .{func.target.cpu.arch});4242 const order = func.air.instructions.items(.data)[@intFromEnum(inst)].fence;
4243 //return func.finishAirBookkeeping();4243 const pred: Mir.Barrier, const succ: Mir.Barrier = switch (order) {
4244 .unordered, .monotonic => unreachable,
4245 .acquire => .{ .r, .rw },
4246 .release => .{ .rw, .r },
4247 .acq_rel => .{ .rw, .rw },
4248 .seq_cst => .{ .rw, .rw },
4249 };
4250
4251 _ = try func.addInst(.{
4252 .tag = .pseudo,
4253 .ops = .pseudo_fence,
4254 .data = .{
4255 .fence = .{
4256 .pred = pred,
4257 .succ = succ,
4258 .fm = if (order == .acq_rel) .tso else .none,
4259 },
4260 },
4261 });
4262 return func.finishAirBookkeeping();
4244}4263}
42454264
4246fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {4265fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
...@@ -6264,12 +6283,13 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {...@@ -6264,12 +6283,13 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
62646283
6265 if (order == .seq_cst) {6284 if (order == .seq_cst) {
6266 _ = try func.addInst(.{6285 _ = try func.addInst(.{
6267 .tag = .fence,6286 .tag = .pseudo,
6268 .ops = .fence,6287 .ops = .pseudo_fence,
6269 .data = .{6288 .data = .{
6270 .fence = .{6289 .fence = .{
6271 .pred = .rw,6290 .pred = .rw,
6272 .succ = .rw,6291 .succ = .rw,
6292 .fm = .none,
6273 },6293 },
6274 },6294 },
6275 });6295 });
...@@ -6284,12 +6304,13 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {...@@ -6284,12 +6304,13 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
6284 // Make sure all previous reads happen before any reading or writing accurs.6304 // Make sure all previous reads happen before any reading or writing accurs.
6285 .seq_cst, .acquire => {6305 .seq_cst, .acquire => {
6286 _ = try func.addInst(.{6306 _ = try func.addInst(.{
6287 .tag = .fence,6307 .tag = .pseudo,
6288 .ops = .fence,6308 .ops = .pseudo_fence,
6289 .data = .{6309 .data = .{
6290 .fence = .{6310 .fence = .{
6291 .pred = .r,6311 .pred = .r,
6292 .succ = .rw,6312 .succ = .rw,
6313 .fm = .none,
6293 },6314 },
6294 },6315 },
6295 });6316 });
...@@ -6313,12 +6334,13 @@ fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOr...@@ -6313,12 +6334,13 @@ fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOr
6313 .unordered, .monotonic => {},6334 .unordered, .monotonic => {},
6314 .release, .seq_cst => {6335 .release, .seq_cst => {
6315 _ = try func.addInst(.{6336 _ = try func.addInst(.{
6316 .tag = .fence,6337 .tag = .pseudo,
6317 .ops = .fence,6338 .ops = .pseudo_fence,
6318 .data = .{6339 .data = .{
6319 .fence = .{6340 .fence = .{
6320 .pred = .rw,6341 .pred = .rw,
6321 .succ = .w,6342 .succ = .w,
6343 .fm = .none,
6322 },6344 },
6323 },6345 },
6324 });6346 });
src/arch/riscv64/Encoding.zig+24-15
...@@ -45,6 +45,11 @@ const AmoWidth = enum(u3) {...@@ -45,6 +45,11 @@ const AmoWidth = enum(u3) {
45 D = 0b011,45 D = 0b011,
46};46};
4747
48const FenceMode = enum(u4) {
49 none = 0b0000,
50 tso = 0b1000,
51};
52
48const Enc = struct {53const Enc = struct {
49 opcode: OpCode,54 opcode: OpCode,
5055
...@@ -58,6 +63,10 @@ const Enc = struct {...@@ -58,6 +63,10 @@ const Enc = struct {
58 funct5: u5,63 funct5: u5,
59 width: AmoWidth,64 width: AmoWidth,
60 },65 },
66 fence: struct {
67 funct3: u3,
68 fm: FenceMode,
69 },
61 /// funct5 + rm + fmt70 /// funct5 + rm + fmt
62 fmt: struct {71 fmt: struct {
63 funct5: u5,72 funct5: u5,
...@@ -210,6 +219,7 @@ pub const Mnemonic = enum {...@@ -210,6 +219,7 @@ pub const Mnemonic = enum {
210219
211 // MISC220 // MISC
212 fence,221 fence,
222 fencetso,
213223
214 // AMO224 // AMO
215 amoswapw,225 amoswapw,
...@@ -406,9 +416,12 @@ pub const Mnemonic = enum {...@@ -406,9 +416,12 @@ pub const Mnemonic = enum {
406 416
407 .unimp => .{ .opcode = .NONE, .data = .{ .f = .{ .funct3 = 0b000 } } },417 .unimp => .{ .opcode = .NONE, .data = .{ .f = .{ .funct3 = 0b000 } } },
408418
419
409 // MISC_MEM420 // MISC_MEM
410421
411 .fence => .{ .opcode = .MISC_MEM, .data = .{ .f = .{ .funct3 = 0b000 } } },422 .fence => .{ .opcode = .MISC_MEM, .data = .{ .fence = .{ .funct3 = 0b000, .fm = .none } } },
423 .fencetso => .{ .opcode = .MISC_MEM, .data = .{ .fence = .{ .funct3 = 0b000, .fm = .tso } } },
424
412425
413 // AMO426 // AMO
414427
...@@ -437,7 +450,6 @@ pub const Mnemonic = enum {...@@ -437,7 +450,6 @@ pub const Mnemonic = enum {
437 .amomaxud => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11100 } } },450 .amomaxud => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11100 } } },
438451
439 452
440
441 // zig fmt: on453 // zig fmt: on
442 };454 };
443 }455 }
...@@ -583,6 +595,7 @@ pub const InstEnc = enum {...@@ -583,6 +595,7 @@ pub const InstEnc = enum {
583 => .system,595 => .system,
584596
585 .fence,597 .fence,
598 .fencetso,
586 => .fence,599 => .fence,
587600
588 .amoswapw,601 .amoswapw,
...@@ -689,7 +702,7 @@ pub const Data = union(InstEnc) {...@@ -689,7 +702,7 @@ pub const Data = union(InstEnc) {
689 rs1: u5 = 0,702 rs1: u5 = 0,
690 succ: u4,703 succ: u4,
691 pred: u4,704 pred: u4,
692 _ignored: u4 = 0,705 fm: u4,
693 },706 },
694 amo: packed struct {707 amo: packed struct {
695 opcode: u7,708 opcode: u7,
...@@ -711,11 +724,9 @@ pub const Data = union(InstEnc) {...@@ -711,11 +724,9 @@ pub const Data = union(InstEnc) {
711724
712 pub fn toU32(self: Data) u32 {725 pub fn toU32(self: Data) u32 {
713 return switch (self) {726 return switch (self) {
714 // zig fmt: off727 .fence => |v| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.rd)) << 7) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.succ)) << 20) + (@as(u32, @intCast(v.pred)) << 24) + (@as(u32, @intCast(v.fm)) << 28),
715 .B => |v| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.imm11)) << 7) + (@as(u32, @intCast(v.imm1_4)) << 8) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.rs2)) << 20) + (@as(u32, @intCast(v.imm5_10)) << 25) + (@as(u32, @intCast(v.imm12)) << 31),
716 inline else => |v| @bitCast(v),728 inline else => |v| @bitCast(v),
717 .system => unreachable,729 .system => unreachable,
718 // zig fmt: on
719 };730 };
720 }731 }
721732
...@@ -869,16 +880,17 @@ pub const Data = union(InstEnc) {...@@ -869,16 +880,17 @@ pub const Data = union(InstEnc) {
869 .fence => {880 .fence => {
870 assert(ops.len == 2);881 assert(ops.len == 2);
871882
872 const succ = ops[0];883 const succ = ops[0].barrier;
873 const pred = ops[1];884 const pred = ops[1].barrier;
874885
875 return .{886 return .{
876 .fence = .{887 .fence = .{
877 .succ = @intFromEnum(succ.barrier),888 .succ = @intFromEnum(succ),
878 .pred = @intFromEnum(pred.barrier),889 .pred = @intFromEnum(pred),
879890
880 .opcode = @intFromEnum(enc.opcode),891 .opcode = @intFromEnum(enc.opcode),
881 .funct3 = enc.data.f.funct3,892 .funct3 = enc.data.fence.funct3,
893 .fm = @intFromEnum(enc.data.fence.fm),
882 },894 },
883 };895 };
884 },896 },
...@@ -891,7 +903,7 @@ pub const Data = union(InstEnc) {...@@ -891,7 +903,7 @@ pub const Data = union(InstEnc) {
891 const rl = ops[3];903 const rl = ops[3];
892 const aq = ops[4];904 const aq = ops[4];
893905
894 const ret: Data = .{906 return .{
895 .amo = .{907 .amo = .{
896 .rd = rd.reg.encodeId(),908 .rd = rd.reg.encodeId(),
897 .rs1 = rs1.reg.encodeId(),909 .rs1 = rs1.reg.encodeId(),
...@@ -906,9 +918,6 @@ pub const Data = union(InstEnc) {...@@ -906,9 +918,6 @@ pub const Data = union(InstEnc) {
906 .funct5 = enc.data.amo.funct5,918 .funct5 = enc.data.amo.funct5,
907 },919 },
908 };920 };
909
910 std.debug.print("ret: {}, {}", .{ ret.amo.rl, rl.barrier == .rl });
911 return ret;
912 },921 },
913922
914 else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}),923 else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}),
src/arch/riscv64/Lower.zig+12-4
...@@ -443,6 +443,18 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index, options: struct {...@@ -443,6 +443,18 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index, options: struct {
443 });443 });
444 },444 },
445445
446 .pseudo_fence => {
447 const fence = inst.data.fence;
448
449 try lower.emit(switch (fence.fm) {
450 .tso => .fencetso,
451 .none => .fence,
452 }, &.{
453 .{ .barrier = fence.succ },
454 .{ .barrier = fence.pred },
455 });
456 },
457
446 else => return lower.fail("TODO lower: psuedo {s}", .{@tagName(inst.ops)}),458 else => return lower.fail("TODO lower: psuedo {s}", .{@tagName(inst.ops)}),
447 },459 },
448 }460 }
...@@ -485,10 +497,6 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -485,10 +497,6 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
485 .{ .reg = inst.data.r_type.rs1 },497 .{ .reg = inst.data.r_type.rs1 },
486 .{ .reg = inst.data.r_type.rs2 },498 .{ .reg = inst.data.r_type.rs2 },
487 },499 },
488 .fence => &.{
489 .{ .barrier = inst.data.fence.succ },
490 .{ .barrier = inst.data.fence.pred },
491 },
492 else => return lower.fail("TODO: generic lower ops {s}", .{@tagName(inst.ops)}),500 else => return lower.fail("TODO: generic lower ops {s}", .{@tagName(inst.ops)}),
493 });501 });
494}502}
src/arch/riscv64/Mir.zig+7-5
...@@ -80,8 +80,6 @@ pub const Inst = struct {...@@ -80,8 +80,6 @@ pub const Inst = struct {
80 sh,80 sh,
81 sb,81 sb,
8282
83 fence,
84
85 // M extension83 // M extension
86 mul,84 mul,
87 mulw,85 mulw,
...@@ -256,6 +254,10 @@ pub const Inst = struct {...@@ -256,6 +254,10 @@ pub const Inst = struct {
256 fence: struct {254 fence: struct {
257 pred: Barrier,255 pred: Barrier,
258 succ: Barrier,256 succ: Barrier,
257 fm: enum {
258 none,
259 tso,
260 },
259 },261 },
260262
261 amo: struct {263 amo: struct {
...@@ -355,7 +357,7 @@ pub const Inst = struct {...@@ -355,7 +357,7 @@ pub const Inst = struct {
355 pseudo_extern_fn_reloc,357 pseudo_extern_fn_reloc,
356358
357 /// IORW, IORW359 /// IORW, IORW
358 fence,360 pseudo_fence,
359361
360 /// Ordering, Src, Addr, Dest362 /// Ordering, Src, Addr, Dest
361 pseudo_amo,363 pseudo_amo,
...@@ -396,8 +398,8 @@ pub const FrameLoc = struct {...@@ -396,8 +398,8 @@ pub const FrameLoc = struct {
396398
397pub const Barrier = enum(u4) {399pub const Barrier = enum(u4) {
398 // Fence400 // Fence
399 r = 0b0001,401 w = 0b0001,
400 w = 0b0010,402 r = 0b0010,
401 rw = 0b0011,403 rw = 0b0011,
402404
403 // Amo405 // Amo
test/behavior/atomics.zig-1
...@@ -42,7 +42,6 @@ test "fence" {...@@ -42,7 +42,6 @@ test "fence" {
42 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO42 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
43 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO43 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
44 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;44 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
45 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
4645
47 var x: i32 = 1234;46 var x: i32 = 1234;
48 @fence(.seq_cst);47 @fence(.seq_cst);
test/behavior/builtin_functions_returning_void_or_noreturn.zig-1
...@@ -11,7 +11,6 @@ test {...@@ -11,7 +11,6 @@ test {
11 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO11 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
12 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO12 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
13 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;13 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
14 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1514
16 var val: u8 = undefined;15 var val: u8 = undefined;
17 try testing.expectEqual({}, @atomicStore(u8, &val, 0, .unordered));16 try testing.expectEqual({}, @atomicStore(u8, &val, 0, .unordered));