authorgravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2024-05-29 17:36:53-07:00
committergravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2024-07-14 23:02:32-07:00
log0460572899482f7aa7ec6d9d177ed48984802c80
tree027b9132b140fbfde4f0d7b3344a84c02b8699ab
parentea084e9519a8e6f14d1bcb5f1fb2cddf842333b6
signaturelock-open Commit is signed but in an unrecognized format.

riscv: `@atomicRmw`

Now we generate debug undefined constants when the user asks for them to dedup across the function decl. This takes 2 instructions instead of 7 in the RISC-V backend. TODO, we need to dedupe across function decl boundaries.

8 files changed, 342 insertions(+), 71 deletions(-)

src/arch/riscv64/CodeGen.zig+125-21
...@@ -117,8 +117,9 @@ const MCValue = union(enum) {...@@ -117,8 +117,9 @@ const MCValue = union(enum) {
117 /// No more references to this value remain.117 /// No more references to this value remain.
118 /// The payload is the value of scope_generation at the point where the death occurred118 /// The payload is the value of scope_generation at the point where the death occurred
119 dead: u32,119 dead: u32,
120 /// The value is undefined.120 /// The value is undefined. Contains a symbol index to an undefined constant. Null means
121 undef,121 /// set the undefined value via immediate instead of a load.
122 undef: ?u32,
122 /// A pointer-sized integer that fits in a register.123 /// A pointer-sized integer that fits in a register.
123 /// If the type is a pointer, this is the pointer address in virtual address space.124 /// If the type is a pointer, this is the pointer address in virtual address space.
124 immediate: u64,125 immediate: u64,
...@@ -1045,6 +1046,7 @@ pub fn addExtraAssumeCapacity(func: *Func, extra: anytype) u32 {...@@ -1045,6 +1046,7 @@ pub fn addExtraAssumeCapacity(func: *Func, extra: anytype) u32 {
1045const required_features = [_]Target.riscv.Feature{1046const required_features = [_]Target.riscv.Feature{
1046 .d,1047 .d,
1047 .m,1048 .m,
1049 .a,
1048};1050};
10491051
1050fn gen(func: *Func) !void {1052fn gen(func: *Func) !void {
...@@ -1631,7 +1633,7 @@ fn computeFrameLayout(func: *Func) !FrameLayout {...@@ -1631,7 +1633,7 @@ fn computeFrameLayout(func: *Func) !FrameLayout {
16311633
1632 // The total frame size is calculated by the amount of s registers you need to save * 8, as each1634 // The total frame size is calculated by the amount of s registers you need to save * 8, as each
1633 // register is 8 bytes, the total allocation sizes, and 16 more register for the spilled ra and s01635 // register is 8 bytes, the total allocation sizes, and 16 more register for the spilled ra and s0
1634 // register. Finally we align the frame size to the align of the base pointer.1636 // register. Finally we align the frame size to the alignment of the base pointer.
1635 const args_frame_size = frame_size[@intFromEnum(FrameIndex.args_frame)];1637 const args_frame_size = frame_size[@intFromEnum(FrameIndex.args_frame)];
1636 const spill_frame_size = frame_size[@intFromEnum(FrameIndex.spill_frame)];1638 const spill_frame_size = frame_size[@intFromEnum(FrameIndex.spill_frame)];
1637 const call_frame_size = frame_size[@intFromEnum(FrameIndex.call_frame)];1639 const call_frame_size = frame_size[@intFromEnum(FrameIndex.call_frame)];
...@@ -2110,7 +2112,7 @@ fn airNot(func: *Func, inst: Air.Inst.Index) !void {...@@ -2110,7 +2112,7 @@ fn airNot(func: *Func, inst: Air.Inst.Index) !void {
2110 });2112 });
2111 },2113 },
2112 .Int => {2114 .Int => {
2113 const size = ty.bitSize(zcu);2115 const size = ty.bitSize(pt);
2114 if (!math.isPowerOfTwo(size))2116 if (!math.isPowerOfTwo(size))
2115 return func.fail("TODO: airNot non-pow 2 int size", .{});2117 return func.fail("TODO: airNot non-pow 2 int size", .{});
21162118
...@@ -3249,7 +3251,7 @@ fn airWrapErrUnionErr(func: *Func, inst: Air.Inst.Index) !void {...@@ -3249,7 +3251,7 @@ fn airWrapErrUnionErr(func: *Func, inst: Air.Inst.Index) !void {
3249 const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(eu_ty, pt));3251 const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(eu_ty, pt));
3250 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, pt));3252 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, pt));
3251 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, pt));3253 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, pt));
3252 try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);3254 try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .{ .undef = null });
3253 const operand = try func.resolveInst(ty_op.operand);3255 const operand = try func.resolveInst(ty_op.operand);
3254 try func.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand);3256 try func.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand);
3255 break :result .{ .load_frame = .{ .index = frame_index } };3257 break :result .{ .load_frame = .{ .index = frame_index } };
...@@ -5627,10 +5629,14 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError!...@@ -5627,10 +5629,14 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
5627 .none,5629 .none,
5628 .dead,5630 .dead,
5629 => unreachable,5631 => unreachable,
5630 .undef => {5632 .undef => |sym_index| {
5631 if (!func.wantSafety())5633 if (!func.wantSafety())
5632 return;5634 return;
56335635
5636 if (sym_index) |index| {
5637 return func.genSetReg(ty, reg, .{ .load_symbol = .{ .sym = index } });
5638 }
5639
5634 switch (abi_size) {5640 switch (abi_size) {
5635 1 => return func.genSetReg(ty, reg, .{ .immediate = 0xAA }),5641 1 => return func.genSetReg(ty, reg, .{ .immediate = 0xAA }),
5636 2 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAA }),5642 2 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAA }),
...@@ -5865,11 +5871,17 @@ fn genSetMem(...@@ -5865,11 +5871,17 @@ fn genSetMem(
5865 .dead,5871 .dead,
5866 .reserved_frame,5872 .reserved_frame,
5867 => unreachable,5873 => unreachable,
5868 .undef => try func.genInlineMemset(5874 .undef => |sym_index| {
5869 dst_ptr_mcv,5875 if (sym_index) |index| {
5870 src_mcv,5876 return func.genSetMem(base, disp, ty, .{ .load_symbol = .{ .sym = index } });
5871 .{ .immediate = abi_size },5877 }
5872 ),5878
5879 try func.genInlineMemset(
5880 dst_ptr_mcv,
5881 src_mcv,
5882 .{ .immediate = abi_size },
5883 );
5884 },
5873 .register_offset,5885 .register_offset,
5874 .memory,5886 .memory,
5875 .indirect,5887 .indirect,
...@@ -6069,12 +6081,82 @@ fn airCmpxchg(func: *Func, inst: Air.Inst.Index) !void {...@@ -6069,12 +6081,82 @@ fn airCmpxchg(func: *Func, inst: Air.Inst.Index) !void {
6069}6081}
60706082
6071fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {6083fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
6072 _ = inst;6084 const zcu = func.pt.zcu;
6073 return func.fail("TODO implement airCmpxchg for {}", .{func.target.cpu.arch});6085 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6086 const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data;
6087
6088 const op = extra.op();
6089 const order = extra.ordering();
6090
6091 const ptr_ty = func.typeOf(pl_op.operand);
6092 const ptr_mcv = try func.resolveInst(pl_op.operand);
6093
6094 const val_ty = func.typeOf(extra.operand);
6095 const val_size = val_ty.abiSize(func.pt);
6096 const val_mcv = try func.resolveInst(extra.operand);
6097
6098 if (!math.isPowerOfTwo(val_size))
6099 return func.fail("TODO: airAtomicRmw non-pow 2", .{});
6100
6101 switch (val_ty.zigTypeTag(zcu)) {
6102 .Int => {},
6103 inline .Bool, .Float, .Enum, .Pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}),
6104 else => unreachable,
6105 }
6106
6107 switch (val_size) {
6108 1, 2 => return func.fail("TODO: airAtomicRmw Int {}", .{val_size}),
6109 4, 8 => {},
6110 else => unreachable,
6111 }
6112
6113 const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv);
6114 defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock);
6115
6116 const val_register, const val_lock = try func.promoteReg(val_ty, val_mcv);
6117 defer if (val_lock) |lock| func.register_manager.unlockReg(lock);
6118
6119 const result_mcv = try func.allocRegOrMem(val_ty, inst, true);
6120 assert(result_mcv == .register); // should fit into 8 bytes
6121
6122 const aq, const rl = switch (order) {
6123 .unordered => unreachable,
6124 .monotonic => .{ false, false },
6125 .acquire => .{ true, false },
6126 .release => .{ false, true },
6127 .acq_rel => .{ true, true },
6128 .seq_cst => .{ true, true },
6129 };
6130
6131 _ = try func.addInst(.{
6132 .tag = .pseudo,
6133 .ops = .pseudo_amo,
6134 .data = .{ .amo = .{
6135 .rd = result_mcv.register,
6136 .rs1 = ptr_register,
6137 .rs2 = val_register,
6138 .aq = if (aq) .aq else .none,
6139 .rl = if (rl) .rl else .none,
6140 .op = switch (op) {
6141 .Xchg => .SWAP,
6142 .Add => .ADD,
6143 .Sub => return func.fail("TODO: airAtomicRmw SUB", .{}),
6144 .And => .AND,
6145 .Nand => return func.fail("TODO: airAtomicRmw NAND", .{}),
6146 .Or => .OR,
6147 .Xor => .XOR,
6148 .Max => .MAX,
6149 .Min => .MIN,
6150 },
6151 .ty = val_ty,
6152 } },
6153 });
6154
6155 return func.finishAir(inst, result_mcv, .{ pl_op.operand, extra.operand, .none });
6074}6156}
60756157
6076fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {6158fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
6077 const zcu = func.bin_file.comp.module.?;6159 const zcu = func.pt.zcu;
6078 const atomic_load = func.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;6160 const atomic_load = func.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
6079 const order: std.builtin.AtomicOrder = atomic_load.order;6161 const order: std.builtin.AtomicOrder = atomic_load.order;
60806162
...@@ -6083,6 +6165,7 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {...@@ -6083,6 +6165,7 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
6083 const ptr_mcv = try func.resolveInst(atomic_load.ptr);6165 const ptr_mcv = try func.resolveInst(atomic_load.ptr);
60846166
6085 const result_mcv = try func.allocRegOrMem(elem_ty, inst, true);6167 const result_mcv = try func.allocRegOrMem(elem_ty, inst, true);
6168 assert(result_mcv == .register); // should be less than 8 bytes
60866169
6087 if (order == .seq_cst) {6170 if (order == .seq_cst) {
6088 _ = try func.addInst(.{6171 _ = try func.addInst(.{
...@@ -6535,19 +6618,40 @@ fn getResolvedInstValue(func: *Func, inst: Air.Inst.Index) *InstTracking {...@@ -6535,19 +6618,40 @@ fn getResolvedInstValue(func: *Func, inst: Air.Inst.Index) *InstTracking {
6535}6618}
65366619
6537fn genTypedValue(func: *Func, val: Value) InnerError!MCValue {6620fn genTypedValue(func: *Func, val: Value) InnerError!MCValue {
6538 const pt = func.pt;6621 const zcu = func.pt.zcu;
6539 const zcu = pt.zcu;6622 const gpa = func.gpa;
6623
6624 const owner_decl_index = zcu.funcOwnerDeclIndex(func.func_index);
6625 const lf = func.bin_file;
6626 const src_loc = func.src_loc;
6627
6628 if (val.isUndef(zcu)) {
6629 const local_sym_index = lf.lowerUnnamedConst(func.pt, val, owner_decl_index) catch |err| {
6630 const msg = try ErrorMsg.create(gpa, src_loc, "lowering unnamed undefined constant failed: {s}", .{@errorName(err)});
6631 func.err_msg = msg;
6632 return error.CodegenFail;
6633 };
6634 switch (lf.tag) {
6635 .elf => {
6636 const elf_file = lf.cast(link.File.Elf).?;
6637 const local = elf_file.symbol(local_sym_index);
6638 return MCValue{ .undef = local.esym_index };
6639 },
6640 else => unreachable,
6641 }
6642 }
6643
6540 const result = try codegen.genTypedValue(6644 const result = try codegen.genTypedValue(
6541 func.bin_file,6645 lf,
6542 pt,6646 func.pt,
6543 func.src_loc,6647 src_loc,
6544 val,6648 val,
6545 zcu.funcOwnerDeclIndex(func.func_index),6649 owner_decl_index,
6546 );6650 );
6547 const mcv: MCValue = switch (result) {6651 const mcv: MCValue = switch (result) {
6548 .mcv => |mcv| switch (mcv) {6652 .mcv => |mcv| switch (mcv) {
6549 .none => .none,6653 .none => .none,
6550 .undef => .undef,6654 .undef => unreachable,
6551 .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } },6655 .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } },
6552 .immediate => |imm| .{ .immediate = imm },6656 .immediate => |imm| .{ .immediate = imm },
6553 .memory => |addr| .{ .memory = addr },6657 .memory => |addr| .{ .memory = addr },
src/arch/riscv64/Encoding.zig+139-21
...@@ -28,7 +28,7 @@ const OpCode = enum(u7) {...@@ -28,7 +28,7 @@ const OpCode = enum(u7) {
28 NONE = 0b00000000,28 NONE = 0b00000000,
29};29};
3030
31const Fmt = enum(u2) {31const FpFmt = enum(u2) {
32 /// 32-bit single-precision32 /// 32-bit single-precision
33 S = 0b00,33 S = 0b00,
34 /// 64-bit double-precision34 /// 64-bit double-precision
...@@ -40,6 +40,11 @@ const Fmt = enum(u2) {...@@ -40,6 +40,11 @@ const Fmt = enum(u2) {
40 Q = 0b11,40 Q = 0b11,
41};41};
4242
43const AmoWidth = enum(u3) {
44 W = 0b010,
45 D = 0b011,
46};
47
43const Enc = struct {48const Enc = struct {
44 opcode: OpCode,49 opcode: OpCode,
4550
...@@ -49,11 +54,15 @@ const Enc = struct {...@@ -49,11 +54,15 @@ const Enc = struct {
49 funct3: u3,54 funct3: u3,
50 funct7: u7,55 funct7: u7,
51 },56 },
57 amo: struct {
58 funct5: u5,
59 width: AmoWidth,
60 },
52 /// funct5 + rm + fmt61 /// funct5 + rm + fmt
53 fmt: struct {62 fmt: struct {
54 funct5: u5,63 funct5: u5,
55 rm: u3,64 rm: u3,
56 fmt: Fmt,65 fmt: FpFmt,
57 },66 },
58 /// funct367 /// funct3
59 f: struct {68 f: struct {
...@@ -202,6 +211,27 @@ pub const Mnemonic = enum {...@@ -202,6 +211,27 @@ pub const Mnemonic = enum {
202 // MISC211 // MISC
203 fence,212 fence,
204213
214 // AMO
215 amoswapw,
216 amoaddw,
217 amoandw,
218 amoorw,
219 amoxorw,
220 amomaxw,
221 amominw,
222 amomaxuw,
223 amominuw,
224
225 amoswapd,
226 amoaddd,
227 amoandd,
228 amoord,
229 amoxord,
230 amomaxd,
231 amomind,
232 amomaxud,
233 amominud,
234
205 pub fn encoding(mnem: Mnemonic) Enc {235 pub fn encoding(mnem: Mnemonic) Enc {
206 return switch (mnem) {236 return switch (mnem) {
207 // zig fmt: off237 // zig fmt: off
...@@ -379,7 +409,34 @@ pub const Mnemonic = enum {...@@ -379,7 +409,34 @@ pub const Mnemonic = enum {
379 // MISC_MEM409 // MISC_MEM
380410
381 .fence => .{ .opcode = .MISC_MEM, .data = .{ .f = .{ .funct3 = 0b000 } } },411 .fence => .{ .opcode = .MISC_MEM, .data = .{ .f = .{ .funct3 = 0b000 } } },
382 412
413 // AMO
414
415 .amoaddw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00000 } } },
416 .amoswapw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00001 } } },
417 // LR.W
418 // SC.W
419 .amoxorw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00100 } } },
420 .amoandw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01100 } } },
421 .amoorw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01000 } } },
422 .amominw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b10000 } } },
423 .amomaxw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b10100 } } },
424 .amominuw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11000 } } },
425 .amomaxuw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11100 } } },
426
427 .amoaddd => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00000 } } },
428 .amoswapd => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00001 } } },
429 // LR.D
430 // SC.D
431 .amoxord => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00100 } } },
432 .amoandd => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01100 } } },
433 .amoord => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01000 } } },
434 .amomind => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b10000 } } },
435 .amomaxd => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b10100 } } },
436 .amominud => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11000 } } },
437 .amomaxud => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11100 } } },
438
439
383440
384 // zig fmt: on441 // zig fmt: on
385 };442 };
...@@ -395,6 +452,7 @@ pub const InstEnc = enum {...@@ -395,6 +452,7 @@ pub const InstEnc = enum {
395 U,452 U,
396 J,453 J,
397 fence,454 fence,
455 amo,
398 /// extras that have unusual op counts456 /// extras that have unusual op counts
399 system,457 system,
400458
...@@ -526,21 +584,43 @@ pub const InstEnc = enum {...@@ -526,21 +584,43 @@ pub const InstEnc = enum {
526584
527 .fence,585 .fence,
528 => .fence,586 => .fence,
587
588 .amoswapw,
589 .amoaddw,
590 .amoandw,
591 .amoorw,
592 .amoxorw,
593 .amomaxw,
594 .amominw,
595 .amomaxuw,
596 .amominuw,
597
598 .amoswapd,
599 .amoaddd,
600 .amoandd,
601 .amoord,
602 .amoxord,
603 .amomaxd,
604 .amomind,
605 .amomaxud,
606 .amominud,
607 => .amo,
529 };608 };
530 }609 }
531610
532 pub fn opsList(enc: InstEnc) [4]std.meta.FieldEnum(Operand) {611 pub fn opsList(enc: InstEnc) [5]std.meta.FieldEnum(Operand) {
533 return switch (enc) {612 return switch (enc) {
534 // zig fmt: off613 // zig fmt: off
535 .R => .{ .reg, .reg, .reg, .none },614 .R => .{ .reg, .reg, .reg, .none, .none, },
536 .R4 => .{ .reg, .reg, .reg, .reg }, 615 .R4 => .{ .reg, .reg, .reg, .reg, .none, },
537 .I => .{ .reg, .reg, .imm, .none },616 .I => .{ .reg, .reg, .imm, .none, .none, },
538 .S => .{ .reg, .reg, .imm, .none },617 .S => .{ .reg, .reg, .imm, .none, .none, },
539 .B => .{ .reg, .reg, .imm, .none },618 .B => .{ .reg, .reg, .imm, .none, .none, },
540 .U => .{ .reg, .imm, .none, .none },619 .U => .{ .reg, .imm, .none, .none, .none, },
541 .J => .{ .reg, .imm, .none, .none },620 .J => .{ .reg, .imm, .none, .none, .none, },
542 .system => .{ .none, .none, .none, .none },621 .system => .{ .none, .none, .none, .none, .none, },
543 .fence => .{ .barrier, .barrier, .none, .none },622 .fence => .{ .barrier, .barrier, .none, .none, .none, },
623 .amo => .{ .reg, .reg, .reg, .barrier, .barrier },
544 // zig fmt: on624 // zig fmt: on
545 };625 };
546 }626 }
...@@ -611,19 +691,29 @@ pub const Data = union(InstEnc) {...@@ -611,19 +691,29 @@ pub const Data = union(InstEnc) {
611 pred: u4,691 pred: u4,
612 _ignored: u4 = 0,692 _ignored: u4 = 0,
613 },693 },
614 system: void,694 amo: packed struct {
695 opcode: u7,
696 rd: u5,
697 funct3: u3,
698 rs1: u5,
699 rs2: u5,
700 rl: bool,
701 aq: bool,
702 funct5: u5,
703 },
704 system: u32,
705
706 comptime {
707 for (std.meta.fields(Data)) |field| {
708 assert(@bitSizeOf(field.type) == 32);
709 }
710 }
615711
616 pub fn toU32(self: Data) u32 {712 pub fn toU32(self: Data) u32 {
617 return switch (self) {713 return switch (self) {
618 // zig fmt: off714 // zig fmt: off
619 .R => |v| @bitCast(v),
620 .R4 => |v| @bitCast(v),
621 .I => |v| @bitCast(v),
622 .S => |v| @bitCast(v),
623 .B => |v| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.imm11)) << 7) + (@as(u32, @intCast(v.imm1_4)) << 8) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.rs2)) << 20) + (@as(u32, @intCast(v.imm5_10)) << 25) + (@as(u32, @intCast(v.imm12)) << 31),715 .B => |v| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.imm11)) << 7) + (@as(u32, @intCast(v.imm1_4)) << 8) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.rs2)) << 20) + (@as(u32, @intCast(v.imm5_10)) << 25) + (@as(u32, @intCast(v.imm12)) << 31),
624 .U => |v| @bitCast(v),716 inline else => |v| @bitCast(v),
625 .J => |v| @bitCast(v),
626 .fence => |v| @bitCast(v),
627 .system => unreachable,717 .system => unreachable,
628 // zig fmt: on718 // zig fmt: on
629 };719 };
...@@ -792,6 +882,34 @@ pub const Data = union(InstEnc) {...@@ -792,6 +882,34 @@ pub const Data = union(InstEnc) {
792 },882 },
793 };883 };
794 },884 },
885 .amo => {
886 assert(ops.len == 5);
887
888 const rd = ops[0];
889 const rs1 = ops[1];
890 const rs2 = ops[2];
891 const rl = ops[3];
892 const aq = ops[4];
893
894 const ret: Data = .{
895 .amo = .{
896 .rd = rd.reg.encodeId(),
897 .rs1 = rs1.reg.encodeId(),
898 .rs2 = rs2.reg.encodeId(),
899
900 // TODO: https://github.com/ziglang/zig/issues/20113
901 .rl = if (rl.barrier == .rl) true else false,
902 .aq = if (aq.barrier == .aq) true else false,
903
904 .opcode = @intFromEnum(enc.opcode),
905 .funct3 = @intFromEnum(enc.data.amo.width),
906 .funct5 = enc.data.amo.funct5,
907 },
908 };
909
910 std.debug.print("ret: {}, {}", .{ ret.amo.rl, rl.barrier == .rl });
911 return ret;
912 },
795913
796 else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}),914 else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}),
797 }915 }
src/arch/riscv64/Lower.zig+26
...@@ -412,6 +412,32 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {...@@ -412,6 +412,32 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
412 });412 });
413 },413 },
414414
415 .pseudo_amo => {
416 const amo = inst.data.amo;
417 const is_d = amo.ty.abiSize(pt) == 8;
418 const is_un = amo.ty.isUnsignedInt(pt.zcu);
419
420 const mnem: Encoding.Mnemonic = switch (amo.op) {
421 // zig fmt: off
422 .SWAP => if (is_d) .amoswapd else .amoswapw,
423 .ADD => if (is_d) .amoaddd else .amoaddw,
424 .AND => if (is_d) .amoandd else .amoandw,
425 .OR => if (is_d) .amoord else .amoorw,
426 .XOR => if (is_d) .amoxord else .amoxorw,
427 .MAX => if (is_d) if (is_un) .amomaxud else .amomaxd else if (is_un) .amomaxuw else .amomaxw,
428 .MIN => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw,
429 // zig fmt: on
430 };
431
432 try lower.emit(mnem, &.{
433 .{ .reg = inst.data.amo.rd },
434 .{ .reg = inst.data.amo.rs1 },
435 .{ .reg = inst.data.amo.rs2 },
436 .{ .barrier = inst.data.amo.rl },
437 .{ .barrier = inst.data.amo.aq },
438 });
439 },
440
415 else => return lower.fail("TODO lower: psuedo {s}", .{@tagName(inst.ops)}),441 else => return lower.fail("TODO lower: psuedo {s}", .{@tagName(inst.ops)}),
416 },442 },
417 }443 }
src/arch/riscv64/Mir.zig+34-2
...@@ -39,8 +39,6 @@ pub const Inst = struct {...@@ -39,8 +39,6 @@ pub const Inst = struct {
39 ecall,39 ecall,
40 unimp,40 unimp,
4141
42 fence,
43
44 add,42 add,
45 addw,43 addw,
46 sub,44 sub,
...@@ -82,6 +80,8 @@ pub const Inst = struct {...@@ -82,6 +80,8 @@ pub const Inst = struct {
82 sh,80 sh,
83 sb,81 sb,
8482
83 fence,
84
85 // M extension85 // M extension
86 mul,86 mul,
87 mulw,87 mulw,
...@@ -136,6 +136,9 @@ pub const Inst = struct {...@@ -136,6 +136,9 @@ pub const Inst = struct {
136 fltd,136 fltd,
137 fled,137 fled,
138138
139 /// A Extension Instructions
140 amo,
141
139 /// A pseudo-instruction. Used for anything that isn't 1:1 with an142 /// A pseudo-instruction. Used for anything that isn't 1:1 with an
140 /// assembly instruction.143 /// assembly instruction.
141 pseudo,144 pseudo,
...@@ -254,6 +257,16 @@ pub const Inst = struct {...@@ -254,6 +257,16 @@ pub const Inst = struct {
254 pred: Barrier,257 pred: Barrier,
255 succ: Barrier,258 succ: Barrier,
256 },259 },
260
261 amo: struct {
262 rd: Register,
263 rs1: Register,
264 rs2: Register,
265 aq: Barrier,
266 rl: Barrier,
267 op: AmoOp,
268 ty: Type,
269 },
257 };270 };
258271
259 pub const Ops = enum {272 pub const Ops = enum {
...@@ -343,6 +356,9 @@ pub const Inst = struct {...@@ -343,6 +356,9 @@ pub const Inst = struct {
343356
344 /// IORW, IORW357 /// IORW, IORW
345 fence,358 fence,
359
360 /// Ordering, Src, Addr, Dest
361 pseudo_amo,
346 };362 };
347363
348 // Make sure we don't accidentally make instructions bigger than expected.364 // Make sure we don't accidentally make instructions bigger than expected.
...@@ -379,9 +395,25 @@ pub const FrameLoc = struct {...@@ -379,9 +395,25 @@ pub const FrameLoc = struct {
379};395};
380396
381pub const Barrier = enum(u4) {397pub const Barrier = enum(u4) {
398 // Fence
382 r = 0b0001,399 r = 0b0001,
383 w = 0b0010,400 w = 0b0010,
384 rw = 0b0011,401 rw = 0b0011,
402
403 // Amo
404 none,
405 aq,
406 rl,
407};
408
409pub const AmoOp = enum(u5) {
410 SWAP,
411 ADD,
412 AND,
413 OR,
414 XOR,
415 MAX,
416 MIN,
385};417};
386418
387/// Returns the requested data, as well as the new index which is at the start of the419/// Returns the requested data, as well as the new index which is at the start of the
src/arch/riscv64/encoder.zig+5-3
...@@ -1,6 +1,6 @@...@@ -1,6 +1,6 @@
1pub const Instruction = struct {1pub const Instruction = struct {
2 encoding: Encoding,2 encoding: Encoding,
3 ops: [4]Operand = .{.none} ** 4,3 ops: [5]Operand = .{.none} ** 5,
44
5 pub const Operand = union(enum) {5 pub const Operand = union(enum) {
6 none,6 none,
...@@ -12,16 +12,18 @@ pub const Instruction = struct {...@@ -12,16 +12,18 @@ pub const Instruction = struct {
1212
13 pub fn new(mnemonic: Encoding.Mnemonic, ops: []const Operand) !Instruction {13 pub fn new(mnemonic: Encoding.Mnemonic, ops: []const Operand) !Instruction {
14 const encoding = (try Encoding.findByMnemonic(mnemonic, ops)) orelse {14 const encoding = (try Encoding.findByMnemonic(mnemonic, ops)) orelse {
15 std.log.err("no encoding found for: {s} [{s} {s} {s}]", .{15 std.log.err("no encoding found for: {s} [{s} {s} {s} {s} {s}]", .{
16 @tagName(mnemonic),16 @tagName(mnemonic),
17 @tagName(if (ops.len > 0) ops[0] else .none),17 @tagName(if (ops.len > 0) ops[0] else .none),
18 @tagName(if (ops.len > 1) ops[1] else .none),18 @tagName(if (ops.len > 1) ops[1] else .none),
19 @tagName(if (ops.len > 2) ops[2] else .none),19 @tagName(if (ops.len > 2) ops[2] else .none),
20 @tagName(if (ops.len > 3) ops[3] else .none),
21 @tagName(if (ops.len > 4) ops[4] else .none),
20 });22 });
21 return error.InvalidInstruction;23 return error.InvalidInstruction;
22 };24 };
2325
24 var result_ops: [4]Operand = .{.none} ** 4;26 var result_ops: [5]Operand = .{.none} ** 5;
25 @memcpy(result_ops[0..ops.len], ops);27 @memcpy(result_ops[0..ops.len], ops);
2628
27 return .{29 return .{
src/codegen.zig+2-1
...@@ -987,8 +987,9 @@ pub fn genTypedValue(...@@ -987,8 +987,9 @@ pub fn genTypedValue(
987987
988 log.debug("genTypedValue: val = {}", .{val.fmtValue(pt, null)});988 log.debug("genTypedValue: val = {}", .{val.fmtValue(pt, null)});
989989
990 if (val.isUndef(zcu))990 if (val.isUndef(zcu)) {
991 return GenResult.mcv(.undef);991 return GenResult.mcv(.undef);
992 }
992993
993 const owner_decl = zcu.declPtr(owner_decl_index);994 const owner_decl = zcu.declPtr(owner_decl_index);
994 const namespace = zcu.namespacePtr(owner_decl.src_namespace);995 const namespace = zcu.namespacePtr(owner_decl.src_namespace);
src/link/Elf/ZigObject.zig+11-7
...@@ -540,8 +540,8 @@ inline fn isGlobal(index: Symbol.Index) bool {...@@ -540,8 +540,8 @@ inline fn isGlobal(index: Symbol.Index) bool {
540540
541pub fn symbol(self: ZigObject, index: Symbol.Index) Symbol.Index {541pub fn symbol(self: ZigObject, index: Symbol.Index) Symbol.Index {
542 const actual_index = index & symbol_mask;542 const actual_index = index & symbol_mask;
543 if (isGlobal(index)) return self.global_symbols.items[actual_index];543 if (isGlobal(index)) return self.globals()[actual_index];
544 return self.local_symbols.items[actual_index];544 return self.locals()[actual_index];
545}545}
546546
547pub fn elfSym(self: *ZigObject, index: Symbol.Index) *elf.Elf64_Sym {547pub fn elfSym(self: *ZigObject, index: Symbol.Index) *elf.Elf64_Sym {
...@@ -1334,11 +1334,15 @@ fn lowerConst(...@@ -1334,11 +1334,15 @@ fn lowerConst(
13341334
1335 const sym_index = try self.addAtom(elf_file);1335 const sym_index = try self.addAtom(elf_file);
13361336
1337 const res = try codegen.generateSymbol(&elf_file.base, pt, src_loc, val, &code_buffer, .{1337 const res = try codegen.generateSymbol(
1338 .none = {},1338 &elf_file.base,
1339 }, .{1339 pt,
1340 .parent_atom_index = sym_index,1340 src_loc,
1341 });1341 val,
1342 &code_buffer,
1343 .{ .none = {} },
1344 .{ .parent_atom_index = sym_index },
1345 );
1342 const code = switch (res) {1346 const code = switch (res) {
1343 .ok => code_buffer.items,1347 .ok => code_buffer.items,
1344 .fail => |em| return .{ .fail = em },1348 .fail => |em| return .{ .fail = em },
test/behavior/atomics.zig-16
...@@ -188,21 +188,6 @@ test "atomic store" {...@@ -188,21 +188,6 @@ test "atomic store" {
188 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO188 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
189 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO189 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
190 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;190 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
191 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
192
193 var x: u32 = 0;
194 @atomicStore(u32, &x, 1, .seq_cst);
195 try expect(@atomicLoad(u32, &x, .seq_cst) == 1);
196 @atomicStore(u32, &x, 12345678, .seq_cst);
197 try expect(@atomicLoad(u32, &x, .seq_cst) == 12345678);
198}
199
200test "atomic store comptime" {
201 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
202 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
203 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
204 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
205 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
206191
207 try comptime testAtomicStore();192 try comptime testAtomicStore();
208 try testAtomicStore();193 try testAtomicStore();
...@@ -451,7 +436,6 @@ test "return @atomicStore, using it as a void value" {...@@ -451,7 +436,6 @@ test "return @atomicStore, using it as a void value" {
451 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO436 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
452 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO437 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
453 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;438 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
454 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
455439
456 const S = struct {440 const S = struct {
457 const A = struct {441 const A = struct {