authorgravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2024-07-26 12:43:47-07:00
committergravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2024-07-26 12:43:47-07:00
log846bd4036154e82fae5d9385c3e349c99840ac5f
treecba663f3e3df9aab800970bdd4b2a2bbde5061f4
parent9752bbfeb3091b5ee812f3892738a8f1731cb4d3
signaturelock-open Commit is signed but in an unrecognized format.

riscv: implement `@cmpxchg*` and remove fixes


3 files changed, 274 insertions(+), 48 deletions(-)

src/arch/riscv64/CodeGen.zig+272-45
......@@ -1287,13 +1287,11 @@ fn gen(func: *Func) !void {
12871287 // ret
12881288 _ = try func.addInst(.{
12891289 .tag = .jalr,
1290 .data = .{
1291 .i_type = .{
1292 .rd = .zero,
1293 .rs1 = .ra,
1294 .imm12 = Immediate.s(0),
1295 },
1296 },
1290 .data = .{ .i_type = .{
1291 .rd = .zero,
1292 .rs1 = .ra,
1293 .imm12 = Immediate.s(0),
1294 } },
12971295 });
12981296
12991297 const frame_layout = try func.computeFrameLayout();
......@@ -1472,14 +1470,11 @@ fn genLazy(func: *Func, lazy_sym: link.File.LazySymbol) InnerError!void {
14721470
14731471 _ = try func.addInst(.{
14741472 .tag = .jalr,
1475
1476 .data = .{
1477 .i_type = .{
1478 .rd = .zero,
1479 .rs1 = .ra,
1480 .imm12 = Immediate.s(0),
1481 },
1482 },
1473 .data = .{ .i_type = .{
1474 .rd = .zero,
1475 .rs1 = .ra,
1476 .imm12 = Immediate.s(0),
1477 } },
14831478 });
14841479 },
14851480 else => return func.fail(
......@@ -1629,8 +1624,8 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
16291624 .struct_field_val=> try func.airStructFieldVal(inst),
16301625 .float_from_int => try func.airFloatFromInt(inst),
16311626 .int_from_float => try func.airIntFromFloat(inst),
1632 .cmpxchg_strong => try func.airCmpxchg(inst),
1633 .cmpxchg_weak => try func.airCmpxchg(inst),
1627 .cmpxchg_strong => try func.airCmpxchg(inst, .strong),
1628 .cmpxchg_weak => try func.airCmpxchg(inst, .weak),
16341629 .atomic_rmw => try func.airAtomicRmw(inst),
16351630 .atomic_load => try func.airAtomicLoad(inst),
16361631 .memcpy => try func.airMemcpy(inst),
......@@ -3527,7 +3522,8 @@ fn airWrapOptional(func: *Func, inst: Air.Inst.Index) !void {
35273522 };
35283523 defer if (pl_lock) |lock| func.register_manager.unlockReg(lock);
35293524
3530 const opt_mcv = try func.allocRegOrMem(opt_ty, inst, true);
3525 const opt_mcv = try func.allocRegOrMem(opt_ty, inst, false);
3526 try func.genCopy(pl_ty, opt_mcv, pl_mcv);
35313527
35323528 if (!same_repr) {
35333529 const pl_abi_size: i32 = @intCast(pl_ty.abiSize(pt));
......@@ -3541,18 +3537,6 @@ fn airWrapOptional(func: *Func, inst: Air.Inst.Index) !void {
35413537 .{ .immediate = 1 },
35423538 );
35433539 },
3544
3545 .register => |opt_reg| {
3546 try func.genBinOp(
3547 .shl,
3548 .{ .immediate = 1 },
3549 Type.u64,
3550 .{ .immediate = 32 },
3551 Type.u64,
3552 opt_reg,
3553 );
3554 try func.genCopy(pl_ty, opt_mcv, pl_mcv);
3555 },
35563540 else => unreachable,
35573541 }
35583542 }
......@@ -5800,6 +5784,7 @@ fn performReloc(func: *Func, inst: Mir.Inst.Index) void {
58005784
58015785 switch (tag) {
58025786 .beq,
5787 .bne,
58035788 => func.mir_instructions.items(.data)[inst].b_type.inst = target,
58045789 .jal => func.mir_instructions.items(.data)[inst].j_type.inst = target,
58055790 .pseudo_j => func.mir_instructions.items(.data)[inst].j_type.inst = target,
......@@ -6047,30 +6032,85 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
60476032 }
60486033 }
60496034
6035 const Label = struct {
6036 target: Mir.Inst.Index = undefined,
6037 pending_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .{},
6038
6039 const Kind = enum { definition, reference };
6040
6041 fn isValid(kind: Kind, name: []const u8) bool {
6042 for (name, 0..) |c, i| switch (c) {
6043 else => return false,
6044 '$' => if (i == 0) return false,
6045 '.' => {},
6046 '0'...'9' => if (i == 0) switch (kind) {
6047 .definition => if (name.len != 1) return false,
6048 .reference => {
6049 if (name.len != 2) return false;
6050 switch (name[1]) {
6051 else => return false,
6052 'B', 'F', 'b', 'f' => {},
6053 }
6054 },
6055 },
6056 '@', 'A'...'Z', '_', 'a'...'z' => {},
6057 };
6058 return name.len > 0;
6059 }
6060 };
6061 var labels: std.StringHashMapUnmanaged(Label) = .{};
6062 defer {
6063 var label_it = labels.valueIterator();
6064 while (label_it.next()) |label| label.pending_relocs.deinit(func.gpa);
6065 labels.deinit(func.gpa);
6066 }
6067
60506068 const asm_source = std.mem.sliceAsBytes(func.air.extra[extra_i..])[0..extra.data.source_len];
60516069 var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");
60526070 next_line: while (line_it.next()) |line| {
60536071 var mnem_it = mem.tokenizeAny(u8, line, " \t");
6054 const instruction: union(enum) { mnem: Mnemonic, pseudo: Pseudo } = while (mnem_it.next()) |mnem_str| {
6072 const mnem_str = while (mnem_it.next()) |mnem_str| {
60556073 if (mem.startsWith(u8, mnem_str, "#")) continue :next_line;
60566074 if (mem.startsWith(u8, mnem_str, "//")) continue :next_line;
6057 if (std.meta.stringToEnum(Mnemonic, mnem_str)) |mnem| {
6058 break .{ .mnem = mnem };
6059 } else if (std.meta.stringToEnum(Pseudo, mnem_str)) |pseudo| {
6060 break .{ .pseudo = pseudo };
6061 } else return func.fail("TODO: airAsm labels, found '{s}'", .{mnem_str});
6075 if (!mem.endsWith(u8, mnem_str, ":")) break mnem_str;
6076 const label_name = mnem_str[0 .. mnem_str.len - ":".len];
6077 if (!Label.isValid(.definition, label_name))
6078 return func.fail("invalid label: '{s}'", .{label_name});
6079
6080 const label_gop = try labels.getOrPut(func.gpa, label_name);
6081 if (!label_gop.found_existing) label_gop.value_ptr.* = .{} else {
6082 const anon = std.ascii.isDigit(label_name[0]);
6083 if (!anon and label_gop.value_ptr.pending_relocs.items.len == 0)
6084 return func.fail("redefined label: '{s}'", .{label_name});
6085 for (label_gop.value_ptr.pending_relocs.items) |pending_reloc|
6086 func.performReloc(pending_reloc);
6087 if (anon)
6088 label_gop.value_ptr.pending_relocs.clearRetainingCapacity()
6089 else
6090 label_gop.value_ptr.pending_relocs.clearAndFree(func.gpa);
6091 }
6092 label_gop.value_ptr.target = @intCast(func.mir_instructions.len);
60626093 } else continue;
60636094
6095 const instruction: union(enum) { mnem: Mnemonic, pseudo: Pseudo } =
6096 if (std.meta.stringToEnum(Mnemonic, mnem_str)) |mnem|
6097 .{ .mnem = mnem }
6098 else if (std.meta.stringToEnum(Pseudo, mnem_str)) |pseudo|
6099 .{ .pseudo = pseudo }
6100 else
6101 return func.fail("invalid mnem str '{s}'", .{mnem_str});
6102
60646103 const Operand = union(enum) {
60656104 none,
60666105 reg: Register,
60676106 imm: Immediate,
6107 inst: Mir.Inst.Index,
60686108 sym: SymbolOffset,
60696109 };
60706110
60716111 var ops: [4]Operand = .{.none} ** 4;
60726112 var last_op = false;
6073 var op_it = mem.splitScalar(u8, mnem_it.rest(), ',');
6113 var op_it = mem.splitAny(u8, mnem_it.rest(), ",(");
60746114 next_op: for (&ops) |*op| {
60756115 const op_str = while (!last_op) {
60766116 const full_str = op_it.next() orelse break :next_op;
......@@ -6109,6 +6149,25 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
61096149 return func.fail("invalid modified '{s}'", .{modifier}),
61106150 else => return func.fail("invalid constraint: '{s}'", .{op_str}),
61116151 };
6152 } else if (mem.endsWith(u8, op_str, ")")) {
6153 const reg = op_str[0 .. op_str.len - ")".len];
6154 const addr_reg = parseRegName(reg) orelse
6155 return func.fail("expected valid register, found '{s}'", .{reg});
6156
6157 op.* = .{ .reg = addr_reg };
6158 } else if (Label.isValid(.reference, op_str)) {
6159 const anon = std.ascii.isDigit(op_str[0]);
6160 const label_gop = try labels.getOrPut(func.gpa, op_str[0..if (anon) 1 else op_str.len]);
6161 if (!label_gop.found_existing) label_gop.value_ptr.* = .{};
6162 if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing)
6163 return func.fail("undefined label: '{s}'", .{op_str});
6164 const pending_relocs = &label_gop.value_ptr.pending_relocs;
6165 if (if (anon)
6166 op_str[1] == 'f' or op_str[1] == 'F'
6167 else
6168 !label_gop.found_existing or pending_relocs.items.len > 0)
6169 try pending_relocs.append(func.gpa, @intCast(func.mir_instructions.len));
6170 op.* = .{ .inst = label_gop.value_ptr.target };
61126171 } else return func.fail("invalid operand: '{s}'", .{op_str});
61136172 } else if (op_it.next()) |op_str| return func.fail("extra operand: '{s}'", .{op_str});
61146173
......@@ -6131,6 +6190,39 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
61316190 }),
61326191 else => error.InvalidInstruction,
61336192 },
6193 .imm => |imm1| switch (ops[2]) {
6194 .reg => |reg2| switch (mnem) {
6195 .sd => try func.addInst(.{
6196 .tag = mnem,
6197 .data = .{ .i_type = .{
6198 .rd = reg2,
6199 .rs1 = reg1,
6200 .imm12 = imm1,
6201 } },
6202 }),
6203 .ld => try func.addInst(.{
6204 .tag = mnem,
6205 .data = .{ .i_type = .{
6206 .rd = reg1,
6207 .rs1 = reg2,
6208 .imm12 = imm1,
6209 } },
6210 }),
6211 else => error.InvalidInstruction,
6212 },
6213 else => error.InvalidInstruction,
6214 },
6215 .none => switch (mnem) {
6216 .jalr => try func.addInst(.{
6217 .tag = mnem,
6218 .data = .{ .i_type = .{
6219 .rd = .zero,
6220 .rs1 = reg1,
6221 .imm12 = Immediate.s(0),
6222 } },
6223 }),
6224 else => error.InvalidInstruction,
6225 },
61346226 else => error.InvalidInstruction,
61356227 },
61366228 else => error.InvalidInstruction,
......@@ -6196,6 +6288,28 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
61966288 } },
61976289 });
61986290 },
6291 .ret => _ = try func.addInst(.{
6292 .tag = .jalr,
6293 .data = .{ .i_type = .{
6294 .rd = .zero,
6295 .rs1 = .ra,
6296 .imm12 = Immediate.s(0),
6297 } },
6298 }),
6299 .beqz => blk: {
6300 if (ops[0] != .reg or ops[1] != .inst) {
6301 break :blk error.InvalidInstruction;
6302 }
6303
6304 _ = try func.addInst(.{
6305 .tag = .beq,
6306 .data = .{ .b_type = .{
6307 .rs1 = ops[0].reg,
6308 .rs2 = .zero,
6309 .inst = ops[1].inst,
6310 } },
6311 });
6312 },
61996313 })) catch |err| {
62006314 switch (err) {
62016315 error.InvalidInstruction => return func.fail(
......@@ -6215,6 +6329,10 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
62156329 }
62166330 }
62176331
6332 var label_it = labels.iterator();
6333 while (label_it.next()) |label| if (label.value_ptr.pending_relocs.items.len > 0)
6334 return func.fail("undefined label: '{s}'", .{label.key_ptr.*});
6335
62186336 for (outputs, args.items[0..outputs.len]) |output, arg_mcv| {
62196337 const extra_bytes = mem.sliceAsBytes(func.air.extra[outputs_extra_i..]);
62206338 const constraint =
......@@ -7203,14 +7321,123 @@ fn airIntFromFloat(func: *Func, inst: Air.Inst.Index) !void {
72037321 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
72047322}
72057323
7206fn airCmpxchg(func: *Func, inst: Air.Inst.Index) !void {
7324fn airCmpxchg(func: *Func, inst: Air.Inst.Index, strength: enum { weak, strong }) !void {
7325 _ = strength; // TODO: do something with this
7326
7327 const pt = func.pt;
72077328 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
7208 const extra = func.air.extraData(Air.Block, ty_pl.payload);
7209 _ = extra;
7210 return func.fail("TODO implement airCmpxchg for {}", .{
7211 func.target.cpu.arch,
7329 const extra = func.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
7330
7331 const ptr_ty = func.typeOf(extra.ptr);
7332 const val_ty = func.typeOf(extra.expected_value);
7333 const val_abi_size: u32 = @intCast(val_ty.abiSize(pt));
7334
7335 switch (val_abi_size) {
7336 1, 2, 4, 8 => {},
7337 else => return func.fail("TODO: airCmpxchg Int size {}", .{val_abi_size}),
7338 }
7339
7340 const succ_order: struct { aq: Mir.Barrier, rl: Mir.Barrier } = switch (extra.successOrder()) {
7341 .unordered,
7342 .release,
7343 .acq_rel,
7344 => unreachable,
7345
7346 .monotonic => .{ .aq = .none, .rl = .none },
7347 .acquire => .{ .aq = .aq, .rl = .none },
7348 .seq_cst => .{ .aq = .aq, .rl = .rl },
7349 };
7350
7351 const ptr_mcv = try func.resolveInst(extra.ptr);
7352 const ptr_reg, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv);
7353 defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock);
7354
7355 const exp_mcv = try func.resolveInst(extra.expected_value);
7356 const exp_reg, const exp_lock = try func.promoteReg(val_ty, exp_mcv);
7357 defer if (exp_lock) |lock| func.register_manager.unlockReg(lock);
7358 try func.truncateRegister(val_ty, exp_reg);
7359
7360 const new_mcv = try func.resolveInst(extra.new_value);
7361 const new_reg, const new_lock = try func.promoteReg(val_ty, new_mcv);
7362 defer if (new_lock) |lock| func.register_manager.unlockReg(lock);
7363 try func.truncateRegister(val_ty, new_reg);
7364
7365 const branch_reg, const branch_lock = try func.allocReg(.int);
7366 defer func.register_manager.unlockReg(branch_lock);
7367
7368 const fallthrough_reg, const fallthrough_lock = try func.allocReg(.int);
7369 defer func.register_manager.unlockReg(fallthrough_lock);
7370
7371 const jump_back = try func.addInst(.{
7372 .tag = if (val_ty.bitSize(pt) <= 32) .lrw else .lrd,
7373 .data = .{ .amo = .{
7374 .aq = succ_order.aq,
7375 .rl = succ_order.rl,
7376 .rd = branch_reg,
7377 .rs1 = ptr_reg,
7378 .rs2 = .zero,
7379 } },
72127380 });
7213 // return func.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
7381 try func.truncateRegister(val_ty, branch_reg);
7382
7383 const jump_forward = try func.addInst(.{
7384 .tag = .bne,
7385 .data = .{ .b_type = .{
7386 .rs1 = branch_reg,
7387 .rs2 = exp_reg,
7388 .inst = undefined,
7389 } },
7390 });
7391
7392 _ = try func.addInst(.{
7393 .tag = if (val_ty.bitSize(pt) <= 32) .scw else .scd,
7394 .data = .{ .amo = .{
7395 .aq = .none,
7396 .rl = succ_order.rl,
7397 .rd = fallthrough_reg,
7398 .rs1 = ptr_reg,
7399 .rs2 = new_reg,
7400 } },
7401 });
7402 try func.truncateRegister(Type.bool, fallthrough_reg);
7403
7404 _ = try func.addInst(.{
7405 .tag = .bne,
7406 .data = .{ .b_type = .{
7407 .rs1 = fallthrough_reg,
7408 .rs2 = .zero,
7409 .inst = jump_back,
7410 } },
7411 });
7412
7413 func.performReloc(jump_forward);
7414
7415 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
7416 const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, false);
7417
7418 const tmp_reg, const tmp_lock = try func.allocReg(.int);
7419 defer func.register_manager.unlockReg(tmp_lock);
7420
7421 try func.genBinOp(
7422 .cmp_neq,
7423 .{ .register = branch_reg },
7424 val_ty,
7425 .{ .register = exp_reg },
7426 val_ty,
7427 tmp_reg,
7428 );
7429
7430 try func.genCopy(val_ty, dst_mcv, .{ .register = branch_reg });
7431 try func.genCopy(
7432 Type.bool,
7433 dst_mcv.address().offset(@intCast(val_abi_size)).deref(),
7434 .{ .register = tmp_reg },
7435 );
7436
7437 break :result dst_mcv;
7438 };
7439
7440 return func.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
72147441}
72157442
72167443fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
......@@ -7234,8 +7461,8 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
72347461 return func.fail("TODO: airAtomicRmw non-pow 2", .{});
72357462
72367463 switch (val_ty.zigTypeTag(pt.zcu)) {
7237 .Int => {},
7238 inline .Bool, .Float, .Enum, .Pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}),
7464 .Enum, .Int => {},
7465 inline .Bool, .Float, .Pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}),
72397466 else => unreachable,
72407467 }
72417468
src/arch/riscv64/mnem.zig+2
......@@ -252,4 +252,6 @@ pub const Pseudo = enum(u8) {
252252 li,
253253 mv,
254254 tail,
255 beqz,
256 ret,
255257};
test/behavior/atomics.zig-3
......@@ -15,7 +15,6 @@ test "cmpxchg" {
1515 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1616 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1717 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
18 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1918
2019 try testCmpxchg();
2120 try comptime testCmpxchg();
......@@ -108,7 +107,6 @@ test "cmpxchg with ignored result" {
108107 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
109108 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
110109 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
111 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
112110
113111 var x: i32 = 1234;
114112
......@@ -153,7 +151,6 @@ test "cmpxchg on a global variable" {
153151 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
154152 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
155153 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
156 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
157154
158155 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
159156 // https://github.com/ziglang/zig/issues/10627