Commit 7ff5709e1b04de4c33988ce6a27bc593dcf20f63 | zig

author	David Rubin <david@vortan.dev> 2024-07-25 23:01:44-07:00
committer	David Rubin <david@vortan.dev> 2024-07-26 04:19:57-07:00
log	7ff5709e1b04de4c33988ce6a27bc593dcf20f63
tree	da08f073bec966042a0815579b5b68ceb9399c29
parent	a1f6a8ef90f0778e4fc5d314eeb1f0a0a93fa53a
signature	Commit is signed but in an unrecognized format.

riscv: implement `lr/sr` loop logic for non-native atomics

7 files changed, 138 insertions(+), 113 deletions(-)

lib
- std
  - start.zig
src
- arch
  - riscv64
    CodeGen.zig
    Lower.zig
    Mir.zig
    abi.zig
    encoding.zig
    mnem.zig

lib/std/start.zig+2-4

...	...	@@ -478,10 +478,8 @@ inline fn callMainWithArgs(argc: usize, argv: [][:0]u8, envp: [][*:0]u8) u8 {
478	478	std.os.argv = argv[0..argc];
479	479	std.os.environ = envp;
480	480
481		if (builtin.zig_backend != .stage2_riscv64) {
482		std.debug.maybeEnableSegfaultHandler();
483		maybeIgnoreSigpipe();
484		}
	481	std.debug.maybeEnableSegfaultHandler();
	482	maybeIgnoreSigpipe();
485	483
486	484	return callMain();
487	485	}

src/arch/riscv64/CodeGen.zig+113-53

...	...	@@ -4717,14 +4717,11 @@ fn airFence(func: *Func, inst: Air.Inst.Index) !void {
4717	4717	};
4718	4718
4719	4719	_ = try func.addInst(.{
4720		.tag = .pseudo_fence,
4721		.data = .{
4722		.fence = .{
4723		.pred = pred,
4724		.succ = succ,
4725		.fm = if (order == .acq_rel) .tso else .none,
4726		},
4727		},
	4720	.tag = if (order == .acq_rel) .fencetso else .fence,
	4721	.data = .{ .fence = .{
	4722	.pred = pred,
	4723	.succ = succ,
	4724	} },
4728	4725	});
4729	4726	return func.finishAirBookkeeping();
4730	4727	}
...	...	@@ -5278,12 +5275,12 @@ fn isNull(func: *Func, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
5278	5275	.dead,
5279	5276	.undef,
5280	5277	.immediate,
5281		.register_pair,
5282	5278	.register_offset,
5283	5279	.lea_frame,
5284	5280	.lea_symbol,
5285	5281	.reserved_frame,
5286	5282	.air_ref,
	5283	.register_pair,
5287	5284	=> unreachable,
5288	5285
5289	5286	.register => \|opt_reg\| {
...	...	@@ -7109,6 +7106,7 @@ fn airCmpxchg(func: *Func, inst: Air.Inst.Index) !void {
7109	7106
7110	7107	fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
7111	7108	const pt = func.pt;
	7109	const zcu = pt.zcu;
7112	7110	const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7113	7111	const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data;
7114	7112
...	...	@@ -7131,11 +7129,11 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
7131	7129	else => unreachable,
7132	7130	}
7133	7131
7134		switch (val_size) {
7135		1, 2 => return func.fail("TODO: airAtomicRmw {s} Int {}", .{ @tagName(op), val_size }),
7136		4, 8 => {},
	7132	const method: enum { amo, loop } = switch (val_size) {
	7133	1, 2 => .loop,
	7134	4, 8 => .amo,
7137	7135	else => unreachable,
7138		}
	7136	};
7139	7137
7140	7138	const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv);
7141	7139	defer if (ptr_lock) \|lock\| func.register_manager.unlockReg(lock);
...	...	@@ -7145,6 +7143,7 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
7145	7143
7146	7144	const result_mcv = try func.allocRegOrMem(val_ty, inst, true);
7147	7145	assert(result_mcv == .register); // should fit into 8 bytes
	7146	const result_reg = result_mcv.register;
7148	7147
7149	7148	const aq, const rl = switch (order) {
7150	7149	.unordered => unreachable,
...	...	@@ -7155,28 +7154,96 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
7155	7154	.seq_cst => .{ true, true },
7156	7155	};
7157	7156
7158		_ = try func.addInst(.{
7159		.tag = .pseudo_amo,
7160		.data = .{ .amo = .{
7161		.rd = result_mcv.register,
7162		.rs1 = ptr_register,
7163		.rs2 = val_register,
7164		.aq = if (aq) .aq else .none,
7165		.rl = if (rl) .rl else .none,
7166		.op = switch (op) {
7167		.Xchg => .SWAP,
7168		.Add => .ADD,
7169		.Sub => return func.fail("TODO: airAtomicRmw SUB", .{}),
7170		.And => .AND,
7171		.Nand => return func.fail("TODO: airAtomicRmw NAND", .{}),
7172		.Or => .OR,
7173		.Xor => .XOR,
7174		.Max => .MAX,
7175		.Min => .MIN,
7176		},
7177		.ty = val_ty,
7178		} },
7179		});
	7157	switch (method) {
	7158	.amo => {
	7159	const is_d = val_ty.abiSize(pt) == 8;
	7160	const is_un = val_ty.isUnsignedInt(zcu);
	7161
	7162	const mnem: Mnemonic = switch (op) {
	7163	// zig fmt: off
	7164	.Xchg => if (is_d) .amoswapd else .amoswapw,
	7165	.Add => if (is_d) .amoaddd else .amoaddw,
	7166	.And => if (is_d) .amoandd else .amoandw,
	7167	.Or => if (is_d) .amoord else .amoorw,
	7168	.Xor => if (is_d) .amoxord else .amoxorw,
	7169	.Max => if (is_d) if (is_un) .amomaxud else .amomaxd else if (is_un) .amomaxuw else .amomaxw,
	7170	.Min => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw,
	7171	else => return func.fail("TODO: airAtomicRmw amo {s}", .{@tagName(op)}),
	7172	// zig fmt: on
	7173	};
	7174
	7175	_ = try func.addInst(.{
	7176	.tag = mnem,
	7177	.data = .{ .amo = .{
	7178	.rd = result_reg,
	7179	.rs1 = ptr_register,
	7180	.rs2 = val_register,
	7181	.aq = if (aq) .aq else .none,
	7182	.rl = if (rl) .rl else .none,
	7183	} },
	7184	});
	7185	},
	7186	.loop => {
	7187	// where we'll jump back when the sc fails
	7188	const jump_back = try func.addInst(.{
	7189	.tag = .lrw,
	7190	.data = .{ .amo = .{
	7191	.rd = result_reg,
	7192	.rs1 = ptr_register,
	7193	.rs2 = .zero,
	7194	.aq = if (aq) .aq else .none,
	7195	.rl = if (rl) .rl else .none,
	7196	} },
	7197	});
	7198
	7199	const after_reg, const after_lock = try func.allocReg(.int);
	7200	defer func.register_manager.unlockReg(after_lock);
	7201
	7202	switch (op) {
	7203	.Add => {
	7204	_ = try func.genBinOp(
	7205	.add,
	7206	.{ .register = result_reg },
	7207	val_ty,
	7208	.{ .register = val_register },
	7209	val_ty,
	7210	after_reg,
	7211	);
	7212	},
	7213	.Sub => {
	7214	_ = try func.genBinOp(
	7215	.sub,
	7216	.{ .register = result_reg },
	7217	val_ty,
	7218	.{ .register = val_register },
	7219	val_ty,
	7220	after_reg,
	7221	);
	7222	},
	7223	else => return func.fail("TODO: airAtomicRmw loop {s}", .{@tagName(op)}),
	7224	}
	7225
	7226	_ = try func.addInst(.{
	7227	.tag = .scw,
	7228	.data = .{ .amo = .{
	7229	.rd = after_reg,
	7230	.rs1 = ptr_register,
	7231	.rs2 = after_reg,
	7232	.aq = if (aq) .aq else .none,
	7233	.rl = if (rl) .rl else .none,
	7234	} },
	7235	});
	7236
	7237	_ = try func.addInst(.{
	7238	.tag = .bne,
	7239	.data = .{ .b_type = .{
	7240	.inst = jump_back,
	7241	.rs1 = after_reg,
	7242	.rs2 = .zero,
	7243	} },
	7244	});
	7245	},
	7246	}
7180	7247
7181	7248	return func.finishAir(inst, result_mcv, .{ pl_op.operand, extra.operand, .none });
7182	7249	}
...	...	@@ -7199,11 +7266,10 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
7199	7266
7200	7267	if (order == .seq_cst) {
7201	7268	_ = try func.addInst(.{
7202		.tag = .pseudo_fence,
	7269	.tag = .fence,
7203	7270	.data = .{ .fence = .{
7204	7271	.pred = .rw,
7205	7272	.succ = .rw,
7206		.fm = .none,
7207	7273	} },
7208	7274	});
7209	7275	}
...	...	@@ -7217,14 +7283,11 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
7217	7283	// Make sure all previous reads happen before any reading or writing accurs.
7218	7284	.seq_cst, .acquire => {
7219	7285	_ = try func.addInst(.{
7220		.tag = .pseudo_fence,
7221		.data = .{
7222		.fence = .{
7223		.pred = .r,
7224		.succ = .rw,
7225		.fm = .none,
7226		},
7227		},
	7286	.tag = .fence,
	7287	.data = .{ .fence = .{
	7288	.pred = .r,
	7289	.succ = .rw,
	7290	} },
7228	7291	});
7229	7292	},
7230	7293	else => unreachable,
...	...	@@ -7249,14 +7312,11 @@ fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOr
7249	7312	.unordered, .monotonic => {},
7250	7313	.release, .seq_cst => {
7251	7314	_ = try func.addInst(.{
7252		.tag = .pseudo_fence,
7253		.data = .{
7254		.fence = .{
7255		.pred = .rw,
7256		.succ = .w,
7257		.fm = .none,
7258		},
7259		},
	7315	.tag = .fence,
	7316	.data = .{ .fence = .{
	7317	.pred = .rw,
	7318	.succ = .w,
	7319	} },
7260	7320	});
7261	7321	},
7262	7322	else => unreachable,

src/arch/riscv64/Lower.zig+11-38

...	...	@@ -446,44 +446,6 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index, options: struct {
446	446	.{ .imm = Immediate.s(0) },
447	447	});
448	448	},
449
450		.pseudo_amo => {
451		const amo = inst.data.amo;
452		const is_d = amo.ty.abiSize(pt) == 8;
453		const is_un = amo.ty.isUnsignedInt(pt.zcu);
454
455		const mnem: Mnemonic = switch (amo.op) {
456		// zig fmt: off
457		.SWAP => if (is_d) .amoswapd else .amoswapw,
458		.ADD => if (is_d) .amoaddd else .amoaddw,
459		.AND => if (is_d) .amoandd else .amoandw,
460		.OR => if (is_d) .amoord else .amoorw,
461		.XOR => if (is_d) .amoxord else .amoxorw,
462		.MAX => if (is_d) if (is_un) .amomaxud else .amomaxd else if (is_un) .amomaxuw else .amomaxw,
463		.MIN => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw,
464		// zig fmt: on
465		};
466
467		try lower.emit(mnem, &.{
468		.{ .reg = inst.data.amo.rd },
469		.{ .reg = inst.data.amo.rs1 },
470		.{ .reg = inst.data.amo.rs2 },
471		.{ .barrier = inst.data.amo.rl },
472		.{ .barrier = inst.data.amo.aq },
473		});
474		},
475
476		.pseudo_fence => {
477		const fence = inst.data.fence;
478
479		try lower.emit(switch (fence.fm) {
480		.tso => .fencetso,
481		.none => .fence,
482		}, &.{
483		.{ .barrier = fence.succ },
484		.{ .barrier = fence.pred },
485		});
486		},
487	449	}
488	450
489	451	return .{
...	...	@@ -524,6 +486,17 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
524	486	.{ .reg = csr.rs1 },
525	487	.{ .reg = csr.rd },
526	488	},
	489	.amo => \|amo\| &.{
	490	.{ .reg = amo.rd },
	491	.{ .reg = amo.rs1 },
	492	.{ .reg = amo.rs2 },
	493	.{ .barrier = amo.rl },
	494	.{ .barrier = amo.aq },
	495	},
	496	.fence => \|fence\| &.{
	497	.{ .barrier = fence.succ },
	498	.{ .barrier = fence.pred },
	499	},
527	500	else => return lower.fail("TODO: generic lower {s}", .{@tagName(inst.data)}),
528	501	});
529	502	}

src/arch/riscv64/Mir.zig-6

...	...	@@ -73,10 +73,6 @@ pub const Inst = struct {
73	73	fence: struct {
74	74	pred: Barrier,
75	75	succ: Barrier,
76		fm: enum {
77		none,
78		tso,
79		},
80	76	},
81	77	amo: struct {
82	78	rd: Register,
...	...	@@ -84,8 +80,6 @@ pub const Inst = struct {
84	80	rs2: Register,
85	81	aq: Barrier,
86	82	rl: Barrier,
87		op: AmoOp,
88		ty: Type,
89	83	},
90	84	csr: struct {
91	85	csr: CSR,

src/arch/riscv64/abi.zig+1-4

...	...	@@ -125,10 +125,7 @@ pub fn classifySystem(ty: Type, pt: Zcu.PerThread) [8]SystemClass {
125	125	result[0] = .integer;
126	126	return result;
127	127	}
128		result[0] = .integer;
129		if (ty.optionalChild(zcu).abiSize(pt) == 0) return result;
130		result[1] = .integer;
131		return result;
	128	return memory_class;
132	129	},
133	130	.Int, .Enum, .ErrorSet => {
134	131	const int_bits = ty.intInfo(pt.zcu).bits;

src/arch/riscv64/encoding.zig+6-6

...	...	@@ -353,6 +353,7 @@ pub const Lir = struct {
353	353	// BRANCH
354	354
355	355	.beq => .{ .opcode = .BRANCH, .format = .B, .data = .{ .f = .{ .funct3 = 0b000 } } },
	356	.bne => .{ .opcode = .BRANCH, .format = .B, .data = .{ .f = .{ .funct3 = 0b001 } } },
356	357
357	358
358	359	// SYSTEM
...	...	@@ -378,8 +379,8 @@ pub const Lir = struct {
378	379
379	380	.amoaddw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00000 } } },
380	381	.amoswapw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00001 } } },
381		// LR.W
382		// SC.W
	382	.lrw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00010 } } },
	383	.scw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00011 } } },
383	384	.amoxorw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00100 } } },
384	385	.amoandw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01100 } } },
385	386	.amoorw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01000 } } },
...	...	@@ -388,10 +389,11 @@ pub const Lir = struct {
388	389	.amominuw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11000 } } },
389	390	.amomaxuw => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11100 } } },
390	391
	392
391	393	.amoaddd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00000 } } },
392	394	.amoswapd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00001 } } },
393		// LR.D
394		// SC.D
	395	.lrd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00010 } } },
	396	.scd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00011 } } },
395	397	.amoxord => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00100 } } },
396	398	.amoandd => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01100 } } },
397	399	.amoord => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01000 } } },
...	...	@@ -434,8 +436,6 @@ pub const Lir = struct {
434	436	.pseudo_compare,
435	437	.pseudo_not,
436	438	.pseudo_extern_fn_reloc,
437		.pseudo_fence,
438		.pseudo_amo,
439	439	.nop,
440	440	=> std.debug.panic("lir: didn't catch pseudo {s}", .{@tagName(mnem)}),
441	441	// zig fmt: on

src/arch/riscv64/mnem.zig+5-2

...	...	@@ -40,6 +40,7 @@ pub const Mnemonic = enum(u16) {
40	40	jal,
41	41
42	42	beq,
	43	bne,
43	44
44	45	// Memory
45	46	lui,
...	...	@@ -178,6 +179,8 @@ pub const Mnemonic = enum(u16) {
178	179	fence,
179	180	fencetso,
180	181
	182	lrw,
	183	scw,
181	184	amoswapw,
182	185	amoaddw,
183	186	amoandw,
...	...	@@ -188,6 +191,8 @@ pub const Mnemonic = enum(u16) {
188	191	amomaxuw,
189	192	amominuw,
190	193
	194	lrd,
	195	scd,
191	196	amoswapd,
192	197	amoaddd,
193	198	amoandd,
...	...	@@ -237,8 +242,6 @@ pub const Mnemonic = enum(u16) {
237	242	pseudo_compare,
238	243	pseudo_not,
239	244	pseudo_extern_fn_reloc,
240		pseudo_fence,
241		pseudo_amo,
242	245	};
243	246
244	247	pub const Pseudo = enum(u8) {