Commit f316cb29cc094c37be191f1eb72ee70eb0dc99ee | zig

author	Jacob Young <jacobly@ziglang.org> 2023-03-20 06:22:38-04:00
committer	Jakub Konka <kubkon@jakubkonka.com> 2023-03-21 08:49:54+01:00
log	f316cb29cc094c37be191f1eb72ee70eb0dc99ee
tree	4da361453ffd24fb0a9d3d9d8a8e1221bb5f3375
parent	3f4569bf187bfe296323aee6fbb59ab374041243

x86_64: implement atomic and fence ops

12 files changed, 436 insertions(+), 155 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Encoding.zig
    Mir.zig
    encoder.zig
    encodings.zig
- register_manager.zig
test
- behavior

src/arch/x86_64/CodeGen.zig+245-92

...	...	@@ -410,6 +410,25 @@ fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
410	410	});
411	411	}
412	412
	413	fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
	414	_ = try self.addInst(.{
	415	.tag = .setcc,
	416	.ops = switch (m) {
	417	.sib => .m_sib_cc,
	418	.rip => .m_rip_cc,
	419	else => unreachable,
	420	},
	421	.data = .{ .x_cc = .{
	422	.payload = switch (m) {
	423	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	424	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	425	else => unreachable,
	426	},
	427	.cc = cc,
	428	} },
	429	});
	430	}
	431
413	432	fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
414	433	_ = try self.addInst(.{
415	434	.tag = .cmovcc,
...	...	@@ -890,7 +909,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
890	909	.breakpoint => try self.airBreakpoint(),
891	910	.ret_addr => try self.airRetAddr(inst),
892	911	.frame_addr => try self.airFrameAddress(inst),
893		.fence => try self.airFence(),
	912	.fence => try self.airFence(inst),
894	913	.cond_br => try self.airCondBr(inst),
895	914	.dbg_stmt => try self.airDbgStmt(inst),
896	915	.fptrunc => try self.airFptrunc(inst),
...	...	@@ -1880,13 +1899,17 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
1880	1899	if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) {
1881	1900	switch (opt_mcv) {
1882	1901	.register => \|reg\| try self.truncateRegister(pl_ty, reg),
	1902	.register_overflow => \|ro\| try self.truncateRegister(pl_ty, ro.reg),
1883	1903	else => {},
1884	1904	}
1885	1905	break :result opt_mcv;
1886	1906	}
1887	1907
1888	1908	const pl_mcv = try self.allocRegOrMem(inst, true);
1889		try self.setRegOrMem(pl_ty, pl_mcv, opt_mcv);
	1909	try self.setRegOrMem(pl_ty, pl_mcv, switch (opt_mcv) {
	1910	else => opt_mcv,
	1911	.register_overflow => \|ro\| .{ .register = ro.reg },
	1912	});
1890	1913	break :result pl_mcv;
1891	1914	};
1892	1915	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	...	@@ -1969,8 +1992,14 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
1969	1992	},
1970	1993	.register => \|reg\| {
1971	1994	// TODO reuse operand
1972		const lock = self.register_manager.lockRegAssumeUnused(reg);
1973		defer self.register_manager.unlockReg(lock);
	1995	self.register_manager.getRegAssumeFree(.rcx, null);
	1996	const rcx_lock =
	1997	if (err_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null;
	1998	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);
	1999
	2000	const eu_lock = self.register_manager.lockReg(reg);
	2001	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
	2002
1974	2003	const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
1975	2004	if (err_off > 0) {
1976	2005	const shift = @intCast(u6, err_off * 8);
...	...	@@ -2018,8 +2047,14 @@ fn genUnwrapErrorUnionPayloadMir(
2018	2047	},
2019	2048	.register => \|reg\| {
2020	2049	// TODO reuse operand
2021		const lock = self.register_manager.lockRegAssumeUnused(reg);
2022		defer self.register_manager.unlockReg(lock);
	2050	self.register_manager.getRegAssumeFree(.rcx, null);
	2051	const rcx_lock =
	2052	if (payload_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null;
	2053	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);
	2054
	2055	const eu_lock = self.register_manager.lockReg(reg);
	2056	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
	2057
2023	2058	const result_reg: Register = if (maybe_inst) \|inst\|
2024	2059	(try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)).register
2025	2060	else
...	...	@@ -3129,7 +3164,12 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
3129	3164	.none => unreachable,
3130	3165	.dead => unreachable,
3131	3166	.unreach => unreachable,
3132		.eflags => unreachable,
	3167	.eflags => \|cc\| {
	3168	try self.asmSetccMemory(Memory.sib(
	3169	Memory.PtrSize.fromSize(abi_size),
	3170	.{ .base = reg.to64(), .disp = 0 },
	3171	), cc);
	3172	},
3133	3173	.undef => {
3134	3174	if (!self.wantSafety()) return; // The already existing value will do just fine.
3135	3175	switch (abi_size) {
...	...	@@ -3598,8 +3638,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3598	3638	},
3599	3639	else => {},
3600	3640	}
3601		assert(self.register_manager.isRegFree(.rcx));
3602		try self.register_manager.getReg(.rcx, null);
	3641	self.register_manager.getRegAssumeFree(.rcx, null);
3603	3642	try self.genSetReg(Type.u8, .rcx, shift);
3604	3643	}
3605	3644
...	...	@@ -3639,8 +3678,7 @@ fn genShiftBinOp(
3639	3678	};
3640	3679	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3641	3680
3642		assert(self.register_manager.isRegFree(.rcx));
3643		try self.register_manager.getReg(.rcx, null);
	3681	self.register_manager.getRegAssumeFree(.rcx, null);
3644	3682	const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx);
3645	3683	defer self.register_manager.unlockReg(rcx_lock);
3646	3684
...	...	@@ -4230,7 +4268,10 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
4230	4268	.base = .rbp,
4231	4269	.disp = -off,
4232	4270	}),
4233		Immediate.u(@intCast(u32, imm)),
	4271	if (math.cast(i32, @bitCast(i64, imm))) \|small\|
	4272	Immediate.s(small)
	4273	else
	4274	Immediate.u(@intCast(u32, imm)),
4234	4275	);
4235	4276	},
4236	4277	64 => {
...	...	@@ -4506,9 +4547,14 @@ fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
4506	4547	return self.finishAir(inst, result, .{ .none, .none, .none });
4507	4548	}
4508	4549
4509		fn airFence(self: *Self) !void {
4510		return self.fail("TODO implement fence() for {}", .{self.target.cpu.arch});
4511		//return self.finishAirBookkeeping();
	4550	fn airFence(self: *Self, inst: Air.Inst.Index) !void {
	4551	const order = self.air.instructions.items(.data)[inst].fence;
	4552	switch (order) {
	4553	.Unordered, .Monotonic => unreachable,
	4554	.Acquire, .Release, .AcqRel => {},
	4555	.SeqCst => try self.asmOpOnly(.mfence),
	4556	}
	4557	return self.finishAirBookkeeping();
4512	4558	}
4513	4559
4514	4560	fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
...	...	@@ -5075,6 +5121,11 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
5075	5121	}
5076	5122
5077	5123	fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
	5124	switch (opt_mcv) {
	5125	.register_overflow => \|ro\| return .{ .eflags = ro.eflags.negate() },
	5126	else => {},
	5127	}
	5128
5078	5129	try self.spillEflagsIfOccupied();
5079	5130	self.eflags_inst = inst;
5080	5131
...	...	@@ -5196,8 +5247,13 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
5196	5247	try self.genBinOpMir(.cmp, Type.anyerror, .{ .stack_offset = offset }, .{ .immediate = 0 });
5197	5248	},
5198	5249	.register => \|reg\| {
5199		const maybe_lock = self.register_manager.lockReg(reg);
5200		defer if (maybe_lock) \|lock\| self.register_manager.unlockReg(lock);
	5250	self.register_manager.getRegAssumeFree(.rcx, null);
	5251	const rcx_lock = if (err_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null;
	5252	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);
	5253
	5254	const eu_lock = self.register_manager.lockReg(reg);
	5255	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
	5256
5201	5257	const tmp_reg = try self.copyToTmpRegister(ty, operand);
5202	5258	if (err_off > 0) {
5203	5259	const shift = @intCast(u6, err_off * 8);
...	...	@@ -5389,69 +5445,6 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
5389	5445	return self.finishAir(inst, result, .{ .none, .none, .none });
5390	5446	}
5391	5447
5392		fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u32 {
5393		const abi_size = @intCast(u32, ty.abiSize(self.target.*));
5394		switch (condition) {
5395		.none => unreachable,
5396		.undef => unreachable,
5397		.dead, .unreach => unreachable,
5398		.eflags => unreachable,
5399		.register => \|cond_reg\| {
5400		try self.spillEflagsIfOccupied();
5401
5402		const cond_reg_lock = self.register_manager.lockReg(cond_reg);
5403		defer if (cond_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
5404
5405		switch (case) {
5406		.none => unreachable,
5407		.undef => unreachable,
5408		.dead, .unreach => unreachable,
5409		.immediate => \|imm\| try self.asmRegisterImmediate(
5410		.xor,
5411		registerAlias(cond_reg, abi_size),
5412		Immediate.u(imm),
5413		),
5414		.register => \|reg\| try self.asmRegisterRegister(
5415		.xor,
5416		registerAlias(cond_reg, abi_size),
5417		registerAlias(reg, abi_size),
5418		),
5419		.stack_offset => {
5420		if (abi_size <= 8) {
5421		const reg = try self.copyToTmpRegister(ty, case);
5422		return self.genCondSwitchMir(ty, condition, .{ .register = reg });
5423		}
5424
5425		return self.fail("TODO implement switch mir when case is stack offset with abi larger than 8 bytes", .{});
5426		},
5427		else => {
5428		return self.fail("TODO implement switch mir when case is {}", .{case});
5429		},
5430		}
5431
5432		const aliased_reg = registerAlias(cond_reg, abi_size);
5433		try self.asmRegisterRegister(.@"test", aliased_reg, aliased_reg);
5434		return self.asmJccReloc(undefined, .ne);
5435		},
5436		.stack_offset => {
5437		try self.spillEflagsIfOccupied();
5438
5439		if (abi_size <= 8) {
5440		const reg = try self.copyToTmpRegister(ty, condition);
5441		const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
5442		defer self.register_manager.unlockReg(reg_lock);
5443		return self.genCondSwitchMir(ty, .{ .register = reg }, case);
5444		}
5445
5446		return self.fail("TODO implement switch mir when condition is stack offset with abi larger than 8 bytes", .{});
5447		},
5448		else => {
5449		return self.fail("TODO implemenent switch mir when condition is {}", .{condition});
5450		},
5451		}
5452		return 0; // TODO
5453		}
5454
5455	5448	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5456	5449	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
5457	5450	const condition = try self.resolveInst(pl_op.operand);
...	...	@@ -5496,8 +5489,10 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5496	5489	defer self.gpa.free(relocs);
5497	5490
5498	5491	for (items, relocs) \|item, *reloc\| {
	5492	try self.spillEflagsIfOccupied();
5499	5493	const item_mcv = try self.resolveInst(item);
5500		reloc.* = try self.genCondSwitchMir(condition_ty, condition, item_mcv);
	5494	try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);
	5495	reloc.* = try self.asmJccReloc(undefined, .ne);
5501	5496	}
5502	5497
5503	5498	// Capture the state of register and stack allocation state so that we can revert to it.
...	...	@@ -6624,26 +6619,184 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6624	6619
6625	6620	fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
6626	6621	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
6627		const extra = self.air.extraData(Air.Block, ty_pl.payload);
6628		_ = extra;
6629		return self.fail("TODO implement x86 airCmpxchg", .{});
6630		// return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
	6622	const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
	6623
	6624	const ptr_ty = self.air.typeOf(extra.ptr);
	6625	const ptr_mcv = try self.resolveInst(extra.ptr);
	6626	const val_ty = self.air.typeOf(extra.expected_value);
	6627
	6628	const exp_mcv = try self.resolveInst(extra.expected_value);
	6629	try self.genSetReg(val_ty, .rax, exp_mcv);
	6630	const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
	6631	defer self.register_manager.unlockReg(rax_lock);
	6632
	6633	const new_mcv = try self.resolveInst(extra.new_value);
	6634	const new_reg = try self.copyToTmpRegister(val_ty, new_mcv);
	6635	const new_lock = self.register_manager.lockRegAssumeUnused(new_reg);
	6636	defer self.register_manager.unlockReg(new_lock);
	6637
	6638	const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
	6639	const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
	6640	const ptr_mem: Memory = switch (ptr_mcv) {
	6641	.register => \|reg\| Memory.sib(ptr_size, .{ .base = reg, .disp = 0 }),
	6642	.ptr_stack_offset => \|off\| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),
	6643	else => Memory.sib(ptr_size, .{
	6644	.base = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
	6645	.disp = 0,
	6646	}),
	6647	};
	6648	const mem_lock = if (ptr_mem.base()) \|reg\| self.register_manager.lockReg(reg) else null;
	6649	defer if (mem_lock) \|lock\| self.register_manager.unlockReg(lock);
	6650
	6651	try self.spillEflagsIfOccupied();
	6652	_ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
	6653	.r1 = new_reg,
	6654	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
	6655	} } });
	6656
	6657	const result: MCValue = result: {
	6658	if (self.liveness.isUnused(inst)) break :result .dead;
	6659
	6660	self.eflags_inst = inst;
	6661	break :result .{ .register_overflow = .{ .reg = .rax, .eflags = .ne } };
	6662	};
	6663	return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
	6664	}
	6665
	6666	fn atomicOp(
	6667	self: *Self,
	6668	dst_reg: Register,
	6669	ptr_mcv: MCValue,
	6670	val_mcv: MCValue,
	6671	ptr_ty: Type,
	6672	val_ty: Type,
	6673	unused: bool,
	6674	op: ?std.builtin.AtomicRmwOp,
	6675	order: std.builtin.AtomicOrder,
	6676	) InnerError!void {
	6677	const dst_lock = self.register_manager.lockReg(dst_reg);
	6678	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	6679
	6680	const ptr_lock = switch (ptr_mcv) {
	6681	.register => \|reg\| self.register_manager.lockReg(reg),
	6682	else => null,
	6683	};
	6684	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
	6685
	6686	const val_lock = switch (val_mcv) {
	6687	.register => \|reg\| self.register_manager.lockReg(reg),
	6688	else => null,
	6689	};
	6690	defer if (val_lock) \|lock\| self.register_manager.unlockReg(lock);
	6691
	6692	const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
	6693	const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
	6694	const ptr_mem: Memory = switch (ptr_mcv) {
	6695	.register => \|reg\| Memory.sib(ptr_size, .{ .base = reg, .disp = 0 }),
	6696	.ptr_stack_offset => \|off\| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),
	6697	else => Memory.sib(ptr_size, .{
	6698	.base = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
	6699	.disp = 0,
	6700	}),
	6701	};
	6702	const mem_lock = if (ptr_mem.base()) \|reg\| self.register_manager.lockReg(reg) else null;
	6703	defer if (mem_lock) \|lock\| self.register_manager.unlockReg(lock);
	6704
	6705	try self.genSetReg(val_ty, dst_reg, val_mcv);
	6706
	6707	const need_loop = val_ty.isRuntimeFloat() or if (op) \|rmw\| switch (rmw) {
	6708	.Xchg, .Add, .Sub => false,
	6709	.And, .Or, .Xor => !unused,
	6710	.Nand, .Max, .Min => true,
	6711	} else false;
	6712	if (!need_loop) {
	6713	const tag: Mir.Inst.Tag = if (op) \|rmw\| switch (rmw) {
	6714	.Xchg => if (unused) .mov else .xchg,
	6715	.Add => if (unused) .add else .xadd,
	6716	.Sub => if (unused) .sub else .xadd,
	6717	.And => .@"and",
	6718	.Or => .@"or",
	6719	.Xor => .xor,
	6720	else => unreachable,
	6721	} else switch (order) {
	6722	.Unordered, .Monotonic, .Release, .AcqRel => .mov,
	6723	.Acquire => unreachable,
	6724	.SeqCst => .xchg,
	6725	};
	6726	if (op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
	6727	try self.genUnOpMir(.neg, val_ty, .{ .register = dst_reg });
	6728	}
	6729	_ = try self.addInst(.{ .tag = tag, .ops = switch (tag) {
	6730	.mov, .xchg => .mr_sib,
	6731	.xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
	6732	else => unreachable,
	6733	}, .data = .{ .rx = .{
	6734	.r1 = registerAlias(dst_reg, val_abi_size),
	6735	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
	6736	} } });
	6737	return;
	6738	}
	6739
	6740	return self.fail("TODO implement x86 atomic loop", .{});
6631	6741	}
6632	6742
6633	6743	fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
6634		_ = inst;
6635		return self.fail("TODO implement x86 airAtomicRmw", .{});
	6744	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
	6745	const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
	6746
	6747	const dst_reg = try self.register_manager.allocReg(inst, gp);
	6748
	6749	const ptr_ty = self.air.typeOf(pl_op.operand);
	6750	const ptr_mcv = try self.resolveInst(pl_op.operand);
	6751
	6752	const val_ty = self.air.typeOf(extra.operand);
	6753	const val_mcv = try self.resolveInst(extra.operand);
	6754
	6755	const unused = self.liveness.isUnused(inst);
	6756	try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering());
	6757	const result: MCValue = if (unused) .dead else .{ .register = dst_reg };
	6758	return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
6636	6759	}
6637	6760
6638	6761	fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
6639		_ = inst;
6640		return self.fail("TODO implement airAtomicLoad for {}", .{self.target.cpu.arch});
	6762	const atomic_load = self.air.instructions.items(.data)[inst].atomic_load;
	6763
	6764	const result: MCValue = result: {
	6765	if (self.liveness.isUnused(inst)) break :result .dead;
	6766
	6767	const ptr_ty = self.air.typeOf(atomic_load.ptr);
	6768	const ptr_mcv = try self.resolveInst(atomic_load.ptr);
	6769	const ptr_lock = switch (ptr_mcv) {
	6770	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
	6771	else => null,
	6772	};
	6773	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
	6774
	6775	const dst_mcv =
	6776	if (self.reuseOperand(inst, atomic_load.ptr, 0, ptr_mcv))
	6777	ptr_mcv
	6778	else
	6779	try self.allocRegOrMem(inst, true);
	6780
	6781	try self.load(dst_mcv, ptr_mcv, ptr_ty);
	6782	break :result dst_mcv;
	6783	};
	6784	return self.finishAir(inst, result, .{ atomic_load.ptr, .none, .none });
6641	6785	}
6642	6786
6643	6787	fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
6644		_ = inst;
6645		_ = order;
6646		return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
	6788	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	6789
	6790	const dst_reg = try self.register_manager.allocReg(null, gp);
	6791
	6792	const ptr_ty = self.air.typeOf(bin_op.lhs);
	6793	const ptr_mcv = try self.resolveInst(bin_op.lhs);
	6794
	6795	const val_ty = self.air.typeOf(bin_op.rhs);
	6796	const val_mcv = try self.resolveInst(bin_op.rhs);
	6797
	6798	try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, true, null, order);
	6799	return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
6647	6800	}
6648	6801
6649	6802	fn airMemset(self: *Self, inst: Air.Inst.Index) !void {

src/arch/x86_64/Emit.zig+91-14

...	...	@@ -87,6 +87,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
87	87	.cdq,
88	88	.cqo,
89	89	.cmp,
	90	.cmpxchg,
90	91	.div,
91	92	.fisttp,
92	93	.fld,
...	...	@@ -95,7 +96,9 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
95	96	.int3,
96	97	.jmp,
97	98	.lea,
	99	.lfence,
98	100	.lzcnt,
	101	.mfence,
99	102	.mov,
100	103	.movzx,
101	104	.mul,
...	...	@@ -110,6 +113,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
110	113	.sal,
111	114	.sar,
112	115	.sbb,
	116	.sfence,
113	117	.shl,
114	118	.shr,
115	119	.sub,
...	...	@@ -117,6 +121,8 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
117	121	.@"test",
118	122	.tzcnt,
119	123	.ud2,
	124	.xadd,
	125	.xchg,
120	126	.xor,
121	127
122	128	.addss,
...	...	@@ -148,6 +154,8 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
148	154	.stos,
149	155	=> try emit.mirString(tag, inst),
150	156
	157	.cmpxchgb => try emit.mirCmpxchgBytes(inst),
	158
151	159	.jmp_reloc => try emit.mirJmpReloc(inst),
152	160
153	161	.call_extern => try emit.mirCallExtern(inst),
...	...	@@ -214,6 +222,20 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
214	222	const ops = emit.mir.instructions.items(.ops)[inst];
215	223	const data = emit.mir.instructions.items(.data)[inst];
216	224
	225	const prefix: Instruction.Prefix = switch (ops) {
	226	.lock_m_sib,
	227	.lock_m_rip,
	228	.lock_mi_u_sib,
	229	.lock_mi_u_rip,
	230	.lock_mi_s_sib,
	231	.lock_mi_s_rip,
	232	.lock_mr_sib,
	233	.lock_mr_rip,
	234	.lock_moffs_rax,
	235	=> .lock,
	236	else => .none,
	237	};
	238
217	239	var op1: Instruction.Operand = .none;
218	240	var op2: Instruction.Operand = .none;
219	241	var op3: Instruction.Operand = .none;
...	...	@@ -252,35 +274,35 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
252	274	op2 = .{ .reg = data.rri.r2 };
253	275	op3 = .{ .imm = imm };
254	276	},
255		.m_sib => {
	277	.m_sib, .lock_m_sib => {
256	278	const msib = emit.mir.extraData(Mir.MemorySib, data.payload).data;
257	279	op1 = .{ .mem = Mir.MemorySib.decode(msib) };
258	280	},
259		.m_rip => {
	281	.m_rip, .lock_m_rip => {
260	282	const mrip = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
261	283	op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
262	284	},
263		.mi_s_sib, .mi_u_sib => {
	285	.mi_s_sib, .mi_u_sib, .lock_mi_s_sib, .lock_mi_u_sib => {
264	286	const msib = emit.mir.extraData(Mir.MemorySib, data.xi.payload).data;
265	287	const imm = switch (ops) {
266		.mi_s_sib => Immediate.s(@bitCast(i32, data.xi.imm)),
267		.mi_u_sib => Immediate.u(data.xi.imm),
	288	.mi_s_sib, .lock_mi_s_sib => Immediate.s(@bitCast(i32, data.xi.imm)),
	289	.mi_u_sib, .lock_mi_u_sib => Immediate.u(data.xi.imm),
268	290	else => unreachable,
269	291	};
270	292	op1 = .{ .mem = Mir.MemorySib.decode(msib) };
271	293	op2 = .{ .imm = imm };
272	294	},
273		.mi_u_rip, .mi_s_rip => {
	295	.mi_u_rip, .mi_s_rip, .lock_mi_u_rip, .lock_mi_s_rip => {
274	296	const mrip = emit.mir.extraData(Mir.MemoryRip, data.xi.payload).data;
275	297	const imm = switch (ops) {
276		.mi_s_rip => Immediate.s(@bitCast(i32, data.xi.imm)),
277		.mi_u_rip => Immediate.u(data.xi.imm),
	298	.mi_s_rip, .lock_mi_s_rip => Immediate.s(@bitCast(i32, data.xi.imm)),
	299	.mi_u_rip, .lock_mi_u_rip => Immediate.u(data.xi.imm),
278	300	else => unreachable,
279	301	};
280	302	op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
281	303	op2 = .{ .imm = imm };
282	304	},
283		.rm_sib, .mr_sib => {
	305	.rm_sib, .mr_sib, .lock_mr_sib => {
284	306	const msib = emit.mir.extraData(Mir.MemorySib, data.rx.payload).data;
285	307	const op_r = .{ .reg = data.rx.r1 };
286	308	const op_m = .{ .mem = Mir.MemorySib.decode(msib) };
...	...	@@ -289,23 +311,23 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
289	311	op1 = op_r;
290	312	op2 = op_m;
291	313	},
292		.mr_sib => {
	314	.mr_sib, .lock_mr_sib => {
293	315	op1 = op_m;
294	316	op2 = op_r;
295	317	},
296	318	else => unreachable,
297	319	}
298	320	},
299		.rm_rip, .mr_rip => {
	321	.rm_rip, .mr_rip, .lock_mr_rip => {
300	322	const mrip = emit.mir.extraData(Mir.MemoryRip, data.rx.payload).data;
301	323	const op_r = .{ .reg = data.rx.r1 };
302	324	const op_m = .{ .mem = Mir.MemoryRip.decode(mrip) };
303	325	switch (ops) {
304		.rm_sib => {
	326	.rm_rip => {
305	327	op1 = op_r;
306	328	op2 = op_m;
307	329	},
308		.mr_sib => {
	330	.mr_rip, .lock_mr_rip => {
309	331	op1 = op_m;
310	332	op2 = op_r;
311	333	},
...	...	@@ -319,6 +341,7 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
319	341	}
320	342
321	343	return emit.encode(mnemonic, .{
	344	.prefix = prefix,
322	345	.op1 = op1,
323	346	.op2 = op2,
324	347	.op3 = op3,
...	...	@@ -348,6 +371,39 @@ fn mirString(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!vo
348	371	}
349	372	}
350	373
	374	fn mirCmpxchgBytes(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	375	const ops = emit.mir.instructions.items(.ops)[inst];
	376	const data = emit.mir.instructions.items(.data)[inst];
	377
	378	var op1: Instruction.Operand = .none;
	379	switch (ops) {
	380	.m_sib, .lock_m_sib => {
	381	const sib = emit.mir.extraData(Mir.MemorySib, data.payload).data;
	382	op1 = .{ .mem = Mir.MemorySib.decode(sib) };
	383	},
	384	.m_rip, .lock_m_rip => {
	385	const rip = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
	386	op1 = .{ .mem = Mir.MemoryRip.decode(rip) };
	387	},
	388	else => unreachable,
	389	}
	390
	391	const mnemonic: Instruction.Mnemonic = switch (op1.mem.bitSize()) {
	392	64 => .cmpxchg8b,
	393	128 => .cmpxchg16b,
	394	else => unreachable,
	395	};
	396
	397	return emit.encode(mnemonic, .{
	398	.prefix = switch (ops) {
	399	.m_sib, .m_rip => .none,
	400	.lock_m_sib, .lock_m_rip => .lock,
	401	else => unreachable,
	402	},
	403	.op1 = op1,
	404	});
	405	}
	406
351	407	fn mirMovMoffs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
352	408	const ops = emit.mir.instructions.items(.ops)[inst];
353	409	const payload = emit.mir.instructions.items(.data)[inst].payload;
...	...	@@ -361,8 +417,13 @@ fn mirMovMoffs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
361	417	.op2 = .{ .mem = Memory.moffs(seg, offset) },
362	418	});
363	419	},
364		.moffs_rax => {
	420	.moffs_rax, .lock_moffs_rax => {
365	421	try emit.encode(.mov, .{
	422	.prefix = switch (ops) {
	423	.moffs_rax => .none,
	424	.lock_moffs_rax => .lock,
	425	else => unreachable,
	426	},
366	427	.op1 = .{ .mem = Memory.moffs(seg, offset) },
367	428	.op2 = .{ .reg = .rax },
368	429	});
...	...	@@ -455,6 +516,22 @@ fn mirSetcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
455	516	.op1 = .{ .reg = data.r1 },
456	517	});
457	518	},
	519	.m_sib_cc => {
	520	const data = emit.mir.instructions.items(.data)[inst].x_cc;
	521	const extra = emit.mir.extraData(Mir.MemorySib, data.payload).data;
	522	const mnemonic = mnemonicFromConditionCode("set", data.cc);
	523	return emit.encode(mnemonic, .{
	524	.op1 = .{ .mem = Mir.MemorySib.decode(extra) },
	525	});
	526	},
	527	.m_rip_cc => {
	528	const data = emit.mir.instructions.items(.data)[inst].x_cc;
	529	const extra = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
	530	const mnemonic = mnemonicFromConditionCode("set", data.cc);
	531	return emit.encode(mnemonic, .{
	532	.op1 = .{ .mem = Mir.MemoryRip.decode(extra) },
	533	});
	534	},
458	535	else => unreachable, // TODO
459	536	}
460	537	}

src/arch/x86_64/Encoding.zig+9-6

...	...	@@ -314,6 +314,7 @@ pub const Mnemonic = enum {
314	314	cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz,
315	315	cmp,
316	316	cmps, cmpsb, cmpsd, cmpsq, cmpsw,
	317	cmpxchg, cmpxchg8b, cmpxchg16b,
317	318	cqo, cwd, cwde,
318	319	div,
319	320	fisttp, fld,
...	...	@@ -321,10 +322,10 @@ pub const Mnemonic = enum {
321	322	ja, jae, jb, jbe, jc, jrcxz, je, jg, jge, jl, jle, jna, jnae, jnb, jnbe,
322	323	jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, js, jz,
323	324	jmp,
324		lea,
	325	lea, lfence,
325	326	lods, lodsb, lodsd, lodsq, lodsw,
326	327	lzcnt,
327		mov,
	328	mfence, mov,
328	329	movs, movsb, movsd, movsq, movsw,
329	330	movsx, movsxd, movzx, mul,
330	331	neg, nop, not,
...	...	@@ -337,10 +338,11 @@ pub const Mnemonic = enum {
337	338	seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
338	339	setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
339	340	setnz, seto, setp, setpe, setpo, sets, setz,
	341	sfence,
340	342	stos, stosb, stosd, stosq, stosw,
341	343	@"test", tzcnt,
342	344	ud2,
343		xor,
	345	xadd, xchg, xor,
344	346	// SSE
345	347	addss,
346	348	cmpss,
...	...	@@ -387,7 +389,7 @@ pub const Op = enum {
387	389	cl,
388	390	r8, r16, r32, r64,
389	391	rm8, rm16, rm32, rm64,
390		m8, m16, m32, m64, m80,
	392	m8, m16, m32, m64, m80, m128,
391	393	rel8, rel16, rel32,
392	394	m,
393	395	moffs,
...	...	@@ -436,6 +438,7 @@ pub const Op = enum {
436	438	32 => .m32,
437	439	64 => .m64,
438	440	80 => .m80,
	441	128 => .m128,
439	442	else => unreachable,
440	443	};
441	444	},
...	...	@@ -473,7 +476,7 @@ pub const Op = enum {
473	476	.imm32, .imm32s, .eax, .r32, .m32, .rm32, .rel32, .xmm_m32 => 32,
474	477	.imm64, .rax, .r64, .m64, .rm64, .xmm_m64 => 64,
475	478	.m80 => 80,
476		.xmm => 128,
	479	.m128, .xmm => 128,
477	480	};
478	481	}
479	482
...	...	@@ -520,7 +523,7 @@ pub const Op = enum {
520	523	// zig fmt: off
521	524	return switch (op) {
522	525	.rm8, .rm16, .rm32, .rm64,
523		.m8, .m16, .m32, .m64, .m80,
	526	.m8, .m16, .m32, .m64, .m80, .m128,
524	527	.m,
525	528	.xmm_m32, .xmm_m64,
526	529	=> true,

src/arch/x86_64/Mir.zig+35-7

...	...	@@ -66,6 +66,10 @@ pub const Inst = struct {
66	66	cqo,
67	67	/// Logical compare
68	68	cmp,
	69	/// Compare and exchange
	70	cmpxchg,
	71	/// Compare and exchange bytes
	72	cmpxchgb,
69	73	/// Unsigned division
70	74	div,
71	75	/// Store integer with truncation
...	...	@@ -82,8 +86,12 @@ pub const Inst = struct {
82	86	jmp,
83	87	/// Load effective address
84	88	lea,
	89	/// Load fence
	90	lfence,
85	91	/// Count the number of leading zero bits
86	92	lzcnt,
	93	/// Memory fence
	94	mfence,
87	95	/// Move
88	96	mov,
89	97	/// Move with sign extension
...	...	@@ -114,6 +122,8 @@ pub const Inst = struct {
114	122	sar,
115	123	/// Integer subtraction with borrow
116	124	sbb,
	125	/// Store fence
	126	sfence,
117	127	/// Logical shift left
118	128	shl,
119	129	/// Logical shift right
...	...	@@ -128,6 +138,10 @@ pub const Inst = struct {
128	138	tzcnt,
129	139	/// Undefined instruction
130	140	ud2,
	141	/// Exchange and add
	142	xadd,
	143	/// Exchange register/memory with register
	144	xchg,
131	145	/// Logical exclusive-or
132	146	xor,
133	147
...	...	@@ -242,10 +256,10 @@ pub const Inst = struct {
242	256	/// Uses `rri` payload.
243	257	rri_u,
244	258	/// Register with condition code (CC).
245		/// Uses `r_c` payload.
	259	/// Uses `r_cc` payload.
246	260	r_cc,
247	261	/// Register, register with condition code (CC).
248		/// Uses `rr_c` payload.
	262	/// Uses `rr_cc` payload.
249	263	rr_cc,
250	264	/// Register, immediate (sign-extended) operands.
251	265	/// Uses `ri` payload.
...	...	@@ -283,6 +297,12 @@ pub const Inst = struct {
283	297	/// Single memory (RIP) operand.
284	298	/// Uses `payload` with extra data of type `MemoryRip`.
285	299	m_rip,
	300	/// Single memory (SIB) operand with condition code (CC).
	301	/// Uses `x_cc` with extra data of type `MemorySib`.
	302	m_sib_cc,
	303	/// Single memory (RIP) operand with condition code (CC).
	304	/// Uses `x_cc` with extra data of type `MemoryRip`.
	305	m_rip_cc,
286	306	/// Memory (SIB), immediate (unsigned) operands.
287	307	/// Uses `xi` payload with extra data of type `MemorySib`.
288	308	mi_u_sib,
...	...	@@ -301,6 +321,12 @@ pub const Inst = struct {
301	321	/// Memory (RIP), register operands.
302	322	/// Uses `rx` payload with extra data of type `MemoryRip`.
303	323	mr_rip,
	324	/// Rax, Memory moffs.
	325	/// Uses `payload` with extra data of type `MemoryMoffs`.
	326	rax_moffs,
	327	/// Memory moffs, rax.
	328	/// Uses `payload` with extra data of type `MemoryMoffs`.
	329	moffs_rax,
304	330	/// Single memory (SIB) operand with lock prefix.
305	331	/// Uses `payload` with extra data of type `MemorySib`.
306	332	lock_m_sib,
...	...	@@ -325,12 +351,9 @@ pub const Inst = struct {
325	351	/// Memory (RIP), register operands with lock prefix.
326	352	/// Uses `rx` payload with extra data of type `MemoryRip`.
327	353	lock_mr_rip,
328		/// Rax, Memory moffs.
	354	/// Memory moffs, rax with lock prefix.
329	355	/// Uses `payload` with extra data of type `MemoryMoffs`.
330		rax_moffs,
331		/// Memory moffs, rax.
332		/// Uses `payload` with extra data of type `MemoryMoffs`.
333		moffs_rax,
	356	lock_moffs_rax,
334	357	/// References another Mir instruction directly.
335	358	/// Uses `inst` payload.
336	359	inst,
...	...	@@ -381,6 +404,11 @@ pub const Inst = struct {
381	404	r2: Register,
382	405	imm: u32,
383	406	},
	407	/// Condition code (CC), followed by custom payload found in extra.
	408	x_cc: struct {
	409	payload: u32,
	410	cc: bits.Condition,
	411	},
384	412	/// Register with condition code (CC).
385	413	r_cc: struct {
386	414	r1: Register,

src/arch/x86_64/encoder.zig+2-1

...	...	@@ -117,7 +117,8 @@ pub const Instruction = struct {
117	117
118	118	pub fn new(mnemonic: Mnemonic, args: Init) !Instruction {
119	119	const encoding = (try Encoding.findByMnemonic(mnemonic, args)) orelse {
120		log.debug("no encoding found for: {s} {s} {s} {s} {s}", .{
	120	log.debug("no encoding found for: {s} {s} {s} {s} {s} {s}", .{
	121	@tagName(args.prefix),
121	122	@tagName(mnemonic),
122	123	@tagName(Encoding.Op.fromOperand(args.op1)),
123	124	@tagName(Encoding.Op.fromOperand(args.op2)),

src/arch/x86_64/encodings.zig+38

...	...	@@ -252,6 +252,15 @@ pub const table = &[_]Entry{
252	252	.{ .cmpsd, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .none },
253	253	.{ .cmpsq, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .long },
254	254
	255	.{ .cmpxchg, .mr, .rm8, .r8, .none, .none, &.{ 0x0f, 0xb0 }, 0, .none },
	256	.{ .cmpxchg, .mr, .rm8, .r8, .none, .none, &.{ 0x0f, 0xb0 }, 0, .rex },
	257	.{ .cmpxchg, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xb1 }, 0, .rex },
	258	.{ .cmpxchg, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xb1 }, 0, .rex },
	259	.{ .cmpxchg, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xb1 }, 0, .long },
	260
	261	.{ .cmpxchg8b , .m, .m64, .none, .none, .none, &.{ 0x0f, 0xc7 }, 1, .none },
	262	.{ .cmpxchg16b, .m, .m128, .none, .none, .none, &.{ 0x0f, 0xc7 }, 1, .long },
	263
255	264	.{ .div, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 6, .none },
256	265	.{ .div, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 6, .rex },
257	266	.{ .div, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 6, .none },
...	...	@@ -328,6 +337,8 @@ pub const table = &[_]Entry{
328	337	.{ .lea, .rm, .r32, .m, .none, .none, &.{ 0x8d }, 0, .none },
329	338	.{ .lea, .rm, .r64, .m, .none, .none, &.{ 0x8d }, 0, .long },
330	339
	340	.{ .lfence, .np, .none, .none, .none, .none, &.{ 0x0f, 0xae, 0xe8 }, 0, .none },
	341
331	342	.{ .lods, .np, .m8, .none, .none, .none, &.{ 0xac }, 0, .none },
332	343	.{ .lods, .np, .m16, .none, .none, .none, &.{ 0xad }, 0, .none },
333	344	.{ .lods, .np, .m32, .none, .none, .none, &.{ 0xad }, 0, .none },
...	...	@@ -341,6 +352,8 @@ pub const table = &[_]Entry{
341	352	.{ .lzcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
342	353	.{ .lzcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .long },
343	354
	355	.{ .mfence, .np, .none, .none, .none, .none, &.{ 0x0f, 0xae, 0xf0 }, 0, .none },
	356
344	357	.{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .none },
345	358	.{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .rex },
346	359	.{ .mov, .mr, .rm16, .r16, .none, .none, &.{ 0x89 }, 0, .none },
...	...	@@ -588,6 +601,8 @@ pub const table = &[_]Entry{
588	601	.{ .setz, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .none },
589	602	.{ .setz, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .rex },
590	603
	604	.{ .sfence, .np, .none, .none, .none, .none, &.{ 0x0f, 0xae, 0xf8 }, 0, .none },
	605
591	606	.{ .shl, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 4, .none },
592	607	.{ .shl, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 4, .rex },
593	608	.{ .shl, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 4, .none },
...	...	@@ -675,6 +690,29 @@ pub const table = &[_]Entry{
675	690
676	691	.{ .ud2, .np, .none, .none, .none, .none, &.{ 0x0f, 0x0b }, 0, .none },
677	692
	693	.{ .xadd, .mr, .rm8, .r8, .none, .none, &.{ 0x0f, 0xc0 }, 0, .none },
	694	.{ .xadd, .mr, .rm8, .r8, .none, .none, &.{ 0x0f, 0xc0 }, 0, .rex },
	695	.{ .xadd, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xc1 }, 0, .none },
	696	.{ .xadd, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xc1 }, 0, .none },
	697	.{ .xadd, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xc1 }, 0, .long },
	698
	699	.{ .xchg, .o, .ax, .r16, .none, .none, &.{ 0x90 }, 0, .none },
	700	.{ .xchg, .o, .r16, .ax, .none, .none, &.{ 0x90 }, 0, .none },
	701	.{ .xchg, .o, .eax, .r32, .none, .none, &.{ 0x90 }, 0, .none },
	702	.{ .xchg, .o, .rax, .r64, .none, .none, &.{ 0x90 }, 0, .long },
	703	.{ .xchg, .o, .r32, .eax, .none, .none, &.{ 0x90 }, 0, .none },
	704	.{ .xchg, .o, .r64, .rax, .none, .none, &.{ 0x90 }, 0, .long },
	705	.{ .xchg, .mr, .rm8, .r8, .none, .none, &.{ 0x86 }, 0, .none },
	706	.{ .xchg, .mr, .rm8, .r8, .none, .none, &.{ 0x86 }, 0, .rex },
	707	.{ .xchg, .rm, .r8, .rm8, .none, .none, &.{ 0x86 }, 0, .none },
	708	.{ .xchg, .rm, .r8, .rm8, .none, .none, &.{ 0x86 }, 0, .rex },
	709	.{ .xchg, .mr, .rm16, .r16, .none, .none, &.{ 0x87 }, 0, .none },
	710	.{ .xchg, .rm, .r16, .rm16, .none, .none, &.{ 0x87 }, 0, .none },
	711	.{ .xchg, .mr, .rm32, .r32, .none, .none, &.{ 0x87 }, 0, .none },
	712	.{ .xchg, .mr, .rm64, .r64, .none, .none, &.{ 0x87 }, 0, .long },
	713	.{ .xchg, .rm, .r32, .rm32, .none, .none, &.{ 0x87 }, 0, .none },
	714	.{ .xchg, .rm, .r64, .rm64, .none, .none, &.{ 0x87 }, 0, .long },
	715
678	716	.{ .xor, .zi, .al, .imm8, .none, .none, &.{ 0x34 }, 0, .none },
679	717	.{ .xor, .zi, .ax, .imm16, .none, .none, &.{ 0x35 }, 0, .none },
680	718	.{ .xor, .zi, .eax, .imm32, .none, .none, &.{ 0x35 }, 0, .none },

src/register_manager.zig+16-24

...	...	@@ -305,40 +305,32 @@ pub fn RegisterManager(
305	305	pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {
306	306	const index = indexOfRegIntoTracked(reg) orelse return;
307	307	log.debug("getReg {} for inst {?}", .{ reg, inst });
308		self.markRegAllocated(reg);
309	308
310		if (inst) \|tracked_inst\|
311		if (!self.isRegFree(reg)) {
312		// Move the instruction that was previously there to a
313		// stack allocation.
314		const spilled_inst = self.registers[index];
315		self.registers[index] = tracked_inst;
316		try self.getFunction().spillInstruction(reg, spilled_inst);
317		} else {
318		self.getRegAssumeFree(reg, tracked_inst);
319		}
320		else {
321		if (!self.isRegFree(reg)) {
322		// Move the instruction that was previously there to a
323		// stack allocation.
324		const spilled_inst = self.registers[index];
325		try self.getFunction().spillInstruction(reg, spilled_inst);
326		self.freeReg(reg);
327		}
328		}
	309	if (!self.isRegFree(reg)) {
	310	self.markRegAllocated(reg);
	311
	312	// Move the instruction that was previously there to a
	313	// stack allocation.
	314	const spilled_inst = self.registers[index];
	315	if (inst) \|tracked_inst\| self.registers[index] = tracked_inst;
	316	try self.getFunction().spillInstruction(reg, spilled_inst);
	317	if (inst == null) self.freeReg(reg);
	318	} else self.getRegAssumeFree(reg, inst);
329	319	}
330	320
331	321	/// Allocates the specified register with the specified
332	322	/// instruction. Asserts that the register is free and no
333	323	/// spilling is necessary.
334		pub fn getRegAssumeFree(self: *Self, reg: Register, inst: Air.Inst.Index) void {
	324	pub fn getRegAssumeFree(self: *Self, reg: Register, inst: ?Air.Inst.Index) void {
335	325	const index = indexOfRegIntoTracked(reg) orelse return;
336		log.debug("getRegAssumeFree {} for inst {}", .{ reg, inst });
	326	log.debug("getRegAssumeFree {} for inst {?}", .{ reg, inst });
337	327	self.markRegAllocated(reg);
338	328
339	329	assert(self.isRegFree(reg));
340		self.registers[index] = inst;
341		self.markRegUsed(reg);
	330	if (inst) \|tracked_inst\| {
	331	self.registers[index] = tracked_inst;
	332	self.markRegUsed(reg);
	333	}
342	334	}
343	335
344	336	/// Marks the specified register as free

test/behavior/atomics.zig-7

...	...	@@ -33,7 +33,6 @@ fn testCmpxchg() !void {
33	33
34	34	test "fence" {
35	35	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
36		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
37	36	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
38	37	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
39	38
...	...	@@ -44,7 +43,6 @@ test "fence" {
44	43
45	44	test "atomicrmw and atomicload" {
46	45	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
47		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
48	46	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
49	47	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
50	48	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...	...	@@ -73,7 +71,6 @@ fn testAtomicLoad(ptr: *u8) !void {
73	71
74	72	test "cmpxchg with ptr" {
75	73	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
76		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
77	74	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
78	75	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
79	76	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...	...	@@ -162,7 +159,6 @@ test "cmpxchg on a global variable" {
162	159
163	160	test "atomic load and rmw with enum" {
164	161	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
165		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
166	162	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
167	163	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
168	164	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...	...	@@ -180,7 +176,6 @@ test "atomic load and rmw with enum" {
180	176
181	177	test "atomic store" {
182	178	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
183		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
184	179	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
185	180	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
186	181	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...	...	@@ -194,7 +189,6 @@ test "atomic store" {
194	189
195	190	test "atomic store comptime" {
196	191	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
197		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
198	192	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
199	193	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
200	194	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...	...	@@ -424,7 +418,6 @@ fn testAtomicsWithType(comptime T: type, a: T, b: T) !void {
424	418
425	419	test "return @atomicStore, using it as a void value" {
426	420	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
427		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
428	421	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
429	422	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
430	423	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO

test/behavior/bugs/13068.zig-1

...	...	@@ -8,7 +8,6 @@ test {
8	8	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
9	9	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
10	10	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
11		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
12	11	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
13	12
14	13	list.items.len = 0;

test/behavior/cast.zig-1

...	...	@@ -655,7 +655,6 @@ test "@floatCast cast down" {
655	655	}
656	656
657	657	test "peer type resolution: unreachable, error set, unreachable" {
658		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
659	658	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
660	659	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
661	660

test/behavior/merge_error_sets.zig-1

...	...	@@ -12,7 +12,6 @@ fn foo() C!void {
12	12	}
13	13
14	14	test "merge error sets" {
15		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
16	15	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
17	16
18	17	if (foo()) {

test/behavior/switch.zig-1

...	...	@@ -228,7 +228,6 @@ const SwitchProngWithVarEnum = union(enum) {
228	228	};
229	229
230	230	test "switch prong with variable" {
231		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
232	231	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
233	232	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
234	233	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO