Commit 0395b35cee8d4082cc40b0dcd0298f797f42309d | zig

author	Andrew Kelley <andrew@ziglang.org> 2021-09-14 21:58:22-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2021-09-14 21:58:22-07:00
log	0395b35cee8d4082cc40b0dcd0298f797f42309d
tree	c4592f4e4cdb555836bb422d485b2c134e3f7547
parent	5d14590ed15ce23e7ef8032f8075dcfe76ba9dd8

stage2: implement cmpxchg and improve comptime eval

 * Implement Sema for `@cmpxchgWeak` and `@cmpxchgStrong`. Both runtime
   and comptime codepaths are implement.
 * Implement Codegen for LLVM backend and C backend.
 * Add LazySrcLoc.node_offset_builtin_call_argX 3...5
 * Sema: rework comptime control flow.
   - `error.ComptimeReturn` is used to signal that a comptime function
     call has returned a result (stored in the Inlining struct).
     `analyzeCall` notices this and handles the result.
   - The ZIR instructions `break_inline`, `block_inline`,
     `condbr_inline` are now redundant and can be deleted. `break`,
     `block`, and `condbr` function equivalently inside a comptime scope.
   - The ZIR instructions `loop` and `repeat` also are modified to
     directly perform comptime control flow inside a comptime scope,
     skipping an unnecessary mechanism for analysis of runtime code.
     This makes Zig perform closer to an interpreter when evaluating
     comptime code.
 * Sema: zirRetErrValue looks at Sema.ret_fn_ty rather than sema.func
   for adding to the inferred error set. This fixes a bug for
    inlined/comptime function calls.
 * Implement ZIR printing for cmpxchg.
 * stage1: make cmpxchg respect --single-threaded
   - Our LLVM C++ API wrapper failed to expose this boolean flag before.
 * Fix AIR printing for struct fields showing incorrect liveness data.

19 files changed, 682 insertions(+), 115 deletions(-)

src
- Air.zig
- AstGen.zig
- Liveness.zig
- Module.zig
- Sema.zig
- Zir.zig
- codegen.zig
- codegen
  - c.zig
  - llvm.zig
  - llvm
    bindings.zig
- link
  - C
    zig.h
- print_air.zig
- stage1
  - codegen.cpp
- target.zig
- type.zig
- zig_llvm.cpp
- zig_llvm.h
test
- behavior
  - atomics.zig
  - atomics_stage1.zig

src/Air.zig+23

...	...	@@ -309,6 +309,10 @@ pub const Inst = struct {
309	309	/// Given a pointer to an array, return a slice.
310	310	/// Uses the `ty_op` field.
311	311	array_to_slice,
	312	/// Uses the `ty_pl` field with payload `Cmpxchg`.
	313	cmpxchg_weak,
	314	/// Uses the `ty_pl` field with payload `Cmpxchg`.
	315	cmpxchg_strong,
312	316
313	317	pub fn fromCmpOp(op: std.math.CompareOperator) Tag {
314	318	return switch (op) {
...	...	@@ -443,6 +447,23 @@ pub const Asm = struct {
443	447	zir_index: u32,
444	448	};
445	449
	450	pub const Cmpxchg = struct {
	451	ptr: Inst.Ref,
	452	expected_value: Inst.Ref,
	453	new_value: Inst.Ref,
	454	/// 0b00000000000000000000000000000XXX - success_order
	455	/// 0b00000000000000000000000000XXX000 - failure_order
	456	flags: u32,
	457
	458	pub fn successOrder(self: Cmpxchg) std.builtin.AtomicOrder {
	459	return @intToEnum(std.builtin.AtomicOrder, @truncate(u3, self.flags));
	460	}
	461
	462	pub fn failureOrder(self: Cmpxchg) std.builtin.AtomicOrder {
	463	return @intToEnum(std.builtin.AtomicOrder, @truncate(u3, self.flags >> 3));
	464	}
	465	};
	466
446	467	pub fn getMainBody(air: Air) []const Air.Inst.Index {
447	468	const body_index = air.extra[@enumToInt(ExtraIndex.main_block)];
448	469	const extra = air.extraData(Block, body_index);
...	...	@@ -507,6 +528,8 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {
507	528	.struct_field_ptr,
508	529	.struct_field_val,
509	530	.ptr_elem_ptr,
	531	.cmpxchg_weak,
	532	.cmpxchg_strong,
510	533	=> return air.getRefType(datas[inst].ty_pl.ty),
511	534
512	535	.not,

src/AstGen.zig+6-5

...	...	@@ -7542,6 +7542,7 @@ fn cmpxchg(
7542	7542	tag: Zir.Inst.Tag,
7543	7543	) InnerError!Zir.Inst.Ref {
7544	7544	const int_type = try typeExpr(gz, scope, params[0]);
	7545	// TODO: allow this to be volatile
7545	7546	const ptr_type = try gz.add(.{ .tag = .ptr_type_simple, .data = .{
7546	7547	.ptr_type_simple = .{
7547	7548	.is_allowzero = false,
...	...	@@ -7553,11 +7554,11 @@ fn cmpxchg(
7553	7554	} });
7554	7555	const result = try gz.addPlNode(tag, node, Zir.Inst.Cmpxchg{
7555	7556	// zig fmt: off
7556		.ptr = try expr(gz, scope, .{ .ty = ptr_type }, params[1]),
7557		.expected_value = try expr(gz, scope, .{ .ty = int_type }, params[2]),
7558		.new_value = try expr(gz, scope, .{ .ty = int_type }, params[3]),
7559		.success_order = try expr(gz, scope, .{ .ty = .atomic_order_type }, params[4]),
7560		.fail_order = try expr(gz, scope, .{ .ty = .atomic_order_type }, params[5]),
	7557	.ptr = try expr(gz, scope, .{ .coerced_ty = ptr_type }, params[1]),
	7558	.expected_value = try expr(gz, scope, .{ .coerced_ty = int_type }, params[2]),
	7559	.new_value = try expr(gz, scope, .{ .coerced_ty = int_type }, params[3]),
	7560	.success_order = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[4]),
	7561	.failure_order = try expr(gz, scope, .{ .coerced_ty = .atomic_order_type }, params[5]),
7561	7562	// zig fmt: on
7562	7563	});
7563	7564	return rvalue(gz, rl, result, node);

src/Liveness.zig+4

...	...	@@ -340,6 +340,10 @@ fn analyzeInst(
340	340	const extra = a.air.extraData(Air.Bin, inst_datas[inst].ty_pl.payload).data;
341	341	return trackOperands(a, new_set, inst, main_tomb, .{ extra.lhs, extra.rhs, .none });
342	342	},
	343	.cmpxchg_strong, .cmpxchg_weak => {
	344	const extra = a.air.extraData(Air.Cmpxchg, inst_datas[inst].ty_pl.payload).data;
	345	return trackOperands(a, new_set, inst, main_tomb, .{ extra.ptr, extra.expected_value, extra.new_value });
	346	},
343	347	.br => {
344	348	const br = inst_datas[inst].br;
345	349	return trackOperands(a, new_set, inst, main_tomb, .{ br.operand, .none, .none });

src/Module.zig+57-37

...	...	@@ -1321,6 +1321,7 @@ pub const Scope = struct {
1321	1321	/// It is shared among all the blocks in an inline or comptime called
1322	1322	/// function.
1323	1323	pub const Inlining = struct {
	1324	comptime_result: Air.Inst.Ref,
1324	1325	merges: Merges,
1325	1326	};
1326	1327
...	...	@@ -1643,36 +1644,12 @@ pub const SrcLoc = struct {
1643	1644	const token_starts = tree.tokens.items(.start);
1644	1645	return token_starts[tok_index];
1645	1646	},
1646		.node_offset_builtin_call_arg0 => \|node_off\| {
1647		const tree = try src_loc.file_scope.getTree(gpa);
1648		const node_datas = tree.nodes.items(.data);
1649		const node_tags = tree.nodes.items(.tag);
1650		const node = src_loc.declRelativeToNodeIndex(node_off);
1651		const param = switch (node_tags[node]) {
1652		.builtin_call_two, .builtin_call_two_comma => node_datas[node].lhs,
1653		.builtin_call, .builtin_call_comma => tree.extra_data[node_datas[node].lhs],
1654		else => unreachable,
1655		};
1656		const main_tokens = tree.nodes.items(.main_token);
1657		const tok_index = main_tokens[param];
1658		const token_starts = tree.tokens.items(.start);
1659		return token_starts[tok_index];
1660		},
1661		.node_offset_builtin_call_arg1 => \|node_off\| {
1662		const tree = try src_loc.file_scope.getTree(gpa);
1663		const node_datas = tree.nodes.items(.data);
1664		const node_tags = tree.nodes.items(.tag);
1665		const node = src_loc.declRelativeToNodeIndex(node_off);
1666		const param = switch (node_tags[node]) {
1667		.builtin_call_two, .builtin_call_two_comma => node_datas[node].rhs,
1668		.builtin_call, .builtin_call_comma => tree.extra_data[node_datas[node].lhs + 1],
1669		else => unreachable,
1670		};
1671		const main_tokens = tree.nodes.items(.main_token);
1672		const tok_index = main_tokens[param];
1673		const token_starts = tree.tokens.items(.start);
1674		return token_starts[tok_index];
1675		},
	1647	.node_offset_builtin_call_arg0 => \|n\| return src_loc.byteOffsetBuiltinCallArg(gpa, n, 0),
	1648	.node_offset_builtin_call_arg1 => \|n\| return src_loc.byteOffsetBuiltinCallArg(gpa, n, 1),
	1649	.node_offset_builtin_call_arg2 => \|n\| return src_loc.byteOffsetBuiltinCallArg(gpa, n, 2),
	1650	.node_offset_builtin_call_arg3 => \|n\| return src_loc.byteOffsetBuiltinCallArg(gpa, n, 3),
	1651	.node_offset_builtin_call_arg4 => \|n\| return src_loc.byteOffsetBuiltinCallArg(gpa, n, 4),
	1652	.node_offset_builtin_call_arg5 => \|n\| return src_loc.byteOffsetBuiltinCallArg(gpa, n, 5),
1676	1653	.node_offset_array_access_index => \|node_off\| {
1677	1654	const tree = try src_loc.file_scope.getTree(gpa);
1678	1655	const node_datas = tree.nodes.items(.data);
...	...	@@ -1965,6 +1942,31 @@ pub const SrcLoc = struct {
1965	1942	},
1966	1943	}
1967	1944	}
	1945
	1946	pub fn byteOffsetBuiltinCallArg(
	1947	src_loc: SrcLoc,
	1948	gpa: *Allocator,
	1949	node_off: i32,
	1950	arg_index: u32,
	1951	) !u32 {
	1952	const tree = try src_loc.file_scope.getTree(gpa);
	1953	const node_datas = tree.nodes.items(.data);
	1954	const node_tags = tree.nodes.items(.tag);
	1955	const node = src_loc.declRelativeToNodeIndex(node_off);
	1956	const param = switch (node_tags[node]) {
	1957	.builtin_call_two, .builtin_call_two_comma => switch (arg_index) {
	1958	0 => node_datas[node].lhs,
	1959	1 => node_datas[node].rhs,
	1960	else => unreachable,
	1961	},
	1962	.builtin_call, .builtin_call_comma => tree.extra_data[node_datas[node].lhs + arg_index],
	1963	else => unreachable,
	1964	};
	1965	const main_tokens = tree.nodes.items(.main_token);
	1966	const tok_index = main_tokens[param];
	1967	const token_starts = tree.tokens.items(.start);
	1968	return token_starts[tok_index];
	1969	}
1968	1970	};
1969	1971
1970	1972	/// Resolving a source location into a byte offset may require doing work
...	...	@@ -2032,6 +2034,10 @@ pub const LazySrcLoc = union(enum) {
2032	2034	node_offset_builtin_call_arg0: i32,
2033	2035	/// Same as `node_offset_builtin_call_arg0` except arg index 1.
2034	2036	node_offset_builtin_call_arg1: i32,
	2037	node_offset_builtin_call_arg2: i32,
	2038	node_offset_builtin_call_arg3: i32,
	2039	node_offset_builtin_call_arg4: i32,
	2040	node_offset_builtin_call_arg5: i32,
2035	2041	/// The source location points to the index expression of an array access
2036	2042	/// expression, found by taking this AST node index offset from the containing
2037	2043	/// Decl AST node, which points to an array access AST node. Next, navigate
...	...	@@ -2157,6 +2163,10 @@ pub const LazySrcLoc = union(enum) {
2157	2163	.node_offset_for_cond,
2158	2164	.node_offset_builtin_call_arg0,
2159	2165	.node_offset_builtin_call_arg1,
	2166	.node_offset_builtin_call_arg2,
	2167	.node_offset_builtin_call_arg3,
	2168	.node_offset_builtin_call_arg4,
	2169	.node_offset_builtin_call_arg5,
2160	2170	.node_offset_array_access_index,
2161	2171	.node_offset_slice_sentinel,
2162	2172	.node_offset_call_func,
...	...	@@ -2205,6 +2215,10 @@ pub const LazySrcLoc = union(enum) {
2205	2215	.node_offset_for_cond,
2206	2216	.node_offset_builtin_call_arg0,
2207	2217	.node_offset_builtin_call_arg1,
	2218	.node_offset_builtin_call_arg2,
	2219	.node_offset_builtin_call_arg3,
	2220	.node_offset_builtin_call_arg4,
	2221	.node_offset_builtin_call_arg5,
2208	2222	.node_offset_array_access_index,
2209	2223	.node_offset_slice_sentinel,
2210	2224	.node_offset_call_func,
...	...	@@ -2246,6 +2260,9 @@ pub const CompileError = error{
2246	2260	/// because the function is generic. This is only seen when analyzing the body of a param
2247	2261	/// instruction.
2248	2262	GenericPoison,
	2263	/// In a comptime scope, a return instruction was encountered. This error is only seen when
	2264	/// doing a comptime function call.
	2265	ComptimeReturn,
2249	2266	};
2250	2267
2251	2268	pub fn deinit(mod: *Module) void {
...	...	@@ -3928,8 +3945,10 @@ pub fn analyzeFnBody(mod: Module, decl: Decl, func: *Fn) SemaError!Air {
3928	3945	log.debug("set {s} to in_progress", .{decl.name});
3929	3946
3930	3947	_ = sema.analyzeBody(&inner_block, fn_info.body) catch \|err\| switch (err) {
	3948	// TODO make these unreachable instead of @panic
3931	3949	error.NeededSourceLocation => @panic("zig compiler bug: NeededSourceLocation"),
3932	3950	error.GenericPoison => @panic("zig compiler bug: GenericPoison"),
	3951	error.ComptimeReturn => @panic("zig compiler bug: ComptimeReturn"),
3933	3952	else => \|e\| return e,
3934	3953	};
3935	3954
...	...	@@ -4534,7 +4553,6 @@ pub const PeerTypeCandidateSrc = union(enum) {
4534	4553	self: PeerTypeCandidateSrc,
4535	4554	gpa: *Allocator,
4536	4555	decl: *Decl,
4537		candidates: usize,
4538	4556	candidate_i: usize,
4539	4557	) ?LazySrcLoc {
4540	4558	@setCold(true);
...	...	@@ -4547,12 +4565,14 @@ pub const PeerTypeCandidateSrc = union(enum) {
4547	4565	return candidate_srcs[candidate_i];
4548	4566	},
4549	4567	.typeof_builtin_call_node_offset => \|node_offset\| {
4550		if (candidates <= 2) {
4551		switch (candidate_i) {
4552		0 => return LazySrcLoc{ .node_offset_builtin_call_arg0 = node_offset },
4553		1 => return LazySrcLoc{ .node_offset_builtin_call_arg1 = node_offset },
4554		else => unreachable,
4555		}
	4568	switch (candidate_i) {
	4569	0 => return LazySrcLoc{ .node_offset_builtin_call_arg0 = node_offset },
	4570	1 => return LazySrcLoc{ .node_offset_builtin_call_arg1 = node_offset },
	4571	2 => return LazySrcLoc{ .node_offset_builtin_call_arg2 = node_offset },
	4572	3 => return LazySrcLoc{ .node_offset_builtin_call_arg3 = node_offset },
	4573	4 => return LazySrcLoc{ .node_offset_builtin_call_arg4 = node_offset },
	4574	5 => return LazySrcLoc{ .node_offset_builtin_call_arg5 = node_offset },
	4575	else => {},
4556	4576	}
4557	4577
4558	4578	const tree = decl.namespace.file_scope.getTree(gpa) catch \|err\| {

src/Sema.zig+230-28

...	...	@@ -159,7 +159,6 @@ pub fn analyzeBody(
159	159	.bit_or => try sema.zirBitwise(block, inst, .bit_or),
160	160	.bitcast => try sema.zirBitcast(block, inst),
161	161	.bitcast_result_ptr => try sema.zirBitcastResultPtr(block, inst),
162		.block => try sema.zirBlock(block, inst),
163	162	.suspend_block => try sema.zirSuspendBlock(block, inst),
164	163	.bool_not => try sema.zirBoolNot(block, inst),
165	164	.bool_br_and => try sema.zirBoolBr(block, inst, false),
...	...	@@ -215,7 +214,6 @@ pub fn analyzeBody(
215	214	.is_non_err_ptr => try sema.zirIsNonErrPtr(block, inst),
216	215	.is_non_null => try sema.zirIsNonNull(block, inst),
217	216	.is_non_null_ptr => try sema.zirIsNonNullPtr(block, inst),
218		.loop => try sema.zirLoop(block, inst),
219	217	.merge_error_sets => try sema.zirMergeErrorSets(block, inst),
220	218	.negate => try sema.zirNegate(block, inst, .sub),
221	219	.negate_wrap => try sema.zirNegate(block, inst, .subwrap),
...	...	@@ -308,8 +306,8 @@ pub fn analyzeBody(
308	306	.shr_exact => try sema.zirShrExact(block, inst),
309	307	.bit_offset_of => try sema.zirBitOffsetOf(block, inst),
310	308	.offset_of => try sema.zirOffsetOf(block, inst),
311		.cmpxchg_strong => try sema.zirCmpxchg(block, inst),
312		.cmpxchg_weak => try sema.zirCmpxchg(block, inst),
	309	.cmpxchg_strong => try sema.zirCmpxchg(block, inst, .cmpxchg_strong),
	310	.cmpxchg_weak => try sema.zirCmpxchg(block, inst, .cmpxchg_weak),
313	311	.splat => try sema.zirSplat(block, inst),
314	312	.reduce => try sema.zirReduce(block, inst),
315	313	.shuffle => try sema.zirShuffle(block, inst),
...	...	@@ -364,16 +362,12 @@ pub fn analyzeBody(
364	362	// Instructions that we know to always be noreturn based solely on their tag.
365	363	// These functions match the return type of analyzeBody so that we can
366	364	// tail call them here.
367		.break_inline => return inst,
368		.condbr => return sema.zirCondbr(block, inst),
369		.@"break" => return sema.zirBreak(block, inst),
370	365	.compile_error => return sema.zirCompileError(block, inst),
371	366	.ret_coerce => return sema.zirRetCoerce(block, inst),
372	367	.ret_node => return sema.zirRetNode(block, inst),
373	368	.ret_load => return sema.zirRetLoad(block, inst),
374	369	.ret_err_value => return sema.zirRetErrValue(block, inst),
375	370	.@"unreachable" => return sema.zirUnreachable(block, inst),
376		.repeat => return sema.zirRepeat(block, inst),
377	371	.panic => return sema.zirPanic(block, inst),
378	372	// zig fmt: on
379	373
...	...	@@ -499,6 +493,28 @@ pub fn analyzeBody(
499	493	},
500	494
501	495	// Special case instructions to handle comptime control flow.
	496	.@"break" => {
	497	if (block.is_comptime) {
	498	return inst; // same as break_inline
	499	} else {
	500	return sema.zirBreak(block, inst);
	501	}
	502	},
	503	.break_inline => return inst,
	504	.repeat => {
	505	if (block.is_comptime) {
	506	// Send comptime control flow back to the beginning of this block.
	507	const src: LazySrcLoc = .{ .node_offset = datas[inst].node };
	508	try sema.emitBackwardBranch(block, src);
	509	i = 0;
	510	continue;
	511	} else {
	512	const src_node = sema.code.instructions.items(.data)[inst].node;
	513	const src: LazySrcLoc = .{ .node_offset = src_node };
	514	try sema.requireRuntimeBlock(block, src);
	515	return always_noreturn;
	516	}
	517	},
502	518	.repeat_inline => {
503	519	// Send comptime control flow back to the beginning of this block.
504	520	const src: LazySrcLoc = .{ .node_offset = datas[inst].node };
...	...	@@ -506,6 +522,34 @@ pub fn analyzeBody(
506	522	i = 0;
507	523	continue;
508	524	},
	525	.loop => blk: {
	526	if (!block.is_comptime) break :blk try sema.zirLoop(block, inst);
	527	// Same as `block_inline`. TODO https://github.com/ziglang/zig/issues/8220
	528	const inst_data = datas[inst].pl_node;
	529	const extra = sema.code.extraData(Zir.Inst.Block, inst_data.payload_index);
	530	const inline_body = sema.code.extra[extra.end..][0..extra.data.body_len];
	531	const break_inst = try sema.analyzeBody(block, inline_body);
	532	const break_data = datas[break_inst].@"break";
	533	if (inst == break_data.block_inst) {
	534	break :blk sema.resolveInst(break_data.operand);
	535	} else {
	536	return break_inst;
	537	}
	538	},
	539	.block => blk: {
	540	if (!block.is_comptime) break :blk try sema.zirBlock(block, inst);
	541	// Same as `block_inline`. TODO https://github.com/ziglang/zig/issues/8220
	542	const inst_data = datas[inst].pl_node;
	543	const extra = sema.code.extraData(Zir.Inst.Block, inst_data.payload_index);
	544	const inline_body = sema.code.extra[extra.end..][0..extra.data.body_len];
	545	const break_inst = try sema.analyzeBody(block, inline_body);
	546	const break_data = datas[break_inst].@"break";
	547	if (inst == break_data.block_inst) {
	548	break :blk sema.resolveInst(break_data.operand);
	549	} else {
	550	return break_inst;
	551	}
	552	},
509	553	.block_inline => blk: {
510	554	// Directly analyze the block body without introducing a new block.
511	555	const inst_data = datas[inst].pl_node;
...	...	@@ -519,6 +563,24 @@ pub fn analyzeBody(
519	563	return break_inst;
520	564	}
521	565	},
	566	.condbr => blk: {
	567	if (!block.is_comptime) return sema.zirCondbr(block, inst);
	568	// Same as condbr_inline. TODO https://github.com/ziglang/zig/issues/8220
	569	const inst_data = datas[inst].pl_node;
	570	const cond_src: LazySrcLoc = .{ .node_offset_if_cond = inst_data.src_node };
	571	const extra = sema.code.extraData(Zir.Inst.CondBr, inst_data.payload_index);
	572	const then_body = sema.code.extra[extra.end..][0..extra.data.then_body_len];
	573	const else_body = sema.code.extra[extra.end + then_body.len ..][0..extra.data.else_body_len];
	574	const cond = try sema.resolveInstConst(block, cond_src, extra.data.condition);
	575	const inline_body = if (cond.val.toBool()) then_body else else_body;
	576	const break_inst = try sema.analyzeBody(block, inline_body);
	577	const break_data = datas[break_inst].@"break";
	578	if (inst == break_data.block_inst) {
	579	break :blk sema.resolveInst(break_data.operand);
	580	} else {
	581	return break_inst;
	582	}
	583	},
522	584	.condbr_inline => blk: {
523	585	const inst_data = datas[inst].pl_node;
524	586	const cond_src: LazySrcLoc = .{ .node_offset_if_cond = inst_data.src_node };
...	...	@@ -1933,16 +1995,6 @@ fn zirCompileLog(
1933	1995	return Air.Inst.Ref.void_value;
1934	1996	}
1935	1997
1936		fn zirRepeat(sema: Sema, block: Scope.Block, inst: Zir.Inst.Index) CompileError!Zir.Inst.Index {
1937		const tracy = trace(@src());
1938		defer tracy.end();
1939
1940		const src_node = sema.code.instructions.items(.data)[inst].node;
1941		const src: LazySrcLoc = .{ .node_offset = src_node };
1942		try sema.requireRuntimeBlock(block, src);
1943		return always_noreturn;
1944		}
1945
1946	1998	fn zirPanic(sema: Sema, block: Scope.Block, inst: Zir.Inst.Index) CompileError!Zir.Inst.Index {
1947	1999	const inst_data = sema.code.instructions.items(.data)[inst].un_node;
1948	2000	const src: LazySrcLoc = inst_data.src();
...	...	@@ -2003,7 +2055,6 @@ fn zirLoop(sema: Sema, parent_block: Scope.Block, inst: Zir.Inst.Index) Compil
2003	2055
2004	2056	_ = try sema.analyzeBody(&loop_block, body);
2005	2057
2006		// Loop repetition is implied so the last instruction may or may not be a noreturn instruction.
2007	2058	try child_block.instructions.append(gpa, loop_inst);
2008	2059
2009	2060	try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.Block).Struct.fields.len +
...	...	@@ -2615,6 +2666,7 @@ fn analyzeCall(
2615	2666	// This one is shared among sub-blocks within the same callee, but not
2616	2667	// shared among the entire inline/comptime call stack.
2617	2668	var inlining: Scope.Block.Inlining = .{
	2669	.comptime_result = undefined,
2618	2670	.merges = .{
2619	2671	.results = .{},
2620	2672	.br_list = .{},
...	...	@@ -2770,8 +2822,13 @@ fn analyzeCall(
2770	2822	}
2771	2823	}
2772	2824
2773		_ = try sema.analyzeBody(&child_block, fn_info.body);
2774		const result = try sema.analyzeBlockBody(block, call_src, &child_block, merges);
	2825	const result = result: {
	2826	_ = sema.analyzeBody(&child_block, fn_info.body) catch \|err\| switch (err) {
	2827	error.ComptimeReturn => break :result inlining.comptime_result,
	2828	else => \|e\| return e,
	2829	};
	2830	break :result try sema.analyzeBlockBody(block, call_src, &child_block, merges);
	2831	};
2775	2832
2776	2833	if (is_comptime_call) {
2777	2834	const result_val = try sema.resolveConstMaybeUndefVal(block, call_src, result);
...	...	@@ -6662,9 +6719,9 @@ fn zirRetErrValue(
6662	6719	const src = inst_data.src();
6663	6720
6664	6721	// Add the error tag to the inferred error set of the in-scope function.
6665		if (sema.func) \|func\| {
6666		if (func.getInferredErrorSet()) \|map\| {
6667		_ = try map.getOrPut(sema.gpa, err_name);
	6722	if (sema.fn_ret_ty.zigTypeTag() == .ErrorUnion) {
	6723	if (sema.fn_ret_ty.errorUnionSet().castTag(.error_set_inferred)) \|payload\| {
	6724	_ = try payload.data.map.getOrPut(sema.gpa, err_name);
6668	6725	}
6669	6726	}
6670	6727	// Return the error code from the function.
...	...	@@ -6699,6 +6756,10 @@ fn zirRetNode(sema: Sema, block: Scope.Block, inst: Zir.Inst.Index) CompileErr
6699	6756	const operand = sema.resolveInst(inst_data.operand);
6700	6757	const src = inst_data.src();
6701	6758
	6759	// TODO: we pass false here for the `need_coercion` boolean, but I'm pretty sure we need
	6760	// to remove this parameter entirely. Observe the problem by looking at the incorrect compile
	6761	// error that occurs when a behavior test case being executed at comptime fails, e.g.
	6762	// `test { comptime foo(); } fn foo() { try expect(false); }`
6702	6763	return sema.analyzeRet(block, operand, src, false);
6703	6764	}
6704	6765
...	...	@@ -6730,6 +6791,10 @@ fn analyzeRet(
6730	6791	try sema.coerce(block, sema.fn_ret_ty, uncasted_operand, src);
6731	6792
6732	6793	if (block.inlining) \|inlining\| {
	6794	if (block.is_comptime) {
	6795	inlining.comptime_result = operand;
	6796	return error.ComptimeReturn;
	6797	}
6733	6798	// We are inlining a function call; rewrite the `ret` as a `break`.
6734	6799	try inlining.merges.results.append(sema.gpa, operand);
6735	6800	_ = try block.addBr(inlining.merges.block_inst, operand);
...	...	@@ -7425,10 +7490,149 @@ fn zirOffsetOf(sema: Sema, block: Scope.Block, inst: Zir.Inst.Index) CompileEr
7425	7490	return sema.mod.fail(&block.base, src, "TODO: Sema.zirOffsetOf", .{});
7426	7491	}
7427	7492
7428		fn zirCmpxchg(sema: Sema, block: Scope.Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
	7493	fn checkAtomicOperandType(
	7494	sema: *Sema,
	7495	block: *Scope.Block,
	7496	ty_src: LazySrcLoc,
	7497	ty: Type,
	7498	) CompileError!void {
	7499	var buffer: Type.Payload.Bits = undefined;
	7500	const target = sema.mod.getTarget();
	7501	const max_atomic_bits = target_util.largestAtomicBits(target);
	7502	const int_ty = switch (ty.zigTypeTag()) {
	7503	.Int => ty,
	7504	.Enum => ty.enumTagType(&buffer),
	7505	.Float => {
	7506	const bit_count = ty.floatBits(target);
	7507	if (bit_count > max_atomic_bits) {
	7508	return sema.mod.fail(
	7509	&block.base,
	7510	ty_src,
	7511	"expected {d}-bit float type or smaller; found {d}-bit float type",
	7512	.{ max_atomic_bits, bit_count },
	7513	);
	7514	}
	7515	return;
	7516	},
	7517	.Bool => return, // Will be treated as `u8`.
	7518	else => return sema.mod.fail(
	7519	&block.base,
	7520	ty_src,
	7521	"expected bool, integer, float, enum, or pointer type; found {}",
	7522	.{ty},
	7523	),
	7524	};
	7525	const bit_count = int_ty.intInfo(target).bits;
	7526	if (bit_count > max_atomic_bits) {
	7527	return sema.mod.fail(
	7528	&block.base,
	7529	ty_src,
	7530	"expected {d}-bit integer type or smaller; found {d}-bit integer type",
	7531	.{ max_atomic_bits, bit_count },
	7532	);
	7533	}
	7534	}
	7535
	7536	fn resolveAtomicOrder(
	7537	sema: *Sema,
	7538	block: *Scope.Block,
	7539	src: LazySrcLoc,
	7540	zir_ref: Zir.Inst.Ref,
	7541	) CompileError!std.builtin.AtomicOrder {
	7542	const atomic_order_ty = try sema.getBuiltinType(block, src, "AtomicOrder");
	7543	const air_ref = sema.resolveInst(zir_ref);
	7544	const coerced = try sema.coerce(block, atomic_order_ty, air_ref, src);
	7545	const val = try sema.resolveConstValue(block, src, coerced);
	7546	return val.toEnum(std.builtin.AtomicOrder);
	7547	}
	7548
	7549	fn zirCmpxchg(
	7550	sema: *Sema,
	7551	block: *Scope.Block,
	7552	inst: Zir.Inst.Index,
	7553	air_tag: Air.Inst.Tag,
	7554	) CompileError!Air.Inst.Ref {
	7555	const mod = sema.mod;
7429	7556	const inst_data = sema.code.instructions.items(.data)[inst].pl_node;
	7557	const extra = sema.code.extraData(Zir.Inst.Cmpxchg, inst_data.payload_index).data;
7430	7558	const src = inst_data.src();
7431		return sema.mod.fail(&block.base, src, "TODO: Sema.zirCmpxchg", .{});
	7559	// zig fmt: off
	7560	const elem_ty_src : LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node };
	7561	const ptr_src : LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node };
	7562	const expected_src : LazySrcLoc = .{ .node_offset_builtin_call_arg2 = inst_data.src_node };
	7563	const new_value_src : LazySrcLoc = .{ .node_offset_builtin_call_arg3 = inst_data.src_node };
	7564	const success_order_src: LazySrcLoc = .{ .node_offset_builtin_call_arg4 = inst_data.src_node };
	7565	const failure_order_src: LazySrcLoc = .{ .node_offset_builtin_call_arg5 = inst_data.src_node };
	7566	// zig fmt: on
	7567	const ptr = sema.resolveInst(extra.ptr);
	7568	const elem_ty = sema.typeOf(ptr).elemType();
	7569	try sema.checkAtomicOperandType(block, elem_ty_src, elem_ty);
	7570	if (elem_ty.zigTypeTag() == .Float) {
	7571	return mod.fail(
	7572	&block.base,
	7573	elem_ty_src,
	7574	"expected bool, integer, enum, or pointer type; found '{}'",
	7575	.{elem_ty},
	7576	);
	7577	}
	7578	const expected_value = try sema.coerce(block, elem_ty, sema.resolveInst(extra.expected_value), expected_src);
	7579	const new_value = try sema.coerce(block, elem_ty, sema.resolveInst(extra.new_value), new_value_src);
	7580	const success_order = try sema.resolveAtomicOrder(block, success_order_src, extra.success_order);
	7581	const failure_order = try sema.resolveAtomicOrder(block, failure_order_src, extra.failure_order);
	7582
	7583	if (@enumToInt(success_order) < @enumToInt(std.builtin.AtomicOrder.Monotonic)) {
	7584	return mod.fail(&block.base, success_order_src, "success atomic ordering must be Monotonic or stricter", .{});
	7585	}
	7586	if (@enumToInt(failure_order) < @enumToInt(std.builtin.AtomicOrder.Monotonic)) {
	7587	return mod.fail(&block.base, failure_order_src, "failure atomic ordering must be Monotonic or stricter", .{});
	7588	}
	7589	if (@enumToInt(failure_order) > @enumToInt(success_order)) {
	7590	return mod.fail(&block.base, failure_order_src, "failure atomic ordering must be no stricter than success", .{});
	7591	}
	7592	if (failure_order == .Release or failure_order == .AcqRel) {
	7593	return mod.fail(&block.base, failure_order_src, "failure atomic ordering must not be Release or AcqRel", .{});
	7594	}
	7595
	7596	const result_ty = try Module.optionalType(sema.arena, elem_ty);
	7597
	7598	// special case zero bit types
	7599	if ((try sema.typeHasOnePossibleValue(block, elem_ty_src, elem_ty)) != null) {
	7600	return sema.addConstant(result_ty, Value.initTag(.null_value));
	7601	}
	7602
	7603	const runtime_src = if (try sema.resolveDefinedValue(block, ptr_src, ptr)) \|ptr_val\| rs: {
	7604	if (try sema.resolveMaybeUndefVal(block, expected_src, expected_value)) \|expected_val\| {
	7605	if (try sema.resolveMaybeUndefVal(block, new_value_src, new_value)) \|new_val\| {
	7606	if (expected_val.isUndef() or new_val.isUndef()) {
	7607	return sema.addConstUndef(result_ty);
	7608	}
	7609	const stored_val = (try ptr_val.pointerDeref(sema.arena)) orelse break :rs ptr_src;
	7610	const result_val = if (stored_val.eql(expected_val, elem_ty)) blk: {
	7611	try sema.storePtr(block, src, ptr, new_value);
	7612	break :blk Value.initTag(.null_value);
	7613	} else try Value.Tag.opt_payload.create(sema.arena, stored_val);
	7614
	7615	return sema.addConstant(result_ty, result_val);
	7616	} else break :rs new_value_src;
	7617	} else break :rs expected_src;
	7618	} else ptr_src;
	7619
	7620	const flags: u32 = @as(u32, @enumToInt(success_order)) \|
	7621	(@as(u32, @enumToInt(failure_order)) << 3);
	7622
	7623	try sema.requireRuntimeBlock(block, runtime_src);
	7624	return block.addInst(.{
	7625	.tag = air_tag,
	7626	.data = .{ .ty_pl = .{
	7627	.ty = try sema.addType(result_ty),
	7628	.payload = try sema.addExtra(Air.Cmpxchg{
	7629	.ptr = ptr,
	7630	.expected_value = expected_value,
	7631	.new_value = new_value,
	7632	.flags = flags,
	7633	}),
	7634	} },
	7635	});
7432	7636	}
7433	7637
7434	7638	fn zirSplat(sema: Sema, block: Scope.Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -9576,13 +9780,11 @@ fn resolvePeerTypes(
9576	9780	const chosen_src = candidate_srcs.resolve(
9577	9781	sema.gpa,
9578	9782	block.src_decl,
9579		instructions.len,
9580	9783	chosen_i,
9581	9784	);
9582	9785	const candidate_src = candidate_srcs.resolve(
9583	9786	sema.gpa,
9584	9787	block.src_decl,
9585		instructions.len,
9586	9788	candidate_i + 1,
9587	9789	);
9588	9790

src/Zir.zig+22-3

...	...	@@ -2778,7 +2778,7 @@ pub const Inst = struct {
2778	2778	expected_value: Ref,
2779	2779	new_value: Ref,
2780	2780	success_order: Ref,
2781		fail_order: Ref,
	2781	failure_order: Ref,
2782	2782	};
2783	2783
2784	2784	pub const AtomicRmw = struct {
...	...	@@ -3054,8 +3054,6 @@ const Writer = struct {
3054	3054	.array_init_ref,
3055	3055	.array_init_anon_ref,
3056	3056	.union_init_ptr,
3057		.cmpxchg_strong,
3058		.cmpxchg_weak,
3059	3057	.shuffle,
3060	3058	.select,
3061	3059	.atomic_rmw,
...	...	@@ -3072,6 +3070,10 @@ const Writer = struct {
3072	3070	.struct_init_ref,
3073	3071	=> try self.writeStructInit(stream, inst),
3074	3072
	3073	.cmpxchg_strong,
	3074	.cmpxchg_weak,
	3075	=> try self.writeCmpxchg(stream, inst),
	3076
3075	3077	.struct_init_anon,
3076	3078	.struct_init_anon_ref,
3077	3079	=> try self.writeStructInitAnon(stream, inst),
...	...	@@ -3474,6 +3476,23 @@ const Writer = struct {
3474	3476	try self.writeSrc(stream, inst_data.src());
3475	3477	}
3476	3478
	3479	fn writeCmpxchg(self: *Writer, stream: anytype, inst: Inst.Index) !void {
	3480	const inst_data = self.code.instructions.items(.data)[inst].pl_node;
	3481	const extra = self.code.extraData(Inst.Cmpxchg, inst_data.payload_index).data;
	3482
	3483	try self.writeInstRef(stream, extra.ptr);
	3484	try stream.writeAll(", ");
	3485	try self.writeInstRef(stream, extra.expected_value);
	3486	try stream.writeAll(", ");
	3487	try self.writeInstRef(stream, extra.new_value);
	3488	try stream.writeAll(", ");
	3489	try self.writeInstRef(stream, extra.success_order);
	3490	try stream.writeAll(", ");
	3491	try self.writeInstRef(stream, extra.failure_order);
	3492	try stream.writeAll(") ");
	3493	try self.writeSrc(stream, inst_data.src());
	3494	}
	3495
3477	3496	fn writeStructInitAnon(self: *Writer, stream: anytype, inst: Inst.Index) !void {
3478	3497	const inst_data = self.code.instructions.items(.data)[inst].pl_node;
3479	3498	const extra = self.code.extraData(Inst.StructInitAnon, inst_data.payload_index);

src/codegen.zig+13

...	...	@@ -857,6 +857,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
857	857	.struct_field_ptr=> try self.airStructFieldPtr(inst),
858	858	.struct_field_val=> try self.airStructFieldVal(inst),
859	859	.array_to_slice => try self.airArrayToSlice(inst),
	860	.cmpxchg_strong => try self.airCmpxchg(inst),
	861	.cmpxchg_weak => try self.airCmpxchg(inst),
860	862
861	863	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
862	864	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
...	...	@@ -4751,6 +4753,17 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
4751	4753	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4752	4754	}
4753	4755
	4756	fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
	4757	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
	4758	const extra = self.air.extraData(Air.Block, ty_pl.payload);
	4759	const result: MCValue = switch (arch) {
	4760	else => return self.fail("TODO implement airCmpxchg for {}", .{
	4761	self.target.cpu.arch,
	4762	}),
	4763	};
	4764	return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
	4765	}
	4766
4754	4767	fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
4755	4768	// First section of indexes correspond to a set number of constant values.
4756	4769	const ref_int = @enumToInt(inst);

src/codegen/c.zig+39

...	...	@@ -911,6 +911,8 @@ fn genBody(o: *Object, body: []const Air.Inst.Index) error{ AnalysisFail, OutOfM
911	911	.wrap_optional => try airWrapOptional(o, inst),
912	912	.struct_field_ptr => try airStructFieldPtr(o, inst),
913	913	.array_to_slice => try airArrayToSlice(o, inst),
	914	.cmpxchg_weak => try airCmpxchg(o, inst, "weak"),
	915	.cmpxchg_strong => try airCmpxchg(o, inst, "strong"),
914	916
915	917	.struct_field_ptr_index_0 => try airStructFieldPtrIndex(o, inst, 0),
916	918	.struct_field_ptr_index_1 => try airStructFieldPtrIndex(o, inst, 1),
...	...	@@ -1878,6 +1880,43 @@ fn airArrayToSlice(o: *Object, inst: Air.Inst.Index) !CValue {
1878	1880	return local;
1879	1881	}
1880	1882
	1883	fn airCmpxchg(o: Object, inst: Air.Inst.Index, flavor: [:0]const u8) !CValue {
	1884	const ty_pl = o.air.instructions.items(.data)[inst].ty_pl;
	1885	const extra = o.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
	1886	const inst_ty = o.air.typeOfIndex(inst);
	1887	const ptr = try o.resolveInst(extra.ptr);
	1888	const expected_value = try o.resolveInst(extra.expected_value);
	1889	const new_value = try o.resolveInst(extra.new_value);
	1890	const local = try o.allocLocal(inst_ty, .Const);
	1891	const writer = o.writer();
	1892
	1893	try writer.print(" = zig_cmpxchg_{s}(", .{flavor});
	1894	try o.writeCValue(writer, ptr);
	1895	try writer.writeAll(", ");
	1896	try o.writeCValue(writer, expected_value);
	1897	try writer.writeAll(", ");
	1898	try o.writeCValue(writer, new_value);
	1899	try writer.writeAll(", ");
	1900	try writeMemoryOrder(writer, extra.successOrder());
	1901	try writer.writeAll(", ");
	1902	try writeMemoryOrder(writer, extra.failureOrder());
	1903	try writer.writeAll(");\n");
	1904
	1905	return local;
	1906	}
	1907
	1908	fn writeMemoryOrder(w: anytype, order: std.builtin.AtomicOrder) !void {
	1909	const str = switch (order) {
	1910	.Unordered => "memory_order_relaxed",
	1911	.Monotonic => "memory_order_consume",
	1912	.Acquire => "memory_order_acquire",
	1913	.Release => "memory_order_release",
	1914	.AcqRel => "memory_order_acq_rel",
	1915	.SeqCst => "memory_order_seq_cst",
	1916	};
	1917	return w.writeAll(str);
	1918	}
	1919
1881	1920	fn IndentWriter(comptime UnderlyingWriter: type) type {
1882	1921	return struct {
1883	1922	const Self = @This();

src/codegen/llvm.zig+93

...	...	@@ -389,6 +389,7 @@ pub const Object = struct {
389	389	.latest_alloca_inst = null,
390	390	.llvm_func = llvm_func,
391	391	.blocks = .{},
	392	.single_threaded = module.comp.bin_file.options.single_threaded,
392	393	};
393	394	defer fg.deinit();
394	395
...	...	@@ -906,6 +907,31 @@ pub const DeclGen = struct {
906	907	// TODO: improve this API, `addAttr(-1, attr_name)`
907	908	self.addAttr(val, std.math.maxInt(llvm.AttributeIndex), attr_name);
908	909	}
	910
	911	/// If the operand type of an atomic operation is not byte sized we need to
	912	/// widen it before using it and then truncate the result.
	913	/// RMW exchange of floating-point values is bitcasted to same-sized integer
	914	/// types to work around a LLVM deficiency when targeting ARM/AArch64.
	915	fn getAtomicAbiType(dg: DeclGen, ty: Type, is_rmw_xchg: bool) ?const llvm.Type {
	916	const target = dg.module.getTarget();
	917	var buffer: Type.Payload.Bits = undefined;
	918	const int_ty = switch (ty.zigTypeTag()) {
	919	.Int => ty,
	920	.Enum => ty.enumTagType(&buffer),
	921	.Float => {
	922	if (!is_rmw_xchg) return null;
	923	return dg.context.intType(@intCast(c_uint, ty.abiSize(target) * 8));
	924	},
	925	.Bool => return dg.context.intType(8),
	926	else => return null,
	927	};
	928	const bit_count = int_ty.intInfo(target).bits;
	929	if (!std.math.isPowerOfTwo(bit_count) or (bit_count % 8) != 0) {
	930	return dg.context.intType(@intCast(c_uint, int_ty.abiSize(target) * 8));
	931	} else {
	932	return null;
	933	}
	934	}
909	935	};
910	936
911	937	pub const FuncGen = struct {
...	...	@@ -940,6 +966,8 @@ pub const FuncGen = struct {
940	966	break_vals: *BreakValues,
941	967	}),
942	968
	969	single_threaded: bool,
	970
943	971	const BreakBasicBlocks = std.ArrayListUnmanaged(*const llvm.BasicBlock);
944	972	const BreakValues = std.ArrayListUnmanaged(*const llvm.Value);
945	973
...	...	@@ -1029,6 +1057,8 @@ pub const FuncGen = struct {
1029	1057	.slice_ptr => try self.airSliceField(inst, 0),
1030	1058	.slice_len => try self.airSliceField(inst, 1),
1031	1059	.array_to_slice => try self.airArrayToSlice(inst),
	1060	.cmpxchg_weak => try self.airCmpxchg(inst, true),
	1061	.cmpxchg_strong => try self.airCmpxchg(inst, false),
1032	1062
1033	1063	.struct_field_ptr => try self.airStructFieldPtr(inst),
1034	1064	.struct_field_val => try self.airStructFieldVal(inst),
...	...	@@ -1975,6 +2005,58 @@ pub const FuncGen = struct {
1975	2005	return null;
1976	2006	}
1977	2007
	2008	fn airCmpxchg(self: FuncGen, inst: Air.Inst.Index, is_weak: bool) !?const llvm.Value {
	2009	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
	2010	const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
	2011	var ptr = try self.resolveInst(extra.ptr);
	2012	var expected_value = try self.resolveInst(extra.expected_value);
	2013	var new_value = try self.resolveInst(extra.new_value);
	2014	const operand_ty = self.air.typeOf(extra.ptr).elemType();
	2015	const opt_abi_ty = self.dg.getAtomicAbiType(operand_ty, false);
	2016	if (opt_abi_ty) \|abi_ty\| {
	2017	// operand needs widening and truncating
	2018	ptr = self.builder.buildBitCast(ptr, abi_ty.pointerType(0), "");
	2019	if (operand_ty.isSignedInt()) {
	2020	expected_value = self.builder.buildSExt(expected_value, abi_ty, "");
	2021	new_value = self.builder.buildSExt(new_value, abi_ty, "");
	2022	} else {
	2023	expected_value = self.builder.buildZExt(expected_value, abi_ty, "");
	2024	new_value = self.builder.buildZExt(new_value, abi_ty, "");
	2025	}
	2026	}
	2027	const success_order = toLlvmAtomicOrdering(extra.successOrder());
	2028	const failure_order = toLlvmAtomicOrdering(extra.failureOrder());
	2029	const result = self.builder.buildCmpXchg(
	2030	ptr,
	2031	expected_value,
	2032	new_value,
	2033	success_order,
	2034	failure_order,
	2035	is_weak,
	2036	self.single_threaded,
	2037	);
	2038
	2039	const optional_ty = self.air.typeOfIndex(inst);
	2040	var buffer: Type.Payload.ElemType = undefined;
	2041	const child_ty = optional_ty.optionalChild(&buffer);
	2042
	2043	var payload = self.builder.buildExtractValue(result, 0, "");
	2044	if (opt_abi_ty != null) {
	2045	payload = self.builder.buildTrunc(payload, try self.dg.llvmType(operand_ty), "");
	2046	}
	2047	const success_bit = self.builder.buildExtractValue(result, 1, "");
	2048
	2049	if (optional_ty.isPtrLikeOptional()) {
	2050	const child_llvm_ty = try self.dg.llvmType(child_ty);
	2051	return self.builder.buildSelect(success_bit, child_llvm_ty.constNull(), payload, "");
	2052	}
	2053
	2054	const optional_llvm_ty = try self.dg.llvmType(optional_ty);
	2055	const non_null_bit = self.builder.buildNot(success_bit, "");
	2056	const partial = self.builder.buildInsertValue(optional_llvm_ty.getUndef(), payload, 0, "");
	2057	return self.builder.buildInsertValue(partial, non_null_bit, 1, "");
	2058	}
	2059
1978	2060	fn getIntrinsic(self: FuncGen, name: []const u8) const llvm.Value {
1979	2061	const id = llvm.lookupIntrinsicID(name.ptr, name.len);
1980	2062	assert(id != 0);
...	...	@@ -2125,3 +2207,14 @@ fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void {
2125	2207	.spirv64 => {},
2126	2208	}
2127	2209	}
	2210
	2211	fn toLlvmAtomicOrdering(atomic_order: std.builtin.AtomicOrder) llvm.AtomicOrdering {
	2212	return switch (atomic_order) {
	2213	.Unordered => .Unordered,
	2214	.Monotonic => .Monotonic,
	2215	.Acquire => .Acquire,
	2216	.Release => .Release,
	2217	.AcqRel => .AcquireRelease,
	2218	.SeqCst => .SequentiallyConsistent,
	2219	};
	2220	}

src/codegen/llvm/bindings.zig+39

...	...	@@ -298,6 +298,14 @@ pub const Builder = opaque {
298	298	Name: [*:0]const u8,
299	299	) *const Value;
300	300
	301	pub const buildSExt = LLVMBuildSExt;
	302	extern fn LLVMBuildSExt(
	303	*const Builder,
	304	Val: *const Value,
	305	DestTy: *const Type,
	306	Name: [*:0]const u8,
	307	) *const Value;
	308
301	309	pub const buildCall = LLVMBuildCall;
302	310	extern fn LLVMBuildCall(
303	311	*const Builder,
...	...	@@ -493,6 +501,27 @@ pub const Builder = opaque {
493	501	Index: c_uint,
494	502	Name: [*:0]const u8,
495	503	) *const Value;
	504
	505	pub const buildCmpXchg = ZigLLVMBuildCmpXchg;
	506	extern fn ZigLLVMBuildCmpXchg(
	507	builder: *const Builder,
	508	ptr: *const Value,
	509	cmp: *const Value,
	510	new_val: *const Value,
	511	success_ordering: AtomicOrdering,
	512	failure_ordering: AtomicOrdering,
	513	is_weak: bool,
	514	is_single_threaded: bool,
	515	) *const Value;
	516
	517	pub const buildSelect = LLVMBuildSelect;
	518	extern fn LLVMBuildSelect(
	519	*const Builder,
	520	If: *const Value,
	521	Then: *const Value,
	522	Else: *const Value,
	523	Name: [*:0]const u8,
	524	) *const Value;
496	525	};
497	526
498	527	pub const IntPredicate = enum(c_uint) {
...	...	@@ -854,3 +883,13 @@ pub const Linkage = enum(c_uint) {
854	883	LinkerPrivate,
855	884	LinkerPrivateWeak,
856	885	};
	886
	887	pub const AtomicOrdering = enum(c_uint) {
	888	NotAtomic = 0,
	889	Unordered = 1,
	890	Monotonic = 2,
	891	Acquire = 4,
	892	Release = 5,
	893	AcquireRelease = 6,
	894	SequentiallyConsistent = 7,
	895	};

src/link/C/zig.h+12

...	...	@@ -60,6 +60,18 @@
60	60	#define zig_breakpoint() raise(SIGTRAP)
61	61	#endif
62	62
	63	#if __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
	64	#include <stdatomic.h>
	65	#define zig_cmpxchg_strong(obj, expected, desired, succ, fail) atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, fail)
	66	#define zig_cmpxchg_weak(obj, expected, desired, succ, fail) atomic_compare_exchange_weak_explicit(obj, expected, desired, succ, fail)
	67	#elif __GNUC__
	68	#define zig_cmpxchg_strong(obj, expected, desired, succ, fail) __sync_val_compare_and_swap(obj, expected, desired)
	69	#define zig_cmpxchg_weak(obj, expected, desired, succ, fail) __sync_val_compare_and_swap(obj, expected, desired)
	70	#else
	71	#define zig_cmpxchg_strong(obj, expected, desired, succ, fail) zig_unimplemented()
	72	#define zig_cmpxchg_weak(obj, expected, desired, succ, fail) zig_unimplemented()
	73	#endif
	74
63	75	#include <stdint.h>
64	76	#include <stddef.h>
65	77	#include <limits.h>

src/print_air.zig+16-1

...	...	@@ -191,6 +191,7 @@ const Writer = struct {
191	191	.br => try w.writeBr(s, inst),
192	192	.cond_br => try w.writeCondBr(s, inst),
193	193	.switch_br => try w.writeSwitchBr(s, inst),
	194	.cmpxchg_weak, .cmpxchg_strong => try w.writeCmpxchg(s, inst),
194	195	}
195	196	}
196	197
...	...	@@ -258,7 +259,21 @@ const Writer = struct {
258	259
259	260	try w.writeOperand(s, inst, 0, extra.lhs);
260	261	try s.writeAll(", ");
261		try w.writeOperand(s, inst, 0, extra.rhs);
	262	try w.writeOperand(s, inst, 1, extra.rhs);
	263	}
	264
	265	fn writeCmpxchg(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
	266	const ty_pl = w.air.instructions.items(.data)[inst].ty_pl;
	267	const extra = w.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
	268
	269	try w.writeOperand(s, inst, 0, extra.ptr);
	270	try s.writeAll(", ");
	271	try w.writeOperand(s, inst, 1, extra.expected_value);
	272	try s.writeAll(", ");
	273	try w.writeOperand(s, inst, 2, extra.new_value);
	274	try s.print(", {s}, {s}", .{
	275	@tagName(extra.successOrder()), @tagName(extra.failureOrder()),
	276	});
262	277	}
263	278
264	279	fn writeConstant(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {

src/stage1/codegen.cpp+1-1

...	...	@@ -5723,7 +5723,7 @@ static LLVMValueRef ir_render_cmpxchg(CodeGen g, Stage1Air executable, Stage1A
5723	5723	LLVMAtomicOrdering failure_order = to_LLVMAtomicOrdering(instruction->failure_order);
5724	5724
5725	5725	LLVMValueRef result_val = ZigLLVMBuildCmpXchg(g->builder, ptr_val, cmp_val, new_val,
5726		success_order, failure_order, instruction->is_weak);
	5726	success_order, failure_order, instruction->is_weak, g->is_single_threaded);
5727	5727
5728	5728	ZigType *optional_type = instruction->base.value->type;
5729	5729	assert(optional_type->id == ZigTypeIdOptional);

src/target.zig+69

...	...	@@ -475,3 +475,72 @@ pub fn clangAssemblerSupportsMcpuArg(target: std.Target) bool {
475	475	pub fn needUnwindTables(target: std.Target) bool {
476	476	return target.os.tag == .windows;
477	477	}
	478
	479	/// TODO this was ported from stage1 but it does not take into account CPU features,
	480	/// which can affect this value. Audit this!
	481	pub fn largestAtomicBits(target: std.Target) u32 {
	482	return switch (target.cpu.arch) {
	483	.avr,
	484	.msp430,
	485	.spu_2,
	486	=> 16,
	487
	488	.arc,
	489	.arm,
	490	.armeb,
	491	.hexagon,
	492	.le32,
	493	.mips,
	494	.mipsel,
	495	.nvptx,
	496	.powerpc,
	497	.powerpcle,
	498	.r600,
	499	.riscv32,
	500	.sparc,
	501	.sparcel,
	502	.tce,
	503	.tcele,
	504	.thumb,
	505	.thumbeb,
	506	.i386,
	507	.xcore,
	508	.amdil,
	509	.hsail,
	510	.spir,
	511	.kalimba,
	512	.lanai,
	513	.shave,
	514	.wasm32,
	515	.renderscript32,
	516	.csky,
	517	.spirv32,
	518	=> 32,
	519
	520	.aarch64,
	521	.aarch64_be,
	522	.aarch64_32,
	523	.amdgcn,
	524	.bpfel,
	525	.bpfeb,
	526	.le64,
	527	.mips64,
	528	.mips64el,
	529	.nvptx64,
	530	.powerpc64,
	531	.powerpc64le,
	532	.riscv64,
	533	.sparcv9,
	534	.s390x,
	535	.amdil64,
	536	.hsail64,
	537	.spir64,
	538	.wasm64,
	539	.renderscript64,
	540	.ve,
	541	.spirv64,
	542	=> 64,
	543
	544	.x86_64 => 128,
	545	};
	546	}

src/type.zig+29

...	...	@@ -2886,6 +2886,35 @@ pub const Type = extern union {
2886	2886	}
2887	2887	}
2888	2888
	2889	/// Returns the integer tag type of the enum.
	2890	pub fn enumTagType(ty: Type, buffer: *Payload.Bits) Type {
	2891	switch (ty.tag()) {
	2892	.enum_full, .enum_nonexhaustive => {
	2893	const enum_full = ty.cast(Payload.EnumFull).?.data;
	2894	return enum_full.tag_ty;
	2895	},
	2896	.enum_simple => {
	2897	const enum_simple = ty.castTag(.enum_simple).?.data;
	2898	buffer.* = .{
	2899	.base = .{ .tag = .int_unsigned },
	2900	.data = std.math.log2_int_ceil(usize, enum_simple.fields.count()),
	2901	};
	2902	return Type.initPayload(&buffer.base);
	2903	},
	2904	.atomic_order,
	2905	.atomic_rmw_op,
	2906	.calling_convention,
	2907	.float_mode,
	2908	.reduce_op,
	2909	.call_options,
	2910	.export_options,
	2911	.extern_options,
	2912	=> @panic("TODO resolve std.builtin types"),
	2913
	2914	else => unreachable,
	2915	}
	2916	}
	2917
2889	2918	pub fn isNonexhaustiveEnum(ty: Type) bool {
2890	2919	return switch (ty.tag()) {
2891	2920	.enum_nonexhaustive => true,

src/zig_llvm.cpp+6-17

...	...	@@ -1087,10 +1087,12 @@ static AtomicOrdering mapFromLLVMOrdering(LLVMAtomicOrdering Ordering) {
1087	1087
1088	1088	LLVMValueRef ZigLLVMBuildCmpXchg(LLVMBuilderRef builder, LLVMValueRef ptr, LLVMValueRef cmp,
1089	1089	LLVMValueRef new_val, LLVMAtomicOrdering success_ordering,
1090		LLVMAtomicOrdering failure_ordering, bool is_weak)
	1090	LLVMAtomicOrdering failure_ordering, bool is_weak, bool is_single_threaded)
1091	1091	{
1092		AtomicCmpXchgInst *inst = unwrap(builder)->CreateAtomicCmpXchg(unwrap(ptr), unwrap(cmp),
1093		unwrap(new_val), mapFromLLVMOrdering(success_ordering), mapFromLLVMOrdering(failure_ordering));
	1092	AtomicCmpXchgInst *inst = unwrap(builder)->CreateAtomicCmpXchg(unwrap(ptr),
	1093	unwrap(cmp), unwrap(new_val),
	1094	mapFromLLVMOrdering(success_ordering), mapFromLLVMOrdering(failure_ordering),
	1095	is_single_threaded ? SyncScope::SingleThread : SyncScope::System);
1094	1096	inst->setWeak(is_weak);
1095	1097	return wrap(inst);
1096	1098	}
...	...	@@ -1308,19 +1310,6 @@ static AtomicRMWInst::BinOp toLLVMRMWBinOp(enum ZigLLVM_AtomicRMWBinOp BinOp) {
1308	1310	}
1309	1311	}
1310	1312
1311		static AtomicOrdering toLLVMOrdering(LLVMAtomicOrdering Ordering) {
1312		switch (Ordering) {
1313		default:
1314		case LLVMAtomicOrderingNotAtomic: return AtomicOrdering::NotAtomic;
1315		case LLVMAtomicOrderingUnordered: return AtomicOrdering::Unordered;
1316		case LLVMAtomicOrderingMonotonic: return AtomicOrdering::Monotonic;
1317		case LLVMAtomicOrderingAcquire: return AtomicOrdering::Acquire;
1318		case LLVMAtomicOrderingRelease: return AtomicOrdering::Release;
1319		case LLVMAtomicOrderingAcquireRelease: return AtomicOrdering::AcquireRelease;
1320		case LLVMAtomicOrderingSequentiallyConsistent: return AtomicOrdering::SequentiallyConsistent;
1321		}
1322		}
1323
1324	1313	inline LLVMAttributeRef wrap(Attribute Attr) {
1325	1314	return reinterpret_cast<LLVMAttributeRef>(Attr.getRawPointer());
1326	1315	}
...	...	@@ -1335,7 +1324,7 @@ LLVMValueRef ZigLLVMBuildAtomicRMW(LLVMBuilderRef B, enum ZigLLVM_AtomicRMWBinOp
1335	1324	{
1336	1325	AtomicRMWInst::BinOp intop = toLLVMRMWBinOp(op);
1337	1326	return wrap(unwrap(B)->CreateAtomicRMW(intop, unwrap(PTR),
1338		unwrap(Val), toLLVMOrdering(ordering),
	1327	unwrap(Val), mapFromLLVMOrdering(ordering),
1339	1328	singleThread ? SyncScope::SingleThread : SyncScope::System));
1340	1329	}
1341	1330

src/zig_llvm.h+1-1

...	...	@@ -148,7 +148,7 @@ ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildSShlSat(LLVMBuilderRef builder, LLVMValueR
148	148
149	149	ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildCmpXchg(LLVMBuilderRef builder, LLVMValueRef ptr, LLVMValueRef cmp,
150	150	LLVMValueRef new_val, LLVMAtomicOrdering success_ordering,
151		LLVMAtomicOrdering failure_ordering, bool is_weak);
	151	LLVMAtomicOrdering failure_ordering, bool is_weak, bool is_single_threaded);
152	152
153	153	ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildNSWShl(LLVMBuilderRef builder, LLVMValueRef LHS, LLVMValueRef RHS,
154	154	const char *name);

test/behavior/atomics.zig+22

...	...	@@ -2,3 +2,25 @@ const std = @import("std");
2	2	const expect = std.testing.expect;
3	3	const expectEqual = std.testing.expectEqual;
4	4	const builtin = @import("builtin");
	5
	6	test "cmpxchg" {
	7	try testCmpxchg();
	8	comptime try testCmpxchg();
	9	}
	10
	11	fn testCmpxchg() !void {
	12	var x: i32 = 1234;
	13	if (@cmpxchgWeak(i32, &x, 99, 5678, .SeqCst, .SeqCst)) \|x1\| {
	14	try expect(x1 == 1234);
	15	} else {
	16	@panic("cmpxchg should have failed");
	17	}
	18
	19	while (@cmpxchgWeak(i32, &x, 1234, 5678, .SeqCst, .SeqCst)) \|x1\| {
	20	try expect(x1 == 1234);
	21	}
	22	try expect(x == 5678);
	23
	24	try expect(@cmpxchgStrong(i32, &x, 5678, 42, .SeqCst, .SeqCst) == null);
	25	try expect(x == 42);
	26	}

test/behavior/atomics_stage1.zig-22

...	...	@@ -3,28 +3,6 @@ const expect = std.testing.expect;
3	3	const expectEqual = std.testing.expectEqual;
4	4	const builtin = @import("builtin");
5	5
6		test "cmpxchg" {
7		try testCmpxchg();
8		comptime try testCmpxchg();
9		}
10
11		fn testCmpxchg() !void {
12		var x: i32 = 1234;
13		if (@cmpxchgWeak(i32, &x, 99, 5678, .SeqCst, .SeqCst)) \|x1\| {
14		try expect(x1 == 1234);
15		} else {
16		@panic("cmpxchg should have failed");
17		}
18
19		while (@cmpxchgWeak(i32, &x, 1234, 5678, .SeqCst, .SeqCst)) \|x1\| {
20		try expect(x1 == 1234);
21		}
22		try expect(x == 5678);
23
24		try expect(@cmpxchgStrong(i32, &x, 5678, 42, .SeqCst, .SeqCst) == null);
25		try expect(x == 42);
26		}
27
28	6	test "fence" {
29	7	var x: i32 = 1234;
30	8	@fence(.SeqCst);